From f0e4a30cd94f2aef52035b98361587810c66df3e Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Mon, 3 Aug 2026 22:55:15 +0600 Subject: [PATCH 1/4] dc-dr: kubectl-dba command group for cross data center DR day-2 operations Adds `kubectl dba dc-dr` with nine commands, each live-verified against a three data center cluster before documenting: switchover planned zero-RPO move of the primary role (hub) status one-shot state + step-by-step switchover progress (hub) abort abort an in-flight switchover via the explicit abort signal accept-data-loss release a failover held by the RPO budget, --yes required handoff move the failover authority; the scope-local failover lever pin-primary break-glass: this DC stays primary and writable, no failover pin-standby this DC never promotes active-dc who holds the primary role, read from the Lease itself debug failover / switchover / fence symptom walkers DC-DR spans three API surfaces, so each command states which it needs. The coordination control plane is a separate apiserver: its kubeconfig is resolved from a file, a ConfigMap, or a Secret (defaulting to the dc-failover/coord-kubeconfig the chart mints), via the shared --coord-* flags. The pin commands act on a spoke by design, which is why they still work when the hub is unreachable. Scope resolution mirrors the operator's own (PlacementPolicy failoverPolicy trigger, then the legacy annotation, then Global) so the CLI can never disagree with the controller about which Lease a database follows. Guards refuse an Arbiter or Witness target, a no-op self-target, an unconfirmed data-loss acceptance, and a handoff into a break-glass-pinned scope. Documented in docs/CLI.md with real captured output. Signed-off-by: Tamal Saha --- docs/CLI.md | 405 +++++++++++++++++++++++++++++++++ pkg/cmds/dcdr.go | 26 +++ pkg/cmds/root.go | 6 + pkg/dcdr/abort.go | 69 ++++++ pkg/dcdr/acceptloss.go | 89 ++++++++ pkg/dcdr/activedc.go | 138 +++++++++++ pkg/dcdr/dcdr.go | 341 ++++++++++++++++++++++++++++ pkg/dcdr/debug.go | 504 +++++++++++++++++++++++++++++++++++++++++ pkg/dcdr/handoff.go | 143 ++++++++++++ pkg/dcdr/pin.go | 180 +++++++++++++++ pkg/dcdr/status.go | 303 +++++++++++++++++++++++++ pkg/dcdr/switchover.go | 100 ++++++++ 12 files changed, 2304 insertions(+) create mode 100644 docs/CLI.md create mode 100644 pkg/cmds/dcdr.go create mode 100644 pkg/dcdr/abort.go create mode 100644 pkg/dcdr/acceptloss.go create mode 100644 pkg/dcdr/activedc.go create mode 100644 pkg/dcdr/dcdr.go create mode 100644 pkg/dcdr/debug.go create mode 100644 pkg/dcdr/handoff.go create mode 100644 pkg/dcdr/pin.go create mode 100644 pkg/dcdr/status.go create mode 100644 pkg/dcdr/switchover.go diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 000000000..23306699d --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,405 @@ +# `kubectl dba dc-dr`: cross data center DR commands + +Day-2 operations for a KubeDB database distributed across data centers: trigger and +monitor planned switchovers, release a failover held by the RPO budget, move the +failover authority, pin a data center, and diagnose a failover that is not happening. + +Every command below was live-verified against a three data center cluster (two Member +DCs plus an Arbiter) before being documented here. + +## Which cluster each command talks to + +DC-DR spans three different API surfaces, and every command documents which one it +needs. Getting this wrong is the most common mistake, so it is stated per command in +`--help` as well. + +| surface | what lives there | how the CLI reaches it | +|---|---|---| +| **hub** | the KubeDB operator, the Postgres CRs, the PlacementPolicies | the ordinary kubeconfig flags: `--kubeconfig`, `$KUBECONFIG`, `--context`, `-n` | +| **coordination control plane** | the `primary-dc*` Leases (the failover authority) | the `--coord-*` flags below | +| **spoke** (one data center) | that DC's agent, its marker ConfigMap, and the human-owned pin ConfigMaps | the ordinary kubeconfig flags, pointed AT that spoke | + +### Reaching the coordination control plane + +It is a separate apiserver, so it needs its own kubeconfig. Commands that read or +write Leases (`active-dc`, `handoff`, `debug failover`, `debug fence`) accept, in +priority order: + +``` +--coord-kubeconfig /path/to/coord.yaml # a file +--coord-kubeconfig-configmap [namespace/]name # key "kubeconfig" in a ConfigMap on the CURRENT cluster +--coord-kubeconfig-secret [namespace/]name # key "kubeconfig" in a Secret on the CURRENT cluster + # default: dc-failover/coord-kubeconfig +--coord-namespace dc-failover # namespace holding the Leases +``` + +The default works out of the box on a standard install: the dr-controlplane chart +mints `dc-failover/coord-kubeconfig` on the hub, so pointing `$KUBECONFIG` at the hub +is usually all that is needed. + +## Command summary + +| command | what it does | talks to | +|---|---|---| +| [`switchover`](#switchover) | planned, zero-RPO move of the primary role | hub | +| [`status`](#status) | one-shot state and switchover progress | hub | +| [`abort`](#abort) | abort an in-flight switchover | hub | +| [`accept-data-loss`](#accept-data-loss) | release a failover held by the RPO budget | hub | +| [`handoff`](#handoff) | move the failover authority (the failover lever) | hub + coordination | +| [`pin-primary`](#pin-primary) | break-glass: this DC stays primary, no failover | spoke | +| [`pin-standby`](#pin-standby) | this DC never promotes | spoke | +| [`active-dc`](#active-dc) | who holds the primary role right now | hub + coordination | +| [`debug`](#debug) | diagnose failover / switchover / fence symptoms | hub + coordination | + +--- + +## switchover + +Planned, zero-RPO move of the primary role to another data center. + +```sh +kubectl dba dc-dr switchover DB_NAME -n NS --to DC +``` + +Sets `dr.kubedb.com/switchover-to`. The operator quiesces the active primary +(write-locks it so its LSN freezes), waits for the target to replay to that exact +LSN, hands off the Lease, and clears the annotation. No committed row is lost. + +**Requires the active primary to be up and reachable.** All three of the operator's +gates measure by dialing it and fail closed, so a dead primary can never be switched +away from; use [`handoff`](#handoff) for that case. + +Refuses up front: a target that is not a Member DC (an Arbiter or Witness can never +hold the primary role), and a target that is already active (prints a no-op notice). + +``` +$ kubectl dba dc-dr switchover cli-test -n demo --to dc-a +Switchover of demo/cli-test to "dc-a" requested (scope primary-dc-clitest, from PlacementPolicy cli-test-pp failoverPolicy trigger (Group "clitest")). +The operator will quiesce, wait for catch-up, and hand off; zero committed rows are lost. +Monitor: kubectl dba dc-dr status cli-test -n demo +Abort: kubectl dba dc-dr abort cli-test -n demo + +$ kubectl dba dc-dr switchover cli-test -n demo --to dc-c +Error: "dc-c" is not a Member data center of this database (members: [dc-a dc-b]); an Arbiter or Witness DC can never become primary +``` + +If the database's scope is Global, the command says so: every database in that scope +switches with it. + +## status + +One-shot DC-DR picture, plus step-by-step switchover progress when one is in flight. + +```sh +kubectl dba dc-dr status DB_NAME -n NS +``` + +**It does not follow.** Run it again to see the next state; that keeps the output +pasteable into a ticket. The step states are derived from the same status fields the +operator's own gates read, so the display cannot drift from the real decision. + +``` +$ kubectl dba dc-dr status pg-dcdr -n demo +Database: demo/pg-dcdr (Ready) +Failover scope: primary-dc + (PlacementPolicy dcdr-postgres failoverPolicy trigger (Global)) +Member DCs: dc-a, dc-b +Active DC: dc-b DR phase: Steady +Protected: true (data center "dc-a" is streaming 0 bytes behind, within the 16777216 byte budget) + +Data centers: + NAME ROLE WRITABLE HEALTHY LAG(BYTES) STREAMER + dc-a Member - true 0 pg-dcdr-dc-a-1 + dc-b Member true true - pg-dcdr-dc-b-1 + dc-c Arbiter - true - + +No switchover in flight. +Trigger one: kubectl dba dc-dr status pg-dcdr -n demo --to +``` + +During a switchover, the progress block replaces that last section: + +``` +Planned switchover to "dc-a", started 11s ago: + [done] 1. target "dc-a" validated: healthy and lag known, within the switchover budget + [done] 2. quiesce requested on the active DC (dc-b) + [NOW] 3. quiesce IN EFFECT: active primary write-locked, its LSN frozen + [pending] 4. target caught up to the frozen LSN (now 0 bytes, needs <= 8192) + [pending] 5. primary-DC Lease handed off to "dc-a" + [pending] 6. old DC demoted to standby, annotations cleared, DR phase back to Steady + + NEXT: waiting for the write-lock to take hold on dc-b. This needs the active primary to be UP + and reachable; a dead primary can never satisfy it (use the failover path). + + Abort: kubectl dba dc-dr abort cli-test -n demo + Re-run this command to see the next step; it does not follow. +``` + +## abort + +```sh +kubectl dba dc-dr abort DB_NAME -n NS +``` + +Sets `dr.kubedb.com/switchover-abort`; its PRESENCE is the signal. The hub clears the +quiesce so the original active DC resumes writes, then removes every switchover +annotation. + +**Do not abort by deleting `switchover-to`.** In a scope shared by several databases +the hub re-propagates that annotation to every sibling on each pass, so a bare removal +silently reappears. A switchover that cannot progress also auto-aborts on its own after +`dr.kubedb.com/switchover-timeout` (default 10m). + +## accept-data-loss + +Releases a cross-DC failover that the RPO budget is holding. + +```sh +kubectl dba dc-dr accept-data-loss DB_NAME -n NS --yes +``` + +When the surviving DC lags more than `spec.replication.bestEffortCrossDCLagBytesForFailover` +(or its lag cannot be measured), promotion is **held**: the lost DC's un-replicated WAL +is unrecoverable, so the choice between an outage and a larger-than-budgeted loss +belongs to a human. This records that decision. + +The command prints the current protection verdict first, and `--yes` is mandatory. Both +promotion paths (the hub gate and the coordinator's data-plane gate) honor it within +seconds, and the operator removes the annotation automatically once the failover it +authorized lands, so it cannot linger and approve a later, unrelated loss. + +``` +$ kubectl dba dc-dr accept-data-loss cli-test -n demo --yes +Current protection verdict: data center "dc-b" is streaming 0 bytes behind, within the 16777216 byte budget +NOTE: the database currently reads protected=true; nothing seems held. Setting the annotation is still +safe: it is only honored by a promotion that the budget is actively refusing, and it is auto-removed after use. +Data-loss acceptance recorded on demo/cli-test. +``` + +## handoff + +Moves the failover authority by annotating the Lease itself. + +```sh +kubectl dba dc-dr handoff (DB_NAME | --lease NAME) --to DC --yes +``` + +This is the **scope-local failover lever**, and the right tool when the active DC's +database is down but its data center is alive (nothing fails over on its own in that +state: the agent renews the Lease on its own health, not the database's). No quiesce +and no catch-up wait happen, so loss is bounded by the RPO budget rather than zero. +For a healthy primary prefer [`switchover`](#switchover). + +It moves **every** database in the scope. Never stop a DC's agent to force a failover +instead: one agent serves every scope that DC holds, so stopping it expires all of them. + +`--lease` works with no database present, which is how orphan scopes are moved. +Refuses a non-Member target, and refuses outright while a break-glass pin holds the +scope. Re-issuing the same target is safe: the command clears and re-sets the +annotation so the agents always see a real transition. + +``` +$ kubectl dba dc-dr handoff cli-test -n demo --to dc-b --yes +Handoff of primary-dc-clitest requested: dc-a -> dc-b. +The holder releases within seconds and dc-b acquires on its next retry tick; the annotation clears itself. +Verify: kubectl dba dc-dr active-dc --lease primary-dc-clitest +``` + +Measured: the target held the role about 6 seconds later. + +## pin-primary + +Break-glass. This data center stays primary and stays writable, even through a +coordination-plane outage. + +```sh +# run against the SPOKE of the DC being pinned +kubectl dba dc-dr pin-primary (--scope LEASE | --db DB) --yes +kubectl dba dc-dr pin-primary --scope LEASE --remove --yes +``` + +Creates the human-owned `-override` ConfigMap on that DC's spoke, which is +precisely why it still works when the hub is unreachable. Two effects: the DC's agent +mirrors the pin onto the Lease so every other Member defers permanently, and the DC's +coordinator forces its leader active regardless of marker state, so a sustained +control-plane outage no longer fences it read-only. + +Without `--yes` it prints what it would do and exits non-zero. + +``` +$ kubectl dba dc-dr pin-primary --scope primary-dc-clitest --yes # against dc-b +ConfigMap dc-failover/primary-dc-clitest-override created on the current cluster. +This data center is now PINNED primary for scope primary-dc-clitest: + - its agent mirrors the pin onto the Lease, so no other Member contends; + - its coordinator keeps the local leader writable even if the marker goes stale. + IMPORTANT: only honored on the scope's LAST KNOWN HOLDER. On any other DC the agent refuses it. + IMPORTANT: while pinned, that DC dying means NO failover happens. +``` + +The pin is then visible from anywhere, and blocks a handoff: + +``` +$ kubectl dba dc-dr active-dc --lease primary-dc-clitest +Active DC: dc-b +PINNED: break-glass override holds this scope on dc-b; it cannot fail over until the + override ConfigMap is removed from that DC's spoke. + +$ kubectl dba dc-dr handoff --lease primary-dc-clitest --to dc-a --yes +Error: scope primary-dc-clitest is PINNED to "dc-b" by a break-glass override; remove that DC's +override ConfigMap first (kubectl dba dc-dr pin-primary --remove), or the handoff cannot complete +``` + +Caveats, all live-verified: it is honored **only on the scope's last known holder** (on +any other DC the agent refuses it and logs why, so it never promotes a standby); while +it stands there is no split-brain protection for the scope and nothing takes over if +that DC dies; and on control-plane recovery the recreated Lease is claimed first-come, +so keep the pin until the authority is back with the right holder, then remove it. + +## pin-standby + +This data center never promotes. + +```sh +# run against the SPOKE of the DC being held +kubectl dba dc-dr pin-standby (--scope LEASE | --db DB) --yes +kubectl dba dc-dr pin-standby --scope LEASE --remove --yes +``` + +Creates `-standby-hold` on that DC's spoke. While it exists the DC never +contends for the Lease, never promotes (it refuses **even an explicit handoff naming +it**, verified live), and its coordinator refuses destructive cross-DC rewinds of the +data it holds. It fails **closed**: an unreadable ConfigMap is treated as held, so a +flaky apiserver never silently drops the protection. + +It is deliberately ignored on the DC that is currently active, because demoting the +active DC without a quiesce is unsafe. Move the primary away with a switchover first. + +Removing the hold takes effect immediately: the agent re-evaluates that scope's +contention the moment the ConfigMap disappears. Verified live, a handoff that had been +vetoed for 30s completed 10 seconds after the hold was released, with no further +action. + +## active-dc + +Who holds the primary role, from the authority itself. + +```sh +kubectl dba dc-dr active-dc DB_NAME -n NS +kubectl dba dc-dr active-dc --lease primary-dc-orders +kubectl dba dc-dr active-dc DB_NAME -n NS -q # just the DC name, for scripts +``` + +Given a database, its scope is resolved exactly as the operator resolves it (the +PlacementPolicy's `failoverPolicy.trigger`, falling back to the legacy +`dr.kubedb.com/failover-group` annotation, else Global) and the matching Lease is read. + +``` +$ kubectl dba dc-dr active-dc pg-dcdr -n demo +Active DC: dc-b +Lease: dc-failover/primary-dc (scope from PlacementPolicy dcdr-postgres failoverPolicy trigger (Global)) +Renewed: 2s ago (lease duration 45s) +Transitions:41 +Members: dc-a,dc-b +``` + +It warns when the Lease is expired, shows an in-flight handoff, shows a break-glass +pin, and notes when the database's own `status.activeDC` disagrees (the status trails +the Lease by a reconcile; the Lease is the authority). + +## debug + +```sh +kubectl dba dc-dr debug failover DB_NAME -n NS # nothing failed over +kubectl dba dc-dr debug switchover DB_NAME -n NS # switchover not completing +kubectl dba dc-dr debug fence DB_NAME -n NS # database up but read-only +``` + +Each walks the real decision chain and reports every condition in order, marking +`OK` / `WARN` / `FAIL`, with a concrete remedy on anything that is not OK, and a count +of blocking conditions at the end. + +`debug failover` checks: the database is distributed and armed; the scope is +registered (an unregistered scope has no Lease, so nothing can ever move); the Lease +exists, who holds it, and whether it is still being renewed; break-glass pins and +in-flight handoffs; each candidate DC's health Lease; the RPO budget verdict; stale or +failed ForceFailOver ops; and the DC-DR conditions including the retry cap. + +Real output from a database whose failovers had been failing overnight: + +``` +$ kubectl dba dc-dr debug failover pg-dcdr -n demo +Diagnosing failover for demo/pg-dcdr + + [OK ] database is DC-DR distributed and armed + [OK ] failover scope resolves to primary-dc + PlacementPolicy dcdr-postgres failoverPolicy trigger (Global) + [WARN] holder "dc-b" is renewing normally, so the authority will NOT move on its own + the Lease was renewed 5s ago, inside its 45s duration: that data center's agent is alive + -> this is by design: no database-level condition (client errors, QPS, lag, a crashed + postgres) ever moves the Lease. If the DATABASE is down but the DC is alive, either let + the DC's own raft promote a local peer, or move the scope deliberately: + kubectl dba dc-dr handoff pg-dcdr -n demo --to --yes + [OK ] no break-glass pin on the Lease + [OK ] candidate DC "dc-a" is alive (health renewed 0s ago) + [OK ] protection is confirmed (RPO budget satisfied) + [WARN] no failed ForceFailOver ops are blocking + failed: pg-dcdr-dcdr-failover-92vjc, ... (5 total) + -> read their status for the real cause, then delete them + [FAIL] the ForceFailOver retry cap is not tripped + 5 consecutive ForceFailOver attempts to promote data center "dc-b" failed; the hub has + stopped creating new attempts and needs manual intervention + -> Fix the underlying failure, delete the failed ops, and it resumes + +1 blocking condition(s) found. +``` + +`debug switchover` reports which gate is holding, and calls out the case people hit +most: every gate measures by dialing the active primary, so an unreachable primary +stalls the switchover permanently (abort and use `handoff`). + +`debug fence` explains a database that is up but refusing writes: whether the +authority is healthy, whether the Lease is being renewed (a stale Lease means every +DC's marker goes stale and all of them fence, by design), and what to do when the +coordination plane itself is the broken thing. + +--- + +## Worked examples + +**Planned move, monitored.** + +```sh +kubectl dba dc-dr switchover pg-dcdr -n demo --to dc-a +kubectl dba dc-dr status pg-dcdr -n demo # re-run until step 6 is done +``` + +**The active DC's database is dead but the DC is alive.** Nothing fails over on its +own; that is by design. Confirm, then move the scope: + +```sh +kubectl dba dc-dr debug failover pg-dcdr -n demo +kubectl dba dc-dr handoff pg-dcdr -n demo --to dc-b --yes +kubectl dba dc-dr active-dc pg-dcdr -n demo +``` + +**A failover is held by the RPO budget.** + +```sh +kubectl dba dc-dr status pg-dcdr -n demo # read protectionMessage +kubectl dba dc-dr accept-data-loss pg-dcdr -n demo --yes +``` + +**The coordination plane is down and the primary must keep writing.** Run against the +active DC's spoke, before its marker goes stale (roughly 90s from the last renewal): + +```sh +kubectl dba dc-dr pin-primary --scope primary-dc --yes --kubeconfig ~/.kube/dc-b.yaml +# ... after recovery, once the Lease is back with the right holder: +kubectl dba dc-dr pin-primary --scope primary-dc --remove --yes --kubeconfig ~/.kube/dc-b.yaml +``` + +**Keep a data center out of the primary role permanently.** + +```sh +kubectl dba dc-dr pin-standby --scope primary-dc --yes --kubeconfig ~/.kube/dc-a.yaml +``` diff --git a/pkg/cmds/dcdr.go b/pkg/cmds/dcdr.go new file mode 100644 index 000000000..60bcedef8 --- /dev/null +++ b/pkg/cmds/dcdr.go @@ -0,0 +1,26 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmds + +import ( + "kubedb.dev/cli/pkg/dcdr" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +// NewCmdDCDR returns the cross data center DR command group. +func NewCmdDCDR(f cmdutil.Factory) *cobra.Command { + return dcdr.NewCmdDCDR(f) +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index a222925bb..1ab74a0f1 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -114,6 +114,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command { NewCmdMSSQL(f), }, }, + { + Message: "Cross Data Center DR Commands", + Commands: []*cobra.Command{ + NewCmdDCDR(f), + }, + }, { Message: "Metric related CMDs", Commands: []*cobra.Command{ diff --git a/pkg/dcdr/abort.go b/pkg/dcdr/abort.go new file mode 100644 index 000000000..09fb82af4 --- /dev/null +++ b/pkg/dcdr/abort.go @@ -0,0 +1,69 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdAbort aborts an in-flight planned switchover. Acts on the HUB cluster. +func NewCmdAbort(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "abort DB_NAME", + Short: i18n.T("Abort an in-flight planned switchover and restore writes to the current active DC"), + Long: templates.LongDesc(` + Sets the dr.kubedb.com/switchover-abort annotation. Its PRESENCE aborts: + the hub clears the quiesce so the original active data center resumes + accepting writes, and removes every switchover annotation once done. + + Do NOT abort by deleting the switchover-to annotation: in a scope shared + by several databases the hub re-propagates it to every sibling each pass, + so a bare removal silently reappears. This explicit abort signal is + propagated and honored scope-wide. A switchover that cannot complete also + auto-aborts on its own after the switchover timeout (default 10m, + dr.kubedb.com/switchover-timeout to override). + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(` + kubectl dba dc-dr abort pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + if db.GetAnnotations()[AnnSwitchoverTo] == "" && db.GetAnnotations()[AnnQuiesceActive] == "" { + cmd.Printf("No switchover appears to be in flight on %s/%s (no %s or %s annotation); setting the abort anyway is harmless.\n", ns, args[0], AnnSwitchoverTo, AnnQuiesceActive) + } + v := "true" + if err := annotateDB(ctx, dyn, ns, args[0], AnnSwitchoverAbort, &v); err != nil { + return err + } + cmd.Printf("Abort requested for %s/%s. The hub restores writes to the current active DC and clears the switchover annotations; verify with:\n", ns, args[0]) + cmd.Printf(" kubectl dba dc-dr status %s -n %s\n", args[0], ns) + return nil + }, + } + return cmd +} diff --git a/pkg/dcdr/acceptloss.go b/pkg/dcdr/acceptloss.go new file mode 100644 index 000000000..9bfff8a11 --- /dev/null +++ b/pkg/dcdr/acceptloss.go @@ -0,0 +1,89 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdAcceptDataLoss releases a cross-DC failover held by the RPO budget. Acts +// on the HUB cluster. +func NewCmdAcceptDataLoss(f cmdutil.Factory) *cobra.Command { + var yes bool + cmd := &cobra.Command{ + Use: "accept-data-loss DB_NAME --yes", + Short: i18n.T("Release a failover held by the RPO budget, explicitly accepting the data loss"), + Long: templates.LongDesc(` + When the surviving data center lags more than + spec.replication.bestEffortCrossDCLagBytesForFailover (or its lag cannot + be measured), the promotion is HELD: the un-replicated WAL of the lost + data center is unrecoverable, so choosing between an outage and a loss + larger than the budget belongs to a human. This command records that + decision by setting dr.kubedb.com/accept-failover-data-loss=true; both + promotion paths (the hub gate and the coordinator's data-plane gate) + honor it within seconds, and the operator removes the annotation + automatically once the failover it authorized lands, so it cannot linger + and approve a later, unrelated loss. + + Where the hold is visible before you decide: + status.disasterRecovery.protectionMessage (the measured lag), + condition DCDRPromotionStalled, and dc-dr status. + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(` + # See what would be lost first + kubectl dba dc-dr status pg-dcdr -n demo + + # Accept it + kubectl dba dc-dr accept-data-loss pg-dcdr -n demo --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + msg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") + protected, protectedFound, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") + if msg != "" { + cmd.Printf("Current protection verdict: %s\n", msg) + } + if protectedFound && protected { + cmd.Printf("NOTE: the database currently reads protected=true; nothing seems held. Setting the annotation is still safe: it is only honored by a promotion that the budget is actively refusing, and it is auto-removed after use.\n") + } + if !yes { + return fmt.Errorf("this authorizes losing committed data beyond the configured budget; re-run with --yes to confirm") + } + v := "true" + if err := annotateDB(ctx, dyn, ns, args[0], AnnAcceptDataLoss, &v); err != nil { + return err + } + cmd.Printf("Data-loss acceptance recorded on %s/%s. The held promotion proceeds within seconds; the annotation is removed automatically once the failover lands.\n", ns, args[0]) + cmd.Printf("Monitor: kubectl dba dc-dr status %s -n %s\n", args[0], ns) + return nil + }, + } + cmd.Flags().BoolVar(&yes, "yes", false, "Confirm accepting data loss beyond the configured RPO budget") + return cmd +} diff --git a/pkg/dcdr/activedc.go b/pkg/dcdr/activedc.go new file mode 100644 index 000000000..74f06c742 --- /dev/null +++ b/pkg/dcdr/activedc.go @@ -0,0 +1,138 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdActiveDC answers "who is primary" from the authoritative source, the Lease +// on the coordination plane, resolving the scope from a database name when given. +func NewCmdActiveDC(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + var leaseName string + var quiet bool + cmd := &cobra.Command{ + Use: "active-dc [DB_NAME] [--lease NAME]", + Short: i18n.T("Print the data center that currently holds the primary role"), + Long: templates.LongDesc(` + Reads the primary-DC Lease from the coordination control plane, the + authority for which data center is active. + + Given a database name, its failover scope is resolved first (the + PlacementPolicy's failoverPolicy trigger, exactly as the operator + resolves it) and the matching Lease is read. Given --lease, that Lease is + read directly, which also works for a scope whose database is gone. + + KUBECONFIG: the hub cluster (to read the Postgres and its + PlacementPolicy, and by default to read the coordination kubeconfig + Secret). The coordination plane itself is reached with the --coord-* + flags.`), + Example: templates.Examples(` + # By database + kubectl dba dc-dr active-dc pg-dcdr -n demo + + # By Lease name, with an explicit coordination kubeconfig file + kubectl dba dc-dr active-dc --lease primary-dc --coord-kubeconfig /tmp/coord.yaml + + # Scriptable: just the DC name + kubectl dba dc-dr active-dc pg-dcdr -n demo -q`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if (len(args) == 0) == (leaseName == "") { + return fmt.Errorf("give exactly one of: a database name, or --lease") + } + ctx := context.Background() + out := cmd.OutOrStdout() + scope := &Scope{LeaseName: leaseName, Source: "--lease flag"} + var ns string + if len(args) == 1 { + db, dyn, dbNS, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + ns = dbNS + scope, err = ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + } + coord, err := cf.CoordClient(ctx, f) + if err != nil { + return err + } + lease, err := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read Lease %s/%s on the coordination plane: %w", cf.LeaseNS, scope.LeaseName, err) + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if quiet { + fmt.Fprintln(out, holder) + return nil + } + fmt.Fprintf(out, "Active DC: %s\n", orNone(holder)) + fmt.Fprintf(out, "Lease: %s/%s (scope from %s)\n", cf.LeaseNS, scope.LeaseName, scope.Source) + if lease.Spec.RenewTime != nil { + age := time.Since(lease.Spec.RenewTime.Time).Round(time.Second) + dur := int32(0) + if lease.Spec.LeaseDurationSeconds != nil { + dur = *lease.Spec.LeaseDurationSeconds + } + fmt.Fprintf(out, "Renewed: %s ago (lease duration %ds)\n", age, dur) + if dur > 0 && age > time.Duration(dur)*time.Second { + fmt.Fprintf(out, " WARNING: the Lease is EXPIRED. Its holder stopped renewing, so a healthy Member DC may acquire it at any moment.\n") + } + } + if lease.Spec.LeaseTransitions != nil { + fmt.Fprintf(out, "Transitions:%d\n", *lease.Spec.LeaseTransitions) + } + if v := lease.Annotations[AnnLeaseMemberDCs]; v != "" { + fmt.Fprintf(out, "Members: %s\n", v) + } + if v := lease.Annotations[AnnLeaseHandoffTo]; v != "" { + fmt.Fprintf(out, "Handoff to: %s (a coordinated handoff is in flight)\n", v) + } + if v := lease.Annotations["dr.open-cluster-management.io/override-hold"]; v != "" { + fmt.Fprintf(out, "PINNED: break-glass override holds this scope on %s; it cannot fail over until the override ConfigMap is removed from that DC's spoke.\n", v) + } + if len(args) == 1 { + // Cross-check the CR's own view, which lags the Lease by a reconcile. + db, _, _, err := getDB(ctx, f, args[0]) + if err == nil { + crActive, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + if crActive != "" && crActive != holder { + fmt.Fprintf(out, "\nNOTE: the database %s/%s still reports activeDC=%s in its status; the Lease is the authority and the status trails it by a reconcile.\n", ns, args[0], crActive) + } + } + } + return nil + }, + } + cmd.Flags().StringVar(&leaseName, "lease", "", "Read this Lease directly instead of resolving a database's scope") + cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Print only the active DC name") + AddCoordFlags(cmd, &cf) + return cmd +} diff --git a/pkg/dcdr/dcdr.go b/pkg/dcdr/dcdr.go new file mode 100644 index 000000000..c820eaa71 --- /dev/null +++ b/pkg/dcdr/dcdr.go @@ -0,0 +1,341 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package dcdr implements `kubectl dba dc-dr`, the day-2 command group for KubeDB +// cross data center disaster recovery (DC-DR). +// +// The commands act on up to three different control planes, and each one documents +// which: +// +// - The HUB cluster (where the KubeDB operator and the database CRs live): the +// ordinary kubeconfig flags / $KUBECONFIG select it, exactly like every other +// kubectl-dba command. +// - The COORDINATION control plane (a separate apiserver holding the primary-DC +// Leases): reached with a dedicated kubeconfig resolved by CoordFlags, from a +// file, a Secret, or a ConfigMap (see AddCoordFlags). +// - A SPOKE cluster (one data center): the pin commands create their marker +// ConfigMaps on a specific DC's spoke, so for those the ordinary kubeconfig +// flags must point AT that spoke. +package dcdr + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// GroupVersionResources of the objects this command group touches. Everything is +// accessed unstructured on purpose: the DC-DR status and PlacementPolicy types are +// not part of the released apimachinery this CLI vendors, and the commands only +// need a handful of well-known paths. +var ( + PostgresGVR = schema.GroupVersionResource{Group: "kubedb.com", Version: "v1", Resource: "postgreses"} + PlacementGVR = schema.GroupVersionResource{Group: "apps.k8s.appscode.com", Version: "v1", Resource: "placementpolicies"} + PgOpsGVR = schema.GroupVersionResource{Group: "ops.kubedb.com", Version: "v1alpha1", Resource: "postgresopsrequests"} +) + +// The annotation and naming contract shared with the KubeDB Postgres operator and +// the dr-controlplane service. Keep byte for byte in sync with +// postgres/pkg/dcdr/helpers.go and dr-controlplane/pkg/leases/names.go. +const ( + AnnSwitchoverTo = "dr.kubedb.com/switchover-to" + AnnSwitchoverAbort = "dr.kubedb.com/switchover-abort" + AnnSwitchoverStart = "dr.kubedb.com/switchover-started" + AnnQuiesceActive = "dr.kubedb.com/quiesce-active" + AnnAcceptDataLoss = "dr.kubedb.com/accept-failover-data-loss" + AnnFailoverGroup = "dr.kubedb.com/failover-group" + AnnMaxLagBytes = "dr.kubedb.com/switchover-max-lag-bytes" + + AnnLeaseHandoffTo = "dr.open-cluster-management.io/handoff-to" + AnnLeaseMemberDCs = "dr.open-cluster-management.io/member-dcs" + + GlobalPrimaryLease = "primary-dc" + GroupLeasePrefix = "primary-dc-" + + OverrideCMSuffix = "-override" + StandbyHoldCMSuffix = "-standby-hold" + + // DefaultCoordNamespace is where the Leases and marker ConfigMaps live, on the + // coordination plane and on every spoke respectively. + DefaultCoordNamespace = "dc-failover" + + // zeroRPOLagBytes mirrors the operator's switchover residual-lag tolerance. + zeroRPOLagBytes = int64(8 * 1024) +) + +// NewCmdDCDR returns the `dc-dr` command group. +func NewCmdDCDR(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "dc-dr", + Short: i18n.T("Cross data center DR operations: switchover, failover, pins, and diagnosis"), + Long: templates.LongDesc(` + Operate a KubeDB database that is distributed across data centers: + trigger and monitor planned switchovers, accept a held failover's data + loss, move the failover authority, pin a data center, and diagnose a + failover that is not happening.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand( + NewCmdSwitchover(f), + NewCmdStatus(f), + NewCmdAbort(f), + NewCmdAcceptDataLoss(f), + NewCmdDebug(f), + NewCmdHandoff(f), + NewCmdPin(f, pinPrimary), + NewCmdPin(f, pinStandby), + NewCmdActiveDC(f), + ) + return cmd +} + +// CoordFlags resolves the kubeconfig of the coordination control plane, the +// separate apiserver that stores the primary-DC Leases. Resolution order: +// +// 1. --coord-kubeconfig: a kubeconfig FILE for the coordination plane. +// 2. --coord-kubeconfig-configmap [ns/]name: read key "kubeconfig" from that +// ConfigMap on the CURRENT cluster. +// 3. --coord-kubeconfig-secret [ns/]name (default dc-failover/coord-kubeconfig, +// the name the dr-controlplane chart mints): read key "kubeconfig" from that +// Secret on the CURRENT cluster. +type CoordFlags struct { + File string + Secret string + ConfigMap string + // LeaseNS is the namespace holding the Leases on the coordination plane. + LeaseNS string +} + +// AddCoordFlags registers the coordination-plane kubeconfig flags on cmd. +func AddCoordFlags(cmd *cobra.Command, cf *CoordFlags) { + cmd.Flags().StringVar(&cf.File, "coord-kubeconfig", "", "Path to a kubeconfig file for the coordination control plane (overrides the secret/configmap sources)") + cmd.Flags().StringVar(&cf.Secret, "coord-kubeconfig-secret", DefaultCoordNamespace+"/coord-kubeconfig", "Secret ([namespace/]name, key \"kubeconfig\") on the current cluster holding the coordination-plane kubeconfig") + cmd.Flags().StringVar(&cf.ConfigMap, "coord-kubeconfig-configmap", "", "ConfigMap ([namespace/]name, key \"kubeconfig\") on the current cluster holding the coordination-plane kubeconfig") + cmd.Flags().StringVar(&cf.LeaseNS, "coord-namespace", DefaultCoordNamespace, "Namespace on the coordination plane that holds the primary-DC Leases") +} + +func splitNSName(s, defaultNS string) (ns, name string) { + if ns, name, found := strings.Cut(s, "/"); found { + return ns, name + } + return defaultNS, s +} + +// CoordClient builds a client for the coordination control plane per the CoordFlags +// resolution order. The Secret/ConfigMap sources are read through f, the CURRENT +// cluster (normally the hub). +func (cf *CoordFlags) CoordClient(ctx context.Context, f cmdutil.Factory) (kubernetes.Interface, error) { + var kubeconfigBytes []byte + switch { + case cf.File != "": + b, err := os.ReadFile(cf.File) + if err != nil { + return nil, fmt.Errorf("failed to read --coord-kubeconfig file: %w", err) + } + kubeconfigBytes = b + case cf.ConfigMap != "": + cur, err := f.KubernetesClientSet() + if err != nil { + return nil, err + } + ns, name := splitNSName(cf.ConfigMap, DefaultCoordNamespace) + cm, err := cur.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to read coordination kubeconfig ConfigMap %s/%s from the current cluster: %w", ns, name, err) + } + kubeconfigBytes = []byte(cm.Data["kubeconfig"]) + default: + cur, err := f.KubernetesClientSet() + if err != nil { + return nil, err + } + ns, name := splitNSName(cf.Secret, DefaultCoordNamespace) + sec, err := cur.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to read coordination kubeconfig Secret %s/%s from the current cluster (is the current kubeconfig pointing at the hub?): %w", ns, name, err) + } + kubeconfigBytes = sec.Data["kubeconfig"] + } + if len(kubeconfigBytes) == 0 { + return nil, fmt.Errorf("resolved an empty coordination-plane kubeconfig (expected key \"kubeconfig\")") + } + cfg, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes) + if err != nil { + return nil, fmt.Errorf("coordination-plane kubeconfig is not valid: %w", err) + } + return kubernetes.NewForConfig(cfg) +} + +// Scope is a database's failover scope resolved to its Lease name. +type Scope struct { + LeaseName string + // Source explains where the scope came from, for human output. + Source string + // MemberDCs are the data-bearing Member DCs from the PlacementPolicy, empty + // when the policy was not resolvable. + MemberDCs []string +} + +// ResolveScopeForDB mirrors the operator's scopeForDB: the PlacementPolicy's +// failoverPolicy.trigger is the source of truth (Group with a name, else Global); +// the dr.kubedb.com/failover-group annotation is consulted only when the policy +// carries no failoverPolicy; otherwise the scope is Global. +func ResolveScopeForDB(ctx context.Context, dyn dynamic.Interface, db *unstructured.Unstructured) (*Scope, error) { + ppName, _, _ := unstructured.NestedString(db.Object, "spec", "podTemplate", "spec", "podPlacementPolicy", "name") + if ppName != "" { + pp, err := dyn.Resource(PlacementGVR).Get(ctx, ppName, metav1.GetOptions{}) + if err == nil { + s := &Scope{MemberDCs: memberDCsFromPP(pp)} + trigger, found, _ := unstructured.NestedMap(pp.Object, "spec", "clusterSpreadConstraint", "failoverPolicy", "trigger") + if found { + scope, _ := trigger["scope"].(string) + group, _ := trigger["group"].(string) + if scope == "Group" && group != "" { + s.LeaseName = GroupLeasePrefix + group + s.Source = fmt.Sprintf("PlacementPolicy %s failoverPolicy trigger (Group %q)", ppName, group) + return s, nil + } + s.LeaseName = GlobalPrimaryLease + s.Source = fmt.Sprintf("PlacementPolicy %s failoverPolicy trigger (Global)", ppName) + return s, nil + } + // Policy exists but registers no failoverPolicy: back-compat annotation. + if g := db.GetAnnotations()[AnnFailoverGroup]; g != "" { + s.LeaseName = GroupLeasePrefix + g + s.Source = fmt.Sprintf("annotation %s (PlacementPolicy %s has no failoverPolicy)", AnnFailoverGroup, ppName) + return s, nil + } + s.LeaseName = GlobalPrimaryLease + s.Source = fmt.Sprintf("default Global (PlacementPolicy %s has no failoverPolicy); WARNING: this scope may not be registered, so no Lease may exist and protection may not be armed", ppName) + return s, nil + } + // The policy is referenced but unreadable: fall through to the annotation, + // but say so. + if g := db.GetAnnotations()[AnnFailoverGroup]; g != "" { + return &Scope{LeaseName: GroupLeasePrefix + g, Source: fmt.Sprintf("annotation %s (PlacementPolicy %s unreadable: %v)", AnnFailoverGroup, ppName, err)}, nil + } + return &Scope{LeaseName: GlobalPrimaryLease, Source: fmt.Sprintf("default Global (PlacementPolicy %s unreadable: %v)", ppName, err)}, nil + } + if g := db.GetAnnotations()[AnnFailoverGroup]; g != "" { + return &Scope{LeaseName: GroupLeasePrefix + g, Source: "annotation " + AnnFailoverGroup}, nil + } + return &Scope{LeaseName: GlobalPrimaryLease, Source: "default Global (no PlacementPolicy set)"}, nil +} + +func memberDCsFromPP(pp *unstructured.Unstructured) []string { + rules, _, _ := unstructured.NestedSlice(pp.Object, "spec", "clusterSpreadConstraint", "distributionRules") + var members []string + for _, r := range rules { + rm, ok := r.(map[string]any) + if !ok { + continue + } + name, _ := rm["clusterName"].(string) + indices, _ := rm["replicaIndices"].([]any) + role, _ := rm["role"].(string) + // A data-bearing Member: explicit role Member, or (older policies) any rule + // with replicaIndices. Arbiter/Witness DCs never hold the primary. + if name == "" { + continue + } + if role == "Member" || (role == "" && len(indices) > 0) { + members = append(members, name) + } + } + return members +} + +// getDB fetches the Postgres CR unstructured, resolving the namespace from the +// factory's kubeconfig flags. +func getDB(ctx context.Context, f cmdutil.Factory, name string) (*unstructured.Unstructured, dynamic.Interface, string, error) { + ns, _, err := f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil, nil, "", err + } + cfg, err := f.ToRESTConfig() + if err != nil { + return nil, nil, "", err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, nil, "", err + } + db, err := dyn.Resource(PostgresGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, nil, "", fmt.Errorf("failed to get Postgres %s/%s: %w", ns, name, err) + } + return db, dyn, ns, nil +} + +// annotateDB merge-patches one annotation onto the Postgres CR (nil value removes). +func annotateDB(ctx context.Context, dyn dynamic.Interface, ns, name, key string, value *string) error { + var v any + if value != nil { + v = *value + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%s}}}`, key, jsonValue(v)) + _, err := dyn.Resource(PostgresGVR).Namespace(ns).Patch(ctx, name, mergePatchType, []byte(patch), metav1.PatchOptions{}) + return err +} + +func jsonValue(v any) string { + if v == nil { + return "null" + } + return fmt.Sprintf("%q", v) +} + +// requireDistributed prints a loud warning when the database is not a DC-DR +// distributed one; the annotations are harmless there but will do nothing. +func requireDistributed(out func(string, ...any), db *unstructured.Unstructured) { + distributed, _, _ := unstructured.NestedBool(db.Object, "spec", "distributed") + if !distributed { + out("WARNING: %s/%s has spec.distributed=false; DC-DR commands have no effect on it\n", db.GetNamespace(), db.GetName()) + } +} + +// markerConfigMap creates or deletes a human-owned marker ConfigMap (break-glass +// override or standby-hold) in the coordination namespace of the CURRENT cluster, +// which for these markers must be the target DC's spoke. +func markerConfigMap(ctx context.Context, f cmdutil.Factory, ns, name string, remove bool) (string, error) { + cs, err := f.KubernetesClientSet() + if err != nil { + return "", err + } + if remove { + if err := cs.CoreV1().ConfigMaps(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return "", err + } + return "deleted", nil + } + _, err = cs.CoreV1().ConfigMaps(ns).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}}, metav1.CreateOptions{}) + if err != nil { + return "", err + } + return "created", nil +} diff --git a/pkg/dcdr/debug.go b/pkg/dcdr/debug.go new file mode 100644 index 000000000..ed00d81b1 --- /dev/null +++ b/pkg/dcdr/debug.go @@ -0,0 +1,504 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdDebug is the `dc-dr debug` group: per-symptom diagnosis walkers. +func NewCmdDebug(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "debug", + Short: i18n.T("Diagnose DC-DR symptoms: failover not happening, switchover stuck, fenced database"), + Long: templates.LongDesc(`Walk the DC-DR decision chain and report, in order, every condition that would stop the thing you are waiting for.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand( + newCmdDebugFailover(f), + newCmdDebugSwitchover(f), + newCmdDebugFence(f), + ) + return cmd +} + +type finding struct { + ok bool + title string + detail string + remedy string + blocker bool +} + +func (fd finding) print(w io.Writer) { + mark := "OK " + if !fd.ok { + mark = "FAIL" + if !fd.blocker { + mark = "WARN" + } + } + fmt.Fprintf(w, " [%s] %s\n", mark, fd.title) + if fd.detail != "" { + fmt.Fprintf(w, " %s\n", fd.detail) + } + if !fd.ok && fd.remedy != "" { + fmt.Fprintf(w, " -> %s\n", fd.remedy) + } +} + +// newCmdDebugFailover answers "the active DC is gone / my database is down and +// nothing failed over". +func newCmdDebugFailover(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + cmd := &cobra.Command{ + Use: "failover DB_NAME", + Short: i18n.T("Diagnose why a cross-DC failover is not happening"), + Long: templates.LongDesc(` + Walks every gate between "something is wrong" and "another data center is + primary", reporting which one is holding: + + 1. is the scope registered at all (no PlacementPolicy failoverPolicy + means no Lease, so nothing can ever move); + 2. does the Lease exist, who holds it, and is it still being renewed + (a renewed Lease means the holder's agent is alive: by design NO + database-level condition, client errors, QPS, lag, or a crashed + postgres, ever moves it); + 3. is a break-glass pin or a standby-hold blocking the move; + 4. is the RPO budget holding the promotion (the accept remedy); + 5. are there stale or failed ForceFailOver ops, or a tripped retry cap, + that make the hub skip evaluation. + + KUBECONFIG: the hub cluster; the coordination plane via --coord-*.`), + Example: templates.Examples(` + kubectl dba dc-dr debug failover pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + out := cmd.OutOrStdout() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + fmt.Fprintf(out, "Diagnosing failover for %s/%s\n\n", ns, args[0]) + + var findings []finding + + // 1. scope registration + distributed, _, _ := unstructured.NestedBool(db.Object, "spec", "distributed") + dcdrOn := db.GetAnnotations()["dr.kubedb.com/enabled"] == "true" + switch { + case !distributed: + findings = append(findings, finding{title: "database is DC-DR distributed", detail: "spec.distributed is false", remedy: "DC-DR does not apply to this database", blocker: true}) + case !dcdrOn: + findings = append(findings, finding{title: "DC-DR is armed on the database", detail: "annotation dr.kubedb.com/enabled is not \"true\"", remedy: "set dr.kubedb.com/enabled=true; without it the per-DC substrate and the coordinator fence are not configured", blocker: true}) + default: + findings = append(findings, finding{ok: true, title: "database is DC-DR distributed and armed"}) + } + if strings.Contains(scope.Source, "WARNING") { + findings = append(findings, finding{title: "failover scope is registered", detail: scope.Source, remedy: "register the scope in the PlacementPolicy (clusterSpreadConstraint.failoverPolicy.trigger); an unregistered scope has no Lease and no protection", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("failover scope resolves to %s", scope.LeaseName), detail: scope.Source}) + } + + // 2. the Lease itself + coord, cerr := cf.CoordClient(ctx, f) + if cerr != nil { + findings = append(findings, finding{title: "coordination plane reachable", detail: cerr.Error(), remedy: "pass --coord-kubeconfig / --coord-kubeconfig-secret; without the Lease no diagnosis of the authority is possible", blocker: true}) + printFindings(out, findings) + return nil + } + lease, lerr := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if lerr != nil { + findings = append(findings, finding{title: "primary-DC Lease exists", detail: lerr.Error(), remedy: "the topology controller creates it from the PlacementPolicy; check the dr-controlplane topology deployment", blocker: true}) + printFindings(out, findings) + return nil + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + dur := int32(45) + if lease.Spec.LeaseDurationSeconds != nil { + dur = *lease.Spec.LeaseDurationSeconds + } + var age time.Duration + if lease.Spec.RenewTime != nil { + age = time.Since(lease.Spec.RenewTime.Time).Round(time.Second) + } + switch { + case holder == "": + findings = append(findings, finding{ok: true, title: "Lease is unheld", detail: "no data center currently holds it; the first healthy Member to contend will acquire it"}) + case age <= time.Duration(dur)*time.Second: + findings = append(findings, finding{ + title: fmt.Sprintf("holder %q is renewing normally, so the authority will NOT move on its own", holder), + detail: fmt.Sprintf("the Lease was renewed %s ago, inside its %ds duration: that data center's agent is alive and healthy", age, dur), + remedy: "this is by design: no database-level condition (client errors, QPS, lag, a crashed postgres) ever moves the Lease. If the DATABASE is down but the DC is alive, either let the DC's own raft promote a local peer, or move the scope deliberately: kubectl dba dc-dr handoff " + args[0] + " -n " + ns + " --to --yes", + }) + default: + findings = append(findings, finding{ok: true, title: fmt.Sprintf("Lease is EXPIRED (holder %q last renewed %s ago, duration %ds)", holder, age, dur), detail: "a healthy Member DC should acquire it within one retry tick"}) + } + + // 3. pins + if pin := lease.Annotations["dr.open-cluster-management.io/override-hold"]; pin != "" { + findings = append(findings, finding{title: "no break-glass pin is blocking the move", detail: fmt.Sprintf("scope is PINNED to %q", pin), remedy: "remove that DC's override ConfigMap: kubectl dba dc-dr pin-primary --scope " + scope.LeaseName + " --remove --yes (run against that DC's spoke)", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: "no break-glass pin on the Lease"}) + } + if ho := lease.Annotations[AnnLeaseHandoffTo]; ho != "" { + findings = append(findings, finding{title: "no handoff is stuck in flight", detail: fmt.Sprintf("handoff-to=%q is still set, so the target has not acquired yet", ho), remedy: "if the target is standby-held it will never take it: kubectl dba dc-dr pin-standby --scope " + scope.LeaseName + " --remove --yes (against that DC's spoke)"}) + } + findings = append(findings, checkStandbyHolds(ctx, coord, cf.LeaseNS, scope, holder)...) + + // 4. the RPO budget + protected, protSet, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") + protMsg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") + if protSet && !protected { + findings = append(findings, finding{ + title: "promotion is not held by the RPO budget", + detail: fmt.Sprintf("protected=false: %s", protMsg), + remedy: "if this is a real failover and the loss is acceptable: kubectl dba dc-dr accept-data-loss " + args[0] + " -n " + ns + " --yes", + }) + } else if protSet { + findings = append(findings, finding{ok: true, title: "protection is confirmed (RPO budget satisfied)"}) + } + if db.GetAnnotations()[AnnAcceptDataLoss] != "" { + findings = append(findings, finding{ok: true, title: "a data-loss acceptance is currently set", detail: "the budget is bypassed for the promotion it authorizes; the operator removes it once that failover lands"}) + } + + // 5. ops objects and conditions + findings = append(findings, checkFailoverOps(ctx, f, ns, args[0])...) + findings = append(findings, checkConditions(db)...) + + printFindings(out, findings) + fmt.Fprintf(out, "\nAlso useful:\n") + fmt.Fprintf(out, " kubectl dba dc-dr status %s -n %s\n", args[0], ns) + fmt.Fprintf(out, " kubectl dba dc-dr active-dc %s -n %s\n", args[0], ns) + return nil + }, + } + AddCoordFlags(cmd, &cf) + return cmd +} + +// checkStandbyHolds reports Member DCs whose health Lease is stale, which is the +// only cross-DC-visible hint that a DC cannot take over. The standby-hold marker +// itself is spoke-local and deliberately invisible from here. +func checkStandbyHolds(ctx context.Context, coord kubernetes.Interface, ns string, scope *Scope, holder string) []finding { + members := scope.MemberDCs + if len(members) == 0 { + return nil + } + var out []finding + for _, dc := range members { + if dc == holder { + continue + } + hl, err := coord.CoordinationV1().Leases(ns).Get(ctx, "dc-health-"+dc, metav1.GetOptions{}) + if err != nil { + out = append(out, finding{title: fmt.Sprintf("candidate DC %q is reporting health", dc), detail: err.Error(), remedy: "a DC with no health Lease has no running agent; it cannot acquire the primary role"}) + continue + } + if hl.Spec.RenewTime != nil { + age := time.Since(hl.Spec.RenewTime.Time).Round(time.Second) + dur := int32(15) + if hl.Spec.LeaseDurationSeconds != nil { + dur = *hl.Spec.LeaseDurationSeconds + } + if age > 3*time.Duration(dur)*time.Second { + out = append(out, finding{title: fmt.Sprintf("candidate DC %q is healthy", dc), detail: fmt.Sprintf("its health Lease is stale (%s old)", age), remedy: "that DC's agent is down or cannot reach the coordination plane; it cannot take over until it returns"}) + continue + } + out = append(out, finding{ok: true, title: fmt.Sprintf("candidate DC %q is alive (health renewed %s ago)", dc, age), detail: "if it still refuses to promote, check for a standby-hold ConfigMap on ITS spoke: kubectl -n " + ns + " get cm " + scope.LeaseName + StandbyHoldCMSuffix}) + } + } + return out +} + +func checkFailoverOps(ctx context.Context, f cmdutil.Factory, ns, dbName string) []finding { + cfg, err := f.ToRESTConfig() + if err != nil { + return nil + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil + } + list, err := dyn.Resource(PgOpsGVR).Namespace(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil + } + var stale, failed, progressing []string + for i := range list.Items { + o := &list.Items[i] + dbRef, _, _ := unstructured.NestedString(o.Object, "spec", "databaseRef", "name") + typ, _, _ := unstructured.NestedString(o.Object, "spec", "type") + if dbRef != dbName || typ != "ForceFailOver" { + continue + } + phase, _, _ := unstructured.NestedString(o.Object, "status", "phase") + switch phase { + case "Failed": + failed = append(failed, o.GetName()) + case "Skipped", "Successful": + if time.Since(o.GetCreationTimestamp().Time) > time.Hour { + stale = append(stale, o.GetName()) + } + case "Progressing", "": + progressing = append(progressing, o.GetName()) + } + } + var out []finding + if len(progressing) > 0 { + out = append(out, finding{ok: true, title: "a ForceFailOver is in progress", detail: strings.Join(progressing, ", ")}) + } + if len(failed) > 0 { + out = append(out, finding{title: "no failed ForceFailOver ops are blocking", detail: "failed: " + strings.Join(failed, ", "), remedy: "read their status for the real cause, then delete them; repeated failures trip the retry cap and the hub stops minting new ones"}) + } + if len(stale) > 0 { + out = append(out, finding{title: "no stale ForceFailOver ops are confusing the hub", detail: "old and completed: " + strings.Join(stale, ", "), remedy: "delete them; a stale op targeting the same DC makes the hub read \"already promoted\" and skip evaluation entirely"}) + } + if len(out) == 0 { + out = append(out, finding{ok: true, title: "no stale or failed ForceFailOver ops"}) + } + return out +} + +func checkConditions(db *unstructured.Unstructured) []finding { + conds, _, _ := unstructured.NestedSlice(db.Object, "status", "conditions") + var out []finding + for _, c := range conds { + cm, ok := c.(map[string]any) + if !ok { + continue + } + typ, _ := cm["type"].(string) + status, _ := cm["status"].(string) + msg, _ := cm["message"].(string) + switch typ { + case "ForceFailOverRetryCapReached": + if status == "True" { + out = append(out, finding{title: "the ForceFailOver retry cap is not tripped", detail: msg, remedy: "the hub has stopped minting new failover ops after repeated failures. Fix the underlying failure, delete the failed ops, and it resumes", blocker: true}) + } + case "DCDRPromotionStalled": + if status == "True" { + out = append(out, finding{title: "no stalled promotion is reported", detail: msg, remedy: "the holder cannot promote; check the coordinator logs on that DC's leader pod"}) + } + case "DCDRFailoverScopeShared": + if status == "True" { + out = append(out, finding{ok: true, title: "NOTE: this scope is SHARED", detail: msg + " (every database in it fails over together)"}) + } + } + } + return out +} + +func printFindings(w io.Writer, fs []finding) { + blocking := 0 + for _, fd := range fs { + fd.print(w) + if !fd.ok && fd.blocker { + blocking++ + } + } + fmt.Fprintf(w, "\n%d blocking condition(s) found.\n", blocking) +} + +// newCmdDebugSwitchover explains a planned switchover that will not complete. +func newCmdDebugSwitchover(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "switchover DB_NAME", + Short: i18n.T("Diagnose a planned switchover that is not completing"), + Long: templates.LongDesc(` + Reports which of the switchover's gates is holding and why, including the + one that surprises people most: every gate measures by dialing the ACTIVE + primary, so a switchover cannot proceed while that primary is unreachable. + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(`kubectl dba dc-dr debug switchover pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + out := cmd.OutOrStdout() + db, _, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + ann := db.GetAnnotations() + target := ann[AnnSwitchoverTo] + if target == "" && ann[AnnQuiesceActive] == "" { + fmt.Fprintf(out, "No switchover is in flight on %s/%s.\n", ns, args[0]) + fmt.Fprintf(out, "Start one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) + return nil + } + var findings []finding + if ann[AnnSwitchoverAbort] != "" { + findings = append(findings, finding{title: "no abort is pending", detail: "the abort annotation is set; the hub is unwinding this switchover", remedy: "wait for the annotations to clear, then retry the switchover"}) + } + if started := ann[AnnSwitchoverStart]; started != "" { + if t, perr := time.Parse(time.RFC3339, started); perr == nil { + el := time.Since(t).Round(time.Second) + fd := finding{ok: true, title: fmt.Sprintf("switchover started %s ago", el)} + if el > 10*time.Minute { + fd = finding{title: "switchover is within its timeout", detail: fmt.Sprintf("running for %s, past the default 10m", el), remedy: "it auto-aborts and restores writes to the original active DC; watch for the annotations to clear"} + } + findings = append(findings, fd) + } + } + activeDC, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + dcs, _, _ := unstructured.NestedSlice(db.Object, "status", "disasterRecovery", "dataCenters") + var targetFound bool + for _, d := range dcs { + dm, ok := d.(map[string]any) + if !ok { + continue + } + name, _ := dm["clusterName"].(string) + if name == target { + targetFound = true + healthy, _ := dm["healthy"].(bool) + if !healthy { + findings = append(findings, finding{title: fmt.Sprintf("target %q is healthy", target), detail: "its health Lease is not fresh", remedy: "the switchover refuses an unhealthy target; fix that DC's agent first", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("target %q is healthy", target)}) + } + if lag, ok := toInt64(dm["lagBytes"]); ok { + if lag > zeroRPOLagBytes { + findings = append(findings, finding{title: "target has caught up to the frozen LSN", detail: fmt.Sprintf("lag is %d bytes, needs <= %d", lag, zeroRPOLagBytes), remedy: "this resolves itself once the quiesce freezes the primary and the target replays; if it never shrinks, the target is not streaming"}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("target lag is %d bytes, within the zero-RPO tolerance", lag)}) + } + } else { + findings = append(findings, finding{title: "target lag is known", detail: "no lagBytes reported for the target", remedy: "the hub measures lag by dialing the ACTIVE primary; an unreachable primary makes this permanently unknown and the switchover can never start. Use the failover path instead: dc-dr handoff", blocker: true}) + } + } + if name == activeDC { + if w, ok := dm["writable"].(bool); ok && !w { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("quiesce is IN EFFECT on %q (write-locked)", activeDC)}) + } else if ann[AnnQuiesceActive] == "true" { + findings = append(findings, finding{title: fmt.Sprintf("quiesce has taken effect on %q", activeDC), detail: "the quiesce was requested but the active DC still reads writable", remedy: "the write-lock is confirmed by dialing the active primary; if that primary is down or unreachable this never flips and the switchover stalls. Abort (dc-dr abort) and use dc-dr handoff instead", blocker: true}) + } + } + } + if target != "" && !targetFound { + findings = append(findings, finding{title: fmt.Sprintf("target %q is a known data center", target), detail: "it is not present in status.disasterRecovery.dataCenters", remedy: "check the target name against the PlacementPolicy's Member DCs", blocker: true}) + } + printFindings(out, findings) + fmt.Fprintf(out, "\n kubectl dba dc-dr status %s -n %s\n", args[0], ns) + fmt.Fprintf(out, " kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + return nil + }, + } + return cmd +} + +// newCmdDebugFence explains a database that is up but refusing writes. +func newCmdDebugFence(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + cmd := &cobra.Command{ + Use: "fence DB_NAME", + Short: i18n.T("Diagnose a database whose primary is fenced read-only"), + Long: templates.LongDesc(` + A DC-DR database goes read-only by design when its local marker is + missing, stale past its TTL, or names another data center, so that at most + one data center is ever writable. This reports which of those applies, + whether the authority itself is healthy, and what to do when the + coordination plane is the thing that is broken. + + KUBECONFIG: the hub cluster; the coordination plane via --coord-*.`), + Example: templates.Examples(`kubectl dba dc-dr debug fence pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + out := cmd.OutOrStdout() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + var findings []finding + coord, cerr := cf.CoordClient(ctx, f) + if cerr != nil { + findings = append(findings, finding{title: "coordination plane reachable", detail: cerr.Error(), remedy: "if the coordination plane is genuinely down, every DC fences read-only after the marker TTL plus its uncertainty hold. To keep the current primary writable through the outage, pin it: kubectl dba dc-dr pin-primary --scope " + scope.LeaseName + " --yes (run against that DC's spoke)", blocker: true}) + printFindings(out, findings) + return nil + } + lease, lerr := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if lerr != nil { + findings = append(findings, finding{title: "primary-DC Lease exists", detail: lerr.Error(), remedy: "with no Lease there is no marker to project, so every DC fences. Check the dr-controlplane topology controller", blocker: true}) + printFindings(out, findings) + return nil + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if holder == "" { + findings = append(findings, finding{title: "the scope has an active data center", detail: "the Lease is currently unheld, so every DC's marker says nobody is active and all of them fence", remedy: "a healthy Member acquires within a retry tick; if none does, check that at least one DC's agent is running", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("the authority says %q is active", holder)}) + } + if lease.Spec.RenewTime != nil { + age := time.Since(lease.Spec.RenewTime.Time).Round(time.Second) + if age > 30*time.Second { + findings = append(findings, finding{title: "the authority is being renewed", detail: fmt.Sprintf("last renewed %s ago; the marker projected onto each spoke goes stale after 30s and the fence closes fail-closed", age), remedy: "the holder's agent cannot write to the coordination plane. Fix that, or pin the primary to ride out the outage: kubectl dba dc-dr pin-primary --scope " + scope.LeaseName + " --yes", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("the authority is fresh (renewed %s ago)", age)}) + } + } + activeDC, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + if activeDC != "" && holder != "" && activeDC != holder { + findings = append(findings, finding{title: "the database agrees with the authority", detail: fmt.Sprintf("status says %q, the Lease says %q", activeDC, holder), remedy: "the status trails the Lease by a reconcile; if it persists, the hub operator is not reconciling this database"}) + } + fmt.Fprintf(out, "Fence diagnosis for %s/%s (scope %s)\n\n", ns, args[0], scope.LeaseName) + printFindings(out, findings) + fmt.Fprintf(out, "\nThe marker each data center actually reads lives on its OWN spoke:\n") + fmt.Fprintf(out, " kubectl --kubeconfig -n %s get cm %s -o jsonpath='{.data}'\n", cf.LeaseNS, scope.LeaseName) + return nil + }, + } + AddCoordFlags(cmd, &cf) + return cmd +} diff --git a/pkg/dcdr/handoff.go b/pkg/dcdr/handoff.go new file mode 100644 index 000000000..fb1ed4e7e --- /dev/null +++ b/pkg/dcdr/handoff.go @@ -0,0 +1,143 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdHandoff moves the failover authority by annotating the Lease itself. Acts +// on the COORDINATION plane (and on the hub only to resolve a database's scope). +func NewCmdHandoff(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + var leaseName, to string + var yes bool + cmd := &cobra.Command{ + Use: "handoff (DB_NAME | --lease NAME) --to DC", + Short: i18n.T("Move the failover authority for a scope by handing off its primary-DC Lease"), + Long: templates.LongDesc(` + Writes dr.open-cluster-management.io/handoff-to on the scope's primary-DC + Lease. The holding data center's agent releases the Lease once, the target + acquires it within a retry tick, and the annotation clears itself. + + This is the scope-local FAILOVER lever, and the correct tool when the + active data center's database is down but its data center is alive: no + quiesce and no catch-up wait happen, so loss is bounded by the RPO budget + rather than zero. For a healthy primary prefer "dc-dr switchover", which + is zero-RPO. + + It moves EVERY database sharing the scope. Do NOT stop a DC's agent to + force a failover instead: one agent serves every scope its DC holds, so + that expires all of them together. + + KUBECONFIG: the hub cluster (to resolve a database's scope and read the + coordination kubeconfig Secret). The Lease is written on the coordination + plane via the --coord-* flags.`), + Example: templates.Examples(` + # Fail a database's scope over to dc-b + kubectl dba dc-dr handoff pg-dcdr -n demo --to dc-b --yes + + # Move a scope by Lease name (works with no database left) + kubectl dba dc-dr handoff --lease primary-dc-orders --to dc-a --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if (len(args) == 0) == (leaseName == "") { + return fmt.Errorf("give exactly one of: a database name, or --lease") + } + if to == "" { + return fmt.Errorf("--to is required (the target data center)") + } + ctx := context.Background() + out := cmd.OutOrStdout() + scope := &Scope{LeaseName: leaseName, Source: "--lease flag"} + if len(args) == 1 { + db, dyn, _, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err = ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + if len(scope.MemberDCs) > 0 && !slices.Contains(scope.MemberDCs, to) { + return fmt.Errorf("%q is not a Member data center of this database (members: %v)", to, scope.MemberDCs) + } + } + coord, err := cf.CoordClient(ctx, f) + if err != nil { + return err + } + lease, err := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read Lease %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if holder == to { + fmt.Fprintf(out, "No-op: %s already holds %s.\n", to, scope.LeaseName) + return nil + } + if members := lease.Annotations[AnnLeaseMemberDCs]; members != "" { + if !slices.Contains(strings.Split(members, ","), to) { + return fmt.Errorf("%q is not listed in the Lease's Member DCs (%s); only a Member can hold the primary role", to, members) + } + } + if pin := lease.Annotations["dr.open-cluster-management.io/override-hold"]; pin != "" && pin != to { + return fmt.Errorf("scope %s is PINNED to %q by a break-glass override; remove that DC's override ConfigMap first (kubectl dba dc-dr pin-primary --remove), or the handoff cannot complete", scope.LeaseName, pin) + } + if !yes { + fmt.Fprintf(out, "Would move %s from %s to %s. Every database in this scope fails over together, without a quiesce (loss bounded by the RPO budget, not zero).\n", scope.LeaseName, orNone(holder), to) + return fmt.Errorf("re-run with --yes to proceed") + } + // Re-requesting the SAME target must still fire an agent-visible event. + // A merge patch that writes an identical value does not bump the + // resourceVersion, so the informers see nothing and the handoff never + // starts: exactly what happens when a target that was standby-held is + // released and the pending handoff has to be re-driven. Clear first, then + // set, so there is always a real transition to observe. + if lease.Annotations[AnnLeaseHandoffTo] == to { + clear := fmt.Sprintf(`{"metadata":{"annotations":{%q:null}}}`, AnnLeaseHandoffTo) + if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, scope.LeaseName, types.MergePatchType, []byte(clear), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to clear the stale handoff annotation on %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) + } + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%q}}}`, AnnLeaseHandoffTo, to) + if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, scope.LeaseName, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to annotate Lease %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) + } + fmt.Fprintf(out, "Handoff of %s requested: %s -> %s.\n", scope.LeaseName, orNone(holder), to) + fmt.Fprintf(out, "The holder releases within seconds and %s acquires on its next retry tick; the annotation clears itself.\n", to) + fmt.Fprintf(out, "Verify: kubectl dba dc-dr active-dc --lease %s\n", scope.LeaseName) + fmt.Fprintf(out, "NOTE: if the target is held by a standby-hold ConfigMap it will refuse; clear it with dc-dr pin-standby --remove.\n") + return nil + }, + } + cmd.Flags().StringVar(&leaseName, "lease", "", "Act on this Lease directly instead of resolving a database's scope") + cmd.Flags().StringVar(&to, "to", "", "Target data center") + cmd.Flags().BoolVar(&yes, "yes", false, "Confirm the handoff (it moves every database in the scope)") + AddCoordFlags(cmd, &cf) + return cmd +} diff --git a/pkg/dcdr/pin.go b/pkg/dcdr/pin.go new file mode 100644 index 000000000..a96b376f6 --- /dev/null +++ b/pkg/dcdr/pin.go @@ -0,0 +1,180 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +type pinKind int + +const ( + pinPrimary pinKind = iota + pinStandby +) + +// NewCmdPin builds either pin-primary (break-glass override: this DC stays primary +// and stays writable even with the control plane gone) or pin-standby +// (standby-hold: this DC never promotes). +// +// Both act on a SPOKE cluster: the marker ConfigMap is human-owned and lives on the +// data center it governs, which is exactly why it still works when the hub is +// unreachable. So the ordinary kubeconfig flags must point at that DC's spoke. +func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { + var scopeName, dbName string + var remove, yes bool + var cf CoordFlags + + use, short, long, example := pinPrimaryTexts() + suffix := OverrideCMSuffix + if kind == pinStandby { + use, short, long, example = pinStandbyTexts() + suffix = StandbyHoldCMSuffix + } + + cmd := &cobra.Command{ + Use: use, + Short: i18n.T(short), + Long: templates.LongDesc(long), + Example: templates.Examples(example), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if (scopeName == "") == (dbName == "") { + return fmt.Errorf("give exactly one of --scope (a primary-DC Lease name) or --db (resolve the scope from a database)") + } + ctx := context.Background() + out := cmd.OutOrStdout() + lease := scopeName + if dbName != "" { + db, dyn, _, err := getDB(ctx, f, dbName) + if err != nil { + return fmt.Errorf("resolving --db (note: with --db the kubeconfig must reach the hub, while creating the marker needs the SPOKE; pass --scope when running against a spoke): %w", err) + } + s, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + lease = s.LeaseName + fmt.Fprintf(out, "Resolved scope %s from database %s (%s).\n", lease, dbName, s.Source) + } + cmName := lease + suffix + if !remove && !yes { + if kind == pinPrimary { + fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) + fmt.Fprintf(out, "This is a STANDING BYPASS of split-brain protection: the local leader stays writable regardless of what the failover authority says, and the scope cannot fail over while it exists.\n") + } else { + fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster; this data center will never promote while it exists.\n", cf.LeaseNS, cmName) + } + return fmt.Errorf("re-run with --yes to proceed (or --remove --yes to clear an existing pin)") + } + if remove && !yes { + fmt.Fprintf(out, "Would delete ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) + return fmt.Errorf("re-run with --yes to proceed") + } + action, err := markerConfigMap(ctx, f, cf.LeaseNS, cmName, remove) + if err != nil { + return err + } + fmt.Fprintf(out, "ConfigMap %s/%s %s on the current cluster.\n", cf.LeaseNS, cmName, action) + if kind == pinPrimary { + if remove { + fmt.Fprintf(out, "The break-glass pin is cleared; normal contention resumes and the override-hold annotation drops from the Lease within seconds.\n") + } else { + fmt.Fprintf(out, "This data center is now PINNED primary for scope %s:\n", lease) + fmt.Fprintf(out, " - its agent mirrors the pin onto the Lease, so no other Member contends;\n") + fmt.Fprintf(out, " - its coordinator keeps the local leader writable even if the marker goes stale (a control-plane outage no longer fences it).\n") + fmt.Fprintf(out, " IMPORTANT: only honored on the scope's LAST KNOWN HOLDER. On any other DC the agent refuses it and logs why; it never promotes a standby.\n") + fmt.Fprintf(out, " IMPORTANT: while pinned, that DC dying means NO failover happens. Remove the pin as soon as the emergency ends:\n") + fmt.Fprintf(out, " kubectl dba dc-dr pin-primary --scope %s --remove --yes\n", lease) + } + } else { + if remove { + fmt.Fprintf(out, "The standby-hold is cleared. NOTE: this DC resumes contending on the NEXT Lease event; if the Lease is idle, a pending handoff may take until the agent's informer resync. Touching the Lease (for example dc-dr handoff --to ) makes it immediate.\n") + } else { + fmt.Fprintf(out, "This data center is now HELD as a standby for scope %s: it never contends for the Lease, never promotes, and refuses destructive cross-DC rewinds of its data.\n", lease) + fmt.Fprintf(out, " It is ignored while this DC is the ACTIVE one (demoting the active DC without a quiesce is unsafe): move the primary away with a switchover first.\n") + fmt.Fprintf(out, " Remove with: kubectl dba dc-dr pin-standby --scope %s --remove --yes\n", lease) + } + } + return nil + }, + } + cmd.Flags().StringVar(&scopeName, "scope", "", "Primary-DC Lease name of the scope (for example primary-dc or primary-dc-orders)") + cmd.Flags().StringVar(&dbName, "db", "", "Resolve the scope from this database instead (requires the kubeconfig to reach the hub)") + cmd.Flags().BoolVar(&remove, "remove", false, "Remove the pin instead of creating it") + cmd.Flags().BoolVar(&yes, "yes", false, "Confirm") + // Only the marker namespace is meaningful here; the pin never touches the + // coordination plane itself. + cmd.Flags().StringVar(&cf.LeaseNS, "coord-namespace", DefaultCoordNamespace, "Namespace on this spoke that holds the marker ConfigMaps") + cmd.Flags().VisitAll(func(fl *pflag.Flag) {}) + return cmd +} + +func pinPrimaryTexts() (use, short, long, example string) { + return "pin-primary (--scope LEASE | --db DB_NAME)", + "Pin this data center as primary (break-glass override): no failover, writable through a control-plane outage", + `Creates the human-owned break-glass override ConfigMap -override in + the coordination namespace of the CURRENT cluster, which must be the data + center you are pinning (its own spoke). + + Two effects, both live-proven: this DC's agent mirrors the pin onto the + Lease so every other Member defers permanently, and this DC's coordinator + forces its leader ACTIVE regardless of marker state, so a sustained + coordination-plane outage no longer fences it read-only after the usual + marker TTL plus uncertainty hold. + + Use it when the failover authority is unreachable and the surviving + primary must keep accepting writes, or as a deliberate "never fail this + scope over" policy. While it stands there is no split-brain protection for + the scope, and nothing takes over if this DC dies. + + KUBECONFIG: the SPOKE of the data center being pinned.`, + `# Keep dc-b primary for the global scope, come what may (run against dc-b) + kubectl dba dc-dr pin-primary --scope primary-dc --yes --kubeconfig ~/.kube/dc-b.yaml + + # Clear it once the emergency is over + kubectl dba dc-dr pin-primary --scope primary-dc --remove --yes --kubeconfig ~/.kube/dc-b.yaml` +} + +func pinStandbyTexts() (use, short, long, example string) { + return "pin-standby (--scope LEASE | --db DB_NAME)", + "Pin this data center as a standby (standby-hold): it never promotes", + `Creates the human-owned standby-hold ConfigMap -standby-hold in the + coordination namespace of the CURRENT cluster, which must be the data + center you are holding down. + + While it exists that DC never contends for the scope's primary-DC Lease, + never promotes (it refuses even an explicit handoff naming it), and its + coordinator refuses destructive cross-DC rewinds or re-seeds of the data + it holds. It fails CLOSED: if the ConfigMap cannot be read the hold is + assumed, so a flaky apiserver never silently drops the protection. + + It is deliberately ignored on the data center that is currently ACTIVE, + because demoting the active DC without a quiesce is unsafe; move the + primary away with a planned switchover first. + + KUBECONFIG: the SPOKE of the data center being held.`, + `# Never let dc-a take the primary role for this scope (run against dc-a) + kubectl dba dc-dr pin-standby --scope primary-dc --yes --kubeconfig ~/.kube/dc-a.yaml + + # Release it + kubectl dba dc-dr pin-standby --scope primary-dc --remove --yes --kubeconfig ~/.kube/dc-a.yaml` +} diff --git a/pkg/dcdr/status.go b/pkg/dcdr/status.go new file mode 100644 index 000000000..688ad69d5 --- /dev/null +++ b/pkg/dcdr/status.go @@ -0,0 +1,303 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// stepState is the rendered state of one switchover step. +type stepState int + +const ( + stepDone stepState = iota + stepCurrent + stepPending + stepSkipped +) + +func (s stepState) mark() string { + switch s { + case stepDone: + return "[done] " + case stepCurrent: + return "[NOW] " + case stepSkipped: + return "[n/a] " + default: + return "[pending]" + } +} + +// NewCmdStatus renders a one-shot picture of a database's DC-DR state and, when a +// switchover is in flight, which of its steps are complete and what happens next. +// Deliberately NOT a watch: it prints once and exits, so the user re-runs it. +func NewCmdStatus(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "status DB_NAME", + Short: i18n.T("Show DC-DR state and switchover progress once (re-run to see further progress)"), + Long: templates.LongDesc(` + Prints the database's failover scope, per data center state, protection + verdict, and, when a planned switchover is in flight, its step-by-step + progress: what has completed, what is happening NOW, and what remains. + + One-shot by design: it does not follow. Run it again to see the next + state, which keeps its output readable in tickets and transcripts. + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(` + kubectl dba dc-dr status pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + out := cmd.OutOrStdout() + ann := db.GetAnnotations() + dbPhase, _, _ := unstructured.NestedString(db.Object, "status", "phase") + activeDC, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + drPhase, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "phase") + protected, protectedSet, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") + protMsg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") + + fmt.Fprintf(out, "Database: %s/%s (%s)\n", ns, args[0], dbPhase) + fmt.Fprintf(out, "Failover scope: %s\n (%s)\n", scope.LeaseName, scope.Source) + if len(scope.MemberDCs) > 0 { + fmt.Fprintf(out, "Member DCs: %s\n", strings.Join(scope.MemberDCs, ", ")) + } + fmt.Fprintf(out, "Active DC: %s DR phase: %s\n", orNone(activeDC), orNone(drPhase)) + if protectedSet { + fmt.Fprintf(out, "Protected: %v", protected) + if protMsg != "" { + fmt.Fprintf(out, " (%s)", protMsg) + } + fmt.Fprintln(out) + } + + // Per-DC table. + dcs, found, _ := unstructured.NestedSlice(db.Object, "status", "disasterRecovery", "dataCenters") + if found && len(dcs) > 0 { + fmt.Fprintf(out, "\nData centers:\n") + fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", "NAME", "ROLE", "WRITABLE", "HEALTHY", "LAG(BYTES)", "STREAMER") + for _, d := range dcs { + dm, ok := d.(map[string]any) + if !ok { + continue + } + name, _ := dm["clusterName"].(string) + role, _ := dm["role"].(string) + writable := boolCell(dm, "writable") + healthy := boolCell(dm, "healthy") + lag := "-" + if v, ok := dm["lagBytes"]; ok { + lag = fmt.Sprintf("%v", v) + } + streamer, _ := dm["crossDCStreamer"].(string) + fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", name, orNone(role), writable, healthy, lag, orNone(streamer)) + } + } + + // Switchover progress, only when one is in flight or was just requested. + target := ann[AnnSwitchoverTo] + quiesced := ann[AnnQuiesceActive] == "true" + aborting := ann[AnnSwitchoverAbort] != "" + started := ann[AnnSwitchoverStart] + if target != "" || quiesced || aborting { + fmt.Fprintf(out, "\nPlanned switchover") + if target != "" { + fmt.Fprintf(out, " to %q", target) + } + if started != "" { + if t, perr := time.Parse(time.RFC3339, started); perr == nil { + fmt.Fprintf(out, ", started %s ago", time.Since(t).Round(time.Second)) + } + } + fmt.Fprintln(out, ":") + if aborting { + fmt.Fprintf(out, " ABORT requested (%s is set). The hub clears the quiesce and restores writes to %s.\n", AnnSwitchoverAbort, orNone(activeDC)) + fmt.Fprintf(out, " Re-run this command until the switchover annotations are gone and DR phase is Steady.\n") + return nil + } + renderSwitchoverSteps(out, target, activeDC, quiesced, dcs) + fmt.Fprintf(out, "\n Abort: kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + fmt.Fprintf(out, " Re-run this command to see the next step; it does not follow.\n") + return nil + } + + fmt.Fprintf(out, "\nNo switchover in flight.\n") + if drPhase == "FailingOver" { + fmt.Fprintf(out, "DR phase is FailingOver with no switchover annotation, so this is an UNPLANNED failover.\n") + fmt.Fprintf(out, " If it is not completing: kubectl dba dc-dr debug failover %s -n %s\n", args[0], ns) + } + if protectedSet && !protected { + fmt.Fprintf(out, "Protection is NOT confirmed. If a promotion is held by the RPO budget:\n") + fmt.Fprintf(out, " kubectl dba dc-dr accept-data-loss %s -n %s --yes\n", args[0], ns) + } + fmt.Fprintf(out, "Trigger one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) + return nil + }, + } + return cmd +} + +// renderSwitchoverSteps prints the operator's switchover sequence with each step's +// state derived from the SAME status fields the operator's own gates read, so the +// output cannot drift from the real decision: +// +// 1 target validated (healthy, lag known and within the switchover budget) +// 2 quiesce requested (annotation set) +// 3 quiesce in effect (active DC's writable flipped false) +// 4 target caught up (target lagBytes <= 8 KiB) +// 5 Lease handed off (activeDC == target) +// 6 old DC demoted, annotations cleared, phase Steady +func renderSwitchoverSteps(out interface{ Write([]byte) (int, error) }, target, activeDC string, quiesced bool, dcs []any) { + var targetLag any + targetHealthy, targetKnown := false, false + activeWritable, activeWritableKnown := true, false + for _, d := range dcs { + dm, ok := d.(map[string]any) + if !ok { + continue + } + name, _ := dm["clusterName"].(string) + if name == target { + targetKnown = true + targetHealthy, _ = dm["healthy"].(bool) + targetLag = dm["lagBytes"] + } + if name == activeDC { + if w, ok := dm["writable"].(bool); ok { + activeWritable, activeWritableKnown = w, true + } + } + } + handedOff := target != "" && activeDC == target + + step1 := stepPending + switch { + case !targetKnown: + step1 = stepCurrent + case targetHealthy && targetLag != nil: + step1 = stepDone + default: + step1 = stepCurrent + } + step2 := stepPending + if quiesced { + step2 = stepDone + } + step3 := stepPending + switch { + case handedOff: + step3 = stepDone + case quiesced && activeWritableKnown && !activeWritable: + step3 = stepDone + case quiesced: + step3 = stepCurrent + } + step4 := stepPending + caughtUp := false + if lag, ok := toInt64(targetLag); ok && lag <= zeroRPOLagBytes { + caughtUp = true + } + switch { + case handedOff: + step4 = stepDone + case step3 == stepDone && caughtUp: + step4 = stepDone + case step3 == stepDone: + step4 = stepCurrent + } + step5 := stepPending + if handedOff { + step5 = stepDone + } else if step4 == stepDone { + step5 = stepCurrent + } + step6 := stepPending + if handedOff { + step6 = stepCurrent + } + + fmt.Fprintf(out, " %s 1. target %q validated: healthy and lag known, within the switchover budget\n", step1.mark(), target) + fmt.Fprintf(out, " %s 2. quiesce requested on the active DC (%s)\n", step2.mark(), orNone(activeDC)) + fmt.Fprintf(out, " %s 3. quiesce IN EFFECT: active primary write-locked, its LSN frozen\n", step3.mark()) + lagText := "unknown" + if targetLag != nil { + lagText = fmt.Sprintf("%v bytes", targetLag) + } + fmt.Fprintf(out, " %s 4. target caught up to the frozen LSN (now %s, needs <= %d)\n", step4.mark(), lagText, zeroRPOLagBytes) + fmt.Fprintf(out, " %s 5. primary-DC Lease handed off to %q\n", step5.mark(), target) + fmt.Fprintf(out, " %s 6. old DC demoted to standby, annotations cleared, DR phase back to Steady\n", step6.mark()) + + fmt.Fprintf(out, "\n NEXT: ") + switch { + case step1 == stepCurrent: + fmt.Fprintf(out, "waiting for the target's health and lag to be observable. If it never becomes healthy the switchover cannot start.\n") + case step3 == stepCurrent: + fmt.Fprintf(out, "waiting for the write-lock to take hold on %s. This needs the active primary to be UP and reachable; a dead primary can never satisfy it (use the failover path).\n", orNone(activeDC)) + case step4 == stepCurrent: + fmt.Fprintf(out, "waiting for %q to replay the last %s. This is the only step whose duration depends on your write volume.\n", target, lagText) + case step5 == stepCurrent: + fmt.Fprintf(out, "handing off the Lease; the target promotes within seconds.\n") + case step6 == stepCurrent: + fmt.Fprintf(out, "the Lease has moved to %q. The old DC self-fences and re-cascades, then the annotations clear and the phase returns to Steady.\n", target) + default: + fmt.Fprintf(out, "the operator picks it up on its next reconcile.\n") + } +} + +func toInt64(v any) (int64, bool) { + switch n := v.(type) { + case int64: + return n, true + case float64: + return int64(n), true + case int: + return int64(n), true + } + return 0, false +} + +func boolCell(m map[string]any, key string) string { + if v, ok := m[key].(bool); ok { + return fmt.Sprintf("%v", v) + } + return "-" +} + +func orNone(s string) string { + if s == "" { + return "" + } + return s +} diff --git a/pkg/dcdr/switchover.go b/pkg/dcdr/switchover.go new file mode 100644 index 000000000..3b254975f --- /dev/null +++ b/pkg/dcdr/switchover.go @@ -0,0 +1,100 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "slices" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +const mergePatchType = types.MergePatchType + +// NewCmdSwitchover triggers a planned, zero-RPO switchover by annotating the +// database. Acts on the HUB cluster (ordinary kubeconfig flags). +func NewCmdSwitchover(f cmdutil.Factory) *cobra.Command { + var to string + cmd := &cobra.Command{ + Use: "switchover DB_NAME --to DC", + Short: i18n.T("Trigger a planned zero-RPO switchover of a distributed database to another data center"), + Long: templates.LongDesc(` + Sets the dr.kubedb.com/switchover-to annotation on the Postgres. The hub + operator then quiesces the active primary (write-locked), waits for the + target data center to catch up to the frozen LSN, hands off the primary-DC + Lease, and clears the annotation. Requires the active primary to be up and + accepting connections: the safety gates measure by dialing it and fail + closed, so a dead primary cannot be switched away from (use the failover + path instead: dc-dr handoff, and dc-dr accept-data-loss if the RPO budget + holds it). + + KUBECONFIG: the hub cluster (where the Postgres CR lives).`), + Example: templates.Examples(` + # Move demo/pg-dcdr to data center dc-a, with zero data loss + kubectl dba dc-dr switchover pg-dcdr -n demo --to dc-a + + # Watch the progress (one-shot, run repeatedly) + kubectl dba dc-dr status pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + if to == "" { + return fmt.Errorf("--to is required (the target data center)") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + requireDistributed(printfErr(cmd), db) + + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + if len(scope.MemberDCs) > 0 && !slices.Contains(scope.MemberDCs, to) { + return fmt.Errorf("%q is not a Member data center of this database (members: %v); an Arbiter or Witness DC can never become primary", to, scope.MemberDCs) + } + if active, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC"); active == to { + cmd.Printf("No-op: %s is already the active data center of %s/%s.\n", to, ns, args[0]) + return nil + } + if err := annotateDB(ctx, dyn, ns, args[0], AnnSwitchoverTo, &to); err != nil { + return err + } + cmd.Printf("Switchover of %s/%s to %q requested (scope %s, from %s).\n", ns, args[0], to, scope.LeaseName, scope.Source) + cmd.Printf("The operator will quiesce, wait for catch-up, and hand off; zero committed rows are lost.\n") + cmd.Printf("Monitor: kubectl dba dc-dr status %s -n %s\n", args[0], ns) + cmd.Printf("Abort: kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + if scope.LeaseName == GlobalPrimaryLease { + cmd.Printf("NOTE: this database follows the GLOBAL scope; every database in that scope switches with it.\n") + } + return nil + }, + } + cmd.Flags().StringVar(&to, "to", "", "Target data center (must be a Member DC of the database's PlacementPolicy)") + return cmd +} + +func printfErr(cmd *cobra.Command) func(string, ...any) { + return func(format string, a ...any) { fmt.Fprintf(cmd.ErrOrStderr(), format, a...) } +} From fc6ee552da4a5a7e60d09a2b9208703735df81df Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Tue, 4 Aug 2026 10:18:38 +0600 Subject: [PATCH 2/4] dc-dr: pin-primary --remove --force clears a dead DC's stuck override-hold The override-hold annotation on the Lease is normally cleared only by the pinned DC's own agent when its override ConfigMap disappears. If that DC dies with the pin standing, the annotation persists and every handoff refuses, which blocks the deliberate promote-the-survivor runbook exactly when it is needed. --remove --force, run from any live cluster with the --coord-* flags, tolerates the failed ConfigMap delete and clears the annotation directly on the Lease. Refuses to run without a reachable coordination plane. Live-proven in the bank two-site dc-loss drill: handoff refused while the pin stood, force cleared it, dr acquired and promoted cleanly. Signed-off-by: Tamal Saha --- docs/CLI.md | 28 ++++++++++++++++++++++++++++ pkg/dcdr/pin.go | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 23306699d..3f56b8ba2 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -216,6 +216,8 @@ coordination-plane outage. # run against the SPOKE of the DC being pinned kubectl dba dc-dr pin-primary (--scope LEASE | --db DB) --yes kubectl dba dc-dr pin-primary --scope LEASE --remove --yes +# the pinned DC is DEAD and its pin is stuck: run from any live cluster +kubectl dba dc-dr pin-primary --scope LEASE --remove --force --yes --coord-kubeconfig coord.yaml ``` Creates the human-owned `-override` ConfigMap on that DC's spoke, which is @@ -255,6 +257,32 @@ it stands there is no split-brain protection for the scope and nothing takes ove that DC dies; and on control-plane recovery the recreated Lease is claimed first-come, so keep the pin until the authority is back with the right holder, then remove it. +### --remove --force: the pinned DC died with its pin standing + +The override-hold annotation on the Lease is normally cleared only by the pinned DC's +own agent (when it sees its ConfigMap gone). If that DC is dead, the annotation stays +and every handoff refuses. `--remove --force` breaks that deadlock: run it from **any +live cluster** with the `--coord-*` flags; the ConfigMap delete on the current cluster +is attempted, a failure is reported and skipped, and the `override-hold` annotation is +then cleared directly on the Lease. Live-proven in the bank dc-loss drill: + +``` +$ kubectl dba dc-dr pin-primary --scope primary-dc-bankpg --remove --force --yes \ + --coord-kubeconfig /tmp/bank-coord.yaml # from dr, dc dead +Could not remove ConfigMap dc-failover/primary-dc-bankpg-override on the current cluster +(configmaps "primary-dc-bankpg-override" not found); proceeding to clear the Lease +annotation because --force is set. +Cleared the override-hold annotation on Lease dc-failover/primary-dc-bankpg directly +(the pinned DC's agent is not alive to do it). +Members now contend normally; the surviving DC can acquire once its holds are removed. +``` + +ONLY for a dead DC. If the pinned DC is alive, its coordinator still honors the local +ConfigMap (it keeps its leader writable), so force-clearing just the annotation while +another DC promotes is a split brain. Alive DC: plain `--remove` on its spoke first. +The stale ConfigMap on the dead cluster is refused by its returning agent (it is no +longer the last known holder), but delete it during failback before re-pinning. + ## pin-standby This data center never promotes. diff --git a/pkg/dcdr/pin.go b/pkg/dcdr/pin.go index a96b376f6..9691b8541 100644 --- a/pkg/dcdr/pin.go +++ b/pkg/dcdr/pin.go @@ -18,7 +18,8 @@ import ( "fmt" "github.com/spf13/cobra" - "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" @@ -40,7 +41,7 @@ const ( // unreachable. So the ordinary kubeconfig flags must point at that DC's spoke. func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { var scopeName, dbName string - var remove, yes bool + var remove, force, yes bool var cf CoordFlags use, short, long, example := pinPrimaryTexts() @@ -91,9 +92,29 @@ func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { } action, err := markerConfigMap(ctx, f, cf.LeaseNS, cmName, remove) if err != nil { - return err + if !(remove && force) { + return err + } + // The pinned data center is dead or unreachable: exactly the situation + // --force exists for. The ConfigMap on its spoke is unreachable garbage; + // what actually blocks the failover is the override-hold annotation on + // the Lease, which only that DC's own (dead) agent would ever clear. + fmt.Fprintf(out, "Could not remove ConfigMap %s/%s on the current cluster (%v); proceeding to clear the Lease annotation because --force is set.\n", cf.LeaseNS, cmName, err) + action = "unreachable, skipped" } fmt.Fprintf(out, "ConfigMap %s/%s %s on the current cluster.\n", cf.LeaseNS, cmName, action) + if kind == pinPrimary && remove && force { + coord, cerr := cf.CoordClient(ctx, f) + if cerr != nil { + return fmt.Errorf("--force needs the coordination plane to clear the override-hold annotation, but it is unreachable: %w", cerr) + } + patch := `{"metadata":{"annotations":{"dr.open-cluster-management.io/override-hold":null}}}` + if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, lease, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to clear override-hold on Lease %s/%s: %w", cf.LeaseNS, lease, err) + } + fmt.Fprintf(out, "Cleared the override-hold annotation on Lease %s/%s directly (the pinned DC's agent is not alive to do it).\n", cf.LeaseNS, lease) + fmt.Fprintf(out, "Members now contend normally; the surviving DC can acquire once its holds are removed.\n") + } if kind == pinPrimary { if remove { fmt.Fprintf(out, "The break-glass pin is cleared; normal contention resumes and the override-hold annotation drops from the Lease within seconds.\n") @@ -120,11 +141,13 @@ func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { cmd.Flags().StringVar(&scopeName, "scope", "", "Primary-DC Lease name of the scope (for example primary-dc or primary-dc-orders)") cmd.Flags().StringVar(&dbName, "db", "", "Resolve the scope from this database instead (requires the kubeconfig to reach the hub)") cmd.Flags().BoolVar(&remove, "remove", false, "Remove the pin instead of creating it") + if kind == pinPrimary { + cmd.Flags().BoolVar(&force, "force", false, "With --remove: also clear the override-hold annotation directly on the Lease, for when the pinned DC is dead and its agent cannot clear it (run against any live cluster; needs the --coord-* flags to reach the coordination plane)") + AddCoordFlags(cmd, &cf) + } else { + cmd.Flags().StringVar(&cf.LeaseNS, "coord-namespace", DefaultCoordNamespace, "Namespace on this spoke that holds the marker ConfigMaps") + } cmd.Flags().BoolVar(&yes, "yes", false, "Confirm") - // Only the marker namespace is meaningful here; the pin never touches the - // coordination plane itself. - cmd.Flags().StringVar(&cf.LeaseNS, "coord-namespace", DefaultCoordNamespace, "Namespace on this spoke that holds the marker ConfigMaps") - cmd.Flags().VisitAll(func(fl *pflag.Flag) {}) return cmd } From 36168ea09f2211792009ccb6050eb9d3100effbe Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Tue, 4 Aug 2026 10:29:11 +0600 Subject: [PATCH 3/4] dc-dr: pin-standby timing caveat, hold refuses the post-failback rewind Signed-off-by: Tamal Saha --- docs/CLI.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 3f56b8ba2..1d224d722 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -280,8 +280,13 @@ Members now contend normally; the surviving DC can acquire once its holds are re ONLY for a dead DC. If the pinned DC is alive, its coordinator still honors the local ConfigMap (it keeps its leader writable), so force-clearing just the annotation while another DC promotes is a split brain. Alive DC: plain `--remove` on its spoke first. -The stale ConfigMap on the dead cluster is refused by its returning agent (it is no -longer the last known holder), but delete it during failback before re-pinning. + +The stale ConfigMap on the dead cluster is a live grenade for failback: the returning +agent refuses to mirror it (no longer the last known holder), but the returning +COORDINATOR honors it unconditionally and brings the old primary back up writable +while the other DC holds the Lease. Live-observed as a real two-primaries window in +the bank drill. Failback step 1, before restoring the database workload on the +returning DC: `pin-primary --scope LEASE --remove --yes` against that spoke. ## pin-standby @@ -307,6 +312,12 @@ contention the moment the ConfigMap disappears. Verified live, a handoff that ha vetoed for 30s completed 10 seconds after the hold was released, with no further action. +Timing caveat, live-burned in the bank failback drill: pin only a DC that is already +streaming. A DC that just lost a failover still needs its one-time rewind onto the +winner's timeline, and the hold refuses exactly that (by design, "data untouched"), +so pinning too early leaves the DC down and fenced instead of standby. Re-apply the +hold only once `pg_stat_replication` on the primary shows that DC's leader. + ## active-dc Who holds the primary role, from the authority itself. From da0a142b2b3be5c4f7602e72d217b66c47715c30 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 7 Aug 2026 19:19:41 +0600 Subject: [PATCH 4/4] dc-dr: satisfy errcheck and staticcheck Discard fmt.Fprint* return values explicitly, matching the repo's existing _, _ = convention, and simplify the pin-primary force guard per De Morgan (QF1001). Signed-off-by: Tamal Saha --- pkg/dcdr/activedc.go | 20 ++++++------ pkg/dcdr/debug.go | 30 ++++++++--------- pkg/dcdr/handoff.go | 12 +++---- pkg/dcdr/pin.go | 42 ++++++++++++------------ pkg/dcdr/status.go | 74 +++++++++++++++++++++--------------------- pkg/dcdr/switchover.go | 2 +- 6 files changed, 90 insertions(+), 90 deletions(-) diff --git a/pkg/dcdr/activedc.go b/pkg/dcdr/activedc.go index 74f06c742..988ea15ba 100644 --- a/pkg/dcdr/activedc.go +++ b/pkg/dcdr/activedc.go @@ -90,33 +90,33 @@ func NewCmdActiveDC(f cmdutil.Factory) *cobra.Command { holder = *lease.Spec.HolderIdentity } if quiet { - fmt.Fprintln(out, holder) + _, _ = fmt.Fprintln(out, holder) return nil } - fmt.Fprintf(out, "Active DC: %s\n", orNone(holder)) - fmt.Fprintf(out, "Lease: %s/%s (scope from %s)\n", cf.LeaseNS, scope.LeaseName, scope.Source) + _, _ = fmt.Fprintf(out, "Active DC: %s\n", orNone(holder)) + _, _ = fmt.Fprintf(out, "Lease: %s/%s (scope from %s)\n", cf.LeaseNS, scope.LeaseName, scope.Source) if lease.Spec.RenewTime != nil { age := time.Since(lease.Spec.RenewTime.Time).Round(time.Second) dur := int32(0) if lease.Spec.LeaseDurationSeconds != nil { dur = *lease.Spec.LeaseDurationSeconds } - fmt.Fprintf(out, "Renewed: %s ago (lease duration %ds)\n", age, dur) + _, _ = fmt.Fprintf(out, "Renewed: %s ago (lease duration %ds)\n", age, dur) if dur > 0 && age > time.Duration(dur)*time.Second { - fmt.Fprintf(out, " WARNING: the Lease is EXPIRED. Its holder stopped renewing, so a healthy Member DC may acquire it at any moment.\n") + _, _ = fmt.Fprintf(out, " WARNING: the Lease is EXPIRED. Its holder stopped renewing, so a healthy Member DC may acquire it at any moment.\n") } } if lease.Spec.LeaseTransitions != nil { - fmt.Fprintf(out, "Transitions:%d\n", *lease.Spec.LeaseTransitions) + _, _ = fmt.Fprintf(out, "Transitions:%d\n", *lease.Spec.LeaseTransitions) } if v := lease.Annotations[AnnLeaseMemberDCs]; v != "" { - fmt.Fprintf(out, "Members: %s\n", v) + _, _ = fmt.Fprintf(out, "Members: %s\n", v) } if v := lease.Annotations[AnnLeaseHandoffTo]; v != "" { - fmt.Fprintf(out, "Handoff to: %s (a coordinated handoff is in flight)\n", v) + _, _ = fmt.Fprintf(out, "Handoff to: %s (a coordinated handoff is in flight)\n", v) } if v := lease.Annotations["dr.open-cluster-management.io/override-hold"]; v != "" { - fmt.Fprintf(out, "PINNED: break-glass override holds this scope on %s; it cannot fail over until the override ConfigMap is removed from that DC's spoke.\n", v) + _, _ = fmt.Fprintf(out, "PINNED: break-glass override holds this scope on %s; it cannot fail over until the override ConfigMap is removed from that DC's spoke.\n", v) } if len(args) == 1 { // Cross-check the CR's own view, which lags the Lease by a reconcile. @@ -124,7 +124,7 @@ func NewCmdActiveDC(f cmdutil.Factory) *cobra.Command { if err == nil { crActive, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") if crActive != "" && crActive != holder { - fmt.Fprintf(out, "\nNOTE: the database %s/%s still reports activeDC=%s in its status; the Lease is the authority and the status trails it by a reconcile.\n", ns, args[0], crActive) + _, _ = fmt.Fprintf(out, "\nNOTE: the database %s/%s still reports activeDC=%s in its status; the Lease is the authority and the status trails it by a reconcile.\n", ns, args[0], crActive) } } } diff --git a/pkg/dcdr/debug.go b/pkg/dcdr/debug.go index ed00d81b1..d31a6dbe9 100644 --- a/pkg/dcdr/debug.go +++ b/pkg/dcdr/debug.go @@ -64,12 +64,12 @@ func (fd finding) print(w io.Writer) { mark = "WARN" } } - fmt.Fprintf(w, " [%s] %s\n", mark, fd.title) + _, _ = fmt.Fprintf(w, " [%s] %s\n", mark, fd.title) if fd.detail != "" { - fmt.Fprintf(w, " %s\n", fd.detail) + _, _ = fmt.Fprintf(w, " %s\n", fd.detail) } if !fd.ok && fd.remedy != "" { - fmt.Fprintf(w, " -> %s\n", fd.remedy) + _, _ = fmt.Fprintf(w, " -> %s\n", fd.remedy) } } @@ -113,7 +113,7 @@ func newCmdDebugFailover(f cmdutil.Factory) *cobra.Command { if err != nil { return err } - fmt.Fprintf(out, "Diagnosing failover for %s/%s\n\n", ns, args[0]) + _, _ = fmt.Fprintf(out, "Diagnosing failover for %s/%s\n\n", ns, args[0]) var findings []finding @@ -204,9 +204,9 @@ func newCmdDebugFailover(f cmdutil.Factory) *cobra.Command { findings = append(findings, checkConditions(db)...) printFindings(out, findings) - fmt.Fprintf(out, "\nAlso useful:\n") - fmt.Fprintf(out, " kubectl dba dc-dr status %s -n %s\n", args[0], ns) - fmt.Fprintf(out, " kubectl dba dc-dr active-dc %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, "\nAlso useful:\n") + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr status %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr active-dc %s -n %s\n", args[0], ns) return nil }, } @@ -334,7 +334,7 @@ func printFindings(w io.Writer, fs []finding) { blocking++ } } - fmt.Fprintf(w, "\n%d blocking condition(s) found.\n", blocking) + _, _ = fmt.Fprintf(w, "\n%d blocking condition(s) found.\n", blocking) } // newCmdDebugSwitchover explains a planned switchover that will not complete. @@ -363,8 +363,8 @@ func newCmdDebugSwitchover(f cmdutil.Factory) *cobra.Command { ann := db.GetAnnotations() target := ann[AnnSwitchoverTo] if target == "" && ann[AnnQuiesceActive] == "" { - fmt.Fprintf(out, "No switchover is in flight on %s/%s.\n", ns, args[0]) - fmt.Fprintf(out, "Start one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) + _, _ = fmt.Fprintf(out, "No switchover is in flight on %s/%s.\n", ns, args[0]) + _, _ = fmt.Fprintf(out, "Start one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) return nil } var findings []finding @@ -420,8 +420,8 @@ func newCmdDebugSwitchover(f cmdutil.Factory) *cobra.Command { findings = append(findings, finding{title: fmt.Sprintf("target %q is a known data center", target), detail: "it is not present in status.disasterRecovery.dataCenters", remedy: "check the target name against the PlacementPolicy's Member DCs", blocker: true}) } printFindings(out, findings) - fmt.Fprintf(out, "\n kubectl dba dc-dr status %s -n %s\n", args[0], ns) - fmt.Fprintf(out, " kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, "\n kubectl dba dc-dr status %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr abort %s -n %s\n", args[0], ns) return nil }, } @@ -492,10 +492,10 @@ func newCmdDebugFence(f cmdutil.Factory) *cobra.Command { if activeDC != "" && holder != "" && activeDC != holder { findings = append(findings, finding{title: "the database agrees with the authority", detail: fmt.Sprintf("status says %q, the Lease says %q", activeDC, holder), remedy: "the status trails the Lease by a reconcile; if it persists, the hub operator is not reconciling this database"}) } - fmt.Fprintf(out, "Fence diagnosis for %s/%s (scope %s)\n\n", ns, args[0], scope.LeaseName) + _, _ = fmt.Fprintf(out, "Fence diagnosis for %s/%s (scope %s)\n\n", ns, args[0], scope.LeaseName) printFindings(out, findings) - fmt.Fprintf(out, "\nThe marker each data center actually reads lives on its OWN spoke:\n") - fmt.Fprintf(out, " kubectl --kubeconfig -n %s get cm %s -o jsonpath='{.data}'\n", cf.LeaseNS, scope.LeaseName) + _, _ = fmt.Fprintf(out, "\nThe marker each data center actually reads lives on its OWN spoke:\n") + _, _ = fmt.Fprintf(out, " kubectl --kubeconfig -n %s get cm %s -o jsonpath='{.data}'\n", cf.LeaseNS, scope.LeaseName) return nil }, } diff --git a/pkg/dcdr/handoff.go b/pkg/dcdr/handoff.go index fb1ed4e7e..fa9e2a936 100644 --- a/pkg/dcdr/handoff.go +++ b/pkg/dcdr/handoff.go @@ -97,7 +97,7 @@ func NewCmdHandoff(f cmdutil.Factory) *cobra.Command { holder = *lease.Spec.HolderIdentity } if holder == to { - fmt.Fprintf(out, "No-op: %s already holds %s.\n", to, scope.LeaseName) + _, _ = fmt.Fprintf(out, "No-op: %s already holds %s.\n", to, scope.LeaseName) return nil } if members := lease.Annotations[AnnLeaseMemberDCs]; members != "" { @@ -109,7 +109,7 @@ func NewCmdHandoff(f cmdutil.Factory) *cobra.Command { return fmt.Errorf("scope %s is PINNED to %q by a break-glass override; remove that DC's override ConfigMap first (kubectl dba dc-dr pin-primary --remove), or the handoff cannot complete", scope.LeaseName, pin) } if !yes { - fmt.Fprintf(out, "Would move %s from %s to %s. Every database in this scope fails over together, without a quiesce (loss bounded by the RPO budget, not zero).\n", scope.LeaseName, orNone(holder), to) + _, _ = fmt.Fprintf(out, "Would move %s from %s to %s. Every database in this scope fails over together, without a quiesce (loss bounded by the RPO budget, not zero).\n", scope.LeaseName, orNone(holder), to) return fmt.Errorf("re-run with --yes to proceed") } // Re-requesting the SAME target must still fire an agent-visible event. @@ -128,10 +128,10 @@ func NewCmdHandoff(f cmdutil.Factory) *cobra.Command { if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, scope.LeaseName, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { return fmt.Errorf("failed to annotate Lease %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) } - fmt.Fprintf(out, "Handoff of %s requested: %s -> %s.\n", scope.LeaseName, orNone(holder), to) - fmt.Fprintf(out, "The holder releases within seconds and %s acquires on its next retry tick; the annotation clears itself.\n", to) - fmt.Fprintf(out, "Verify: kubectl dba dc-dr active-dc --lease %s\n", scope.LeaseName) - fmt.Fprintf(out, "NOTE: if the target is held by a standby-hold ConfigMap it will refuse; clear it with dc-dr pin-standby --remove.\n") + _, _ = fmt.Fprintf(out, "Handoff of %s requested: %s -> %s.\n", scope.LeaseName, orNone(holder), to) + _, _ = fmt.Fprintf(out, "The holder releases within seconds and %s acquires on its next retry tick; the annotation clears itself.\n", to) + _, _ = fmt.Fprintf(out, "Verify: kubectl dba dc-dr active-dc --lease %s\n", scope.LeaseName) + _, _ = fmt.Fprintf(out, "NOTE: if the target is held by a standby-hold ConfigMap it will refuse; clear it with dc-dr pin-standby --remove.\n") return nil }, } diff --git a/pkg/dcdr/pin.go b/pkg/dcdr/pin.go index 9691b8541..d9210c21e 100644 --- a/pkg/dcdr/pin.go +++ b/pkg/dcdr/pin.go @@ -74,35 +74,35 @@ func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { return err } lease = s.LeaseName - fmt.Fprintf(out, "Resolved scope %s from database %s (%s).\n", lease, dbName, s.Source) + _, _ = fmt.Fprintf(out, "Resolved scope %s from database %s (%s).\n", lease, dbName, s.Source) } cmName := lease + suffix if !remove && !yes { if kind == pinPrimary { - fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) - fmt.Fprintf(out, "This is a STANDING BYPASS of split-brain protection: the local leader stays writable regardless of what the failover authority says, and the scope cannot fail over while it exists.\n") + _, _ = fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) + _, _ = fmt.Fprintf(out, "This is a STANDING BYPASS of split-brain protection: the local leader stays writable regardless of what the failover authority says, and the scope cannot fail over while it exists.\n") } else { - fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster; this data center will never promote while it exists.\n", cf.LeaseNS, cmName) + _, _ = fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster; this data center will never promote while it exists.\n", cf.LeaseNS, cmName) } return fmt.Errorf("re-run with --yes to proceed (or --remove --yes to clear an existing pin)") } if remove && !yes { - fmt.Fprintf(out, "Would delete ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) + _, _ = fmt.Fprintf(out, "Would delete ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) return fmt.Errorf("re-run with --yes to proceed") } action, err := markerConfigMap(ctx, f, cf.LeaseNS, cmName, remove) if err != nil { - if !(remove && force) { + if !remove || !force { return err } // The pinned data center is dead or unreachable: exactly the situation // --force exists for. The ConfigMap on its spoke is unreachable garbage; // what actually blocks the failover is the override-hold annotation on // the Lease, which only that DC's own (dead) agent would ever clear. - fmt.Fprintf(out, "Could not remove ConfigMap %s/%s on the current cluster (%v); proceeding to clear the Lease annotation because --force is set.\n", cf.LeaseNS, cmName, err) + _, _ = fmt.Fprintf(out, "Could not remove ConfigMap %s/%s on the current cluster (%v); proceeding to clear the Lease annotation because --force is set.\n", cf.LeaseNS, cmName, err) action = "unreachable, skipped" } - fmt.Fprintf(out, "ConfigMap %s/%s %s on the current cluster.\n", cf.LeaseNS, cmName, action) + _, _ = fmt.Fprintf(out, "ConfigMap %s/%s %s on the current cluster.\n", cf.LeaseNS, cmName, action) if kind == pinPrimary && remove && force { coord, cerr := cf.CoordClient(ctx, f) if cerr != nil { @@ -112,27 +112,27 @@ func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, lease, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { return fmt.Errorf("failed to clear override-hold on Lease %s/%s: %w", cf.LeaseNS, lease, err) } - fmt.Fprintf(out, "Cleared the override-hold annotation on Lease %s/%s directly (the pinned DC's agent is not alive to do it).\n", cf.LeaseNS, lease) - fmt.Fprintf(out, "Members now contend normally; the surviving DC can acquire once its holds are removed.\n") + _, _ = fmt.Fprintf(out, "Cleared the override-hold annotation on Lease %s/%s directly (the pinned DC's agent is not alive to do it).\n", cf.LeaseNS, lease) + _, _ = fmt.Fprintf(out, "Members now contend normally; the surviving DC can acquire once its holds are removed.\n") } if kind == pinPrimary { if remove { - fmt.Fprintf(out, "The break-glass pin is cleared; normal contention resumes and the override-hold annotation drops from the Lease within seconds.\n") + _, _ = fmt.Fprintf(out, "The break-glass pin is cleared; normal contention resumes and the override-hold annotation drops from the Lease within seconds.\n") } else { - fmt.Fprintf(out, "This data center is now PINNED primary for scope %s:\n", lease) - fmt.Fprintf(out, " - its agent mirrors the pin onto the Lease, so no other Member contends;\n") - fmt.Fprintf(out, " - its coordinator keeps the local leader writable even if the marker goes stale (a control-plane outage no longer fences it).\n") - fmt.Fprintf(out, " IMPORTANT: only honored on the scope's LAST KNOWN HOLDER. On any other DC the agent refuses it and logs why; it never promotes a standby.\n") - fmt.Fprintf(out, " IMPORTANT: while pinned, that DC dying means NO failover happens. Remove the pin as soon as the emergency ends:\n") - fmt.Fprintf(out, " kubectl dba dc-dr pin-primary --scope %s --remove --yes\n", lease) + _, _ = fmt.Fprintf(out, "This data center is now PINNED primary for scope %s:\n", lease) + _, _ = fmt.Fprintf(out, " - its agent mirrors the pin onto the Lease, so no other Member contends;\n") + _, _ = fmt.Fprintf(out, " - its coordinator keeps the local leader writable even if the marker goes stale (a control-plane outage no longer fences it).\n") + _, _ = fmt.Fprintf(out, " IMPORTANT: only honored on the scope's LAST KNOWN HOLDER. On any other DC the agent refuses it and logs why; it never promotes a standby.\n") + _, _ = fmt.Fprintf(out, " IMPORTANT: while pinned, that DC dying means NO failover happens. Remove the pin as soon as the emergency ends:\n") + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr pin-primary --scope %s --remove --yes\n", lease) } } else { if remove { - fmt.Fprintf(out, "The standby-hold is cleared. NOTE: this DC resumes contending on the NEXT Lease event; if the Lease is idle, a pending handoff may take until the agent's informer resync. Touching the Lease (for example dc-dr handoff --to ) makes it immediate.\n") + _, _ = fmt.Fprintf(out, "The standby-hold is cleared. NOTE: this DC resumes contending on the NEXT Lease event; if the Lease is idle, a pending handoff may take until the agent's informer resync. Touching the Lease (for example dc-dr handoff --to ) makes it immediate.\n") } else { - fmt.Fprintf(out, "This data center is now HELD as a standby for scope %s: it never contends for the Lease, never promotes, and refuses destructive cross-DC rewinds of its data.\n", lease) - fmt.Fprintf(out, " It is ignored while this DC is the ACTIVE one (demoting the active DC without a quiesce is unsafe): move the primary away with a switchover first.\n") - fmt.Fprintf(out, " Remove with: kubectl dba dc-dr pin-standby --scope %s --remove --yes\n", lease) + _, _ = fmt.Fprintf(out, "This data center is now HELD as a standby for scope %s: it never contends for the Lease, never promotes, and refuses destructive cross-DC rewinds of its data.\n", lease) + _, _ = fmt.Fprintf(out, " It is ignored while this DC is the ACTIVE one (demoting the active DC without a quiesce is unsafe): move the primary away with a switchover first.\n") + _, _ = fmt.Fprintf(out, " Remove with: kubectl dba dc-dr pin-standby --scope %s --remove --yes\n", lease) } } return nil diff --git a/pkg/dcdr/status.go b/pkg/dcdr/status.go index 688ad69d5..7820ea496 100644 --- a/pkg/dcdr/status.go +++ b/pkg/dcdr/status.go @@ -89,25 +89,25 @@ func NewCmdStatus(f cmdutil.Factory) *cobra.Command { protected, protectedSet, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") protMsg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") - fmt.Fprintf(out, "Database: %s/%s (%s)\n", ns, args[0], dbPhase) - fmt.Fprintf(out, "Failover scope: %s\n (%s)\n", scope.LeaseName, scope.Source) + _, _ = fmt.Fprintf(out, "Database: %s/%s (%s)\n", ns, args[0], dbPhase) + _, _ = fmt.Fprintf(out, "Failover scope: %s\n (%s)\n", scope.LeaseName, scope.Source) if len(scope.MemberDCs) > 0 { - fmt.Fprintf(out, "Member DCs: %s\n", strings.Join(scope.MemberDCs, ", ")) + _, _ = fmt.Fprintf(out, "Member DCs: %s\n", strings.Join(scope.MemberDCs, ", ")) } - fmt.Fprintf(out, "Active DC: %s DR phase: %s\n", orNone(activeDC), orNone(drPhase)) + _, _ = fmt.Fprintf(out, "Active DC: %s DR phase: %s\n", orNone(activeDC), orNone(drPhase)) if protectedSet { - fmt.Fprintf(out, "Protected: %v", protected) + _, _ = fmt.Fprintf(out, "Protected: %v", protected) if protMsg != "" { - fmt.Fprintf(out, " (%s)", protMsg) + _, _ = fmt.Fprintf(out, " (%s)", protMsg) } - fmt.Fprintln(out) + _, _ = fmt.Fprintln(out) } // Per-DC table. dcs, found, _ := unstructured.NestedSlice(db.Object, "status", "disasterRecovery", "dataCenters") if found && len(dcs) > 0 { - fmt.Fprintf(out, "\nData centers:\n") - fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", "NAME", "ROLE", "WRITABLE", "HEALTHY", "LAG(BYTES)", "STREAMER") + _, _ = fmt.Fprintf(out, "\nData centers:\n") + _, _ = fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", "NAME", "ROLE", "WRITABLE", "HEALTHY", "LAG(BYTES)", "STREAMER") for _, d := range dcs { dm, ok := d.(map[string]any) if !ok { @@ -122,7 +122,7 @@ func NewCmdStatus(f cmdutil.Factory) *cobra.Command { lag = fmt.Sprintf("%v", v) } streamer, _ := dm["crossDCStreamer"].(string) - fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", name, orNone(role), writable, healthy, lag, orNone(streamer)) + _, _ = fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", name, orNone(role), writable, healthy, lag, orNone(streamer)) } } @@ -132,37 +132,37 @@ func NewCmdStatus(f cmdutil.Factory) *cobra.Command { aborting := ann[AnnSwitchoverAbort] != "" started := ann[AnnSwitchoverStart] if target != "" || quiesced || aborting { - fmt.Fprintf(out, "\nPlanned switchover") + _, _ = fmt.Fprintf(out, "\nPlanned switchover") if target != "" { - fmt.Fprintf(out, " to %q", target) + _, _ = fmt.Fprintf(out, " to %q", target) } if started != "" { if t, perr := time.Parse(time.RFC3339, started); perr == nil { - fmt.Fprintf(out, ", started %s ago", time.Since(t).Round(time.Second)) + _, _ = fmt.Fprintf(out, ", started %s ago", time.Since(t).Round(time.Second)) } } - fmt.Fprintln(out, ":") + _, _ = fmt.Fprintln(out, ":") if aborting { - fmt.Fprintf(out, " ABORT requested (%s is set). The hub clears the quiesce and restores writes to %s.\n", AnnSwitchoverAbort, orNone(activeDC)) - fmt.Fprintf(out, " Re-run this command until the switchover annotations are gone and DR phase is Steady.\n") + _, _ = fmt.Fprintf(out, " ABORT requested (%s is set). The hub clears the quiesce and restores writes to %s.\n", AnnSwitchoverAbort, orNone(activeDC)) + _, _ = fmt.Fprintf(out, " Re-run this command until the switchover annotations are gone and DR phase is Steady.\n") return nil } renderSwitchoverSteps(out, target, activeDC, quiesced, dcs) - fmt.Fprintf(out, "\n Abort: kubectl dba dc-dr abort %s -n %s\n", args[0], ns) - fmt.Fprintf(out, " Re-run this command to see the next step; it does not follow.\n") + _, _ = fmt.Fprintf(out, "\n Abort: kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, " Re-run this command to see the next step; it does not follow.\n") return nil } - fmt.Fprintf(out, "\nNo switchover in flight.\n") + _, _ = fmt.Fprintf(out, "\nNo switchover in flight.\n") if drPhase == "FailingOver" { - fmt.Fprintf(out, "DR phase is FailingOver with no switchover annotation, so this is an UNPLANNED failover.\n") - fmt.Fprintf(out, " If it is not completing: kubectl dba dc-dr debug failover %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, "DR phase is FailingOver with no switchover annotation, so this is an UNPLANNED failover.\n") + _, _ = fmt.Fprintf(out, " If it is not completing: kubectl dba dc-dr debug failover %s -n %s\n", args[0], ns) } if protectedSet && !protected { - fmt.Fprintf(out, "Protection is NOT confirmed. If a promotion is held by the RPO budget:\n") - fmt.Fprintf(out, " kubectl dba dc-dr accept-data-loss %s -n %s --yes\n", args[0], ns) + _, _ = fmt.Fprintf(out, "Protection is NOT confirmed. If a promotion is held by the RPO budget:\n") + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr accept-data-loss %s -n %s --yes\n", args[0], ns) } - fmt.Fprintf(out, "Trigger one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) + _, _ = fmt.Fprintf(out, "Trigger one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) return nil }, } @@ -248,31 +248,31 @@ func renderSwitchoverSteps(out interface{ Write([]byte) (int, error) }, target, step6 = stepCurrent } - fmt.Fprintf(out, " %s 1. target %q validated: healthy and lag known, within the switchover budget\n", step1.mark(), target) - fmt.Fprintf(out, " %s 2. quiesce requested on the active DC (%s)\n", step2.mark(), orNone(activeDC)) - fmt.Fprintf(out, " %s 3. quiesce IN EFFECT: active primary write-locked, its LSN frozen\n", step3.mark()) + _, _ = fmt.Fprintf(out, " %s 1. target %q validated: healthy and lag known, within the switchover budget\n", step1.mark(), target) + _, _ = fmt.Fprintf(out, " %s 2. quiesce requested on the active DC (%s)\n", step2.mark(), orNone(activeDC)) + _, _ = fmt.Fprintf(out, " %s 3. quiesce IN EFFECT: active primary write-locked, its LSN frozen\n", step3.mark()) lagText := "unknown" if targetLag != nil { lagText = fmt.Sprintf("%v bytes", targetLag) } - fmt.Fprintf(out, " %s 4. target caught up to the frozen LSN (now %s, needs <= %d)\n", step4.mark(), lagText, zeroRPOLagBytes) - fmt.Fprintf(out, " %s 5. primary-DC Lease handed off to %q\n", step5.mark(), target) - fmt.Fprintf(out, " %s 6. old DC demoted to standby, annotations cleared, DR phase back to Steady\n", step6.mark()) + _, _ = fmt.Fprintf(out, " %s 4. target caught up to the frozen LSN (now %s, needs <= %d)\n", step4.mark(), lagText, zeroRPOLagBytes) + _, _ = fmt.Fprintf(out, " %s 5. primary-DC Lease handed off to %q\n", step5.mark(), target) + _, _ = fmt.Fprintf(out, " %s 6. old DC demoted to standby, annotations cleared, DR phase back to Steady\n", step6.mark()) - fmt.Fprintf(out, "\n NEXT: ") + _, _ = fmt.Fprintf(out, "\n NEXT: ") switch { case step1 == stepCurrent: - fmt.Fprintf(out, "waiting for the target's health and lag to be observable. If it never becomes healthy the switchover cannot start.\n") + _, _ = fmt.Fprintf(out, "waiting for the target's health and lag to be observable. If it never becomes healthy the switchover cannot start.\n") case step3 == stepCurrent: - fmt.Fprintf(out, "waiting for the write-lock to take hold on %s. This needs the active primary to be UP and reachable; a dead primary can never satisfy it (use the failover path).\n", orNone(activeDC)) + _, _ = fmt.Fprintf(out, "waiting for the write-lock to take hold on %s. This needs the active primary to be UP and reachable; a dead primary can never satisfy it (use the failover path).\n", orNone(activeDC)) case step4 == stepCurrent: - fmt.Fprintf(out, "waiting for %q to replay the last %s. This is the only step whose duration depends on your write volume.\n", target, lagText) + _, _ = fmt.Fprintf(out, "waiting for %q to replay the last %s. This is the only step whose duration depends on your write volume.\n", target, lagText) case step5 == stepCurrent: - fmt.Fprintf(out, "handing off the Lease; the target promotes within seconds.\n") + _, _ = fmt.Fprintf(out, "handing off the Lease; the target promotes within seconds.\n") case step6 == stepCurrent: - fmt.Fprintf(out, "the Lease has moved to %q. The old DC self-fences and re-cascades, then the annotations clear and the phase returns to Steady.\n", target) + _, _ = fmt.Fprintf(out, "the Lease has moved to %q. The old DC self-fences and re-cascades, then the annotations clear and the phase returns to Steady.\n", target) default: - fmt.Fprintf(out, "the operator picks it up on its next reconcile.\n") + _, _ = fmt.Fprintf(out, "the operator picks it up on its next reconcile.\n") } } diff --git a/pkg/dcdr/switchover.go b/pkg/dcdr/switchover.go index 3b254975f..4baf8c7a3 100644 --- a/pkg/dcdr/switchover.go +++ b/pkg/dcdr/switchover.go @@ -96,5 +96,5 @@ func NewCmdSwitchover(f cmdutil.Factory) *cobra.Command { } func printfErr(cmd *cobra.Command) func(string, ...any) { - return func(format string, a ...any) { fmt.Fprintf(cmd.ErrOrStderr(), format, a...) } + return func(format string, a ...any) { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), format, a...) } }