Dcdr cli

pkg/cmds/dcdr.go

@@ -0,0 +1,26 @@
+/*
+Copyright AppsCode Inc. and Contributors
+Licensed under the AppsCode Community License 1.0.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmds
+
+import (
+	"kubedb.dev/cli/pkg/dcdr"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+)
+
+// NewCmdDCDR returns the cross data center DR command group.
+func NewCmdDCDR(f cmdutil.Factory) *cobra.Command {
+	return dcdr.NewCmdDCDR(f)
+}

pkg/cmds/root.go

@@ -114,6 +114,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command {
 				NewCmdMSSQL(f),
 			},
 		},
+		{
+			Message: "Cross Data Center DR Commands",
+			Commands: []*cobra.Command{
+				NewCmdDCDR(f),
+			},
+		},
 		{
 			Message: "Metric related CMDs",
 			Commands: []*cobra.Command{

pkg/dcdr/abort.go

@@ -0,0 +1,69 @@
+/*
+Copyright AppsCode Inc. and Contributors
+Licensed under the AppsCode Community License 1.0.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dcdr
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+)
+
+// NewCmdAbort aborts an in-flight planned switchover. Acts on the HUB cluster.
+func NewCmdAbort(f cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "abort DB_NAME",
+		Short: i18n.T("Abort an in-flight planned switchover and restore writes to the current active DC"),
+		Long: templates.LongDesc(`
+			Sets the dr.kubedb.com/switchover-abort annotation. Its PRESENCE aborts:
+			the hub clears the quiesce so the original active data center resumes
+			accepting writes, and removes every switchover annotation once done.
+
+			Do NOT abort by deleting the switchover-to annotation: in a scope shared
+			by several databases the hub re-propagates it to every sibling each pass,
+			so a bare removal silently reappears. This explicit abort signal is
+			propagated and honored scope-wide. A switchover that cannot complete also
+			auto-aborts on its own after the switchover timeout (default 10m,
+			dr.kubedb.com/switchover-timeout to override).
+
+			KUBECONFIG: the hub cluster.`),
+		Example: templates.Examples(`
+			kubectl dba dc-dr abort pg-dcdr -n demo`),
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return fmt.Errorf("exactly one database name is required")
+			}
+			ctx := context.Background()
+			db, dyn, ns, err := getDB(ctx, f, args[0])
+			if err != nil {
+				return err
+			}
+			if db.GetAnnotations()[AnnSwitchoverTo] == "" && db.GetAnnotations()[AnnQuiesceActive] == "" {
+				cmd.Printf("No switchover appears to be in flight on %s/%s (no %s or %s annotation); setting the abort anyway is harmless.\n", ns, args[0], AnnSwitchoverTo, AnnQuiesceActive)
+			}
+			v := "true"
+			if err := annotateDB(ctx, dyn, ns, args[0], AnnSwitchoverAbort, &v); err != nil {
+				return err
+			}
+			cmd.Printf("Abort requested for %s/%s. The hub restores writes to the current active DC and clears the switchover annotations; verify with:\n", ns, args[0])
+			cmd.Printf("  kubectl dba dc-dr status %s -n %s\n", args[0], ns)
+			return nil
+		},
+	}
+	return cmd
+}

pkg/dcdr/acceptloss.go

@@ -0,0 +1,89 @@
+/*
+Copyright AppsCode Inc. and Contributors
+Licensed under the AppsCode Community License 1.0.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dcdr
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+)
+
+// NewCmdAcceptDataLoss releases a cross-DC failover held by the RPO budget. Acts
+// on the HUB cluster.
+func NewCmdAcceptDataLoss(f cmdutil.Factory) *cobra.Command {
+	var yes bool
+	cmd := &cobra.Command{
+		Use:   "accept-data-loss DB_NAME --yes",
+		Short: i18n.T("Release a failover held by the RPO budget, explicitly accepting the data loss"),
+		Long: templates.LongDesc(`
+			When the surviving data center lags more than
+			spec.replication.bestEffortCrossDCLagBytesForFailover (or its lag cannot
+			be measured), the promotion is HELD: the un-replicated WAL of the lost
+			data center is unrecoverable, so choosing between an outage and a loss
+			larger than the budget belongs to a human. This command records that
+			decision by setting dr.kubedb.com/accept-failover-data-loss=true; both
+			promotion paths (the hub gate and the coordinator's data-plane gate)
+			honor it within seconds, and the operator removes the annotation
+			automatically once the failover it authorized lands, so it cannot linger
+			and approve a later, unrelated loss.
+
+			Where the hold is visible before you decide:
+			status.disasterRecovery.protectionMessage (the measured lag),
+			condition DCDRPromotionStalled, and dc-dr status.
+
+			KUBECONFIG: the hub cluster.`),
+		Example: templates.Examples(`
+			# See what would be lost first
+			kubectl dba dc-dr status pg-dcdr -n demo
+
+			# Accept it
+			kubectl dba dc-dr accept-data-loss pg-dcdr -n demo --yes`),
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return fmt.Errorf("exactly one database name is required")
+			}
+			ctx := context.Background()
+			db, dyn, ns, err := getDB(ctx, f, args[0])
+			if err != nil {
+				return err
+			}
+			msg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage")
+			protected, protectedFound, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected")
+			if msg != "" {
+				cmd.Printf("Current protection verdict: %s\n", msg)
+			}
+			if protectedFound && protected {
+				cmd.Printf("NOTE: the database currently reads protected=true; nothing seems held. Setting the annotation is still safe: it is only honored by a promotion that the budget is actively refusing, and it is auto-removed after use.\n")
+			}
+			if !yes {
+				return fmt.Errorf("this authorizes losing committed data beyond the configured budget; re-run with --yes to confirm")
+			}
+			v := "true"
+			if err := annotateDB(ctx, dyn, ns, args[0], AnnAcceptDataLoss, &v); err != nil {
+				return err
+			}
+			cmd.Printf("Data-loss acceptance recorded on %s/%s. The held promotion proceeds within seconds; the annotation is removed automatically once the failover lands.\n", ns, args[0])
+			cmd.Printf("Monitor:  kubectl dba dc-dr status %s -n %s\n", args[0], ns)
+			return nil
+		},
+	}
+	cmd.Flags().BoolVar(&yes, "yes", false, "Confirm accepting data loss beyond the configured RPO budget")
+	return cmd
+}

pkg/dcdr/activedc.go

@@ -0,0 +1,138 @@
+/*
+Copyright AppsCode Inc. and Contributors
+Licensed under the AppsCode Community License 1.0.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dcdr
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+)
+
+// NewCmdActiveDC answers "who is primary" from the authoritative source, the Lease
+// on the coordination plane, resolving the scope from a database name when given.
+func NewCmdActiveDC(f cmdutil.Factory) *cobra.Command {
+	var cf CoordFlags
+	var leaseName string
+	var quiet bool
+	cmd := &cobra.Command{
+		Use:   "active-dc [DB_NAME] [--lease NAME]",
+		Short: i18n.T("Print the data center that currently holds the primary role"),
+		Long: templates.LongDesc(`
+			Reads the primary-DC Lease from the coordination control plane, the
+			authority for which data center is active.
+
+			Given a database name, its failover scope is resolved first (the
+			PlacementPolicy's failoverPolicy trigger, exactly as the operator
+			resolves it) and the matching Lease is read. Given --lease, that Lease is
+			read directly, which also works for a scope whose database is gone.
+
+			KUBECONFIG: the hub cluster (to read the Postgres and its
+			PlacementPolicy, and by default to read the coordination kubeconfig
+			Secret). The coordination plane itself is reached with the --coord-*
+			flags.`),
+		Example: templates.Examples(`
+			# By database
+			kubectl dba dc-dr active-dc pg-dcdr -n demo
+
+			# By Lease name, with an explicit coordination kubeconfig file
+			kubectl dba dc-dr active-dc --lease primary-dc --coord-kubeconfig /tmp/coord.yaml
+
+			# Scriptable: just the DC name
+			kubectl dba dc-dr active-dc pg-dcdr -n demo -q`),
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if (len(args) == 0) == (leaseName == "") {
+				return fmt.Errorf("give exactly one of: a database name, or --lease")
+			}
+			ctx := context.Background()
+			out := cmd.OutOrStdout()
+			scope := &Scope{LeaseName: leaseName, Source: "--lease flag"}
+			var ns string
+			if len(args) == 1 {
+				db, dyn, dbNS, err := getDB(ctx, f, args[0])
+				if err != nil {
+					return err
+				}
+				ns = dbNS
+				scope, err = ResolveScopeForDB(ctx, dyn, db)
+				if err != nil {
+					return err
+				}
+			}
+			coord, err := cf.CoordClient(ctx, f)
+			if err != nil {
+				return err
+			}
+			lease, err := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{})
+			if err != nil {
+				return fmt.Errorf("failed to read Lease %s/%s on the coordination plane: %w", cf.LeaseNS, scope.LeaseName, err)
+			}
+			holder := ""
+			if lease.Spec.HolderIdentity != nil {
+				holder = *lease.Spec.HolderIdentity
+			}
+			if quiet {
+				_, _ = fmt.Fprintln(out, holder)
+				return nil
+			}
+			_, _ = fmt.Fprintf(out, "Active DC:  %s\n", orNone(holder))
+			_, _ = fmt.Fprintf(out, "Lease:      %s/%s  (scope from %s)\n", cf.LeaseNS, scope.LeaseName, scope.Source)
+			if lease.Spec.RenewTime != nil {
+				age := time.Since(lease.Spec.RenewTime.Time).Round(time.Second)
+				dur := int32(0)
+				if lease.Spec.LeaseDurationSeconds != nil {
+					dur = *lease.Spec.LeaseDurationSeconds
+				}
+				_, _ = fmt.Fprintf(out, "Renewed:    %s ago (lease duration %ds)\n", age, dur)
+				if dur > 0 && age > time.Duration(dur)*time.Second {
+					_, _ = fmt.Fprintf(out, "  WARNING: the Lease is EXPIRED. Its holder stopped renewing, so a healthy Member DC may acquire it at any moment.\n")
+				}
+			}
+			if lease.Spec.LeaseTransitions != nil {
+				_, _ = fmt.Fprintf(out, "Transitions:%d\n", *lease.Spec.LeaseTransitions)
+			}
+			if v := lease.Annotations[AnnLeaseMemberDCs]; v != "" {
+				_, _ = fmt.Fprintf(out, "Members:    %s\n", v)
+			}
+			if v := lease.Annotations[AnnLeaseHandoffTo]; v != "" {
+				_, _ = fmt.Fprintf(out, "Handoff to: %s (a coordinated handoff is in flight)\n", v)
+			}
+			if v := lease.Annotations["dr.open-cluster-management.io/override-hold"]; v != "" {
+				_, _ = fmt.Fprintf(out, "PINNED:     break-glass override holds this scope on %s; it cannot fail over until the override ConfigMap is removed from that DC's spoke.\n", v)
+			}
+			if len(args) == 1 {
+				// Cross-check the CR's own view, which lags the Lease by a reconcile.
+				db, _, _, err := getDB(ctx, f, args[0])
+				if err == nil {
+					crActive, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC")
+					if crActive != "" && crActive != holder {
+						_, _ = fmt.Fprintf(out, "\nNOTE: the database %s/%s still reports activeDC=%s in its status; the Lease is the authority and the status trails it by a reconcile.\n", ns, args[0], crActive)
+					}
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&leaseName, "lease", "", "Read this Lease directly instead of resolving a database's scope")
+	cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Print only the active DC name")
+	AddCoordFlags(cmd, &cf)
+	return cmd
+}