From b911d6009c716c22469070746ea46afcf1cdfd1d Mon Sep 17 00:00:00 2001 From: Minal Kyada Date: Thu, 9 Jul 2026 15:05:27 -0700 Subject: [PATCH] node-decomission-resume-path-fix --- .../tcm/sequences/SingleNodeSequences.java | 1 + .../distributed/test/DecommissionTest.java | 4 +- .../test/ring/DecommissionTest.java | 41 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java index 313700aaafff..82a1059b6ee9 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java +++ b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java @@ -92,6 +92,7 @@ static void decommission(boolean shutdownNetworking, boolean force) else if (InProgressSequences.isLeave(inProgress)) { logger.info("Resuming decommission @ {} (current epoch = {}): {}", inProgress.latestModification, metadata.epoch, inProgress.status()); + StorageService.instance.clearTransientMode(); } else { diff --git a/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java index ad0120513b47..41f8274ae570 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; @@ -141,6 +142,7 @@ public void testDecommissionAfterNodeRestart() throws Throwable public static class BB { + static final AtomicBoolean injected = new AtomicBoolean(false); public static void install(ClassLoader classLoader, Integer num) { if (num == 2) @@ -157,7 +159,7 @@ public static void install(ClassLoader classLoader, Integer num) public static void execute(NodeId leaving, PlacementDeltas startLeave, PlacementDeltas midLeave, PlacementDeltas finishLeave, @SuperCall Callable zuper) throws ExecutionException, InterruptedException { - if (!StorageService.instance.isDecommissionFailed()) + if (injected.compareAndSet(false, true)) throw new ExecutionException(new RuntimeException("simulated error in prepareUnbootstrapStreaming")); try diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java index 307956828efc..f6ba5c33a173 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java @@ -53,8 +53,10 @@ import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.transformations.PrepareLeave; import org.apache.cassandra.tcm.transformations.Startup; import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.FBUtilities; @@ -62,6 +64,8 @@ import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCommit; +import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits; import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack; import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology; import static org.apache.cassandra.distributed.test.ring.BootstrapTest.populate; @@ -84,6 +88,43 @@ public void testResumableDecom() throws IOException } } + @Test + public void testOperationModeOnDecomResume() throws Exception + { + // Node 2's first decomission attempt fails mid-stream (injected by BB), leaving it in + // DECOMISSION_FAILED. On resume, operationMode() must transition back to LEAVING before + // MID_LEAVE is committed. We pause the CMS just before that commit to assert the mode + // in the window where streaming has finished but the epoch has not yet advanced. + try (Cluster cluster = builder().withNodes(3) + .withConfig(config -> config.with(NETWORK, GOSSIP)) + .withInstanceInitializer(BB::install) + .start()) + { + populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM); + + IInvokableInstance cmsNode = cluster.get(1); + IInvokableInstance leavingNode = cluster.get(2); + + // --force required: cie_internal keyspace has RF=3, decommission would fail replication check otherwise + leavingNode.nodetoolResult("decommission", "--force").asserts().failure(); + leavingNode.runOnInstance(() -> assertEquals(StorageService.Mode.DECOMMISSION_FAILED, StorageService.instance.operationMode())); + + Callable midLeavePaused = pauseBeforeCommit(cmsNode, e -> e instanceof PrepareLeave.MidLeave); + + Thread resumeThread = new Thread(() -> leavingNode.nodetoolResult("decommission").asserts().success()); + resumeThread.start(); + midLeavePaused.call(); + + leavingNode.runOnInstance(() -> + assertEquals("operationMode during resumed decommission streaming should be LEAVING, not DECOMMISSION_FAILED", + StorageService.Mode.LEAVING, + StorageService.instance.operationMode())); + + unpauseCommits(cmsNode); + resumeThread.join(TimeUnit.MINUTES.toMillis(2)); + } + } + @Test public void testAddressReuseAfterDecommission() throws IOException, ExecutionException, InterruptedException {