From f4cbede6de559c24e0c40de113076aef91734662 Mon Sep 17 00:00:00 2001 From: Nidhi Nandwani Date: Wed, 19 Aug 2026 05:57:30 +0000 Subject: [PATCH 1/4] feat(storage): Parameterized test setup for RCU Integration Testing --- .../cloud/storage/spi/v1/HttpStorageRpc.java | 23 +- .../it/runner/CrossRunIntersection.java | 160 +++++++++-- .../storage/it/runner/StorageITRunner.java | 62 +++-- .../it/runner/annotations/Backend.java | 2 + .../it/runner/annotations/Colocation.java | 22 ++ .../it/runner/annotations/CrossRun.java | 12 + .../it/runner/annotations/LocationType.java | 23 ++ .../it/runner/annotations/SingleBackend.java | 4 + .../it/runner/registry/BackendResources.java | 248 ++++++++++++++++-- .../it/runner/registry/BucketInfoShim.java | 42 +++ .../storage/it/runner/registry/Registry.java | 5 +- 11 files changed, 539 insertions(+), 64 deletions(-) create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index ec7d8e02e411..0ee0dcfe4e10 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -158,11 +158,26 @@ public HttpStorageRpc(StorageOptions options, JsonFactory jsonFactory) { initializer = censusHttpModule.getHttpRequestInitializer(initializer); initializer = new InvocationIdInitializer(initializer, applicationName, tm); batchRequestInitializer = censusHttpModule.getHttpRequestInitializer(null); - storage = + String host = options.getHost(); + Storage.Builder storageBuilder = new Storage.Builder(transport, jsonFactory, initializer) - .setRootUrl(options.getHost()) - .setApplicationName(applicationName) - .build(); + .setApplicationName(applicationName); + if (host != null) { + java.net.URI uri = java.net.URI.create(host); + String path = uri.getPath(); + if (path != null && !path.isEmpty() && !"/".equals(path)) { + String rootUrl = host.substring(0, host.indexOf(path)); + String servicePath = path.startsWith("/") ? path.substring(1) : path; + if (!servicePath.endsWith("/")) { + servicePath += "/"; + } + storageBuilder.setRootUrl(rootUrl); + storageBuilder.setServicePath(servicePath); + } else { + storageBuilder.setRootUrl(host); + } + } + storage = storageBuilder.build(); } public Storage getStorage() { diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java index a48ba9d3ce5f..8aa30c50290a 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java @@ -20,10 +20,14 @@ import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; +import java.util.Collections; import java.util.Locale; +import java.util.Set; import java.util.Objects; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; @@ -39,10 +43,18 @@ public final class CrossRunIntersection { private final @Nullable Backend backend; private final @Nullable Transport transport; + private final @Nullable LocationType locationType; + private final @Nullable Colocation colocation; - private CrossRunIntersection(@Nullable Backend backend, @Nullable Transport transport) { + private CrossRunIntersection( + @Nullable Backend backend, + @Nullable Transport transport, + @Nullable LocationType locationType, + @Nullable Colocation colocation) { this.backend = backend; this.transport = transport; + this.locationType = locationType; + this.colocation = colocation; } @Nullable @@ -55,11 +67,21 @@ public Transport getTransport() { return transport; } + @Nullable + public LocationType getLocationType() { + return locationType; + } + + @Nullable + public Colocation getColocation() { + return colocation; + } + public CrossRunIntersection clearBackend() { if (backend == null) { return this; } else { - return new CrossRunIntersection(null, transport); + return new CrossRunIntersection(null, transport, locationType, colocation); } } @@ -67,7 +89,23 @@ public CrossRunIntersection clearTransport() { if (transport == null) { return this; } else { - return new CrossRunIntersection(backend, null); + return new CrossRunIntersection(backend, null, locationType, colocation); + } + } + + public CrossRunIntersection clearLocationType() { + if (locationType == null) { + return this; + } else { + return new CrossRunIntersection(backend, transport, null, colocation); + } + } + + public CrossRunIntersection clearColocation() { + if (colocation == null) { + return this; + } else { + return new CrossRunIntersection(backend, transport, locationType, null); } } @@ -76,7 +114,7 @@ public CrossRunIntersection withBackend(Backend backend) { if (this.backend == backend) { return this; } else { - return new CrossRunIntersection(backend, transport); + return new CrossRunIntersection(backend, transport, locationType, colocation); } } @@ -85,7 +123,25 @@ public CrossRunIntersection withTransport(Transport transport) { if (this.transport == transport) { return this; } else { - return new CrossRunIntersection(backend, transport); + return new CrossRunIntersection(backend, transport, locationType, colocation); + } + } + + public CrossRunIntersection withLocationType(LocationType locationType) { + requireNonNull(locationType, "locationType must be non null"); + if (this.locationType == locationType) { + return this; + } else { + return new CrossRunIntersection(backend, transport, locationType, colocation); + } + } + + public CrossRunIntersection withColocation(Colocation colocation) { + requireNonNull(colocation, "colocation must be non null"); + if (this.colocation == colocation) { + return this; + } else { + return new CrossRunIntersection(backend, transport, locationType, colocation); } } @@ -107,6 +163,20 @@ public boolean anyMatch(CrossRunIntersection other) { l = l.clearTransport(); } + if (l.locationType == null) { + r = r.clearLocationType(); + } + if (r.locationType == null) { + l = l.clearLocationType(); + } + + if (l.colocation == null) { + r = r.clearColocation(); + } + if (r.colocation == null) { + l = l.clearColocation(); + } + return l.equals(r); } @@ -119,7 +189,9 @@ public boolean anyMatch(CrossRunIntersection other) { public String fmtSuiteName() { String t = transport != null ? transport.toString() : "NULL_TRANSPORT"; String b = backend != null ? backend.toString() : "NULL_BACKEND"; - return String.format(Locale.US, "[%s][%s]", t, b); + String lt = locationType != null ? locationType.toString() : "NULL_LOCATION"; + String c = colocation != null ? colocation.toString() : "NULL_COLOCATION"; + return String.format(Locale.US, "[%s][%s][%s][%s]", t, b, lt, c); } @Override @@ -131,12 +203,15 @@ public boolean equals(Object o) { return false; } CrossRunIntersection crossRunIntersection = (CrossRunIntersection) o; - return backend == crossRunIntersection.backend && transport == crossRunIntersection.transport; + return backend == crossRunIntersection.backend + && transport == crossRunIntersection.transport + && locationType == crossRunIntersection.locationType + && colocation == crossRunIntersection.colocation; } @Override public int hashCode() { - return Objects.hash(backend, transport); + return Objects.hash(backend, transport, locationType, colocation); } @Override @@ -144,41 +219,74 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("backend", backend) .add("transport", transport) + .add("locationType", locationType) + .add("colocation", colocation) .toString(); } - public static CrossRunIntersection of(@Nullable Backend t, @Nullable Transport s) { - return new CrossRunIntersection(t, s); + public static CrossRunIntersection of(@Nullable Backend b, @Nullable Transport t) { + return new CrossRunIntersection(b, t, null, null); + } + + public static CrossRunIntersection of( + @Nullable Backend b, + @Nullable Transport t, + @Nullable LocationType lt, + @Nullable Colocation c) { + return new CrossRunIntersection(b, t, lt, c); } public static ImmutableSet expand(CrossRun.Ignore i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); - return expand(backends, transports); + ImmutableSet locations = ImmutableSet.copyOf(i.locations()); + ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); + return expand(backends, transports, locations, colocations); } public static ImmutableSet expand(CrossRun.Exclude i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); - return expand(backends, transports); + ImmutableSet locations = ImmutableSet.copyOf(i.locations()); + ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); + return expand(backends, transports, locations, colocations); } public static ImmutableSet expand( - ImmutableSet backends, ImmutableSet<@Nullable Transport> transports) { - if (backends.isEmpty() && transports.isEmpty()) { + ImmutableSet backends, + ImmutableSet<@Nullable Transport> transports, + ImmutableSet<@Nullable LocationType> locations, + ImmutableSet<@Nullable Colocation> colocations) { + if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty() && colocations.isEmpty()) { return ImmutableSet.of(); - } else if (!backends.isEmpty() && !transports.isEmpty()) { - return backends.stream() - .flatMap(t -> transports.stream().map(s -> new CrossRunIntersection(t, s))) - .collect(ImmutableSet.toImmutableSet()); - } else if (!backends.isEmpty()) { - return backends.stream() - .map(t -> new CrossRunIntersection(t, null)) - .collect(ImmutableSet.toImmutableSet()); - } else { - return transports.stream() - .map(s -> new CrossRunIntersection(null, s)) - .collect(ImmutableSet.toImmutableSet()); } + + Set<@Nullable Backend> bSet = + backends.isEmpty() ? Collections.singleton((Backend) null) : backends; + Set<@Nullable Transport> tSet = + transports.isEmpty() ? Collections.singleton((Transport) null) : transports; + Set<@Nullable LocationType> lSet = + locations.isEmpty() ? Collections.singleton((LocationType) null) : locations; + Set<@Nullable Colocation> cSet = + colocations.isEmpty() ? Collections.singleton((Colocation) null) : colocations; + + return bSet.stream() + .flatMap( + b -> + tSet.stream() + .flatMap( + t -> + lSet.stream() + .flatMap( + l -> + cSet.stream() + .map(c -> new CrossRunIntersection(b, t, l, c))))) + .filter( + i -> + !(i.backend == null + && i.transport == null + && i.locationType == null + && i.colocation == null)) + .collect(ImmutableSet.toImmutableSet()); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java index 208892bb4dea..0cb06fa66414 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java @@ -23,6 +23,8 @@ import com.google.cloud.storage.it.runner.annotations.Parameterized; import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; +import com.google.cloud.storage.it.runner.annotations.Colocation; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.registry.Registry; import com.google.common.collect.ImmutableList; @@ -166,7 +168,16 @@ private static List computeRunners(Class klass, Registry registry) .flatMap( b -> ImmutableSet.copyOf(crossRun.transports()).stream() - .map(t -> CrossRunIntersection.of(b, t))) + .flatMap( + t -> + ImmutableSet.copyOf(crossRun.locations()).stream() + .flatMap( + l -> + ImmutableSet.copyOf(crossRun.colocations()).stream() + .map( + c -> + CrossRunIntersection.of( + b, t, l, c))))) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); @@ -187,23 +198,38 @@ private static List computeRunners(Class klass, Registry registry) .collect(ImmutableList.toImmutableList())); } else { Backend backend = singleBackend.value(); - CrossRunIntersection crossRunIntersection = CrossRunIntersection.of(backend, null); - TestInitializer ti = registry.newTestInitializerForCell(crossRunIntersection); - if (parameters != null) { - return SneakyException.unwrap( - () -> - parameters.stream() - .map( - param -> - StorageITLeafRunner.unsafeOf( - testClass, - crossRunIntersection, - fmtParam(param), - ti.andThen(setFieldTo(testClass, param)))) - .collect(ImmutableList.toImmutableList())); - } else { - return ImmutableList.of(StorageITLeafRunner.of(testClass, crossRunIntersection, null, ti)); - } + boolean isDefault = + singleBackend.locations().length == 1 + && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD + && singleBackend.colocations().length == 1 + && singleBackend.colocations()[0] == Colocation.CO_LOCATED; + + return SneakyException.unwrap( + () -> + ImmutableSet.copyOf(singleBackend.locations()).stream() + .flatMap( + l -> + ImmutableSet.copyOf(singleBackend.colocations()).stream() + .map(c -> CrossRunIntersection.of(backend, null, l, c))) + .flatMap( + c -> { + TestInitializer ti = registry.newTestInitializerForCell(c); + if (parameters != null) { + return parameters.stream() + .map( + param -> + StorageITLeafRunner.unsafeOf( + testClass, + c, + isDefault ? fmtParam(param) : fmtParam(c, param), + ti.andThen(setFieldTo(testClass, param)))); + } else { + return Stream.of( + StorageITLeafRunner.unsafeOf( + testClass, c, isDefault ? null : c.fmtSuiteName(), ti)); + } + }) + .collect(ImmutableList.toImmutableList())); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java index 6e6c1b37b4bc..56e77a5931d7 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java @@ -20,6 +20,8 @@ public enum Backend { /** Use the "Production" GCS endpoints */ PROD, + /** Use the GCS Pre-prod (Staging) endpoints */ + PREPROD, /** Use the test bench as a backend */ TEST_BENCH } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java new file mode 100644 index 000000000000..b3982eb2bc76 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java @@ -0,0 +1,22 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it.runner.annotations; + +public enum Colocation { + CO_LOCATED, + NON_CO_LOCATED +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java index 960e56e94a72..fe6c31b0b757 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java @@ -39,6 +39,10 @@ Transport[] transports(); + LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; + + Colocation[] colocations() default {Colocation.CO_LOCATED}; + /** * Exclude a method from being included in the generated test suite if it's backend and transport * match with those defined. When matching, if the empty set is defined as a value it will be @@ -53,6 +57,10 @@ Backend[] backends() default {}; TransportCompatibility.Transport[] transports() default {}; + + LocationType[] locations() default {}; + + Colocation[] colocations() default {}; } /** @@ -69,6 +77,10 @@ Backend[] backends() default {}; TransportCompatibility.Transport[] transports() default {}; + + LocationType[] locations() default {}; + + Colocation[] colocations() default {}; } /** diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java new file mode 100644 index 000000000000..f9c47b924a8b --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java @@ -0,0 +1,23 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it.runner.annotations; + +public enum LocationType { + REGIONAL_STANDARD, + REGIONAL_RAPID, + ZONAL_RAPID +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java index d9d98b9ac1ea..09fc2d6a17be 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java @@ -34,4 +34,8 @@ @Retention(RetentionPolicy.RUNTIME) public @interface SingleBackend { Backend value(); + + LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; + + Colocation[] colocations() default {Colocation.CO_LOCATED}; } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java index bfe9a5768754..f82357b533aa 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java @@ -35,8 +35,14 @@ import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.GrpcPlainRequestLoggingInterceptor; +import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.BucketType; +import com.google.cloud.storage.it.runner.annotations.Colocation; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.junit.runners.model.FrameworkField; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.storage.control.v2.StorageControlClient; @@ -48,20 +54,23 @@ import java.util.Locale; import java.util.UUID; + /** The set of resources which are defined for a single backend. */ final class BackendResources implements ManagedLifecycle { private final Backend backend; private final ProtectedBucketNames protectedBucketNames; - + private final ConcurrentMap dynamicBuckets; private final ImmutableList> registryEntries; private BackendResources( Backend backend, ProtectedBucketNames protectedBucketNames, + ConcurrentMap dynamicBuckets, ImmutableList> registryEntries) { this.backend = backend; this.protectedBucketNames = protectedBucketNames; + this.dynamicBuckets = dynamicBuckets; this.registryEntries = registryEntries; } @@ -80,6 +89,8 @@ public void start() {} @Override public void stop() { protectedBucketNames.stop(); + dynamicBuckets.values().forEach(BucketInfoShim::stop); + dynamicBuckets.clear(); } @Override @@ -93,6 +104,8 @@ static BackendResources of( TestRunScopedInstance otelSdk, TestRunScopedInstance zone) { ProtectedBucketNames protectedBucketNames = new ProtectedBucketNames(); + ConcurrentMap dynamicBuckets = new ConcurrentHashMap<>(); + TestRunScopedInstance storageJson = TestRunScopedInstance.of( "fixture/STORAGE/[JSON][" + backend.name() + "]", @@ -106,6 +119,13 @@ static BackendResources of( .setHost(Registry.getInstance().testBench().getBaseUri()) .setProjectId("test-project-id"); break; + case PREPROD: + optionsBuilder = + StorageOptions.http() + .setHost("https://storage-preprod-test-unified.googleusercontent.com/storage/v1_preprod/") + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); + break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry optionsBuilder = StorageOptions.http().setOpenTelemetry(otelSdk.get().get()); @@ -130,6 +150,13 @@ static BackendResources of( .setAttemptDirectPath(false) .setProjectId("test-project-id"); break; + case PREPROD: + optionsBuilder = + StorageOptions.grpc() + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); + break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry optionsBuilder = StorageOptions.grpc().setOpenTelemetry(otelSdk.get().get()); @@ -168,6 +195,18 @@ static BackendResources of( .setEndpoint(endpoint) .setTransportChannelProvider(instantiatingGrpcChannelProvider); break; + case PREPROD: + String preProdEndpoint = "storage-preprod-test-grpc.googleusercontent.com:443"; + builder = + StorageControlSettings.newBuilder() + .setEndpoint(preProdEndpoint) + .setTransportChannelProvider( + StorageControlStubSettings.defaultGrpcTransportProviderBuilder() + .setInterceptorProvider( + GrpcPlainRequestLoggingInterceptor.getInterceptorProvider()) + .setEndpoint(preProdEndpoint) + .build()); + break; default: // PROD, java8 doesn't have exhaustive checking for enum switch builder = StorageControlSettings.newBuilder() @@ -186,20 +225,12 @@ static BackendResources of( throw new RuntimeException(e); } }); - TestRunScopedInstance bucket = + TestRunScopedInstance bucket = TestRunScopedInstance.of( - "fixture/BUCKET/[" + backend.name() + "]", - () -> { - String bucketName = - String.format(Locale.US, "java-storage-grpc-%s", UUID.randomUUID()); - protectedBucketNames.add(bucketName); - return new BucketInfoShim( - BucketInfo.newBuilder(bucketName) - .setLocation(zone.get().get().getRegion()) - .build(), - storageJson.get().getStorage(), - ctrl.get().getCtrl()); - }); + "fixture/BUCKET/[" + backend.name() + "]/DYNAMIC", + () -> + new DynamicBucketLifecycle( + backend, storageJson, ctrl, zone, protectedBucketNames, dynamicBuckets)); TestRunScopedInstance bucketRp = TestRunScopedInstance.of( "fixture/BUCKET/[" + backend.name() + "]/REQUESTER_PAYS", @@ -278,7 +309,18 @@ static BackendResources of( TestRunScopedInstance objectsFixture = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]", - () -> new ObjectsFixture(storageJson.get().getStorage(), bucket.get().getBucketInfo())); + () -> + new ObjectsFixture( + storageJson.get().getStorage(), + bucket + .get() + .resolve( + null, + CrossRunIntersection.of( + backend, + null, + LocationType.REGIONAL_STANDARD, + Colocation.CO_LOCATED)))); TestRunScopedInstance objectsFixtureRp = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]/REQUESTER_PAYS", @@ -298,6 +340,7 @@ static BackendResources of( return new BackendResources( backend, protectedBucketNames, + dynamicBuckets, ImmutableList.of( RegistryEntry.of( 40, Storage.class, storageJson, transportAndBackendAre(Transport.HTTP, backend)), @@ -343,4 +386,179 @@ static BackendResources of( backendIs(backend).and(bucketTypeIs(BucketType.HNS))), RegistryEntry.of(100, KmsFixture.class, kmsFixture, backendIs(backend)))); } + + private static final class BucketKey { + private final LocationType locationType; + private final Colocation colocation; + + private BucketKey(LocationType locationType, Colocation colocation) { + this.locationType = locationType; + this.colocation = colocation; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BucketKey)) { + return false; + } + BucketKey bucketKey = (BucketKey) o; + return locationType == bucketKey.locationType && colocation == bucketKey.colocation; + } + + @Override + public int hashCode() { + return java.util.Objects.hash(locationType, colocation); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("locationType", locationType) + .add("colocation", colocation) + .toString(); + } + } + + private static final class DynamicBucketLifecycle + implements Registry.StatelessManagedLifecycle { + private final Backend backend; + private final TestRunScopedInstance storageJson; + private final TestRunScopedInstance ctrl; + private final TestRunScopedInstance zone; + private final ProtectedBucketNames protectedBucketNames; + private final ConcurrentMap dynamicBuckets; + + private DynamicBucketLifecycle( + Backend backend, + TestRunScopedInstance storageJson, + TestRunScopedInstance ctrl, + TestRunScopedInstance zone, + ProtectedBucketNames protectedBucketNames, + ConcurrentMap dynamicBuckets) { + this.backend = backend; + this.storageJson = storageJson; + this.ctrl = ctrl; + this.zone = zone; + this.protectedBucketNames = protectedBucketNames; + this.dynamicBuckets = dynamicBuckets; + } + + @Override + public BucketInfo resolve(FrameworkField ff, CrossRunIntersection crossRunIntersection) { + LocationType lt = crossRunIntersection.getLocationType(); + Colocation col = crossRunIntersection.getColocation(); + + if (lt == null) { + lt = LocationType.REGIONAL_STANDARD; + } + if (col == null) { + col = Colocation.CO_LOCATED; + } + + BucketKey key = new BucketKey(lt, col); + BucketInfoShim shim = dynamicBuckets.computeIfAbsent(key, this::createBucketShim); + return (BucketInfo) shim.get(); + } + + private BucketInfoShim createBucketShim(BucketKey key) { + Zone z = zone.get().get(); + String region = z.getRegion(); + String zoneName = z.getZone(); + + String rAlt; + String vAlt; + if ("us-east1".equals(region)) { + rAlt = "us-central1"; + vAlt = "us-central1-a"; + } else { + rAlt = "us-east1"; + vAlt = "us-east1-b"; + } + + String targetRegion = region; + String targetZone = zoneName; + + if (key.colocation == Colocation.NON_CO_LOCATED) { + targetRegion = rAlt; + targetZone = vAlt; + } + + BucketInfo.Builder builder; + String prefix; + + switch (key.locationType) { + case REGIONAL_STANDARD: + prefix = "java-storage-reg-std"; + builder = BucketInfo.newBuilder("").setLocation(targetRegion); + break; + case REGIONAL_RAPID: + prefix = "java-storage-reg-rapid"; + builder = + BucketInfo.newBuilder("") + .setLocation(targetRegion) + .setHierarchicalNamespace( + HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()); + break; + case ZONAL_RAPID: + prefix = "java-storage-zon-rapid"; + builder = + BucketInfo.newBuilder("") + .setLocation(targetRegion) + .setCustomPlacementConfig( + CustomPlacementConfig.newBuilder() + .setDataLocations(ImmutableList.of(targetZone)) + .build()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .setHierarchicalNamespace( + HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()); + break; + default: + throw new IllegalArgumentException("Unknown location type: " + key.locationType); + } + + String bucketName = + String.format( + Locale.US, + "%s-%s-%s", + prefix, + backend.name().toLowerCase(Locale.US), + UUID.randomUUID().toString().substring(0, 8)); + + builder.setName(bucketName); + protectedBucketNames.add(bucketName); + + BucketInfoShim shim = + new BucketInfoShim( + builder.build(), + key.locationType, + targetZone, + storageJson.get().getStorage(), + ctrl.get().getCtrl()); + + shim.start(); + return shim; + } + } + + private static String getPreprodProjectId() { + String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + if (projectId == null || projectId.isEmpty()) { + projectId = System.getProperty("google.cloud.project"); + } + if (projectId == null || projectId.isEmpty()) { + projectId = "gcs-hyd-connector-benchmarks"; + } + return projectId; + } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java index 0c981895038b..8a0e3b71f7d2 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java @@ -22,6 +22,10 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.it.BucketCleaner; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import com.google.protobuf.Duration; +import com.google.storage.control.v2.BucketName; +import com.google.storage.control.v2.RapidCache; import com.google.storage.control.v2.StorageControlClient; import java.util.Locale; @@ -29,13 +33,26 @@ final class BucketInfoShim implements ManagedLifecycle { private final BucketInfo bucketInfo; + private final LocationType locationType; + private final String targetZone; private final Storage s; private final StorageControlClient ctrl; private BucketInfo createdBucket; BucketInfoShim(BucketInfo bucketInfo, Storage s, StorageControlClient ctrl) { + this(bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); + } + + BucketInfoShim( + BucketInfo bucketInfo, + LocationType locationType, + String targetZone, + Storage s, + StorageControlClient ctrl) { this.bucketInfo = bucketInfo; + this.locationType = locationType; + this.targetZone = targetZone; this.s = s; this.ctrl = ctrl; } @@ -53,6 +70,31 @@ public Object get() { public void start() { try { createdBucket = s.create(bucketInfo).asBucketInfo(); + if (locationType == LocationType.REGIONAL_RAPID) { + if (ctrl == null) { + throw new IllegalStateException( + "StorageControlClient is required for REGIONAL_RAPID but was not provided"); + } + String cacheName = + String.format( + Locale.US, + "projects/_/buckets/%s/rapidCaches/%s", + createdBucket.getName(), + targetZone); + RapidCache rapidCache = + RapidCache.newBuilder() + .setName(cacheName) + .setZone(targetZone) + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours + .build(); + try { + ctrl.createRapidCacheAsync(BucketName.format("_", createdBucket.getName()), rapidCache) + .get(); + } catch (Exception e) { + throw new RuntimeException("Failed to create Rapid Cache: " + e.getMessage(), e); + } + } } catch (StorageException se) { String msg = se.getMessage().toLowerCase(Locale.US); if (se.getCode() == 400 && (msg.contains("not a valid zone in location")) diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java index c232cd32de6c..0db457ae507f 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java @@ -91,6 +91,8 @@ public final class Registry extends RunListener { private final BackendResources prodBackendResources = BackendResources.of(Backend.PROD, otelSdk, zone); + private final BackendResources preProdBackendResources = + BackendResources.of(Backend.PREPROD, otelSdk, zone); private final BackendResources testBenchBackendResource = BackendResources.of(Backend.TEST_BENCH, otelSdk, zone); @@ -104,6 +106,7 @@ public final class Registry extends RunListener { registryEntry(3, Backend.class, CrossRunIntersection::getBackend), registryEntry(4, Transport.class, CrossRunIntersection::getTransport)) .addAll(prodBackendResources.getRegistryEntries()) + .addAll(preProdBackendResources.getRegistryEntries()) .addAll(testBenchBackendResource.getRegistryEntries()) .build(); @@ -281,7 +284,7 @@ private void shutdown() { } @FunctionalInterface - private interface StatelessManagedLifecycle extends ManagedLifecycle { + interface StatelessManagedLifecycle extends ManagedLifecycle { T resolve(FrameworkField ff, CrossRunIntersection crossRunIntersection); @Override From 5dc69d5671d30b9d73caeb5a99e691a40a7ad4f1 Mon Sep 17 00:00:00 2001 From: Nidhi Nandwani Date: Thu, 20 Aug 2026 09:35:28 +0000 Subject: [PATCH 2/4] add integration test files --- .../storage/ITObjectReadSessionFakeTest.java | 204 ++++++- .../cloud/storage/ITRcuBidiWriteTest.java | 552 ++++++++++++++++++ .../cloud/storage/it/ITRapidCacheTest.java | 220 +++++++ .../storage/it/ITRcuBidiReadTempTest.java | 155 +++++ .../cloud/storage/it/ITRcuBidiReadTest.java | 359 ++++++++++++ .../it/runner/CrossRunIntersection.java | 84 +-- .../storage/it/runner/StorageITRunner.java | 19 +- .../it/runner/annotations/Colocation.java | 22 - .../it/runner/annotations/CrossRun.java | 6 - .../it/runner/annotations/SingleBackend.java | 1 - .../it/runner/registry/BackendResources.java | 82 ++- .../it/runner/registry/BucketInfoShim.java | 37 +- .../storage/it/runner/registry/Registry.java | 13 +- 13 files changed, 1609 insertions(+), 145 deletions(-) create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java delete mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java index 1cb32704982b..eaeb4019bcee 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java @@ -210,8 +210,16 @@ public void bidiReadObjectRedirectedError() throws Exception { req3, respond -> respond.onNext(res2))); + GrpcRequestAuditing requestAuditing = new GrpcRequestAuditing(); try (FakeServer fakeServer = FakeServer.of(fake); - Storage storage = fakeServer.getGrpcStorageOptions().toBuilder().build().getService()) { + Storage storage = + fakeServer.getGrpcStorageOptions().toBuilder() + .setGrpcInterceptorProvider( + () -> + ImmutableList.of( + requestAuditing, GrpcPlainRequestLoggingInterceptor.getInstance())) + .build() + .getService()) { BlobId id = BlobId.of("b", "o"); ApiFuture futureBlobDescriptor = storage.blobReadSession(id); @@ -222,6 +230,12 @@ public void bidiReadObjectRedirectedError() throws Exception { .get(1, TimeUnit.SECONDS); assertThat(xxd(actual)).isEqualTo(xxd(content.getBytes())); + + requestAuditing + .assertRequestHeader("x-goog-request-params") + .containsExactly( + "bucket=projects/_/buckets/b", + "bucket=projects/_/buckets/b&routing_token=" + routingToken); } } } @@ -1805,6 +1819,193 @@ static ObjectReadSessionImpl getObjectReadSessionImpl(BlobReadSession bd) { return orsi; } + private static Consumer> onRedirect( + BidiReadHandle handle, String token) { + return respond -> { + BidiReadObjectRedirectedError redirect = + BidiReadObjectRedirectedError.newBuilder() + .setReadHandle(handle) + .setRoutingToken(token) + .build(); + + com.google.rpc.Status grpcStatusDetails = + com.google.rpc.Status.newBuilder() + .setCode(com.google.rpc.Code.UNAVAILABLE_VALUE) + .setMessage("redirect") + .addDetails(Any.pack(redirect)) + .build(); + + Metadata trailers = new Metadata(); + trailers.put(GRPC_STATUS_DETAILS_KEY, grpcStatusDetails); + StatusRuntimeException statusRuntimeException = + Status.UNAVAILABLE.withDescription("redirect").asRuntimeException(trailers); + respond.onError(statusRuntimeException); + }; + } + + @Test + public void bidiReadObjectRedirectedError_redirectCounterResetOnResponse() throws Exception { + BidiReadHandle handle1 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-1")).build(); + BidiReadHandle handle2 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-2")).build(); + BidiReadHandle handle3 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-3")).build(); + BidiReadHandle handle4 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-4")).build(); + BidiReadHandle handle5 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-5")).build(); + + BidiReadObjectRequest req_read_1 = read(1, 10, 5); + + BidiReadObjectRequest req_read_1_redirected_1 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle1) + .setRoutingToken("token-1") + .build()) + .addReadRanges(getReadRange(1, 10, 5)) + .build(); + + BidiReadObjectRequest req_read_2 = + BidiReadObjectRequest.newBuilder() + .addReadRanges(getReadRange(2, 15, 5)) + .build(); + + BidiReadObjectRequest req_read_2_redirected_2 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle2) + .setRoutingToken("token-2") + .build()) + .addReadRanges(getReadRange(2, 15, 5)) + .build(); + + BidiReadObjectRequest req_read_3 = + BidiReadObjectRequest.newBuilder() + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + BidiReadObjectRequest req_read_3_redirected_3 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle3) + .setRoutingToken("token-3") + .build()) + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + BidiReadObjectRequest req_read_3_redirected_4 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle4) + .setRoutingToken("token-4") + .build()) + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + BidiReadObjectRequest req_read_3_redirected_5 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle5) + .setRoutingToken("token-5") + .build()) + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + ChecksummedTestContent content1 = ChecksummedTestContent.of(ALL_OBJECT_BYTES, 10, 5); + BidiReadObjectResponse res_read_1 = + BidiReadObjectResponse.newBuilder() + .setMetadata(METADATA) + .addObjectDataRanges( + ObjectRangeData.newBuilder() + .setChecksummedData(content1.asChecksummedData()) + .setReadRange(getReadRange(1, 10, 5)) + .setRangeEnd(true) + .build()) + .build(); + + ChecksummedTestContent content2 = ChecksummedTestContent.of(ALL_OBJECT_BYTES, 15, 5); + BidiReadObjectResponse res_read_2 = + BidiReadObjectResponse.newBuilder() + .setMetadata(METADATA) + .addObjectDataRanges( + ObjectRangeData.newBuilder() + .setChecksummedData(content2.asChecksummedData()) + .setReadRange(getReadRange(2, 15, 5)) + .setRangeEnd(true) + .build()) + .build(); + + ChecksummedTestContent content3 = ChecksummedTestContent.of(ALL_OBJECT_BYTES, 20, 5); + BidiReadObjectResponse res_read_3 = + BidiReadObjectResponse.newBuilder() + .setMetadata(METADATA) + .addObjectDataRanges( + ObjectRangeData.newBuilder() + .setChecksummedData(content3.asChecksummedData()) + .setReadRange(getReadRange(3, 20, 5)) + .setRangeEnd(true) + .build()) + .build(); + + FakeStorage fake = + FakeStorage.of( + ImmutableMap.>>builder() + .put(REQ_OPEN, respond -> respond.onNext(RES_OPEN)) + .put(req_read_1, onRedirect(handle1, "token-1")) + .put(req_read_1_redirected_1, respond -> respond.onNext(res_read_1)) + .put(req_read_2, onRedirect(handle2, "token-2")) + .put(req_read_2_redirected_2, respond -> respond.onNext(res_read_2)) + .put(req_read_3, onRedirect(handle3, "token-3")) + .put(req_read_3_redirected_3, onRedirect(handle4, "token-4")) + .put(req_read_3_redirected_4, onRedirect(handle5, "token-5")) + .put(req_read_3_redirected_5, respond -> respond.onNext(res_read_3)) + .build()); + + try (FakeServer fakeServer = FakeServer.of(fake); + Storage storage = fakeServer.getGrpcStorageOptions().toBuilder().build().getService()) { + + BlobId id = BlobId.of("b", "o"); + ApiFuture futureBlobDescriptor = storage.blobReadSession(id); + + try (BlobReadSession bd = futureBlobDescriptor.get(5, TimeUnit.SECONDS)) { + // Read 1 (should trigger Redirect 1, then succeed) + byte[] actual1 = + bd.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(10L, 5L))) + .get(1, TimeUnit.SECONDS); + assertThat(xxd(actual1)).isEqualTo(xxd(content1.getBytes())); + + // Read 2 (should trigger Redirect 2, then succeed) + byte[] actual2 = + bd.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(15L, 5L))) + .get(1, TimeUnit.SECONDS); + assertThat(xxd(actual2)).isEqualTo(xxd(content2.getBytes())); + + // Read 3 (should trigger Redirect 3, Redirect 4, Redirect 5, then succeed) + byte[] actual3 = + bd.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(20L, 5L))) + .get(1, TimeUnit.SECONDS); + assertThat(xxd(actual3)).isEqualTo(xxd(content3.getBytes())); + } + } + } + static final class FakeStorage extends StorageImplBase { private final Map>> db; @@ -1823,6 +2024,7 @@ public void onNext(BidiReadObjectRequest req) { if (db.containsKey(req)) { db.get(req).accept(respond); } else { + System.err.println("FakeStorage: UNEXPECTED REQUEST:\n" + req); respond.onError(TestUtils.apiException(Code.UNIMPLEMENTED, "Unexpected request")); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java new file mode 100644 index 000000000000..dff3a407d577 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java @@ -0,0 +1,552 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import static com.google.cloud.storage.TestUtils.assertAll; +import static com.google.cloud.storage.TestUtils.xxd; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +import com.google.api.core.ApiFuture; +import com.google.cloud.storage.BlobAppendableUpload.AppendableUploadWriteableByteChannel; +import com.google.cloud.storage.BlobAppendableUploadConfig.CloseAction; +import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown; +import com.google.cloud.storage.FlushPolicy.MaxFlushSizeFlushPolicy; +import com.google.cloud.storage.FlushPolicy.MinFlushSizeFlushPolicy; +import com.google.cloud.storage.ITRcuBidiWriteTest.UploadConfigParameters; +import com.google.cloud.storage.MetadataField.PartRange; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.ChecksummedTestContent; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import com.google.cloud.storage.it.runner.annotations.Parameterized; +import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; +import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; +import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Paths; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + backends = {Backend.PROD}, + transports = Transport.GRPC, + locations = { + LocationType.REGIONAL_RAPID + }) +@Parameterized(UploadConfigParameters.class) +public final class ITRcuBidiWriteTest { + + @Inject public Generator generator; + + @Inject public Storage storage; + + @Inject public BucketInfo bucket; + + @Inject public Backend backend; + + @Parameter public Param p; + + @Test + public void appendableUpload_emptyObject() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + assumeTrue( + "only run once", + p.content.length() == UploadConfigParameters.objectSizes.get(0) + && p.uploadConfig.getCloseAction() == UploadConfigParameters.closeActions.get(0) + && p.uploadConfig.getFlushPolicy().equals(UploadConfigParameters.flushPolicies.get(0))); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + upload.open().close(); + + BlobInfo actual = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(actual.getSize()).isEqualTo(0); + assertThat(actual.getCrc32c()) + .isEqualTo(Utils.crc32cCodec.encode(Crc32cValue.zero().getValue())); + + assumeFalse( + "Testbench doesn't handle {read_id: 1, read_offset: 0} for a 0 byte object", + backend == Backend.TEST_BENCH); + byte[] actualBytes = readAllBytes(actual); + assertThat(xxd(actualBytes)).isEqualTo(xxd(new byte[0])); + } + + @Test + public void appendableUpload_bytes() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + checkTestbenchIssue733(); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + // cut out the middle + 1 byte + int length = p.content.length(); + int mid = length / 2; + ChecksummedTestContent a1 = p.content.slice(0, mid); + ChecksummedTestContent a2 = p.content.slice(mid + 1, length - mid - 1); + ChecksummedTestContent a1_a2 = a1.concat(a2); + Crc32cLengthKnown c1_c2 = Crc32cValue.of(a1_a2.getCrc32c(), a1_a2.length()); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written1 = Buffers.emptyTo(ByteBuffer.wrap(a1.getBytes()), channel); + assertThat(written1).isEqualTo(a1.length()); + int written2 = Buffers.emptyTo(ByteBuffer.wrap(a2.getBytes()), channel); + assertThat(written2).isEqualTo(a2.length()); + } + + BlobInfo actual = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(actual.getSize()).isEqualTo(c1_c2.getLength()); + assertThat(actual.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(c1_c2.getValue())); + + byte[] actualBytes = readAllBytes(actual); + assertThat(xxd(actualBytes)).isEqualTo(xxd(a1_a2.getBytes())); + } + + @Test + public void explicitFlush() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + checkTestbenchIssue733(); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + ByteBuffer src = p.content.asByteBuffer(); + ByteBuffer zed = src.slice(); + zed.limit(zed.position() + 1); + src.position(src.position() + 1); + + int written = channel.write(zed); + assertThat(written).isEqualTo(1); + channel.flush(); + + written = StorageChannelUtils.blockingEmptyTo(src, channel); + assertThat(written).isEqualTo(p.content.length() - 1); + } + + BlobInfo gen1 = upload.getResult().get(3, TimeUnit.SECONDS); + assertThat(gen1.getSize()).isEqualTo(p.content.length()); + assertThat(gen1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + } + + @Test + // Pending work in testbench: https://github.com/googleapis/storage-testbench/issues/723 + // manually verified internally on 2025-03-25 + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void appendableBlobUploadTakeover() throws Exception { + + List chunks = p.content.chunkup((p.content.length() / 2) + 1); + assertThat(chunks).hasSize(2); + + ChecksummedTestContent c1 = chunks.get(0); + ChecksummedTestContent c2 = chunks.get(1); + + BlobId id = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + BlobAppendableUploadConfig doNotFinalizeConfig = + p.uploadConfig.withCloseAction(CloseAction.CLOSE_WITHOUT_FINALIZING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(id).setStorageClass(StorageClass.valueOf("RAPID")).build(), + doNotFinalizeConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(c1.getBytes()), channel); + assertThat(written).isEqualTo(c1.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(c1.length()); + assertThat(done1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(c1.getCrc32c())); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(c2.getBytes()), channel); + assertThat(written).isEqualTo(c2.length()); + } + BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); + + assertThat(done2.getSize()).isEqualTo(p.content.length()); + assertThat(done2.getCrc32c()).isAnyOf(Utils.crc32cCodec.encode(p.content.getCrc32c()), null); + } + + @Test + public void testUploadFileUsingAppendable() throws Exception { + checkTestbenchIssue733(); + + String objectName = UUID.randomUUID().toString(); + String fileName = + ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy.noPrefix() + .fmtName(objectName, PartRange.of(1)); + BlobId bid = BlobId.of(bucket.getName(), objectName); + int fileSize = p.content.length(); + try (TmpFile tmpFile = + TmpFile.of(Paths.get(System.getProperty("java.io.tmpdir")), fileName + ".", ".bin")) { + try (SeekableByteChannel w = tmpFile.writer()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), w); + assertThat(written).isEqualTo(p.content.length()); + } + + BlobAppendableUpload appendable = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (SeekableByteChannel r = tmpFile.reader(); + AppendableUploadWriteableByteChannel w = appendable.open()) { + long copied = ByteStreams.copy(r, w); + assertThat(copied).isEqualTo(fileSize); + } + BlobInfo bi = appendable.getResult().get(5, TimeUnit.SECONDS); + assertThat(bi.getSize()).isEqualTo(fileSize); + } + } + + @Test + // Pending work in testbench: https://github.com/googleapis/storage-testbench/issues/723 + // manually verified internally on 2025-03-25 + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverJustToFinalizeWorks() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + assertThat(done1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + takeOver.open().finalizeAndClose(); + + BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); + assertAll( + () -> assertThat(done2).isNotNull(), + () -> assertThat(done2.getSize()).isEqualTo(p.content.length()), + () -> assertThat(done2.getCrc32c()).isNotNull()); + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void explicitFinalizeWithCorrectChecksum() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + String expectedCrc = Utils.crc32cCodec.encode(p.content.getCrc32c()); + channel.finalizeAndClose(expectedCrc); + } + + BlobInfo gen1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(gen1.getSize()).isEqualTo(p.content.length()); + assertThat(gen1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void explicitFinalizeWithIncorrectChecksumFails() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); + try { + channel.finalizeAndClose(badCrc); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (IOException e) { + assertThat(e.getMessage().toLowerCase()).contains("mismatch"); + } + } + + try { + upload.getResult().get(5, TimeUnit.SECONDS); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (ExecutionException e) { + // The server rejects it + assertThat(e.getCause().getMessage().toLowerCase()).contains("mismatch"); + } + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverJustToFinalizeWithIncorrectChecksumFails() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + assertThat(done1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + try { + channel.finalizeAndClose(badCrc); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (IOException e) { + assertThat(e.getMessage().toLowerCase()).contains("mismatch"); + } + } + + try { + takeOver.getResult().get(5, TimeUnit.SECONDS); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (ExecutionException e) { + // The server rejects it + assertThat(e.getCause().getMessage().toLowerCase()).contains("mismatch"); + } + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverAndAppendWithCorrectChecksumWorks() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + ChecksummedTestContent fullContent = p.content.concat(p.content); + String expectedCrc = Utils.crc32cCodec.encode(fullContent.getCrc32c()); + channel.finalizeAndClose(expectedCrc); + } + + BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); + assertThat(done2.getSize()).isEqualTo(p.content.length() * 2); + ChecksummedTestContent fullContent = p.content.concat(p.content); + assertThat(done2.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(fullContent.getCrc32c())); + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverAndAppendWithIncorrectChecksumFails() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); + try { + channel.finalizeAndClose(badCrc); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (IOException e) { + assertThat(e.getMessage().toLowerCase()).contains("mismatch"); + } + } + + try { + takeOver.getResult().get(5, TimeUnit.SECONDS); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (ExecutionException e) { + assertThat(e.getCause().getMessage().toLowerCase()).contains("mismatch"); + } + } + + private void checkTestbenchIssue733() { + if (backend == Backend.TEST_BENCH + && p.uploadConfig.getCloseAction() == CloseAction.FINALIZE_WHEN_CLOSING) { + int estimatedMessageCount = 0; + FlushPolicy flushPolicy = p.uploadConfig.getFlushPolicy(); + if (flushPolicy instanceof MinFlushSizeFlushPolicy) { + MinFlushSizeFlushPolicy min = (MinFlushSizeFlushPolicy) flushPolicy; + estimatedMessageCount = p.content.length() / min.getMinFlushSize(); + } else if (flushPolicy instanceof MaxFlushSizeFlushPolicy) { + MaxFlushSizeFlushPolicy max = (MaxFlushSizeFlushPolicy) flushPolicy; + estimatedMessageCount = p.content.length() / max.getMaxFlushSize(); + } + // if our int division results in a partial message, ensure we are counting at least one + // message. We have a separate test specifically for empty objects. + estimatedMessageCount = Math.max(estimatedMessageCount, 1); + assumeTrue( + "testbench broken https://github.com/googleapis/storage-testbench/issues/733", + estimatedMessageCount > 1); + } + } + + private byte[] readAllBytes(BlobInfo actual) + throws IOException, InterruptedException, ExecutionException, TimeoutException { + ApiFuture blobReadSessionFuture = storage.blobReadSession(actual.getBlobId()); + try (BlobReadSession read = blobReadSessionFuture.get(2_372, TimeUnit.MILLISECONDS)) { + ApiFuture futureBytes = read.readAs(ReadProjectionConfigs.asFutureBytes()); + return futureBytes.get(2_273, TimeUnit.MILLISECONDS); + } + } + + public static final class UploadConfigParameters implements ParametersProvider { + + private static final ImmutableList flushPolicies = + ImmutableList.of( + FlushPolicy.minFlushSize(1_000), + FlushPolicy.minFlushSize(1_000).withMaxPendingBytes(5_000), + FlushPolicy.maxFlushSize(500_000), + FlushPolicy.minFlushSize(), + FlushPolicy.maxFlushSize()); + private static final ImmutableList closeActions = + ImmutableList.copyOf(CloseAction.values()); + public static final ImmutableList objectSizes = + ImmutableList.of(5, 500, 5_000, 500_000, 5_000_000); + + @Override + public ImmutableList parameters() { + ImmutableList.Builder builder = ImmutableList.builder(); + for (FlushPolicy fp : flushPolicies) { + for (CloseAction ca : closeActions) { + for (int size : objectSizes) { + Param param = + new Param( + ChecksummedTestContent.gen(size), + BlobAppendableUploadConfig.of().withFlushPolicy(fp).withCloseAction(ca)); + builder.add(param); + } + } + } + return builder.build(); + } + } + + public static final class Param { + private final ChecksummedTestContent content; + private final BlobAppendableUploadConfig uploadConfig; + + private Param(ChecksummedTestContent content, BlobAppendableUploadConfig uploadConfig) { + this.content = content; + this.uploadConfig = uploadConfig; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("byteCount", content) + .add("uploadConfig", uploadConfig) + .toString(); + } + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java new file mode 100644 index 000000000000..d871d9054ba0 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java @@ -0,0 +1,220 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.StatusCode; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.BucketInfo.CustomPlacementConfig; +import com.google.cloud.storage.BucketInfo.HierarchicalNamespace; +import com.google.cloud.storage.BucketInfo.IamConfiguration; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Duration; +import com.google.protobuf.FieldMask; +import com.google.storage.control.v2.BucketName; +import com.google.storage.control.v2.RapidCache; +import com.google.storage.control.v2.StorageControlClient; +import com.google.storage.control.v2.StorageControlSettings; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class ITRapidCacheTest { + + private static final String PROJECT_ID = "gcs-hyd-connector-benchmarks"; + private static StorageControlClient controlClient; + private static Storage storageClient; + private static String bucketName; + + // Shared Cache Details + private static String cacheId; + private static String cacheName; + + @BeforeClass + public static void setUpClass() throws Exception { + // Initialize standard Storage client for preprod (gRPC) + storageClient = StorageOptions.grpc() + .setProjectId(PROJECT_ID) + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .build() + .getService(); + + // Initialize StorageControl client for preprod (gRPC) + StorageControlSettings controlSettings = StorageControlSettings.newBuilder() + .setEndpoint("storage-preprod-test-grpc.googleusercontent.com:443") + .build(); + controlClient = StorageControlClient.create(controlSettings); + + // Create HNS enabled regional bucket in preprod us-central1 + bucketName = "java-storage-preprod-rapid-" + UUID.randomUUID().toString().substring(0, 8); + BucketInfo bucketInfo = BucketInfo.newBuilder(bucketName) + .setLocation("us-central1") + .setHierarchicalNamespace(HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build(); + storageClient.create(bucketInfo); + + // Define shared cache ID (forced to be the zone name by the backend) + cacheId = "us-central1-a"; + cacheName = String.format("projects/_/buckets/%s/rapidCaches/%s", bucketName, cacheId); + } + + @AfterClass + public static void tearDownClass() throws Exception { + if (storageClient != null && bucketName != null) { + try { + storageClient.delete(bucketName); + } catch (Exception e) { + System.err.println("Failed to clean up preprod bucket: " + e.getMessage()); + } + } + if (controlClient != null) { + controlClient.close(); + } + } + + // --- Test Cases (Alphabetical Sort Order matches Logical Lifecycle) --- + + @Test + public void createRapidCache() throws Exception { + RapidCache rapidCache = RapidCache.newBuilder() + .setName(cacheName) + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours + .build(); + + RapidCache created = controlClient.createRapidCacheAsync( + BucketName.format("_", bucketName), rapidCache).get(); + + assertThat(created).isNotNull(); + assertThat(created.getName()).isEqualTo(cacheName); + assertThat(created.getState()).isEqualTo("running"); + } + + @Test + public void createRapidCache_duplicate() throws Exception { + RapidCache rapidCache = RapidCache.newBuilder() + .setName(cacheName) // Use the same name as the shared cache + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .build(); + + try { + controlClient.createRapidCacheAsync( + BucketName.format("_", bucketName), rapidCache).get(); + fail("Expected AlreadyExists exception"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(ApiException.class); + ApiException apiException = (ApiException) e.getCause(); + assertThat(apiException.getStatusCode().getCode()).isEqualTo(StatusCode.Code.ALREADY_EXISTS); + } + } + + @Test + public void createRapidCache_invalidConfig() { + String invalidCacheId = "invalid-cache-" + UUID.randomUUID().toString().substring(0, 8); + String invalidCacheName = String.format("projects/_/buckets/%s/rapidCaches/%s", bucketName, invalidCacheId); + + RapidCache rapidCache = RapidCache.newBuilder() + .setName(invalidCacheName) + .setZone("invalid-zone") + .setCacheType("rapid-cache-ultra") + .build(); + + try { + controlClient.createRapidCacheAsync( + BucketName.format("_", bucketName), rapidCache).get(); + fail("Expected InvalidArgument exception"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(ApiException.class); + ApiException apiException = (ApiException) e.getCause(); + assertThat(apiException.getStatusCode().getCode()).isEqualTo(StatusCode.Code.INVALID_ARGUMENT); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Interrupted"); + } + } + + @Test + public void getRapidCache() throws Exception { + RapidCache retrieved = controlClient.getRapidCache(cacheName); + assertThat(retrieved).isNotNull(); + assertThat(retrieved.getName()).isEqualTo(cacheName); + assertThat(retrieved.getState()).isEqualTo("running"); + } + + @Test + public void getRapidCache_nonExistent() { + String nonExistentCacheName = String.format("projects/_/buckets/%s/rapidCaches/non-existent-cache", bucketName); + + try { + controlClient.getRapidCache(nonExistentCacheName); + fail("Expected NotFound exception"); + } catch (ApiException e) { + assertThat(e.getStatusCode().getCode()).isEqualTo(StatusCode.Code.NOT_FOUND); + } + } + + @Test + public void listRapidCaches() throws Exception { + StorageControlClient.ListRapidCachesPagedResponse response = + controlClient.listRapidCaches(BucketName.format("_", bucketName)); + + List names = new ArrayList<>(); + for (RapidCache rc : response.iterateAll()) { + names.add(rc.getName()); + } + + assertThat(names).contains(cacheName); + } + + @Test + @Ignore("b/483013082: UpdateRapidCache returns 500 Internal error in PreProd") + public void updateRapidCache() throws Exception { + RapidCache toUpdate = RapidCache.newBuilder() + .setName(cacheName) + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(172800).build()) // 48h + .build(); + + FieldMask updateMask = FieldMask.newBuilder().addPaths("ttl").build(); + + // No try-catch wrapping of ExecutionException. Let the test fail naturally if update fails. + RapidCache updated = controlClient.updateRapidCacheAsync(toUpdate, updateMask).get(); + assertThat(updated.getTtl().getSeconds()).isEqualTo(172800); + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java new file mode 100644 index 000000000000..0da263be7a24 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.api.core.ApiFuture; +import com.google.cloud.storage.AsyncSessionClosedException; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BlobReadSession; +import com.google.cloud.storage.RangeSpec; +import com.google.cloud.storage.ReadProjectionConfigs; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.ZeroCopySupport.DisposableByteString; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public final class ITRcuBidiReadTempTest { + + private static final String PROJECT_ID = "gcs-hyd-connector-benchmarks"; + private static final String BUCKET_NAME = "java-storage-reg-rapid-preprod-3fe2bb58"; // Reusing active bucket with running cache + private static Storage storage; + + @BeforeClass + public static void setUpClass() throws Exception { + System.out.println("Initializing storage client pointing to pre-prod endpoint..."); + storage = StorageOptions.grpc() + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .setProjectId(PROJECT_ID) + .build() + .getService(); + } + + @AfterClass + public static void tearDownClass() throws Exception { + // No cleanup of BUCKET_NAME since it is a shared pre-created bucket + } + + @Test + public void readPostStreamClose() throws Exception { + System.out.println("Running readPostStreamClose against bucket " + BUCKET_NAME); + + // Generate 5MB of random data + int dataSize = 5 * 1024 * 1024; + byte[] data = new byte[dataSize]; + new Random().nextBytes(data); + + BlobId blobId = BlobId.of(BUCKET_NAME, "test-bidi-read-close-temp-" + UUID.randomUUID()); + storage.create(BlobInfo.newBuilder(blobId).build(), data); + + try { + ApiFuture futureSession = storage.blobReadSession(blobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start a future read for the entire object + ApiFuture readFuture = session.readAs(ReadProjectionConfigs.asFutureBytes()); + + // Close the session immediately while the transfer is in flight + session.close(); + + // Resolving the future should now fail since the session is closed + try { + readFuture.get(5, TimeUnit.SECONDS); + fail("Expected ExecutionException when reading after session close"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + assertThat(cause.getCause()).isInstanceOf(AsyncSessionClosedException.class); + System.out.println("Successfully caught expected exception: " + cause.getMessage()); + } + } + } finally { + storage.delete(blobId); + } + } + + @Test + public void zeroCopyRangeReads() throws Exception { + System.out.println("Running zeroCopyRangeReads against bucket " + BUCKET_NAME); + + // Generate 1MB of random data + int dataSize = 1024 * 1024; + byte[] data = new byte[dataSize]; + new Random().nextBytes(data); + + BlobId blobId = BlobId.of(BUCKET_NAME, "test-bidi-zero-copy-temp-" + UUID.randomUUID()); + storage.create(BlobInfo.newBuilder(blobId).build(), data); + + try { + // Define 3 non-overlapping ranges + RangeSpec r1 = RangeSpec.of(0, 1000); + RangeSpec r2 = RangeSpec.of(50000, 250000); + RangeSpec r3 = RangeSpec.of(800000, 10000); + + ApiFuture futureSession = storage.blobReadSession(blobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start concurrent zero-copy range reads + ApiFuture f1 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r1)); + ApiFuture f2 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r2)); + ApiFuture f3 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r3)); + + // Resolve and verify Range 1 + try (DisposableByteString d1 = f1.get(10, TimeUnit.SECONDS)) { + assertThat(d1).isNotNull(); + byte[] expected = Arrays.copyOfRange(data, 0, 1000); + assertThat(d1.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 2 + try (DisposableByteString d2 = f2.get(10, TimeUnit.SECONDS)) { + assertThat(d2).isNotNull(); + byte[] expected = Arrays.copyOfRange(data, 50000, 50000 + 250000); + assertThat(d2.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 3 + try (DisposableByteString d3 = f3.get(10, TimeUnit.SECONDS)) { + assertThat(d3).isNotNull(); + byte[] expected = Arrays.copyOfRange(data, 800000, 800000 + 10000); + assertThat(d3.byteString().toByteArray()).isEqualTo(expected); + } + System.out.println("Successfully validated concurrent zero-copy range reads!"); + } + } finally { + storage.delete(blobId); + } + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java new file mode 100644 index 000000000000..4f9b995acd5b --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java @@ -0,0 +1,359 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.OutOfRangeException; +import com.google.cloud.storage.AsyncSessionClosedException; +import com.google.cloud.storage.BlobAppendableUpload; +import com.google.cloud.storage.BlobAppendableUpload.AppendableUploadWriteableByteChannel; +import com.google.cloud.storage.BlobAppendableUploadConfig; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BlobReadSession; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.RangeSpec; +import com.google.cloud.storage.ReadProjectionConfigs; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.ZeroCopySupport.DisposableByteString; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + backends = {Backend.PROD}, + transports = {Transport.GRPC}, + locations = { + LocationType.REGIONAL_RAPID + }) +public final class ITRcuBidiReadTest { + + @Inject public Storage storage; + @Inject public BucketInfo bucket; + @Inject public Backend backend; + @Inject public Transport transport; + + private static boolean initialized = false; + private static Storage staticStorage; + private static BlobId closeTestBlobId; + private static BlobId zeroCopyTestBlobId; + private static BlobId multipleRangeTestBlobId; + private static BlobId outOfRangeTestBlobId; + + private static byte[] closeTestData; + private static byte[] zeroCopyTestData; + private static byte[] multipleRangeTestData; + private static byte[] outOfRangeTestData; + + @Before + public void setUp() throws Exception { + if (initialized) { + return; + } + initialized = true; + staticStorage = storage; + + // Generate test data + closeTestData = new byte[5 * 1024 * 1024]; + new Random().nextBytes(closeTestData); + + zeroCopyTestData = new byte[1024 * 1024]; + new Random().nextBytes(zeroCopyTestData); + + multipleRangeTestData = new byte[2 * 1024 * 1024]; + new Random().nextBytes(multipleRangeTestData); + + outOfRangeTestData = new byte[100 * 1024]; + new Random().nextBytes(outOfRangeTestData); + + closeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-read-close-" + UUID.randomUUID()); + zeroCopyTestBlobId = BlobId.of(bucket.getName(), "test-bidi-zero-copy-" + UUID.randomUUID()); + multipleRangeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-multiple-range-" + UUID.randomUUID()); + outOfRangeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-out-of-range-" + UUID.randomUUID()); + + System.out.println("Pre-creating objects for read integration tests..."); + createObjectForWarming(closeTestBlobId, closeTestData); + createObjectForWarming(zeroCopyTestBlobId, zeroCopyTestData); + createObjectForWarming(multipleRangeTestBlobId, multipleRangeTestData); + createObjectForWarming(outOfRangeTestBlobId, outOfRangeTestData); + + if (bucket.getName().contains("reg-rapid")) { + System.out.println("Regional Rapid bucket detected. Triggering Ingest-On-Read on all objects..."); + triggerIngestOnRead(closeTestBlobId); + triggerIngestOnRead(zeroCopyTestBlobId); + triggerIngestOnRead(multipleRangeTestBlobId); + triggerIngestOnRead(outOfRangeTestBlobId); + + System.out.println("Sleeping for 30 minutes to allow background uptiering to RZ..."); + Thread.sleep(30 * 60 * 1000); // 30 minutes + System.out.println("Wake up! Continuing with integration test execution."); + } + } + + private void createObjectForWarming(BlobId blobId, byte[] data) throws Exception { + StorageClass storageClass = bucket.getStorageClass(); + if (StorageClass.valueOf("RAPID").equals(storageClass)) { + System.out.println("Bucket is ZONAL_RAPID, writing via Appendable upload with RAPID storage class..."); + BlobInfo info = BlobInfo.newBuilder(blobId) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(); + BlobAppendableUploadConfig config = BlobAppendableUploadConfig.of(); + BlobAppendableUpload upload = + storage.blobAppendableUpload(info, config, BlobWriteOption.doesNotExist()); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + ByteBuffer buffer = ByteBuffer.wrap(data); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.finalizeAndClose(); + } + upload.getResult().get(10, TimeUnit.SECONDS); + } else { + System.out.println("Writing object " + blobId + " via standard create..."); + storage.create(BlobInfo.newBuilder(blobId).build(), data); + } + } + + private void triggerIngestOnRead(BlobId blobId) { + try { + ApiFuture futureSession = storage.blobReadSession(blobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + ApiFuture readFuture = session.readAs( + ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 100))); + readFuture.get(10, TimeUnit.SECONDS); + } + } catch (Exception e) { + System.out.println("Warning: Ingest-on-read trigger failed for " + blobId + ": " + e.getMessage()); + } + } + + @AfterClass + public static void tearDownClass() { + if (staticStorage != null) { + System.out.println("Cleaning up pre-created integration test objects..."); + try { + if (closeTestBlobId != null) staticStorage.delete(closeTestBlobId); + if (zeroCopyTestBlobId != null) staticStorage.delete(zeroCopyTestBlobId); + if (multipleRangeTestBlobId != null) staticStorage.delete(multipleRangeTestBlobId); + if (outOfRangeTestBlobId != null) staticStorage.delete(outOfRangeTestBlobId); + } catch (Exception e) { + System.out.println("Error cleaning up integration test objects: " + e.getMessage()); + } + } + } + + @Test + public void readPostStreamClose() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: readPostStreamClose against bucket " + bucket.getName()); + + ApiFuture futureSession = storage.blobReadSession(closeTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start a future read for the entire object + ApiFuture readFuture = session.readAs(ReadProjectionConfigs.asFutureBytes()); + + // Close the session immediately while the transfer is in flight + session.close(); + + // Resolving the future should now fail since the session is closed + try { + readFuture.get(5, TimeUnit.SECONDS); + Assert.fail("Expected ExecutionException when reading after session close"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + assertThat(cause.getCause()).isInstanceOf(AsyncSessionClosedException.class); + System.out.println(">>> SUCCESS: readPostStreamClose verified AsyncSessionClosedException."); + } + } + } + + @Test + public void zeroCopyRangeReads() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: zeroCopyRangeReads against bucket " + bucket.getName()); + + try { + // Define 3 non-overlapping ranges + RangeSpec r1 = RangeSpec.of(0, 1000); + RangeSpec r2 = RangeSpec.of(50000, 250000); + RangeSpec r3 = RangeSpec.of(800000, 10000); + + ApiFuture futureSession = storage.blobReadSession(zeroCopyTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start concurrent zero-copy range reads + ApiFuture f1 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r1)); + ApiFuture f2 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r2)); + ApiFuture f3 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r3)); + + // Resolve and verify Range 1 + try (DisposableByteString d1 = f1.get(10, TimeUnit.SECONDS)) { + assertThat(d1).isNotNull(); + byte[] expected = Arrays.copyOfRange(zeroCopyTestData, 0, 1000); + assertThat(d1.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 2 + try (DisposableByteString d2 = f2.get(10, TimeUnit.SECONDS)) { + assertThat(d2).isNotNull(); + byte[] expected = Arrays.copyOfRange(zeroCopyTestData, 50000, 50000 + 250000); + assertThat(d2.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 3 + try (DisposableByteString d3 = f3.get(10, TimeUnit.SECONDS)) { + assertThat(d3).isNotNull(); + byte[] expected = Arrays.copyOfRange(zeroCopyTestData, 800000, 800000 + 10000); + assertThat(d3.byteString().toByteArray()).isEqualTo(expected); + } + System.out.println(">>> SUCCESS: zeroCopyRangeReads concurrent offsets verified."); + } + } finally { + // Do not delete + } + } + + @Test + public void multipleRangedRead() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: multipleRangedRead against bucket " + bucket.getName()); + + try { + // Define 4 non-overlapping ranges (each 512KB) + int rangeSize = 512 * 1024; + RangeSpec r1 = RangeSpec.of(0, rangeSize); + RangeSpec r2 = RangeSpec.of(rangeSize, rangeSize); + RangeSpec r3 = RangeSpec.of(2 * rangeSize, rangeSize); + RangeSpec r4 = RangeSpec.of(3 * rangeSize, rangeSize); + + ApiFuture futureSession = storage.blobReadSession(multipleRangeTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start concurrent byte range reads + ApiFuture f1 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r1)); + ApiFuture f2 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r2)); + ApiFuture f3 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r3)); + ApiFuture f4 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r4)); + + // Resolve and verify Range 1 + byte[] b1 = f1.get(10, TimeUnit.SECONDS); + assertThat(b1).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 0, rangeSize)); + + // Resolve and verify Range 2 + byte[] b2 = f2.get(10, TimeUnit.SECONDS); + assertThat(b2).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, rangeSize, 2 * rangeSize)); + + // Resolve and verify Range 3 + byte[] b3 = f3.get(10, TimeUnit.SECONDS); + assertThat(b3).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 2 * rangeSize, 3 * rangeSize)); + + // Resolve and verify Range 4 + byte[] b4 = f4.get(10, TimeUnit.SECONDS); + assertThat(b4).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 3 * rangeSize, 4 * rangeSize)); + + System.out.println(">>> SUCCESS: multipleRangedRead concurrent offsets verified."); + } + } finally { + // Do not delete + } + } + + @Test + public void readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404"); + + String nonExistentBucketName = "java-storage-non-existent-bucket-" + UUID.randomUUID(); + BlobId blobId = BlobId.of(nonExistentBucketName, "someobject"); + + ApiFuture futureObjectReadSession = storage.blobReadSession(blobId); + + try { + futureObjectReadSession.get(10, TimeUnit.SECONDS); + Assert.fail("Expected ExecutionException when reading from non-existent bucket"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + StorageException se = (StorageException) cause; + assertThat(se.getCode()).isIn(Arrays.asList(404, 403)); + System.out.println(">>> SUCCESS: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404 verified StorageException 404 or 403."); + } + } + + @Test + public void outOfRange() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: outOfRange against bucket " + bucket.getName()); + + try { + ApiFuture futureSession = storage.blobReadSession(outOfRangeTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start a valid range read on the session first to verify it succeeds + ApiFuture fValid = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 1000))); + byte[] bytes = fValid.get(10, TimeUnit.SECONDS); + assertThat(bytes).isEqualTo(Arrays.copyOfRange(outOfRangeTestData, 0, 1000)); + + // Start an out-of-bounds range read (offset > size) + ApiFuture fOutOfRange = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.beginAt(100 * 1024 + 1000))); + + // Verify that resolving it throws OutOfRangeException + try { + fOutOfRange.get(10, TimeUnit.SECONDS); + Assert.fail("Expected ExecutionException for out-of-bounds range read"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + assertThat(cause.getCause()).isInstanceOf(OutOfRangeException.class); + } + System.out.println(">>> SUCCESS: outOfRange verified valid read success and subsequent out of range exception."); + } + } finally { + // Do not delete + } + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java index 8aa30c50290a..814f89ba6196 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java @@ -20,7 +20,6 @@ import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.annotations.Backend; -import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.CrossRun; import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.common.base.MoreObjects; @@ -44,17 +43,14 @@ public final class CrossRunIntersection { private final @Nullable Backend backend; private final @Nullable Transport transport; private final @Nullable LocationType locationType; - private final @Nullable Colocation colocation; private CrossRunIntersection( @Nullable Backend backend, @Nullable Transport transport, - @Nullable LocationType locationType, - @Nullable Colocation colocation) { + @Nullable LocationType locationType) { this.backend = backend; this.transport = transport; this.locationType = locationType; - this.colocation = colocation; } @Nullable @@ -72,16 +68,11 @@ public LocationType getLocationType() { return locationType; } - @Nullable - public Colocation getColocation() { - return colocation; - } - public CrossRunIntersection clearBackend() { if (backend == null) { return this; } else { - return new CrossRunIntersection(null, transport, locationType, colocation); + return new CrossRunIntersection(null, transport, locationType); } } @@ -89,7 +80,7 @@ public CrossRunIntersection clearTransport() { if (transport == null) { return this; } else { - return new CrossRunIntersection(backend, null, locationType, colocation); + return new CrossRunIntersection(backend, null, locationType); } } @@ -97,15 +88,7 @@ public CrossRunIntersection clearLocationType() { if (locationType == null) { return this; } else { - return new CrossRunIntersection(backend, transport, null, colocation); - } - } - - public CrossRunIntersection clearColocation() { - if (colocation == null) { - return this; - } else { - return new CrossRunIntersection(backend, transport, locationType, null); + return new CrossRunIntersection(backend, transport, null); } } @@ -114,7 +97,7 @@ public CrossRunIntersection withBackend(Backend backend) { if (this.backend == backend) { return this; } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); + return new CrossRunIntersection(backend, transport, locationType); } } @@ -123,7 +106,7 @@ public CrossRunIntersection withTransport(Transport transport) { if (this.transport == transport) { return this; } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); + return new CrossRunIntersection(backend, transport, locationType); } } @@ -132,16 +115,7 @@ public CrossRunIntersection withLocationType(LocationType locationType) { if (this.locationType == locationType) { return this; } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); - } - } - - public CrossRunIntersection withColocation(Colocation colocation) { - requireNonNull(colocation, "colocation must be non null"); - if (this.colocation == colocation) { - return this; - } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); + return new CrossRunIntersection(backend, transport, locationType); } } @@ -170,13 +144,6 @@ public boolean anyMatch(CrossRunIntersection other) { l = l.clearLocationType(); } - if (l.colocation == null) { - r = r.clearColocation(); - } - if (r.colocation == null) { - l = l.clearColocation(); - } - return l.equals(r); } @@ -190,8 +157,7 @@ public String fmtSuiteName() { String t = transport != null ? transport.toString() : "NULL_TRANSPORT"; String b = backend != null ? backend.toString() : "NULL_BACKEND"; String lt = locationType != null ? locationType.toString() : "NULL_LOCATION"; - String c = colocation != null ? colocation.toString() : "NULL_COLOCATION"; - return String.format(Locale.US, "[%s][%s][%s][%s]", t, b, lt, c); + return String.format(Locale.US, "[%s][%s][%s]", t, b, lt); } @Override @@ -205,13 +171,12 @@ public boolean equals(Object o) { CrossRunIntersection crossRunIntersection = (CrossRunIntersection) o; return backend == crossRunIntersection.backend && transport == crossRunIntersection.transport - && locationType == crossRunIntersection.locationType - && colocation == crossRunIntersection.colocation; + && locationType == crossRunIntersection.locationType; } @Override public int hashCode() { - return Objects.hash(backend, transport, locationType, colocation); + return Objects.hash(backend, transport, locationType); } @Override @@ -220,44 +185,39 @@ public String toString() { .add("backend", backend) .add("transport", transport) .add("locationType", locationType) - .add("colocation", colocation) .toString(); } public static CrossRunIntersection of(@Nullable Backend b, @Nullable Transport t) { - return new CrossRunIntersection(b, t, null, null); + return new CrossRunIntersection(b, t, null); } public static CrossRunIntersection of( @Nullable Backend b, @Nullable Transport t, - @Nullable LocationType lt, - @Nullable Colocation c) { - return new CrossRunIntersection(b, t, lt, c); + @Nullable LocationType lt) { + return new CrossRunIntersection(b, t, lt); } public static ImmutableSet expand(CrossRun.Ignore i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); ImmutableSet locations = ImmutableSet.copyOf(i.locations()); - ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); - return expand(backends, transports, locations, colocations); + return expand(backends, transports, locations); } public static ImmutableSet expand(CrossRun.Exclude i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); ImmutableSet locations = ImmutableSet.copyOf(i.locations()); - ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); - return expand(backends, transports, locations, colocations); + return expand(backends, transports, locations); } public static ImmutableSet expand( ImmutableSet backends, ImmutableSet<@Nullable Transport> transports, - ImmutableSet<@Nullable LocationType> locations, - ImmutableSet<@Nullable Colocation> colocations) { - if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty() && colocations.isEmpty()) { + ImmutableSet<@Nullable LocationType> locations) { + if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty()) { return ImmutableSet.of(); } @@ -267,8 +227,6 @@ public static ImmutableSet expand( transports.isEmpty() ? Collections.singleton((Transport) null) : transports; Set<@Nullable LocationType> lSet = locations.isEmpty() ? Collections.singleton((LocationType) null) : locations; - Set<@Nullable Colocation> cSet = - colocations.isEmpty() ? Collections.singleton((Colocation) null) : colocations; return bSet.stream() .flatMap( @@ -277,16 +235,12 @@ public static ImmutableSet expand( .flatMap( t -> lSet.stream() - .flatMap( - l -> - cSet.stream() - .map(c -> new CrossRunIntersection(b, t, l, c))))) + .map(l -> new CrossRunIntersection(b, t, l)))) .filter( i -> !(i.backend == null && i.transport == null - && i.locationType == null - && i.colocation == null)) + && i.locationType == null)) .collect(ImmutableSet.toImmutableSet()); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java index 0cb06fa66414..4e04768d58ab 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java @@ -23,7 +23,6 @@ import com.google.cloud.storage.it.runner.annotations.Parameterized; import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; -import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.registry.Registry; @@ -171,13 +170,10 @@ private static List computeRunners(Class klass, Registry registry) .flatMap( t -> ImmutableSet.copyOf(crossRun.locations()).stream() - .flatMap( + .map( l -> - ImmutableSet.copyOf(crossRun.colocations()).stream() - .map( - c -> - CrossRunIntersection.of( - b, t, l, c))))) + CrossRunIntersection.of( + b, t, l)))) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); @@ -200,17 +196,12 @@ private static List computeRunners(Class klass, Registry registry) Backend backend = singleBackend.value(); boolean isDefault = singleBackend.locations().length == 1 - && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD - && singleBackend.colocations().length == 1 - && singleBackend.colocations()[0] == Colocation.CO_LOCATED; + && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD; return SneakyException.unwrap( () -> ImmutableSet.copyOf(singleBackend.locations()).stream() - .flatMap( - l -> - ImmutableSet.copyOf(singleBackend.colocations()).stream() - .map(c -> CrossRunIntersection.of(backend, null, l, c))) + .map(l -> CrossRunIntersection.of(backend, null, l)) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java deleted file mode 100644 index b3982eb2bc76..000000000000 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.storage.it.runner.annotations; - -public enum Colocation { - CO_LOCATED, - NON_CO_LOCATED -} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java index fe6c31b0b757..9926ce4ac0c6 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java @@ -41,8 +41,6 @@ LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; - Colocation[] colocations() default {Colocation.CO_LOCATED}; - /** * Exclude a method from being included in the generated test suite if it's backend and transport * match with those defined. When matching, if the empty set is defined as a value it will be @@ -59,8 +57,6 @@ TransportCompatibility.Transport[] transports() default {}; LocationType[] locations() default {}; - - Colocation[] colocations() default {}; } /** @@ -79,8 +75,6 @@ TransportCompatibility.Transport[] transports() default {}; LocationType[] locations() default {}; - - Colocation[] colocations() default {}; } /** diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java index 09fc2d6a17be..348620aa6a67 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java @@ -37,5 +37,4 @@ LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; - Colocation[] colocations() default {Colocation.CO_LOCATED}; } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java index f82357b533aa..ca1eaa25287e 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java @@ -38,7 +38,6 @@ import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.BucketType; -import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.LocationType; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -61,16 +60,25 @@ final class BackendResources implements ManagedLifecycle { private final Backend backend; private final ProtectedBucketNames protectedBucketNames; private final ConcurrentMap dynamicBuckets; + private final TestRunScopedInstance storageJson; + private final TestRunScopedInstance storageGrpc; + private final TestRunScopedInstance ctrl; private final ImmutableList> registryEntries; private BackendResources( Backend backend, ProtectedBucketNames protectedBucketNames, ConcurrentMap dynamicBuckets, + TestRunScopedInstance storageJson, + TestRunScopedInstance storageGrpc, + TestRunScopedInstance ctrl, ImmutableList> registryEntries) { this.backend = backend; this.protectedBucketNames = protectedBucketNames; this.dynamicBuckets = dynamicBuckets; + this.storageJson = storageJson; + this.storageGrpc = storageGrpc; + this.ctrl = ctrl; this.registryEntries = registryEntries; } @@ -98,6 +106,14 @@ public String toString() { return MoreObjects.toStringHelper(this).add("backend", backend).toString(); } + public Storage getStorage(Transport transport) { + return transport == Transport.GRPC ? storageGrpc.get().getStorage() : storageJson.get().getStorage(); + } + + public StorageControlClient getStorageControlClient() { + return ctrl.get().getCtrl(); + } + @SuppressWarnings("SwitchStatementWithTooFewBranches") static BackendResources of( Backend backend, @@ -128,7 +144,10 @@ static BackendResources of( break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry - optionsBuilder = StorageOptions.http().setOpenTelemetry(otelSdk.get().get()); + optionsBuilder = + StorageOptions.http() + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); break; } HttpStorageOptions built = optionsBuilder.build(); @@ -159,7 +178,10 @@ static BackendResources of( break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry - optionsBuilder = StorageOptions.grpc().setOpenTelemetry(otelSdk.get().get()); + optionsBuilder = + StorageOptions.grpc() + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); break; } GrpcStorageOptions built = @@ -239,6 +261,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-rp-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation(zone.get().get().getRegion()) .setRequesterPays(true) @@ -254,6 +277,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-v-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation(zone.get().get().getRegion()) .setVersioningEnabled(true) @@ -269,6 +293,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-hns-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation(zone.get().get().getRegion()) .setHierarchicalNamespace( @@ -289,6 +314,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-rapid-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation("us-central1") .setCustomPlacementConfig( @@ -319,8 +345,7 @@ static BackendResources of( CrossRunIntersection.of( backend, null, - LocationType.REGIONAL_STANDARD, - Colocation.CO_LOCATED)))); + LocationType.REGIONAL_STANDARD)))); TestRunScopedInstance objectsFixtureRp = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]/REQUESTER_PAYS", @@ -341,6 +366,9 @@ static BackendResources of( backend, protectedBucketNames, dynamicBuckets, + storageJson, + storageGrpc, + ctrl, ImmutableList.of( RegistryEntry.of( 40, Storage.class, storageJson, transportAndBackendAre(Transport.HTTP, backend)), @@ -389,11 +417,9 @@ static BackendResources of( private static final class BucketKey { private final LocationType locationType; - private final Colocation colocation; - private BucketKey(LocationType locationType, Colocation colocation) { + private BucketKey(LocationType locationType) { this.locationType = locationType; - this.colocation = colocation; } @Override @@ -405,19 +431,18 @@ public boolean equals(Object o) { return false; } BucketKey bucketKey = (BucketKey) o; - return locationType == bucketKey.locationType && colocation == bucketKey.colocation; + return locationType == bucketKey.locationType; } @Override public int hashCode() { - return java.util.Objects.hash(locationType, colocation); + return java.util.Objects.hash(locationType); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("locationType", locationType) - .add("colocation", colocation) .toString(); } } @@ -449,16 +474,12 @@ private DynamicBucketLifecycle( @Override public BucketInfo resolve(FrameworkField ff, CrossRunIntersection crossRunIntersection) { LocationType lt = crossRunIntersection.getLocationType(); - Colocation col = crossRunIntersection.getColocation(); if (lt == null) { lt = LocationType.REGIONAL_STANDARD; } - if (col == null) { - col = Colocation.CO_LOCATED; - } - BucketKey key = new BucketKey(lt, col); + BucketKey key = new BucketKey(lt); BucketInfoShim shim = dynamicBuckets.computeIfAbsent(key, this::createBucketShim); return (BucketInfo) shim.get(); } @@ -468,22 +489,20 @@ private BucketInfoShim createBucketShim(BucketKey key) { String region = z.getRegion(); String zoneName = z.getZone(); - String rAlt; - String vAlt; - if ("us-east1".equals(region)) { - rAlt = "us-central1"; - vAlt = "us-central1-a"; - } else { - rAlt = "us-east1"; - vAlt = "us-east1-b"; - } - String targetRegion = region; String targetZone = zoneName; - if (key.colocation == Colocation.NON_CO_LOCATED) { - targetRegion = rAlt; - targetZone = vAlt; + Storage storageClientToUse = storageJson.get().getStorage(); + StorageControlClient controlClientToUse = ctrl.get().getCtrl(); + + if (key.locationType == LocationType.REGIONAL_RAPID) { + targetRegion = "us-central1"; + targetZone = "us-central1-a"; + if (backend == Backend.PROD) { + BackendResources preprod = Registry.getInstance().getPreProdBackendResources(); + storageClientToUse = preprod.getStorage(Transport.GRPC); + controlClientToUse = preprod.getStorageControlClient(); + } } BucketInfo.Builder builder; @@ -540,11 +559,12 @@ private BucketInfoShim createBucketShim(BucketKey key) { BucketInfoShim shim = new BucketInfoShim( + backend, builder.build(), key.locationType, targetZone, - storageJson.get().getStorage(), - ctrl.get().getCtrl()); + storageClientToUse, + controlClientToUse); shim.start(); return shim; diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java index 8a0e3b71f7d2..69be2b41df30 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java @@ -22,6 +22,7 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.it.BucketCleaner; +import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.protobuf.Duration; import com.google.storage.control.v2.BucketName; @@ -32,6 +33,7 @@ /** Shim to lift a BucketInfo to be a managed bucket instance */ final class BucketInfoShim implements ManagedLifecycle { + private final Backend backend; private final BucketInfo bucketInfo; private final LocationType locationType; private final String targetZone; @@ -41,15 +43,21 @@ final class BucketInfoShim implements ManagedLifecycle { private BucketInfo createdBucket; BucketInfoShim(BucketInfo bucketInfo, Storage s, StorageControlClient ctrl) { - this(bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); + this(Backend.PROD, bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); + } + + BucketInfoShim(Backend backend, BucketInfo bucketInfo, Storage s, StorageControlClient ctrl) { + this(backend, bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); } BucketInfoShim( + Backend backend, BucketInfo bucketInfo, LocationType locationType, String targetZone, Storage s, StorageControlClient ctrl) { + this.backend = backend; this.bucketInfo = bucketInfo; this.locationType = locationType; this.targetZone = targetZone; @@ -63,13 +71,22 @@ public BucketInfo getBucketInfo() { @Override public Object get() { - return bucketInfo; + return createdBucket != null ? createdBucket : bucketInfo; } @Override public void start() { try { + if (locationType == LocationType.REGIONAL_RAPID && backend != Backend.TEST_BENCH) { + System.out.println(">>> REUSING static pre-created RCU bucket java-storage-reg-rapid-preprod-3fe2bb58 for REGIONAL_RAPID test!"); + createdBucket = BucketInfo.newBuilder("java-storage-reg-rapid-preprod-3fe2bb58") + .setLocation("US-CENTRAL1") + .build(); + return; + } + System.out.println("Starting resource creation for LocationType: " + locationType + " in zone: " + targetZone); createdBucket = s.create(bucketInfo).asBucketInfo(); + System.out.println("Successfully created bucket: " + createdBucket.getName() + " (Location: " + createdBucket.getLocation() + ")"); if (locationType == LocationType.REGIONAL_RAPID) { if (ctrl == null) { throw new IllegalStateException( @@ -89,17 +106,29 @@ public void start() { .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours .build(); try { + System.out.println("Submitting CreateRapidCache LRO for bucket: " + createdBucket.getName() + " in zone: " + targetZone); ctrl.createRapidCacheAsync(BucketName.format("_", createdBucket.getName()), rapidCache) - .get(); + .get(30, java.util.concurrent.TimeUnit.SECONDS); + System.out.println("Successfully created Rapid Cache in zone: " + targetZone); + } catch (java.util.concurrent.TimeoutException te) { + System.out.println("WARNING: CreateRapidCache LRO timed out after 30s. Skipping test."); + assumeTrue( + "Skipping test because Rapid Cache creation LRO timed out (30s) in zone: " + targetZone, + false); } catch (Exception e) { - throw new RuntimeException("Failed to create Rapid Cache: " + e.getMessage(), e); + System.out.println("WARNING: CreateRapidCache LRO failed: " + e.getMessage() + ". Skipping test."); + assumeTrue( + "Skipping test due to failure during Rapid Cache creation: " + e.getMessage(), + false); } } } catch (StorageException se) { String msg = se.getMessage().toLowerCase(Locale.US); + System.out.println("StorageException caught during resource creation: " + msg); if (se.getCode() == 400 && (msg.contains("not a valid zone in location")) || msg.contains("custom placement config") || msg.contains("zonal")) { + System.out.println("Skipping test: setup unavailable in current zone."); assumeTrue( "Skipping test due to bucket setup unavailable in current zone. (" + msg + ")", false); } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java index 0db457ae507f..8bdb507f71c3 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java @@ -16,8 +16,10 @@ package com.google.cloud.storage.it.runner.registry; +import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.TestInitializer; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -106,7 +108,6 @@ public final class Registry extends RunListener { registryEntry(3, Backend.class, CrossRunIntersection::getBackend), registryEntry(4, Transport.class, CrossRunIntersection::getTransport)) .addAll(prodBackendResources.getRegistryEntries()) - .addAll(preProdBackendResources.getRegistryEntries()) .addAll(testBenchBackendResource.getRegistryEntries()) .build(); @@ -157,6 +158,10 @@ TestBench testBench() { return testBench.get(); } + BackendResources getPreProdBackendResources() { + return preProdBackendResources; + } + @Nullable public Description getCurrentTest() { return currentTest.get().desc; @@ -224,6 +229,12 @@ public Object resolve(FrameworkField ff, CrossRunIntersection crossRunIntersecti } else { finalCrossRunIntersection = crossRunIntersection; } + if (ff.getType() == Storage.class + && finalCrossRunIntersection.getLocationType() == LocationType.REGIONAL_RAPID + && finalCrossRunIntersection.getBackend() == Backend.PROD) { + return preProdBackendResources.getStorage(finalCrossRunIntersection.getTransport()); + } + Optional> first = entries.stream() .filter(re -> re.getPredicate().test(ff, finalCrossRunIntersection)) From ee8a73799cca0a6836af6f9f7ac394aacef777c2 Mon Sep 17 00:00:00 2001 From: Nidhi Nandwani Date: Fri, 21 Aug 2026 09:02:56 +0000 Subject: [PATCH 3/4] test: add bidiWriteObjectRedirectedError_redirectCounterResetOnResponse to ITAppendableUploadFakeTest Verify that successful responses on the BidiWriteObject stream (such as a successful chunk persist or reconnect state lookup response) correctly reset the client's consecutive redirect counter to 0. This ensures that the client is not blocked by the max consecutive redirect limit (3) when redirects are spread out. [Generated-by: AI] --- .../storage/ITAppendableUploadFakeTest.java | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java index 78d07c4d9425..c113eb1d702a 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java @@ -291,6 +291,211 @@ public void bidiWriteObjectRedirectedError_maxAttempts() throws Exception { } } + @Test + public void bidiWriteObjectRedirectedError_redirectCounterResetOnResponse() throws Exception { + String routingToken1 = "routingToken1"; + String routingToken2 = "routingToken2"; + String routingToken3 = "routingToken3"; + String routingToken4 = "routingToken4"; + String routingToken5 = "routingToken5"; + + BidiWriteHandle writeHandle1 = + BidiWriteHandle.newBuilder() + .setHandle(ByteString.copyFromUtf8("handle-1")) + .build(); + BidiWriteHandle writeHandle2 = + BidiWriteHandle.newBuilder() + .setHandle(ByteString.copyFromUtf8("handle-2")) + .build(); + BidiWriteHandle writeHandle3 = + BidiWriteHandle.newBuilder() + .setHandle(ByteString.copyFromUtf8("handle-3")) + .build(); + + // 1. First write of "ABC". + BidiWriteObjectRequest req_open_abc = BidiUploadTestUtils.withFlushAndStateLookup(open_abc); + BidiWriteObjectResponse res_abc_h1 = res_abc.toBuilder().setWriteHandle(writeHandle1).build(); + + // 2. Second write of "DEF". + BidiWriteObjectRequest req_def = BidiUploadTestUtils.withFlushAndStateLookup(def); + BidiWriteObjectResponse res_def_success = + BidiWriteObjectResponse.newBuilder() + .setWriteHandle(writeHandle1) + .setPersistedSize(6) + .build(); + + // 3. Reconnect to routingToken1 + BidiWriteObjectRequest reconnect_token1 = + BidiWriteObjectRequest.newBuilder() + .setAppendObjectSpec( + AppendObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(METADATA.getGeneration()) + .setWriteHandle(writeHandle1) + .setRoutingToken(routingToken1) + .build()) + .setStateLookup(true) + .build(); + BidiWriteObjectResponse res_lookup_token1 = + BidiWriteObjectResponse.newBuilder() + .setWriteHandle(writeHandle1) + .setPersistedSize(3) + .build(); + + // 5. Third write of "GHI". + BidiWriteObjectRequest req_ghi = BidiUploadTestUtils.withFlushAndStateLookup(ghi); + BidiWriteObjectResponse res_ghi_success = + BidiWriteObjectResponse.newBuilder() + .setWriteHandle(writeHandle2) + .setPersistedSize(9) + .build(); + + // 6. Reconnect to routingToken2 + BidiWriteObjectRequest reconnect_token2 = + BidiWriteObjectRequest.newBuilder() + .setAppendObjectSpec( + AppendObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(METADATA.getGeneration()) + .setWriteHandle(writeHandle1) + .setRoutingToken(routingToken2) + .build()) + .setStateLookup(true) + .build(); + BidiWriteObjectResponse res_lookup_token2 = + BidiWriteObjectResponse.newBuilder() + .setWriteHandle(writeHandle2) + .setPersistedSize(6) + .build(); + + // 8. Fourth write of "J" (finalize/finish write) + BidiWriteObjectRequest req_j_finish = j_finish; + BidiWriteObjectResponse res_j_final = resource_10.toBuilder().setWriteHandle(writeHandle3).build(); + + // 9. Reconnect to routingToken3 + BidiWriteObjectRequest reconnect_token3 = + BidiWriteObjectRequest.newBuilder() + .setAppendObjectSpec( + AppendObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(METADATA.getGeneration()) + .setWriteHandle(writeHandle2) + .setRoutingToken(routingToken3) + .build()) + .setStateLookup(true) + .build(); + + // 10. Reconnect to routingToken4 + BidiWriteObjectRequest reconnect_token4 = + BidiWriteObjectRequest.newBuilder() + .setAppendObjectSpec( + AppendObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(METADATA.getGeneration()) + .setWriteHandle(writeHandle2) + .setRoutingToken(routingToken4) + .build()) + .setStateLookup(true) + .build(); + + // 11. Reconnect to routingToken5 + BidiWriteObjectRequest reconnect_token5 = + BidiWriteObjectRequest.newBuilder() + .setAppendObjectSpec( + AppendObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(METADATA.getGeneration()) + .setWriteHandle(writeHandle2) + .setRoutingToken(routingToken5) + .build()) + .setStateLookup(true) + .build(); + BidiWriteObjectResponse res_lookup_token5 = + BidiWriteObjectResponse.newBuilder() + .setWriteHandle(writeHandle3) + .setPersistedSize(9) + .build(); + + AtomicInteger defCounter = new AtomicInteger(); + Consumer> defHandler = + respond -> { + if (defCounter.getAndIncrement() == 0) { + respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken1))); + } else { + respond.onNext(res_def_success); + } + }; + + AtomicInteger ghiCounter = new AtomicInteger(); + Consumer> ghiHandler = + respond -> { + if (ghiCounter.getAndIncrement() == 0) { + respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken2))); + } else { + respond.onNext(res_ghi_success); + } + }; + + AtomicInteger jCounter = new AtomicInteger(); + Consumer> jHandler = + respond -> { + if (jCounter.getAndIncrement() == 0) { + respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken3))); + } else { + respond.onNext(res_j_final); + respond.onCompleted(); + } + }; + + FakeStorage fake = + FakeStorage.of( + ImmutableMap.>>builder() + .put(req_open_abc, respond -> respond.onNext(res_abc_h1)) + .put(req_def, defHandler) + .put(reconnect_token1, respond -> respond.onNext(res_lookup_token1)) + .put(req_ghi, ghiHandler) + .put(reconnect_token2, respond -> respond.onNext(res_lookup_token2)) + .put(req_j_finish, jHandler) + .put(reconnect_token3, respond -> respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken4)))) + .put(reconnect_token4, respond -> respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken5)))) + .put(reconnect_token5, respond -> respond.onNext(res_lookup_token5)) + .build()); + + try (FakeServer fakeServer = FakeServer.of(fake); + Storage storage = + fakeServer.getGrpcStorageOptions().toBuilder() + .setRetrySettings( + fakeServer.getGrpcStorageOptions().getRetrySettings().toBuilder() + .setRetryDelayMultiplier(1.0) + .setInitialRetryDelayDuration(Duration.ofMillis(10)) + .build()) + .build() + .getService()) { + + BlobId id = BlobId.of("b", "o"); + BlobAppendableUploadConfig config = + BlobAppendableUploadConfig.of() + .withFlushPolicy(FlushPolicy.maxFlushSize(3)) + .withCloseAction(CloseAction.FINALIZE_WHEN_CLOSING); + BlobAppendableUpload b = + storage.blobAppendableUpload(BlobInfo.newBuilder(id).build(), config); + try (AppendableUploadWriteableByteChannel channel = b.open()) { + ByteBuffer wrap = ByteBuffer.wrap(content.getBytes()); + Buffers.emptyTo(wrap, channel); + } + + // Verification + ApiFuture resultFuture = b.getResult(); + BlobInfo finalMetadata = resultFuture.get(3, TimeUnit.SECONDS); + assertThat(finalMetadata.getSize()).isEqualTo(10L); + } + } + /** * We use a small segmenter (3 byte segments) and flush "ABCDEFGHIJ". We make sure that this * resolves to segments of "ABC"/"DEF"/"GHI"/"J". From 61f4ee8eb812361ff981f9284521e2603dcdf01f Mon Sep 17 00:00:00 2001 From: Nidhi Nandwani Date: Fri, 21 Aug 2026 09:34:27 +0000 Subject: [PATCH 4/4] style: format java files using spotify-java-formatter Reformat modified and new test/runner files to comply with Spotify Java format checks in the CI pipelines. [Generated-by: AI] --- .../storage/ITAppendableUploadFakeTest.java | 31 ++--- .../storage/ITObjectReadSessionFakeTest.java | 26 +++-- .../cloud/storage/ITRcuBidiWriteTest.java | 28 +++-- .../cloud/storage/it/ITRapidCacheTest.java | 106 +++++++++--------- .../storage/it/ITRcuBidiReadTempTest.java | 15 +-- .../cloud/storage/it/ITRcuBidiReadTest.java | 65 ++++++----- .../it/runner/CrossRunIntersection.java | 17 +-- .../storage/it/runner/StorageITRunner.java | 7 +- .../it/runner/annotations/SingleBackend.java | 1 - .../it/runner/registry/BackendResources.java | 22 ++-- .../it/runner/registry/BucketInfoShim.java | 38 +++++-- .../storage/it/runner/registry/Registry.java | 2 +- 12 files changed, 193 insertions(+), 165 deletions(-) diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java index c113eb1d702a..10604170938e 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadFakeTest.java @@ -300,17 +300,11 @@ public void bidiWriteObjectRedirectedError_redirectCounterResetOnResponse() thro String routingToken5 = "routingToken5"; BidiWriteHandle writeHandle1 = - BidiWriteHandle.newBuilder() - .setHandle(ByteString.copyFromUtf8("handle-1")) - .build(); + BidiWriteHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-1")).build(); BidiWriteHandle writeHandle2 = - BidiWriteHandle.newBuilder() - .setHandle(ByteString.copyFromUtf8("handle-2")) - .build(); + BidiWriteHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-2")).build(); BidiWriteHandle writeHandle3 = - BidiWriteHandle.newBuilder() - .setHandle(ByteString.copyFromUtf8("handle-3")) - .build(); + BidiWriteHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-3")).build(); // 1. First write of "ABC". BidiWriteObjectRequest req_open_abc = BidiUploadTestUtils.withFlushAndStateLookup(open_abc); @@ -372,7 +366,8 @@ public void bidiWriteObjectRedirectedError_redirectCounterResetOnResponse() thro // 8. Fourth write of "J" (finalize/finish write) BidiWriteObjectRequest req_j_finish = j_finish; - BidiWriteObjectResponse res_j_final = resource_10.toBuilder().setWriteHandle(writeHandle3).build(); + BidiWriteObjectResponse res_j_final = + resource_10.toBuilder().setWriteHandle(writeHandle3).build(); // 9. Reconnect to routingToken3 BidiWriteObjectRequest reconnect_token3 = @@ -454,15 +449,25 @@ public void bidiWriteObjectRedirectedError_redirectCounterResetOnResponse() thro FakeStorage fake = FakeStorage.of( - ImmutableMap.>>builder() + ImmutableMap + .>> + builder() .put(req_open_abc, respond -> respond.onNext(res_abc_h1)) .put(req_def, defHandler) .put(reconnect_token1, respond -> respond.onNext(res_lookup_token1)) .put(req_ghi, ghiHandler) .put(reconnect_token2, respond -> respond.onNext(res_lookup_token2)) .put(req_j_finish, jHandler) - .put(reconnect_token3, respond -> respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken4)))) - .put(reconnect_token4, respond -> respond.onError(packRedirectIntoAbortedException(makeRedirect(routingToken5)))) + .put( + reconnect_token3, + respond -> + respond.onError( + packRedirectIntoAbortedException(makeRedirect(routingToken4)))) + .put( + reconnect_token4, + respond -> + respond.onError( + packRedirectIntoAbortedException(makeRedirect(routingToken5)))) .put(reconnect_token5, respond -> respond.onNext(res_lookup_token5)) .build()); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java index eaeb4019bcee..d9c965a57b45 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java @@ -1845,11 +1845,16 @@ private static Consumer> onRedirect( @Test public void bidiReadObjectRedirectedError_redirectCounterResetOnResponse() throws Exception { - BidiReadHandle handle1 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-1")).build(); - BidiReadHandle handle2 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-2")).build(); - BidiReadHandle handle3 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-3")).build(); - BidiReadHandle handle4 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-4")).build(); - BidiReadHandle handle5 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-5")).build(); + BidiReadHandle handle1 = + BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-1")).build(); + BidiReadHandle handle2 = + BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-2")).build(); + BidiReadHandle handle3 = + BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-3")).build(); + BidiReadHandle handle4 = + BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-4")).build(); + BidiReadHandle handle5 = + BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-5")).build(); BidiReadObjectRequest req_read_1 = read(1, 10, 5); @@ -1867,9 +1872,7 @@ public void bidiReadObjectRedirectedError_redirectCounterResetOnResponse() throw .build(); BidiReadObjectRequest req_read_2 = - BidiReadObjectRequest.newBuilder() - .addReadRanges(getReadRange(2, 15, 5)) - .build(); + BidiReadObjectRequest.newBuilder().addReadRanges(getReadRange(2, 15, 5)).build(); BidiReadObjectRequest req_read_2_redirected_2 = BidiReadObjectRequest.newBuilder() @@ -1885,9 +1888,7 @@ public void bidiReadObjectRedirectedError_redirectCounterResetOnResponse() throw .build(); BidiReadObjectRequest req_read_3 = - BidiReadObjectRequest.newBuilder() - .addReadRanges(getReadRange(3, 20, 5)) - .build(); + BidiReadObjectRequest.newBuilder().addReadRanges(getReadRange(3, 20, 5)).build(); BidiReadObjectRequest req_read_3_redirected_3 = BidiReadObjectRequest.newBuilder() @@ -1966,7 +1967,8 @@ public void bidiReadObjectRedirectedError_redirectCounterResetOnResponse() throw FakeStorage fake = FakeStorage.of( - ImmutableMap.>>builder() + ImmutableMap + .>>builder() .put(REQ_OPEN, respond -> respond.onNext(RES_OPEN)) .put(req_read_1, onRedirect(handle1, "token-1")) .put(req_read_1_redirected_1, respond -> respond.onNext(res_read_1)) diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java index dff3a407d577..29d7d1f3c647 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java @@ -59,9 +59,7 @@ @CrossRun( backends = {Backend.PROD}, transports = Transport.GRPC, - locations = { - LocationType.REGIONAL_RAPID - }) + locations = {LocationType.REGIONAL_RAPID}) @Parameterized(UploadConfigParameters.class) public final class ITRcuBidiWriteTest { @@ -88,7 +86,8 @@ public void appendableUpload_emptyObject() storage.blobAppendableUpload( BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); upload.open().close(); @@ -113,7 +112,8 @@ public void appendableUpload_bytes() storage.blobAppendableUpload( BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); // cut out the middle + 1 byte int length = p.content.length(); @@ -147,7 +147,8 @@ public void explicitFlush() storage.blobAppendableUpload( BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); try (AppendableUploadWriteableByteChannel channel = upload.open()) { ByteBuffer src = p.content.asByteBuffer(); @@ -200,7 +201,8 @@ public void appendableBlobUploadTakeover() throws Exception { storage.blobAppendableUpload( BlobInfo.newBuilder(done1.getBlobId()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { int written = Buffers.emptyTo(ByteBuffer.wrap(c2.getBytes()), channel); assertThat(written).isEqualTo(c2.length()); @@ -268,7 +270,8 @@ public void takeoverJustToFinalizeWorks() throws Exception { storage.blobAppendableUpload( BlobInfo.newBuilder(done1.getBlobId()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); takeOver.open().finalizeAndClose(); BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); @@ -363,7 +366,8 @@ public void takeoverJustToFinalizeWithIncorrectChecksumFails() throws Exception storage.blobAppendableUpload( BlobInfo.newBuilder(done1.getBlobId()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { @@ -407,7 +411,8 @@ public void takeoverAndAppendWithCorrectChecksumWorks() throws Exception { storage.blobAppendableUpload( BlobInfo.newBuilder(done1.getBlobId()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); @@ -447,7 +452,8 @@ public void takeoverAndAppendWithIncorrectChecksumFails() throws Exception { storage.blobAppendableUpload( BlobInfo.newBuilder(done1.getBlobId()) .setStorageClass(StorageClass.valueOf("RAPID")) - .build(), p.uploadConfig); + .build(), + p.uploadConfig); try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java index d871d9054ba0..256d4795e5bc 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java @@ -22,20 +22,16 @@ import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode; import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.BucketInfo.CustomPlacementConfig; import com.google.cloud.storage.BucketInfo.HierarchicalNamespace; import com.google.cloud.storage.BucketInfo.IamConfiguration; import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageClass; import com.google.cloud.storage.StorageOptions; -import com.google.common.collect.ImmutableList; import com.google.protobuf.Duration; import com.google.protobuf.FieldMask; import com.google.storage.control.v2.BucketName; import com.google.storage.control.v2.RapidCache; import com.google.storage.control.v2.StorageControlClient; import com.google.storage.control.v2.StorageControlSettings; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -62,28 +58,29 @@ public class ITRapidCacheTest { @BeforeClass public static void setUpClass() throws Exception { // Initialize standard Storage client for preprod (gRPC) - storageClient = StorageOptions.grpc() - .setProjectId(PROJECT_ID) - .setHost("storage-preprod-test-grpc.googleusercontent.com:443") - .build() - .getService(); + storageClient = + StorageOptions.grpc() + .setProjectId(PROJECT_ID) + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .build() + .getService(); // Initialize StorageControl client for preprod (gRPC) - StorageControlSettings controlSettings = StorageControlSettings.newBuilder() - .setEndpoint("storage-preprod-test-grpc.googleusercontent.com:443") - .build(); + StorageControlSettings controlSettings = + StorageControlSettings.newBuilder() + .setEndpoint("storage-preprod-test-grpc.googleusercontent.com:443") + .build(); controlClient = StorageControlClient.create(controlSettings); // Create HNS enabled regional bucket in preprod us-central1 bucketName = "java-storage-preprod-rapid-" + UUID.randomUUID().toString().substring(0, 8); - BucketInfo bucketInfo = BucketInfo.newBuilder(bucketName) - .setLocation("us-central1") - .setHierarchicalNamespace(HierarchicalNamespace.newBuilder().setEnabled(true).build()) - .setIamConfiguration( - IamConfiguration.newBuilder() - .setIsUniformBucketLevelAccessEnabled(true) - .build()) - .build(); + BucketInfo bucketInfo = + BucketInfo.newBuilder(bucketName) + .setLocation("us-central1") + .setHierarchicalNamespace(HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder().setIsUniformBucketLevelAccessEnabled(true).build()) + .build(); storageClient.create(bucketInfo); // Define shared cache ID (forced to be the zone name by the backend) @@ -109,15 +106,16 @@ public static void tearDownClass() throws Exception { @Test public void createRapidCache() throws Exception { - RapidCache rapidCache = RapidCache.newBuilder() - .setName(cacheName) - .setZone("us-central1-a") - .setCacheType("rapid-cache-ultra") - .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours - .build(); + RapidCache rapidCache = + RapidCache.newBuilder() + .setName(cacheName) + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours + .build(); - RapidCache created = controlClient.createRapidCacheAsync( - BucketName.format("_", bucketName), rapidCache).get(); + RapidCache created = + controlClient.createRapidCacheAsync(BucketName.format("_", bucketName), rapidCache).get(); assertThat(created).isNotNull(); assertThat(created.getName()).isEqualTo(cacheName); @@ -126,15 +124,15 @@ public void createRapidCache() throws Exception { @Test public void createRapidCache_duplicate() throws Exception { - RapidCache rapidCache = RapidCache.newBuilder() - .setName(cacheName) // Use the same name as the shared cache - .setZone("us-central1-a") - .setCacheType("rapid-cache-ultra") - .build(); + RapidCache rapidCache = + RapidCache.newBuilder() + .setName(cacheName) // Use the same name as the shared cache + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .build(); try { - controlClient.createRapidCacheAsync( - BucketName.format("_", bucketName), rapidCache).get(); + controlClient.createRapidCacheAsync(BucketName.format("_", bucketName), rapidCache).get(); fail("Expected AlreadyExists exception"); } catch (ExecutionException e) { assertThat(e.getCause()).isInstanceOf(ApiException.class); @@ -146,22 +144,24 @@ public void createRapidCache_duplicate() throws Exception { @Test public void createRapidCache_invalidConfig() { String invalidCacheId = "invalid-cache-" + UUID.randomUUID().toString().substring(0, 8); - String invalidCacheName = String.format("projects/_/buckets/%s/rapidCaches/%s", bucketName, invalidCacheId); + String invalidCacheName = + String.format("projects/_/buckets/%s/rapidCaches/%s", bucketName, invalidCacheId); - RapidCache rapidCache = RapidCache.newBuilder() - .setName(invalidCacheName) - .setZone("invalid-zone") - .setCacheType("rapid-cache-ultra") - .build(); + RapidCache rapidCache = + RapidCache.newBuilder() + .setName(invalidCacheName) + .setZone("invalid-zone") + .setCacheType("rapid-cache-ultra") + .build(); try { - controlClient.createRapidCacheAsync( - BucketName.format("_", bucketName), rapidCache).get(); + controlClient.createRapidCacheAsync(BucketName.format("_", bucketName), rapidCache).get(); fail("Expected InvalidArgument exception"); } catch (ExecutionException e) { assertThat(e.getCause()).isInstanceOf(ApiException.class); ApiException apiException = (ApiException) e.getCause(); - assertThat(apiException.getStatusCode().getCode()).isEqualTo(StatusCode.Code.INVALID_ARGUMENT); + assertThat(apiException.getStatusCode().getCode()) + .isEqualTo(StatusCode.Code.INVALID_ARGUMENT); } catch (InterruptedException e) { Thread.currentThread().interrupt(); fail("Interrupted"); @@ -178,7 +178,8 @@ public void getRapidCache() throws Exception { @Test public void getRapidCache_nonExistent() { - String nonExistentCacheName = String.format("projects/_/buckets/%s/rapidCaches/non-existent-cache", bucketName); + String nonExistentCacheName = + String.format("projects/_/buckets/%s/rapidCaches/non-existent-cache", bucketName); try { controlClient.getRapidCache(nonExistentCacheName); @@ -190,7 +191,7 @@ public void getRapidCache_nonExistent() { @Test public void listRapidCaches() throws Exception { - StorageControlClient.ListRapidCachesPagedResponse response = + StorageControlClient.ListRapidCachesPagedResponse response = controlClient.listRapidCaches(BucketName.format("_", bucketName)); List names = new ArrayList<>(); @@ -204,12 +205,13 @@ public void listRapidCaches() throws Exception { @Test @Ignore("b/483013082: UpdateRapidCache returns 500 Internal error in PreProd") public void updateRapidCache() throws Exception { - RapidCache toUpdate = RapidCache.newBuilder() - .setName(cacheName) - .setZone("us-central1-a") - .setCacheType("rapid-cache-ultra") - .setTtl(Duration.newBuilder().setSeconds(172800).build()) // 48h - .build(); + RapidCache toUpdate = + RapidCache.newBuilder() + .setName(cacheName) + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(172800).build()) // 48h + .build(); FieldMask updateMask = FieldMask.newBuilder().addPaths("ttl").build(); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java index 0da263be7a24..05dedcf0d875 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java @@ -30,7 +30,6 @@ import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.ZeroCopySupport.DisposableByteString; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Random; import java.util.UUID; @@ -43,17 +42,19 @@ public final class ITRcuBidiReadTempTest { private static final String PROJECT_ID = "gcs-hyd-connector-benchmarks"; - private static final String BUCKET_NAME = "java-storage-reg-rapid-preprod-3fe2bb58"; // Reusing active bucket with running cache + private static final String BUCKET_NAME = + "java-storage-reg-rapid-preprod-3fe2bb58"; // Reusing active bucket with running cache private static Storage storage; @BeforeClass public static void setUpClass() throws Exception { System.out.println("Initializing storage client pointing to pre-prod endpoint..."); - storage = StorageOptions.grpc() - .setHost("storage-preprod-test-grpc.googleusercontent.com:443") - .setProjectId(PROJECT_ID) - .build() - .getService(); + storage = + StorageOptions.grpc() + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .setProjectId(PROJECT_ID) + .build() + .getService(); } @AfterClass diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java index 4f9b995acd5b..71834d63b5ef 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java @@ -27,13 +27,13 @@ import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.BlobReadSession; -import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.StorageClass; -import com.google.cloud.storage.StorageException; import com.google.cloud.storage.RangeSpec; import com.google.cloud.storage.ReadProjectionConfigs; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageException; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.ZeroCopySupport.DisposableByteString; import com.google.cloud.storage.it.runner.StorageITRunner; @@ -41,9 +41,7 @@ import com.google.cloud.storage.it.runner.annotations.CrossRun; import com.google.cloud.storage.it.runner.annotations.Inject; import com.google.cloud.storage.it.runner.annotations.LocationType; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Random; import java.util.UUID; @@ -60,9 +58,7 @@ @CrossRun( backends = {Backend.PROD}, transports = {Transport.GRPC}, - locations = { - LocationType.REGIONAL_RAPID - }) + locations = {LocationType.REGIONAL_RAPID}) public final class ITRcuBidiReadTest { @Inject public Storage storage; @@ -105,8 +101,10 @@ public void setUp() throws Exception { closeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-read-close-" + UUID.randomUUID()); zeroCopyTestBlobId = BlobId.of(bucket.getName(), "test-bidi-zero-copy-" + UUID.randomUUID()); - multipleRangeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-multiple-range-" + UUID.randomUUID()); - outOfRangeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-out-of-range-" + UUID.randomUUID()); + multipleRangeTestBlobId = + BlobId.of(bucket.getName(), "test-bidi-multiple-range-" + UUID.randomUUID()); + outOfRangeTestBlobId = + BlobId.of(bucket.getName(), "test-bidi-out-of-range-" + UUID.randomUUID()); System.out.println("Pre-creating objects for read integration tests..."); createObjectForWarming(closeTestBlobId, closeTestData); @@ -115,7 +113,8 @@ public void setUp() throws Exception { createObjectForWarming(outOfRangeTestBlobId, outOfRangeTestData); if (bucket.getName().contains("reg-rapid")) { - System.out.println("Regional Rapid bucket detected. Triggering Ingest-On-Read on all objects..."); + System.out.println( + "Regional Rapid bucket detected. Triggering Ingest-On-Read on all objects..."); triggerIngestOnRead(closeTestBlobId); triggerIngestOnRead(zeroCopyTestBlobId); triggerIngestOnRead(multipleRangeTestBlobId); @@ -130,10 +129,10 @@ public void setUp() throws Exception { private void createObjectForWarming(BlobId blobId, byte[] data) throws Exception { StorageClass storageClass = bucket.getStorageClass(); if (StorageClass.valueOf("RAPID").equals(storageClass)) { - System.out.println("Bucket is ZONAL_RAPID, writing via Appendable upload with RAPID storage class..."); - BlobInfo info = BlobInfo.newBuilder(blobId) - .setStorageClass(StorageClass.valueOf("RAPID")) - .build(); + System.out.println( + "Bucket is ZONAL_RAPID, writing via Appendable upload with RAPID storage class..."); + BlobInfo info = + BlobInfo.newBuilder(blobId).setStorageClass(StorageClass.valueOf("RAPID")).build(); BlobAppendableUploadConfig config = BlobAppendableUploadConfig.of(); BlobAppendableUpload upload = storage.blobAppendableUpload(info, config, BlobWriteOption.doesNotExist()); @@ -155,12 +154,14 @@ private void triggerIngestOnRead(BlobId blobId) { try { ApiFuture futureSession = storage.blobReadSession(blobId); try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { - ApiFuture readFuture = session.readAs( - ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 100))); + ApiFuture readFuture = + session.readAs( + ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 100))); readFuture.get(10, TimeUnit.SECONDS); } } catch (Exception e) { - System.out.println("Warning: Ingest-on-read trigger failed for " + blobId + ": " + e.getMessage()); + System.out.println( + "Warning: Ingest-on-read trigger failed for " + blobId + ": " + e.getMessage()); } } @@ -200,7 +201,8 @@ public void readPostStreamClose() throws Exception { Throwable cause = e.getCause(); assertThat(cause).isInstanceOf(StorageException.class); assertThat(cause.getCause()).isInstanceOf(AsyncSessionClosedException.class); - System.out.println(">>> SUCCESS: readPostStreamClose verified AsyncSessionClosedException."); + System.out.println( + ">>> SUCCESS: readPostStreamClose verified AsyncSessionClosedException."); } } } @@ -284,15 +286,18 @@ public void multipleRangedRead() throws Exception { // Resolve and verify Range 2 byte[] b2 = f2.get(10, TimeUnit.SECONDS); - assertThat(b2).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, rangeSize, 2 * rangeSize)); + assertThat(b2) + .isEqualTo(Arrays.copyOfRange(multipleRangeTestData, rangeSize, 2 * rangeSize)); // Resolve and verify Range 3 byte[] b3 = f3.get(10, TimeUnit.SECONDS); - assertThat(b3).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 2 * rangeSize, 3 * rangeSize)); + assertThat(b3) + .isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 2 * rangeSize, 3 * rangeSize)); // Resolve and verify Range 4 byte[] b4 = f4.get(10, TimeUnit.SECONDS); - assertThat(b4).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 3 * rangeSize, 4 * rangeSize)); + assertThat(b4) + .isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 3 * rangeSize, 4 * rangeSize)); System.out.println(">>> SUCCESS: multipleRangedRead concurrent offsets verified."); } @@ -304,7 +309,8 @@ public void multipleRangedRead() throws Exception { @Test public void readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404() throws Exception { Assume.assumeTrue(transport == Transport.GRPC); - System.out.println(">>> START: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404"); + System.out.println( + ">>> START: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404"); String nonExistentBucketName = "java-storage-non-existent-bucket-" + UUID.randomUUID(); BlobId blobId = BlobId.of(nonExistentBucketName, "someobject"); @@ -319,7 +325,8 @@ public void readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404() t assertThat(cause).isInstanceOf(StorageException.class); StorageException se = (StorageException) cause; assertThat(se.getCode()).isIn(Arrays.asList(404, 403)); - System.out.println(">>> SUCCESS: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404 verified StorageException 404 or 403."); + System.out.println( + ">>> SUCCESS: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404 verified StorageException 404 or 403."); } } @@ -333,13 +340,16 @@ public void outOfRange() throws Exception { try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { // Start a valid range read on the session first to verify it succeeds ApiFuture fValid = - session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 1000))); + session.readAs( + ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 1000))); byte[] bytes = fValid.get(10, TimeUnit.SECONDS); assertThat(bytes).isEqualTo(Arrays.copyOfRange(outOfRangeTestData, 0, 1000)); // Start an out-of-bounds range read (offset > size) ApiFuture fOutOfRange = - session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.beginAt(100 * 1024 + 1000))); + session.readAs( + ReadProjectionConfigs.asFutureBytes() + .withRangeSpec(RangeSpec.beginAt(100 * 1024 + 1000))); // Verify that resolving it throws OutOfRangeException try { @@ -350,7 +360,8 @@ public void outOfRange() throws Exception { assertThat(cause).isInstanceOf(StorageException.class); assertThat(cause.getCause()).isInstanceOf(OutOfRangeException.class); } - System.out.println(">>> SUCCESS: outOfRange verified valid read success and subsequent out of range exception."); + System.out.println( + ">>> SUCCESS: outOfRange verified valid read success and subsequent out of range exception."); } } finally { // Do not delete diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java index 814f89ba6196..1cb7e8e826a6 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java @@ -26,8 +26,8 @@ import com.google.common.collect.ImmutableSet; import java.util.Collections; import java.util.Locale; -import java.util.Set; import java.util.Objects; +import java.util.Set; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; import org.checkerframework.checker.nullness.qual.Nullable; @@ -193,9 +193,7 @@ public static CrossRunIntersection of(@Nullable Backend b, @Nullable Transport t } public static CrossRunIntersection of( - @Nullable Backend b, - @Nullable Transport t, - @Nullable LocationType lt) { + @Nullable Backend b, @Nullable Transport t, @Nullable LocationType lt) { return new CrossRunIntersection(b, t, lt); } @@ -232,15 +230,8 @@ public static ImmutableSet expand( .flatMap( b -> tSet.stream() - .flatMap( - t -> - lSet.stream() - .map(l -> new CrossRunIntersection(b, t, l)))) - .filter( - i -> - !(i.backend == null - && i.transport == null - && i.locationType == null)) + .flatMap(t -> lSet.stream().map(l -> new CrossRunIntersection(b, t, l)))) + .filter(i -> !(i.backend == null && i.transport == null && i.locationType == null)) .collect(ImmutableSet.toImmutableSet()); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java index 4e04768d58ab..ba5d0153580b 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java @@ -19,11 +19,11 @@ import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.CrossRun; import com.google.cloud.storage.it.runner.annotations.CrossRun.AllowClassRule; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.ParallelFriendly; import com.google.cloud.storage.it.runner.annotations.Parameterized; import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; -import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.registry.Registry; import com.google.common.collect.ImmutableList; @@ -170,10 +170,7 @@ private static List computeRunners(Class klass, Registry registry) .flatMap( t -> ImmutableSet.copyOf(crossRun.locations()).stream() - .map( - l -> - CrossRunIntersection.of( - b, t, l)))) + .map(l -> CrossRunIntersection.of(b, t, l)))) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java index 348620aa6a67..4db0c7942b32 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java @@ -36,5 +36,4 @@ Backend value(); LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; - } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java index ca1eaa25287e..abe2214e1d67 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java @@ -39,9 +39,6 @@ import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.BucketType; import com.google.cloud.storage.it.runner.annotations.LocationType; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import org.junit.runners.model.FrameworkField; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.storage.control.v2.StorageControlClient; @@ -52,7 +49,9 @@ import java.net.URI; import java.util.Locale; import java.util.UUID; - +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.junit.runners.model.FrameworkField; /** The set of resources which are defined for a single backend. */ final class BackendResources implements ManagedLifecycle { @@ -107,7 +106,9 @@ public String toString() { } public Storage getStorage(Transport transport) { - return transport == Transport.GRPC ? storageGrpc.get().getStorage() : storageJson.get().getStorage(); + return transport == Transport.GRPC + ? storageGrpc.get().getStorage() + : storageJson.get().getStorage(); } public StorageControlClient getStorageControlClient() { @@ -138,7 +139,8 @@ static BackendResources of( case PREPROD: optionsBuilder = StorageOptions.http() - .setHost("https://storage-preprod-test-unified.googleusercontent.com/storage/v1_preprod/") + .setHost( + "https://storage-preprod-test-unified.googleusercontent.com/storage/v1_preprod/") .setProjectId(getPreprodProjectId()) .setOpenTelemetry(otelSdk.get().get()); break; @@ -343,9 +345,7 @@ static BackendResources of( .resolve( null, CrossRunIntersection.of( - backend, - null, - LocationType.REGIONAL_STANDARD)))); + backend, null, LocationType.REGIONAL_STANDARD)))); TestRunScopedInstance objectsFixtureRp = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]/REQUESTER_PAYS", @@ -441,9 +441,7 @@ public int hashCode() { @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("locationType", locationType) - .toString(); + return MoreObjects.toStringHelper(this).add("locationType", locationType).toString(); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java index 69be2b41df30..308c43b7d409 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java @@ -78,15 +78,26 @@ public Object get() { public void start() { try { if (locationType == LocationType.REGIONAL_RAPID && backend != Backend.TEST_BENCH) { - System.out.println(">>> REUSING static pre-created RCU bucket java-storage-reg-rapid-preprod-3fe2bb58 for REGIONAL_RAPID test!"); - createdBucket = BucketInfo.newBuilder("java-storage-reg-rapid-preprod-3fe2bb58") - .setLocation("US-CENTRAL1") - .build(); + System.out.println( + ">>> REUSING static pre-created RCU bucket java-storage-reg-rapid-preprod-3fe2bb58 for REGIONAL_RAPID test!"); + createdBucket = + BucketInfo.newBuilder("java-storage-reg-rapid-preprod-3fe2bb58") + .setLocation("US-CENTRAL1") + .build(); return; } - System.out.println("Starting resource creation for LocationType: " + locationType + " in zone: " + targetZone); + System.out.println( + "Starting resource creation for LocationType: " + + locationType + + " in zone: " + + targetZone); createdBucket = s.create(bucketInfo).asBucketInfo(); - System.out.println("Successfully created bucket: " + createdBucket.getName() + " (Location: " + createdBucket.getLocation() + ")"); + System.out.println( + "Successfully created bucket: " + + createdBucket.getName() + + " (Location: " + + createdBucket.getLocation() + + ")"); if (locationType == LocationType.REGIONAL_RAPID) { if (ctrl == null) { throw new IllegalStateException( @@ -106,20 +117,25 @@ public void start() { .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours .build(); try { - System.out.println("Submitting CreateRapidCache LRO for bucket: " + createdBucket.getName() + " in zone: " + targetZone); + System.out.println( + "Submitting CreateRapidCache LRO for bucket: " + + createdBucket.getName() + + " in zone: " + + targetZone); ctrl.createRapidCacheAsync(BucketName.format("_", createdBucket.getName()), rapidCache) .get(30, java.util.concurrent.TimeUnit.SECONDS); System.out.println("Successfully created Rapid Cache in zone: " + targetZone); } catch (java.util.concurrent.TimeoutException te) { System.out.println("WARNING: CreateRapidCache LRO timed out after 30s. Skipping test."); assumeTrue( - "Skipping test because Rapid Cache creation LRO timed out (30s) in zone: " + targetZone, + "Skipping test because Rapid Cache creation LRO timed out (30s) in zone: " + + targetZone, false); } catch (Exception e) { - System.out.println("WARNING: CreateRapidCache LRO failed: " + e.getMessage() + ". Skipping test."); + System.out.println( + "WARNING: CreateRapidCache LRO failed: " + e.getMessage() + ". Skipping test."); assumeTrue( - "Skipping test due to failure during Rapid Cache creation: " + e.getMessage(), - false); + "Skipping test due to failure during Rapid Cache creation: " + e.getMessage(), false); } } } catch (StorageException se) { diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java index 8bdb507f71c3..66c8781f57ca 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java @@ -19,11 +19,11 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TransportCompatibility.Transport; -import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.TestInitializer; import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.annotations.StorageFixture; import com.google.common.base.Joiner;