From c62fcb75a910e8d8406d205829629b851b1c6248 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 21:12:50 +0000 Subject: [PATCH 01/15] chore: release 2.1.7-SNAPSHOT (#1044) :robot: I have created a release \*beep\* \*boop\* --- ### Updating meta-information for bleeding-edge SNAPSHOT release. --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- gapic-google-cloud-storage-v2/pom.xml | 4 ++-- google-cloud-storage/pom.xml | 4 ++-- grpc-google-cloud-storage-v2/pom.xml | 4 ++-- pom.xml | 8 ++++---- proto-google-cloud-storage-v2/pom.xml | 4 ++-- versions.txt | 8 ++++---- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml index 1547030bf..937f789ac 100644 --- a/gapic-google-cloud-storage-v2/pom.xml +++ b/gapic-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.1.6-alpha + 2.1.7-alpha-SNAPSHOT gapic-google-cloud-storage-v2 GRPC library for gapic-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.1.6 + 2.1.7-SNAPSHOT diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index b5e705145..208c6d46d 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 2.1.6 + 2.1.7-SNAPSHOT jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 2.1.6 + 2.1.7-SNAPSHOT google-cloud-storage diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml index 4e14cd780..a0565660b 100644 --- a/grpc-google-cloud-storage-v2/pom.xml +++ b/grpc-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-v2 - 2.1.6-alpha + 2.1.7-alpha-SNAPSHOT grpc-google-cloud-storage-v2 GRPC library for grpc-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.1.6 + 2.1.7-SNAPSHOT diff --git a/pom.xml b/pom.xml index c461cc25e..3285250ce 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 2.1.6 + 2.1.7-SNAPSHOT Storage Parent https://github.com/googleapis/java-storage @@ -93,17 +93,17 @@ com.google.api.grpc proto-google-cloud-storage-v2 - 2.1.6-alpha + 2.1.7-alpha-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-v2 - 2.1.6-alpha + 2.1.7-alpha-SNAPSHOT com.google.api.grpc gapic-google-cloud-storage-v2 - 2.1.6-alpha + 2.1.7-alpha-SNAPSHOT com.google.cloud diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml index abcc1d3f3..2b2a4d476 100644 --- a/proto-google-cloud-storage-v2/pom.xml +++ b/proto-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-v2 - 2.1.6-alpha + 2.1.7-alpha-SNAPSHOT proto-google-cloud-storage-v2 PROTO library for proto-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.1.6 + 2.1.7-SNAPSHOT diff --git a/versions.txt b/versions.txt index f7427a34f..7da1b65ac 100644 --- a/versions.txt +++ b/versions.txt @@ -1,7 +1,7 @@ # Format: # module:released-version:current-version -google-cloud-storage:2.1.6:2.1.6 -gapic-google-cloud-storage-v2:2.1.6-alpha:2.1.6-alpha -grpc-google-cloud-storage-v2:2.1.6-alpha:2.1.6-alpha -proto-google-cloud-storage-v2:2.1.6-alpha:2.1.6-alpha +google-cloud-storage:2.1.6:2.1.7-SNAPSHOT +gapic-google-cloud-storage-v2:2.1.6-alpha:2.1.7-alpha-SNAPSHOT +grpc-google-cloud-storage-v2:2.1.6-alpha:2.1.7-alpha-SNAPSHOT +proto-google-cloud-storage-v2:2.1.6-alpha:2.1.7-alpha-SNAPSHOT From ea1eeb9575019f266808d49c2b937d49a7fcb7b5 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Mon, 27 Sep 2021 13:23:13 -0400 Subject: [PATCH 02/15] chore(retry): introduce RetryAlgorithmManager (#1029) ### Summary * RetryAlgorithmManager provides an abstraction point for determining which ExceptionHandler should be used for a specific call. * Implement LegacyRetryAlgorithmManager for which all methods return BaseService.EXCEPTION_HANDLER which is hardcoded everywhere previously. * Stub out NewRetryAlgorithmManager which currently extends LegacyRetryAlgorithmManager, but allows for overriding individual methods in an incremental manner. ### Refactoring * All usages of BaseService.EXCEPTION_HANDLER in StorageImpl, Blob, Bucket etc now instead use RetryAlgorithmManager to resolve a handler per method. * Refactor Blob to use Retrying#run instead of runWithRetries * Split starting of a resumable upload session out of BlobWriteChannel to ResumableMedia. This decouple the ability to instantiate a BlobWriteChannel independent of performing an RPC to resolve the uploadId. This split is also necessary to allow differing configuration of ExceptionHandler for open vs write of a resumable upload session. * Refactor StorageException#translateAndThrow to allow for an option which doesn't throw, new StorageException#coalesce which attempts to coalesce to a BaseServiceException but doesn't throw. * Relax several easymock assertions on StorageOptions method. (Calling get on an immutable object shouldn't be so concerned with invocation count) Related to #1024 --- .../java/com/google/cloud/storage/Blob.java | 48 +--- .../google/cloud/storage/BlobReadChannel.java | 27 +- .../cloud/storage/BlobWriteChannel.java | 156 +++++----- .../com/google/cloud/storage/CopyWriter.java | 1 + .../storage/LegacyRetryAlgorithmManager.java | 247 ++++++++++++++++ .../storage/NewRetryAlgorithmManager.java | 20 ++ .../google/cloud/storage/ResumableMedia.java | 74 +++++ .../cloud/storage/RetryAlgorithmManager.java | 118 ++++++++ .../com/google/cloud/storage/Retrying.java | 20 +- .../cloud/storage/StorageException.java | 21 +- .../com/google/cloud/storage/StorageImpl.java | 266 ++++++++++++++---- .../google/cloud/storage/StorageOptions.java | 35 ++- .../com/google/cloud/storage/BlobTest.java | 109 +++---- .../cloud/storage/BlobWriteChannelTest.java | 161 ++++++----- .../com/google/cloud/storage/BucketTest.java | 90 +++--- .../PackagePrivateMethodWorkarounds.java | 4 + .../cloud/storage/ResumableMediaTest.java | 46 +++ .../cloud/storage/SerializationTest.java | 8 +- .../cloud/storage/StorageBatchTest.java | 19 +- .../google/cloud/storage/StorageImplTest.java | 12 - .../conformance/retry/RetryTestFixture.java | 2 + 21 files changed, 1102 insertions(+), 382 deletions(-) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index abc61eb1f..4fbbe069a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -16,7 +16,6 @@ package com.google.cloud.storage; -import static com.google.cloud.RetryHelper.runWithRetries; import static com.google.cloud.storage.Blob.BlobSourceOption.toGetOptions; import static com.google.cloud.storage.Blob.BlobSourceOption.toSourceOptions; import static com.google.common.base.Preconditions.checkNotNull; @@ -25,9 +24,8 @@ import com.google.api.services.storage.model.StorageObject; import com.google.auth.ServiceAccountSigner; import com.google.auth.ServiceAccountSigner.SigningException; +import com.google.cloud.ExceptionHandler; import com.google.cloud.ReadChannel; -import com.google.cloud.RetryHelper; -import com.google.cloud.Tuple; import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.Storage.BlobTargetOption; @@ -35,7 +33,6 @@ import com.google.cloud.storage.Storage.CopyRequest; import com.google.cloud.storage.Storage.SignUrlOption; import com.google.cloud.storage.spi.v1.StorageRpc; -import com.google.common.base.Function; import com.google.common.io.BaseEncoding; import com.google.common.io.CountingOutputStream; import java.io.IOException; @@ -50,6 +47,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.function.Function; /** * An object in Google Cloud Storage. A {@code Blob} object includes the {@code BlobId} instance, @@ -78,18 +76,9 @@ public class Blob extends BlobInfo { private static final long serialVersionUID = -6806832496717441434L; private final StorageOptions options; + private final RetryAlgorithmManager retryAlgorithmManager; private transient Storage storage; - static final Function, Blob> BLOB_FROM_PB_FUNCTION = - new Function, Blob>() { - @Override - public Blob apply(Tuple pb) { - return Blob.fromPb(pb.x(), pb.y()); - } - }; - - private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; - /** Class for specifying blob source options when {@code Blob} methods are used. */ public static class BlobSourceOption extends Option { @@ -258,26 +247,18 @@ public void downloadTo(Path path, BlobSourceOption... options) { public void downloadTo(OutputStream outputStream, BlobSourceOption... options) { final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream); final StorageRpc storageRpc = this.options.getStorageRpcV1(); + StorageObject pb = getBlobId().toPb(); final Map requestOptions = StorageImpl.optionMap(getBlobId(), options); - try { - runWithRetries( - callable( - new Runnable() { - @Override - public void run() { - storageRpc.read( - getBlobId().toPb(), - requestOptions, - countingOutputStream.getCount(), - countingOutputStream); - } - }), - this.options.getRetrySettings(), - StorageImpl.EXCEPTION_HANDLER, - this.options.getClock()); - } catch (RetryHelper.RetryHelperException e) { - StorageException.translateAndThrow(e); - } + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForObjectsGet(pb, requestOptions); + Retrying.run( + this.options, + exceptionHandler, + callable( + () -> { + storageRpc.read( + pb, requestOptions, countingOutputStream.getCount(), countingOutputStream); + }), + Function.identity()); } /** @@ -506,6 +487,7 @@ public Blob build() { super(infoBuilder); this.storage = checkNotNull(storage); this.options = storage.getOptions(); + this.retryAlgorithmManager = storage.getOptions().getRetryAlgorithmManager(); } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java index c5fce5bd1..59d6188dd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java @@ -19,6 +19,7 @@ import static com.google.cloud.RetryHelper.runWithRetries; import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.ExceptionHandler; import com.google.cloud.ReadChannel; import com.google.cloud.RestorableState; import com.google.cloud.RetryHelper; @@ -31,7 +32,6 @@ import java.nio.channels.ClosedChannelException; import java.util.Map; import java.util.Objects; -import java.util.concurrent.Callable; /** Default implementation for ReadChannel. */ class BlobReadChannel implements ReadChannel { @@ -41,6 +41,7 @@ class BlobReadChannel implements ReadChannel { private final StorageOptions serviceOptions; private final BlobId blob; private final Map requestOptions; + private final RetryAlgorithmManager retryAlgorithmManager; private String lastEtag; private long position; private boolean isOpen; @@ -57,6 +58,7 @@ class BlobReadChannel implements ReadChannel { this.serviceOptions = serviceOptions; this.blob = blob; this.requestOptions = requestOptions; + this.retryAlgorithmManager = serviceOptions.getRetryAlgorithmManager(); isOpen = true; storageRpc = serviceOptions.getStorageRpcV1(); storageObject = blob.toPb(); @@ -119,24 +121,21 @@ public int read(ByteBuffer byteBuffer) throws IOException { } final int toRead = Math.max(byteBuffer.remaining(), chunkSize); try { + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsGet(storageObject, requestOptions); Tuple result = runWithRetries( - new Callable>() { - @Override - public Tuple call() { - return storageRpc.read(storageObject, requestOptions, position, toRead); - } - }, + () -> storageRpc.read(storageObject, requestOptions, position, toRead), serviceOptions.getRetrySettings(), - StorageImpl.EXCEPTION_HANDLER, + exceptionHandler, serviceOptions.getClock()); - if (result.y().length > 0 && lastEtag != null && !Objects.equals(result.x(), lastEtag)) { - StringBuilder messageBuilder = new StringBuilder(); - messageBuilder.append("Blob ").append(blob).append(" was updated while reading"); - throw new IOException(messageBuilder.toString()); + String etag = result.x(); + byte[] bytes = result.y(); + if (bytes.length > 0 && lastEtag != null && !Objects.equals(etag, lastEtag)) { + throw new IOException("Blob " + blob + " was updated while reading"); } - lastEtag = result.x(); - buffer = result.y(); + lastEtag = etag; + buffer = bytes; } catch (RetryHelper.RetryHelperException e) { throw new IOException(e); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index 6207ef662..fd7e73e38 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -17,47 +17,41 @@ package com.google.cloud.storage; import static com.google.cloud.RetryHelper.runWithRetries; +import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.callable; import com.google.api.services.storage.model.StorageObject; import com.google.cloud.BaseWriteChannel; +import com.google.cloud.ExceptionHandler; import com.google.cloud.RestorableState; import com.google.cloud.RetryHelper; import com.google.cloud.WriteChannel; -import com.google.cloud.storage.spi.v1.StorageRpc; import java.math.BigInteger; -import java.net.URL; -import java.util.Map; -import java.util.concurrent.Callable; +import java.util.function.Supplier; +import org.checkerframework.checker.nullness.qual.NonNull; /** Write channel implementation to upload Google Cloud Storage blobs. */ class BlobWriteChannel extends BaseWriteChannel { - BlobWriteChannel(StorageOptions options, BlobInfo blob, Map optionsMap) { - this(options, blob, open(options, blob, optionsMap)); - } - - BlobWriteChannel(StorageOptions options, URL signedURL) { - this(options, open(signedURL, options)); - } - - BlobWriteChannel(StorageOptions options, BlobInfo blobInfo, String uploadId) { - super(options, blobInfo, uploadId); - } - - BlobWriteChannel(StorageOptions options, String uploadId) { - super(options, null, uploadId); - } - - // Contains metadata of the updated object or null if upload is not completed. - private StorageObject storageObject; - + private final ExceptionHandler exceptionHandlerForWrite; // Detect if flushBuffer() is being retried or not. // TODO: I don't think this is thread safe, and there's probably a better way to detect a retry // occuring. private boolean retrying = false; private boolean checkingForLastChunk = false; + // Contains metadata of the updated object or null if upload is not completed. + private StorageObject storageObject; + + BlobWriteChannel( + StorageOptions storageOptions, + BlobInfo blobInfo, + String uploadId, + ExceptionHandler exceptionHandlerForWrite) { + super(storageOptions, blobInfo, uploadId); + this.exceptionHandlerForWrite = exceptionHandlerForWrite; + } + boolean isRetrying() { return retrying; } @@ -275,7 +269,7 @@ public void run() { } }), getOptions().getRetrySettings(), - StorageImpl.EXCEPTION_HANDLER, + exceptionHandlerForWrite, getOptions().getClock()); } catch (RetryHelper.RetryHelperException e) { throw StorageException.translateAndThrow(e); @@ -283,83 +277,73 @@ public void run() { } protected StateImpl.Builder stateBuilder() { - return StateImpl.builder(getOptions(), getEntity(), getUploadId()); + return StateImpl.builder(getOptions(), getEntity(), getUploadId()) + .setResultRetryAlgorithm(exceptionHandlerForWrite); } - private static String open( - final StorageOptions options, - final BlobInfo blob, - final Map optionsMap) { - try { - return runWithRetries( - new Callable() { - @Override - public String call() { - return options.getStorageRpcV1().open(blob.toPb(), optionsMap); - } - }, - options.getRetrySettings(), - StorageImpl.EXCEPTION_HANDLER, - options.getClock()); - } catch (RetryHelper.RetryHelperException e) { - throw StorageException.translateAndThrow(e); - } + static Builder newBuilder() { + return new Builder(); } - private static String open(final URL signedURL, final StorageOptions options) { - try { - return runWithRetries( - new Callable() { - @Override - public String call() { - if (!isValidSignedURL(signedURL.getQuery())) { - throw new StorageException(2, "invalid signedURL"); - } - return options.getStorageRpcV1().open(signedURL.toString()); - } - }, - options.getRetrySettings(), - StorageImpl.EXCEPTION_HANDLER, - options.getClock()); - } catch (RetryHelper.RetryHelperException e) { - throw StorageException.translateAndThrow(e); + static final class Builder { + private StorageOptions storageOptions; + private BlobInfo blobInfo; + private Supplier<@NonNull String> uploadIdSupplier; + private ExceptionHandler putExceptionHandler; + + public Builder setStorageOptions(StorageOptions storageOptions) { + this.storageOptions = storageOptions; + return this; } - } - private static boolean isValidSignedURL(String signedURLQuery) { - boolean isValid = true; - if (signedURLQuery.startsWith("X-Goog-Algorithm=")) { - if (!signedURLQuery.contains("&X-Goog-Credential=") - || !signedURLQuery.contains("&X-Goog-Date=") - || !signedURLQuery.contains("&X-Goog-Expires=") - || !signedURLQuery.contains("&X-Goog-SignedHeaders=") - || !signedURLQuery.contains("&X-Goog-Signature=")) { - isValid = false; - } - } else if (signedURLQuery.startsWith("GoogleAccessId=")) { - if (!signedURLQuery.contains("&Expires=") || !signedURLQuery.contains("&Signature=")) { - isValid = false; - } - } else { - isValid = false; + public Builder setBlobInfo(BlobInfo blobInfo) { + this.blobInfo = blobInfo; + return this; + } + + public Builder setUploadIdSupplier(Supplier uploadIdSupplier) { + this.uploadIdSupplier = uploadIdSupplier; + return this; + } + + public Builder setPutExceptionHandler(ExceptionHandler putExceptionHandler) { + this.putExceptionHandler = putExceptionHandler; + return this; + } + + BlobWriteChannel build() { + String uploadId = requireNonNull(uploadIdSupplier, "uploadId must be non null").get(); + return new BlobWriteChannel( + requireNonNull(storageOptions, "storageOptions must be non null"), + blobInfo, + requireNonNull(uploadId, "uploadId must be non null"), + requireNonNull(putExceptionHandler, "putExceptionHandler must be non null")); } - return isValid; } static class StateImpl extends BaseWriteChannel.BaseState { private static final long serialVersionUID = -9028324143780151286L; + private final ExceptionHandler exceptionHandler; + StateImpl(Builder builder) { super(builder); + this.exceptionHandler = builder.exceptionHandler; } static class Builder extends BaseWriteChannel.BaseState.Builder { + private ExceptionHandler exceptionHandler; private Builder(StorageOptions options, BlobInfo blobInfo, String uploadId) { super(options, blobInfo, uploadId); } + public Builder setResultRetryAlgorithm(ExceptionHandler exceptionHandler) { + this.exceptionHandler = exceptionHandler; + return this; + } + @Override public RestorableState build() { return new StateImpl(this); @@ -372,9 +356,19 @@ static Builder builder(StorageOptions options, BlobInfo blobInfo, String uploadI @Override public WriteChannel restore() { - BlobWriteChannel channel = new BlobWriteChannel(serviceOptions, entity, uploadId); - channel.restore(this); - return channel; + try { + BlobWriteChannel channel = + BlobWriteChannel.newBuilder() + .setStorageOptions(serviceOptions) + .setBlobInfo(entity) + .setUploadIdSupplier(() -> uploadId) + .setPutExceptionHandler(exceptionHandler) + .build(); + channel.restore(this); + return channel; + } catch (Exception e) { + throw StorageException.coalesce(e); + } } } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java index 47aa6b9f8..f3dcdfdee 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java @@ -100,6 +100,7 @@ public void copyChunk() { this.rewriteResponse = Retrying.run( serviceOptions, + serviceOptions.getRetryAlgorithmManager().getForObjectsCopy(), () -> storageRpc.continueRewrite(rewriteResponse), Function.identity()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java new file mode 100644 index 000000000..fd88c3333 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java @@ -0,0 +1,247 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.HmacKeyMetadata; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.api.services.storage.model.Policy; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.BaseService; +import com.google.cloud.ExceptionHandler; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest; +import java.util.List; +import java.util.Map; + +class LegacyRetryAlgorithmManager implements RetryAlgorithmManager { + + @Override + public ExceptionHandler getForBucketAclCreate( + BucketAccessControl pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclDelete(String pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclGet(String pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclUpdate( + BucketAccessControl pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclList(String pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsCreate(Bucket pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsDelete(Bucket pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsGet(Bucket pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsUpdate(Bucket pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsList(Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsLockRetentionPolicy( + Bucket pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsGetIamPolicy( + String bucket, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsSetIamPolicy( + String bucket, Policy pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsTestIamPermissions( + String bucket, List permissions, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclCreate(ObjectAccessControl pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclDelete(String pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclGet(String pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclUpdate(ObjectAccessControl pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclList(String pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyCreate(String pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyDelete( + HmacKeyMetadata pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyGet(String accessId, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyUpdate( + HmacKeyMetadata pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyList(Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclCreate(ObjectAccessControl aclPb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclDelete( + String bucket, String name, Long generation, String pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclList(String bucket, String name, Long generation) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclGet( + String bucket, String name, Long generation, String pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclUpdate(ObjectAccessControl aclPb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCreate( + StorageObject pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsDelete( + StorageObject pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsGet(StorageObject pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsUpdate( + StorageObject pb, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsList(String bucket, Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsRewrite(RewriteRequest pb) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCopy() { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCompose( + List sources, StorageObject target, Map targetOptions) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForResumableUploadSessionCreate(Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForResumableUploadSessionWrite(Map optionsMap) { + return BaseService.EXCEPTION_HANDLER; + } + + @Override + public ExceptionHandler getForServiceAccountGet(String pb) { + return BaseService.EXCEPTION_HANDLER; + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java new file mode 100644 index 000000000..0c3e1f8e7 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java @@ -0,0 +1,20 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +final class NewRetryAlgorithmManager extends LegacyRetryAlgorithmManager + implements RetryAlgorithmManager {} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java new file mode 100644 index 000000000..d0e212ef3 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.core.ApiFutures; +import com.google.cloud.ExceptionHandler; +import com.google.cloud.storage.spi.v1.StorageRpc; +import java.net.URL; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; + +final class ResumableMedia { + + static Supplier startUploadForBlobInfo( + final StorageOptions storageOptions, + final BlobInfo blob, + final Map optionsMap, + ExceptionHandler exceptionHandler) { + return () -> + Retrying.run( + storageOptions, + exceptionHandler, + () -> storageOptions.getStorageRpcV1().open(blob.toPb(), optionsMap), + Function.identity()); + } + + static Supplier startUploadForSignedUrl( + final StorageOptions storageOptions, final URL signedURL, ExceptionHandler exceptionHandler) { + if (!isValidSignedURL(signedURL.getQuery())) { + ApiFutures.immediateFailedFuture(new StorageException(2, "invalid signedURL")); + } + return () -> + Retrying.run( + storageOptions, + exceptionHandler, + () -> storageOptions.getStorageRpcV1().open(signedURL.toString()), + Function.identity()); + } + + private static boolean isValidSignedURL(String signedURLQuery) { + boolean isValid = true; + if (signedURLQuery.startsWith("X-Goog-Algorithm=")) { + if (!signedURLQuery.contains("&X-Goog-Credential=") + || !signedURLQuery.contains("&X-Goog-Date=") + || !signedURLQuery.contains("&X-Goog-Expires=") + || !signedURLQuery.contains("&X-Goog-SignedHeaders=") + || !signedURLQuery.contains("&X-Goog-Signature=")) { + isValid = false; + } + } else if (signedURLQuery.startsWith("GoogleAccessId=")) { + if (!signedURLQuery.contains("&Expires=") || !signedURLQuery.contains("&Signature=")) { + isValid = false; + } + } else { + isValid = false; + } + return isValid; + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java new file mode 100644 index 000000000..884c4d4eb --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java @@ -0,0 +1,118 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.HmacKeyMetadata; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.api.services.storage.model.Policy; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.ExceptionHandler; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest; +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +interface RetryAlgorithmManager extends Serializable { + ExceptionHandler getForBucketAclCreate( + BucketAccessControl pb, Map optionsMap); + + ExceptionHandler getForBucketAclDelete(String pb, Map optionsMap); + + ExceptionHandler getForBucketAclGet(String pb, Map optionsMap); + + ExceptionHandler getForBucketAclUpdate( + BucketAccessControl pb, Map optionsMap); + + ExceptionHandler getForBucketAclList(String pb, Map optionsMap); + + ExceptionHandler getForBucketsCreate(Bucket pb, Map optionsMap); + + ExceptionHandler getForBucketsDelete(Bucket pb, Map optionsMap); + + ExceptionHandler getForBucketsGet(Bucket pb, Map optionsMap); + + ExceptionHandler getForBucketsUpdate(Bucket pb, Map optionsMap); + + ExceptionHandler getForBucketsList(Map optionsMap); + + ExceptionHandler getForBucketsLockRetentionPolicy( + Bucket pb, Map optionsMap); + + ExceptionHandler getForBucketsGetIamPolicy(String bucket, Map optionsMap); + + ExceptionHandler getForBucketsSetIamPolicy( + String bucket, Policy pb, Map optionsMap); + + ExceptionHandler getForBucketsTestIamPermissions( + String bucket, List permissions, Map optionsMap); + + ExceptionHandler getForDefaultObjectAclCreate(ObjectAccessControl pb); + + ExceptionHandler getForDefaultObjectAclDelete(String pb); + + ExceptionHandler getForDefaultObjectAclGet(String pb); + + ExceptionHandler getForDefaultObjectAclUpdate(ObjectAccessControl pb); + + ExceptionHandler getForDefaultObjectAclList(String pb); + + ExceptionHandler getForHmacKeyCreate(String pb, Map optionsMap); + + ExceptionHandler getForHmacKeyDelete(HmacKeyMetadata pb, Map optionsMap); + + ExceptionHandler getForHmacKeyGet(String accessId, Map optionsMap); + + ExceptionHandler getForHmacKeyUpdate(HmacKeyMetadata pb, Map optionsMap); + + ExceptionHandler getForHmacKeyList(Map optionsMap); + + ExceptionHandler getForObjectAclCreate(ObjectAccessControl aclPb); + + ExceptionHandler getForObjectAclDelete(String bucket, String name, Long generation, String pb); + + ExceptionHandler getForObjectAclList(String bucket, String name, Long generation); + + ExceptionHandler getForObjectAclGet(String bucket, String name, Long generation, String pb); + + ExceptionHandler getForObjectAclUpdate(ObjectAccessControl aclPb); + + ExceptionHandler getForObjectsCreate(StorageObject pb, Map optionsMap); + + ExceptionHandler getForObjectsDelete(StorageObject pb, Map optionsMap); + + ExceptionHandler getForObjectsGet(StorageObject pb, Map optionsMap); + + ExceptionHandler getForObjectsUpdate(StorageObject pb, Map optionsMap); + + ExceptionHandler getForObjectsList(String bucket, Map optionsMap); + + ExceptionHandler getForObjectsRewrite(RewriteRequest pb); + + ExceptionHandler getForObjectsCopy(); + + ExceptionHandler getForObjectsCompose( + List sources, StorageObject target, Map targetOptions); + + ExceptionHandler getForResumableUploadSessionCreate(Map optionsMap); + /** Resumable upload has differing 429 handling */ + ExceptionHandler getForResumableUploadSessionWrite(Map optionsMap); + + ExceptionHandler getForServiceAccountGet(String pb); +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java index 3cae94ea7..603d43aeb 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java @@ -21,7 +21,7 @@ import com.google.api.core.ApiClock; import com.google.api.gax.retrying.ResultRetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.BaseService; +import com.google.cloud.ExceptionHandler; import com.google.cloud.RetryHelper.RetryHelperException; import java.util.concurrent.Callable; import java.util.function.Function; @@ -33,25 +33,27 @@ final class Retrying { * RetrySettings, ResultRetryAlgorithm, ApiClock)} that gives us centralized error translation and * reduces some duplication in how we resolved the {@link RetrySettings} and {@link ApiClock}. * + * @param The result type of {@code c} + * @param The result type of any mapping that takes place via {@code f} * @param options The {@link StorageOptions} which {@link RetrySettings} and {@link ApiClock} will * be resolved from. + * @param exceptionHandler The {@link ExceptionHandler} to use when determining if a retry is + * possible * @param c The {@link Callable} which will be passed to runWithRetries producing some {@code T}, * can optionally return null * @param f A post process mapping {@link Function} which can be used to transform the result from * {@code c} if it is successful and non-null - * @param The result type of {@code c} - * @param The result type of any mapping that takes place via {@code f} * @return A {@code U} (possibly null) after applying {@code f} to the result of {@code c} * @throws StorageException if {@code c} fails due to any retry exhaustion */ - static U run(StorageOptions options, Callable c, Function f) { + static U run( + StorageOptions options, ExceptionHandler exceptionHandler, Callable c, Function f) { try { - T answer = - runWithRetries( - c, options.getRetrySettings(), BaseService.EXCEPTION_HANDLER, options.getClock()); - return answer == null ? null : f.apply(answer); + T result = + runWithRetries(c, options.getRetrySettings(), exceptionHandler, options.getClock()); + return result == null ? null : f.apply(result); } catch (RetryHelperException e) { - throw StorageException.translateAndThrow(e); + throw StorageException.coalesce(e); } } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageException.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageException.java index f4b8d0c42..d0b3e9c70 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageException.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageException.java @@ -78,7 +78,26 @@ public StorageException(GoogleJsonError error) { */ public static StorageException translateAndThrow(RetryHelperException ex) { BaseServiceException.translate(ex); - throw new StorageException(UNKNOWN_CODE, ex.getMessage(), ex.getCause()); + throw getStorageException(ex); + } + + private static StorageException getStorageException(Throwable t) { + return new StorageException(UNKNOWN_CODE, t.getMessage(), t.getCause()); + } + + /** + * Attempt to find an Exception which is a {@link BaseServiceException} If neither {@code t} or + * {@code t.getCause()} are a {@code BaseServiceException} a {@link StorageException} will be + * created with an unknown status code. + */ + static BaseServiceException coalesce(Throwable t) { + if (t instanceof BaseServiceException) { + return (BaseServiceException) t; + } + if (t.getCause() instanceof BaseServiceException) { + return (BaseServiceException) t.getCause(); + } + return getStorageException(t); } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 2d7f9675c..483387a0e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -39,6 +39,7 @@ import com.google.auth.ServiceAccountSigner; import com.google.cloud.BaseService; import com.google.cloud.BatchResult; +import com.google.cloud.ExceptionHandler; import com.google.cloud.PageImpl; import com.google.cloud.PageImpl.NextPageFetcher; import com.google.cloud.Policy; @@ -52,6 +53,7 @@ import com.google.cloud.storage.PostPolicyV4.PostFieldsV4; import com.google.cloud.storage.PostPolicyV4.PostPolicyV4Document; import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest; import com.google.common.base.CharMatcher; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -104,18 +106,25 @@ final class StorageImpl extends BaseService implements Storage { private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; private static final int MIN_BUFFER_SIZE = 256 * 1024; + private final RetryAlgorithmManager retryAlgorithmManager; private final StorageRpc storageRpc; StorageImpl(StorageOptions options) { super(options); - storageRpc = options.getStorageRpcV1(); + this.retryAlgorithmManager = options.getRetryAlgorithmManager(); + this.storageRpc = options.getStorageRpcV1(); } @Override public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb(); final Map optionsMap = optionMap(bucketInfo, options); - return run(() -> storageRpc.create(bucketPb, optionsMap), (b) -> Bucket.fromPb(this, b)); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsCreate(bucketPb, optionsMap); + return run( + exceptionHandler, + () -> storageRpc.create(bucketPb, optionsMap), + (b) -> Bucket.fromPb(this, b)); } @Override @@ -183,7 +192,10 @@ private Blob internalCreate( Preconditions.checkNotNull(content); final StorageObject blobPb = info.toPb(); final Map optionsMap = optionMap(info, options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsCreate(blobPb, optionsMap); return run( + exceptionHandler, () -> storageRpc.create( blobPb, new ByteArrayInputStream(content, offset, length), optionsMap), @@ -248,7 +260,12 @@ private static void uploadHelper(ReadableByteChannel reader, WriteChannel writer public Bucket get(String bucket, BucketGetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb(); final Map optionsMap = optionMap(options); - return run(() -> storageRpc.get(bucketPb, optionsMap), (b) -> Bucket.fromPb(this, b)); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsGet(bucketPb, optionsMap); + return run( + exceptionHandler, + () -> storageRpc.get(bucketPb, optionsMap), + (b) -> Bucket.fromPb(this, b)); } @Override @@ -260,7 +277,12 @@ public Blob get(String bucket, String blob, BlobGetOption... options) { public Blob get(BlobId blob, BlobGetOption... options) { final StorageObject storedObject = blob.toPb(); final Map optionsMap = optionMap(blob, options); - return run(() -> storageRpc.get(storedObject, optionsMap), (x) -> Blob.fromPb(this, x)); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsGet(storedObject, optionsMap); + return run( + exceptionHandler, + () -> storageRpc.get(storedObject, optionsMap), + (x) -> Blob.fromPb(this, x)); } @Override @@ -315,16 +337,21 @@ private static class HmacKeyMetadataPageFetcher implements NextPageFetcher options; - HmacKeyMetadataPageFetcher(StorageOptions serviceOptions, Map options) { + HmacKeyMetadataPageFetcher( + StorageOptions serviceOptions, + RetryAlgorithmManager retryAlgorithmManager, + Map options) { this.serviceOptions = serviceOptions; + this.retryAlgorithmManager = retryAlgorithmManager; this.options = options; } @Override public Page getNextPage() { - return listHmacKeys(serviceOptions, options); + return listHmacKeys(serviceOptions, retryAlgorithmManager, options); } } @@ -340,8 +367,11 @@ public Page list(final String bucket, BlobListOption... options) { private static Page listBuckets( final StorageOptions serviceOptions, final Map optionsMap) { + ExceptionHandler exceptionHandler = + serviceOptions.getRetryAlgorithmManager().getForBucketsList(optionsMap); return Retrying.run( serviceOptions, + exceptionHandler, () -> serviceOptions.getStorageRpcV1().list(optionsMap), (result) -> { String cursor = result.x(); @@ -359,8 +389,11 @@ private static Page listBlobs( final String bucket, final StorageOptions serviceOptions, final Map optionsMap) { + ExceptionHandler exceptionHandler = + serviceOptions.getRetryAlgorithmManager().getForObjectsList(bucket, optionsMap); return Retrying.run( serviceOptions, + exceptionHandler, () -> serviceOptions.getStorageRpcV1().list(bucket, optionsMap), (result) -> { String cursor = result.x(); @@ -379,14 +412,24 @@ private static Page listBlobs( public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb(); final Map optionsMap = optionMap(bucketInfo, options); - return run(() -> storageRpc.patch(bucketPb, optionsMap), (x) -> Bucket.fromPb(this, x)); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsUpdate(bucketPb, optionsMap); + return run( + exceptionHandler, + () -> storageRpc.patch(bucketPb, optionsMap), + (x) -> Bucket.fromPb(this, x)); } @Override public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { final StorageObject storageObject = blobInfo.toPb(); final Map optionsMap = optionMap(blobInfo, options); - return run(() -> storageRpc.patch(storageObject, optionsMap), (x) -> Blob.fromPb(this, x)); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsUpdate(storageObject, optionsMap); + return run( + exceptionHandler, + () -> storageRpc.patch(storageObject, optionsMap), + (x) -> Blob.fromPb(this, x)); } @Override @@ -398,7 +441,10 @@ public Blob update(BlobInfo blobInfo) { public boolean delete(String bucket, BucketSourceOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb(); final Map optionsMap = optionMap(options); - return run(() -> storageRpc.delete(bucketPb, optionsMap), Function.identity()); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsDelete(bucketPb, optionsMap); + return run( + exceptionHandler, () -> storageRpc.delete(bucketPb, optionsMap), Function.identity()); } @Override @@ -410,7 +456,10 @@ public boolean delete(String bucket, String blob, BlobSourceOption... options) { public boolean delete(BlobId blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); final Map optionsMap = optionMap(blob, options); - return run(() -> storageRpc.delete(storageObject, optionsMap), Function.identity()); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsDelete(storageObject, optionsMap); + return run( + exceptionHandler, () -> storageRpc.delete(storageObject, optionsMap), Function.identity()); } @Override @@ -438,8 +487,12 @@ public Blob compose(final ComposeRequest composeRequest) { composeRequest.getTarget().getGeneration(), composeRequest.getTarget().getMetageneration(), composeRequest.getTargetOptions()); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsCompose(sources, target, targetOptions); return run( - () -> storageRpc.compose(sources, target, targetOptions), (x) -> Blob.fromPb(this, x)); + exceptionHandler, + () -> storageRpc.compose(sources, target, targetOptions), + (x) -> Blob.fromPb(this, x)); } @Override @@ -454,16 +507,18 @@ public CopyWriter copy(final CopyRequest copyRequest) { copyRequest.getTarget().getGeneration(), copyRequest.getTarget().getMetageneration(), copyRequest.getTargetOptions()); + RewriteRequest rewriteRequest = + new RewriteRequest( + source, + sourceOptions, + copyRequest.overrideInfo(), + targetObject, + targetOptions, + copyRequest.getMegabytesCopiedPerChunk()); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForObjectsRewrite(rewriteRequest); return run( - () -> - storageRpc.openRewrite( - new StorageRpc.RewriteRequest( - source, - sourceOptions, - copyRequest.overrideInfo(), - targetObject, - targetOptions, - copyRequest.getMegabytesCopiedPerChunk())), + exceptionHandler, + () -> storageRpc.openRewrite(rewriteRequest), (r) -> new CopyWriter(getOptions(), r)); } @@ -476,7 +531,10 @@ public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... optio public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); final Map optionsMap = optionMap(blob, options); - return run(() -> storageRpc.load(storageObject, optionsMap), Function.identity()); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectsGet(storageObject, optionsMap); + return run( + exceptionHandler, () -> storageRpc.load(storageObject, optionsMap), Function.identity()); } @Override @@ -504,12 +562,33 @@ public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) { @Override public BlobWriteChannel writer(URL signedURL) { - return new BlobWriteChannel(getOptions(), signedURL); + ExceptionHandler forResumableUploadSessionCreate = + retryAlgorithmManager.getForResumableUploadSessionCreate( + Collections + .emptyMap()); // TODO: is it possible to know if a signed url is configured to have + // a constraint which makes it idempotent? + return BlobWriteChannel.newBuilder() + .setStorageOptions(getOptions()) + .setUploadIdSupplier( + ResumableMedia.startUploadForSignedUrl( + getOptions(), signedURL, forResumableUploadSessionCreate)) + .setPutExceptionHandler( + retryAlgorithmManager.getForResumableUploadSessionWrite(optionMap())) + .build(); } private BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) { final Map optionsMap = optionMap(blobInfo, options); - return new BlobWriteChannel(getOptions(), blobInfo, optionsMap); + return BlobWriteChannel.newBuilder() + .setStorageOptions(getOptions()) + .setUploadIdSupplier( + ResumableMedia.startUploadForBlobInfo( + getOptions(), + blobInfo, + optionsMap, + retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap))) + .setPutExceptionHandler(retryAlgorithmManager.getForResumableUploadSessionWrite(optionsMap)) + .build(); } @Override @@ -987,8 +1066,10 @@ public void error(StorageException exception) { @Override public Acl getAcl(final String bucket, final Entity entity, BucketSourceOption... options) { + String pb = entity.toPb(); final Map optionsMap = optionMap(options); - return run(() -> storageRpc.getAcl(bucket, entity.toPb(), optionsMap), Acl::fromPb); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForBucketAclGet(pb, optionsMap); + return run(exceptionHandler, () -> storageRpc.getAcl(bucket, pb, optionsMap), Acl::fromPb); } @Override @@ -999,8 +1080,11 @@ public Acl getAcl(final String bucket, final Entity entity) { @Override public boolean deleteAcl( final String bucket, final Entity entity, BucketSourceOption... options) { + final String pb = entity.toPb(); final Map optionsMap = optionMap(options); - return run(() -> storageRpc.deleteAcl(bucket, entity.toPb(), optionsMap), Function.identity()); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForBucketAclDelete(pb, optionsMap); + return run( + exceptionHandler, () -> storageRpc.deleteAcl(bucket, pb, optionsMap), Function.identity()); } @Override @@ -1012,7 +1096,9 @@ public boolean deleteAcl(final String bucket, final Entity entity) { public Acl createAcl(String bucket, Acl acl, BucketSourceOption... options) { final BucketAccessControl aclPb = acl.toBucketPb().setBucket(bucket); final Map optionsMap = optionMap(options); - return run(() -> storageRpc.createAcl(aclPb, optionsMap), Acl::fromPb); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketAclCreate(aclPb, optionsMap); + return run(exceptionHandler, () -> storageRpc.createAcl(aclPb, optionsMap), Acl::fromPb); } @Override @@ -1024,7 +1110,9 @@ public Acl createAcl(String bucket, Acl acl) { public Acl updateAcl(String bucket, Acl acl, BucketSourceOption... options) { final BucketAccessControl aclPb = acl.toBucketPb().setBucket(bucket); final Map optionsMap = optionMap(options); - return run(() -> storageRpc.patchAcl(aclPb, optionsMap), Acl::fromPb); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketAclUpdate(aclPb, optionsMap); + return run(exceptionHandler, () -> storageRpc.patchAcl(aclPb, optionsMap), Acl::fromPb); } @Override @@ -1035,7 +1123,10 @@ public Acl updateAcl(String bucket, Acl acl) { @Override public List listAcls(final String bucket, BucketSourceOption... options) { final Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketAclList(bucket, optionsMap); return run( + exceptionHandler, () -> storageRpc.listAcls(bucket, optionsMap), (answer) -> answer.stream() @@ -1050,29 +1141,38 @@ public List listAcls(final String bucket) { @Override public Acl getDefaultAcl(final String bucket, final Entity entity) { - return run(() -> storageRpc.getDefaultAcl(bucket, entity.toPb()), Acl::fromPb); + String pb = entity.toPb(); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForDefaultObjectAclGet(pb); + return run(exceptionHandler, () -> storageRpc.getDefaultAcl(bucket, pb), Acl::fromPb); } @Override public boolean deleteDefaultAcl(final String bucket, final Entity entity) { - return run(() -> storageRpc.deleteDefaultAcl(bucket, entity.toPb()), Function.identity()); + String pb = entity.toPb(); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForDefaultObjectAclDelete(pb); + return run( + exceptionHandler, () -> storageRpc.deleteDefaultAcl(bucket, pb), Function.identity()); } @Override public Acl createDefaultAcl(String bucket, Acl acl) { final ObjectAccessControl aclPb = acl.toObjectPb().setBucket(bucket); - return run(() -> storageRpc.createDefaultAcl(aclPb), Acl::fromPb); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForDefaultObjectAclCreate(aclPb); + return run(exceptionHandler, () -> storageRpc.createDefaultAcl(aclPb), Acl::fromPb); } @Override public Acl updateDefaultAcl(String bucket, Acl acl) { final ObjectAccessControl aclPb = acl.toObjectPb().setBucket(bucket); - return run(() -> storageRpc.patchDefaultAcl(aclPb), Acl::fromPb); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForDefaultObjectAclUpdate(aclPb); + return run(exceptionHandler, () -> storageRpc.patchDefaultAcl(aclPb), Acl::fromPb); } @Override public List listDefaultAcls(final String bucket) { + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForDefaultObjectAclList(bucket); return run( + exceptionHandler, () -> storageRpc.listDefaultAcls(bucket), (answer) -> answer.stream() @@ -1082,19 +1182,27 @@ public List listDefaultAcls(final String bucket) { @Override public Acl getAcl(final BlobId blob, final Entity entity) { + String bucket = blob.getBucket(); + String name = blob.getName(); + Long generation = blob.getGeneration(); + String pb = entity.toPb(); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectAclGet(bucket, name, generation, pb); return run( - () -> - storageRpc.getAcl( - blob.getBucket(), blob.getName(), blob.getGeneration(), entity.toPb()), - Acl::fromPb); + exceptionHandler, () -> storageRpc.getAcl(bucket, name, generation, pb), Acl::fromPb); } @Override public boolean deleteAcl(final BlobId blob, final Entity entity) { + String bucket = blob.getBucket(); + String name = blob.getName(); + Long generation = blob.getGeneration(); + String pb = entity.toPb(); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectAclDelete(bucket, name, generation, pb); return run( - () -> - storageRpc.deleteAcl( - blob.getBucket(), blob.getName(), blob.getGeneration(), entity.toPb()), + exceptionHandler, + () -> storageRpc.deleteAcl(bucket, name, generation, pb), Function.identity()); } @@ -1105,7 +1213,8 @@ public Acl createAcl(final BlobId blob, final Acl acl) { .setBucket(blob.getBucket()) .setObject(blob.getName()) .setGeneration(blob.getGeneration()); - return run(() -> storageRpc.createAcl(aclPb), Acl::fromPb); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForObjectAclCreate(aclPb); + return run(exceptionHandler, () -> storageRpc.createAcl(aclPb), Acl::fromPb); } @Override @@ -1115,13 +1224,20 @@ public Acl updateAcl(BlobId blob, Acl acl) { .setBucket(blob.getBucket()) .setObject(blob.getName()) .setGeneration(blob.getGeneration()); - return run(() -> storageRpc.patchAcl(aclPb), Acl::fromPb); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForObjectAclUpdate(aclPb); + return run(exceptionHandler, () -> storageRpc.patchAcl(aclPb), Acl::fromPb); } @Override public List listAcls(final BlobId blob) { + String bucket = blob.getBucket(); + String name = blob.getName(); + Long generation = blob.getGeneration(); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForObjectAclList(bucket, name, generation); return run( - () -> storageRpc.listAcls(blob.getBucket(), blob.getName(), blob.getGeneration()), + exceptionHandler, + () -> storageRpc.listAcls(bucket, name, generation), (answer) -> answer.stream() .map(Acl.FROM_OBJECT_PB_FUNCTION) @@ -1130,26 +1246,35 @@ public List listAcls(final BlobId blob) { public HmacKey createHmacKey( final ServiceAccount serviceAccount, final CreateHmacKeyOption... options) { - return run( - () -> storageRpc.createHmacKey(serviceAccount.getEmail(), optionMap(options)), - HmacKey::fromPb); + String pb = serviceAccount.getEmail(); + Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForHmacKeyCreate(pb, optionsMap); + return run(exceptionHandler, () -> storageRpc.createHmacKey(pb, optionsMap), HmacKey::fromPb); } @Override public Page listHmacKeys(ListHmacKeysOption... options) { - return listHmacKeys(getOptions(), optionMap(options)); + return listHmacKeys(getOptions(), retryAlgorithmManager, optionMap(options)); } @Override public HmacKeyMetadata getHmacKey(final String accessId, final GetHmacKeyOption... options) { - return run(() -> storageRpc.getHmacKey(accessId, optionMap(options)), HmacKeyMetadata::fromPb); + Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForHmacKeyGet(accessId, optionsMap); + return run( + exceptionHandler, + () -> storageRpc.getHmacKey(accessId, optionMap(options)), + HmacKeyMetadata::fromPb); } private HmacKeyMetadata updateHmacKey( final HmacKeyMetadata hmacKeyMetadata, final UpdateHmacKeyOption... options) { + com.google.api.services.storage.model.HmacKeyMetadata pb = hmacKeyMetadata.toPb(); + Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForHmacKeyUpdate(pb, optionsMap); return run( - () -> storageRpc.updateHmacKey(hmacKeyMetadata.toPb(), optionMap(options)), - HmacKeyMetadata::fromPb); + exceptionHandler, () -> storageRpc.updateHmacKey(pb, optionsMap), HmacKeyMetadata::fromPb); } @Override @@ -1168,19 +1293,27 @@ public HmacKeyMetadata updateHmacKeyState( @Override public void deleteHmacKey(final HmacKeyMetadata metadata, final DeleteHmacKeyOption... options) { + com.google.api.services.storage.model.HmacKeyMetadata pb = metadata.toPb(); + Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForHmacKeyDelete(pb, optionsMap); run( + exceptionHandler, (Callable) () -> { - storageRpc.deleteHmacKey(metadata.toPb(), optionMap(options)); + storageRpc.deleteHmacKey(pb, optionsMap); return null; }, Function.identity()); } private static Page listHmacKeys( - final StorageOptions serviceOptions, final Map options) { + final StorageOptions serviceOptions, + final RetryAlgorithmManager retryAlgorithmManager, + final Map options) { + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForHmacKeyList(options); return Retrying.run( serviceOptions, + exceptionHandler, () -> serviceOptions.getStorageRpcV1().listHmacKeys(options), (result) -> { String cursor = result.x(); @@ -1189,23 +1322,33 @@ private static Page listHmacKeys( ? ImmutableList.of() : Iterables.transform(result.y(), HmacKeyMetadata::fromPb); return new PageImpl<>( - new HmacKeyMetadataPageFetcher(serviceOptions, options), cursor, metadata); + new HmacKeyMetadataPageFetcher(serviceOptions, retryAlgorithmManager, options), + cursor, + metadata); }); } @Override public Policy getIamPolicy(final String bucket, BucketSourceOption... options) { final Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsGetIamPolicy(bucket, optionsMap); return run( - () -> storageRpc.getIamPolicy(bucket, optionsMap), PolicyHelper::convertFromApiPolicy); + exceptionHandler, + () -> storageRpc.getIamPolicy(bucket, optionsMap), + PolicyHelper::convertFromApiPolicy); } @Override public Policy setIamPolicy( final String bucket, final Policy policy, BucketSourceOption... options) { + com.google.api.services.storage.model.Policy pb = convertToApiPolicy(policy); final Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsSetIamPolicy(bucket, pb, optionsMap); return run( - () -> storageRpc.setIamPolicy(bucket, convertToApiPolicy(policy), optionsMap), + exceptionHandler, + () -> storageRpc.setIamPolicy(bucket, pb, optionsMap), PolicyHelper::convertFromApiPolicy); } @@ -1213,7 +1356,10 @@ public Policy setIamPolicy( public List testIamPermissions( final String bucket, final List permissions, BucketSourceOption... options) { final Map optionsMap = optionMap(options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsTestIamPermissions(bucket, permissions, optionsMap); return run( + exceptionHandler, () -> storageRpc.testIamPermissions(bucket, permissions, optionsMap), (response) -> { final Set heldPermissions = @@ -1230,17 +1376,23 @@ public List testIamPermissions( public Bucket lockRetentionPolicy(BucketInfo bucketInfo, BucketTargetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb(); final Map optionsMap = optionMap(bucketInfo, options); + ExceptionHandler exceptionHandler = + retryAlgorithmManager.getForBucketsLockRetentionPolicy(bucketPb, optionsMap); return run( - () -> storageRpc.lockRetentionPolicy(bucketPb, optionsMap), (x) -> Bucket.fromPb(this, x)); + exceptionHandler, + () -> storageRpc.lockRetentionPolicy(bucketPb, optionsMap), + (x) -> Bucket.fromPb(this, x)); } @Override public ServiceAccount getServiceAccount(final String projectId) { - return run(() -> storageRpc.getServiceAccount(projectId), ServiceAccount::fromPb); + ExceptionHandler exceptionHandler = retryAlgorithmManager.getForServiceAccountGet(projectId); + return run( + exceptionHandler, () -> storageRpc.getServiceAccount(projectId), ServiceAccount::fromPb); } - private U run(Callable c, Function f) { - return Retrying.run(getOptions(), c, f); + private U run(ExceptionHandler exceptionHandler, Callable c, Function f) { + return Retrying.run(getOptions(), exceptionHandler, c, f); } private static void addToOptionMap( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java index 4a25e549b..0cf60a787 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java @@ -25,6 +25,7 @@ import com.google.cloud.storage.spi.StorageRpcFactory; import com.google.cloud.storage.spi.v1.HttpStorageRpc; import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import java.util.Set; @@ -36,6 +37,8 @@ public class StorageOptions extends ServiceOptions { private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE); private static final String DEFAULT_HOST = "https://storage.googleapis.com"; + private final RetryAlgorithmManager retryAlgorithmManager; + public static class DefaultStorageFactory implements StorageFactory { private static final StorageFactory INSTANCE = new DefaultStorageFactory(); @@ -58,6 +61,8 @@ public ServiceRpc create(StorageOptions options) { public static class Builder extends ServiceOptions.Builder { + private RetryAlgorithmManager retryAlgorithmManager; + private Builder() {} private Builder(StorageOptions options) { @@ -73,14 +78,30 @@ public Builder setTransportOptions(TransportOptions transportOptions) { return super.setTransportOptions(transportOptions); } + Builder setUseLegacyRetryAlgorithms() { + return setRetryAlgorithmManager(new LegacyRetryAlgorithmManager()); + } + + Builder setUseDefaultRetryAlgorithms() { + return setRetryAlgorithmManager(new NewRetryAlgorithmManager()); + } + + private Builder setRetryAlgorithmManager(RetryAlgorithmManager retryAlgorithmManager) { + this.retryAlgorithmManager = retryAlgorithmManager; + return this; + } + @Override public StorageOptions build() { - return new StorageOptions(this); + return new StorageOptions(this, new StorageDefaults()); } } - private StorageOptions(Builder builder) { - super(StorageFactory.class, StorageRpcFactory.class, builder, new StorageDefaults()); + private StorageOptions(Builder builder, StorageDefaults serviceDefaults) { + super(StorageFactory.class, StorageRpcFactory.class, builder, serviceDefaults); + this.retryAlgorithmManager = + MoreObjects.firstNonNull( + builder.retryAlgorithmManager, serviceDefaults.getRetryAlgorithmManager()); } private static class StorageDefaults implements ServiceDefaults { @@ -99,6 +120,10 @@ public StorageRpcFactory getDefaultRpcFactory() { public TransportOptions getDefaultTransportOptions() { return getDefaultHttpTransportOptions(); } + + public RetryAlgorithmManager getRetryAlgorithmManager() { + return new LegacyRetryAlgorithmManager(); + } } public static HttpTransportOptions getDefaultHttpTransportOptions() { @@ -121,6 +146,10 @@ protected StorageRpc getStorageRpcV1() { return (StorageRpc) getRpc(); } + protected RetryAlgorithmManager getRetryAlgorithmManager() { + return retryAlgorithmManager; + } + /** Returns a default {@code StorageOptions} instance. */ public static StorageOptions getDefaultInstance() { return newBuilder().build(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index 36752a4dc..555349752 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -169,6 +169,8 @@ public long millisTime() { private Blob expectedBlob; private Storage serviceMockReturnsOptions = createMock(Storage.class); private StorageOptions mockOptions = createMock(StorageOptions.class); + private final RetryAlgorithmManager retryAlgorithmManager = + StorageOptions.getDefaultInstance().getRetryAlgorithmManager(); @Before public void setUp() { @@ -180,8 +182,10 @@ public void tearDown() throws Exception { verify(storage); } - private void initializeExpectedBlob(int optionsCalls) { - expect(serviceMockReturnsOptions.getOptions()).andReturn(mockOptions).times(optionsCalls); + private void initializeExpectedBlob() { + expect(serviceMockReturnsOptions.getOptions()).andReturn(mockOptions).anyTimes(); + expect(mockOptions.getRetryAlgorithmManager()).andReturn(retryAlgorithmManager).anyTimes(); + replay(mockOptions); replay(serviceMockReturnsOptions); expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(BLOB_INFO)); } @@ -192,9 +196,9 @@ private void initializeBlob() { @Test public void testExists_True() throws Exception { - initializeExpectedBlob(1); + initializeExpectedBlob(); Storage.BlobGetOption[] expectedOptions = {Storage.BlobGetOption.fields()}; - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.get(expectedBlob.getBlobId(), expectedOptions)).andReturn(expectedBlob); replay(storage); initializeBlob(); @@ -204,7 +208,7 @@ public void testExists_True() throws Exception { @Test public void testExists_False() throws Exception { Storage.BlobGetOption[] expectedOptions = {Storage.BlobGetOption.fields()}; - expect(storage.getOptions()).andReturn(null); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.get(BLOB_INFO.getBlobId(), expectedOptions)).andReturn(null); replay(storage); initializeBlob(); @@ -213,9 +217,9 @@ public void testExists_False() throws Exception { @Test public void testContent() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); byte[] content = {1, 2}; - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.readAllBytes(BLOB_INFO.getBlobId())).andReturn(content); replay(storage); initializeBlob(); @@ -224,9 +228,9 @@ public void testContent() throws Exception { @Test public void testContentWithDecryptionKey() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); byte[] content = {1, 2}; - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect( storage.readAllBytes( BLOB_INFO.getBlobId(), Storage.BlobSourceOption.decryptionKey(BASE64_KEY))) @@ -240,9 +244,9 @@ public void testContentWithDecryptionKey() throws Exception { @Test public void testReload() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); Blob expectedReloadedBlob = expectedBlob.toBuilder().setCacheControl("c").build(); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.get(BLOB_INFO_NO_GENERATION.getBlobId(), new Storage.BlobGetOption[0])) .andReturn(expectedReloadedBlob); replay(storage); @@ -253,8 +257,8 @@ public void testReload() throws Exception { @Test public void testReloadNull() throws Exception { - initializeExpectedBlob(1); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.get(BLOB_INFO_NO_GENERATION.getBlobId(), new Storage.BlobGetOption[0])) .andReturn(null); replay(storage); @@ -265,10 +269,10 @@ public void testReloadNull() throws Exception { @Test public void testReloadWithOptions() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); Blob expectedReloadedBlob = expectedBlob.toBuilder().setCacheControl("c").build(); Storage.BlobGetOption[] options = {Storage.BlobGetOption.metagenerationMatch(42L)}; - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.get(BLOB_INFO_NO_GENERATION.getBlobId(), options)) .andReturn(expectedReloadedBlob); replay(storage); @@ -279,9 +283,9 @@ public void testReloadWithOptions() throws Exception { @Test public void testUpdate() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); Blob expectedUpdatedBlob = expectedBlob.toBuilder().setCacheControl("c").build(); - expect(storage.getOptions()).andReturn(mockOptions).times(2); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.update(eq(expectedUpdatedBlob), new Storage.BlobTargetOption[0])) .andReturn(expectedUpdatedBlob); replay(storage); @@ -293,8 +297,8 @@ public void testUpdate() throws Exception { @Test public void testDelete() throws Exception { - initializeExpectedBlob(2); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.delete(BLOB_INFO.getBlobId(), new Storage.BlobSourceOption[0])).andReturn(true); replay(storage); initializeBlob(); @@ -303,11 +307,11 @@ public void testDelete() throws Exception { @Test public void testCopyToBucket() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); BlobInfo target = BlobInfo.newBuilder(BlobId.of("bt", "n")).build(); CopyWriter copyWriter = createMock(CopyWriter.class); Capture capturedCopyRequest = Capture.newInstance(); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter); replay(storage); initializeBlob(); @@ -322,11 +326,11 @@ public void testCopyToBucket() throws Exception { @Test public void testCopyTo() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); BlobInfo target = BlobInfo.newBuilder(BlobId.of("bt", "nt")).build(); CopyWriter copyWriter = createMock(CopyWriter.class); Capture capturedCopyRequest = Capture.newInstance(); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter); replay(storage); initializeBlob(); @@ -341,12 +345,12 @@ public void testCopyTo() throws Exception { @Test public void testCopyToBlobId() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); BlobInfo target = BlobInfo.newBuilder(BlobId.of("bt", "nt")).build(); BlobId targetId = BlobId.of("bt", "nt"); CopyWriter copyWriter = createMock(CopyWriter.class); Capture capturedCopyRequest = Capture.newInstance(); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter); replay(storage); initializeBlob(); @@ -361,9 +365,9 @@ public void testCopyToBlobId() throws Exception { @Test public void testReader() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); ReadChannel channel = createMock(ReadChannel.class); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.reader(BLOB_INFO.getBlobId())).andReturn(channel); replay(storage); initializeBlob(); @@ -372,9 +376,9 @@ public void testReader() throws Exception { @Test public void testReaderWithDecryptionKey() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); ReadChannel channel = createMock(ReadChannel.class); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect( storage.reader( BLOB_INFO.getBlobId(), Storage.BlobSourceOption.decryptionKey(BASE64_KEY))) @@ -388,9 +392,9 @@ public void testReaderWithDecryptionKey() throws Exception { @Test public void testWriter() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); BlobWriteChannel channel = createMock(BlobWriteChannel.class); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.writer(eq(expectedBlob))).andReturn(channel); replay(storage); initializeBlob(); @@ -399,9 +403,9 @@ public void testWriter() throws Exception { @Test public void testWriterWithEncryptionKey() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); BlobWriteChannel channel = createMock(BlobWriteChannel.class); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.encryptionKey(BASE64_KEY)))) .andReturn(channel) .times(2); @@ -413,9 +417,9 @@ public void testWriterWithEncryptionKey() throws Exception { @Test public void testWriterWithKmsKeyName() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); BlobWriteChannel channel = createMock(BlobWriteChannel.class); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.kmsKeyName(KMS_KEY_NAME)))) .andReturn(channel); replay(storage); @@ -425,9 +429,9 @@ public void testWriterWithKmsKeyName() throws Exception { @Test public void testSignUrl() throws Exception { - initializeExpectedBlob(2); + initializeExpectedBlob(); URL url = new URL("http://localhost:123/bla"); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.signUrl(expectedBlob, 100, TimeUnit.SECONDS)).andReturn(url); replay(storage); initializeBlob(); @@ -436,8 +440,8 @@ public void testSignUrl() throws Exception { @Test public void testGetAcl() throws Exception { - initializeExpectedBlob(1); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.getAcl(BLOB_INFO.getBlobId(), User.ofAllAuthenticatedUsers())).andReturn(ACL); replay(storage); initializeBlob(); @@ -446,8 +450,8 @@ public void testGetAcl() throws Exception { @Test public void testDeleteAcl() throws Exception { - initializeExpectedBlob(1); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.deleteAcl(BLOB_INFO.getBlobId(), User.ofAllAuthenticatedUsers())) .andReturn(true); replay(storage); @@ -457,8 +461,8 @@ public void testDeleteAcl() throws Exception { @Test public void testCreateAcl() throws Exception { - initializeExpectedBlob(1); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); Acl returnedAcl = ACL.toBuilder().setEtag("ETAG").setId("ID").build(); expect(storage.createAcl(BLOB_INFO.getBlobId(), ACL)).andReturn(returnedAcl); replay(storage); @@ -468,8 +472,8 @@ public void testCreateAcl() throws Exception { @Test public void testUpdateAcl() throws Exception { - initializeExpectedBlob(1); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); Acl returnedAcl = ACL.toBuilder().setEtag("ETAG").setId("ID").build(); expect(storage.updateAcl(BLOB_INFO.getBlobId(), ACL)).andReturn(returnedAcl); replay(storage); @@ -479,8 +483,8 @@ public void testUpdateAcl() throws Exception { @Test public void testListAcls() throws Exception { - initializeExpectedBlob(1); - expect(storage.getOptions()).andReturn(mockOptions); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); expect(storage.listAcls(BLOB_INFO.getBlobId())).andReturn(ACLS); replay(storage); initializeBlob(); @@ -489,8 +493,10 @@ public void testListAcls() throws Exception { @Test public void testToBuilder() { - expect(storage.getOptions()).andReturn(mockOptions).times(6); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); + expect(mockOptions.getRetryAlgorithmManager()).andReturn(retryAlgorithmManager).anyTimes(); replay(storage); + replay(mockOptions); Blob fullBlob = new Blob(storage, new BlobInfo.BuilderImpl(FULL_BLOB_INFO)); assertEquals(fullBlob, fullBlob.toBuilder().build()); Blob simpleBlob = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO)); @@ -501,8 +507,8 @@ public void testToBuilder() { @Test public void testBuilder() { - initializeExpectedBlob(4); - expect(storage.getOptions()).andReturn(mockOptions).times(6); + initializeExpectedBlob(); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); replay(storage); Blob.Builder builder = new Blob.Builder(new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO))); Blob blob = @@ -609,11 +615,12 @@ public void testBuilder() { private StorageRpc prepareForDownload() { StorageRpc mockStorageRpc = createNiceMock(StorageRpc.class); - expect(storage.getOptions()).andReturn(mockOptions); + expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); replay(storage); expect(mockOptions.getStorageRpcV1()).andReturn(mockStorageRpc); expect(mockOptions.getRetrySettings()).andReturn(RETRY_SETTINGS); expect(mockOptions.getClock()).andReturn(API_CLOCK); + expect(mockOptions.getRetryAlgorithmManager()).andReturn(retryAlgorithmManager).anyTimes(); replay(mockOptions); blob = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO)); return mockStorageRpc; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index d25bd64a3..d9a70897f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -35,6 +35,8 @@ import static org.junit.Assert.fail; import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.ExceptionHandler; +import com.google.cloud.NoCredentials; import com.google.cloud.RestorableState; import com.google.cloud.WriteChannel; import com.google.cloud.storage.spi.StorageRpcFactory; @@ -47,6 +49,7 @@ import java.net.URL; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collections; import java.util.Map; import java.util.Random; import org.easymock.Capture; @@ -76,6 +79,7 @@ public class BlobWriteChannelTest { private StorageRpcFactory rpcFactoryMock; private StorageRpc storageRpcMock; private BlobWriteChannel writer; + private RetryAlgorithmManager retryAlgorithmManager; @Before public void setUp() { @@ -87,7 +91,9 @@ public void setUp() { StorageOptions.newBuilder() .setProjectId("projectid") .setServiceRpcFactory(rpcFactoryMock) + .setCredentials(NoCredentials.getInstance()) .build(); + retryAlgorithmManager = options.getRetryAlgorithmManager(); } @After @@ -99,7 +105,7 @@ public void tearDown() throws Exception { public void testCreate() { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertTrue(writer.isOpen()); assertNull(writer.getStorageObject()); } @@ -110,7 +116,7 @@ public void testCreateRetryableError() { .andThrow(socketClosedException); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertTrue(writer.isOpen()); assertNull(writer.getStorageObject()); } @@ -121,7 +127,7 @@ public void testCreateNonRetryableError() { .andThrow(new RuntimeException()); replay(storageRpcMock); try { - new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + newWriter(); Assert.fail(); } catch (RuntimeException ex) { assertNotNull(ex.getMessage()); @@ -129,15 +135,15 @@ public void testCreateNonRetryableError() { } @Test - public void testWriteWithoutFlush() throws IOException { + public void testWriteWithoutFlush() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE))); } @Test - public void testWriteWithFlushRetryChunk() throws IOException { + public void testWriteWithFlushRetryChunk() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -161,7 +167,7 @@ public void testWriteWithFlushRetryChunk() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); assertTrue(writer.isOpen()); @@ -170,7 +176,7 @@ public void testWriteWithFlushRetryChunk() throws IOException { } @Test - public void testWriteWithRetryFullChunk() throws IOException { + public void testWriteWithRetryFullChunk() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -198,7 +204,7 @@ public void testWriteWithRetryFullChunk() throws IOException { eq(true))) .andReturn(BLOB_INFO.toPb()); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); writer.close(); @@ -208,7 +214,7 @@ public void testWriteWithRetryFullChunk() throws IOException { } @Test - public void testWriteWithRemoteProgressMade() throws IOException { + public void testWriteWithRemoteProgressMade() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -233,7 +239,7 @@ public void testWriteWithRemoteProgressMade() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); assertTrue(writer.isOpen()); @@ -242,7 +248,7 @@ public void testWriteWithRemoteProgressMade() throws IOException { } @Test - public void testWriteWithDriftRetryCase4() throws IOException { + public void testWriteWithDriftRetryCase4() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -266,7 +272,7 @@ public void testWriteWithDriftRetryCase4() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); assertArrayEquals(buffer.array(), capturedBuffer.getValue()); @@ -279,7 +285,7 @@ public void testWriteWithDriftRetryCase4() throws IOException { } @Test - public void testWriteWithUnreachableRemoteOffset() throws IOException { + public void testWriteWithUnreachableRemoteOffset() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -294,7 +300,7 @@ public void testWriteWithUnreachableRemoteOffset() throws IOException { .andThrow(socketClosedException); expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(MIN_CHUNK_SIZE + 10L); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); try { writer.write(buffer); @@ -308,7 +314,7 @@ public void testWriteWithUnreachableRemoteOffset() throws IOException { } @Test - public void testWriteWithRetryAndObjectMetadata() throws IOException { + public void testWriteWithRetryAndObjectMetadata() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -339,7 +345,7 @@ public void testWriteWithRetryAndObjectMetadata() throws IOException { expect(storageRpcMock.queryCompletedResumableUpload(eq(UPLOAD_ID), eq((long) MIN_CHUNK_SIZE))) .andReturn(BLOB_INFO.toPb().setSize(BigInteger.valueOf(MIN_CHUNK_SIZE))); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); writer.close(); @@ -349,7 +355,7 @@ public void testWriteWithRetryAndObjectMetadata() throws IOException { } @Test - public void testWriteWithUploadCompletedByAnotherClient() throws IOException { + public void testWriteWithUploadCompletedByAnotherClient() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -374,7 +380,7 @@ public void testWriteWithUploadCompletedByAnotherClient() throws IOException { expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L); expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); try { writer.write(buffer); @@ -390,7 +396,7 @@ public void testWriteWithUploadCompletedByAnotherClient() throws IOException { } @Test - public void testWriteWithLocalOffsetGoingBeyondRemoteOffset() throws IOException { + public void testWriteWithLocalOffsetGoingBeyondRemoteOffset() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -414,7 +420,7 @@ public void testWriteWithLocalOffsetGoingBeyondRemoteOffset() throws IOException .andThrow(socketClosedException); expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(0L); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); try { writer.write(buffer); @@ -428,7 +434,7 @@ public void testWriteWithLocalOffsetGoingBeyondRemoteOffset() throws IOException } @Test - public void testGetCurrentUploadOffset() throws IOException { + public void testGetCurrentUploadOffset() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -462,7 +468,7 @@ public void testGetCurrentUploadOffset() throws IOException { eq(true))) .andReturn(BLOB_INFO.toPb()); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); writer.close(); @@ -472,7 +478,7 @@ public void testGetCurrentUploadOffset() throws IOException { } @Test - public void testWriteWithLastFlushRetryChunkButCompleted() throws IOException { + public void testWriteWithLastFlushRetryChunkButCompleted() throws Exception { ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); Capture capturedBuffer = Capture.newInstance(); expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); @@ -489,7 +495,7 @@ public void testWriteWithLastFlushRetryChunkButCompleted() throws IOException { expect(storageRpcMock.queryCompletedResumableUpload(eq(UPLOAD_ID), eq((long) MIN_CHUNK_SIZE))) .andReturn(BLOB_INFO.toPb().setSize(BigInteger.valueOf(MIN_CHUNK_SIZE))); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); writer.close(); assertFalse(writer.isRetrying()); @@ -501,7 +507,7 @@ public void testWriteWithLastFlushRetryChunkButCompleted() throws IOException { } @Test - public void testWriteWithFlush() throws IOException { + public void testWriteWithFlush() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -514,7 +520,7 @@ public void testWriteWithFlush() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.setChunkSize(CUSTOM_CHUNK_SIZE); ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); @@ -523,7 +529,7 @@ public void testWriteWithFlush() throws IOException { } @Test - public void testWritesAndFlush() throws IOException { + public void testWritesAndFlush() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -536,7 +542,7 @@ public void testWritesAndFlush() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; for (int i = 0; i < buffers.length; i++) { buffers[i] = randomBuffer(MIN_CHUNK_SIZE); @@ -552,7 +558,7 @@ public void testWritesAndFlush() throws IOException { } @Test - public void testCloseWithoutFlush() throws IOException { + public void testCloseWithoutFlush() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -560,7 +566,7 @@ public void testCloseWithoutFlush() throws IOException { eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertTrue(writer.isOpen()); writer.close(); assertArrayEquals(new byte[0], capturedBuffer.getValue()); @@ -569,7 +575,7 @@ public void testCloseWithoutFlush() throws IOException { } @Test - public void testCloseWithFlush() throws IOException { + public void testCloseWithFlush() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); @@ -583,7 +589,7 @@ public void testCloseWithFlush() throws IOException { eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertTrue(writer.isOpen()); writer.write(buffer); writer.close(); @@ -594,7 +600,7 @@ public void testCloseWithFlush() throws IOException { } @Test - public void testWriteClosed() throws IOException { + public void testWriteClosed() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -602,7 +608,7 @@ public void testWriteClosed() throws IOException { eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.close(); try { writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE)); @@ -614,7 +620,7 @@ public void testWriteClosed() throws IOException { } @Test - public void testSaveAndRestore() throws IOException { + public void testSaveAndRestore() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); @@ -631,7 +637,7 @@ public void testSaveAndRestore() throws IOException { replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); ByteBuffer buffer2 = randomBuffer(DEFAULT_CHUNK_SIZE); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); assertEquals(DEFAULT_CHUNK_SIZE, writer.write(buffer1)); assertArrayEquals(buffer1.array(), capturedBuffer.getValues().get(0)); assertEquals(new Long(0L), capturedPosition.getValues().get(0)); @@ -643,7 +649,7 @@ public void testSaveAndRestore() throws IOException { } @Test - public void testSaveAndRestoreClosed() throws IOException { + public void testSaveAndRestoreClosed() throws Exception { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -651,7 +657,7 @@ public void testSaveAndRestoreClosed() throws IOException { eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); writer.close(); RestorableState writerState = writer.capture(); RestorableState expectedWriterState = @@ -670,10 +676,10 @@ public void testSaveAndRestoreClosed() throws IOException { public void testStateEquals() { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID).times(2); replay(storageRpcMock); - writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = newWriter(); // avoid closing when you don't want partial writes to GCS upon failure @SuppressWarnings("resource") - WriteChannel writer2 = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + WriteChannel writer2 = newWriter(); RestorableState state = writer.capture(); RestorableState state2 = writer2.capture(); assertEquals(state, state2); @@ -682,15 +688,7 @@ public void testStateEquals() { } @Test - public void testWriteWithSignedURLAndWithoutFlush() throws IOException { - expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); - replay(storageRpcMock); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); - assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE))); - } - - @Test - public void testWriteWithSignedURLAndWithFlush() throws IOException { + public void testWriteWithSignedURLAndWithFlush() throws Exception { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -703,7 +701,7 @@ public void testWriteWithSignedURLAndWithFlush() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer = newWriterForSignedUrl(); writer.setChunkSize(CUSTOM_CHUNK_SIZE); ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); @@ -711,7 +709,7 @@ public void testWriteWithSignedURLAndWithFlush() throws IOException { } @Test - public void testWriteWithSignedURLAndFlush() throws IOException { + public void testWriteWithSignedURLAndFlush() throws Exception { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -724,7 +722,7 @@ public void testWriteWithSignedURLAndFlush() throws IOException { eq(false))) .andReturn(null); replay(storageRpcMock); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer = newWriterForSignedUrl(); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; for (int i = 0; i < buffers.length; i++) { buffers[i] = randomBuffer(MIN_CHUNK_SIZE); @@ -739,7 +737,7 @@ public void testWriteWithSignedURLAndFlush() throws IOException { } @Test - public void testCloseWithSignedURLWithoutFlush() throws IOException { + public void testCloseWithSignedURLWithoutFlush() throws Exception { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -747,7 +745,7 @@ public void testCloseWithSignedURLWithoutFlush() throws IOException { eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer = newWriterForSignedUrl(); assertTrue(writer.isOpen()); writer.close(); assertArrayEquals(new byte[0], capturedBuffer.getValue()); @@ -755,7 +753,7 @@ public void testCloseWithSignedURLWithoutFlush() throws IOException { } @Test - public void testCloseWithSignedURLWithFlush() throws IOException { + public void testCloseWithSignedURLWithFlush() throws Exception { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); @@ -769,7 +767,7 @@ public void testCloseWithSignedURLWithFlush() throws IOException { eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer = newWriterForSignedUrl(); assertTrue(writer.isOpen()); writer.write(buffer); writer.close(); @@ -779,7 +777,7 @@ public void testCloseWithSignedURLWithFlush() throws IOException { } @Test - public void testWriteWithSignedURLClosed() throws IOException { + public void testWriteWithSignedURLClosed() throws Exception { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( @@ -787,7 +785,7 @@ public void testWriteWithSignedURLClosed() throws IOException { eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer = newWriterForSignedUrl(); writer.close(); try { writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE)); @@ -798,7 +796,7 @@ public void testWriteWithSignedURLClosed() throws IOException { } @Test - public void testSaveAndRestoreWithSignedURL() throws IOException { + public void testSaveAndRestoreWithSignedURL() throws Exception { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); @@ -815,7 +813,7 @@ public void testSaveAndRestoreWithSignedURL() throws IOException { replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); ByteBuffer buffer2 = randomBuffer(DEFAULT_CHUNK_SIZE); - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); + writer = newWriterForSignedUrl(); assertEquals(DEFAULT_CHUNK_SIZE, writer.write(buffer1)); assertArrayEquals(buffer1.array(), capturedBuffer.getValues().get(0)); assertEquals(new Long(0L), capturedPosition.getValues().get(0)); @@ -826,18 +824,35 @@ public void testSaveAndRestoreWithSignedURL() throws IOException { assertEquals(new Long(DEFAULT_CHUNK_SIZE), capturedPosition.getValues().get(1)); } - @Test - public void testRuntimeExceptionWithSignedURL() throws MalformedURLException { - String exceptionMessage = "invalid signedURL"; - expect(new BlobWriteChannel(options, new URL(SIGNED_URL))) - .andThrow(new RuntimeException(exceptionMessage)); - replay(storageRpcMock); - try { - writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); - Assert.fail(); - } catch (StorageException ex) { - assertNotNull(ex.getMessage()); - } + private BlobWriteChannel newWriter() { + Map optionsMap = EMPTY_RPC_OPTIONS; + ExceptionHandler createResultExceptionHandler = + retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap); + ExceptionHandler writeResultExceptionHandler = + retryAlgorithmManager.getForResumableUploadSessionWrite(optionsMap); + return BlobWriteChannel.newBuilder() + .setStorageOptions(options) + .setBlobInfo(BLOB_INFO) + .setUploadIdSupplier( + ResumableMedia.startUploadForBlobInfo( + options, BLOB_INFO, optionsMap, createResultExceptionHandler)) + .setPutExceptionHandler(writeResultExceptionHandler) + .build(); + } + + private BlobWriteChannel newWriterForSignedUrl() throws MalformedURLException { + Map optionsMap = Collections.emptyMap(); + ExceptionHandler createResultExceptionHandler = + retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap); + ExceptionHandler writeResultExceptionHandler = + retryAlgorithmManager.getForResumableUploadSessionWrite(optionsMap); + return BlobWriteChannel.newBuilder() + .setStorageOptions(options) + .setUploadIdSupplier( + ResumableMedia.startUploadForSignedUrl( + options, new URL(SIGNED_URL), createResultExceptionHandler)) + .setPutExceptionHandler(writeResultExceptionHandler) + .build(); } private static ByteBuffer randomBuffer(int size) { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index e8a8e3f46..5d9e622f1 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -136,6 +136,8 @@ public class BucketTest { private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; private static final Key KEY = new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); + private final RetryAlgorithmManager retryAlgorithmManager = + StorageOptions.getDefaultInstance().getRetryAlgorithmManager(); private Storage storage; private Storage serviceMockReturnsOptions = createMock(Storage.class); @@ -154,9 +156,11 @@ public void tearDown() throws Exception { verify(storage); } - private void initializeExpectedBucket(int optionsCalls) { - expect(serviceMockReturnsOptions.getOptions()).andReturn(mockOptions).times(optionsCalls); + private void initializeExpectedBucket() { + expect(serviceMockReturnsOptions.getOptions()).andReturn(mockOptions).anyTimes(); replay(serviceMockReturnsOptions); + expect(mockOptions.getRetryAlgorithmManager()).andReturn(retryAlgorithmManager).anyTimes(); + replay(mockOptions); expectedBucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(BUCKET_INFO)); blobResults = ImmutableList.of( @@ -177,7 +181,7 @@ private void initializeBucket() { @Test public void testExists_True() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); Storage.BucketGetOption[] expectedOptions = {Storage.BucketGetOption.fields()}; expect(storage.getOptions()).andReturn(mockOptions); expect(storage.get(BUCKET_INFO.getName(), expectedOptions)).andReturn(expectedBucket); @@ -188,7 +192,7 @@ public void testExists_True() throws Exception { @Test public void testExists_False() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); Storage.BucketGetOption[] expectedOptions = {Storage.BucketGetOption.fields()}; expect(storage.getOptions()).andReturn(mockOptions); expect(storage.get(BUCKET_INFO.getName(), expectedOptions)).andReturn(null); @@ -199,7 +203,7 @@ public void testExists_False() throws Exception { @Test public void testReload() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BucketInfo updatedInfo = BUCKET_INFO.toBuilder().setNotFoundPage("p").build(); Bucket expectedUpdatedBucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(updatedInfo)); @@ -213,7 +217,7 @@ public void testReload() throws Exception { @Test public void testReloadNull() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.get(BUCKET_INFO.getName())).andReturn(null); replay(storage); @@ -223,7 +227,7 @@ public void testReloadNull() throws Exception { @Test public void testReloadWithOptions() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BucketInfo updatedInfo = BUCKET_INFO.toBuilder().setNotFoundPage("p").build(); Bucket expectedUpdatedBucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(updatedInfo)); @@ -238,7 +242,7 @@ public void testReloadWithOptions() throws Exception { @Test public void testUpdate() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); Bucket expectedUpdatedBucket = expectedBucket.toBuilder().setNotFoundPage("p").build(); expect(storage.getOptions()).andReturn(mockOptions).times(2); expect(storage.update(expectedUpdatedBucket)).andReturn(expectedUpdatedBucket); @@ -251,7 +255,7 @@ public void testUpdate() throws Exception { @Test public void testDelete() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.delete(BUCKET_INFO.getName())).andReturn(true); replay(storage); @@ -261,7 +265,7 @@ public void testDelete() throws Exception { @Test public void testList() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); PageImpl expectedBlobPage = new PageImpl<>(null, "c", blobResults); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.list(BUCKET_INFO.getName())).andReturn(expectedBlobPage); @@ -280,7 +284,7 @@ public void testList() throws Exception { @Test public void testGet() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); Blob expectedBlob = new Blob( serviceMockReturnsOptions, @@ -296,7 +300,7 @@ public void testGet() throws Exception { @Test public void testGetAllArray() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); List blobIds = Lists.transform( @@ -315,7 +319,7 @@ public BlobId apply(Blob blob) { @Test public void testGetAllIterable() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); List blobIds = Lists.transform( @@ -334,7 +338,7 @@ public BlobId apply(Blob blob) { @Test public void testCreate() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder("b", "n").setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -348,7 +352,7 @@ public void testCreate() throws Exception { @Test public void testCreateNoContentType() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder("b", "n").build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -362,7 +366,7 @@ public void testCreateNoContentType() throws Exception { @Test public void testCreateWithOptions() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n", 42L)) .setContentType(CONTENT_TYPE) @@ -399,7 +403,7 @@ public void testCreateWithOptions() throws Exception { @Test public void testCreateWithEncryptionKey() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n")).setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -415,7 +419,7 @@ public void testCreateWithEncryptionKey() throws Exception { @Test public void testCreateWithKmsKeyName() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n")).setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -432,7 +436,7 @@ public void testCreateWithKmsKeyName() throws Exception { @Test public void testCreateNotExists() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n", 0L)).setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); @@ -448,7 +452,7 @@ public void testCreateNotExists() throws Exception { @Test public void testCreateWithWrongGenerationOptions() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); replay(storage); initializeBucket(); @@ -468,7 +472,7 @@ public void testCreateWithWrongGenerationOptions() throws Exception { @Test public void testCreateWithWrongMetagenerationOptions() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); replay(storage); initializeBucket(); @@ -489,7 +493,7 @@ public void testCreateWithWrongMetagenerationOptions() throws Exception { @Test @SuppressWarnings({"unchecked", "deprecation"}) public void testCreateFromStream() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder("b", "n").setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -505,7 +509,7 @@ public void testCreateFromStream() throws Exception { @Test @SuppressWarnings({"unchecked", "deprecation"}) public void testCreateFromStreamNoContentType() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder("b", "n").build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -521,7 +525,7 @@ public void testCreateFromStreamNoContentType() throws Exception { @Test @SuppressWarnings({"unchecked", "deprecation"}) public void testCreateFromStreamWithOptions() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n", 42L)) .setContentType(CONTENT_TYPE) @@ -566,7 +570,7 @@ public void testCreateFromStreamWithOptions() throws Exception { @Test @SuppressWarnings({"unchecked", "deprecation"}) public void testCreateFromStreamWithEncryptionKey() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n")).setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); byte[] content = {0xD, 0xE, 0xA, 0xD}; @@ -584,7 +588,7 @@ public void testCreateFromStreamWithEncryptionKey() throws Exception { @Test @SuppressWarnings({"unchecked", "deprecation"}) public void testCreateFromStreamNotExists() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); BlobInfo info = BlobInfo.newBuilder(BlobId.of("b", "n", 0L)).setContentType(CONTENT_TYPE).build(); Blob expectedBlob = new Blob(serviceMockReturnsOptions, new BlobInfo.BuilderImpl(info)); @@ -602,7 +606,7 @@ public void testCreateFromStreamNotExists() throws Exception { @Test public void testCreateFromStreamWithWrongGenerationOptions() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); replay(storage); initializeBucket(); @@ -623,7 +627,7 @@ public void testCreateFromStreamWithWrongGenerationOptions() throws Exception { @Test public void testCreateFromStreamWithWrongMetagenerationOptions() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); replay(storage); initializeBucket(); @@ -644,7 +648,7 @@ public void testCreateFromStreamWithWrongMetagenerationOptions() throws Exceptio @Test public void testGetAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.getAcl(BUCKET_INFO.getName(), User.ofAllAuthenticatedUsers())).andReturn(ACL); replay(storage); @@ -654,7 +658,7 @@ public void testGetAcl() throws Exception { @Test public void testDeleteAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.deleteAcl(BUCKET_INFO.getName(), User.ofAllAuthenticatedUsers())) .andReturn(true); @@ -665,7 +669,7 @@ public void testDeleteAcl() throws Exception { @Test public void testCreateAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); Acl returnedAcl = ACL.toBuilder().setEtag("ETAG").setId("ID").build(); expect(storage.createAcl(BUCKET_INFO.getName(), ACL)).andReturn(returnedAcl); @@ -676,7 +680,7 @@ public void testCreateAcl() throws Exception { @Test public void testUpdateAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); Acl returnedAcl = ACL.toBuilder().setEtag("ETAG").setId("ID").build(); expect(storage.updateAcl(BUCKET_INFO.getName(), ACL)).andReturn(returnedAcl); @@ -687,7 +691,7 @@ public void testUpdateAcl() throws Exception { @Test public void testListAcls() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.listAcls(BUCKET_INFO.getName())).andReturn(ACLS); replay(storage); @@ -697,7 +701,7 @@ public void testListAcls() throws Exception { @Test public void testGetDefaultAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.getDefaultAcl(BUCKET_INFO.getName(), User.ofAllAuthenticatedUsers())) .andReturn(ACL); @@ -708,7 +712,7 @@ public void testGetDefaultAcl() throws Exception { @Test public void testDeleteDefaultAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.deleteDefaultAcl(BUCKET_INFO.getName(), User.ofAllAuthenticatedUsers())) .andReturn(true); @@ -719,7 +723,7 @@ public void testDeleteDefaultAcl() throws Exception { @Test public void testCreateDefaultAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); Acl returnedAcl = ACL.toBuilder().setEtag("ETAG").setId("ID").build(); expect(storage.createDefaultAcl(BUCKET_INFO.getName(), ACL)).andReturn(returnedAcl); @@ -730,7 +734,7 @@ public void testCreateDefaultAcl() throws Exception { @Test public void testUpdateDefaultAcl() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); Acl returnedAcl = ACL.toBuilder().setEtag("ETAG").setId("ID").build(); expect(storage.updateDefaultAcl(BUCKET_INFO.getName(), ACL)).andReturn(returnedAcl); @@ -741,7 +745,7 @@ public void testUpdateDefaultAcl() throws Exception { @Test public void testListDefaultAcls() throws Exception { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.listDefaultAcls(BUCKET_INFO.getName())).andReturn(ACLS); replay(storage); @@ -751,7 +755,7 @@ public void testListDefaultAcls() throws Exception { @Test public void testLockRetention() throws Exception { - initializeExpectedBucket(5); + initializeExpectedBucket(); Bucket expectedRetentionLockedBucket = expectedBucket .toBuilder() @@ -790,7 +794,7 @@ public void testToBuilder() { @Test @SuppressWarnings({"unchecked", "deprecation"}) public void testBuilder() { - initializeExpectedBucket(4); + initializeExpectedBucket(); expect(storage.getOptions()).andReturn(mockOptions).times(4); replay(storage); Bucket.Builder builder = @@ -854,7 +858,7 @@ public void testBuilder() { @Test public void testDeleteLifecycleRules() { - initializeExpectedBucket(6); + initializeExpectedBucket(); Bucket bucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(FULL_BUCKET_INFO)); assertThat(bucket.getLifecycleRules()).hasSize(1); @@ -870,7 +874,7 @@ public void testDeleteLifecycleRules() { @Test public void testUpdateBucketLogging() { - initializeExpectedBucket(6); + initializeExpectedBucket(); BucketInfo.Logging logging = BucketInfo.Logging.newBuilder() .setLogBucket("logs-bucket") @@ -893,7 +897,7 @@ public void testUpdateBucketLogging() { @Test public void testRemoveBucketCORS() { - initializeExpectedBucket(6); + initializeExpectedBucket(); List origins = ImmutableList.of(Cors.Origin.of("http://cloud.google.com")); List httpMethods = ImmutableList.of(HttpMethod.GET); List responseHeaders = ImmutableList.of("Content-Type"); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java index 0706df1a7..fb86d5d19 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java @@ -38,4 +38,8 @@ public static Blob blobCopyWithStorage(Blob b, Storage s) { BlobInfo.BuilderImpl builder = (BlobInfo.BuilderImpl) BlobInfo.fromPb(b.toPb()).toBuilder(); return new Blob(s, builder); } + + public static StorageOptions.Builder useNewRetryAlgorithmManager(StorageOptions.Builder builder) { + return builder.setUseDefaultRetryAlgorithms(); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java new file mode 100644 index 000000000..47b165dc8 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertNotNull; + +import com.google.cloud.BaseService; +import com.google.cloud.ExceptionHandler; +import java.net.URL; +import org.junit.Assert; +import org.junit.Test; + +public final class ResumableMediaTest { + private static final String SIGNED_URL = + "http://www.test.com/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; + + private final ExceptionHandler createResultExceptionHandler = BaseService.EXCEPTION_HANDLER; + + @Test + public void startUploadForSignedUrl_expectStorageException_whenUrlInvalid() throws Exception { + try { + ResumableMedia.startUploadForSignedUrl( + StorageOptions.newBuilder().build(), + new URL(SIGNED_URL), + createResultExceptionHandler) + .get(); + Assert.fail(); + } catch (StorageException ex) { + assertNotNull(ex.getMessage()); + } + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java index ae49f13f0..6d4876d47 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java @@ -17,6 +17,7 @@ package com.google.cloud.storage; import com.google.cloud.BaseSerializationTest; +import com.google.cloud.ExceptionHandler; import com.google.cloud.NoCredentials; import com.google.cloud.PageImpl; import com.google.cloud.ReadChannel; @@ -99,12 +100,17 @@ protected Serializable[] serializableObjects() { @Override protected Restorable[] restorableObjects() { StorageOptions options = StorageOptions.newBuilder().setProjectId("p2").build(); + ExceptionHandler exceptionHandler = + options.getRetryAlgorithmManager().getForResumableUploadSessionWrite(EMPTY_RPC_OPTIONS); ReadChannel reader = new BlobReadChannel(options, BlobId.of("b", "n"), EMPTY_RPC_OPTIONS); // avoid closing when you don't want partial writes to GCS upon failure @SuppressWarnings("resource") BlobWriteChannel writer = new BlobWriteChannel( - options, BlobInfo.newBuilder(BlobId.of("b", "n")).build(), "upload-id"); + options, + BlobInfo.newBuilder(BlobId.of("b", "n")).build(), + "upload-id", + exceptionHandler); return new Restorable[] {reader, writer}; } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageBatchTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageBatchTest.java index 3943e1409..978cffe0c 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageBatchTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageBatchTest.java @@ -54,6 +54,8 @@ public class StorageBatchTest { BlobTargetOption.generationMatch(), BlobTargetOption.metagenerationMatch() }; private static final GoogleJsonError GOOGLE_JSON_ERROR = new GoogleJsonError(); + private final RetryAlgorithmManager retryAlgorithmManager = + StorageOptions.getDefaultInstance().getRetryAlgorithmManager(); private StorageOptions optionsMock; private StorageRpc storageRpcMock; @@ -67,6 +69,9 @@ public void setUp() { storageRpcMock = EasyMock.createMock(StorageRpc.class); batchMock = EasyMock.createMock(RpcBatch.class); EasyMock.expect(optionsMock.getStorageRpcV1()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.getRetryAlgorithmManager()) + .andReturn(retryAlgorithmManager) + .anyTimes(); EasyMock.expect(storageRpcMock.createBatch()).andReturn(batchMock); EasyMock.replay(optionsMock, storageRpcMock, batchMock, storage); storageBatch = new StorageBatch(optionsMock); @@ -165,8 +170,11 @@ public void testUpdate() { @Test public void testUpdateWithOptions() { EasyMock.reset(storage, batchMock, optionsMock); - EasyMock.expect(storage.getOptions()).andReturn(optionsMock).times(2); - EasyMock.expect(optionsMock.getService()).andReturn(storage); + EasyMock.expect(storage.getOptions()).andReturn(optionsMock).anyTimes(); + EasyMock.expect(optionsMock.getService()).andReturn(storage).anyTimes(); + EasyMock.expect(optionsMock.getRetryAlgorithmManager()) + .andReturn(retryAlgorithmManager) + .anyTimes(); Capture> callback = Capture.newInstance(); Capture> capturedOptions = Capture.newInstance(); batchMock.addPatch( @@ -216,8 +224,11 @@ public void testGet() { @Test public void testGetWithOptions() { EasyMock.reset(storage, batchMock, optionsMock); - EasyMock.expect(storage.getOptions()).andReturn(optionsMock).times(2); - EasyMock.expect(optionsMock.getService()).andReturn(storage); + EasyMock.expect(storage.getOptions()).andReturn(optionsMock).anyTimes(); + EasyMock.expect(optionsMock.getService()).andReturn(storage).anyTimes(); + EasyMock.expect(optionsMock.getRetryAlgorithmManager()) + .andReturn(retryAlgorithmManager) + .anyTimes(); Capture> callback = Capture.newInstance(); Capture> capturedOptions = Capture.newInstance(); batchMock.addGet( diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 590fe37af..5619e5f7b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -38,7 +38,6 @@ import com.google.cloud.Identity; import com.google.cloud.Policy; import com.google.cloud.ServiceOptions; -import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Project; import com.google.cloud.storage.Acl.Project.ProjectRole; import com.google.cloud.storage.Acl.Role; @@ -55,7 +54,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.security.InvalidKeyException; @@ -2148,16 +2146,6 @@ public void testRuntimeException() { } } - @Test - public void testWriterWithSignedURL() throws MalformedURLException { - EasyMock.expect(storageRpcMock.open(SIGNED_URL)).andReturn("upload-id"); - EasyMock.replay(storageRpcMock); - initializeService(); - WriteChannel writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); - assertNotNull(writer); - assertTrue(writer.isOpen()); - } - @Test public void testV4PostPolicy() { EasyMock.replay(storageRpcMock); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RetryTestFixture.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RetryTestFixture.java index cb0415c1e..79a19a124 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RetryTestFixture.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RetryTestFixture.java @@ -23,6 +23,7 @@ import com.google.cloud.NoCredentials; import com.google.cloud.conformance.storage.v1.InstructionList; import com.google.cloud.conformance.storage.v1.Method; +import com.google.cloud.storage.PackagePrivateMethodWorkarounds; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.conformance.retry.TestBench.RetryTestResource; @@ -138,6 +139,7 @@ private Storage newStorage(boolean forTest) { .setHost(testBench.getBaseUri()) .setCredentials(NoCredentials.getInstance()) .setProjectId("conformance-tests"); + builder = PackagePrivateMethodWorkarounds.useNewRetryAlgorithmManager(builder); if (forTest) { builder.setHeaderProvider( new FixedHeaderProvider() { From 2e7f041ef7873ede8c2d1ebb526639d1aff61963 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 27 Sep 2021 21:38:21 +0200 Subject: [PATCH 03/15] chore(deps): update dependency com.google.cloud:google-cloud-storage to v2.1.6 (#1045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-storage](https://togithub.com/googleapis/java-storage) | `2.1.5` -> `2.1.6` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/compatibility-slim/2.1.5)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/confidence-slim/2.1.5)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/java-storage ### [`v2.1.6`](https://togithub.com/googleapis/java-storage/blob/master/CHANGELOG.md#​216-httpswwwgithubcomgoogleapisjava-storagecomparev215v216-2021-09-24) [Compare Source](https://togithub.com/googleapis/java-storage/compare/v2.1.5...v2.1.6)
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-storage). --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 1690c2f4d..3fbd529a4 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-storage - 2.1.5 + 2.1.6 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index cd1a26d92..b692e70de 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-storage - 2.1.5 + 2.1.6 From 6d73e4631777542996a0ea815b482f5c19a8927d Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Tue, 28 Sep 2021 11:51:47 -0400 Subject: [PATCH 04/15] fix: update PAP to use inherited instead of unspecified (#1051) --- .../com/google/cloud/storage/BucketInfo.java | 26 +++++++++++++------ .../cloud/storage/it/ITStorageTest.java | 4 +-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 2bc3df390..83a836f78 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -113,13 +113,19 @@ public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) */ public enum PublicAccessPrevention { ENFORCED("enforced"), - /** Default value for Public Access Prevention */ - UNSPECIFIED("unspecified"), + /** + * Default value for Public Access Prevention + * + * @deprecated use {@link #INHERITED} + */ + @Deprecated + UNSPECIFIED("inherited"), /** * If the api returns a value that isn't defined in {@link PublicAccessPrevention} this value * will be returned. */ - UNKNOWN(null); + UNKNOWN(null), + INHERITED("inherited"); private final String value; @@ -133,10 +139,14 @@ public String getValue() { public static PublicAccessPrevention parse(String value) { String upper = value.toUpperCase(); - try { - return valueOf(upper); - } catch (IllegalArgumentException ignore) { - return UNKNOWN; + switch (upper) { + case "ENFORCED": + return ENFORCED; + case "UNSPECIFIED": + case "INHERITED": + return INHERITED; + default: + return UNKNOWN; } } } @@ -300,7 +310,7 @@ Builder setUniformBucketLevelAccessLockedTime(Long uniformBucketLevelAccessLocke /** * Sets the bucket's Public Access Prevention configuration. Currently supported options are - * {@link PublicAccessPrevention#UNSPECIFIED} or {@link PublicAccessPrevention#ENFORCED} + * {@link PublicAccessPrevention#INHERITED} or {@link PublicAccessPrevention#ENFORCED} * * @see public-access-prevention diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index f8f51d386..73ef7e9b8 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -3296,7 +3296,7 @@ private Bucket generatePublicAccessPreventionBucket(String bucketName, boolean e .setPublicAccessPrevention( enforced ? BucketInfo.PublicAccessPrevention.ENFORCED - : BucketInfo.PublicAccessPrevention.UNSPECIFIED) + : BucketInfo.PublicAccessPrevention.INHERITED) .build()) .build()); } @@ -3389,7 +3389,7 @@ public void testUBLAWithPublicAccessPreventionOnBucket() throws Exception { Bucket bucket = generatePublicAccessPreventionBucket(papBucket, false); assertEquals( bucket.getIamConfiguration().getPublicAccessPrevention(), - BucketInfo.PublicAccessPrevention.UNSPECIFIED); + BucketInfo.PublicAccessPrevention.INHERITED); assertFalse(bucket.getIamConfiguration().isUniformBucketLevelAccessEnabled()); assertFalse(bucket.getIamConfiguration().isBucketPolicyOnlyEnabled()); From 2c79005d29ee0b279850c7008b1afbb302f9c90d Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 28 Sep 2021 18:45:07 +0200 Subject: [PATCH 05/15] deps: update dependency com.google.apis:google-api-services-storage to v1-rev20210918-1.32.1 (#1046) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3285250ce..308367e59 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ com.google.apis google-api-services-storage - v1-rev20210914-1.32.1 + v1-rev20210918-1.32.1 org.easymock From 980dbd0cff319de9782e7a5efbf4e75e31c6df49 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Tue, 28 Sep 2021 12:45:59 -0400 Subject: [PATCH 06/15] test: add positive case for ResumableMedia#startUploadForSignedUrl (#1049) * cleanup dangling future --- .../google/cloud/storage/ResumableMedia.java | 3 +-- .../cloud/storage/ResumableMediaTest.java | 26 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java index d0e212ef3..78a0e5629 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java @@ -16,7 +16,6 @@ package com.google.cloud.storage; -import com.google.api.core.ApiFutures; import com.google.cloud.ExceptionHandler; import com.google.cloud.storage.spi.v1.StorageRpc; import java.net.URL; @@ -42,7 +41,7 @@ static Supplier startUploadForBlobInfo( static Supplier startUploadForSignedUrl( final StorageOptions storageOptions, final URL signedURL, ExceptionHandler exceptionHandler) { if (!isValidSignedURL(signedURL.getQuery())) { - ApiFutures.immediateFailedFuture(new StorageException(2, "invalid signedURL")); + throw new StorageException(2, "invalid signedURL"); } return () -> Retrying.run( diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java index 47b165dc8..d6fc7d674 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ResumableMediaTest.java @@ -16,17 +16,24 @@ package com.google.cloud.storage; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.cloud.BaseService; import com.google.cloud.ExceptionHandler; +import com.google.cloud.storage.spi.v1.StorageRpc; import java.net.URL; +import java.util.function.Supplier; import org.junit.Assert; import org.junit.Test; public final class ResumableMediaTest { - private static final String SIGNED_URL = - "http://www.test.com/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; + private static final String SIGNED_URL_INVALID = + "http://localhost/test-bucket/test1.txt?GoogAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; + private static final String SIGNED_URL_VALID = + "http://localhost/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; private final ExceptionHandler createResultExceptionHandler = BaseService.EXCEPTION_HANDLER; @@ -35,7 +42,7 @@ public void startUploadForSignedUrl_expectStorageException_whenUrlInvalid() thro try { ResumableMedia.startUploadForSignedUrl( StorageOptions.newBuilder().build(), - new URL(SIGNED_URL), + new URL(SIGNED_URL_INVALID), createResultExceptionHandler) .get(); Assert.fail(); @@ -43,4 +50,17 @@ public void startUploadForSignedUrl_expectStorageException_whenUrlInvalid() thro assertNotNull(ex.getMessage()); } } + + @Test + public void startUploadForSignedUrl_whenUrlValid() throws Exception { + StorageRpc rpc = mock(StorageRpc.class); + StorageOptions options = StorageOptions.newBuilder().setServiceRpcFactory(opts -> rpc).build(); + + URL url = new URL(SIGNED_URL_VALID); + when(rpc.open(url.toString())).thenReturn("upload-id"); + + Supplier uploadIdSupplier = + ResumableMedia.startUploadForSignedUrl(options, url, createResultExceptionHandler); + assertThat(uploadIdSupplier.get()).isEqualTo("upload-id"); + } } From 3716105b5ea83378e5cacb4c4cd480b6dfe0fece Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Tue, 28 Sep 2021 12:46:36 -0400 Subject: [PATCH 07/15] test: retry conformance tests fixes, round 3 (#1050) * add applicability predicate for setIamPolicy mappings * add metageneration precondition mapping for setIamPolicy * add applicability predicate for hmacKey.update mappings * fix objects.delete mappings to use stateful blob * fix objects.delete to use generationMatch * add applicability predicate for objects.patch mappings --- .../conformance/retry/RpcMethodMappings.java | 141 ++++++------- .../retry/testNamesWhichShouldSucceed.txt | 185 +++++++++++++++--- 2 files changed, 232 insertions(+), 94 deletions(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java index af756b6ae..f2d188a1f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java @@ -612,6 +612,20 @@ private static void lockRetentionPolicy(ArrayList a) { private static void setIamPolicy(ArrayList a) { a.add( RpcMethodMapping.newBuilder(18, buckets.setIamPolicy) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) + .withTest( + (ctx, c) -> + ctx.map( + state -> + state.with( + ctx.getStorage() + .setIamPolicy( + state.getBucket().getName(), + Policy.newBuilder().build())))) + .build()); // TODO: configure policy + a.add( + RpcMethodMapping.newBuilder(240, buckets.setIamPolicy) + .withApplicable(TestRetryConformance::isPreconditionsProvided) .withTest( (ctx, c) -> ctx.map( @@ -619,7 +633,10 @@ private static void setIamPolicy(ArrayList a) { state.with( ctx.getStorage() .setIamPolicy( - c.getBucketName(), Policy.newBuilder().build())))) + state.getBucket().getName(), + Policy.newBuilder().build(), + BucketSourceOption.metagenerationMatch( + state.getBucket().getMetageneration()))))) .build()); // TODO: configure policy } @@ -829,6 +846,7 @@ private static void list(ArrayList a) { private static void update(ArrayList a) { a.add( RpcMethodMapping.newBuilder(29, hmacKey.update) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) .withTest( (ctx, c) -> ctx.map( @@ -1005,59 +1023,52 @@ private static void delete(ArrayList a) { RpcMethodMapping.newBuilder(36, objects.delete) .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) .withTest( - blobIdWithoutGeneration.andThen( - (ctx, c) -> - ctx.map( - state -> state.with(ctx.getStorage().delete(state.getBlobId()))))) + (ctx, c) -> + ctx.map( + state -> + state.with(ctx.getStorage().delete(state.getBlob().getBlobId())))) .build()); // TODO: Why does this exist, varargs should suffice a.add( RpcMethodMapping.newBuilder(37, objects.delete) .withApplicable(TestRetryConformance::isPreconditionsProvided) .withTest( - blobIdWithoutGeneration.andThen( - (ctx, c) -> - ctx.map( - state -> - state.with( - ctx.getStorage() - .delete( - state.getBlobId(), - BlobSourceOption.metagenerationMatch(1L)))))) + (ctx, c) -> + ctx.map( + state -> + state.with( + ctx.getStorage() + .delete( + state.getBlob().getBlobId(), + BlobSourceOption.generationMatch())))) .build()); // TODO: Correct arg? a.add( RpcMethodMapping.newBuilder(38, objects.delete) .withApplicable(TestRetryConformance::isPreconditionsProvided) .withTest( - blobIdWithoutGeneration.andThen( - (ctx, c) -> - ctx.map( - state -> - state.with( - ctx.getStorage() - .delete( - state.getBlobId().getBucket(), - state.getBlobId().getName(), - BlobSourceOption.metagenerationMatch(1L)))))) + (ctx, c) -> + ctx.map( + state -> + state.with( + ctx.getStorage() + .delete( + state.getBlob().getBlobId().getBucket(), + state.getBlob().getBlobId().getName(), + BlobSourceOption.generationMatch( + state.getBlob().getGeneration()))))) .build()); // TODO: Correct arg? a.add( RpcMethodMapping.newBuilder(67, objects.delete) - .withTest( - blobIdWithoutGeneration - .andThen(Rpc.blobWithGeneration) - .andThen((ctx, c) -> ctx.peek(state -> state.getBlob().delete()))) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) + .withTest((ctx, c) -> ctx.peek(state -> state.getBlob().delete())) .build()); a.add( RpcMethodMapping.newBuilder(68, objects.delete) + .withApplicable(TestRetryConformance::isPreconditionsProvided) .withTest( - blobIdWithoutGeneration - .andThen(Rpc.blobWithGeneration) - .andThen( - (ctx, c) -> - ctx.peek( - state -> - state - .getBlob() - .delete(Blob.BlobSourceOption.metagenerationMatch())))) + (ctx, c) -> + ctx.peek( + state -> + state.getBlob().delete(Blob.BlobSourceOption.generationMatch()))) .build()); } @@ -1770,51 +1781,41 @@ private static void list(ArrayList a) { private static void patch(ArrayList a) { a.add( RpcMethodMapping.newBuilder(56, objects.patch) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) .withTest( - blobIdWithoutGeneration - .andThen(Rpc.blobWithGeneration) - .andThen( - (ctx, c) -> - ctx.map( - state -> - state.with( - ctx.getStorage().update(ctx.getState().getBlob()))))) + (ctx, c) -> + ctx.map( + state -> state.with(ctx.getStorage().update(ctx.getState().getBlob())))) .build()); // TODO: Why does this exist, varargs should suffice a.add( RpcMethodMapping.newBuilder(57, objects.patch) + .withApplicable(TestRetryConformance::isPreconditionsProvided) .withTest( - blobIdWithoutGeneration - .andThen(Rpc.blobWithGeneration) - .andThen( - (ctx, c) -> - ctx.map( - state -> - state.with( - ctx.getStorage() - .update( - ctx.getState().getBlob(), - BlobTargetOption.metagenerationMatch()))))) + (ctx, c) -> + ctx.map( + state -> + state.with( + ctx.getStorage() + .update( + ctx.getState().getBlob(), + BlobTargetOption.metagenerationMatch())))) .build()); a.add( RpcMethodMapping.newBuilder(79, objects.patch) - .withTest( - blobIdWithoutGeneration - .andThen(Rpc.blobWithGeneration) - .andThen((ctx, c) -> ctx.peek(state -> state.getBlob().update()))) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) + .withTest((ctx, c) -> ctx.peek(state -> state.getBlob().update())) .build()); a.add( RpcMethodMapping.newBuilder(80, objects.patch) + .withApplicable(TestRetryConformance::isPreconditionsProvided) .withTest( - blobIdWithoutGeneration - .andThen(Rpc.blobWithGeneration) - .andThen( - (ctx, c) -> - ctx.map( - state -> - state.with( - state - .getBlob() - .update(BlobTargetOption.generationMatch()))))) + (ctx, c) -> + ctx.map( + state -> + state.with( + state + .getBlob() + .update(BlobTargetOption.metagenerationMatch())))) .build()); // TODO: Correct arg? } diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt index 2b3f8f2bc..2dcb832dd 100644 --- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt +++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt @@ -2,7 +2,7 @@ # Each test name present in the file will be expected to pass in a CI environment # # This list can be regenerated by running the following command in a shell: -# mvn test && xq '//testsuite/testcase[not(./error) and not(./failure) and not(./skipped)]/@name' google-cloud-storage/target/failsafe-reports/TEST-com.google.cloud.storage.conformance.retry.ITRetryConformanceTest-sponge_log.xml | tail -n+2 | head -n-1 | sed 's# test\[##g' | sed 's#\]##g' | sort | tee -a google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt +# mvn -Penable-integration-test clean verify && xq '//testsuite/testcase[not(./error) and not(./failure) and not(./skipped)]/@name' google-cloud-storage/target/failsafe-reports/TEST-com.google.cloud.storage.conformance.retry.ITRetryConformanceTest-sponge_log.xml | tail -n+2 | head -n-1 | sed 's# test\[##g' | sed 's#\]##g' | sort | tee -a google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt # where xq is the package from https://github.com/jeffbr13/xq TestRetryConformance/1-[return-503_return-503]-storage.bucket_acl.get-3 @@ -14,15 +14,16 @@ TestRetryConformance/1-[return-503_return-503]-storage.bucket_acl.list-90 TestRetryConformance/1-[return-503_return-503]-storage.buckets.delete-11 TestRetryConformance/1-[return-503_return-503]-storage.buckets.delete-92 TestRetryConformance/1-[return-503_return-503]-storage.buckets.get-12 +TestRetryConformance/1-[return-503_return-503]-storage.buckets.get-94 TestRetryConformance/1-[return-503_return-503]-storage.buckets.get-96 TestRetryConformance/1-[return-503_return-503]-storage.buckets.getIamPolicy-13 TestRetryConformance/1-[return-503_return-503]-storage.buckets.insert-14 -TestRetryConformance/1-[return-503_return-503]-storage.buckets.lockRetentionPolicy-100 TestRetryConformance/1-[return-503_return-503]-storage.buckets.testIamPermissions-19 TestRetryConformance/1-[return-503_return-503]-storage.default_object_acl.get-103 TestRetryConformance/1-[return-503_return-503]-storage.default_object_acl.get-21 TestRetryConformance/1-[return-503_return-503]-storage.default_object_acl.list-105 TestRetryConformance/1-[return-503_return-503]-storage.default_object_acl.list-23 +TestRetryConformance/1-[return-503_return-503]-storage.hmacKey.delete-26 TestRetryConformance/1-[return-503_return-503]-storage.hmacKey.get-27 TestRetryConformance/1-[return-503_return-503]-storage.hmacKey.list-28 TestRetryConformance/1-[return-503_return-503]-storage.object_acl.get-31 @@ -38,6 +39,7 @@ TestRetryConformance/1-[return-503_return-503]-storage.objects.get-42 TestRetryConformance/1-[return-503_return-503]-storage.objects.get-43 TestRetryConformance/1-[return-503_return-503]-storage.objects.get-44 TestRetryConformance/1-[return-503_return-503]-storage.objects.get-45 +TestRetryConformance/1-[return-503_return-503]-storage.objects.get-60 TestRetryConformance/1-[return-503_return-503]-storage.objects.get-69 TestRetryConformance/1-[return-503_return-503]-storage.objects.get-70 TestRetryConformance/1-[return-503_return-503]-storage.objects.get-71 @@ -57,15 +59,16 @@ TestRetryConformance/1-[return-reset-connection_return-503]-storage.bucket_acl.l TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.delete-11 TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.delete-92 TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.get-12 +TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.get-94 TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.get-96 TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.getIamPolicy-13 TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.insert-14 -TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.lockRetentionPolicy-100 TestRetryConformance/1-[return-reset-connection_return-503]-storage.buckets.testIamPermissions-19 TestRetryConformance/1-[return-reset-connection_return-503]-storage.default_object_acl.get-103 TestRetryConformance/1-[return-reset-connection_return-503]-storage.default_object_acl.get-21 TestRetryConformance/1-[return-reset-connection_return-503]-storage.default_object_acl.list-105 TestRetryConformance/1-[return-reset-connection_return-503]-storage.default_object_acl.list-23 +TestRetryConformance/1-[return-reset-connection_return-503]-storage.hmacKey.delete-26 TestRetryConformance/1-[return-reset-connection_return-503]-storage.hmacKey.get-27 TestRetryConformance/1-[return-reset-connection_return-503]-storage.hmacKey.list-28 TestRetryConformance/1-[return-reset-connection_return-503]-storage.object_acl.get-31 @@ -81,6 +84,7 @@ TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get- TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-43 TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-44 TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-45 +TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-60 TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-69 TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-70 TestRetryConformance/1-[return-reset-connection_return-503]-storage.objects.get-71 @@ -100,15 +104,16 @@ TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.delete-11 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.delete-92 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.get-12 +TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.get-94 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.get-96 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.getIamPolicy-13 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.insert-14 -TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.lockRetentionPolicy-100 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.buckets.testIamPermissions-19 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.default_object_acl.get-103 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.default_object_acl.get-21 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.default_object_acl.list-105 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.default_object_acl.list-23 +TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.hmacKey.delete-26 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.hmacKey.get-27 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.hmacKey.list-28 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.object_acl.get-31 @@ -124,6 +129,7 @@ TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-43 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-44 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-45 +TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-60 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-69 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-70 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.get-71 @@ -135,40 +141,31 @@ TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.objects.list-55 TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.serviceaccount.get-59 TestRetryConformance/2-[return-503_return-503]-storage.buckets.patch-101 -TestRetryConformance/2-[return-503_return-503]-storage.hmacKey.update-29 +TestRetryConformance/2-[return-503_return-503]-storage.buckets.patch-122 TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-37 TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-38 -TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-67 TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-68 TestRetryConformance/2-[return-503_return-503]-storage.objects.insert-46 TestRetryConformance/2-[return-503_return-503]-storage.objects.insert-47 -TestRetryConformance/2-[return-503_return-503]-storage.objects.patch-56 TestRetryConformance/2-[return-503_return-503]-storage.objects.patch-57 -TestRetryConformance/2-[return-503_return-503]-storage.objects.patch-79 TestRetryConformance/2-[return-503_return-503]-storage.objects.patch-80 TestRetryConformance/2-[return-reset-connection_return-503]-storage.buckets.patch-101 -TestRetryConformance/2-[return-reset-connection_return-503]-storage.hmacKey.update-29 +TestRetryConformance/2-[return-reset-connection_return-503]-storage.buckets.patch-122 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-37 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-38 -TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-67 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-68 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.insert-46 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.insert-47 -TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.patch-56 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.patch-57 -TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.patch-79 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.patch-80 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.buckets.patch-101 -TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.hmacKey.update-29 +TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.buckets.patch-122 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-37 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-38 -TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-67 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-68 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.insert-46 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.insert-47 -TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-56 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-57 -TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-79 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-80 TestRetryConformance/3-[return-503]-storage.objects.insert-110 TestRetryConformance/3-[return-503]-storage.objects.insert-111 @@ -198,7 +195,6 @@ TestRetryConformance/5-[return-400]-storage.buckets.get-94 TestRetryConformance/5-[return-400]-storage.buckets.get-96 TestRetryConformance/5-[return-400]-storage.buckets.getIamPolicy-13 TestRetryConformance/5-[return-400]-storage.buckets.insert-14 -TestRetryConformance/5-[return-400]-storage.buckets.lockRetentionPolicy-100 TestRetryConformance/5-[return-400]-storage.buckets.patch-101 TestRetryConformance/5-[return-400]-storage.buckets.patch-17 TestRetryConformance/5-[return-400]-storage.buckets.setIamPolicy-18 @@ -213,6 +209,7 @@ TestRetryConformance/5-[return-400]-storage.default_object_acl.list-105 TestRetryConformance/5-[return-400]-storage.default_object_acl.list-23 TestRetryConformance/5-[return-400]-storage.default_object_acl.patch-106 TestRetryConformance/5-[return-400]-storage.default_object_acl.patch-24 +TestRetryConformance/5-[return-400]-storage.hmacKey.create-25 TestRetryConformance/5-[return-400]-storage.hmacKey.delete-26 TestRetryConformance/5-[return-400]-storage.hmacKey.get-27 TestRetryConformance/5-[return-400]-storage.hmacKey.list-28 @@ -230,7 +227,6 @@ TestRetryConformance/5-[return-400]-storage.object_acl.patch-66 TestRetryConformance/5-[return-400]-storage.objects.compose-35 TestRetryConformance/5-[return-400]-storage.objects.delete-36 TestRetryConformance/5-[return-400]-storage.objects.delete-67 -TestRetryConformance/5-[return-400]-storage.objects.delete-68 TestRetryConformance/5-[return-400]-storage.objects.get-107 TestRetryConformance/5-[return-400]-storage.objects.get-239 TestRetryConformance/5-[return-400]-storage.objects.get-39 @@ -238,6 +234,8 @@ TestRetryConformance/5-[return-400]-storage.objects.get-40 TestRetryConformance/5-[return-400]-storage.objects.get-41 TestRetryConformance/5-[return-400]-storage.objects.get-42 TestRetryConformance/5-[return-400]-storage.objects.get-43 +TestRetryConformance/5-[return-400]-storage.objects.get-44 +TestRetryConformance/5-[return-400]-storage.objects.get-45 TestRetryConformance/5-[return-400]-storage.objects.get-60 TestRetryConformance/5-[return-400]-storage.objects.get-69 TestRetryConformance/5-[return-400]-storage.objects.get-70 @@ -259,9 +257,7 @@ TestRetryConformance/5-[return-400]-storage.objects.insert-116 TestRetryConformance/5-[return-400]-storage.objects.insert-117 TestRetryConformance/5-[return-400]-storage.objects.list-55 TestRetryConformance/5-[return-400]-storage.objects.patch-56 -TestRetryConformance/5-[return-400]-storage.objects.patch-57 TestRetryConformance/5-[return-400]-storage.objects.patch-79 -TestRetryConformance/5-[return-400]-storage.objects.patch-80 TestRetryConformance/5-[return-400]-storage.objects.rewrite-58 TestRetryConformance/5-[return-400]-storage.objects.rewrite-81 TestRetryConformance/5-[return-400]-storage.objects.rewrite-82 @@ -292,7 +288,6 @@ TestRetryConformance/5-[return-401]-storage.buckets.get-94 TestRetryConformance/5-[return-401]-storage.buckets.get-96 TestRetryConformance/5-[return-401]-storage.buckets.getIamPolicy-13 TestRetryConformance/5-[return-401]-storage.buckets.insert-14 -TestRetryConformance/5-[return-401]-storage.buckets.lockRetentionPolicy-100 TestRetryConformance/5-[return-401]-storage.buckets.patch-101 TestRetryConformance/5-[return-401]-storage.buckets.patch-17 TestRetryConformance/5-[return-401]-storage.buckets.setIamPolicy-18 @@ -307,6 +302,7 @@ TestRetryConformance/5-[return-401]-storage.default_object_acl.list-105 TestRetryConformance/5-[return-401]-storage.default_object_acl.list-23 TestRetryConformance/5-[return-401]-storage.default_object_acl.patch-106 TestRetryConformance/5-[return-401]-storage.default_object_acl.patch-24 +TestRetryConformance/5-[return-401]-storage.hmacKey.create-25 TestRetryConformance/5-[return-401]-storage.hmacKey.delete-26 TestRetryConformance/5-[return-401]-storage.hmacKey.get-27 TestRetryConformance/5-[return-401]-storage.hmacKey.list-28 @@ -324,7 +320,6 @@ TestRetryConformance/5-[return-401]-storage.object_acl.patch-66 TestRetryConformance/5-[return-401]-storage.objects.compose-35 TestRetryConformance/5-[return-401]-storage.objects.delete-36 TestRetryConformance/5-[return-401]-storage.objects.delete-67 -TestRetryConformance/5-[return-401]-storage.objects.delete-68 TestRetryConformance/5-[return-401]-storage.objects.get-107 TestRetryConformance/5-[return-401]-storage.objects.get-239 TestRetryConformance/5-[return-401]-storage.objects.get-39 @@ -332,6 +327,8 @@ TestRetryConformance/5-[return-401]-storage.objects.get-40 TestRetryConformance/5-[return-401]-storage.objects.get-41 TestRetryConformance/5-[return-401]-storage.objects.get-42 TestRetryConformance/5-[return-401]-storage.objects.get-43 +TestRetryConformance/5-[return-401]-storage.objects.get-44 +TestRetryConformance/5-[return-401]-storage.objects.get-45 TestRetryConformance/5-[return-401]-storage.objects.get-60 TestRetryConformance/5-[return-401]-storage.objects.get-69 TestRetryConformance/5-[return-401]-storage.objects.get-70 @@ -353,9 +350,7 @@ TestRetryConformance/5-[return-401]-storage.objects.insert-116 TestRetryConformance/5-[return-401]-storage.objects.insert-117 TestRetryConformance/5-[return-401]-storage.objects.list-55 TestRetryConformance/5-[return-401]-storage.objects.patch-56 -TestRetryConformance/5-[return-401]-storage.objects.patch-57 TestRetryConformance/5-[return-401]-storage.objects.patch-79 -TestRetryConformance/5-[return-401]-storage.objects.patch-80 TestRetryConformance/5-[return-401]-storage.objects.rewrite-58 TestRetryConformance/5-[return-401]-storage.objects.rewrite-81 TestRetryConformance/5-[return-401]-storage.objects.rewrite-82 @@ -364,3 +359,145 @@ TestRetryConformance/5-[return-401]-storage.objects.rewrite-84 TestRetryConformance/5-[return-401]-storage.objects.rewrite-85 TestRetryConformance/5-[return-401]-storage.objects.rewrite-86 TestRetryConformance/5-[return-401]-storage.serviceaccount.get-59 +TestRetryConformance/6-[return-503_return-400]-storage.bucket_acl.get-3 +TestRetryConformance/6-[return-503_return-400]-storage.bucket_acl.get-4 +TestRetryConformance/6-[return-503_return-400]-storage.bucket_acl.get-88 +TestRetryConformance/6-[return-503_return-400]-storage.bucket_acl.list-7 +TestRetryConformance/6-[return-503_return-400]-storage.bucket_acl.list-8 +TestRetryConformance/6-[return-503_return-400]-storage.bucket_acl.list-90 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.delete-11 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.delete-92 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.delete-93 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.get-12 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.get-95 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.get-97 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.getIamPolicy-13 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.lockRetentionPolicy-100 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.patch-101 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.patch-122 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.setIamPolicy-240 +TestRetryConformance/6-[return-503_return-400]-storage.buckets.testIamPermissions-19 +TestRetryConformance/6-[return-503_return-400]-storage.default_object_acl.get-103 +TestRetryConformance/6-[return-503_return-400]-storage.default_object_acl.get-21 +TestRetryConformance/6-[return-503_return-400]-storage.default_object_acl.list-105 +TestRetryConformance/6-[return-503_return-400]-storage.default_object_acl.list-23 +TestRetryConformance/6-[return-503_return-400]-storage.hmacKey.delete-26 +TestRetryConformance/6-[return-503_return-400]-storage.hmacKey.get-27 +TestRetryConformance/6-[return-503_return-400]-storage.hmacKey.list-28 +TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-31 +TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-63 +TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-33 +TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-65 +TestRetryConformance/6-[return-503_return-400]-storage.objects.compose-35 +TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-37 +TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-38 +TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-68 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-39 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-40 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-41 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-42 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-43 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-44 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-45 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-61 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-69 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-70 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-71 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-72 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-73 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-74 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-75 +TestRetryConformance/6-[return-503_return-400]-storage.objects.get-76 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-118 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-119 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-46 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-47 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-49 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-50 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-51 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-52 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-53 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-77 +TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-78 +TestRetryConformance/6-[return-503_return-400]-storage.objects.list-55 +TestRetryConformance/6-[return-503_return-400]-storage.objects.patch-57 +TestRetryConformance/6-[return-503_return-400]-storage.objects.patch-80 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-58 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-81 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-82 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-83 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-84 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-85 +TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-86 +TestRetryConformance/6-[return-503_return-400]-storage.serviceaccount.get-59 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-3 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-4 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-88 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.list-7 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.list-8 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.list-90 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.delete-11 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.delete-92 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.delete-93 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.get-12 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.get-95 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.get-97 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.getIamPolicy-13 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.lockRetentionPolicy-100 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.patch-101 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.patch-122 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.setIamPolicy-240 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.buckets.testIamPermissions-19 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.default_object_acl.get-103 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.default_object_acl.get-21 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.default_object_acl.list-105 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.default_object_acl.list-23 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.hmacKey.delete-26 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.hmacKey.get-27 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.hmacKey.list-28 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.get-31 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.get-63 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-33 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-65 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.compose-35 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-37 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-38 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-68 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-39 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-40 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-41 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-42 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-43 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-44 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-45 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-61 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-69 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-70 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-71 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-72 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-73 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-74 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-75 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.get-76 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-118 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-119 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-46 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-47 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-49 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-50 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-51 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-52 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-53 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-77 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-78 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.list-55 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.patch-57 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.patch-80 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-58 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-81 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-82 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-83 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-84 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-85 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-86 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.serviceaccount.get-59 From 7a1a9288df5fd3b35dbc03e73b0ba39274c507e1 Mon Sep 17 00:00:00 2001 From: Neenu Shaji Date: Wed, 29 Sep 2021 10:30:16 -0400 Subject: [PATCH 08/15] chore: change branch master to main in github configurations (#1059) --- .github/blunderbuss.yml | 2 +- .github/sync-repo-settings.yaml | 2 +- .github/workflows/ci.yaml | 2 +- README.md | 26 +++++++++++++------------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml index 1a23ea42b..2176b0543 100644 --- a/.github/blunderbuss.yml +++ b/.github/blunderbuss.yml @@ -1,5 +1,5 @@ # Configuration for the Blunderbuss GitHub app. For more info see -# https://github.com/googleapis/repo-automation-bots/tree/master/packages/blunderbuss +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/blunderbuss assign_prs_by: - labels: - samples diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 583b39c63..85d52c141 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -2,7 +2,7 @@ rebaseMergeAllowed: false squashMergeAllowed: true mergeCommitAllowed: false branchProtectionRules: - - pattern: master + - pattern: main isAdminEnforced: true requiredApprovingReviewCount: 1 requiresCodeOwnerReviews: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3becb5c02..3932a70d4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: ci jobs: diff --git a/README.md b/README.md index d3ac39937..024add90e 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Blob blob = storage.create(blobInfo, "a simple blob".getBytes(UTF_8)); ``` A complete example for creating a blob can be found at -[CreateBlob.java](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/CreateBlob.java). +[CreateBlob.java](https://github.com/googleapis/google-cloud-java/tree/main/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/CreateBlob.java). At this point, you will be able to see your newly created bucket and blob on the Google Developers Console. @@ -164,7 +164,7 @@ String contentString = new String(content, UTF_8); ``` A complete example for accessing blobs can be found at -[CreateBlob.java](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/CreateBlob.java). +[CreateBlob.java](https://github.com/googleapis/google-cloud-java/tree/main/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/CreateBlob.java). #### Updating data Another thing we may want to do is update a blob. The following snippet shows how to update a Storage blob if it exists. @@ -182,7 +182,7 @@ if (blob != null) { ``` The complete source code can be found at -[UpdateBlob.java](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/UpdateBlob.java). +[UpdateBlob.java](https://github.com/googleapis/google-cloud-java/tree/main/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/UpdateBlob.java). #### Listing buckets and contents of buckets Suppose that you've added more buckets and blobs, and now you want to see the names of your buckets and the contents @@ -205,7 +205,7 @@ for (Bucket bucket : storage.list().iterateAll()) { #### Complete source code In -[CreateAndListBucketsAndBlobs.java](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/CreateAndListBucketsAndBlobs.java) +[CreateAndListBucketsAndBlobs.java](https://github.com/googleapis/google-cloud-java/tree/main/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/CreateAndListBucketsAndBlobs.java) we put together examples creating and listing buckets and blobs into one program. The program assumes that you are running on Compute Engine or from your own desktop. To run the example on App Engine, simply move the code from the main method to your application's servlet class and change the print statements to @@ -213,21 +213,21 @@ display on your webpage. ### Example Applications -- [`StorageExample`](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/StorageExample.java) is a simple command line interface that provides some of Cloud Storage's functionality. Read more about using the application on the [`StorageExample` docs page](https://github.com/googleapis/google-cloud-java/blob/master/google-cloud-examples/README.md). -- [`Bookshelf`](https://github.com/GoogleCloudPlatform/getting-started-java/tree/master/bookshelf) - An App Engine application that manages a virtual bookshelf. +- [`StorageExample`](https://github.com/googleapis/google-cloud-java/tree/main/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/StorageExample.java) is a simple command line interface that provides some of Cloud Storage's functionality. Read more about using the application on the [`StorageExample` docs page](https://github.com/googleapis/google-cloud-java/blob/main/google-cloud-examples/README.md). +- [`Bookshelf`](https://github.com/GoogleCloudPlatform/getting-started-java/tree/main/bookshelf) - An App Engine application that manages a virtual bookshelf. - This app uses `google-cloud` to interface with Cloud Datastore and Cloud Storage. It also uses Cloud SQL, another Google Cloud Platform service. -- [`Flexible Environment/Storage example`](https://github.com/GoogleCloudPlatform/java-docs-samples/tree/master/flexible/cloudstorage) - An app that uploads files to a public Cloud Storage bucket on the App Engine Flexible Environment runtime. +- [`Flexible Environment/Storage example`](https://github.com/GoogleCloudPlatform/java-docs-samples/tree/main/flexible/cloudstorage) - An app that uploads files to a public Cloud Storage bucket on the App Engine Flexible Environment runtime. ## Samples -Samples are in the [`samples/`](https://github.com/googleapis/java-storage/tree/master/samples) directory. +Samples are in the [`samples/`](https://github.com/googleapis/java-storage/tree/main/samples) directory. | Sample | Source Code | Try it | | --------------------------- | --------------------------------- | ------ | -| Quickstart Sample | [source code](https://github.com/googleapis/java-storage/blob/master/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/storage/QuickstartSample.java) | +| Quickstart Sample | [source code](https://github.com/googleapis/java-storage/blob/main/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/storage/QuickstartSample.java) | @@ -336,10 +336,10 @@ Java is a registered trademark of Oracle and/or its affiliates. [developer-console]: https://console.developers.google.com/ [create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects [cloud-sdk]: https://cloud.google.com/sdk/ -[troubleshooting]: https://github.com/googleapis/google-cloud-common/blob/master/troubleshooting/readme.md#troubleshooting -[contributing]: https://github.com/googleapis/java-storage/blob/master/CONTRIBUTING.md -[code-of-conduct]: https://github.com/googleapis/java-storage/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct -[license]: https://github.com/googleapis/java-storage/blob/master/LICENSE +[troubleshooting]: https://github.com/googleapis/google-cloud-common/blob/main/troubleshooting/readme.md#troubleshooting +[contributing]: https://github.com/googleapis/java-storage/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/googleapis/java-storage/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/googleapis/java-storage/blob/main/LICENSE [enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing [enable-api]: https://console.cloud.google.com/flows/enableapi?apiid=storage.googleapis.com [libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM From 575cf1286b4683c65bd15dace06eefed427b424a Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Wed, 29 Sep 2021 11:19:55 -0400 Subject: [PATCH 09/15] chore: refactor PostPolicyV4Test to be compatible with behavior change of Collections.unmodifiableMap from jdk 17 (#1016) --- .../com/google/cloud/storage/PostPolicyV4Test.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/PostPolicyV4Test.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/PostPolicyV4Test.java index 8213f6093..eed13cd3e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/PostPolicyV4Test.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/PostPolicyV4Test.java @@ -17,7 +17,6 @@ package com.google.cloud.storage; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -33,8 +32,7 @@ public class PostPolicyV4Test { private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - private void assertNotSameButEqual(Map expected, Map returned) { - assertNotSame(expected, returned); + private void assertMapsEquals(Map expected, Map returned) { assertEquals("map sizes", expected.size(), returned.size()); for (String key : expected.keySet()) { assertEquals("value of $" + key, expected.get(key), returned.get(key)); @@ -78,7 +76,7 @@ public void testPostPolicyV4_of() { String url = "http://example.com"; PostPolicyV4 policy = PostPolicyV4.of(url, ALL_FIELDS); assertEquals(url, policy.getUrl()); - assertNotSameButEqual(ALL_FIELDS, policy.getFields()); + assertMapsEquals(ALL_FIELDS, policy.getFields()); } @Test @@ -115,7 +113,7 @@ public void testPostPolicyV4_ofInvalidField() { @Test public void testPostFieldsV4_of() { PostPolicyV4.PostFieldsV4 fields = PostPolicyV4.PostFieldsV4.of(ALL_FIELDS); - assertNotSameButEqual(ALL_FIELDS, fields.getFieldsMap()); + assertMapsEquals(ALL_FIELDS, fields.getFieldsMap()); } @Test @@ -157,7 +155,7 @@ public void testPostPolicyV4_builder() { expectedUpdated.put("content-type", "new-content-type"); expectedUpdated.put("success_action_status", "42"); Map updated = builder.build().getFieldsMap(); - assertNotSameButEqual(expectedUpdated, updated); + assertMapsEquals(expectedUpdated, updated); } @Test From 8717cd69fdb0096737f148ceffaf16a50a1e4f4e Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 29 Sep 2021 08:27:30 -0700 Subject: [PATCH 10/15] build: add jdk 17 to java units and dependency builds (#1048) * chore(java): rename master branch to main Source-Author: Neenu Shaji Source-Date: Mon Sep 27 10:04:11 2021 -0400 Source-Repo: googleapis/synthtool Source-Sha: 67ab4f9f4272ad13f4b809de47fd0dec05f425ad Source-Link: https://github.com/googleapis/synthtool/commit/67ab4f9f4272ad13f4b809de47fd0dec05f425ad * build: add jdk 17 to java units and dependency builds * update dependencies.sh to not pass MaxPermSize when jdk 17 is used. MaxPermSize is an unrecognized flag in jdk 17. Source-Author: BenWhitehead Source-Date: Mon Sep 27 11:34:46 2021 -0400 Source-Repo: googleapis/synthtool Source-Sha: a4be3384ccb92364795d981f2863f6986fcee620 Source-Link: https://github.com/googleapis/synthtool/commit/a4be3384ccb92364795d981f2863f6986fcee620 --- .github/workflows/ci.yaml | 19 ++++++++++++------- .kokoro/dependencies.sh | 23 ++++++++++++++++++++++- synth.metadata | 4 ++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3932a70d4..2425d7234 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,14 +9,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [8, 11] + java: [8, 11, 17] steps: - uses: actions/checkout@v2 - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: ${{matrix.java}} - run: java -version - run: .kokoro/build.sh @@ -29,8 +30,9 @@ jobs: - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: 8 - run: java -version - run: .kokoro/build.bat @@ -40,14 +42,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [8, 11] + java: [8, 11, 17] steps: - uses: actions/checkout@v2 - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: ${{matrix.java}} - run: java -version - run: .kokoro/dependencies.sh @@ -58,8 +61,9 @@ jobs: - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: 8 - run: java -version - run: .kokoro/build.sh @@ -72,8 +76,9 @@ jobs: - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: 8 - run: java -version - run: .kokoro/build.sh diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index 9030ba8f9..9a5105d7e 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -28,7 +28,28 @@ source ${scriptDir}/common.sh java -version echo $JOB_TYPE -export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=128m" +function determineMavenOpts() { + local javaVersion=$( + # filter down to the version line, then pull out the version between quotes, + # then trim the version number down to its minimal number (removing any + # update or suffix number). + java -version 2>&1 | grep "version" \ + | sed -E 's/^.*"(.*?)".*$/\1/g' \ + | sed -E 's/^(1\.[0-9]\.0).*$/\1/g' + ) + + case $javaVersion in + "17") + # MaxPermSize is no longer supported as of jdk 17 + echo -n "-Xmx1024m" + ;; + *) + echo -n "-Xmx1024m -XX:MaxPermSize=128m" + ;; + esac +} + +export MAVEN_OPTS=$(determineMavenOpts) # this should run maven enforcer retry_with_backoff 3 10 \ diff --git a/synth.metadata b/synth.metadata index 56fd80e98..40a98002e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "d6d95456f1e300ca4a9dc4b1c47940dea7b3a82a" + "sha": "2e7f041ef7873ede8c2d1ebb526639d1aff61963" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "0752ff727a19a467dffed335d5e59303689cf0d1" + "sha": "a4be3384ccb92364795d981f2863f6986fcee620" } } ], From 35cb2f76c28596b86de06f2f43d388bcc5f830ce Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Fri, 1 Oct 2021 14:21:48 -0400 Subject: [PATCH 11/15] chore(retry): implement new retry exception handlers (#1058) * add idempotent handler * add idempotent resumable write handler * add non-idempotent handler test count summary shift after this change failure -> success: 55 error -> success: 24 success -> failure: 26 --- .../com/google/cloud/storage/CopyWriter.java | 3 +- .../storage/LegacyRetryAlgorithmManager.java | 7 +- .../storage/NewRetryAlgorithmManager.java | 340 +++++++++++++++++- .../cloud/storage/RetryAlgorithmManager.java | 4 +- .../google/cloud/storage/StorageOptions.java | 2 +- .../retry/testNamesWhichShouldSucceed.txt | 105 ++++-- 6 files changed, 422 insertions(+), 39 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java index f3dcdfdee..d621d1492 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java @@ -97,10 +97,11 @@ public long getTotalBytesCopied() { */ public void copyChunk() { if (!isDone()) { + RewriteRequest rewriteRequest = rewriteResponse.rewriteRequest; this.rewriteResponse = Retrying.run( serviceOptions, - serviceOptions.getRetryAlgorithmManager().getForObjectsCopy(), + serviceOptions.getRetryAlgorithmManager().getForObjectsRewrite(rewriteRequest), () -> storageRpc.continueRewrite(rewriteResponse), Function.identity()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java index fd88c3333..f6ba9d381 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java @@ -219,14 +219,9 @@ public ExceptionHandler getForObjectsRewrite(RewriteRequest pb) { return BaseService.EXCEPTION_HANDLER; } - @Override - public ExceptionHandler getForObjectsCopy() { - return BaseService.EXCEPTION_HANDLER; - } - @Override public ExceptionHandler getForObjectsCompose( - List sources, StorageObject target, Map targetOptions) { + List sources, StorageObject target, Map optionsMap) { return BaseService.EXCEPTION_HANDLER; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java index 0c3e1f8e7..8f84990f2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java @@ -16,5 +16,341 @@ package com.google.cloud.storage; -final class NewRetryAlgorithmManager extends LegacyRetryAlgorithmManager - implements RetryAlgorithmManager {} +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.HmacKeyMetadata; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.api.services.storage.model.Policy; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.BaseServiceException; +import com.google.cloud.ExceptionHandler; +import com.google.cloud.ExceptionHandler.Interceptor; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +final class NewRetryAlgorithmManager implements RetryAlgorithmManager { + + private static final Interceptor INTERCEPTOR_IDEMPOTENT = + new InterceptorImpl( + true, + ImmutableSet.builder() + .add(408) + .add(429) + .add(500) + .add(502) + .add(503) + .add(504) + .build()); + private static final Interceptor INTERCEPTOR_IDEMPOTENT_RESUMABLE = + new InterceptorImpl( + true, + ImmutableSet.builder().add(408).add(500).add(502).add(503).add(504).build()); + private static final Interceptor INTERCEPTOR_NON_IDEMPOTENT = + new InterceptorImpl(false, ImmutableSet.builder().build()); + + private static final ExceptionHandler IDEMPOTENT_HANDLER = + ExceptionHandler.newBuilder() + .retryOn(RuntimeException.class) + .addInterceptors(INTERCEPTOR_IDEMPOTENT) + .build(); + + private static final ExceptionHandler IDEMPOTENT_RESUMABLE_UPLOAD_HANDLER = + ExceptionHandler.newBuilder() + .retryOn(RuntimeException.class) + .addInterceptors(INTERCEPTOR_IDEMPOTENT_RESUMABLE) + .build(); + + private static final ExceptionHandler NON_IDEMPOTENT_HANDLER = + ExceptionHandler.newBuilder() + .retryOn(RuntimeException.class) + .addInterceptors(INTERCEPTOR_NON_IDEMPOTENT) + .build(); + + @Override + public ExceptionHandler getForBucketAclCreate( + BucketAccessControl pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclDelete(String pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclGet(String pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclUpdate( + BucketAccessControl pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclList(String pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsCreate(Bucket pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsDelete(Bucket pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsGet(Bucket pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsUpdate(Bucket pb, Map optionsMap) { + // TODO: Include etag when it is supported by the library + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsList(Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsLockRetentionPolicy( + Bucket pb, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsGetIamPolicy( + String bucket, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsSetIamPolicy( + String bucket, Policy pb, Map optionsMap) { + // TODO: Include etag when it is supported by the library + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsTestIamPermissions( + String bucket, List permissions, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclCreate(ObjectAccessControl pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclDelete(String pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclGet(String pb) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclUpdate(ObjectAccessControl pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclList(String pb) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyCreate(String pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyDelete( + HmacKeyMetadata pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyGet(String accessId, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyUpdate( + HmacKeyMetadata pb, Map optionsMap) { + // TODO: Include etag when it is supported by the library + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyList(Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclCreate(ObjectAccessControl aclPb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclDelete( + String bucket, String name, Long generation, String pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclList(String bucket, String name, Long generation) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclGet( + String bucket, String name, Long generation, String pb) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclUpdate(ObjectAccessControl aclPb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCreate( + StorageObject pb, Map optionsMap) { + if (pb.getGeneration() != null && pb.getGeneration() == 0) { + return IDEMPOTENT_HANDLER; + } + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsDelete( + StorageObject pb, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsGet(StorageObject pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsUpdate( + StorageObject pb, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsList(String bucket, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsRewrite(RewriteRequest pb) { + return (pb.sourceOptions.containsKey(StorageRpc.Option.IF_SOURCE_GENERATION_MATCH) + || pb.sourceOptions.containsKey(StorageRpc.Option.IF_GENERATION_MATCH)) + && pb.targetOptions.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCompose( + List sources, StorageObject target, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForResumableUploadSessionCreate(Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForResumableUploadSessionWrite(Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_RESUMABLE_UPLOAD_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForServiceAccountGet(String pb) { + return IDEMPOTENT_HANDLER; + } + + private static class InterceptorImpl implements Interceptor { + + private final boolean idempotent; + private final ImmutableSet retryableCodes; + + private InterceptorImpl(boolean idempotent, ImmutableSet retryableCodes) { + this.idempotent = idempotent; + this.retryableCodes = retryableCodes; + } + + @Override + public RetryResult afterEval(Exception exception, RetryResult retryResult) { + return RetryResult.CONTINUE_EVALUATION; + } + + @Override + public RetryResult beforeEval(Exception exception) { + + // first check if an IO exception has been wrapped by a StorageException, fallback to + // a general BaseServiceException to check status code + if (exception instanceof StorageException) { + StorageException storageException = (StorageException) exception; + Throwable cause = storageException.getCause(); + //noinspection StatementWithEmptyBody + if (cause instanceof GoogleJsonResponseException) { + // this is handled by the case for BaseServiceException below + } else if (cause instanceof IOException) { + IOException ioException = (IOException) cause; + return BaseServiceException.isRetryable(idempotent, ioException) + ? RetryResult.RETRY + : RetryResult.NO_RETRY; + } + } + + if (exception instanceof BaseServiceException) { + int code = ((BaseServiceException) exception).getCode(); + if (retryableCodes.contains(code)) { + return RetryResult.RETRY; + } else { + return RetryResult.NO_RETRY; + } + } + return RetryResult.CONTINUE_EVALUATION; + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java index 884c4d4eb..aa72abe03 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java @@ -105,10 +105,8 @@ ExceptionHandler getForBucketsTestIamPermissions( ExceptionHandler getForObjectsRewrite(RewriteRequest pb); - ExceptionHandler getForObjectsCopy(); - ExceptionHandler getForObjectsCompose( - List sources, StorageObject target, Map targetOptions); + List sources, StorageObject target, Map optionsMap); ExceptionHandler getForResumableUploadSessionCreate(Map optionsMap); /** Resumable upload has differing 429 handling */ diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java index 0cf60a787..8450df6f6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java @@ -146,7 +146,7 @@ protected StorageRpc getStorageRpcV1() { return (StorageRpc) getRpc(); } - protected RetryAlgorithmManager getRetryAlgorithmManager() { + RetryAlgorithmManager getRetryAlgorithmManager() { return retryAlgorithmManager; } diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt index 2dcb832dd..c631fba3b 100644 --- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt +++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt @@ -167,12 +167,91 @@ TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.insert-47 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-57 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-80 +TestRetryConformance/3-[return-503]-storage.buckets.patch-17 +TestRetryConformance/3-[return-503]-storage.buckets.setIamPolicy-18 +TestRetryConformance/3-[return-503]-storage.hmacKey.update-29 +TestRetryConformance/3-[return-503]-storage.objects.compose-35 +TestRetryConformance/3-[return-503]-storage.objects.delete-36 +TestRetryConformance/3-[return-503]-storage.objects.delete-67 +TestRetryConformance/3-[return-503]-storage.objects.insert-108 +TestRetryConformance/3-[return-503]-storage.objects.insert-109 TestRetryConformance/3-[return-503]-storage.objects.insert-110 TestRetryConformance/3-[return-503]-storage.objects.insert-111 +TestRetryConformance/3-[return-503]-storage.objects.insert-112 +TestRetryConformance/3-[return-503]-storage.objects.insert-113 TestRetryConformance/3-[return-503]-storage.objects.insert-114 +TestRetryConformance/3-[return-503]-storage.objects.insert-115 +TestRetryConformance/3-[return-503]-storage.objects.insert-116 +TestRetryConformance/3-[return-503]-storage.objects.insert-117 +TestRetryConformance/3-[return-503]-storage.objects.patch-56 +TestRetryConformance/3-[return-503]-storage.objects.patch-79 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-58 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-81 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-82 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-83 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-84 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-85 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-86 +TestRetryConformance/3-[return-reset-connection]-storage.buckets.patch-17 +TestRetryConformance/3-[return-reset-connection]-storage.buckets.setIamPolicy-18 +TestRetryConformance/3-[return-reset-connection]-storage.hmacKey.update-29 +TestRetryConformance/3-[return-reset-connection]-storage.objects.compose-35 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-108 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-109 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-110 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-111 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-112 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-113 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-114 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-115 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-116 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-117 +TestRetryConformance/3-[return-reset-connection]-storage.objects.patch-56 +TestRetryConformance/3-[return-reset-connection]-storage.objects.patch-79 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-58 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-81 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-82 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-83 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-84 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-85 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-86 +TestRetryConformance/4-[return-503]-storage.bucket_acl.delete-1 +TestRetryConformance/4-[return-503]-storage.bucket_acl.delete-2 +TestRetryConformance/4-[return-503]-storage.bucket_acl.delete-87 +TestRetryConformance/4-[return-503]-storage.bucket_acl.insert-5 +TestRetryConformance/4-[return-503]-storage.bucket_acl.insert-6 +TestRetryConformance/4-[return-503]-storage.bucket_acl.insert-89 +TestRetryConformance/4-[return-503]-storage.bucket_acl.patch-10 +TestRetryConformance/4-[return-503]-storage.bucket_acl.patch-9 +TestRetryConformance/4-[return-503]-storage.bucket_acl.patch-91 +TestRetryConformance/4-[return-503]-storage.default_object_acl.delete-102 +TestRetryConformance/4-[return-503]-storage.default_object_acl.delete-20 +TestRetryConformance/4-[return-503]-storage.default_object_acl.insert-104 +TestRetryConformance/4-[return-503]-storage.default_object_acl.insert-22 +TestRetryConformance/4-[return-503]-storage.default_object_acl.patch-106 +TestRetryConformance/4-[return-503]-storage.default_object_acl.patch-24 +TestRetryConformance/4-[return-503]-storage.hmacKey.create-25 +TestRetryConformance/4-[return-503]-storage.object_acl.delete-30 +TestRetryConformance/4-[return-503]-storage.object_acl.delete-62 +TestRetryConformance/4-[return-503]-storage.object_acl.insert-32 +TestRetryConformance/4-[return-503]-storage.object_acl.insert-64 +TestRetryConformance/4-[return-503]-storage.object_acl.patch-34 +TestRetryConformance/4-[return-503]-storage.object_acl.patch-66 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.insert-5 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.insert-6 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.insert-89 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.patch-10 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.patch-9 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.patch-91 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.insert-104 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.insert-22 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.patch-106 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.patch-24 +TestRetryConformance/4-[return-reset-connection]-storage.hmacKey.create-25 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.insert-32 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.insert-64 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.patch-34 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.patch-66 TestRetryConformance/5-[return-400]-storage.bucket_acl.delete-1 TestRetryConformance/5-[return-400]-storage.bucket_acl.delete-2 TestRetryConformance/5-[return-400]-storage.bucket_acl.delete-87 @@ -388,7 +467,6 @@ TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-31 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-63 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-33 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-65 -TestRetryConformance/6-[return-503_return-400]-storage.objects.compose-35 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-37 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-38 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-68 @@ -412,23 +490,9 @@ TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-118 TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-119 TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-46 TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-47 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-49 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-50 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-51 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-52 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-53 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-77 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-78 TestRetryConformance/6-[return-503_return-400]-storage.objects.list-55 TestRetryConformance/6-[return-503_return-400]-storage.objects.patch-57 TestRetryConformance/6-[return-503_return-400]-storage.objects.patch-80 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-58 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-81 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-82 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-83 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-84 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-85 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-86 TestRetryConformance/6-[return-503_return-400]-storage.serviceaccount.get-59 TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-3 TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-4 @@ -459,7 +523,6 @@ TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.g TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.get-63 TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-33 TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-65 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.compose-35 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-37 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-38 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-68 @@ -486,18 +549,8 @@ TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.inse TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-49 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-50 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-51 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-52 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-53 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-77 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-78 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.list-55 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.patch-57 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.patch-80 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-58 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-81 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-82 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-83 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-84 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-85 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-86 TestRetryConformance/6-[return-reset-connection_return-401]-storage.serviceaccount.get-59 From 97b1a2ebe411e48e2df095fe5518a867c5136851 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Oct 2021 22:25:56 +0200 Subject: [PATCH 12/15] deps: update kms.version to v0.93.0 (#1061) --- google-cloud-storage/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 208c6d46d..fb3e3ddac 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -16,7 +16,7 @@ google-cloud-storage - 0.92.2 + 0.93.0 From 193a30a6878ab24512d8ac1f4596115effeb66eb Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 1 Oct 2021 22:44:12 +0200 Subject: [PATCH 13/15] chore(deps): update dependency com.google.cloud:libraries-bom to v23.1.0 (#1060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:libraries-bom](https://togithub.com/GoogleCloudPlatform/cloud-opensource-java) | `23.0.0` -> `23.1.0` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/23.1.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/23.1.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/23.1.0/compatibility-slim/23.0.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/23.1.0/confidence-slim/23.0.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-storage). --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 2a6ae9cdf..4f56565a6 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 23.0.0 + 23.1.0 pom import From 4478b9d8df123d429a3643484403cd294d34dfc0 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Mon, 4 Oct 2021 12:16:17 -0400 Subject: [PATCH 14/15] test(retry): fix objects.compose tests (#1062) * fix incorrect evaluation of idempotency for objects.compose (now generation, previously metageneration) * add mapping for non-idempotent objects.compose invocation * add CtxFunction to construct a ComposeRequest relative to the state * add ComposeRequest handling to state --- .../storage/NewRetryAlgorithmManager.java | 2 +- .../conformance/retry/CtxFunctions.java | 30 ++++++++++++++++++- .../conformance/retry/RpcMethodMappings.java | 23 ++++++++------ .../storage/conformance/retry/State.java | 14 +++++++++ .../retry/testNamesWhichShouldSucceed.txt | 13 +++++--- 5 files changed, 67 insertions(+), 15 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java index 8f84990f2..4dd6390df 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java @@ -284,7 +284,7 @@ public ExceptionHandler getForObjectsRewrite(RewriteRequest pb) { @Override public ExceptionHandler getForObjectsCompose( List sources, StorageObject target, Map optionsMap) { - return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) ? IDEMPOTENT_HANDLER : NON_IDEMPOTENT_HANDLER; } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/CtxFunctions.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/CtxFunctions.java index 5bf0c22bd..d51de7a66 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/CtxFunctions.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/CtxFunctions.java @@ -29,6 +29,8 @@ import com.google.cloud.storage.BucketInfo; import com.google.cloud.storage.HmacKey; import com.google.cloud.storage.ServiceAccount; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.Storage.ComposeRequest; import com.google.cloud.storage.conformance.retry.Functions.CtxFunction; import com.google.common.base.Joiner; import java.util.HashSet; @@ -46,9 +48,35 @@ */ final class CtxFunctions { - private static final class Util { + static final class Util { private static final CtxFunction blobIdAndBlobInfo = (ctx, c) -> ctx.map(state -> state.with(BlobInfo.newBuilder(state.getBlobId()).build())); + + static final CtxFunction composeRequest = + (ctx, c) -> + ctx.map( + state -> { + Blob blob = state.getBlob(); + String bucket = blob.getBucket(); + final BlobInfo target; + if (c.isPreconditionsProvided()) { + target = BlobInfo.newBuilder(BlobId.of(bucket, "blob-full", 0L)).build(); + } else { + target = BlobInfo.newBuilder(BlobId.of(bucket, "blob-full")).build(); + } + ComposeRequest.Builder builder = + ComposeRequest.newBuilder() + // source bucket is resolved from the target, as compose must be within + // the same bucket + .addSource(blob.getName(), blob.getGeneration()) + .addSource(blob.getName(), blob.getGeneration()) + .setTarget(target); + if (c.isPreconditionsProvided()) { + builder = builder.setTargetOptions(BlobTargetOption.generationMatch()); + } + ComposeRequest r = builder.build(); + return state.with(r); + }); } static final class Local { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java index f2d188a1f..c76d2d004 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java @@ -23,7 +23,6 @@ import static com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup.defaultSetup; import static com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup.serviceAccount; import static com.google.common.base.Predicates.not; -import static com.google.common.collect.Lists.newArrayList; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertTrue; @@ -45,12 +44,12 @@ import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.Storage.BucketSourceOption; import com.google.cloud.storage.Storage.BucketTargetOption; -import com.google.cloud.storage.Storage.ComposeRequest; import com.google.cloud.storage.Storage.CopyRequest; import com.google.cloud.storage.Storage.SignUrlOption; import com.google.cloud.storage.Storage.UriScheme; import com.google.cloud.storage.conformance.retry.CtxFunctions.Local; import com.google.cloud.storage.conformance.retry.CtxFunctions.Rpc; +import com.google.cloud.storage.conformance.retry.CtxFunctions.Util; import com.google.cloud.storage.conformance.retry.RpcMethod.storage.bucket_acl; import com.google.cloud.storage.conformance.retry.RpcMethod.storage.buckets; import com.google.cloud.storage.conformance.retry.RpcMethod.storage.default_object_acl; @@ -1824,17 +1823,23 @@ private static void update(ArrayList a) {} private static void compose(ArrayList a) { a.add( RpcMethodMapping.newBuilder(35, objects.compose) + .withApplicable(TestRetryConformance::isPreconditionsProvided) + .withSetup(defaultSetup.andThen(Util.composeRequest)) .withTest( (ctx, c) -> ctx.map( state -> - state.with( - ctx.getStorage() - .compose( - ComposeRequest.of( - c.getBucketName(), - newArrayList("blob-part-1", "blob-part-2"), - "blob-full"))))) + state.with(ctx.getStorage().compose(state.getComposeRequest())))) + .build()); + a.add( + RpcMethodMapping.newBuilder(241, objects.compose) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) + .withSetup(defaultSetup.andThen(Util.composeRequest)) + .withTest( + (ctx, c) -> + ctx.map( + state -> + state.with(ctx.getStorage().compose(state.getComposeRequest())))) .build()); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/State.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/State.java index 8dafa7c17..4f7552977 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/State.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/State.java @@ -30,6 +30,7 @@ import com.google.cloud.storage.HmacKey; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.ServiceAccount; +import com.google.cloud.storage.Storage.ComposeRequest; import com.google.cloud.storage.conformance.retry.Functions.VoidFunction; import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.Immutable; @@ -75,6 +76,7 @@ final class State { new Key<>("testIamPermissionsResults"); private static final Key> KEY_ACLS = new Key<>("acls"); private static final Key KEY_BYTES = new Key<>("bytes"); + private static final Key KEY_COMPOSE_REQUEST = new Key<>("composeRequest"); private final ImmutableMap, Object> data; @@ -293,6 +295,18 @@ public State consume(Page page) { return newStateWith(KEY_LIST_OBJECTS, collect); } + public State with(ComposeRequest composeRequest) { + return newStateWith(KEY_COMPOSE_REQUEST, composeRequest); + } + + public ComposeRequest getComposeRequest() { + return getValue(KEY_COMPOSE_REQUEST); + } + + public boolean hasComposeRequest() { + return hasValue(KEY_COMPOSE_REQUEST); + } + private T getValue(Key key) { Object o = data.get(key); requireNonNull(o, () -> String.format("%s was not found in state", key.name)); diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt index c631fba3b..7a183b7ea 100644 --- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt +++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt @@ -142,6 +142,7 @@ TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/1-[return-reset-connection_return-reset-connection]-storage.serviceaccount.get-59 TestRetryConformance/2-[return-503_return-503]-storage.buckets.patch-101 TestRetryConformance/2-[return-503_return-503]-storage.buckets.patch-122 +TestRetryConformance/2-[return-503_return-503]-storage.objects.compose-35 TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-37 TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-38 TestRetryConformance/2-[return-503_return-503]-storage.objects.delete-68 @@ -151,6 +152,7 @@ TestRetryConformance/2-[return-503_return-503]-storage.objects.patch-57 TestRetryConformance/2-[return-503_return-503]-storage.objects.patch-80 TestRetryConformance/2-[return-reset-connection_return-503]-storage.buckets.patch-101 TestRetryConformance/2-[return-reset-connection_return-503]-storage.buckets.patch-122 +TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.compose-35 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-37 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-38 TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.delete-68 @@ -160,6 +162,7 @@ TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.patc TestRetryConformance/2-[return-reset-connection_return-503]-storage.objects.patch-80 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.buckets.patch-101 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.buckets.patch-122 +TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.compose-35 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-37 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-38 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.delete-68 @@ -170,7 +173,7 @@ TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/3-[return-503]-storage.buckets.patch-17 TestRetryConformance/3-[return-503]-storage.buckets.setIamPolicy-18 TestRetryConformance/3-[return-503]-storage.hmacKey.update-29 -TestRetryConformance/3-[return-503]-storage.objects.compose-35 +TestRetryConformance/3-[return-503]-storage.objects.compose-241 TestRetryConformance/3-[return-503]-storage.objects.delete-36 TestRetryConformance/3-[return-503]-storage.objects.delete-67 TestRetryConformance/3-[return-503]-storage.objects.insert-108 @@ -195,7 +198,7 @@ TestRetryConformance/3-[return-503]-storage.objects.rewrite-86 TestRetryConformance/3-[return-reset-connection]-storage.buckets.patch-17 TestRetryConformance/3-[return-reset-connection]-storage.buckets.setIamPolicy-18 TestRetryConformance/3-[return-reset-connection]-storage.hmacKey.update-29 -TestRetryConformance/3-[return-reset-connection]-storage.objects.compose-35 +TestRetryConformance/3-[return-reset-connection]-storage.objects.compose-241 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-108 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-109 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-110 @@ -303,7 +306,7 @@ TestRetryConformance/5-[return-400]-storage.object_acl.list-33 TestRetryConformance/5-[return-400]-storage.object_acl.list-65 TestRetryConformance/5-[return-400]-storage.object_acl.patch-34 TestRetryConformance/5-[return-400]-storage.object_acl.patch-66 -TestRetryConformance/5-[return-400]-storage.objects.compose-35 +TestRetryConformance/5-[return-400]-storage.objects.compose-241 TestRetryConformance/5-[return-400]-storage.objects.delete-36 TestRetryConformance/5-[return-400]-storage.objects.delete-67 TestRetryConformance/5-[return-400]-storage.objects.get-107 @@ -396,7 +399,7 @@ TestRetryConformance/5-[return-401]-storage.object_acl.list-33 TestRetryConformance/5-[return-401]-storage.object_acl.list-65 TestRetryConformance/5-[return-401]-storage.object_acl.patch-34 TestRetryConformance/5-[return-401]-storage.object_acl.patch-66 -TestRetryConformance/5-[return-401]-storage.objects.compose-35 +TestRetryConformance/5-[return-401]-storage.objects.compose-241 TestRetryConformance/5-[return-401]-storage.objects.delete-36 TestRetryConformance/5-[return-401]-storage.objects.delete-67 TestRetryConformance/5-[return-401]-storage.objects.get-107 @@ -467,6 +470,7 @@ TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-31 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-63 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-33 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-65 +TestRetryConformance/6-[return-503_return-400]-storage.objects.compose-35 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-37 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-38 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-68 @@ -523,6 +527,7 @@ TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.g TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.get-63 TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-33 TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-65 +TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.compose-35 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-37 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-38 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-68 From bf3207b5c3347845cafe5cde87fa204cd07bec07 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 17:36:31 +0000 Subject: [PATCH 15/15] chore: release 2.1.7 (#1055) :robot: I have created a release \*beep\* \*boop\* --- ### [2.1.7](https://www.github.com/googleapis/java-storage/compare/v2.1.6...v2.1.7) (2021-10-04) ### Bug Fixes * update PAP to use inherited instead of unspecified ([#1051](https://www.github.com/googleapis/java-storage/issues/1051)) ([6d73e46](https://www.github.com/googleapis/java-storage/commit/6d73e4631777542996a0ea815b482f5c19a8927d)) ### Dependencies * update dependency com.google.apis:google-api-services-storage to v1-rev20210918-1.32.1 ([#1046](https://www.github.com/googleapis/java-storage/issues/1046)) ([2c79005](https://www.github.com/googleapis/java-storage/commit/2c79005d29ee0b279850c7008b1afbb302f9c90d)) * update kms.version to v0.93.0 ([#1061](https://www.github.com/googleapis/java-storage/issues/1061)) ([97b1a2e](https://www.github.com/googleapis/java-storage/commit/97b1a2ebe411e48e2df095fe5518a867c5136851)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 13 +++++++++++++ gapic-google-cloud-storage-v2/pom.xml | 4 ++-- google-cloud-storage/pom.xml | 4 ++-- grpc-google-cloud-storage-v2/pom.xml | 4 ++-- pom.xml | 8 ++++---- proto-google-cloud-storage-v2/pom.xml | 4 ++-- versions.txt | 8 ++++---- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f6d405e0..2a5853102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +### [2.1.7](https://www.github.com/googleapis/java-storage/compare/v2.1.6...v2.1.7) (2021-10-04) + + +### Bug Fixes + +* update PAP to use inherited instead of unspecified ([#1051](https://www.github.com/googleapis/java-storage/issues/1051)) ([6d73e46](https://www.github.com/googleapis/java-storage/commit/6d73e4631777542996a0ea815b482f5c19a8927d)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20210918-1.32.1 ([#1046](https://www.github.com/googleapis/java-storage/issues/1046)) ([2c79005](https://www.github.com/googleapis/java-storage/commit/2c79005d29ee0b279850c7008b1afbb302f9c90d)) +* update kms.version to v0.93.0 ([#1061](https://www.github.com/googleapis/java-storage/issues/1061)) ([97b1a2e](https://www.github.com/googleapis/java-storage/commit/97b1a2ebe411e48e2df095fe5518a867c5136851)) + ### [2.1.6](https://www.github.com/googleapis/java-storage/compare/v2.1.5...v2.1.6) (2021-09-24) diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml index 937f789ac..98578b24f 100644 --- a/gapic-google-cloud-storage-v2/pom.xml +++ b/gapic-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.1.7-alpha-SNAPSHOT + 2.1.7-alpha gapic-google-cloud-storage-v2 GRPC library for gapic-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.1.7-SNAPSHOT + 2.1.7 diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index fb3e3ddac..27e97af4b 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 2.1.7-SNAPSHOT + 2.1.7 jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 2.1.7-SNAPSHOT + 2.1.7 google-cloud-storage diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml index a0565660b..497e681d2 100644 --- a/grpc-google-cloud-storage-v2/pom.xml +++ b/grpc-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-v2 - 2.1.7-alpha-SNAPSHOT + 2.1.7-alpha grpc-google-cloud-storage-v2 GRPC library for grpc-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.1.7-SNAPSHOT + 2.1.7 diff --git a/pom.xml b/pom.xml index 308367e59..d1f77e822 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 2.1.7-SNAPSHOT + 2.1.7 Storage Parent https://github.com/googleapis/java-storage @@ -93,17 +93,17 @@ com.google.api.grpc proto-google-cloud-storage-v2 - 2.1.7-alpha-SNAPSHOT + 2.1.7-alpha com.google.api.grpc grpc-google-cloud-storage-v2 - 2.1.7-alpha-SNAPSHOT + 2.1.7-alpha com.google.api.grpc gapic-google-cloud-storage-v2 - 2.1.7-alpha-SNAPSHOT + 2.1.7-alpha com.google.cloud diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml index 2b2a4d476..c3f5109c3 100644 --- a/proto-google-cloud-storage-v2/pom.xml +++ b/proto-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-v2 - 2.1.7-alpha-SNAPSHOT + 2.1.7-alpha proto-google-cloud-storage-v2 PROTO library for proto-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.1.7-SNAPSHOT + 2.1.7 diff --git a/versions.txt b/versions.txt index 7da1b65ac..7ceab6429 100644 --- a/versions.txt +++ b/versions.txt @@ -1,7 +1,7 @@ # Format: # module:released-version:current-version -google-cloud-storage:2.1.6:2.1.7-SNAPSHOT -gapic-google-cloud-storage-v2:2.1.6-alpha:2.1.7-alpha-SNAPSHOT -grpc-google-cloud-storage-v2:2.1.6-alpha:2.1.7-alpha-SNAPSHOT -proto-google-cloud-storage-v2:2.1.6-alpha:2.1.7-alpha-SNAPSHOT +google-cloud-storage:2.1.7:2.1.7 +gapic-google-cloud-storage-v2:2.1.7-alpha:2.1.7-alpha +grpc-google-cloud-storage-v2:2.1.7-alpha:2.1.7-alpha +proto-google-cloud-storage-v2:2.1.7-alpha:2.1.7-alpha