Skip to content

Commit 7beb99d

Browse files
dmitry-fadmitry-fa
authored
feat: add upload functionality (#214)
feat: add upload functionality * feat: add upload functionality * fix: review comments
1 parent be74072 commit 7beb99d

File tree

3 files changed

+405
-0
lines changed

3 files changed

+405
-0
lines changed

‎google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOperations.java

+188
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.cloud.WriteChannel;
20+
import java.io.IOException;
21+
import java.io.InputStream;
22+
import java.nio.ByteBuffer;
23+
import java.nio.channels.Channels;
24+
import java.nio.channels.ReadableByteChannel;
25+
import java.nio.file.Files;
26+
import java.nio.file.Path;
27+
28+
/** Utility methods to perform various operations with the Storage such as upload. */
29+
public final class StorageOperations {
30+
31+
private final Storage storage;
32+
private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024;
33+
private static final int MIN_BUFFER_SIZE = 256 * 1024;
34+
35+
/**
36+
* Creates a new StorageOperations instance associated with the given storage.
37+
*
38+
* @param storage the Storage
39+
*/
40+
public StorageOperations(Storage storage) {
41+
this.storage = storage;
42+
}
43+
44+
/**
45+
* Uploads {@code path} to the blob using {@link Storage#writer}. By default any MD5 and CRC32C
46+
* values in the given {@code blobInfo} are ignored unless requested via the {@link
47+
* Storage.BlobWriteOption#md5Match()} and {@link Storage.BlobWriteOption#crc32cMatch()} options.
48+
* Folder upload is not supported.
49+
*
50+
* <p>Example of uploading a file:
51+
*
52+
* <pre>{@code
53+
* String bucketName = "my-unique-bucket";
54+
* String fileName = "readme.txt";
55+
* BlobId blobId = BlobId.of(bucketName, fileName);
56+
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
57+
* new StorageOperations(storage).upload(blobInfo, Paths.get(fileName));
58+
* }</pre>
59+
*
60+
* @param blobInfo blob to create
61+
* @param path file to upload
62+
* @param options blob write options
63+
* @throws IOException on I/O error
64+
* @throws StorageException on failure
65+
* @see #upload(BlobInfo, Path, int, Storage.BlobWriteOption...)
66+
*/
67+
public void upload(BlobInfo blobInfo, Path path, Storage.BlobWriteOption... options)
68+
throws IOException {
69+
upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options);
70+
}
71+
72+
/**
73+
* Uploads {@code path} to the blob using {@link Storage#writer} and {@code bufferSize}. By
74+
* default any MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested
75+
* via the {@link Storage.BlobWriteOption#md5Match()} and {@link
76+
* Storage.BlobWriteOption#crc32cMatch()} options. Folder upload is not supported.
77+
*
78+
* <p>{@link #upload(BlobInfo, Path, Storage.BlobWriteOption...)} invokes this method with a
79+
* buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the
80+
* upload performance but require more memory. This can cause an OutOfMemoryError or add
81+
* significant garbage collection overhead. Smaller buffer sizes reduce memory consumption, that
82+
* is noticeable when uploading many objects in parallel. Buffer sizes less than 256 KiB are
83+
* treated as 256 KiB.
84+
*
85+
* <p>Example of uploading a humongous file:
86+
*
87+
* <pre>{@code
88+
* BlobId blobId = BlobId.of(bucketName, blobName);
89+
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("video/webm").build();
90+
*
91+
* int largeBufferSize = 150 * 1024 * 1024;
92+
* Path file = Paths.get("humongous.file");
93+
* new StorageOperations(storage).upload(blobInfo, file, largeBufferSize);
94+
* }</pre>
95+
*
96+
* @param blobInfo blob to create
97+
* @param path file to upload
98+
* @param bufferSize size of the buffer I/O operations
99+
* @param options blob write options
100+
* @throws IOException on I/O error
101+
* @throws StorageException on failure
102+
*/
103+
public void upload(
104+
BlobInfo blobInfo, Path path, int bufferSize, Storage.BlobWriteOption... options)
105+
throws IOException {
106+
if (Files.isDirectory(path)) {
107+
throw new StorageException(0, path + " is a directory");
108+
}
109+
try (InputStream input = Files.newInputStream(path)) {
110+
upload(blobInfo, input, bufferSize, options);
111+
}
112+
}
113+
114+
/**
115+
* Reads bytes from an input stream and uploads those bytes to the blob using {@link
116+
* Storage#writer}. By default any MD5 and CRC32C values in the given {@code blobInfo} are ignored
117+
* unless requested via the {@link Storage.BlobWriteOption#md5Match()} and {@link
118+
* Storage.BlobWriteOption#crc32cMatch()} options.
119+
*
120+
* <p>Example of uploading data with CRC32C checksum:
121+
*
122+
* <pre>{@code
123+
* BlobId blobId = BlobId.of(bucketName, blobName);
124+
* byte[] content = "Hello, world".getBytes(StandardCharsets.UTF_8);
125+
* Hasher hasher = Hashing.crc32c().newHasher().putBytes(content);
126+
* String crc32c = BaseEncoding.base64().encode(Ints.toByteArray(hasher.hash().asInt()));
127+
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(crc32c).build();
128+
* new StorageOperations(storage).upload(blobInfo, new ByteArrayInputStream(content),
129+
* Storage.BlobWriteOption.crc32cMatch());
130+
* }</pre>
131+
*
132+
* @param blobInfo blob to create
133+
* @param content input stream to read from
134+
* @param options blob write options
135+
* @throws IOException on I/O error
136+
* @throws StorageException on failure
137+
* @see #upload(BlobInfo, InputStream, int, Storage.BlobWriteOption...)
138+
*/
139+
public void upload(BlobInfo blobInfo, InputStream content, Storage.BlobWriteOption... options)
140+
throws IOException {
141+
upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options);
142+
}
143+
144+
/**
145+
* Reads bytes from an input stream and uploads those bytes to the blob using {@link
146+
* Storage#writer} and {@code bufferSize}. By default any MD5 and CRC32C values in the given
147+
* {@code blobInfo} are ignored unless requested via the {@link
148+
* Storage.BlobWriteOption#md5Match()} and {@link Storage.BlobWriteOption#crc32cMatch()} options.
149+
*
150+
* <p>{@link #upload(BlobInfo, InputStream, Storage.BlobWriteOption...)} )} invokes this method
151+
* with a buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might
152+
* improve the upload performance but require more memory. This can cause an OutOfMemoryError or
153+
* add significant garbage collection overhead. Smaller buffer sizes reduce memory consumption,
154+
* that is noticeable when uploading many objects in parallel. Buffer sizes less than 256 KiB are
155+
* treated as 256 KiB.
156+
*
157+
* @param blobInfo blob to create
158+
* @param content input stream to read from
159+
* @param bufferSize size of the buffer I/O operations
160+
* @param options blob write options
161+
* @throws IOException on I/O error
162+
* @throws StorageException on failure
163+
*/
164+
public void upload(
165+
BlobInfo blobInfo, InputStream content, int bufferSize, Storage.BlobWriteOption... options)
166+
throws IOException {
167+
try (WriteChannel writer = storage.writer(blobInfo, options)) {
168+
upload(Channels.newChannel(content), writer, bufferSize);
169+
}
170+
}
171+
172+
/*
173+
* Uploads the given content to the storage using specified write channel and the given buffer
174+
* size. This method does not close any channels.
175+
*/
176+
private static void upload(ReadableByteChannel reader, WriteChannel writer, int bufferSize)
177+
throws IOException {
178+
bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE);
179+
ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
180+
writer.setChunkSize(bufferSize);
181+
182+
while (reader.read(buffer) >= 0) {
183+
buffer.flip();
184+
writer.write(buffer);
185+
buffer.clear();
186+
}
187+
}
188+
}

‎google-cloud-storage/src/test/java/com/google/cloud/storage/StorageOperationsTest.java

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import static org.easymock.EasyMock.anyObject;
20+
import static org.easymock.EasyMock.createStrictMock;
21+
import static org.easymock.EasyMock.eq;
22+
import static org.easymock.EasyMock.expect;
23+
import static org.easymock.EasyMock.replay;
24+
import static org.easymock.EasyMock.verify;
25+
import static org.junit.Assert.assertEquals;
26+
import static org.junit.Assert.assertSame;
27+
import static org.junit.Assert.fail;
28+
29+
import com.google.cloud.WriteChannel;
30+
import java.io.ByteArrayInputStream;
31+
import java.io.IOException;
32+
import java.io.InputStream;
33+
import java.nio.ByteBuffer;
34+
import java.nio.file.Files;
35+
import java.nio.file.NoSuchFileException;
36+
import java.nio.file.Path;
37+
import java.nio.file.Paths;
38+
import org.junit.After;
39+
import org.junit.Before;
40+
import org.junit.Test;
41+
42+
public class StorageOperationsTest {
43+
private Storage storage;
44+
private StorageOperations storageOperations;
45+
46+
private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder("b", "n").build();
47+
private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024;
48+
private static final int MIN_BUFFER_SIZE = 256 * 1024;
49+
50+
@Before
51+
public void setUp() {
52+
storage = createStrictMock(Storage.class);
53+
storageOperations = new StorageOperations(storage);
54+
}
55+
56+
@After
57+
public void tearDown() throws Exception {
58+
verify(storage);
59+
}
60+
61+
@Test
62+
public void testUploadFromNonExistentFile() {
63+
replay(storage);
64+
String fileName = "non_existing_file.txt";
65+
try {
66+
storageOperations.upload(BLOB_INFO, Paths.get(fileName));
67+
storageOperations.upload(BLOB_INFO, Paths.get(fileName), -1);
68+
fail();
69+
} catch (IOException e) {
70+
assertEquals(NoSuchFileException.class, e.getClass());
71+
assertEquals(fileName, e.getMessage());
72+
}
73+
}
74+
75+
@Test
76+
public void testUploadFromDirectory() throws IOException {
77+
replay(storage);
78+
Path dir = Files.createTempDirectory("unit_");
79+
try {
80+
storageOperations.upload(BLOB_INFO, dir);
81+
storageOperations.upload(BLOB_INFO, dir, -2);
82+
fail();
83+
} catch (StorageException e) {
84+
assertEquals(dir + " is a directory", e.getMessage());
85+
}
86+
}
87+
88+
private void prepareForUpload(BlobInfo blobInfo, byte[] bytes, Storage.BlobWriteOption... options)
89+
throws Exception {
90+
prepareForUpload(blobInfo, bytes, DEFAULT_BUFFER_SIZE, options);
91+
}
92+
93+
private void prepareForUpload(
94+
BlobInfo blobInfo, byte[] bytes, int bufferSize, Storage.BlobWriteOption... options)
95+
throws Exception {
96+
WriteChannel channel = createStrictMock(WriteChannel.class);
97+
ByteBuffer expectedByteBuffer = ByteBuffer.wrap(bytes, 0, bytes.length);
98+
channel.setChunkSize(bufferSize);
99+
expect(channel.write(expectedByteBuffer)).andReturn(bytes.length);
100+
channel.close();
101+
replay(channel);
102+
expect(storage.writer(blobInfo, options)).andReturn(channel);
103+
replay(storage);
104+
}
105+
106+
@Test
107+
public void testUploadFromFile() throws Exception {
108+
byte[] dataToSend = {1, 2, 3};
109+
prepareForUpload(BLOB_INFO, dataToSend);
110+
Path tempFile = Files.createTempFile("testUpload", ".tmp");
111+
Files.write(tempFile, dataToSend);
112+
storageOperations.upload(BLOB_INFO, tempFile);
113+
}
114+
115+
@Test
116+
public void testUploadFromStream() throws Exception {
117+
byte[] dataToSend = {1, 2, 3, 4, 5};
118+
Storage.BlobWriteOption[] options =
119+
new Storage.BlobWriteOption[] {Storage.BlobWriteOption.crc32cMatch()};
120+
prepareForUpload(BLOB_INFO, dataToSend, options);
121+
InputStream input = new ByteArrayInputStream(dataToSend);
122+
storageOperations.upload(BLOB_INFO, input, options);
123+
}
124+
125+
@Test
126+
public void testUploadSmallBufferSize() throws Exception {
127+
byte[] dataToSend = new byte[100_000];
128+
prepareForUpload(BLOB_INFO, dataToSend, MIN_BUFFER_SIZE);
129+
InputStream input = new ByteArrayInputStream(dataToSend);
130+
int smallBufferSize = 100;
131+
storageOperations.upload(BLOB_INFO, input, smallBufferSize);
132+
}
133+
134+
@Test
135+
public void testUploadFromIOException() throws Exception {
136+
IOException ioException = new IOException("message");
137+
WriteChannel channel = createStrictMock(WriteChannel.class);
138+
channel.setChunkSize(DEFAULT_BUFFER_SIZE);
139+
expect(channel.write((ByteBuffer) anyObject())).andThrow(ioException);
140+
replay(channel);
141+
expect(storage.writer(eq(BLOB_INFO))).andReturn(channel);
142+
replay(storage);
143+
InputStream input = new ByteArrayInputStream(new byte[10]);
144+
try {
145+
storageOperations.upload(BLOB_INFO, input);
146+
fail();
147+
} catch (IOException e) {
148+
assertSame(e, ioException);
149+
}
150+
}
151+
152+
@Test
153+
public void testUploadMultiplePortions() throws Exception {
154+
int totalSize = 400_000;
155+
int bufferSize = 300_000;
156+
byte[] dataToSend = new byte[totalSize];
157+
dataToSend[0] = 42;
158+
dataToSend[bufferSize] = 43;
159+
160+
WriteChannel channel = createStrictMock(WriteChannel.class);
161+
channel.setChunkSize(bufferSize);
162+
expect(channel.write(ByteBuffer.wrap(dataToSend, 0, bufferSize))).andReturn(1);
163+
expect(channel.write(ByteBuffer.wrap(dataToSend, bufferSize, totalSize - bufferSize)))
164+
.andReturn(2);
165+
channel.close();
166+
replay(channel);
167+
expect(storage.writer(BLOB_INFO)).andReturn(channel);
168+
replay(storage);
169+
170+
InputStream input = new ByteArrayInputStream(dataToSend);
171+
storageOperations.upload(BLOB_INFO, input, bufferSize);
172+
}
173+
}

0 commit comments

Comments
 (0)