docs: add sample and sample test for transfer manager (#1027)

* add sample and sample test for transfer manager download blob as chunks concurrently method

* chore: modify format for int

* chore: refactor transfer manager sample names and tests

---------

Co-authored-by: Andrew Gorcester <[email protected]>

Author: MiaCY

samples/snippets/snippets_test.py

@@ -72,7 +72,10 @@
 import storage_set_bucket_default_kms_key
 import storage_set_client_endpoint
 import storage_set_metadata
-import storage_transfer_manager
+import storage_transfer_manager_download_all_blobs
+import storage_transfer_manager_download_chunks_concurrently
+import storage_transfer_manager_upload_directory
+import storage_transfer_manager_upload_many_blobs
 import storage_upload_file
 import storage_upload_from_memory
 import storage_upload_from_stream
@@ -686,7 +689,7 @@ def test_transfer_manager_snippets(test_bucket, capsys):
             with open(os.path.join(uploads, name), "w") as f:
                 f.write(name)
 
-        storage_transfer_manager.upload_many_blobs_with_transfer_manager(
+        storage_transfer_manager_upload_many_blobs.upload_many_blobs_with_transfer_manager(
             test_bucket.name,
             BLOB_NAMES,
             source_directory="{}/".format(uploads),
@@ -699,7 +702,7 @@ def test_transfer_manager_snippets(test_bucket, capsys):
 
     with tempfile.TemporaryDirectory() as downloads:
         # Download the files.
-        storage_transfer_manager.download_all_blobs_with_transfer_manager(
+        storage_transfer_manager_download_all_blobs.download_all_blobs_with_transfer_manager(
             test_bucket.name,
             destination_directory=os.path.join(downloads, ""),
             threads=2,
@@ -729,11 +732,34 @@ def test_transfer_manager_directory_upload(test_bucket, capsys):
             with open(os.path.join(uploads, name), "w") as f:
                 f.write(name)
 
-        storage_transfer_manager.upload_directory_with_transfer_manager(
+        storage_transfer_manager_upload_directory.upload_directory_with_transfer_manager(
             test_bucket.name, source_directory="{}/".format(uploads)
         )
         out, _ = capsys.readouterr()
 
         assert "Found {}".format(len(BLOB_NAMES)) in out
         for name in BLOB_NAMES:
             assert "Uploaded {}".format(name) in out
+
+
+def test_transfer_manager_download_chunks_concurrently(test_bucket, capsys):
+    BLOB_NAME = "test_file.txt"
+
+    with tempfile.NamedTemporaryFile() as file:
+        file.write(b"test")
+
+        storage_upload_file.upload_blob(
+            test_bucket.name, file.name, BLOB_NAME
+        )
+
+    with tempfile.TemporaryDirectory() as downloads:
+        # Download the file.
+        storage_transfer_manager_download_chunks_concurrently.download_chunks_concurrently(
+            test_bucket.name,
+            BLOB_NAME,
+            os.path.join(downloads, BLOB_NAME),
+            processes=8,
+        )
+        out, _ = capsys.readouterr()
+
+        assert "Downloaded {} to {}".format(BLOB_NAME, os.path.join(downloads, BLOB_NAME)) in out

samples/snippets/storage_transfer_manager.py

@@ -0,0 +1,65 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def download_all_blobs_with_transfer_manager(
+    bucket_name, destination_directory="", threads=4
+):
+    """Download all of the blobs in a bucket, concurrently in a thread pool.
+
+    The filename of each blob once downloaded is derived from the blob name and
+    the `destination_directory `parameter. For complete control of the filename
+    of each blob, use transfer_manager.download_many() instead.
+
+    Directories will be created automatically as needed, for instance to
+    accommodate blob names that include slashes.
+    """
+
+    # The ID of your GCS bucket
+    # bucket_name = "your-bucket-name"
+
+    # The directory on your computer to which to download all of the files. This
+    # string is prepended (with os.path.join()) to the name of each blob to form
+    # the full path. Relative paths and absolute paths are both accepted. An
+    # empty string means "the current working directory". Note that this
+    # parameter allows accepts directory traversal ("../" etc.) and is not
+    # intended for unsanitized end user input.
+    # destination_directory = ""
+
+    # The number of threads to use for the operation. The performance impact of
+    # this value depends on the use case, but generally, smaller files benefit
+    # from more threads and larger files don't benefit from more threads. Too
+    # many threads can slow operations, especially with large files, due to
+    # contention over the Python GIL.
+    # threads=4
+
+    from google.cloud.storage import Client, transfer_manager
+
+    storage_client = Client()
+    bucket = storage_client.bucket(bucket_name)
+
+    blob_names = [blob.name for blob in bucket.list_blobs()]
+
+    results = transfer_manager.download_many_to_path(
+        bucket, blob_names, destination_directory=destination_directory, threads=threads
+    )
+
+    for name, result in zip(blob_names, results):
+        # The results list is either `None` or an exception for each blob in
+        # the input list, in order.
+
+        if isinstance(result, Exception):
+            print("Failed to download {} due to exception: {}".format(name, result))
+        else:
+            print("Downloaded {} to {}.".format(name, destination_directory + name))

samples/snippets/storage_transfer_manager_download_all_blobs.py

@@ -0,0 +1,44 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def download_chunks_concurrently(bucket_name, blob_name, filename, processes=8):
+    """Download a single file in chunks, concurrently."""
+
+    # The ID of your GCS bucket
+    # bucket_name = "your-bucket-name"
+
+    # The file to be downloaded
+    # blob_name = "target-file"
+
+    # The destination filename or path
+    # filename = ""
+
+    # The maximum number of worker processes that should be used to handle the
+    # workload of downloading the blob concurrently. PROCESS worker type uses more
+    # system resources (both memory and CPU) and can result in faster operations
+    # when working with large files. The optimal number of workers depends heavily
+    # on the specific use case. Refer to the docstring of the underlining method
+    # for more details.
+    # processes=8
+
+    from google.cloud.storage import Client, transfer_manager
+
+    storage_client = Client()
+    bucket = storage_client.bucket(bucket_name)
+    blob = bucket.blob(blob_name)
+
+    transfer_manager.download_chunks_concurrently(blob, filename, max_workers=processes)
+
+    print("Downloaded {} to {}.".format(blob_name, filename))