Add 'Client.list_blobs(bucket_or_name)'. (#8375)

Gurov Ilya · tswast · commit c737b7459464 · 2019-06-28T16:24:43.000-07:00
* Union create-or-use bucket snippets into one helper

* Deleted test_list_blobs - looks like it's a full copy of test_list_blobs_defaults

* Add new method client.list_blobs that's the same as bucket.list_blobs()

* Redact comment and delete unittest runner

* Add deprecation for bucket.list_blobs

* Changed warn to comment

* Bring method docs to Google-style, use mock connection instead of class

* Nit on docing

Add backticks for sphinx
diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py
@@ -721,6 +721,9 @@ def list_blobs(
     ):
         """Return an iterator used to find blobs in the bucket.
 
+        .. note::
+          Direct use of this method is deprecated. Use ``Client.list_blobs`` instead.
+
         If :attr:`user_project` is set, bills the API request to that project.
 
         :type max_results: int
diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py
@@ -153,6 +153,26 @@ def _pop_batch(self):
         """
         return self._batch_stack.pop()
 
+    def _bucket_arg_to_bucket(self, bucket_or_name):
+        """Helper to return given bucket or create new by name.
+
+        Args:
+            bucket_or_name (Union[ \
+                :class:`~google.cloud.storage.bucket.Bucket`, \
+                 str, \
+            ]):
+                The bucket resource to pass or name to create.
+
+        Returns:
+            google.cloud.storage.bucket.Bucket
+                The newly created bucket or the given one.
+        """
+        if isinstance(bucket_or_name, Bucket):
+            bucket = bucket_or_name
+        else:
+            bucket = Bucket(self, name=bucket_or_name)
+        return bucket
+
     @property
     def current_batch(self):
         """Currently-active batch.
@@ -252,12 +272,7 @@ def get_bucket(self, bucket_or_name):
             >>> bucket = client.get_bucket(bucket)  # API request.
 
         """
-
-        bucket = None
-        if isinstance(bucket_or_name, Bucket):
-            bucket = bucket_or_name
-        else:
-            bucket = Bucket(self, name=bucket_or_name)
+        bucket = self._bucket_arg_to_bucket(bucket_or_name)
 
         bucket.reload(client=self)
         return bucket
@@ -299,7 +314,7 @@ def create_bucket(self, bucket_or_name, requester_pays=None, project=None):
                 Optional. Whether requester pays for API requests for this
                 bucket and its blobs.
             project (str):
-                Optional. the project under which the  bucket is to be created.
+                Optional. the project under which the bucket is to be created.
                 If not passed, uses the project set on the client.
 
         Returns:
@@ -331,12 +346,7 @@ def create_bucket(self, bucket_or_name, requester_pays=None, project=None):
             >>> bucket = client.create_bucket(bucket)  # API request.
 
         """
-
-        bucket = None
-        if isinstance(bucket_or_name, Bucket):
-            bucket = bucket_or_name
-        else:
-            bucket = Bucket(self, name=bucket_or_name)
+        bucket = self._bucket_arg_to_bucket(bucket_or_name)
 
         if requester_pays is not None:
             bucket.requester_pays = requester_pays
@@ -394,6 +404,80 @@ def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None):
 
             blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end)
 
+    def list_blobs(
+        self,
+        bucket_or_name,
+        max_results=None,
+        page_token=None,
+        prefix=None,
+        delimiter=None,
+        versions=None,
+        projection="noAcl",
+        fields=None,
+    ):
+        """Return an iterator used to find blobs in the bucket.
+
+        If :attr:`user_project` is set, bills the API request to that project.
+
+        Args:
+            bucket_or_name (Union[ \
+                :class:`~google.cloud.storage.bucket.Bucket`, \
+                 str, \
+            ]):
+                The bucket resource to pass or name to create.
+
+            max_results (int):
+                (Optional) The maximum number of blobs in each page of results
+                from this request. Non-positive values are ignored. Defaults to
+                a sensible value set by the API.
+
+            page_token (str):
+                (Optional) If present, return the next batch of blobs, using the
+                value, which must correspond to the ``nextPageToken`` value
+                returned in the previous response.  Deprecated: use the ``pages``
+                property of the returned iterator instead of manually passing the
+                token.
+
+            prefix (str):
+                (Optional) prefix used to filter blobs.
+
+            delimiter (str):
+                (Optional) Delimiter, used with ``prefix`` to
+                emulate hierarchy.
+
+            versions (bool):
+                (Optional) Whether object versions should be returned
+                as separate blobs.
+
+            projection (str):
+                (Optional) If used, must be 'full' or 'noAcl'.
+                Defaults to ``'noAcl'``. Specifies the set of
+                properties to return.
+
+            fields (str):
+                (Optional) Selector specifying which fields to include
+                in a partial response. Must be a list of fields. For
+                example to get a partial response with just the next
+                page token and the name and language of each blob returned:
+                ``'items(name,contentLanguage),nextPageToken'``.
+                See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
+
+        Returns:
+            Iterator of all :class:`~google.cloud.storage.blob.Blob`
+            in this bucket matching the arguments.
+        """
+        bucket = self._bucket_arg_to_bucket(bucket_or_name)
+        return bucket.list_blobs(
+            max_results=max_results,
+            page_token=page_token,
+            prefix=prefix,
+            delimiter=delimiter,
+            versions=versions,
+            projection=projection,
+            fields=fields,
+            client=self
+        )
+
     def list_buckets(
         self,
         max_results=None,
diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py
@@ -782,19 +782,6 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
         self.assertEqual(kw["path"], "/b/%s/o" % NAME)
         self.assertEqual(kw["query_params"], EXPECTED)
 
-    def test_list_blobs(self):
-        NAME = "name"
-        connection = _Connection({"items": []})
-        client = _Client(connection)
-        bucket = self._make_one(client=client, name=NAME)
-        iterator = bucket.list_blobs()
-        blobs = list(iterator)
-        self.assertEqual(blobs, [])
-        kw, = connection._requested
-        self.assertEqual(kw["method"], "GET")
-        self.assertEqual(kw["path"], "/b/%s/o" % NAME)
-        self.assertEqual(kw["query_params"], {"projection": "noAcl"})
-
     def test_list_notifications(self):
         from google.cloud.storage.notification import BucketNotification
         from google.cloud.storage.notification import _TOPIC_REF_FMT
diff --git a/storage/tests/unit/test_client.py b/storage/tests/unit/test_client.py
@@ -28,6 +28,16 @@ def _make_credentials():
     return mock.Mock(spec=google.auth.credentials.Credentials)
 
 
+def _make_connection(*responses):
+    import google.cloud.storage._http
+    from google.cloud.exceptions import NotFound
+
+    mock_conn = mock.create_autospec(google.cloud.storage._http.Connection)
+    mock_conn.user_agent = "testing 1.2.3"
+    mock_conn.api_request.side_effect = list(responses) + [NotFound("miss")]
+    return mock_conn
+
+
 def _make_response(status=http_client.OK, content=b"", headers={}):
     response = requests.Response()
     response.status_code = status
@@ -639,6 +649,83 @@ def test_download_blob_to_file_with_invalid_uri(self):
                     "http://bucket_name/path/to/object", file_obj
                 )
 
+    def test_list_blobs(self):
+        from google.cloud.storage.bucket import Bucket
+        BUCKET_NAME = "bucket-name"
+
+        credentials = _make_credentials()
+        client = self._make_one(project="PROJECT", credentials=credentials)
+        connection = _make_connection({"items": []})
+
+        with mock.patch(
+            'google.cloud.storage.client.Client._connection',
+            new_callable=mock.PropertyMock
+        ) as client_mock:
+            client_mock.return_value = connection
+
+            bucket_obj = Bucket(client, BUCKET_NAME)
+            iterator = client.list_blobs(bucket_obj)
+            blobs = list(iterator)
+
+            self.assertEqual(blobs, [])
+            connection.api_request.assert_called_once_with(
+                method="GET",
+                path="/b/%s/o" % BUCKET_NAME,
+                query_params={"projection": "noAcl"}
+            )
+
+    def test_list_blobs_w_all_arguments_and_user_project(self):
+        from google.cloud.storage.bucket import Bucket
+        BUCKET_NAME = "name"
+        USER_PROJECT = "user-project-123"
+        MAX_RESULTS = 10
+        PAGE_TOKEN = "ABCD"
+        PREFIX = "subfolder"
+        DELIMITER = "/"
+        VERSIONS = True
+        PROJECTION = "full"
+        FIELDS = "items/contentLanguage,nextPageToken"
+        EXPECTED = {
+            "maxResults": 10,
+            "pageToken": PAGE_TOKEN,
+            "prefix": PREFIX,
+            "delimiter": DELIMITER,
+            "versions": VERSIONS,
+            "projection": PROJECTION,
+            "fields": FIELDS,
+            "userProject": USER_PROJECT,
+        }
+
+        credentials = _make_credentials()
+        client = self._make_one(project=USER_PROJECT, credentials=credentials)
+        connection = _make_connection({"items": []})
+
+        with mock.patch(
+            'google.cloud.storage.client.Client._connection',
+            new_callable=mock.PropertyMock
+        ) as client_mock:
+            client_mock.return_value = connection
+
+            bucket = Bucket(client, BUCKET_NAME, user_project=USER_PROJECT)
+            iterator = client.list_blobs(
+                bucket_or_name=bucket,
+                max_results=MAX_RESULTS,
+                page_token=PAGE_TOKEN,
+                prefix=PREFIX,
+                delimiter=DELIMITER,
+                versions=VERSIONS,
+                projection=PROJECTION,
+                fields=FIELDS,
+            )
+            blobs = list(iterator)
+
+            self.assertEqual(blobs, [])
+            connection.api_request.assert_called_once_with(
+                method="GET",
+                path="/b/%s/o" % BUCKET_NAME,
+                query_params=EXPECTED
+            )
+
     def test_list_buckets_wo_project(self):
         CREDENTIALS = _make_credentials()
         client = self._make_one(project=None, credentials=CREDENTIALS)
