From 027766ac683614212320c97706fa159a8a8c6941eb04196fd39f44bc7156949f Mon Sep 17 00:00:00 2001 From: Matej Cepl Date: Wed, 9 Jun 2021 16:11:12 +0000 Subject: [PATCH] Revert OBS-URL: https://build.opensuse.org/package/show/devel:languages:python/python-google-cloud-storage?expand=0&rev=24 --- ...eal_client_in_bucket_blob_unit_tests.patch | 13029 ---------------- no-network.patch | 604 + python-google-cloud-storage.changes | 8 - python-google-cloud-storage.spec | 2 +- 4 files changed, 605 insertions(+), 13038 deletions(-) delete mode 100644 416-avoid_real_client_in_bucket_blob_unit_tests.patch create mode 100644 no-network.patch diff --git a/416-avoid_real_client_in_bucket_blob_unit_tests.patch b/416-avoid_real_client_in_bucket_blob_unit_tests.patch deleted file mode 100644 index f79eec8..0000000 --- a/416-avoid_real_client_in_bucket_blob_unit_tests.patch +++ /dev/null @@ -1,13029 +0,0 @@ ---- - .github/.OwlBot.lock.yaml | 3 - .github/.OwlBot.yaml | 19 - .github/header-checker-lint.yml | 15 - .kokoro/samples/python3.6/periodic-head.cfg | 11 - .kokoro/samples/python3.7/periodic-head.cfg | 11 - .kokoro/samples/python3.8/periodic-head.cfg | 11 - .kokoro/test-samples-against-head.sh | 28 - .kokoro/test-samples-impl.sh | 102 - LICENSE | 7 - MANIFEST.in | 4 - SECURITY.md | 7 - google/cloud/storage/_helpers.py | 23 - google/cloud/storage/acl.py | 41 - google/cloud/storage/blob.py | 77 - google/cloud/storage/bucket.py | 177 - - google/cloud/storage/client.py | 446 +++ - google/cloud/storage/hmac_key.py | 33 - google/cloud/storage/notification.py | 49 - google/cloud/storage/retry.py | 10 - setup.py | 13 - tests/unit/test__helpers.py | 503 ++-- - tests/unit/test_acl.py | 510 ++-- - tests/unit/test_blob.py | 2474 ++++++++++---------- - tests/unit/test_bucket.py | 3299 +++++++++++++++------------- - tests/unit/test_client.py | 1932 +++++++++------- - tests/unit/test_hmac_key.py | 227 - - tests/unit/test_notification.py | 251 +- - tests/unit/test_retry.py | 21 - 28 files changed, 5785 insertions(+), 4519 deletions(-) - ---- /dev/null -+++ b/.github/.OwlBot.lock.yaml -@@ -0,0 +1,3 @@ -+docker: -+ image: gcr.io/repo-automation-bots/owlbot-python:latest -+ digest: sha256:c66ba3c8d7bc8566f47df841f98cd0097b28fff0b1864c86f5817f4c8c3e8600 ---- /dev/null -+++ b/.github/.OwlBot.yaml -@@ -0,0 +1,19 @@ -+# Copyright 2021 Google LLC -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# http://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+ -+docker: -+ image: gcr.io/repo-automation-bots/owlbot-python:latest -+ -+begin-after-commit-hash: 6acf4a0a797f1082027985c55c4b14b60f673dd7 -+ ---- /dev/null -+++ b/.github/header-checker-lint.yml -@@ -0,0 +1,15 @@ -+{"allowedCopyrightHolders": ["Google LLC"], -+ "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"], -+ "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt", "**/__init__.py", "samples/**/constraints.txt", "samples/**/constraints-test.txt"], -+ "sourceFileExtensions": [ -+ "ts", -+ "js", -+ "java", -+ "sh", -+ "Dockerfile", -+ "yaml", -+ "py", -+ "html", -+ "txt" -+ ] -+} -\ No newline at end of file ---- /dev/null -+++ b/.kokoro/samples/python3.6/periodic-head.cfg -@@ -0,0 +1,11 @@ -+# Format: //devtools/kokoro/config/proto/build.proto -+ -+env_vars: { -+ key: "INSTALL_LIBRARY_FROM_SOURCE" -+ value: "True" -+} -+ -+env_vars: { -+ key: "TRAMPOLINE_BUILD_FILE" -+ value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" -+} ---- /dev/null -+++ b/.kokoro/samples/python3.7/periodic-head.cfg -@@ -0,0 +1,11 @@ -+# Format: //devtools/kokoro/config/proto/build.proto -+ -+env_vars: { -+ key: "INSTALL_LIBRARY_FROM_SOURCE" -+ value: "True" -+} -+ -+env_vars: { -+ key: "TRAMPOLINE_BUILD_FILE" -+ value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" -+} ---- /dev/null -+++ b/.kokoro/samples/python3.8/periodic-head.cfg -@@ -0,0 +1,11 @@ -+# Format: //devtools/kokoro/config/proto/build.proto -+ -+env_vars: { -+ key: "INSTALL_LIBRARY_FROM_SOURCE" -+ value: "True" -+} -+ -+env_vars: { -+ key: "TRAMPOLINE_BUILD_FILE" -+ value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" -+} ---- /dev/null -+++ b/.kokoro/test-samples-against-head.sh -@@ -0,0 +1,28 @@ -+#!/bin/bash -+# Copyright 2020 Google LLC -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# https://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+ -+# A customized test runner for samples. -+# -+# For periodic builds, you can specify this file for testing against head. -+ -+# `-e` enables the script to automatically fail when a command fails -+# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -+set -eo pipefail -+# Enables `**` to include files nested inside sub-folders -+shopt -s globstar -+ -+cd github/python-storage -+ -+exec .kokoro/test-samples-impl.sh ---- /dev/null -+++ b/.kokoro/test-samples-impl.sh -@@ -0,0 +1,102 @@ -+#!/bin/bash -+# Copyright 2021 Google LLC -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# https://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+ -+ -+# `-e` enables the script to automatically fail when a command fails -+# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -+set -eo pipefail -+# Enables `**` to include files nested inside sub-folders -+shopt -s globstar -+ -+# Exit early if samples directory doesn't exist -+if [ ! -d "./samples" ]; then -+ echo "No tests run. `./samples` not found" -+ exit 0 -+fi -+ -+# Disable buffering, so that the logs stream through. -+export PYTHONUNBUFFERED=1 -+ -+# Debug: show build environment -+env | grep KOKORO -+ -+# Install nox -+python3.6 -m pip install --upgrade --quiet nox -+ -+# Use secrets acessor service account to get secrets -+if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then -+ gcloud auth activate-service-account \ -+ --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ -+ --project="cloud-devrel-kokoro-resources" -+fi -+ -+# This script will create 3 files: -+# - testing/test-env.sh -+# - testing/service-account.json -+# - testing/client-secrets.json -+./scripts/decrypt-secrets.sh -+ -+source ./testing/test-env.sh -+export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json -+ -+# For cloud-run session, we activate the service account for gcloud sdk. -+gcloud auth activate-service-account \ -+ --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" -+ -+export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json -+ -+echo -e "\n******************** TESTING PROJECTS ********************" -+ -+# Switch to 'fail at end' to allow all tests to complete before exiting. -+set +e -+# Use RTN to return a non-zero value if the test fails. -+RTN=0 -+ROOT=$(pwd) -+# Find all requirements.txt in the samples directory (may break on whitespace). -+for file in samples/**/requirements.txt; do -+ cd "$ROOT" -+ # Navigate to the project folder. -+ file=$(dirname "$file") -+ cd "$file" -+ -+ echo "------------------------------------------------------------" -+ echo "- testing $file" -+ echo "------------------------------------------------------------" -+ -+ # Use nox to execute the tests for the project. -+ python3.6 -m nox -s "$RUN_TESTS_SESSION" -+ EXIT=$? -+ -+ # If this is a periodic build, send the test log to the FlakyBot. -+ # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. -+ if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then -+ chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot -+ $KOKORO_GFILE_DIR/linux_amd64/flakybot -+ fi -+ -+ if [[ $EXIT -ne 0 ]]; then -+ RTN=1 -+ echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" -+ else -+ echo -e "\n Testing completed.\n" -+ fi -+ -+done -+cd "$ROOT" -+ -+# Workaround for Kokoro permissions issue: delete secrets -+rm testing/{test-env.sh,client-secrets.json,service-account.json} -+ -+exit "$RTN" ---- a/LICENSE -+++ b/LICENSE -@@ -1,6 +1,7 @@ -- Apache License -+ -+ Apache License - Version 2.0, January 2004 -- https://www.apache.org/licenses/ -+ http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -@@ -192,7 +193,7 @@ - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - -- https://www.apache.org/licenses/LICENSE-2.0 -+ http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, ---- a/MANIFEST.in -+++ b/MANIFEST.in -@@ -16,10 +16,10 @@ - - # Generated by synthtool. DO NOT EDIT! - include README.rst LICENSE --recursive-include google *.json *.proto -+recursive-include google *.json *.proto py.typed - recursive-include tests * - global-exclude *.py[co] - global-exclude __pycache__ - - # Exclude scripts for samples readmegen --prune scripts/readme-gen -\ No newline at end of file -+prune scripts/readme-gen ---- /dev/null -+++ b/SECURITY.md -@@ -0,0 +1,7 @@ -+# Security Policy -+ -+To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). -+ -+The Google Security Team will respond within 5 working days of your report on g.co/vulnz. -+ -+We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. ---- a/google/cloud/storage/_helpers.py -+++ b/google/cloud/storage/_helpers.py -@@ -215,14 +215,13 @@ class _PropertyMixin(object): - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) -- api_response = client._connection.api_request( -- method="GET", -- path=self.path, -+ api_response = client._get_resource( -+ self.path, - query_params=query_params, - headers=self._encryption_headers(), -- _target_object=self, - timeout=timeout, - retry=retry, -+ _target_object=self, - ) - self._set_properties(api_response) - -@@ -332,10 +331,9 @@ class _PropertyMixin(object): - update_properties = {key: self._properties[key] for key in self._changes} - - # Make the API call. -- api_response = client._connection.api_request( -- method="PATCH", -- path=self.path, -- data=update_properties, -+ api_response = client._patch_resource( -+ self.path, -+ update_properties, - query_params=query_params, - _target_object=self, - timeout=timeout, -@@ -418,14 +416,13 @@ class _PropertyMixin(object): - if_metageneration_not_match=if_metageneration_not_match, - ) - -- api_response = client._connection.api_request( -- method="PUT", -- path=self.path, -- data=self._properties, -+ api_response = client._put_resource( -+ self.path, -+ self._properties, - query_params=query_params, -- _target_object=self, - timeout=timeout, - retry=retry, -+ _target_object=self, - ) - self._set_properties(api_response) - ---- a/google/cloud/storage/acl.py -+++ b/google/cloud/storage/acl.py -@@ -85,6 +85,7 @@ when sending metadata for ACLs to the AP - """ - - from google.cloud.storage.constants import _DEFAULT_TIMEOUT -+from google.cloud.storage.retry import DEFAULT_RETRY - - - class _ACLEntity(object): -@@ -206,6 +207,7 @@ class ACL(object): - - # Subclasses must override to provide these attributes (typically, - # as properties). -+ client = None - reload_path = None - save_path = None - user_project = None -@@ -430,7 +432,7 @@ class ACL(object): - client = self.client - return client - -- def reload(self, client=None, timeout=_DEFAULT_TIMEOUT): -+ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): - """Reload the ACL data from Cloud Storage. - - If :attr:`user_project` is set, bills the API request to that project. -@@ -445,6 +447,15 @@ class ACL(object): - - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. -+ -+ :type retry: :class:`~google.api_core.retry.Retry` -+ :param retry: (Optional) How to retry the RPC. -+ -+ A None value will disable retries. -+ -+ A google.api_core.retry.Retry value will enable retries, -+ and the object will define retriable response codes and errors -+ and configure backoff and timeout options. - """ - path = self.reload_path - client = self._require_client(client) -@@ -455,10 +466,11 @@ class ACL(object): - - self.entities.clear() - -- found = client._connection.api_request( -- method="GET", path=path, query_params=query_params, timeout=timeout, -+ found = client._get_resource( -+ path, query_params=query_params, timeout=timeout, retry=retry, - ) - self.loaded = True -+ - for entry in found.get("items", ()): - self.add_entity(self.entity_from_dict(entry)) - -@@ -483,8 +495,19 @@ class ACL(object): - - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. -+ -+ :type retry: :class:`~google.api_core.retry.Retry` -+ :param retry: (Optional) How to retry the RPC. -+ -+ A None value will disable retries. -+ -+ A google.api_core.retry.Retry value will enable retries, -+ and the object will define retriable response codes and errors -+ and configure backoff and timeout options. - """ -+ client = self._require_client(client) - query_params = {"projection": "full"} -+ - if predefined is not None: - acl = [] - query_params[self._PREDEFINED_QUERY_PARAM] = predefined -@@ -493,18 +516,20 @@ class ACL(object): - query_params["userProject"] = self.user_project - - path = self.save_path -- client = self._require_client(client) - -- result = client._connection.api_request( -- method="PATCH", -- path=path, -- data={self._URL_PATH_ELEM: list(acl)}, -+ result = client._patch_resource( -+ path, -+ {self._URL_PATH_ELEM: list(acl)}, - query_params=query_params, - timeout=timeout, -+ retry=None, - ) -+ - self.entities.clear() -+ - for entry in result.get(self._URL_PATH_ELEM, ()): - self.add_entity(self.entity_from_dict(entry)) -+ - self.loaded = True - - def save(self, acl=None, client=None, timeout=_DEFAULT_TIMEOUT): ---- a/google/cloud/storage/blob.py -+++ b/google/cloud/storage/blob.py -@@ -704,20 +704,19 @@ class Blob(_PropertyMixin): - try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. -- client._connection.api_request( -- method="GET", -- path=self.path, -+ client._get_resource( -+ self.path, - query_params=query_params, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) -+ except NotFound: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. -- return True -- except NotFound: - return False -+ return True - - def delete( - self, -@@ -2829,13 +2828,12 @@ class Blob(_PropertyMixin): - if requested_policy_version is not None: - query_params["optionsRequestedPolicyVersion"] = requested_policy_version - -- info = client._connection.api_request( -- method="GET", -- path="%s/iam" % (self.path,), -+ info = client._get_resource( -+ "%s/iam" % (self.path,), - query_params=query_params, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - return Policy.from_api_repr(info) - -@@ -2900,16 +2898,16 @@ class Blob(_PropertyMixin): - if self.user_project is not None: - query_params["userProject"] = self.user_project - -+ path = "{}/iam".format(self.path) - resource = policy.to_api_repr() - resource["resourceId"] = self.path -- info = client._connection.api_request( -- method="PUT", -- path="%s/iam" % (self.path,), -+ info = client._put_resource( -+ path, -+ resource, - query_params=query_params, -- data=resource, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - return Policy.from_api_repr(info) - -@@ -2970,37 +2968,53 @@ class Blob(_PropertyMixin): - query_params["userProject"] = self.user_project - - path = "%s/iam/testPermissions" % (self.path,) -- resp = client._connection.api_request( -- method="GET", -- path=path, -+ resp = client._get_resource( -+ path, - query_params=query_params, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - - return resp.get("permissions", []) - -- def make_public(self, client=None): -+ def make_public(self, client=None, timeout=_DEFAULT_TIMEOUT): - """Update blob's ACL, granting read access to anonymous users. - - :type client: :class:`~google.cloud.storage.client.Client` or - ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back - to the ``client`` stored on the blob's bucket. -+ -+ :type timeout: float or tuple -+ :param timeout: (Optional) The amount of time, in seconds, to wait -+ for the server response. The timeout applies to each underlying -+ request. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. - """ - self.acl.all().grant_read() -- self.acl.save(client=client) -+ self.acl.save(client=client, timeout=timeout) - -- def make_private(self, client=None): -+ def make_private(self, client=None, timeout=_DEFAULT_TIMEOUT): - """Update blob's ACL, revoking read access for anonymous users. - - :type client: :class:`~google.cloud.storage.client.Client` or - ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back - to the ``client`` stored on the blob's bucket. -+ -+ :type timeout: float or tuple -+ :param timeout: (Optional) The amount of time, in seconds, to wait -+ for the server response. The timeout applies to each underlying -+ request. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. - """ - self.acl.all().revoke_read() -- self.acl.save(client=client) -+ self.acl.save(client=client, timeout=timeout) - - def compose( - self, -@@ -3118,14 +3132,13 @@ class Blob(_PropertyMixin): - "sourceObjects": source_objects, - "destination": self._properties.copy(), - } -- api_response = client._connection.api_request( -- method="POST", -- path=self.path + "/compose", -+ api_response = client._post_resource( -+ "{}/compose".format(self.path), -+ request, - query_params=query_params, -- data=request, -- _target_object=self, - timeout=timeout, - retry=retry, -+ _target_object=self, - ) - self._set_properties(api_response) - -@@ -3269,15 +3282,15 @@ class Blob(_PropertyMixin): - if_source_metageneration_not_match=if_source_metageneration_not_match, - ) - -- api_response = client._connection.api_request( -- method="POST", -- path=source.path + "/rewriteTo" + self.path, -+ path = "{}/rewriteTo{}".format(source.path, self.path) -+ api_response = client._post_resource( -+ path, -+ self._properties, - query_params=query_params, -- data=self._properties, - headers=headers, -- _target_object=self, - timeout=timeout, - retry=retry, -+ _target_object=self, - ) - rewritten = int(api_response["totalBytesRewritten"]) - size = int(api_response["objectSize"]) ---- a/google/cloud/storage/bucket.py -+++ b/google/cloud/storage/bucket.py -@@ -17,14 +17,12 @@ - import base64 - import copy - import datetime --import functools - import json - import warnings - - import six - from six.moves.urllib.parse import urlsplit - --from google.api_core import page_iterator - from google.api_core import datetime_helpers - from google.cloud._helpers import _datetime_to_rfc3339 - from google.cloud._helpers import _NOW -@@ -786,20 +784,19 @@ class Bucket(_PropertyMixin): - try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. -- client._connection.api_request( -- method="GET", -- path=self.path, -+ client._get_resource( -+ self.path, - query_params=query_params, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) -+ except NotFound: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. -- return True -- except NotFound: - return False -+ return True - - def create( - self, -@@ -1066,9 +1063,9 @@ class Bucket(_PropertyMixin): - # Call the superclass method. - super(Bucket, self).patch( - client=client, -- timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, -+ timeout=timeout, - retry=retry, - ) - -@@ -1393,14 +1390,8 @@ class Bucket(_PropertyMixin): - """ - client = self._require_client(client) - path = self.path + "/notificationConfigs" -- api_request = functools.partial( -- client._connection.api_request, timeout=timeout, retry=retry -- ) -- iterator = page_iterator.HTTPIterator( -- client=client, -- api_request=api_request, -- path=path, -- item_to_value=_item_to_notification, -+ iterator = client._list_resource( -+ path, _item_to_notification, timeout=timeout, retry=retry, - ) - iterator.bucket = self - return iterator -@@ -1467,9 +1458,9 @@ class Bucket(_PropertyMixin): - self, - force=False, - client=None, -- timeout=_DEFAULT_TIMEOUT, - if_metageneration_match=None, - if_metageneration_not_match=None, -+ timeout=_DEFAULT_TIMEOUT, - retry=DEFAULT_RETRY, - ): - """Delete this bucket. -@@ -1497,13 +1488,6 @@ class Bucket(_PropertyMixin): - :param client: (Optional) The client to use. If not passed, falls back - to the ``client`` stored on the current bucket. - -- :type timeout: float or tuple -- :param timeout: (Optional) The amount of time, in seconds, to wait -- for the server response on each request. -- -- Can also be passed as a tuple (connect_timeout, read_timeout). -- See :meth:`requests.Session.request` documentation for details. -- - :type if_metageneration_match: long - :param if_metageneration_match: (Optional) Make the operation conditional on whether the - blob's current metageneration matches the given value. -@@ -1512,6 +1496,13 @@ class Bucket(_PropertyMixin): - :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the - blob's current metageneration does not match the given value. - -+ :type timeout: float or tuple -+ :param timeout: (Optional) The amount of time, in seconds, to wait -+ for the server response on each request. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ - :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy - :param retry: (Optional) How to retry the RPC. A None value will disable retries. - A google.api_core.retry.Retry value will enable retries, and the object will -@@ -1546,6 +1537,7 @@ class Bucket(_PropertyMixin): - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, - client=client, - timeout=timeout, -+ retry=retry, - ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: -@@ -1559,19 +1551,22 @@ class Bucket(_PropertyMixin): - - # Ignore 404 errors on delete. - self.delete_blobs( -- blobs, on_error=lambda blob: None, client=client, timeout=timeout -+ blobs, -+ on_error=lambda blob: None, -+ client=client, -+ timeout=timeout, -+ retry=retry, - ) - - # We intentionally pass `_target_object=None` since a DELETE - # request has no response value (whether in a standard request or - # in a batch request). -- client._connection.api_request( -- method="DELETE", -- path=self.path, -+ client._delete_resource( -+ self.path, - query_params=query_params, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - - def delete_blob( -@@ -1678,13 +1673,12 @@ class Bucket(_PropertyMixin): - # We intentionally pass `_target_object=None` since a DELETE - # request has no response value (whether in a standard request or - # in a batch request). -- client._connection.api_request( -- method="DELETE", -- path=blob.path, -+ client._delete_resource( -+ blob.path, - query_params=query_params, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - - def delete_blobs( -@@ -1803,11 +1797,11 @@ class Bucket(_PropertyMixin): - self.delete_blob( - blob_name, - client=client, -- timeout=timeout, - if_generation_match=next(if_generation_match, None), - if_generation_not_match=next(if_generation_not_match, None), - if_metageneration_match=next(if_metageneration_match, None), - if_metageneration_not_match=next(if_metageneration_not_match, None), -+ timeout=timeout, - retry=retry, - ) - except NotFound: -@@ -1984,13 +1978,13 @@ class Bucket(_PropertyMixin): - - new_blob = Blob(bucket=destination_bucket, name=new_name) - api_path = blob.path + "/copyTo" + new_blob.path -- copy_result = client._connection.api_request( -- method="POST", -- path=api_path, -+ copy_result = client._post_resource( -+ api_path, -+ None, - query_params=query_params, -- _target_object=new_blob, - timeout=timeout, - retry=retry, -+ _target_object=new_blob, - ) - - if not preserve_acl: -@@ -2004,7 +1998,6 @@ class Bucket(_PropertyMixin): - blob, - new_name, - client=None, -- timeout=_DEFAULT_TIMEOUT, - if_generation_match=None, - if_generation_not_match=None, - if_metageneration_match=None, -@@ -2013,6 +2006,7 @@ class Bucket(_PropertyMixin): - if_source_generation_not_match=None, - if_source_metageneration_match=None, - if_source_metageneration_not_match=None, -+ timeout=_DEFAULT_TIMEOUT, - retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - ): - """Rename the given blob using copy and delete operations. -@@ -2042,14 +2036,6 @@ class Bucket(_PropertyMixin): - :param client: (Optional) The client to use. If not passed, falls back - to the ``client`` stored on the current bucket. - -- :type timeout: float or tuple -- :param timeout: (Optional) The amount of time, in seconds, to wait -- for the server response. The timeout applies to each individual -- request. -- -- Can also be passed as a tuple (connect_timeout, read_timeout). -- See :meth:`requests.Session.request` documentation for details. -- - :type if_generation_match: long - :param if_generation_match: (Optional) Makes the operation - conditional on whether the destination -@@ -2111,6 +2097,14 @@ class Bucket(_PropertyMixin): - does not match the given value. - Also used in the delete request. - -+ :type timeout: float or tuple -+ :param timeout: (Optional) The amount of time, in seconds, to wait -+ for the server response. The timeout applies to each individual -+ request. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ - :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy - :param retry: (Optional) How to retry the RPC. A None value will disable retries. - A google.api_core.retry.Retry value will enable retries, and the object will -@@ -2361,7 +2355,13 @@ class Bucket(_PropertyMixin): - elif action_type == "SetStorageClass": - yield LifecycleRuleSetStorageClass.from_api_repr(rule) - else: -- raise ValueError("Unknown lifecycle rule: {}".format(rule)) -+ warnings.warn( -+ "Unknown lifecycle rule type received: {}. Please upgrade to the latest version of google-cloud-storage.".format( -+ rule -+ ), -+ UserWarning, -+ stacklevel=1, -+ ) - - @lifecycle_rules.setter - def lifecycle_rules(self, rules): -@@ -2876,13 +2876,12 @@ class Bucket(_PropertyMixin): - if requested_policy_version is not None: - query_params["optionsRequestedPolicyVersion"] = requested_policy_version - -- info = client._connection.api_request( -- method="GET", -- path="%s/iam" % (self.path,), -+ info = client._get_resource( -+ "%s/iam" % (self.path,), - query_params=query_params, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - return Policy.from_api_repr(info) - -@@ -2939,17 +2938,19 @@ class Bucket(_PropertyMixin): - if self.user_project is not None: - query_params["userProject"] = self.user_project - -+ path = "{}/iam".format(self.path) - resource = policy.to_api_repr() - resource["resourceId"] = self.path -- info = client._connection.api_request( -- method="PUT", -- path="%s/iam" % (self.path,), -+ -+ info = client._put_resource( -+ path, -+ resource, - query_params=query_params, -- data=resource, -- _target_object=None, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) -+ - return Policy.from_api_repr(info) - - def test_iam_permissions( -@@ -3002,22 +3003,17 @@ class Bucket(_PropertyMixin): - query_params["userProject"] = self.user_project - - path = "%s/iam/testPermissions" % (self.path,) -- resp = client._connection.api_request( -- method="GET", -- path=path, -+ resp = client._get_resource( -+ path, - query_params=query_params, - timeout=timeout, - retry=retry, -+ _target_object=None, - ) - return resp.get("permissions", []) - - def make_public( -- self, -- recursive=False, -- future=False, -- client=None, -- timeout=_DEFAULT_TIMEOUT, -- retry=DEFAULT_RETRY, -+ self, recursive=False, future=False, client=None, timeout=_DEFAULT_TIMEOUT, - ): - """Update bucket's ACL, granting read access to anonymous users. - -@@ -3041,20 +3037,6 @@ class Bucket(_PropertyMixin): - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. - -- :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy -- :param retry: (Optional) How to retry the RPC. A None value will disable retries. -- A google.api_core.retry.Retry value will enable retries, and the object will -- define retriable response codes and errors and configure backoff and timeout options. -- -- A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -- activates it only if certain conditions are met. This class exists to provide safe defaults -- for RPC calls that are not technically safe to retry normally (due to potential data -- duplication or other side-effects) but become safe to retry if a condition such as -- if_metageneration_match is set. -- -- See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -- information on retry types and how to configure them. -- - :raises ValueError: - If ``recursive`` is True, and the bucket contains more than 256 - blobs. This is to prevent extremely long runtime of this -@@ -3080,7 +3062,6 @@ class Bucket(_PropertyMixin): - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, - client=client, - timeout=timeout, -- retry=retry, - ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: -@@ -3098,12 +3079,7 @@ class Bucket(_PropertyMixin): - blob.acl.save(client=client, timeout=timeout) - - def make_private( -- self, -- recursive=False, -- future=False, -- client=None, -- timeout=_DEFAULT_TIMEOUT, -- retry=DEFAULT_RETRY, -+ self, recursive=False, future=False, client=None, timeout=_DEFAULT_TIMEOUT, - ): - """Update bucket's ACL, revoking read access for anonymous users. - -@@ -3128,20 +3104,6 @@ class Bucket(_PropertyMixin): - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. - -- :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy -- :param retry: (Optional) How to retry the RPC. A None value will disable retries. -- A google.api_core.retry.Retry value will enable retries, and the object will -- define retriable response codes and errors and configure backoff and timeout options. -- -- A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -- activates it only if certain conditions are met. This class exists to provide safe defaults -- for RPC calls that are not technically safe to retry normally (due to potential data -- duplication or other side-effects) but become safe to retry if a condition such as -- if_metageneration_match is set. -- -- See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -- information on retry types and how to configure them. -- - :raises ValueError: - If ``recursive`` is True, and the bucket contains more than 256 - blobs. This is to prevent extremely long runtime of this -@@ -3167,7 +3129,6 @@ class Bucket(_PropertyMixin): - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, - client=client, - timeout=timeout, -- retry=retry, - ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: -@@ -3220,7 +3181,7 @@ class Bucket(_PropertyMixin): - to attach the signature. - """ - client = self._require_client(client) -- credentials = client._base_connection.credentials -+ credentials = client._credentials - _signing.ensure_signed_credentials(credentials) - - if expiration is None: -@@ -3302,13 +3263,13 @@ class Bucket(_PropertyMixin): - query_params["userProject"] = self.user_project - - path = "/b/{}/lockRetentionPolicy".format(self.name) -- api_response = client._connection.api_request( -- method="POST", -- path=path, -+ api_response = client._post_resource( -+ path, -+ None, - query_params=query_params, -- _target_object=self, - timeout=timeout, - retry=retry, -+ _target_object=self, - ) - self._set_properties(api_response) - ---- a/google/cloud/storage/client.py -+++ b/google/cloud/storage/client.py -@@ -284,10 +284,9 @@ class Client(ClientWithProject): - """ - if project is None: - project = self.project -+ - path = "/projects/%s/serviceAccount" % (project,) -- api_response = self._base_connection.api_request( -- method="GET", path=path, timeout=timeout, retry=retry, -- ) -+ api_response = self._get_resource(path, timeout=timeout, retry=retry) - return api_response["email_address"] - - def bucket(self, bucket_name, user_project=None): -@@ -321,6 +320,376 @@ class Client(ClientWithProject): - """ - return Batch(client=self) - -+ def _get_resource( -+ self, -+ path, -+ query_params=None, -+ headers=None, -+ timeout=_DEFAULT_TIMEOUT, -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ): -+ """Helper for bucket / blob methods making API 'GET' calls. -+ -+ Args: -+ path str: -+ The path of the resource to fetch. -+ -+ query_params Optional[dict]: -+ HTTP query parameters to be passed -+ -+ headers Optional[dict]: -+ HTTP headers to be passed -+ -+ timeout (Optional[Union[float, Tuple[float, float]]]): -+ The amount of time, in seconds, to wait for the server response. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ -+ retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): -+ How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. -+ -+ _target_object (Union[ \ -+ :class:`~google.cloud.storage.bucket.Bucket`, \ -+ :class:`~google.cloud.storage.bucket.blob`, \ -+ ]): -+ Object to which future data is to be applied -- only relevant -+ in the context of a batch. -+ -+ Returns: -+ dict -+ The JSON resource fetched -+ -+ Raises: -+ google.cloud.exceptions.NotFound -+ If the bucket is not found. -+ """ -+ return self._connection.api_request( -+ method="GET", -+ path=path, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=_target_object, -+ ) -+ -+ def _list_resource( -+ self, -+ path, -+ item_to_value, -+ page_token=None, -+ max_results=None, -+ extra_params=None, -+ page_start=page_iterator._do_nothing_page_start, -+ timeout=_DEFAULT_TIMEOUT, -+ retry=DEFAULT_RETRY, -+ ): -+ api_request = functools.partial( -+ self._connection.api_request, timeout=timeout, retry=retry -+ ) -+ return page_iterator.HTTPIterator( -+ client=self, -+ api_request=api_request, -+ path=path, -+ item_to_value=item_to_value, -+ page_token=page_token, -+ max_results=max_results, -+ extra_params=extra_params, -+ page_start=page_start, -+ ) -+ -+ def _patch_resource( -+ self, -+ path, -+ data, -+ query_params=None, -+ headers=None, -+ timeout=_DEFAULT_TIMEOUT, -+ retry=None, -+ _target_object=None, -+ ): -+ """Helper for bucket / blob methods making API 'PATCH' calls. -+ -+ Args: -+ path str: -+ The path of the resource to fetch. -+ -+ data dict: -+ The data to be patched. -+ -+ query_params Optional[dict]: -+ HTTP query parameters to be passed -+ -+ headers Optional[dict]: -+ HTTP headers to be passed -+ -+ timeout (Optional[Union[float, Tuple[float, float]]]): -+ The amount of time, in seconds, to wait for the server response. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ -+ retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): -+ How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. -+ -+ _target_object (Union[ \ -+ :class:`~google.cloud.storage.bucket.Bucket`, \ -+ :class:`~google.cloud.storage.bucket.blob`, \ -+ ]): -+ Object to which future data is to be applied -- only relevant -+ in the context of a batch. -+ -+ Returns: -+ dict -+ The JSON resource fetched -+ -+ Raises: -+ google.cloud.exceptions.NotFound -+ If the bucket is not found. -+ """ -+ return self._connection.api_request( -+ method="PATCH", -+ path=path, -+ data=data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=_target_object, -+ ) -+ -+ def _put_resource( -+ self, -+ path, -+ data, -+ query_params=None, -+ headers=None, -+ timeout=_DEFAULT_TIMEOUT, -+ retry=None, -+ _target_object=None, -+ ): -+ """Helper for bucket / blob methods making API 'PUT' calls. -+ -+ Args: -+ path str: -+ The path of the resource to fetch. -+ -+ data dict: -+ The data to be patched. -+ -+ query_params Optional[dict]: -+ HTTP query parameters to be passed -+ -+ headers Optional[dict]: -+ HTTP headers to be passed -+ -+ timeout (Optional[Union[float, Tuple[float, float]]]): -+ The amount of time, in seconds, to wait for the server response. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ -+ retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): -+ How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. -+ -+ _target_object (Union[ \ -+ :class:`~google.cloud.storage.bucket.Bucket`, \ -+ :class:`~google.cloud.storage.bucket.blob`, \ -+ ]): -+ Object to which future data is to be applied -- only relevant -+ in the context of a batch. -+ -+ Returns: -+ dict -+ The JSON resource fetched -+ -+ Raises: -+ google.cloud.exceptions.NotFound -+ If the bucket is not found. -+ """ -+ return self._connection.api_request( -+ method="PUT", -+ path=path, -+ data=data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=_target_object, -+ ) -+ -+ def _post_resource( -+ self, -+ path, -+ data, -+ query_params=None, -+ headers=None, -+ timeout=_DEFAULT_TIMEOUT, -+ retry=None, -+ _target_object=None, -+ ): -+ """Helper for bucket / blob methods making API 'POST' calls. -+ -+ Args: -+ path str: -+ The path of the resource to which to post. -+ -+ data dict: -+ The data to be posted. -+ -+ query_params Optional[dict]: -+ HTTP query parameters to be passed -+ -+ headers Optional[dict]: -+ HTTP headers to be passed -+ -+ timeout (Optional[Union[float, Tuple[float, float]]]): -+ The amount of time, in seconds, to wait for the server response. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ -+ retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): -+ How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. -+ -+ _target_object (Union[ \ -+ :class:`~google.cloud.storage.bucket.Bucket`, \ -+ :class:`~google.cloud.storage.bucket.blob`, \ -+ ]): -+ Object to which future data is to be applied -- only relevant -+ in the context of a batch. -+ -+ Returns: -+ dict -+ The JSON resource returned from the post. -+ -+ Raises: -+ google.cloud.exceptions.NotFound -+ If the bucket is not found. -+ """ -+ return self._connection.api_request( -+ method="POST", -+ path=path, -+ data=data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=_target_object, -+ ) -+ -+ def _delete_resource( -+ self, -+ path, -+ query_params=None, -+ headers=None, -+ timeout=_DEFAULT_TIMEOUT, -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ): -+ """Helper for bucket / blob methods making API 'DELETE' calls. -+ -+ Args: -+ path str: -+ The path of the resource to delete. -+ -+ query_params Optional[dict]: -+ HTTP query parameters to be passed -+ -+ headers Optional[dict]: -+ HTTP headers to be passed -+ -+ timeout (Optional[Union[float, Tuple[float, float]]]): -+ The amount of time, in seconds, to wait for the server response. -+ -+ Can also be passed as a tuple (connect_timeout, read_timeout). -+ See :meth:`requests.Session.request` documentation for details. -+ -+ retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): -+ How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. -+ -+ _target_object (Union[ \ -+ :class:`~google.cloud.storage.bucket.Bucket`, \ -+ :class:`~google.cloud.storage.bucket.blob`, \ -+ ]): -+ Object to which future data is to be applied -- only relevant -+ in the context of a batch. -+ -+ Returns: -+ dict -+ The JSON resource fetched -+ -+ Raises: -+ google.cloud.exceptions.NotFound -+ If the bucket is not found. -+ """ -+ return self._connection.api_request( -+ method="DELETE", -+ path=path, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=_target_object, -+ ) -+ - def get_bucket( - self, - bucket_or_name, -@@ -602,14 +971,13 @@ class Client(ClientWithProject): - if location is not None: - properties["location"] = location - -- api_response = self._connection.api_request( -- method="POST", -- path="/b", -+ api_response = self._post_resource( -+ "/b", -+ properties, - query_params=query_params, -- data=properties, -- _target_object=bucket, - timeout=timeout, - retry=retry, -+ _target_object=bucket, - ) - - bucket._set_properties(api_response) -@@ -871,14 +1239,9 @@ class Client(ClientWithProject): - extra_params["userProject"] = bucket.user_project - - path = bucket.path + "/o" -- api_request = functools.partial( -- self._connection.api_request, timeout=timeout, retry=retry -- ) -- iterator = page_iterator.HTTPIterator( -- client=self, -- api_request=api_request, -- path=path, -- item_to_value=_item_to_blob, -+ iterator = self._list_resource( -+ path, -+ _item_to_blob, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, -@@ -985,18 +1348,14 @@ class Client(ClientWithProject): - if fields is not None: - extra_params["fields"] = fields - -- api_request = functools.partial( -- self._connection.api_request, retry=retry, timeout=timeout -- ) -- -- return page_iterator.HTTPIterator( -- client=self, -- api_request=api_request, -- path="/b", -- item_to_value=_item_to_bucket, -+ return self._list_resource( -+ "/b", -+ _item_to_bucket, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, -+ timeout=timeout, -+ retry=retry, - ) - - def create_hmac_key( -@@ -1005,6 +1364,7 @@ class Client(ClientWithProject): - project_id=None, - user_project=None, - timeout=_DEFAULT_TIMEOUT, -+ retry=None, - ): - """Create an HMAC key for a service account. - -@@ -1025,6 +1385,20 @@ class Client(ClientWithProject): - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. - -+ :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy -+ :param retry: (Optional) How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. -+ - :rtype: - Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] - :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. -@@ -1038,12 +1412,8 @@ class Client(ClientWithProject): - if user_project is not None: - qs_params["userProject"] = user_project - -- api_response = self._connection.api_request( -- method="POST", -- path=path, -- query_params=qs_params, -- timeout=timeout, -- retry=None, -+ api_response = self._post_resource( -+ path, None, query_params=qs_params, timeout=timeout, retry=retry, - ) - metadata = HMACKeyMetadata(self) - metadata._properties = api_response["metadata"] -@@ -1122,17 +1492,13 @@ class Client(ClientWithProject): - if user_project is not None: - extra_params["userProject"] = user_project - -- api_request = functools.partial( -- self._connection.api_request, timeout=timeout, retry=retry -- ) -- -- return page_iterator.HTTPIterator( -- client=self, -- api_request=api_request, -- path=path, -- item_to_value=_item_to_hmac_key_metadata, -+ return self._list_resource( -+ path, -+ _item_to_hmac_key_metadata, - max_results=max_results, - extra_params=extra_params, -+ timeout=timeout, -+ retry=retry, - ) - - def get_hmac_key_metadata( ---- a/google/cloud/storage/hmac_key.py -+++ b/google/cloud/storage/hmac_key.py -@@ -222,12 +222,8 @@ class HMACKeyMetadata(object): - if self.user_project is not None: - qs_params["userProject"] = self.user_project - -- self._client._connection.api_request( -- method="GET", -- path=self.path, -- query_params=qs_params, -- timeout=timeout, -- retry=retry, -+ self._client._get_resource( -+ self.path, query_params=qs_params, timeout=timeout, retry=retry, - ) - except NotFound: - return False -@@ -266,12 +262,8 @@ class HMACKeyMetadata(object): - if self.user_project is not None: - qs_params["userProject"] = self.user_project - -- self._properties = self._client._connection.api_request( -- method="GET", -- path=self.path, -- query_params=qs_params, -- timeout=timeout, -- retry=retry, -+ self._properties = self._client._get_resource( -+ self.path, query_params=qs_params, timeout=timeout, retry=retry, - ) - - def update(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY_IF_ETAG_IN_JSON): -@@ -306,13 +298,8 @@ class HMACKeyMetadata(object): - qs_params["userProject"] = self.user_project - - payload = {"state": self.state} -- self._properties = self._client._connection.api_request( -- method="PUT", -- path=self.path, -- data=payload, -- query_params=qs_params, -- timeout=timeout, -- retry=retry, -+ self._properties = self._client._put_resource( -+ self.path, payload, query_params=qs_params, timeout=timeout, retry=retry, - ) - - def delete(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): -@@ -349,10 +336,6 @@ class HMACKeyMetadata(object): - if self.user_project is not None: - qs_params["userProject"] = self.user_project - -- self._client._connection.api_request( -- method="DELETE", -- path=self.path, -- query_params=qs_params, -- timeout=timeout, -- retry=retry, -+ self._client._delete_resource( -+ self.path, query_params=qs_params, timeout=timeout, retry=retry, - ) ---- a/google/cloud/storage/notification.py -+++ b/google/cloud/storage/notification.py -@@ -233,7 +233,7 @@ class BucketNotification(object): - self._properties.clear() - self._properties.update(response) - -- def create(self, client=None, timeout=_DEFAULT_TIMEOUT): -+ def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): - """API wrapper: create the notification. - - See: -@@ -251,6 +251,20 @@ class BucketNotification(object): - - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. -+ -+ :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy -+ :param retry: (Optional) How to retry the RPC. A None value will disable retries. -+ A google.api_core.retry.Retry value will enable retries, and the object will -+ define retriable response codes and errors and configure backoff and timeout options. -+ -+ A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and -+ activates it only if certain conditions are met. This class exists to provide safe defaults -+ for RPC calls that are not technically safe to retry normally (due to potential data -+ duplication or other side-effects) but become safe to retry if a condition such as -+ if_metageneration_match is set. -+ -+ See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for -+ information on retry types and how to configure them. - """ - if self.notification_id is not None: - raise ValueError( -@@ -266,13 +280,8 @@ class BucketNotification(object): - path = "/b/{}/notificationConfigs".format(self.bucket.name) - properties = self._properties.copy() - properties["topic"] = _TOPIC_REF_FMT.format(self.topic_project, self.topic_name) -- self._properties = client._connection.api_request( -- method="POST", -- path=path, -- query_params=query_params, -- data=properties, -- timeout=timeout, -- retry=None, -+ self._properties = client._post_resource( -+ path, properties, query_params=query_params, timeout=timeout, retry=retry, - ) - - def exists(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): -@@ -323,12 +332,8 @@ class BucketNotification(object): - query_params["userProject"] = self.bucket.user_project - - try: -- client._connection.api_request( -- method="GET", -- path=self.path, -- query_params=query_params, -- timeout=timeout, -- retry=retry, -+ client._get_resource( -+ self.path, query_params=query_params, timeout=timeout, retry=retry, - ) - except NotFound: - return False -@@ -381,12 +386,8 @@ class BucketNotification(object): - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - -- response = client._connection.api_request( -- method="GET", -- path=self.path, -- query_params=query_params, -- timeout=timeout, -- retry=retry, -+ response = client._get_resource( -+ self.path, query_params=query_params, timeout=timeout, retry=retry, - ) - self._set_properties(response) - -@@ -437,12 +438,8 @@ class BucketNotification(object): - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - -- client._connection.api_request( -- method="DELETE", -- path=self.path, -- query_params=query_params, -- timeout=timeout, -- retry=retry, -+ client._delete_resource( -+ self.path, query_params=query_params, timeout=timeout, retry=retry, - ) - - ---- a/google/cloud/storage/retry.py -+++ b/google/cloud/storage/retry.py -@@ -21,7 +21,14 @@ from google.auth import exceptions as au - import json - - --_RETRYABLE_TYPES = ( -+# ConnectionError is a built-in exception only in Python3 and not in Python2. -+try: -+ _RETRYABLE_STDLIB_TYPES = (ConnectionError,) -+except NameError: -+ _RETRYABLE_STDLIB_TYPES = () -+ -+ -+_RETRYABLE_TYPES = _RETRYABLE_STDLIB_TYPES + ( - api_exceptions.TooManyRequests, # 429 - api_exceptions.InternalServerError, # 500 - api_exceptions.BadGateway, # 502 -@@ -30,6 +37,7 @@ _RETRYABLE_TYPES = ( - requests.ConnectionError, - ) - -+ - # Some retriable errors don't have their own custom exception in api_core. - _ADDITIONAL_RETRYABLE_STATUS_CODES = (408,) - ---- a/setup.py -+++ b/setup.py -@@ -17,16 +17,6 @@ import os - - import setuptools - --# Disable version normalization performed by setuptools.setup() --try: -- # Try the approach of using sic(), added in setuptools 46.1.0 -- from setuptools import sic --except ImportError: -- # Try the approach of replacing packaging.version.Version -- sic = lambda v: v -- import packaging -- packaging.version.Version = packaging.version.LegacyVersion -- - # Package metadata. - - name = "google-cloud-storage" -@@ -41,6 +31,7 @@ dependencies = [ - "google-cloud-core >= 1.4.1, < 2.0dev", - "google-resumable-media >= 1.2.0, < 2.0dev", - "requests >= 2.18.0, < 3.0.0dev", -+ "googleapis-common-protos < 1.53.0; python_version<'3.0'", - ] - extras = {} - -@@ -72,7 +63,7 @@ if "google.cloud" in packages: - - setuptools.setup( - name=name, -- version=sic(version), -+ version=version, - description=description, - long_description=readme, - author="Google LLC", ---- a/tests/unit/test__helpers.py -+++ b/tests/unit/test__helpers.py -@@ -66,6 +66,7 @@ class Test_PropertyMixin(unittest.TestCa - class Derived(self._get_target_class()): - - client = None -+ _actual_encryption_headers = None - - @property - def path(self): -@@ -75,6 +76,9 @@ class Test_PropertyMixin(unittest.TestCa - def user_project(self): - return user_project - -+ def _encryption_headers(self): -+ return self._actual_encryption_headers or {} -+ - return Derived - - def test_path_is_abstract(self): -@@ -105,119 +109,130 @@ class Test_PropertyMixin(unittest.TestCa - derived = self._derivedClass("/path", user_project)() - self.assertEqual(derived._query_params, {"userProject": user_project}) - -- def test_reload(self): -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ def test_reload_w_defaults(self): -+ path = "/path" -+ response = {"foo": "Foo"} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = response -+ derived = self._derivedClass(path)() - # Make sure changes is not a set instance before calling reload - # (which will clear / replace it with an empty set), checked below. - derived._changes = object() -- derived.reload(client=client, timeout=42) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/path", -- "query_params": {"projection": "noAcl"}, -- "headers": {}, -- "_target_object": derived, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- }, -- ) -- self.assertEqual(derived._changes, set()) -+ derived.client = client - -- def test_reload_with_generation_match(self): -- GENERATION_NUMBER = 9 -- METAGENERATION_NUMBER = 6 -+ derived.reload() -+ -+ self.assertEqual(derived._properties, response) -+ self.assertEqual(derived._changes, set()) - -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} # no encryption headers by default -+ client._get_resource.assert_called_once_with( -+ path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=derived, -+ ) -+ -+ def test_reload_w_generation_match_w_timeout(self): -+ generation_number = 9 -+ metageneration_number = 6 -+ path = "/path" -+ timeout = 42 -+ response = {"foo": "Foo"} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = response -+ derived = self._derivedClass(path)() - # Make sure changes is not a set instance before calling reload - # (which will clear / replace it with an empty set), checked below. - derived._changes = object() -+ derived.client = client -+ - derived.reload( -- client=client, -- timeout=42, -- if_generation_match=GENERATION_NUMBER, -- if_metageneration_match=METAGENERATION_NUMBER, -- ) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/path", -- "query_params": { -- "projection": "noAcl", -- "ifGenerationMatch": GENERATION_NUMBER, -- "ifMetagenerationMatch": METAGENERATION_NUMBER, -- }, -- "headers": {}, -- "_target_object": derived, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- }, -+ if_generation_match=generation_number, -+ if_metageneration_match=metageneration_number, -+ timeout=timeout, - ) -+ -+ self.assertEqual(derived._properties, response) - self.assertEqual(derived._changes, set()) - -- def test_reload_w_user_project(self): -+ expected_query_params = { -+ "projection": "noAcl", -+ "ifGenerationMatch": generation_number, -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ expected_headers = {} # no encryption headers by default -+ client._get_resource.assert_called_once_with( -+ path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=derived, -+ ) -+ -+ def test_reload_w_user_project_w_retry(self): - user_project = "user-project-123" -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path", user_project)() -+ path = "/path" -+ retry = mock.Mock(spec=[]) -+ response = {"foo": "Foo"} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = response -+ derived = self._derivedClass(path, user_project)() - # Make sure changes is not a set instance before calling reload - # (which will clear / replace it with an empty set), checked below. - derived._changes = object() -- derived.reload(client=client) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/path", -- "query_params": {"projection": "noAcl", "userProject": user_project}, -- "headers": {}, -- "_target_object": derived, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- }, -- ) -+ derived.client = client -+ -+ derived.reload(retry=retry) -+ -+ self.assertEqual(derived._properties, response) - self.assertEqual(derived._changes, set()) - -- def test_reload_w_projection(self): -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ expected_query_params = { -+ "projection": "noAcl", -+ "userProject": user_project, -+ } -+ expected_headers = {} # no encryption headers by default -+ client._get_resource.assert_called_once_with( -+ path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=derived, -+ ) -+ -+ def test_reload_w_projection_w_explicit_client_w_enc_header(self): -+ path = "/path" -+ response = {"foo": "Foo"} -+ encryption_headers = {"bar": "Bar"} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = response -+ derived = self._derivedClass(path)() - # Make sure changes is not a set instance before calling reload - # (which will clear / replace it with an empty set), checked below. - derived._changes = object() -- derived.reload(projection="full", client=client, timeout=42) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/path", -- "query_params": {"projection": "full"}, -- "headers": {}, -- "_target_object": derived, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- }, -- ) -+ derived._actual_encryption_headers = encryption_headers -+ -+ derived.reload(projection="full", client=client) -+ -+ self.assertEqual(derived._properties, response) - self.assertEqual(derived._changes, set()) - -+ expected_query_params = {"projection": "full"} -+ client._get_resource.assert_called_once_with( -+ path, -+ query_params=expected_query_params, -+ headers=encryption_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=derived, -+ ) -+ - def test__set_properties(self): - mixin = self._make_one() - self.assertEqual(mixin._properties, {}) -@@ -230,182 +245,204 @@ class Test_PropertyMixin(unittest.TestCa - derived._patch_property("foo", "Foo") - self.assertEqual(derived._properties, {"foo": "Foo"}) - -- def test_patch(self): -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ def test_patch_w_defaults(self): -+ path = "/path" -+ api_response = {"foo": "Foo"} -+ derived = self._derivedClass(path)() - # Make sure changes is non-empty, so we can observe a change. -- BAR = object() -- BAZ = object() -- derived._properties = {"bar": BAR, "baz": BAZ} -+ bar = object() -+ baz = object() -+ derived._properties = {"bar": bar, "baz": baz} - derived._changes = set(["bar"]) # Ignore baz. -- derived.patch(client=client, timeout=42) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/path", -- "query_params": {"projection": "full"}, -- # Since changes does not include `baz`, we don't see it sent. -- "data": {"bar": BAR}, -- "_target_object": derived, -- "timeout": 42, -- "retry": DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -- }, -- ) -+ client = derived.client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ -+ derived.patch() -+ -+ self.assertEqual(derived._properties, api_response) - # Make sure changes get reset by patch(). - self.assertEqual(derived._changes, set()) - -- def test_patch_with_metageneration_match(self): -- GENERATION_NUMBER = 9 -- METAGENERATION_NUMBER = 6 -+ expected_data = {"bar": bar} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=derived, -+ ) - -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ def test_patch_w_metageneration_match_w_timeout_w_retry(self): -+ path = "/path" -+ api_response = {"foo": "Foo"} -+ derived = self._derivedClass(path)() - # Make sure changes is non-empty, so we can observe a change. -- BAR = object() -- BAZ = object() -- derived._properties = {"bar": BAR, "baz": BAZ} -+ bar = object() -+ baz = object() -+ derived._properties = {"bar": bar, "baz": baz} - derived._changes = set(["bar"]) # Ignore baz. -+ client = derived.client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ generation_number = 9 -+ metageneration_number = 6 -+ - derived.patch( -- client=client, -- timeout=42, -- if_generation_match=GENERATION_NUMBER, -- if_metageneration_match=METAGENERATION_NUMBER, -+ if_generation_match=generation_number, -+ if_metageneration_match=metageneration_number, -+ timeout=timeout, -+ retry=retry, - ) -+ - self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/path", -- "query_params": { -- "projection": "full", -- "ifGenerationMatch": GENERATION_NUMBER, -- "ifMetagenerationMatch": METAGENERATION_NUMBER, -- }, -- # Since changes does not include `baz`, we don't see it sent. -- "data": {"bar": BAR}, -- "_target_object": derived, -- "timeout": 42, -- "retry": DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -- }, -- ) - # Make sure changes get reset by patch(). - self.assertEqual(derived._changes, set()) - -- def test_patch_w_user_project(self): -+ expected_data = {"bar": bar} -+ expected_query_params = { -+ "projection": "full", -+ "ifGenerationMatch": generation_number, -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ client._patch_resource.assert_called_once_with( -+ path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=derived, -+ ) -+ -+ def test_patch_w_user_project_w_explicit_client(self): -+ path = "/path" - user_project = "user-project-123" -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path", user_project)() -+ api_response = {"foo": "Foo"} -+ derived = self._derivedClass(path, user_project)() - # Make sure changes is non-empty, so we can observe a change. -- BAR = object() -- BAZ = object() -- derived._properties = {"bar": BAR, "baz": BAZ} -+ bar = object() -+ baz = object() -+ derived._properties = {"bar": bar, "baz": baz} - derived._changes = set(["bar"]) # Ignore baz. -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ - derived.patch(client=client) -+ - self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/path", -- "query_params": {"projection": "full", "userProject": user_project}, -- # Since changes does not include `baz`, we don't see it sent. -- "data": {"bar": BAR}, -- "_target_object": derived, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -- }, -- ) - # Make sure changes get reset by patch(). - self.assertEqual(derived._changes, set()) - -- def test_update(self): -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ expected_data = {"bar": bar} -+ expected_query_params = { -+ "projection": "full", -+ "userProject": user_project, -+ } -+ client._patch_resource.assert_called_once_with( -+ path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=derived, -+ ) -+ -+ def test_update_w_defaults(self): -+ path = "/path" -+ api_response = {"foo": "Foo"} -+ derived = self._derivedClass(path)() - # Make sure changes is non-empty, so we can observe a change. -- BAR = object() -- BAZ = object() -- derived._properties = {"bar": BAR, "baz": BAZ} -+ bar = object() -+ baz = object() -+ expected_data = derived._properties = {"bar": bar, "baz": baz} - derived._changes = set(["bar"]) # Update sends 'baz' anyway. -- derived.update(client=client, timeout=42) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "/path") -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["data"], {"bar": BAR, "baz": BAZ}) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED) -- # Make sure changes get reset by patch(). -+ client = derived.client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response -+ -+ derived.update() -+ -+ self.assertEqual(derived._properties, api_response) -+ # Make sure changes get reset by update(). - self.assertEqual(derived._changes, set()) - -- def test_update_with_metageneration_not_match(self): -- GENERATION_NUMBER = 6 -+ expected_query_params = {"projection": "full"} -+ client._put_resource.assert_called_once_with( -+ path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=derived, -+ ) - -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path")() -+ def test_update_with_metageneration_not_match_w_timeout_w_retry(self): -+ path = "/path" -+ generation_number = 6 -+ api_response = {"foo": "Foo"} -+ derived = self._derivedClass(path)() - # Make sure changes is non-empty, so we can observe a change. -- BAR = object() -- BAZ = object() -- derived._properties = {"bar": BAR, "baz": BAZ} -+ bar = object() -+ baz = object() -+ expected_data = derived._properties = {"bar": bar, "baz": baz} - derived._changes = set(["bar"]) # Update sends 'baz' anyway. -+ client = derived.client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response -+ timeout = 42 -+ - derived.update( -- client=client, timeout=42, if_metageneration_not_match=GENERATION_NUMBER -+ if_metageneration_not_match=generation_number, timeout=timeout, - ) -+ - self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "/path") -- self.assertEqual( -- kw[0]["query_params"], -- {"projection": "full", "ifMetagenerationNotMatch": GENERATION_NUMBER}, -- ) -- self.assertEqual(kw[0]["data"], {"bar": BAR, "baz": BAZ}) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED) - # Make sure changes get reset by patch(). - self.assertEqual(derived._changes, set()) - -- def test_update_w_user_project(self): -+ expected_query_params = { -+ "projection": "full", -+ "ifMetagenerationNotMatch": generation_number, -+ } -+ client._put_resource.assert_called_once_with( -+ path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=derived, -+ ) -+ -+ def test_update_w_user_project_w_retry_w_explicit_client(self): - user_project = "user-project-123" -- connection = _Connection({"foo": "Foo"}) -- client = _Client(connection) -- derived = self._derivedClass("/path", user_project)() -+ path = "/path" -+ api_response = {"foo": "Foo"} -+ derived = self._derivedClass(path, user_project)() - # Make sure changes is non-empty, so we can observe a change. -- BAR = object() -- BAZ = object() -- derived._properties = {"bar": BAR, "baz": BAZ} -+ bar = object() -+ baz = object() -+ expected_data = derived._properties = {"bar": bar, "baz": baz} - derived._changes = set(["bar"]) # Update sends 'baz' anyway. -- derived.update(client=client) -- self.assertEqual(derived._properties, {"foo": "Foo"}) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "/path") -- self.assertEqual( -- kw[0]["query_params"], {"projection": "full", "userProject": user_project} -- ) -- self.assertEqual(kw[0]["data"], {"bar": BAR, "baz": BAZ}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED) -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response -+ retry = mock.Mock(spec=[]) -+ -+ derived.update(client=client, retry=retry) - # Make sure changes get reset by patch(). - self.assertEqual(derived._changes, set()) - -+ expected_query_params = { -+ "projection": "full", -+ "userProject": user_project, -+ } -+ client._put_resource.assert_called_once_with( -+ path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=derived, -+ ) -+ - - class Test__scalar_property(unittest.TestCase): - def _call_fut(self, fieldName): -@@ -556,17 +593,6 @@ class Test__bucket_bound_hostname_url(un - self.assertEqual(self._call_fut(host=HOST, scheme=SCHEME), EXPECTED_URL) - - --class _Connection(object): -- def __init__(self, *responses): -- self._responses = responses -- self._requested = [] -- -- def api_request(self, **kw): -- self._requested.append(kw) -- response, self._responses = self._responses[0], self._responses[1:] -- return response -- -- - class _MD5Hash(object): - def __init__(self, digest_val): - self.digest_val = digest_val -@@ -598,8 +624,3 @@ class _Base64(object): - def b64encode(self, value): - self._called_b64encode.append(value) - return value -- -- --class _Client(object): -- def __init__(self, connection): -- self._connection = connection ---- a/tests/unit/test_acl.py -+++ b/tests/unit/test_acl.py -@@ -14,6 +14,10 @@ - - import unittest - -+import mock -+ -+from google.cloud.storage.retry import DEFAULT_RETRY -+ - - class Test_ACLEntity(unittest.TestCase): - @staticmethod -@@ -530,270 +534,345 @@ class Test_ACL(unittest.TestCase): - entity = acl.entity(TYPE, ID) - self.assertEqual(acl.get_entities(), [entity]) - -- def test_reload_missing(self): -+ def test_reload_missing_w_defaults(self): - # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/652 -- ROLE = "role" -- connection = _Connection({}) -- client = _Client(connection) -- acl = self._make_one() -- acl.reload_path = "/testing/acl" -- acl.loaded = True -- acl.entity("allUsers", ROLE) -- acl.reload(client=client, timeout=42) -+ class Derived(self._get_target_class()): -+ client = None -+ -+ role = "role" -+ reload_path = "/testing/acl" -+ api_response = {} -+ acl = Derived() -+ acl.reload_path = reload_path -+ acl.loaded = True -+ acl.entity("allUsers", role) -+ client = acl.client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ -+ acl.reload() -+ - self.assertEqual(list(acl), []) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/testing/acl", -- "query_params": {}, -- "timeout": 42, -- }, -+ -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ reload_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) - -- def test_reload_empty_result_clears_local(self): -- ROLE = "role" -- connection = _Connection({"items": []}) -- client = _Client(connection) -- acl = self._make_one() -- acl.reload_path = "/testing/acl" -- acl.loaded = True -- acl.entity("allUsers", ROLE) -+ def test_reload_w_empty_result_w_timeout_w_retry_w_explicit_client(self): -+ role = "role" -+ reload_path = "/testing/acl" -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ api_response = {"items": []} -+ acl = self._make_one() -+ acl.reload_path = reload_path -+ acl.loaded = True -+ acl.entity("allUsers", role) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response - -- acl.reload(client=client) -+ acl.reload(client=client, timeout=timeout, retry=retry) - - self.assertTrue(acl.loaded) - self.assertEqual(list(acl), []) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/testing/acl", -- "query_params": {}, -- "timeout": self._get_default_timeout(), -- }, -+ -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ reload_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, - ) - -- def test_reload_nonempty_result_w_user_project(self): -- ROLE = "role" -- USER_PROJECT = "user-project-123" -- connection = _Connection({"items": [{"entity": "allUsers", "role": ROLE}]}) -- client = _Client(connection) -- acl = self._make_one() -- acl.reload_path = "/testing/acl" -- acl.loaded = True -- acl.user_project = USER_PROJECT -+ def test_reload_w_nonempty_result_w_user_project(self): -+ role = "role" -+ reload_path = "/testing/acl" -+ user_project = "user-project-123" -+ api_response = {"items": [{"entity": "allUsers", "role": role}]} -+ acl = self._make_one() -+ acl.reload_path = reload_path -+ acl.loaded = True -+ acl.user_project = user_project -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response - - acl.reload(client=client) - - self.assertTrue(acl.loaded) -- self.assertEqual(list(acl), [{"entity": "allUsers", "role": ROLE}]) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "/testing/acl", -- "query_params": {"userProject": USER_PROJECT}, -- "timeout": self._get_default_timeout(), -- }, -+ self.assertEqual(list(acl), [{"entity": "allUsers", "role": role}]) -+ -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ reload_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) - - def test_save_none_set_none_passed(self): -- connection = _Connection() -- client = _Client(connection) -+ save_path = "/testing" -+ client = mock.Mock(spec=["_patch_resource"]) - acl = self._make_one() -- acl.save_path = "/testing" -+ acl.save_path = save_path -+ - acl.save(client=client) -- kw = connection._requested -- self.assertEqual(len(kw), 0) - -- def test_save_existing_missing_none_passed(self): -- connection = _Connection({}) -- client = _Client(connection) -- acl = self._make_one() -- acl.save_path = "/testing" -+ client._patch_resource.assert_not_called() -+ -+ def test_save_w_empty_response_w_defaults(self): -+ class Derived(self._get_target_class()): -+ client = None -+ -+ save_path = "/testing" -+ api_response = {} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ acl = Derived() -+ acl.client = client -+ acl.save_path = save_path - acl.loaded = True -- acl.save(client=client, timeout=42) -+ -+ acl.save() -+ - self.assertEqual(list(acl), []) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/testing") -- self.assertEqual(kw[0]["data"], {"acl": []}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], 42) -- -- def test_save_no_acl(self): -- ROLE = "role" -- AFTER = [{"entity": "allUsers", "role": ROLE}] -- connection = _Connection({"acl": AFTER}) -- client = _Client(connection) -- acl = self._make_one() -- acl.save_path = "/testing" -- acl.loaded = True -- acl.entity("allUsers").grant(ROLE) -- acl.save(client=client) -- self.assertEqual(list(acl), AFTER) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/testing") -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/testing", -- "query_params": {"projection": "full"}, -- "data": {"acl": AFTER}, -- "timeout": self._get_default_timeout(), -- }, -+ -+ expected_data = {"acl": []} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ def test_save_no_acl_w_timeout(self): -+ save_path = "/testing" -+ role = "role" -+ expected_acl = [{"entity": "allUsers", "role": role}] -+ api_response = {"acl": expected_acl} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ acl = self._make_one() -+ acl.save_path = save_path -+ acl.loaded = True -+ acl.entity("allUsers").grant(role) -+ timeout = 42 -+ -+ acl.save(client=client, timeout=timeout) -+ -+ self.assertEqual(list(acl), expected_acl) -+ -+ expected_data = api_response -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=None, - ) - - def test_save_w_acl_w_user_project(self): -- ROLE1 = "role1" -- ROLE2 = "role2" -- STICKY = {"entity": "allUsers", "role": ROLE2} -- USER_PROJECT = "user-project-123" -- new_acl = [{"entity": "allUsers", "role": ROLE1}] -- connection = _Connection({"acl": [STICKY] + new_acl}) -- client = _Client(connection) -+ save_path = "/testing" -+ user_project = "user-project-123" -+ role1 = "role1" -+ role2 = "role2" -+ sticky = {"entity": "allUsers", "role": role2} -+ new_acl = [{"entity": "allUsers", "role": role1}] -+ api_response = {"acl": [sticky] + new_acl} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response - acl = self._make_one() -- acl.save_path = "/testing" -+ acl.save_path = save_path - acl.loaded = True -- acl.user_project = USER_PROJECT -+ acl.user_project = user_project - - acl.save(new_acl, client=client) - - entries = list(acl) - self.assertEqual(len(entries), 2) -- self.assertTrue(STICKY in entries) -+ self.assertTrue(sticky in entries) - self.assertTrue(new_acl[0] in entries) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/testing", -- "query_params": {"projection": "full", "userProject": USER_PROJECT}, -- "data": {"acl": new_acl}, -- "timeout": self._get_default_timeout(), -- }, -+ -+ expected_data = {"acl": new_acl} -+ expected_query_params = {"projection": "full", "userProject": user_project} -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, - ) - - def test_save_prefefined_invalid(self): -- connection = _Connection() -- client = _Client(connection) -+ save_path = "/testing" -+ client = mock.Mock(spec=["_patch_resource"]) - acl = self._make_one() -- acl.save_path = "/testing" -+ acl.save_path = save_path - acl.loaded = True -+ - with self.assertRaises(ValueError): - acl.save_predefined("bogus", client=client) - -- def test_save_predefined_valid(self): -- PREDEFINED = "private" -- connection = _Connection({"acl": []}) -- client = _Client(connection) -- acl = self._make_one() -- acl.save_path = "/testing" -+ client._patch_resource.assert_not_called() -+ -+ def test_save_predefined_w_defaults(self): -+ class Derived(self._get_target_class()): -+ client = None -+ -+ save_path = "/testing" -+ predefined = "private" -+ api_response = {"acl": []} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ acl = Derived() -+ acl.save_path = save_path - acl.loaded = True -- acl.save_predefined(PREDEFINED, client=client, timeout=42) -+ acl.client = client -+ -+ acl.save_predefined(predefined) -+ - entries = list(acl) - self.assertEqual(len(entries), 0) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/testing", -- "query_params": {"projection": "full", "predefinedAcl": PREDEFINED}, -- "data": {"acl": []}, -- "timeout": 42, -- }, -- ) -- -- def test_save_predefined_w_XML_alias(self): -- PREDEFINED_XML = "project-private" -- PREDEFINED_JSON = "projectPrivate" -- connection = _Connection({"acl": []}) -- client = _Client(connection) -+ -+ expected_data = {"acl": []} -+ expected_query_params = { -+ "projection": "full", -+ "predefinedAcl": predefined, -+ } -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ def test_save_predefined_w_XML_alias_w_timeout(self): -+ save_path = "/testing" -+ predefined_xml = "project-private" -+ predefined_json = "projectPrivate" -+ api_response = {"acl": []} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response - acl = self._make_one() -- acl.save_path = "/testing" -+ acl.save_path = save_path - acl.loaded = True -- acl.save_predefined(PREDEFINED_XML, client=client) -+ timeout = 42 -+ -+ acl.save_predefined(predefined_xml, client=client, timeout=timeout) -+ - entries = list(acl) - self.assertEqual(len(entries), 0) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/testing", -- "query_params": { -- "projection": "full", -- "predefinedAcl": PREDEFINED_JSON, -- }, -- "data": {"acl": []}, -- "timeout": self._get_default_timeout(), -- }, -+ -+ expected_data = {"acl": []} -+ expected_query_params = { -+ "projection": "full", -+ "predefinedAcl": predefined_json, -+ } -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=None, - ) - -- def test_save_predefined_valid_w_alternate_query_param(self): -+ def test_save_predefined_w_alternate_query_param(self): - # Cover case where subclass overrides _PREDEFINED_QUERY_PARAM -- PREDEFINED = "publicRead" -- connection = _Connection({"acl": []}) -- client = _Client(connection) -+ save_path = "/testing" -+ predefined = "publicRead" -+ api_response = {"acl": []} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response - acl = self._make_one() -- acl.save_path = "/testing" -+ acl.save_path = save_path - acl.loaded = True - acl._PREDEFINED_QUERY_PARAM = "alternate" -- acl.save_predefined(PREDEFINED, client=client) -+ -+ acl.save_predefined(predefined, client=client) -+ - entries = list(acl) - self.assertEqual(len(entries), 0) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/testing", -- "query_params": {"projection": "full", "alternate": PREDEFINED}, -- "data": {"acl": []}, -- "timeout": self._get_default_timeout(), -- }, -+ -+ expected_data = {"acl": []} -+ expected_query_params = { -+ "projection": "full", -+ "alternate": predefined, -+ } -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, - ) - -- def test_clear(self): -- ROLE1 = "role1" -- ROLE2 = "role2" -- STICKY = {"entity": "allUsers", "role": ROLE2} -- connection = _Connection({"acl": [STICKY]}) -- client = _Client(connection) -- acl = self._make_one() -- acl.save_path = "/testing" -- acl.loaded = True -- acl.entity("allUsers", ROLE1) -- acl.clear(client=client, timeout=42) -- self.assertEqual(list(acl), [STICKY]) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "PATCH", -- "path": "/testing", -- "query_params": {"projection": "full"}, -- "data": {"acl": []}, -- "timeout": 42, -- }, -+ def test_clear_w_defaults(self): -+ class Derived(self._get_target_class()): -+ client = None -+ -+ save_path = "/testing" -+ role1 = "role1" -+ role2 = "role2" -+ sticky = {"entity": "allUsers", "role": role2} -+ api_response = {"acl": [sticky]} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ acl = Derived() -+ acl.client = client -+ acl.save_path = save_path -+ acl.loaded = True -+ acl.entity("allUsers", role1) -+ -+ acl.clear() -+ -+ self.assertEqual(list(acl), [sticky]) -+ -+ expected_data = {"acl": []} -+ expected_query_params = { -+ "projection": "full", -+ } -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ def test_clear_w_explicit_client_w_timeout(self): -+ save_path = "/testing" -+ role1 = "role1" -+ role2 = "role2" -+ sticky = {"entity": "allUsers", "role": role2} -+ api_response = {"acl": [sticky]} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ acl = self._make_one() -+ acl.save_path = save_path -+ acl.loaded = True -+ acl.entity("allUsers", role1) -+ timeout = 42 -+ -+ acl.clear(client=client, timeout=timeout) -+ -+ self.assertEqual(list(acl), [sticky]) -+ -+ expected_data = {"acl": []} -+ expected_query_params = { -+ "projection": "full", -+ } -+ client._patch_resource.assert_called_once_with( -+ save_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=None, - ) - - -@@ -905,22 +984,3 @@ class _Bucket(object): - @property - def path(self): - return "/b/%s" % self.name -- -- --class _Connection(object): -- _delete_ok = False -- -- def __init__(self, *responses): -- self._responses = responses -- self._requested = [] -- self._deleted = [] -- -- def api_request(self, **kw): -- self._requested.append(kw) -- response, self._responses = self._responses[0], self._responses[1:] -- return response -- -- --class _Client(object): -- def __init__(self, connection): -- self._connection = connection ---- a/tests/unit/test_blob.py -+++ b/tests/unit/test_blob.py -@@ -27,6 +27,7 @@ import six - from six.moves import http_client - - from google.cloud.storage.retry import DEFAULT_RETRY -+from google.cloud.storage.retry import DEFAULT_RETRY_IF_ETAG_IN_JSON - from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED - - -@@ -56,7 +57,7 @@ class Test_Blob(unittest.TestCase): - def _make_client(*args, **kw): - from google.cloud.storage.client import Client - -- return Client(*args, **kw) -+ return mock.create_autospec(Client, instance=True, **kw) - - def test_ctor_wo_encryption_key(self): - BLOB_NAME = "blob-name" -@@ -421,10 +422,10 @@ class Test_Blob(unittest.TestCase): - def test_generate_signed_url_w_invalid_version(self): - BLOB_NAME = "blob-name" - EXPIRATION = "2014-10-16T20:34:37.000Z" -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) -+ - with self.assertRaises(ValueError): - blob.generate_signed_url(EXPIRATION, version="nonesuch") - -@@ -463,8 +464,13 @@ class Test_Blob(unittest.TestCase): - if expiration is None: - expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + delta - -- connection = _Connection() -- client = _Client(connection) -+ if credentials is None: -+ expected_creds = _make_credentials() -+ client = self._make_client(_credentials=expected_creds) -+ else: -+ expected_creds = credentials -+ client = self._make_client(_credentials=object()) -+ - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket, encryption_key=encryption_key) - -@@ -499,11 +505,6 @@ class Test_Blob(unittest.TestCase): - - self.assertEqual(signed_uri, signer.return_value) - -- if credentials is None: -- expected_creds = _Connection.credentials -- else: -- expected_creds = credentials -- - encoded_name = blob_name.encode("utf-8") - quoted_name = parse.quote(encoded_name, safe=b"/~") - -@@ -688,117 +689,108 @@ class Test_Blob(unittest.TestCase): - credentials = object() - self._generate_signed_url_v4_helper(credentials=credentials) - -- def test_exists_miss(self): -- NONESUCH = "nonesuch" -- not_found_response = ({"status": http_client.NOT_FOUND}, b"") -- connection = _Connection(not_found_response) -- client = _Client(connection) -+ def test_exists_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound -+ -+ blob_name = "nonesuch" -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.side_effect = NotFound("testing") - bucket = _Bucket(client) -- blob = self._make_one(NONESUCH, bucket=bucket) -- self.assertFalse(blob.exists(timeout=42)) -- self.assertEqual(len(connection._requested), 1) -- self.assertEqual( -- connection._requested[0], -- { -- "method": "GET", -- "path": "/b/name/o/{}".format(NONESUCH), -- "query_params": {"fields": "name"}, -- "_target_object": None, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- }, -+ blob = self._make_one(blob_name, bucket=bucket) -+ -+ self.assertFalse(blob.exists()) -+ -+ expected_query_params = {"fields": "name"} -+ client._get_resource.assert_called_once_with( -+ blob.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) - -- def test_exists_hit_w_user_project(self): -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- found_response = ({"status": http_client.OK}, b"") -- connection = _Connection(found_response) -- client = _Client(connection) -- bucket = _Bucket(client, user_project=USER_PROJECT) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -- bucket._blobs[BLOB_NAME] = 1 -- self.assertTrue(blob.exists()) -- self.assertEqual(len(connection._requested), 1) -- self.assertEqual( -- connection._requested[0], -- { -- "method": "GET", -- "path": "/b/name/o/{}".format(BLOB_NAME), -- "query_params": {"fields": "name", "userProject": USER_PROJECT}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- }, -+ def test_exists_hit_w_user_project_w_timeout(self): -+ blob_name = "blob-name" -+ user_project = "user-project-123" -+ timeout = 42 -+ api_response = {"name": blob_name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = _Bucket(client, user_project=user_project) -+ blob = self._make_one(blob_name, bucket=bucket) -+ -+ self.assertTrue(blob.exists(timeout=timeout)) -+ -+ expected_query_params = {"fields": "name", "userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ blob.path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) - -- def test_exists_hit_w_generation(self): -- BLOB_NAME = "blob-name" -- GENERATION = 123456 -- found_response = ({"status": http_client.OK}, b"") -- connection = _Connection(found_response) -- client = _Client(connection) -+ def test_exists_hit_w_generation_w_retry(self): -+ blob_name = "blob-name" -+ generation = 123456 -+ api_response = {"name": blob_name} -+ retry = mock.Mock(spec=[]) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response - bucket = _Bucket(client) -- blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) -- bucket._blobs[BLOB_NAME] = 1 -- self.assertTrue(blob.exists()) -- self.assertEqual(len(connection._requested), 1) -- self.assertEqual( -- connection._requested[0], -- { -- "method": "GET", -- "path": "/b/name/o/{}".format(BLOB_NAME), -- "query_params": {"fields": "name", "generation": GENERATION}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- }, -+ blob = self._make_one(blob_name, bucket=bucket, generation=generation) -+ -+ self.assertTrue(blob.exists(retry=retry)) -+ -+ expected_query_params = {"fields": "name", "generation": generation} -+ client._get_resource.assert_called_once_with( -+ blob.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=None, - ) - - def test_exists_w_generation_match(self): -- BLOB_NAME = "blob-name" -- GENERATION_NUMBER = 123456 -- METAGENERATION_NUMBER = 6 -- -- found_response = ({"status": http_client.OK}, b"") -- connection = _Connection(found_response) -- client = _Client(connection) -+ blob_name = "blob-name" -+ generation_number = 123456 -+ metageneration_number = 6 -+ api_response = {"name": blob_name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response - bucket = _Bucket(client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -- bucket._blobs[BLOB_NAME] = 1 -+ blob = self._make_one(blob_name, bucket=bucket) -+ - self.assertTrue( - blob.exists( -- if_generation_match=GENERATION_NUMBER, -- if_metageneration_match=METAGENERATION_NUMBER, -+ if_generation_match=generation_number, -+ if_metageneration_match=metageneration_number, -+ retry=None, - ) - ) -- self.assertEqual(len(connection._requested), 1) -- self.assertEqual( -- connection._requested[0], -- { -- "method": "GET", -- "path": "/b/name/o/{}".format(BLOB_NAME), -- "query_params": { -- "fields": "name", -- "ifGenerationMatch": GENERATION_NUMBER, -- "ifMetagenerationMatch": METAGENERATION_NUMBER, -- }, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- }, -+ -+ expected_query_params = { -+ "fields": "name", -+ "ifGenerationMatch": generation_number, -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ client._get_resource.assert_called_once_with( -+ blob.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ _target_object=None, - ) - - def test_delete_wo_generation(self): - BLOB_NAME = "blob-name" -- not_found_response = ({"status": http_client.NOT_FOUND}, b"") -- connection = _Connection(not_found_response) -- client = _Client(connection) -+ client = self._make_client() - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - bucket._blobs[BLOB_NAME] = 1 -+ - blob.delete() -- self.assertFalse(blob.exists()) -+ - self.assertEqual( - bucket._deleted, - [ -@@ -819,14 +811,13 @@ class Test_Blob(unittest.TestCase): - def test_delete_w_generation(self): - BLOB_NAME = "blob-name" - GENERATION = 123456 -- not_found_response = ({"status": http_client.NOT_FOUND}, b"") -- connection = _Connection(not_found_response) -- client = _Client(connection) -+ client = self._make_client() - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) - bucket._blobs[BLOB_NAME] = 1 -+ - blob.delete(timeout=42) -- self.assertFalse(blob.exists()) -+ - self.assertEqual( - bucket._deleted, - [ -@@ -847,14 +838,13 @@ class Test_Blob(unittest.TestCase): - def test_delete_w_generation_match(self): - BLOB_NAME = "blob-name" - GENERATION = 123456 -- not_found_response = ({"status": http_client.NOT_FOUND}, b"") -- connection = _Connection(not_found_response) -- client = _Client(connection) -+ client = self._make_client() - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) - bucket._blobs[BLOB_NAME] = 1 -+ - blob.delete(timeout=42, if_generation_match=GENERATION) -- self.assertFalse(blob.exists()) -+ - self.assertEqual( - bucket._deleted, - [ -@@ -1031,6 +1021,99 @@ class Test_Blob(unittest.TestCase): - response.request = requests.Request("POST", "http://example.com").prepare() - return response - -+ def test__extract_headers_from_download_gzipped(self): -+ blob_name = "blob-name" -+ client = mock.Mock(spec=["_http"]) -+ bucket = _Bucket(client) -+ blob = self._make_one(blob_name, bucket=bucket) -+ -+ response = self._mock_requests_response( -+ http_client.OK, -+ headers={ -+ "Content-Type": "application/json", -+ "Content-Language": "ko-kr", -+ "Cache-Control": "max-age=1337;public", -+ "Content-Encoding": "gzip", -+ "X-Goog-Storage-Class": "STANDARD", -+ "X-Goog-Hash": "crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==", -+ }, -+ # { "x": 5 } gzipped -+ content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00", -+ ) -+ blob._extract_headers_from_download(response) -+ -+ self.assertEqual(blob.content_type, "application/json") -+ self.assertEqual(blob.content_language, "ko-kr") -+ self.assertEqual(blob.content_encoding, "gzip") -+ self.assertEqual(blob.cache_control, "max-age=1337;public") -+ self.assertEqual(blob.storage_class, "STANDARD") -+ self.assertEqual(blob.md5_hash, "CS9tHYTtyFntzj7B9nkkJQ==") -+ self.assertEqual(blob.crc32c, "4gcgLQ==") -+ -+ def test__extract_headers_from_download_empty(self): -+ blob_name = "blob-name" -+ client = mock.Mock(spec=["_http"]) -+ bucket = _Bucket(client) -+ blob = self._make_one(blob_name, bucket=bucket) -+ -+ response = self._mock_requests_response( -+ http_client.OK, -+ headers={ -+ "Content-Type": "application/octet-stream", -+ "Content-Language": "en-US", -+ "Cache-Control": "max-age=1337;public", -+ "Content-Encoding": "gzip", -+ "X-Goog-Storage-Class": "STANDARD", -+ "X-Goog-Hash": "crc32c=4/c+LQ==,md5=CS9tHYTt/+ntzj7B9nkkJQ==", -+ }, -+ content=b"", -+ ) -+ blob._extract_headers_from_download(response) -+ self.assertEqual(blob.content_type, "application/octet-stream") -+ self.assertEqual(blob.content_language, "en-US") -+ self.assertEqual(blob.md5_hash, "CS9tHYTt/+ntzj7B9nkkJQ==") -+ self.assertEqual(blob.crc32c, "4/c+LQ==") -+ -+ def test__extract_headers_from_download_w_hash_response_header_none(self): -+ blob_name = "blob-name" -+ md5_hash = "CS9tHYTtyFntzj7B9nkkJQ==" -+ crc32c = "4gcgLQ==" -+ client = mock.Mock(spec=["_http"]) -+ bucket = _Bucket(client) -+ properties = { -+ "md5Hash": md5_hash, -+ "crc32c": crc32c, -+ } -+ blob = self._make_one(blob_name, bucket=bucket, properties=properties) -+ -+ response = self._mock_requests_response( -+ http_client.OK, -+ headers={"X-Goog-Hash": ""}, -+ # { "x": 5 } gzipped -+ content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00", -+ ) -+ blob._extract_headers_from_download(response) -+ -+ self.assertEqual(blob.md5_hash, md5_hash) -+ self.assertEqual(blob.crc32c, crc32c) -+ -+ def test__extract_headers_from_download_w_response_headers_not_match(self): -+ blob_name = "blob-name" -+ client = mock.Mock(spec=["_http"]) -+ bucket = _Bucket(client) -+ blob = self._make_one(blob_name, bucket=bucket) -+ -+ response = self._mock_requests_response( -+ http_client.OK, -+ headers={"X-Goog-Hash": "bogus=4gcgLQ==,"}, -+ # { "x": 5 } gzipped -+ content=b"", -+ ) -+ blob._extract_headers_from_download(response) -+ -+ self.assertIsNone(blob.md5_hash) -+ self.assertIsNone(blob.crc32c) -+ - def _do_download_helper_wo_chunks(self, w_range, raw_download, timeout=None): - blob_name = "blob-name" - client = mock.Mock() -@@ -1121,7 +1204,7 @@ class Test_Blob(unittest.TestCase): - self, w_range, raw_download, timeout=None, checksum="md5" - ): - blob_name = "blob-name" -- client = mock.Mock(_credentials=_make_credentials(), spec=["_credentials"]) -+ client = self._make_client() - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) - blob._CHUNK_SIZE_MULTIPLE = 1 -@@ -1224,41 +1307,32 @@ class Test_Blob(unittest.TestCase): - patch.assert_not_called() - - def test_download_to_file_with_failure(self): -- import requests -- from google.resumable_media import InvalidResponse -- from google.cloud import exceptions -- -- raw_response = requests.Response() -- raw_response.status_code = http_client.NOT_FOUND -- raw_request = requests.Request("GET", "http://example.com") -- raw_response.request = raw_request.prepare() -- grmp_response = InvalidResponse(raw_response) -+ from google.cloud.exceptions import NotFound - - blob_name = "blob-name" -- media_link = "http://test.invalid" - client = self._make_client() -+ client.download_blob_to_file.side_effect = NotFound("testing") - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) -- blob._properties["mediaLink"] = media_link -- blob._do_download = mock.Mock() -- blob._do_download.side_effect = grmp_response -- - file_obj = io.BytesIO() -- with self.assertRaises(exceptions.NotFound): -+ -+ with self.assertRaises(NotFound): - blob.download_to_file(file_obj) - - self.assertEqual(file_obj.tell(), 0) - -- headers = {"accept-encoding": "gzip"} -- blob._do_download.assert_called_once_with( -- client._http, -+ expected_timeout = self._get_default_timeout() -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - file_obj, -- media_link, -- headers, -- None, -- None, -- False, -- timeout=self._get_default_timeout(), -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=False, -+ timeout=expected_timeout, - checksum="md5", - ) - -@@ -1267,7 +1341,6 @@ class Test_Blob(unittest.TestCase): - client = self._make_client() - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) -- blob._do_download = mock.Mock() - file_obj = io.BytesIO() - - blob.download_to_file(file_obj) -@@ -1275,49 +1348,41 @@ class Test_Blob(unittest.TestCase): - # Make sure the media link is still unknown. - self.assertIsNone(blob.media_link) - -- expected_url = ( -- "https://storage.googleapis.com/download/storage/v1/b/" -- "name/o/blob-name?alt=media" -- ) -- headers = {"accept-encoding": "gzip"} -- blob._do_download.assert_called_once_with( -- client._http, -+ expected_timeout = self._get_default_timeout() -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - file_obj, -- expected_url, -- headers, -- None, -- None, -- False, -- timeout=self._get_default_timeout(), -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=False, -+ timeout=expected_timeout, - checksum="md5", - ) - - def test_download_to_file_w_generation_match(self): -- GENERATION_NUMBER = 6 -- HEADERS = {"accept-encoding": "gzip"} -- EXPECTED_URL = ( -- "https://storage.googleapis.com/download/storage/v1/b/" -- "name/o/blob-name?alt=media&ifGenerationNotMatch={}".format( -- GENERATION_NUMBER -- ) -- ) -- -+ generation_number = 6 - client = self._make_client() - blob = self._make_one("blob-name", bucket=_Bucket(client)) -- blob._do_download = mock.Mock() - file_obj = io.BytesIO() - -- blob.download_to_file(file_obj, if_generation_not_match=GENERATION_NUMBER) -+ blob.download_to_file(file_obj, if_generation_not_match=generation_number) - -- blob._do_download.assert_called_once_with( -- client._http, -+ expected_timeout = self._get_default_timeout() -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - file_obj, -- EXPECTED_URL, -- HEADERS, -- None, -- None, -- False, -- timeout=self._get_default_timeout(), -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=generation_number, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=False, -+ timeout=expected_timeout, - checksum="md5", - ) - -@@ -1331,7 +1396,6 @@ class Test_Blob(unittest.TestCase): - if use_chunks: - blob._CHUNK_SIZE_MULTIPLE = 1 - blob.chunk_size = 3 -- blob._do_download = mock.Mock() - - if timeout is None: - expected_timeout = self._get_default_timeout() -@@ -1346,15 +1410,16 @@ class Test_Blob(unittest.TestCase): - else: - blob.download_to_file(file_obj, **timeout_kwarg) - -- headers = {"accept-encoding": "gzip"} -- blob._do_download.assert_called_once_with( -- client._http, -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - file_obj, -- media_link, -- headers, -- None, -- None, -- raw_download, -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=raw_download, - timeout=expected_timeout, - checksum="md5", - ) -@@ -1384,13 +1449,11 @@ class Test_Blob(unittest.TestCase): - blob_name = "blob-name" - client = self._make_client() - bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = {"mediaLink": media_link} -+ properties = {} - if updated is not None: - properties["updated"] = updated - - blob = self._make_one(blob_name, bucket=bucket, properties=properties) -- blob._do_download = mock.Mock() - - with _NamedTemporaryFile() as temp: - if timeout is None: -@@ -1412,51 +1475,22 @@ class Test_Blob(unittest.TestCase): - - expected_timeout = self._get_default_timeout() if timeout is None else timeout - -- headers = {"accept-encoding": "gzip"} -- blob._do_download.assert_called_once_with( -- client._http, -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - mock.ANY, -- media_link, -- headers, -- None, -- None, -- raw_download, -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=raw_download, - timeout=expected_timeout, - checksum="md5", - ) -- stream = blob._do_download.mock_calls[0].args[1] -+ stream = client.download_blob_to_file.mock_calls[0].args[1] - self.assertEqual(stream.name, temp.name) - -- def test_download_to_filename_w_generation_match(self): -- from google.cloud._testing import _NamedTemporaryFile -- -- GENERATION_NUMBER = 6 -- MEDIA_LINK = "http://example.com/media/" -- EXPECTED_LINK = MEDIA_LINK + "?ifGenerationMatch={}".format(GENERATION_NUMBER) -- HEADERS = {"accept-encoding": "gzip"} -- -- client = self._make_client() -- -- blob = self._make_one( -- "blob-name", bucket=_Bucket(client), properties={"mediaLink": MEDIA_LINK} -- ) -- blob._do_download = mock.Mock() -- -- with _NamedTemporaryFile() as temp: -- blob.download_to_filename(temp.name, if_generation_match=GENERATION_NUMBER) -- -- blob._do_download.assert_called_once_with( -- client._http, -- mock.ANY, -- EXPECTED_LINK, -- HEADERS, -- None, -- None, -- False, -- timeout=self._get_default_timeout(), -- checksum="md5", -- ) -- - def test_download_to_filename_w_updated_wo_raw(self): - updated = "2014-12-06T13:13:50.690Z" - self._download_to_filename_helper(updated=updated, raw_download=False) -@@ -1476,18 +1510,41 @@ class Test_Blob(unittest.TestCase): - updated=None, raw_download=False, timeout=9.58 - ) - -+ def test_download_to_filename_w_generation_match(self): -+ from google.cloud._testing import _NamedTemporaryFile -+ -+ generation_number = 6 -+ client = self._make_client() -+ blob = self._make_one("blob-name", bucket=_Bucket(client)) -+ -+ with _NamedTemporaryFile() as temp: -+ blob.download_to_filename(temp.name, if_generation_match=generation_number) -+ -+ expected_timeout = self._get_default_timeout() -+ client.download_blob_to_file.assert_called_once_with( -+ blob, -+ mock.ANY, -+ start=None, -+ end=None, -+ if_generation_match=generation_number, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=False, -+ timeout=expected_timeout, -+ checksum="md5", -+ ) -+ stream = client.download_blob_to_file.mock_calls[0].args[1] -+ self.assertEqual(stream.name, temp.name) -+ - def test_download_to_filename_corrupted(self): - from google.resumable_media import DataCorruption - - blob_name = "blob-name" - client = self._make_client() - bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = {"mediaLink": media_link} -- -- blob = self._make_one(blob_name, bucket=bucket, properties=properties) -- blob._do_download = mock.Mock() -- blob._do_download.side_effect = DataCorruption("testing") -+ blob = self._make_one(blob_name, bucket=bucket) -+ client.download_blob_to_file.side_effect = DataCorruption("testing") - - # Try to download into a temporary file (don't use - # `_NamedTemporaryFile` it will try to remove after the file is -@@ -1502,64 +1559,28 @@ class Test_Blob(unittest.TestCase): - # Make sure the file was cleaned up. - self.assertFalse(os.path.exists(filename)) - -- headers = {"accept-encoding": "gzip"} -- blob._do_download.assert_called_once_with( -- client._http, -+ expected_timeout = self._get_default_timeout() -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - mock.ANY, -- media_link, -- headers, -- None, -- None, -- False, -- timeout=self._get_default_timeout(), -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=False, -+ timeout=expected_timeout, - checksum="md5", - ) -- stream = blob._do_download.mock_calls[0].args[1] -+ stream = client.download_blob_to_file.mock_calls[0].args[1] - self.assertEqual(stream.name, filename) - -- def test_download_to_filename_w_key(self): -- from google.cloud._testing import _NamedTemporaryFile -- from google.cloud.storage.blob import _get_encryption_headers -- -- blob_name = "blob-name" -- # Create a fake client/bucket and use them in the Blob() constructor. -- client = self._make_client() -- bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = {"mediaLink": media_link} -- key = b"aa426195405adee2c8081bb9e7e74b19" -- blob = self._make_one( -- blob_name, bucket=bucket, properties=properties, encryption_key=key -- ) -- blob._do_download = mock.Mock() -- -- with _NamedTemporaryFile() as temp: -- blob.download_to_filename(temp.name) -- -- headers = {"accept-encoding": "gzip"} -- headers.update(_get_encryption_headers(key)) -- blob._do_download.assert_called_once_with( -- client._http, -- mock.ANY, -- media_link, -- headers, -- None, -- None, -- False, -- timeout=self._get_default_timeout(), -- checksum="md5", -- ) -- stream = blob._do_download.mock_calls[0].args[1] -- self.assertEqual(stream.name, temp.name) -- - def _download_as_bytes_helper(self, raw_download, timeout=None): - blob_name = "blob-name" - client = self._make_client() - bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = {"mediaLink": media_link} -- blob = self._make_one(blob_name, bucket=bucket, properties=properties) -- blob._do_download = mock.Mock() -+ blob = self._make_one(blob_name, bucket=bucket) - - if timeout is None: - expected_timeout = self._get_default_timeout() -@@ -1569,113 +1590,30 @@ class Test_Blob(unittest.TestCase): - fetched = blob.download_as_bytes(raw_download=raw_download, timeout=timeout) - self.assertEqual(fetched, b"") - -- headers = {"accept-encoding": "gzip"} -- blob._do_download.assert_called_once_with( -- client._http, -+ client.download_blob_to_file.assert_called_once_with( -+ blob, - mock.ANY, -- media_link, -- headers, -- None, -- None, -- raw_download, -+ start=None, -+ end=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ raw_download=raw_download, - timeout=expected_timeout, - checksum="md5", - ) -- stream = blob._do_download.mock_calls[0].args[1] -+ stream = client.download_blob_to_file.mock_calls[0].args[1] - self.assertIsInstance(stream, io.BytesIO) - -- def test_download_as_string_w_response_headers(self): -- blob_name = "blob-name" -- client = mock.Mock(spec=["_http"]) -- bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = {"mediaLink": media_link} -- blob = self._make_one(blob_name, bucket=bucket, properties=properties) -- -- response = self._mock_requests_response( -- http_client.OK, -- headers={ -- "Content-Type": "application/json", -- "Content-Language": "ko-kr", -- "Cache-Control": "max-age=1337;public", -- "Content-Encoding": "gzip", -- "X-Goog-Storage-Class": "STANDARD", -- "X-Goog-Hash": "crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==", -- }, -- # { "x": 5 } gzipped -- content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00", -- ) -- blob._extract_headers_from_download(response) -- -- self.assertEqual(blob.content_type, "application/json") -- self.assertEqual(blob.content_language, "ko-kr") -- self.assertEqual(blob.content_encoding, "gzip") -- self.assertEqual(blob.cache_control, "max-age=1337;public") -- self.assertEqual(blob.storage_class, "STANDARD") -- self.assertEqual(blob.md5_hash, "CS9tHYTtyFntzj7B9nkkJQ==") -- self.assertEqual(blob.crc32c, "4gcgLQ==") -- -- response = self._mock_requests_response( -- http_client.OK, -- headers={ -- "Content-Type": "application/octet-stream", -- "Content-Language": "en-US", -- "Cache-Control": "max-age=1337;public", -- "Content-Encoding": "gzip", -- "X-Goog-Storage-Class": "STANDARD", -- "X-Goog-Hash": "crc32c=4/c+LQ==,md5=CS9tHYTt/+ntzj7B9nkkJQ==", -- }, -- content=b"", -- ) -- blob._extract_headers_from_download(response) -- self.assertEqual(blob.content_type, "application/octet-stream") -- self.assertEqual(blob.content_language, "en-US") -- self.assertEqual(blob.md5_hash, "CS9tHYTt/+ntzj7B9nkkJQ==") -- self.assertEqual(blob.crc32c, "4/c+LQ==") -- -- def test_download_as_string_w_hash_response_header_none(self): -- blob_name = "blob-name" -- md5_hash = "CS9tHYTtyFntzj7B9nkkJQ==" -- crc32c = "4gcgLQ==" -- client = mock.Mock(spec=["_http"]) -- bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = { -- "mediaLink": media_link, -- "md5Hash": md5_hash, -- "crc32c": crc32c, -- } -- blob = self._make_one(blob_name, bucket=bucket, properties=properties) -- -- response = self._mock_requests_response( -- http_client.OK, -- headers={"X-Goog-Hash": ""}, -- # { "x": 5 } gzipped -- content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00", -- ) -- blob._extract_headers_from_download(response) -- -- self.assertEqual(blob.md5_hash, md5_hash) -- self.assertEqual(blob.crc32c, crc32c) -- -- def test_download_as_string_w_response_headers_not_match(self): -- blob_name = "blob-name" -- client = mock.Mock(spec=["_http"]) -- bucket = _Bucket(client) -- media_link = "http://example.com/media/" -- properties = {"mediaLink": media_link} -- blob = self._make_one(blob_name, bucket=bucket, properties=properties) -+ def test_download_as_bytes_wo_raw(self): -+ self._download_as_bytes_helper(raw_download=False) - -- response = self._mock_requests_response( -- http_client.OK, -- headers={"X-Goog-Hash": "bogus=4gcgLQ==,"}, -- # { "x": 5 } gzipped -- content=b"", -- ) -- blob._extract_headers_from_download(response) -+ def test_download_as_bytes_w_raw(self): -+ self._download_as_bytes_helper(raw_download=True) - -- self.assertIsNone(blob.md5_hash) -- self.assertIsNone(blob.crc32c) -+ def test_download_as_bytes_w_custom_timeout(self): -+ self._download_as_bytes_helper(raw_download=False, timeout=9.58) - - def test_download_as_bytes_w_generation_match(self): - GENERATION_NUMBER = 6 -@@ -1704,15 +1642,6 @@ class Test_Blob(unittest.TestCase): - checksum="md5", - ) - -- def test_download_as_bytes_wo_raw(self): -- self._download_as_bytes_helper(raw_download=False) -- -- def test_download_as_bytes_w_raw(self): -- self._download_as_bytes_helper(raw_download=True) -- -- def test_download_as_byte_w_custom_timeout(self): -- self._download_as_bytes_helper(raw_download=False, timeout=9.58) -- - def _download_as_text_helper( - self, - raw_download, -@@ -1737,7 +1666,8 @@ class Test_Blob(unittest.TestCase): - payload = expected_value.encode() - - blob_name = "blob-name" -- bucket = _Bucket() -+ bucket_client = self._make_client() -+ bucket = _Bucket(bucket_client) - - properties = {} - if charset is not None: -@@ -3146,139 +3076,128 @@ class Test_Blob(unittest.TestCase): - self.assertIn(message, exc_info.exception.message) - self.assertEqual(exc_info.exception.errors, []) - -- def test_get_iam_policy(self): -+ def test_get_iam_policy_defaults(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - -- BLOB_NAME = "blob-name" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- EDITOR1 = "domain:google.com" -- EDITOR2 = "user:phred@example.com" -- VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" -- VIEWER2 = "user:phred@example.com" -- RETURNED = { -- "resourceId": PATH, -- "etag": ETAG, -- "version": VERSION, -+ blob_name = "blob-name" -+ path = "/b/name/o/%s" % (blob_name,) -+ etag = "DEADBEEF" -+ version = 1 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ editor1 = "domain:google.com" -+ editor2 = "user:phred@example.com" -+ viewer1 = "serviceAccount:1234-abcdef@service.example.com" -+ viewer2 = "user:phred@example.com" -+ api_response = { -+ "resourceId": path, -+ "etag": etag, -+ "version": version, - "bindings": [ -- {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, -- {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, -- {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, -+ {"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}, -+ {"role": STORAGE_EDITOR_ROLE, "members": [editor1, editor2]}, -+ {"role": STORAGE_VIEWER_ROLE, "members": [viewer1, viewer2]}, - ], - } -- after = ({"status": http_client.OK}, RETURNED) -- EXPECTED = { -- binding["role"]: set(binding["members"]) for binding in RETURNED["bindings"] -+ expected_policy = { -+ binding["role"]: set(binding["members"]) -+ for binding in api_response["bindings"] - } -- connection = _Connection(after) -- client = _Client(connection) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) - -- policy = blob.get_iam_policy(timeout=42) -+ policy = blob.get_iam_policy() - - self.assertIsInstance(policy, Policy) -- self.assertEqual(policy.etag, RETURNED["etag"]) -- self.assertEqual(policy.version, RETURNED["version"]) -- self.assertEqual(dict(policy), EXPECTED) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "%s/iam" % (PATH,), -- "query_params": {}, -- "_target_object": None, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- }, -+ self.assertEqual(policy.etag, api_response["etag"]) -+ self.assertEqual(policy.version, api_response["version"]) -+ self.assertEqual(dict(policy), expected_policy) -+ -+ expected_path = "%s/iam" % (path,) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) - -- def test_get_iam_policy_w_requested_policy_version(self): -- from google.cloud.storage.iam import STORAGE_OWNER_ROLE -+ def test_get_iam_policy_w_user_project_w_timeout(self): -+ from google.api_core.iam import Policy - -- BLOB_NAME = "blob-name" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- RETURNED = { -- "resourceId": PATH, -- "etag": ETAG, -- "version": VERSION, -- "bindings": [{"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}], -+ blob_name = "blob-name" -+ user_project = "user-project-123" -+ timeout = 42 -+ path = "/b/name/o/%s" % (blob_name,) -+ etag = "DEADBEEF" -+ version = 1 -+ api_response = { -+ "resourceId": path, -+ "etag": etag, -+ "version": version, -+ "bindings": [], - } -- after = ({"status": http_client.OK}, RETURNED) -- connection = _Connection(after) -- client = _Client(connection) -- bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ expected_policy = {} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = _Bucket(client=client, user_project=user_project) -+ blob = self._make_one(blob_name, bucket=bucket) - -- blob.get_iam_policy(requested_policy_version=3) -+ policy = blob.get_iam_policy(timeout=42) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "%s/iam" % (PATH,), -- "query_params": {"optionsRequestedPolicyVersion": 3}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- }, -+ self.assertIsInstance(policy, Policy) -+ self.assertEqual(policy.etag, api_response["etag"]) -+ self.assertEqual(policy.version, api_response["version"]) -+ self.assertEqual(dict(policy), expected_policy) -+ -+ expected_path = "%s/iam" % (path,) -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) - -- def test_get_iam_policy_w_user_project(self): -- from google.api_core.iam import Policy -+ def test_get_iam_policy_w_requested_policy_version(self): -+ from google.cloud.storage.iam import STORAGE_OWNER_ROLE - -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- RETURNED = { -- "resourceId": PATH, -- "etag": ETAG, -- "version": VERSION, -- "bindings": [], -+ blob_name = "blob-name" -+ path = "/b/name/o/%s" % (blob_name,) -+ etag = "DEADBEEF" -+ version = 3 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ api_response = { -+ "resourceId": path, -+ "etag": etag, -+ "version": version, -+ "bindings": [{"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}], - } -- after = ({"status": http_client.OK}, RETURNED) -- EXPECTED = {} -- connection = _Connection(after) -- client = _Client(connection) -- bucket = _Bucket(client=client, user_project=USER_PROJECT) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = _Bucket(client=client) -+ blob = self._make_one(blob_name, bucket=bucket) - -- policy = blob.get_iam_policy() -+ policy = blob.get_iam_policy(requested_policy_version=version) - -- self.assertIsInstance(policy, Policy) -- self.assertEqual(policy.etag, RETURNED["etag"]) -- self.assertEqual(policy.version, RETURNED["version"]) -- self.assertEqual(dict(policy), EXPECTED) -+ self.assertEqual(policy.version, version) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "GET", -- "path": "%s/iam" % (PATH,), -- "query_params": {"userProject": USER_PROJECT}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- }, -+ expected_path = "%s/iam" % (path,) -+ expected_query_params = {"optionsRequestedPolicyVersion": version} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) - - def test_set_iam_policy(self): -@@ -3288,928 +3207,1073 @@ class Test_Blob(unittest.TestCase): - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - -- BLOB_NAME = "blob-name" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- EDITOR1 = "domain:google.com" -- EDITOR2 = "user:phred@example.com" -- VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" -- VIEWER2 = "user:phred@example.com" -- BINDINGS = [ -- {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, -- {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, -- {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, -+ blob_name = "blob-name" -+ path = "/b/name/o/%s" % (blob_name,) -+ etag = "DEADBEEF" -+ version = 1 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ editor1 = "domain:google.com" -+ editor2 = "user:phred@example.com" -+ viewer1 = "serviceAccount:1234-abcdef@service.example.com" -+ viewer2 = "user:phred@example.com" -+ bindings = [ -+ {"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}, -+ {"role": STORAGE_EDITOR_ROLE, "members": [editor1, editor2]}, -+ {"role": STORAGE_VIEWER_ROLE, "members": [viewer1, viewer2]}, - ] -- RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} -- after = ({"status": http_client.OK}, RETURNED) -+ api_response = {"etag": etag, "version": version, "bindings": bindings} - policy = Policy() -- for binding in BINDINGS: -+ for binding in bindings: - policy[binding["role"]] = binding["members"] - -- connection = _Connection(after) -- client = _Client(connection) -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) - -- returned = blob.set_iam_policy(policy, timeout=42) -+ returned = blob.set_iam_policy(policy) - -- self.assertEqual(returned.etag, ETAG) -- self.assertEqual(returned.version, VERSION) -+ self.assertEqual(returned.etag, etag) -+ self.assertEqual(returned.version, version) - self.assertEqual(dict(returned), dict(policy)) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {}) -- self.assertEqual(kw[0]["timeout"], 42) -- sent = kw[0]["data"] -- self.assertEqual(sent["resourceId"], PATH) -- self.assertEqual(len(sent["bindings"]), len(BINDINGS)) -+ expected_path = "%s/iam" % (path,) -+ expected_data = { -+ "resourceId": path, -+ "bindings": mock.ANY, -+ } -+ expected_query_params = {} -+ client._put_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_ETAG_IN_JSON, -+ _target_object=None, -+ ) -+ -+ sent_bindings = client._put_resource.call_args.args[1]["bindings"] - key = operator.itemgetter("role") - for found, expected in zip( -- sorted(sent["bindings"], key=key), sorted(BINDINGS, key=key) -+ sorted(sent_bindings, key=key), sorted(bindings, key=key) - ): - self.assertEqual(found["role"], expected["role"]) - self.assertEqual(sorted(found["members"]), sorted(expected["members"])) - -- def test_set_iam_policy_w_user_project(self): -+ def test_set_iam_policy_w_user_project_w_explicit_client_w_timeout_retry(self): - from google.api_core.iam import Policy - -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- BINDINGS = [] -- RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} -- after = ({"status": http_client.OK}, RETURNED) -+ blob_name = "blob-name" -+ user_project = "user-project-123" -+ path = "/b/name/o/%s" % (blob_name,) -+ etag = "DEADBEEF" -+ version = 1 -+ bindings = [] - policy = Policy() - -- connection = _Connection(after) -- client = _Client(connection) -- bucket = _Bucket(client=client, user_project=USER_PROJECT) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ api_response = {"etag": etag, "version": version, "bindings": bindings} -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response -+ bucket = _Bucket(client=None, user_project=user_project) -+ blob = self._make_one(blob_name, bucket=bucket) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- returned = blob.set_iam_policy(policy) -+ returned = blob.set_iam_policy( -+ policy, client=client, timeout=timeout, retry=retry, -+ ) - -- self.assertEqual(returned.etag, ETAG) -- self.assertEqual(returned.version, VERSION) -+ self.assertEqual(returned.etag, etag) -+ self.assertEqual(returned.version, version) - self.assertEqual(dict(returned), dict(policy)) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) -- self.assertEqual(kw[0]["data"], {"resourceId": PATH}) -+ expected_path = "%s/iam" % (path,) -+ expected_data = { # bindings omitted -+ "resourceId": path, -+ } -+ expected_query_params = {"userProject": user_project} -+ client._put_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=None, -+ ) - -- def test_test_iam_permissions(self): -+ def test_test_iam_permissions_defaults(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - -- BLOB_NAME = "blob-name" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- PERMISSIONS = [ -+ blob_name = "blob-name" -+ permissions = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] -- ALLOWED = PERMISSIONS[1:] -- RETURNED = {"permissions": ALLOWED} -- after = ({"status": http_client.OK}, RETURNED) -- connection = _Connection(after) -- client = _Client(connection) -+ expected = permissions[1:] -+ api_response = {"permissions": expected} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) - -- allowed = blob.test_iam_permissions(PERMISSIONS, timeout=42) -+ found = blob.test_iam_permissions(permissions) - -- self.assertEqual(allowed, ALLOWED) -+ self.assertEqual(found, expected) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {"permissions": PERMISSIONS}) -- self.assertEqual(kw[0]["timeout"], 42) -+ expected_path = "/b/name/o/%s/iam/testPermissions" % (blob_name,) -+ expected_query_params = {"permissions": permissions} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) - -- def test_test_iam_permissions_w_user_project(self): -+ def test_test_iam_permissions_w_user_project_w_timeout_w_retry(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- PATH = "/b/name/o/%s" % (BLOB_NAME,) -- PERMISSIONS = [ -+ blob_name = "blob-name" -+ user_project = "user-project-123" -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ permissions = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] -- ALLOWED = PERMISSIONS[1:] -- RETURNED = {"permissions": ALLOWED} -- after = ({"status": http_client.OK}, RETURNED) -- connection = _Connection(after) -- client = _Client(connection) -- bucket = _Bucket(client=client, user_project=USER_PROJECT) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ expected = permissions[1:] -+ api_response = {"permissions": expected} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = _Bucket(client=client, user_project=user_project) -+ blob = self._make_one(blob_name, bucket=bucket) - -- allowed = blob.test_iam_permissions(PERMISSIONS) -+ found = blob.test_iam_permissions(permissions, timeout=timeout, retry=retry) - -- self.assertEqual(allowed, ALLOWED) -+ self.assertEqual(found, expected) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) -- self.assertEqual( -- kw[0]["query_params"], -- {"permissions": PERMISSIONS, "userProject": USER_PROJECT}, -+ expected_path = "/b/name/o/%s/iam/testPermissions" % (blob_name,) -+ expected_query_params = { -+ "permissions": permissions, -+ "userProject": user_project, -+ } -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=None, - ) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - -- def test_make_public(self): -+ def test_make_public_w_defaults(self): - from google.cloud.storage.acl import _ACLEntity - -- BLOB_NAME = "blob-name" -+ blob_name = "blob-name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] -- after = ({"status": http_client.OK}, {"acl": permissive}) -- connection = _Connection(after) -- client = _Client(connection) -+ api_response = {"acl": permissive} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) - blob.acl.loaded = True -+ - blob.make_public() -+ - self.assertEqual(list(blob.acl), permissive) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/name/o/%s" % BLOB_NAME) -- self.assertEqual(kw[0]["data"], {"acl": permissive}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - -- def test_make_private(self): -- BLOB_NAME = "blob-name" -+ expected_patch_data = {"acl": permissive} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ blob.path, -+ expected_patch_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ def test_make_public_w_timeout(self): -+ from google.cloud.storage.acl import _ACLEntity -+ -+ blob_name = "blob-name" -+ permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] -+ api_response = {"acl": permissive} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ bucket = _Bucket(client=client) -+ blob = self._make_one(blob_name, bucket=bucket) -+ blob.acl.loaded = True -+ timeout = 42 -+ -+ blob.make_public(timeout=timeout) -+ -+ self.assertEqual(list(blob.acl), permissive) -+ -+ expected_patch_data = {"acl": permissive} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ blob.path, -+ expected_patch_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=None, -+ ) -+ -+ def test_make_private_w_defaults(self): -+ blob_name = "blob-name" - no_permissions = [] -- after = ({"status": http_client.OK}, {"acl": no_permissions}) -- connection = _Connection(after) -- client = _Client(connection) -+ api_response = {"acl": no_permissions} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) - blob.acl.loaded = True -+ - blob.make_private() -+ -+ self.assertEqual(list(blob.acl), no_permissions) -+ -+ expected_patch_data = {"acl": no_permissions} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ blob.path, -+ expected_patch_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ def test_make_private_w_timeout(self): -+ blob_name = "blob-name" -+ no_permissions = [] -+ api_response = {"acl": no_permissions} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ bucket = _Bucket(client=client) -+ blob = self._make_one(blob_name, bucket=bucket) -+ blob.acl.loaded = True -+ timeout = 42 -+ -+ blob.make_private(timeout=timeout) -+ - self.assertEqual(list(blob.acl), no_permissions) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/name/o/%s" % BLOB_NAME) -- self.assertEqual(kw[0]["data"], {"acl": no_permissions}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -+ -+ expected_patch_data = {"acl": no_permissions} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ blob.path, -+ expected_patch_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=None, -+ ) - - def test_compose_wo_content_type_set(self): -- SOURCE_1 = "source-1" -- SOURCE_2 = "source-2" -- DESTINATION = "destination" -- RESOURCE = {} -- after = ({"status": http_client.OK}, RESOURCE) -- connection = _Connection(after) -- client = _Client(connection) -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ api_response = {} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = _Bucket(client=client) -- source_1 = self._make_one(SOURCE_1, bucket=bucket) -- source_2 = self._make_one(SOURCE_2, bucket=bucket) -- destination = self._make_one(DESTINATION, bucket=bucket) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) - # no destination.content_type set - - destination.compose(sources=[source_1, source_2]) - - self.assertIsNone(destination.content_type) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "POST", -- "path": "/b/name/o/%s/compose" % DESTINATION, -- "query_params": {}, -- "data": { -- "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], -- "destination": {}, -- }, -- "_target_object": destination, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -- }, -+ expected_path = "/b/name/o/%s/compose" % destination_name -+ expected_data = { -+ "sourceObjects": [{"name": source_1_name}, {"name": source_2_name}], -+ "destination": {}, -+ } -+ expected_query_params = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=destination, - ) - -- def test_compose_minimal_w_user_project(self): -- SOURCE_1 = "source-1" -- SOURCE_2 = "source-2" -- DESTINATION = "destination" -- RESOURCE = {"etag": "DEADBEEF"} -- USER_PROJECT = "user-project-123" -- after = ({"status": http_client.OK}, RESOURCE) -- connection = _Connection(after) -- client = _Client(connection) -- bucket = _Bucket(client=client, user_project=USER_PROJECT) -- source_1 = self._make_one(SOURCE_1, bucket=bucket) -- source_2 = self._make_one(SOURCE_2, bucket=bucket) -- destination = self._make_one(DESTINATION, bucket=bucket) -+ def test_compose_minimal_w_user_project_w_timeout(self): -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ api_response = {"etag": "DEADBEEF"} -+ user_project = "user-project-123" -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response -+ bucket = _Bucket(client=client, user_project=user_project) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) - destination.content_type = "text/plain" -+ timeout = 42 - -- destination.compose(sources=[source_1, source_2], timeout=42) -+ destination.compose(sources=[source_1, source_2], timeout=timeout) - - self.assertEqual(destination.etag, "DEADBEEF") - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "POST", -- "path": "/b/name/o/%s/compose" % DESTINATION, -- "query_params": {"userProject": USER_PROJECT}, -- "data": { -- "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], -- "destination": {"contentType": "text/plain"}, -- }, -- "_target_object": destination, -- "timeout": 42, -- "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -- }, -+ expected_path = "/b/name/o/%s/compose" % destination_name -+ expected_data = { -+ "sourceObjects": [{"name": source_1_name}, {"name": source_2_name}], -+ "destination": {"contentType": "text/plain"}, -+ } -+ expected_query_params = {"userProject": user_project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=destination, - ) - -- def test_compose_w_additional_property_changes(self): -- SOURCE_1 = "source-1" -- SOURCE_2 = "source-2" -- DESTINATION = "destination" -- RESOURCE = {"etag": "DEADBEEF"} -- after = ({"status": http_client.OK}, RESOURCE) -- connection = _Connection(after) -- client = _Client(connection) -+ def test_compose_w_additional_property_changes_w_retry(self): -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ api_response = {"etag": "DEADBEEF"} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = _Bucket(client=client) -- source_1 = self._make_one(SOURCE_1, bucket=bucket) -- source_2 = self._make_one(SOURCE_2, bucket=bucket) -- destination = self._make_one(DESTINATION, bucket=bucket) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) - destination.content_type = "text/plain" - destination.content_language = "en-US" - destination.metadata = {"my-key": "my-value"} -+ retry = mock.Mock(spec=[]) - -- destination.compose(sources=[source_1, source_2]) -+ destination.compose(sources=[source_1, source_2], retry=retry) - - self.assertEqual(destination.etag, "DEADBEEF") - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "POST", -- "path": "/b/name/o/%s/compose" % DESTINATION, -- "query_params": {}, -- "data": { -- "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], -- "destination": { -- "contentType": "text/plain", -- "contentLanguage": "en-US", -- "metadata": {"my-key": "my-value"}, -- }, -- }, -- "_target_object": destination, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ expected_path = "/b/name/o/%s/compose" % destination_name -+ expected_data = { -+ "sourceObjects": [{"name": source_1_name}, {"name": source_2_name}], -+ "destination": { -+ "contentType": "text/plain", -+ "contentLanguage": "en-US", -+ "metadata": {"my-key": "my-value"}, - }, -+ } -+ expected_query_params = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=destination, - ) - - def test_compose_w_generation_match(self): -- SOURCE_1 = "source-1" -- SOURCE_2 = "source-2" -- DESTINATION = "destination" -- RESOURCE = {} -- GENERATION_NUMBERS = [6, 9] -- METAGENERATION_NUMBERS = [7, 1] -- -- after = ({"status": http_client.OK}, RESOURCE) -- connection = _Connection(after) -- client = _Client(connection) -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ api_response = {} -+ generation_numbers = [6, 9] -+ metageneration_numbers = [7, 1] -+ -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = _Bucket(client=client) -- source_1 = self._make_one(SOURCE_1, bucket=bucket) -- source_2 = self._make_one(SOURCE_2, bucket=bucket) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) - -- destination = self._make_one(DESTINATION, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) - destination.compose( - sources=[source_1, source_2], -- if_generation_match=GENERATION_NUMBERS, -- if_metageneration_match=METAGENERATION_NUMBERS, -+ if_generation_match=generation_numbers, -+ if_metageneration_match=metageneration_numbers, - ) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "POST", -- "path": "/b/name/o/%s/compose" % DESTINATION, -- "query_params": {}, -- "data": { -- "sourceObjects": [ -- { -- "name": source_1.name, -- "objectPreconditions": { -- "ifGenerationMatch": GENERATION_NUMBERS[0], -- "ifMetagenerationMatch": METAGENERATION_NUMBERS[0], -- }, -- }, -- { -- "name": source_2.name, -- "objectPreconditions": { -- "ifGenerationMatch": GENERATION_NUMBERS[1], -- "ifMetagenerationMatch": METAGENERATION_NUMBERS[1], -- }, -- }, -- ], -- "destination": {}, -+ expected_path = "/b/name/o/%s/compose" % destination_name -+ expected_data = { -+ "sourceObjects": [ -+ { -+ "name": source_1_name, -+ "objectPreconditions": { -+ "ifGenerationMatch": generation_numbers[0], -+ "ifMetagenerationMatch": metageneration_numbers[0], -+ }, - }, -- "_target_object": destination, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -- }, -+ { -+ "name": source_2_name, -+ "objectPreconditions": { -+ "ifGenerationMatch": generation_numbers[1], -+ "ifMetagenerationMatch": metageneration_numbers[1], -+ }, -+ }, -+ ], -+ "destination": {}, -+ } -+ expected_query_params = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=destination, - ) - - def test_compose_w_generation_match_bad_length(self): -- SOURCE_1 = "source-1" -- SOURCE_2 = "source-2" -- DESTINATION = "destination" -- GENERATION_NUMBERS = [6] -- METAGENERATION_NUMBERS = [7] -- -- after = ({"status": http_client.OK}, {}) -- connection = _Connection(after) -- client = _Client(connection) -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ generation_numbers = [6] -+ client = mock.Mock(spec=["_post_resource"]) - bucket = _Bucket(client=client) -- source_1 = self._make_one(SOURCE_1, bucket=bucket) -- source_2 = self._make_one(SOURCE_2, bucket=bucket) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) - -- destination = self._make_one(DESTINATION, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) - - with self.assertRaises(ValueError): - destination.compose( -- sources=[source_1, source_2], if_generation_match=GENERATION_NUMBERS -+ sources=[source_1, source_2], if_generation_match=generation_numbers - ) -+ -+ client._post_resource.assert_not_called() -+ -+ def test_compose_w_metageneration_match_bad_length(self): -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ metageneration_numbers = [7] -+ client = mock.Mock(spec=["_post_resource"]) -+ bucket = _Bucket(client=client) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) -+ - with self.assertRaises(ValueError): - destination.compose( - sources=[source_1, source_2], -- if_metageneration_match=METAGENERATION_NUMBERS, -+ if_metageneration_match=metageneration_numbers, - ) - -+ client._post_resource.assert_not_called() -+ - def test_compose_w_generation_match_nones(self): -- SOURCE_1 = "source-1" -- SOURCE_2 = "source-2" -- DESTINATION = "destination" -- GENERATION_NUMBERS = [6, None] -- -- after = ({"status": http_client.OK}, {}) -- connection = _Connection(after) -- client = _Client(connection) -+ source_1_name = "source-1" -+ source_2_name = "source-2" -+ destination_name = "destination" -+ generation_numbers = [6, None] -+ api_response = {} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = _Bucket(client=client) -- source_1 = self._make_one(SOURCE_1, bucket=bucket) -- source_2 = self._make_one(SOURCE_2, bucket=bucket) -+ source_1 = self._make_one(source_1_name, bucket=bucket) -+ source_2 = self._make_one(source_2_name, bucket=bucket) -+ destination = self._make_one(destination_name, bucket=bucket) - -- destination = self._make_one(DESTINATION, bucket=bucket) - destination.compose( -- sources=[source_1, source_2], if_generation_match=GENERATION_NUMBERS -+ sources=[source_1, source_2], if_generation_match=generation_numbers - ) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual( -- kw[0], -- { -- "method": "POST", -- "path": "/b/name/o/%s/compose" % DESTINATION, -- "query_params": {}, -- "data": { -- "sourceObjects": [ -- { -- "name": source_1.name, -- "objectPreconditions": { -- "ifGenerationMatch": GENERATION_NUMBERS[0] -- }, -- }, -- {"name": source_2.name}, -- ], -- "destination": {}, -+ expected_path = "/b/name/o/%s/compose" % destination_name -+ expected_data = { -+ "sourceObjects": [ -+ { -+ "name": source_1_name, -+ "objectPreconditions": { -+ "ifGenerationMatch": generation_numbers[0], -+ }, - }, -- "_target_object": destination, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -- }, -+ {"name": source_2_name}, -+ ], -+ "destination": {}, -+ } -+ expected_query_params = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=destination, - ) - -- def test_rewrite_response_without_resource(self): -- SOURCE_BLOB = "source" -- DEST_BLOB = "dest" -- DEST_BUCKET = "other-bucket" -- TOKEN = "TOKEN" -- RESPONSE = { -- "totalBytesRewritten": 33, -- "objectSize": 42, -+ def test_rewrite_w_response_wo_resource(self): -+ source_name = "source" -+ dest_name = "dest" -+ other_bucket_name = "other-bucket" -+ bytes_rewritten = 33 -+ object_size = 52 -+ rewrite_token = "TOKEN" -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": False, -- "rewriteToken": TOKEN, -+ "rewriteToken": rewrite_token, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - source_bucket = _Bucket(client=client) -- source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) -- dest_bucket = _Bucket(client=client, name=DEST_BUCKET) -- dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) -+ source_blob = self._make_one(source_name, bucket=source_bucket) -+ dest_bucket = _Bucket(client=client, name=other_bucket_name) -+ dest_blob = self._make_one(dest_name, bucket=dest_bucket) - - token, rewritten, size = dest_blob.rewrite(source_blob) - -- self.assertEqual(token, TOKEN) -- self.assertEqual(rewritten, 33) -- self.assertEqual(size, 42) -- -- def test_rewrite_w_generations(self): -- SOURCE_BLOB = "source" -- SOURCE_GENERATION = 42 -- DEST_BLOB = "dest" -- DEST_BUCKET = "other-bucket" -- DEST_GENERATION = 43 -- TOKEN = "TOKEN" -- RESPONSE = { -- "totalBytesRewritten": 33, -- "objectSize": 42, -+ self.assertEqual(token, rewrite_token) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) -+ -+ expected_path = "/b/%s/o/%s/rewriteTo/b/%s/o/%s" % ( -+ source_bucket.name, -+ source_name, -+ other_bucket_name, -+ dest_name, -+ ) -+ expected_data = {} -+ expected_query_params = {} -+ expected_headers = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=dest_blob, -+ ) -+ -+ def test_rewrite_w_generations_w_timeout(self): -+ source_name = "source" -+ source_generation = 22 -+ dest_name = "dest" -+ other_bucket_name = "other-bucket" -+ dest_generation = 23 -+ bytes_rewritten = 33 -+ object_size = 52 -+ rewrite_token = "TOKEN" -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": False, -- "rewriteToken": TOKEN, -+ "rewriteToken": rewrite_token, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - source_bucket = _Bucket(client=client) - source_blob = self._make_one( -- SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION -+ source_name, bucket=source_bucket, generation=source_generation - ) -- dest_bucket = _Bucket(client=client, name=DEST_BUCKET) -+ dest_bucket = _Bucket(client=client, name=other_bucket_name) - dest_blob = self._make_one( -- DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION -+ dest_name, bucket=dest_bucket, generation=dest_generation - ) -+ timeout = 42 - -- token, rewritten, size = dest_blob.rewrite(source_blob, timeout=42) -+ token, rewritten, size = dest_blob.rewrite(source_blob, timeout=timeout) - -- self.assertEqual(token, TOKEN) -- self.assertEqual(rewritten, 33) -- self.assertEqual(size, 42) -+ self.assertEqual(token, rewrite_token) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) -+ -+ expected_path = "/b/%s/o/%s/rewriteTo/b/%s/o/%s" % ( -+ source_bucket.name, -+ source_name, -+ other_bucket_name, -+ dest_name, -+ ) -+ expected_data = {"generation": dest_generation} -+ expected_query_params = {"sourceGeneration": source_generation} -+ expected_headers = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=timeout, -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=dest_blob, -+ ) - -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual( -- kw["path"], -- "/b/%s/o/%s/rewriteTo/b/%s/o/%s" -- % ( -- (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) -- ), -- ) -- self.assertEqual(kw["query_params"], {"sourceGeneration": SOURCE_GENERATION}) -- self.assertEqual(kw["timeout"], 42) -- -- def test_rewrite_w_generation_match(self): -- SOURCE_BLOB = "source" -- SOURCE_GENERATION_NUMBER = 42 -- DEST_BLOB = "dest" -- DEST_BUCKET = "other-bucket" -- DEST_GENERATION_NUMBER = 16 -- TOKEN = "TOKEN" -- RESPONSE = { -- "totalBytesRewritten": 33, -- "objectSize": 42, -+ def test_rewrite_w_generation_match_w_retry(self): -+ source_name = "source" -+ source_generation = 42 -+ dest_name = "dest" -+ other_bucket_name = "other-bucket" -+ dest_generation = 16 -+ bytes_rewritten = 33 -+ object_size = 52 -+ rewrite_token = "TOKEN" -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": False, -- "rewriteToken": TOKEN, -+ "rewriteToken": rewrite_token, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - source_bucket = _Bucket(client=client) - source_blob = self._make_one( -- SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION_NUMBER -+ source_name, bucket=source_bucket, generation=source_generation - ) -- dest_bucket = _Bucket(client=client, name=DEST_BUCKET) -+ dest_bucket = _Bucket(client=client, name=other_bucket_name) - dest_blob = self._make_one( -- DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION_NUMBER -+ dest_name, bucket=dest_bucket, generation=dest_generation - ) -+ retry = mock.Mock(spec=[]) -+ - token, rewritten, size = dest_blob.rewrite( - source_blob, -- timeout=42, - if_generation_match=dest_blob.generation, - if_source_generation_match=source_blob.generation, -+ retry=retry, - ) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual( -- kw["path"], -- "/b/%s/o/%s/rewriteTo/b/%s/o/%s" -- % ( -- (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) -- ), -- ) -- self.assertEqual( -- kw["query_params"], -- { -- "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, -- "ifGenerationMatch": DEST_GENERATION_NUMBER, -- "sourceGeneration": SOURCE_GENERATION_NUMBER, -- }, -+ -+ self.assertEqual(token, rewrite_token) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) -+ -+ expected_path = "/b/%s/o/%s/rewriteTo/b/%s/o/%s" % ( -+ source_bucket.name, -+ source_name, -+ other_bucket_name, -+ dest_name, -+ ) -+ expected_data = {"generation": dest_generation} -+ expected_query_params = { -+ "ifSourceGenerationMatch": source_generation, -+ "ifGenerationMatch": dest_generation, -+ "sourceGeneration": source_generation, -+ } -+ expected_headers = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=dest_blob, - ) -- self.assertEqual(kw["timeout"], 42) - - def test_rewrite_other_bucket_other_name_no_encryption_partial(self): -- SOURCE_BLOB = "source" -- DEST_BLOB = "dest" -- DEST_BUCKET = "other-bucket" -- TOKEN = "TOKEN" -- RESPONSE = { -- "totalBytesRewritten": 33, -- "objectSize": 42, -+ source_name = "source" -+ dest_name = "dest" -+ other_bucket_name = "other-bucket" -+ bytes_rewritten = 33 -+ object_size = 52 -+ rewrite_token = "TOKEN" -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": False, -- "rewriteToken": TOKEN, -- "resource": {"etag": "DEADBEEF"}, -+ "rewriteToken": rewrite_token, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - source_bucket = _Bucket(client=client) -- source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) -- dest_bucket = _Bucket(client=client, name=DEST_BUCKET) -- dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) -+ source_blob = self._make_one(source_name, bucket=source_bucket) -+ dest_bucket = _Bucket(client=client, name=other_bucket_name) -+ dest_blob = self._make_one(dest_name, bucket=dest_bucket) - - token, rewritten, size = dest_blob.rewrite(source_blob) - -- self.assertEqual(token, TOKEN) -- self.assertEqual(rewritten, 33) -- self.assertEqual(size, 42) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/%s/o/%s" % ( -- SOURCE_BLOB, -- DEST_BUCKET, -- DEST_BLOB, -- ) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual(kw[0]["query_params"], {}) -- SENT = {} -- self.assertEqual(kw[0]["data"], SENT) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- -- headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) -- self.assertNotIn("X-Goog-Encryption-Algorithm", headers) -- self.assertNotIn("X-Goog-Encryption-Key", headers) -- self.assertNotIn("X-Goog-Encryption-Key-Sha256", headers) -+ self.assertEqual(token, rewrite_token) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) -+ -+ expected_path = "/b/name/o/%s/rewriteTo/b/%s/o/%s" % ( -+ source_name, -+ other_bucket_name, -+ dest_name, -+ ) -+ expected_query_params = {} -+ expected_data = {} -+ expected_headers = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=dest_blob, -+ ) - - def test_rewrite_same_name_no_old_key_new_key_done_w_user_project(self): -- KEY = b"01234567890123456789012345678901" # 32 bytes -- KEY_B64 = base64.b64encode(KEY).rstrip().decode("ascii") -- KEY_HASH = hashlib.sha256(KEY).digest() -- KEY_HASH_B64 = base64.b64encode(KEY_HASH).rstrip().decode("ascii") -- BLOB_NAME = "blob" -- USER_PROJECT = "user-project-123" -- RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 42, -+ blob_name = "blob" -+ user_project = "user-project-123" -+ key = b"01234567890123456789012345678901" # 32 bytes -+ key_b64 = base64.b64encode(key).rstrip().decode("ascii") -+ key_hash = hashlib.sha256(key).digest() -+ key_hash_b64 = base64.b64encode(key_hash).rstrip().decode("ascii") -+ bytes_rewritten = object_size = 52 -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": True, - "resource": {"etag": "DEADBEEF"}, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -- bucket = _Bucket(client=client, user_project=USER_PROJECT) -- plain = self._make_one(BLOB_NAME, bucket=bucket) -- encrypted = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=KEY) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response -+ bucket = _Bucket(client=client, user_project=user_project) -+ plain = self._make_one(blob_name, bucket=bucket) -+ encrypted = self._make_one(blob_name, bucket=bucket, encryption_key=key) - - token, rewritten, size = encrypted.rewrite(plain) - - self.assertIsNone(token) -- self.assertEqual(rewritten, 42) -- self.assertEqual(size, 42) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) -- SENT = {} -- self.assertEqual(kw[0]["data"], SENT) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- -- headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) -- self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") -- self.assertEqual(headers["X-Goog-Encryption-Key"], KEY_B64) -- self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], KEY_HASH_B64) -+ expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) -+ expected_query_params = {"userProject": user_project} -+ expected_data = {} -+ expected_headers = { -+ "X-Goog-Encryption-Algorithm": "AES256", -+ "X-Goog-Encryption-Key": key_b64, -+ "X-Goog-Encryption-Key-Sha256": key_hash_b64, -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=encrypted, -+ ) - - def test_rewrite_same_name_no_key_new_key_w_token(self): -- SOURCE_KEY = b"01234567890123456789012345678901" # 32 bytes -- SOURCE_KEY_B64 = base64.b64encode(SOURCE_KEY).rstrip().decode("ascii") -- SOURCE_KEY_HASH = hashlib.sha256(SOURCE_KEY).digest() -- SOURCE_KEY_HASH_B64 = base64.b64encode(SOURCE_KEY_HASH).rstrip().decode("ascii") -- DEST_KEY = b"90123456789012345678901234567890" # 32 bytes -- DEST_KEY_B64 = base64.b64encode(DEST_KEY).rstrip().decode("ascii") -- DEST_KEY_HASH = hashlib.sha256(DEST_KEY).digest() -- DEST_KEY_HASH_B64 = base64.b64encode(DEST_KEY_HASH).rstrip().decode("ascii") -- BLOB_NAME = "blob" -- TOKEN = "TOKEN" -- RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 42, -+ blob_name = "blob" -+ source_key = b"01234567890123456789012345678901" # 32 bytes -+ source_key_b64 = base64.b64encode(source_key).rstrip().decode("ascii") -+ source_key_hash = hashlib.sha256(source_key).digest() -+ source_key_hash_b64 = base64.b64encode(source_key_hash).rstrip().decode("ascii") -+ dest_key = b"90123456789012345678901234567890" # 32 bytes -+ dest_key_b64 = base64.b64encode(dest_key).rstrip().decode("ascii") -+ dest_key_hash = hashlib.sha256(dest_key).digest() -+ dest_key_hash_b64 = base64.b64encode(dest_key_hash).rstrip().decode("ascii") -+ previous_token = "TOKEN" -+ bytes_rewritten = object_size = 52 -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": True, - "resource": {"etag": "DEADBEEF"}, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = _Bucket(client=client) -- source = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=SOURCE_KEY) -- dest = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=DEST_KEY) -+ source = self._make_one(blob_name, bucket=bucket, encryption_key=source_key) -+ dest = self._make_one(blob_name, bucket=bucket, encryption_key=dest_key) - -- token, rewritten, size = dest.rewrite(source, token=TOKEN) -+ token, rewritten, size = dest.rewrite(source, token=previous_token) - - self.assertIsNone(token) -- self.assertEqual(rewritten, 42) -- self.assertEqual(size, 42) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual(kw[0]["query_params"], {"rewriteToken": TOKEN}) -- SENT = {} -- self.assertEqual(kw[0]["data"], SENT) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- -- headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} -- self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") -- self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], SOURCE_KEY_B64) -- self.assertEqual( -- headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], SOURCE_KEY_HASH_B64 -+ expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) -+ expected_data = {} -+ expected_query_params = {"rewriteToken": previous_token} -+ expected_headers = { -+ "X-Goog-Copy-Source-Encryption-Algorithm": "AES256", -+ "X-Goog-Copy-Source-Encryption-Key": source_key_b64, -+ "X-Goog-Copy-Source-Encryption-Key-Sha256": source_key_hash_b64, -+ "X-Goog-Encryption-Algorithm": "AES256", -+ "X-Goog-Encryption-Key": dest_key_b64, -+ "X-Goog-Encryption-Key-Sha256": dest_key_hash_b64, -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=dest, - ) -- self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") -- self.assertEqual(headers["X-Goog-Encryption-Key"], DEST_KEY_B64) -- self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], DEST_KEY_HASH_B64) - - def test_rewrite_same_name_w_old_key_new_kms_key(self): -- SOURCE_KEY = b"01234567890123456789012345678901" # 32 bytes -- SOURCE_KEY_B64 = base64.b64encode(SOURCE_KEY).rstrip().decode("ascii") -- SOURCE_KEY_HASH = hashlib.sha256(SOURCE_KEY).digest() -- SOURCE_KEY_HASH_B64 = base64.b64encode(SOURCE_KEY_HASH).rstrip().decode("ascii") -- DEST_KMS_RESOURCE = ( -+ blob_name = "blob" -+ source_key = b"01234567890123456789012345678901" # 32 bytes -+ source_key_b64 = base64.b64encode(source_key).rstrip().decode("ascii") -+ source_key_hash = hashlib.sha256(source_key).digest() -+ source_key_hash_b64 = base64.b64encode(source_key_hash).rstrip().decode("ascii") -+ dest_kms_resource = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) -- BLOB_NAME = "blob" -- RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 42, -+ bytes_rewritten = object_size = 42 -+ api_response = { -+ "totalBytesRewritten": bytes_rewritten, -+ "objectSize": object_size, - "done": True, - "resource": {"etag": "DEADBEEF"}, - } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = _Bucket(client=client) -- source = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=SOURCE_KEY) -- dest = self._make_one(BLOB_NAME, bucket=bucket, kms_key_name=DEST_KMS_RESOURCE) -+ source = self._make_one(blob_name, bucket=bucket, encryption_key=source_key) -+ dest = self._make_one(blob_name, bucket=bucket, kms_key_name=dest_kms_resource) - - token, rewritten, size = dest.rewrite(source) - - self.assertIsNone(token) -- self.assertEqual(rewritten, 42) -- self.assertEqual(size, 42) -+ self.assertEqual(rewritten, bytes_rewritten) -+ self.assertEqual(size, object_size) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual( -- kw[0]["query_params"], {"destinationKmsKeyName": DEST_KMS_RESOURCE} -- ) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- SENT = {"kmsKeyName": DEST_KMS_RESOURCE} -- self.assertEqual(kw[0]["data"], SENT) -- -- headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} -- self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") -- self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], SOURCE_KEY_B64) -- self.assertEqual( -- headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], SOURCE_KEY_HASH_B64 -+ expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) -+ expected_data = {"kmsKeyName": dest_kms_resource} -+ expected_query_params = {"destinationKmsKeyName": dest_kms_resource} -+ expected_headers = { -+ "X-Goog-Copy-Source-Encryption-Algorithm": "AES256", -+ "X-Goog-Copy-Source-Encryption-Key": source_key_b64, -+ "X-Goog-Copy-Source-Encryption-Key-Sha256": source_key_hash_b64, -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=dest, - ) - - def test_update_storage_class_invalid(self): -- BLOB_NAME = "blob-name" -+ blob_name = "blob-name" - bucket = _Bucket() -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) -+ blob.rewrite = mock.Mock(spec=[]) -+ - with self.assertRaises(ValueError): - blob.update_storage_class(u"BOGUS") - -- def test_update_storage_class_large_file(self): -- BLOB_NAME = "blob-name" -- STORAGE_CLASS = u"NEARLINE" -- TOKEN = "TOKEN" -- INCOMPLETE_RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 84, -- "done": False, -- "rewriteToken": TOKEN, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- COMPLETE_RESPONSE = { -- "totalBytesRewritten": 84, -- "objectSize": 84, -- "done": True, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- response_1 = ({"status": http_client.OK}, INCOMPLETE_RESPONSE) -- response_2 = ({"status": http_client.OK}, COMPLETE_RESPONSE) -- connection = _Connection(response_1, response_2) -- client = _Client(connection) -+ blob.rewrite.assert_not_called() -+ -+ def _update_storage_class_multi_pass_helper(self, **kw): -+ blob_name = "blob-name" -+ storage_class = u"NEARLINE" -+ rewrite_token = "TOKEN" -+ bytes_rewritten = 42 -+ object_size = 84 -+ client = mock.Mock(spec=[]) - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) -+ blob.rewrite = mock.Mock(spec=[]) -+ blob.rewrite.side_effect = [ -+ (rewrite_token, bytes_rewritten, object_size), -+ (None, object_size, object_size), -+ ] - -- blob.update_storage_class("NEARLINE") -+ expected_i_g_m = kw.get("if_generation_match") -+ expected_i_g_n_m = kw.get("if_generation_not_match") -+ expected_i_m_m = kw.get("if_metageneration_match") -+ expected_i_m_n_m = kw.get("if_metageneration_not_match") -+ expected_i_s_g_m = kw.get("if_source_generation_match") -+ expected_i_s_g_n_m = kw.get("if_source_generation_not_match") -+ expected_i_s_m_m = kw.get("if_source_metageneration_match") -+ expected_i_s_m_n_m = kw.get("if_source_metageneration_not_match") -+ expected_timeout = kw.get("timeout", self._get_default_timeout()) -+ expected_retry = kw.get("retry", DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - -- self.assertEqual(blob.storage_class, "NEARLINE") -+ blob.update_storage_class(storage_class, **kw) - -- def test_update_storage_class_with_custom_timeout(self): -- BLOB_NAME = "blob-name" -- STORAGE_CLASS = u"NEARLINE" -- TOKEN = "TOKEN" -- INCOMPLETE_RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 84, -- "done": False, -- "rewriteToken": TOKEN, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- COMPLETE_RESPONSE = { -- "totalBytesRewritten": 84, -- "objectSize": 84, -- "done": True, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- response_1 = ({"status": http_client.OK}, INCOMPLETE_RESPONSE) -- response_2 = ({"status": http_client.OK}, COMPLETE_RESPONSE) -- connection = _Connection(response_1, response_2) -- client = _Client(connection) -- bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ self.assertEqual(blob.storage_class, storage_class) - -- blob.update_storage_class("NEARLINE", timeout=9.58) -+ call1 = mock.call( -+ blob, -+ if_generation_match=expected_i_g_m, -+ if_generation_not_match=expected_i_g_n_m, -+ if_metageneration_match=expected_i_m_m, -+ if_metageneration_not_match=expected_i_m_n_m, -+ if_source_generation_match=expected_i_s_g_m, -+ if_source_generation_not_match=expected_i_s_g_n_m, -+ if_source_metageneration_match=expected_i_s_m_m, -+ if_source_metageneration_not_match=expected_i_s_m_n_m, -+ timeout=expected_timeout, -+ retry=expected_retry, -+ ) -+ call2 = mock.call( -+ blob, -+ token=rewrite_token, -+ if_generation_match=expected_i_g_m, -+ if_generation_not_match=expected_i_g_n_m, -+ if_metageneration_match=expected_i_m_m, -+ if_metageneration_not_match=expected_i_m_n_m, -+ if_source_generation_match=expected_i_s_g_m, -+ if_source_generation_not_match=expected_i_s_g_n_m, -+ if_source_metageneration_match=expected_i_s_m_m, -+ if_source_metageneration_not_match=expected_i_s_m_n_m, -+ timeout=expected_timeout, -+ retry=expected_retry, -+ ) -+ blob.rewrite.assert_has_calls([call1, call2]) - -- self.assertEqual(blob.storage_class, "NEARLINE") -+ def test_update_storage_class_multi_pass_w_defaults(self): -+ self._update_storage_class_multi_pass_helper() - -- kw = connection._requested -- self.assertEqual(len(kw), 2) -+ def test_update_storage_class_multi_pass_w_i_g_m(self): -+ generation = 16 -+ self._update_storage_class_multi_pass_helper(if_generation_match=generation) - -- for kw_item in kw: -- self.assertIn("timeout", kw_item) -- self.assertEqual(kw_item["timeout"], 9.58) -- -- def test_update_storage_class_wo_encryption_key(self): -- BLOB_NAME = "blob-name" -- STORAGE_CLASS = u"NEARLINE" -- RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 42, -- "done": True, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -- bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ def test_update_storage_class_multi_pass_w_i_g_n_m(self): -+ generation = 16 -+ self._update_storage_class_multi_pass_helper(if_generation_not_match=generation) - -- blob.update_storage_class("NEARLINE") -+ def test_update_storage_class_multi_pass_w_i_m_m(self): -+ metageneration = 16 -+ self._update_storage_class_multi_pass_helper( -+ if_metageneration_match=metageneration, -+ ) - -- self.assertEqual(blob.storage_class, "NEARLINE") -+ def test_update_storage_class_multi_pass_w_i_m_n_m(self): -+ metageneration = 16 -+ self._update_storage_class_multi_pass_helper( -+ if_metageneration_not_match=metageneration, -+ ) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual(kw[0]["query_params"], {}) -- SENT = {"storageClass": STORAGE_CLASS} -- self.assertEqual(kw[0]["data"], SENT) -- -- headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} -- # Blob has no key, and therefore the relevant headers are not sent. -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) -- self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) -- self.assertNotIn("X-Goog-Encryption-Algorithm", headers) -- self.assertNotIn("X-Goog-Encryption-Key", headers) -- self.assertNotIn("X-Goog-Encryption-Key-Sha256", headers) -- -- def test_update_storage_class_w_encryption_key_w_user_project(self): -- BLOB_NAME = "blob-name" -- BLOB_KEY = b"01234567890123456789012345678901" # 32 bytes -- BLOB_KEY_B64 = base64.b64encode(BLOB_KEY).rstrip().decode("ascii") -- BLOB_KEY_HASH = hashlib.sha256(BLOB_KEY).digest() -- BLOB_KEY_HASH_B64 = base64.b64encode(BLOB_KEY_HASH).rstrip().decode("ascii") -- STORAGE_CLASS = u"NEARLINE" -- USER_PROJECT = "user-project-123" -- RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 42, -- "done": True, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -- bucket = _Bucket(client=client, user_project=USER_PROJECT) -- blob = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=BLOB_KEY) -+ def test_update_storage_class_multi_pass_w_i_s_g_m(self): -+ generation = 16 -+ self._update_storage_class_multi_pass_helper( -+ if_source_generation_match=generation -+ ) - -- blob.update_storage_class("NEARLINE") -+ def test_update_storage_class_multi_pass_w_i_s_g_n_m(self): -+ generation = 16 -+ self._update_storage_class_multi_pass_helper( -+ if_source_generation_not_match=generation -+ ) - -- self.assertEqual(blob.storage_class, "NEARLINE") -+ def test_update_storage_class_multi_pass_w_i_s_m_m(self): -+ metageneration = 16 -+ self._update_storage_class_multi_pass_helper( -+ if_source_metageneration_match=metageneration, -+ ) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) -- SENT = {"storageClass": STORAGE_CLASS} -- self.assertEqual(kw[0]["data"], SENT) -- -- headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} -- # Blob has key, and therefore the relevant headers are sent. -- self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") -- self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], BLOB_KEY_B64) -- self.assertEqual( -- headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64 -+ def test_update_storage_class_multi_pass_w_i_s_m_n_m(self): -+ metageneration = 16 -+ self._update_storage_class_multi_pass_helper( -+ if_source_metageneration_not_match=metageneration, - ) -- self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") -- self.assertEqual(headers["X-Goog-Encryption-Key"], BLOB_KEY_B64) -- self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64) - -- def test_update_storage_class_w_generation_match(self): -- BLOB_NAME = "blob-name" -- STORAGE_CLASS = u"NEARLINE" -- GENERATION_NUMBER = 6 -- SOURCE_GENERATION_NUMBER = 9 -- RESPONSE = { -- "totalBytesRewritten": 42, -- "objectSize": 42, -- "done": True, -- "resource": {"storageClass": STORAGE_CLASS}, -- } -- response = ({"status": http_client.OK}, RESPONSE) -- connection = _Connection(response) -- client = _Client(connection) -+ def test_update_storage_class_multi_pass_w_timeout(self): -+ timeout = 42 -+ self._update_storage_class_multi_pass_helper(timeout=timeout) -+ -+ def test_update_storage_class_multi_pass_w_retry(self): -+ retry = mock.Mock(spec=[]) -+ self._update_storage_class_multi_pass_helper(retry=retry) -+ -+ def _update_storage_class_single_pass_helper(self, **kw): -+ blob_name = "blob-name" -+ storage_class = u"NEARLINE" -+ object_size = 84 -+ client = mock.Mock(spec=[]) - bucket = _Bucket(client=client) -- blob = self._make_one(BLOB_NAME, bucket=bucket) -+ blob = self._make_one(blob_name, bucket=bucket) -+ blob.rewrite = mock.Mock(spec=[]) -+ blob.rewrite.return_value = (None, object_size, object_size) - -- blob.update_storage_class( -- "NEARLINE", -- if_generation_match=GENERATION_NUMBER, -- if_source_generation_match=SOURCE_GENERATION_NUMBER, -+ expected_i_g_m = kw.get("if_generation_match") -+ expected_i_g_n_m = kw.get("if_generation_not_match") -+ expected_i_m_m = kw.get("if_metageneration_match") -+ expected_i_m_n_m = kw.get("if_metageneration_not_match") -+ expected_i_s_g_m = kw.get("if_source_generation_match") -+ expected_i_s_g_n_m = kw.get("if_source_generation_not_match") -+ expected_i_s_m_m = kw.get("if_source_metageneration_match") -+ expected_i_s_m_n_m = kw.get("if_source_metageneration_not_match") -+ expected_timeout = kw.get("timeout", self._get_default_timeout()) -+ expected_retry = kw.get("retry", DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ -+ blob.update_storage_class(storage_class, **kw) -+ -+ self.assertEqual(blob.storage_class, storage_class) -+ -+ blob.rewrite.assert_called_once_with( -+ blob, -+ if_generation_match=expected_i_g_m, -+ if_generation_not_match=expected_i_g_n_m, -+ if_metageneration_match=expected_i_m_m, -+ if_metageneration_not_match=expected_i_m_n_m, -+ if_source_generation_match=expected_i_s_g_m, -+ if_source_generation_not_match=expected_i_s_g_n_m, -+ if_source_metageneration_match=expected_i_s_m_m, -+ if_source_metageneration_not_match=expected_i_s_m_n_m, -+ timeout=expected_timeout, -+ retry=expected_retry, - ) - -- self.assertEqual(blob.storage_class, "NEARLINE") -+ def test_update_storage_class_single_pass_w_defaults(self): -+ self._update_storage_class_single_pass_helper() - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "POST") -- PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) -- self.assertEqual(kw[0]["path"], PATH) -- self.assertEqual( -- kw[0]["query_params"], -- { -- "ifGenerationMatch": GENERATION_NUMBER, -- "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, -- }, -+ def test_update_storage_class_single_pass_w_i_g_m(self): -+ generation = 16 -+ self._update_storage_class_single_pass_helper(if_generation_match=generation) -+ -+ def test_update_storage_class_single_pass_w_i_g_n_m(self): -+ generation = 16 -+ self._update_storage_class_single_pass_helper( -+ if_generation_not_match=generation -+ ) -+ -+ def test_update_storage_class_single_pass_w_i_m_m(self): -+ metageneration = 16 -+ self._update_storage_class_single_pass_helper( -+ if_metageneration_match=metageneration, -+ ) -+ -+ def test_update_storage_class_single_pass_w_i_m_n_m(self): -+ metageneration = 16 -+ self._update_storage_class_single_pass_helper( -+ if_metageneration_not_match=metageneration, -+ ) -+ -+ def test_update_storage_class_single_pass_w_i_s_g_m(self): -+ generation = 16 -+ self._update_storage_class_single_pass_helper( -+ if_source_generation_match=generation - ) -- SENT = {"storageClass": STORAGE_CLASS} -- self.assertEqual(kw[0]["data"], SENT) -+ -+ def test_update_storage_class_single_pass_w_i_s_g_n_m(self): -+ generation = 16 -+ self._update_storage_class_single_pass_helper( -+ if_source_generation_not_match=generation -+ ) -+ -+ def test_update_storage_class_single_pass_w_i_s_m_m(self): -+ metageneration = 16 -+ self._update_storage_class_single_pass_helper( -+ if_source_metageneration_match=metageneration, -+ ) -+ -+ def test_update_storage_class_single_pass_w_i_s_m_n_m(self): -+ metageneration = 16 -+ self._update_storage_class_single_pass_helper( -+ if_source_metageneration_not_match=metageneration, -+ ) -+ -+ def test_update_storage_class_single_pass_w_timeout(self): -+ timeout = 42 -+ self._update_storage_class_single_pass_helper(timeout=timeout) -+ -+ def test_update_storage_class_single_pass_w_retry(self): -+ retry = mock.Mock(spec=[]) -+ self._update_storage_class_single_pass_helper(retry=retry) - - def test_cache_control_getter(self): - BLOB_NAME = "blob-name" -@@ -4677,8 +4741,7 @@ class Test_Blob(unittest.TestCase): - def test_from_string_w_valid_uri(self): - from google.cloud.storage.blob import Blob - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - uri = "gs://BUCKET_NAME/b" - blob = Blob.from_string(uri, client) - -@@ -4690,8 +4753,7 @@ class Test_Blob(unittest.TestCase): - def test_from_string_w_invalid_uri(self): - from google.cloud.storage.blob import Blob - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - - with pytest.raises(ValueError, match="URI scheme must be gs"): - Blob.from_string("http://bucket_name/b", client) -@@ -4699,8 +4761,7 @@ class Test_Blob(unittest.TestCase): - def test_from_string_w_domain_name_bucket(self): - from google.cloud.storage.blob import Blob - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - uri = "gs://buckets.example.com/b" - blob = Blob.from_string(uri, client) - -@@ -4888,30 +4949,12 @@ class _Connection(object): - USER_AGENT = "testing 1.2.3" - credentials = object() - -- def __init__(self, *responses): -- self._responses = responses[:] -- self._requested = [] -- self._signed = [] -- -- def _respond(self, **kw): -- self._requested.append(kw) -- response, self._responses = self._responses[0], self._responses[1:] -- return response -- -- def api_request(self, **kw): -- from google.cloud.exceptions import NotFound -- -- info, content = self._respond(**kw) -- if info.get("status") == http_client.NOT_FOUND: -- raise NotFound(info) -- return content -- - - class _Bucket(object): - def __init__(self, client=None, name="name", user_project=None): - if client is None: -- connection = _Connection() -- client = _Client(connection) -+ client = Test_Blob._make_client() -+ - self.client = client - self._blobs = {} - self._copied = [] -@@ -4946,16 +4989,3 @@ class _Bucket(object): - retry, - ) - ) -- -- --class _Client(object): -- def __init__(self, connection): -- self._base_connection = connection -- -- @property -- def _connection(self): -- return self._base_connection -- -- @property -- def _credentials(self): -- return self._base_connection.credentials ---- a/tests/unit/test_bucket.py -+++ b/tests/unit/test_bucket.py -@@ -19,19 +19,11 @@ import mock - import pytest - - from google.cloud.storage.retry import DEFAULT_RETRY -+from google.cloud.storage.retry import DEFAULT_RETRY_IF_ETAG_IN_JSON - from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED - from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED - - --def _make_connection(*responses): -- import google.cloud.storage._http -- -- mock_connection = mock.create_autospec(google.cloud.storage._http.Connection) -- mock_connection.user_agent = "testing 1.2.3" -- mock_connection.api_request.side_effect = list(responses) -- return mock_connection -- -- - def _create_signing_credentials(): - import google.auth.credentials - -@@ -45,6 +37,79 @@ def _create_signing_credentials(): - return credentials - - -+class Test__blobs_page_start(unittest.TestCase): -+ @staticmethod -+ def _call_fut(iterator, page, response): -+ from google.cloud.storage.bucket import _blobs_page_start -+ -+ return _blobs_page_start(iterator, page, response) -+ -+ def test_wo_any_prefixes(self): -+ iterator = mock.Mock(spec=["prefixes"], prefixes=set()) -+ page = mock.Mock(spec=["prefixes"]) -+ response = {} -+ -+ self._call_fut(iterator, page, response) -+ -+ self.assertEqual(page.prefixes, ()) -+ self.assertEqual(iterator.prefixes, set()) -+ -+ def test_w_prefixes(self): -+ iterator_prefixes = set(["foo/", "qux/"]) -+ iterator = mock.Mock(spec=["prefixes"], prefixes=iterator_prefixes) -+ page = mock.Mock(spec=["prefixes"]) -+ page_prefixes = ["foo/", "bar/", "baz/"] -+ response = {"prefixes": page_prefixes} -+ -+ self._call_fut(iterator, page, response) -+ -+ self.assertEqual(page.prefixes, tuple(page_prefixes)) -+ self.assertEqual(iterator.prefixes, iterator_prefixes.union(page_prefixes)) -+ -+ -+class Test__item_to_blob(unittest.TestCase): -+ @staticmethod -+ def _call_fut(iterator, item): -+ from google.cloud.storage.bucket import _item_to_blob -+ -+ return _item_to_blob(iterator, item) -+ -+ def test_wo_extra_properties(self): -+ from google.cloud.storage.blob import Blob -+ -+ blob_name = "blob-name" -+ bucket = mock.Mock(spec=[]) -+ iterator = mock.Mock(spec=["bucket"], bucket=bucket) -+ item = {"name": blob_name} -+ -+ blob = self._call_fut(iterator, item) -+ -+ self.assertIsInstance(blob, Blob) -+ self.assertIs(blob.bucket, bucket) -+ self.assertEqual(blob.name, blob_name) -+ self.assertEqual(blob._properties, item) -+ -+ def test_w_extra_properties(self): -+ from google.cloud.storage.blob import Blob -+ -+ blob_name = "blob-name" -+ bucket = mock.Mock(spec=[]) -+ iterator = mock.Mock(spec=["bucket"], bucket=bucket) -+ item = { -+ "name": blob_name, -+ "generation": 123, -+ "contentType": "text/plain", -+ "contentLanguage": "en-US", -+ } -+ -+ blob = self._call_fut(iterator, item) -+ -+ self.assertIsInstance(blob, Blob) -+ self.assertIs(blob.bucket, bucket) -+ self.assertEqual(blob.name, blob_name) -+ self.assertEqual(blob._properties, item) -+ -+ - class Test_LifecycleRuleConditions(unittest.TestCase): - @staticmethod - def _get_target_class(): -@@ -450,15 +515,14 @@ class Test_Bucket(unittest.TestCase): - return _DEFAULT_TIMEOUT - - @staticmethod -- def _make_client(*args, **kw): -+ def _make_client(**kw): - from google.cloud.storage.client import Client - -- return Client(*args, **kw) -+ return mock.create_autospec(Client, instance=True, **kw) - - def _make_one(self, client=None, name=None, properties=None, user_project=None): - if client is None: -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - if user_project is None: - bucket = self._get_target_class()(client, name=name) - else: -@@ -490,8 +554,7 @@ class Test_Bucket(unittest.TestCase): - def test_ctor_w_user_project(self): - NAME = "name" - USER_PROJECT = "user-project-123" -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - bucket = self._make_one(client, name=NAME, user_project=USER_PROJECT) - self.assertEqual(bucket.name, NAME) - self.assertEqual(bucket._properties, {}) -@@ -583,7 +646,7 @@ class Test_Bucket(unittest.TestCase): - PROJECT = "PROJECT" - BUCKET_NAME = "BUCKET_NAME" - TOPIC_NAME = "TOPIC_NAME" -- client = _Client(_Connection(), project=PROJECT) -+ client = self._make_client(project=PROJECT) - bucket = self._make_one(client, name=BUCKET_NAME) - - notification = bucket.notification(TOPIC_NAME) -@@ -611,7 +674,7 @@ class Test_Bucket(unittest.TestCase): - CUSTOM_ATTRIBUTES = {"attr1": "value1", "attr2": "value2"} - EVENT_TYPES = [OBJECT_FINALIZE_EVENT_TYPE, OBJECT_DELETE_EVENT_TYPE] - BLOB_NAME_PREFIX = "blob-name-prefix/" -- client = _Client(_Connection(), project=PROJECT) -+ client = self._make_client(project=PROJECT) - bucket = self._make_one(client, name=BUCKET_NAME) - - notification = bucket.notification( -@@ -650,95 +713,72 @@ class Test_Bucket(unittest.TestCase): - bucket._user_project = USER_PROJECT - self.assertEqual(bucket.user_project, USER_PROJECT) - -- def test_exists_miss(self): -+ def test_exists_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- class _FakeConnection(object): -- -- _called_with = [] -- -- @classmethod -- def api_request(cls, *args, **kwargs): -- cls._called_with.append((args, kwargs)) -- raise NotFound(args) -- -- BUCKET_NAME = "bucket-name" -- bucket = self._make_one(name=BUCKET_NAME) -- client = _Client(_FakeConnection) -- self.assertFalse(bucket.exists(client=client, timeout=42)) -- expected_called_kwargs = { -- "method": "GET", -- "path": bucket.path, -- "query_params": {"fields": "name"}, -- "_target_object": None, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- } -- expected_cw = [((), expected_called_kwargs)] -- self.assertEqual(_FakeConnection._called_with, expected_cw) -- -- def test_exists_with_metageneration_match(self): -- class _FakeConnection(object): -- -- _called_with = [] -- -- @classmethod -- def api_request(cls, *args, **kwargs): -- cls._called_with.append((args, kwargs)) -- # exists() does not use the return value -- return object() -+ bucket_name = "bucket-name" -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.side_effect = NotFound("testing") -+ bucket = self._make_one(client, name=bucket_name) -+ -+ self.assertFalse(bucket.exists()) -+ -+ expected_query_params = {"fields": "name"} -+ client._get_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) - -- BUCKET_NAME = "bucket-name" -- METAGENERATION_NUMBER = 6 -+ def test_exists_w_metageneration_match_w_timeout(self): -+ bucket_name = "bucket-name" -+ metageneration_number = 6 -+ timeout = 42 -+ api_response = {"name": bucket_name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client, name=bucket_name) - -- bucket = self._make_one(name=BUCKET_NAME) -- client = _Client(_FakeConnection) - self.assertTrue( -- bucket.exists( -- client=client, timeout=42, if_metageneration_match=METAGENERATION_NUMBER -- ) -+ bucket.exists(timeout=42, if_metageneration_match=metageneration_number) - ) -- expected_called_kwargs = { -- "method": "GET", -- "path": bucket.path, -- "query_params": { -- "fields": "name", -- "ifMetagenerationMatch": METAGENERATION_NUMBER, -- }, -- "_target_object": None, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- } -- expected_cw = [((), expected_called_kwargs)] -- self.assertEqual(_FakeConnection._called_with, expected_cw) -- -- def test_exists_hit_w_user_project(self): -- USER_PROJECT = "user-project-123" -- -- class _FakeConnection(object): -- -- _called_with = [] - -- @classmethod -- def api_request(cls, *args, **kwargs): -- cls._called_with.append((args, kwargs)) -- # exists() does not use the return value -- return object() -- -- BUCKET_NAME = "bucket-name" -- bucket = self._make_one(name=BUCKET_NAME, user_project=USER_PROJECT) -- client = _Client(_FakeConnection) -- self.assertTrue(bucket.exists(client=client)) -- expected_called_kwargs = { -- "method": "GET", -- "path": bucket.path, -- "query_params": {"fields": "name", "userProject": USER_PROJECT}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -+ expected_query_params = { -+ "fields": "name", -+ "ifMetagenerationMatch": metageneration_number, - } -- expected_cw = [((), expected_called_kwargs)] -- self.assertEqual(_FakeConnection._called_with, expected_cw) -+ client._get_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) -+ -+ def test_exists_hit_w_user_project_w_retry_w_explicit_client(self): -+ bucket_name = "bucket-name" -+ user_project = "user-project-123" -+ retry = mock.Mock(spec=[]) -+ api_response = {"name": bucket_name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(name=bucket_name, user_project=user_project) -+ -+ self.assertTrue(bucket.exists(client=client, retry=retry)) -+ -+ expected_query_params = { -+ "fields": "name", -+ "userProject": user_project, -+ } -+ client._get_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=None, -+ ) - - def test_acl_property(self): - from google.cloud.storage.acl import BucketACL -@@ -765,671 +805,920 @@ class Test_Bucket(unittest.TestCase): - bucket = self._make_one(name=NAME) - self.assertEqual(bucket.path, "/b/%s" % NAME) - -- def test_get_blob_miss(self): -- NAME = "name" -- NONESUCH = "nonesuch" -- connection = _Connection() -- client = _Client(connection) -- bucket = self._make_one(name=NAME) -- result = bucket.get_blob(NONESUCH, client=client, timeout=42) -+ def test_get_blob_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound -+ from google.cloud.storage.blob import Blob -+ -+ name = "name" -+ blob_name = "nonesuch" -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.side_effect = NotFound("testing") -+ bucket = self._make_one(client, name=name) -+ -+ result = bucket.get_blob(blob_name) -+ - self.assertIsNone(result) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) -- self.assertEqual(kw["timeout"], 42) -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=mock.ANY, -+ ) -+ -+ target = client._get_resource.call_args[1]["_target_object"] -+ self.assertIsInstance(target, Blob) -+ self.assertIs(target.bucket, bucket) -+ self.assertEqual(target.name, blob_name) - - def test_get_blob_hit_w_user_project(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- connection = _Connection({"name": BLOB_NAME}) -- client = _Client(connection) -- bucket = self._make_one(name=NAME, user_project=USER_PROJECT) -- blob = bucket.get_blob(BLOB_NAME, client=client) -+ from google.cloud.storage.blob import Blob -+ -+ name = "name" -+ blob_name = "blob-name" -+ user_project = "user-project-123" -+ api_response = {"name": blob_name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client, name=name, user_project=user_project) -+ -+ blob = bucket.get_blob(blob_name, client=client) -+ -+ self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) -- self.assertEqual(blob.name, BLOB_NAME) -- (kw,) = connection._requested -- expected_qp = {"userProject": USER_PROJECT, "projection": "noAcl"} -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw["query_params"], expected_qp) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- -- def test_get_blob_hit_w_generation(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- GENERATION = 1512565576797178 -- connection = _Connection({"name": BLOB_NAME, "generation": GENERATION}) -- client = _Client(connection) -- bucket = self._make_one(name=NAME) -- blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) -+ self.assertEqual(blob.name, blob_name) -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = { -+ "userProject": user_project, -+ "projection": "noAcl", -+ } -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=blob, -+ ) -+ -+ def test_get_blob_hit_w_generation_w_timeout(self): -+ from google.cloud.storage.blob import Blob -+ -+ name = "name" -+ blob_name = "blob-name" -+ generation = 1512565576797178 -+ timeout = 42 -+ api_response = {"name": blob_name, "generation": generation} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client, name=name) -+ -+ blob = bucket.get_blob(blob_name, generation=generation, timeout=timeout) -+ -+ self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) -- self.assertEqual(blob.name, BLOB_NAME) -- self.assertEqual(blob.generation, GENERATION) -- (kw,) = connection._requested -- expected_qp = {"generation": GENERATION, "projection": "noAcl"} -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw["query_params"], expected_qp) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- -- def test_get_blob_w_generation_match(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- GENERATION = 1512565576797178 -+ self.assertEqual(blob.name, blob_name) -+ self.assertEqual(blob.generation, generation) - -- connection = _Connection({"name": BLOB_NAME, "generation": GENERATION}) -- client = _Client(connection) -- bucket = self._make_one(name=NAME) -- blob = bucket.get_blob(BLOB_NAME, client=client, if_generation_match=GENERATION) -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = { -+ "generation": generation, -+ "projection": "noAcl", -+ } -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=blob, -+ ) -+ -+ def test_get_blob_w_generation_match_w_retry(self): -+ from google.cloud.storage.blob import Blob -+ -+ name = "name" -+ blob_name = "blob-name" -+ generation = 1512565576797178 -+ retry = mock.Mock(spec=[]) -+ api_response = {"name": blob_name, "generation": generation} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client, name=name) -+ -+ blob = bucket.get_blob(blob_name, if_generation_match=generation, retry=retry) - -+ self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) -- self.assertEqual(blob.name, BLOB_NAME) -- self.assertEqual(blob.generation, GENERATION) -- (kw,) = connection._requested -- expected_qp = {"ifGenerationMatch": GENERATION, "projection": "noAcl"} -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw["query_params"], expected_qp) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -+ self.assertEqual(blob.name, blob_name) -+ self.assertEqual(blob.generation, generation) -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = { -+ "ifGenerationMatch": generation, -+ "projection": "noAcl", -+ } -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=blob, -+ ) - -- def test_get_blob_hit_with_kwargs(self): -+ def test_get_blob_hit_with_kwargs_w_explicit_client(self): -+ from google.cloud.storage.blob import Blob - from google.cloud.storage.blob import _get_encryption_headers - -- NAME = "name" -- BLOB_NAME = "blob-name" -- CHUNK_SIZE = 1024 * 1024 -- KEY = b"01234567890123456789012345678901" # 32 bytes -+ name = "name" -+ blob_name = "blob-name" -+ chunk_size = 1024 * 1024 -+ key = b"01234567890123456789012345678901" # 32 bytes -+ api_response = {"name": blob_name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(name=name) - -- connection = _Connection({"name": BLOB_NAME}) -- client = _Client(connection) -- bucket = self._make_one(name=NAME) - blob = bucket.get_blob( -- BLOB_NAME, client=client, encryption_key=KEY, chunk_size=CHUNK_SIZE -+ blob_name, client=client, encryption_key=key, chunk_size=chunk_size - ) -+ -+ self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) -- self.assertEqual(blob.name, BLOB_NAME) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw["headers"], _get_encryption_headers(KEY)) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(blob.chunk_size, CHUNK_SIZE) -- self.assertEqual(blob._encryption_key, KEY) -+ self.assertEqual(blob.name, blob_name) -+ self.assertEqual(blob.chunk_size, chunk_size) -+ self.assertEqual(blob._encryption_key, key) - -- def test_list_blobs_defaults(self): -- NAME = "name" -- connection = _Connection({"items": []}) -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = { -+ "projection": "noAcl", -+ } -+ expected_headers = _get_encryption_headers(key) -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=blob, -+ ) -+ -+ def test_list_blobs_w_defaults(self): -+ name = "name" - client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -+ client.list_blobs = mock.Mock(spec=[]) -+ bucket = self._make_one(client=client, name=name) -+ - iterator = bucket.list_blobs() -- blobs = list(iterator) -- self.assertEqual(blobs, []) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o" % NAME) -- self.assertEqual(kw["query_params"], {"projection": "noAcl"}) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) - -- def test_list_blobs_w_all_arguments_and_user_project(self): -- NAME = "name" -- USER_PROJECT = "user-project-123" -- MAX_RESULTS = 10 -- PAGE_TOKEN = "ABCD" -- PREFIX = "subfolder" -- DELIMITER = "/" -- START_OFFSET = "c" -- END_OFFSET = "g" -- INCLUDE_TRAILING_DELIMITER = True -- VERSIONS = True -- PROJECTION = "full" -- FIELDS = "items/contentLanguage,nextPageToken" -- EXPECTED = { -- "maxResults": 10, -- "pageToken": PAGE_TOKEN, -- "prefix": PREFIX, -- "delimiter": DELIMITER, -- "startOffset": START_OFFSET, -- "endOffset": END_OFFSET, -- "includeTrailingDelimiter": INCLUDE_TRAILING_DELIMITER, -- "versions": VERSIONS, -- "projection": PROJECTION, -- "fields": FIELDS, -- "userProject": USER_PROJECT, -- } -- connection = _Connection({"items": []}) -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(name=NAME, user_project=USER_PROJECT) -+ self.assertIs(iterator, client.list_blobs.return_value) -+ -+ expected_page_token = None -+ expected_max_results = None -+ expected_prefix = None -+ expected_delimiter = None -+ expected_start_offset = None -+ expected_end_offset = None -+ expected_include_trailing_delimiter = None -+ expected_versions = None -+ expected_projection = "noAcl" -+ expected_fields = None -+ client.list_blobs.assert_called_once_with( -+ bucket, -+ max_results=expected_max_results, -+ page_token=expected_page_token, -+ prefix=expected_prefix, -+ delimiter=expected_delimiter, -+ start_offset=expected_start_offset, -+ end_offset=expected_end_offset, -+ include_trailing_delimiter=expected_include_trailing_delimiter, -+ versions=expected_versions, -+ projection=expected_projection, -+ fields=expected_fields, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) -+ -+ def test_list_blobs_w_explicit(self): -+ name = "name" -+ max_results = 10 -+ page_token = "ABCD" -+ prefix = "subfolder" -+ delimiter = "/" -+ start_offset = "c" -+ end_offset = "g" -+ include_trailing_delimiter = True -+ versions = True -+ projection = "full" -+ fields = "items/contentLanguage,nextPageToken" -+ bucket = self._make_one(client=None, name=name) -+ other_client = self._make_client() -+ other_client.list_blobs = mock.Mock(spec=[]) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ - iterator = bucket.list_blobs( -- max_results=MAX_RESULTS, -- page_token=PAGE_TOKEN, -- prefix=PREFIX, -- delimiter=DELIMITER, -- start_offset=START_OFFSET, -- end_offset=END_OFFSET, -- include_trailing_delimiter=INCLUDE_TRAILING_DELIMITER, -- versions=VERSIONS, -- projection=PROJECTION, -- fields=FIELDS, -- client=client, -- timeout=42, -+ max_results=max_results, -+ page_token=page_token, -+ prefix=prefix, -+ delimiter=delimiter, -+ start_offset=start_offset, -+ end_offset=end_offset, -+ include_trailing_delimiter=include_trailing_delimiter, -+ versions=versions, -+ projection=projection, -+ fields=fields, -+ client=other_client, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ self.assertIs(iterator, other_client.list_blobs.return_value) -+ -+ expected_page_token = page_token -+ expected_max_results = max_results -+ expected_prefix = prefix -+ expected_delimiter = delimiter -+ expected_start_offset = start_offset -+ expected_end_offset = end_offset -+ expected_include_trailing_delimiter = include_trailing_delimiter -+ expected_versions = versions -+ expected_projection = projection -+ expected_fields = fields -+ other_client.list_blobs.assert_called_once_with( -+ bucket, -+ max_results=expected_max_results, -+ page_token=expected_page_token, -+ prefix=expected_prefix, -+ delimiter=expected_delimiter, -+ start_offset=expected_start_offset, -+ end_offset=expected_end_offset, -+ include_trailing_delimiter=expected_include_trailing_delimiter, -+ versions=expected_versions, -+ projection=expected_projection, -+ fields=expected_fields, -+ timeout=timeout, -+ retry=retry, - ) -- blobs = list(iterator) -- self.assertEqual(blobs, []) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "GET") -- self.assertEqual(kw["path"], "/b/%s/o" % NAME) -- self.assertEqual(kw["query_params"], EXPECTED) -- self.assertEqual(kw["timeout"], 42) - -- def test_list_notifications(self): -- from google.cloud.storage.notification import BucketNotification -- from google.cloud.storage.notification import _TOPIC_REF_FMT -- from google.cloud.storage.notification import ( -- JSON_API_V1_PAYLOAD_FORMAT, -- NONE_PAYLOAD_FORMAT, -+ def test_list_notifications_w_defaults(self): -+ from google.cloud.storage.bucket import _item_to_notification -+ -+ bucket_name = "name" -+ client = self._make_client() -+ client._list_resource = mock.Mock(spec=[]) -+ bucket = self._make_one(client=client, name=bucket_name) -+ -+ iterator = bucket.list_notifications() -+ -+ self.assertIs(iterator, client._list_resource.return_value) -+ self.assertIs(iterator.bucket, bucket) -+ -+ expected_path = "/b/{}/notificationConfigs".format(bucket_name) -+ expected_item_to_value = _item_to_notification -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) - -- NAME = "name" -+ def test_list_notifications_w_explicit(self): -+ from google.cloud.storage.bucket import _item_to_notification - -- topic_refs = [("my-project-123", "topic-1"), ("other-project-456", "topic-2")] -+ bucket_name = "name" -+ other_client = self._make_client() -+ other_client._list_resource = mock.Mock(spec=[]) -+ bucket = self._make_one(client=None, name=bucket_name) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- resources = [ -- { -- "topic": _TOPIC_REF_FMT.format(*topic_refs[0]), -- "id": "1", -- "etag": "DEADBEEF", -- "selfLink": "https://example.com/notification/1", -- "payload_format": NONE_PAYLOAD_FORMAT, -- }, -- { -- "topic": _TOPIC_REF_FMT.format(*topic_refs[1]), -- "id": "2", -- "etag": "FACECABB", -- "selfLink": "https://example.com/notification/2", -- "payload_format": JSON_API_V1_PAYLOAD_FORMAT, -- }, -- ] -- connection = _Connection({"items": resources}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- -- notifications = list(bucket.list_notifications(timeout=42)) -- -- req_args = client._connection._requested[0] -- self.assertEqual(req_args.get("timeout"), 42) -- -- self.assertEqual(len(notifications), len(resources)) -- for notification, resource, topic_ref in zip( -- notifications, resources, topic_refs -- ): -- self.assertIsInstance(notification, BucketNotification) -- self.assertEqual(notification.topic_project, topic_ref[0]) -- self.assertEqual(notification.topic_name, topic_ref[1]) -- self.assertEqual(notification.notification_id, resource["id"]) -- self.assertEqual(notification.etag, resource["etag"]) -- self.assertEqual(notification.self_link, resource["selfLink"]) -- self.assertEqual( -- notification.custom_attributes, resource.get("custom_attributes") -- ) -- self.assertEqual(notification.event_types, resource.get("event_types")) -- self.assertEqual( -- notification.blob_name_prefix, resource.get("blob_name_prefix") -- ) -- self.assertEqual( -- notification.payload_format, resource.get("payload_format") -- ) -+ iterator = bucket.list_notifications( -+ client=other_client, timeout=timeout, retry=retry, -+ ) -+ -+ self.assertIs(iterator, other_client._list_resource.return_value) -+ self.assertIs(iterator.bucket, bucket) -+ -+ expected_path = "/b/{}/notificationConfigs".format(bucket_name) -+ expected_item_to_value = _item_to_notification -+ other_client._list_resource.assert_called_once_with( -+ expected_path, expected_item_to_value, timeout=timeout, retry=retry, -+ ) -+ -+ def test_get_notification_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound -+ -+ project = "my-project-123" -+ name = "name" -+ notification_id = "1" -+ -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.side_effect = NotFound("testing") -+ client.project = project -+ bucket = self._make_one(client=client, name=name) -+ -+ with self.assertRaises(NotFound): -+ bucket.get_notification(notification_id=notification_id) -+ -+ expected_path = "/b/{}/notificationConfigs/{}".format(name, notification_id) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) - -- def test_get_notification(self): -+ def test_get_notification_hit_w_explicit_w_user_project(self): -+ from google.cloud.storage.notification import BucketNotification - from google.cloud.storage.notification import _TOPIC_REF_FMT - from google.cloud.storage.notification import JSON_API_V1_PAYLOAD_FORMAT - -- NAME = "name" -- ETAG = "FACECABB" -- NOTIFICATION_ID = "1" -- SELF_LINK = "https://example.com/notification/1" -- resources = { -+ project = "my-project-123" -+ user_project = "user-project-456" -+ name = "name" -+ etag = "FACECABB" -+ notification_id = "1" -+ self_link = "https://example.com/notification/1" -+ api_response = { - "topic": _TOPIC_REF_FMT.format("my-project-123", "topic-1"), -- "id": NOTIFICATION_ID, -- "etag": ETAG, -- "selfLink": SELF_LINK, -+ "id": notification_id, -+ "etag": etag, -+ "selfLink": self_link, - "payload_format": JSON_API_V1_PAYLOAD_FORMAT, - } -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.return_value = api_response -+ client.project = project -+ bucket = self._make_one(client=client, name=name, user_project=user_project) -+ -+ notification = bucket.get_notification( -+ notification_id=notification_id, timeout=timeout, retry=retry, -+ ) - -- connection = _make_connection(resources) -- client = _Client(connection, project="my-project-123") -- bucket = self._make_one(client=client, name=NAME) -- notification = bucket.get_notification(notification_id=NOTIFICATION_ID) -- -- self.assertEqual(notification.notification_id, NOTIFICATION_ID) -- self.assertEqual(notification.etag, ETAG) -- self.assertEqual(notification.self_link, SELF_LINK) -+ self.assertIsInstance(notification, BucketNotification) -+ self.assertEqual(notification.notification_id, notification_id) -+ self.assertEqual(notification.etag, etag) -+ self.assertEqual(notification.self_link, self_link) - self.assertIsNone(notification.custom_attributes) - self.assertIsNone(notification.event_types) - self.assertIsNone(notification.blob_name_prefix) - self.assertEqual(notification.payload_format, JSON_API_V1_PAYLOAD_FORMAT) - -- def test_get_notification_miss(self): -+ expected_path = "/b/{}/notificationConfigs/{}".format(name, notification_id) -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ def test_delete_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- response = NotFound("testing") -- connection = _make_connection(response) -- client = _Client(connection, project="my-project-123") -- bucket = self._make_one(client=client, name="name") -+ name = "name" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.side_effect = NotFound("testing") -+ bucket = self._make_one(client=client, name=name) -+ - with self.assertRaises(NotFound): -- bucket.get_notification(notification_id="1") -+ bucket.delete() - -- def test_delete_miss(self): -- from google.cloud.exceptions import NotFound -+ expected_query_params = {} -+ client._delete_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) - -- NAME = "name" -- connection = _Connection() -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- self.assertRaises(NotFound, bucket.delete) -- expected_cw = [ -- { -- "method": "DELETE", -- "path": bucket.path, -- "query_params": {}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- ] -- self.assertEqual(connection._deleted_buckets, expected_cw) -+ def test_delete_hit_w_metageneration_match_w_explicit_client(self): -+ name = "name" -+ metageneration_number = 6 -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=None, name=name) -+ -+ result = bucket.delete( -+ client=client, if_metageneration_match=metageneration_number, -+ ) - -- def test_delete_hit_with_user_project(self): -- NAME = "name" -- USER_PROJECT = "user-project-123" -- GET_BLOBS_RESP = {"items": []} -- connection = _Connection(GET_BLOBS_RESP) -- connection._delete_bucket = True -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) -- result = bucket.delete(force=True, timeout=42) - self.assertIsNone(result) -- expected_cw = [ -- { -- "method": "DELETE", -- "path": bucket.path, -- "_target_object": None, -- "query_params": {"userProject": USER_PROJECT}, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- } -- ] -- self.assertEqual(connection._deleted_buckets, expected_cw) - -- def test_delete_force_delete_blobs(self): -- NAME = "name" -- BLOB_NAME1 = "blob-name1" -- BLOB_NAME2 = "blob-name2" -- GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} -- DELETE_BLOB1_RESP = DELETE_BLOB2_RESP = {} -- connection = _Connection(GET_BLOBS_RESP, DELETE_BLOB1_RESP, DELETE_BLOB2_RESP) -- connection._delete_bucket = True -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -- result = bucket.delete(force=True) -+ expected_query_params = {"ifMetagenerationMatch": metageneration_number} -+ client._delete_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) -+ -+ def test_delete_hit_w_force_w_user_project_w_explicit_timeout_retry(self): -+ name = "name" -+ user_project = "user-project-123" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name, user_project=user_project) -+ bucket.list_blobs = mock.Mock(return_value=iter([])) -+ bucket.delete_blobs = mock.Mock(return_value=None) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ -+ result = bucket.delete(force=True, timeout=timeout, retry=retry) -+ - self.assertIsNone(result) -- expected_cw = [ -- { -- "method": "DELETE", -- "path": bucket.path, -- "query_params": {}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- ] -- self.assertEqual(connection._deleted_buckets, expected_cw) - -- def test_delete_with_metageneration_match(self): -- NAME = "name" -- BLOB_NAME1 = "blob-name1" -- BLOB_NAME2 = "blob-name2" -- GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} -- DELETE_BLOB1_RESP = DELETE_BLOB2_RESP = {} -- METAGENERATION_NUMBER = 6 -- -- connection = _Connection(GET_BLOBS_RESP, DELETE_BLOB1_RESP, DELETE_BLOB2_RESP) -- connection._delete_bucket = True -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- result = bucket.delete(if_metageneration_match=METAGENERATION_NUMBER) -+ bucket.list_blobs.assert_called_once_with( -+ max_results=bucket._MAX_OBJECTS_FOR_ITERATION + 1, -+ client=client, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ bucket.delete_blobs.assert_called_once_with( -+ [], on_error=mock.ANY, client=client, timeout=timeout, retry=retry, -+ ) -+ -+ expected_query_params = {"userProject": user_project} -+ client._delete_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=None, -+ ) -+ -+ def test_delete_hit_w_force_delete_blobs(self): -+ name = "name" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name) -+ blobs = [mock.Mock(spec=[]), mock.Mock(spec=[])] -+ bucket.list_blobs = mock.Mock(return_value=iter(blobs)) -+ bucket.delete_blobs = mock.Mock(return_value=None) -+ -+ result = bucket.delete(force=True) -+ - self.assertIsNone(result) -- expected_cw = [ -- { -- "method": "DELETE", -- "path": bucket.path, -- "query_params": {"ifMetagenerationMatch": METAGENERATION_NUMBER}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- ] -- self.assertEqual(connection._deleted_buckets, expected_cw) - -- def test_delete_force_miss_blobs(self): -- NAME = "name" -- BLOB_NAME = "blob-name1" -- GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME}]} -- # Note the connection does not have a response for the blob. -- connection = _Connection(GET_BLOBS_RESP) -- connection._delete_bucket = True -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -+ bucket.list_blobs.assert_called_once_with( -+ max_results=bucket._MAX_OBJECTS_FOR_ITERATION + 1, -+ client=client, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) -+ -+ bucket.delete_blobs.assert_called_once_with( -+ blobs, -+ on_error=mock.ANY, -+ client=client, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) -+ -+ expected_query_params = {} -+ client._delete_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) -+ -+ def test_delete_w_force_w_user_project_w_miss_on_blob(self): -+ from google.cloud.exceptions import NotFound -+ -+ name = "name" -+ blob_name = "blob-name" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name) -+ blob = mock.Mock(spec=["name"]) -+ blob.name = blob_name -+ blobs = [blob] -+ bucket.list_blobs = mock.Mock(return_value=iter(blobs)) -+ bucket.delete_blob = mock.Mock(side_effect=NotFound("testing")) -+ - result = bucket.delete(force=True) -+ - self.assertIsNone(result) -- expected_cw = [ -- { -- "method": "DELETE", -- "path": bucket.path, -- "query_params": {}, -- "_target_object": None, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- ] -- self.assertEqual(connection._deleted_buckets, expected_cw) - -- def test_delete_too_many(self): -- NAME = "name" -- BLOB_NAME1 = "blob-name1" -- BLOB_NAME2 = "blob-name2" -- GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} -- connection = _Connection(GET_BLOBS_RESP) -- connection._delete_bucket = True -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -+ bucket.delete_blob.assert_called_once_with( -+ blob_name, -+ client=client, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) - -+ expected_query_params = {} -+ client._delete_resource.assert_called_once_with( -+ bucket.path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) -+ -+ def test_delete_w_too_many(self): -+ name = "name" -+ blob_name1 = "blob-name1" -+ blob_name2 = "blob-name2" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name) -+ blob1 = mock.Mock(spec=["name"]) -+ blob1.name = blob_name1 -+ blob2 = mock.Mock(spec=["name"]) -+ blob2.name = blob_name2 -+ blobs = [blob1, blob2] -+ bucket.list_blobs = mock.Mock(return_value=iter(blobs)) -+ bucket.delete_blobs = mock.Mock() - # Make the Bucket refuse to delete with 2 objects. - bucket._MAX_OBJECTS_FOR_ITERATION = 1 -- self.assertRaises(ValueError, bucket.delete, force=True) -- self.assertEqual(connection._deleted_buckets, []) - -- def test_delete_blob_miss(self): -+ with self.assertRaises(ValueError): -+ bucket.delete(force=True) -+ -+ bucket.delete_blobs.assert_not_called() -+ -+ def test_delete_blob_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- NAME = "name" -- NONESUCH = "nonesuch" -- connection = _Connection() -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- self.assertRaises(NotFound, bucket.delete_blob, NONESUCH) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "DELETE") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) -- self.assertEqual(kw["query_params"], {}) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -+ name = "name" -+ blob_name = "nonesuch" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.side_effect = NotFound("testing") -+ bucket = self._make_one(client=client, name=name) -+ -+ with self.assertRaises(NotFound): -+ bucket.delete_blob(blob_name) -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = {} -+ client._delete_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=None, -+ ) -+ -+ def test_delete_blob_hit_w_user_project_w_timeout(self): -+ name = "name" -+ blob_name = "blob-name" -+ user_project = "user-project-123" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name, user_project=user_project) -+ timeout = 42 -+ -+ result = bucket.delete_blob(blob_name, timeout=timeout) - -- def test_delete_blob_hit_with_user_project(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) -- result = bucket.delete_blob(BLOB_NAME, timeout=42) - self.assertIsNone(result) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "DELETE") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) -- self.assertEqual(kw["timeout"], 42) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- def test_delete_blob_hit_with_generation(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- GENERATION = 1512565576797178 -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = {"userProject": user_project} -+ client._delete_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=None, -+ ) -+ -+ def test_delete_blob_hit_w_generation_w_retry(self): -+ name = "name" -+ blob_name = "blob-name" -+ generation = 1512565576797178 -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name) -+ retry = mock.Mock(spec=[]) -+ -+ result = bucket.delete_blob(blob_name, generation=generation, retry=retry) -+ - self.assertIsNone(result) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "DELETE") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw["query_params"], {"generation": GENERATION}) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- def test_delete_blob_with_generation_match(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- GENERATION = 6 -- METAGENERATION = 9 -- -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = {"generation": generation} -+ client._delete_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=None, -+ ) -+ -+ def test_delete_blob_hit_w_generation_match(self): -+ name = "name" -+ blob_name = "blob-name" -+ generation = 6 -+ metageneration = 9 -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket = self._make_one(client=client, name=name) -+ - result = bucket.delete_blob( -- BLOB_NAME, -- if_generation_match=GENERATION, -- if_metageneration_match=METAGENERATION, -+ blob_name, -+ if_generation_match=generation, -+ if_metageneration_match=metageneration, - ) - - self.assertIsNone(result) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "DELETE") -- self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual( -- kw["query_params"], -- {"ifGenerationMatch": GENERATION, "ifMetagenerationMatch": METAGENERATION}, -+ -+ expected_path = "/b/%s/o/%s" % (name, blob_name) -+ expected_query_params = { -+ "ifGenerationMatch": generation, -+ "ifMetagenerationMatch": metageneration, -+ } -+ client._delete_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=None, - ) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - - def test_delete_blobs_empty(self): -- NAME = "name" -- connection = _Connection() -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ name = "name" -+ bucket = self._make_one(client=None, name=name) -+ bucket.delete_blob = mock.Mock() -+ - bucket.delete_blobs([]) -- self.assertEqual(connection._requested, []) - -- def test_delete_blobs_hit_w_user_project(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- USER_PROJECT = "user-project-123" -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) -- bucket.delete_blobs([BLOB_NAME], timeout=42) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "DELETE") -- self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- def test_delete_blobs_w_generation_match(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- BLOB_NAME2 = "blob-name2" -- GENERATION_NUMBER = 6 -- GENERATION_NUMBER2 = 9 -- -- connection = _Connection({}, {}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- bucket.delete_blobs( -- [BLOB_NAME, BLOB_NAME2], -- timeout=42, -- if_generation_match=[GENERATION_NUMBER, GENERATION_NUMBER2], -- ) -- kw = connection._requested -- self.assertEqual(len(kw), 2) -+ bucket.delete_blob.assert_not_called() - -- self.assertEqual(kw[0]["method"], "DELETE") -- self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual( -- kw[0]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER} -- ) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- self.assertEqual(kw[1]["method"], "DELETE") -- self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME2)) -- self.assertEqual(kw[1]["timeout"], 42) -- self.assertEqual( -- kw[1]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER2} -+ def test_delete_blobs_hit_w_explicit_client_w_timeout(self): -+ name = "name" -+ blob_name = "blob-name" -+ client = mock.Mock(spec=[]) -+ bucket = self._make_one(client=None, name=name) -+ bucket.delete_blob = mock.Mock() -+ timeout = 42 -+ -+ bucket.delete_blobs([blob_name], client=client, timeout=timeout) -+ -+ bucket.delete_blob.assert_called_once_with( -+ blob_name, -+ client=client, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=timeout, -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - ) -- self.assertEqual(kw[1]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - - def test_delete_blobs_w_generation_match_wrong_len(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- BLOB_NAME2 = "blob-name2" -- GENERATION_NUMBER = 6 -- -- connection = _Connection() -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ name = "name" -+ blob_name = "blob-name" -+ blob_name2 = "blob-name2" -+ generation_number = 6 -+ bucket = self._make_one(client=None, name=name) -+ bucket.delete_blob = mock.Mock() -+ - with self.assertRaises(ValueError): - bucket.delete_blobs( -- [BLOB_NAME, BLOB_NAME2], -- timeout=42, -- if_generation_not_match=[GENERATION_NUMBER], -+ [blob_name, blob_name2], if_generation_not_match=[generation_number], - ) - -+ bucket.delete_blob.assert_not_called() -+ -+ def test_delete_blobs_w_generation_match_w_retry(self): -+ name = "name" -+ blob_name = "blob-name" -+ blob_name2 = "blob-name2" -+ generation_number = 6 -+ generation_number2 = 9 -+ client = mock.Mock(spec=[]) -+ bucket = self._make_one(client=client, name=name) -+ bucket.delete_blob = mock.Mock() -+ retry = mock.Mock(spec=[]) -+ -+ bucket.delete_blobs( -+ [blob_name, blob_name2], -+ if_generation_match=[generation_number, generation_number2], -+ retry=retry, -+ ) -+ -+ call_1 = mock.call( -+ blob_name, -+ client=None, -+ if_generation_match=generation_number, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ ) -+ call_2 = mock.call( -+ blob_name2, -+ client=None, -+ if_generation_match=generation_number2, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ ) -+ bucket.delete_blob.assert_has_calls([call_1, call_2]) -+ - def test_delete_blobs_w_generation_match_none(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- BLOB_NAME2 = "blob-name2" -- GENERATION_NUMBER = 6 -- GENERATION_NUMBER2 = None -- -- connection = _Connection({}, {}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ name = "name" -+ blob_name = "blob-name" -+ blob_name2 = "blob-name2" -+ generation_number = 6 -+ generation_number2 = None -+ client = mock.Mock(spec=[]) -+ bucket = self._make_one(client=client, name=name) -+ bucket.delete_blob = mock.Mock() -+ - bucket.delete_blobs( -- [BLOB_NAME, BLOB_NAME2], -- timeout=42, -- if_generation_match=[GENERATION_NUMBER, GENERATION_NUMBER2], -+ [blob_name, blob_name2], -+ if_generation_match=[generation_number, generation_number2], - ) -- kw = connection._requested -- self.assertEqual(len(kw), 2) - -- self.assertEqual(kw[0]["method"], "DELETE") -- self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual( -- kw[0]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER} -+ call_1 = mock.call( -+ blob_name, -+ client=None, -+ if_generation_match=generation_number, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ ) -+ call_2 = mock.call( -+ blob_name2, -+ client=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - ) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- self.assertEqual(kw[1]["method"], "DELETE") -- self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME2)) -- self.assertEqual(kw[1]["timeout"], 42) -- self.assertEqual(kw[1]["query_params"], {}) -- self.assertEqual(kw[1]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ bucket.delete_blob.assert_has_calls([call_1, call_2]) - -- def test_delete_blobs_miss_no_on_error(self): -+ def test_delete_blobs_miss_wo_on_error(self): - from google.cloud.exceptions import NotFound - -- NAME = "name" -- BLOB_NAME = "blob-name" -- NONESUCH = "nonesuch" -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- self.assertRaises(NotFound, bucket.delete_blobs, [BLOB_NAME, NONESUCH]) -- kw = connection._requested -- self.assertEqual(len(kw), 2) -- self.assertEqual(kw[0]["method"], "DELETE") -- self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- self.assertEqual(kw[1]["method"], "DELETE") -- self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) -- self.assertEqual(kw[1]["timeout"], self._get_default_timeout()) -- self.assertEqual(kw[1]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ name = "name" -+ blob_name = "blob-name" -+ blob_name2 = "nonesuch" -+ client = mock.Mock(spec=[]) -+ bucket = self._make_one(client=client, name=name) -+ bucket.delete_blob = mock.Mock() -+ bucket.delete_blob.side_effect = [None, NotFound("testing")] -+ -+ with self.assertRaises(NotFound): -+ bucket.delete_blobs([blob_name, blob_name2]) -+ -+ call_1 = mock.call( -+ blob_name, -+ client=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ ) -+ call_2 = mock.call( -+ blob_name2, -+ client=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ ) -+ bucket.delete_blob.assert_has_calls([call_1, call_2]) - - def test_delete_blobs_miss_w_on_error(self): -- NAME = "name" -- BLOB_NAME = "blob-name" -- NONESUCH = "nonesuch" -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ from google.cloud.exceptions import NotFound -+ -+ name = "name" -+ blob_name = "blob-name" -+ blob_name2 = "nonesuch" -+ client = mock.Mock(spec=[]) -+ bucket = self._make_one(client=client, name=name) -+ bucket.delete_blob = mock.Mock() -+ bucket.delete_blob.side_effect = [None, NotFound("testing")] -+ - errors = [] -- bucket.delete_blobs([BLOB_NAME, NONESUCH], errors.append) -- self.assertEqual(errors, [NONESUCH]) -- kw = connection._requested -- self.assertEqual(len(kw), 2) -- self.assertEqual(kw[0]["method"], "DELETE") -- self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- self.assertEqual(kw[0]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- self.assertEqual(kw[1]["method"], "DELETE") -- self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) -- self.assertEqual(kw[1]["timeout"], self._get_default_timeout()) -- self.assertEqual(kw[1]["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- def test_reload_bucket_w_metageneration_match(self): -- NAME = "name" -- METAGENERATION_NUMBER = 9 -- -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -- -- bucket.reload(if_metageneration_match=METAGENERATION_NUMBER) -- -- self.assertEqual(len(connection._requested), 1) -- req = connection._requested[0] -- self.assertEqual(req["method"], "GET") -- self.assertEqual(req["path"], "/b/%s" % NAME) -- self.assertEqual(req["timeout"], self._get_default_timeout()) -- self.assertEqual( -- req["query_params"], -- {"projection": "noAcl", "ifMetagenerationMatch": METAGENERATION_NUMBER}, -+ bucket.delete_blobs([blob_name, blob_name2], on_error=errors.append) -+ -+ self.assertEqual(errors, [blob_name2]) -+ -+ call_1 = mock.call( -+ blob_name, -+ client=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ ) -+ call_2 = mock.call( -+ blob_name2, -+ client=None, -+ if_generation_match=None, -+ if_generation_not_match=None, -+ if_metageneration_match=None, -+ if_metageneration_not_match=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ ) -+ bucket.delete_blob.assert_has_calls([call_1, call_2]) -+ -+ def test_reload_w_metageneration_match(self): -+ name = "name" -+ metageneration_number = 9 -+ api_response = {"name": name} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client, name=name) -+ -+ bucket.reload(if_metageneration_match=metageneration_number) -+ -+ expected_path = "/b/%s" % (name,) -+ expected_query_params = { -+ "projection": "noAcl", -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) - -- def test_reload_bucket_w_generation_match(self): -- connection = _Connection({}) -- client = _Client(connection) -+ def test_reload_w_generation_match(self): -+ client = self._make_client() - bucket = self._make_one(client=client, name="name") - - with self.assertRaises(TypeError): - bucket.reload(if_generation_match=6) - -- def test_update_bucket_w_metageneration_match(self): -- NAME = "name" -- METAGENERATION_NUMBER = 9 -- -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ def test_update_w_metageneration_match(self): -+ name = "name" -+ metageneration_number = 9 -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = {} -+ bucket = self._make_one(client=client, name=name) - -- bucket.update(if_metageneration_match=METAGENERATION_NUMBER) -+ bucket.update(if_metageneration_match=metageneration_number) - -- self.assertEqual(len(connection._requested), 1) -- req = connection._requested[0] -- self.assertEqual(req["method"], "PUT") -- self.assertEqual(req["path"], "/b/%s" % NAME) -- self.assertEqual(req["timeout"], self._get_default_timeout()) -- self.assertEqual( -- req["query_params"], -- {"projection": "full", "ifMetagenerationMatch": METAGENERATION_NUMBER}, -+ expected_query_params = { -+ "projection": "full", -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ client._put_resource.assert_called_once_with( -+ bucket.path, -+ bucket._properties, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=bucket, - ) -- self.assertEqual(req["retry"], DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED) - -- def test_update_bucket_w_generation_match(self): -- connection = _Connection({}) -- client = _Client(connection) -- bucket = self._make_one(client=client, name="name") -+ def test_update_w_generation_match(self): -+ name = "name" -+ generation_number = 6 -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = {} -+ bucket = self._make_one(client=client, name=name) - - with self.assertRaises(TypeError): -- bucket.update(if_generation_match=6) -+ bucket.update(if_generation_match=generation_number) -+ -+ client._put_resource.assert_not_called() - - @staticmethod - def _make_blob(bucket_name, blob_name): -@@ -1441,277 +1730,287 @@ class Test_Bucket(unittest.TestCase): - return blob - - def test_copy_blobs_wo_name(self): -- SOURCE = "source" -- DEST = "dest" -- BLOB_NAME = "blob-name" -- connection = _Connection({}) -- client = _Client(connection) -- source = self._make_one(client=client, name=SOURCE) -- dest = self._make_one(client=client, name=DEST) -- blob = self._make_blob(SOURCE, BLOB_NAME) -+ source_name = "source" -+ dest_name = "dest" -+ blob_name = "blob-name" -+ api_response = {} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response -+ source = self._make_one(client=client, name=source_name) -+ dest = self._make_one(client=client, name=dest_name) -+ blob = self._make_blob(source_name, blob_name) - -- new_blob = source.copy_blob(blob, dest, timeout=42) -+ new_blob = source.copy_blob(blob, dest) - - self.assertIs(new_blob.bucket, dest) -- self.assertEqual(new_blob.name, BLOB_NAME) -+ self.assertEqual(new_blob.name, blob_name) -+ -+ expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -+ source_name, blob_name, dest_name, blob_name -+ ) -+ expected_data = None -+ expected_query_params = {} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=new_blob, -+ ) - -- (kw,) = connection._requested -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- SOURCE, BLOB_NAME, DEST, BLOB_NAME -- ) -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual(kw["query_params"], {}) -- self.assertEqual(kw["timeout"], 42) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- def test_copy_blobs_source_generation(self): -- SOURCE = "source" -- DEST = "dest" -- BLOB_NAME = "blob-name" -- GENERATION = 1512565576797178 -- -- connection = _Connection({}) -- client = _Client(connection) -- source = self._make_one(client=client, name=SOURCE) -- dest = self._make_one(client=client, name=DEST) -- blob = self._make_blob(SOURCE, BLOB_NAME) -+ def test_copy_blob_w_source_generation_w_timeout(self): -+ source_name = "source" -+ dest_name = "dest" -+ blob_name = "blob-name" -+ generation = 1512565576797178 -+ api_response = {} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response -+ source = self._make_one(client=client, name=source_name) -+ dest = self._make_one(client=client, name=dest_name) -+ blob = self._make_blob(source_name, blob_name) -+ timeout = 42 - -- new_blob = source.copy_blob(blob, dest, source_generation=GENERATION) -+ new_blob = source.copy_blob( -+ blob, dest, source_generation=generation, timeout=timeout, -+ ) - - self.assertIs(new_blob.bucket, dest) -- self.assertEqual(new_blob.name, BLOB_NAME) -+ self.assertEqual(new_blob.name, blob_name) -+ -+ expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -+ source_name, blob_name, dest_name, blob_name -+ ) -+ expected_data = None -+ expected_query_params = {"sourceGeneration": generation} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=new_blob, -+ ) - -- (kw,) = connection._requested -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- SOURCE, BLOB_NAME, DEST, BLOB_NAME -- ) -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual(kw["query_params"], {"sourceGeneration": GENERATION}) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- def test_copy_blobs_w_generation_match(self): -- SOURCE = "source" -- DEST = "dest" -- BLOB_NAME = "blob-name" -- GENERATION_NUMBER = 6 -- SOURCE_GENERATION_NUMBER = 9 -- -- connection = _Connection({}) -- client = _Client(connection) -- source = self._make_one(client=client, name=SOURCE) -- dest = self._make_one(client=client, name=DEST) -- blob = self._make_blob(SOURCE, BLOB_NAME) -+ def test_copy_blob_w_generation_match_w_retry(self): -+ source_name = "source" -+ dest_name = "dest" -+ blob_name = "blob-name" -+ generation_number = 6 -+ source_generation_number = 9 -+ api_response = {} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response -+ source = self._make_one(client=client, name=source_name) -+ dest = self._make_one(client=client, name=dest_name) -+ blob = self._make_blob(source_name, blob_name) -+ retry = mock.Mock(spec=[]) - - new_blob = source.copy_blob( - blob, - dest, -- if_generation_match=GENERATION_NUMBER, -- if_source_generation_match=SOURCE_GENERATION_NUMBER, -+ if_generation_match=generation_number, -+ if_source_generation_match=source_generation_number, -+ retry=retry, - ) - self.assertIs(new_blob.bucket, dest) -- self.assertEqual(new_blob.name, BLOB_NAME) -+ self.assertEqual(new_blob.name, blob_name) - -- (kw,) = connection._requested -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- SOURCE, BLOB_NAME, DEST, BLOB_NAME -+ expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -+ source_name, blob_name, dest_name, blob_name - ) -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual( -- kw["query_params"], -- { -- "ifGenerationMatch": GENERATION_NUMBER, -- "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, -- }, -+ expected_data = None -+ expected_query_params = { -+ "ifGenerationMatch": generation_number, -+ "ifSourceGenerationMatch": source_generation_number, -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=new_blob, - ) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - -- def test_copy_blobs_preserve_acl(self): -+ def test_copy_blob_w_preserve_acl_false_w_explicit_client(self): - from google.cloud.storage.acl import ObjectACL - -- SOURCE = "source" -- DEST = "dest" -- BLOB_NAME = "blob-name" -- NEW_NAME = "new_name" -- -- connection = _Connection({}, {}) -- client = _Client(connection) -- source = self._make_one(client=client, name=SOURCE) -- dest = self._make_one(client=client, name=DEST) -- blob = self._make_blob(SOURCE, BLOB_NAME) -+ source_name = "source" -+ dest_name = "dest" -+ blob_name = "blob-name" -+ new_name = "new_name" -+ post_api_response = {} -+ patch_api_response = {} -+ client = mock.Mock(spec=["_post_resource", "_patch_resource"]) -+ client._post_resource.return_value = post_api_response -+ client._patch_resource.return_value = patch_api_response -+ source = self._make_one(client=None, name=source_name) -+ dest = self._make_one(client=None, name=dest_name) -+ blob = self._make_blob(source_name, blob_name) - - new_blob = source.copy_blob( -- blob, dest, NEW_NAME, client=client, preserve_acl=False -+ blob, dest, new_name, client=client, preserve_acl=False - ) - - self.assertIs(new_blob.bucket, dest) -- self.assertEqual(new_blob.name, NEW_NAME) -+ self.assertEqual(new_blob.name, new_name) - self.assertIsInstance(new_blob.acl, ObjectACL) - -- kw1, kw2 = connection._requested -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- SOURCE, BLOB_NAME, DEST, NEW_NAME -- ) -- NEW_BLOB_PATH = "/b/{}/o/{}".format(DEST, NEW_NAME) -- -- self.assertEqual(kw1["method"], "POST") -- self.assertEqual(kw1["path"], COPY_PATH) -- self.assertEqual(kw1["query_params"], {}) -- self.assertEqual(kw1["timeout"], self._get_default_timeout()) -- self.assertEqual(kw1["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -- -- self.assertEqual(kw2["method"], "PATCH") -- self.assertEqual(kw2["path"], NEW_BLOB_PATH) -- self.assertEqual(kw2["query_params"], {"projection": "full"}) -- self.assertEqual(kw2["timeout"], self._get_default_timeout()) -- -- def test_copy_blobs_w_name_and_user_project(self): -- SOURCE = "source" -- DEST = "dest" -- BLOB_NAME = "blob-name" -- NEW_NAME = "new_name" -- USER_PROJECT = "user-project-123" -- connection = _Connection({}) -- client = _Client(connection) -- source = self._make_one(client=client, name=SOURCE, user_project=USER_PROJECT) -- dest = self._make_one(client=client, name=DEST) -- blob = self._make_blob(SOURCE, BLOB_NAME) -- -- new_blob = source.copy_blob(blob, dest, NEW_NAME) -+ expected_copy_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -+ source_name, blob_name, dest_name, new_name -+ ) -+ expected_copy_data = None -+ expected_copy_query_params = {} -+ client._post_resource.assert_called_once_with( -+ expected_copy_path, -+ expected_copy_data, -+ query_params=expected_copy_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=new_blob, -+ ) - -- self.assertIs(new_blob.bucket, dest) -- self.assertEqual(new_blob.name, NEW_NAME) -+ expected_patch_path = "/b/{}/o/{}".format(dest_name, new_name) -+ expected_patch_data = {"acl": []} -+ expected_patch_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ expected_patch_path, -+ expected_patch_data, -+ query_params=expected_patch_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) - -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- SOURCE, BLOB_NAME, DEST, NEW_NAME -+ def test_copy_blob_w_name_and_user_project(self): -+ source_name = "source" -+ dest_name = "dest" -+ blob_name = "blob-name" -+ new_name = "new_name" -+ user_project = "user-project-123" -+ api_response = {} -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response -+ source = self._make_one( -+ client=client, name=source_name, user_project=user_project - ) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ dest = self._make_one(client=client, name=dest_name) -+ blob = self._make_blob(source_name, blob_name) - -- def test_rename_blob(self): -- BUCKET_NAME = "BUCKET_NAME" -- BLOB_NAME = "blob-name" -- NEW_BLOB_NAME = "new-blob-name" -- DATA = {"name": NEW_BLOB_NAME} -- connection = _Connection(DATA) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=BUCKET_NAME) -- blob = self._make_blob(BUCKET_NAME, BLOB_NAME) -- -- renamed_blob = bucket.rename_blob( -- blob, NEW_BLOB_NAME, client=client, timeout=42 -- ) -- -- self.assertIs(renamed_blob.bucket, bucket) -- self.assertEqual(renamed_blob.name, NEW_BLOB_NAME) -- -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- BUCKET_NAME, BLOB_NAME, BUCKET_NAME, NEW_BLOB_NAME -- ) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual(kw["query_params"], {}) -- self.assertEqual(kw["timeout"], 42) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ new_blob = source.copy_blob(blob, dest, new_name) - -- blob.delete.assert_called_once_with( -- client=client, -- timeout=42, -- if_generation_match=None, -- if_generation_not_match=None, -- if_metageneration_match=None, -- if_metageneration_not_match=None, -+ self.assertIs(new_blob.bucket, dest) -+ self.assertEqual(new_blob.name, new_name) -+ -+ expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -+ source_name, blob_name, dest_name, new_name -+ ) -+ expected_data = None -+ expected_query_params = {"userProject": user_project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -+ _target_object=new_blob, - ) - -- def test_rename_blob_with_generation_match(self): -- BUCKET_NAME = "BUCKET_NAME" -- BLOB_NAME = "blob-name" -- NEW_BLOB_NAME = "new-blob-name" -- DATA = {"name": NEW_BLOB_NAME} -- GENERATION_NUMBER = 6 -- SOURCE_GENERATION_NUMBER = 7 -- SOURCE_METAGENERATION_NUMBER = 9 -- -- connection = _Connection(DATA) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=BUCKET_NAME) -- blob = self._make_blob(BUCKET_NAME, BLOB_NAME) -+ def _rename_blob_helper(self, explicit_client=False, same_name=False, **kw): -+ bucket_name = "BUCKET_NAME" -+ blob_name = "blob-name" -+ -+ if same_name: -+ new_blob_name = blob_name -+ else: -+ new_blob_name = "new-blob-name" - -- renamed_blob = bucket.rename_blob( -+ client = mock.Mock(spec=[]) -+ kw = kw.copy() -+ -+ if explicit_client: -+ bucket = self._make_one(client=None, name=bucket_name) -+ expected_client = kw["client"] = client -+ else: -+ bucket = self._make_one(client=client, name=bucket_name) -+ expected_client = None -+ -+ expected_i_g_m = kw.get("if_generation_match") -+ expected_i_g_n_m = kw.get("if_generation_not_match") -+ expected_i_m_m = kw.get("if_metageneration_match") -+ expected_i_m_n_m = kw.get("if_metageneration_not_match") -+ expected_i_s_g_m = kw.get("if_source_generation_match") -+ expected_i_s_g_n_m = kw.get("if_source_generation_not_match") -+ expected_i_s_m_m = kw.get("if_source_metageneration_match") -+ expected_i_s_m_n_m = kw.get("if_source_metageneration_not_match") -+ expected_timeout = kw.get("timeout", self._get_default_timeout()) -+ expected_retry = kw.get("retry", DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ -+ bucket.copy_blob = mock.Mock(spec=[]) -+ blob = self._make_blob(bucket_name, blob_name) -+ -+ renamed_blob = bucket.rename_blob(blob, new_blob_name, **kw) -+ -+ self.assertIs(renamed_blob, bucket.copy_blob.return_value) -+ -+ bucket.copy_blob.assert_called_once_with( - blob, -- NEW_BLOB_NAME, -- client=client, -- timeout=42, -- if_generation_match=GENERATION_NUMBER, -- if_source_generation_match=SOURCE_GENERATION_NUMBER, -- if_source_metageneration_not_match=SOURCE_METAGENERATION_NUMBER, -+ bucket, -+ new_blob_name, -+ client=expected_client, -+ if_generation_match=expected_i_g_m, -+ if_generation_not_match=expected_i_g_n_m, -+ if_metageneration_match=expected_i_m_m, -+ if_metageneration_not_match=expected_i_m_n_m, -+ if_source_generation_match=expected_i_s_g_m, -+ if_source_generation_not_match=expected_i_s_g_n_m, -+ if_source_metageneration_match=expected_i_s_m_m, -+ if_source_metageneration_not_match=expected_i_s_m_n_m, -+ timeout=expected_timeout, -+ retry=expected_retry, - ) - -- self.assertIs(renamed_blob.bucket, bucket) -- self.assertEqual(renamed_blob.name, NEW_BLOB_NAME) -+ if same_name: -+ blob.delete.assert_not_called() -+ else: -+ blob.delete.assert_called_once_with( -+ client=expected_client, -+ if_generation_match=expected_i_s_g_m, -+ if_generation_not_match=expected_i_s_g_n_m, -+ if_metageneration_match=expected_i_s_m_m, -+ if_metageneration_not_match=expected_i_s_m_n_m, -+ timeout=expected_timeout, -+ retry=expected_retry, -+ ) - -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- BUCKET_NAME, BLOB_NAME, BUCKET_NAME, NEW_BLOB_NAME -- ) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual( -- kw["query_params"], -- { -- "ifGenerationMatch": GENERATION_NUMBER, -- "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, -- "ifSourceMetagenerationNotMatch": SOURCE_METAGENERATION_NUMBER, -- }, -- ) -- self.assertEqual(kw["timeout"], 42) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ def test_rename_blob_w_defaults(self): -+ self._rename_blob_helper() - -- blob.delete.assert_called_once_with( -- client=client, -- timeout=42, -- if_generation_match=SOURCE_GENERATION_NUMBER, -- if_generation_not_match=None, -- if_metageneration_match=None, -- if_metageneration_not_match=SOURCE_METAGENERATION_NUMBER, -- retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, -- ) -+ def test_rename_blob_w_explicit_client(self): -+ self._rename_blob_helper(explicit_client=True) - -- def test_rename_blob_to_itself(self): -- BUCKET_NAME = "BUCKET_NAME" -- BLOB_NAME = "blob-name" -- DATA = {"name": BLOB_NAME} -- connection = _Connection(DATA) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=BUCKET_NAME) -- blob = self._make_blob(BUCKET_NAME, BLOB_NAME) -- -- renamed_blob = bucket.rename_blob(blob, BLOB_NAME) -- -- self.assertIs(renamed_blob.bucket, bucket) -- self.assertEqual(renamed_blob.name, BLOB_NAME) -- -- COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( -- BUCKET_NAME, BLOB_NAME, BUCKET_NAME, BLOB_NAME -- ) -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], COPY_PATH) -- self.assertEqual(kw["query_params"], {}) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) -- self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) -+ def test_rename_blob_w_generation_match(self): -+ generation_number = 6 -+ source_generation_number = 7 -+ source_metageneration_number = 9 -+ -+ self._rename_blob_helper( -+ if_generation_match=generation_number, -+ if_source_generation_match=source_generation_number, -+ if_source_metageneration_not_match=source_metageneration_number, -+ ) -+ -+ def test_rename_blob_w_timeout(self): -+ timeout = 42 -+ self._rename_blob_helper(timeout=timeout) -+ -+ def test_rename_blob_w_retry(self): -+ retry = mock.Mock(spec={}) -+ self._rename_blob_helper(retry=retry) - -- blob.delete.assert_not_called() -+ def test_rename_blob_to_itself(self): -+ self._rename_blob_helper(same_name=True) - - def test_etag(self): - ETAG = "ETAG" -@@ -1783,15 +2082,22 @@ class Test_Bucket(unittest.TestCase): - self.assertTrue(config.uniform_bucket_level_access_enabled) - self.assertEqual(config.uniform_bucket_level_access_locked_time, now) - -- def test_lifecycle_rules_getter_unknown_action_type(self): -+ @mock.patch("warnings.warn") -+ def test_lifecycle_rules_getter_unknown_action_type(self, mock_warn): - NAME = "name" - BOGUS_RULE = {"action": {"type": "Bogus"}, "condition": {"age": 42}} - rules = [BOGUS_RULE] - properties = {"lifecycle": {"rule": rules}} - bucket = self._make_one(name=NAME, properties=properties) - -- with self.assertRaises(ValueError): -- list(bucket.lifecycle_rules) -+ list(bucket.lifecycle_rules) -+ mock_warn.assert_called_with( -+ "Unknown lifecycle rule type received: {}. Please upgrade to the latest version of google-cloud-storage.".format( -+ BOGUS_RULE -+ ), -+ UserWarning, -+ stacklevel=1, -+ ) - - def test_lifecycle_rules_getter(self): - from google.cloud.storage.bucket import ( -@@ -2005,23 +2311,39 @@ class Test_Bucket(unittest.TestCase): - self.assertEqual(bucket.labels, {"color": "red"}) - - # Make sure that a patch call correctly removes the flavor label. -- client = mock.NonCallableMock(spec=("_connection",)) -- client._connection = mock.NonCallableMock(spec=("api_request",)) -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = {} -+ - bucket.patch(client=client) -- client._connection.api_request.assert_called() -- _, _, kwargs = client._connection.api_request.mock_calls[0] -- self.assertEqual(len(kwargs["data"]["labels"]), 2) -- self.assertEqual(kwargs["data"]["labels"]["color"], "red") -- self.assertIsNone(kwargs["data"]["labels"]["flavor"]) -- self.assertEqual(kwargs["timeout"], self._get_default_timeout()) -+ -+ expected_patch_data = { -+ "labels": {"color": "red", "flavor": None}, -+ } -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ bucket.path, -+ expected_patch_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=bucket, -+ ) - - # A second patch call should be a no-op for labels. -- client._connection.api_request.reset_mock() -+ client._patch_resource.reset_mock() -+ - bucket.patch(client=client, timeout=42) -- client._connection.api_request.assert_called() -- _, _, kwargs = client._connection.api_request.mock_calls[0] -- self.assertNotIn("labels", kwargs["data"]) -- self.assertEqual(kwargs["timeout"], 42) -+ -+ expected_patch_data = {} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ bucket.path, -+ expected_patch_data, -+ query_params=expected_query_params, -+ timeout=42, -+ retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, -+ _target_object=bucket, -+ ) - - def test_location_type_getter_unset(self): - bucket = self._make_one() -@@ -2302,23 +2624,22 @@ class Test_Bucket(unittest.TestCase): - self.assertEqual(bucket.versioning_enabled, True) - - @mock.patch("warnings.warn") -- def test_create_deprecated(self, mock_warn): -- PROJECT = "PROJECT" -- BUCKET_NAME = "bucket-name" -- DATA = {"name": BUCKET_NAME} -- connection = _make_connection(DATA) -- client = self._make_client(project=PROJECT) -- client._base_connection = connection -+ def test_create_w_defaults_deprecated(self, mock_warn): -+ bucket_name = "bucket-name" -+ api_response = {"name": bucket_name} -+ client = mock.Mock(spec=["create_bucket"]) -+ client.create_bucket.return_value = api_response -+ bucket = self._make_one(client=client, name=bucket_name) - -- bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket.create() - -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": PROJECT}, -- data=DATA, -- _target_object=bucket, -+ client.create_bucket.assert_called_once_with( -+ bucket_or_name=bucket, -+ project=None, -+ user_project=None, -+ location=None, -+ predefined_acl=None, -+ predefined_default_object_acl=None, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, - ) -@@ -2331,26 +2652,40 @@ class Test_Bucket(unittest.TestCase): - ) - - @mock.patch("warnings.warn") -- def test_create_w_user_project(self, mock_warn): -- PROJECT = "PROJECT" -- BUCKET_NAME = "bucket-name" -- DATA = {"name": BUCKET_NAME} -- connection = _make_connection(DATA) -- client = self._make_client(project=PROJECT) -- client._base_connection = connection -+ def test_create_w_explicit_deprecated(self, mock_warn): -+ project = "PROJECT" -+ location = "eu" -+ user_project = "USER_PROJECT" -+ bucket_name = "bucket-name" -+ predefined_acl = "authenticatedRead" -+ predefined_default_object_acl = "bucketOwnerFullControl" -+ api_response = {"name": bucket_name} -+ client = mock.Mock(spec=["create_bucket"]) -+ client.create_bucket.return_value = api_response -+ bucket = self._make_one(client=None, name=bucket_name) -+ bucket._user_project = user_project -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- bucket = self._make_one(client=client, name=BUCKET_NAME) -- bucket._user_project = "USER_PROJECT" -- bucket.create() -- -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": PROJECT, "userProject": "USER_PROJECT"}, -- data=DATA, -- _target_object=bucket, -- timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -+ bucket.create( -+ client=client, -+ project=project, -+ location=location, -+ predefined_acl=predefined_acl, -+ predefined_default_object_acl=predefined_default_object_acl, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ client.create_bucket.assert_called_once_with( -+ bucket_or_name=bucket, -+ project=project, -+ user_project=user_project, -+ location=location, -+ predefined_acl=predefined_acl, -+ predefined_default_object_acl=predefined_default_object_acl, -+ timeout=timeout, -+ retry=retry, - ) - - mock_warn.assert_called_with( -@@ -2408,345 +2743,400 @@ class Test_Bucket(unittest.TestCase): - bucket.disable_website() - self.assertEqual(bucket._properties, UNSET) - -- def test_get_iam_policy(self): -+ def test_get_iam_policy_defaults(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - -- NAME = "name" -- PATH = "/b/%s" % (NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- EDITOR1 = "domain:google.com" -- EDITOR2 = "user:phred@example.com" -- VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" -- VIEWER2 = "user:phred@example.com" -- RETURNED = { -- "resourceId": PATH, -- "etag": ETAG, -- "version": VERSION, -+ bucket_name = "name" -+ path = "/b/%s" % (bucket_name,) -+ etag = "DEADBEEF" -+ version = 1 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ editor1 = "domain:google.com" -+ editor2 = "user:phred@example.com" -+ viewer1 = "serviceAccount:1234-abcdef@service.example.com" -+ viewer2 = "user:phred@example.com" -+ api_response = { -+ "resourceId": path, -+ "etag": etag, -+ "version": version, - "bindings": [ -- {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, -- {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, -- {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, -+ {"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}, -+ {"role": STORAGE_EDITOR_ROLE, "members": [editor1, editor2]}, -+ {"role": STORAGE_VIEWER_ROLE, "members": [viewer1, viewer2]}, - ], - } -- EXPECTED = { -- binding["role"]: set(binding["members"]) for binding in RETURNED["bindings"] -- } -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME) -+ expected_policy = { -+ binding["role"]: set(binding["members"]) -+ for binding in api_response["bindings"] -+ } -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=bucket_name) - -- policy = bucket.get_iam_policy(timeout=42) -+ policy = bucket.get_iam_policy() - - self.assertIsInstance(policy, Policy) -- self.assertEqual(policy.etag, RETURNED["etag"]) -- self.assertEqual(policy.version, RETURNED["version"]) -- self.assertEqual(dict(policy), EXPECTED) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {}) -- self.assertEqual(kw[0]["timeout"], 42) -+ self.assertEqual(policy.etag, api_response["etag"]) -+ self.assertEqual(policy.version, api_response["version"]) -+ self.assertEqual(dict(policy), expected_policy) -+ -+ expected_path = "/b/%s/iam" % (bucket_name,) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) - -- def test_get_iam_policy_w_user_project(self): -+ def test_get_iam_policy_w_user_project_w_timeout(self): - from google.api_core.iam import Policy - -- NAME = "name" -- USER_PROJECT = "user-project-123" -- PATH = "/b/%s" % (NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- RETURNED = { -- "resourceId": PATH, -- "etag": ETAG, -- "version": VERSION, -+ bucket_name = "name" -+ timeout = 42 -+ user_project = "user-project-123" -+ path = "/b/%s" % (bucket_name,) -+ etag = "DEADBEEF" -+ version = 1 -+ api_response = { -+ "resourceId": path, -+ "etag": etag, -+ "version": version, - "bindings": [], - } -- EXPECTED = {} -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) -+ expected_policy = {} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one( -+ client=client, name=bucket_name, user_project=user_project -+ ) - -- policy = bucket.get_iam_policy() -+ policy = bucket.get_iam_policy(timeout=timeout) - - self.assertIsInstance(policy, Policy) -- self.assertEqual(policy.etag, RETURNED["etag"]) -- self.assertEqual(policy.version, RETURNED["version"]) -- self.assertEqual(dict(policy), EXPECTED) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -+ self.assertEqual(policy.etag, api_response["etag"]) -+ self.assertEqual(policy.version, api_response["version"]) -+ self.assertEqual(dict(policy), expected_policy) -+ -+ expected_path = "/b/%s/iam" % (bucket_name,) -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) - -- def test_get_iam_policy_w_requested_policy_version(self): -+ def test_get_iam_policy_w_requested_policy_version_w_retry(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - -- NAME = "name" -- PATH = "/b/%s" % (NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- RETURNED = { -- "resourceId": PATH, -- "etag": ETAG, -- "version": VERSION, -- "bindings": [{"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}], -- } -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME) -- -- policy = bucket.get_iam_policy(requested_policy_version=3) -- -- self.assertEqual(policy.version, VERSION) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {"optionsRequestedPolicyVersion": 3}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -+ bucket_name = "name" -+ path = "/b/%s" % (bucket_name,) -+ etag = "DEADBEEF" -+ version = 3 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ api_response = { -+ "resourceId": path, -+ "etag": etag, -+ "version": version, -+ "bindings": [{"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}], -+ } -+ retry = mock.Mock(spec=[]) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=bucket_name) -+ -+ policy = bucket.get_iam_policy(requested_policy_version=3, retry=retry) -+ -+ self.assertEqual(policy.version, version) -+ -+ expected_path = "/b/%s/iam" % (bucket_name,) -+ expected_query_params = {"optionsRequestedPolicyVersion": version} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=None, -+ ) - -- def test_set_iam_policy(self): -+ def test_set_iam_policy_w_defaults(self): - import operator - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - -- NAME = "name" -- PATH = "/b/%s" % (NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- EDITOR1 = "domain:google.com" -- EDITOR2 = "user:phred@example.com" -- VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" -- VIEWER2 = "user:phred@example.com" -- BINDINGS = [ -- {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, -- {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, -- {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, -+ name = "name" -+ etag = "DEADBEEF" -+ version = 1 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ editor1 = "domain:google.com" -+ editor2 = "user:phred@example.com" -+ viewer1 = "serviceAccount:1234-abcdef@service.example.com" -+ viewer2 = "user:phred@example.com" -+ bindings = [ -+ {"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}, -+ {"role": STORAGE_EDITOR_ROLE, "members": [editor1, editor2]}, -+ {"role": STORAGE_VIEWER_ROLE, "members": [viewer1, viewer2]}, - ] -- RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} - policy = Policy() -- for binding in BINDINGS: -+ for binding in bindings: - policy[binding["role"]] = binding["members"] - -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME) -+ api_response = {"etag": etag, "version": version, "bindings": bindings} -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=name) - -- returned = bucket.set_iam_policy(policy, timeout=42) -+ returned = bucket.set_iam_policy(policy) - -- self.assertEqual(returned.etag, ETAG) -- self.assertEqual(returned.version, VERSION) -+ self.assertEqual(returned.etag, etag) -+ self.assertEqual(returned.version, version) - self.assertEqual(dict(returned), dict(policy)) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {}) -- self.assertEqual(kw[0]["timeout"], 42) -- sent = kw[0]["data"] -- self.assertEqual(sent["resourceId"], PATH) -- self.assertEqual(len(sent["bindings"]), len(BINDINGS)) -+ expected_path = "%s/iam" % (bucket.path,) -+ expected_data = { -+ "resourceId": bucket.path, -+ "bindings": mock.ANY, -+ } -+ expected_query_params = {} -+ client._put_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_ETAG_IN_JSON, -+ _target_object=None, -+ ) -+ -+ sent_bindings = client._put_resource.call_args.args[1]["bindings"] - key = operator.itemgetter("role") - for found, expected in zip( -- sorted(sent["bindings"], key=key), sorted(BINDINGS, key=key) -+ sorted(sent_bindings, key=key), sorted(bindings, key=key) - ): - self.assertEqual(found["role"], expected["role"]) - self.assertEqual(sorted(found["members"]), sorted(expected["members"])) - -- def test_set_iam_policy_w_user_project(self): -+ def test_set_iam_policy_w_user_project_w_expl_client_w_timeout_retry(self): - import operator - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - -- NAME = "name" -- USER_PROJECT = "user-project-123" -- PATH = "/b/%s" % (NAME,) -- ETAG = "DEADBEEF" -- VERSION = 1 -- OWNER1 = "user:phred@example.com" -- OWNER2 = "group:cloud-logs@google.com" -- EDITOR1 = "domain:google.com" -- EDITOR2 = "user:phred@example.com" -- VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" -- VIEWER2 = "user:phred@example.com" -- BINDINGS = [ -- {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, -- {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, -- {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, -+ name = "name" -+ user_project = "user-project-123" -+ etag = "DEADBEEF" -+ version = 1 -+ owner1 = "user:phred@example.com" -+ owner2 = "group:cloud-logs@google.com" -+ editor1 = "domain:google.com" -+ editor2 = "user:phred@example.com" -+ viewer1 = "serviceAccount:1234-abcdef@service.example.com" -+ viewer2 = "user:phred@example.com" -+ bindings = [ -+ {"role": STORAGE_OWNER_ROLE, "members": [owner1, owner2]}, -+ {"role": STORAGE_EDITOR_ROLE, "members": [editor1, editor2]}, -+ {"role": STORAGE_VIEWER_ROLE, "members": [viewer1, viewer2]}, - ] -- RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} - policy = Policy() -- for binding in BINDINGS: -+ for binding in bindings: - policy[binding["role"]] = binding["members"] - -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) -+ api_response = {"etag": etag, "version": version, "bindings": bindings} -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = api_response -+ bucket = self._make_one(client=None, name=name, user_project=user_project) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- returned = bucket.set_iam_policy(policy) -+ returned = bucket.set_iam_policy( -+ policy, client=client, timeout=timeout, retry=retry -+ ) - -- self.assertEqual(returned.etag, ETAG) -- self.assertEqual(returned.version, VERSION) -+ self.assertEqual(returned.etag, etag) -+ self.assertEqual(returned.version, version) - self.assertEqual(dict(returned), dict(policy)) - -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PUT") -- self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -- sent = kw[0]["data"] -- self.assertEqual(sent["resourceId"], PATH) -- self.assertEqual(len(sent["bindings"]), len(BINDINGS)) -+ expected_path = "%s/iam" % (bucket.path,) -+ expected_data = { -+ "resourceId": bucket.path, -+ "bindings": mock.ANY, -+ } -+ expected_query_params = {"userProject": user_project} -+ client._put_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=None, -+ ) -+ -+ sent_bindings = client._put_resource.call_args.args[1]["bindings"] - key = operator.itemgetter("role") - for found, expected in zip( -- sorted(sent["bindings"], key=key), sorted(BINDINGS, key=key) -+ sorted(sent_bindings, key=key), sorted(bindings, key=key) - ): - self.assertEqual(found["role"], expected["role"]) - self.assertEqual(sorted(found["members"]), sorted(expected["members"])) - -- def test_test_iam_permissions(self): -+ def test_test_iam_permissions_defaults(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - -- NAME = "name" -- PATH = "/b/%s" % (NAME,) -- PERMISSIONS = [ -+ name = "name" -+ permissions = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] -- ALLOWED = PERMISSIONS[1:] -- RETURNED = {"permissions": ALLOWED} -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME) -- -- allowed = bucket.test_iam_permissions(PERMISSIONS, timeout=42) -- -- self.assertEqual(allowed, ALLOWED) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) -- self.assertEqual(kw[0]["query_params"], {"permissions": PERMISSIONS}) -- self.assertEqual(kw[0]["timeout"], 42) -+ expected = permissions[1:] -+ api_response = {"permissions": expected} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=name) -+ -+ found = bucket.test_iam_permissions(permissions) -+ -+ self.assertEqual(found, expected) - -- def test_test_iam_permissions_w_user_project(self): -+ expected_path = "/b/%s/iam/testPermissions" % (name,) -+ expected_query_params = {} -+ expected_query_params = {"permissions": permissions} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, -+ ) -+ -+ def test_test_iam_permissions_w_user_project_w_timeout_w_retry(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - -- NAME = "name" -- USER_PROJECT = "user-project-123" -- PATH = "/b/%s" % (NAME,) -- PERMISSIONS = [ -+ name = "name" -+ user_project = "user-project-123" -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ permissions = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] -- ALLOWED = PERMISSIONS[1:] -- RETURNED = {"permissions": ALLOWED} -- connection = _Connection(RETURNED) -- client = _Client(connection, None) -- bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) -- -- allowed = bucket.test_iam_permissions(PERMISSIONS) -- -- self.assertEqual(allowed, ALLOWED) -- -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "GET") -- self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) -- self.assertEqual( -- kw[0]["query_params"], -- {"permissions": PERMISSIONS, "userProject": USER_PROJECT}, -+ expected = permissions[1:] -+ api_response = {"permissions": expected} -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=name, user_project=user_project) -+ -+ found = bucket.test_iam_permissions(permissions, timeout=timeout, retry=retry) -+ -+ self.assertEqual(found, expected) -+ -+ expected_path = "/b/%s/iam/testPermissions" % (name,) -+ expected_query_params = { -+ "permissions": permissions, -+ "userProject": user_project, -+ } -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=None, - ) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def test_make_public_defaults(self): - from google.cloud.storage.acl import _ACLEntity - -- NAME = "name" -+ name = "name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] -- after = {"acl": permissive, "defaultObjectAcl": []} -- connection = _Connection(after) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ api_response = {"acl": permissive, "defaultObjectAcl": []} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True -+ - bucket.make_public() -+ - self.assertEqual(list(bucket.acl), permissive) - self.assertEqual(list(bucket.default_object_acl), []) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[0]["data"], {"acl": after["acl"]}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -+ -+ expected_path = bucket.path -+ expected_data = {"acl": permissive} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) - - def _make_public_w_future_helper(self, default_object_acl_loaded=True): - from google.cloud.storage.acl import _ACLEntity - -- NAME = "name" -+ name = "name" -+ get_api_response = {"items": []} - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] -- after1 = {"acl": permissive, "defaultObjectAcl": []} -- after2 = {"acl": permissive, "defaultObjectAcl": permissive} -- if default_object_acl_loaded: -- num_requests = 2 -- connection = _Connection(after1, after2) -- else: -- num_requests = 3 -- # We return the same value for default_object_acl.reload() -- # to consume. -- connection = _Connection(after1, after1, after2) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ acl_patched_response = {"acl": permissive, "defaultObjectAcl": []} -+ dac_patched_response = {"acl": permissive, "defaultObjectAcl": permissive} -+ client = mock.Mock(spec=["_get_resource", "_patch_resource"]) -+ client._get_resource.return_value = get_api_response -+ client._patch_resource.side_effect = [ -+ acl_patched_response, -+ dac_patched_response, -+ ] -+ -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = default_object_acl_loaded -+ - bucket.make_public(future=True) -+ - self.assertEqual(list(bucket.acl), permissive) - self.assertEqual(list(bucket.default_object_acl), permissive) -- kw = connection._requested -- self.assertEqual(len(kw), num_requests) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[0]["data"], {"acl": permissive}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -+ -+ self.assertEqual(len(client._patch_resource.call_args_list), 2) -+ expected_acl_data = {"acl": permissive} -+ expected_dac_data = {"defaultObjectAcl": permissive} -+ expected_kw = { -+ "query_params": {"projection": "full"}, -+ "timeout": self._get_default_timeout(), -+ "retry": None, -+ } -+ client._patch_resource.assert_has_calls( -+ [ -+ ((bucket.path, expected_acl_data), expected_kw), -+ ((bucket.path, expected_dac_data), expected_kw), -+ ] -+ ) -+ - if not default_object_acl_loaded: -- self.assertEqual(kw[1]["method"], "GET") -- self.assertEqual(kw[1]["path"], "/b/%s/defaultObjectAcl" % NAME) -- # Last could be 1 or 2 depending on `default_object_acl_loaded`. -- self.assertEqual(kw[-1]["method"], "PATCH") -- self.assertEqual(kw[-1]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[-1]["data"], {"defaultObjectAcl": permissive}) -- self.assertEqual(kw[-1]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[-1]["timeout"], self._get_default_timeout()) -+ expected_path = "/b/%s/defaultObjectAcl" % (name,) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) -+ else: -+ client._get_resource.assert_not_called() - - def test_make_public_w_future(self): - self._make_public_w_future_helper(default_object_acl_loaded=True) -@@ -2782,119 +3172,153 @@ class Test_Bucket(unittest.TestCase): - (self._bucket, self._name, self._granted, client, timeout) - ) - -- def item_to_blob(self, item): -- return _Blob(self.bucket, item["name"]) -- -- NAME = "name" -- BLOB_NAME = "blob-name" -+ name = "name" -+ blob_name = "blob-name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] -- after = {"acl": permissive, "defaultObjectAcl": []} -- connection = _Connection(after, {"items": [{"name": BLOB_NAME}]}) -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -+ -+ patch_acl_response = {"acl": permissive, "defaultObjectAcl": []} -+ client = mock.Mock(spec=["list_blobs", "_patch_resource"]) -+ client._patch_resource.return_value = patch_acl_response -+ -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - -- with mock.patch("google.cloud.storage.client._item_to_blob", new=item_to_blob): -- bucket.make_public(recursive=True, timeout=42, retry=DEFAULT_RETRY) -+ list_blobs_response = iter([_Blob(bucket, blob_name)]) -+ client.list_blobs.return_value = list_blobs_response -+ -+ timeout = 42 -+ -+ bucket.make_public(recursive=True, timeout=timeout) - - self.assertEqual(list(bucket.acl), permissive) - self.assertEqual(list(bucket.default_object_acl), []) -- self.assertEqual(_saved, [(bucket, BLOB_NAME, True, None, 42)]) -- kw = connection._requested -- self.assertEqual(len(kw), 2) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[0]["data"], {"acl": permissive}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual(kw[1]["method"], "GET") -- self.assertEqual(kw[1]["path"], "/b/%s/o" % NAME) -- self.assertEqual(kw[1]["retry"], DEFAULT_RETRY) -- max_results = bucket._MAX_OBJECTS_FOR_ITERATION + 1 -- self.assertEqual( -- kw[1]["query_params"], {"maxResults": max_results, "projection": "full"} -+ self.assertEqual(_saved, [(bucket, blob_name, True, None, timeout)]) -+ -+ expected_patch_data = {"acl": permissive} -+ expected_patch_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ bucket.path, -+ expected_patch_data, -+ query_params=expected_patch_query_params, -+ timeout=timeout, -+ retry=None, - ) -- self.assertEqual(kw[1]["timeout"], 42) -+ client.list_blobs.assert_called_once() - - def test_make_public_recursive_too_many(self): - from google.cloud.storage.acl import _ACLEntity - -- PERMISSIVE = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] -- AFTER = {"acl": PERMISSIVE, "defaultObjectAcl": []} -+ permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] - -- NAME = "name" -- BLOB_NAME1 = "blob-name1" -- BLOB_NAME2 = "blob-name2" -- GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} -- connection = _Connection(AFTER, GET_BLOBS_RESP) -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -+ name = "name" -+ blob1 = mock.Mock(spec=[]) -+ blob2 = mock.Mock(spec=[]) -+ patch_acl_response = {"acl": permissive, "defaultObjectAcl": []} -+ list_blobs_response = iter([blob1, blob2]) -+ client = mock.Mock(spec=["list_blobs", "_patch_resource"]) -+ client.list_blobs.return_value = list_blobs_response -+ client._patch_resource.return_value = patch_acl_response -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - - # Make the Bucket refuse to make_public with 2 objects. - bucket._MAX_OBJECTS_FOR_ITERATION = 1 -- self.assertRaises(ValueError, bucket.make_public, recursive=True) -+ -+ with self.assertRaises(ValueError): -+ bucket.make_public(recursive=True) -+ -+ expected_path = bucket.path -+ expected_data = {"acl": permissive} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ client.list_blobs.assert_called_once() - - def test_make_private_defaults(self): -- NAME = "name" -+ name = "name" - no_permissions = [] -- after = {"acl": no_permissions, "defaultObjectAcl": []} -- connection = _Connection(after) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ api_response = {"acl": no_permissions, "defaultObjectAcl": []} -+ client = mock.Mock(spec=["_patch_resource"]) -+ client._patch_resource.return_value = api_response -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True -+ - bucket.make_private() -+ - self.assertEqual(list(bucket.acl), no_permissions) - self.assertEqual(list(bucket.default_object_acl), []) -- kw = connection._requested -- self.assertEqual(len(kw), 1) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[0]["data"], {"acl": after["acl"]}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -+ -+ expected_path = bucket.path -+ expected_data = {"acl": no_permissions} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) - - def _make_private_w_future_helper(self, default_object_acl_loaded=True): -- NAME = "name" -+ name = "name" - no_permissions = [] -- after1 = {"acl": no_permissions, "defaultObjectAcl": []} -- after2 = {"acl": no_permissions, "defaultObjectAcl": no_permissions} -- if default_object_acl_loaded: -- num_requests = 2 -- connection = _Connection(after1, after2) -- else: -- num_requests = 3 -- # We return the same value for default_object_acl.reload() -- # to consume. -- connection = _Connection(after1, after1, after2) -- client = _Client(connection) -- bucket = self._make_one(client=client, name=NAME) -+ get_api_response = {"items": []} -+ acl_patched_response = {"acl": no_permissions, "defaultObjectAcl": []} -+ dac_patched_response = { -+ "acl": no_permissions, -+ "defaultObjectAcl": no_permissions, -+ } -+ client = mock.Mock(spec=["_get_resource", "_patch_resource"]) -+ client._get_resource.return_value = get_api_response -+ client._patch_resource.side_effect = [ -+ acl_patched_response, -+ dac_patched_response, -+ ] -+ -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = default_object_acl_loaded -+ - bucket.make_private(future=True) -+ - self.assertEqual(list(bucket.acl), no_permissions) - self.assertEqual(list(bucket.default_object_acl), no_permissions) -- kw = connection._requested -- self.assertEqual(len(kw), num_requests) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[0]["data"], {"acl": no_permissions}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) -+ -+ self.assertEqual(len(client._patch_resource.call_args_list), 2) -+ expected_acl_data = {"acl": no_permissions} -+ expected_dac_data = {"defaultObjectAcl": no_permissions} -+ expected_kw = { -+ "query_params": {"projection": "full"}, -+ "timeout": self._get_default_timeout(), -+ "retry": None, -+ } -+ client._patch_resource.assert_has_calls( -+ [ -+ ((bucket.path, expected_acl_data), expected_kw), -+ ((bucket.path, expected_dac_data), expected_kw), -+ ] -+ ) -+ - if not default_object_acl_loaded: -- self.assertEqual(kw[1]["method"], "GET") -- self.assertEqual(kw[1]["path"], "/b/%s/defaultObjectAcl" % NAME) -- # Last could be 1 or 2 depending on `default_object_acl_loaded`. -- self.assertEqual(kw[-1]["method"], "PATCH") -- self.assertEqual(kw[-1]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[-1]["data"], {"defaultObjectAcl": no_permissions}) -- self.assertEqual(kw[-1]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[-1]["timeout"], self._get_default_timeout()) -+ expected_path = "/b/%s/defaultObjectAcl" % (name,) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ ) -+ else: -+ client._get_resource.assert_not_called() - - def test_make_private_w_future(self): - self._make_private_w_future_helper(default_object_acl_loaded=True) -@@ -2928,150 +3352,83 @@ class Test_Bucket(unittest.TestCase): - (self._bucket, self._name, self._granted, client, timeout) - ) - -- def item_to_blob(self, item): -- return _Blob(self.bucket, item["name"]) -- -- NAME = "name" -- BLOB_NAME = "blob-name" -+ name = "name" -+ blob_name = "blob-name" - no_permissions = [] -- after = {"acl": no_permissions, "defaultObjectAcl": []} -- connection = _Connection(after, {"items": [{"name": BLOB_NAME}]}) -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -- bucket.acl.loaded = True -- bucket.default_object_acl.loaded = True -- -- with mock.patch("google.cloud.storage.client._item_to_blob", new=item_to_blob): -- bucket.make_private(recursive=True, timeout=42, retry=DEFAULT_RETRY) -- self.assertEqual(list(bucket.acl), no_permissions) -- self.assertEqual(list(bucket.default_object_acl), []) -- self.assertEqual(_saved, [(bucket, BLOB_NAME, False, None, 42)]) -- kw = connection._requested -- self.assertEqual(len(kw), 2) -- self.assertEqual(kw[0]["method"], "PATCH") -- self.assertEqual(kw[0]["path"], "/b/%s" % NAME) -- self.assertEqual(kw[0]["data"], {"acl": no_permissions}) -- self.assertEqual(kw[0]["query_params"], {"projection": "full"}) -- self.assertEqual(kw[0]["timeout"], 42) -- self.assertEqual(kw[1]["method"], "GET") -- self.assertEqual(kw[1]["path"], "/b/%s/o" % NAME) -- self.assertEqual(kw[1]["retry"], DEFAULT_RETRY) -- max_results = bucket._MAX_OBJECTS_FOR_ITERATION + 1 -- self.assertEqual( -- kw[1]["query_params"], {"maxResults": max_results, "projection": "full"} -- ) -- self.assertEqual(kw[1]["timeout"], 42) - -- def test_make_private_recursive_too_many(self): -- NO_PERMISSIONS = [] -- AFTER = {"acl": NO_PERMISSIONS, "defaultObjectAcl": []} -+ patch_acl_response = {"acl": no_permissions, "defaultObjectAcl": []} -+ client = mock.Mock(spec=["list_blobs", "_patch_resource"]) -+ client._patch_resource.return_value = patch_acl_response - -- NAME = "name" -- BLOB_NAME1 = "blob-name1" -- BLOB_NAME2 = "blob-name2" -- GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} -- connection = _Connection(AFTER, GET_BLOBS_RESP) -- client = self._make_client() -- client._base_connection = connection -- bucket = self._make_one(client=client, name=NAME) -+ bucket = self._make_one(client=client, name=name) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - -- # Make the Bucket refuse to make_private with 2 objects. -- bucket._MAX_OBJECTS_FOR_ITERATION = 1 -- self.assertRaises(ValueError, bucket.make_private, recursive=True) -- -- def test_page_empty_response(self): -- from google.api_core import page_iterator -- -- connection = _Connection() -- client = self._make_client() -- client._base_connection = connection -- name = "name" -- bucket = self._make_one(client=client, name=name) -- iterator = bucket.list_blobs() -- page = page_iterator.Page(iterator, (), None) -- iterator._page = page -- blobs = list(page) -- self.assertEqual(blobs, []) -- self.assertEqual(iterator.prefixes, set()) -+ list_blobs_response = iter([_Blob(bucket, blob_name)]) -+ client.list_blobs.return_value = list_blobs_response - -- def test_page_non_empty_response(self): -- import six -- from google.cloud.storage.blob import Blob -+ timeout = 42 - -- blob_name = "blob-name" -- response = {"items": [{"name": blob_name}], "prefixes": ["foo"]} -- connection = _Connection() -- client = self._make_client() -- client._base_connection = connection -- name = "name" -- bucket = self._make_one(client=client, name=name) -+ bucket.make_private(recursive=True, timeout=42) - -- def fake_response(): -- return response -+ self.assertEqual(list(bucket.acl), no_permissions) -+ self.assertEqual(list(bucket.default_object_acl), []) -+ self.assertEqual(_saved, [(bucket, blob_name, False, None, timeout)]) - -- iterator = bucket.list_blobs() -- iterator._get_next_page_response = fake_response -+ expected_patch_data = {"acl": no_permissions} -+ expected_patch_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ bucket.path, -+ expected_patch_data, -+ query_params=expected_patch_query_params, -+ timeout=timeout, -+ retry=None, -+ ) - -- page = six.next(iterator.pages) -- self.assertEqual(page.prefixes, ("foo",)) -- self.assertEqual(page.num_items, 1) -- blob = six.next(page) -- self.assertEqual(page.remaining, 0) -- self.assertIsInstance(blob, Blob) -- self.assertEqual(blob.name, blob_name) -- self.assertEqual(iterator.prefixes, set(["foo"])) -+ client.list_blobs.assert_called_once() - -- def test_cumulative_prefixes(self): -- import six -- from google.cloud.storage.blob import Blob -+ def test_make_private_recursive_too_many(self): -+ no_permissions = [] - -- BLOB_NAME = "blob-name1" -- response1 = { -- "items": [{"name": BLOB_NAME}], -- "prefixes": ["foo"], -- "nextPageToken": "s39rmf9", -- } -- response2 = {"items": [], "prefixes": ["bar"]} -- client = self._make_client() - name = "name" -+ blob1 = mock.Mock(spec=[]) -+ blob2 = mock.Mock(spec=[]) -+ patch_acl_response = {"acl": no_permissions, "defaultObjectAcl": []} -+ list_blobs_response = iter([blob1, blob2]) -+ client = mock.Mock(spec=["list_blobs", "_patch_resource"]) -+ client.list_blobs.return_value = list_blobs_response -+ client._patch_resource.return_value = patch_acl_response - bucket = self._make_one(client=client, name=name) -- responses = [response1, response2] -+ bucket.acl.loaded = True -+ bucket.default_object_acl.loaded = True - -- def fake_response(): -- return responses.pop(0) -+ # Make the Bucket refuse to make_private with 2 objects. -+ bucket._MAX_OBJECTS_FOR_ITERATION = 1 - -- iterator = bucket.list_blobs() -- iterator._get_next_page_response = fake_response -+ with self.assertRaises(ValueError): -+ bucket.make_private(recursive=True) - -- # Parse first response. -- pages_iter = iterator.pages -- page1 = six.next(pages_iter) -- self.assertEqual(page1.prefixes, ("foo",)) -- self.assertEqual(page1.num_items, 1) -- blob = six.next(page1) -- self.assertEqual(page1.remaining, 0) -- self.assertIsInstance(blob, Blob) -- self.assertEqual(blob.name, BLOB_NAME) -- self.assertEqual(iterator.prefixes, set(["foo"])) -- # Parse second response. -- page2 = six.next(pages_iter) -- self.assertEqual(page2.prefixes, ("bar",)) -- self.assertEqual(page2.num_items, 0) -- self.assertEqual(iterator.prefixes, set(["foo", "bar"])) -+ expected_path = bucket.path -+ expected_data = {"acl": no_permissions} -+ expected_query_params = {"projection": "full"} -+ client._patch_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ ) -+ -+ client.list_blobs.assert_called_once() - -- def _test_generate_upload_policy_helper(self, **kwargs): -+ def _generate_upload_policy_helper(self, **kwargs): - import base64 - import json - - credentials = _create_signing_credentials() - credentials.signer_email = mock.sentinel.signer_email - credentials.sign_bytes.return_value = b"DEADBEEF" -- connection = _Connection() -- connection.credentials = credentials -- client = _Client(connection) -+ client = self._make_client(_credentials=credentials) - name = "name" - bucket = self._make_one(client=client, name=name) - -@@ -3108,7 +3465,7 @@ class Test_Bucket(unittest.TestCase): - def test_generate_upload_policy(self, now): - from google.cloud._helpers import _datetime_to_rfc3339 - -- _, policy = self._test_generate_upload_policy_helper() -+ _, policy = self._generate_upload_policy_helper() - - self.assertEqual( - policy["expiration"], -@@ -3120,15 +3477,13 @@ class Test_Bucket(unittest.TestCase): - - expiration = datetime.datetime(1990, 5, 29) - -- _, policy = self._test_generate_upload_policy_helper(expiration=expiration) -+ _, policy = self._generate_upload_policy_helper(expiration=expiration) - - self.assertEqual(policy["expiration"], _datetime_to_rfc3339(expiration)) - - def test_generate_upload_policy_bad_credentials(self): - credentials = object() -- connection = _Connection() -- connection.credentials = credentials -- client = _Client(connection) -+ client = self._make_client(_credentials=credentials) - name = "name" - bucket = self._make_one(client=client, name=name) - -@@ -3136,10 +3491,7 @@ class Test_Bucket(unittest.TestCase): - bucket.generate_upload_policy([]) - - def test_lock_retention_policy_no_policy_set(self): -- credentials = object() -- connection = _Connection() -- connection.credentials = credentials -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) - name = "name" - bucket = self._make_one(client=client, name=name) - bucket._properties["metageneration"] = 1234 -@@ -3147,11 +3499,10 @@ class Test_Bucket(unittest.TestCase): - with self.assertRaises(ValueError): - bucket.lock_retention_policy() - -+ client._post_resource.assert_not_called() -+ - def test_lock_retention_policy_no_metageneration(self): -- credentials = object() -- connection = _Connection() -- connection.credentials = credentials -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) - name = "name" - bucket = self._make_one(client=client, name=name) - bucket._properties["retentionPolicy"] = { -@@ -3162,11 +3513,10 @@ class Test_Bucket(unittest.TestCase): - with self.assertRaises(ValueError): - bucket.lock_retention_policy() - -+ client._post_resource.assert_not_called() -+ - def test_lock_retention_policy_already_locked(self): -- credentials = object() -- connection = _Connection() -- connection.credentials = credentials -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) - name = "name" - bucket = self._make_one(client=client, name=name) - bucket._properties["metageneration"] = 1234 -@@ -3179,74 +3529,92 @@ class Test_Bucket(unittest.TestCase): - with self.assertRaises(ValueError): - bucket.lock_retention_policy() - -- def test_lock_retention_policy_ok(self): -+ client._post_resource.assert_not_called() -+ -+ def test_lock_retention_policy_ok_w_timeout_w_retry(self): - name = "name" -- response = { -+ effective_time = "2018-03-01T16:46:27.123456Z" -+ one_hundred_days = 86400 * 100 # seconds in 100 days -+ metageneration = 1234 -+ api_response = { - "name": name, -- "metageneration": 1235, -+ "metageneration": metageneration + 1, - "retentionPolicy": { -- "effectiveTime": "2018-03-01T16:46:27.123456Z", -+ "effectiveTime": effective_time, - "isLocked": True, -- "retentionPeriod": 86400 * 100, # 100 days -+ "retentionPeriod": one_hundred_days, - }, - } -- credentials = object() -- connection = _Connection(response) -- connection.credentials = credentials -- client = _Client(connection) -+ metageneration = 1234 -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = self._make_one(client=client, name=name) -- bucket._properties["metageneration"] = 1234 -+ bucket._properties["metageneration"] = metageneration - bucket._properties["retentionPolicy"] = { -- "effectiveTime": "2018-03-01T16:46:27.123456Z", -- "retentionPeriod": 86400 * 100, # 100 days -+ "effectiveTime": effective_time, -+ "retentionPeriod": one_hundred_days, - } -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- bucket.lock_retention_policy(timeout=42) -+ bucket.lock_retention_policy(timeout=timeout, retry=retry) - -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) -- self.assertEqual(kw["query_params"], {"ifMetagenerationMatch": 1234}) -- self.assertEqual(kw["timeout"], 42) -+ expected_path = "/b/{}/lockRetentionPolicy".format(name) -+ expected_data = None -+ expected_query_params = {"ifMetagenerationMatch": metageneration} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ _target_object=bucket, -+ ) - - def test_lock_retention_policy_w_user_project(self): - name = "name" - user_project = "user-project-123" -- response = { -+ metageneration = 1234 -+ effective_time = "2018-03-01T16:46:27.123456Z" -+ one_hundred_days = 86400 * 100 # seconds in 100 days -+ api_response = { - "name": name, -- "metageneration": 1235, -+ "metageneration": metageneration + 1, - "retentionPolicy": { -- "effectiveTime": "2018-03-01T16:46:27.123456Z", -+ "effectiveTime": effective_time, - "isLocked": True, -- "retentionPeriod": 86400 * 100, # 100 days -+ "retentionPeriod": one_hundred_days, - }, - } -- credentials = object() -- connection = _Connection(response) -- connection.credentials = credentials -- client = _Client(connection) -+ client = mock.Mock(spec=["_post_resource"]) -+ client._post_resource.return_value = api_response - bucket = self._make_one(client=client, name=name, user_project=user_project) - bucket._properties["metageneration"] = 1234 - bucket._properties["retentionPolicy"] = { -- "effectiveTime": "2018-03-01T16:46:27.123456Z", -- "retentionPeriod": 86400 * 100, # 100 days -+ "effectiveTime": effective_time, -+ "retentionPeriod": one_hundred_days, - } - - bucket.lock_retention_policy() - -- (kw,) = connection._requested -- self.assertEqual(kw["method"], "POST") -- self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) -- self.assertEqual( -- kw["query_params"], -- {"ifMetagenerationMatch": 1234, "userProject": user_project}, -+ expected_path = "/b/{}/lockRetentionPolicy".format(name) -+ expected_data = None -+ expected_query_params = { -+ "ifMetagenerationMatch": metageneration, -+ "userProject": user_project, -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) -- self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_generate_signed_url_w_invalid_version(self): - expiration = "2014-10-16T20:34:37.000Z" -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - bucket = self._make_one(name="bucket_name", client=client) - with self.assertRaises(ValueError): - bucket.generate_signed_url(expiration, version="nonesuch") -@@ -3282,8 +3650,7 @@ class Test_Bucket(unittest.TestCase): - if expiration is None: - expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + delta - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client(_credentials=credentials) - bucket = self._make_one(name=bucket_name, client=client) - - if version is None: -@@ -3343,11 +3710,12 @@ class Test_Bucket(unittest.TestCase): - def test_get_bucket_from_string_w_valid_uri(self): - from google.cloud.storage.bucket import Bucket - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - BUCKET_NAME = "BUCKET_NAME" - uri = "gs://" + BUCKET_NAME -+ - bucket = Bucket.from_string(uri, client) -+ - self.assertIsInstance(bucket, Bucket) - self.assertIs(bucket.client, client) - self.assertEqual(bucket.name, BUCKET_NAME) -@@ -3355,8 +3723,7 @@ class Test_Bucket(unittest.TestCase): - def test_get_bucket_from_string_w_invalid_uri(self): - from google.cloud.storage.bucket import Bucket - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - - with pytest.raises(ValueError, match="URI scheme must be gs"): - Bucket.from_string("http://bucket_name", client) -@@ -3364,11 +3731,12 @@ class Test_Bucket(unittest.TestCase): - def test_get_bucket_from_string_w_domain_name_bucket(self): - from google.cloud.storage.bucket import Bucket - -- connection = _Connection() -- client = _Client(connection) -+ client = self._make_client() - BUCKET_NAME = "buckets.example.com" - uri = "gs://" + BUCKET_NAME -+ - bucket = Bucket.from_string(uri, client) -+ - self.assertIsInstance(bucket, Bucket) - self.assertIs(bucket.client, client) - self.assertEqual(bucket.name, BUCKET_NAME) -@@ -3474,51 +3842,32 @@ class Test_Bucket(unittest.TestCase): - self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") - - --class _Connection(object): -- _delete_bucket = False -+class Test__item_to_notification(unittest.TestCase): -+ def _call_fut(self, iterator, item): -+ from google.cloud.storage.bucket import _item_to_notification - -- def __init__(self, *responses): -- self._responses = responses -- self._requested = [] -- self._deleted_buckets = [] -- self.credentials = None -- -- @staticmethod -- def _is_bucket_path(path): -- # Now just ensure the path only has /b/ and one more segment. -- return path.startswith("/b/") and path.count("/") == 2 -- -- def api_request(self, **kw): -- from google.cloud.exceptions import NotFound -- -- self._requested.append(kw) -- -- method = kw.get("method") -- path = kw.get("path", "") -- if method == "DELETE" and self._is_bucket_path(path): -- self._deleted_buckets.append(kw) -- if self._delete_bucket: -- return -- else: -- raise NotFound("miss") -- -- try: -- response, self._responses = self._responses[0], self._responses[1:] -- except IndexError: -- raise NotFound("miss") -- else: -- return response -+ return _item_to_notification(iterator, item) - -+ def test_it(self): -+ from google.cloud.storage.notification import BucketNotification -+ from google.cloud.storage.notification import _TOPIC_REF_FMT -+ from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT - --class _Client(object): -- def __init__(self, connection, project=None): -- self._base_connection = connection -- self.project = project -+ iterator = mock.Mock(spec=["bucket"]) -+ project = "my-project-123" -+ topic = "topic-1" -+ item = { -+ "topic": _TOPIC_REF_FMT.format(project, topic), -+ "id": "1", -+ "etag": "DEADBEEF", -+ "selfLink": "https://example.com/notification/1", -+ "payload_format": NONE_PAYLOAD_FORMAT, -+ } - -- @property -- def _connection(self): -- return self._base_connection -+ notification = self._call_fut(iterator, item) - -- @property -- def _credentials(self): -- return self._base_connection.credentials -+ self.assertIsInstance(notification, BucketNotification) -+ self.assertIs(notification._bucket, iterator.bucket) -+ self.assertEqual(notification._topic_name, topic) -+ self.assertEqual(notification._topic_project, project) -+ self.assertEqual(notification._properties, item) ---- a/tests/unit/test_client.py -+++ b/tests/unit/test_client.py -@@ -39,9 +39,12 @@ _POST_POLICY_TESTS = [test for test in _ - _FAKE_CREDENTIALS = Credentials.from_service_account_info(_SERVICE_ACCOUNT_JSON) - - --def _make_credentials(): -+def _make_credentials(project=None): - import google.auth.credentials - -+ if project is not None: -+ return mock.Mock(spec=google.auth.credentials.Credentials, project_id=project) -+ - return mock.Mock(spec=google.auth.credentials.Credentials) - - -@@ -174,14 +177,9 @@ class TestClient(unittest.TestCase): - from google.cloud.storage._http import Connection - - PROJECT = "PROJECT" -- credentials = _make_credentials() -- -- ddp_patch = mock.patch( -- "google.cloud.client._determine_default_project", return_value=PROJECT -- ) -+ credentials = _make_credentials(project=PROJECT) - -- with ddp_patch: -- client = self._make_one(credentials=credentials) -+ client = self._make_one(credentials=credentials) - - self.assertEqual(client.project, PROJECT) - self.assertIsInstance(client._connection, Connection) -@@ -221,7 +219,8 @@ class TestClient(unittest.TestCase): - self.assertIs(client._connection._client_info, client_info) - - def test_ctor_mtls(self): -- credentials = _make_credentials() -+ PROJECT = "PROJECT" -+ credentials = _make_credentials(project=PROJECT) - - client = self._make_one(credentials=credentials) - self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, True) -@@ -411,339 +410,675 @@ class TestClient(unittest.TestCase): - self.assertIsInstance(batch, Batch) - self.assertIs(batch._client, client) - -- def test_get_bucket_with_string_miss(self): -+ def test__get_resource_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ project = "PROJECT" -+ path = "/path/to/something" -+ credentials = _make_credentials() - -- NONESUCH = "nonesuch" -- http = _make_requests_session( -- [_make_json_response({}, status=http_client.NOT_FOUND)] -- ) -- client._http_internal = http -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() - - with self.assertRaises(NotFound): -- client.get_bucket(NONESUCH, timeout=42) -+ client._get_resource(path) - -- http.request.assert_called_once_with( -- method="GET", url=mock.ANY, data=mock.ANY, headers=mock.ANY, timeout=42 -+ connection.api_request.assert_called_once_with( -+ method="GET", -+ path=path, -+ query_params=None, -+ headers=None, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -+ -+ def test__get_resource_hit_w_explicit(self): -+ project = "PROJECT" -+ path = "/path/to/something" -+ query_params = {"foo": "Foo"} -+ headers = {"bar": "Bar"} -+ timeout = 100 -+ retry = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ -+ client = self._make_one(project=project, credentials=credentials) -+ expected = mock.Mock(spec={}) -+ connection = client._base_connection = _make_connection(expected) -+ target = mock.Mock(spec={}) -+ -+ found = client._get_resource( - path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", NONESUCH]), -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_get_bucket_with_string_hit(self): -- from google.cloud.storage.bucket import Bucket -+ self.assertIs(found, expected) - -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ connection.api_request.assert_called_once_with( -+ method="GET", -+ path=path, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, -+ ) - -- BUCKET_NAME = "bucket-name" -- data = {"name": BUCKET_NAME} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -+ def test__list_resource_w_defaults(self): -+ import functools -+ from google.api_core.page_iterator import HTTPIterator -+ from google.api_core.page_iterator import _do_nothing_page_start - -- bucket = client.get_bucket(BUCKET_NAME) -+ project = "PROJECT" -+ path = "/path/to/list/resource" -+ item_to_value = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() - -- self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, BUCKET_NAME) -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -+ iterator = client._list_resource(path=path, item_to_value=item_to_value,) -+ -+ self.assertIsInstance(iterator, HTTPIterator) -+ self.assertIs(iterator.client, client) -+ self.assertIsInstance(iterator.api_request, functools.partial) -+ self.assertIs(iterator.api_request.func, connection.api_request) -+ self.assertEqual(iterator.api_request.args, ()) -+ expected_keywords = { -+ "timeout": self._get_default_timeout(), -+ "retry": DEFAULT_RETRY, -+ } -+ self.assertEqual(iterator.api_request.keywords, expected_keywords) -+ self.assertEqual(iterator.path, path) -+ self.assertEqual(iterator.next_page_token, None) -+ self.assertEqual(iterator.max_results, None) -+ self.assertIs(iterator._page_start, _do_nothing_page_start) -+ -+ def test__list_resource_w_explicit(self): -+ import functools -+ from google.api_core.page_iterator import HTTPIterator -+ -+ project = "PROJECT" -+ path = "/path/to/list/resource" -+ item_to_value = mock.Mock(spec=[]) -+ page_token = "PAGE-TOKEN" -+ max_results = 47 -+ extra_params = {"foo": "Foo"} -+ page_start = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() -+ -+ iterator = client._list_resource( -+ path=path, -+ item_to_value=item_to_value, -+ page_token=page_token, -+ max_results=max_results, -+ extra_params=extra_params, -+ page_start=page_start, -+ ) -+ -+ self.assertIsInstance(iterator, HTTPIterator) -+ self.assertIs(iterator.client, client) -+ self.assertIsInstance(iterator.api_request, functools.partial) -+ self.assertIs(iterator.api_request.func, connection.api_request) -+ self.assertEqual(iterator.api_request.args, ()) -+ expected_keywords = { -+ "timeout": self._get_default_timeout(), -+ "retry": DEFAULT_RETRY, -+ } -+ self.assertEqual(iterator.api_request.keywords, expected_keywords) -+ self.assertEqual(iterator.path, path) -+ self.assertEqual(iterator.next_page_token, page_token) -+ self.assertEqual(iterator.max_results, max_results) -+ self.assertIs(iterator._page_start, page_start) -+ -+ def test__patch_resource_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound -+ -+ project = "PROJECT" -+ path = "/path/to/something" -+ credentials = _make_credentials() -+ data = {"baz": "Baz"} -+ -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() -+ -+ with self.assertRaises(NotFound): -+ client._patch_resource(path, data) -+ -+ connection.api_request.assert_called_once_with( -+ method="PATCH", -+ path=path, -+ data=data, -+ query_params=None, -+ headers=None, - timeout=self._get_default_timeout(), -+ retry=None, -+ _target_object=None, - ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -+ -+ def test__patch_resource_hit_w_explicit(self): -+ project = "PROJECT" -+ path = "/path/to/something" -+ data = {"baz": "Baz"} -+ query_params = {"foo": "Foo"} -+ headers = {"bar": "Bar"} -+ timeout = 100 -+ retry = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ -+ client = self._make_one(project=project, credentials=credentials) -+ expected = mock.Mock(spec={}) -+ connection = client._base_connection = _make_connection(expected) -+ target = mock.Mock(spec={}) -+ -+ found = client._patch_resource( - path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", BUCKET_NAME]), -+ data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_get_bucket_with_metageneration_match(self): -- from google.cloud.storage.bucket import Bucket -+ self.assertIs(found, expected) - -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- METAGENERATION_NUMBER = 6 -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ connection.api_request.assert_called_once_with( -+ method="PATCH", -+ path=path, -+ data=data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, -+ ) - -- BUCKET_NAME = "bucket-name" -- data = {"name": BUCKET_NAME} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -+ def test__put_resource_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound - -- bucket = client.get_bucket( -- BUCKET_NAME, if_metageneration_match=METAGENERATION_NUMBER -- ) -- self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, BUCKET_NAME) -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -+ project = "PROJECT" -+ path = "/path/to/something" -+ credentials = _make_credentials() -+ data = {"baz": "Baz"} -+ -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() -+ -+ with self.assertRaises(NotFound): -+ client._put_resource(path, data) -+ -+ connection.api_request.assert_called_once_with( -+ method="PUT", -+ path=path, -+ data=data, -+ query_params=None, -+ headers=None, - timeout=self._get_default_timeout(), -+ retry=None, -+ _target_object=None, - ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -+ -+ def test__put_resource_hit_w_explicit(self): -+ project = "PROJECT" -+ path = "/path/to/something" -+ data = {"baz": "Baz"} -+ query_params = {"foo": "Foo"} -+ headers = {"bar": "Bar"} -+ timeout = 100 -+ retry = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ -+ client = self._make_one(project=project, credentials=credentials) -+ expected = mock.Mock(spec={}) -+ connection = client._base_connection = _make_connection(expected) -+ target = mock.Mock(spec={}) -+ -+ found = client._put_resource( - path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", BUCKET_NAME]), -+ data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["ifMetagenerationMatch"], str(METAGENERATION_NUMBER)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_get_bucket_with_object_miss(self): -+ self.assertIs(found, expected) -+ -+ connection.api_request.assert_called_once_with( -+ method="PUT", -+ path=path, -+ data=data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, -+ ) -+ -+ def test__post_resource_miss_w_defaults(self): - from google.cloud.exceptions import NotFound -- from google.cloud.storage.bucket import Bucket - - project = "PROJECT" -+ path = "/path/to/something" -+ credentials = _make_credentials() -+ data = {"baz": "Baz"} -+ -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() -+ -+ with self.assertRaises(NotFound): -+ client._post_resource(path, data) -+ -+ connection.api_request.assert_called_once_with( -+ method="POST", -+ path=path, -+ data=data, -+ query_params=None, -+ headers=None, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ _target_object=None, -+ ) -+ -+ def test__post_resource_hit_w_explicit(self): -+ project = "PROJECT" -+ path = "/path/to/something" -+ data = {"baz": "Baz"} -+ query_params = {"foo": "Foo"} -+ headers = {"bar": "Bar"} -+ timeout = 100 -+ retry = mock.Mock(spec=[]) - credentials = _make_credentials() -+ - client = self._make_one(project=project, credentials=credentials) -+ expected = mock.Mock(spec={}) -+ connection = client._base_connection = _make_connection(expected) -+ target = mock.Mock(spec={}) - -- nonesuch = "nonesuch" -- bucket_obj = Bucket(client, nonesuch) -- http = _make_requests_session( -- [_make_json_response({}, status=http_client.NOT_FOUND)] -+ found = client._post_resource( -+ path, -+ data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, - ) -- client._http_internal = http -+ -+ self.assertIs(found, expected) -+ -+ connection.api_request.assert_called_once_with( -+ method="POST", -+ path=path, -+ data=data, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, -+ ) -+ -+ def test__delete_resource_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound -+ -+ project = "PROJECT" -+ path = "/path/to/something" -+ credentials = _make_credentials() -+ -+ client = self._make_one(project=project, credentials=credentials) -+ connection = client._base_connection = _make_connection() - - with self.assertRaises(NotFound): -- client.get_bucket(bucket_obj) -+ client._delete_resource(path) - -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -+ connection.api_request.assert_called_once_with( -+ method="DELETE", -+ path=path, -+ query_params=None, -+ headers=None, - timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=None, - ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -+ -+ def test__delete_resource_hit_w_explicit(self): -+ project = "PROJECT" -+ path = "/path/to/something" -+ query_params = {"foo": "Foo"} -+ headers = {"bar": "Bar"} -+ timeout = 100 -+ retry = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ -+ client = self._make_one(project=project, credentials=credentials) -+ expected = mock.Mock(spec={}) -+ connection = client._base_connection = _make_connection(expected) -+ target = mock.Mock(spec={}) -+ -+ found = client._delete_resource( - path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", nonesuch]), -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, -+ ) -+ -+ self.assertIs(found, expected) -+ -+ connection.api_request.assert_called_once_with( -+ method="DELETE", -+ path=path, -+ query_params=query_params, -+ headers=headers, -+ timeout=timeout, -+ retry=retry, -+ _target_object=target, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_get_bucket_with_object_hit(self): -+ def test_get_bucket_miss_w_string_w_defaults(self): -+ from google.cloud.exceptions import NotFound - from google.cloud.storage.bucket import Bucket - - project = "PROJECT" - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock() -+ client._get_resource.side_effect = NotFound("testing") -+ bucket_name = "nonesuch" -+ -+ with self.assertRaises(NotFound): -+ client.get_bucket(bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=mock.ANY, -+ ) -+ -+ target = client._get_resource.call_args[1]["_target_object"] -+ self.assertIsInstance(target, Bucket) -+ self.assertEqual(target.name, bucket_name) -+ -+ def test_get_bucket_hit_w_string_w_timeout(self): -+ from google.cloud.storage.bucket import Bucket - -+ project = "PROJECT" - bucket_name = "bucket-name" -- bucket_obj = Bucket(client, bucket_name) -- data = {"name": bucket_name} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -+ timeout = 42 -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) - -- bucket = client.get_bucket(bucket_obj) -+ bucket = client.get_bucket(bucket_name, timeout=timeout) - - self.assertIsInstance(bucket, Bucket) - self.assertEqual(bucket.name, bucket_name) -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=self._get_default_timeout(), -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", bucket_name]), -+ -+ def test_get_bucket_hit_w_string_w_metageneration_match(self): -+ from google.cloud.storage.bucket import Bucket -+ -+ project = "PROJECT" -+ bucket_name = "bucket-name" -+ metageneration_number = 6 -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) -+ -+ bucket = client.get_bucket( -+ bucket_name, if_metageneration_match=metageneration_number - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_get_bucket_default_retry(self): -+ self.assertIsInstance(bucket, Bucket) -+ self.assertEqual(bucket.name, bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = { -+ "projection": "noAcl", -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, -+ ) -+ -+ def test_get_bucket_miss_w_object_w_retry(self): -+ from google.cloud.exceptions import NotFound - from google.cloud.storage.bucket import Bucket -- from google.cloud.storage._http import Connection - -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ project = "PROJECT" -+ bucket_name = "nonesuch" -+ retry = mock.Mock(spec=[]) -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(side_effect=NotFound("testing")) -+ bucket_obj = Bucket(client, bucket_name) -+ -+ with self.assertRaises(NotFound): -+ client.get_bucket(bucket_obj, retry=retry) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=retry, -+ _target_object=mock.ANY, -+ ) - -+ target = client._get_resource.call_args[1]["_target_object"] -+ self.assertIsInstance(target, Bucket) -+ self.assertEqual(target.name, bucket_name) -+ -+ def test_get_bucket_hit_w_object_defaults(self): -+ from google.cloud.storage.bucket import Bucket -+ -+ project = "PROJECT" - bucket_name = "bucket-name" -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) - bucket_obj = Bucket(client, bucket_name) - -- with mock.patch.object(Connection, "api_request") as req: -- client.get_bucket(bucket_obj) -+ bucket = client.get_bucket(bucket_obj) - -- req.assert_called_once_with( -- method="GET", -- path=mock.ANY, -- query_params=mock.ANY, -- headers=mock.ANY, -- _target_object=bucket_obj, -- timeout=mock.ANY, -+ self.assertIsInstance(bucket, Bucket) -+ self.assertEqual(bucket.name, bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) - -- def test_get_bucket_respects_retry_override(self): -+ def test_get_bucket_hit_w_object_w_retry_none(self): - from google.cloud.storage.bucket import Bucket -- from google.cloud.storage._http import Connection -- -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) - -+ project = "PROJECT" - bucket_name = "bucket-name" -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) - bucket_obj = Bucket(client, bucket_name) - -- with mock.patch.object(Connection, "api_request") as req: -- client.get_bucket(bucket_obj, retry=None) -+ bucket = client.get_bucket(bucket_obj, retry=None) - -- req.assert_called_once_with( -- method="GET", -- path=mock.ANY, -- query_params=mock.ANY, -- headers=mock.ANY, -- _target_object=bucket_obj, -- timeout=mock.ANY, -+ self.assertIsInstance(bucket, Bucket) -+ self.assertEqual(bucket.name, bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), - retry=None, -+ _target_object=bucket, - ) - -- def test_lookup_bucket_miss(self): -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ def test_lookup_bucket_miss_w_defaults(self): -+ from google.cloud.exceptions import NotFound -+ from google.cloud.storage.bucket import Bucket - -- NONESUCH = "nonesuch" -- http = _make_requests_session( -- [_make_json_response({}, status=http_client.NOT_FOUND)] -- ) -- client._http_internal = http -+ project = "PROJECT" -+ bucket_name = "nonesuch" -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(side_effect=NotFound("testing")) - -- bucket = client.lookup_bucket(NONESUCH, timeout=42) -+ bucket = client.lookup_bucket(bucket_name) - - self.assertIsNone(bucket) -- http.request.assert_called_once_with( -- method="GET", url=mock.ANY, data=mock.ANY, headers=mock.ANY, timeout=42 -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", NONESUCH]), -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=mock.ANY, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_lookup_bucket_hit(self): -- from google.cloud.storage.bucket import Bucket -+ target = client._get_resource.call_args[1]["_target_object"] -+ self.assertIsInstance(target, Bucket) -+ self.assertEqual(target.name, bucket_name) - -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ def test_lookup_bucket_hit_w_timeout(self): -+ from google.cloud.storage.bucket import Bucket - -- BUCKET_NAME = "bucket-name" -- data = {"name": BUCKET_NAME} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -+ project = "PROJECT" -+ bucket_name = "bucket-name" -+ timeout = 42 -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) - -- bucket = client.lookup_bucket(BUCKET_NAME) -+ bucket = client.lookup_bucket(bucket_name, timeout=timeout) - - self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, BUCKET_NAME) -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=self._get_default_timeout(), -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", BUCKET_NAME]), -+ self.assertEqual(bucket.name, bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=timeout, -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") - -- def test_lookup_bucket_with_metageneration_match(self): -+ def test_lookup_bucket_hit_w_metageneration_match(self): - from google.cloud.storage.bucket import Bucket - -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- METAGENERATION_NUMBER = 6 -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- -- BUCKET_NAME = "bucket-name" -- data = {"name": BUCKET_NAME} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -+ project = "PROJECT" -+ bucket_name = "bucket-name" -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ metageneration_number = 6 -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) - - bucket = client.lookup_bucket( -- BUCKET_NAME, if_metageneration_match=METAGENERATION_NUMBER -+ bucket_name, if_metageneration_match=metageneration_number - ) -+ - self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, BUCKET_NAME) -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -+ self.assertEqual(bucket.name, bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = { -+ "projection": "noAcl", -+ "ifMetagenerationMatch": metageneration_number, -+ } -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, - timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join(["", "storage", client._connection.API_VERSION, "b", BUCKET_NAME]), -- ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["projection"], "noAcl") -- self.assertEqual(parms["ifMetagenerationMatch"], str(METAGENERATION_NUMBER)) - -- def test_lookup_bucket_default_retry(self): -+ def test_lookup_bucket_hit_w_retry(self): - from google.cloud.storage.bucket import Bucket -- from google.cloud.storage._http import Connection -- -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) - -+ project = "PROJECT" - bucket_name = "bucket-name" -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._get_resource = mock.Mock(return_value=api_response) - bucket_obj = Bucket(client, bucket_name) - -- with mock.patch.object(Connection, "api_request") as req: -- client.lookup_bucket(bucket_obj) -- req.assert_called_once_with( -- method="GET", -- path=mock.ANY, -- query_params=mock.ANY, -- headers=mock.ANY, -- _target_object=bucket_obj, -- timeout=mock.ANY, -- retry=DEFAULT_RETRY, -- ) -+ bucket = client.lookup_bucket(bucket_obj, retry=None) -+ -+ self.assertIsInstance(bucket, Bucket) -+ self.assertEqual(bucket.name, bucket_name) -+ -+ expected_path = "/b/%s" % (bucket_name,) -+ expected_query_params = {"projection": "noAcl"} -+ expected_headers = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ headers=expected_headers, -+ timeout=self._get_default_timeout(), -+ retry=None, -+ _target_object=bucket, -+ ) - - def test_create_bucket_w_missing_client_project(self): - credentials = _make_credentials() -@@ -752,7 +1087,7 @@ class TestClient(unittest.TestCase): - with self.assertRaises(ValueError): - client.create_bucket("bucket") - -- def test_create_bucket_w_conflict(self): -+ def test_create_bucket_w_conflict_w_user_project(self): - from google.cloud.exceptions import Conflict - - project = "PROJECT" -@@ -760,62 +1095,60 @@ class TestClient(unittest.TestCase): - other_project = "OTHER_PROJECT" - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -- connection = _make_connection() -- client._base_connection = connection -- connection.api_request.side_effect = Conflict("testing") -+ client._post_resource = mock.Mock() -+ client._post_resource.side_effect = Conflict("testing") - - bucket_name = "bucket-name" -- data = {"name": bucket_name} - - with self.assertRaises(Conflict): - client.create_bucket( - bucket_name, project=other_project, user_project=user_project - ) - -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": other_project, "userProject": user_project}, -- data=data, -- _target_object=mock.ANY, -+ expected_path = "/b" -+ expected_data = {"name": bucket_name} -+ expected_query_params = { -+ "project": other_project, -+ "userProject": user_project, -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, -+ _target_object=mock.ANY, - ) - - @mock.patch("warnings.warn") -- def test_create_requester_pays_deprecated(self, mock_warn): -+ def test_create_bucket_w_requester_pays_deprecated(self, mock_warn): - from google.cloud.storage.bucket import Bucket - -+ bucket_name = "bucket-name" - project = "PROJECT" - credentials = _make_credentials() -+ api_respone = {"name": bucket_name, "billing": {"requesterPays": True}} - client = self._make_one(project=project, credentials=credentials) -- bucket_name = "bucket-name" -- json_expected = {"name": bucket_name, "billing": {"requesterPays": True}} -- http = _make_requests_session([_make_json_response(json_expected)]) -- client._http_internal = http -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_respone - - bucket = client.create_bucket(bucket_name, requester_pays=True) - - self.assertIsInstance(bucket, Bucket) - self.assertEqual(bucket.name, bucket_name) - self.assertTrue(bucket.requester_pays) -- http.request.assert_called_once_with( -- method="POST", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=mock.ANY, -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, "/".join(["", "storage", client._connection.API_VERSION, "b"]) -+ -+ expected_path = "/b" -+ expected_data = api_respone -+ expected_query_params = {"project": project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=mock.ANY, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["project"], project) -- json_sent = http.request.call_args_list[0][1]["data"] -- self.assertEqual(json_expected, json.loads(json_sent)) - - mock_warn.assert_called_with( - "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", -@@ -828,31 +1161,40 @@ class TestClient(unittest.TestCase): - bucket_name = "bucket-name" - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -+ client._post_resource = mock.Mock() - - with self.assertRaises(ValueError): - client.create_bucket(bucket_name, predefined_acl="bogus") - -- def test_create_bucket_w_predefined_acl_valid(self): -+ client._post_resource.assert_not_called() -+ -+ def test_create_bucket_w_predefined_acl_valid_w_timeout(self): - project = "PROJECT" - bucket_name = "bucket-name" -- data = {"name": bucket_name} -- -+ api_response = {"name": bucket_name} - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -- connection = _make_connection(data) -- client._base_connection = connection -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response -+ timeout = 42 -+ - bucket = client.create_bucket( -- bucket_name, predefined_acl="publicRead", timeout=42 -+ bucket_name, predefined_acl="publicRead", timeout=timeout, - ) - -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": project, "predefinedAcl": "publicRead"}, -- data=data, -- _target_object=bucket, -- timeout=42, -+ expected_path = "/b" -+ expected_data = api_response -+ expected_query_params = { -+ "project": project, -+ "predefinedAcl": "publicRead", -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, - retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) - - def test_create_bucket_w_predefined_default_object_acl_invalid(self): -@@ -861,93 +1203,98 @@ class TestClient(unittest.TestCase): - - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -+ client._post_resource = mock.Mock() - - with self.assertRaises(ValueError): - client.create_bucket(bucket_name, predefined_default_object_acl="bogus") - -- def test_create_bucket_w_predefined_default_object_acl_valid(self): -+ client._post_resource.assert_not_called() -+ -+ def test_create_bucket_w_predefined_default_object_acl_valid_w_retry(self): - project = "PROJECT" - bucket_name = "bucket-name" -- data = {"name": bucket_name} -- -+ api_response = {"name": bucket_name} - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -- connection = _make_connection(data) -- client._base_connection = connection -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response -+ retry = mock.Mock(spec=[]) -+ - bucket = client.create_bucket( -- bucket_name, predefined_default_object_acl="publicRead" -+ bucket_name, predefined_default_object_acl="publicRead", retry=retry, - ) - -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={ -- "project": project, -- "predefinedDefaultObjectAcl": "publicRead", -- }, -- data=data, -- _target_object=bucket, -+ expected_path = "/b" -+ expected_data = api_response -+ expected_query_params = { -+ "project": project, -+ "predefinedDefaultObjectAcl": "publicRead", -+ } -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, - timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -+ retry=retry, -+ _target_object=bucket, - ) - - def test_create_bucket_w_explicit_location(self): - project = "PROJECT" - bucket_name = "bucket-name" - location = "us-central1" -- data = {"location": location, "name": bucket_name} -- -- connection = _make_connection( -- data, "{'location': 'us-central1', 'name': 'bucket-name'}" -- ) -- -+ api_response = {"location": location, "name": bucket_name} - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -- client._base_connection = connection -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response - - bucket = client.create_bucket(bucket_name, location=location) - -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- data=data, -- _target_object=bucket, -- query_params={"project": project}, -+ self.assertEqual(bucket.location, location) -+ -+ expected_path = "/b" -+ expected_data = {"location": location, "name": bucket_name} -+ expected_query_params = {"project": project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) -- self.assertEqual(bucket.location, location) - - def test_create_bucket_w_explicit_project(self): -- from google.cloud.storage.client import Client -- -- PROJECT = "PROJECT" -- OTHER_PROJECT = "other-project-123" -- BUCKET_NAME = "bucket-name" -- DATA = {"name": BUCKET_NAME} -- connection = _make_connection(DATA) -+ project = "PROJECT" -+ other_project = "other-project-123" -+ bucket_name = "bucket-name" -+ api_response = {"name": bucket_name} -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response - -- client = Client(project=PROJECT) -- client._base_connection = connection -+ bucket = client.create_bucket(bucket_name, project=other_project) - -- bucket = client.create_bucket(BUCKET_NAME, project=OTHER_PROJECT) -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": OTHER_PROJECT}, -- data=DATA, -- _target_object=bucket, -+ expected_path = "/b" -+ expected_data = api_response -+ expected_query_params = {"project": other_project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) - -- def test_create_w_extra_properties(self): -- from google.cloud.storage.client import Client -+ def test_create_bucket_w_extra_properties(self): - from google.cloud.storage.bucket import Bucket - -- BUCKET_NAME = "bucket-name" -- PROJECT = "PROJECT" -- CORS = [ -+ bucket_name = "bucket-name" -+ project = "PROJECT" -+ cors = [ - { - "maxAgeSeconds": 60, - "methods": ["*"], -@@ -955,157 +1302,82 @@ class TestClient(unittest.TestCase): - "responseHeader": ["X-Custom-Header"], - } - ] -- LIFECYCLE_RULES = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] -- LOCATION = "eu" -- LABELS = {"color": "red", "flavor": "cherry"} -- STORAGE_CLASS = "NEARLINE" -- DATA = { -- "name": BUCKET_NAME, -- "cors": CORS, -- "lifecycle": {"rule": LIFECYCLE_RULES}, -- "location": LOCATION, -- "storageClass": STORAGE_CLASS, -+ lifecycle_rules = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] -+ location = "eu" -+ labels = {"color": "red", "flavor": "cherry"} -+ storage_class = "NEARLINE" -+ api_response = { -+ "name": bucket_name, -+ "cors": cors, -+ "lifecycle": {"rule": lifecycle_rules}, -+ "location": location, -+ "storageClass": storage_class, - "versioning": {"enabled": True}, - "billing": {"requesterPays": True}, -- "labels": LABELS, -+ "labels": labels, - } -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response - -- connection = _make_connection(DATA) -- client = Client(project=PROJECT) -- client._base_connection = connection -- -- bucket = Bucket(client=client, name=BUCKET_NAME) -- bucket.cors = CORS -- bucket.lifecycle_rules = LIFECYCLE_RULES -- bucket.storage_class = STORAGE_CLASS -+ bucket = Bucket(client=client, name=bucket_name) -+ bucket.cors = cors -+ bucket.lifecycle_rules = lifecycle_rules -+ bucket.storage_class = storage_class - bucket.versioning_enabled = True - bucket.requester_pays = True -- bucket.labels = LABELS -- client.create_bucket(bucket, location=LOCATION) -- -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": PROJECT}, -- data=DATA, -- _target_object=bucket, -- timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -- ) -+ bucket.labels = labels - -- def test_create_hit(self): -- from google.cloud.storage.client import Client -- -- PROJECT = "PROJECT" -- BUCKET_NAME = "bucket-name" -- DATA = {"name": BUCKET_NAME} -- connection = _make_connection(DATA) -- client = Client(project=PROJECT) -- client._base_connection = connection -- -- bucket = client.create_bucket(BUCKET_NAME) -+ client.create_bucket(bucket, location=location) - -- connection.api_request.assert_called_once_with( -- method="POST", -- path="/b", -- query_params={"project": PROJECT}, -- data=DATA, -- _target_object=bucket, -+ expected_path = "/b" -+ expected_data = api_response -+ expected_query_params = {"project": project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) - -- def test_create_bucket_w_string_success(self): -- from google.cloud.storage.bucket import Bucket -- -+ def test_create_bucket_w_name_only(self): - project = "PROJECT" -- credentials = _make_credentials() -- client = self._make_one(project=project, credentials=credentials) -- - bucket_name = "bucket-name" -- json_expected = {"name": bucket_name} -- data = json_expected -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -- -- bucket = client.create_bucket(bucket_name) -- -- self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, bucket_name) -- http.request.assert_called_once_with( -- method="POST", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=mock.ANY, -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, "/".join(["", "storage", client._connection.API_VERSION, "b"]), -- ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["project"], project) -- json_sent = http.request.call_args_list[0][1]["data"] -- self.assertEqual(json_expected, json.loads(json_sent)) -- -- def test_create_bucket_w_object_success(self): -- from google.cloud.storage.bucket import Bucket -- -- project = "PROJECT" -+ api_response = {"name": bucket_name} - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response - -- bucket_name = "bucket-name" -- bucket_obj = Bucket(client, bucket_name) -- bucket_obj.storage_class = "COLDLINE" -- bucket_obj.requester_pays = True -- -- json_expected = { -- "name": bucket_name, -- "billing": {"requesterPays": True}, -- "storageClass": "COLDLINE", -- } -- data = json_expected -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -- -- bucket = client.create_bucket(bucket_obj) -+ bucket = client.create_bucket(bucket_name) - -- self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, bucket_name) -- self.assertTrue(bucket.requester_pays) -- http.request.assert_called_once_with( -- method="POST", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=mock.ANY, -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, "/".join(["", "storage", client._connection.API_VERSION, "b"]), -+ expected_path = "/b" -+ expected_data = api_response -+ expected_query_params = {"project": project} -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, -+ _target_object=bucket, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["project"], project) -- json_sent = http.request.call_args_list[0][1]["data"] -- self.assertEqual(json_expected, json.loads(json_sent)) - - def test_download_blob_to_file_with_failure(self): - from google.resumable_media import InvalidResponse - from google.cloud.storage.blob import Blob - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - -+ project = "PROJECT" - raw_response = requests.Response() - raw_response.status_code = http_client.NOT_FOUND - raw_request = requests.Request("GET", "http://example.com") - raw_response.request = raw_request.prepare() - grmp_response = InvalidResponse(raw_response) -- -- credentials = _make_credentials() -+ credentials = _make_credentials(project=project) - client = self._make_one(credentials=credentials) - blob = mock.create_autospec(Blob) - blob._encryption_key = None -@@ -1136,7 +1408,7 @@ class TestClient(unittest.TestCase): - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - - project = "PROJECT" -- credentials = _make_credentials() -+ credentials = _make_credentials(project=project) - client = self._make_one(project=project, credentials=credentials) - blob = mock.Mock() - file_obj = io.BytesIO() -@@ -1164,7 +1436,7 @@ class TestClient(unittest.TestCase): - - def test_download_blob_to_file_with_invalid_uri(self): - project = "PROJECT" -- credentials = _make_credentials() -+ credentials = _make_credentials(project=project) - client = self._make_one(project=project, credentials=credentials) - file_obj = io.BytesIO() - -@@ -1175,7 +1447,8 @@ class TestClient(unittest.TestCase): - from google.cloud.storage.blob import Blob - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - -- credentials = _make_credentials() -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) - client = self._make_one(credentials=credentials) - blob = mock.create_autospec(Blob) - blob._encryption_key = None -@@ -1216,303 +1489,237 @@ class TestClient(unittest.TestCase): - def test_download_blob_to_file_w_chunks_w_raw(self): - self._download_blob_to_file_helper(use_chunks=True, raw_download=True) - -- def test_list_blobs(self): -+ def test_list_blobs_w_defaults_w_bucket_obj(self): - from google.cloud.storage.bucket import Bucket -+ from google.cloud.storage.bucket import _blobs_page_start -+ from google.cloud.storage.bucket import _item_to_blob - -- BUCKET_NAME = "bucket-name" -- -+ project = "PROJECT" -+ bucket_name = "bucket-name" - credentials = _make_credentials() -- client = self._make_one(project="PROJECT", credentials=credentials) -- connection = _make_connection({"items": []}) -- -- with mock.patch( -- "google.cloud.storage.client.Client._connection", -- new_callable=mock.PropertyMock, -- ) as client_mock: -- client_mock.return_value = connection -- -- bucket_obj = Bucket(client, BUCKET_NAME) -- iterator = client.list_blobs(bucket_obj) -- blobs = list(iterator) -- -- self.assertEqual(blobs, []) -- connection.api_request.assert_called_once_with( -- method="GET", -- path="/b/%s/o" % BUCKET_NAME, -- query_params={"projection": "noAcl"}, -- timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -- ) -+ client = self._make_one(project=project, credentials=credentials) -+ client._list_resource = mock.Mock(spec=[]) -+ bucket = Bucket(client, bucket_name) - -- def test_list_blobs_w_all_arguments_and_user_project(self): -- from google.cloud.storage.bucket import Bucket -+ iterator = client.list_blobs(bucket) - -- BUCKET_NAME = "name" -- USER_PROJECT = "user-project-123" -- MAX_RESULTS = 10 -- PAGE_TOKEN = "ABCD" -- PREFIX = "subfolder" -- DELIMITER = "/" -- START_OFFSET = "c" -- END_OFFSET = "g" -- INCLUDE_TRAILING_DELIMITER = True -- VERSIONS = True -- PROJECTION = "full" -- FIELDS = "items/contentLanguage,nextPageToken" -- EXPECTED = { -- "maxResults": 10, -- "pageToken": PAGE_TOKEN, -- "prefix": PREFIX, -- "delimiter": DELIMITER, -- "startOffset": START_OFFSET, -- "endOffset": END_OFFSET, -- "includeTrailingDelimiter": INCLUDE_TRAILING_DELIMITER, -- "versions": VERSIONS, -- "projection": PROJECTION, -- "fields": FIELDS, -- "userProject": USER_PROJECT, -- } -+ self.assertIs(iterator, client._list_resource.return_value) -+ self.assertIs(iterator.bucket, bucket) -+ self.assertEqual(iterator.prefixes, set()) -+ -+ expected_path = "/b/{}/o".format(bucket_name) -+ expected_item_to_value = _item_to_blob -+ expected_page_token = None -+ expected_max_results = None -+ expected_extra_params = {"projection": "noAcl"} -+ expected_page_start = _blobs_page_start -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ page_token=expected_page_token, -+ max_results=expected_max_results, -+ extra_params=expected_extra_params, -+ page_start=expected_page_start, -+ ) -+ -+ def test_list_blobs_w_explicit_w_user_project(self): -+ from google.cloud.storage.bucket import _blobs_page_start -+ from google.cloud.storage.bucket import _item_to_blob - -+ project = "PROJECT" -+ user_project = "user-project-123" -+ bucket_name = "name" -+ max_results = 10 -+ page_token = "ABCD" -+ prefix = "subfolder" -+ delimiter = "/" -+ start_offset = "c" -+ end_offset = "g" -+ include_trailing_delimiter = True -+ versions = True -+ projection = "full" -+ fields = "items/contentLanguage,nextPageToken" - credentials = _make_credentials() -- client = self._make_one(project=USER_PROJECT, credentials=credentials) -- connection = _make_connection({"items": []}) -- -- with mock.patch( -- "google.cloud.storage.client.Client._connection", -- new_callable=mock.PropertyMock, -- ) as client_mock: -- client_mock.return_value = connection -- -- bucket = Bucket(client, BUCKET_NAME, user_project=USER_PROJECT) -- iterator = client.list_blobs( -- bucket_or_name=bucket, -- max_results=MAX_RESULTS, -- page_token=PAGE_TOKEN, -- prefix=PREFIX, -- delimiter=DELIMITER, -- start_offset=START_OFFSET, -- end_offset=END_OFFSET, -- include_trailing_delimiter=INCLUDE_TRAILING_DELIMITER, -- versions=VERSIONS, -- projection=PROJECTION, -- fields=FIELDS, -- timeout=42, -- ) -- blobs = list(iterator) -- -- self.assertEqual(blobs, []) -- connection.api_request.assert_called_once_with( -- method="GET", -- path="/b/%s/o" % BUCKET_NAME, -- query_params=EXPECTED, -- timeout=42, -- retry=DEFAULT_RETRY, -- ) -+ client = self._make_one(project=project, credentials=credentials) -+ client._list_resource = mock.Mock(spec=[]) -+ client._bucket_arg_to_bucket = mock.Mock(spec=[]) -+ bucket = client._bucket_arg_to_bucket.return_value = mock.Mock( -+ spec=["path", "user_project"], -+ ) -+ bucket.path = "/b/{}".format(bucket_name) -+ bucket.user_project = user_project -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ -+ iterator = client.list_blobs( -+ bucket_or_name=bucket_name, -+ max_results=max_results, -+ page_token=page_token, -+ prefix=prefix, -+ delimiter=delimiter, -+ start_offset=start_offset, -+ end_offset=end_offset, -+ include_trailing_delimiter=include_trailing_delimiter, -+ versions=versions, -+ projection=projection, -+ fields=fields, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ self.assertIs(iterator, client._list_resource.return_value) -+ self.assertIs(iterator.bucket, bucket) -+ self.assertEqual(iterator.prefixes, set()) -+ -+ expected_path = "/b/{}/o".format(bucket_name) -+ expected_item_to_value = _item_to_blob -+ expected_page_token = page_token -+ expected_max_results = max_results -+ expected_extra_params = { -+ "projection": projection, -+ "prefix": prefix, -+ "delimiter": delimiter, -+ "startOffset": start_offset, -+ "endOffset": end_offset, -+ "includeTrailingDelimiter": include_trailing_delimiter, -+ "versions": versions, -+ "fields": fields, -+ "userProject": user_project, -+ } -+ expected_page_start = _blobs_page_start -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ page_token=expected_page_token, -+ max_results=expected_max_results, -+ extra_params=expected_extra_params, -+ page_start=expected_page_start, -+ ) - - def test_list_buckets_wo_project(self): -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=None, credentials=CREDENTIALS) -+ credentials = _make_credentials() -+ client = self._make_one(project=None, credentials=credentials) - - with self.assertRaises(ValueError): - client.list_buckets() - -- def test_list_buckets_empty(self): -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- -- http = _make_requests_session([_make_json_response({})]) -- client._http_internal = http -- -- buckets = list(client.list_buckets()) -- -- self.assertEqual(len(buckets), 0) -- -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=mock.ANY, -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, "/".join(["", "storage", client._connection.API_VERSION, "b"]) -- ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["project"], PROJECT) -- self.assertEqual(parms["projection"], "noAcl") -- -- def test_list_buckets_explicit_project(self): -- PROJECT = "PROJECT" -- OTHER_PROJECT = "OTHER_PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- -- http = _make_requests_session([_make_json_response({})]) -- client._http_internal = http -- -- buckets = list(client.list_buckets(project=OTHER_PROJECT)) -- -- self.assertEqual(len(buckets), 0) -- -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=mock.ANY, -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, "/".join(["", "storage", client._connection.API_VERSION, "b"]) -- ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["project"], str(OTHER_PROJECT)) -- self.assertEqual(parms["projection"], "noAcl") -+ def test_list_buckets_w_defaults(self): -+ from google.cloud.storage.client import _item_to_bucket - -- def test_list_buckets_non_empty(self): -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- -- BUCKET_NAME = "bucket-name" -- -- data = {"items": [{"name": BUCKET_NAME}]} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -+ project = "PROJECT" -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._list_resource = mock.Mock(spec=[]) - -- buckets = list(client.list_buckets()) -+ iterator = client.list_buckets() - -- self.assertEqual(len(buckets), 1) -- self.assertEqual(buckets[0].name, BUCKET_NAME) -+ self.assertIs(iterator, client._list_resource.return_value) - -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -+ expected_path = "/b" -+ expected_item_to_value = _item_to_bucket -+ expected_page_token = None -+ expected_max_results = None -+ expected_extra_params = { -+ "project": project, -+ "projection": "noAcl", -+ } -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ page_token=expected_page_token, -+ max_results=expected_max_results, -+ extra_params=expected_extra_params, - timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) - -- def test_list_buckets_all_arguments(self): -- PROJECT = "foo-bar" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- -- MAX_RESULTS = 10 -- PAGE_TOKEN = "ABCD" -- PREFIX = "subfolder" -- PROJECTION = "full" -- FIELDS = "items/id,nextPageToken" -- -- data = {"items": []} -- http = _make_requests_session([_make_json_response(data)]) -- client._http_internal = http -- iterator = client.list_buckets( -- max_results=MAX_RESULTS, -- page_token=PAGE_TOKEN, -- prefix=PREFIX, -- projection=PROJECTION, -- fields=FIELDS, -- timeout=42, -- ) -- buckets = list(iterator) -- self.assertEqual(buckets, []) -- http.request.assert_called_once_with( -- method="GET", url=mock.ANY, data=mock.ANY, headers=mock.ANY, timeout=42 -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, "/".join(["", "storage", client._connection.API_VERSION, "b"]) -- ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["project"], PROJECT) -- self.assertEqual(parms["maxResults"], str(MAX_RESULTS)) -- self.assertEqual(parms["pageToken"], PAGE_TOKEN) -- self.assertEqual(parms["prefix"], PREFIX) -- self.assertEqual(parms["projection"], PROJECTION) -- self.assertEqual(parms["fields"], FIELDS) -- -- def test_list_buckets_page_empty_response(self): -- from google.api_core import page_iterator -- -- project = "PROJECT" -- credentials = _make_credentials() -- client = self._make_one(project=project, credentials=credentials) -- iterator = client.list_buckets() -- page = page_iterator.Page(iterator, (), None) -- iterator._page = page -- self.assertEqual(list(page), []) -- -- def test_list_buckets_page_non_empty_response(self): -- import six -- from google.cloud.storage.bucket import Bucket -+ def test_list_buckets_w_explicit(self): -+ from google.cloud.storage.client import _item_to_bucket - -- project = "PROJECT" -+ project = "foo-bar" -+ other_project = "OTHER_PROJECT" -+ max_results = 10 -+ page_token = "ABCD" -+ prefix = "subfolder" -+ projection = "full" -+ fields = "items/id,nextPageToken" - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) -+ client._list_resource = mock.Mock(spec=[]) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- blob_name = "bucket-name" -- response = {"items": [{"name": blob_name}]} -- -- def fake_response(): -- return response -- -- iterator = client.list_buckets() -- iterator._get_next_page_response = fake_response -- -- page = six.next(iterator.pages) -- self.assertEqual(page.num_items, 1) -- bucket = six.next(page) -- self.assertEqual(page.remaining, 0) -- self.assertIsInstance(bucket, Bucket) -- self.assertEqual(bucket.name, blob_name) -+ iterator = client.list_buckets( -+ project=other_project, -+ max_results=max_results, -+ page_token=page_token, -+ prefix=prefix, -+ projection=projection, -+ fields=fields, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ self.assertIs(iterator, client._list_resource.return_value) -+ -+ expected_path = "/b" -+ expected_item_to_value = _item_to_bucket -+ expected_page_token = page_token -+ expected_max_results = max_results -+ expected_extra_params = { -+ "project": other_project, -+ "prefix": prefix, -+ "projection": projection, -+ "fields": fields, -+ } -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ page_token=expected_page_token, -+ max_results=expected_max_results, -+ extra_params=expected_extra_params, -+ timeout=timeout, -+ retry=retry, -+ ) - - def _create_hmac_key_helper( -- self, explicit_project=None, user_project=None, timeout=None -+ self, explicit_project=None, user_project=None, timeout=None, retry=None, - ): - import datetime - from pytz import UTC - from google.cloud.storage.hmac_key import HMACKeyMetadata - -- PROJECT = "PROJECT" -- ACCESS_ID = "ACCESS-ID" -- CREDENTIALS = _make_credentials() -- EMAIL = "storage-user-123@example.com" -- SECRET = "a" * 40 -+ project = "PROJECT" -+ access_id = "ACCESS-ID" -+ credentials = _make_credentials() -+ email = "storage-user-123@example.com" -+ secret = "a" * 40 - now = datetime.datetime.utcnow().replace(tzinfo=UTC) - now_stamp = "{}Z".format(now.isoformat()) - - if explicit_project is not None: - expected_project = explicit_project - else: -- expected_project = PROJECT -+ expected_project = project - -- RESOURCE = { -+ api_response = { - "kind": "storage#hmacKey", - "metadata": { -- "accessId": ACCESS_ID, -+ "accessId": access_id, - "etag": "ETAG", -- "id": "projects/{}/hmacKeys/{}".format(PROJECT, ACCESS_ID), -+ "id": "projects/{}/hmacKeys/{}".format(project, access_id), - "project": expected_project, - "state": "ACTIVE", -- "serviceAccountEmail": EMAIL, -+ "serviceAccountEmail": email, - "timeCreated": now_stamp, - "updated": now_stamp, - }, -- "secret": SECRET, -+ "secret": secret, - } - -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- http = _make_requests_session([_make_json_response(RESOURCE)]) -- client._http_internal = http -+ client = self._make_one(project=project, credentials=credentials) -+ client._post_resource = mock.Mock() -+ client._post_resource.return_value = api_response - - kwargs = {} - if explicit_project is not None: -@@ -1522,43 +1729,37 @@ class TestClient(unittest.TestCase): - kwargs["user_project"] = user_project - - if timeout is None: -- timeout = self._get_default_timeout() -- kwargs["timeout"] = timeout -+ expected_timeout = self._get_default_timeout() -+ else: -+ expected_timeout = kwargs["timeout"] = timeout -+ -+ if retry is None: -+ expected_retry = None -+ else: -+ expected_retry = kwargs["retry"] = retry - -- metadata, secret = client.create_hmac_key(service_account_email=EMAIL, **kwargs) -+ metadata, secret = client.create_hmac_key(service_account_email=email, **kwargs) - - self.assertIsInstance(metadata, HMACKeyMetadata) -+ - self.assertIs(metadata._client, client) -- self.assertEqual(metadata._properties, RESOURCE["metadata"]) -- self.assertEqual(secret, RESOURCE["secret"]) -+ self.assertEqual(metadata._properties, api_response["metadata"]) -+ self.assertEqual(secret, api_response["secret"]) - -- qs_params = {"serviceAccountEmail": EMAIL} -+ expected_path = "/projects/{}/hmacKeys".format(expected_project) -+ expected_data = None -+ expected_query_params = {"serviceAccountEmail": email} - - if user_project is not None: -- qs_params["userProject"] = user_project -+ expected_query_params["userProject"] = user_project - -- http.request.assert_called_once_with( -- method="POST", url=mock.ANY, data=None, headers=mock.ANY, timeout=timeout -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join( -- [ -- "", -- "storage", -- client._connection.API_VERSION, -- "projects", -- expected_project, -- "hmacKeys", -- ] -- ), -+ client._post_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=expected_timeout, -+ retry=expected_retry, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- for param, expected in qs_params.items(): -- self.assertEqual(parms[param], expected) - - def test_create_hmac_key_defaults(self): - self._create_hmac_key_helper() -@@ -1566,113 +1767,83 @@ class TestClient(unittest.TestCase): - def test_create_hmac_key_explicit_project(self): - self._create_hmac_key_helper(explicit_project="other-project-456") - -- def test_create_hmac_key_user_project(self): -- self._create_hmac_key_helper(user_project="billed-project", timeout=42) -- -- def test_list_hmac_keys_defaults_empty(self): -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -- -- http = _make_requests_session([_make_json_response({})]) -- client._http_internal = http -- -- metadatas = list(client.list_hmac_keys()) -+ def test_create_hmac_key_w_user_project(self): -+ self._create_hmac_key_helper(user_project="billed-project") - -- self.assertEqual(len(metadatas), 0) -+ def test_create_hmac_key_w_timeout(self): -+ self._create_hmac_key_helper(timeout=42) - -- http.request.assert_called_once_with( -- method="GET", -- url=mock.ANY, -- data=None, -- headers=mock.ANY, -- timeout=self._get_default_timeout(), -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join( -- [ -- "", -- "storage", -- client._connection.API_VERSION, -- "projects", -- PROJECT, -- "hmacKeys", -- ] -- ), -- ) -+ def test_create_hmac_key_w_retry(self): -+ self._create_hmac_key_helper(retry=mock.Mock(spec=[])) - -- def test_list_hmac_keys_explicit_non_empty(self): -- from google.cloud.storage.hmac_key import HMACKeyMetadata -+ def test_list_hmac_keys_w_defaults(self): -+ from google.cloud.storage.client import _item_to_hmac_key_metadata - -- PROJECT = "PROJECT" -- OTHER_PROJECT = "other-project-456" -- MAX_RESULTS = 3 -- EMAIL = "storage-user-123@example.com" -- ACCESS_ID = "ACCESS-ID" -- USER_PROJECT = "billed-project" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) -+ project = "PROJECT" -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._list_resource = mock.Mock(spec=[]) - -- response = { -- "kind": "storage#hmacKeysMetadata", -- "items": [ -- { -- "kind": "storage#hmacKeyMetadata", -- "accessId": ACCESS_ID, -- "serviceAccountEmail": EMAIL, -- } -- ], -- } -+ iterator = client.list_hmac_keys() - -- http = _make_requests_session([_make_json_response(response)]) -- client._http_internal = http -+ self.assertIs(iterator, client._list_resource.return_value) - -- metadatas = list( -- client.list_hmac_keys( -- max_results=MAX_RESULTS, -- service_account_email=EMAIL, -- show_deleted_keys=True, -- project_id=OTHER_PROJECT, -- user_project=USER_PROJECT, -- timeout=42, -- ) -+ expected_path = "/projects/{}/hmacKeys".format(project) -+ expected_item_to_value = _item_to_hmac_key_metadata -+ expected_max_results = None -+ expected_extra_params = {} -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ max_results=expected_max_results, -+ extra_params=expected_extra_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) - -- self.assertEqual(len(metadatas), len(response["items"])) -- -- for metadata, resource in zip(metadatas, response["items"]): -- self.assertIsInstance(metadata, HMACKeyMetadata) -- self.assertIs(metadata._client, client) -- self.assertEqual(metadata._properties, resource) -+ def test_list_hmac_keys_w_explicit(self): -+ from google.cloud.storage.client import _item_to_hmac_key_metadata - -- http.request.assert_called_once_with( -- method="GET", url=mock.ANY, data=None, headers=mock.ANY, timeout=42 -- ) -- _, kwargs = http.request.call_args -- scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) -- self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) -- self.assertEqual( -- path, -- "/".join( -- [ -- "", -- "storage", -- client._connection.API_VERSION, -- "projects", -- OTHER_PROJECT, -- "hmacKeys", -- ] -- ), -+ project = "PROJECT" -+ other_project = "other-project-456" -+ max_results = 3 -+ show_deleted_keys = True -+ service_account_email = "storage-user-123@example.com" -+ user_project = "billed-project" -+ credentials = _make_credentials() -+ client = self._make_one(project=project, credentials=credentials) -+ client._list_resource = mock.Mock(spec=[]) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ -+ iterator = client.list_hmac_keys( -+ max_results=max_results, -+ service_account_email=service_account_email, -+ show_deleted_keys=show_deleted_keys, -+ project_id=other_project, -+ user_project=user_project, -+ timeout=timeout, -+ retry=retry, -+ ) -+ -+ self.assertIs(iterator, client._list_resource.return_value) -+ -+ expected_path = "/projects/{}/hmacKeys".format(other_project) -+ expected_item_to_value = _item_to_hmac_key_metadata -+ expected_max_results = max_results -+ expected_extra_params = { -+ "serviceAccountEmail": service_account_email, -+ "showDeletedKeys": show_deleted_keys, -+ "userProject": user_project, -+ } -+ client._list_resource.assert_called_once_with( -+ expected_path, -+ expected_item_to_value, -+ max_results=expected_max_results, -+ extra_params=expected_extra_params, -+ timeout=timeout, -+ retry=retry, - ) -- parms = dict(urlparse.parse_qsl(qs)) -- self.assertEqual(parms["maxResults"], str(MAX_RESULTS)) -- self.assertEqual(parms["serviceAccountEmail"], EMAIL) -- self.assertEqual(parms["showDeletedKeys"], "True") -- self.assertEqual(parms["userProject"], USER_PROJECT) - - def test_get_hmac_key_metadata_wo_project(self): - from google.cloud.storage.hmac_key import HMACKeyMetadata -@@ -1786,7 +1957,9 @@ class TestClient(unittest.TestCase): - EXPECTED_SIGN = "5369676e61747572655f6279746573" - EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiYnVja2V0IjoiYnVja2V0LW5hbWUifSx7ImtleSI6Im9iamVjdC1uYW1lIn0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMzEyVDExNDcxNloifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdEBtYWlsLmNvbS8yMDIwMDMxMi9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAzLTI2VDAwOjAwOjEwWiJ9" - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() - with dtstamps_patch, now_patch, expire_secs_patch: -@@ -1864,7 +2037,9 @@ class TestClient(unittest.TestCase): - EXPECTED_SIGN = "5369676e61747572655f6279746573" - EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiZmllbGQxIjoiVmFsdWUxIn0seyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() - with dtstamps_patch, now_patch, expire_secs_patch: -@@ -1902,7 +2077,9 @@ class TestClient(unittest.TestCase): - - BUCKET_NAME = "bucket-name" - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, _, _ = _time_functions_patches() - with dtstamps_patch: -@@ -1920,7 +2097,9 @@ class TestClient(unittest.TestCase): - def test_get_signed_policy_v4_bucket_bound_hostname(self): - import datetime - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, _, _ = _time_functions_patches() - with dtstamps_patch: -@@ -1936,7 +2115,9 @@ class TestClient(unittest.TestCase): - def test_get_signed_policy_v4_bucket_bound_hostname_with_scheme(self): - import datetime - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, _, _ = _time_functions_patches() - with dtstamps_patch: -@@ -1954,7 +2135,9 @@ class TestClient(unittest.TestCase): - BUCKET_NAME = "bucket-name" - EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() - with dtstamps_patch, now_patch, expire_secs_patch: -@@ -1978,7 +2161,9 @@ class TestClient(unittest.TestCase): - EXPECTED_SIGN = "0c4003044105" - EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiYnVja2V0IjoiYnVja2V0LW5hbWUifSx7ImtleSI6Im9iamVjdC1uYW1lIn0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMzEyVDExNDcxNloifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdEBtYWlsLmNvbS8yMDIwMDMxMi9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAzLTI2VDAwOjAwOjEwWiJ9" - -- client = self._make_one(project="PROJECT") -+ project = "PROJECT" -+ credentials = _make_credentials(project=project) -+ client = self._make_one(credentials=credentials) - - dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() - with dtstamps_patch, now_patch, expire_secs_patch: -@@ -2013,32 +2198,57 @@ class TestClient(unittest.TestCase): - self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) - self.assertEqual(fields["policy"], EXPECTED_POLICY) - -- def test_list_buckets_retries_error(self): -- PROJECT = "PROJECT" -- CREDENTIALS = _make_credentials() -- client = self._make_one(project=PROJECT, credentials=CREDENTIALS) - -- BUCKET_NAME = "bucket-name" -+class Test__item_to_bucket(unittest.TestCase): -+ def _call_fut(self, iterator, item): -+ from google.cloud.storage.client import _item_to_bucket - -- data = {"items": [{"name": BUCKET_NAME}]} -- http = _make_requests_session( -- [exceptions.InternalServerError("mock error"), _make_json_response(data)] -- ) -- client._http_internal = http -+ return _item_to_bucket(iterator, item) - -- buckets = list(client.list_buckets()) -+ def test_w_empty_item(self): -+ from google.cloud.storage.bucket import Bucket - -- self.assertEqual(len(buckets), 1) -- self.assertEqual(buckets[0].name, BUCKET_NAME) -+ iterator = mock.Mock(spec=["client"]) -+ item = {} - -- call = mock.call( -- method="GET", -- url=mock.ANY, -- data=mock.ANY, -- headers=mock.ANY, -- timeout=self._get_default_timeout(), -- ) -- http.request.assert_has_calls([call, call]) -+ bucket = self._call_fut(iterator, item) -+ -+ self.assertIsInstance(bucket, Bucket) -+ self.assertIs(bucket.client, iterator.client) -+ self.assertIsNone(bucket.name) -+ -+ def test_w_name(self): -+ from google.cloud.storage.bucket import Bucket -+ -+ name = "name" -+ iterator = mock.Mock(spec=["client"]) -+ item = {"name": name} -+ -+ bucket = self._call_fut(iterator, item) -+ -+ self.assertIsInstance(bucket, Bucket) -+ self.assertIs(bucket.client, iterator.client) -+ self.assertEqual(bucket.name, name) -+ -+ -+class Test__item_to_hmac_key_metadata(unittest.TestCase): -+ def _call_fut(self, iterator, item): -+ from google.cloud.storage.client import _item_to_hmac_key_metadata -+ -+ return _item_to_hmac_key_metadata(iterator, item) -+ -+ def test_it(self): -+ from google.cloud.storage.hmac_key import HMACKeyMetadata -+ -+ access_id = "ABCDE" -+ iterator = mock.Mock(spec=["client"]) -+ item = {"id": access_id} -+ -+ metadata = self._call_fut(iterator, item) -+ -+ self.assertIsInstance(metadata, HMACKeyMetadata) -+ self.assertIs(metadata._client, iterator.client) -+ self.assertEqual(metadata._properties, item) - - - @pytest.mark.parametrize("test_data", _POST_POLICY_TESTS) ---- a/tests/unit/test_hmac_key.py -+++ b/tests/unit/test_hmac_key.py -@@ -218,31 +218,29 @@ class TestHMACKeyMetadata(unittest.TestC - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) - self.assertEqual(metadata.path, expected_path) - -- def test_exists_miss_no_project_set(self): -+ def test_exists_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - - access_id = "ACCESS-ID" -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.side_effect = NotFound("testing") -- client = _Client(connection) -+ project = "PROJECT" -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.side_effect = NotFound("testing") -+ client.project = project - metadata = self._make_one(client) - metadata._properties["accessId"] = access_id - -- self.assertFalse(metadata.exists(timeout=42)) -+ self.assertFalse(metadata.exists()) - -- expected_path = "/projects/{}/hmacKeys/{}".format( -- client.DEFAULT_PROJECT, access_id -+ expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) -- expected_kwargs = { -- "method": "GET", -- "path": expected_path, -- "query_params": {}, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) - -- def test_exists_hit_w_project_set(self): -+ def test_exists_hit_w_explicit_w_user_project(self): - project = "PROJECT-ID" - access_id = "ACCESS-ID" - user_project = "billed-project" -@@ -252,49 +250,47 @@ class TestHMACKeyMetadata(unittest.TestC - "accessId": access_id, - "serviceAccountEmail": email, - } -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.return_value = resource -- client = _Client(connection) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = resource - metadata = self._make_one(client, user_project=user_project) - metadata._properties["accessId"] = access_id - metadata._properties["projectId"] = project - -- self.assertTrue(metadata.exists()) -+ self.assertTrue(metadata.exists(timeout=timeout, retry=retry)) - - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -- expected_kwargs = { -- "method": "GET", -- "path": expected_path, -- "query_params": {"userProject": user_project}, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ ) - -- def test_reload_miss_no_project_set(self): -+ def test_reload_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - - access_id = "ACCESS-ID" -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.side_effect = NotFound("testing") -- client = _Client(connection) -+ project = "PROJECT" -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.side_effect = NotFound("testing") -+ client.project = project - metadata = self._make_one(client) - metadata._properties["accessId"] = access_id - - with self.assertRaises(NotFound): -- metadata.reload(timeout=42) -+ metadata.reload() - -- expected_path = "/projects/{}/hmacKeys/{}".format( -- client.DEFAULT_PROJECT, access_id -+ expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) -- expected_kwargs = { -- "method": "GET", -- "path": expected_path, -- "query_params": {}, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) - - def test_reload_hit_w_project_set(self): - project = "PROJECT-ID" -@@ -306,55 +302,54 @@ class TestHMACKeyMetadata(unittest.TestC - "accessId": access_id, - "serviceAccountEmail": email, - } -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.return_value = resource -- client = _Client(connection) -+ timeout = 42 -+ retry = mock.Mock(spec=[]) -+ client = mock.Mock(spec=["_get_resource"]) -+ client._get_resource.return_value = resource - metadata = self._make_one(client, user_project=user_project) - metadata._properties["accessId"] = access_id - metadata._properties["projectId"] = project - -- metadata.reload() -+ metadata.reload(timeout=timeout, retry=retry) - - self.assertEqual(metadata._properties, resource) - - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -- expected_kwargs = { -- "method": "GET", -- "path": expected_path, -- "query_params": {"userProject": user_project}, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ ) - -- def test_update_miss_no_project_set(self): -+ def test_update_miss_no_project_set_w_defaults(self): - from google.cloud.exceptions import NotFound - -+ project = "PROJECT" - access_id = "ACCESS-ID" -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.side_effect = NotFound("testing") -- client = _Client(connection) -+ client = mock.Mock(spec=["_put_resource", "project"]) -+ client._put_resource.side_effect = NotFound("testing") -+ client.project = project - metadata = self._make_one(client) - metadata._properties["accessId"] = access_id - metadata.state = "INACTIVE" - - with self.assertRaises(NotFound): -- metadata.update(timeout=42) -+ metadata.update() - -- expected_path = "/projects/{}/hmacKeys/{}".format( -- client.DEFAULT_PROJECT, access_id -+ expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -+ expected_data = {"state": "INACTIVE"} -+ expected_query_params = {} -+ client._put_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY_IF_ETAG_IN_JSON, - ) -- expected_kwargs = { -- "method": "PUT", -- "path": expected_path, -- "data": {"state": "INACTIVE"}, -- "query_params": {}, -- "timeout": 42, -- "retry": DEFAULT_RETRY_IF_ETAG_IN_JSON, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) - -- def test_update_hit_w_project_set(self): -+ def test_update_hit_w_project_set_w_timeout_w_retry(self): - project = "PROJECT-ID" - access_id = "ACCESS-ID" - user_project = "billed-project" -@@ -365,86 +360,90 @@ class TestHMACKeyMetadata(unittest.TestC - "serviceAccountEmail": email, - "state": "ACTIVE", - } -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.return_value = resource -- client = _Client(connection) -+ client = mock.Mock(spec=["_put_resource"]) -+ client._put_resource.return_value = resource - metadata = self._make_one(client, user_project=user_project) - metadata._properties["accessId"] = access_id - metadata._properties["projectId"] = project - metadata.state = "ACTIVE" -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- metadata.update() -+ metadata.update(timeout=42, retry=retry) - - self.assertEqual(metadata._properties, resource) - - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -- expected_kwargs = { -- "method": "PUT", -- "path": expected_path, -- "data": {"state": "ACTIVE"}, -- "query_params": {"userProject": user_project}, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY_IF_ETAG_IN_JSON, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) -+ expected_data = {"state": "ACTIVE"} -+ expected_query_params = {"userProject": user_project} -+ client._put_resource.assert_called_once_with( -+ expected_path, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ ) - - def test_delete_not_inactive(self): -- metadata = self._make_one() -+ client = mock.Mock(spec=["_delete_resource", "project"]) -+ client.project = "PROJECT" -+ metadata = self._make_one(client) -+ - for state in ("ACTIVE", "DELETED"): - metadata._properties["state"] = state - - with self.assertRaises(ValueError): - metadata.delete() - -- def test_delete_miss_no_project_set(self): -+ client._delete_resource.assert_not_called() -+ -+ def test_delete_miss_no_project_set_w_defaults(self): - from google.cloud.exceptions import NotFound - - access_id = "ACCESS-ID" -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.side_effect = NotFound("testing") -- client = _Client(connection) -+ client = mock.Mock(spec=["_delete_resource", "project"]) -+ client._delete_resource.side_effect = NotFound("testing") -+ client.project = "PROJECT" - metadata = self._make_one(client) - metadata._properties["accessId"] = access_id - metadata.state = "INACTIVE" - - with self.assertRaises(NotFound): -- metadata.delete(timeout=42) -+ metadata.delete() - -- expected_path = "/projects/{}/hmacKeys/{}".format( -- client.DEFAULT_PROJECT, access_id -+ expected_path = "/projects/{}/hmacKeys/{}".format(client.project, access_id) -+ expected_query_params = {} -+ client._delete_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), -+ retry=DEFAULT_RETRY, - ) -- expected_kwargs = { -- "method": "DELETE", -- "path": expected_path, -- "query_params": {}, -- "timeout": 42, -- "retry": DEFAULT_RETRY, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) - -- def test_delete_hit_w_project_set(self): -+ def test_delete_hit_w_project_set_w_explicit_timeout_retry(self): - project = "PROJECT-ID" - access_id = "ACCESS-ID" - user_project = "billed-project" -- connection = mock.Mock(spec=["api_request"]) -- connection.api_request.return_value = {} -- client = _Client(connection) -+ client = mock.Mock(spec=["_delete_resource", "project"]) -+ client.project = "CLIENT-PROJECT" -+ client._delete_resource.return_value = {} - metadata = self._make_one(client, user_project=user_project) - metadata._properties["accessId"] = access_id - metadata._properties["projectId"] = project - metadata.state = "INACTIVE" -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- metadata.delete() -+ metadata.delete(timeout=timeout, retry=retry) - - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) -- expected_kwargs = { -- "method": "DELETE", -- "path": expected_path, -- "query_params": {"userProject": user_project}, -- "timeout": self._get_default_timeout(), -- "retry": DEFAULT_RETRY, -- } -- connection.api_request.assert_called_once_with(**expected_kwargs) -+ expected_query_params = {"userProject": user_project} -+ client._delete_resource.assert_called_once_with( -+ expected_path, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, -+ ) - - - class _Client(object): ---- a/tests/unit/test_notification.py -+++ b/tests/unit/test_notification.py -@@ -231,7 +231,8 @@ class TestBucketNotification(unittest.Te - self.assertEqual(notification.self_link, self.SELF_LINK) - - def test_create_w_existing_notification_id(self): -- client = self._make_client() -+ client = mock.Mock(spec=["_post_resource", "project"]) -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - notification._properties["id"] = self.NOTIFICATION_ID -@@ -239,20 +240,23 @@ class TestBucketNotification(unittest.Te - with self.assertRaises(ValueError): - notification.create() - -+ client._post_resource.assert_not_called() -+ - def test_create_w_defaults(self): - from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT - -- client = self._make_client() -- bucket = self._make_bucket(client) -- notification = self._make_one(bucket, self.TOPIC_NAME) -- api_request = client._connection.api_request -- api_request.return_value = { -+ api_response = { - "topic": self.TOPIC_REF, - "id": self.NOTIFICATION_ID, - "etag": self.ETAG, - "selfLink": self.SELF_LINK, - "payload_format": NONE_PAYLOAD_FORMAT, - } -+ client = mock.Mock(spec=["_post_resource", "project"]) -+ client.project = self.BUCKET_PROJECT -+ client._post_resource.return_value = api_response -+ bucket = self._make_bucket(client) -+ notification = self._make_one(bucket, self.TOPIC_NAME) - - notification.create() - -@@ -264,32 +268,22 @@ class TestBucketNotification(unittest.Te - self.assertIsNone(notification.blob_name_prefix) - self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT) - -- data = {"topic": self.TOPIC_REF, "payload_format": NONE_PAYLOAD_FORMAT} -- api_request.assert_called_once_with( -- method="POST", -- path=self.CREATE_PATH, -- query_params={}, -- data=data, -+ expected_data = { -+ "topic": self.TOPIC_REF, -+ "payload_format": NONE_PAYLOAD_FORMAT, -+ } -+ expected_query_params = {} -+ client._post_resource.assert_called_once_with( -+ self.CREATE_PATH, -+ expected_data, -+ query_params=expected_query_params, - timeout=self._get_default_timeout(), - retry=None, - ) - -- def test_create_w_explicit_client(self): -- USER_PROJECT = "user-project-123" -- client = self._make_client() -- alt_client = self._make_client() -- bucket = self._make_bucket(client, user_project=USER_PROJECT) -- notification = self._make_one( -- bucket, -- self.TOPIC_NAME, -- topic_project=self.TOPIC_ALT_PROJECT, -- custom_attributes=self.CUSTOM_ATTRIBUTES, -- event_types=self.event_types(), -- blob_name_prefix=self.BLOB_NAME_PREFIX, -- payload_format=self.payload_format(), -- ) -- api_request = alt_client._connection.api_request -- api_request.return_value = { -+ def test_create_w_explicit_client_w_timeout_w_retry(self): -+ user_project = "user-project-123" -+ api_response = { - "topic": self.TOPIC_ALT_REF, - "custom_attributes": self.CUSTOM_ATTRIBUTES, - "event_types": self.event_types(), -@@ -299,8 +293,23 @@ class TestBucketNotification(unittest.Te - "etag": self.ETAG, - "selfLink": self.SELF_LINK, - } -+ bucket = self._make_bucket(client=None, user_project=user_project) -+ notification = self._make_one( -+ bucket, -+ self.TOPIC_NAME, -+ topic_project=self.TOPIC_ALT_PROJECT, -+ custom_attributes=self.CUSTOM_ATTRIBUTES, -+ event_types=self.event_types(), -+ blob_name_prefix=self.BLOB_NAME_PREFIX, -+ payload_format=self.payload_format(), -+ ) -+ client = mock.Mock(spec=["_post_resource", "project"]) -+ client.project = self.BUCKET_PROJECT -+ client._post_resource.return_value = api_response -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- notification.create(client=alt_client, timeout=42) -+ notification.create(client=client, timeout=timeout, retry=retry) - - self.assertEqual(notification.custom_attributes, self.CUSTOM_ATTRIBUTES) - self.assertEqual(notification.event_types, self.event_types()) -@@ -310,121 +319,135 @@ class TestBucketNotification(unittest.Te - self.assertEqual(notification.etag, self.ETAG) - self.assertEqual(notification.self_link, self.SELF_LINK) - -- data = { -+ expected_data = { - "topic": self.TOPIC_ALT_REF, - "custom_attributes": self.CUSTOM_ATTRIBUTES, - "event_types": self.event_types(), - "object_name_prefix": self.BLOB_NAME_PREFIX, - "payload_format": self.payload_format(), - } -- api_request.assert_called_once_with( -- method="POST", -- path=self.CREATE_PATH, -- query_params={"userProject": USER_PROJECT}, -- data=data, -- timeout=42, -- retry=None, -+ expected_query_params = {"userProject": user_project} -+ client._post_resource.assert_called_once_with( -+ self.CREATE_PATH, -+ expected_data, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, - ) - - def test_exists_wo_notification_id(self): -- client = self._make_client() -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - - with self.assertRaises(ValueError): - notification.exists() - -- def test_exists_miss(self): -+ client._get_resource.assert_not_called() -+ -+ def test_exists_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- client = self._make_client() -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.side_effect = NotFound("testing") -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - notification._properties["id"] = self.NOTIFICATION_ID -- api_request = client._connection.api_request -- api_request.side_effect = NotFound("testing") - -- self.assertFalse(notification.exists(timeout=42)) -+ self.assertFalse(notification.exists()) - -- api_request.assert_called_once_with( -- method="GET", -- path=self.NOTIFICATION_PATH, -- query_params={}, -- timeout=42, -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ self.NOTIFICATION_PATH, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, - ) - -- def test_exists_hit(self): -- USER_PROJECT = "user-project-123" -- client = self._make_client() -- bucket = self._make_bucket(client, user_project=USER_PROJECT) -- notification = self._make_one(bucket, self.TOPIC_NAME) -- notification._properties["id"] = self.NOTIFICATION_ID -- api_request = client._connection.api_request -- api_request.return_value = { -+ def test_exists_hit_w_explicit_w_user_project(self): -+ user_project = "user-project-123" -+ api_response = { - "topic": self.TOPIC_REF, - "id": self.NOTIFICATION_ID, - "etag": self.ETAG, - "selfLink": self.SELF_LINK, - } -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.return_vale = api_response -+ client.project = self.BUCKET_PROJECT -+ bucket = self._make_bucket(client, user_project=user_project) -+ notification = self._make_one(bucket, self.TOPIC_NAME) -+ notification._properties["id"] = self.NOTIFICATION_ID -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- self.assertTrue(notification.exists(client=client)) -+ self.assertTrue( -+ notification.exists(client=client, timeout=timeout, retry=retry) -+ ) - -- api_request.assert_called_once_with( -- method="GET", -- path=self.NOTIFICATION_PATH, -- query_params={"userProject": USER_PROJECT}, -- timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ self.NOTIFICATION_PATH, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, - ) - - def test_reload_wo_notification_id(self): -- client = self._make_client() -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - - with self.assertRaises(ValueError): - notification.reload() - -- def test_reload_miss(self): -+ client._get_resource.assert_not_called() -+ -+ def test_reload_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- client = self._make_client() -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.side_effect = NotFound("testing") -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - notification._properties["id"] = self.NOTIFICATION_ID -- api_request = client._connection.api_request -- api_request.side_effect = NotFound("testing") - - with self.assertRaises(NotFound): -- notification.reload(timeout=42) -+ notification.reload() - -- api_request.assert_called_once_with( -- method="GET", -- path=self.NOTIFICATION_PATH, -- query_params={}, -- timeout=42, -+ expected_query_params = {} -+ client._get_resource.assert_called_once_with( -+ self.NOTIFICATION_PATH, -+ query_params=expected_query_params, -+ timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, - ) - -- def test_reload_hit(self): -+ def test_reload_hit_w_explicit_w_user_project(self): - from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT - -- USER_PROJECT = "user-project-123" -- client = self._make_client() -- bucket = self._make_bucket(client, user_project=USER_PROJECT) -- notification = self._make_one(bucket, self.TOPIC_NAME) -- notification._properties["id"] = self.NOTIFICATION_ID -- api_request = client._connection.api_request -- api_request.return_value = { -+ user_project = "user-project-123" -+ api_response = { - "topic": self.TOPIC_REF, - "id": self.NOTIFICATION_ID, - "etag": self.ETAG, - "selfLink": self.SELF_LINK, - "payload_format": NONE_PAYLOAD_FORMAT, - } -+ client = mock.Mock(spec=["_get_resource", "project"]) -+ client._get_resource.return_value = api_response -+ client.project = self.BUCKET_PROJECT -+ bucket = self._make_bucket(client, user_project=user_project) -+ notification = self._make_one(bucket, self.TOPIC_NAME) -+ notification._properties["id"] = self.NOTIFICATION_ID -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- notification.reload(client=client) -+ notification.reload(client=client, timeout=timeout, retry=retry) - - self.assertEqual(notification.etag, self.ETAG) - self.assertEqual(notification.self_link, self.SELF_LINK) -@@ -433,60 +456,64 @@ class TestBucketNotification(unittest.Te - self.assertIsNone(notification.blob_name_prefix) - self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT) - -- api_request.assert_called_once_with( -- method="GET", -- path=self.NOTIFICATION_PATH, -- query_params={"userProject": USER_PROJECT}, -- timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -+ expected_query_params = {"userProject": user_project} -+ client._get_resource.assert_called_once_with( -+ self.NOTIFICATION_PATH, -+ query_params=expected_query_params, -+ timeout=timeout, -+ retry=retry, - ) - - def test_delete_wo_notification_id(self): -- client = self._make_client() -+ client = mock.Mock(spec=["_delete_resource", "project"]) -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - - with self.assertRaises(ValueError): - notification.delete() - -- def test_delete_miss(self): -+ client._delete_resource.assert_not_called() -+ -+ def test_delete_miss_w_defaults(self): - from google.cloud.exceptions import NotFound - -- client = self._make_client() -+ client = mock.Mock(spec=["_delete_resource", "project"]) -+ client._delete_resource.side_effect = NotFound("testing") -+ client.project = self.BUCKET_PROJECT - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - notification._properties["id"] = self.NOTIFICATION_ID -- api_request = client._connection.api_request -- api_request.side_effect = NotFound("testing") - - with self.assertRaises(NotFound): -- notification.delete(timeout=42) -+ notification.delete() - -- api_request.assert_called_once_with( -- method="DELETE", -- path=self.NOTIFICATION_PATH, -+ client._delete_resource.assert_called_once_with( -+ self.NOTIFICATION_PATH, - query_params={}, -- timeout=42, -+ timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, - ) - -- def test_delete_hit(self): -- USER_PROJECT = "user-project-123" -- client = self._make_client() -- bucket = self._make_bucket(client, user_project=USER_PROJECT) -+ def test_delete_hit_w_explicit_client_timeout_retry(self): -+ user_project = "user-project-123" -+ client = mock.Mock(spec=["_delete_resource"]) -+ client._delete_resource.return_value = None -+ bucket_client = mock.Mock(spec=["project"]) -+ bucket_client.project = self.BUCKET_PROJECT -+ bucket = self._make_bucket(bucket_client, user_project=user_project) - notification = self._make_one(bucket, self.TOPIC_NAME) - notification._properties["id"] = self.NOTIFICATION_ID -- api_request = client._connection.api_request -- api_request.return_value = None -+ timeout = 42 -+ retry = mock.Mock(spec=[]) - -- notification.delete(client=client) -+ notification.delete(client=client, timeout=timeout, retry=retry) - -- api_request.assert_called_once_with( -- method="DELETE", -- path=self.NOTIFICATION_PATH, -- query_params={"userProject": USER_PROJECT}, -- timeout=self._get_default_timeout(), -- retry=DEFAULT_RETRY, -+ client._delete_resource.assert_called_once_with( -+ self.NOTIFICATION_PATH, -+ query_params={"userProject": user_project}, -+ timeout=timeout, -+ retry=retry, - ) - - ---- a/tests/unit/test_retry.py -+++ b/tests/unit/test_retry.py -@@ -19,6 +19,14 @@ from google.cloud.storage import _helper - import mock - - -+try: -+ ConnectionError -+except NameError: -+ _HAS_STDLIB_CONNECTION_ERROR = False -+else: -+ _HAS_STDLIB_CONNECTION_ERROR = True -+ -+ - class Test_should_retry(unittest.TestCase): - def _call_fut(self, exc): - from google.cloud.storage import retry -@@ -56,9 +64,22 @@ class Test_should_retry(unittest.TestCas - self.assertFalse(self._call_fut(exc)) - - def test_w_requests_connection_error(self): -+ import requests -+ -+ exc = requests.ConnectionError() -+ self.assertTrue(self._call_fut(exc)) -+ -+ def test_miss_w_stdlib_error(self): - exc = ValueError("testing") - self.assertFalse(self._call_fut(exc)) - -+ @unittest.skipUnless( -+ _HAS_STDLIB_CONNECTION_ERROR, "No builtin 'ConnectionError' in Python 2", -+ ) -+ def test_w_stdlib_connection_error(self): -+ exc = ConnectionError() -+ self.assertTrue(self._call_fut(exc)) -+ - - class TestConditionalRetryPolicy(unittest.TestCase): - def _make_one(self, retry_policy, conditional_predicate, required_kwargs): diff --git a/no-network.patch b/no-network.patch new file mode 100644 index 0000000..7542b8a --- /dev/null +++ b/no-network.patch @@ -0,0 +1,604 @@ +--- + setup.cfg | 3 ++ + tests/unit/test_blob.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++ + tests/unit/test_bucket.py | 15 +++++++++++ + tests/unit/test_client.py | 16 ++++++++++++ + 4 files changed, 94 insertions(+) + +--- a/setup.cfg ++++ b/setup.cfg +@@ -5,3 +5,6 @@ universal = 1 + tag_build = + tag_date = 0 + ++[tool:pytest] ++markers = ++ network: marks tests which require network connection +--- a/tests/unit/test_blob.py ++++ b/tests/unit/test_blob.py +@@ -1223,6 +1223,7 @@ class Test_Blob(unittest.TestCase): + ) + patch.assert_not_called() + ++ @pytest.mark.network + def test_download_to_file_with_failure(self): + import requests + from google.resumable_media import InvalidResponse +@@ -1262,6 +1263,7 @@ class Test_Blob(unittest.TestCase): + checksum="md5", + ) + ++ @pytest.mark.network + def test_download_to_file_wo_media_link(self): + blob_name = "blob-name" + client = self._make_client() +@@ -1292,6 +1294,7 @@ class Test_Blob(unittest.TestCase): + checksum="md5", + ) + ++ @pytest.mark.network + def test_download_to_file_w_generation_match(self): + GENERATION_NUMBER = 6 + HEADERS = {"accept-encoding": "gzip"} +@@ -1359,18 +1362,23 @@ class Test_Blob(unittest.TestCase): + checksum="md5", + ) + ++ @pytest.mark.network + def test_download_to_file_wo_chunks_wo_raw(self): + self._download_to_file_helper(use_chunks=False, raw_download=False) + ++ @pytest.mark.network + def test_download_to_file_w_chunks_wo_raw(self): + self._download_to_file_helper(use_chunks=True, raw_download=False) + ++ @pytest.mark.network + def test_download_to_file_wo_chunks_w_raw(self): + self._download_to_file_helper(use_chunks=False, raw_download=True) + ++ @pytest.mark.network + def test_download_to_file_w_chunks_w_raw(self): + self._download_to_file_helper(use_chunks=True, raw_download=True) + ++ @pytest.mark.network + def test_download_to_file_w_custom_timeout(self): + self._download_to_file_helper( + use_chunks=False, raw_download=False, timeout=9.58 +@@ -1427,6 +1435,7 @@ class Test_Blob(unittest.TestCase): + stream = blob._do_download.mock_calls[0].args[1] + self.assertEqual(stream.name, temp.name) + ++ @pytest.mark.network + def test_download_to_filename_w_generation_match(self): + from google.cloud._testing import _NamedTemporaryFile + +@@ -1457,25 +1466,31 @@ class Test_Blob(unittest.TestCase): + checksum="md5", + ) + ++ @pytest.mark.network + def test_download_to_filename_w_updated_wo_raw(self): + updated = "2014-12-06T13:13:50.690Z" + self._download_to_filename_helper(updated=updated, raw_download=False) + ++ @pytest.mark.network + def test_download_to_filename_wo_updated_wo_raw(self): + self._download_to_filename_helper(updated=None, raw_download=False) + ++ @pytest.mark.network + def test_download_to_filename_w_updated_w_raw(self): + updated = "2014-12-06T13:13:50.690Z" + self._download_to_filename_helper(updated=updated, raw_download=True) + ++ @pytest.mark.network + def test_download_to_filename_wo_updated_w_raw(self): + self._download_to_filename_helper(updated=None, raw_download=True) + ++ @pytest.mark.network + def test_download_to_filename_w_custom_timeout(self): + self._download_to_filename_helper( + updated=None, raw_download=False, timeout=9.58 + ) + ++ @pytest.mark.network + def test_download_to_filename_corrupted(self): + from google.resumable_media import DataCorruption + +@@ -1517,6 +1532,7 @@ class Test_Blob(unittest.TestCase): + stream = blob._do_download.mock_calls[0].args[1] + self.assertEqual(stream.name, filename) + ++ @pytest.mark.network + def test_download_to_filename_w_key(self): + from google.cloud._testing import _NamedTemporaryFile + from google.cloud.storage.blob import _get_encryption_headers +@@ -1677,6 +1693,7 @@ class Test_Blob(unittest.TestCase): + self.assertIsNone(blob.md5_hash) + self.assertIsNone(blob.crc32c) + ++ @pytest.mark.network + def test_download_as_bytes_w_generation_match(self): + GENERATION_NUMBER = 6 + MEDIA_LINK = "http://example.com/media/" +@@ -1704,12 +1721,15 @@ class Test_Blob(unittest.TestCase): + checksum="md5", + ) + ++ @pytest.mark.network + def test_download_as_bytes_wo_raw(self): + self._download_as_bytes_helper(raw_download=False) + ++ @pytest.mark.network + def test_download_as_bytes_w_raw(self): + self._download_as_bytes_helper(raw_download=True) + ++ @pytest.mark.network + def test_download_as_byte_w_custom_timeout(self): + self._download_as_bytes_helper(raw_download=False, timeout=9.58) + +@@ -1860,6 +1880,7 @@ class Test_Blob(unittest.TestCase): + charset=charset, + ) + ++ @pytest.mark.network + @mock.patch("warnings.warn") + def test_download_as_string(self, mock_warn): + MEDIA_LINK = "http://example.com/media/" +@@ -2129,25 +2150,30 @@ class Test_Blob(unittest.TestCase): + "POST", upload_url, data=payload, headers=headers, timeout=expected_timeout + ) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_no_size(self, mock_get_boundary): + self._do_multipart_success(mock_get_boundary, predefined_acl="private") + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_no_size_mtls(self, mock_get_boundary): + self._do_multipart_success( + mock_get_boundary, predefined_acl="private", mtls=True + ) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_size(self, mock_get_boundary): + self._do_multipart_success(mock_get_boundary, size=10) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_user_project(self, mock_get_boundary): + user_project = "user-project-123" + self._do_multipart_success(mock_get_boundary, user_project=user_project) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_kms(self, mock_get_boundary): + kms_resource = ( +@@ -2158,6 +2184,7 @@ class Test_Blob(unittest.TestCase): + ) + self._do_multipart_success(mock_get_boundary, kms_key_name=kms_resource) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_kms_with_version(self, mock_get_boundary): + kms_resource = ( +@@ -2169,26 +2196,31 @@ class Test_Blob(unittest.TestCase): + ) + self._do_multipart_success(mock_get_boundary, kms_key_name=kms_resource) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_retry(self, mock_get_boundary): + self._do_multipart_success(mock_get_boundary, num_retries=8) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_generation_match(self, mock_get_boundary): + self._do_multipart_success( + mock_get_boundary, if_generation_match=4, if_metageneration_match=4 + ) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_custom_timeout(self, mock_get_boundary): + self._do_multipart_success(mock_get_boundary, timeout=9.58) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_generation_not_match(self, mock_get_boundary): + self._do_multipart_success( + mock_get_boundary, if_generation_not_match=4, if_metageneration_not_match=4 + ) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_client(self, mock_get_boundary): + transport = self._mock_transport(http_client.OK, {}) +@@ -2196,6 +2228,7 @@ class Test_Blob(unittest.TestCase): + client._connection.API_BASE_URL = "https://storage.googleapis.com" + self._do_multipart_success(mock_get_boundary, client=client) + ++ @pytest.mark.network + @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_metadata(self, mock_get_boundary): + self._do_multipart_success(mock_get_boundary, metadata={"test": "test"}) +@@ -2403,25 +2436,32 @@ class Test_Blob(unittest.TestCase): + timeout=expected_timeout, + ) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_metadata(self): + self._initiate_resumable_helper(metadata={"test": "test"}) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_custom_timeout(self): + self._initiate_resumable_helper(timeout=9.58) + ++ @pytest.mark.network + def test__initiate_resumable_upload_no_size(self): + self._initiate_resumable_helper() + ++ @pytest.mark.network + def test__initiate_resumable_upload_no_size_mtls(self): + self._initiate_resumable_helper(mtls=True) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_size(self): + self._initiate_resumable_helper(size=10000) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_user_project(self): + user_project = "user-project-123" + self._initiate_resumable_helper(user_project=user_project) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_kms(self): + kms_resource = ( + "projects/test-project-123/" +@@ -2431,6 +2471,7 @@ class Test_Blob(unittest.TestCase): + ) + self._initiate_resumable_helper(kms_key_name=kms_resource) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_kms_with_version(self): + kms_resource = ( + "projects/test-project-123/" +@@ -2441,33 +2482,41 @@ class Test_Blob(unittest.TestCase): + ) + self._initiate_resumable_helper(kms_key_name=kms_resource) + ++ @pytest.mark.network + def test__initiate_resumable_upload_without_chunk_size(self): + self._initiate_resumable_helper(blob_chunk_size=None) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_chunk_size(self): + one_mb = 1048576 + self._initiate_resumable_helper(chunk_size=one_mb) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_extra_headers(self): + extra_headers = {"origin": "http://not-in-kansas-anymore.invalid"} + self._initiate_resumable_helper(extra_headers=extra_headers) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_retry(self): + self._initiate_resumable_helper(num_retries=11) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_generation_match(self): + self._initiate_resumable_helper( + if_generation_match=4, if_metageneration_match=4 + ) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_generation_not_match(self): + self._initiate_resumable_helper( + if_generation_not_match=4, if_metageneration_not_match=4 + ) + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_predefined_acl(self): + self._initiate_resumable_helper(predefined_acl="private") + ++ @pytest.mark.network + def test__initiate_resumable_upload_with_client(self): + resumable_url = "http://test.invalid?upload_id=hey-you" + response_headers = {"location": resumable_url} +@@ -2699,21 +2748,27 @@ class Test_Blob(unittest.TestCase): + ) + self.assertEqual(transport.request.mock_calls, [call0, call1, call2]) + ++ @pytest.mark.network + def test__do_resumable_upload_with_custom_timeout(self): + self._do_resumable_helper(timeout=9.58) + ++ @pytest.mark.network + def test__do_resumable_upload_no_size(self): + self._do_resumable_helper() + ++ @pytest.mark.network + def test__do_resumable_upload_with_size(self): + self._do_resumable_helper(use_size=True) + ++ @pytest.mark.network + def test__do_resumable_upload_with_retry(self): + self._do_resumable_helper(num_retries=6) + ++ @pytest.mark.network + def test__do_resumable_upload_with_predefined_acl(self): + self._do_resumable_helper(predefined_acl="private") + ++ @pytest.mark.network + def test__do_resumable_upload_with_data_corruption(self): + from google.resumable_media import DataCorruption + +@@ -3121,15 +3176,19 @@ class Test_Blob(unittest.TestCase): + timeout=expected_timeout, + ) + ++ @pytest.mark.network + def test_create_resumable_upload_session(self): + self._create_resumable_upload_session_helper() + ++ @pytest.mark.network + def test_create_resumable_upload_session_with_custom_timeout(self): + self._create_resumable_upload_session_helper(timeout=9.58) + ++ @pytest.mark.network + def test_create_resumable_upload_session_with_origin(self): + self._create_resumable_upload_session_helper(origin="http://google.com") + ++ @pytest.mark.network + def test_create_resumable_upload_session_with_failure(self): + from google.resumable_media import InvalidResponse + from google.cloud import exceptions +@@ -4709,6 +4768,7 @@ class Test_Blob(unittest.TestCase): + self.assertEqual(blob.name, "b") + self.assertEqual(blob.bucket.name, "buckets.example.com") + ++ @pytest.mark.network + def test_open(self): + from io import TextIOWrapper + from google.cloud.storage.fileio import BlobReader +--- a/tests/unit/test_bucket.py ++++ b/tests/unit/test_bucket.py +@@ -857,6 +857,7 @@ class Test_Bucket(unittest.TestCase): + self.assertEqual(blob.chunk_size, CHUNK_SIZE) + self.assertEqual(blob._encryption_key, KEY) + ++ @pytest.mark.network + def test_list_blobs_defaults(self): + NAME = "name" + connection = _Connection({"items": []}) +@@ -872,6 +873,7 @@ class Test_Bucket(unittest.TestCase): + self.assertEqual(kw["query_params"], {"projection": "noAcl"}) + self.assertEqual(kw["timeout"], self._get_default_timeout()) + ++ @pytest.mark.network + def test_list_blobs_w_all_arguments_and_user_project(self): + NAME = "name" + USER_PROJECT = "user-project-123" +@@ -1041,6 +1043,7 @@ class Test_Bucket(unittest.TestCase): + ] + self.assertEqual(connection._deleted_buckets, expected_cw) + ++ @pytest.mark.network + def test_delete_hit_with_user_project(self): + NAME = "name" + USER_PROJECT = "user-project-123" +@@ -1064,6 +1067,7 @@ class Test_Bucket(unittest.TestCase): + ] + self.assertEqual(connection._deleted_buckets, expected_cw) + ++ @pytest.mark.network + def test_delete_force_delete_blobs(self): + NAME = "name" + BLOB_NAME1 = "blob-name1" +@@ -1115,6 +1119,7 @@ class Test_Bucket(unittest.TestCase): + ] + self.assertEqual(connection._deleted_buckets, expected_cw) + ++ @pytest.mark.network + def test_delete_force_miss_blobs(self): + NAME = "name" + BLOB_NAME = "blob-name1" +@@ -1139,6 +1144,7 @@ class Test_Bucket(unittest.TestCase): + ] + self.assertEqual(connection._deleted_buckets, expected_cw) + ++ @pytest.mark.network + def test_delete_too_many(self): + NAME = "name" + BLOB_NAME1 = "blob-name1" +@@ -2301,6 +2307,7 @@ class Test_Bucket(unittest.TestCase): + bucket = self._make_one(name=NAME, properties=before) + self.assertEqual(bucket.versioning_enabled, True) + ++ @pytest.mark.network + @mock.patch("warnings.warn") + def test_create_deprecated(self, mock_warn): + PROJECT = "PROJECT" +@@ -2330,6 +2337,7 @@ class Test_Bucket(unittest.TestCase): + stacklevel=1, + ) + ++ @pytest.mark.network + @mock.patch("warnings.warn") + def test_create_w_user_project(self, mock_warn): + PROJECT = "PROJECT" +@@ -2754,6 +2762,7 @@ class Test_Bucket(unittest.TestCase): + def test_make_public_w_future_reload_default(self): + self._make_public_w_future_helper(default_object_acl_loaded=False) + ++ @pytest.mark.network + def test_make_public_recursive(self): + from google.cloud.storage.acl import _ACLEntity + +@@ -2818,6 +2827,7 @@ class Test_Bucket(unittest.TestCase): + ) + self.assertEqual(kw[1]["timeout"], 42) + ++ @pytest.mark.network + def test_make_public_recursive_too_many(self): + from google.cloud.storage.acl import _ACLEntity + +@@ -2902,6 +2912,7 @@ class Test_Bucket(unittest.TestCase): + def test_make_private_w_future_reload_default(self): + self._make_private_w_future_helper(default_object_acl_loaded=False) + ++ @pytest.mark.network + def test_make_private_recursive(self): + _saved = [] + +@@ -2963,6 +2974,7 @@ class Test_Bucket(unittest.TestCase): + ) + self.assertEqual(kw[1]["timeout"], 42) + ++ @pytest.mark.network + def test_make_private_recursive_too_many(self): + NO_PERMISSIONS = [] + AFTER = {"acl": NO_PERMISSIONS, "defaultObjectAcl": []} +@@ -2982,6 +2994,7 @@ class Test_Bucket(unittest.TestCase): + bucket._MAX_OBJECTS_FOR_ITERATION = 1 + self.assertRaises(ValueError, bucket.make_private, recursive=True) + ++ @pytest.mark.network + def test_page_empty_response(self): + from google.api_core import page_iterator + +@@ -2997,6 +3010,7 @@ class Test_Bucket(unittest.TestCase): + self.assertEqual(blobs, []) + self.assertEqual(iterator.prefixes, set()) + ++ @pytest.mark.network + def test_page_non_empty_response(self): + import six + from google.cloud.storage.blob import Blob +@@ -3024,6 +3038,7 @@ class Test_Bucket(unittest.TestCase): + self.assertEqual(blob.name, blob_name) + self.assertEqual(iterator.prefixes, set(["foo"])) + ++ @pytest.mark.network + def test_cumulative_prefixes(self): + import six + from google.cloud.storage.blob import Blob +--- a/tests/unit/test_client.py ++++ b/tests/unit/test_client.py +@@ -220,6 +220,7 @@ class TestClient(unittest.TestCase): + self.assertEqual(list(client._batch_stack), []) + self.assertIs(client._connection._client_info, client_info) + ++ @pytest.mark.network + def test_ctor_mtls(self): + credentials = _make_credentials() + +@@ -918,6 +919,7 @@ class TestClient(unittest.TestCase): + ) + self.assertEqual(bucket.location, location) + ++ @pytest.mark.network + def test_create_bucket_w_explicit_project(self): + from google.cloud.storage.client import Client + +@@ -941,6 +943,7 @@ class TestClient(unittest.TestCase): + retry=DEFAULT_RETRY, + ) + ++ @pytest.mark.network + def test_create_w_extra_properties(self): + from google.cloud.storage.client import Client + from google.cloud.storage.bucket import Bucket +@@ -993,6 +996,7 @@ class TestClient(unittest.TestCase): + retry=DEFAULT_RETRY, + ) + ++ @pytest.mark.network + def test_create_hit(self): + from google.cloud.storage.client import Client + +@@ -1094,6 +1098,7 @@ class TestClient(unittest.TestCase): + json_sent = http.request.call_args_list[0][1]["data"] + self.assertEqual(json_expected, json.loads(json_sent)) + ++ @pytest.mark.network + def test_download_blob_to_file_with_failure(self): + from google.resumable_media import InvalidResponse + from google.cloud.storage.blob import Blob +@@ -1204,15 +1209,19 @@ class TestClient(unittest.TestCase): + timeout=_DEFAULT_TIMEOUT, + ) + ++ @pytest.mark.network + def test_download_blob_to_file_wo_chunks_wo_raw(self): + self._download_blob_to_file_helper(use_chunks=False, raw_download=False) + ++ @pytest.mark.network + def test_download_blob_to_file_w_chunks_wo_raw(self): + self._download_blob_to_file_helper(use_chunks=True, raw_download=False) + ++ @pytest.mark.network + def test_download_blob_to_file_wo_chunks_w_raw(self): + self._download_blob_to_file_helper(use_chunks=False, raw_download=True) + ++ @pytest.mark.network + def test_download_blob_to_file_w_chunks_w_raw(self): + self._download_blob_to_file_helper(use_chunks=True, raw_download=True) + +@@ -1778,6 +1787,7 @@ class TestClient(unittest.TestCase): + parms = dict(urlparse.parse_qsl(qs)) + self.assertEqual(parms["userProject"], USER_PROJECT) + ++ @pytest.mark.network + def test_get_signed_policy_v4(self): + import datetime + +@@ -1855,6 +1865,7 @@ class TestClient(unittest.TestCase): + self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) + self.assertEqual(fields["policy"], EXPECTED_POLICY) + ++ @pytest.mark.network + def test_get_signed_policy_v4_with_fields(self): + import datetime + +@@ -1897,6 +1908,7 @@ class TestClient(unittest.TestCase): + self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) + self.assertEqual(fields["policy"], EXPECTED_POLICY) + ++ @pytest.mark.network + def test_get_signed_policy_v4_virtual_hosted_style(self): + import datetime + +@@ -1917,6 +1929,7 @@ class TestClient(unittest.TestCase): + policy["url"], "https://{}.storage.googleapis.com/".format(BUCKET_NAME) + ) + ++ @pytest.mark.network + def test_get_signed_policy_v4_bucket_bound_hostname(self): + import datetime + +@@ -1933,6 +1946,7 @@ class TestClient(unittest.TestCase): + ) + self.assertEqual(policy["url"], "https://bucket.bound_hostname") + ++ @pytest.mark.network + def test_get_signed_policy_v4_bucket_bound_hostname_with_scheme(self): + import datetime + +@@ -1950,6 +1964,7 @@ class TestClient(unittest.TestCase): + ) + self.assertEqual(policy["url"], "http://bucket.bound_hostname/") + ++ @pytest.mark.network + def test_get_signed_policy_v4_no_expiration(self): + BUCKET_NAME = "bucket-name" + EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" +@@ -1970,6 +1985,7 @@ class TestClient(unittest.TestCase): + ) + self.assertEqual(policy["fields"]["policy"], EXPECTED_POLICY) + ++ @pytest.mark.network + def test_get_signed_policy_v4_with_access_token(self): + import datetime + diff --git a/python-google-cloud-storage.changes b/python-google-cloud-storage.changes index c48ec55..bfa7719 100644 --- a/python-google-cloud-storage.changes +++ b/python-google-cloud-storage.changes @@ -1,11 +1,3 @@ -------------------------------------------------------------------- -Wed Jun 9 15:19:45 UTC 2021 - Matej Cepl - -- Remove no-network.patch and replace it - with the upstream attempt for solution - 416-avoid_real_client_in_bucket_blob_unit_tests.patch - (gh#googleapis/python-storage#457). - ------------------------------------------------------------------- Fri Jun 4 07:09:47 UTC 2021 - Matej Cepl diff --git a/python-google-cloud-storage.spec b/python-google-cloud-storage.spec index 7b01f97..5b36f72 100644 --- a/python-google-cloud-storage.spec +++ b/python-google-cloud-storage.spec @@ -29,7 +29,7 @@ Source: https://files.pythonhosted.org/packages/source/g/google-cloud-st Patch0: no-sic.patch # PATCH-FIX-UPSTREAM no-network.patch gh#googleapis/python-storage#457 mcepl@suse.com # mark tests as requiring network -Patch1: 416-avoid_real_client_in_bucket_blob_unit_tests.patch +Patch1: no-network.patch BuildRequires: %{python_module google-auth >= 1.11.0} BuildRequires: %{python_module google-cloud-core >= 1.4.1} BuildRequires: %{python_module google-resumable-media >= 1.2.0}