From ba24b41dcefbf3e0517164beb03bacdd8e9e7235 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 28 Feb 2020 19:10:38 +0000 Subject: [PATCH 1/5] feat: add client_options support for api endpoint override --- googleapiclient/discovery.py | 19 ++++++++++++- setup.py | 1 + tests/test_discovery.py | 52 ++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/googleapiclient/discovery.py b/googleapiclient/discovery.py index 87403b9e80d..3158fb3eb9c 100644 --- a/googleapiclient/discovery.py +++ b/googleapiclient/discovery.py @@ -46,6 +46,7 @@ # Third-party imports import httplib2 import uritemplate +import google.api_core.client_options # Local imports from googleapiclient import _auth @@ -176,6 +177,7 @@ def build( credentials=None, cache_discovery=True, cache=None, + client_options=None, ): """Construct a Resource for interacting with an API. @@ -202,6 +204,8 @@ def build( cache_discovery: Boolean, whether or not to cache the discovery doc. cache: googleapiclient.discovery_cache.base.CacheBase, an optional cache object for the discovery documents. + client_options: Dictionary or google.api_core.client_options, Client options to set user + options on the client. API endpoint should be set through client_options. Returns: A Resource object with methods for interacting with the service. @@ -228,6 +232,7 @@ def build( model=model, requestBuilder=requestBuilder, credentials=credentials, + client_options=client_options ) except HttpError as e: if e.resp.status == http_client.NOT_FOUND: @@ -304,6 +309,7 @@ def build_from_document( model=None, requestBuilder=HttpRequest, credentials=None, + client_options=None ): """Create a Resource for interacting with an API. @@ -328,6 +334,8 @@ def build_from_document( credentials: oauth2client.Credentials or google.auth.credentials.Credentials, credentials to be used for authentication. + client_options: Dictionary or google.api_core.client_options, Client options to set user + options on the client. API endpoint should be set through client_options. Returns: A Resource object with methods for interacting with the service. @@ -350,7 +358,16 @@ def build_from_document( ) raise InvalidJsonError() - base = urljoin(service["rootUrl"], service["servicePath"]) + # If an API Endpoint is provided on client options, use that as the base URL + base = urljoin(service['rootUrl'], service["servicePath"]) + if client_options: + if type(client_options) == dict: + client_options = google.api_core.client_options.from_dict( + client_options + ) + if client_options.api_endpoint: + base = client_options.api_endpoint + schema = Schemas(service) # If the http client is not specified, then we must construct an http client diff --git a/setup.py b/setup.py index 643df2ed7cc..011384d9cc4 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ "httplib2>=0.9.2,<1dev", "google-auth>=1.4.1", "google-auth-httplib2>=0.0.3", + "google-api-core>=1.13.0,<2dev", "six>=1.6.1,<2dev", "uritemplate>=3.0.0,<4dev", ] diff --git a/tests/test_discovery.py b/tests/test_discovery.py index f85035ef424..08234e5399c 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -515,6 +515,25 @@ def test_building_with_developer_key_skips_adc(self): # application default credentials were used. self.assertNotIsInstance(plus._http, google_auth_httplib2.AuthorizedHttp) + def test_api_endpoint_override_from_client_options(self): + discovery = open(datafile("plus.json")).read() + api_endpoint = "https://foo.googleapis.com/" + options = google.api_core.client_options.ClientOptions( + api_endpoint=api_endpoint + ) + plus = build_from_document(discovery, client_options=options) + + self.assertEquals(plus._baseUrl, api_endpoint) + + def test_api_endpoint_override_from_client_options(self): + discovery = open(datafile("plus.json")).read() + api_endpoint = "https://foo.googleapis.com/" + plus = build_from_document( + discovery, client_options={"api_endpoint": api_endpoint} + ) + + self.assertEquals(plus._baseUrl, api_endpoint) + class DiscoveryFromHttp(unittest.TestCase): def setUp(self): @@ -588,6 +607,39 @@ def test_discovery_loading_from_v2_discovery_uri(self): zoo = build("zoo", "v1", http=http, cache_discovery=False) self.assertTrue(hasattr(zoo, "animals")) + def test_api_endpoint_override_from_client_options(self): + http = HttpMockSequence( + [ + ({"status": "404"}, "Not found"), + ({"status": "200"}, open(datafile("zoo.json"), "rb").read()), + ] + ) + api_endpoint = "https://foo.googleapis.com/" + options = google.api_core.client_options.ClientOptions( + api_endpoint=api_endpoint + ) + zoo = build( + "zoo", "v1", http=http, cache_discovery=False, client_options=options + ) + self.assertEquals(zoo._baseUrl, api_endpoint) + + def test_api_endpoint_override_from_client_options_dict(self): + http = HttpMockSequence( + [ + ({"status": "404"}, "Not found"), + ({"status": "200"}, open(datafile("zoo.json"), "rb").read()), + ] + ) + api_endpoint = "https://foo.googleapis.com/" + zoo = build( + "zoo", + "v1", + http=http, + cache_discovery=False, + client_options={"api_endpoint": api_endpoint}, + ) + self.assertEquals(zoo._baseUrl, api_endpoint) + class DiscoveryFromAppEngineCache(unittest.TestCase): def test_appengine_memcache(self): From 3ba1df2c26fef86b2310af45a26ac103803ac507 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 28 Feb 2020 19:14:49 +0000 Subject: [PATCH 2/5] chore: blacken --- tests/test_discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 08234e5399c..0a0388a60dc 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -525,7 +525,7 @@ def test_api_endpoint_override_from_client_options(self): self.assertEquals(plus._baseUrl, api_endpoint) - def test_api_endpoint_override_from_client_options(self): + def test_api_endpoint_override_from_client_options_dict(self): discovery = open(datafile("plus.json")).read() api_endpoint = "https://foo.googleapis.com/" plus = build_from_document( From cea71d1fb4934b64c9bb2650376321b85359e03d Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 28 Feb 2020 21:29:09 +0000 Subject: [PATCH 3/5] chore: move kokoro configs to expected path --- .kokoro/{ => continuous}/continuous.cfg | 0 .kokoro/{ => presubmit}/presubmit.cfg | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .kokoro/{ => continuous}/continuous.cfg (100%) rename .kokoro/{ => presubmit}/presubmit.cfg (100%) diff --git a/.kokoro/continuous.cfg b/.kokoro/continuous/continuous.cfg similarity index 100% rename from .kokoro/continuous.cfg rename to .kokoro/continuous/continuous.cfg diff --git a/.kokoro/presubmit.cfg b/.kokoro/presubmit/presubmit.cfg similarity index 100% rename from .kokoro/presubmit.cfg rename to .kokoro/presubmit/presubmit.cfg From 25a38934143ba6ff0563ffc4e13a2f854e3e0913 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 28 Feb 2020 22:13:33 +0000 Subject: [PATCH 4/5] chore: add more kokoro configs --- .kokoro/continuous/common.cfg | 27 +++++++++++++++++++++++++++ .kokoro/presubmit/common.cfg | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 .kokoro/continuous/common.cfg create mode 100644 .kokoro/presubmit/common.cfg diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg new file mode 100644 index 00000000000..613998c4451 --- /dev/null +++ b/.kokoro/continuous/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "google-api-python-client/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-api-python-client/.kokoro/build.sh" +} diff --git a/.kokoro/presubmit/common.cfg b/.kokoro/presubmit/common.cfg new file mode 100644 index 00000000000..613998c4451 --- /dev/null +++ b/.kokoro/presubmit/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "google-api-python-client/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-api-python-client/.kokoro/build.sh" +} From f726a1152296f0ffebaeb9ad8972381c12ea0d7d Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 11 Mar 2020 21:12:16 +0000 Subject: [PATCH 5/5] chore: replace all `assertEquals` with `assertEqual` --- tests/test_discovery.py | 100 ++++++++++++++++++++-------------------- tests/test_http.py | 4 +- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 0a0388a60dc..6400f2147e1 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -466,7 +466,7 @@ def test_building_with_base_remembers_base(self): plus = build_from_document( discovery, base=base, credentials=self.MOCK_CREDENTIALS ) - self.assertEquals("https://www.googleapis.com/plus/v1/", plus._baseUrl) + self.assertEqual("https://www.googleapis.com/plus/v1/", plus._baseUrl) def test_building_with_optional_http_with_authorization(self): discovery = open(datafile("plus.json")).read() @@ -503,7 +503,7 @@ def test_building_with_explicit_http(self): plus = build_from_document( discovery, base="https://www.googleapis.com/", http=http ) - self.assertEquals(plus._http, http) + self.assertEqual(plus._http, http) def test_building_with_developer_key_skips_adc(self): discovery = open(datafile("plus.json")).read() @@ -523,7 +523,7 @@ def test_api_endpoint_override_from_client_options(self): ) plus = build_from_document(discovery, client_options=options) - self.assertEquals(plus._baseUrl, api_endpoint) + self.assertEqual(plus._baseUrl, api_endpoint) def test_api_endpoint_override_from_client_options_dict(self): discovery = open(datafile("plus.json")).read() @@ -532,7 +532,7 @@ def test_api_endpoint_override_from_client_options_dict(self): discovery, client_options={"api_endpoint": api_endpoint} ) - self.assertEquals(plus._baseUrl, api_endpoint) + self.assertEqual(plus._baseUrl, api_endpoint) class DiscoveryFromHttp(unittest.TestCase): @@ -621,7 +621,7 @@ def test_api_endpoint_override_from_client_options(self): zoo = build( "zoo", "v1", http=http, cache_discovery=False, client_options=options ) - self.assertEquals(zoo._baseUrl, api_endpoint) + self.assertEqual(zoo._baseUrl, api_endpoint) def test_api_endpoint_override_from_client_options_dict(self): http = HttpMockSequence( @@ -638,7 +638,7 @@ def test_api_endpoint_override_from_client_options_dict(self): cache_discovery=False, client_options={"api_endpoint": api_endpoint}, ) - self.assertEquals(zoo._baseUrl, api_endpoint) + self.assertEqual(zoo._baseUrl, api_endpoint) class DiscoveryFromAppEngineCache(unittest.TestCase): @@ -980,8 +980,8 @@ def test_simple_media_upload_no_max_size_provided(self): self.http = HttpMock(datafile("zoo.json"), {"status": "200"}) zoo = build("zoo", "v1", http=self.http) request = zoo.animals().crossbreed(media_body=datafile("small.png")) - self.assertEquals("image/png", request.headers["content-type"]) - self.assertEquals(b"PNG", request.body[1:4]) + self.assertEqual("image/png", request.headers["content-type"]) + self.assertEqual(b"PNG", request.body[1:4]) def test_simple_media_raise_correct_exceptions(self): self.http = HttpMock(datafile("zoo.json"), {"status": "200"}) @@ -1004,8 +1004,8 @@ def test_simple_media_good_upload(self): zoo = build("zoo", "v1", http=self.http) request = zoo.animals().insert(media_body=datafile("small.png")) - self.assertEquals("image/png", request.headers["content-type"]) - self.assertEquals(b"PNG", request.body[1:4]) + self.assertEqual("image/png", request.headers["content-type"]) + self.assertEqual(b"PNG", request.body[1:4]) assertUrisEqual( self, "https://www.googleapis.com/upload/zoo/v1/animals?uploadType=media&alt=json", @@ -1025,8 +1025,8 @@ def test_simple_media_unknown_mimetype(self): request = zoo.animals().insert( media_body=datafile("small-png"), media_mime_type="image/png" ) - self.assertEquals("image/png", request.headers["content-type"]) - self.assertEquals(b"PNG", request.body[1:4]) + self.assertEqual("image/png", request.headers["content-type"]) + self.assertEqual(b"PNG", request.body[1:4]) assertUrisEqual( self, "https://www.googleapis.com/upload/zoo/v1/animals?uploadType=media&alt=json", @@ -1097,13 +1097,13 @@ def test_resumable_multipart_media_good_upload(self): media_upload = MediaFileUpload(datafile("small.png"), resumable=True) request = zoo.animals().insert(media_body=media_upload, body={}) self.assertTrue(request.headers["content-type"].startswith("application/json")) - self.assertEquals('{"data": {}}', request.body) - self.assertEquals(media_upload, request.resumable) + self.assertEqual('{"data": {}}', request.body) + self.assertEqual(media_upload, request.resumable) - self.assertEquals("image/png", request.resumable.mimetype()) + self.assertEqual("image/png", request.resumable.mimetype()) self.assertNotEquals(request.body, None) - self.assertEquals(request.resumable_uri, None) + self.assertEqual(request.resumable_uri, None) http = HttpMockSequence( [ @@ -1130,32 +1130,32 @@ def test_resumable_multipart_media_good_upload(self): ) status, body = request.next_chunk(http=http) - self.assertEquals(None, body) + self.assertEqual(None, body) self.assertTrue(isinstance(status, MediaUploadProgress)) - self.assertEquals(0, status.resumable_progress) + self.assertEqual(0, status.resumable_progress) # Two requests should have been made and the resumable_uri should have been # updated for each one. - self.assertEquals(request.resumable_uri, "http://upload.example.com/2") - self.assertEquals(media_upload, request.resumable) - self.assertEquals(0, request.resumable_progress) + self.assertEqual(request.resumable_uri, "http://upload.example.com/2") + self.assertEqual(media_upload, request.resumable) + self.assertEqual(0, request.resumable_progress) # This next chuck call should upload the first chunk status, body = request.next_chunk(http=http) - self.assertEquals(request.resumable_uri, "http://upload.example.com/3") - self.assertEquals(media_upload, request.resumable) - self.assertEquals(13, request.resumable_progress) + self.assertEqual(request.resumable_uri, "http://upload.example.com/3") + self.assertEqual(media_upload, request.resumable) + self.assertEqual(13, request.resumable_progress) # This call will upload the next chunk status, body = request.next_chunk(http=http) - self.assertEquals(request.resumable_uri, "http://upload.example.com/4") - self.assertEquals(media_upload.size() - 1, request.resumable_progress) - self.assertEquals('{"data": {}}', request.body) + self.assertEqual(request.resumable_uri, "http://upload.example.com/4") + self.assertEqual(media_upload.size() - 1, request.resumable_progress) + self.assertEqual('{"data": {}}', request.body) # Final call to next_chunk should complete the upload. status, body = request.next_chunk(http=http) - self.assertEquals(body, {"foo": "bar"}) - self.assertEquals(status, None) + self.assertEqual(body, {"foo": "bar"}) + self.assertEqual(status, None) def test_resumable_media_good_upload(self): """Not a multipart upload.""" @@ -1164,12 +1164,12 @@ def test_resumable_media_good_upload(self): media_upload = MediaFileUpload(datafile("small.png"), resumable=True) request = zoo.animals().insert(media_body=media_upload, body=None) - self.assertEquals(media_upload, request.resumable) + self.assertEqual(media_upload, request.resumable) - self.assertEquals("image/png", request.resumable.mimetype()) + self.assertEqual("image/png", request.resumable.mimetype()) - self.assertEquals(request.body, None) - self.assertEquals(request.resumable_uri, None) + self.assertEqual(request.body, None) + self.assertEqual(request.resumable_uri, None) http = HttpMockSequence( [ @@ -1195,26 +1195,26 @@ def test_resumable_media_good_upload(self): ) status, body = request.next_chunk(http=http) - self.assertEquals(None, body) + self.assertEqual(None, body) self.assertTrue(isinstance(status, MediaUploadProgress)) - self.assertEquals(13, status.resumable_progress) + self.assertEqual(13, status.resumable_progress) # Two requests should have been made and the resumable_uri should have been # updated for each one. - self.assertEquals(request.resumable_uri, "http://upload.example.com/2") + self.assertEqual(request.resumable_uri, "http://upload.example.com/2") - self.assertEquals(media_upload, request.resumable) - self.assertEquals(13, request.resumable_progress) + self.assertEqual(media_upload, request.resumable) + self.assertEqual(13, request.resumable_progress) status, body = request.next_chunk(http=http) - self.assertEquals(request.resumable_uri, "http://upload.example.com/3") - self.assertEquals(media_upload.size() - 1, request.resumable_progress) - self.assertEquals(request.body, None) + self.assertEqual(request.resumable_uri, "http://upload.example.com/3") + self.assertEqual(media_upload.size() - 1, request.resumable_progress) + self.assertEqual(request.body, None) # Final call to next_chunk should complete the upload. status, body = request.next_chunk(http=http) - self.assertEquals(body, {"foo": "bar"}) - self.assertEquals(status, None) + self.assertEqual(body, {"foo": "bar"}) + self.assertEqual(status, None) def test_resumable_media_good_upload_from_execute(self): """Not a multipart upload.""" @@ -1253,7 +1253,7 @@ def test_resumable_media_good_upload_from_execute(self): ) body = request.execute(http=http) - self.assertEquals(body, {"foo": "bar"}) + self.assertEqual(body, {"foo": "bar"}) def test_resumable_media_fail_unknown_response_code_first_request(self): """Not a multipart upload.""" @@ -1299,7 +1299,7 @@ def test_resumable_media_fail_unknown_response_code_subsequent_request(self): ) status, body = request.next_chunk(http=http) - self.assertEquals( + self.assertEqual( status.resumable_progress, 7, "Should have first checked length and then tried to PUT more.", @@ -1623,9 +1623,9 @@ def test_resumable_media_upload_no_content(self): media_upload = MediaFileUpload(datafile("empty"), resumable=True) request = zoo.animals().insert(media_body=media_upload, body=None) - self.assertEquals(media_upload, request.resumable) - self.assertEquals(request.body, None) - self.assertEquals(request.resumable_uri, None) + self.assertEqual(media_upload, request.resumable) + self.assertEqual(request.body, None) + self.assertEqual(request.resumable_uri, None) http = HttpMockSequence( [ @@ -1642,9 +1642,9 @@ def test_resumable_media_upload_no_content(self): ) status, body = request.next_chunk(http=http) - self.assertEquals(None, body) + self.assertEqual(None, body) self.assertTrue(isinstance(status, MediaUploadProgress)) - self.assertEquals(0, status.progress()) + self.assertEqual(0, status.progress()) class Next(unittest.TestCase): diff --git a/tests/test_http.py b/tests/test_http.py index 1b0caa5b707..f1f260d520a 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1122,7 +1122,7 @@ def setUp(self): def test_id_to_from_content_id_header(self): batch = BatchHttpRequest() - self.assertEquals("12", batch._header_to_id(batch._id_to_header("12"))) + self.assertEqual("12", batch._header_to_id(batch._id_to_header("12"))) def test_invalid_content_id_header(self): batch = BatchHttpRequest() @@ -1646,7 +1646,7 @@ def test_build_http_default_timeout_can_be_overridden(self): def test_build_http_default_timeout_can_be_set_to_zero(self): socket.setdefaulttimeout(0) http = build_http() - self.assertEquals(http.timeout, 0) + self.assertEqual(http.timeout, 0) if __name__ == "__main__":