From f50fa53512ff376566594796b931366afa41ed82 Mon Sep 17 00:00:00 2001 From: Chris Heald Date: Mon, 5 Jun 2017 16:09:37 -0700 Subject: [PATCH 1/2] Add a thread_safe flag to get_client, which will cause the client to create a new HTTP object per request. This is much slower since it has to authorize credentials for each request, but it is threadsafe. --- bigquery/client.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/bigquery/client.py b/bigquery/client.py index b8971cd..ae9d3c4 100644 --- a/bigquery/client.py +++ b/bigquery/client.py @@ -1,5 +1,6 @@ import calendar import json +import apiclient from logging import getLogger, NullHandler from collections import defaultdict from datetime import datetime, timedelta @@ -55,7 +56,8 @@ def get_client(project_id=None, credentials=None, service_url=None, service_account=None, private_key=None, private_key_file=None, json_key=None, json_key_file=None, - readonly=True, swallow_results=True): + readonly=True, swallow_results=True, + thread_safe=False): """Return a singleton instance of BigQueryClient. Either AssertionCredentials or a service account and private key combination need to be provided in order to authenticate requests to BigQuery. @@ -94,6 +96,10 @@ def get_client(project_id=None, credentials=None, swallow_results : bool If set to False, then return the actual response value instead of converting to boolean. Default True. + thread_safe : bool + If set to True, create a new HTTP client per request to work around + httplib2's lack of thread safety. This is less performant, so don't + set it unless needed. Default False. Returns ------- @@ -145,7 +151,8 @@ def get_client(project_id=None, credentials=None, project_id = json_key['project_id'] bq_service = _get_bq_service(credentials=credentials, - service_url=service_url) + service_url=service_url, + thread_safe=thread_safe) return BigQueryClient(bq_service, project_id, swallow_results) @@ -164,16 +171,21 @@ def get_projects(bq_service): return projects -def _get_bq_service(credentials=None, service_url=None): +def _get_bq_service(credentials=None, service_url=None, thread_safe=False): """Construct an authorized BigQuery service object.""" assert credentials, 'Must provide ServiceAccountCredentials' - - http = credentials.authorize(Http()) - service = build('bigquery', 'v2', http=http, - discoveryServiceUrl=service_url) - - return service + if thread_safe: + # Create a new Http() object for every request + def build_request(http, *args, **kwargs): + http = credentials.authorize(Http()) + return apiclient.http.HttpRequest(http, *args, **kwargs) + return build('bigquery', 'v2', discoveryServiceUrl=service_url, + requestBuilder=build_request, credentials=credentials) + else: + http = credentials.authorize(Http()) + return build('bigquery', 'v2', http=http, + discoveryServiceUrl=service_url) def _credentials(): From c7de85244ab771b98eef57ea2699ab16af541f79 Mon Sep 17 00:00:00 2001 From: Chris Heald Date: Fri, 30 Jun 2017 13:20:48 -0700 Subject: [PATCH 2/2] fork the version to be mash-specific --- bigquery/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/version.py b/bigquery/version.py index 6c371de..86fd9b2 100644 --- a/bigquery/version.py +++ b/bigquery/version.py @@ -1 +1 @@ -__version__ = '1.11.2' +__version__ = '1.11.2-mash'