Merge branch 'master' into pr-http-py3

Author: chmeyers

.build-bot.json

@@ -0,0 +1,5 @@
+# .coveragerc to control coverage.py
+[run]
+
+[report]
+include = */site-packages/google/datalab/*

.coveragerc

@@ -2,9 +2,17 @@
 *.pyi
 *.map
 *.egg-info
+*.iml
 .idea
+.DS_Store
 MANIFEST
 build
 .coverage
 dist
 datalab.magics.rst
+datalab/notebook/static/*.js
+google/datalab/notebook/static/*.js
+
+# Test files
+.tox/
+.cache/

.gitignore

@@ -1,32 +1,25 @@
-sudo: required
-
-dist: trusty
-
 language: python
-
-python:
-  - 2.7
-  - 3.5
+dist: trusty
+sudo: false
+
+matrix:
+  include:
+  - python: 2.7
+    env: TOX_ENV=py27
+  - python: 3.5
+    env: TOX_ENV=py35
+  - python: 2.7
+    env: TOX_ENV=flake8
+  - python: 2.7
+    env: TOX_ENV=coveralls
 
 before_install:
-  - sudo apt-get install -y python-setuptools
   - npm install -g typescript
   - tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts
-  - pip install -U pip
-  - pip install -U setuptools
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -U google-cloud-dataflow==0.5.5; fi
-  - pip install .
-  - pip install solutionbox/structured_data/
-  - pip install solutionbox/image_classification/  
-  - pip install -U tensorflow==1.0.0
-  - pip install -U protobuf==3.1.0  #v3.2.0 has the "python: double free or corruption" bug.
-  - pip install flake8
-
-before_script:
-  - flake8 .
-
+  # We use tox for actually running tests.
+  - pip install --upgrade pip tox
+  
 script:
-  - python ./tests/main.py
-  - python ./legacy_tests/main.py
-
+  # tox reads its configuration from tox.ini.
+  - tox -e $TOX_ENV
 

.travis.yml

@@ -20,6 +20,22 @@ frustration later on.
 All submissions, including submissions by project members, require review. We
 use Github pull requests for this purpose.
 
+### Running tests
+We use [`tox`](https://tox.readthedocs.io/) for running our tests. To run tests
+before sending out a pull request, just
+[install tox](https://http://tox.readthedocs.io/en/latest/install.html) and run
+
+```shell
+$ tox
+```
+
+to run tests under all supported environments. (This will skip any environments
+for which no interpreter is available.) `tox -l` will provide a list of all
+supported environments.
+
+`tox` will run all tests referenced by `tests/main.py` and
+`legacy_tests/main.py`.
+
 ### The small print
 Contributions made by corporations are covered by a different agreement than
 the one above, the

CONTRIBUTING.md

@@ -1,64 +1,80 @@
-# datalab
+# datalab [![Build Status](https://travis-ci.org/googledatalab/pydatalab.svg?branch=master)](https://travis-ci.org/googledatalab/pydatalab) [![PyPI Package](https://img.shields.io/pypi/v/datalab.svg)](https://pypi.python.org/pypi/datalab)
 
-Google datalab Python package. Used in [Google Cloud Datalab]
-(https://github.com/GoogleCloudPlatform/datalab) and can be used
-in [Jupyter Notebook](http://jupyter.org/).
+[Google Cloud Datalab](https://cloud.google.com/datalab/) Python package. Used in
+[Google Cloud Datalab](https://github.com/GoogleCloudPlatform/datalab) and can
+be used in [Jupyter Notebook](http://jupyter.org/).
 
-This adds a number of Python modules such as datalab.bigquery, 
-datalab.storage, etc, for accessing [Google Cloud Platform services]
-(https://cloud.google.com/) as well as adding some new cell magics such as `%chart`,
-`%bigquery`, `%storage`, etc.
+This adds a number of Python modules such as `google.datalab.bigquery`,
+`google.datalab.storage`, etc, for accessing
+[Google Cloud Platform services](https://cloud.google.com/) as well as adding
+some new cell magics such as `%chart`, `%bigquery`, `%storage`, etc.
 
-See [https://github.com/googledatalab/notebooks](https://github.com/googledatalab/notebooks) 
+See
+[https://github.com/googledatalab/notebooks](https://github.com/googledatalab/notebooks)
 for samples of using this package.
 
-## Prerequisites
-
-You will need the [Typscript compiler](https://www.typescriptlang.org/) installed.
-In future we should be installable from PyPI.
-
 ## Installation
 
-First:
+This package is available on PyPI as `datalab`:
 
-    git clone https://github.com/googledatalab/pydatalab.git
-    cd pydatalab
+    pip install datalab
 
-Then do one of the folowing:
+## Using in Jupyter
 
-    ./install-virtualenv.sh  # For use in Python virtual environments
-    ./install-no-virtualenv.sh  # For installing in a non-virtual environment
+After datalab installation, enable datalab's frontend in Jupyter by running:
 
-You can ignore the message about running `jupyter nbextension enable`; it is not required.
+    jupyter nbextension install --py datalab.notebook --sys-prefix
 
-## Using in Jupyter
+See further details [Jupyter Kernel and Notebook Extensions](https://github.com/googledatalab/pydatalab/wiki/Jupyter-Kernel-and-Notebook-Extensions).
 
-In a notebook cell, enable with:
+Then in a notebook cell, enable datalab's magics with:
 
-    %load_ext datalab.kernel
+    %load_ext google.datalab.kernel
+    
+(Note: If you hit an error "module traceback cannot be imported", try setting the following environment variable: CLOUDSDK_PYTHON_SITEPACKAGES=1)
 
 Alternatively add this to your `ipython_config.py` file in your profile:
 
     c = get_config()
     c.InteractiveShellApp.extensions = [
-        'datalab.kernel'
+        'google.datalab.kernel'
     ]
 
-You will typically put this under `~/.ipython/profile_default`. 
-See [http://ipython.readthedocs.io/en/stable/development/config.html]
-(http://ipython.readthedocs.io/en/stable/development/config.html)
+You will typically put this under `~/.ipython/profile_default`. See
+[the IPython docs](http://ipython.readthedocs.io/en/stable/development/config.html)
 for more about IPython profiles.
 
-If you want to access Google Cloud Platform services such as BigQuery,
-you will also need to install [gcloud]
-(https://cloud.google.com/sdk/gcloud). You will need to use gcloud
-to authenticate; e.g. with:
+If you want to access Google Cloud Platform services such as BigQuery, you
+will also need to install [gcloud](https://cloud.google.com/sdk/gcloud). You
+will need to use `gcloud` to authenticate; e.g. with:
 
     gcloud auth login
 
 You will also need to set the project ID to use; either set a `PROJECT_ID`
-environment variable to the project name, or call `set_datalab_project_id(name)`
-from within your notebook.
+environment variable to the project name, or call
+`set_datalab_project_id(name)` from within your notebook.
 
 ## Documentation
-You can read the Sphinx generated docs at: [http://googledatalab.github.io/pydatalab/](http://googledatalab.github.io/pydatalab/)
+
+You can read the Sphinx generated docs at:
+[http://googledatalab.github.io/pydatalab/](http://googledatalab.github.io/pydatalab/)
+
+## Development installation
+
+If you'd like to work on the package, it's useful to be able to install from
+source. You will need the
+[Typescript compiler](https://www.typescriptlang.org/) installed.
+
+First:
+
+    git clone https://github.com/googledatalab/pydatalab.git
+    cd pydatalab
+
+Then do one of the folowing:
+
+    ./install-virtualenv.sh  # For use in Python virtual environments
+    ./install-no-virtualenv.sh  # For installing in a non-virtual environment
+
+You can ignore the message about running `jupyter nbextension enable`; it is
+not required.
+

README.md

@@ -16,11 +16,16 @@
 
 from __future__ import absolute_import
 from __future__ import unicode_literals
-import oauth2client.client
 import json
 import os
 import subprocess
 
+import oauth2client.client
+import google.auth
+import google.auth.exceptions
+import google.auth.credentials
+import google.auth._oauth2client
+
 
 # TODO(ojarjur): This limits the APIs against which Datalab can be called
 # (when using a service account with a credentials file) to only being those
@@ -51,6 +56,19 @@ def get_config_dir():
   return config_dir
 
 
+def _convert_oauth2client_creds(credentials):
+  new_credentials = google.oauth2.credentials.Credentials(
+    token=credentials.access_token,
+    refresh_token=credentials.refresh_token,
+    token_uri=credentials.token_uri,
+    client_id=credentials.client_id,
+    client_secret=credentials.client_secret,
+    scopes=credentials.scopes)
+
+  new_credentials._expires = credentials.token_expiry
+  return new_credentials
+
+
 def get_credentials():
   """ Get the credentials to use. We try application credentials first, followed by
       user credentials. The path to the application credentials can be overridden
@@ -61,9 +79,8 @@ def get_credentials():
       overriding these the defaults should suffice.
   """
   try:
-    credentials = oauth2client.client.GoogleCredentials.get_application_default()
-    if credentials.create_scoped_required():
-      credentials = credentials.create_scoped(CREDENTIAL_SCOPES)
+    credentials, _ = google.auth.default()
+    credentials = google.auth.credentials.with_scopes_if_required(credentials, CREDENTIAL_SCOPES)
     return credentials
   except Exception as e:
 
@@ -75,9 +92,10 @@ def get_credentials():
       # Use the first gcloud one we find
       for entry in creds['data']:
         if entry['key']['type'] == 'google-cloud-sdk':
-          return oauth2client.client.OAuth2Credentials.from_json(json.dumps(entry['credential']))
+          creds = oauth2client.client.OAuth2Credentials.from_json(json.dumps(entry['credential']))
+          return _convert_oauth2client_creds(creds)
 
-    if type(e) == oauth2client.client.ApplicationDefaultCredentialsError:
+    if type(e) == google.auth.exceptions.DefaultCredentialsError:
       # If we are in Datalab container, change the message to be about signing in.
       if _in_datalab_docker():
         raise Exception('No application credentials found. Perhaps you should sign in.')

datalab/context/_utils.py

@@ -26,5 +26,5 @@ def make_client(project_id=None, context=None):
       project=project_id,
       credentials=context.credentials,
   )
-  client._connection_class.USER_AGENT = 'pydatalab/v0'
+  client._connection.USER_AGENT = 'pydatalab/v0'
   return client

datalab/stackdriver/monitoring/_utils.py

@@ -26,6 +26,7 @@
 import urllib.parse
 import urllib.error
 import httplib2
+import google_auth_httplib2
 import logging
 
 
@@ -132,7 +133,7 @@ def request(url, args=None, data=None, headers=None, method=None,
     if credentials is not None:
       # Make a copy of the shared http instance before we modify it.
       http = copy.copy(http)
-      http = credentials.authorize(http)
+      http = google_auth_httplib2.AuthorizedHttp(credentials)
     if stats is not None:
       stats['duration'] = datetime.datetime.utcnow()
 

datalab/utils/_http.py

@@ -36,7 +36,7 @@ def __init__(self, project_id, credentials, config=None):
     """
     self._project_id = project_id
     self._credentials = credentials
-    self._config = config or Context._get_default_config()
+    self._config = config if config is not None else Context._get_default_config()
 
   @property
   def credentials(self):