[GCP/Auth] Use the application key in the env var (#2116)

* [GCP] use the application key in the env var

* rename

* lint

* Add doc and fix the test

* fix test

* format

* longer waiting time for azure

* fix

* Address comments

* format

Author: Michaelvll

docs/source/cloud-setup/cloud-auth.rst

@@ -45,3 +45,25 @@ Example of mixing a default profile and an SSO profile:
 
     $ # A cluster launched under a different profile.
     $ AWS_PROFILE=AdministratorAccess-12345 sky launch --cloud aws -c my-sso-cluster
+
+
+GCP
+-------------------------------
+
+.. _gcp-service-account:
+
+GCP Service Account
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`GCP Service Account <https://cloud.google.com/iam/docs/service-account-overview>`__ is supported.
+
+To use it to access GCP with SkyPilot, you need to setup the credentials:
+
+1. Download the key for the service account from the `GCP console <https://console.cloud.google.com/iam-admin/serviceaccounts>`__.
+2. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the key file, and configure the gcloud CLI tool:
+
+.. code-block:: console
+
+    $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key.json
+    $ gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+    $ gcloud config set project your-project-id

docs/source/getting-started/installation.rst

@@ -93,6 +93,8 @@ Note: if you encounter *Authorization Error (Error 400: invalid_request)* with t
 
   If you are using multiple GCP projects, list all the projects by :code:`gcloud project list` and activate one by :code:`gcloud config set project <PROJECT_ID>` (See `GCP docs <https://cloud.google.com/sdk/gcloud/reference/config/set>`_).
 
+To use service account to access GCP for SkyPilot, see :ref:`here<gcp-service-account>` for instructions.
+
 **Optional**: To create a new GCP user with minimal permissions for SkyPilot, see :ref:`GCP User Creation <cloud-permissions-gcp>`.
 
 Azure

sky/clouds/gcp.py

@@ -22,6 +22,12 @@
 
 logger = sky_logging.init_logger(__name__)
 
+# Env var pointing to any service account key. If it exists, this path takes
+# priority over the DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH below, and will be
+# used instead for SkyPilot-launched instances. This is the same behavior as
+# gcloud:
+# https://cloud.google.com/docs/authentication/provide-credentials-adc#local-key
+_GCP_APPLICATION_CREDENTIAL_ENV = 'GOOGLE_APPLICATION_CREDENTIALS'
 DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH: str = os.path.expanduser(
     '~/.config/gcloud/'
     'application_default_credentials.json')
@@ -44,7 +50,6 @@
 # Minimum set of files under ~/.config/gcloud that grant GCP access.
 _CREDENTIAL_FILES = [
     'credentials.db',
-    'application_default_credentials.json',
     'access_tokens.db',
     'configurations',
     'legacy_credentials',
@@ -470,6 +475,24 @@ def get_vcpus_mem_from_instance_type(
         return service_catalog.get_vcpus_mem_from_instance_type(instance_type,
                                                                 clouds='gcp')
 
+    @classmethod
+    def _find_application_key_path(cls) -> str:
+        # Check the application default credentials in the environment variable.
+        # If the file does not exist, fallback to the default path.
+        application_key_path = os.environ.get(_GCP_APPLICATION_CREDENTIAL_ENV,
+                                              None)
+        if application_key_path is not None:
+            if not os.path.isfile(os.path.expanduser(application_key_path)):
+                raise FileNotFoundError(
+                    f'{_GCP_APPLICATION_CREDENTIAL_ENV}={application_key_path},'
+                    ' but the file does not exist.')
+            return application_key_path
+        if (not os.path.isfile(
+                os.path.expanduser(DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH))):
+            # Fallback to the default application credential path.
+            raise FileNotFoundError(DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH)
+        return DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH
+
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to this cloud."""
@@ -485,10 +508,12 @@ def check_credentials(cls) -> Tuple[bool, Optional[str]]:
             for file in [
                     '~/.config/gcloud/access_tokens.db',
                     '~/.config/gcloud/credentials.db',
-                    '~/.config/gcloud/application_default_credentials.json'
             ]:
                 if not os.path.isfile(os.path.expanduser(file)):
                     raise FileNotFoundError(file)
+
+            cls._find_application_key_path()
+
             # Check the installation of google-cloud-sdk.
             _run_output('gcloud --version')
 
@@ -500,7 +525,7 @@ def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         except (auth.exceptions.DefaultCredentialsError,
                 subprocess.CalledProcessError,
                 exceptions.CloudUserIdentityError, FileNotFoundError,
-                ImportError):
+                ImportError) as e:
             # See also: https://stackoverflow.com/a/53307505/1165051
             return False, (
                 'GCP tools are not installed or credentials are not set. '
@@ -515,6 +540,7 @@ def check_credentials(cls) -> Tuple[bool, Optional[str]]:
                 '  $ gcloud auth application-default login\n    '
                 'For more info: '
                 'https://skypilot.readthedocs.io/en/latest/getting-started/installation.html'  # pylint: disable=line-too-long
+                f'\nDetails: {common_utils.format_exception(e, use_bracket=True)}'
             )
 
         # Check APIs.
@@ -611,6 +637,10 @@ def get_credential_file_mounts(self) -> Dict[str, str]:
             f'~/.config/gcloud/{filename}': f'~/.config/gcloud/{filename}'
             for filename in _CREDENTIAL_FILES
         }
+        # Upload the application key path to the default path, so that
+        # autostop and GCS can be accessed on the remote cluster.
+        credentials[DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH] = (
+            self._find_application_key_path())
         credentials[GCP_CONFIG_SKY_BACKUP_PATH] = GCP_CONFIG_SKY_BACKUP_PATH
         return credentials
 

tests/conftest.py

@@ -189,6 +189,9 @@ def enable_all_clouds(monkeypatch):
         prefix='tmp_backup_config_default', delete=False)
     monkeypatch.setattr('sky.clouds.gcp.GCP_CONFIG_SKY_BACKUP_PATH',
                         config_file_backup.name)
+    monkeypatch.setattr(
+        'sky.clouds.gcp.DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH',
+        config_file_backup.name)
     monkeypatch.setenv('OCI_CONFIG', config_file_backup.name)
 
 

tests/test_optimizer_dryruns.py

@@ -42,6 +42,8 @@ def _test_parse_accelerators(spec, expected_accelerators):
 # clouds are enabled, so we monkeypatch the `sky.global_user_state` module
 # to return all three clouds. We also monkeypatch `sky.check.check` so that
 # when the optimizer tries calling it to update enabled_clouds, it does not
+# TODO: Keep the cloud enabling in sync with the fixture enable_all_clouds
+# in tests/conftest.py
 # raise exceptions.
 def _make_resources(
     monkeypatch,
@@ -61,6 +63,9 @@ def _make_resources(
         prefix='tmp_backup_config_default', delete=False)
     monkeypatch.setattr('sky.clouds.gcp.GCP_CONFIG_SKY_BACKUP_PATH',
                         config_file_backup.name)
+    monkeypatch.setattr(
+        'sky.clouds.gcp.DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH',
+        config_file_backup.name)
     monkeypatch.setenv('OCI_CONFIG', config_file_backup.name)
 
     # Should create Resources here, since it uses the enabled clouds.

tests/test_smoke.py

@@ -1267,8 +1267,8 @@ def test_azure_start_stop():
             f'sky start -y {name} -i 1',
             f'sky exec {name} examples/azure_start_stop.yaml',
             f'sky logs {name} 3 --status',  # Ensure the job succeeded.
-            'sleep 180',
-            f'sky status -r {name} | grep "INIT\|STOPPED"'
+            'sleep 200',
+            f's=$(sky status -r {name}) | echo $s && echo $s | grep "INIT\|STOPPED"'
         ],
         f'sky down -y {name}',
         timeout=30 * 60,  # 30 mins