adding owlbot and notebook ci

.cloud-build/CheckPythonVersion.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# # Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+MINIMUM_MAJOR_VERSION = 3
+MINIMUM_MINOR_VERSION = 5
+
+if (
+    sys.version_info.major >= MINIMUM_MAJOR_VERSION
+    or sys.version_info.minor >= MINIMUM_MINOR_VERSION
+):
+    print(f"Python version acceptable: {sys.version}")
+    exit(0)
+else:
+    print(
+        f"Error: Python version less than {MINIMUM_MAJOR_VERSION}.{MINIMUM_MINOR_VERSION}"
+    )
+    exit(1)

.cloud-build/cleanup/cleanup-cloudbuild.yaml

@@ -0,0 +1,8 @@
+steps:  
+  # Install Python dependencies and run cleanup script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 -m pip install -U -r .cloud-build/cleanup/cleanup-requirements.txt && python3 .cloud-build/cleanup/cleanup.py'
+timeout: 86400s

.cloud-build/cleanup/cleanup.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# # Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+from resource_cleanup_manager import (
+    ResourceCleanupManager,
+    DatasetResourceCleanupManager,
+    EndpointResourceCleanupManager,
+    ModelResourceCleanupManager,
+)
+
+
+def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
+    for manager in managers:
+        type_name = manager.type_name
+
+        print(f"Fetching {type_name}'s...")
+        resources = manager.list()
+        print(f"Found {len(resources)} {type_name}'s")
+        for resource in resources:
+            if not manager.is_deletable(resource):
+                continue
+
+            if is_dry_run:
+                resource_name = manager.resource_name(resource)
+                print(f"Will delete '{type_name}': {resource_name}")
+            else:
+                try:
+                    manager.delete(resource)
+                except Exception as exception:
+                    print(exception)
+
+
+is_dry_run = False
+
+# List of all cleanup managers
+managers = [
+    DatasetResourceCleanupManager(),
+    EndpointResourceCleanupManager(),
+    ModelResourceCleanupManager(),
+]
+
+run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/requirements.txt

@@ -0,0 +1 @@
+google-cloud-aiplatform==1.12.1

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# # Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+from google.cloud import aiplatform
+from typing import Any
+from proto.datetime_helpers import DatetimeWithNanoseconds
+from google.cloud.aiplatform import base
+
+# If a resource was updated within this number of seconds, do not delete.
+RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
+
+
+class ResourceCleanupManager(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def type_name(str) -> str:
+        pass
+
+    @abc.abstractmethod
+    def list(self) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def resource_name(self, resource: Any) -> str:
+        pass
+
+    @abc.abstractmethod
+    def delete(self, resource: Any):
+        pass
+
+    @abc.abstractmethod
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        pass
+
+    def is_deletable(self, resource: Any) -> bool:
+        time_difference = self.get_seconds_since_modification(resource)
+
+        if self.resource_name(resource).startswith("perm"):
+            print(f"Skipping '{resource}' due to name starting with 'perm'.")
+            return False
+
+        # Check that it wasn't created too recently, to prevent race conditions
+        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
+            print(
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+            )
+            return False
+
+        return True
+
+
+class VertexAIResourceCleanupManager(ResourceCleanupManager):
+    @property
+    @abc.abstractmethod
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
+        pass
+
+    @property
+    def type_name(self) -> str:
+        return self.vertex_ai_resource._resource_noun
+
+    def list(self) -> Any:
+        return self.vertex_ai_resource.list()
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.display_name
+
+    def delete(self, resource):
+        resource.delete()
+
+    def get_seconds_since_modification(self, resource: Any) -> bool:
+        update_time = resource.update_time
+        current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
+        return (current_time - update_time).total_seconds()
+
+
+class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.datasets._Dataset
+
+
+class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Endpoint
+
+    def delete(self, resource):
+        resource.delete(force=True)
+
+
+class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Model

.cloud-build/execute_changed_notebooks_cli.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A CLI to process changed notebooks and execute them on Google Cloud Build"""
+
+import argparse
+import pathlib
+import execute_changed_notebooks_helper
+
+
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("String value expected.")
+
+
+parser = argparse.ArgumentParser(description="Run changed notebooks.")
+parser.add_argument(
+    "--test_paths_file",
+    type=pathlib.Path,
+    help="The path to the file that has newline-limited folders of notebooks that should be tested.",
+    required=True,
+)
+parser.add_argument(
+    "--base_branch",
+    help="The base git branch to diff against to find changed files.",
+    required=False,
+)
+parser.add_argument(
+    "--container_uri",
+    type=str,
+    help="The container uri to run each notebook in.",
+    required=True,
+)
+parser.add_argument(
+    "--variable_project_id",
+    type=str,
+    help="The GCP project id. This is used to inject a variable value into the notebook before running.",
+    required=True,
+)
+parser.add_argument(
+    "--variable_region",
+    type=str,
+    help="The GCP region. This is used to inject a variable value into the notebook before running.",
+    required=True,
+)
+parser.add_argument(
+    "--staging_bucket",
+    type=str,
+    help="The GCS bucket for staging temporary files.",
+    required=True,
+)
+parser.add_argument(
+    "--artifacts_bucket",
+    type=str,
+    help="The GCP directory for storing executed notebooks.",
+    required=True,
+)
+parser.add_argument(
+    "--should_parallelize",
+    type=str2bool,
+    nargs="?",
+    const=True,
+    default=True,
+    help="Should run notebooks in parallel.",
+)
+
+args = parser.parse_args()
+
+notebooks = execute_changed_notebooks_helper.get_changed_notebooks(
+    test_paths_file=args.test_paths_file,
+    base_branch=args.base_branch,
+)
+
+execute_changed_notebooks_helper.process_and_execute_notebooks(
+    notebooks=notebooks,
+    container_uri=args.container_uri,
+    staging_bucket=args.staging_bucket,
+    artifacts_bucket=args.artifacts_bucket,
+    variable_project_id=args.variable_project_id,
+    variable_region=args.variable_region,
+    should_parallelize=args.should_parallelize,
+)