diff --git a/infra/gcp/lib.sh b/infra/gcp/lib.sh index 4470ea3d92c..4a94138cd77 100755 --- a/infra/gcp/lib.sh +++ b/infra/gcp/lib.sh @@ -16,20 +16,8 @@ # This is a library of functions used to create GCP stuff. -function _color() { - tput setf "$1" || true -} - -function _nocolor() { - tput sgr0 || true -} - -function color() { - _color "$1" - shift - echo "$@" - _nocolor -} +. "$(dirname "${BASH_SOURCE[0]}")/lib_util.sh" +. "$(dirname "${BASH_SOURCE[0]}")/lib_gcs.sh" # The group that admins all GCR repos. GCR_ADMINS="k8s-infra-artifact-admins@kubernetes.io" @@ -185,81 +173,6 @@ function ensure_gcr_repo() { gsutil bucketpolicyonly set on "${bucket}" } -# Ensure the bucket exists -# $1: The GCP project -# $2: The bucket -function ensure_gcs_bucket() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_gcs_bucket(project, bucket) requires 2 arguments" >&2 - return 1 - fi - project="$1" - bucket="$2" - location="us" - - if ! gsutil ls "${bucket}" >/dev/null 2>&1; then - gsutil mb -p "${project}" -l "${location}" "${bucket}" - fi - gsutil bucketpolicyonly set on "${bucket}" -} - -# Ensure the bucket exists and is world-readable -# $1: The GCP project -# $2: The bucket -function ensure_public_gcs_bucket() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_public_gcs_bucket(project, bucket) requires 2 arguments" >&2 - return 1 - fi - project="$1" - bucket="$2" - - ensure_gcs_bucket "${project}" "${bucket}" - gsutil iam ch allUsers:objectViewer "${bucket}" -} - -# Ensure the bucket exists and is NOT world-accessible -# $1: The GCP project -# $2: The bucket -function ensure_private_gcs_bucket() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_private_gcs_bucket(project, bucket) requires 2 arguments" >&2 - return 1 - fi - project="$1" - bucket="$2" - - ensure_gcs_bucket "${project}" "${bucket}" - gsutil iam ch -d allUsers "${bucket}" -} - -# Sets the web policy on the bucket, including a default index.html page -# $1: The bucket -function ensure_gcs_web_policy() { - if [ $# -lt 1 -o -z "$1" ]; then - echo "ensure_gcs_web_policy(bucket) requires 1 argument" >&2 - return 1 - fi - bucket="$1" - - gsutil web set -m index.html "${bucket}" -} - -# Copies any static content into the bucket -# $1: The bucket -# $2: The source directory -function upload_gcs_static_content() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "upload_gcs_static_content(bucket, dir) requires 2 arguments" >&2 - return 1 - fi - bucket="$1" - srcdir="$2" - - # Checksum data to avoid no-op syncs. - gsutil rsync -c "${srcdir}" "${bucket}" -} - # Grant project viewer privileges to a principal # $1: The GCP project # $2: The group email @@ -431,44 +344,6 @@ function empower_group_to_gcr() { empower_group_to_write_gcs_bucket "${group}" "${bucket}" } -# Grant write privileges on a bucket to a group -# $1: The googlegroups group -# $2: The bucket -function empower_group_to_write_gcs_bucket() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "empower_group_to_write_gcs_bucket(group_name, bucket) requires 2 arguments" >&2 - return 1 - fi - group="$1" - bucket="$2" - - gsutil iam ch \ - "group:${group}:objectAdmin" \ - "${bucket}" - gsutil iam ch \ - "group:${group}:legacyBucketReader" \ - "${bucket}" -} - -# Grant admin privileges on a bucket to a group -# $1: The googlegroups group -# $2: The bucket -function empower_group_to_admin_gcs_bucket() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "empower_group_to_admin_gcs_bucket(group_name, bucket) requires 2 arguments" >&2 - return 1 - fi - group="$1" - bucket="$2" - - gsutil iam ch \ - "group:${group}:objectAdmin" \ - "${bucket}" - gsutil iam ch \ - "group:${group}:legacyBucketOwner" \ - "${bucket}" -} - # Grant Cloud Run privileges to a group. # $1: The GCP project # $2: The googlegroups group @@ -616,47 +491,6 @@ function empower_artifact_auditor_invoker() { --region=us-central1 } -# Ensure the bucket retention policy is set -# $1: The GCS bucket -# $2: The retention -function ensure_gcs_bucket_retention() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_gcs_bucket_retention(bucket, retention) requires 2 arguments" >&2 - return 1 - fi - bucket="$1" - retention="$2" - - gsutil retention set "${retention}" "${bucket}" -} - -# Ensure the bucket auto-deletion policy is set -# $1: The GCS bucket -# $2: The auto-deletion policy -function ensure_gcs_bucket_auto_deletion() { - if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_gcs_bucket_auto_deletion(bucket, auto_delettion_days) requires 2 arguments" >&2 - return 1 - fi - bucket="$1" - auto_deletion_days="$2" - - echo " - { - \"rule\": [ - { - \"condition\": { - \"age\": ${auto_deletion_days} - }, - \"action\": { - \"type\": \"Delete\" - } - } - ] - } - " | gsutil lifecycle set /dev/stdin "${bucket}" -} - # Create a service account # $1: The GCP project # $2: The account name (e.g. "foo-manager") diff --git a/infra/gcp/lib_gcs.sh b/infra/gcp/lib_gcs.sh new file mode 100755 index 00000000000..19797056079 --- /dev/null +++ b/infra/gcp/lib_gcs.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GCS utility functions +# +# This is intended to be very general-purpose and "low-level". Higher-level +# policy does not belong here. +# +# This MUST NOT be used directly. Source it via lib.sh instead. + +# Ensure that a bucket exists +# $1: The GCP project +# $2: The bucket +function _ensure_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "_ensure_gcs_bucket(project, bucket) requires 2 arguments" >&2 + return 1 + fi + project="$1" + bucket="$2" + location="us" + + if ! gsutil ls "${bucket}" >/dev/null 2>&1; then + gsutil mb -p "${project}" -l "${location}" "${bucket}" + fi + gsutil bucketpolicyonly set on "${bucket}" +} + +# Ensure the bucket exists and is world-readable +# $1: The GCP project +# $2: The bucket +function ensure_public_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_public_gcs_bucket(project, bucket) requires 2 arguments" >&2 + return 1 + fi + project="$1" + bucket="$2" + + _ensure_gcs_bucket "${project}" "${bucket}" + gsutil iam ch allUsers:objectViewer "${bucket}" +} + +# Ensure the bucket exists and is NOT world-accessible +# $1: The GCP project +# $2: The bucket +function ensure_private_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_private_gcs_bucket(project, bucket) requires 2 arguments" >&2 + return 1 + fi + project="$1" + bucket="$2" + + _ensure_gcs_bucket "${project}" "${bucket}" + gsutil iam ch -d allUsers "${bucket}" +} + +# Sets the web policy on the bucket, including a default index.html page +# $1: The bucket +function ensure_gcs_web_policy() { + if [ $# -lt 1 -o -z "$1" ]; then + echo "ensure_gcs_web_policy(bucket) requires 1 argument" >&2 + return 1 + fi + bucket="$1" + + gsutil web set -m index.html "${bucket}" +} + +# Copies any static content into the bucket +# $1: The bucket +# $2: The source directory +function upload_gcs_static_content() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "upload_gcs_static_content(bucket, dir) requires 2 arguments" >&2 + return 1 + fi + bucket="$1" + srcdir="$2" + + # Checksum data to avoid no-op syncs. + gsutil rsync -c "${srcdir}" "${bucket}" +} + +# Ensure the bucket retention policy is set +# $1: The GCS bucket +# $2: The retention +function ensure_gcs_bucket_retention() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_gcs_bucket_retention(bucket, retention) requires 2 arguments" >&2 + return 1 + fi + bucket="$1" + retention="$2" + + gsutil retention set "${retention}" "${bucket}" +} + +# Ensure the bucket auto-deletion policy is set +# $1: The GCS bucket +# $2: The auto-deletion policy +function ensure_gcs_bucket_auto_deletion() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "ensure_gcs_bucket_auto_deletion(bucket, auto_delettion_days) requires 2 arguments" >&2 + return 1 + fi + bucket="$1" + auto_deletion_days="$2" + + echo " + { + \"rule\": [ + { + \"condition\": { + \"age\": ${auto_deletion_days} + }, + \"action\": { + \"type\": \"Delete\" + } + } + ] + } + " | gsutil lifecycle set /dev/stdin "${bucket}" +} + +# Grant write privileges on a bucket to a principal +# $1: The principal (group: or serviceAccount: or ...) +# $2: The bucket +function _empower_principal_to_write_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "_empower_principal_to_write_gcs_bucket(principal, bucket) requires 2 arguments" >&2 + return 1 + fi + principal="$1" + bucket="$2" + + gsutil iam ch \ + "${principal}:objectAdmin" \ + "${bucket}" + gsutil iam ch \ + "${principal}:legacyBucketReader" \ + "${bucket}" +} + +# Grant admin privileges on a bucket to a principal +# $1: The principal (group: or serviceAccount: or ...) +# $2: The bucket +function _empower_principal_to_admin_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "_empower_principal_to_admin_gcs_bucket(principal, bucket) requires 2 arguments" >&2 + return 1 + fi + principal="$1" + bucket="$2" + + gsutil iam ch \ + "${principal}:objectAdmin" \ + "${bucket}" + gsutil iam ch \ + "${principal}:legacyBucketOwner" \ + "${bucket}" +} + +# Grant write privileges on a bucket to a group +# $1: The googlegroups group email +# $2: The bucket +function empower_group_to_write_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "empower_group_to_write_gcs_bucket(group_email, bucket) requires 2 arguments" >&2 + return 1 + fi + group="$1" + bucket="$2" + + _empower_principal_to_write_gcs_bucket "group:${group}" "${bucket}" +} + +# Grant admin privileges on a bucket to a group +# $1: The googlegroups group email +# $2: The bucket +function empower_group_to_admin_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "empower_group_to_admin_gcs_bucket(group_email, bucket) requires 2 arguments" >&2 + return 1 + fi + group="$1" + bucket="$2" + + _empower_principal_to_admin_gcs_bucket "group:${group}" "${bucket}" +} + +# Grant write privileges on a bucket to a service account +# $1: The service account email +# $2: The bucket +function empower_svcacct_to_write_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "empower_svcacct_to_write_gcs_bucket(svcacct_email, bucket) requires 2 arguments" >&2 + return 1 + fi + svcacct="$1" + bucket="$2" + + _empower_principal_to_write_gcs_bucket "serviceAccount:${svcacct}" "${bucket}" +} + +# Grant admin privileges on a bucket to a service account +# $1: The service account email +# $2: The bucket +function empower_svcacct_to_admin_gcs_bucket() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "empower_svcacct_to_admin_gcs_bucket(svcacct_email, bucket) requires 2 arguments" >&2 + return 1 + fi + svcacct="$1" + bucket="$2" + + _empower_principal_to_admin_gcs_bucket "serviceAccount:${svcacct}" "${bucket}" +} diff --git a/infra/gcp/lib_util.sh b/infra/gcp/lib_util.sh new file mode 100755 index 00000000000..49679cd32c1 --- /dev/null +++ b/infra/gcp/lib_util.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generic utility functions +# +# This is intended to be very general-purpose and "low-level". Higher-level +# policy does not belong here. +# +# This MUST NOT be used directly. Source it via lib.sh instead. + +function _color() { + tput setf "$1" || true +} + +function _nocolor() { + tput sgr0 || true +} + +# Print the arguments in a given color +# $1: The color code (numeric, see `tput setf`) +# $2+: The things to print +function color() { + _color "$1" + shift + echo "$@" + _nocolor +}