From 8328a144a84566b3dc1f00577b430bd05921ba36 Mon Sep 17 00:00:00 2001 From: abdhumal Date: Fri, 2 Feb 2024 18:01:14 +0530 Subject: [PATCH 1/2] add an automated CI workflow to sync ODH-notebooks with Codeflare-SDK release --- .github/workflows/odh-notebooks-sync.yml | 116 ++++++++++++++++++ .../workflows/project-codeflare-release.yml | 17 +++ 2 files changed, 133 insertions(+) create mode 100644 .github/workflows/odh-notebooks-sync.yml diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml new file mode 100644 index 000000000..789184c09 --- /dev/null +++ b/.github/workflows/odh-notebooks-sync.yml @@ -0,0 +1,116 @@ +# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release. +name: Sync with codeflare-sdk release +on: + workflow_dispatch: + inputs: + upstream-repository-organization: + required: true + description: "Owner of target upstream notebooks repository used to open a PR against" + default: "opendatahub-io" + + codeflare-repository-organization: + required: true + description: "Owner of origin notebooks repository used to open a PR" + default: "project-codeflare" + + codeflare_sdk_release_version: + required: true + description: "Provide version of the Codeflare-SDK release" + +env: + BRANCH_NAME: main + CODEFLARE_RELEASE_VERSION: ${{ github.event.inputs.codeflare_sdk_release_version }} + UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }} + UPSTREAM_OWNER: ${{ github.event.inputs.upstream-repository-organization }} + REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }} + REPO_NAME: notebooks + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_NAME }} + + - name: Clone repository and Sync + run: | + git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME + cd $REPO_NAME + git remote add upstream https://github.com/$UPSTREAM_OWNER/$REPO_NAME.git + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" + git remote -v + git pull upstream main && git push origin main + + - name: Setup Python environment + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.9 + + - name: Install pipenv + run: pip install pipenv + + - name: Update Pipfiles in accordance with Codeflare-SDK latest release + run: | + # list all Pipfile paths having Codeflare-SDK listed + paths+=($(grep -rl 'codeflare-sdk = "~=.*"')) + # Extracting only directories from file paths, excluding a `.gitworkflow` directory + directories=() + exclude_directories=( + ".git/objects/pack" + ".github/workflows/", + ) + for path in "${paths[@]}"; do + current_dir=$(dirname "$path") + #Check if current_dir is not in exclude_directories list + if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then + #Check if Pipfile exists in current_dir + if [ -f "$current_dir/Pipfile" ];then + directories+=("$current_dir") + fi + fi + done + # Remove duplicates + directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + # Print the directories for verification + echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" + for dir in "${directories[@]}"; do + echo "- $dir" + done + # iterate over the directories and update Pipfile + counter=0 + total=${#directories[@]} + for dir in "${directories[@]}"; do + counter=$((counter+1)) + echo "--Processing directory $counter '$dir' of total $total" + cd "$dir" && pipenv install codeflare-sdk~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - + echo "$((total-counter)) directories remaining.." + done + + - name: Push changes + run: | + cd $REPO_NAME + git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \ + git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff && + git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git + git push origin ${{ env.UPDATER_BRANCH }} + + - name: Create Pull Request + run: | + gh pr create --repo $UPSTREAM_OWNER/$REPO_NAME \ + --title "$pr_title" \ + --body "$pr_body" \ + --head $REPO_OWNER:$UPDATER_BRANCH \ + --base $BRANCH_NAME + env: + pr_title: "[Digest Updater Action] Update notebook's pipfile to sync with Codeflare-SDK release" + pr_body: | + :rocket: This is an automated Pull Request. + + This PR updates the `Pipfile` to sync with latest Codeflare-SDK release. + + :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.UPDATER_BRANCH }}` branch after merging the changes diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 77d0a0a1f..7ed8899cf 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -38,6 +38,10 @@ on: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' + upstream-repository-organization: + description: 'Owner of target upstream notebooks repository used to open a PR against' + required: false + default: 'opendatahub-io' jobs: release-parameters: @@ -152,6 +156,19 @@ jobs: shell: bash if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} + - name: Sync ODH Notebooks + run: | + gh workflow run odh-notebooks-sync.yml \ + --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ + --ref ${{ github.ref }} \ + --field upstream-repository-organization=${{ github.event.inputs.upstream-repository-organization }} + --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ + --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + if: ${{ env.SDK_RELEASE_STATUS_CODE == '200' }} + release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] runs-on: ubuntu-latest From c198109c1a17cec70c8fc8f5c98b9aaf64399873 Mon Sep 17 00:00:00 2001 From: abdhumal Date: Fri, 9 Feb 2024 17:40:25 +0530 Subject: [PATCH 2/2] validate if codeflare-sdk version exists --- .github/workflows/odh-notebooks-sync.yml | 86 +++++++++++-------- .../workflows/project-codeflare-release.yml | 6 +- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml index 789184c09..016608f26 100644 --- a/.github/workflows/odh-notebooks-sync.yml +++ b/.github/workflows/odh-notebooks-sync.yml @@ -51,45 +51,61 @@ jobs: 3.8 3.9 - - name: Install pipenv - run: pip install pipenv + - name: Install pipenv and pip-versions + run: pip install pipenv pip-versions - name: Update Pipfiles in accordance with Codeflare-SDK latest release run: | - # list all Pipfile paths having Codeflare-SDK listed - paths+=($(grep -rl 'codeflare-sdk = "~=.*"')) - # Extracting only directories from file paths, excluding a `.gitworkflow` directory - directories=() - exclude_directories=( - ".git/objects/pack" - ".github/workflows/", - ) - for path in "${paths[@]}"; do - current_dir=$(dirname "$path") - #Check if current_dir is not in exclude_directories list - if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then - #Check if Pipfile exists in current_dir - if [ -f "$current_dir/Pipfile" ];then - directories+=("$current_dir") + package_name=codeflare-sdk + # Get the list of available versions for the package + if ! versions=$(pipenv run pip-versions list $package_name);then + echo "Failed to retrieve versions for $package_name" + exit 1 + fi + # Check if the desired version exists in the list + if echo "$versions" | grep -q "${CODEFLARE_RELEASE_VERSION}"; then + echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name" + # list all Pipfile paths having Codeflare-SDK listed + paths+=($(grep -rl "${package_name} = \"~=.*\"")) + # Extracting only directories from file paths, excluding a `.gitworkflow` directory + directories=() + exclude_directories=( + ".git/objects/pack" + ".github/workflows/", + ) + for path in "${paths[@]}"; do + current_dir=$(dirname "$path") + #Check if current_dir is not in exclude_directories list + if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then + #Check if Pipfile exists in current_dir + if [ -f "$current_dir/Pipfile" ];then + directories+=("$current_dir") + fi fi - fi - done - # Remove duplicates - directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - # Print the directories for verification - echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" - for dir in "${directories[@]}"; do - echo "- $dir" - done - # iterate over the directories and update Pipfile - counter=0 - total=${#directories[@]} - for dir in "${directories[@]}"; do - counter=$((counter+1)) - echo "--Processing directory $counter '$dir' of total $total" - cd "$dir" && pipenv install codeflare-sdk~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - - echo "$((total-counter)) directories remaining.." - done + done + # Remove duplicates + directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + # Print the directories for verification + echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" + for dir in "${directories[@]}"; do + echo "- $dir" + done + # iterate over the directories and update Pipfile + counter=0 + total=${#directories[@]} + for dir in "${directories[@]}"; do + counter=$((counter+1)) + echo "--Processing directory $counter '$dir' of total $total" + cd "$dir" && pipenv install ${package_name}~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - + echo "$((total-counter)) directories remaining.." + done + else + versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//') + versions="${versions_list%,}" + echo "Version '${CODEFLARE_RELEASE_VERSION}' is not available for $package_name" + echo "Available versions for $package_name: $versions" + exit 1 + fi - name: Push changes run: | diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 7ed8899cf..48a70ef47 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -38,10 +38,6 @@ on: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' - upstream-repository-organization: - description: 'Owner of target upstream notebooks repository used to open a PR against' - required: false - default: 'opendatahub-io' jobs: release-parameters: @@ -161,7 +157,7 @@ jobs: gh workflow run odh-notebooks-sync.yml \ --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ --ref ${{ github.ref }} \ - --field upstream-repository-organization=${{ github.event.inputs.upstream-repository-organization }} + --field upstream-repository-organization=opendatahub-io --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} env: