From 26a99c37501815ae27870e7a9f3018890e5d6eda Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 29 May 2024 12:22:21 -0500 Subject: [PATCH 1/4] extension code mirror template --- eng/ci/code-mirror.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 eng/ci/code-mirror.yml diff --git a/eng/ci/code-mirror.yml b/eng/ci/code-mirror.yml new file mode 100644 index 0000000..fec4062 --- /dev/null +++ b/eng/ci/code-mirror.yml @@ -0,0 +1,18 @@ +trigger: + branches: + include: + - dev + - release/* + +resources: + repositories: + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: ci/variables/cfs.yml@eng + +extends: + template: ci/code-mirror.yml@eng \ No newline at end of file From fb8abc57ea62abaf9b570b9f627403e9f9f7b9d9 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 18 Jun 2024 16:28:12 -0500 Subject: [PATCH 2/4] official, public build, starting tests --- eng/ci/code-mirror.yml | 2 +- eng/ci/official-build.yml | 50 ++++++++++++++++++++++++++++++++++++++ eng/ci/public-build.yml | 43 ++++++++++++++++++++++++++++++++ eng/templates/build.yml | 34 ++++++++++++++++++++++++++ eng/templates/ci-tests.yml | 14 +++++++++++ 5 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 eng/ci/official-build.yml create mode 100644 eng/ci/public-build.yml create mode 100644 eng/templates/build.yml create mode 100644 eng/templates/ci-tests.yml diff --git a/eng/ci/code-mirror.yml b/eng/ci/code-mirror.yml index fec4062..ee145b2 100644 --- a/eng/ci/code-mirror.yml +++ b/eng/ci/code-mirror.yml @@ -15,4 +15,4 @@ variables: - template: ci/variables/cfs.yml@eng extends: - template: ci/code-mirror.yml@eng \ No newline at end of file + template: ci/code-mirror.yml@eng diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml new file mode 100644 index 0000000..2723cd5 --- /dev/null +++ b/eng/ci/official-build.yml @@ -0,0 +1,50 @@ +trigger: + batch: true + branches: + include: + - dev + - release/* + +# CI only, does not trigger on PRs. +pr: none + +schedules: + - cron: '0 0 * * MON' + displayName: At 12:00 AM, only on Monday + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: ci/variables/build.yml@eng + - template: ci/variables/cfs.yml@eng + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: Build + jobs: + - template: /eng/templates/build.yml@self + + - stage: RunTests + dependsOn: Build + jobs: + - template: /eng/templates/ci-tests.yml@self diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml new file mode 100644 index 0000000..cac005a --- /dev/null +++ b/eng/ci/public-build.yml @@ -0,0 +1,43 @@ +trigger: + batch: true + branches: + include: + - dev + +pr: + branches: + include: + - dev + +schedules: + - cron: '0 0 * * MON' + displayName: At 12:00 AM, only on Monday + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc-public + image: 1es-windows-2022 + os: windows + + stages: + - stage: Build + jobs: + - template: /eng/templates/build.yml@self + + - stage: RunTests + dependsOn: Build + jobs: + - template: /eng/templates/ci-tests.yml@self diff --git a/eng/templates/build.yml b/eng/templates/build.yml new file mode 100644 index 0000000..3b6420f --- /dev/null +++ b/eng/templates/build.yml @@ -0,0 +1,34 @@ +jobs: + - job: "Build" + displayName: 'Build Python Extensions' + + pool: + name: 1es-pool-azfunc-public + image: 1es-windows-2022 + os: windows + + strategy: + matrix: + base_extension: + EXTENSION_DIRECTORY: 'azurefunctions-extensions-base' + EXTENSION_NAME: 'Base' + blob_extension: + EXTENSION_DIRECTORY: 'azurefunctions-extensions-bindings-blob' + EXTENSION_NAME: 'Blob' + http_extension: + EXTENSION_DIRECTORY: 'azurefunctions-extensions-http' + EXTENSION_NAME: 'Http' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + - bash: | + python --version + displayName: 'Check python version' + - bash: | + python -m pip install -U pip + pip install build twine + cd ${{ EXTENSION_DIRECTORY }} + python -m build + displayName: 'Build $(EXTENSION_NAME) Extension' diff --git a/eng/templates/ci-tests.yml b/eng/templates/ci-tests.yml new file mode 100644 index 0000000..8907aa9 --- /dev/null +++ b/eng/templates/ci-tests.yml @@ -0,0 +1,14 @@ +jobs: + - job: "TestPython" + displayName: "Run Extension Unit Tests" + + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + strategy: + matrix: + base-38: + PYTHON_VERSION: '3.8' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-base' From 141eed88c828cf9f334da1a47805f597414bb464 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 24 Jun 2024 14:31:37 -0500 Subject: [PATCH 3/4] builds & test --- .github/workflows/ci_ut_ext_base_workflow.yml | 87 ------------------- .github/workflows/ci_ut_ext_blob_workflow.yml | 81 ----------------- .../workflows/ci_ut_ext_fastapi_workflow.yml | 80 ----------------- eng/ci/public-build.yml | 5 -- eng/templates/build.yml | 7 +- eng/templates/ci-tests.yml | 60 +++++++++++-- 6 files changed, 56 insertions(+), 264 deletions(-) delete mode 100644 .github/workflows/ci_ut_ext_base_workflow.yml delete mode 100644 .github/workflows/ci_ut_ext_blob_workflow.yml delete mode 100644 .github/workflows/ci_ut_ext_fastapi_workflow.yml diff --git a/.github/workflows/ci_ut_ext_base_workflow.yml b/.github/workflows/ci_ut_ext_base_workflow.yml deleted file mode 100644 index 8aa0562..0000000 --- a/.github/workflows/ci_ut_ext_base_workflow.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: CI Run for Python Extension Base - -on: - push: - branches: - - dev - - main - - release/* - paths: - - 'azurefunctions-extensions-base/**' - pull_request: - branches: - - dev - - main - - release/* - paths: - - 'azurefunctions-extensions-base/**' - -jobs: - build: - name: "Python Extension Base CI Run" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [ 3.8, 3.9, "3.10", "3.11" ] - permissions: read-all - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - working-directory: azurefunctions-extensions-base - run: | - python -m pip install --upgrade pip - python -m pip install -U -e .[dev] - - - name: Run Unit Tests - working-directory: azurefunctions-extensions-base - env: - AzureWebJobsStorage: ${{ secrets.AzureWebJobsStorage }} - run: | - python -m pytest -q --instafail --cov=. --cov-report xml --cov-branch --ignore tests/ tests - - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v3 - # env: - # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./azurefunctions-extensions-base/coverage.xml - flags: unittests - name: codecov - fail_ci_if_error: false - # TODO: upgrade to v4 when it is not breaking https://github.com/codecov/codecov-action/issues/1336 - - send-notification: - runs-on: ubuntu-latest - needs: build - if: always() - steps: - - name: Notify dedicated teams channel on failure - if: ${{ always() && needs.build.result == 'failure' }} - uses: jdcargile/ms-teams-notification@v1.4 - with: - github-token: ${{ github.token }} - ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} - notification-summary: "Python Extension Base CI Failed" - notification-color: FF0000 - timezone: America/Denver - verbose-logging: false - - - name: Notify dedicated teams channel on success - if: ${{ always() && needs.build.result == 'success' }} - uses: jdcargile/ms-teams-notification@v1.4 - with: - github-token: ${{ github.token }} - ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} - notification-summary: "Python Extension Base CI Passed" - notification-color: 008000 - timezone: America/Denver - verbose-logging: false - \ No newline at end of file diff --git a/.github/workflows/ci_ut_ext_blob_workflow.yml b/.github/workflows/ci_ut_ext_blob_workflow.yml deleted file mode 100644 index c8c6123..0000000 --- a/.github/workflows/ci_ut_ext_blob_workflow.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: CI Run for Python Extension Blob - -on: - push: - branches: [ dev, main, release/* ] - paths: - - 'azurefunctions-extensions-bindings-blob/**' - pull_request: - branches: [ dev, main, release/* ] - paths: - - 'azurefunctions-extensions-bindings-blob/**' - -jobs: - build: - name: "Python Extension Blob CI Run" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [ 3.9, "3.10", "3.11" ] - permissions: read-all - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - working-directory: azurefunctions-extensions-bindings-blob - run: | - python -m pip install --upgrade pip - python -m pip install -U -e .[dev] - - - name: Run Unit Tests - working-directory: azurefunctions-extensions-bindings-blob - env: - AzureWebJobsStorage: ${{ secrets.AzureWebJobsStorage }} - run: | - python -m pytest -q --instafail --cov=. --cov-report xml --cov-branch --ignore tests/ tests - - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v3 - # env: - # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./azurefunctions-extensions-bindings-blob/coverage.xml - flags: unittests - name: codecov - fail_ci_if_error: false - # TODO: upgrade to v4 when it is not breaking https://github.com/codecov/codecov-action/issues/1336 - - - send-notification: - runs-on: ubuntu-latest - needs: build - if: always() - steps: - - name: Notify dedicated teams channel on failure - if: ${{ always() && needs.build.result == 'failure' }} - uses: jdcargile/ms-teams-notification@v1.4 - with: - github-token: ${{ github.token }} - ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} - notification-summary: "Python Extension Blob CI Failed" - notification-color: FF0000 - timezone: America/Denver - verbose-logging: false - - - name: Notify dedicated teams channel on success - if: ${{ always() && needs.build.result == 'success' }} - uses: jdcargile/ms-teams-notification@v1.4 - with: - github-token: ${{ github.token }} - ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} - notification-summary: "Python Extension Blob CI Passed" - notification-color: 008000 - timezone: America/Denver - verbose-logging: false \ No newline at end of file diff --git a/.github/workflows/ci_ut_ext_fastapi_workflow.yml b/.github/workflows/ci_ut_ext_fastapi_workflow.yml deleted file mode 100644 index 9d99887..0000000 --- a/.github/workflows/ci_ut_ext_fastapi_workflow.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: CI Run for Python Extension FastApi - -on: - push: - branches: [ dev, main, release/* ] - paths: - - 'azurefunctions-extensions-http-fastapi/**' - pull_request: - branches: [ dev, main, release/* ] - paths: - - 'azurefunctions-extensions-http-fastapi/**' - -jobs: - build: - name: "Python Extension UT FastApi CI Run" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [ 3.8, 3.9, "3.10", "3.11" ] - permissions: read-all - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - working-directory: azurefunctions-extensions-http-fastapi - run: | - python -m pip install --upgrade pip - python -m pip install -U -e .[dev] - - - name: Run Unit Tests - working-directory: azurefunctions-extensions-http-fastapi - env: - AzureWebJobsStorage: ${{ secrets.AzureWebJobsStorage }} - run: | - python -m pytest -q --instafail --cov=. --cov-report xml --cov-branch --ignore tests/ tests - - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v3 - # env: - # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./azurefunctions-extensions-http-fastapi/coverage.xml - flags: unittests - name: codecov - fail_ci_if_error: false - # TODO: upgrade to v4 when it is not breaking https://github.com/codecov/codecov-action/issues/1336 - - send-notification: - runs-on: ubuntu-latest - needs: build - if: always() - steps: - - name: Notify dedicated teams channel on failure - if: ${{ always() && needs.build.result == 'failure' }} - uses: jdcargile/ms-teams-notification@v1.4 - with: - github-token: ${{ github.token }} - ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} - notification-summary: "Python Extension FastApi CI Failed" - notification-color: FF0000 - timezone: America/Denver - verbose-logging: false - - - name: Notify dedicated teams channel on success - if: ${{ always() && needs.build.result == 'success' }} - uses: jdcargile/ms-teams-notification@v1.4 - with: - github-token: ${{ github.token }} - ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} - notification-summary: "Python Extension FastApi CI Passed" - notification-color: 008000 - timezone: America/Denver - verbose-logging: false \ No newline at end of file diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index cac005a..b7c6b92 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -36,8 +36,3 @@ extends: - stage: Build jobs: - template: /eng/templates/build.yml@self - - - stage: RunTests - dependsOn: Build - jobs: - - template: /eng/templates/ci-tests.yml@self diff --git a/eng/templates/build.yml b/eng/templates/build.yml index 3b6420f..a03bf7b 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -2,11 +2,6 @@ jobs: - job: "Build" displayName: 'Build Python Extensions' - pool: - name: 1es-pool-azfunc-public - image: 1es-windows-2022 - os: windows - strategy: matrix: base_extension: @@ -29,6 +24,6 @@ jobs: - bash: | python -m pip install -U pip pip install build twine - cd ${{ EXTENSION_DIRECTORY }} + cd $(EXTENSION_DIRECTORY) python -m build displayName: 'Build $(EXTENSION_NAME) Extension' diff --git a/eng/templates/ci-tests.yml b/eng/templates/ci-tests.yml index 8907aa9..bbf3738 100644 --- a/eng/templates/ci-tests.yml +++ b/eng/templates/ci-tests.yml @@ -2,13 +2,63 @@ jobs: - job: "TestPython" displayName: "Run Extension Unit Tests" - pool: - name: 1es-pool-azfunc - image: 1es-ubuntu-22.04 - os: linux - strategy: matrix: base-38: PYTHON_VERSION: '3.8' EXTENSION_DIRECTORY: 'azurefunctions-extensions-base' + EXTENSION_NAME: 'Base' + base-39: + PYTHON_VERSION: '3.9' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-base' + EXTENSION_NAME: 'Base' + base-310: + PYTHON_VERSION: '3.10' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-base' + EXTENSION_NAME: 'Base' + base-311: + PYTHON_VERSION: '3.11' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-base' + EXTENSION_NAME: 'Base' + blob-39: + PYTHON_VERSION: '3.9' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-blob' + EXTENSION_NAME: 'Blob' + blob-310: + PYTHON_VERSION: '3.10' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-blob' + EXTENSION_NAME: 'Blob' + blob-311: + PYTHON_VERSION: '3.11' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-blob' + EXTENSION_NAME: 'Blob' + http-38: + PYTHON_VERSION: '3.8' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-http' + EXTENSION_NAME: 'HTTP' + http-39: + PYTHON_VERSION: '3.9' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-http' + EXTENSION_NAME: 'HTTP' + http-310: + PYTHON_VERSION: '3.10' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-http' + EXTENSION_NAME: 'HTTP' + http-311: + PYTHON_VERSION: '3.11' + EXTENSION_DIRECTORY: 'azurefunctions-extensions-http' + EXTENSION_NAME: 'HTTP' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + - bash: | + python -m pip install --upgrade pip + python -m pip install -U -e .[dev] + displayName: 'Install dependencies' + - bash: | + python -m pytest -q --instafail --cov=. --cov-report xml --cov-branch --ignore $(EXTENSION_DIRECTORY)/tests/ + env: + AzureWebJobsStorage: $(AzureWebJobsStorage) + displayName: "Running $(EXTENSION_NAME) $(PYTHON_VERSION) Python Extension Tests" From bc4173b3b04cf48a007640526a9e75f04b064981 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 25 Jun 2024 14:31:09 -0500 Subject: [PATCH 4/4] absolute paths --- eng/ci/code-mirror.yml | 4 ++-- eng/ci/official-build.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/ci/code-mirror.yml b/eng/ci/code-mirror.yml index ee145b2..d69d024 100644 --- a/eng/ci/code-mirror.yml +++ b/eng/ci/code-mirror.yml @@ -12,7 +12,7 @@ resources: ref: refs/tags/release variables: - - template: ci/variables/cfs.yml@eng + - template: /ci/variables/cfs.yml@eng extends: - template: ci/code-mirror.yml@eng + template: /ci/code-mirror.yml@eng diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 2723cd5..c538aa5 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -28,8 +28,8 @@ resources: ref: refs/tags/release variables: - - template: ci/variables/build.yml@eng - - template: ci/variables/cfs.yml@eng + - template: /ci/variables/build.yml@eng + - template: /ci/variables/cfs.yml@eng extends: template: v1/1ES.Official.PipelineTemplate.yml@1es