diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml similarity index 61% rename from .github/actions/build/action.yml.j2 rename to .github/actions/build/action.yml index 59a6ba291..952fb9cd1 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml @@ -22,33 +22,14 @@ inputs: upload-enabled: required: true type: boolean + python-version: + required: true + type: string runs: using: composite steps: -<% for package_id, package_info in packages.items() %> - - name: Download <> (artifacts) - uses: ./.github/actions/download-artifacts - with: - artifact-repo: "<>" - artifact-name: "<>" - target-device: "${{ inputs.target-device }}" - git_sha: "<>" - host-platform: ${{ inputs.host-platform }} - dest-dir: ${{ env.ARTIFACTS_DIR }} - dependencies-workflow: <> -<% endfor %> - -<% if packages %> - - - name: Display structure of downloaded artifacts - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - pwd - ls -lahR ${{ env.ARTIFACTS_DIR }} -<% endif %> - - if: ${{ inputs.use-container }} name: Build (in container) shell: bash --noprofile --norc -xeuo pipefail {0} @@ -60,14 +41,16 @@ runs: -e AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY \ -e GITHUB_TOKEN \ - -e ARTIFACTS_DIR="$ARTIFACTS_DIR" \ + -e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \ + -e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \ -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \ -e USE_CUDA="$USE_CUDA" \ -e REPO_DIR="$REPO_DIR" \ -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \ -e PYTHON_VERSION="$PYTHON_VERSION" \ -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \ - -v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \ + -v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \ + -v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \ --rm "${{ inputs.docker-image }}" \ /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}" @@ -77,14 +60,26 @@ runs: run: | "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}" - - name: Display structure of the artifacts folder (post build) + - name: Display structure of the bindings artifacts folder (post build) + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }} + ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }} + + - name: Upload bindings build artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.BINDINGS_ARTIFACT_NAME }} + path: ${{ env.BINDINGS_ARTIFACTS_DIR }} + + - name: Display structure of the core artifacts folder (post build) shell: bash --noprofile --norc -xeuo pipefail {0} run: | - sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }} - ls -lahR ${{ env.ARTIFACTS_DIR }} + sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }} + ls -lahR ${{ env.CORE_ARTIFACTS_DIR }} - - name: Upload build artifacts + - name: Upload core build artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACTS_DIR }} + name: ${{ env.CORE_ARTIFACT_NAME }} + path: ${{ env.CORE_ARTIFACTS_DIR }} diff --git a/.github/actions/download-artifacts/action.yml b/.github/actions/download-artifacts/action.yml deleted file mode 100644 index c3dffa02c..000000000 --- a/.github/actions/download-artifacts/action.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: download-artifacts - -description: Download dependencies (artifacts) - -inputs: - artifact-repo: - type: string - require: true - artifact-name: - type: string - require: true - target-device: - type: string - required: true - git_sha: - type: string - required: true - host-platform: - type: string - required: true - dest-dir: - type: string - required: true - dependencies-workflow: - required: true - type: string - -runs: - using: composite - steps: - - - id: cache - name: Cache conda artifacts - uses: actions/cache@v4 - with: - key: "nvidia/{ inputs.artifact-repo }}@${{ inputs.host-platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}" - path: ${{ inputs.dest-dir }} - - - if: steps.cache.outputs.cache-hit != 'true' - name: Download ${{ inputs.artifact-repo }} artifacts - uses: dawidd6/action-download-artifact@v3 - with: - path: ${{ inputs.dest-dir }} - repo: nvidia/${{ inputs.artifact-repo }} - check_artifacts: true - commit: ${{ inputs.git_sha }} - workflow_conclusion: "" - workflow: ${{ inputs.dependencies-workflow }} - name: ${{ inputs.artifact-name }} - skip_unpack: true - if_no_artifact_found: fail - allow_forks: false - - - if: steps.cache.outputs.cache-hit != 'true' - name: Unpack artifact - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - cd ${{ inputs.dest-dir }} - unzip *.zip diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index adeb48df1..c2a8407c3 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -20,7 +20,7 @@ inputs: required: true type: boolean python-version: - required: false + required: true type: string runs: @@ -57,12 +57,16 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - echo "ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV - echo "ARTIFACTS_DIR=$(realpath "$(pwd)/dist")" >> $GITHUB_ENV + PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') + + echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV + echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV + echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV + echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV - echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV \ No newline at end of file + echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index 2c43d03fc..d38cb8e30 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -24,6 +24,9 @@ jobs: - release upload-enabled: - false + python-version: + #TODO cover the whole python and cuda matrix + - 3.12 uses: ./.github/workflows/gh-build-and-test.yml with: @@ -32,4 +35,5 @@ jobs: build-mode: ${{ matrix.build-mode }} build-type: ci upload-enabled: ${{ matrix.upload-enabled }} + python-version: ${{ matrix.python-version }} secrets: inherit diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index 4376776d1..430fbf5b5 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -16,6 +16,9 @@ on: upload-enabled: type: boolean required: true + python-version: + type: string + required: true jobs: build: if: ${{ github.repository_owner == 'nvidia' }} @@ -31,4 +34,5 @@ jobs: dependencies-file: "" build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} + python-version: ${{ inputs.python-version }} secrets: inherit diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 922e04ec6..c60e0c2ac 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -33,7 +33,7 @@ on: required: true type: boolean python-version: - required: false + required: true type: string jobs: @@ -63,27 +63,6 @@ jobs: upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} - - name: Render templates - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - pip -q install jinja2 - - DEPENDENCIES_FILE="" - - if [ -z "${{ inputs.dependencies-file }}" ]; then - DEPENDENCIES_FILE="${REPO_DIR}/continuous_integration/no_dependencies.json" - else - DEPENDENCIES_FILE="${REPO_DIR}/${{ inputs.dependencies-file }}" - fi - - ${REPO_DIR}/continuous_integration/scripts/render-template.py .github/actions/build/action.yml.j2 "${DEPENDENCIES_FILE}" .github/actions/build/action.yml - - - name: Dump templates - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - echo ${REPO_DIR}/.github/actions/build/action.yml - cat ${REPO_DIR}/.github/actions/build/action.yml - - name: Call build action uses: ./.github/actions/build with: diff --git a/continuous_integration/no_dependencies.json b/continuous_integration/no_dependencies.json deleted file mode 100644 index e2d7bd793..000000000 --- a/continuous_integration/no_dependencies.json +++ /dev/null @@ -1 +0,0 @@ -{ "packages" : {} } diff --git a/continuous_integration/scripts/build b/continuous_integration/scripts/build index 5db25e675..618edd5f5 100755 --- a/continuous_integration/scripts/build +++ b/continuous_integration/scripts/build @@ -3,11 +3,13 @@ build_ci() { set -xeou pipefail - cd "${REPO_DIR}" - export CUDA_HOME="${CONDA_PREFIX}/targets/x86_64-linux" export PARALLEL_LEVEL=$(nproc --ignore 1) + cd "${REPO_DIR}/cuda_bindings" + python setup.py bdist_wheel + + cd "${REPO_DIR}/cuda_core" python setup.py bdist_wheel } diff --git a/continuous_integration/scripts/generate-environment b/continuous_integration/scripts/generate-environment new file mode 100755 index 000000000..8bf2c38d8 --- /dev/null +++ b/continuous_integration/scripts/generate-environment @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Function to generate environment.yml +generate_environment_yml() { + local python_version=$1 + local cuda_version=$2 + local output_path=$3 + + cat < "${output_path}/environment.yml" +name: cuda_python +channels: + - defaults + - conda-forge +dependencies: + - python=${python_version} + - cython + - pytest + - numpy + - setuptools + - wheel + - pip + - cuda-version=${cuda_version} + - cuda-cudart-static + - cuda-driver-dev + - cuda-cudart-dev + - cuda-profiler-api + - cuda-nvrtc-dev + - cuda-nvcc + - pip: + - pytest-benchmark + - pyclibrary + - versioneer==0.29 + - tomli; python_version < "3.11" + - pywin32; sys_platform == 'win32' +EOF +} \ No newline at end of file diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env index 1294f0389..37539b378 100755 --- a/continuous_integration/scripts/make-conda-env +++ b/continuous_integration/scripts/make-conda-env @@ -3,24 +3,16 @@ set -x make_ci_env() { - mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" -} - -make_test_env() { - . conda-utils - - mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" - - activate_conda_env - - pip install "${ARTIFACTS_DIR}"/*.whl - + #TODO wire cuda version as a top level matrix argument + generate_environment_yml "${PYTHON_VERSION}" 12.6 . + mamba env create -n "${CONDA_ENV}" -f ./environment.yml } make_conda_env() { set -xeuo pipefail . setup-utils; + . generate-environment set_base_defs; case "$1" in diff --git a/continuous_integration/scripts/render-template.py b/continuous_integration/scripts/render-template.py deleted file mode 100755 index b887e361d..000000000 --- a/continuous_integration/scripts/render-template.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -from jinja2 import Environment, FileSystemLoader -import os -import re - -# TODO: make this work for arbitrary context. ie. implement replace_using_context() -def replace_placeholder(source_str, variable_name, variable_value): - # Escaping any regex special characters in variable_name - variable_name_escaped = re.escape(variable_name) - - # Using regular expression to replace ${variable_name} with actual variable_value - # \s* means any amount of whitespace (including none) - # pattern = rf'\$\{{\s*\{{\s*{variable_name_escaped}\s*\}}\s*\}}' - pattern = rf'<<\s*{variable_name_escaped}\s*>>' - return re.sub(pattern, variable_value.strip(), source_str) - -# Setup command-line argument parsing -parser = argparse.ArgumentParser(description='Render a Jinja2 template using a JSON context.') -parser.add_argument('template_file', type=str, help='Path to the Jinja2 template file (with .j2 extension).') -parser.add_argument('json_file', type=str, help='Path to the JSON file to use as the rendering context.') -parser.add_argument('output_file', type=str, help='Path to the output file.') - -args = parser.parse_args() - -# Load JSON file as the rendering context -with open(args.json_file, 'r') as file: - context = json.load(file) - -# Setup Jinja2 environment and load the template -env = Environment( - loader=FileSystemLoader(searchpath='./'), - variable_start_string='<<', - variable_end_string='>>', - block_start_string='<%', - block_end_string='%>', - comment_start_string='<#', - comment_end_string='#>') -env.filters['replace_placeholder'] = replace_placeholder - -template = env.get_template(args.template_file) - -# Render the template with the context -rendered_content = template.render(context) -# print(rendered_content) - -with open(args.output_file, 'w') as file: - file.write(rendered_content) - -print(f'Template rendered successfully. Output saved to {args.output_file}')