From cebca45c1257892ff12b863b9937df8658c05647 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Fri, 12 Aug 2022 18:56:22 -0700 Subject: [PATCH 1/3] Dockerize CI --- .github/workflows/buildAndTest.yml | 114 ++++++-------------- .gitignore | 2 + build_tools/docker/Dockerfile | 40 +++++++ build_tools/docker/run_cmake_build.sh | 27 +++++ build_tools/docker/run_cmake_build_oot.sh | 39 +++++++ build_tools/docker/run_docker.sh | 10 ++ build_tools/docker/run_integration_tests.sh | 15 +++ build_tools/docker/run_unit_tests.sh | 13 +++ development.md | 22 ++++ 9 files changed, 201 insertions(+), 81 deletions(-) create mode 100644 build_tools/docker/Dockerfile create mode 100755 build_tools/docker/run_cmake_build.sh create mode 100755 build_tools/docker/run_cmake_build_oot.sh create mode 100755 build_tools/docker/run_docker.sh create mode 100755 build_tools/docker/run_integration_tests.sh create mode 100755 build_tools/docker/run_unit_tests.sh diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml index 113c9e8e7116..9ac5579d3b76 100644 --- a/.github/workflows/buildAndTest.yml +++ b/.github/workflows/buildAndTest.yml @@ -57,62 +57,36 @@ jobs: with: cache-suffix: ${{ matrix.os-arch }}-${{ matrix.llvm-build }}-${{ matrix.torch-binary }} - - name: Configure os-arch='ubuntu-x86_64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}' + - name: Build docker image + if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} + run: | + docker build -f build_tools/docker/Dockerfile \ + -t torch-mlir-cmake:ci \ + . + + - name: Build torch-mlir (os-arch='ubuntu-x86_64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}') # Fastest build, most used dev flow if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }} run: | - cmake -GNinja -Bbuild \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_LINKER=lld \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$GITHUB_WORKSPACE" \ - -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="${GITHUB_WORKSPACE}/externals/llvm-external-projects/torch-mlir-dialects" \ - -DLLVM_TARGETS_TO_BUILD=host \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DTORCH_MLIR_ENABLE_LTC=ON \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \ - -DPython3_EXECUTABLE="$(which python)" \ - $GITHUB_WORKSPACE/externals/llvm-project/llvm - - - name: Configure os-arch='ubuntu-x86_64' llvm-build='out-of-tree' torch-binary='${{ matrix.torch-binary }}' + docker run --rm \ + -v "$(pwd)":/opt/src/torch-mlir \ + -e CCACHE_DIR=/opt/src/torch-mlir/.ccache \ + -e TORCH_BINARY="${{ matrix.torch-binary }}" \ + torch-mlir-cmake:ci \ + ./build_tools/docker/run_cmake_build.sh + + - name: Build torch-mlir (os-arch='ubuntu-x86_64' llvm-build='out-of-tree' torch-binary='${{ matrix.torch-binary }}') # Most elaborate build, but cached if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'out-of-tree' }} run: | - cmake -GNinja -Bllvm-build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_LINKER=lld \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_TARGETS_TO_BUILD=host \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE="$(which python)" \ - $GITHUB_WORKSPACE/externals/llvm-project/llvm - cmake --build llvm-build - - cmake -GNinja -Bbuild \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_LINKER=lld \ - -DLLVM_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm/" \ - -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir/" \ - -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \ - -DPython3_EXECUTABLE="$(which python)" \ - $GITHUB_WORKSPACE - - - name: Configure os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}' + docker run --rm \ + -v "$(pwd)":/opt/src/torch-mlir \ + -e CCACHE_DIR=/opt/src/torch-mlir/.ccache \ + -e TORCH_BINARY="${{ matrix.torch-binary }}" \ + torch-mlir-cmake:ci \ + ./build_tools/docker/run_cmake_build_oot.sh + + - name: Build torch-mlir (os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}') # cross compile, can't test arm64 if: ${{ matrix.os-arch == 'macos-arm64' && matrix.llvm-build == 'in-tree' }} run: | @@ -138,42 +112,20 @@ jobs: -DMACOSX_DEPLOYMENT_TARGET=12.0 \ -DPython3_EXECUTABLE="$(which python)" \ $GITHUB_WORKSPACE/externals/llvm-project/llvm - - - name: Build torch-mlir - if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} - run: | - cmake --build build - - - name: Build torch-mlir (cross-compile) - if: ${{ matrix.os-arch == 'macos-arm64' }} - run: | cmake --build build_arm64 - name: Run torch-mlir unit tests if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} run: | - cmake --build build --target check-torch-mlir-all - - - name: Run refbackend e2e integration tests - if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }} - run: | - export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir" - python -m e2e_testing.torchscript.main --config=refbackend -v - - - name: Run eager_mode e2e integration tests - if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }} - run: | - export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir" - python -m e2e_testing.torchscript.main --config=eager_mode -v - - - name: Run tosa e2e integration tests - if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }} - run: | - export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir" - python -m e2e_testing.torchscript.main --config=tosa -v + docker run --rm \ + -v "$(pwd)":/opt/src/torch-mlir \ + torch-mlir-cmake:ci \ + ./build_tools/docker/run_unit_tests.sh - - name: Run lazy_tensor_core e2e integration tests + - name: Run torch-mlir integration tests if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }} run: | - export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir" - python -m e2e_testing.torchscript.main --config=lazy_tensor_core -v + docker run --rm \ + -v "$(pwd)":/opt/src/torch-mlir \ + torch-mlir-cmake:ci \ + ./build_tools/docker/run_integration_tests.sh diff --git a/.gitignore b/.gitignore index 543d617fe889..497c8ece16d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.swp .cache/ +.ccache .vscode .env *.code-workspace @@ -10,6 +11,7 @@ externals/pytorch/ libtorch* /build/ +/llvm-build/ __pycache__ *.pyc diff --git a/build_tools/docker/Dockerfile b/build_tools/docker/Dockerfile new file mode 100644 index 000000000000..dc453c1546bd --- /dev/null +++ b/build_tools/docker/Dockerfile @@ -0,0 +1,40 @@ +ARG BASE_IMG=ubuntu:18.04 +FROM ${BASE_IMG} as dev-base + +ARG ARCH="x86_64" +ARG TARGETARCH="amd64" +ARG BAZEL_VERSION=4.2.1 + +# Install basic packages +RUN apt-get update && \ + apt-get install -y \ + python3.8 \ + python3.8-dev \ + cmake \ + ccache \ + ninja-build \ + git \ + python3-pip \ + wget \ + clang \ + automake \ + libtool \ + curl \ + make \ + unzip + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10 + +# Install bazel +RUN wget -q https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-${ARCH} -O /usr/bin/bazel \ + && chmod a+x /usr/bin/bazel + +COPY externals/llvm-project/mlir/python/requirements.txt /opt/app/mlir-requirements.txt +COPY requirements.txt /opt/app/torch-mlir-requirements.txt +WORKDIR /opt/app +RUN python -m pip install --upgrade pip +RUN python -m pip install -r mlir-requirements.txt +RUN python -m pip install -r torch-mlir-requirements.txt + +WORKDIR /opt/src/torch-mlir diff --git a/build_tools/docker/run_cmake_build.sh b/build_tools/docker/run_cmake_build.sh new file mode 100755 index 000000000000..c846cb73e412 --- /dev/null +++ b/build_tools/docker/run_cmake_build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +torch_binary="${TORCH_BINARY:-ON}" + + +# Configure cmake to build torch-mlir in-tree +cmake -GNinja -Bbuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_LINKER=lld \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \ + -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$(pwd)" \ + -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="$(pwd)/externals/llvm-external-projects/torch-mlir-dialects" \ + -DLLVM_TARGETS_TO_BUILD=host \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DTORCH_MLIR_ENABLE_LTC=ON \ + -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${torch_binary}" \ + -DPython3_EXECUTABLE="$(which python)" \ + externals/llvm-project/llvm + +# Build torch-mlir +cmake --build build diff --git a/build_tools/docker/run_cmake_build_oot.sh b/build_tools/docker/run_cmake_build_oot.sh new file mode 100755 index 000000000000..c5e3bcde1137 --- /dev/null +++ b/build_tools/docker/run_cmake_build_oot.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +torch_binary="${TORCH_BINARY:-ON}" + + +# Configure cmake to build torch-mlir out-of-tree +cmake -GNinja -Bllvm-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_LINKER=lld \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_TARGETS_TO_BUILD=host \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DPython3_EXECUTABLE="$(which python)" \ + externals/llvm-project/llvm + +# Build llvm +cmake --build llvm-build + +# TODO: Reenable LTC once OOT build is successful (https://github.com/llvm/torch-mlir/issues/1154) +cmake -GNinja -Bbuild \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_LINKER=lld \ + -DLLVM_DIR="$(pwd)/llvm-build/lib/cmake/llvm/" \ + -DMLIR_DIR="$(pwd)/llvm-build/lib/cmake/mlir/" \ + -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \ + -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${torch_binary}" \ + -DPython3_EXECUTABLE="$(which python)" \ + . + +# Build torch-mlir +cmake --build build diff --git a/build_tools/docker/run_docker.sh b/build_tools/docker/run_docker.sh new file mode 100755 index 000000000000..7d2580aadf28 --- /dev/null +++ b/build_tools/docker/run_docker.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +docker build -f build_tools/docker/Dockerfile \ + -t torch-mlir-cmake:dev \ + . + +docker run -it \ + -v "$(pwd)":/opt/src/torch-mlir \ + -e CCACHE_DIR=/opt/src/torch-mlir/.ccache \ + torch-mlir-cmake:dev diff --git a/build_tools/docker/run_integration_tests.sh b/build_tools/docker/run_integration_tests.sh new file mode 100755 index 000000000000..4a6af4c3e574 --- /dev/null +++ b/build_tools/docker/run_integration_tests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export PYTHONPATH="$(pwd)/build/tools/torch-mlir/python_packages/torch_mlir" + +# refbackend e2e tests +python -m e2e_testing.torchscript.main --config=refbackend -v -s + +# eagermode backend e2e tests +python -m e2e_testing.torchscript.main --config=eager_mode -v -s + +# tosa backend e2e tests +python -m e2e_testing.torchscript.main --config=tosa -v -s + +# ltc backend e2e tests +python -m e2e_testing.torchscript.main --config=lazy_tensor_core -v -s diff --git a/build_tools/docker/run_unit_tests.sh b/build_tools/docker/run_unit_tests.sh new file mode 100755 index 000000000000..af761a0e377f --- /dev/null +++ b/build_tools/docker/run_unit_tests.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Full suite +# cmake --build build --target check-torch-mlir-all + +# Run torch-mlir unit tests. +cmake --build build --target check-torch-mlir + +# Run torch-mlir-python unit tests. +cmake --build build --target check-torch-mlir-python + +# Run torch-mlir-dialects unit tests. +cmake --build build --target check-torch-mlir-dialects diff --git a/development.md b/development.md index 59b4c52fe619..bb8448aa951f 100644 --- a/development.md +++ b/development.md @@ -8,6 +8,28 @@ cd torch-mlir git submodule update --init ``` +## Docker Setup for CMake Builds + Tests (Ubuntu_x86-64 only) + +We provide a minimal self-contained docker setup with the required dependencies +for a CMake build of torch-mlir. This allows easy and consistent setup for reproducing +CI locally and local validation of PRs using unit tests and python integration tests. +It is only supported for the most used developer workflows using Ubuntu_x86-64 platform, +and does not support cross-compiling on MacOS-arm-64 platform. + +```shell +# Build an image and launch an interactive docker container +./build_tools/docker/run_docker.sh + +# Run cmake build (either in-tree or out-of-tree) +./build_tools/docker/run_cmake_build{_oot}.sh + +# Run torch-mlir unit tests (+ python + dialect LIT tests) +./build_tools/docker/run_unit_tests.sh + +# Run torch-mlir integration tests +./build_tools/docker/run_integration_tests.sh +``` + ## Setup your Python VirtualEnvironment and Dependencies Also, ensure that you have the appropriate `python-dev` package installed From 5dd888f3a7ed12c3c65063c16e7965b22ca84125 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Mon, 15 Aug 2022 22:17:09 -0700 Subject: [PATCH 2/3] Bazel build within docker --- .github/workflows/bazelBuildAndTest.yml | 19 +++++++++++-------- build_tools/docker/Dockerfile | 2 +- build_tools/docker/run_bazel_build.sh | 3 +++ utils/bazel/.bazelrc | 8 ++++---- 4 files changed, 19 insertions(+), 13 deletions(-) create mode 100755 build_tools/docker/run_bazel_build.sh diff --git a/.github/workflows/bazelBuildAndTest.yml b/.github/workflows/bazelBuildAndTest.yml index dbceedef316c..8e27068bffb6 100644 --- a/.github/workflows/bazelBuildAndTest.yml +++ b/.github/workflows/bazelBuildAndTest.yml @@ -20,20 +20,23 @@ jobs: runs-on: ubuntu-22.04 steps: - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Checkout torch-mlir uses: actions/checkout@v2 with: submodules: 'true' - - name: Build with bazel + - name: Build docker image + run: | + docker build -f build_tools/docker/Dockerfile \ + -t torch-mlir-bazel:ci \ + . + + - name: Bazel build torch-mlir run: | - cd $GITHUB_WORKSPACE/utils/bazel - bazel build @torch-mlir//... + docker run --rm \ + -v "$(pwd)":/opt/src/torch-mlir \ + torch-mlir-bazel:ci \ + ./build_tools/docker/run_bazel_build.sh - name: Send mail if: failure() diff --git a/build_tools/docker/Dockerfile b/build_tools/docker/Dockerfile index dc453c1546bd..384bbf2be983 100644 --- a/build_tools/docker/Dockerfile +++ b/build_tools/docker/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && \ git \ python3-pip \ wget \ - clang \ + clang-10 \ automake \ libtool \ curl \ diff --git a/build_tools/docker/run_bazel_build.sh b/build_tools/docker/run_bazel_build.sh new file mode 100755 index 000000000000..c0705206231e --- /dev/null +++ b/build_tools/docker/run_bazel_build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +cd "$(pwd)/utils/bazel" && bazel build @torch-mlir//... diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index 797af847ddf6..fb6209820e8d 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -2,10 +2,10 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -build --action_env=CC=clang -build --action_env=CXX=clang++ -build --cxxopt=-std=c++17 -build --host_cxxopt=-std=c++17 +build --action_env=CC=clang-10 +build --action_env=CXX=clang-cpp-10 +build --cxxopt=-std=c++17 +build --host_cxxopt=-std=c++17 build --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0 build --cxxopt=-U__GXX_ABI_VERSION build --cxxopt=-D__GXX_ABI_VERSION=1011 From 13054a31bff4c53ccba1b4ea687b5b3e87ccc8f7 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Tue, 16 Aug 2022 00:59:02 -0700 Subject: [PATCH 3/3] update-alternatives to point to the correct clang --- build_tools/docker/Dockerfile | 3 +++ utils/bazel/.bazelrc | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build_tools/docker/Dockerfile b/build_tools/docker/Dockerfile index 384bbf2be983..a628930f124e 100644 --- a/build_tools/docker/Dockerfile +++ b/build_tools/docker/Dockerfile @@ -26,6 +26,9 @@ RUN apt-get update && \ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10 +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 10 +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 10 + # Install bazel RUN wget -q https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-${ARCH} -O /usr/bin/bazel \ && chmod a+x /usr/bin/bazel diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index fb6209820e8d..15d329dccc5c 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -2,8 +2,8 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -build --action_env=CC=clang-10 -build --action_env=CXX=clang-cpp-10 +build --action_env=CC=clang +build --action_env=CXX=clang++ build --cxxopt=-std=c++17 build --host_cxxopt=-std=c++17 build --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0