From f14e949420bc58dc7df822a18cba2d6b4554069f Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 12:18:42 -0700 Subject: [PATCH 01/10] Add unit test step and refactor m1 logic --- .github/workflows/build-m1-binaries.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 422c41fb8f..9436cf5072 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -26,13 +26,15 @@ jobs: echo $PATH . ~/miniconda3/etc/profile.d/conda.sh set -ex - export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") + . packaging/pkg_helpers.bash + setup_build_version WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel - conda run -p ${ENV_NAME} delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} + export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" + conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - name: Test wheel shell: arch -arch arm64 bash {0} @@ -43,18 +45,20 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)" conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchaudio-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} From e8fcdbc44d6a7f4ecea8c035830ddbabde23189d Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 12:21:17 -0700 Subject: [PATCH 02/10] Add Unit test step --- .github/workflows/build-m1-binaries.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 9436cf5072..e132919b50 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -51,6 +51,21 @@ jobs: conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)" conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')" conda env remove -p ${ENV_NAME} + - name: Unit Test wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl + # Test torch is importable, by changing cwd and running import commands + cd audio/test/ + pytest torchaudio_unittest + conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 From 4963c07b31ca7abe952b3cb0e392083fc7cc4e53 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 12:28:10 -0700 Subject: [PATCH 03/10] Testing with absolute path --- .github/workflows/build-m1-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index e132919b50..177e45b216 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -63,7 +63,7 @@ jobs: conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands - cd audio/test/ + cd /Users/ec2-user/runner/_work/audio/test/ pytest torchaudio_unittest conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub From 2b416a1a4a9b562269da25f9fa340403846a7c66 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 12:34:55 -0700 Subject: [PATCH 04/10] More testing --- .github/workflows/build-m1-binaries.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 177e45b216..641c5560da 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -36,7 +36,7 @@ jobs: export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - - name: Test wheel + - name: Unit Test wheel shell: arch -arch arm64 bash {0} env: ENV_NAME: conda-test-env-${{ github.run_id }} @@ -48,10 +48,10 @@ jobs: conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands - conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)" - conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')" + cd ~/runner/_work/audio/audio/test/ + pytest torchaudio_unittest conda env remove -p ${ENV_NAME} - - name: Unit Test wheel + - name: Test wheel shell: arch -arch arm64 bash {0} env: ENV_NAME: conda-test-env-${{ github.run_id }} @@ -63,8 +63,8 @@ jobs: conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands - cd /Users/ec2-user/runner/_work/audio/test/ - pytest torchaudio_unittest + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} From 3b2abb57f58491b2596ad5dc99cb5fc7d5b5ccda Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 12:43:48 -0700 Subject: [PATCH 05/10] Make sure to add pytest --- .github/workflows/build-m1-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 641c5560da..776c16eb32 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -29,7 +29,7 @@ jobs: . packaging/pkg_helpers.bash setup_build_version WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja libpng openjpeg wheel pkg-config + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake pytest ninja libpng openjpeg wheel pkg-config conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel From 9144aa730e3dd2c0942db8ae44398b13c6e5aff2 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 12:54:19 -0700 Subject: [PATCH 06/10] Add m1 integration test --- .github/workflows/build-m1-binaries.yml | 15 --------- .github/workflows/integration-test-m1.yml | 38 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/integration-test-m1.yml diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 776c16eb32..66532f3dee 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -36,21 +36,6 @@ jobs: export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - - name: Unit Test wheel - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-test-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly - conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl - # Test torch is importable, by changing cwd and running import commands - cd ~/runner/_work/audio/audio/test/ - pytest torchaudio_unittest - conda env remove -p ${ENV_NAME} - name: Test wheel shell: arch -arch arm64 bash {0} env: diff --git a/.github/workflows/integration-test-m1.yml b/.github/workflows/integration-test-m1.yml new file mode 100644 index 0000000000..95eb152ffb --- /dev/null +++ b/.github/workflows/integration-test-m1.yml @@ -0,0 +1,38 @@ +name: Integration Test + +on: + pull_request: + branches: [ main ] + + workflow_dispatch: + +jobs: + build: + + runs-on: macos-m1-11 + strategy: + fail-fast: false + matrix: + python-version: [ 3.8 ] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4 + sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev + - name: Install packages + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + python -m pip install --quiet pytest requests cmake ninja deep-phonemizer sentencepiece + python setup.py install + env: + USE_FFMPEG: true + - name: Run integration test + run: | + cd test && pytest integration_tests -v --use-tmp-hub-dir From 2c993fdcb5c0cb85eb60287a26d4615dd67c4524 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 13:21:01 -0700 Subject: [PATCH 07/10] More changes to audio test --- .github/workflows/build-m1-binaries.yml | 19 ++++++++++++ .github/workflows/integration-test-m1.yml | 38 ----------------------- 2 files changed, 19 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/integration-test-m1.yml diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 66532f3dee..fb5bf67e3c 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -36,6 +36,25 @@ jobs: export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} + - name: Unit Test wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl + # Test torch is importable, by changing cwd and running import commands + conda run -p ${ENV_NAME} python3 -m pip install --quiet --upgrade pip + conda run -p ${ENV_NAME} python3 -m pip install --quiet pytest requests cmake ninja deep-phonemizer sentencepiece + cd ~/runner/_work/audio/audio/ + conda run -p ${ENV_NAME} python3 setup.py install + cd test + conda run -p ${ENV_NAME} pytest torchaudio_unittest + conda env remove -p ${ENV_NAME} - name: Test wheel shell: arch -arch arm64 bash {0} env: diff --git a/.github/workflows/integration-test-m1.yml b/.github/workflows/integration-test-m1.yml deleted file mode 100644 index 95eb152ffb..0000000000 --- a/.github/workflows/integration-test-m1.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Integration Test - -on: - pull_request: - branches: [ main ] - - workflow_dispatch: - -jobs: - build: - - runs-on: macos-m1-11 - strategy: - fail-fast: false - matrix: - python-version: [ 3.8 ] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4 - sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev - - name: Install packages - run: | - python -m pip install --quiet --upgrade pip - python -m pip install --quiet --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - python -m pip install --quiet pytest requests cmake ninja deep-phonemizer sentencepiece - python setup.py install - env: - USE_FFMPEG: true - - name: Run integration test - run: | - cd test && pytest integration_tests -v --use-tmp-hub-dir From c7398aa01765c1cc2643c3d2397a63d22fe16625 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 13:30:16 -0700 Subject: [PATCH 08/10] M1 test avoid one package --- .github/workflows/build-m1-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index fb5bf67e3c..78782e818e 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -49,7 +49,7 @@ jobs: conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run -p ${ENV_NAME} python3 -m pip install --quiet --upgrade pip - conda run -p ${ENV_NAME} python3 -m pip install --quiet pytest requests cmake ninja deep-phonemizer sentencepiece + conda run -p ${ENV_NAME} python3 -m pip install --quiet pytest requests cmake ninja deep-phonemizer cd ~/runner/_work/audio/audio/ conda run -p ${ENV_NAME} python3 setup.py install cd test From 34fd294849cfe7df6934d7a1700368a0fa078096 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 13:59:36 -0700 Subject: [PATCH 09/10] Don't include unit test --- .github/workflows/build-m1-binaries.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 78782e818e..66532f3dee 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -36,25 +36,6 @@ jobs: export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - - name: Unit Test wheel - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-test-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly - conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl - # Test torch is importable, by changing cwd and running import commands - conda run -p ${ENV_NAME} python3 -m pip install --quiet --upgrade pip - conda run -p ${ENV_NAME} python3 -m pip install --quiet pytest requests cmake ninja deep-phonemizer - cd ~/runner/_work/audio/audio/ - conda run -p ${ENV_NAME} python3 setup.py install - cd test - conda run -p ${ENV_NAME} pytest torchaudio_unittest - conda env remove -p ${ENV_NAME} - name: Test wheel shell: arch -arch arm64 bash {0} env: From 86643784f76e6630b380f4e2105739f4269eea08 Mon Sep 17 00:00:00 2001 From: atalman Date: Fri, 3 Jun 2022 06:55:31 -0700 Subject: [PATCH 10/10] Fix lint warning address comments --- .github/workflows/build-m1-binaries.yml | 2 +- packaging/pkg_helpers.bash | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 66532f3dee..5757410063 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -29,7 +29,7 @@ jobs: . packaging/pkg_helpers.bash setup_build_version WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake pytest ninja libpng openjpeg wheel pkg-config + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja wheel pkg-config conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index b1eac3dfd5..44e3e0318f 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -105,10 +105,11 @@ setup_build_version() { # version.txt for some reason has `a` character after major.minor.rev # command below yields 0.10.0 from version.txt containing 0.10.0a0 _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) - export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else - export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" + BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi + export BUILD_VERSION } # Set some useful variables for OS X, if applicable