diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml
index 422c41fb8f..5757410063 100644
--- a/.github/workflows/build-m1-binaries.yml
+++ b/.github/workflows/build-m1-binaries.yml
@@ -26,13 +26,15 @@ jobs:
           echo $PATH
           . ~/miniconda3/etc/profile.d/conda.sh
           set -ex
-          export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d")
+          . packaging/pkg_helpers.bash
+          setup_build_version
           WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl
-          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja libpng openjpeg wheel pkg-config
-          conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly
+          conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja wheel pkg-config
+          conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly
           conda run -p ${ENV_NAME} python3 -mpip install delocate
           conda run -p ${ENV_NAME} python3 setup.py bdist_wheel
-          conda run -p ${ENV_NAME} delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME}
+          export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')"
+          conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME}
           conda env remove -p ${ENV_NAME}
       - name: Test wheel
         shell: arch -arch arm64 bash {0}
@@ -43,18 +45,20 @@ jobs:
           . ~/miniconda3/etc/profile.d/conda.sh
           set -ex
           conda create -yp ${ENV_NAME} python=${PY_VERS} numpy
-          conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly
+          conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly
           conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl
           # Test torch is importable, by changing cwd and running import commands
           conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)"
           conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')"
           conda env remove -p ${ENV_NAME}
       - name: Upload wheel to GitHub
+        if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }}
         uses: actions/upload-artifact@v3
         with:
           name: torchaudio-py${{ matrix.py_vers }}-macos11-m1
           path: dist/
       - name: Upload wheel to S3
+        if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }}
         shell: arch -arch arm64 bash {0}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
index b1eac3dfd5..44e3e0318f 100644
--- a/packaging/pkg_helpers.bash
+++ b/packaging/pkg_helpers.bash
@@ -105,10 +105,11 @@ setup_build_version() {
     # version.txt for some reason has `a` character after major.minor.rev
     # command below yields 0.10.0 from version.txt containing 0.10.0a0
     _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" )
-    export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX"
+    BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX"
   else
-    export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
+    BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
   fi
+  export BUILD_VERSION
 }
 
 # Set some useful variables for OS X, if applicable