diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
deleted file mode 100644
index 8678f4adbf..0000000000
--- a/.github/workflows/build.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: PyPI Nightly Build
-
-on:
-  schedule:
-    - cron: '0 0 * * *'  # Runs at midnight UTC every day
-  workflow_dispatch:
-    inputs:
-      build-type:
-        description: 'Choose build type: nightly or release'
-        required: true
-        default: 'release'
-        options:
-          - nightly
-          - release
-
-jobs:
-  build-and-publish:
-    runs-on: linux.g5.12xlarge.nvidia.gpu
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-        pip install -r requirements.txt
-
-    - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
-      with:
-        runner: ${{ matrix.runs-on }}
-        gpu-arch-type: "cuda"
-        gpu-arch-version: "12.1"
-        script: |
-          conda create -n venv python=3.9 -y
-          conda activate venv
-          echo "::group::Install newer objcopy that supports --set-section-alignment"
-          yum install -y  devtoolset-10-binutils
-          export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
-          python -m pip install --upgrade pip
-          pip install torch
-          pip install -r requirements.txt
-          pip install -r dev-requirements.txt
-
-    - name: Build package
-      run: |
-        if [ "${{ github.event_name }}" = "schedule" ]; then
-          export TORCHAO_NIGHTLY=1
-        elif [ "${{ github.event.inputs['build-type'] }}" = "nightly" ]; then
-          export TORCHAO_NIGHTLY=1
-        fi
-        python setup.py sdist bdist_wheel
-        pytest test --verbose -s
-    - name: Publish package to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
-        repository_url: https://upload.pypi.org/legacy/
-        packages_dir: dist/
-
-    # - name: Open issue on failure
-    #   if: ${{ failure() && github.event_name  == 'schedule' }}
-    #   uses: dacbd/create-issue-action@v1
-    #   with:
-    #     token: ${{ secrets.GITHUB_TOKEN }}
-    #     title: Nightly Build failed
-    #     body:  Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why
-    #     assignees: ''
diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
new file mode 100644
index 0000000000..d81f2f8434
--- /dev/null
+++ b/.github/workflows/build_wheels_linux.yml
@@ -0,0 +1,45 @@
+# From https://github.com/pytorch/test-infra/wiki/Using-Nova-Reusable-Build-Workflows
+name: Build Linux Wheels
+
+on:
+  pull_request:
+    paths:
+      - build/packaging/**
+      - .github/workflows/build_wheels_linux.yml
+  schedule:
+    - cron: '0 0 * * *'  # Runs at midnight UTC every day
+  workflow_dispatch:
+
+jobs:
+  generate-matrix:
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: linux
+      with-cpu: enable
+      with-cuda: enable
+      with-rocm: disable
+
+  build:
+    needs: generate-matrix
+    permissions:
+      id-token: write
+      contents: read
+    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
+    with:
+      # Set the ref to an empty string instead of the default nightly because
+      # torchao doesn't have nightly branch setup yet, instead the build is
+      # triggered daily from main with a schedule
+      repository: pytorch/ao
+      ref: ""
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      env-var-script: packaging/env_var_script_linux.sh
+      pre-script: packaging/pre_build_script.sh
+      post-script: packaging/post_build_script.sh
+      smoke-test-script: packaging/smoke_test.py
+      package-name: torchao
+      trigger-event: ${{ github.event_name }}
+      # This is the CUDA version to be uploaded to torchao-nightly pypi
+      upload-to-pypi: cu121
+    secrets:
+      PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/packaging/env_var_script_linux.sh b/packaging/env_var_script_linux.sh
new file mode 100644
index 0000000000..3d3394fbd5
--- /dev/null
+++ b/packaging/env_var_script_linux.sh
@@ -0,0 +1,19 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This file is sourced into the environment before building a pip wheel. It
+# should typically only contain shell variable assignments. Be sure to export
+# any variables so that subprocesses will see them.
+if [[ ${CHANNEL:-nightly} == "nightly" ]]; then
+  export TORCHAO_NIGHTLY=1
+fi
+
+# Set ARCH list so that we can build fp16 with SM75+, the logic is copied from
+# pytorch/builder
+TORCH_CUDA_ARCH_LIST="8.0;8.6"
+if [[ ${CU_VERSION:-} == "cu124" ]]; then
+  TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST};9.0"
+fi
diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh
new file mode 100644
index 0000000000..e036fcbcfe
--- /dev/null
+++ b/packaging/post_build_script.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eux
+
+WHEEL_NAME=$(ls dist/)
+
+pushd dist
+# Prepare manywheel
+auditwheel repair --plat manylinux2014_x86_64 -w . \
+    --exclude libtorch.so \
+    --exclude libtorch_python.so \
+    --exclude libtorch_cuda.so \
+    --exclude libtorch_cpu.so \
+    --exclude libc10.so \
+    --exclude libc10_cuda.so \
+    --exclude libcudart.so.12 \
+    --exclude libcudart.so.11.0 \
+    "${WHEEL_NAME}"
+
+ls -lah .
+# Clean up the linux_x86_64 wheel
+rm "${WHEEL_NAME}"
+popd
+
+MANYWHEEL_NAME=$(ls dist/)
+# Try to install the new wheel
+pip install "dist/${MANYWHEEL_NAME}"
+# and validating it by running the unit tests. Some tests are failing here and
+# there, so let's add more of them later
+pytest -v test/test_ops.py
diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh
new file mode 100644
index 0000000000..366752f7ed
--- /dev/null
+++ b/packaging/pre_build_script.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eux
+
+echo "This script is run before building torchao binaries"
+
+pip install setuptools wheel twine auditwheel
+pip install -r requirements.txt
+pip install -r dev-requirements.txt
diff --git a/packaging/smoke_test.py b/packaging/smoke_test.py
new file mode 100644
index 0000000000..2c2c873512
--- /dev/null
+++ b/packaging/smoke_test.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import subprocess
+import torchao.ops
+
+
+def main():
+    """
+    Run torchao binary smoke tests like importing and performing simple ops
+    """
+    print(dir(torchao.ops))
+
+
+if __name__ == "__main__":
+    main()