diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
index 6ce3dd70b..c4de395c8 100755
--- a/.github/scripts/validate_binaries.sh
+++ b/.github/scripts/validate_binaries.sh
@@ -2,49 +2,31 @@ if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then
     curl ${MATRIX_INSTALLATION} -o libtorch.zip
     unzip libtorch.zip
 else
-    #special case for Python 3.11
-    if [[ ${MATRIX_PYTHON_VERSION} == '3.11' ]]; then
-        conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION}
-        conda activate ${ENV_NAME}
-
-        INSTALLATION=${MATRIX_INSTALLATION/"-c pytorch"/"-c malfet -c pytorch"}
-        INSTALLATION=${INSTALLATION/"pytorch-cuda"/"pytorch-${MATRIX_CHANNEL}::pytorch-cuda"}
-        INSTALLATION=${INSTALLATION/"conda install"/"conda install -y"}
-
-        eval $INSTALLATION
-        python ./test/smoke_test/smoke_test.py
+    # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package
+    if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then
+        conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+        INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"}
+        INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""}
+        INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"}
+        conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI}
+        conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly
         conda deactivate
-        conda env remove -n ${ENV_NAME}
-    else
-
-
-
-        # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package
-        if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then
-            conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
-            INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"}
-            INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""}
-            INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"}
-            conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI}
-            conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly
-            conda deactivate
-            conda env remove -p ${ENV_NAME}_pypi
-        fi
-
-        # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
-        conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
-        conda activate ${ENV_NAME}
-        INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
-        eval $INSTALLATION
+        conda env remove -p ${ENV_NAME}_pypi
+    fi
 
-        if [[ ${TARGET_OS} == 'linux' ]]; then
-            export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
-            export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
-            ${PWD}/check_binary.sh
-        fi
+    # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
+    conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+    conda activate ${ENV_NAME}
+    INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
+    eval $INSTALLATION
 
-        python  ./test/smoke_test/smoke_test.py
-        conda deactivate
-        conda env remove -n ${ENV_NAME}
+    if [[ ${TARGET_OS} == 'linux' ]]; then
+        export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
+        export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
+        ${PWD}/check_binary.sh
     fi
+
+    python  ./test/smoke_test/smoke_test.py
+    conda deactivate
+    conda env remove -n ${ENV_NAME}
 fi
diff --git a/aarch64_linux/build_aarch64_wheel.py b/aarch64_linux/build_aarch64_wheel.py
index f7b70208a..5bf8450ee 100755
--- a/aarch64_linux/build_aarch64_wheel.py
+++ b/aarch64_linux/build_aarch64_wheel.py
@@ -211,14 +211,10 @@ def install_condaforge_python(host: RemoteHost, python_version="3.8") -> None:
         # Python-3.6 EOLed and not compatible with conda-4.11
         install_condaforge(host, suffix="download/4.10.3-10/Miniforge3-4.10.3-10-Linux-aarch64.sh")
         host.run_cmd(f"conda install -y python={python_version} numpy pyyaml")
-    elif python_version == "3.11":
-        install_condaforge(host, suffix="download/4.11.0-4/Miniforge3-4.11.0-4-Linux-aarch64.sh")
-        # Pytorch-1.10 or older are not compatible with setuptools=59.6 or newer
-        host.run_cmd(f"conda install -y python={python_version} numpy pyyaml setuptools=59.8.0 -c malfet")
     else:
         install_condaforge(host, suffix="download/4.11.0-4/Miniforge3-4.11.0-4-Linux-aarch64.sh")
         # Pytorch-1.10 or older are not compatible with setuptools=59.6 or newer
-        host.run_cmd(f"conda install -y python={python_version} numpy pyyaml setuptools=59.5.0")
+        host.run_cmd(f"conda install -y python={python_version} numpy pyyaml setuptools>=59.5.0")
 
 
 def build_OpenBLAS(host: RemoteHost, git_clone_flags: str = "") -> None:
diff --git a/conda/build_pytorch.sh b/conda/build_pytorch.sh
index adde9fb27..b9bdcf89b 100755
--- a/conda/build_pytorch.sh
+++ b/conda/build_pytorch.sh
@@ -299,12 +299,6 @@ else
     export CONDA_BUILD_EXTRA_ARGS=""
 fi
 
-if [[ "$DESIRED_PYTHON" == "3.11" ]]; then
-    # TODO: Remove me when numpy is available in default channel
-    # or copy numpy to pytorch channel
-    export CONDA_BUILD_EXTRA_ARGS="-c malfet ${CONDA_BUILD_EXTRA_ARGS}"
-fi
-
 # Build PyTorch with Gloo's TCP_TLS transport
 if [[ "$(uname)" == 'Linux' ]]; then
     export USE_GLOO_WITH_OPENSSL=1