diff --git a/.circleci/config.yml b/.circleci/config.yml
index e2f964f723..b27052977f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -61,7 +61,7 @@ binary_common: &binary_common
     python_version:
       description: "Python version to build against (e.g., 3.8)"
       type: string
-  environment:
+  environment: &environment
     PYTHON_VERSION: << parameters.python_version >>
     BUILD_VERSION: << parameters.build_version >>
     PYTORCH_VERSION: << parameters.pytorch_version >>
@@ -432,6 +432,7 @@ jobs:
       image: ubuntu-1604-cuda-10.1:201909-23
     resource_class: gpu.small
     environment:
+      <<: *environment
       image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
     steps:
       - checkout
@@ -444,9 +445,12 @@ jobs:
           keys:
             - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
+      - run:
+          name: Pull Docker image
+          command: docker pull --quiet "${image_name}"
       - run:
           name: Setup
-          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
+          command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
           key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
@@ -509,6 +513,7 @@ jobs:
     executor:
       name: windows-gpu
     environment:
+      <<: *environment
       CUDA_VERSION: "10.1"
     steps:
       - checkout
@@ -760,40 +765,16 @@ workflows:
           requires:
           - download_third_parties_nix
       - unittest_linux_gpu:
-          filters:
-            branches:
-              only:
-              - master
-              - nightly
-              - release/*
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
           name: unittest_linux_gpu_py3.6
           python_version: '3.6'
           requires:
           - download_third_parties_nix
       - unittest_linux_gpu:
-          filters:
-            branches:
-              only:
-              - master
-              - nightly
-              - release/*
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
           name: unittest_linux_gpu_py3.7
           python_version: '3.7'
           requires:
           - download_third_parties_nix
       - unittest_linux_gpu:
-          filters:
-            branches:
-              only:
-              - master
-              - nightly
-              - release/*
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
           name: unittest_linux_gpu_py3.8
           python_version: '3.8'
           requires:
@@ -808,36 +789,12 @@ workflows:
           name: unittest_windows_cpu_py3.8
           python_version: '3.8'
       - unittest_windows_gpu:
-          filters:
-            branches:
-              only:
-              - master
-              - nightly
-              - release/*
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
           name: unittest_windows_gpu_py3.6
           python_version: '3.6'
       - unittest_windows_gpu:
-          filters:
-            branches:
-              only:
-              - master
-              - nightly
-              - release/*
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
           name: unittest_windows_gpu_py3.7
           python_version: '3.7'
       - unittest_windows_gpu:
-          filters:
-            branches:
-              only:
-              - master
-              - nightly
-              - release/*
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
           name: unittest_windows_gpu_py3.8
           python_version: '3.8'
       - unittest_macos_cpu:
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
index b9fccbaf30..296be53463 100644
--- a/.circleci/config.yml.in
+++ b/.circleci/config.yml.in
@@ -61,7 +61,7 @@ binary_common: &binary_common
     python_version:
       description: "Python version to build against (e.g., 3.8)"
       type: string
-  environment:
+  environment: &environment
     PYTHON_VERSION: << parameters.python_version >>
     BUILD_VERSION: << parameters.build_version >>
     PYTORCH_VERSION: << parameters.pytorch_version >>
@@ -432,6 +432,7 @@ jobs:
       image: ubuntu-1604-cuda-10.1:201909-23
     resource_class: gpu.small
     environment:
+      <<: *environment
       image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
     steps:
       - checkout
@@ -444,9 +445,12 @@ jobs:
           keys:
             - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
+      - run:
+          name: Pull Docker image
+          command: docker pull --quiet "${image_name}"
       - run:
           name: Setup
-          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
+          command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
           key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
@@ -509,6 +513,7 @@ jobs:
     executor:
       name: windows-gpu
     environment:
+      <<: *environment
       CUDA_VERSION: "10.1"
     steps:
       - checkout
diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
index 2f4d4088d9..7527dc8c69 100755
--- a/.circleci/regenerate.py
+++ b/.circleci/regenerate.py
@@ -147,9 +147,6 @@ def unittest_workflows(indentation=6):
                     "python_version": python_version,
                 }
 
-                if device_type == 'gpu':
-                    job['filters'] = gen_filter_branch_tree('master', 'nightly', 'release/*')
-
                 if os_type != "windows":
                     job['requires'] = ['download_third_parties_nix']
 
diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh
index 6e203dea27..26292a00d5 100755
--- a/.circleci/unittest/linux/scripts/setup_env.sh
+++ b/.circleci/unittest/linux/scripts/setup_env.sh
@@ -21,20 +21,26 @@ esac
 # 1. Install conda at ./conda
 if [ ! -d "${conda_dir}" ]; then
     printf "* Installing conda\n"
-    wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
+    wget --quiet -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
     bash ./miniconda.sh -b -f -p "${conda_dir}"
+    eval "$("${conda_dir}/bin/conda" shell.bash hook)"
+    conda update --quiet -y conda
+    printf "* Updating the base Python version to %s\n" "${PYTHON_VERSION}"
+    conda install --quiet -y python="${PYTHON_VERSION}"
+else
+    eval "$("${conda_dir}/bin/conda" shell.bash hook)"
 fi
-eval "$(${conda_dir}/bin/conda shell.bash hook)"
+
 
 # 2. Create test environment at ./env
 if [ ! -d "${env_dir}" ]; then
-    printf "* Creating a test environment\n"
-    conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
+    printf "* Creating a test environment with PYTHON_VERSION=%s\n" "${PYTHON_VERSION}\n"
+    conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}"
 fi
 conda activate "${env_dir}"
 
 # 3. Install minimal build tools
-pip install cmake ninja
+pip --quiet install cmake ninja
 
 # 4. Buld codecs
 mkdir -p third_party/build
diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh
index 71b556d2bd..929a98cb8f 100644
--- a/.circleci/unittest/windows/scripts/setup_env.sh
+++ b/.circleci/unittest/windows/scripts/setup_env.sh
@@ -19,17 +19,21 @@ if [ ! -d "${conda_dir}" ]; then
     printf "* Installing conda\n"
     export tmp_conda="$(echo $conda_dir | tr '/' '\\')"
     export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe"
-    curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
+    curl --silent --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
     "$this_dir/install_conda.bat"
     unset tmp_conda
     unset miniconda_exe
+    eval "$("${conda_dir}/Scripts/conda.exe" 'shell.bash' 'hook')"
+    conda update --quiet -y conda
+    printf "* Updating the base Python version to %s\n" "${PYTHON_VERSION}"
+    conda install --quiet -y python="$PYTHON_VERSION"
+else
+    eval "$("${conda_dir}/Scripts/conda.exe" 'shell.bash' 'hook')"
 fi
 
-eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')"
-
 # 2. Create test environment at ./env
 if [ ! -d "${env_dir}" ]; then
-    printf "* Creating a test environment\n"
-    conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
+    printf "* Creating a test environment with PYTHON_VERSION=%s\n" "${PYTHON_VERSION}"
+    conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}"
 fi
 conda activate "${env_dir}"