diff --git a/.circleci/config.yml b/.circleci/config.yml
index 94ac3e3fd23..94f097b520b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -359,6 +359,39 @@ jobs:
       - run_tests_selective:
           file_or_dir: test/test_prototype_*.py
 
+  # The JPEG reference tests will only pass if torchvision is built against the same JPEG library as Pillow. Starting
+  # from Pillow==9, the Pillow wheels are built against libjpeg-turbo on all platforms whereas torchvision is built
+  # against libjpeg. By installing Pillow from conda-forge, it uses whatever jpeg library is already present instead of
+  # packaging one.
+  unittest_jpeg_ref:
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: xlarge
+    steps:
+      - run:
+          name: Prepare mamba
+          command: |
+            mv ~/.bashrc ~/._bashrc
+            conda init bash
+            cat ~/.bashrc >> $BASH_ENV
+            mv ~/._bashrc ~/.bashrc
+      - checkout
+      - run:
+          name: Create environment
+          command: |
+            mamba env create -n jpeg-ref -f .circleci/unittest/jpeg-ref-env.yml
+            echo 'conda activate jpeg-ref' >> $BASH_ENV
+      - run:
+          name: Install torchvision
+          command: pip install -v --no-build-isolation --editable .
+      - run:
+          name: Enable JPEG ref tests
+          command: echo 'export PYTORCH_TEST_JPEG_REF=1' >> $BASH_ENV
+      - run_tests_selective:
+          file_or_dir: >
+            test/test_image.py::test_encode_jpeg
+            test/test_image.py::test_write_jpeg
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -1573,6 +1606,7 @@ workflows:
       - unittest_torchhub
       - unittest_onnx
       - unittest_prototype
+      - unittest_jpeg_ref
       - unittest_linux_cpu:
           cu_version: cpu
           name: unittest_linux_cpu_py3.7
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
index cb79ba7b224..f652bde3ec4 100644
--- a/.circleci/config.yml.in
+++ b/.circleci/config.yml.in
@@ -359,6 +359,39 @@ jobs:
       - run_tests_selective:
           file_or_dir: test/test_prototype_*.py
 
+  # The JPEG reference tests will only pass if torchvision is built against the same JPEG library as Pillow. Starting
+  # from Pillow==9, the Pillow wheels are built against libjpeg-turbo on all platforms whereas torchvision is built
+  # against libjpeg. By installing Pillow from conda-forge, it uses whatever jpeg library is already present instead of
+  # packaging one.
+  unittest_jpeg_ref:
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: xlarge
+    steps:
+      - run:
+          name: Prepare mamba
+          command: |
+            mv ~/.bashrc ~/._bashrc
+            conda init bash
+            cat ~/.bashrc >> $BASH_ENV
+            mv ~/._bashrc ~/.bashrc
+      - checkout
+      - run:
+          name: Create environment
+          command: |
+            mamba env create -n jpeg-ref -f .circleci/unittest/jpeg-ref-env.yml
+            echo 'conda activate jpeg-ref' >> $BASH_ENV
+      - run:
+          name: Install torchvision
+          command: pip install -v --no-build-isolation --editable .
+      - run:
+          name: Enable JPEG ref tests
+          command: echo 'export PYTORCH_TEST_JPEG_REF=1' >> $BASH_ENV
+      - run_tests_selective:
+          file_or_dir: >
+            test/test_image.py::test_encode_jpeg
+            test/test_image.py::test_write_jpeg
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -1093,6 +1126,7 @@ workflows:
       - unittest_torchhub
       - unittest_onnx
       - unittest_prototype
+      - unittest_jpeg_ref
       {{ unittest_workflows() }}
 
   cmake:
diff --git a/.circleci/unittest/jpeg-ref-env.yml b/.circleci/unittest/jpeg-ref-env.yml
new file mode 100644
index 00000000000..dc39b0ce7f7
--- /dev/null
+++ b/.circleci/unittest/jpeg-ref-env.yml
@@ -0,0 +1,19 @@
+channels:
+  - pytorch-nightly
+  - conda-forge
+
+dependencies:
+  - python == 3.7.*
+  - setuptools
+  - compilers
+  - ninja
+  - cmake
+
+  - cpuonly
+  - pytorch
+
+  - numpy
+  - requests
+  - libpng
+  - jpeg
+  - pillow >=5.3.0, !=8.3.*
diff --git a/test/common_utils.py b/test/common_utils.py
index e20e2c658bc..932c8c0c052 100644
--- a/test/common_utils.py
+++ b/test/common_utils.py
@@ -16,6 +16,17 @@
 
 
 def get_bool_env_var(name, *, exist_ok=False, default=False):
+    """Gets the value of a boolean environment variable, by mapping common bool-ish
+    values to a :class:`bool`.
+
+    Args:
+        name: Name of the environment variable.
+        exist_ok: Returns ``True`` if the environment variable exists regardless of the
+            value it holds. This is useful for checking if an environment variable
+            exists even if it holds no value.
+        default: Value to return if the environment variable doesn't exist. Defaults to
+            ``False``.
+    """
     value = os.getenv(name)
     if value is None:
         return default
@@ -216,4 +227,16 @@ def _test_fn_on_batch(batch_tensors, fn, scripted_fn_atol=1e-8, **fn_kwargs):
 
 
 def run_on_env_var(name, *, skip_reason=None, exist_ok=False, default=False):
+    """Decorator for tests to only run them if an environment variable is ``True``-ish.
+
+    Args:
+        name: Name of the environment variable.
+        skip_reason: Reason to skip the test if environment variable is not ``True``-ish.
+        exist_ok: Returns ``True`` if the environment variable exists regardless of the
+            value it holds. This is useful for checking if an environment variable
+            exists even if it holds no value.
+        default: Value to return if the environment variable doesn't exist. Defaults to
+            ``False``.
+
+    """
     return pytest.mark.skipif(not get_bool_env_var(name, exist_ok=exist_ok, default=default), reason=skip_reason)
diff --git a/test/test_image.py b/test/test_image.py
index f9e88e8ad2f..804f98b8deb 100644
--- a/test/test_image.py
+++ b/test/test_image.py
@@ -9,6 +9,7 @@
 import torch
 import torchvision.transforms.functional as F
 from common_utils import needs_cuda, assert_equal
+from common_utils import run_on_env_var
 from PIL import Image, __version__ as PILLOW_VERSION
 from torchvision.io.image import (
     decode_png,
@@ -478,8 +479,19 @@ def test_write_jpeg_reference(img_path, tmpdir):
     assert_equal(torch_bytes, pil_bytes)
 
 
-# TODO: Remove the skip. See https://github.com/pytorch/vision/issues/5162.
-@pytest.mark.skip("this test fails because PIL uses libjpeg-turbo")
+run_if_test_jpeg_ref = run_on_env_var(
+    "PYTORCH_TEST_JPEG_REF",
+    skip_reason=(
+        "JPEG reference tests compare `torchvision` JPEG encoding against `Pillow`'s. "
+        "By default `torchvision` is build against `libjpeg` while `Pillow` on wheels are built against "
+        "`libjpeg-turbo` on all platform starting from `Pillow >= 9`. For `Pillow < 9`, only the windows wheel was "
+        "built against `libjpeg-turbo`. Make sure to use the same underlying library (by running `Pillow < 9` on Linux "
+        "or macOS or by installing `Pillow` from conda-forge) and set PYTORCH_TEST_JPEG_REF=1 to run the tests."
+    ),
+)
+
+
+@run_if_test_jpeg_ref
 @pytest.mark.parametrize(
     "img_path",
     [pytest.param(jpeg_path, id=_get_safe_image_name(jpeg_path)) for jpeg_path in get_images(ENCODE_JPEG, ".jpg")],
@@ -498,8 +510,7 @@ def test_encode_jpeg(img_path):
         assert_equal(encoded_jpeg_torch, encoded_jpeg_pil)
 
 
-# TODO: Remove the skip. See https://github.com/pytorch/vision/issues/5162.
-@pytest.mark.skip("this test fails because PIL uses libjpeg-turbo")
+@run_if_test_jpeg_ref
 @pytest.mark.parametrize(
     "img_path",
     [pytest.param(jpeg_path, id=_get_safe_image_name(jpeg_path)) for jpeg_path in get_images(ENCODE_JPEG, ".jpg")],