5269 5291 Update PyTorch base docker to 22.09 (#5293)

Fixes #5269 #5291 .

### Description

This PR updated the PyTorch base docker to 22.09.

### Types of changes
<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Non-breaking change (fix or new feature that would not break
existing functionality).
- [ ] Breaking change (fix or new feature that would cause existing
functionality to change).
- [ ] New tests added to cover the changes.
- [ ] Integration tests passed locally by running `./runtests.sh -f -u
--net --coverage`.
- [ ] Quick tests passed locally by running `./runtests.sh --quick
--unittests --disttests`.
- [ ] In-line docstrings updated.
- [ ] Documentation updated, tested `make html` command in the `docs/`
folder.

Signed-off-by: Nic Ma <[email protected]>
Signed-off-by: monai-bot <[email protected]>
Signed-off-by: Wenqi Li <[email protected]>
Co-authored-by: monai-bot <[email protected]>
Co-authored-by: Wenqi Li <[email protected]>

Author: 3 people

.github/workflows/cron.yml

@@ -62,7 +62,7 @@ jobs:
     if: github.repository == 'Project-MONAI/MONAI'
     strategy:
       matrix:
-        container: ["pytorch:21.02", "pytorch:21.10", "pytorch:22.08"]  # 21.02, 21.10 for backward comp.
+        container: ["pytorch:21.02", "pytorch:21.10", "pytorch:22.09"]  # 21.02, 21.10 for backward comp.
     container:
       image: nvcr.io/nvidia/${{ matrix.container }}-py3  # testing with the latest pytorch base image
       options: "--gpus all"
@@ -106,7 +106,7 @@ jobs:
     if: github.repository == 'Project-MONAI/MONAI'
     strategy:
       matrix:
-        container: ["pytorch:21.02", "pytorch:21.10", "pytorch:22.08"]  # 21.02, 21.10 for backward comp.
+        container: ["pytorch:21.02", "pytorch:21.10", "pytorch:22.09"]  # 21.02, 21.10 for backward comp.
     container:
       image: nvcr.io/nvidia/${{ matrix.container }}-py3  # testing with the latest pytorch base image
       options: "--gpus all"
@@ -204,7 +204,7 @@ jobs:
     if: github.repository == 'Project-MONAI/MONAI'
     needs: cron-gpu  # so that monai itself is verified first
     container:
-      image: nvcr.io/nvidia/pytorch:22.08-py3  # testing with the latest pytorch base image
+      image: nvcr.io/nvidia/pytorch:22.09-py3  # testing with the latest pytorch base image
       options: "--gpus all --ipc=host"
     runs-on: [self-hosted, linux, x64, common]
     steps:

.github/workflows/pythonapp-gpu.yml

@@ -45,11 +45,11 @@ jobs:
             # 21.10: 1.10.0a0+0aef44c
             pytorch: "-h"
             base: "nvcr.io/nvidia/pytorch:21.10-py3"
-          - environment: PT112+CUDA117
+          - environment: PT112+CUDA118
             # we explicitly set pytorch to -h to avoid pip install error
-            # 22.08: 1.13.0a0+d321be6
+            # 22.09: 1.13.0a0+d0d6b1f
             pytorch: "-h"
-            base: "nvcr.io/nvidia/pytorch:22.08-py3"
+            base: "nvcr.io/nvidia/pytorch:22.09-py3"
           - environment: PT110+CUDA102
             pytorch: "torch==1.10.2 torchvision==0.11.3"
             base: "nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04"
@@ -64,7 +64,7 @@ jobs:
     - uses: actions/checkout@v3
     - name: apt install
       run: |
-        # workaround for https://github.com/Project-MONAI/MONAI/issues/4200
+        # FIXME: workaround for https://github.com/Project-MONAI/MONAI/issues/4200
         apt-key del 7fa2af80 && rm -rf /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list
         apt-get update
         apt-get install -y wget

Dockerfile

@@ -11,7 +11,7 @@
 
 # To build with a different base image
 # please run `docker build` using the `--build-arg PYTORCH_IMAGE=...` flag.
-ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:22.08-py3
+ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:22.09-py3
 FROM ${PYTORCH_IMAGE}
 
 LABEL maintainer="[email protected]"

docs/requirements.txt

@@ -33,3 +33,4 @@ pydicom
 h5py
 nni
 optuna
+opencv-python-headless

docs/source/apps.rst

@@ -191,6 +191,11 @@ Applications
 `Reconstruction`
 ----------------
 
+FastMRIReader
+~~~~~~~~~~~~~
+.. autoclass:: monai.apps.reconstruction.fastmri_reader.FastMRIReader
+  :members:
+
 `ConvertToTensorComplex`
 ~~~~~~~~~~~~~~~~~~~~~~~~
 .. autofunction:: monai.apps.reconstruction.complex_utils.convert_to_tensor_complex

docs/source/data.rst

@@ -153,12 +153,6 @@ PILReader
   :members:
 
 
-FastMRIReader
-~~~~~~~~~~~~~
-.. autoclass:: monai.apps.reconstruction.fastmri_reader.FastMRIReader
-  :members:
-
-
 Image writer
 ------------
 
@@ -352,12 +346,12 @@ Video datasets
 
 VideoDataset
 ~~~~~~~~~~~~
-.. autoclass:: monai.data.VideoDataset
+.. autoclass:: monai.data.video_dataset.VideoDataset
 
 VideoFileDataset
 ~~~~~~~~~~~~~~~~
-.. autoclass:: monai.data.VideoFileDataset
+.. autoclass:: monai.data.video_dataset.VideoFileDataset
 
 CameraDataset
 ~~~~~~~~~~~~~
-.. autoclass:: monai.data.CameraDataset
+.. autoclass:: monai.data.video_dataset.CameraDataset

monai/__init__.py

@@ -39,7 +39,17 @@
 
 # handlers_* have some external decorators the users may not have installed
 # *.so files and folder "_C" may not exist when the cpp extensions are not compiled
-excludes = "(^(monai.handlers))|(^(monai.bundle))|(^(monai.fl))|((\\.so)$)|(^(monai._C))|(.*(__main__)$)"
+excludes = "|".join(
+    [
+        "(^(monai.handlers))",
+        "(^(monai.bundle))",
+        "(^(monai.fl))",
+        "((\\.so)$)",
+        "(^(monai._C))",
+        "(.*(__main__)$)",
+        "(.*(video_dataset)$)",
+    ]
+)
 
 # load directory modules only, skip loading individual files
 load_submodules(sys.modules[__name__], False, exclude_pattern=excludes)

monai/data/__init__.py

@@ -106,7 +106,9 @@
     worker_init_fn,
     zoom_affine,
 )
-from .video_dataset import CameraDataset, VideoDataset, VideoFileDataset
+
+# FIXME: workaround for https://github.com/Project-MONAI/MONAI/issues/5291
+# from .video_dataset import CameraDataset, VideoDataset, VideoFileDataset
 from .wsi_datasets import MaskedPatchWSIDataset, PatchWSIDataset, SlidingPatchWSIDataset
 from .wsi_reader import BaseWSIReader, CuCIMWSIReader, OpenSlideWSIReader, TiffFileWSIReader, WSIReader
 

tests/__init__.py

@@ -29,7 +29,7 @@ def _enter_pr_4800(self):
     return self
 
 
-# workaround for https://bugs.python.org/issue29620
+# FIXME: workaround for https://bugs.python.org/issue29620
 try:
     # Suppression for issue #494:  tests/__init__.py:34: error: Cannot assign to a method
     unittest.case._AssertWarnsContext.__enter__ = _enter_pr_4800  # type: ignore

tests/test_cv2_dist.py

@@ -0,0 +1,46 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+import torch.distributed as dist
+from torch.cuda.amp import autocast
+
+# FIXME: test for the workaround of https://github.com/Project-MONAI/MONAI/issues/5291
+from monai.config.deviceconfig import print_config
+from tests.utils import skip_if_no_cuda
+
+
+def main_worker(rank, ngpus_per_node):
+    dist.init_process_group(backend="nccl", init_method="tcp://127.0.0.1:12345", world_size=ngpus_per_node, rank=rank)
+    # `benchmark = True` is not compatible with openCV in PyTorch 22.09 docker for multi-gpu training
+    torch.backends.cudnn.benchmark = True
+
+    model = torch.nn.Conv3d(in_channels=1, out_channels=32, kernel_size=3, bias=True).to(rank)
+    model = torch.nn.parallel.DistributedDataParallel(
+        model, device_ids=[rank], output_device=rank, find_unused_parameters=False
+    )
+    x = torch.ones(1, 1, 192, 192, 192).to(rank)
+    with autocast(enabled=True):
+        model(x)
+
+
+@skip_if_no_cuda
+class TestCV2Dist(unittest.TestCase):
+    def test_cv2_cuda_ops(self):
+        print_config()
+        ngpus_per_node = torch.cuda.device_count()
+        torch.multiprocessing.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node,))
+
+
+if __name__ == "__main__":
+    unittest.main()