Adding MViT v2 architecture (#6105)

* Adding mvitv2 architecture

* Fixing memory issues on tests and minor refactorings.

* Adding input validation

* Adding docs and minor refactoring

* Add `min_temporal_size` in the supported meta-data.

* Switch Tuple[int, int, int] with List[int] to support easier the 2D case

* Adding more docs and references

* Change naming conventions of classes to follow the same pattern as MobileNetV3

* Fix test breakage.

* Update todos

* Performance optimizations.

Author: datumbox

docs/source/models.rst

@@ -459,6 +459,7 @@ pre-trained weights:
 .. toctree::
    :maxdepth: 1
 
+   models/video_mvitv2
    models/video_resnet
 
 |

docs/source/models/video_mvitv2.rst

@@ -0,0 +1,28 @@
+Video ResNet
+============
+
+.. currentmodule:: torchvision.models.video
+
+The MViT V2 model is based on the
+`MViTv2: Improved Multiscale Vision Transformers for Classification and Detection
+<https://arxiv.org/abs/2112.01526>`__ and `Multiscale Vision Transformers
+<https://arxiv.org/abs/2104.11227>`__ papers.
+
+
+Model builders
+--------------
+
+The following model builders can be used to instantiate a MViTV2 model, with or
+without pre-trained weights. All the model builders internally rely on the
+``torchvision.models.video.MViTV2`` base class. Please refer to the `source
+code
+<https://github.com/pytorch/vision/blob/main/torchvision/models/video/mvitv2.py>`_ for
+more details about this class.
+
+.. autosummary::
+    :toctree: generated/
+    :template: function.rst
+
+    mvit_v2_t
+    mvit_v2_s
+    mvit_v2_b

test/expect/ModelTester.test_mvit_v2_b_expect.pkl

@@ -87,6 +87,7 @@ def test_schema_meta_validation(model_fn):
         "license",
         "_metrics",
         "min_size",
+        "min_temporal_size",
         "num_params",
         "recipe",
         "unquantized",

test/expect/ModelTester.test_mvit_v2_s_expect.pkl

@@ -309,6 +309,15 @@ def _check_input_backprop(model, inputs):
         "image_size": 56,
         "input_shape": (1, 3, 56, 56),
     },
+    "mvit_v2_t": {
+        "input_shape": (1, 3, 16, 224, 224),
+    },
+    "mvit_v2_s": {
+        "input_shape": (1, 3, 16, 224, 224),
+    },
+    "mvit_v2_b": {
+        "input_shape": (1, 3, 32, 224, 224),
+    },
 }
 # speeding up slow models:
 slow_models = [
@@ -338,6 +347,7 @@ def _check_input_backprop(model, inputs):
 skipped_big_models = {
     "vit_h_14",
     "regnet_y_128gf",
+    "mvit_v2_b",
 }
 
 # The following contains configuration and expected values to be used tests that are model specific
@@ -830,6 +840,8 @@ def test_video_model(model_fn, dev):
         "num_classes": 50,
     }
     model_name = model_fn.__name__
+    if dev == "cuda" and SKIP_BIG_MODEL and model_name in skipped_big_models:
+        pytest.skip("Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model")
     kwargs = {**defaults, **_model_params.get(model_name, {})}
     num_classes = kwargs.get("num_classes")
     input_shape = kwargs.pop("input_shape")

test/expect/ModelTester.test_mvit_v2_t_expect.pkl

@@ -1 +1,2 @@
+from .mvitv2 import *
 from .resnet import *