Revert D33405355: [fbsync] add contribution instructions for prototype datasets (#5133)

Differential Revision:
D33405355

Original commit changeset: 864ae4642a93

Original Phabricator Diff: D33405355

fbshipit-source-id: 0127385422452c5ab77d43b1605557484bb6a30b

Author: Haonan Sun

torchvision/prototype/datasets/_builtin/README.md

@@ -136,11 +136,8 @@ def _make_datapipe(
         return Mapper(dp, functools.partial(self._collate_and_decode_sample, decoder=decoder))
 
     def _generate_categories(self, root: pathlib.Path) -> List[str]:
-        resources = self.resources(self.default_config)
-
-        dp = resources[0].load(root)
+        dp = self.resources(self.default_config)[0].load(pathlib.Path(root) / self.name)
         dp = Filter(dp, self._is_not_background_image)
-
         return sorted({pathlib.Path(path).parent.name for path, _ in dp})
 
 
@@ -192,9 +189,6 @@ def _make_datapipe(
         return Mapper(dp, functools.partial(self._collate_and_decode_sample, decoder=decoder))
 
     def _generate_categories(self, root: pathlib.Path) -> List[str]:
-        resources = self.resources(self.default_config)
-
-        dp = resources[0].load(root)
+        dp = self.resources(self.default_config)[0].load(pathlib.Path(root) / self.name)
         dir_names = {pathlib.Path(path).parent.name for path, _ in dp}
-
         return [name.split(".")[1] for name in sorted(dir_names)]

torchvision/prototype/datasets/_builtin/caltech.py

@@ -92,12 +92,9 @@ def _make_datapipe(
         return Mapper(dp, functools.partial(self._collate_and_decode, decoder=decoder))
 
     def _generate_categories(self, root: pathlib.Path) -> List[str]:
-        resources = self.resources(self.default_config)
-
-        dp = resources[0].load(root)
+        dp = self.resources(self.default_config)[0].load(pathlib.Path(root) / self.name)
         dp = Filter(dp, path_comparator("name", self._META_FILE_NAME))
         dp = Mapper(dp, self._unpickle)
-
         return cast(List[str], next(iter(dp))[self._CATEGORIES_KEY])
 
 

torchvision/prototype/datasets/_builtin/cifar.py

@@ -238,7 +238,7 @@ def _generate_categories(self, root: pathlib.Path) -> Tuple[Tuple[str, str]]:
         config = self.default_config
         resources = self.resources(config)
 
-        dp = resources[1].load(root)
+        dp = resources[1].load(pathlib.Path(root) / self.name)
         dp = Filter(
             dp,
             functools.partial(self._filter_meta_files, split=config.split, year=config.year, annotations="instances"),

torchvision/prototype/datasets/_builtin/coco.py

@@ -177,8 +177,7 @@ def _make_datapipe(
 
     def _generate_categories(self, root: pathlib.Path) -> List[Tuple[str, ...]]:
         resources = self.resources(self.default_config)
-
-        devkit_dp = resources[1].load(root)
+        devkit_dp = resources[1].load(root / self.name)
         devkit_dp = Filter(devkit_dp, path_comparator("name", "meta.mat"))
 
         meta = next(iter(devkit_dp))[1]

torchvision/prototype/datasets/_builtin/imagenet.py

@@ -4,7 +4,7 @@
 import operator
 import pathlib
 import string
-from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, cast, BinaryIO, Union, Sequence
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, cast, BinaryIO
 
 import torch
 from torchdata.datapipes.iter import (
@@ -78,7 +78,7 @@ def __iter__(self) -> Iterator[torch.Tensor]:
 
 
 class _MNISTBase(Dataset):
-    _URL_BASE: Union[str, Sequence[str]]
+    _URL_BASE: str
 
     @abc.abstractmethod
     def _files_and_checksums(self, config: DatasetConfig) -> Tuple[Tuple[str, str], Tuple[str, str]]:
@@ -90,15 +90,8 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]:
             labels_sha256,
         ) = self._files_and_checksums(config)
 
-        url_bases = self._URL_BASE
-        if isinstance(url_bases, str):
-            url_bases = (url_bases,)
-
-        images_urls = [f"{url_base}/{images_file}" for url_base in url_bases]
-        images = HttpResource(images_urls[0], sha256=images_sha256, mirrors=images_urls[1:])
-
-        labels_urls = [f"{url_base}/{labels_file}" for url_base in url_bases]
-        labels = HttpResource(labels_urls[0], sha256=images_sha256, mirrors=labels_urls[1:])
+        images = HttpResource(f"{self._URL_BASE}/{images_file}", sha256=images_sha256)
+        labels = HttpResource(f"{self._URL_BASE}/{labels_file}", sha256=labels_sha256)
 
         return [images, labels]
 
@@ -158,10 +151,7 @@ def _make_info(self) -> DatasetInfo:
             ),
         )
 
-    _URL_BASE: Union[str, Sequence[str]] = (
-        "http://yann.lecun.com/exdb/mnist",
-        "https://ossci-datasets.s3.amazonaws.com/mnist/",
-    )
+    _URL_BASE = "http://yann.lecun.com/exdb/mnist"
     _CHECKSUMS = {
         "train-images-idx3-ubyte.gz": "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609",
         "train-labels-idx1-ubyte.gz": "3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c",

torchvision/prototype/datasets/_builtin/mnist.py

@@ -156,9 +156,7 @@ def _make_datapipe(
         return Mapper(dp, functools.partial(self._collate_and_decode_sample, config=config, decoder=decoder))
 
     def _generate_categories(self, root: pathlib.Path) -> Tuple[str, ...]:
-        resources = self.resources(self.default_config)
-
-        dp = resources[0].load(root)
+        dp = self.resources(self.default_config)[0].load(pathlib.Path(root) / self.name)
         dp = Filter(dp, path_comparator("name", "category_names.m"))
         dp = LineReader(dp)
         dp = Mapper(dp, bytes.decode, input_col=1)

torchvision/prototype/datasets/_builtin/sbd.py

@@ -11,7 +11,7 @@
 
 
 def main(*names, force=False):
-    home = pathlib.Path(datasets.home())
+    root = pathlib.Path(datasets.home())
 
     for name in names:
         path = BUILTIN_DIR / f"{name}.categories"
@@ -20,14 +20,13 @@ def main(*names, force=False):
 
         dataset = find(name)
         try:
-            categories = dataset._generate_categories(home / name)
+            categories = dataset._generate_categories(root)
         except NotImplementedError:
             continue
 
-        with open(path, "w") as file:
-            writer = csv.writer(file, lineterminator="\n")
+        with open(path, "w", newline="") as file:
             for category in categories:
-                writer.writerow((category,) if isinstance(category, str) else category)
+                csv.writer(file).writerow((category,) if isinstance(category, str) else category)
 
 
 def parse_args(argv=None):

torchvision/prototype/datasets/generate_category_files.py

@@ -6,7 +6,7 @@
 import itertools
 import os
 import pathlib
-from typing import Any, Callable, Dict, List, Optional, Sequence, Union, Tuple, Collection
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union, Tuple
 
 import torch
 from torch.utils.data import IterDataPipe
@@ -33,7 +33,7 @@ def __init__(
         name: str,
         *,
         type: Union[str, DatasetType],
-        dependencies: Collection[str] = (),
+        dependencies: Sequence[str] = (),
         categories: Optional[Union[int, Sequence[str], str, pathlib.Path]] = None,
         citation: Optional[str] = None,
         homepage: Optional[str] = None,

torchvision/prototype/datasets/utils/_dataset.py

@@ -136,14 +136,14 @@ def _check_sha256(self, path: pathlib.Path, *, chunk_size: int = 1024 * 1024) ->
 
 class HttpResource(OnlineResource):
     def __init__(
-        self, url: str, *, file_name: Optional[str] = None, mirrors: Sequence[str] = (), **kwargs: Any
+        self, url: str, *, file_name: Optional[str] = None, mirrors: Optional[Sequence[str]] = None, **kwargs: Any
     ) -> None:
         super().__init__(file_name=file_name or pathlib.Path(urlparse(url).path).name, **kwargs)
         self.url = url
         self.mirrors = mirrors
 
     def _download(self, root: pathlib.Path) -> None:
-        for url in itertools.chain((self.url,), self.mirrors):
+        for url in itertools.chain((self.url,), self.mirrors or ()):
             try:
                 download_url(url, str(root), filename=self.file_name, md5=None)
             # TODO: make this more precise