Updating TorchData DataPipe API usages

NivekT · NivekT · commit 2b195ad08ff9 · 2022-03-25T11:30:38.000-04:00
diff --git a/torchtext/datasets/amazonreviewfull.py b/torchtext/datasets/amazonreviewfull.py
@@ -70,7 +70,7 @@ def AmazonReviewFull(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py
@@ -67,7 +67,7 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/cc100.py b/torchtext/datasets/cc100.py
@@ -161,7 +161,7 @@ def CC100(root: str, language_code: str = "en"):
     cache_decompressed_dp = cache_compressed_dp.on_disk_cache(
         filepath_fn=lambda x: os.path.join(root, os.path.basename(x).rstrip(".xz"))
     )
-    cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_xz()
+    cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").load_from_xz()
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb")
 
     data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8").readlines(return_path=False)
diff --git a/torchtext/datasets/dbpedia.py b/torchtext/datasets/dbpedia.py
@@ -66,7 +66,7 @@ def DBpedia(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/enwik9.py b/torchtext/datasets/enwik9.py
@@ -48,7 +48,7 @@ def EnWik9(root: str):
     cache_decompressed_dp = cache_compressed_dp.on_disk_cache(
         filepath_fn=lambda x: os.path.join(root, os.path.splitext(_PATH)[0])
     )
-    cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_zip()
+    cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").load_from_zip()
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
     data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
diff --git a/torchtext/datasets/imdb.py b/torchtext/datasets/imdb.py
@@ -62,7 +62,7 @@ def IMDB(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: [os.path.join(root, decompressed_folder, split, label) for label in labels]
     )
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b")
-    cache_decompressed_dp = cache_decompressed_dp.read_from_tar()
+    cache_decompressed_dp = cache_decompressed_dp.load_from_tar()
 
     def filter_imdb_data(key, fname):
         # eg. fname = "aclImdb/train/neg/12416_3.txt"
diff --git a/torchtext/datasets/iwslt2016.py b/torchtext/datasets/iwslt2016.py
@@ -125,7 +125,7 @@
 # avoid additional conditional imports.
 def _filter_clean_cache(cache_decompressed_dp, full_filepath, uncleaned_filename):
     cache_inner_decompressed_dp = cache_decompressed_dp.on_disk_cache(filepath_fn=lambda x: full_filepath)
-    cache_inner_decompressed_dp = FileOpener(cache_inner_decompressed_dp, mode="b").read_from_tar()
+    cache_inner_decompressed_dp = FileOpener(cache_inner_decompressed_dp, mode="b").load_from_tar()
     cache_inner_decompressed_dp = cache_inner_decompressed_dp.filter(
         lambda x: os.path.basename(uncleaned_filename) in x[0]
     )
@@ -263,7 +263,7 @@ def IWSLT2016(
     cache_decompressed_dp = cache_compressed_dp.on_disk_cache(filepath_fn=lambda x: inner_iwslt_tar)
     cache_decompressed_dp = (
         FileOpener(cache_decompressed_dp, mode="b")
-        .read_from_tar()
+        .load_from_tar()
         .filter(lambda x: os.path.basename(inner_iwslt_tar) in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
diff --git a/torchtext/datasets/iwslt2017.py b/torchtext/datasets/iwslt2017.py
@@ -104,7 +104,7 @@
 # avoid additional conditional imports.
 def _filter_clean_cache(cache_decompressed_dp, full_filepath, uncleaned_filename):
     cache_inner_decompressed_dp = cache_decompressed_dp.on_disk_cache(filepath_fn=lambda x: full_filepath)
-    cache_inner_decompressed_dp = FileOpener(cache_inner_decompressed_dp, mode="b").read_from_tar()
+    cache_inner_decompressed_dp = FileOpener(cache_inner_decompressed_dp, mode="b").load_from_tar()
     cache_inner_decompressed_dp = cache_inner_decompressed_dp.filter(
         lambda x: os.path.basename(uncleaned_filename) in x[0]
     )
@@ -208,7 +208,7 @@ def IWSLT2017(root=".data", split=("train", "valid", "test"), language_pair=("de
     )
 
     cache_decompressed_dp = cache_compressed_dp.on_disk_cache(filepath_fn=lambda x: inner_iwslt_tar)
-    cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_tar()
+    cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").load_from_tar()
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
     src_filename = file_path_by_lang_and_split[src_language][split]
diff --git a/torchtext/datasets/multi30k.py b/torchtext/datasets/multi30k.py
@@ -84,7 +84,7 @@ def Multi30k(root: str, split: Union[Tuple[str], str], language_pair: Tuple[str]
     )
     src_cache_decompressed_dp = (
         FileOpener(src_cache_decompressed_dp, mode="b")
-        .read_from_tar()
+        .load_from_tar()
         .filter(lambda x: f"{_PREFIX[split]}.{language_pair[0]}" in x[0])
     )
     src_cache_decompressed_dp = src_cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
@@ -94,7 +94,7 @@ def Multi30k(root: str, split: Union[Tuple[str], str], language_pair: Tuple[str]
     )
     tgt_cache_decompressed_dp = (
         FileOpener(tgt_cache_decompressed_dp, mode="b")
-        .read_from_tar()
+        .load_from_tar()
         .filter(lambda x: f"{_PREFIX[split]}.{language_pair[1]}" in x[0])
     )
     tgt_cache_decompressed_dp = tgt_cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
diff --git a/torchtext/datasets/sogounews.py b/torchtext/datasets/sogounews.py
@@ -70,7 +70,7 @@ def SogouNews(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/sst2.py b/torchtext/datasets/sst2.py
@@ -73,7 +73,7 @@ def SST2(root, split):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/udpos.py b/torchtext/datasets/udpos.py
@@ -61,7 +61,7 @@ def UDPOS(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/wikitext103.py b/torchtext/datasets/wikitext103.py
@@ -66,7 +66,7 @@ def WikiText103(root: str, split: Union[Tuple[str], str]):
     )
     # Extract zip and filter the appropriate split file
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
     data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
diff --git a/torchtext/datasets/wikitext2.py b/torchtext/datasets/wikitext2.py
@@ -66,7 +66,7 @@ def WikiText2(root: str, split: Union[Tuple[str], str]):
     )
     # Extract zip and filter the appropriate split file
     cache_decompressed_dp = (
-        FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+        FileOpener(cache_decompressed_dp, mode="b").load_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
     data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py
@@ -67,7 +67,7 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b")
-    cache_decompressed_dp = cache_decompressed_dp.read_from_tar()
+    cache_decompressed_dp = cache_decompressed_dp.load_from_tar()
     cache_decompressed_dp = cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
diff --git a/torchtext/datasets/yelpreviewfull.py b/torchtext/datasets/yelpreviewfull.py
@@ -67,7 +67,7 @@ def YelpReviewFull(root: str, split: Union[Tuple[str], str]):
         filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])
     )
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b")
-    cache_decompressed_dp = cache_decompressed_dp.read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
+    cache_decompressed_dp = cache_decompressed_dp.load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
     data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
diff --git a/torchtext/datasets/yelpreviewpolarity.py b/torchtext/datasets/yelpreviewpolarity.py
@@ -67,7 +67,7 @@ def YelpReviewPolarity(root: str, split: Union[Tuple[str], str]):
     )
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b")
 
-    cache_decompressed_dp = cache_decompressed_dp.read_from_tar()
+    cache_decompressed_dp = cache_decompressed_dp.load_from_tar()
 
     cache_decompressed_dp = cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)

Original file line number	Diff line number	Diff line change
`@@ -70,7 +70,7 @@ def AmazonReviewFull(root: str, split: Union[Tuple[str], str]):`
`70`	`70`	`filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])`
`71`	`71`	`)`
`72`	`72`	`cache_decompressed_dp = (`
`73`		`- FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
	`73`	`+ FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
`74`	`74`	`)`
`75`	`75`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`76`	`76`
Original file line number	Diff line number	Diff line change
`@@ -67,7 +67,7 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]):`
`67`	`67`	`filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])`
`68`	`68`	`)`
`69`	`69`	`cache_decompressed_dp = (`
`70`		`- FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
	`70`	`+ FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
`71`	`71`	`)`
`72`	`72`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`73`	`73`
Original file line number	Diff line number	Diff line change
`@@ -161,7 +161,7 @@ def CC100(root: str, language_code: str = "en"):`
`161`	`161`	`cache_decompressed_dp = cache_compressed_dp.on_disk_cache(`
`162`	`162`	`filepath_fn=lambda x: os.path.join(root, os.path.basename(x).rstrip(".xz"))`
`163`	`163`	`)`
`164`		`- cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_xz()`
	`164`	`+ cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").load_from_xz()`
`165`	`165`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb")`
`166`	`166`
`167`	`167`	`data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8").readlines(return_path=False)`
Original file line number	Diff line number	Diff line change
`@@ -66,7 +66,7 @@ def DBpedia(root: str, split: Union[Tuple[str], str]):`
`66`	`66`	`filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])`
`67`	`67`	`)`
`68`	`68`	`cache_decompressed_dp = (`
`69`		`- FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
	`69`	`+ FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
`70`	`70`	`)`
`71`	`71`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`72`	`72`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ def EnWik9(root: str):`
`48`	`48`	`cache_decompressed_dp = cache_compressed_dp.on_disk_cache(`
`49`	`49`	`filepath_fn=lambda x: os.path.join(root, os.path.splitext(_PATH)[0])`
`50`	`50`	`)`
`51`		`- cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_zip()`
	`51`	`+ cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").load_from_zip()`
`52`	`52`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`53`	`53`
`54`	`54`	`data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")`
Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,7 @@ def IMDB(root: str, split: Union[Tuple[str], str]):`
`62`	`62`	`filepath_fn=lambda x: [os.path.join(root, decompressed_folder, split, label) for label in labels]`
`63`	`63`	`)`
`64`	`64`	`cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b")`
`65`		`- cache_decompressed_dp = cache_decompressed_dp.read_from_tar()`
	`65`	`+ cache_decompressed_dp = cache_decompressed_dp.load_from_tar()`
`66`	`66`
`67`	`67`	`def filter_imdb_data(key, fname):`
`68`	`68`	`# eg. fname = "aclImdb/train/neg/12416_3.txt"`
Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ def Multi30k(root: str, split: Union[Tuple[str], str], language_pair: Tuple[str]`
`84`	`84`	`)`
`85`	`85`	`src_cache_decompressed_dp = (`
`86`	`86`	`FileOpener(src_cache_decompressed_dp, mode="b")`
`87`		`- .read_from_tar()`
	`87`	`+ .load_from_tar()`
`88`	`88`	`.filter(lambda x: f"{_PREFIX[split]}.{language_pair[0]}" in x[0])`
`89`	`89`	`)`
`90`	`90`	`src_cache_decompressed_dp = src_cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`@@ -94,7 +94,7 @@ def Multi30k(root: str, split: Union[Tuple[str], str], language_pair: Tuple[str]`
`94`	`94`	`)`
`95`	`95`	`tgt_cache_decompressed_dp = (`
`96`	`96`	`FileOpener(tgt_cache_decompressed_dp, mode="b")`
`97`		`- .read_from_tar()`
	`97`	`+ .load_from_tar()`
`98`	`98`	`.filter(lambda x: f"{_PREFIX[split]}.{language_pair[1]}" in x[0])`
`99`	`99`	`)`
`100`	`100`	`tgt_cache_decompressed_dp = tgt_cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
Original file line number	Diff line number	Diff line change
`@@ -70,7 +70,7 @@ def SogouNews(root: str, split: Union[Tuple[str], str]):`
`70`	`70`	`filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])`
`71`	`71`	`)`
`72`	`72`	`cache_decompressed_dp = (`
`73`		`- FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
	`73`	`+ FileOpener(cache_decompressed_dp, mode="b").load_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
`74`	`74`	`)`
`75`	`75`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`76`	`76`
Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ def SST2(root, split):`
`73`	`73`	`filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])`
`74`	`74`	`)`
`75`	`75`	`cache_decompressed_dp = (`
`76`		`- FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
	`76`	`+ FileOpener(cache_decompressed_dp, mode="b").load_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
`77`	`77`	`)`
`78`	`78`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`79`	`79`
Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ def UDPOS(root: str, split: Union[Tuple[str], str]):`
`61`	`61`	`filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])`
`62`	`62`	`)`
`63`	`63`	`cache_decompressed_dp = (`
`64`		`- FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
	`64`	`+ FileOpener(cache_decompressed_dp, mode="b").load_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])`
`65`	`65`	`)`
`66`	`66`	`cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)`
`67`	`67`