From 6f74eb0f6bba7d9f74295cda7baa713b92911d73 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 28 Oct 2022 21:15:17 +0530 Subject: [PATCH 1/8] new mypy version released n support for recursive datatype --- .github/workflows/lint.yml | 5 +---- torchdata/datapipes/iter/util/tfrecordloader.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dc8c64d24..e883f4119 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,9 +2,6 @@ name: Lint on: push: - branches: - - main - - release/* pull_request: jobs: @@ -53,7 +50,7 @@ jobs: pip3 install --pre torch -f "${{ steps.pytorch_channel.outputs.value }}" - name: Install dependencies run: | - pip3 install requests mypy==0.812 graphviz numpy + pip3 install requests mypy==0.982 graphviz numpy - name: Build TorchData run: | python setup.py develop diff --git a/torchdata/datapipes/iter/util/tfrecordloader.py b/torchdata/datapipes/iter/util/tfrecordloader.py index 00b146b09..184db1417 100644 --- a/torchdata/datapipes/iter/util/tfrecordloader.py +++ b/torchdata/datapipes/iter/util/tfrecordloader.py @@ -45,7 +45,7 @@ def prod(xs): # Note, reccursive types not supported by mypy at the moment # TODO(640): uncomment as soon as it becomes supported # https://github.com/python/mypy/issues/731 -# BinaryData = Union[str, List['BinaryData']] +BinaryData = Union[str, List['BinaryData']] TFRecordBinaryData = Union[str, List[str], List[List[str]], List[List[List[Any]]]] TFRecordExampleFeature = Union[torch.Tensor, List[torch.Tensor], TFRecordBinaryData] TFRecordExample = Dict[str, TFRecordExampleFeature] From 1713a6136680fd29d824406ebaf547aeb1851ee5 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 28 Oct 2022 21:18:59 +0530 Subject: [PATCH 2/8] small change --- .github/workflows/lint.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e883f4119..61ddfbdb1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,6 +2,10 @@ name: Lint on: push: + branches: + - main + - release/* + -abhi-glitchhg pull_request: jobs: From fccb7c3844550849389ee5c6d01a794f7b83c90d Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 28 Oct 2022 21:20:25 +0530 Subject: [PATCH 3/8] letsee --- .github/workflows/lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 61ddfbdb1..1f91443fa 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,7 +5,6 @@ on: branches: - main - release/* - -abhi-glitchhg pull_request: jobs: From 1e609197fde290ccd9c1a1c9d1e68104920e9d75 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 28 Oct 2022 21:26:46 +0530 Subject: [PATCH 4/8] lets go with 0.981 rn --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1f91443fa..25034cf2b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -53,7 +53,7 @@ jobs: pip3 install --pre torch -f "${{ steps.pytorch_channel.outputs.value }}" - name: Install dependencies run: | - pip3 install requests mypy==0.982 graphviz numpy + pip3 install requests mypy==0.981 graphviz numpy - name: Build TorchData run: | python setup.py develop From 14097bd4ab7755110a41d5cfa0b6aff19b7d6afa Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 29 Oct 2022 08:24:45 +0530 Subject: [PATCH 5/8] added --install-types flag in ci to install missing package stubs --- .github/workflows/lint.yml | 2 +- torchdata/datapipes/iter/util/tfrecordloader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 25034cf2b..ee57667cd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -64,7 +64,7 @@ jobs: run: | set -eux STATUS= - if ! mypy --config=mypy.ini; then + if ! mypy --config=mypy.ini --install-types --non-interactive; then STATUS=fail fi if [ -n "$STATUS" ]; then diff --git a/torchdata/datapipes/iter/util/tfrecordloader.py b/torchdata/datapipes/iter/util/tfrecordloader.py index 184db1417..19728dfc3 100644 --- a/torchdata/datapipes/iter/util/tfrecordloader.py +++ b/torchdata/datapipes/iter/util/tfrecordloader.py @@ -45,7 +45,7 @@ def prod(xs): # Note, reccursive types not supported by mypy at the moment # TODO(640): uncomment as soon as it becomes supported # https://github.com/python/mypy/issues/731 -BinaryData = Union[str, List['BinaryData']] +BinaryData = Union[str, List["BinaryData"]] TFRecordBinaryData = Union[str, List[str], List[List[str]], List[List[List[Any]]]] TFRecordExampleFeature = Union[torch.Tensor, List[torch.Tensor], TFRecordBinaryData] TFRecordExample = Dict[str, TFRecordExampleFeature] From 459a1a9e5345420cceff858aa4054ca7ff48c728 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Nov 2022 00:01:47 +0530 Subject: [PATCH 6/8] enable recursive aliases --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ee57667cd..5babb540d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -64,7 +64,7 @@ jobs: run: | set -eux STATUS= - if ! mypy --config=mypy.ini --install-types --non-interactive; then + if ! mypy --config=mypy.ini --enable-recursive-aliases --install-types --non-interactive; then STATUS=fail fi if [ -n "$STATUS" ]; then From 115d17a45af4b27c22d34294dea586aa5a05b4dd Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Nov 2022 19:16:10 +0530 Subject: [PATCH 7/8] supress the warning --- torchdata/datapipes/iter/util/randomsplitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/datapipes/iter/util/randomsplitter.py b/torchdata/datapipes/iter/util/randomsplitter.py index 27732314f..b56f98f2e 100644 --- a/torchdata/datapipes/iter/util/randomsplitter.py +++ b/torchdata/datapipes/iter/util/randomsplitter.py @@ -101,7 +101,7 @@ def __init__( self._rng = random.Random(self._seed) self._lengths: List[int] = [] - def draw(self) -> T: + def draw(self) -> T: # type: ignore selected_key = self._rng.choices(self.keys, self.weights)[0] index = self.key_to_index[selected_key] self.weights[index] -= 1 From 5765f6cf55e2961fe824fdb1f8fc75cac34e91b4 Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Thu, 10 Nov 2022 15:06:40 +0530 Subject: [PATCH 8/8] Update .github/workflows/lint.yml --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5babb540d..12d6614d9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -53,7 +53,7 @@ jobs: pip3 install --pre torch -f "${{ steps.pytorch_channel.outputs.value }}" - name: Install dependencies run: | - pip3 install requests mypy==0.981 graphviz numpy + pip3 install requests mypy==0.990 graphviz numpy - name: Build TorchData run: | python setup.py develop