Skip to content

Commit 3d2d0f2

Browse files
authored
MANIFEST.in and setup.py clean-up (Lightning-AI#7614)
1 parent 94390ab commit 3d2d0f2

File tree

10 files changed

+114
-86
lines changed

10 files changed

+114
-86
lines changed

.github/workflows/ci_pkg-install.yml

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,11 @@ jobs:
2626

2727
- name: Prepare env
2828
run: |
29-
pip install check-manifest "twine==3.2" setuptools wheel
29+
pip install "twine==3.2" setuptools wheel
3030
3131
- name: Create package
3232
run: |
33-
check-manifest
34-
# python setup.py check --metadata --strict
33+
python setup.py check --metadata --strict
3534
python setup.py sdist bdist_wheel
3635
3736
- name: Check package

MANIFEST.in

Lines changed: 3 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -11,69 +11,8 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
15-
# Manifest syntax https://docs.python.org/2/distutils/sourcedist.html
16-
graft wheelhouse
17-
18-
recursive-exclude __pycache__ *.py[cod] *.orig
19-
20-
# Include the README and CHANGELOG
21-
include *.md
22-
23-
# Include the license file
24-
include LICENSE
25-
26-
# Include the citation info
27-
include *.cff
28-
29-
exclude *.sh
30-
exclude *.svg
31-
recursive-include pytorch_lightning *.py
32-
33-
# Include marker file for PEP 561
34-
include pytorch_lightning/py.typed
35-
36-
# include examples
37-
recursive-include pl_examples *.py *.md *.sh *.txt *.toml
38-
39-
# exclude tests from package
40-
recursive-exclude tests *
41-
recursive-exclude site *
42-
exclude tests
43-
44-
# Exclude the documentation files
45-
recursive-exclude docs *
46-
exclude docs
47-
recursive-include docs/source/_static/images/logos/ *
48-
recursive-include docs/source/_static/images/general/ pl_overview* tf_* tutorial_* PTL101_*
49-
50-
# Include the Requirements
14+
include pytorch_lightning/py.typed # marker file for PEP 561
15+
include CHANGELOG.md
5116
recursive-include requirements *.txt
52-
recursive-exclude requirements *.sh *.py
5317
include requirements.txt
54-
include pyproject.toml
55-
56-
# Exclude build configs
57-
exclude *.yml
58-
exclude *.yaml
59-
exclude *.toml
60-
exclude *.jsonnet
61-
62-
# Exclude pyright config
63-
exclude .pyrightconfig.json
64-
65-
# Exclude submodules
66-
exclude .gitmodules
67-
exclude _notebooks
68-
69-
# Exclude Makefile
70-
exclude Makefile
71-
72-
prune .git
73-
prune .github
74-
prune .circleci
75-
prune temp*
76-
prune test*
77-
prune benchmark*
78-
prune dockers
79-
prune legacy
18+
include *.cff # citation info

pl_examples/basic_examples/mnist_datamodule.py

Lines changed: 99 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,18 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14+
import logging
1415
import os
1516
import platform
16-
from typing import Optional
17+
import random
18+
import time
19+
import urllib
20+
from typing import Optional, Tuple
1721
from urllib.error import HTTPError
1822
from warnings import warn
1923

20-
from torch.utils.data import DataLoader, random_split
24+
import torch
25+
from torch.utils.data import DataLoader, Dataset, random_split
2126

2227
from pl_examples import _DATASETS_PATH
2328
from pytorch_lightning import LightningDataModule
@@ -27,6 +32,97 @@
2732
from torchvision import transforms as transform_lib
2833

2934

35+
class _MNIST(Dataset):
36+
"""Carbon copy of ``tests.helpers.datasets.MNIST``.
37+
38+
We cannot import the tests as they are not distributed with the package.
39+
See https://github.com/PyTorchLightning/pytorch-lightning/pull/7614#discussion_r671183652 for more context.
40+
"""
41+
42+
RESOURCES = (
43+
"https://pl-public-data.s3.amazonaws.com/MNIST/processed/training.pt",
44+
"https://pl-public-data.s3.amazonaws.com/MNIST/processed/test.pt",
45+
)
46+
47+
TRAIN_FILE_NAME = "training.pt"
48+
TEST_FILE_NAME = "test.pt"
49+
cache_folder_name = "complete"
50+
51+
def __init__(
52+
self, root: str, train: bool = True, normalize: tuple = (0.1307, 0.3081), download: bool = True, **kwargs
53+
):
54+
super().__init__()
55+
self.root = root
56+
self.train = train # training set or test set
57+
self.normalize = normalize
58+
59+
self.prepare_data(download)
60+
61+
data_file = self.TRAIN_FILE_NAME if self.train else self.TEST_FILE_NAME
62+
self.data, self.targets = self._try_load(os.path.join(self.cached_folder_path, data_file))
63+
64+
def __getitem__(self, idx: int) -> Tuple[torch.Tensor, int]:
65+
img = self.data[idx].float().unsqueeze(0)
66+
target = int(self.targets[idx])
67+
68+
if self.normalize is not None and len(self.normalize) == 2:
69+
img = self.normalize_tensor(img, *self.normalize)
70+
71+
return img, target
72+
73+
def __len__(self) -> int:
74+
return len(self.data)
75+
76+
@property
77+
def cached_folder_path(self) -> str:
78+
return os.path.join(self.root, "MNIST", self.cache_folder_name)
79+
80+
def _check_exists(self, data_folder: str) -> bool:
81+
existing = True
82+
for fname in (self.TRAIN_FILE_NAME, self.TEST_FILE_NAME):
83+
existing = existing and os.path.isfile(os.path.join(data_folder, fname))
84+
return existing
85+
86+
def prepare_data(self, download: bool = True):
87+
if download and not self._check_exists(self.cached_folder_path):
88+
self._download(self.cached_folder_path)
89+
if not self._check_exists(self.cached_folder_path):
90+
raise RuntimeError("Dataset not found.")
91+
92+
def _download(self, data_folder: str) -> None:
93+
os.makedirs(data_folder, exist_ok=True)
94+
for url in self.RESOURCES:
95+
logging.info(f"Downloading {url}")
96+
fpath = os.path.join(data_folder, os.path.basename(url))
97+
urllib.request.urlretrieve(url, fpath)
98+
99+
@staticmethod
100+
def _try_load(path_data, trials: int = 30, delta: float = 1.0):
101+
"""Resolving loading from the same time from multiple concurrent processes."""
102+
res, exception = None, None
103+
assert trials, "at least some trial has to be set"
104+
assert os.path.isfile(path_data), f"missing file: {path_data}"
105+
for _ in range(trials):
106+
try:
107+
res = torch.load(path_data)
108+
# todo: specify the possible exception
109+
except Exception as e:
110+
exception = e
111+
time.sleep(delta * random.random())
112+
else:
113+
break
114+
if exception is not None:
115+
# raise the caught exception
116+
raise exception
117+
return res
118+
119+
@staticmethod
120+
def normalize_tensor(tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0) -> torch.Tensor:
121+
mean = torch.as_tensor(mean, dtype=tensor.dtype, device=tensor.device)
122+
std = torch.as_tensor(std, dtype=tensor.dtype, device=tensor.device)
123+
return tensor.sub(mean).div(std)
124+
125+
30126
def MNIST(*args, **kwargs):
31127
torchvision_mnist_available = not bool(os.getenv("PL_USE_MOCKED_MNIST", False))
32128
if torchvision_mnist_available:
@@ -39,7 +135,7 @@ def MNIST(*args, **kwargs):
39135
torchvision_mnist_available = False
40136
if not torchvision_mnist_available:
41137
print("`torchvision.datasets.MNIST` not available. Using our hosted version")
42-
from tests.helpers.datasets import MNIST
138+
MNIST = _MNIST
43139
return MNIST(*args, **kwargs)
44140

45141

pl_examples/run_examples.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash
22
set -ex
33

4+
export PYTHONPATH="${PYTHONPATH}:$(pwd)"
45
dir_path=$(dirname "${BASH_SOURCE[0]}")
56
args="
67
--data.batch_size=32

pl_examples/test_examples.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414
from unittest import mock
1515

1616
import pytest
17+
import torch
1718

1819
from pl_examples import _DALI_AVAILABLE
19-
from tests.helpers.runif import RunIf
20+
from pytorch_lightning.utilities.imports import _IS_WINDOWS
2021

2122
ARGS_DEFAULT = (
2223
"--trainer.default_root_dir %(tmpdir)s "
@@ -31,7 +32,8 @@
3132

3233

3334
@pytest.mark.skipif(not _DALI_AVAILABLE, reason="Nvidia DALI required")
34-
@RunIf(min_gpus=1, skip_windows=True)
35+
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
36+
@pytest.mark.skipif(_IS_WINDOWS, reason="Not supported on Windows")
3537
@pytest.mark.parametrize("cli_args", [ARGS_GPU])
3638
def test_examples_mnist_dali(tmpdir, cli_args):
3739
from pl_examples.integration_examples.dali_image_classifier import cli_main

requirements/test.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ coverage>5.2.0
22
codecov>=2.1
33
pytest>=6.0
44
pytest-rerunfailures>=10.2
5-
check-manifest
65
twine==3.2
76
mypy>=0.900
87
flake8>=3.9.2

setup.cfg

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,6 @@ ignore =
7373
W503 # Ignore "Line break occurred before a binary operator"
7474
E203 # Ignore "whitespace before ':'"
7575

76-
# setup.cfg or tox.ini
77-
[check-manifest]
78-
ignore =
79-
*.yml
80-
.github
81-
.github/*
82-
.circleci
83-
8476

8577
[metadata]
8678
license_file = LICENSE

setup.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ def _load_py_module(fname, pkg="pytorch_lightning"):
7474
url=about.__homepage__,
7575
download_url="https://github.com/PyTorchLightning/pytorch-lightning",
7676
license=about.__license__,
77-
packages=find_packages(exclude=["tests", "tests/*", "benchmarks", "legacy", "legacy/*"]),
77+
packages=find_packages(exclude=["tests*", "pl_examples*", "legacy*"]),
78+
include_package_data=True,
7879
long_description=long_description,
7980
long_description_content_type="text/markdown",
80-
include_package_data=True,
8181
zip_safe=False,
8282
keywords=["deep learning", "pytorch", "AI"],
8383
python_requires=">=3.6",

tests/helpers/datasets.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
from typing import Optional, Sequence, Tuple
2020

2121
import torch
22-
from torch import Tensor
2322
from torch.utils.data import Dataset
2423

2524

@@ -70,7 +69,7 @@ def __init__(
7069
data_file = self.TRAIN_FILE_NAME if self.train else self.TEST_FILE_NAME
7170
self.data, self.targets = self._try_load(os.path.join(self.cached_folder_path, data_file))
7271

73-
def __getitem__(self, idx: int) -> Tuple[Tensor, int]:
72+
def __getitem__(self, idx: int) -> Tuple[torch.Tensor, int]:
7473
img = self.data[idx].float().unsqueeze(0)
7574
target = int(self.targets[idx])
7675

@@ -126,7 +125,7 @@ def _try_load(path_data, trials: int = 30, delta: float = 1.0):
126125
return res
127126

128127
@staticmethod
129-
def normalize_tensor(tensor: Tensor, mean: float = 0.0, std: float = 1.0) -> Tensor:
128+
def normalize_tensor(tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0) -> torch.Tensor:
130129
mean = torch.as_tensor(mean, dtype=tensor.dtype, device=tensor.device)
131130
std = torch.as_tensor(std, dtype=tensor.dtype, device=tensor.device)
132131
return tensor.sub(mean).div(std)

tests/special_tests.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ fi
8181
# report+="Ran\ttests/plugins/environments/torch_elastic_deadlock.py\n"
8282

8383
# test that a user can manually launch individual processes
84+
export PYTHONPATH="${PYTHONPATH}:$(pwd)"
8485
args="--trainer.gpus 2 --trainer.strategy ddp --trainer.max_epochs=1 --trainer.limit_train_batches=1 --trainer.limit_val_batches=1 --trainer.limit_test_batches=1"
8586
MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=1 python pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py ${args} &
8687
MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=0 python pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py ${args}

0 commit comments

Comments
 (0)