From b0a84c6b5d45e47e0a77898b58733e3fd166e477 Mon Sep 17 00:00:00 2001 From: jithunnair-amd Date: Mon, 16 Jul 2018 18:40:39 -0500 Subject: [PATCH 1/6] Disable certain unit tests in test_utils for ROCm builds to establish baseline tests for ROCm CI; add override so user can run all tests if desired --- .jenkins/pytorch/build.sh | 1 + test/common.py | 3 ++- test/test_utils.py | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh index bb06df22868ea..3e400d5993441 100755 --- a/.jenkins/pytorch/build.sh +++ b/.jenkins/pytorch/build.sh @@ -33,6 +33,7 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then export HCC_AMDGPU_TARGET=gfx900 export LANG=C.UTF-8 export LC_ALL=C.UTF-8 + export PYTORCH_BUILT_WITH_ROCM=1 sudo chown -R jenkins:jenkins /usr/local rm -rf "$(dirname "${BASH_SOURCE[0]}")/../../../pytorch_amd/" || true diff --git a/test/common.py b/test/common.py index 5debc66905624..e15b3f91ef021 100644 --- a/test/common.py +++ b/test/common.py @@ -73,7 +73,8 @@ def run_tests(argv=UNITTEST_ARGS): NO_MULTIPROCESSING_SPAWN = os.environ.get('NO_MULTIPROCESSING_SPAWN', '0') == '1' TEST_WITH_ASAN = os.getenv('PYTORCH_TEST_WITH_ASAN', '0') == '1' TEST_WITH_UBSAN = os.getenv('PYTORCH_TEST_WITH_UBSAN', '0') == '1' - +BUILT_WITH_ROCM = os.getenv('PYTORCH_BUILT_WITH_ROCM', '0') == '1' +OVERRIDE_DISABLED_UNIT_TESTS_ROCM = os.getenv('PYTORCH_OVERRIDE_DISABLED_UNIT_TESTS_ROCM', '0') == '1' def skipIfNoLapack(fn): @wraps(fn) diff --git a/test/test_utils.py b/test/test_utils.py index e8c33ca761c7e..1cda4e57a44d3 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -19,7 +19,8 @@ from torch.utils.trainer.plugins.plugin import Plugin from torch.autograd._functions.utils import prepare_onnx_paddings from torch.autograd._functions.utils import check_onnx_broadcast -from common import IS_WINDOWS, IS_PPC +from common import IS_WINDOWS, IS_PPC, BUILT_WITH_ROCM, OVERRIDE_DISABLED_UNIT_TESTS_ROCM + HAS_CUDA = torch.cuda.is_available() @@ -412,6 +413,7 @@ def test_cpu(self): @unittest.skipIf(not HAS_CFFI or not HAS_CUDA, "ffi tests require cffi package") @unittest.skipIf(IS_WINDOWS, "ffi doesn't currently work on Windows") + @unittest.skipIf(BUILT_WITH_ROCM and not OVERRIDE_DISABLED_UNIT_TESTS_ROCM, "Test disabled for ROCm") def test_gpu(self): create_extension( name='gpulib', @@ -615,6 +617,7 @@ def test_bottleneck_cpu_only(self): self._check_cuda(out) @unittest.skipIf(not HAS_CUDA, 'No CUDA') + @unittest.skipIf(BUILT_WITH_ROCM and not OVERRIDE_DISABLED_UNIT_TESTS_ROCM, "Test disabled for ROCm") def test_bottleneck_cuda(self): rc, out, err = self._run_bottleneck('bottleneck/test_cuda.py') self.assertEqual(rc, 0, 'Run failed with\n{}'.format(err)) From 62ebece3716f914a3b78c3268cd1f4c71ad7e15d Mon Sep 17 00:00:00 2001 From: jithunnair-amd Date: Wed, 18 Jul 2018 10:32:40 -0500 Subject: [PATCH 2/6] Remove OVERRIDE_DISABLED_UNIT_TESTS_ROCM define as one can just NOT define BUILT_WITH_ROCM to run all unit tests --- test/common.py | 2 +- test/test_utils.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/test/common.py b/test/common.py index e15b3f91ef021..8ee3b2be85515 100644 --- a/test/common.py +++ b/test/common.py @@ -74,7 +74,7 @@ def run_tests(argv=UNITTEST_ARGS): TEST_WITH_ASAN = os.getenv('PYTORCH_TEST_WITH_ASAN', '0') == '1' TEST_WITH_UBSAN = os.getenv('PYTORCH_TEST_WITH_UBSAN', '0') == '1' BUILT_WITH_ROCM = os.getenv('PYTORCH_BUILT_WITH_ROCM', '0') == '1' -OVERRIDE_DISABLED_UNIT_TESTS_ROCM = os.getenv('PYTORCH_OVERRIDE_DISABLED_UNIT_TESTS_ROCM', '0') == '1' + def skipIfNoLapack(fn): @wraps(fn) diff --git a/test/test_utils.py b/test/test_utils.py index 1cda4e57a44d3..dcafe5103f9c0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -19,8 +19,7 @@ from torch.utils.trainer.plugins.plugin import Plugin from torch.autograd._functions.utils import prepare_onnx_paddings from torch.autograd._functions.utils import check_onnx_broadcast -from common import IS_WINDOWS, IS_PPC, BUILT_WITH_ROCM, OVERRIDE_DISABLED_UNIT_TESTS_ROCM - +from common import IS_WINDOWS, IS_PPC, BUILT_WITH_ROCM HAS_CUDA = torch.cuda.is_available() @@ -413,7 +412,7 @@ def test_cpu(self): @unittest.skipIf(not HAS_CFFI or not HAS_CUDA, "ffi tests require cffi package") @unittest.skipIf(IS_WINDOWS, "ffi doesn't currently work on Windows") - @unittest.skipIf(BUILT_WITH_ROCM and not OVERRIDE_DISABLED_UNIT_TESTS_ROCM, "Test disabled for ROCm") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_gpu(self): create_extension( name='gpulib', @@ -617,7 +616,7 @@ def test_bottleneck_cpu_only(self): self._check_cuda(out) @unittest.skipIf(not HAS_CUDA, 'No CUDA') - @unittest.skipIf(BUILT_WITH_ROCM and not OVERRIDE_DISABLED_UNIT_TESTS_ROCM, "Test disabled for ROCm") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_bottleneck_cuda(self): rc, out, err = self._run_bottleneck('bottleneck/test_cuda.py') self.assertEqual(rc, 0, 'Run failed with\n{}'.format(err)) From 7f3302c4ea7cd2986c2414912d429b6c81345279 Mon Sep 17 00:00:00 2001 From: jithunnair-amd Date: Wed, 18 Jul 2018 16:26:13 -0500 Subject: [PATCH 3/6] Skip 3 tests in test_autograd that currently don't work for ROCm builds --- test/test_autograd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_autograd.py b/test/test_autograd.py index 3b519ea5a8687..88b7954be3d7b 100644 --- a/test/test_autograd.py +++ b/test/test_autograd.py @@ -14,7 +14,7 @@ from torch.autograd.function import once_differentiable from torch.autograd.profiler import profile from common import TEST_MKL, TestCase, run_tests, skipIfNoLapack, \ - suppress_warnings, skipIfNoZeroSize + suppress_warnings, skipIfNoZeroSize, BUILT_WITH_ROCM from torch.autograd import Variable, Function, detect_anomaly from torch.autograd.function import InplaceFunction from torch.testing import make_non_contiguous, randn_like @@ -1561,6 +1561,7 @@ def test_pyscalar_conversions(self): self._test_pyscalar_conversions(lambda x: x.cuda(), lambda x: long(x)) @unittest.skipIf(not torch.cuda.is_available(), "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_pin_memory(self): x = torch.randn(2, 2, requires_grad=True) self.assertEqual(x, x.pin_memory()) @@ -2359,6 +2360,7 @@ def f3(dt): f(dt) @unittest.skipIf(not torch.cuda.is_available(), "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_set_requires_grad_only_for_floats_cuda(self): self._test_set_requires_grad_only_for_floats(self, True) @@ -2366,6 +2368,7 @@ def test_set_requires_grad_only_for_floats(self): self._test_set_requires_grad_only_for_floats(self, False) @unittest.skipIf(not torch.cuda.is_available(), "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_rnn_backward_to_input_but_not_parameters_cuda(self): # this checks whether it is possible to not require # weight parameters, but require inputs, see #7722 From 2eed71c0d1742dba31227271a281c2237c6aca95 Mon Sep 17 00:00:00 2001 From: jithunnair-amd Date: Wed, 18 Jul 2018 17:00:20 -0500 Subject: [PATCH 4/6] Skip 1 test in test_optim that currently doesn't work for ROCm builds --- test/test_optim.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_optim.py b/test/test_optim.py index 57fe9e5da5394..67328919c32df 100644 --- a/test/test_optim.py +++ b/test/test_optim.py @@ -10,7 +10,7 @@ from torch.autograd import Variable from torch import sparse from torch.optim.lr_scheduler import LambdaLR, StepLR, MultiStepLR, ExponentialLR, CosineAnnealingLR, ReduceLROnPlateau -from common import TestCase, run_tests, TEST_WITH_UBSAN +from common import TestCase, run_tests, TEST_WITH_UBSAN, BUILT_WITH_ROCM def rosenbrock(tensor): @@ -437,6 +437,7 @@ def test_asgd(self): with self.assertRaisesRegex(ValueError, "Invalid weight_decay value: -0.5"): optim.ASGD(None, lr=1e-2, weight_decay=-0.5) + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_rprop(self): self._test_rosenbrock( lambda params: optim.Rprop(params, lr=1e-3), From ca0a693990cef22afbcc13aa42ec3ae84fb605ea Mon Sep 17 00:00:00 2001 From: jithunnair-amd Date: Wed, 18 Jul 2018 17:59:45 -0500 Subject: [PATCH 5/6] Skip 10 tests in test_jit that currently don't work for ROCm builds --- test/test_jit.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/test_jit.py b/test/test_jit.py index 0663b41b67e08..f93985281522b 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -9,7 +9,7 @@ from torch.autograd.function import traceable from torch.testing import assert_allclose from torch.onnx import OperatorExportTypes -from common import TestCase, run_tests, IS_WINDOWS, TEST_WITH_UBSAN +from common import TestCase, run_tests, IS_WINDOWS, TEST_WITH_UBSAN, BUILT_WITH_ROCM from textwrap import dedent import os import io @@ -344,6 +344,7 @@ def forward(self, x): # TODO: Fuser doesn't work at all when inputs require grad. Fix that @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_lstm_fusion_cuda(self): inputs = get_lstm_inputs('cuda') ge = self.checkTrace(LSTMCellF, inputs) @@ -367,6 +368,7 @@ def test_lstm_fusion_cpu(self): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_lstm_fusion_concat(self): inputs = get_lstm_inputs('cuda') ge = self.checkTrace(LSTMCellC, inputs) @@ -374,6 +376,7 @@ def test_lstm_fusion_concat(self): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_concat_fusion(self): hx = torch.randn(3, 20, dtype=torch.float, device='cuda') cx = torch.randn(3, 20, dtype=torch.float, device='cuda') @@ -386,6 +389,7 @@ def foo(hx, cx): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_fusion_distribute(self): def f(x, y): z1, z2 = (x + y).chunk(2, dim=1) @@ -407,6 +411,7 @@ def fn_test_comparison_gt_lt(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_comparison_gt_lt(self): x = torch.randn(4, 4, dtype=torch.float, device='cuda') y = torch.randn(4, 4, dtype=torch.float, device='cuda') @@ -415,6 +420,7 @@ def test_comparison_gt_lt(self): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_comparison_ge_le(self): def f(x, y): mask = (x >= 0).type_as(x) @@ -434,6 +440,7 @@ def fn_test_relu(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_relu(self): x = torch.randn(4, 4, dtype=torch.float, device='cuda') y = torch.randn(4, 4, dtype=torch.float, device='cuda') @@ -446,6 +453,7 @@ def fn_test_exp(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_exp(self): x = torch.randn(4, 4, dtype=torch.float, device='cuda') y = torch.randn(4, 4, dtype=torch.float, device='cuda') @@ -790,6 +798,7 @@ def doit(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "cpp tests require CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_cpp(self): # rather than rebuild assertExpected in cpp, # just glob all the cpp outputs into one file for now @@ -2005,6 +2014,7 @@ def test_tensor_number_math(self): self._test_tensor_number_math() @unittest.skipIf(not RUN_CUDA, "No CUDA") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_tensor_number_math_cuda(self): self._test_tensor_number_math(device='cuda') From 651d407bbb4498cb67be71f9e1d68a6269b52b10 Mon Sep 17 00:00:00 2001 From: jithunnair-amd Date: Wed, 18 Jul 2018 19:58:50 -0500 Subject: [PATCH 6/6] Skip 8 tests in test_dataloader that currently don't work for ROCm builds --- test/test_dataloader.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/test_dataloader.py b/test/test_dataloader.py index ef636a1ec77e7..d9a03f3401158 100644 --- a/test/test_dataloader.py +++ b/test/test_dataloader.py @@ -13,7 +13,7 @@ from torch.utils.data import Dataset, TensorDataset, DataLoader, ConcatDataset from torch.utils.data.dataset import random_split from torch.utils.data.dataloader import default_collate, ExceptionWrapper, MANAGER_STATUS_CHECK_INTERVAL -from common import TestCase, run_tests, TEST_NUMPY, IS_WINDOWS, NO_MULTIPROCESSING_SPAWN +from common import TestCase, run_tests, TEST_NUMPY, IS_WINDOWS, NO_MULTIPROCESSING_SPAWN, BUILT_WITH_ROCM # We cannot import TEST_CUDA from common_nn here, because if we do that, # the TEST_CUDNN line from common_nn will be executed multiple times @@ -338,12 +338,14 @@ def test_growing_dataset(self): self.assertEqual(len(dataloader_shuffle), 5) @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_sequential_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, pin_memory=True) for input, target in loader: self.assertTrue(input.is_pinned()) self.assertTrue(target.is_pinned()) + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_multiple_dataloaders(self): loader1_it = iter(DataLoader(self.dataset, num_workers=1)) loader2_it = iter(DataLoader(self.dataset, num_workers=2)) @@ -444,6 +446,7 @@ def test_batch_sampler(self): self._test_batch_sampler(num_workers=4) @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_shuffle_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, shuffle=True, num_workers=4, pin_memory=True) for input, target in loader: @@ -476,6 +479,7 @@ def test_error_workers(self): @unittest.skipIf(IS_WINDOWS, "FIXME: stuck test") @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_partial_workers(self): "check that workers exit even if the iterator is not exhausted" loader = iter(DataLoader(self.dataset, batch_size=2, num_workers=4, pin_memory=True)) @@ -529,6 +533,7 @@ def _is_process_alive(pid, pname): "spawn start method is not supported in Python 2, \ but we need it for creating another process with CUDA") @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_manager_unclean_exit(self): '''there might be ConnectionResetError or leaked semaphore warning (due to dirty process exit), \ but they are all safe to ignore''' @@ -632,6 +637,7 @@ def setUp(self): self.dataset = StringDataset() @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_shuffle_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, shuffle=True, num_workers=4, pin_memory=True) for batch_ndx, (s, n) in enumerate(loader): @@ -675,6 +681,7 @@ def test_sequential_batch(self): self.assertEqual(n[1], idx + 1) @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, pin_memory=True) for batch_ndx, sample in enumerate(loader): @@ -714,6 +721,7 @@ def _run_ind_worker_queue_test(self, batch_size, num_workers): if current_worker_idx == num_workers: current_worker_idx = 0 + @unittest.skipIf(BUILT_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_ind_worker_queue(self): for batch_size in (8, 16, 32, 64): for num_workers in range(1, 6):