Skip to content

Commit d12d6d0

Browse files
bertmaherfacebook-github-bot
bertmaher
authored and
facebook-github-bot
committed
Register backend-parameterized tests inside lib/Backends (#3476)
Summary: This diff shows a possible approach to moving our backend parameterization of tests into lib/Backends. The basic problem we're trying to solve is that vendors maintaining private backends may want to use, e.g., the OperatorTest suite to verify conformance, but currently the "ENABLED_BACKENDS" mechanism requires us to list each supported backend, for each test, in the source code. Obviously, MyStealthModeStartup is not going to want that name appearing in the open-source operator test, so they've got to maintain a private, forked version of OperatorTest that enables all the tests relevant to their architecture. By moving the parameterization into lib/Backends/*, we just have a test consisting of a simple source file listing the instantiated test cases, e.g., ``` INSTANTIATE_GLOW_BACKEND_TEST(MyStealthModeStartup, OperatorTest, Conv2D); ``` Obviously the provided test `NewTest` is just a sample that wouldn't land; I ripped one of the cases out of OperatorTest for ease of editing. I'll say: - There's a bit more ceremony involved than I'd strictly like (mostly because the CMake lib list is pretty unwieldy). - It feels odd to define all the tests in a header file, and put just the instantiates in a .cpp file. I couldn't see a good way around this, though, since gtest requires you to instantiate entire test suites, rather than individual tests, and we need finer-grain control over test enablement. - I don't love making the include path start with "tests/", maybe there's some way around this. Pull Request resolved: #3476 Test Plan: eventually, run all the tests. Reviewed By: rdzhabarov Differential Revision: D17215560 Pulled By: bertmaher fbshipit-source-id: 5562b9725e6489d624508eb68594af289eacfe7f
1 parent fc9d1b1 commit d12d6d0

File tree

66 files changed

+2253
-566
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+2253
-566
lines changed

‎CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ include(GlowExternalBackends)
5353
include(SanitizerSupport)
5454
include(CoverageSupport)
5555
include(DoxygenSupport)
56+
include(FindBackends)
57+
5658
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/CMakeGraphVizOptions.cmake
5759
${CMAKE_CURRENT_BINARY_DIR}/CMakeGraphVizOptions.cmake COPYONLY)
5860

‎cmake/modules/FindBackends.cmake

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
set(GLOW_BACKENDS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib/Backends")
2+
file(GLOB subdirs RELATIVE "${GLOW_BACKENDS_DIR}" "${GLOW_BACKENDS_DIR}/*")
3+
foreach(object ${subdirs})
4+
if(IS_DIRECTORY "${GLOW_BACKENDS_DIR}/${object}")
5+
set(backendEnabled "GLOW_WITH_${object}")
6+
string(TOUPPER "${backendEnabled}" backendEnabled)
7+
if(NOT DEFINED ${backendEnabled} OR ${backendEnabled})
8+
list(APPEND GLOW_BACKENDS "${object}")
9+
endif()
10+
endif()
11+
endforeach()

‎cmake/modules/GlowTestSupport.cmake

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,33 @@ function(add_glow_test)
5454
set_property(TEST ${ARG_NAME} PROPERTY LABELS EXPENSIVE)
5555
endif()
5656
endfunction()
57+
58+
# Adds a backend-parameterized test. These tests can be instantiated for any
59+
# backend present in lib/Backends, and allow similar functionality to easily be
60+
# tested across all defined backends.
61+
function(add_backend_test)
62+
set(options UNOPT EXPENSIVE)
63+
set(oneValueArgs BACKEND TEST)
64+
set(multiValueArgs PRIVATE)
65+
cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
66+
set(test "${ARG_BACKEND}${ARG_TEST}")
67+
set(SOURCES "${ARG_TEST}.cpp")
68+
set(BLACKLIST_SOURCE "${GLOW_BACKENDS_DIR}/${ARG_BACKEND}/tests/${test}.cpp")
69+
if(NOT EXISTS ${BLACKLIST_SOURCE})
70+
return()
71+
endif()
72+
list(APPEND SOURCES ${BLACKLIST_SOURCE})
73+
add_executable("${test}" ${SOURCES})
74+
target_compile_definitions("${test}" PRIVATE -DGLOW_TEST_BACKEND=${ARG_BACKEND})
75+
target_link_libraries("${test}"
76+
PRIVATE BackendTestUtils TestMain ${ARG_PRIVATE})
77+
if(${ARG_EXPENSIVE})
78+
set(ARG_EXPENSIVE EXPENSIVE)
79+
else()
80+
set(ARG_EXPENSIVE)
81+
endif()
82+
add_glow_test("${test}" ${ARG_EXPENSIVE} ${GLOW_BINARY_DIR}/tests/${test} --gtest_output=xml:${test}.xml )
83+
if(${ARG_UNOPT})
84+
list(APPEND UNOPT_TESTS ./tests/${test} -optimize-ir=false &&)
85+
endif()
86+
endfunction()

‎docs/Backends.md

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,31 @@ The `tools/ClassGen/Backends/CPU/CPUSpecificNodes.h` and
214214
`tools/ClassGen/Backends/CPU/CPUSpecificInstrs.h` files are included in
215215
`tools/ClassGen/NodeGen.cpp` and `tools/ClassGen/InstrGen.cpp`, respectively.
216216

217+
#### Backend Parameterized Tests
218+
219+
Glow provides several test suites that are parameterized by backend. An
220+
example of such a suite is `tests/unittests/OperatorTest.cpp`, which defines
221+
small tests of Glow operators. These tests can be executed against any backend
222+
to check compliance.
223+
224+
These tests will only be run for a backend if a corresponding
225+
`lib/Backends/$BACKEND/tests` directory is found and contains a corresponding
226+
`${BACKEND}${TEST}.cpp` file containing a blacklist definition, e.g.:
227+
```
228+
std::set<std::string> glow::backendTestBlacklist = {};
229+
```
230+
231+
This blacklist can be used to exclude any unsupported tests while a backend is
232+
a work-in-progress. See the Interpreter and CPU backends for examples of
233+
setting up and using these tests. To bootstrap a blacklist, we recommend using
234+
a simple shell script to check which tests already work:
235+
```
236+
for test in $(tests/ExampleBackendOperatorTest --gtest_list_tests); do
237+
if ! tests/ExampleBackendOperatorTest --gtest_filter="$test" >& /dev/null; then
238+
echo $test
239+
fi
240+
done
241+
```
217242

218243
#### External backends
219244

@@ -223,4 +248,4 @@ An external backend is provided as a single source directory. It can then be dev
223248

224249
The external backend mechanism is for instance convenient for adding closed source backends to Glow.
225250

226-
The structure of external backends is defined [here](https://github.com/pytorch/glow/blob/master/docs/ExternalBackend.md).
251+
The structure of external backends is defined [here](https://github.com/pytorch/glow/blob/master/docs/ExternalBackend.md).

‎lib/Backends/CPU/tests/CPUBackendCorrectnessTest.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {};

‎lib/Backends/CPU/tests/CPUBackendTest.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {};

‎lib/Backends/CPU/tests/CPUDeviceManagerTest.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {};

‎lib/Backends/CPU/tests/CPUGradCheckTest.cpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {
21+
"gradientCheckAdaptiveAvgPool/0",
22+
"gradientCheckCrossEntropyLoss/0",
23+
};

‎lib/Backends/CPU/tests/CPUGraphOptzTest.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {};

‎lib/Backends/CPU/tests/CPUMLTest.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {};

‎lib/Backends/CPU/tests/CPUOperatorGradTest.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {};

‎lib/Backends/CPU/tests/CPUOperatorTest.cpp

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tests/unittests/BackendTestUtils.h"
17+
18+
using namespace glow;
19+
20+
std::set<std::string> glow::backendTestBlacklist = {
21+
"less_float16Cases/0",
22+
"less_int64Cases/0",
23+
"ResizeNearest_Float/0",
24+
"ResizeNearest_Float16/0",
25+
"ResizeNearest_Int8/0",
26+
"ResizeNearest_Int16/0",
27+
"ResizeNearest_Int32/0",
28+
"replaceNaN_Float16/0",
29+
"Logit_Float16/0",
30+
"FP16Add/0",
31+
"FP16Matmul/0",
32+
"batchedReduceAdd_Float16/0",
33+
"batchedReduceZeroDimResult_Float16/0",
34+
"batchedReduceAddWithAxis_Float16/0",
35+
"ReluSimple_Float16/0",
36+
"PReluSimple_Float16/0",
37+
"GatherDataFloat16IdxInt32/0",
38+
"GatherDataFloat16IdxInt64/0",
39+
"GatherRangesDataFloat16IdxInt32/0",
40+
"GatherRangesDataFloat16IdxInt64/0",
41+
"FP16Transpose2Dims/0",
42+
"Transpose3Dims_Float16/0",
43+
"ArithAdd_int32_t/0",
44+
"ArithAdd_int64_t/0",
45+
"ArithAdd_float16_t/0",
46+
"ArithSub_int32_t/0",
47+
"ArithSub_int64_t/0",
48+
"ArithSub_float16_t/0",
49+
"ArithMul_int32_t/0",
50+
"ArithMul_int64_t/0",
51+
"ArithMul_float16_t/0",
52+
"ArithMax_int32_t/0",
53+
"ArithMax_int64_t/0",
54+
"ArithMax_float16_t/0",
55+
"ArithMin_int32_t/0",
56+
"ArithMin_int64_t/0",
57+
"ArithMin_float16_t/0",
58+
"convTest_Float16/0",
59+
"FP16Max/0",
60+
"concatVectors_Int32/0",
61+
"concatVectors_Float16/0",
62+
"concatVectorsRepeated_Int32/0",
63+
"concatVectorsRepeated_Float16/0",
64+
"sliceVectors_Float16/0",
65+
"sliceConcatVectors_Float16/0",
66+
"ExpandDims_Float16/0",
67+
"Split_Float16/0",
68+
"Fp16Splat/0",
69+
"GroupConv3D/0",
70+
"NonCubicPaddingConv3D/0",
71+
"FP16AvgPool/0",
72+
"AdaptiveAvgPool/0",
73+
"FP16AdaptiveAvgPool/0",
74+
"Int8AdaptiveAvgPool/0",
75+
"AdaptiveAvgPoolNonSquare/0",
76+
"FP16MaxPool/0",
77+
"NonCubicKernelConv3D/0",
78+
"NonCubicKernelConv3DQuantized/0",
79+
"NonCubicStrideConv3D/0",
80+
"FP16BatchAdd/0",
81+
"Sigmoid_Float16/0",
82+
"testBatchAdd_Float16/0",
83+
"SparseLengthsSum_Float16/0",
84+
"SparseLengthsSumI8/0",
85+
"SparseLengthsWeightedSum_1D_Float16/0",
86+
"SparseLengthsWeightedSum_2D_Float16/0",
87+
"SparseLengthsWeightedSumI8/0",
88+
"RowwiseQuantizedSparseLengthsWeightedSum_Float16_AccumFloat/0",
89+
"RowwiseQuantizedSparseLengthsWeightedSum_Float16_AccumFloat16/0",
90+
"RowwiseQuantizedSparseLengthsSum_Float16_AccumFloat/0",
91+
"RowwiseQuantizedSparseLengthsSum_Float16_AccumFloat16/0",
92+
"FusedRowwiseQuantizedSparseLengthsWeightedSum_Float16_AccumFloat/0",
93+
"FusedRowwiseQuantizedSparseLengthsWeightedSum_Float16_AccumFloat16/0",
94+
"FusedRowwiseQuantizedSparseLengthsWeightedSum_ConvertedFloat16/0",
95+
"FusedRowwiseQuantizedSparseLengthsSum_Float16_AccumFloat/0",
96+
"FusedRowwiseQuantizedSparseLengthsSum_Float16_AccumFloat16/0",
97+
"SparseToDenseMask1/0",
98+
"SparseToDenseMask2/0",
99+
"FP16Reshape/0",
100+
"sliceReshape_Float16/0",
101+
"Flatten_Float16Ty/0",
102+
"Bucketize/0",
103+
"FP16SoftMax/0",
104+
"BatchOneHotDataFloat/0",
105+
"BatchOneHotDataFloat16/0",
106+
"BatchOneHotDataInt64/0",
107+
"BatchOneHotDataInt32/0",
108+
"BatchOneHotDataInt8/0",
109+
"dotProduct1D_Float16/0",
110+
"dotProduct2D_Float16/0",
111+
"BatchBoxCox_Float/0",
112+
"BatchBoxCox_Float16/0",
113+
"ConvertFrom_FloatTy_To_Float16Ty/0",
114+
"ConvertFrom_FloatTy_To_Int32ITy/0",
115+
"ConvertFrom_FloatTy_To_Int64ITy/0",
116+
"ConvertFrom_Float16Ty_To_FloatTy/0",
117+
"ConvertFrom_Float16Ty_To_Float16Ty/0",
118+
"ConvertFrom_Float16Ty_To_Int32ITy/0",
119+
"ConvertFrom_Float16Ty_To_Int64ITy/0",
120+
"ConvertFrom_Int32ITy_To_FloatTy/0",
121+
"ConvertFrom_Int32ITy_To_Float16Ty/0",
122+
"ConvertFrom_Int32ITy_To_Int64ITy/0",
123+
"ConvertFrom_Int64ITy_To_FloatTy/0",
124+
"ConvertFrom_Int64ITy_To_Float16Ty/0",
125+
"ConvertFrom_Int64ITy_To_Int32ITy/0",
126+
"ConvertFrom_FloatTy_To_Float16Ty_AndBack/0",
127+
"ConvertFrom_FloatTy_To_Int32ITy_AndBack/0",
128+
"ConvertFrom_FloatTy_To_Int64ITy_AndBack/0",
129+
"ConvertFrom_Float16Ty_To_FloatTy_AndBack/0",
130+
"ConvertFrom_Float16Ty_To_Float16Ty_AndBack/0",
131+
"ConvertFrom_Float16Ty_To_Int32ITy_AndBack/0",
132+
"ConvertFrom_Float16Ty_To_Int64ITy_AndBack/0",
133+
"ConvertFrom_Int32ITy_To_FloatTy_AndBack/0",
134+
"ConvertFrom_Int32ITy_To_Float16Ty_AndBack/0",
135+
"ConvertFrom_Int64ITy_To_FloatTy_AndBack/0",
136+
"ConvertFrom_Int64ITy_To_Float16Ty_AndBack/0",
137+
"ConvertFrom_Int64ITy_To_Int32ITy_AndBack/0",
138+
"BasicDivNetFloatVsInt8/0",
139+
"BasicAddNetFloatVsFloat16/0",
140+
"BasicSubNetFloatVsFloat16/0",
141+
"BasicMulNetFloatVsFloat16/0",
142+
"BasicDivNetFloatVsFloat16/0",
143+
"BasicMaxNetFloatVsFloat16/0",
144+
"BasicMinNetFloatVsFloat16/0",
145+
"Int16ConvolutionDepth10/0",
146+
"Int16ConvolutionDepth8/0",
147+
"FP16ConvolutionDepth10/0",
148+
"FP16ConvolutionDepth8/0",
149+
"FC_Float16/0",
150+
"Tanh_Float16/0",
151+
"Exp_Float16/0",
152+
"rowwiseQuantizedSLWSTest/0",
153+
};

0 commit comments

Comments
 (0)