Fix OSS build + separate test build into its own CMakeLists.txt (#3724)

Summary:
Pull Request resolved: #3724

## Context

As title. In the next diff, the vulkan test binary will be added to CI.

imported-using-ghimport

Test Plan: Imported from OSS

Reviewed By: kirklandsign

Differential Revision: D57747739

Pulled By: SS-JIA

fbshipit-source-id: 0829abb08849d5349751c63d2c0c5a5dc39ca553

Author: SS-JIA

backends/vulkan/CMakeLists.txt

@@ -40,52 +40,43 @@ include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 
 include(cmake/ShaderLibrary.cmake)
 
-# Third party libs
+# Third party include paths
 
 set(VULKAN_THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/third-party)
 
 set(VULKAN_HEADERS_PATH ${VULKAN_THIRD_PARTY_PATH}/Vulkan-Headers/include)
 set(VOLK_PATH ${VULKAN_THIRD_PARTY_PATH}/volk)
 set(VMA_PATH ${VULKAN_THIRD_PARTY_PATH}/VulkanMemoryAllocator)
 
+set(COMMON_INCLUDES ${EXECUTORCH_ROOT}/.. ${VULKAN_HEADERS_PATH} ${VOLK_PATH}
+                    ${VMA_PATH}
+)
+
 # Compile settings
 
 set(VULKAN_CXX_FLAGS "")
 list(APPEND VULKAN_CXX_FLAGS "-DUSE_VULKAN_WRAPPER")
 list(APPEND VULKAN_CXX_FLAGS "-DUSE_VULKAN_VOLK")
 
-# Vulkan Compute API
+# vulkan API files
 
-file(GLOB vulkan_api_cpp ${RUNTIME_PATH}/api/*.cpp)
+file(GLOB_RECURSE vulkan_api_cpp ${RUNTIME_PATH}/api/*)
 list(APPEND vulkan_api_cpp ${VOLK_PATH}/volk.c)
 
-set(COMMON_INCLUDES ${EXECUTORCH_ROOT}/.. ${VULKAN_HEADERS_PATH} ${VOLK_PATH}
-                    ${VMA_PATH}
-)
-
-# vulkan_graph_lib
+# vulkan ComputeGraph files
 
 file(GLOB_RECURSE vulkan_graph_cpp ${RUNTIME_PATH}/graph/*)
 list(APPEND vulkan_graph_cpp ${vulkan_api_cpp})
 
-add_library(vulkan_graph_lib STATIC ${vulkan_graph_cpp})
-target_include_directories(vulkan_graph_lib PRIVATE ${COMMON_INCLUDES})
-target_compile_options(vulkan_graph_lib PRIVATE ${VULKAN_CXX_FLAGS})
-# Link this library with --whole-archive due to dynamic operator registrations
-target_link_options_shared_lib(vulkan_graph_lib)
-
-# vulkan_standard_shaders
+# Standard GLSL shader library
 
 set(VULKAN_GRAPH_SHADERS_PATH ${RUNTIME_PATH}/graph/ops/glsl/)
-# Declares a vulkan_standard_shaders library containing compiled SPIR-V shaders
-# from the above path
-vulkan_shader_library(${VULKAN_GRAPH_SHADERS_PATH} vulkan_standard_shaders)
-# preserve the path of the generated cpp file
-set(vulkan_standard_shaders_cpp
-    ${CMAKE_BINARY_DIR}/vulkan_standard_shaders/spv.cpp
-)
+# Generates a spv.cpp file containing compiled GLSL shaders
+gen_vulkan_shader_lib_cpp(${VULKAN_GRAPH_SHADERS_PATH})
+# Save the path of the generated cpp file
+set(vulkan_standard_shaders_cpp ${generated_spv_cpp})
 
-# Generate Files from flatc
+# Generate Vulkan Delegate Schema Files from flatc
 
 set(SCHEMA_INCLUDE_DIR ${CMAKE_BINARY_DIR}/schema/include)
 
@@ -151,31 +142,6 @@ endif()
 
 # Test targets
 
-if(EXECUTORCH_BUILD_GTESTS)
-  set(TEST_UTILS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/test/utils)
-  file(GLOB TEST_UTILS_CPP ${CMAKE_CURRENT_SOURCE_DIR}/test/utils/*.cpp)
-
-  set(TEST_SHADERS_PATH ${CMAKE_CURRENT_SOURCE_DIR}/test/glsl)
-  vulkan_shader_library(${TEST_SHADERS_PATH} test_shaderlib)
-
-  # vulkan_compute_api_test
-  set(COMPUTE_API_TEST_CPP
-      ${CMAKE_CURRENT_SOURCE_DIR}/test/vulkan_compute_api_test.cpp
-  )
-
-  add_executable(
-    vulkan_compute_api_test ${COMPUTE_API_TEST_CPP} ${TEST_UTILS_CPP}
-  )
-  target_include_directories(
-    vulkan_compute_api_test PRIVATE ${COMMON_INCLUDES} ${TEST_UTILS_HEADERS}
-  )
-  target_link_libraries(
-    vulkan_compute_api_test PRIVATE gtest_main vulkan_graph_lib
-                                    vulkan_standard_shaders test_shaderlib
-  )
-  target_compile_options(vulkan_compute_api_test PRIVATE ${VULKAN_CXX_FLAGS})
-endif()
-
 install(
   TARGETS vulkan_backend
   DESTINATION lib

backends/vulkan/cmake/ShaderLibrary.cmake

@@ -14,6 +14,16 @@
 #
 # The targets in this file will be built if EXECUTORCH_BUILD_VULKAN is ON
 
+if(NOT PYTHON_EXECUTABLE)
+  message(
+    "WARNING: PYTHON_EXECUTABLE is not set! A failure is likely imminent."
+  )
+endif()
+
+if(NOT EXECUTORCH_ROOT)
+  message("WARNING: EXECUTORCH_ROOT is not set! A failure is likely imminent.")
+endif()
+
 if(ANDROID)
   if(NOT ANDROID_NDK)
     message(FATAL_ERROR "ANDROID_NDK not set")
@@ -37,30 +47,67 @@ endif()
 # Required to enable linking with --whole-archive
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 
-# Convenience macro to generate a SPIR-V shader library target. Given the path
-# to the shaders to compile and the name of the library, it will create a static
-# library containing the generated SPIR-V shaders. The generated_spv_cpp
-# variable can be used to reference the generated CPP file outside the macro.
-macro(VULKAN_SHADER_LIBRARY shaders_path library_name)
+function(gen_vulkan_shader_lib_cpp shaders_path)
   set(VULKAN_SHADERGEN_ENV "")
-  set(VULKAN_SHADERGEN_OUT_PATH ${CMAKE_BINARY_DIR}/${library_name})
+  set(VULKAN_SHADERGEN_OUT_PATH ${CMAKE_BINARY_DIR}/${ARGV1})
 
   execute_process(
     COMMAND
       "${PYTHON_EXECUTABLE}"
-      ${CMAKE_CURRENT_SOURCE_DIR}/runtime/api/gen_vulkan_spv.py --glsl-path
-      ${shaders_path} --output-path ${VULKAN_SHADERGEN_OUT_PATH}
+      ${EXECUTORCH_ROOT}/backends/vulkan/runtime/api/gen_vulkan_spv.py
+      --glsl-path ${shaders_path} --output-path ${VULKAN_SHADERGEN_OUT_PATH}
       --glslc-path=${GLSLC_PATH} --tmp-dir-path=${VULKAN_SHADERGEN_OUT_PATH}
       --env ${VULKAN_GEN_ARG_ENV}
     RESULT_VARIABLE error_code
   )
-  set(ENV{PYTHONPATH} ${PYTHONPATH})
+
+  set(generated_spv_cpp
+      ${VULKAN_SHADERGEN_OUT_PATH}/spv.cpp
+      PARENT_SCOPE
+  )
+endfunction()
+
+function(vulkan_shader_lib library_name generated_spv_cpp)
+  add_library(${library_name} STATIC ${generated_spv_cpp})
+  target_include_directories(
+    ${library_name}
+    PRIVATE
+      ${EXECUTORCH_ROOT}/..
+      ${EXECUTORCH_ROOT}/backends/vulkan/third-party/Vulkan-Headers/include
+      ${EXECUTORCH_ROOT}/backends/vulkan/third-party/volk
+  )
+  target_link_libraries(${library_name} vulkan_backend)
+  target_compile_options(${library_name} PRIVATE ${VULKAN_CXX_FLAGS})
+  # Link this library with --whole-archive due to dynamic shader registrations
+  target_link_options_shared_lib(${library_name})
+endfunction()
+
+# Convenience macro to generate a SPIR-V shader library target. Given the path
+# to the shaders to compile and the name of the library, it will create a static
+# library containing the generated SPIR-V shaders. The generated_spv_cpp
+# variable can be used to reference the generated CPP file outside the macro.
+macro(vulkan_shader_library shaders_path library_name)
+  set(VULKAN_SHADERGEN_ENV "")
+  set(VULKAN_SHADERGEN_OUT_PATH ${CMAKE_BINARY_DIR}/${library_name})
+
+  # execute_process( COMMAND "${PYTHON_EXECUTABLE}"
+  # ${EXECUTORCH_ROOT}/backends/vulkan/runtime/api/gen_vulkan_spv.py --glsl-path
+  # ${shaders_path} --output-path ${VULKAN_SHADERGEN_OUT_PATH}
+  # --glslc-path=${GLSLC_PATH} --tmp-dir-path=${VULKAN_SHADERGEN_OUT_PATH} --env
+  # ${VULKAN_GEN_ARG_ENV} RESULT_VARIABLE error_code ) set(ENV{PYTHONPATH}
+  # ${PYTHONPATH})
 
   set(generated_spv_cpp ${VULKAN_SHADERGEN_OUT_PATH}/spv.cpp)
 
   add_library(${library_name} STATIC ${generated_spv_cpp})
-  target_include_directories(${library_name} PRIVATE ${COMMON_INCLUDES})
-  target_link_libraries(${library_name} vulkan_graph_lib)
+  target_include_directories(
+    ${library_name}
+    PRIVATE
+      ${EXECUTORCH_ROOT}/..
+      ${EXECUTORCH_ROOT}/backends/vulkan/third-party/Vulkan-Headers/include
+      ${EXECUTORCH_ROOT}/backends/vulkan/third-party/volk
+  )
+  target_link_libraries(${library_name} vulkan_backend)
   target_compile_options(${library_name} PRIVATE ${VULKAN_CXX_FLAGS})
   # Link this library with --whole-archive due to dynamic shader registrations
   target_link_options_shared_lib(${library_name})

backends/vulkan/runtime/api/Pipeline.cpp

@@ -159,6 +159,10 @@ bool operator==(const SpecVar& lhs, const SpecVar& rhs) {
   return false;
 }
 
+bool operator!=(const SpecVar& lhs, const SpecVar& rhs) {
+  return !(lhs == rhs);
+}
+
 SpecVarList::SpecVarList() {}
 
 SpecVarList::SpecVarList(std::initializer_list<SpecVar> init_list) {

backends/vulkan/runtime/api/Pipeline.h

@@ -55,6 +55,8 @@ struct SpecVar final {
 
 bool operator==(const SpecVar& lhs, const SpecVar& rhs);
 
+bool operator!=(const SpecVar& lhs, const SpecVar& rhs);
+
 class SpecVarList final {
   std::vector<SpecVar> vars;
 

backends/vulkan/test/CMakeLists.txt

@@ -0,0 +1,89 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# ### Editing this file ###
+#
+# This file should be formatted with
+# ~~~
+# cmake-format -i CMakeLists.txt
+# ~~~
+# It should also be cmake-lint clean.
+#
+# The targets in this file will be built if EXECUTORCH_BUILD_VULKAN is ON
+
+cmake_minimum_required(VERSION 3.19)
+project(executorch)
+
+find_package(executorch CONFIG REQUIRED COMPONENTS vulkan_backend)
+find_package(GTest CONFIG REQUIRED)
+
+# Only build tests if Vulkan was compiled
+find_library(LIB_VULKAN_BACKEND vulkan_backend)
+
+if(LIB_VULKAN_BACKEND)
+
+if(NOT EXECUTORCH_ROOT)
+  set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
+endif()
+
+if(NOT PYTHON_EXECUTABLE)
+  set(PYTHON_EXECUTABLE python3)
+endif()
+
+# Include this file to access target_link_options_shared_lib This is required to
+# provide access to target_link_options_shared_lib which allows libraries to be
+# linked with the --whole-archive flag. This is required for libraries that
+# perform dynamic registration via static initialization.
+include(${EXECUTORCH_ROOT}/build/Utils.cmake)
+
+include(../cmake/ShaderLibrary.cmake)
+
+# Third party include paths
+
+set(VULKAN_THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../third-party)
+
+set(GTEST_INCLUDE_PATH
+    ${EXECUTORCH_ROOT}/third-party/googletest/googletest/include set
+    (PYTORCH_PATH ${EXECUTORCH_ROOT}/third-party/pytorch)
+)
+set(VULKAN_HEADERS_PATH ${VULKAN_THIRD_PARTY_PATH}/Vulkan-Headers/include)
+set(VOLK_PATH ${VULKAN_THIRD_PARTY_PATH}/volk)
+set(VMA_PATH ${VULKAN_THIRD_PARTY_PATH}/VulkanMemoryAllocator)
+
+set(COMMON_INCLUDES ${EXECUTORCH_ROOT}/.. ${VULKAN_HEADERS_PATH} ${VOLK_PATH}
+                    ${VMA_PATH} ${GTEST_INCLUDE_PATH} ${PYTORCH_PATH}
+)
+
+# Test Utility files
+
+set(TEST_UTILS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/utils)
+file(GLOB TEST_UTILS_CPP ${CMAKE_CURRENT_SOURCE_DIR}/utils/*.cpp)
+
+# Test shaders
+
+set(TEST_SHADERS_PATH ${CMAKE_CURRENT_SOURCE_DIR}/glsl)
+gen_vulkan_shader_lib_cpp(${TEST_SHADERS_PATH})
+vulkan_shader_lib(test_shaderlib ${generated_spv_cpp})
+
+# API Test binary
+
+set(COMPUTE_API_TEST_CPP
+    ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_compute_api_test.cpp
+)
+
+target_link_options_shared_lib(vulkan_backend)
+
+add_executable(
+  vulkan_compute_api_test ${COMPUTE_API_TEST_CPP} ${TEST_UTILS_CPP}
+)
+target_include_directories(vulkan_compute_api_test PRIVATE ${COMMON_INCLUDES})
+target_link_libraries(
+  vulkan_compute_api_test PRIVATE GTest::gtest_main vulkan_backend executorch
+                                  test_shaderlib
+)
+target_compile_options(vulkan_compute_api_test PRIVATE ${VULKAN_CXX_FLAGS})
+
+endif()

backends/vulkan/test/vulkan_compute_api_test.cpp

@@ -11,7 +11,7 @@
 #include <utility>
 #include <vector>
 
-#include <c10/util/Half.h>
+#include <executorch/runtime/core/portable_type/half.h>
 
 #include <executorch/backends/vulkan/runtime/api/api.h>
 
@@ -120,6 +120,7 @@ std::vector<int64_t> get_reference_strides(
           return {};
       }
   }
+  return {};
 }
 
 TEST_F(VulkanComputeAPITest, calculate_tensor_strides_test) {
@@ -347,7 +348,7 @@ TEST_F(VulkanComputeAPITest, test_buffer_float16) {
   if (!api::context()->adapter_ptr()->has_full_float16_buffers_support()) {
     GTEST_SKIP();
   }
-  test_storage_buffer_type<c10::Half, api::kHalf>(16);
+  test_storage_buffer_type<torch::executor::Half, api::kHalf>(16);
 }
 
 TEST_F(VulkanComputeAPITest, test_buffer_int8) {
@@ -1628,7 +1629,7 @@ TEST(VulkanToFromGPUShaderTest, to_gpu_and_from_gpu_test_texture) {
 
   for (auto& sizes : to_test) {
     RUN_TESTS(float, api::kFloat)
-    RUN_TESTS(c10::Half, api::kHalf)
+    RUN_TESTS(torch::executor::Half, api::kHalf)
   }
 #undef RUN_TESTS
 }

runtime/core/portable_type/targets.bzl

@@ -24,6 +24,7 @@ def define_common_targets():
         # Only should be depended on by kernel_types:kernel_types, but various suffixes like Android and Static
         # mean I cant just expose visibility to a single rule.
         visibility = [
+            "//executorch/backends/...",
             "//executorch/runtime/core/exec_aten/...",
             "//executorch/runtime/core/portable_type/test/...",
         ],

test/run_oss_cpp_tests.sh

@@ -38,7 +38,12 @@ build_and_run_test() {
     -DCMAKE_PREFIX_PATH="$(pwd)/third-party/googletest/build" \
     -Bcmake-out/"${test_dir}"
   cmake --build cmake-out/"${test_dir}" -j9
-  for t in cmake-out/"${test_dir}"/*test; do ./"$t"; done
+
+  for t in cmake-out/"${test_dir}"/*test; do
+    if [ -e "$t" ]; then
+      ./"$t";
+    fi
+  done
 }
 
 probe_tests() {