diff --git a/CMakeLists.txt b/CMakeLists.txt index a1c499e8e4279..830cd631038bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,95 @@ else() any compiler host sources written in Swift") endif() +# A convenience pattern to match Darwin platforms. Example: +# if(SWIFT_HOST_VARIANT MATCHES "${SWIFT_DARWIN_VARIANTS}") +# ... +# endif() +set(SWIFT_DARWIN_VARIANTS "^(macosx|iphoneos|iphonesimulator|appletvos|appletvsimulator|watchos|watchsimulator)") +set(SWIFT_DARWIN_EMBEDDED_VARIANTS "^(iphoneos|iphonesimulator|appletvos|appletvsimulator|watchos|watchsimulator)") + +# A convenient list to match Darwin SDKs. Example: +# if("${SWIFT_HOST_VARIANT_SDK}" IN_LIST SWIFT_DARWIN_PLATFORMS) +# ... +# endif() +set(SWIFT_DARWIN_PLATFORMS "IOS" "IOS_SIMULATOR" "TVOS" "TVOS_SIMULATOR" "WATCHOS" "WATCHOS_SIMULATOR" "OSX") + +set(SWIFT_APPLE_PLATFORMS ${SWIFT_DARWIN_PLATFORMS}) +if(SWIFT_FREESTANDING_FLAVOR STREQUAL "apple") + list(APPEND SWIFT_APPLE_PLATFORMS "FREESTANDING") + if(SWIFT_FREESTANDING_IS_DARWIN) + list(APPEND SWIFT_DARWIN_PLATFORMS "FREESTANDING") + endif() +endif() + +# If SWIFT_HOST_VARIANT_SDK not given, try to detect from the CMAKE_SYSTEM_NAME. +if(SWIFT_HOST_VARIANT_SDK) + set(SWIFT_HOST_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}") +else() + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") + set(SWIFT_HOST_VARIANT_SDK_default "LINUX") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + set(SWIFT_HOST_VARIANT_SDK_default "FREEBSD") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD") + set(SWIFT_HOST_VARIANT_SDK_default "OPENBSD") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "CYGWIN") + set(SWIFT_HOST_VARIANT_SDK_default "CYGWIN") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows") + set(SWIFT_HOST_VARIANT_SDK_default "WINDOWS") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Haiku") + set(SWIFT_HOST_VARIANT_SDK_default "HAIKU") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") + set(SWIFT_HOST_VARIANT_SDK_default "ANDROID") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") + set(SWIFT_HOST_VARIANT_SDK_default "OSX") + else() + message(FATAL_ERROR "Unable to detect SDK for host system: ${CMAKE_SYSTEM_NAME}") + endif() +endif() + +# If SWIFT_HOST_VARIANT_ARCH not given, try to detect from the CMAKE_SYSTEM_PROCESSOR. +if(SWIFT_HOST_VARIANT_ARCH) + set(SWIFT_HOST_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}") +else() + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(SWIFT_HOST_VARIANT_ARCH_default "x86_64") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64|arm64") + if(SWIFT_HOST_VARIANT_SDK_default STREQUAL OSX) + set(SWIFT_HOST_VARIANT_ARCH_default "arm64") + else() + set(SWIFT_HOST_VARIANT_ARCH_default "aarch64") + endif() + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc64") + set(SWIFT_HOST_VARIANT_ARCH_default "powerpc64") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc") + set(SWIFT_HOST_VARIANT_ARCH_default "powerpc") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc64le") + set(SWIFT_HOST_VARIANT_ARCH_default "powerpc64le") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "s390x") + set(SWIFT_HOST_VARIANT_ARCH_default "s390x") + elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "armv5|armv5te") + set(SWIFT_HOST_VARIANT_ARCH_default "armv5") + # FIXME: Only matches v6l/v7l - by far the most common variants + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "armv6l") + set(SWIFT_HOST_VARIANT_ARCH_default "armv6") + elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "armv7l|armv7-a") + set(SWIFT_HOST_VARIANT_ARCH_default "armv7") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "IA64") + set(SWIFT_HOST_VARIANT_ARCH_default "itanium") + elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "(x86|i686)") + set(SWIFT_HOST_VARIANT_ARCH_default "i686") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "wasm32") + set(SWIFT_HOST_VARIANT_ARCH_default "wasm32") + else() + message(FATAL_ERROR "Unrecognized architecture on host system: ${CMAKE_SYSTEM_PROCESSOR}") + endif() +endif() + +set(SWIFT_HOST_VARIANT_SDK "${SWIFT_HOST_VARIANT_SDK_default}" CACHE STRING + "Deployment sdk for Swift host tools (the compiler).") +set(SWIFT_HOST_VARIANT_ARCH "${SWIFT_HOST_VARIANT_ARCH_default}" CACHE STRING + "Deployment arch for Swift host tools (the compiler).") + # # User-configurable options that control the inclusion and default build # behavior for components which may not strictly be necessary (tools, examples, @@ -103,6 +192,16 @@ option(SWIFT_STDLIB_ENABLE_UNICODE_DATA NOTE: Disabling this will cause many String methods to crash." TRUE) +include(Threading) + +threading_package_default("${SWIFT_HOST_VARIANT_SDK}" + SWIFT_THREADING_PACKAGE_default) + +set(SWIFT_THREADING_PACKAGE "${SWIFT_THREADING_PACKAGE_default}" + CACHE STRING + "The threading package to use. Must be one of 'none', 'pthreads', + 'darwin', 'linux', 'win32', 'c11'.") + option(SWIFT_BUILD_DYNAMIC_SDK_OVERLAY "Build dynamic variants of the Swift SDK overlay" TRUE) @@ -671,27 +770,6 @@ include_directories(BEFORE ${SWIFT_INCLUDE_DIR} ) -# A convenience pattern to match Darwin platforms. Example: -# if(SWIFT_HOST_VARIANT MATCHES "${SWIFT_DARWIN_VARIANTS}") -# ... -# endif() -set(SWIFT_DARWIN_VARIANTS "^(macosx|iphoneos|iphonesimulator|appletvos|appletvsimulator|watchos|watchsimulator)") -set(SWIFT_DARWIN_EMBEDDED_VARIANTS "^(iphoneos|iphonesimulator|appletvos|appletvsimulator|watchos|watchsimulator)") - -# A convenient list to match Darwin SDKs. Example: -# if("${SWIFT_HOST_VARIANT_SDK}" IN_LIST SWIFT_DARWIN_PLATFORMS) -# ... -# endif() -set(SWIFT_DARWIN_PLATFORMS "IOS" "IOS_SIMULATOR" "TVOS" "TVOS_SIMULATOR" "WATCHOS" "WATCHOS_SIMULATOR" "OSX") - -set(SWIFT_APPLE_PLATFORMS ${SWIFT_DARWIN_PLATFORMS}) -if(SWIFT_FREESTANDING_FLAVOR STREQUAL "apple") - list(APPEND SWIFT_APPLE_PLATFORMS "FREESTANDING") - if(SWIFT_FREESTANDING_IS_DARWIN) - list(APPEND SWIFT_DARWIN_PLATFORMS "FREESTANDING") - endif() -endif() - # Configuration flags passed to all of our invocations of gyb. Try to # avoid making up new variable names here if you can find a CMake # variable that will do the job. @@ -708,74 +786,6 @@ if(XCODE) swift_common_xcode_cxx_config() endif() -# If SWIFT_HOST_VARIANT_SDK not given, try to detect from the CMAKE_SYSTEM_NAME. -if(SWIFT_HOST_VARIANT_SDK) - set(SWIFT_HOST_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}") -else() - if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - set(SWIFT_HOST_VARIANT_SDK_default "LINUX") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") - set(SWIFT_HOST_VARIANT_SDK_default "FREEBSD") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD") - set(SWIFT_HOST_VARIANT_SDK_default "OPENBSD") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "CYGWIN") - set(SWIFT_HOST_VARIANT_SDK_default "CYGWIN") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows") - set(SWIFT_HOST_VARIANT_SDK_default "WINDOWS") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Haiku") - set(SWIFT_HOST_VARIANT_SDK_default "HAIKU") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") - set(SWIFT_HOST_VARIANT_SDK_default "ANDROID") - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - set(SWIFT_HOST_VARIANT_SDK_default "OSX") - else() - message(FATAL_ERROR "Unable to detect SDK for host system: ${CMAKE_SYSTEM_NAME}") - endif() -endif() - -# If SWIFT_HOST_VARIANT_ARCH not given, try to detect from the CMAKE_SYSTEM_PROCESSOR. -if(SWIFT_HOST_VARIANT_ARCH) - set(SWIFT_HOST_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}") -else() - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") - set(SWIFT_HOST_VARIANT_ARCH_default "x86_64") - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64|arm64") - if(SWIFT_HOST_VARIANT_SDK_default STREQUAL OSX) - set(SWIFT_HOST_VARIANT_ARCH_default "arm64") - else() - set(SWIFT_HOST_VARIANT_ARCH_default "aarch64") - endif() - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc64") - set(SWIFT_HOST_VARIANT_ARCH_default "powerpc64") - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc") - set(SWIFT_HOST_VARIANT_ARCH_default "powerpc") - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc64le") - set(SWIFT_HOST_VARIANT_ARCH_default "powerpc64le") - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "s390x") - set(SWIFT_HOST_VARIANT_ARCH_default "s390x") - elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "armv5|armv5te") - set(SWIFT_HOST_VARIANT_ARCH_default "armv5") - # FIXME: Only matches v6l/v7l - by far the most common variants - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "armv6l") - set(SWIFT_HOST_VARIANT_ARCH_default "armv6") - elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "armv7l|armv7-a") - set(SWIFT_HOST_VARIANT_ARCH_default "armv7") - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "IA64") - set(SWIFT_HOST_VARIANT_ARCH_default "itanium") - elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "(x86|i686)") - set(SWIFT_HOST_VARIANT_ARCH_default "i686") - elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "wasm32") - set(SWIFT_HOST_VARIANT_ARCH_default "wasm32") - else() - message(FATAL_ERROR "Unrecognized architecture on host system: ${CMAKE_SYSTEM_PROCESSOR}") - endif() -endif() - -set(SWIFT_HOST_VARIANT_SDK "${SWIFT_HOST_VARIANT_SDK_default}" CACHE STRING - "Deployment sdk for Swift host tools (the compiler).") -set(SWIFT_HOST_VARIANT_ARCH "${SWIFT_HOST_VARIANT_ARCH_default}" CACHE STRING - "Deployment arch for Swift host tools (the compiler).") - # Which default linker to use. Prefer LLVM_USE_LINKER if it set, otherwise use # our own defaults. This should only be possible in a unified (not stand alone) # build environment. @@ -1037,11 +1047,12 @@ if(SWIFT_BUILD_STDLIB OR SWIFT_BUILD_SDK_OVERLAY) message(STATUS " Leak Detection Checker Entrypoints: ${SWIFT_RUNTIME_ENABLE_LEAK_CHECKER}") message(STATUS "") + message(STATUS "Threading Package: ${SWIFT_THREADING_PACKAGE}") message(STATUS "Differentiable Programming Support: ${SWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING}") - message(STATUS "Concurrency Support: ${SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY}") - message(STATUS "Distributed Support: ${SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED}") - message(STATUS "String Processing Support: ${SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING}") - message(STATUS "Unicode Support: ${SWIFT_STDLIB_ENABLE_UNICODE_DATA}") + message(STATUS "Concurrency Support: ${SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY}") + message(STATUS "Distributed Support: ${SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED}") + message(STATUS "String Processing Support: ${SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING}") + message(STATUS "Unicode Support: ${SWIFT_STDLIB_ENABLE_UNICODE_DATA}") message(STATUS "") else() message(STATUS "Not building Swift standard library, SDK overlays, and runtime") @@ -1115,6 +1126,9 @@ endif() if(SWIFT_BUILD_STDLIB) add_subdirectory(stdlib) else() + # Some of the things below depend on the threading library + add_subdirectory(stdlib/public/Threading) + if(SWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT) add_subdirectory(stdlib/toolchain) endif() diff --git a/cmake/caches/Runtime-WASI-wasm32.cmake b/cmake/caches/Runtime-WASI-wasm32.cmake index f41634a44fc3c..213682f7b3b8f 100644 --- a/cmake/caches/Runtime-WASI-wasm32.cmake +++ b/cmake/caches/Runtime-WASI-wasm32.cmake @@ -23,4 +23,4 @@ set(SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY YES CACHE BOOL "") # build with the host compiler set(SWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER YES CACHE BOOL "") -set(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME TRUE CACHE BOOL "") +set(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY TRUE CACHE BOOL "") diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index 5cf79d8e3c1ca..949010484bd84 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -4,6 +4,7 @@ include(SwiftXcodeSupport) include(SwiftWindowsSupport) include(SwiftAndroidSupport) include(SwiftCXXUtils) +include(Threading) function(_swift_gyb_target_sources target scope) file(GLOB GYB_UNICODE_DATA ${SWIFT_SOURCE_DIR}/utils/UnicodeData/*) @@ -308,6 +309,10 @@ function(_add_host_variant_c_compile_flags target) $<$:SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS>) endif() + threading_package_name("${SWIFT_HOST_VARIANT_SDK}" _threading_package) + target_compile_definitions(${target} PRIVATE + "SWIFT_THREADING_${_threading_package}") + if(SWIFT_ANALYZE_CODE_COVERAGE) target_compile_options(${target} PRIVATE $<$:-fprofile-instr-generate -fcoverage-mapping>) diff --git a/cmake/modules/AddSwiftUnittests.cmake b/cmake/modules/AddSwiftUnittests.cmake index 4b9dcf11197b3..e19a1b6a8cff6 100644 --- a/cmake/modules/AddSwiftUnittests.cmake +++ b/cmake/modules/AddSwiftUnittests.cmake @@ -1,5 +1,6 @@ include(AddSwift) +include(Threading) add_custom_target(SwiftUnitTests) @@ -47,11 +48,15 @@ function(add_swift_unittest test_dirname) endif() # some headers switch their inline implementations based on - # SWIFT_STDLIB_SINGLE_THREADED_RUNTIME definition - if(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) + # SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY and + # SWIFT_THREADING_PACKAGE definitions + if(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) target_compile_definitions("${test_dirname}" PRIVATE - SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) + SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) endif() + threading_package_name("${SWIFT_HOST_VARIANT_SDK}" _threading_package) + target_compile_definitions("${test_dirname}" PRIVATE + "SWIFT_THREADING_${_threading_package}") if(NOT SWIFT_COMPILER_IS_MSVC_LIKE) if(SWIFT_USE_LINKER) diff --git a/cmake/modules/Threading.cmake b/cmake/modules/Threading.cmake new file mode 100644 index 0000000000000..f439ed0ba90b0 --- /dev/null +++ b/cmake/modules/Threading.cmake @@ -0,0 +1,29 @@ +# Get the default threading package for the platform +function(threading_package_default sdk out_var) + if(sdk IN_LIST SWIFT_DARWIN_PLATFORMS) + set("${out_var}" "darwin" PARENT_SCOPE) + elseif(sdk STREQUAL "LINUX") + set("${out_var}" "linux" PARENT_SCOPE) + elseif(sdk STREQUAL "WINDOWS") + set("${out_var}" "win32" PARENT_SCOPE) + elseif(sdk STREQUAL "WASI") + set("${out_var}" "none" PARENT_SCOPE) + else() + set("${out_var}" "pthreads" PARENT_SCOPE) + endif() +endfunction() + +# Given the threading package, find the name for the preprocessor +# define that we need to make. Also deals with the default platform +# setting. +function(threading_package_name sdk out_var) + precondition(SWIFT_HOST_VARIANT_SDK) + precondition(SWIFT_DARWIN_PLATFORMS) + + string(TOUPPER "${SWIFT_THREADING_PACKAGE}" package) + if(package STREQUAL "") + threading_package_default(package) + string(TOUPPER "${package}" package) + endif() + set("${out_var}" "${package}" PARENT_SCOPE) +endfunction() diff --git a/include/swift/Basic/Compiler.h b/include/swift/Basic/Compiler.h index e181a24139d82..e2ad49ec4aa2d 100644 --- a/include/swift/Basic/Compiler.h +++ b/include/swift/Basic/Compiler.h @@ -162,4 +162,20 @@ #define SWIFT_ASM_LABEL_WITH_PREFIX(STRING) \ SWIFT_ASM_LABEL_RAW(SWIFT_SYMBOL_PREFIX_STRING STRING) +// SWIFT_FORMAT(fmt,first) marks a function as taking a format string argument +// at argument `fmt`, with the first argument for the format string as `first`. +#if __has_attribute(format) +#define SWIFT_FORMAT(fmt, first) __attribute__((format(printf, fmt, first))) +#else +#define SWIFT_FORMAT(fmt, first) +#endif + +// SWIFT_VFORMAT(fmt) marks a function as taking a format string argument at +// argument `fmt`, with the arguments in a `va_list`. +#if __has_attribute(format) +#define SWIFT_VFORMAT(fmt) __attribute__((format(printf, fmt, 0))) +#else +#define SWIFT_VFORMAT(fmt) +#endif + #endif // SWIFT_BASIC_COMPILER_H diff --git a/include/swift/Basic/Lazy.h b/include/swift/Basic/Lazy.h index 87eecbe295c3c..4a99b78fc802f 100644 --- a/include/swift/Basic/Lazy.h +++ b/include/swift/Basic/Lazy.h @@ -14,78 +14,27 @@ #define SWIFT_BASIC_LAZY_H #include -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME -// No dependencies on single-threaded environments. -#elif defined(__APPLE__) -#include -#elif defined(__wasi__) -// No pthread on wasi, see https://bugs.swift.org/browse/SR-12097 for more details. -#else -#include -#endif + #include "swift/Basic/Malloc.h" #include "swift/Basic/type_traits.h" - -#if defined(__wasi__) -// Temporary single-threaded stub. Should be replaced with a thread-safe version -// as soon as the WASI SDK allows it. See https://bugs.swift.org/browse/SR-12766. -inline void wasi_call_once(int *flag, void *context, void (*func)(void *)) { - switch (*flag) { - case 0: - *flag = 1; - func(context); - return; - case 1: - return; - default: - assert(false && "wasi_call_once got invalid flag"); - abort(); - } -} -#endif +#include "swift/Threading/Once.h" namespace swift { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - using OnceToken_t = bool; -# define SWIFT_ONCE_F(TOKEN, FUNC, CONTEXT) \ - if (!TOKEN) { TOKEN = true; (FUNC)(CONTEXT); } -#elif defined(__APPLE__) - using OnceToken_t = dispatch_once_t; -# define SWIFT_ONCE_F(TOKEN, FUNC, CONTEXT) \ - ::dispatch_once_f(&TOKEN, CONTEXT, FUNC) -#elif defined(__CYGWIN__) - // _swift_once_f() is declared in Private.h. - // This prototype is copied instead including the header file. - void _swift_once_f(uintptr_t *predicate, void *context, - void (*function)(void *)); - using OnceToken_t = unsigned long; -# define SWIFT_ONCE_F(TOKEN, FUNC, CONTEXT) \ - _swift_once_f(&TOKEN, CONTEXT, FUNC) -#elif defined(__wasi__) - using OnceToken_t = int; -# define SWIFT_ONCE_F(TOKEN, FUNC, CONTEXT) \ - ::wasi_call_once(&TOKEN, CONTEXT, FUNC) -#else - using OnceToken_t = std::once_flag; -# define SWIFT_ONCE_F(TOKEN, FUNC, CONTEXT) \ - ::std::call_once(TOKEN, FUNC, CONTEXT) -#endif - /// A template for lazily-constructed, zero-initialized, leaked-on-exit /// global objects. template class Lazy { alignas(T) char Value[sizeof(T)] = { 0 }; - OnceToken_t OnceToken = {}; + swift::once_t OnceToken = {}; static void defaultInitCallback(void *ValueAddr) { ::new (ValueAddr) T(); } - + public: using Type = T; - + T &get(void (*initCallback)(void *) = defaultInitCallback); template @@ -109,7 +58,7 @@ template inline T &Lazy::get(void (*initCallback)(void*)) { static_assert(std::is_literal_type>::value, "Lazy must be a literal type"); - SWIFT_ONCE_F(OnceToken, initCallback, &Value); + swift::once(OnceToken, initCallback, &Value); return unsafeGetAlreadyInitialized(); } @@ -125,7 +74,7 @@ template inline T &Lazy::getWithInit(Arg1 &&arg1) { } } data{&Value, static_cast(arg1)}; - SWIFT_ONCE_F(OnceToken, &Data::init, &data); + swift::once(OnceToken, &Data::init, &data); return unsafeGetAlreadyInitialized(); } diff --git a/include/swift/Runtime/Atomic.h b/include/swift/Runtime/Atomic.h index 378fc8481cce9..e5509995d6cfc 100644 --- a/include/swift/Runtime/Atomic.h +++ b/include/swift/Runtime/Atomic.h @@ -20,6 +20,7 @@ #include "swift/Runtime/Config.h" #include #include +#include #if defined(_WIN64) #include #endif @@ -48,7 +49,7 @@ namespace impl { /// /// TODO: should we make this use non-atomic operations when the runtime /// is single-threaded? -template +template class alignas(Size) atomic_impl { std::atomic value; public: diff --git a/include/swift/Runtime/AtomicWaitQueue.h b/include/swift/Runtime/AtomicWaitQueue.h index bfad6c71ccb5b..88a566d7d971a 100644 --- a/include/swift/Runtime/AtomicWaitQueue.h +++ b/include/swift/Runtime/AtomicWaitQueue.h @@ -21,7 +21,7 @@ #include "swift/Runtime/Heap.h" #include "swift/Runtime/HeapObject.h" -#include "swift/Runtime/Mutex.h" +#include "swift/Threading/Mutex.h" #include namespace swift { diff --git a/include/swift/Runtime/Concurrency.h b/include/swift/Runtime/Concurrency.h index 176ce73890044..f6a78d8bc3541 100644 --- a/include/swift/Runtime/Concurrency.h +++ b/include/swift/Runtime/Concurrency.h @@ -26,7 +26,7 @@ #pragma clang diagnostic ignored "-Wreturn-type-c-linkage" // Does the runtime use a cooperative global executor? -#if defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +#if defined(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) #define SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR 1 #else #define SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR 0 diff --git a/include/swift/Runtime/Concurrent.h b/include/swift/Runtime/Concurrent.h index 4121e88f2c603..400b6a9f3f316 100644 --- a/include/swift/Runtime/Concurrent.h +++ b/include/swift/Runtime/Concurrent.h @@ -11,17 +11,17 @@ //===----------------------------------------------------------------------===// #ifndef SWIFT_RUNTIME_CONCURRENTUTILS_H #define SWIFT_RUNTIME_CONCURRENTUTILS_H -#include +#include "Atomic.h" +#include "Debug.h" +#include "swift/Threading/Mutex.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Support/Allocator.h" #include #include #include +#include #include #include -#include "llvm/ADT/Hashing.h" -#include "llvm/Support/Allocator.h" -#include "Atomic.h" -#include "Debug.h" -#include "Mutex.h" #if defined(__FreeBSD__) || defined(__CYGWIN__) || defined(__HAIKU__) #include @@ -624,7 +624,7 @@ using llvm::hash_value; /// process. It has no destructor, to avoid generating useless global destructor /// calls. The memory it allocates can be freed by calling clear() with no /// outstanding readers, but this won't destroy the static mutex it uses. -template +template struct ConcurrentReadableHashMap { // We don't call destructors. Make sure the elements will put up with this. static_assert(std::is_trivially_destructible::value, @@ -1171,7 +1171,7 @@ template struct HashMapElementWrapper { /// by allocating them separately and storing pointers to them. The elements of /// the hash table are instances of HashMapElementWrapper. A new getOrInsert /// method is provided that directly returns the stable element pointer. -template +template struct StableAddressConcurrentReadableHashMap : public ConcurrentReadableHashMap, MutexTy> { // Implicitly trivial destructor. diff --git a/include/swift/Runtime/Config.h b/include/swift/Runtime/Config.h index 3baae757440a1..41a02738161e8 100644 --- a/include/swift/Runtime/Config.h +++ b/include/swift/Runtime/Config.h @@ -232,22 +232,6 @@ extern uintptr_t __COMPATIBILITY_LIBRARIES_CANNOT_CHECK_THE_IS_SWIFT_BIT_DIRECTL // so changing this value is not sufficient. #define SWIFT_DEFAULT_LLVM_CC llvm::CallingConv::C -// SWIFT_FORMAT(fmt,first) marks a function as taking a format string argument -// at argument `fmt`, with the first argument for the format string as `first`. -#if __has_attribute(format) -#define SWIFT_FORMAT(fmt,first) __attribute__((format(printf, fmt, first))) -#else -#define SWIFT_FORMAT(fmt,first) -#endif - -// SWIFT_VFORMAT(fmt) marks a function as taking a format string argument at -// argument `fmt`, with the arguments in a `va_list`. -#if __has_attribute(format) -#define SWIFT_VFORMAT(fmt) __attribute__((format(printf, fmt, 0))) -#else -#define SWIFT_VFORMAT(fmt) -#endif - /// Should we use absolute function pointers instead of relative ones? /// WebAssembly target uses it by default. #ifndef SWIFT_COMPACT_ABSOLUTE_FUNCTION_POINTER diff --git a/include/swift/Runtime/EnvironmentVariables.h b/include/swift/Runtime/EnvironmentVariables.h index 51d78e31d3110..d462e3b5a6312 100644 --- a/include/swift/Runtime/EnvironmentVariables.h +++ b/include/swift/Runtime/EnvironmentVariables.h @@ -14,7 +14,7 @@ // //===----------------------------------------------------------------------===// -#include "../Basic/Lazy.h" +#include "swift/Threading/Once.h" namespace swift { namespace runtime { @@ -22,17 +22,17 @@ namespace environment { void initialize(void *); -extern OnceToken_t initializeToken; +extern swift::once_t initializeToken; // Declare backing variables. #define VARIABLE(name, type, defaultValue, help) extern type name ## _variable; #include "../../../stdlib/public/runtime/EnvironmentVariables.def" // Define getter functions. -#define VARIABLE(name, type, defaultValue, help) \ - inline type name() { \ - SWIFT_ONCE_F(initializeToken, initialize, nullptr); \ - return name ## _variable; \ +#define VARIABLE(name, type, defaultValue, help) \ + inline type name() { \ + swift::once(initializeToken, initialize, nullptr); \ + return name##_variable; \ } #include "../../../stdlib/public/runtime/EnvironmentVariables.def" diff --git a/include/swift/Runtime/Mutex.h b/include/swift/Runtime/Mutex.h deleted file mode 100644 index 16682fbe7af43..0000000000000 --- a/include/swift/Runtime/Mutex.h +++ /dev/null @@ -1,646 +0,0 @@ -//===--- Mutex.h - Mutex and ReadWriteLock ----------------------*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Mutex, ReadWriteLock, and Scoped lock abstractions for use in -// Swift runtime. -// -// We intentionally do not provide a condition-variable abstraction. -// Traditional condition-variable interfaces are subject to unavoidable -// priority inversions, as well as making poor use of threads. -// Prefer AtomicWaitQueue. -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_MUTEX_H -#define SWIFT_RUNTIME_MUTEX_H - -#include -#include - -#if __has_include() -#include -#endif - -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME -#include "swift/Runtime/MutexSingleThreaded.h" -#elif defined(_POSIX_THREADS) -#include "swift/Runtime/MutexPThread.h" -#elif defined(_WIN32) -#include "swift/Runtime/MutexWin32.h" -#elif defined(__wasi__) -#include "swift/Runtime/MutexSingleThreaded.h" -#else -#error "Implement equivalent of MutexPThread.h/cpp for your platform." -#endif - -namespace swift { - -/// Compile time adjusted stack based object that locks/unlocks the supplied -/// Mutex type. Use the provided typedefs instead of this directly. -template class ScopedLockT { - ScopedLockT() = delete; - ScopedLockT(const ScopedLockT &) = delete; - ScopedLockT &operator=(const ScopedLockT &) = delete; - ScopedLockT(ScopedLockT &&) = delete; - ScopedLockT &operator=(ScopedLockT &&) = delete; - -public: - explicit ScopedLockT(T &l) : Lock(l) { - if (Inverted) { - Lock.unlock(); - } else { - Lock.lock(); - } - } - - ~ScopedLockT() { - if (Inverted) { - Lock.lock(); - } else { - Lock.unlock(); - } - } - -private: - T &Lock; -}; - -/// A Mutex object that supports `BasicLockable` and `Lockable` C++ concepts. -/// See http://en.cppreference.com/w/cpp/concept/BasicLockable -/// See http://en.cppreference.com/w/cpp/concept/Lockable -/// -/// This is NOT a recursive mutex. -class Mutex { - - Mutex(const Mutex &) = delete; - Mutex &operator=(const Mutex &) = delete; - Mutex(Mutex &&) = delete; - Mutex &operator=(Mutex &&) = delete; - -public: - /// Constructs a non-recursive mutex. - /// - /// If `checked` is true the mutex will attempt to check for misuse and - /// fatalError when detected. If `checked` is false (the default) the - /// mutex will make little to no effort to check for misuse (more efficient). - explicit Mutex(bool checked = false) { - MutexPlatformHelper::init(Handle, checked); - } - ~Mutex() { MutexPlatformHelper::destroy(Handle); } - - /// The lock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Blocks the calling thread until exclusive ownership of the mutex - /// can be obtained. - /// - Prior m.unlock() operations on the same mutex synchronize-with - /// this lock operation. - /// - The behavior is undefined if the calling thread already owns - /// the mutex (likely a deadlock). - /// - Does not throw exceptions but will halt on error (fatalError). - void lock() { MutexPlatformHelper::lock(Handle); } - - /// The unlock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Releases the calling thread's ownership of the mutex and - /// synchronizes-with the subsequent successful lock operations on - /// the same object. - /// - The behavior is undefined if the calling thread does not own - /// the mutex. - /// - Does not throw exceptions but will halt on error (fatalError). - void unlock() { MutexPlatformHelper::unlock(Handle); } - - /// The try_lock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Attempts to obtain exclusive ownership of the mutex for the calling - /// thread without blocking. If ownership is not obtained, returns - /// immediately. The function is allowed to spuriously fail and return - /// even if the mutex is not currently owned by another thread. - /// - If try_lock() succeeds, prior unlock() operations on the same object - /// synchronize-with this operation. lock() does not synchronize with a - /// failed try_lock() - /// - The behavior is undefined if the calling thread already owns - /// the mutex (likely a deadlock)? - /// - Does not throw exceptions but will halt on error (fatalError). - bool try_lock() { return MutexPlatformHelper::try_lock(Handle); } - - /// Acquires lock before calling the supplied critical section and releases - /// lock on return from critical section. - /// - /// This call can block while waiting for the lock to become available. - /// - /// For example the following mutates value while holding the mutex lock. - /// - /// ``` - /// mutex.lock([&value] { value++; }); - /// ``` - /// - /// Precondition: Mutex not held by this thread, undefined otherwise. - template - auto withLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - ScopedLock guard(*this); - return std::forward(criticalSection)(); - } - - /// A stack based object that locks the supplied mutex on construction - /// and unlocks it on destruction. - /// - /// Precondition: Mutex unlocked by this thread, undefined otherwise. - typedef ScopedLockT ScopedLock; - - /// A stack based object that unlocks the supplied mutex on construction - /// and relocks it on destruction. - /// - /// Precondition: Mutex locked by this thread, undefined otherwise. - typedef ScopedLockT ScopedUnlock; - -private: - MutexHandle Handle; -}; - -/// Compile time adjusted stack based object that locks/unlocks the supplied -/// ReadWriteLock type. Use the provided typedefs instead of this directly. -template class ScopedRWLockT { - - ScopedRWLockT() = delete; - ScopedRWLockT(const ScopedRWLockT &) = delete; - ScopedRWLockT &operator=(const ScopedRWLockT &) = delete; - ScopedRWLockT(ScopedRWLockT &&) = delete; - ScopedRWLockT &operator=(ScopedRWLockT &&) = delete; - -public: - explicit ScopedRWLockT(T &l) : Lock(l) { - if (Inverted) { - if (Read) { - Lock.readUnlock(); - } else { - Lock.writeUnlock(); - } - } else { - if (Read) { - Lock.readLock(); - } else { - Lock.writeLock(); - } - } - } - - ~ScopedRWLockT() { - if (Inverted) { - if (Read) { - Lock.readLock(); - } else { - Lock.writeLock(); - } - } else { - if (Read) { - Lock.readUnlock(); - } else { - Lock.writeUnlock(); - } - } - } - -private: - T &Lock; -}; - -class ReadWriteLock; -class StaticReadWriteLock; - -/// A stack based object that unlocks the supplied ReadWriteLock on -/// construction and locks it for reading on destruction. -/// -/// Precondition: ReadWriteLock unlocked by this thread, undefined otherwise. -typedef ScopedRWLockT ScopedReadLock; -typedef ScopedRWLockT StaticScopedReadLock; - -/// A stack based object that unlocks the supplied ReadWriteLock on -/// construction and locks it for reading on destruction. -/// -/// Precondition: ReadWriteLock unlocked by this thread, undefined -/// otherwise. -typedef ScopedRWLockT ScopedReadUnlock; -typedef ScopedRWLockT StaticScopedReadUnlock; - -/// A stack based object that unlocks the supplied ReadWriteLock on -/// construction and locks it for reading on destruction. -/// -/// Precondition: ReadWriteLock unlocked by this thread, undefined otherwise. -typedef ScopedRWLockT ScopedWriteLock; -typedef ScopedRWLockT StaticScopedWriteLock; - -/// A stack based object that unlocks the supplied ReadWriteLock on -/// construction and locks it for writing on destruction. -/// -/// Precondition: ReadWriteLock unlocked by this thread, undefined otherwise. -typedef ScopedRWLockT ScopedWriteUnlock; -typedef ScopedRWLockT StaticScopedWriteUnlock; - -/// A Read / Write lock object that has semantics similar to `BasicLockable` -/// and `Lockable` C++ concepts however it supports multiple concurrent -/// threads holding the reader lock as long as the write lock isn't held and -/// only one thread can hold the write local at the same time. -/// -/// If you need static allocated ReadWriteLock use StaticReadWriteLock. -/// -/// See http://en.cppreference.com/w/cpp/concept/BasicLockable -/// See http://en.cppreference.com/w/cpp/concept/Lockable -/// -/// This is NOT a recursive mutex. -class ReadWriteLock { - - ReadWriteLock(const ReadWriteLock &) = delete; - ReadWriteLock &operator=(const ReadWriteLock &) = delete; - ReadWriteLock(ReadWriteLock &&) = delete; - ReadWriteLock &operator=(ReadWriteLock &&) = delete; - -public: - ReadWriteLock() { ReadWriteLockPlatformHelper::init(Handle); } - ~ReadWriteLock() { ReadWriteLockPlatformHelper::destroy(Handle); } - - /// The readLock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Blocks the calling thread while the write lock is held by another - /// thread and once the read lock is acquired by the calling thread - /// other threads are prevented from acquiring the write lock. - /// - Multiple threads can hold the read lock at the same time. - /// - Prior unlock() operations on the same lock synchronize-with - /// this lock operation. - /// - The behavior is undefined if the calling thread already owns - /// the read or write lock (likely a deadlock). - /// - Does not throw exceptions but will halt on error (fatalError). - /// - /// Callers must not mutate the data protected by the ReadWriteLock while - /// holding the read lock, the write lock must be used. - void readLock() { ReadWriteLockPlatformHelper::readLock(Handle); } - - /// The try_readLock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Attempts to obtain the read lock without blocking the calling thread. - /// If ownership is not obtained, returns immediately. The function is - /// allowed to spuriously fail and return even if the lock is not - /// currently owned by another thread. - /// - If try_readLock() succeeds, prior unlock() operations on the same - /// object synchronize-with this operation. unlock() does not synchronize - /// with a failed try_readLock(). - /// - The behavior is undefined if the calling thread already owns - /// the read or write lock (likely a deadlock)? - /// - Does not throw exceptions but will halt on error (fatalError). - /// - /// Callers must not mutate the data protected by the ReadWriteLock while - /// holding the read lock, the write lock must be used. - bool try_readLock() { - return ReadWriteLockPlatformHelper::try_readLock(Handle); - } - - /// The readUnlock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Releases the calling thread's ownership of the read lock - /// and synchronizes-with the subsequent successful lock operations on - /// the same object. - /// - The behavior is undefined if the calling thread does not own - /// the read lock. - /// - Does not throw exceptions but will halt on error (fatalError). - void readUnlock() { ReadWriteLockPlatformHelper::readUnlock(Handle); } - - /// The writeLock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Blocks the calling thread while the write lock or a read lock is held - /// by another thread and once the write lock is acquired by the calling - /// thread other threads are prevented from acquiring the write lock or a - /// read lock. - /// - Only one thread can hold the write lock at the same time. - /// - Prior unlock() operations on the same lock synchronize-with - /// this lock operation. - /// - The behavior is undefined if the calling thread already owns - /// the read or write lock (likely a deadlock). - /// - Does not throw exceptions but will halt on error (fatalError). - void writeLock() { ReadWriteLockPlatformHelper::writeLock(Handle); } - - /// The try_writeLock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Attempts to obtain the write lock without blocking the calling thread. - /// If ownership is not obtained, returns immediately. The function is - /// allowed to spuriously fail and return even if the lock is not - /// currently owned by another thread. - /// - If try_writeLock() succeeds, prior unlock() operations on the same - /// object synchronize-with this operation. unlock() does not synchronize - /// with a failed try_writeLock(). - /// - The behavior is undefined if the calling thread already owns - /// the read or write lock (likely a deadlock)? - /// - Does not throw exceptions but will halt on error (fatalError). - bool try_writeLock() { - return ReadWriteLockPlatformHelper::try_writeLock(Handle); - } - - /// The writeUnlock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Releases the calling thread's ownership of the write lock - /// and synchronizes-with the subsequent successful lock operations on - /// the same object. - /// - The behavior is undefined if the calling thread does not own - /// the write lock. - /// - Does not throw exceptions but will halt on error (fatalError). - void writeUnlock() { ReadWriteLockPlatformHelper::writeUnlock(Handle); } - - /// Acquires read lock before calling the supplied critical section and - /// releases lock on return from critical section. Callers must not mutate - /// the data protected by the ReadWriteLock while holding the read lock, the - /// write lock must be used. - /// - /// This call can block while waiting for the lock to become available. - /// - /// For example the following reads the cached value while holding - /// the read lock. - /// - /// ``` - /// rw.withReadLock([&value] { value = cachedValue; }); - /// ``` - /// - /// Precondition: ReadWriteLock not held by this thread, undefined otherwise. - template - auto withReadLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - ScopedReadLock guard(*this); - return std::forward(criticalSection)(); - } - - /// Acquires write lock before calling the supplied critical section and - /// releases lock on return from critical section. - /// - /// This call can block while waiting for the lock to become available. - /// - /// For example the following updates the cached value while holding - /// the write lock. - /// - /// ``` - /// rw.withWriteLock([&newValue] { cachedValue = newValue }); - /// ``` - /// - /// Precondition: ReadWriteLock not held by this thread, undefined otherwise. - template - auto withWriteLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - ScopedWriteLock guard(*this); - return std::forward(criticalSection)(); - } - -private: - ReadWriteLockHandle Handle; -}; - -/// A static allocation variant of Mutex. -/// -/// Use Mutex instead unless you need static allocation. -class StaticMutex { - - StaticMutex(const StaticMutex &) = delete; - StaticMutex &operator=(const StaticMutex &) = delete; - StaticMutex(StaticMutex &&) = delete; - StaticMutex &operator=(StaticMutex &&) = delete; - -public: -#if SWIFT_MUTEX_SUPPORTS_CONSTEXPR - constexpr -#endif - StaticMutex() - : Handle(MutexPlatformHelper::staticInit()) { - } - - /// See Mutex::lock - void lock() { MutexPlatformHelper::lock(Handle); } - - /// See Mutex::unlock - void unlock() { MutexPlatformHelper::unlock(Handle); } - - /// See Mutex::try_lock - bool try_lock() { return MutexPlatformHelper::try_lock(Handle); } - - /// See Mutex::lock - template - auto withLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - ScopedLock guard(*this); - return std::forward(criticalSection)(); - } - - /// A stack based object that locks the supplied mutex on construction - /// and unlocks it on destruction. - /// - /// Precondition: Mutex unlocked by this thread, undefined otherwise. - typedef ScopedLockT ScopedLock; - - /// A stack based object that unlocks the supplied mutex on construction - /// and relocks it on destruction. - /// - /// Precondition: Mutex locked by this thread, undefined otherwise. - typedef ScopedLockT ScopedUnlock; - -private: - MutexHandle Handle; -}; - -/// A static allocation variant of ReadWriteLock. -/// -/// Use ReadWriteLock instead unless you need static allocation. -class StaticReadWriteLock { - - StaticReadWriteLock(const StaticReadWriteLock &) = delete; - StaticReadWriteLock &operator=(const StaticReadWriteLock &) = delete; - StaticReadWriteLock(StaticReadWriteLock &&) = delete; - StaticReadWriteLock &operator=(StaticReadWriteLock &&) = delete; - -public: -#if SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR - constexpr -#endif - StaticReadWriteLock() - : Handle(ReadWriteLockPlatformHelper::staticInit()) { - } - - /// See ReadWriteLock::readLock - void readLock() { ReadWriteLockPlatformHelper::readLock(Handle); } - - /// See ReadWriteLock::try_readLock - bool try_readLock() { - return ReadWriteLockPlatformHelper::try_readLock(Handle); - } - - /// See ReadWriteLock::readUnlock - void readUnlock() { ReadWriteLockPlatformHelper::readUnlock(Handle); } - - /// See ReadWriteLock::writeLock - void writeLock() { ReadWriteLockPlatformHelper::writeLock(Handle); } - - /// See ReadWriteLock::try_writeLock - bool try_writeLock() { - return ReadWriteLockPlatformHelper::try_writeLock(Handle); - } - - /// See ReadWriteLock::writeUnlock - void writeUnlock() { ReadWriteLockPlatformHelper::writeUnlock(Handle); } - - /// See ReadWriteLock::withReadLock - template - auto withReadLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - StaticScopedReadLock guard(*this); - return std::forward(criticalSection)(); - } - - /// See ReadWriteLock::withWriteLock - template - auto withWriteLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - StaticScopedWriteLock guard(*this); - return std::forward(criticalSection)(); - } - -private: - ReadWriteLockHandle Handle; -}; - -/// A Mutex object that supports `BasicLockable` C++ concepts. It is -/// considered -/// unsafe to use because it doesn't do any error checking. It is only for -/// use in pathways that deal with reporting fatalErrors to avoid the -/// potential -/// for recursive fatalErrors that could happen if you used Mutex. -/// -/// Always use Mutex, unless in the above mentioned error pathway situation. -class StaticUnsafeMutex { - - StaticUnsafeMutex(const StaticUnsafeMutex &) = delete; - StaticUnsafeMutex &operator=(const StaticUnsafeMutex &) = delete; - StaticUnsafeMutex(StaticUnsafeMutex &&) = delete; - StaticUnsafeMutex &operator=(StaticUnsafeMutex &&) = delete; - -public: -#if SWIFT_MUTEX_SUPPORTS_CONSTEXPR - constexpr -#endif - StaticUnsafeMutex() - : Handle(MutexPlatformHelper::staticInit()) { - } - - /// The lock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Blocks the calling thread until exclusive ownership of the mutex - /// can be obtained. - /// - Prior m.unlock() operations on the same mutex synchronize-with - /// this lock operation. - /// - The behavior is undefined if the calling thread already owns - /// the mutex (likely a deadlock). - /// - Ignores errors that may happen, undefined when an error happens. - void lock() { MutexPlatformHelper::unsafeLock(Handle); } - - /// The unlock() method has the following properties: - /// - Behaves as an atomic operation. - /// - Releases the calling thread's ownership of the mutex and - /// synchronizes-with the subsequent successful lock operations on - /// the same object. - /// - The behavior is undefined if the calling thread does not own - /// the mutex. - /// - Ignores errors that may happen, undefined when an error happens. - void unlock() { MutexPlatformHelper::unsafeUnlock(Handle); } - - template - auto withLock(CriticalSection &&criticalSection) - -> decltype(std::forward(criticalSection)()) { - ScopedLock guard(*this); - return std::forward(criticalSection)(); - } - - typedef ScopedLockT ScopedLock; - typedef ScopedLockT ScopedUnlock; - -private: - MutexHandle Handle; -}; - -/// An indirect variant of a Mutex. This allocates the mutex on the heap, for -/// places where having the mutex inline takes up too much space. Used for -/// SmallMutex on platforms where Mutex is large. -class IndirectMutex { - IndirectMutex(const IndirectMutex &) = delete; - IndirectMutex &operator=(const IndirectMutex &) = delete; - IndirectMutex(IndirectMutex &&) = delete; - IndirectMutex &operator=(IndirectMutex &&) = delete; - -public: - explicit IndirectMutex(bool checked = false) { Ptr = new Mutex(checked); } - ~IndirectMutex() { delete Ptr; } - - void lock() { Ptr->lock(); } - - void unlock() { Ptr->unlock(); } - - bool try_lock() { return Ptr->try_lock(); } - - template - auto withLock(CriticalSection &&criticalSection) - -> decltype(criticalSection()) { - return Ptr->withLock(std::forward(criticalSection)); - } - - /// A stack based object that locks the supplied mutex on construction - /// and unlocks it on destruction. - /// - /// Precondition: Mutex unlocked by this thread, undefined otherwise. - typedef ScopedLockT ScopedLock; - - /// A stack based object that unlocks the supplied mutex on construction - /// and relocks it on destruction. - /// - /// Precondition: Mutex locked by this thread, undefined otherwise. - typedef ScopedLockT ScopedUnlock; - -private: - Mutex *Ptr; -}; - -/// A "small" mutex, which is pointer sized or smaller, for places where the -/// mutex is stored inline with limited storage space. This uses a normal Mutex -/// when that is small, and otherwise uses IndirectMutex. -using SmallMutex = - std::conditional_t; - -// Enforce literal requirements for static variants. -#if SWIFT_MUTEX_SUPPORTS_CONSTEXPR -static_assert(std::is_literal_type::value, - "StaticMutex must be literal type"); -static_assert(std::is_literal_type::value, - "StaticUnsafeMutex must be literal type"); -#else -// Your platform doesn't currently support statically allocated Mutex -// you will possibly see global-constructors warnings -#endif - -#if SWIFT_CONDITION_SUPPORTS_CONSTEXPR -static_assert(std::is_literal_type::value, - "StaticConditionVariable must be literal type"); -#else -// Your platform doesn't currently support statically allocated ConditionVar -// you will possibly see global-constructors warnings -#endif - -#if SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR -static_assert(std::is_literal_type::value, - "StaticReadWriteLock must be literal type"); -#else -// Your platform doesn't currently support statically allocated ReadWriteLocks -// you will possibly see global-constructors warnings -#endif -} - -#endif diff --git a/include/swift/Runtime/MutexPThread.h b/include/swift/Runtime/MutexPThread.h deleted file mode 100644 index 5d467d05a20d3..0000000000000 --- a/include/swift/Runtime/MutexPThread.h +++ /dev/null @@ -1,143 +0,0 @@ -//===--- MutexPThread.h - Supports Mutex.h using PThreads -------*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Mutex, Read/Write lock, and Scoped lock implementations -// using PThreads. -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_MUTEX_PHTREAD_H -#define SWIFT_RUNTIME_MUTEX_PHTREAD_H - -#include - -#if defined(__APPLE__) && defined(__MACH__) -#include -#define HAS_OS_UNFAIR_LOCK 1 -#endif - -namespace swift { - -typedef pthread_rwlock_t ReadWriteLockHandle; - -#if HAS_OS_UNFAIR_LOCK -typedef os_unfair_lock MutexHandle; -#else -typedef pthread_mutex_t MutexHandle; -#endif - -#if defined(__CYGWIN__) || defined(__HAIKU__) || defined(__wasi__) -// At the moment CYGWIN pthreads implementation doesn't support the use of -// constexpr for static allocation versions. The way they define things -// results in a reinterpret_cast which violates constexpr. -// WASI currently doesn't support threading/locking at all. -#define SWIFT_MUTEX_SUPPORTS_CONSTEXPR 0 -#define SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR 0 -#else -#define SWIFT_MUTEX_SUPPORTS_CONSTEXPR 1 -#define SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR 1 -#endif - -/// PThread low-level implementation that supports Mutex -/// found in Mutex.h -/// -/// See Mutex -struct MutexPlatformHelper { -#if SWIFT_MUTEX_SUPPORTS_CONSTEXPR - static constexpr -#else - static -#endif - MutexHandle - staticInit() { -#if HAS_OS_UNFAIR_LOCK - return OS_UNFAIR_LOCK_INIT; -#else - return PTHREAD_MUTEX_INITIALIZER; -#endif - } - - static void init(MutexHandle &mutex, bool checked = false); - static void destroy(MutexHandle &mutex); - static void lock(MutexHandle &mutex); - static void unlock(MutexHandle &mutex); - static bool try_lock(MutexHandle &mutex); - -#if HAS_OS_UNFAIR_LOCK - // os_unfair_lock always checks for errors, so just call through. - static void unsafeLock(MutexHandle &mutex) { - lock(mutex); - } - static void unsafeUnlock(MutexHandle &mutex) { - unlock(mutex); - } -#else - // Skip error checking for the unsafe versions. - static void unsafeLock(MutexHandle &mutex) { - (void)pthread_mutex_lock(&mutex); - } - static void unsafeUnlock(MutexHandle &mutex) { - (void)pthread_mutex_unlock(&mutex); - } -#endif -}; - -#if HAS_OS_UNFAIR_LOCK - -inline void MutexPlatformHelper::init(os_unfair_lock &lock, bool checked) { - (void)checked; // Unfair locks are always checked. - lock = OS_UNFAIR_LOCK_INIT; -} - -inline void MutexPlatformHelper::destroy(os_unfair_lock &lock) {} - -inline void MutexPlatformHelper::lock(os_unfair_lock &lock) { - os_unfair_lock_lock(&lock); -} - -inline void MutexPlatformHelper::unlock(os_unfair_lock &lock) { - os_unfair_lock_unlock(&lock); -} - -inline bool MutexPlatformHelper::try_lock(os_unfair_lock &lock) { - return os_unfair_lock_trylock(&lock); -} - -#endif - -/// PThread low-level implementation that supports ReadWriteLock -/// found in Mutex.h -/// -/// See ReadWriteLock -struct ReadWriteLockPlatformHelper { -#if SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR - static constexpr -#else - static -#endif - ReadWriteLockHandle - staticInit() { - return PTHREAD_RWLOCK_INITIALIZER; - }; - - static void init(ReadWriteLockHandle &rwlock); - static void destroy(ReadWriteLockHandle &rwlock); - static void readLock(ReadWriteLockHandle &rwlock); - static bool try_readLock(ReadWriteLockHandle &rwlock); - static void readUnlock(ReadWriteLockHandle &rwlock); - static void writeLock(ReadWriteLockHandle &rwlock); - static bool try_writeLock(ReadWriteLockHandle &rwlock); - static void writeUnlock(ReadWriteLockHandle &rwlock); -}; -} - -#endif diff --git a/include/swift/Runtime/MutexSingleThreaded.h b/include/swift/Runtime/MutexSingleThreaded.h deleted file mode 100644 index 3eee91f414ce9..0000000000000 --- a/include/swift/Runtime/MutexSingleThreaded.h +++ /dev/null @@ -1,54 +0,0 @@ -//===--- MutexSingleThreaded.h - --------------------------------*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// No-op implementation of locks for single-threaded environments. -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_MUTEX_SINGLE_THREADED_H -#define SWIFT_RUNTIME_MUTEX_SINGLE_THREADED_H - -#include "swift/Runtime/Debug.h" - -namespace swift { - -typedef void* MutexHandle; -typedef void* ReadWriteLockHandle; - -#define SWIFT_MUTEX_SUPPORTS_CONSTEXPR 1 -#define SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR 1 - -struct MutexPlatformHelper { - static constexpr MutexHandle staticInit() { return nullptr; } - static void init(MutexHandle &mutex, bool checked = false) {} - static void destroy(MutexHandle &mutex) {} - static void lock(MutexHandle &mutex) {} - static void unlock(MutexHandle &mutex) {} - static bool try_lock(MutexHandle &mutex) { return true; } - static void unsafeLock(MutexHandle &mutex) {} - static void unsafeUnlock(MutexHandle &mutex) {} -}; - -struct ReadWriteLockPlatformHelper { - static constexpr ReadWriteLockHandle staticInit() { return nullptr; } - static void init(ReadWriteLockHandle &rwlock) {} - static void destroy(ReadWriteLockHandle &rwlock) {} - static void readLock(ReadWriteLockHandle &rwlock) {} - static bool try_readLock(ReadWriteLockHandle &rwlock) { return true; } - static void readUnlock(ReadWriteLockHandle &rwlock) {} - static void writeLock(ReadWriteLockHandle &rwlock) {} - static bool try_writeLock(ReadWriteLockHandle &rwlock) { return true; } - static void writeUnlock(ReadWriteLockHandle &rwlock) {} -}; -} - -#endif diff --git a/include/swift/Runtime/MutexWin32.h b/include/swift/Runtime/MutexWin32.h deleted file mode 100644 index 4e1f0b680d832..0000000000000 --- a/include/swift/Runtime/MutexWin32.h +++ /dev/null @@ -1,78 +0,0 @@ -//===--- MutexWin32.h - -----------------------------------------*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Mutex and Read/Write lock implementations using Windows Slim -// Reader/Writer Locks and Conditional Variables. -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_MUTEX_WIN32_H -#define SWIFT_RUNTIME_MUTEX_WIN32_H - -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include - -namespace swift { - -typedef SRWLOCK MutexHandle; -typedef SRWLOCK ReadWriteLockHandle; - -#define SWIFT_MUTEX_SUPPORTS_CONSTEXPR 1 -#define SWIFT_READWRITELOCK_SUPPORTS_CONSTEXPR 1 - -struct MutexPlatformHelper { - static constexpr MutexHandle staticInit() { return SRWLOCK_INIT; } - static void init(MutexHandle &mutex, bool checked = false) { - InitializeSRWLock(&mutex); - } - static void destroy(MutexHandle &mutex) {} - static void lock(MutexHandle &mutex) { AcquireSRWLockExclusive(&mutex); } - static void unlock(MutexHandle &mutex) { ReleaseSRWLockExclusive(&mutex); } - static bool try_lock(MutexHandle &mutex) { - return TryAcquireSRWLockExclusive(&mutex) != 0; - } - // The unsafe versions don't do error checking. - static void unsafeLock(MutexHandle &mutex) { - AcquireSRWLockExclusive(&mutex); - } - static void unsafeUnlock(MutexHandle &mutex) { - ReleaseSRWLockExclusive(&mutex); - } -}; - -struct ReadWriteLockPlatformHelper { - static constexpr ReadWriteLockHandle staticInit() { return SRWLOCK_INIT; } - static void init(ReadWriteLockHandle &rwlock) { InitializeSRWLock(&rwlock); } - static void destroy(ReadWriteLockHandle &rwlock) {} - static void readLock(ReadWriteLockHandle &rwlock) { - AcquireSRWLockShared(&rwlock); - } - static bool try_readLock(ReadWriteLockHandle &rwlock) { - return TryAcquireSRWLockShared(&rwlock) != 0; - } - static void readUnlock(ReadWriteLockHandle &rwlock) { - ReleaseSRWLockShared(&rwlock); - } - static void writeLock(ReadWriteLockHandle &rwlock) { - AcquireSRWLockExclusive(&rwlock); - } - static bool try_writeLock(ReadWriteLockHandle &rwlock) { - return TryAcquireSRWLockExclusive(&rwlock) != 0; - } - static void writeUnlock(ReadWriteLockHandle &rwlock) { - ReleaseSRWLockExclusive(&rwlock); - } -}; -} - -#endif diff --git a/include/swift/Runtime/Once.h b/include/swift/Runtime/Once.h index f9931edf5b9bf..55921d76dac41 100644 --- a/include/swift/Runtime/Once.h +++ b/include/swift/Runtime/Once.h @@ -17,35 +17,18 @@ #ifndef SWIFT_RUNTIME_ONCE_H #define SWIFT_RUNTIME_ONCE_H -#include "swift/Runtime/HeapObject.h" -#include +#include "swift/Threading/Once.h" namespace swift { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - -typedef bool swift_once_t; - -#elif defined(__APPLE__) - -// On OS X and iOS, swift_once_t matches dispatch_once_t. -typedef long swift_once_t; - -#elif defined(__CYGWIN__) - -// On Cygwin, std::once_flag can not be used because it is larger than the -// platform word. -typedef uintptr_t swift_once_t; -#else - -// On other platforms swift_once_t is std::once_flag -typedef std::once_flag swift_once_t; - -#endif +typedef swift::once_t swift_once_t; /// Runs the given function with the given context argument exactly once. /// The predicate argument must point to a global or static variable of static /// extent of type swift_once_t. +/// +/// Within the runtime, you should be using swift::once, which will be faster; +/// this is exposed so that the compiler can generate calls to it. SWIFT_RUNTIME_EXPORT void swift_once(swift_once_t *predicate, void (*fn)(void *), void *context); diff --git a/include/swift/Runtime/ThreadLocal.h b/include/swift/Runtime/ThreadLocal.h deleted file mode 100644 index f952b2d5a4b2b..0000000000000 --- a/include/swift/Runtime/ThreadLocal.h +++ /dev/null @@ -1,176 +0,0 @@ -//===--- ThreadLocal.h - Thread-local storage -------------------*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Declarations and macros for working with thread-local storage in the -// Swift runtime. -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_THREADLOCAL_H -#define SWIFT_RUNTIME_THREADLOCAL_H - -#include -#include "ThreadLocalStorage.h" - -/// SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL - Does the current configuration -/// allow the use of SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL? -#if SWIFT_STDLIB_SINGLE_THREADED_RUNTIME -// We define SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL to nothing in this -// configuration and just use a global variable, so this is okay. -#define SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL 1 -#elif defined(__APPLE__) -// The pthread TLS APIs work better than C++ TLS on Apple platforms. -#define SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL 0 -#elif __has_feature(tls) -// If __has_feature reports that TLS is available, use it. -#define SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL 1 -#elif !defined(__clang__) -// If we're not using Clang, assume that __has_feature is unreliable -// and that we can safely use TLS. -#else -// Otherwise we can't use TLS and have to fall back on something else. -#define SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL 0 -#endif - -/// SWIFT_RUNTIME_THREAD_LOCAL - Declare that something is a -/// thread-local variable in the runtime. -#if SWIFT_STDLIB_SINGLE_THREADED_RUNTIME -// In a single-threaded runtime, thread-locals are global. -#define SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL -#elif defined(__GNUC__) -// In GCC-compatible compilers, we prefer __thread because it's understood -// to guarantee a constant initializer, which permits more efficient access -// patterns. -#define SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL __thread -#else -// Otherwise, just fall back on the standard C++ feature. -#define SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL thread_local -#endif - -// Implementation of SWIFT_RUNTIME_DECLARE_THREAD_LOCAL -#if !SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL -#include -#include -#endif - -namespace swift { -// Validate a type stored in thread-local storage, using static asserts. Such -// types must fit in a pointer and be trivially copyable/destructible. -#define VALIDATE_THREAD_LOCAL_TYPE(T) \ - static_assert(sizeof(T) <= sizeof(void *), \ - "cannot store more than a pointer"); \ - static_assert(std::is_trivially_copyable::value, \ - "ThreadLocal values must be trivially copyable"); \ - static_assert(std::is_trivially_destructible::value, \ - "ThreadLocal cleanup is not supported, stored types must be " \ - "trivially destructible"); - -// A wrapper class for thread-local storage. -// -// - On platforms that report SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL -// above, an object of this type is declared with -// SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL. This makes the object -// itself thread-local, and no internal support is required. -// -// Note that this includes platforms that set -// SWIFT_STDLIB_SINGLE_THREADED_RUNTIME, for which -// SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL is empty; -// thread-local declarations then create an ordinary global. -// -// - On platforms that don't report SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL, -// we have to simulate thread-local storage. Fortunately, all of -// these platforms (at least for now) support pthread_getspecific. -#if SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL -template -class ThreadLocal { - VALIDATE_THREAD_LOCAL_TYPE(T) - - T value; - -public: - constexpr ThreadLocal() {} - - T get() { return value; } - - void set(T newValue) { value = newValue; } -}; -#else -// A wrapper around a pthread_key_t that is lazily initialized using -// dispatch_once. -class ThreadLocalKey { - // We rely on the zero-initialization of objects with static storage - // duration. - dispatch_once_t once; - pthread_key_t key; - -public: - pthread_key_t getKey() { - dispatch_once_f(&once, &key, [](void *ctx) { - pthread_key_create(reinterpret_cast(ctx), nullptr); - }); - return key; - } -}; - -// A type representing a constant pthread_key_t, for use on platforms that -// provide reserved keys. -template -class ConstantThreadLocalKey { -public: - pthread_key_t getKey() { return constantKey; } -}; - -template -class ThreadLocal { - VALIDATE_THREAD_LOCAL_TYPE(T) - - Key key; - -public: - constexpr ThreadLocal() {} - - T get() { - void *storedValue = SWIFT_THREAD_GETSPECIFIC(key.getKey()); - T value; - memcpy(&value, &storedValue, sizeof(T)); - return value; - } - - void set(T newValue) { - void *storedValue; - memcpy(&storedValue, &newValue, sizeof(T)); - SWIFT_THREAD_SETSPECIFIC(key.getKey(), storedValue); - } -}; -#endif - -} // end namespace swift - -/// SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(TYPE, NAME) - Declare a variable -/// to be a thread-local variable. The declaration must have static -/// storage duration; it may be prefixed with "static". -/// -/// Because of the fallback path, the default-initialization of the -/// type must be equivalent to a bitwise zero-initialization, and the -/// type must be small and trivially copyable and destructible. -#if SWIFT_RUNTIME_SUPPORTS_THREAD_LOCAL -#define SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(TYPE, NAME, KEY) \ - SWIFT_RUNTIME_ATTRIBUTE_THREAD_LOCAL swift::ThreadLocal NAME -#elif SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC -#define SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(TYPE, NAME, KEY) \ - swift::ThreadLocal> NAME -#else -#define SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(TYPE, NAME, KEY) \ - swift::ThreadLocal NAME -#endif - -#endif diff --git a/include/swift/Runtime/ThreadLocalStorage.h b/include/swift/Runtime/ThreadLocalStorage.h deleted file mode 100644 index 5cfab5ad65ae2..0000000000000 --- a/include/swift/Runtime/ThreadLocalStorage.h +++ /dev/null @@ -1,128 +0,0 @@ -//===--- ThreadLocalStorage.h - Thread-local storage interface. --*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_THREADLOCALSTORAGE_H -#define SWIFT_RUNTIME_THREADLOCALSTORAGE_H - -#include "swift/Runtime/Config.h" - -// Depending on the target, we may be able to use dedicated TSD keys or -// thread_local variables. When dedicated TSD keys aren't available, -// wrap the target's API for thread-local data for things that don't want -// to use thread_local. - -// On Apple platforms, we have dedicated TSD keys. -#if defined(__APPLE__) -# define SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC 1 -#endif - -#if SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC -// Use reserved TSD keys. -# if __has_include() -# include -# else -// We still need to use the SPI for setting the destructor, so declare it here. -extern "C" int pthread_key_init_np(int key, void (*destructor)(void *)); -# endif - -// If the keys are not available from the header, define them ourselves. The values match -// what tsd_private.h provides. -# ifndef __PTK_FRAMEWORK_SWIFT_KEY0 -# define __PTK_FRAMEWORK_SWIFT_KEY0 100 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY1 -# define __PTK_FRAMEWORK_SWIFT_KEY1 101 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY2 -# define __PTK_FRAMEWORK_SWIFT_KEY2 102 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY3 -# define __PTK_FRAMEWORK_SWIFT_KEY3 103 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY4 -# define __PTK_FRAMEWORK_SWIFT_KEY4 104 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY5 -# define __PTK_FRAMEWORK_SWIFT_KEY5 105 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY6 -# define __PTK_FRAMEWORK_SWIFT_KEY6 106 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY7 -# define __PTK_FRAMEWORK_SWIFT_KEY7 107 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY8 -# define __PTK_FRAMEWORK_SWIFT_KEY8 108 -# endif -# ifndef __PTK_FRAMEWORK_SWIFT_KEY9 -# define __PTK_FRAMEWORK_SWIFT_KEY9 109 -# endif - - -# define SWIFT_RUNTIME_TLS_KEY __PTK_FRAMEWORK_SWIFT_KEY0 -# define SWIFT_STDLIB_TLS_KEY __PTK_FRAMEWORK_SWIFT_KEY1 -# define SWIFT_COMPATIBILITY_50_TLS_KEY __PTK_FRAMEWORK_SWIFT_KEY2 -# define SWIFT_CONCURRENCY_TASK_KEY __PTK_FRAMEWORK_SWIFT_KEY3 -# define SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY __PTK_FRAMEWORK_SWIFT_KEY4 -# define SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY \ - __PTK_FRAMEWORK_SWIFT_KEY5 - -#endif - -// If the reserved key path didn't already provide get/setspecific macros, -// wrap the platform's APIs. -#ifndef SWIFT_THREAD_GETSPECIFIC - -// Pick the right typedef for the key. -# if defined(__linux__) -# if defined(__ANDROID__) -typedef int __swift_thread_key_t; -# else -typedef unsigned int __swift_thread_key_t; -# endif -# elif defined(__FreeBSD__) -typedef int __swift_thread_key_t; -# elif defined(__OpenBSD__) -typedef int __swift_thread_key_t; -# elif defined(_WIN32) -typedef unsigned long __swift_thread_key_t; -# elif defined(__HAIKU__) -typedef int __swift_thread_key_t; -# else -typedef unsigned long __swift_thread_key_t; -# endif - -# if defined(_WIN32) && !defined(__CYGWIN__) -// Windows has its own flavor of API. -# include -# define WIN32_LEAN_AND_MEAN -# include - -#include - -static_assert(std::is_same<__swift_thread_key_t, DWORD>::value, - "__swift_thread_key_t is not a DWORD"); - -# define SWIFT_THREAD_KEY_CREATE _stdlib_thread_key_create -# define SWIFT_THREAD_GETSPECIFIC FlsGetValue -# define SWIFT_THREAD_SETSPECIFIC(key, value) (FlsSetValue(key, value) == FALSE) - -# elif !defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) -// Otherwise use the pthread API. -# include -# define SWIFT_THREAD_KEY_CREATE pthread_key_create -# define SWIFT_THREAD_GETSPECIFIC pthread_getspecific -# define SWIFT_THREAD_SETSPECIFIC pthread_setspecific -# endif -#endif - -#endif // SWIFT_RUNTIME_THREADLOCALSTORAGE_H diff --git a/include/swift/Runtime/VoucherShims.h b/include/swift/Runtime/VoucherShims.h index 64662cea7c275..997cb07b991be 100644 --- a/include/swift/Runtime/VoucherShims.h +++ b/include/swift/Runtime/VoucherShims.h @@ -22,8 +22,7 @@ // swift-corelibs-libdispatch has os/voucher_private.h but it doesn't work for // us yet, so only look for it on Apple platforms. We also don't need vouchers // in the single threaded concurrency runtime. -#if __APPLE__ && !SWIFT_STDLIB_SINGLE_THREADED_RUNTIME \ - && __has_include() +#if __APPLE__ && !SWIFT_THREADING_NONE && __has_include() #define SWIFT_HAS_VOUCHER_HEADER 1 #include #endif diff --git a/include/swift/Threading/Errors.h b/include/swift/Threading/Errors.h new file mode 100644 index 0000000000000..34dec45d67d6b --- /dev/null +++ b/include/swift/Threading/Errors.h @@ -0,0 +1,35 @@ +//==--- Errors.h - Threading implementation error handling ----- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Declares some support routines for error handling in the threading code +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_ERRORS_H +#define SWIFT_THREADING_ERRORS_H + +#include + +#include "swift/Basic/Compiler.h" + +namespace swift { +namespace threading { + +// Users of the threading library are expected to provide this function. +SWIFT_ATTRIBUTE_NORETURN +SWIFT_FORMAT(1, 2) +void fatal(const char *msg, ...); + +} // namespace threading +} // namespace swift + +#endif // SWIFT_THREADING_ERRORS_H diff --git a/include/swift/Threading/Impl.h b/include/swift/Threading/Impl.h new file mode 100644 index 0000000000000..f734a310b5928 --- /dev/null +++ b/include/swift/Threading/Impl.h @@ -0,0 +1,72 @@ +//===--- Impl.h - Threading abstraction implementation -------- -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Includes the relevant implementation file based on the selected threading +// package. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_H +#define SWIFT_THREADING_IMPL_H + +#include "TLSKeys.h" + +namespace swift { +namespace threading_impl { + +struct stack_bounds { + void *low; + void *high; +}; + +} // namespace swift +} // namespace threading_impl + + +// Try to autodetect if we aren't told what to do +#if !SWIFT_THREADING_NONE && !SWIFT_THREADING_DARWIN && \ + !SWIFT_THREADING_LINUX && !SWIFT_THREADING_PTHREADS && \ + !SWIFT_THREADING_C11 && !SWIFT_THREADING_WIN32 +#ifdef __APPLE__ +#define SWIFT_THREADING_DARWIN 1 +#elif defined(__linux__) +#define SWIFT_THREADING_LINUX 1 +#elif defined(_WIN32) +#define SWIFT_THREADING_WIN32 1 +#elif defined(__wasi__) +#define SWIFT_THREADING_NONE 1 +#elif __has_include() +#define SWIFT_THREADING_C11 1 +#elif __has_include() +#define SWIFT_THREADING_PTHREADS 1 +#else +#error Unable to autodetect threading package. Please define SWIFT_THREADING_x as appropriate for your platform. +#endif +#endif + +#if SWIFT_THREADING_NONE +#include "Impl/Nothreads.h" +#elif SWIFT_THREADING_DARWIN +#include "Impl/Darwin.h" +#elif SWIFT_THREADING_LINUX +#include "Impl/Linux.h" +#elif SWIFT_THREADING_PTHREADS +#include "Impl/Pthreads.h" +#elif SWIFT_THREADING_C11 +#include "Impl/C11.h" +#elif SWIFT_THREADING_WIN32 +#include "Impl/Win32.h" +#else +#error You need to implement Threading/Impl.h for your threading package. +#endif + +#endif // SWIFT_THREADING_IMPL_H diff --git a/include/swift/Threading/Impl/C11.h b/include/swift/Threading/Impl/C11.h new file mode 100644 index 0000000000000..40782c233f714 --- /dev/null +++ b/include/swift/Threading/Impl/C11.h @@ -0,0 +1,188 @@ +//==--- C11.h - Threading abstraction implementation ----------- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for C11 threads +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_C11_H +#define SWIFT_THREADING_IMPL_C11_H + +#include +#include + +#include + +#include "llvm/ADT/Optional.h" + +#include "swift/Threading/Errors.h" + +namespace swift { +namespace threading_impl { + +#define SWIFT_C11THREADS_CHECK(expr) \ + do { \ + int res_ = (expr); \ + if (res_ != thrd_success) \ + swift::threading::fatal(#expr " failed with error %d\n", res_); \ + } while (0) + +#define SWIFT_C11THREADS_RETURN_TRUE_OR_FALSE(expr) \ + do { \ + int res_ = (expr); \ + switch (res_) { \ + case thrd_success: \ + return true; \ + case thrd_busy: \ + return false; \ + default: \ + swift::threading::fatal(#expr " failed with error (%d)\n", res_); \ + } \ + } while (0) + +// .. Thread related things .................................................. + +using thread_id = ::thrd_t; + +inline thread_id thread_get_current() { return ::thrd_current(); } +bool thread_is_main(); +inline bool threads_same(thread_id a, thread_id b) { + return ::thrd_equal(a, b); +} +inline llvm::Optional thread_get_current_stack_bounds() { + return {}; +} + +// .. Mutex support .......................................................... + +using mutex_handle = ::mtx_t; + +inline void mutex_init(mutex_handle &handle, bool checked = false) { + SWIFT_C11THREADS_CHECK(::mtx_init(&handle, ::mtx_plain)); +} +inline void mutex_destroy(mutex_handle &handle) { ::mtx_destroy(&handle); } + +inline void mutex_lock(mutex_handle &handle) { + SWIFT_C11THREADS_CHECK(::mtx_lock(&handle)); +} +inline void mutex_unlock(mutex_handle &handle) { + SWIFT_C11THREADS_CHECK(::mtx_unlock(&handle)); +} +inline bool mutex_try_lock(mutex_handle &handle) { + SWIFT_C11THREADS_RETURN_TRUE_OR_FALSE(::mtx_trylock(&handle)); +} + +inline void mutex_unsafe_lock(mutex_handle &handle) { + (void)::mtx_lock(&handle); +} +inline void mutex_unsafe_unlock(mutex_handle &handle) { + (void)::mtx_unlock(&handle); +} + +struct lazy_mutex_handle { + ::mtx_t mutex; + std::int32_t once; // -1 = initialized, 0 = uninitialized, 1 = initializing +}; + +inline constexpr lazy_mutex_handle lazy_mutex_initializer() { + return (lazy_mutex_handle){}; +} +inline void lazy_mutex_init(lazy_mutex_handle &handle) { + // Sadly, we can't use call_once() for this as it doesn't have a context + if (std::atomic_load_explicit((std::atomic *)&handle.once, + std::memory_order_acquire) < 0) + return; + + int zero = 0; + if (std::atomic_compare_exchange_strong_explicit( + (std::atomic *)&handle.once, &zero, 1, + std::memory_order_relaxed, std::memory_order_relaxed)) { + SWIFT_C11THREADS_CHECK(::mtx_init(&handle.mutex, ::mtx_plain)); + std::atomic_store_explicit((std::atomic *)&handle.once, -1, + std::memory_order_release); + return; + } + + while (std::atomic_load_explicit((std::atomic *)&handle.once, + std::memory_order_acquire) >= 0) { + // Just spin; ::mtx_init() is very likely to be fast + } +} + +inline void lazy_mutex_destroy(lazy_mutex_handle &handle) { + if (std::atomic_load_explicit((std::atomic *)&handle.once, + std::memory_order_acquire) < 0) + ::mtx_destroy(&handle.mutex); +} + +inline void lazy_mutex_lock(lazy_mutex_handle &handle) { + lazy_mutex_init(handle); + SWIFT_C11THREADS_CHECK(::mtx_lock(&handle.mutex)); +} +inline void lazy_mutex_unlock(lazy_mutex_handle &handle) { + lazy_mutex_init(handle); + SWIFT_C11THREADS_CHECK(::mtx_unlock(&handle.mutex)); +} +inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { + lazy_mutex_init(handle); + SWIFT_C11THREADS_RETURN_TRUE_OR_FALSE(::mtx_trylock(&handle.mutex)); +} + +inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) { + lazy_mutex_init(handle); + (void)::mtx_lock(&handle.mutex); +} +inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) { + lazy_mutex_init(handle); + (void)::mtx_unlock(&handle.mutex); +} + +// .. Once ................................................................... + +typedef std::atomic once_t; + +void once_slow(once_t &predicate, void (*fn)(void *), void *context); + +inline void once_impl(once_t &predicate, void (*fn)(void *), void *context) { + // Sadly we can't use call_once() for this (no context) + if (std::atomic_load_explicit(&predicate, std::memory_order_acquire) < 0) + return; + + once_slow(predicate, fn, context); +} + +// .. Thread local storage ................................................... + +// Get rid of this, because it causes clashes with TokenKinds.def +#undef thread_local + +// We *can* use the C++ version though +#if __cplusplus >= 201103L || __has_feature(cxx_thread_local) +#define SWIFT_THREAD_LOCAL thread_local +#endif + +using tls_key_t = ::tss_t; +using tls_dtor_t = void (*)(void *); + +inline bool tls_alloc(tls_key_t &key, tls_dtor_t dtor) { + return ::tss_create(&key, dtor) == thrd_success; +} + +inline void *tls_get(tls_key_t key) { return ::tss_get(key); } + +inline void tls_set(tls_key_t key, void *ptr) { ::tss_set(key, ptr); } + +} // namespace threading_impl + +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_C11_H diff --git a/include/swift/Threading/Impl/Darwin.h b/include/swift/Threading/Impl/Darwin.h new file mode 100644 index 0000000000000..cdbb90c4a89cb --- /dev/null +++ b/include/swift/Threading/Impl/Darwin.h @@ -0,0 +1,204 @@ +//==--- Darwin.h - Threading abstraction implementation -------- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for Apple platforms +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_DARWIN_H +#define SWIFT_THREADING_IMPL_DARWIN_H + +#include +#include +#include + +#include "llvm/ADT/Optional.h" + +#include "swift/Threading/Errors.h" + +namespace swift { +namespace threading_impl { + +// .. Thread related things .................................................. + +using thread_id = ::pthread_t; + +inline thread_id thread_get_current() { return ::pthread_self(); } + +inline bool thread_is_main() { return pthread_main_np(); } + +inline bool threads_same(thread_id a, thread_id b) { + return ::pthread_equal(a, b); +} + +inline llvm::Optional thread_get_current_stack_bounds() { + stack_bounds result; + pthread_t thread = pthread_self(); + + // On Apple platforms, pthread_get_stackaddr_np() gets the address of the + // *end* of the stack (i.e. the highest address in stack space), *NOT* the + // address of the *base* of the stack (the lowest address). + result.high = pthread_get_stackaddr_np(thread); + result.low = (char *)result.high - pthread_get_stacksize_np(thread); + + return result; +} + +// .. Mutex support .......................................................... + +using mutex_handle = ::os_unfair_lock; + +inline void mutex_init(mutex_handle &handle, bool checked = false) { + handle = OS_UNFAIR_LOCK_INIT; +} +inline void mutex_destroy(mutex_handle &handle) {} + +inline void mutex_lock(mutex_handle &handle) { ::os_unfair_lock_lock(&handle); } +inline void mutex_unlock(mutex_handle &handle) { + ::os_unfair_lock_unlock(&handle); +} +inline bool mutex_try_lock(mutex_handle &handle) { + return ::os_unfair_lock_trylock(&handle); +} + +inline void mutex_unsafe_lock(mutex_handle &handle) { + ::os_unfair_lock_lock(&handle); +} +inline void mutex_unsafe_unlock(mutex_handle &handle) { + ::os_unfair_lock_unlock(&handle); +} + +using lazy_mutex_handle = ::os_unfair_lock; + +// We don't need to be lazy here because Darwin has OS_UNFAIR_LOCK_INIT. +inline constexpr lazy_mutex_handle lazy_mutex_initializer() { + return OS_UNFAIR_LOCK_INIT; +} +inline void lazy_mutex_destroy(lazy_mutex_handle &handle) {} + +inline void lazy_mutex_lock(lazy_mutex_handle &handle) { + ::os_unfair_lock_lock(&handle); +} +inline void lazy_mutex_unlock(lazy_mutex_handle &handle) { + ::os_unfair_lock_unlock(&handle); +} +inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { + return ::os_unfair_lock_trylock(&handle); +} + +inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) { + ::os_unfair_lock_lock(&handle); +} +inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) { + ::os_unfair_lock_unlock(&handle); +} + +// .. Once ................................................................... + +using once_t = ::dispatch_once_t; + +inline void once_impl(once_t &predicate, void (*fn)(void *), void *context) { + dispatch_once_f(&predicate, context, fn); +} + +// .. Thread local storage ................................................... + +// On Darwin, we want to use the reserved keys +#define SWIFT_THREADING_USE_RESERVED_TLS_KEYS 1 + +#if __has_include() +#include +#else +#define __PTK_FRAMEWORK_SWIFT_KEY0 100 +#define __PTK_FRAMEWORK_SWIFT_KEY1 101 +#define __PTK_FRAMEWORK_SWIFT_KEY2 102 +#define __PTK_FRAMEWORK_SWIFT_KEY3 103 +#define __PTK_FRAMEWORK_SWIFT_KEY4 104 +#define __PTK_FRAMEWORK_SWIFT_KEY5 105 +#define __PTK_FRAMEWORK_SWIFT_KEY6 106 +#define __PTK_FRAMEWORK_SWIFT_KEY7 107 +#define __PTK_FRAMEWORK_SWIFT_KEY8 108 +#define __PTK_FRAMEWORK_SWIFT_KEY9 109 + +extern "C" { + +extern int pthread_key_init_np(int, void (*)(void *)); + +inline bool _pthread_has_direct_tsd() { return false; } +inline void *_pthread_getspecific_direct(pthread_key_t k) { + return pthread_getspecific(k); +} +inline void _pthread_setspecific_direct(pthread_key_t k, void *v) { + pthread_setspecific(k, v); +} +} +#endif + +#define SWIFT_TLS_DECLARE_DTOR(name) void name(void *) + +using tls_key_t = pthread_key_t; +using tls_dtor_t = void (*)(void *); + +inline tls_key_t tls_get_key(tls_key k) { + switch (k) { + case tls_key::runtime: + return __PTK_FRAMEWORK_SWIFT_KEY0; + case tls_key::stdlib: + return __PTK_FRAMEWORK_SWIFT_KEY1; + case tls_key::compatibility50: + return __PTK_FRAMEWORK_SWIFT_KEY2; + case tls_key::concurrency_task: + return __PTK_FRAMEWORK_SWIFT_KEY3; + case tls_key::concurrency_executor_tracking_info: + return __PTK_FRAMEWORK_SWIFT_KEY4; + case tls_key::concurrency_fallback: + return __PTK_FRAMEWORK_SWIFT_KEY5; + } +} + +inline bool tls_init(tls_key_t key, tls_dtor_t dtor) { + return pthread_key_init_np(key, dtor) == 0; +} + +inline bool tls_init(tls_key key, tls_dtor_t dtor) { + return tls_init(tls_get_key(key), dtor); +} + +inline bool tls_alloc(tls_key_t &key, tls_dtor_t dtor) { + return pthread_key_create(&key, dtor) == 0; +} + +inline void *tls_get(tls_key_t key) { + if (_pthread_has_direct_tsd()) + return _pthread_getspecific_direct(key); + else + return pthread_getspecific(key); +} + +inline void *tls_get(tls_key key) { return tls_get(tls_get_key(key)); } + +inline void tls_set(tls_key_t key, void *value) { + if (_pthread_has_direct_tsd()) + _pthread_setspecific_direct(key, value); + else + pthread_setspecific(key, value); +} + +inline void tls_set(tls_key key, void *value) { + tls_set(tls_get_key(key), value); +} + +} // namespace threading_impl + +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_DARWIN_H diff --git a/include/swift/Threading/Impl/Linux.h b/include/swift/Threading/Impl/Linux.h new file mode 100644 index 0000000000000..f631dc9fb4061 --- /dev/null +++ b/include/swift/Threading/Impl/Linux.h @@ -0,0 +1,174 @@ +//==--- Pthreads.h - Threading abstraction implementation ------ -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for Linux +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_LINUX_H +#define SWIFT_THREADING_IMPL_LINUX_H + +#include +#include + +#include +#include + +#include "llvm/ADT/Optional.h" + +#include "swift/Threading/Errors.h" + +#include "Linux/ulock.h" + +namespace swift { +namespace threading_impl { + +#define SWIFT_LINUXTHREADS_CHECK(expr) \ + do { \ + int res_ = (expr); \ + if (res_ != 0) \ + swift::threading::fatal(#expr " failed with error %d\n", res_); \ + } while (0) + +#define SWIFT_LINUXTHREADS_RETURN_TRUE_OR_FALSE(expr) \ + do { \ + int res_ = (expr); \ + switch (res_) { \ + case 0: \ + return true; \ + case EBUSY: \ + return false; \ + default: \ + swift::threading::fatal(#expr " failed with error (%d)\n", res_); \ + } \ + } while (0) + +// .. Thread related things .................................................. + +using thread_id = ::pthread_t; + +inline thread_id thread_get_current() { return ::pthread_self(); } + +bool thread_is_main(); + +inline bool threads_same(thread_id a, thread_id b) { + return ::pthread_equal(a, b); +} + +llvm::Optional thread_get_current_stack_bounds(); + +// .. Mutex support .......................................................... + +using mutex_handle = ::pthread_mutex_t; + +inline void mutex_init(mutex_handle &handle, bool checked = false) { + if (!checked) { + handle = PTHREAD_MUTEX_INITIALIZER; + } else { + ::pthread_mutexattr_t attr; + SWIFT_LINUXTHREADS_CHECK(::pthread_mutexattr_init(&attr)); + SWIFT_LINUXTHREADS_CHECK( + ::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK)); + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_init(&handle, &attr)); + SWIFT_LINUXTHREADS_CHECK(::pthread_mutexattr_destroy(&attr)); + } +} +inline void mutex_destroy(mutex_handle &handle) { + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_destroy(&handle)); +} + +inline void mutex_lock(mutex_handle &handle) { + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_lock(&handle)); +} +inline void mutex_unlock(mutex_handle &handle) { + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_unlock(&handle)); +} +inline bool mutex_try_lock(mutex_handle &handle) { + SWIFT_LINUXTHREADS_RETURN_TRUE_OR_FALSE(::pthread_mutex_trylock(&handle)); +} + +inline void mutex_unsafe_lock(mutex_handle &handle) { + (void)::pthread_mutex_lock(&handle); +} +inline void mutex_unsafe_unlock(mutex_handle &handle) { + (void)::pthread_mutex_unlock(&handle); +} + +using lazy_mutex_handle = ::pthread_mutex_t; + +// We don't actually need to be lazy here because pthreads has +// PTHREAD_MUTEX_INITIALIZER. +inline constexpr lazy_mutex_handle lazy_mutex_initializer() { + return PTHREAD_MUTEX_INITIALIZER; +} +inline void lazy_mutex_destroy(lazy_mutex_handle &handle) { + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_destroy(&handle)); +} + +inline void lazy_mutex_lock(lazy_mutex_handle &handle) { + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_lock(&handle)); +} +inline void lazy_mutex_unlock(lazy_mutex_handle &handle) { + SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_unlock(&handle)); +} +inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { + SWIFT_LINUXTHREADS_RETURN_TRUE_OR_FALSE(::pthread_mutex_trylock(&handle)); +} + +inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) { + (void)::pthread_mutex_lock(&handle); +} +inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) { + (void)::pthread_mutex_unlock(&handle); +} + +// .. Once ................................................................... + +struct once_t { + std::atomic flag; + linux::ulock_t lock; +}; + +void once_slow(once_t &predicate, void (*fn)(void *), void *context); + +inline void once_impl(once_t &predicate, void (*fn)(void *), void *context) { + // Sadly we can't use ::pthread_once() for this (no context) + if (predicate.flag.load(std::memory_order_acquire) < 0) + return; + + once_slow(predicate, fn, context); +} + +// .. Thread local storage ................................................... + +#if __cplusplus >= 201103L || __has_feature(cxx_thread_local) +#define SWIFT_THREAD_LOCAL thread_local +#endif + +using tls_key_t = pthread_key_t; +using tls_dtor_t = void (*)(void *); + +inline bool tls_alloc(tls_key_t &key, tls_dtor_t dtor) { + return pthread_key_create(&key, dtor) == 0; +} + +inline void *tls_get(tls_key_t key) { return pthread_getspecific(key); } + +inline void tls_set(tls_key_t key, void *value) { + pthread_setspecific(key, value); +} + +} // namespace threading_impl + +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_PTHREADS_H diff --git a/include/swift/Threading/Impl/Linux/ulock.h b/include/swift/Threading/Impl/Linux/ulock.h new file mode 100644 index 0000000000000..a9020965ec5d4 --- /dev/null +++ b/include/swift/Threading/Impl/Linux/ulock.h @@ -0,0 +1,94 @@ +//==--- ulock.h - 32-bit futex-based lock for Linux ------------ -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements a 32-bit futex-based locking primitive with priority inversion +// support. +// +// This has comparable performance to pthread_mutex_t (on x86-64, it's slower +// under contention, but not much different otherwise; other architectures may +// vary), but it only takes up 32 bits instead of 40 *bytes*. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_LINUX_ULOCK_H +#define SWIFT_THREADING_IMPL_LINUX_ULOCK_H + +// This file is specific to Linux; we're just going to assume we can use +// various GCC/Clang extensions here. + +#include +#include +#include +#include +#include + +#include +#include + +namespace swift { +namespace threading_impl { +namespace linux { + +typedef std::int32_t ulock_t; + +#define ULOCK_INITIALIZER 0 +#define ulock_fastpath(x) __builtin_expect((x), true) + +inline int ulock_get_tid(void) { + static __thread int tid; + if (tid == 0) + tid = syscall(SYS_gettid); + return tid; +} + +inline int ulock_futex(ulock_t *lock, int op) { + return syscall(SYS_futex, lock, op | FUTEX_PRIVATE_FLAG, 0, nullptr, nullptr, + 0); +} + +inline void ulock_lock(ulock_t *lock) { + const ulock_t tid = ulock_get_tid(); + do { + ulock_t zero = 0; + if (ulock_fastpath(__atomic_compare_exchange_n( + lock, &zero, tid, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))) + return; + + } while (ulock_futex(lock, FUTEX_LOCK_PI) != 0); +} + +inline bool ulock_trylock(ulock_t *lock) { + ulock_t zero = 0; + if (ulock_fastpath(__atomic_compare_exchange_n(lock, &zero, ulock_get_tid(), + true, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED))) + return true; + + return ulock_futex(lock, FUTEX_TRYLOCK_PI) == 0; +} + +inline void ulock_unlock(ulock_t *lock) { + const ulock_t tid = ulock_get_tid(); + do { + ulock_t expected = tid; + if (ulock_fastpath(__atomic_compare_exchange_n( + lock, &expected, 0, true, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) + return; + + } while (ulock_futex(lock, FUTEX_UNLOCK_PI) != 0); +} + +} // namespace linux +} // namespace threading_impl +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_LINUX_ULOCK_H diff --git a/include/swift/Threading/Impl/Nothreads.h b/include/swift/Threading/Impl/Nothreads.h new file mode 100644 index 0000000000000..576f1e2f87dad --- /dev/null +++ b/include/swift/Threading/Impl/Nothreads.h @@ -0,0 +1,80 @@ +//==--- Nothreads.h - Threading abstraction implementation ----- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for platforms without threading +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_NOTHREADS_H +#define SWIFT_THREADING_IMPL_NOTHREADS_H + +#include "llvm/ADT/Optional.h" + +namespace swift { +namespace threading_impl { + +// .. Thread related things .................................................. + +using thread_id = unsigned; + +inline thread_id thread_get_current() { return 0; } +inline bool thread_is_main() { return true; } +inline bool threads_same(thread_id a, thread_id b) { return a == b; } +inline llvm::Optional thread_get_current_stack_bounds() { + return {}; +} + +// .. Mutex support .......................................................... + +using mutex_handle = unsigned; + +inline void mutex_init(mutex_handle &handle, bool checked = false) {} +inline void mutex_destroy(mutex_handle &handle) {} +inline void mutex_lock(mutex_handle &handle) {} +inline void mutex_unlock(mutex_handle &handle) {} +inline bool mutex_try_lock(mutex_handle &handle) { return true; } + +inline void mutex_unsafe_lock(mutex_handle &handle) {} +inline void mutex_unsafe_unlock(mutex_handle &handle) {} + +using lazy_mutex_handle = unsigned; + +inline constexpr lazy_mutex_handle lazy_mutex_initializer() { return 0; } +inline void lazy_mutex_destroy(lazy_mutex_handle &handle) {} +inline void lazy_mutex_lock(lazy_mutex_handle &handle) {} +inline void lazy_mutex_unlock(lazy_mutex_handle &handle) {} +inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { return true; } + +inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) {} +inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) {} + +// .. Once ................................................................... + +typedef bool once_t; + +inline void once_impl(once_t &predicate, void (*fn)(void *), void *ctx) { + if (!predicate) { + predicate = true; + fn(ctx); + } +} + +// .. Thread local storage ................................................... + +// If we have no threads, we can use the simple version of TLS +#define SWIFT_THREAD_LOCAL + +} // namespace threading_impl + +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_NOTHREADS_H diff --git a/include/swift/Threading/Impl/Pthreads.h b/include/swift/Threading/Impl/Pthreads.h new file mode 100644 index 0000000000000..09dd4816b9b3a --- /dev/null +++ b/include/swift/Threading/Impl/Pthreads.h @@ -0,0 +1,169 @@ +//==--- Pthreads.h - Threading abstraction implementation ------ -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for plain pthreads +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_PTHREADS_H +#define SWIFT_THREADING_IMPL_PTHREADS_H + +#include +#include + +#include +#include + +#include "llvm/ADT/Optional.h" + +#include "swift/Threading/Errors.h" + +namespace swift { +namespace threading_impl { + +#define SWIFT_PTHREADS_CHECK(expr) \ + do { \ + int res_ = (expr); \ + if (res_ != 0) \ + swift::threading::fatal(#expr " failed with error %d\n", res_); \ + } while (0) + +#define SWIFT_PTHREADS_RETURN_TRUE_OR_FALSE(expr) \ + do { \ + int res_ = (expr); \ + switch (res_) { \ + case 0: \ + return true; \ + case EBUSY: \ + return false; \ + default: \ + swift::threading::fatal(#expr " failed with error (%d)\n", res_); \ + } \ + } while (0) + +// .. Thread related things .................................................. + +using thread_id = ::pthread_t; + +inline thread_id thread_get_current() { return ::pthread_self(); } + +bool thread_is_main(); + +inline bool threads_same(thread_id a, thread_id b) { + return ::pthread_equal(a, b); +} + +llvm::Optional thread_get_current_stack_bounds(); + +// .. Mutex support .......................................................... + +using mutex_handle = ::pthread_mutex_t; + +inline void mutex_init(mutex_handle &handle, bool checked = false) { + if (!checked) { + handle = PTHREAD_MUTEX_INITIALIZER; + } else { + ::pthread_mutexattr_t attr; + SWIFT_PTHREADS_CHECK(::pthread_mutexattr_init(&attr)); + SWIFT_PTHREADS_CHECK( + ::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK)); + SWIFT_PTHREADS_CHECK(::pthread_mutex_init(&handle, &attr)); + SWIFT_PTHREADS_CHECK(::pthread_mutexattr_destroy(&attr)); + } +} +inline void mutex_destroy(mutex_handle &handle) { + SWIFT_PTHREADS_CHECK(::pthread_mutex_destroy(&handle)); +} + +inline void mutex_lock(mutex_handle &handle) { + SWIFT_PTHREADS_CHECK(::pthread_mutex_lock(&handle)); +} +inline void mutex_unlock(mutex_handle &handle) { + SWIFT_PTHREADS_CHECK(::pthread_mutex_unlock(&handle)); +} +inline bool mutex_try_lock(mutex_handle &handle) { + SWIFT_PTHREADS_RETURN_TRUE_OR_FALSE(::pthread_mutex_trylock(&handle)); +} + +inline void mutex_unsafe_lock(mutex_handle &handle) { + (void)::pthread_mutex_lock(&handle); +} +inline void mutex_unsafe_unlock(mutex_handle &handle) { + (void)::pthread_mutex_unlock(&handle); +} + +using lazy_mutex_handle = ::pthread_mutex_t; + +// We don't actually need to be lazy here because pthreads has +// PTHREAD_MUTEX_INITIALIZER. +inline constexpr lazy_mutex_handle lazy_mutex_initializer() { + return PTHREAD_MUTEX_INITIALIZER; +} +inline void lazy_mutex_destroy(lazy_mutex_handle &handle) { + SWIFT_PTHREADS_CHECK(::pthread_mutex_destroy(&handle)); +} + +inline void lazy_mutex_lock(lazy_mutex_handle &handle) { + SWIFT_PTHREADS_CHECK(::pthread_mutex_lock(&handle)); +} +inline void lazy_mutex_unlock(lazy_mutex_handle &handle) { + SWIFT_PTHREADS_CHECK(::pthread_mutex_unlock(&handle)); +} +inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { + SWIFT_PTHREADS_RETURN_TRUE_OR_FALSE(::pthread_mutex_trylock(&handle)); +} + +inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) { + (void)::pthread_mutex_lock(&handle); +} +inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) { + (void)::pthread_mutex_unlock(&handle); +} + +// .. Once ................................................................... + +using once_t = std::atomic; + +void once_slow(once_t &predicate, void (*fn)(void *), void *context); + +inline void once_impl(once_t &predicate, void (*fn)(void *), void *context) { + // Sadly we can't use ::pthread_once() for this (no context) + if (predicate.load(std::memory_order_acquire) < 0) + return; + + once_slow(predicate, fn, context); +} + +// .. Thread local storage ................................................... + +#if __cplusplus >= 201103L || __has_feature(cxx_thread_local) +#define SWIFT_THREAD_LOCAL thread_local +#endif + +using tls_key_t = pthread_key_t; +using tls_dtor_t = void (*)(void *); + +inline bool tls_alloc(tls_key_t &key, tls_dtor_t dtor) { + return pthread_key_create(&key, dtor) == 0; +} + +inline void *tls_get(tls_key_t key) { return pthread_getspecific(key); } + +inline void tls_set(tls_key_t key, void *value) { + pthread_setspecific(key, value); +} + +} // namespace threading_impl + +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_PTHREADS_H diff --git a/include/swift/Threading/Impl/Win32.h b/include/swift/Threading/Impl/Win32.h new file mode 100644 index 0000000000000..8ae8cc181defd --- /dev/null +++ b/include/swift/Threading/Impl/Win32.h @@ -0,0 +1,131 @@ +//==--- Win32.h - Threading abstraction implementation --------- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for Windows threads +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_WIN32_H +#define SWIFT_THREADING_IMPL_WIN32_H + +#include "Win32/Win32Defs.h" + +#include + +#include "llvm/ADT/Optional.h" + +namespace swift { +namespace threading_impl { + +// .. Thread related things .................................................. + +using thread_id = ::DWORD; + +inline thread_id thread_get_current() { return ::GetCurrentThreadId(); } +bool thread_is_main(); +inline bool threads_same(thread_id a, thread_id b) { return a == b; } +llvm::Optional thread_get_current_stack_bounds(); + +// .. Mutex support .......................................................... + +using mutex_handle = SWIFT_SRWLOCK; + +inline void mutex_init(mutex_handle &handle, bool checked = false) { + handle = SRWLOCK_INIT; +} +inline void mutex_destroy(mutex_handle &handle) {} + +inline void mutex_lock(mutex_handle &handle) { + AcquireSRWLockExclusive(&handle); +} +inline void mutex_unlock(mutex_handle &handle) { + ReleaseSRWLockExclusive(&handle); +} +inline bool mutex_try_lock(mutex_handle &handle) { + return !!TryAcquireSRWLockExclusive(&handle); +} + +inline void mutex_unsafe_lock(mutex_handle &handle) { + AcquireSRWLockExclusive(&handle); +} +inline void mutex_unsafe_unlock(mutex_handle &handle) { + ReleaseSRWLockExclusive(&handle); +} + +using lazy_mutex_handle = SWIFT_SRWLOCK; + +// We don't need to be lazy here because Win32 has SRWLOCK_INIT. +inline constexpr lazy_mutex_handle lazy_mutex_initializer() { + return SRWLOCK_INIT; +} +inline void lazy_mutex_destroy(lazy_mutex_handle &handle) {} + +inline void lazy_mutex_lock(lazy_mutex_handle &handle) { + AcquireSRWLockExclusive(&handle); +} +inline void lazy_mutex_unlock(lazy_mutex_handle &handle) { + ReleaseSRWLockExclusive(&handle); +} +inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { + return !!TryAcquireSRWLockExclusive(&handle); +} + +inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) { + AcquireSRWLockExclusive(&handle); +} +inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) { + ReleaseSRWLockExclusive(&handle); +} + +// .. Once ................................................................... + +typedef std::atomic once_t; + +void once_slow(once_t &predicate, void (*fn)(void *), void *context); + +inline void once_impl(once_t &predicate, void (*fn)(void *), void *context) { + // Using INIT_ONCE is slower than doing this. + if (predicate.load(std::memory_order_acquire) < 0) + return; + + once_slow(predicate, fn, context); +} + +// .. Thread local storage ................................................... + +#ifdef __clang__ +#if __has_feature(cxx_thread_local) +#define SWIFT_THREAD_LOCAL thread_local +#endif +#elif __cplusplus >= 201103L +#define SWIFT_THREAD_LOCAL thread_local +#endif + +#define SWIFT_TLS_DECLARE_DTOR(name) void NTAPI name(void *) + +using tls_key_t = ::DWORD; +using tls_dtor_t = ::PFLS_CALLBACK_FUNCTION; + +inline bool tls_alloc(tls_key_t &key, tls_dtor_t dtor) { + key = ::FlsAlloc(dtor); + return key != FLS_OUT_OF_INDEXES; +} + +inline void *tls_get(tls_key_t key) { return ::FlsGetValue(key); } + +inline void tls_set(tls_key_t key, void *value) { ::FlsSetValue(key, value); } + +} // namespace threading_impl + +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_WIN32_H diff --git a/include/swift/Threading/Impl/Win32/Win32Defs.h b/include/swift/Threading/Impl/Win32/Win32Defs.h new file mode 100644 index 0000000000000..644e27ff4609d --- /dev/null +++ b/include/swift/Threading/Impl/Win32/Win32Defs.h @@ -0,0 +1,92 @@ +//==--- Win32Defs.h - Windows API definitions ------------------ -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// We cannot include from the Threading headers because they get +// included all over the place and defines a large number of +// obnoxious macros. Instead, this header declares *just* what we need. +// +// If you need in a file, please make sure to include it *before* +// this file, or you'll get errors about RTL_SRWLOCK. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_IMPL_WIN32_DEFS_H +#define SWIFT_THREADING_IMPL_WIN32_DEFS_H + +#define DECLSPEC_IMPORT __declspec(dllimport) +#define WINBASEAPI DECLSPEC_IMPORT +#define WINAPI __stdcall +#define NTAPI __stdcall + +// #defines VOID rather than typedefing it(!) Changing that +// to use a typedef instead isn't problematic later on, so let's do that. +#undef VOID + +typedef void VOID, *PVOID; +typedef unsigned char BYTE; +typedef BYTE BOOLEAN; +typedef int BOOL; +typedef unsigned long DWORD; + +typedef VOID(NTAPI *PFLS_CALLBACK_FUNCTION)(PVOID lpFlsData); + +typedef struct _RTL_SRWLOCK *PRTL_SRWLOCK; +typedef PRTL_SRWLOCK PSRWLOCK; + +// These have to be #defines, to avoid problems with +#define RTL_SRWLOCK_INIT \ + { 0 } +#define SRWLOCK_INIT RTL_SRWLOCK_INIT +#define FLS_OUT_OF_INDEXES ((DWORD)0xFFFFFFFF) + +extern "C" { +WINBASEAPI DWORD WINAPI GetCurrentThreadId(VOID); + +WINBASEAPI VOID WINAPI InitializeSRWLock(PSRWLOCK SRWLock); +WINBASEAPI VOID WINAPI ReleaseSRWLockExclusive(PSRWLOCK SRWLock); +WINBASEAPI VOID WINAPI AcquireSRWLockExclusive(PSRWLOCK SRWLock); +WINBASEAPI BOOLEAN WINAPI TryAcquireSRWLockExclusive(PSRWLOCK SRWLock); + +WINBASEAPI DWORD WINAPI FlsAlloc(PFLS_CALLBACK_FUNCTION lpCallback); +WINBASEAPI PVOID WINAPI FlsGetValue(DWORD dwFlsIndex); +WINBASEAPI BOOL WINAPI FlsSetValue(DWORD dwFlsIndex, PVOID lpFlsData); +WINBASEAPI BOOL WINAPI FlsFree(DWORD dwFlsIndex); +} + +namespace swift { +namespace threading_impl { + +// We do this because we can't declare _RTL_SRWLOCK here in case someone +// later includes +struct SWIFT_SRWLOCK { + PVOID Ptr; +}; + +typedef SWIFT_SRWLOCK *PSWIFT_SRWLOCK; + +inline VOID InitializeSRWLock(PSWIFT_SRWLOCK SRWLock) { + ::InitializeSRWLock(reinterpret_cast(SRWLock)); +} +inline VOID ReleaseSRWLockExclusive(PSWIFT_SRWLOCK SRWLock) { + ::ReleaseSRWLockExclusive(reinterpret_cast(SRWLock)); +} +inline VOID AcquireSRWLockExclusive(PSWIFT_SRWLOCK SRWLock) { + ::AcquireSRWLockExclusive(reinterpret_cast(SRWLock)); +} +inline BOOLEAN TryAcquireSRWLockExclusive(PSWIFT_SRWLOCK SRWLock) { + return ::TryAcquireSRWLockExclusive(reinterpret_cast(SRWLock)); +} + +} // namespace threading_impl +} // namespace swift + +#endif // SWIFT_THREADING_IMPL_WIN32_DEFS_H diff --git a/include/swift/Threading/Mutex.h b/include/swift/Threading/Mutex.h new file mode 100644 index 0000000000000..49fefae2df5c0 --- /dev/null +++ b/include/swift/Threading/Mutex.h @@ -0,0 +1,280 @@ +//===--- Mutex.h - Mutex and ScopedLock ----------------------- -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Provides a system-independent Mutex abstraction, as well as some +// related utilities like ScopedLock. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_MUTEX_H +#define SWIFT_THREADING_MUTEX_H + +#include +#include + +#include "Impl.h" + +namespace swift { + +// -- ScopedLock --------------------------------------------------------------- + +/// Compile time adjusted stack based object that locks/unlocks the supplied +/// Mutex type. Use the provided typedefs instead of this directly. +template +class ScopedLockT { + ScopedLockT() = delete; + ScopedLockT(const ScopedLockT &) = delete; + ScopedLockT &operator=(const ScopedLockT &) = delete; + ScopedLockT(ScopedLockT &&) = delete; + ScopedLockT &operator=(ScopedLockT &&) = delete; + +public: + explicit ScopedLockT(T &l) : Lock(l) { + if (Inverted) { + Lock.unlock(); + } else { + Lock.lock(); + } + } + + ~ScopedLockT() { + if (Inverted) { + Lock.lock(); + } else { + Lock.unlock(); + } + } + +private: + T &Lock; +}; + +// -- Mutex -------------------------------------------------------------------- + +/// A Mutex object that supports `BasicLockable` and `Lockable` C++ concepts. +/// See http://en.cppreference.com/w/cpp/concept/BasicLockable +/// See http://en.cppreference.com/w/cpp/concept/Lockable +/// +/// This is NOT a recursive mutex. +class Mutex { + + Mutex(const Mutex &) = delete; + Mutex &operator=(const Mutex &) = delete; + Mutex(Mutex &&) = delete; + Mutex &operator=(Mutex &&) = delete; + +public: + /// Constructs a non-recursive mutex. + /// + /// If `checked` is true the mutex will attempt to check for misuse and + /// fatalError when detected. If `checked` is false (the default) the + /// mutex will make little to no effort to check for misuse (more efficient). + explicit Mutex(bool checked = false) { + threading_impl::mutex_init(Handle, checked); + } + ~Mutex() { threading_impl::mutex_destroy(Handle); } + + /// The lock() method has the following properties: + /// - Behaves as an atomic operation. + /// - Blocks the calling thread until exclusive ownership of the mutex + /// can be obtained. + /// - Prior m.unlock() operations on the same mutex synchronize-with + /// this lock operation. + /// - The behavior is undefined if the calling thread already owns + /// the mutex (likely a deadlock). + /// - Does not throw exceptions but will halt on error (fatalError). + void lock() { threading_impl::mutex_lock(Handle); } + + /// The unlock() method has the following properties: + /// - Behaves as an atomic operation. + /// - Releases the calling thread's ownership of the mutex and + /// synchronizes-with the subsequent successful lock operations on + /// the same object. + /// - The behavior is undefined if the calling thread does not own + /// the mutex. + /// - Does not throw exceptions but will halt on error (fatalError). + void unlock() { threading_impl::mutex_unlock(Handle); } + + /// The try_lock() method has the following properties: + /// - Behaves as an atomic operation. + /// - Attempts to obtain exclusive ownership of the mutex for the calling + /// thread without blocking. If ownership is not obtained, returns + /// immediately. The function is allowed to spuriously fail and return + /// even if the mutex is not currently owned by another thread. + /// - If try_lock() succeeds, prior unlock() operations on the same object + /// synchronize-with this operation. lock() does not synchronize with a + /// failed try_lock() + /// - The behavior is undefined if the calling thread already owns + /// the mutex (likely a deadlock)? + /// - Does not throw exceptions but will halt on error (fatalError). + bool try_lock() { return threading_impl::mutex_try_lock(Handle); } + + /// Acquires lock before calling the supplied critical section and releases + /// lock on return from critical section. + /// + /// This call can block while waiting for the lock to become available. + /// + /// For example the following mutates value while holding the mutex lock. + /// + /// ``` + /// mutex.lock([&value] { value++; }); + /// ``` + /// + /// Precondition: Mutex not held by this thread, undefined otherwise. + template + auto withLock(CriticalSection &&criticalSection) + -> decltype(std::forward(criticalSection)()) { + ScopedLock guard(*this); + return std::forward(criticalSection)(); + } + + /// A stack based object that locks the supplied mutex on construction + /// and unlocks it on destruction. + /// + /// Precondition: Mutex unlocked by this thread, undefined otherwise. + typedef ScopedLockT ScopedLock; + + /// A stack based object that unlocks the supplied mutex on construction + /// and relocks it on destruction. + /// + /// Precondition: Mutex locked by this thread, undefined otherwise. + typedef ScopedLockT ScopedUnlock; + +protected: + threading_impl::mutex_handle Handle; +}; + +/// An unsafe variant of the above (for use in the error handling path) +/// +/// This is used to ensure that we can't infinitely recurse if the mutex +/// itself generates errors. +class UnsafeMutex : public Mutex { +public: + UnsafeMutex() : Mutex() {} + + void lock() { threading_impl::mutex_unsafe_unlock(Handle); } + void unlock() { threading_impl::mutex_unsafe_unlock(Handle); } +}; + +/// A lazily initialized variant of Mutex. +/// +/// Use Mutex instead unless you need static allocation. LazyMutex *may* +/// be entirely statically initialized, on some platforms, but on others +/// it might be a little larger than and slightly slower than Mutex. +class LazyMutex { + + LazyMutex(const LazyMutex &) = delete; + LazyMutex &operator=(const LazyMutex &) = delete; + LazyMutex(LazyMutex &&) = delete; + LazyMutex &operator=(LazyMutex &&) = delete; + +public: + constexpr LazyMutex() : Handle(threading_impl::lazy_mutex_initializer()) {} + + // No destructor; this is intentional; this class is for STATIC allocation + // and you don't need to delete mutexes on termination. + + /// See Mutex::lock + void lock() { threading_impl::lazy_mutex_lock(Handle); } + + /// See Mutex::unlock + void unlock() { threading_impl::lazy_mutex_unlock(Handle); } + + /// See Mutex::try_lock + bool try_lock() { return threading_impl::lazy_mutex_try_lock(Handle); } + + /// See Mutex::withLock + template + auto withLock(CriticalSection &&criticalSection) + -> decltype(std::forward(criticalSection)()) { + ScopedLock guard(*this); + return std::forward(criticalSection)(); + } + + /// A stack based object that locks the supplied mutex on construction + /// and unlocks it on destruction. + /// + /// Precondition: Mutex unlocked by this thread, undefined otherwise. + typedef ScopedLockT ScopedLock; + + /// A stack based object that unlocks the supplied mutex on construction + /// and relocks it on destruction. + /// + /// Precondition: Mutex locked by this thread, undefined otherwise. + typedef ScopedLockT ScopedUnlock; + +protected: + threading_impl::lazy_mutex_handle Handle; +}; + +/// An unsafe variant of the above (for use in the error handling path) +/// +/// This is used to ensure that we can't infinitely recurse if the mutex +/// itself generates errors. +class LazyUnsafeMutex : public LazyMutex { +public: + constexpr LazyUnsafeMutex() : LazyMutex() {} + + void lock() { threading_impl::lazy_mutex_unsafe_lock(Handle); } + void unlock() { threading_impl::lazy_mutex_unsafe_unlock(Handle); } +}; + +/// An indirect variant of a Mutex. This allocates the mutex on the heap, for +/// places where having the mutex inline takes up too much space. Used for +/// SmallMutex on platforms where Mutex is large. +class IndirectMutex { + IndirectMutex(const IndirectMutex &) = delete; + IndirectMutex &operator=(const IndirectMutex &) = delete; + IndirectMutex(IndirectMutex &&) = delete; + IndirectMutex &operator=(IndirectMutex &&) = delete; + +public: + explicit IndirectMutex(bool checked = false) { Ptr = new Mutex(checked); } + ~IndirectMutex() { delete Ptr; } + + void lock() { Ptr->lock(); } + + void unlock() { Ptr->unlock(); } + + bool try_lock() { return Ptr->try_lock(); } + + template + auto withLock(CriticalSection &&criticalSection) + -> decltype(criticalSection()) { + return Ptr->withLock(std::forward(criticalSection)); + } + + /// A stack based object that locks the supplied mutex on construction + /// and unlocks it on destruction. + /// + /// Precondition: Mutex unlocked by this thread, undefined otherwise. + typedef ScopedLockT ScopedLock; + + /// A stack based object that unlocks the supplied mutex on construction + /// and relocks it on destruction. + /// + /// Precondition: Mutex locked by this thread, undefined otherwise. + typedef ScopedLockT ScopedUnlock; + +private: + Mutex *Ptr; +}; + +/// A "small" mutex, which is pointer sized or smaller, for places where the +/// mutex is stored inline with limited storage space. This uses a normal Mutex +/// when that is small, and otherwise uses IndirectMutex. +using SmallMutex = + std::conditional_t; + +} // namespace swift + +#endif // SWIFT_THREADING_MUTEX_H diff --git a/include/swift/Threading/Once.h b/include/swift/Threading/Once.h new file mode 100644 index 0000000000000..4291126bb0610 --- /dev/null +++ b/include/swift/Threading/Once.h @@ -0,0 +1,47 @@ +//===--- Once.h - Runtime support for lazy initialization -------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift runtime functions in support of lazy initialization. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_ONCE_H +#define SWIFT_THREADING_ONCE_H + +#include "Impl.h" + +namespace swift { + +using once_t = threading_impl::once_t; + +/// Runs the given function with the given context argument exactly once. +/// The predicate argument must refer to a global or static variable of static +/// extent of type swift::once_t. +inline void once(once_t &predicate, void (*fn)(void *), + void *context = nullptr) { + threading_impl::once_impl(predicate, fn, context); +} + +/// Executes the given callable exactly once. +/// The predicate argument must refer to a global or static variable of static +/// extent of type swift::once_t. +template +inline void once(once_t &predicate, const Callable &callable) { + once(predicate, [](void *ctx) { + const Callable &callable = *(const Callable*)(ctx); + callable(); + }, (void *)(&callable)); +} + +} // namespace swift + +#endif // SWIFT_THREADING_ONCE_H diff --git a/include/swift/Threading/TLSKeys.h b/include/swift/Threading/TLSKeys.h new file mode 100644 index 0000000000000..a9feaae389aea --- /dev/null +++ b/include/swift/Threading/TLSKeys.h @@ -0,0 +1,29 @@ +//===--- TLSKeys.h - Reserved TLS keys ------------------------ -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_TLSKEYS_H +#define SWIFT_THREADING_TLSKEYS_H + +namespace swift { + +enum class tls_key { + runtime, + stdlib, + compatibility50, + concurrency_task, + concurrency_executor_tracking_info, + concurrency_fallback +}; + +} // namespace swift + +#endif // SWIFT_THREADING_TLSKEYS_H diff --git a/include/swift/Threading/Thread.h b/include/swift/Threading/Thread.h new file mode 100644 index 0000000000000..ecf96ef82429e --- /dev/null +++ b/include/swift/Threading/Thread.h @@ -0,0 +1,80 @@ +//===--- Thread.h - Thread abstraction ------------------------ -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Provides an abstract Thread class that identifies a system thread, +// and can fetch the current and main threads as well as being comparable +// with other Thread instances. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_THREAD_H +#define SWIFT_THREADING_THREAD_H + +#include "llvm/ADT/Optional.h" + +#include "Impl.h" + +namespace swift { + +/// Identifies a thread +class Thread { +public: + using Id = threading_impl::thread_id; + using StackBounds = threading_impl::stack_bounds; + +private: + Id id_; + +public: + Thread() {} + explicit Thread(Id platformId) : id_(platformId) {} + Thread(const Thread &other) : id_(other.id_) {} + Thread(Thread &&other) : id_(std::move(other.id_)) {} + + Thread &operator=(const Thread &other) { + id_ = other.id_; + return *this; + } + + Thread &operator=(Thread &&other) { + id_ = other.id_; + return *this; + } + + /// Returns the platform specific thread ID + Id platformThreadId() const { return id_; } + + /// Returns the currently executing thread + static Thread current() { + return Thread(threading_impl::thread_get_current()); + } + + /// Returns true iff executed on the main thread + static bool onMainThread() { return threading_impl::thread_is_main(); } + + /// Returns true if the two Thread values are equal + bool operator==(const Thread &other) const { + return threading_impl::threads_same(id_, other.id_); + } + bool operator!=(const Thread &other) const { + return !threading_impl::threads_same(id_, other.id_); + } + + // Retrieve the bounds of the current thread's stack + static llvm::Optional stackBounds() { + return threading_impl::thread_get_current_stack_bounds(); + } +}; + +} // namespace swift + +#endif // SWIFT_THREADING_THREAD_H diff --git a/include/swift/Threading/ThreadLocalStorage.h b/include/swift/Threading/ThreadLocalStorage.h new file mode 100644 index 0000000000000..0bbf0fae55bc7 --- /dev/null +++ b/include/swift/Threading/ThreadLocalStorage.h @@ -0,0 +1,198 @@ +//===--- ThreadLocalStorage.h - Thread-local storage interface. -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_THREADING_THREADLOCALSTORAGE_H +#define SWIFT_THREADING_THREADLOCALSTORAGE_H + +#include + +#include "Errors.h" +#include "Impl.h" +#include "Once.h" +#include "TLSKeys.h" + +namespace swift { + +// -- Low-level TLS functions ------------------------------------------------- + +#if !SWIFT_THREADING_NONE +using tls_key_t = threading_impl::tls_key_t; +using tls_dtor_t = threading_impl::tls_dtor_t; + +#if SWIFT_THREADING_USE_RESERVED_TLS_KEYS +using threading_impl::tls_get_key; +using threading_impl::tls_init; + +/// tls_init_once() - Initialize TLS, once only +inline void tls_init_once(once_t &token, tls_key_t key, tls_dtor_t dtor) { + const struct tls_init_info { + tls_key_t &k; + tls_dtor_t d; + } info = {key, dtor}; + once( + token, + [](void *ctx) { + const struct tls_init_info *pinfo = + static_cast(ctx); + if (!tls_init(pinfo->k, pinfo->d)) + swift::threading::fatal("tls_init_once() failed to set destructor"); + }, + (void *)&info); +} + +inline void tls_init_once(once_t &token, tls_key key, tls_dtor_t dtor) { + tls_init_once(token, tls_get_key(key), dtor); +} +#endif // SWIFT_THREADING_USE_RESERVED_TLS_KEYS + +using threading_impl::tls_alloc; +using threading_impl::tls_get; +using threading_impl::tls_set; + +/// tls_alloc_once() - Allocate TLS key, once only +inline void tls_alloc_once(once_t &token, tls_key_t &key, tls_dtor_t dtor) { + const struct tls_init_info { + tls_key_t &k; + tls_dtor_t d; + } info = {key, dtor}; + once( + token, + [](void *ctx) { + const struct tls_init_info *pinfo = + static_cast(ctx); + if (!tls_alloc(pinfo->k, pinfo->d)) + swift::threading::fatal("tls_alloc_once() failed to allocate key"); + }, + (void *)&info); +} +#endif // !SWIFT_THREADING_NONE + +// -- High-level TLS classes -------------------------------------------------- + +// Validate a type stored in thread-local storage, using static asserts. Such +// types must fit in a pointer and be trivially copyable/destructible. +#define VALIDATE_THREAD_LOCAL_TYPE(T) \ + static_assert(sizeof(T) <= sizeof(void *), \ + "cannot store more than a pointer"); \ + static_assert(std::is_trivially_copyable::value, \ + "ThreadLocal values must be trivially copyable"); \ + static_assert(std::is_trivially_destructible::value, \ + "ThreadLocal cleanup is not supported, stored types must be " \ + "trivially destructible"); + +// A wrapper class for thread-local storage. +// +// - On platforms that define SWIFT_THREAD_LOCAL, an object of this type +// is declared with SWIFT_THREAD_LOCAL. This makes the object +// itself thread-local, and no internal support is required. +// +// Note that this includes platforms that don't support threading, +// for which SWIFT_THREAD_LOCAL is empty; thread-local declarations +// then create an ordinary global. +// +// - On platforms that don't define SWIFT_THREAD_LOCAL, we have to simulate +// thread-local storage. Fortunately, all of these platforms (at least +// for now) support pthread_getspecific or similar. +#ifdef SWIFT_THREAD_LOCAL +template +class ThreadLocal { + VALIDATE_THREAD_LOCAL_TYPE(T) + + T value; + +public: + constexpr ThreadLocal() {} + + T get() { return value; } + + void set(T newValue) { value = newValue; } +}; +#else +// A wrapper around a TLS key that is lazily initialized using swift::once. +class ThreadLocalKey { + // We rely on the zero-initialization of objects with static storage + // duration. + once_t onceFlag; + tls_key_t key; + +public: + threading_impl::tls_key_t getKey() { + once( + onceFlag, + [](void *ctx) { + tls_key_t *pkey = reinterpret_cast(ctx); + tls_alloc(*pkey, nullptr); + }, + &key); + return key; + } +}; + +#if SWIFT_THREADING_USE_RESERVED_TLS_KEYS +// A type representing a constant TLS key, for use on platforms +// that provide reserved keys. +template +class ConstantThreadLocalKey { +public: + tls_key_t getKey() { return tls_get_key(constantKey); } +}; +#endif + +template +class ThreadLocal { + VALIDATE_THREAD_LOCAL_TYPE(T) + + Key key; + +public: + constexpr ThreadLocal() {} + + T get() { + void *storedValue = tls_get(key.getKey()); + T value; + memcpy(&value, &storedValue, sizeof(T)); + return value; + } + + void set(T newValue) { + void *storedValue; + memcpy(&storedValue, &newValue, sizeof(T)); + tls_set(key.getKey(), storedValue); + } +}; +#endif + +} // end namespace swift + +/// SWIFT_THREAD_LOCAL_TYPE(TYPE, KEY) - Declare a variable +/// to be a thread-local variable. The declaration must have static +/// storage duration; it may be prefixed with "static". +/// +/// For example +/// +/// static SWIFT_THREAD_LOCAL_TYPE(int, SWIFT_RESERVED_TLS_KEY_T_9) frobble; +/// +/// Because of the fallback path, the default-initialization of the +/// type must be equivalent to a bitwise zero-initialization, and the +/// type must be small and trivially copyable and destructible. +#ifdef SWIFT_THREAD_LOCAL +#define SWIFT_THREAD_LOCAL_TYPE(TYPE, KEY) \ + SWIFT_THREAD_LOCAL swift::ThreadLocal +#elif SWIFT_THREADING_USE_RESERVED_TLS_KEYS +#define SWIFT_THREAD_LOCAL_TYPE(TYPE, KEY) \ + swift::ThreadLocal> +#else +#define SWIFT_THREAD_LOCAL_TYPE(TYPE, KEY) \ + swift::ThreadLocal +#endif + +#endif // SWIFT_THREADING_THREADLOCALSTORAGE_H diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 71f673185e08d..a5edfe8773a17 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -50,3 +50,5 @@ add_subdirectory(SymbolGraphGen) add_subdirectory(Syntax) add_subdirectory(SyntaxParse) add_subdirectory(TBDGen) +add_subdirectory(Threading) + diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt index 6976fc2e9d919..ec94b5fc8b121 100644 --- a/lib/FrontendTool/CMakeLists.txt +++ b/lib/FrontendTool/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries(swiftFrontendTool PRIVATE swiftSIL swiftSILGen swiftSILOptimizer - swiftTBDGen) + swiftTBDGen + swiftThreading) set_swift_llvm_is_available(swiftFrontendTool) diff --git a/lib/Threading/C11.cpp b/lib/Threading/C11.cpp new file mode 100644 index 0000000000000..d1f5b20b1c75a --- /dev/null +++ b/lib/Threading/C11.cpp @@ -0,0 +1,89 @@ +//==--- C11.cpp - Threading abstraction implementation --------- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for C11 threads +// +//===----------------------------------------------------------------------===// + +#if SWIFT_THREADING_C11 + +#include "swift/Threading/Impl.h" +#include "swift/Threading/Errors.h" + +namespace { + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wglobal-constructors" + +class C11ThreadingHelper { +private: + thrd_t mainThread_; + mtx_t onceMutex_; + cnd_t onceCond_; + +public: + C11ThreadingHelper() { + mainThread_ = thrd_current(); + SWIFT_C11THREADS_CHECK(::mtx_init(&onceMutex_, ::mtx_plain)); + SWIFT_C11THREADS_CHECK(::cnd_init(&onceCond_)); + } + + thrd_t main_thread() const { return mainThread_; } + + void once_lock() { SWIFT_C11THREADS_CHECK(mtx_lock(&onceMutex_)); } + void once_unlock() { SWIFT_C11THREADS_CHECK(mtx_unlock(&onceMutex_)); } + void once_broadcast() { SWIFT_C11THREADS_CHECK(cnd_broadcast(&onceCond_)); } + void once_wait() { + SWIFT_C11THREADS_CHECK(mtx_lock(&onceMutex_)); + SWIFT_C11THREADS_CHECK(cnd_wait(&onceCond_, &onceMutex_)); + SWIFT_C11THREADS_CHECK(mtx_unlock(&onceMutex_)); + } +}; + +C11ThreadingHelper helper; + +#pragma clang diagnostic pop + +} // namespace + +using namespace swift; +using namespace threading_impl; + +bool swift::threading_impl::thread_is_main() { + return thrd_equal(thrd_current(), helper.main_thread()); +} + +void swift::threading_impl::once_slow(once_t &predicate, void (*fn)(void *), + void *context) { + std::int64_t zero = 0; + if (std::atomic_compare_exchange_strong_explicit(&predicate, &zero, 1, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + fn(context); + + std::atomic_store_explicit(&predicate, -1, std::memory_order_release); + + helper.once_lock(); + helper.once_unlock(); + helper.once_broadcast(); + return; + } + + helper.once_lock(); + while (std::atomic_load_explicit(&predicate, std::memory_order_acquire) >= + 0) { + helper.once_wait(); + } + helper.once_unlock(); +} + +#endif // SWIFT_THREADING_C11 diff --git a/lib/Threading/CMakeLists.txt b/lib/Threading/CMakeLists.txt new file mode 100644 index 0000000000000..c43deec244bef --- /dev/null +++ b/lib/Threading/CMakeLists.txt @@ -0,0 +1,13 @@ +# If you update this, you also need to update the CMakeLists.txt file in +# stdlib/public/Threading + +# Note that it is *not* an error that Errors.cpp is only listed here. +# It shouldn't be in stdlib/public/Threading because that is an OBJECT_LIBRARY +# and things that use that should be defining their own fatal error handler. + +add_swift_host_library(swiftThreading STATIC + C11.cpp + Linux.cpp + Pthreads.cpp + Win32.cpp + Errors.cpp) diff --git a/lib/Threading/Errors.cpp b/lib/Threading/Errors.cpp new file mode 100644 index 0000000000000..7897b5155866f --- /dev/null +++ b/lib/Threading/Errors.cpp @@ -0,0 +1,41 @@ +//==--- Errors.cpp - Threading implementation error handling --- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Provides a fallback definition of swift::threading::fatal(). You may +// care to provide your own definition elsewhere, to tie the threading code's +// error handling into the relevant code. +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#include "swift/Threading/Errors.h" + +namespace swift { +namespace threading { + +SWIFT_ATTRIBUTE_NORETURN +SWIFT_FORMAT(1, 2) +void fatal(const char *msg, ...) { + std::va_list val; + + va_start(val, msg); + std::vfprintf(stderr, msg, val); + va_end(val); + + std::abort(); +} + +} // namespace threading +} // namespace swift diff --git a/lib/Threading/Linux.cpp b/lib/Threading/Linux.cpp new file mode 100644 index 0000000000000..0a9876ece908f --- /dev/null +++ b/lib/Threading/Linux.cpp @@ -0,0 +1,81 @@ +//==--- Linux.cpp - Threading abstraction implementation ------- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for Linux +// +//===----------------------------------------------------------------------===// + +#if SWIFT_THREADING_LINUX + +#include "swift/Threading/Impl.h" +#include "swift/Threading/Errors.h" + +namespace { + +#pragma clang diagnostic push +#pragma GCC diagnostic ignored "-Wglobal-constructors" + +class MainThreadRememberer { +private: + pthread_t mainThread_; + +public: + MainThreadRememberer() { mainThread_ = pthread_self(); } + + pthread_t main_thread() const { return mainThread_; } +}; + +MainThreadRememberer rememberer; + +#pragma clang diagnostic pop + +} // namespace + +using namespace swift; +using namespace threading_impl; + +bool swift::threading_impl::thread_is_main() { + return pthread_equal(pthread_self(), rememberer.main_thread()); +} + +void swift::threading_impl::once_slow(once_t &predicate, void (*fn)(void *), + void *context) { + linux::ulock_lock(&predicate.lock); + if (predicate.flag.load(std::memory_order_acquire) == 0) { + fn(context); + predicate.flag.store(-1, std::memory_order_release); + } + linux::ulock_unlock(&predicate.lock); +} + +llvm::Optional +swift::threading_impl::thread_get_current_stack_bounds() { + pthread_attr_t attr; + size_t size = 0; + void *begin = nullptr; + + if (!pthread_getattr_np(pthread_self(), &attr)) { + if (!pthread_attr_getstack(&attr, &begin, &size)) { + stack_bounds result = { begin, (char *)begin + size }; + + pthread_attr_destroy(&attr); + + return result; + } + + pthread_attr_destroy(&attr); + } + + return {}; +} + +#endif // SWIFT_THREADING_LINUX diff --git a/lib/Threading/Pthreads.cpp b/lib/Threading/Pthreads.cpp new file mode 100644 index 0000000000000..21f34f86953e8 --- /dev/null +++ b/lib/Threading/Pthreads.cpp @@ -0,0 +1,120 @@ +//==--- Pthreads.cpp - Threading abstraction implementation ---- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for plain pthreads +// +//===----------------------------------------------------------------------===// + +#if SWIFT_THREADING_PTHREADS + +#if defined(__FreeBSD__) || defined(__OpenBSD__) +#include +#endif + +#include "swift/Threading/Impl.h" +#include "swift/Threading/Errors.h" + +namespace { + +#pragma clang diagnostic push +#pragma GCC diagnostic ignored "-Wglobal-constructors" + +class MainThreadRememberer { +private: + pthread_t mainThread_; + +public: + MainThreadRememberer() { mainThread_ = pthread_self(); } + + pthread_t main_thread() const { return mainThread_; } +}; + +MainThreadRememberer rememberer; +pthread_mutex_t onceMutex = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t onceCond = PTHREAD_COND_INITIALIZER; + +#pragma clang diagnostic pop + +} // namespace + +using namespace swift; +using namespace threading_impl; + +bool swift::threading_impl::thread_is_main() { + return pthread_equal(pthread_self(), rememberer.main_thread()); +} + +void swift::threading_impl::once_slow(once_t &predicate, void (*fn)(void *), + void *context) { + std::int64_t zero = 0; + if (predicate.compare_exchange_strong(zero, (std::int64_t)1, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + fn(context); + + predicate.store((std::int64_t)-1, std::memory_order_release); + + pthread_mutex_lock(&onceMutex); + pthread_mutex_unlock(&onceMutex); + pthread_cond_broadcast(&onceCond); + return; + } + + pthread_mutex_lock(&onceMutex); + while (predicate.load(std::memory_order_acquire) >= (std::int64_t)0) { + pthread_cond_wait(&onceCond); + } + pthread_mutex_unlock(&onceMutex); +} + +#if defined(__OpenBSD__) +llvm::Optional +swift::threading_impl::thread_get_current_stack_bounds() { + stack_t sinfo; + + if (!pthread_stackseg_np(pthread_self(), &sinfo)) { + stack_bounds result = { + (char *)sinfo.ss_sp - sinfo.ss_size, + sinfo.ss_sp + }; + return result; + } + + return {}; +} +#else +llvm::Optional +swift::threading_impl::thread_get_current_stack_bounds() { + pthread_attr_t attr; + size_t size = 0; + void *begin = nullptr; + +#if defined(__FreeBSD__) + if (pthread_attr_init(&attr)) + return {}; +#endif + + if (!pthread_getattr_np(pthread_self(), &attr)) { + if (!pthread_attr_getstack(&attr, &begin, &size)) { + stack_bounds result = { begin, (char *)begin + size }; + pthread_attr_destroy(&attr); + return result; + } + + pthread_attr_destroy(&attr); + } + + return {}; +} +#endif + +#endif // SWIFT_THREADING_PTHREADS diff --git a/lib/Threading/Win32.cpp b/lib/Threading/Win32.cpp new file mode 100644 index 0000000000000..a1fb7a3e7636b --- /dev/null +++ b/lib/Threading/Win32.cpp @@ -0,0 +1,122 @@ +//==--- Win32.cpp - Threading abstraction implementation ------- -*-C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Implements threading support for Windows threads +// +//===----------------------------------------------------------------------===// + +#if SWIFT_THREADING_WIN32 + +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include + +#include "swift/Threading/Errors.h" +#include "swift/Threading/Impl.h" + +namespace { + +#pragma clang diagnostic push +#pragma GCC diagnostic ignored "-Wglobal-constructors" + +class MainThreadRememberer { +private: + DWORD dwMainThread_; + +public: + MainThreadRememberer() { dwMainThread_ = ::GetCurrentThreadId(); } + + DWORD main_thread() const { return dwMainThread_; } +}; + +MainThreadRememberer rememberer; + +// Prior to Windows 8, we have to use a global lock +#if _WIN32_WINNT < 0x0602 +SRWLOCK onceMutex = SRWLOCK_INIT; +CONDITION_VARIABLE onceCond = CONDITION_VARIABLE_INIT; +#endif + +#pragma clang diagnostic pop + +} // namespace + +using namespace swift; +using namespace threading_impl; + +bool swift::threading_impl::thread_is_main() { + return ::GetCurrentThreadId() == rememberer.main_thread(); +} + +void swift::threading_impl::once_slow(once_t &predicate, void (*fn)(void *), + void *context) { + std::int64_t expected = 0; + if (predicate.compare_exchange_strong(expected, (std::int64_t)1, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + fn(context); + + predicate.store((std::int64_t)-1, std::memory_order_release); + +#if _WIN32_WINNT >= 0x0602 + // On Windows 8, use WakeByAddressAll() to wake waiters + WakeByAddressAll(&predicate); +#else + // On Windows 7 and earlier, we use a global lock and condition variable; + // this will wake *all* waiters on *all* onces, which might result in a + // thundering herd problem, but it's the best we can do. + AcquireSRWLockExclusive(&onceMutex); + WakeAllConditionVariable(&onceCond); + ReleaseSRWLockExclusive(&onceMutex); +#endif + return; + } + +#if _WIN32_WINNT >= 0x0602 + // On Windows 8, loop waiting on the address until it changes to -1 + while (expected >= 0) { + WaitOnAddress(&predicate, &expected, 8, INFINITE); + expected = predicate.load(std::memory_order_acquire); + } +#else + // On Windows 7 and earlier, wait on the global condition variable + AcquireSRWLockExclusive(&onceMutex); + while (predicate.load(std::memory_order_acquire) >= 0) { + SleepConditionVariableSRW(&onceCond, &onceMutex, INFINITE, 0); + } + ReleaseSRWLockExclusive(&onceMutex); +#endif +} + +llvm::Optional +swift::threading_impl::thread_get_current_stack_bounds() { +#if _WIN32_WINNT >= 0x0602 + ULONG_PTR lowLimit = 0; + ULONG_PTR highLimit = 0; + + GetCurrentThreadStackLimits(&lowLimit, &highLimit); + + stack_bounds result = { (void *)lowLimit, (void *)highLimit }; +#else + MEMORY_BASIC_INFORMATION mbi; + VirtualQuery(&mbi, &mbi, sizeof(mbi)); + + stack_bounds result = { + mbi.AllocationBase, + (char *)mbi.BaseAddress + mbi.RegionSize + }; +#endif + + return result; +} + +#endif // SWIFT_THREADING_WIN32 diff --git a/stdlib/CMakeLists.txt b/stdlib/CMakeLists.txt index e8e3e22826a2b..7e0d2d27666a6 100644 --- a/stdlib/CMakeLists.txt +++ b/stdlib/CMakeLists.txt @@ -174,8 +174,8 @@ function(swift_create_stdlib_targets name variant define_all_alias) endfunction() if("${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}" STREQUAL "singlethreaded" - AND NOT SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) - message(SEND_ERROR "Cannot enable the single-threaded global executor without enabling SWIFT_STDLIB_SINGLE_THREADED_RUNTIME") + AND NOT SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) + message(SEND_ERROR "Cannot enable the single-threaded global executor without enabling SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY") endif() swift_create_stdlib_targets("swift-stdlib" "" TRUE) diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index 5e6cad7a2e9a8..36463c7c77aa1 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -1,6 +1,7 @@ include(AddSwift) include(SwiftSource) +include(Threading) function(add_dependencies_multiple_targets) cmake_parse_arguments( @@ -350,10 +351,13 @@ function(_add_target_variant_c_compile_flags) list(APPEND result "-DSWIFT_STDLIB_HAS_LOCALE") endif() - if(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) - list(APPEND result "-DSWIFT_STDLIB_SINGLE_THREADED_RUNTIME") + if(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) + list(APPEND result "-DSWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY") endif() + threading_package_name("${CFLAGS_SDK}" _threading_package) + list(APPEND result "-DSWIFT_THREADING_${_threading_package}") + if(SWIFT_STDLIB_OS_VERSIONING) list(APPEND result "-DSWIFT_RUNTIME_OS_VERSIONING") endif() diff --git a/stdlib/cmake/modules/StdlibOptions.cmake b/stdlib/cmake/modules/StdlibOptions.cmake index 2d0f2fa63a3f1..04e5e6c842e84 100644 --- a/stdlib/cmake/modules/StdlibOptions.cmake +++ b/stdlib/cmake/modules/StdlibOptions.cmake @@ -1,9 +1,24 @@ include_guard(GLOBAL) include(${CMAKE_CURRENT_LIST_DIR}/../../../cmake/modules/SwiftUtils.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/../../../cmake/modules/Threading.cmake) + precondition(SWIFT_HOST_VARIANT_SDK) precondition(SWIFT_DARWIN_PLATFORMS) +# +----------------------------------------------------------------------+ +# | | +# | NOTE: It makes no sense setting defaults here on the basis of | +# | SWIFT_HOST_VARIANT_SDK, because the stdlib is a *TARGET* | +# | library, not a host library. | +# | | +# | Rather, if you have a default to set, you need to do that | +# | in AddSwiftStdlib.cmake, in an appropriate place, | +# | likely on the basis of CFLAGS_SDK, SWIFTLIB_SINGLE_SDK or | +# | similar. | +# | | +# +----------------------------------------------------------------------+ + if("${SWIFT_HOST_VARIANT_SDK}" MATCHES "CYGWIN") set(SWIFT_STDLIB_SUPPORTS_BACKTRACE_REPORTING_default FALSE) elseif("${SWIFT_HOST_VARIANT_SDK}" MATCHES "HAIKU") @@ -166,16 +181,26 @@ option(SWIFT_STDLIB_HAS_ENVIRON "Build stdlib assuming the platform supports environment variables." TRUE) -option(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +option(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY "Build the standard libraries assuming that they will be used in an environment with only a single thread." FALSE) -if(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +if(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) set(SWIFT_CONCURRENCY_GLOBAL_EXECUTOR_default "singlethreaded") else() set(SWIFT_CONCURRENCY_GLOBAL_EXECUTOR_default "dispatch") endif() +include(Threading) + +threading_package_default("${SWIFT_HOST_VARIANT_SDK}" + SWIFT_THREADING_PACKAGE_default) + +set(SWIFT_THREADING_PACKAGE "${SWIFT_THREADING_PACKAGE_default}" + CACHE STRING + "The threading package to use. Must be one of 'none', 'pthreads', + 'darwin', 'linux', 'win32', 'c11'.") + set(SWIFT_CONCURRENCY_GLOBAL_EXECUTOR "${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR_default}" CACHE STRING "Build the concurrency library to use the given global executor (options: dispatch, singlethreaded, hooked)") diff --git a/stdlib/cmake/modules/SwiftSource.cmake b/stdlib/cmake/modules/SwiftSource.cmake index 6286db16ddafb..c5c6366bcb5ec 100644 --- a/stdlib/cmake/modules/SwiftSource.cmake +++ b/stdlib/cmake/modules/SwiftSource.cmake @@ -1,5 +1,6 @@ include(macCatalystUtils) include(SwiftUtils) +include(Threading) function(_compute_lto_swift_flag option out_var) string(TOLOWER "${option}" lowercase_option) @@ -322,10 +323,13 @@ function(_add_target_variant_swift_compile_flags list(APPEND result "-Xcc" "-DSWIFT_STDLIB_HAS_ENVIRON") endif() - if(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) - list(APPEND result "-D" "SWIFT_STDLIB_SINGLE_THREADED_RUNTIME") + if(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) + list(APPEND result "-D" "SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY") endif() + threading_package_name("${sdk}" _threading_package) + list(APPEND result "-D" "SWIFT_THREADING_${_threading_package}") + set("${result_var_name}" "${result}" PARENT_SCOPE) endfunction() @@ -479,7 +483,7 @@ function(_compile_swift_files list(APPEND swift_flags "-Xfrontend" "-library-level" "-Xfrontend" "api") endif() - if(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) + if(SWIFT_THREADING_PACKAGE STREQUAL "none") list(APPEND swift_flags "-Xfrontend" "-assume-single-threaded") endif() diff --git a/stdlib/private/StdlibUnittest/RaceTest.swift b/stdlib/private/StdlibUnittest/RaceTest.swift index 0380579c644c4..008f17a52000a 100644 --- a/stdlib/private/StdlibUnittest/RaceTest.swift +++ b/stdlib/private/StdlibUnittest/RaceTest.swift @@ -619,12 +619,6 @@ class _InterruptibleSleep { } #endif -#if os(Windows) -typealias ThreadHandle = HANDLE -#else -typealias ThreadHandle = pthread_t -#endif - public func runRaceTest( _: RT.Type, trials: Int, diff --git a/stdlib/public/BackDeployConcurrency/Actor.cpp b/stdlib/public/BackDeployConcurrency/Actor.cpp index c14628bfd445c..31722e4c8c658 100644 --- a/stdlib/public/BackDeployConcurrency/Actor.cpp +++ b/stdlib/public/BackDeployConcurrency/Actor.cpp @@ -17,22 +17,14 @@ #include "ConcurrencyRuntime.h" -#ifdef _WIN32 -// On Windows, an include below triggers an indirect include of minwindef.h -// which contains a definition of the `max` macro, generating an error in our -// use of std::max in this file. This define prevents those macros from being -// defined. -#define NOMINMAX -#endif - #include "CompatibilityOverride.h" +#include "swift/Basic/ListMerger.h" #include "swift/Runtime/Atomic.h" #include "swift/Runtime/Casting.h" -#include "swift/Runtime/Once.h" -#include "swift/Runtime/Mutex.h" -#include "swift/Runtime/ThreadLocal.h" -#include "swift/Runtime/ThreadLocalStorage.h" -#include "swift/Basic/ListMerger.h" +#include "swift/Threading/Once.h" +#include "swift/Threading/Mutex.h" +#include "swift/Threading/Thread.h" +#include "swift/Threading/ThreadLocalStorage.h" #ifndef SWIFT_CONCURRENCY_BACK_DEPLOYMENT #include "llvm/Config/config.h" #else @@ -71,19 +63,8 @@ #include #endif -#if HAVE_PTHREAD_H -#include - -// Only use __has_include since HAVE_PTHREAD_NP_H is not provided. -#if __has_include() -#include -#endif -#endif - #if defined(_WIN32) #include -#include -#include #endif #if SWIFT_OBJC_INTEROP @@ -129,9 +110,9 @@ class ExecutorTrackingInfo { /// the right executor. It would make sense for that to be a /// separate thread-local variable (or whatever is most efficient /// on the target platform). - static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, ActiveInfoInThread, - SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY); + static SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_executor_tracking_info) + ActiveInfoInThread; /// The active executor. ExecutorRef ActiveExecutor = ExecutorRef::generic(); @@ -154,7 +135,7 @@ class ExecutorTrackingInfo { /// Unconditionally initialize a fresh tracking state on the /// current state, shadowing any previous tracking state. - /// leave() must be called beforet the object goes out of scope. + /// leave() must be called before the object goes out of scope. void enterAndShadow(ExecutorRef currentExecutor) { ActiveExecutor = currentExecutor; SavedInfo = ActiveInfoInThread.get(); @@ -197,8 +178,8 @@ class ExecutorTrackingInfo { class ActiveTask { /// A thread-local variable pointing to the active tracking /// information about the current thread, if any. - static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(Pointer, Value, - SWIFT_CONCURRENCY_TASK_KEY); + static SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_task) Value; public: static void set(AsyncTask *task) { Value.set(task); } @@ -206,15 +187,12 @@ class ActiveTask { }; /// Define the thread-locals. -SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, - ActiveTask::Value, - SWIFT_CONCURRENCY_TASK_KEY); +SWIFT_THREAD_LOCAL_TYPE(Pointer, tls_key::concurrency_task) +ActiveTask::Value; -SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, - ExecutorTrackingInfo::ActiveInfoInThread, - SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY); +SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_executor_tracking_info) +ExecutorTrackingInfo::ActiveInfoInThread; } // end anonymous namespace @@ -238,7 +216,7 @@ void swift::runJobInEstablishedExecutorContext(Job *job) { task->runInFullyEstablishedContext(); assert(ActiveTask::get() == nullptr && - "active task wasn't cleared before susspending?"); + "active task wasn't cleared before suspending?"); } else { // There's no extra bookkeeping to do for simple jobs besides swapping in // the voucher. @@ -281,30 +259,20 @@ static ExecutorRef swift_task_getCurrentExecutorImpl() { return result; } -#if defined(_WIN32) -static HANDLE __initialPthread = INVALID_HANDLE_VALUE; -#endif - /// Determine whether we are currently executing on the main thread /// independently of whether we know that we are on the main actor. static bool isExecutingOnMainThread() { -#if defined(__linux__) - return syscall(SYS_gettid) == getpid(); -#elif defined(_WIN32) - if (__initialPthread == INVALID_HANDLE_VALUE) { - DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), - GetCurrentProcess(), &__initialPthread, 0, FALSE, - DUPLICATE_SAME_ACCESS); - } - - return __initialPthread == GetCurrentThread(); +#if SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY + return true; #else - return pthread_main_np() == 1; + return Thread::onMainThread(); #endif } JobPriority swift::swift_task_getCurrentThreadPriority() { -#if defined(__APPLE__) +#if SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY + return JobPriority::UserInitiated; +#elif defined(__APPLE__) return static_cast(qos_class_self()); #else if (isExecutingOnMainThread()) @@ -344,8 +312,8 @@ void swift::swift_task_reportUnexpectedExecutor( const unsigned char *file, uintptr_t fileLength, bool fileIsASCII, uintptr_t line, ExecutorRef executor) { // Make sure we have an appropriate log level. - static swift_once_t logLevelToken; - swift_once(&logLevelToken, checkUnexpectedExecutorLogLevel, nullptr); + static swift::once_t logLevelToken; + swift::once(logLevelToken, checkUnexpectedExecutorLogLevel, nullptr); bool isFatalError = false; switch (unexpectedExecutorLogLevel) { diff --git a/stdlib/public/BackDeployConcurrency/AsyncLet.cpp b/stdlib/public/BackDeployConcurrency/AsyncLet.cpp index 048ed25c16a51..e994e08b6dd30 100644 --- a/stdlib/public/BackDeployConcurrency/AsyncLet.cpp +++ b/stdlib/public/BackDeployConcurrency/AsyncLet.cpp @@ -17,8 +17,9 @@ #include "CompatibilityOverride.h" #include "ConcurrencyRuntime.h" #include "swift/ABI/Metadata.h" -#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/Heap.h" #include "swift/Runtime/HeapObject.h" +#include "swift/Threading/Mutex.h" #include "llvm/ADT/PointerIntPair.h" #include "AsyncLet.h" #include "Debug.h" diff --git a/stdlib/public/BackDeployConcurrency/AsyncStream.cpp b/stdlib/public/BackDeployConcurrency/AsyncStream.cpp index 14b7caf0ad463..389ebd3d7389a 100644 --- a/stdlib/public/BackDeployConcurrency/AsyncStream.cpp +++ b/stdlib/public/BackDeployConcurrency/AsyncStream.cpp @@ -10,30 +10,24 @@ // //===----------------------------------------------------------------------===// -#include "swift/Runtime/Mutex.h" +#include + +#include "swift/Threading/Mutex.h" namespace swift { // return the size in words for the given mutex primitive extern "C" size_t _swift_async_stream_lock_size() { - size_t words = sizeof(MutexHandle) / sizeof(void *); + size_t words = sizeof(Mutex) / sizeof(void *); if (words < 1) { return 1; } return words; } -extern "C" -void _swift_async_stream_lock_init(MutexHandle &lock) { - MutexPlatformHelper::init(lock); +extern "C" void _swift_async_stream_lock_init(Mutex &lock) { + new (&lock) Mutex(); } -extern "C" -void _swift_async_stream_lock_lock(MutexHandle &lock) { - MutexPlatformHelper::lock(lock); -} - -extern "C" -void _swift_async_stream_lock_unlock(MutexHandle &lock) { - MutexPlatformHelper::unlock(lock); -} +extern "C" void _swift_async_stream_lock_lock(Mutex &lock) { lock.lock(); } +extern "C" void _swift_async_stream_lock_unlock(Mutex &lock) { lock.unlock(); } } diff --git a/stdlib/public/BackDeployConcurrency/CMakeLists.txt b/stdlib/public/BackDeployConcurrency/CMakeLists.txt index 5da354361110d..740aefe472f76 100644 --- a/stdlib/public/BackDeployConcurrency/CMakeLists.txt +++ b/stdlib/public/BackDeployConcurrency/CMakeLists.txt @@ -16,7 +16,7 @@ cmake_minimum_required(VERSION 3.19.6) include("${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/modules/StandaloneOverlay.cmake") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules") set(SWIFT_STDLIB_STABLE_ABI TRUE) -set(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME FALSE) +set(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY FALSE) set(SWIFT_CONCURRENCY_GLOBAL_EXECUTOR "dispatch") include(AddSwiftStdlib) @@ -99,6 +99,7 @@ add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} I AsyncThrowingFlatMapSequence.swift AsyncThrowingMapSequence.swift AsyncThrowingPrefixWhileSequence.swift + ConditionVariable.cpp GlobalActor.swift MainActor.swift PartialAsyncTask.swift @@ -114,7 +115,7 @@ add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} I TaskLocal.swift TaskSleep.swift ThreadSanitizer.cpp - Mutex.cpp + ThreadingError.cpp AsyncStreamBuffer.swift AsyncStream.swift AsyncThrowingStream.swift @@ -123,6 +124,8 @@ add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} I ${swift_concurrency_extra_sources} ../Concurrency/linker-support/magic-symbols-for-install-name.c + INCORPORATE_OBJECT_LIBRARIES_SHARED_ONLY + swiftThreading LINK_LIBRARIES ${swift_concurrency_link_libraries} C_COMPILE_FLAGS diff --git a/stdlib/public/BackDeployConcurrency/Mutex.cpp b/stdlib/public/BackDeployConcurrency/ConditionVariable.cpp similarity index 54% rename from stdlib/public/BackDeployConcurrency/Mutex.cpp rename to stdlib/public/BackDeployConcurrency/ConditionVariable.cpp index f42d21ee0a385..b9472290239ff 100644 --- a/stdlib/public/BackDeployConcurrency/Mutex.cpp +++ b/stdlib/public/BackDeployConcurrency/ConditionVariable.cpp @@ -1,4 +1,4 @@ -//===--- Mutex.cpp - Mutex support code -----------------------------------===// +//===--- ConditionVariable.cpp - A condition variable ---------------------===// // // This source file is part of the Swift.org open source project // @@ -11,21 +11,51 @@ //===----------------------------------------------------------------------===// #include "Error.h" - -#define SWIFT_FATAL_ERROR swift_Concurrency_fatalError - -// Include the runtime's mutex support code. -// FIXME: figure out some reasonable way to share this stuff - #include "ConditionVariable.h" -#include "../runtime/MutexPThread.cpp" -#include "../runtime/MutexWin32.cpp" -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - #include "swift/Runtime/MutexSingleThreaded.h" -#endif using namespace swift; +#define fatalError swift_Concurrency_fatalError + +#define reportError(PThreadFunction) \ + do { \ + int errorcode = PThreadFunction; \ + if (errorcode != 0) { \ + fatalError(/* flags = */ 0, "'%s' failed with error '%s'(%d)\n", \ + #PThreadFunction, errorName(errorcode), errorcode); \ + } \ + } while (false) + +#define returnTrueOrReportError(PThreadFunction, returnFalseOnEBUSY) \ + do { \ + int errorcode = PThreadFunction; \ + if (errorcode == 0) \ + return true; \ + if (returnFalseOnEBUSY && errorcode == EBUSY) \ + return false; \ + fatalError(/* flags = */ 0, "'%s' failed with error '%s'(%d)\n", \ + #PThreadFunction, errorName(errorcode), errorcode); \ + } while (false) + +static const char *errorName(int errorcode) { + switch (errorcode) { + case EINVAL: + return "EINVAL"; + case EPERM: + return "EPERM"; + case EDEADLK: + return "EDEADLK"; + case ENOMEM: + return "ENOMEM"; + case EAGAIN: + return "EAGAIN"; + case EBUSY: + return "EBUSY"; + default: + return ""; + } +} + void ConditionPlatformHelper::init(pthread_cond_t &condition) { reportError(pthread_cond_init(&condition, nullptr)); } diff --git a/stdlib/public/BackDeployConcurrency/ConditionVariable.h b/stdlib/public/BackDeployConcurrency/ConditionVariable.h index acc5360d6531f..7a157032f53ee 100644 --- a/stdlib/public/BackDeployConcurrency/ConditionVariable.h +++ b/stdlib/public/BackDeployConcurrency/ConditionVariable.h @@ -18,7 +18,7 @@ #ifndef SWIFT_RUNTIME_CONDITION_VARIABLE_H #define SWIFT_RUNTIME_CONDITION_VARIABLE_H -#include "swift/Runtime/Mutex.h" +#include "swift/Threading/Mutex.h" #include #include diff --git a/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc b/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc index acbe281a6a2b1..0b01d4d7b9299 100644 --- a/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc +++ b/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc @@ -118,7 +118,7 @@ extern "C" void dispatch_queue_set_width(dispatch_queue_t dq, long width); static dispatch_queue_t getGlobalQueue(JobPriority priority) { size_t numericPriority = static_cast(priority); if (numericPriority >= globalQueueCacheCount) - swift_Concurrency_fatalError(0, "invalid job priority %#zx"); + swift_Concurrency_fatalError(0, "invalid job priority %#zx", numericPriority); #ifdef SWIFT_CONCURRENCY_BACK_DEPLOYMENT std::memory_order loadOrder = std::memory_order_acquire; diff --git a/stdlib/public/BackDeployConcurrency/Error.cpp b/stdlib/public/BackDeployConcurrency/Error.cpp index 2fbe4b0865c0b..fa427552853f0 100644 --- a/stdlib/public/BackDeployConcurrency/Error.cpp +++ b/stdlib/public/BackDeployConcurrency/Error.cpp @@ -10,10 +10,27 @@ // //===----------------------------------------------------------------------===// +#include "swift/Threading/Errors.h" +#include + #include "Error.h" // swift::fatalError is not exported from libswiftCore and not shared, so define another // internal function instead. -SWIFT_NORETURN void swift::swift_Concurrency_fatalError(uint32_t flags, const char *format, ...) { +SWIFT_NORETURN +SWIFT_VFORMAT(2) +void swift::swift_Concurrency_fatalErrorv(uint32_t flags, const char *format, + va_list val) { + vfprintf(stderr, format, val); abort(); } + +SWIFT_NORETURN +SWIFT_FORMAT(2, 3) +void swift::swift_Concurrency_fatalError(uint32_t flags, const char *format, + ...) { + va_list val; + + va_start(val, format); + swift_Concurrency_fatalErrorv(flags, format, val); +} diff --git a/stdlib/public/BackDeployConcurrency/Error.h b/stdlib/public/BackDeployConcurrency/Error.h index 1e0fd3ffd339c..85e8a621399a1 100644 --- a/stdlib/public/BackDeployConcurrency/Error.h +++ b/stdlib/public/BackDeployConcurrency/Error.h @@ -17,13 +17,19 @@ #ifndef SWIFT_CONCURRENCY_ERRORS_H #define SWIFT_CONCURRENCY_ERRORS_H +#include "swift/Basic/Compiler.h" + #include "../SwiftShims/Visibility.h" +#include #include -#include +#include namespace swift { -SWIFT_NORETURN void swift_Concurrency_fatalError(uint32_t flags, const char *format, ...); +SWIFT_NORETURN SWIFT_FORMAT(2, 3) void swift_Concurrency_fatalError( + uint32_t flags, const char *format, ...); +SWIFT_NORETURN SWIFT_VFORMAT(2) void swift_Concurrency_fatalErrorv( + uint32_t flags, const char *format, va_list val); } // namespace swift diff --git a/stdlib/public/BackDeployConcurrency/Executor.swift b/stdlib/public/BackDeployConcurrency/Executor.swift index 35862d5bf223f..13e2811b83ef5 100644 --- a/stdlib/public/BackDeployConcurrency/Executor.swift +++ b/stdlib/public/BackDeployConcurrency/Executor.swift @@ -93,7 +93,7 @@ func _checkExpectedExecutor(_filenameStart: Builtin.RawPointer, _filenameStart, _filenameLength, _filenameIsASCII, _line, _executor) } -#if !SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#if !SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY // This must take a DispatchQueueShim, not something like AnyObject, // or else SILGen will emit a retain/release in unoptimized builds, // which won't work because DispatchQueues aren't actually diff --git a/stdlib/public/BackDeployConcurrency/Task.cpp b/stdlib/public/BackDeployConcurrency/Task.cpp index 86f0f5e5958d4..4f44c4bb7ed7d 100644 --- a/stdlib/public/BackDeployConcurrency/Task.cpp +++ b/stdlib/public/BackDeployConcurrency/Task.cpp @@ -17,8 +17,8 @@ #include "CompatibilityOverride.h" #include "ConcurrencyRuntime.h" #include "swift/ABI/Metadata.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/HeapObject.h" +#include "swift/Threading/Mutex.h" #include "Task.h" #include "TaskGroupPrivate.h" #include "TaskLocal.h" diff --git a/stdlib/public/BackDeployConcurrency/TaskGroup.cpp b/stdlib/public/BackDeployConcurrency/TaskGroup.cpp index a0bfd8768f401..359d2d8603dcb 100644 --- a/stdlib/public/BackDeployConcurrency/TaskGroup.cpp +++ b/stdlib/public/BackDeployConcurrency/TaskGroup.cpp @@ -26,8 +26,8 @@ #include "swift/Basic/STLExtras.h" #include "ConcurrencyRuntime.h" #include "swift/Runtime/Config.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/HeapObject.h" +#include "swift/Threading/Mutex.h" #include "Debug.h" #include "bitset" #include "string" @@ -42,6 +42,8 @@ #include #endif +#include + using namespace swift; /******************************************************************************/ diff --git a/stdlib/public/BackDeployConcurrency/TaskLocal.cpp b/stdlib/public/BackDeployConcurrency/TaskLocal.cpp index 31ba53a669931..520f3efcb4ea3 100644 --- a/stdlib/public/BackDeployConcurrency/TaskLocal.cpp +++ b/stdlib/public/BackDeployConcurrency/TaskLocal.cpp @@ -14,10 +14,9 @@ #include "swift/Runtime/Atomic.h" #include "swift/Runtime/Casting.h" #include "swift/Runtime/Once.h" -#include "swift/Runtime/Mutex.h" #include "ConcurrencyRuntime.h" -#include "swift/Runtime/ThreadLocal.h" -#include "swift/Runtime/ThreadLocalStorage.h" +#include "swift/Threading/Mutex.h" +#include "swift/Threading/ThreadLocalStorage.h" #include "swift/ABI/Metadata.h" #include "llvm/ADT/PointerIntPair.h" #include "Actor.h" @@ -32,14 +31,8 @@ #include #endif -#if HAVE_PTHREAD_H -#include -#endif - #if defined(_WIN32) #include -#include -#include #endif using namespace swift; @@ -58,9 +51,8 @@ template struct Pointer { /// THIS IS RUNTIME INTERNAL AND NOT ABI. class FallbackTaskLocalStorage { - static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, Value, - SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY); + static SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_fallback) Value; public: static void set(TaskLocal::Storage *task) { Value.set(task); } @@ -68,9 +60,10 @@ class FallbackTaskLocalStorage { }; /// Define the thread-locals. -SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, FallbackTaskLocalStorage::Value, - SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY); +SWIFT_THREAD_LOCAL_TYPE( + Pointer, + tls_key::concurrency_fallback) +FallbackTaskLocalStorage::Value; // ==== ABI -------------------------------------------------------------------- diff --git a/stdlib/public/BackDeployConcurrency/TaskPrivate.h b/stdlib/public/BackDeployConcurrency/TaskPrivate.h index 71a773e40a287..8e581202ae6cf 100644 --- a/stdlib/public/BackDeployConcurrency/TaskPrivate.h +++ b/stdlib/public/BackDeployConcurrency/TaskPrivate.h @@ -24,6 +24,7 @@ #include "swift/Runtime/Error.h" #include "swift/Runtime/Exclusivity.h" #include "swift/Runtime/HeapObject.h" +#include "swift/Threading/Thread.h" #include "Task.h" #define SWIFT_FATAL_ERROR swift_Concurrency_fatalError @@ -46,27 +47,13 @@ namespace swift { #if 0 #define SWIFT_TASK_DEBUG_LOG(fmt, ...) \ fprintf(stderr, "[%lu] [%s:%d](%s) " fmt "\n", \ - (unsigned long)_swift_get_thread_id(), \ + (unsigned long)Thread::current()::platformThreadId(), \ __FILE__, __LINE__, __FUNCTION__, \ __VA_ARGS__) #else #define SWIFT_TASK_DEBUG_LOG(fmt, ...) (void)0 #endif -#if defined(_WIN32) -using ThreadID = decltype(GetCurrentThreadId()); -#else -using ThreadID = decltype(pthread_self()); -#endif - -inline ThreadID _swift_get_thread_id() { -#if defined(_WIN32) - return GetCurrentThreadId(); -#else - return pthread_self(); -#endif -} - class AsyncTask; class TaskGroup; @@ -109,7 +96,7 @@ void _swift_tsan_release(void *addr); /// executors. #define DISPATCH_QUEUE_GLOBAL_EXECUTOR (void *)1 -#if !defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +#if !defined(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) inline SerialExecutorWitnessTable * _swift_task_getDispatchQueueSerialExecutorWitnessTable() { extern SerialExecutorWitnessTable wtable diff --git a/stdlib/public/BackDeployConcurrency/TaskStatus.cpp b/stdlib/public/BackDeployConcurrency/TaskStatus.cpp index b85f378913a0b..10195bd3d86b8 100644 --- a/stdlib/public/BackDeployConcurrency/TaskStatus.cpp +++ b/stdlib/public/BackDeployConcurrency/TaskStatus.cpp @@ -17,8 +17,8 @@ #include "CompatibilityOverride.h" #include "ConcurrencyRuntime.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/AtomicWaitQueue.h" +#include "swift/Threading/Mutex.h" #include "TaskStatus.h" #include "TaskPrivate.h" #include @@ -36,7 +36,7 @@ ActiveTaskStatus::getStatusRecordParent(TaskStatusRecord *ptr) { /// A lock used to protect management of task-specific status /// record locks. -static StaticMutex StatusRecordLockLock; +static LazyMutex StatusRecordLockLock; namespace { @@ -62,7 +62,7 @@ namespace { /// must acquire the global status-record lock, find this record /// (which should be the innermost record), and wait for an unlock. class StatusRecordLockRecord : - public AtomicWaitQueue, + public AtomicWaitQueue, public TaskStatusRecord { public: StatusRecordLockRecord(TaskStatusRecord *parent) diff --git a/stdlib/public/Concurrency/Mutex.cpp b/stdlib/public/BackDeployConcurrency/ThreadingError.cpp similarity index 54% rename from stdlib/public/Concurrency/Mutex.cpp rename to stdlib/public/BackDeployConcurrency/ThreadingError.cpp index ccf4ec939d882..512bd1128beca 100644 --- a/stdlib/public/Concurrency/Mutex.cpp +++ b/stdlib/public/BackDeployConcurrency/ThreadingError.cpp @@ -1,4 +1,4 @@ -//===--- Mutex.cpp - Mutex support code -----------------------------------===// +//===--- ThreadingError.cpp - Error handling support code -----------------===// // // This source file is part of the Swift.org open source project // @@ -10,15 +10,17 @@ // //===----------------------------------------------------------------------===// -#include "Error.h" +#include "swift/Threading/Errors.h" +#include -#define SWIFT_FATAL_ERROR swift_Concurrency_fatalError +#include "Error.h" -// Include the runtime's mutex support code. -// FIXME: figure out some reasonable way to share this stuff +// Handle fatal errors from the threading library +SWIFT_ATTRIBUTE_NORETURN +SWIFT_FORMAT(1, 2) +void swift::threading::fatal(const char *format, ...) { + va_list val; -#include "../runtime/MutexPThread.cpp" -#include "../runtime/MutexWin32.cpp" -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - #include "swift/Runtime/MutexSingleThreaded.h" -#endif + va_start(val, format); + swift_Concurrency_fatalErrorv(0, format, val); +} diff --git a/stdlib/public/CMakeLists.txt b/stdlib/public/CMakeLists.txt index 73e8183d4a78e..c122134e92baa 100644 --- a/stdlib/public/CMakeLists.txt +++ b/stdlib/public/CMakeLists.txt @@ -64,6 +64,7 @@ endif() add_subdirectory(SwiftShims) add_subdirectory(CommandLineSupport) +add_subdirectory(Threading) # This static library is shared across swiftCore and swiftReflection if(SWIFT_BUILD_STDLIB OR SWIFT_BUILD_REMOTE_MIRROR) diff --git a/stdlib/public/Concurrency/Actor.cpp b/stdlib/public/Concurrency/Actor.cpp index 535f02cb371fe..ddeef8578791b 100644 --- a/stdlib/public/Concurrency/Actor.cpp +++ b/stdlib/public/Concurrency/Actor.cpp @@ -19,27 +19,19 @@ #include #include -#ifdef _WIN32 -// On Windows, an include below triggers an indirect include of minwindef.h -// which contains a definition of the `max` macro, generating an error in our -// use of std::max in this file. This define prevents those macros from being -// defined. -#define NOMINMAX -#endif - #include "../CompatibilityOverride/CompatibilityOverride.h" -#include "swift/Runtime/Atomic.h" -#include "swift/Runtime/AccessibleFunction.h" -#include "swift/Runtime/Casting.h" -#include "swift/Runtime/Once.h" -#include "swift/Runtime/Mutex.h" -#include "swift/Runtime/ThreadLocal.h" -#include "swift/Runtime/ThreadLocalStorage.h" -#include "swift/Runtime/DispatchShims.h" -#include "swift/ABI/Task.h" #include "swift/ABI/Actor.h" +#include "swift/ABI/Task.h" #include "swift/Basic/ListMerger.h" #include "swift/Concurrency/Actor.h" +#include "swift/Runtime/AccessibleFunction.h" +#include "swift/Runtime/Atomic.h" +#include "swift/Runtime/Casting.h" +#include "swift/Runtime/DispatchShims.h" +#include "swift/Threading/Mutex.h" +#include "swift/Threading/Once.h" +#include "swift/Threading/Thread.h" +#include "swift/Threading/ThreadLocalStorage.h" #ifdef SWIFT_CONCURRENCY_BACK_DEPLOYMENT // All platforms where we care about back deployment have a known // configurations. @@ -68,19 +60,8 @@ #include #endif -#if defined(_POSIX_THREADS) -#include - -// Only use __has_include since HAVE_PTHREAD_NP_H is not provided. -#if __has_include() -#include -#endif -#endif - #if defined(_WIN32) #include -#include -#include #endif #if SWIFT_OBJC_INTEROP @@ -126,9 +107,9 @@ class ExecutorTrackingInfo { /// the right executor. It would make sense for that to be a /// separate thread-local variable (or whatever is most efficient /// on the target platform). - static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, ActiveInfoInThread, - SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY); + static SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_executor_tracking_info) + ActiveInfoInThread; /// The active executor. ExecutorRef ActiveExecutor = ExecutorRef::generic(); @@ -194,8 +175,8 @@ class ExecutorTrackingInfo { class ActiveTask { /// A thread-local variable pointing to the active tracking /// information about the current thread, if any. - static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(Pointer, Value, - SWIFT_CONCURRENCY_TASK_KEY); + static SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_task) Value; public: static void set(AsyncTask *task) { Value.set(task); } @@ -203,15 +184,12 @@ class ActiveTask { }; /// Define the thread-locals. -SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, - ActiveTask::Value, - SWIFT_CONCURRENCY_TASK_KEY); +SWIFT_THREAD_LOCAL_TYPE(Pointer, tls_key::concurrency_task) +ActiveTask::Value; -SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, - ExecutorTrackingInfo::ActiveInfoInThread, - SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY); +SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_executor_tracking_info) +ExecutorTrackingInfo::ActiveInfoInThread; } // end anonymous namespace @@ -280,34 +258,18 @@ static ExecutorRef swift_task_getCurrentExecutorImpl() { return result; } -#if defined(_WIN32) -static HANDLE __initialPthread = INVALID_HANDLE_VALUE; -#endif - /// Determine whether we are currently executing on the main thread /// independently of whether we know that we are on the main actor. static bool isExecutingOnMainThread() { -#if SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - return true; -#elif defined(__linux__) - return syscall(SYS_gettid) == getpid(); -#elif defined(_WIN32) - if (__initialPthread == INVALID_HANDLE_VALUE) { - DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), - GetCurrentProcess(), &__initialPthread, 0, FALSE, - DUPLICATE_SAME_ACCESS); - } - - return __initialPthread == GetCurrentThread(); -#elif defined(__wasi__) +#if SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY return true; #else - return pthread_main_np() == 1; + return Thread::onMainThread(); #endif } JobPriority swift::swift_task_getCurrentThreadPriority() { -#if SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#if SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY return JobPriority::UserInitiated; #elif defined(__APPLE__) return static_cast(qos_class_self()); @@ -351,8 +313,8 @@ void swift::swift_task_reportUnexpectedExecutor( const unsigned char *file, uintptr_t fileLength, bool fileIsASCII, uintptr_t line, ExecutorRef executor) { // Make sure we have an appropriate log level. - static swift_once_t logLevelToken; - swift_once(&logLevelToken, checkUnexpectedExecutorLogLevel, nullptr); + static swift::once_t logLevelToken; + swift::once(logLevelToken, checkUnexpectedExecutorLogLevel, nullptr); bool isFatalError = false; switch (unexpectedExecutorLogLevel) { diff --git a/stdlib/public/Concurrency/AsyncLet.cpp b/stdlib/public/Concurrency/AsyncLet.cpp index eb2d91fbcfadf..55759a512b551 100644 --- a/stdlib/public/Concurrency/AsyncLet.cpp +++ b/stdlib/public/Concurrency/AsyncLet.cpp @@ -14,17 +14,20 @@ // //===----------------------------------------------------------------------===// -#include "../CompatibilityOverride/CompatibilityOverride.h" #include "swift/Runtime/Concurrency.h" + +#include "../CompatibilityOverride/CompatibilityOverride.h" +#include "Debug.h" +#include "TaskPrivate.h" + #include "swift/ABI/AsyncLet.h" #include "swift/ABI/Metadata.h" #include "swift/ABI/Task.h" #include "swift/ABI/TaskOptions.h" -#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/Heap.h" #include "swift/Runtime/HeapObject.h" +#include "swift/Threading/Mutex.h" #include "llvm/ADT/PointerIntPair.h" -#include "TaskPrivate.h" -#include "Debug.h" #if !defined(_WIN32) && !defined(__wasi__) && __has_include() #include diff --git a/stdlib/public/Concurrency/AsyncStream.cpp b/stdlib/public/Concurrency/AsyncStream.cpp index 14b7caf0ad463..389ebd3d7389a 100644 --- a/stdlib/public/Concurrency/AsyncStream.cpp +++ b/stdlib/public/Concurrency/AsyncStream.cpp @@ -10,30 +10,24 @@ // //===----------------------------------------------------------------------===// -#include "swift/Runtime/Mutex.h" +#include + +#include "swift/Threading/Mutex.h" namespace swift { // return the size in words for the given mutex primitive extern "C" size_t _swift_async_stream_lock_size() { - size_t words = sizeof(MutexHandle) / sizeof(void *); + size_t words = sizeof(Mutex) / sizeof(void *); if (words < 1) { return 1; } return words; } -extern "C" -void _swift_async_stream_lock_init(MutexHandle &lock) { - MutexPlatformHelper::init(lock); +extern "C" void _swift_async_stream_lock_init(Mutex &lock) { + new (&lock) Mutex(); } -extern "C" -void _swift_async_stream_lock_lock(MutexHandle &lock) { - MutexPlatformHelper::lock(lock); -} - -extern "C" -void _swift_async_stream_lock_unlock(MutexHandle &lock) { - MutexPlatformHelper::unlock(lock); -} +extern "C" void _swift_async_stream_lock_lock(Mutex &lock) { lock.lock(); } +extern "C" void _swift_async_stream_lock_unlock(Mutex &lock) { lock.unlock(); } } diff --git a/stdlib/public/Concurrency/CMakeLists.txt b/stdlib/public/Concurrency/CMakeLists.txt index 06752c398d077..d41069b01546a 100644 --- a/stdlib/public/Concurrency/CMakeLists.txt +++ b/stdlib/public/Concurrency/CMakeLists.txt @@ -25,6 +25,13 @@ endif() set(SWIFT_RUNTIME_CONCURRENCY_C_FLAGS) set(SWIFT_RUNTIME_CONCURRENCY_SWIFT_FLAGS) +set(swift_concurrency_private_link_libraries) +if(CMAKE_SYSTEM_NAME STREQUAL Windows) + list(APPEND swift_concurrency_private_link_libraries Synchronization) +endif() + +set(swift_concurrency_incorporate_object_libraries_so swiftThreading) + if("${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}" STREQUAL "dispatch") if(NOT CMAKE_SYSTEM_NAME STREQUAL Darwin) include_directories(AFTER @@ -44,7 +51,6 @@ else() message(FATAL_ERROR "Invalid value for SWIFT_CONCURRENCY_GLOBAL_EXECUTOR (\"${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}\").") endif() - if(NOT SWIFT_CONCURRENCY_USES_DISPATCH) endif() @@ -111,8 +117,8 @@ add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} I TaskLocal.swift TaskSleep.swift ThreadSanitizer.cpp + ThreadingError.cpp TracingSignpost.cpp - Mutex.cpp AsyncStreamBuffer.swift AsyncStream.swift AsyncThrowingStream.swift @@ -128,6 +134,9 @@ add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} I SWIFT_MODULE_DEPENDS_HAIKU Glibc SWIFT_MODULE_DEPENDS_WINDOWS CRT + PRIVATE_LINK_LIBRARIES ${swift_concurrency_private_link_libraries} + INCORPORATE_OBJECT_LIBRARIES_SHARED_ONLY + ${swift_concurrency_incorporate_object_libraries_so} LINK_LIBRARIES ${swift_concurrency_link_libraries} C_COMPILE_FLAGS diff --git a/stdlib/public/Concurrency/DispatchGlobalExecutor.inc b/stdlib/public/Concurrency/DispatchGlobalExecutor.inc index d5613f804cd48..511e4b96ba958 100644 --- a/stdlib/public/Concurrency/DispatchGlobalExecutor.inc +++ b/stdlib/public/Concurrency/DispatchGlobalExecutor.inc @@ -116,7 +116,7 @@ extern "C" void dispatch_queue_set_width(dispatch_queue_t dq, long width); static dispatch_queue_t getGlobalQueue(JobPriority priority) { size_t numericPriority = static_cast(priority); if (numericPriority >= globalQueueCacheCount) - swift_Concurrency_fatalError(0, "invalid job priority %#zx"); + swift_Concurrency_fatalError(0, "invalid job priority %#zx", numericPriority); #ifdef SWIFT_CONCURRENCY_BACK_DEPLOYMENT std::memory_order loadOrder = std::memory_order_acquire; diff --git a/stdlib/public/Concurrency/Error.cpp b/stdlib/public/Concurrency/Error.cpp index 2fbe4b0865c0b..fa427552853f0 100644 --- a/stdlib/public/Concurrency/Error.cpp +++ b/stdlib/public/Concurrency/Error.cpp @@ -10,10 +10,27 @@ // //===----------------------------------------------------------------------===// +#include "swift/Threading/Errors.h" +#include + #include "Error.h" // swift::fatalError is not exported from libswiftCore and not shared, so define another // internal function instead. -SWIFT_NORETURN void swift::swift_Concurrency_fatalError(uint32_t flags, const char *format, ...) { +SWIFT_NORETURN +SWIFT_VFORMAT(2) +void swift::swift_Concurrency_fatalErrorv(uint32_t flags, const char *format, + va_list val) { + vfprintf(stderr, format, val); abort(); } + +SWIFT_NORETURN +SWIFT_FORMAT(2, 3) +void swift::swift_Concurrency_fatalError(uint32_t flags, const char *format, + ...) { + va_list val; + + va_start(val, format); + swift_Concurrency_fatalErrorv(flags, format, val); +} diff --git a/stdlib/public/Concurrency/Error.h b/stdlib/public/Concurrency/Error.h index 1e0fd3ffd339c..85e8a621399a1 100644 --- a/stdlib/public/Concurrency/Error.h +++ b/stdlib/public/Concurrency/Error.h @@ -17,13 +17,19 @@ #ifndef SWIFT_CONCURRENCY_ERRORS_H #define SWIFT_CONCURRENCY_ERRORS_H +#include "swift/Basic/Compiler.h" + #include "../SwiftShims/Visibility.h" +#include #include -#include +#include namespace swift { -SWIFT_NORETURN void swift_Concurrency_fatalError(uint32_t flags, const char *format, ...); +SWIFT_NORETURN SWIFT_FORMAT(2, 3) void swift_Concurrency_fatalError( + uint32_t flags, const char *format, ...); +SWIFT_NORETURN SWIFT_VFORMAT(2) void swift_Concurrency_fatalErrorv( + uint32_t flags, const char *format, va_list val); } // namespace swift diff --git a/stdlib/public/Concurrency/Executor.swift b/stdlib/public/Concurrency/Executor.swift index 35862d5bf223f..13e2811b83ef5 100644 --- a/stdlib/public/Concurrency/Executor.swift +++ b/stdlib/public/Concurrency/Executor.swift @@ -93,7 +93,7 @@ func _checkExpectedExecutor(_filenameStart: Builtin.RawPointer, _filenameStart, _filenameLength, _filenameIsASCII, _line, _executor) } -#if !SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#if !SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY // This must take a DispatchQueueShim, not something like AnyObject, // or else SILGen will emit a retain/release in unoptimized builds, // which won't work because DispatchQueues aren't actually diff --git a/stdlib/public/Concurrency/Task.cpp b/stdlib/public/Concurrency/Task.cpp index 6baeaaff94bfa..3aca194ebf3ca 100644 --- a/stdlib/public/Concurrency/Task.cpp +++ b/stdlib/public/Concurrency/Task.cpp @@ -14,19 +14,25 @@ // //===----------------------------------------------------------------------===// +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#endif + #include "../CompatibilityOverride/CompatibilityOverride.h" -#include "swift/Runtime/Concurrency.h" +#include "Debug.h" +#include "Error.h" +#include "TaskGroupPrivate.h" +#include "TaskPrivate.h" +#include "Tracing.h" +#include "swift/ABI/Metadata.h" #include "swift/ABI/Task.h" #include "swift/ABI/TaskLocal.h" #include "swift/ABI/TaskOptions.h" -#include "swift/ABI/Metadata.h" -#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/Concurrency.h" #include "swift/Runtime/HeapObject.h" -#include "TaskGroupPrivate.h" -#include "TaskPrivate.h" -#include "Tracing.h" -#include "Debug.h" -#include "Error.h" +#include "swift/Threading/Mutex.h" #include #include diff --git a/stdlib/public/Concurrency/TaskGroup.cpp b/stdlib/public/Concurrency/TaskGroup.cpp index 60ff5d8670c21..ce4bd34991748 100644 --- a/stdlib/public/Concurrency/TaskGroup.cpp +++ b/stdlib/public/Concurrency/TaskGroup.cpp @@ -16,24 +16,29 @@ #include "../CompatibilityOverride/CompatibilityOverride.h" -#include "swift/ABI/TaskGroup.h" -#include "swift/ABI/Task.h" -#include "swift/ABI/Metadata.h" -#include "swift/ABI/HeapObject.h" -#include "TaskPrivate.h" +#include "Debug.h" #include "TaskGroupPrivate.h" +#include "TaskPrivate.h" +#include "bitset" +#include "queue" // TODO: remove and replace with usage of our mpsc queue +#include "string" +#include "swift/ABI/HeapObject.h" +#include "swift/ABI/Metadata.h" +#include "swift/ABI/Task.h" +#include "swift/ABI/TaskGroup.h" #include "swift/Basic/RelativePointer.h" #include "swift/Basic/STLExtras.h" #include "swift/Runtime/Concurrency.h" #include "swift/Runtime/Config.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/HeapObject.h" -#include "Debug.h" -#include "bitset" -#include "string" -#include "queue" // TODO: remove and replace with usage of our mpsc queue +#include "swift/Threading/Mutex.h" #include #include + +#if !SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY +#include +#endif + #include #if SWIFT_CONCURRENCY_ENABLE_DISPATCH #include @@ -279,8 +284,7 @@ class TaskGroupImpl: public TaskGroupTaskStatusRecord { }; private: - -#if !SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#if !SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY // TODO: move to lockless via the status atomic (make readyQueue an mpsc_queue_t) mutable std::mutex mutex_; diff --git a/stdlib/public/Concurrency/TaskLocal.cpp b/stdlib/public/Concurrency/TaskLocal.cpp index dec28b157a012..8512fc271ed2d 100644 --- a/stdlib/public/Concurrency/TaskLocal.cpp +++ b/stdlib/public/Concurrency/TaskLocal.cpp @@ -10,20 +10,17 @@ // //===----------------------------------------------------------------------===// +#include "swift/ABI/TaskLocal.h" #include "../CompatibilityOverride/CompatibilityOverride.h" +#include "TaskPrivate.h" +#include "swift/ABI/Actor.h" +#include "swift/ABI/Metadata.h" +#include "swift/ABI/Task.h" #include "swift/Runtime/Atomic.h" #include "swift/Runtime/Casting.h" -#include "swift/Runtime/Once.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/Concurrency.h" -#include "swift/Runtime/ThreadLocal.h" -#include "swift/Runtime/ThreadLocalStorage.h" -#include "swift/ABI/TaskLocal.h" -#include "swift/ABI/Task.h" -#include "swift/ABI/Actor.h" -#include "swift/ABI/Metadata.h" +#include "swift/Threading/ThreadLocalStorage.h" #include "llvm/ADT/PointerIntPair.h" -#include "TaskPrivate.h" #include #include @@ -33,14 +30,8 @@ #include #endif -#if HAVE_PTHREAD_H -#include -#endif - #if defined(_WIN32) #include -#include -#include #endif using namespace swift; @@ -59,9 +50,8 @@ template struct Pointer { /// THIS IS RUNTIME INTERNAL AND NOT ABI. class FallbackTaskLocalStorage { - static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, Value, - SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY); + static SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_fallback) Value; public: static void set(TaskLocal::Storage *task) { Value.set(task); } @@ -69,9 +59,9 @@ class FallbackTaskLocalStorage { }; /// Define the thread-locals. -SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( - Pointer, FallbackTaskLocalStorage::Value, - SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY); +SWIFT_THREAD_LOCAL_TYPE(Pointer, + tls_key::concurrency_fallback) +FallbackTaskLocalStorage::Value; // ==== ABI -------------------------------------------------------------------- diff --git a/stdlib/public/Concurrency/TaskPrivate.h b/stdlib/public/Concurrency/TaskPrivate.h index cdda19c5d3056..f9c3906319440 100644 --- a/stdlib/public/Concurrency/TaskPrivate.h +++ b/stdlib/public/Concurrency/TaskPrivate.h @@ -27,22 +27,13 @@ #include "swift/Runtime/Error.h" #include "swift/Runtime/Exclusivity.h" #include "swift/Runtime/HeapObject.h" +#include "swift/Threading/Thread.h" #include #include #define SWIFT_FATAL_ERROR swift_Concurrency_fatalError #include "../runtime/StackAllocator.h" -#if HAVE_PTHREAD_H -#include -#endif -#if defined(_WIN32) -#define WIN32_LEAN_AND_MEAN -#define VC_EXTRA_LEAN -#define NOMINMAX -#include -#endif - namespace swift { // Set to 1 to enable helpful debug spew to stderr @@ -50,24 +41,8 @@ namespace swift { #if 0 #define SWIFT_TASK_DEBUG_LOG(fmt, ...) \ fprintf(stderr, "[%lu] [%s:%d](%s) " fmt "\n", \ - (unsigned long)_swift_get_thread_id(), \ - __FILE__, __LINE__, __FUNCTION__, \ - __VA_ARGS__) - -#if defined(_WIN32) -using ThreadID = decltype(GetCurrentThreadId()); -#else -using ThreadID = decltype(pthread_self()); -#endif - -inline ThreadID _swift_get_thread_id() { -#if defined(_WIN32) - return GetCurrentThreadId(); -#else - return pthread_self(); -#endif -} - + (unsigned long)Thread::current()::platformThreadId(), __FILE__, \ + __LINE__, __FUNCTION__, __VA_ARGS__) #else #define SWIFT_TASK_DEBUG_LOG(fmt, ...) (void)0 #endif @@ -114,7 +89,7 @@ void _swift_tsan_release(void *addr); /// executors. #define DISPATCH_QUEUE_GLOBAL_EXECUTOR (void *)1 -#if !defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +#if !SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY inline SerialExecutorWitnessTable * _swift_task_getDispatchQueueSerialExecutorWitnessTable() { extern SerialExecutorWitnessTable wtable diff --git a/stdlib/public/Concurrency/TaskStatus.cpp b/stdlib/public/Concurrency/TaskStatus.cpp index 7c85aba19a000..99dc553eb1b26 100644 --- a/stdlib/public/Concurrency/TaskStatus.cpp +++ b/stdlib/public/Concurrency/TaskStatus.cpp @@ -15,12 +15,12 @@ // //===----------------------------------------------------------------------===// -#include "../CompatibilityOverride/CompatibilityOverride.h" -#include "swift/Runtime/Concurrency.h" -#include "swift/Runtime/Mutex.h" -#include "swift/Runtime/AtomicWaitQueue.h" #include "swift/ABI/TaskStatus.h" +#include "../CompatibilityOverride/CompatibilityOverride.h" #include "TaskPrivate.h" +#include "swift/Runtime/AtomicWaitQueue.h" +#include "swift/Runtime/Concurrency.h" +#include "swift/Threading/Mutex.h" #include using namespace swift; @@ -36,7 +36,7 @@ ActiveTaskStatus::getStatusRecordParent(TaskStatusRecord *ptr) { /// A lock used to protect management of task-specific status /// record locks. -static StaticMutex StatusRecordLockLock; +static LazyMutex StatusRecordLockLock; namespace { @@ -61,9 +61,9 @@ namespace { /// it sees that the locked bit is set in the `Status` field, it /// must acquire the global status-record lock, find this record /// (which should be the innermost record), and wait for an unlock. -class StatusRecordLockRecord : - public AtomicWaitQueue, - public TaskStatusRecord { +class StatusRecordLockRecord + : public AtomicWaitQueue, + public TaskStatusRecord { public: StatusRecordLockRecord(TaskStatusRecord *parent) : TaskStatusRecord(TaskStatusRecordKind::Private_RecordLock, parent) { @@ -77,7 +77,6 @@ class StatusRecordLockRecord : return record->getKind() == TaskStatusRecordKind::Private_RecordLock; } }; - } /// Wait for a task's status record lock to be unlocked. diff --git a/stdlib/public/Concurrency/ThreadingError.cpp b/stdlib/public/Concurrency/ThreadingError.cpp new file mode 100644 index 0000000000000..512bd1128beca --- /dev/null +++ b/stdlib/public/Concurrency/ThreadingError.cpp @@ -0,0 +1,26 @@ +//===--- ThreadingError.cpp - Error handling support code -----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Threading/Errors.h" +#include + +#include "Error.h" + +// Handle fatal errors from the threading library +SWIFT_ATTRIBUTE_NORETURN +SWIFT_FORMAT(1, 2) +void swift::threading::fatal(const char *format, ...) { + va_list val; + + va_start(val, format); + swift_Concurrency_fatalErrorv(0, format, val); +} diff --git a/stdlib/public/Concurrency/TracingSignpost.cpp b/stdlib/public/Concurrency/TracingSignpost.cpp index 3efaf4d0f7b80..798e9c52eac21 100644 --- a/stdlib/public/Concurrency/TracingSignpost.cpp +++ b/stdlib/public/Concurrency/TracingSignpost.cpp @@ -35,7 +35,7 @@ namespace trace { os_log_t ActorLog; os_log_t TaskLog; -OnceToken_t LogsToken; +swift::once_t LogsToken; void setupLogs(void *unused) { ActorLog = os_log_create(SWIFT_LOG_CONCURRENCY_ACTOR_SUBSYSTEM, diff --git a/stdlib/public/Concurrency/TracingSignpost.h b/stdlib/public/Concurrency/TracingSignpost.h index a4ac6ae86c2f7..ab9d98e52d0db 100644 --- a/stdlib/public/Concurrency/TracingSignpost.h +++ b/stdlib/public/Concurrency/TracingSignpost.h @@ -69,7 +69,7 @@ namespace trace { extern os_log_t ActorLog; extern os_log_t TaskLog; -extern OnceToken_t LogsToken; +extern swift::once_t LogsToken; void setupLogs(void *unused); @@ -80,7 +80,7 @@ void setupLogs(void *unused); do { \ if (!SWIFT_RUNTIME_WEAK_CHECK(os_signpost_enabled)) \ return __VA_ARGS__; \ - SWIFT_ONCE_F(LogsToken, setupLogs, nullptr); \ + swift::once(LogsToken, setupLogs, nullptr); \ } while (0) // Every function does ENSURE_LOGS() before making any os_signpost calls, so diff --git a/stdlib/public/Threading/CMakeLists.txt b/stdlib/public/Threading/CMakeLists.txt new file mode 100644 index 0000000000000..e6f193bd01074 --- /dev/null +++ b/stdlib/public/Threading/CMakeLists.txt @@ -0,0 +1,21 @@ +# This is the counterpart to lib/Threading/CMakeLists.txt. Any updates +# need to be reflected in both places. + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules") +include(AddSwiftStdlib) + +add_swift_target_library(swiftThreading OBJECT_LIBRARY + "${SWIFT_SOURCE_DIR}/lib/Threading/C11.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Linux.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Pthreads.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Win32.cpp" + INSTALL_IN_COMPONENT never_install) + +# This is only used by the compatibility libraries +add_swift_target_library(swiftThreadingWithFatal OBJECT_LIBRARY + "${SWIFT_SOURCE_DIR}/lib/Threading/C11.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Linux.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Pthreads.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Win32.cpp" + "${SWIFT_SOURCE_DIR}/lib/Threading/Errors.cpp" + INSTALL_IN_COMPONENT never_install) diff --git a/stdlib/public/core/CMakeLists.txt b/stdlib/public/core/CMakeLists.txt index f33f2da1362e9..3c457451fdca6 100644 --- a/stdlib/public/core/CMakeLists.txt +++ b/stdlib/public/core/CMakeLists.txt @@ -260,7 +260,7 @@ elseif(SWIFT_PRIMARY_VARIANT_SDK STREQUAL LINUX) list(APPEND swift_core_private_link_libraries) endif() elseif(SWIFT_PRIMARY_VARIANT_SDK STREQUAL WINDOWS) - list(APPEND swift_core_private_link_libraries shell32;DbgHelp) + list(APPEND swift_core_private_link_libraries shell32;DbgHelp;Synchronization) endif() option(SWIFT_CHECK_ESSENTIAL_STDLIB @@ -299,6 +299,7 @@ list(APPEND swift_core_incorporate_object_libraries swiftRuntime) list(APPEND swift_core_incorporate_object_libraries swiftLLVMSupport) list(APPEND swift_core_incorporate_object_libraries swiftDemangling) list(APPEND swift_core_incorporate_object_libraries swiftStdlibStubs) +list(APPEND swift_core_incorporate_object_libraries swiftThreading) if(SWIFT_STDLIB_HAS_COMMANDLINE) list(APPEND swift_core_incorporate_object_libraries swiftCommandLineSupport) endif() diff --git a/stdlib/public/runtime/Bincompat.cpp b/stdlib/public/runtime/Bincompat.cpp index 31c1fef243c2c..40002e1807bf8 100644 --- a/stdlib/public/runtime/Bincompat.cpp +++ b/stdlib/public/runtime/Bincompat.cpp @@ -16,8 +16,9 @@ #include "swift/Runtime/Config.h" #include "swift/Runtime/Bincompat.h" -#include "swift/Runtime/Once.h" +#include "swift/Runtime/Debug.h" #include "swift/Runtime/EnvironmentVariables.h" +#include "swift/Threading/Once.h" #include "../SwiftShims/RuntimeShims.h" #include @@ -96,8 +97,8 @@ static void checkBinCompatEnvironmentVariable(void *context) { extern "C" __swift_bool _swift_stdlib_isExecutableLinkedOnOrAfter( _SwiftStdlibVersion version ) { - static OnceToken_t getenvToken; - SWIFT_ONCE_F(getenvToken, checkBinCompatEnvironmentVariable, nullptr); + static once_t getenvToken; + swift::once(getenvToken, checkBinCompatEnvironmentVariable, nullptr); if (binCompatVersionOverride._value > 0) { return version._value <= binCompatVersionOverride._value; diff --git a/stdlib/public/runtime/CMakeLists.txt b/stdlib/public/runtime/CMakeLists.txt index e87bda0ed6057..1664fb14d1cab 100644 --- a/stdlib/public/runtime/CMakeLists.txt +++ b/stdlib/public/runtime/CMakeLists.txt @@ -33,7 +33,6 @@ set(swift_runtime_sources Bincompat.cpp Casting.cpp CrashReporter.cpp - CygwinPort.cpp Demangle.cpp DynamicCast.cpp Enum.cpp @@ -59,8 +58,6 @@ set(swift_runtime_sources KnownMetadata.cpp Metadata.cpp MetadataLookup.cpp - MutexPThread.cpp - MutexWin32.cpp Numeric.cpp Once.cpp Portability.cpp diff --git a/stdlib/public/runtime/Casting.cpp b/stdlib/public/runtime/Casting.cpp index 864f662655086..4c46d9a653355 100644 --- a/stdlib/public/runtime/Casting.cpp +++ b/stdlib/public/runtime/Casting.cpp @@ -16,14 +16,15 @@ //===----------------------------------------------------------------------===// #include "swift/Runtime/Casting.h" -#include "../SwiftShims/RuntimeShims.h" -#include "../SwiftShims/GlobalObjects.h" #include "../CompatibilityOverride/CompatibilityOverride.h" +#include "../SwiftShims/GlobalObjects.h" +#include "../SwiftShims/RuntimeShims.h" #include "ErrorObject.h" #include "ExistentialMetadataImpl.h" #include "Private.h" #include "SwiftHashableSupport.h" #include "swift/Basic/Lazy.h" +#include "swift/Basic/Unreachable.h" #include "swift/Demangling/Demangler.h" #include "swift/Runtime/Config.h" #include "swift/Runtime/Debug.h" @@ -31,13 +32,7 @@ #include "swift/Runtime/ExistentialContainer.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" -#if defined(__wasi__) -# define SWIFT_CASTING_SUPPORTS_MUTEX 0 -#else -# define SWIFT_CASTING_SUPPORTS_MUTEX 1 -# include "swift/Runtime/Mutex.h" -#endif -#include "swift/Basic/Unreachable.h" +#include "swift/Threading/Mutex.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerIntPair.h" #if SWIFT_OBJC_INTEROP @@ -144,10 +139,8 @@ enum class TypeNameKind { using TypeNameCacheKey = llvm::PointerIntPair; -#if SWIFT_CASTING_SUPPORTS_MUTEX -static StaticReadWriteLock TypeNameCacheLock; -static StaticReadWriteLock MangledToPrettyFunctionNameCacheLock; -#endif +static LazyMutex TypeNameCacheLock; +static LazyMutex MangledToPrettyFunctionNameCacheLock; /// Cache containing rendered names for Metadata. /// Access MUST be protected using `TypeNameCacheLock`. @@ -166,9 +159,7 @@ swift::swift_getTypeName(const Metadata *type, bool qualified) { // Attempt read-only lookup of cache entry. { - #if SWIFT_CASTING_SUPPORTS_MUTEX - StaticScopedReadLock guard(TypeNameCacheLock); - #endif + LazyMutex::ScopedLock guard(TypeNameCacheLock); auto found = cache.find(key); if (found != cache.end()) { @@ -179,9 +170,7 @@ swift::swift_getTypeName(const Metadata *type, bool qualified) { // Read-only lookup failed to find item, we may need to create it. { - #if SWIFT_CASTING_SUPPORTS_MUTEX - StaticScopedWriteLock guard(TypeNameCacheLock); - #endif + LazyMutex::ScopedLock guard(TypeNameCacheLock); // Do lookup again just to make sure it wasn't created by another // thread before we acquired the write lock. @@ -212,9 +201,7 @@ swift::swift_getMangledTypeName(const Metadata *type) { // Attempt read-only lookup of cache entry. { - #if SWIFT_CASTING_SUPPORTS_MUTEX - StaticScopedReadLock guard(TypeNameCacheLock); - #endif + LazyMutex::ScopedLock guard(TypeNameCacheLock); auto found = cache.find(key); if (found != cache.end()) { @@ -225,9 +212,7 @@ swift::swift_getMangledTypeName(const Metadata *type) { // Read-only cache lookup failed, we may need to create it. { - #if SWIFT_CASTING_SUPPORTS_MUTEX - StaticScopedWriteLock guard(TypeNameCacheLock); - #endif + LazyMutex::ScopedLock guard(TypeNameCacheLock); // Do lookup again just to make sure it wasn't created by another // thread before we acquired the write lock. @@ -270,9 +255,7 @@ swift::swift_getFunctionFullNameFromMangledName( auto &cache = MangledToPrettyFunctionNameCache.get(); // Attempt read-only lookup of cache entry. { - #if SWIFT_CASTING_SUPPORTS_MUTEX - StaticScopedReadLock guard(MangledToPrettyFunctionNameCacheLock); - #endif + LazyMutex::ScopedLock guard(MangledToPrettyFunctionNameCacheLock); auto found = cache.find(mangledName); if (found != cache.end()) { @@ -387,9 +370,7 @@ swift::swift_getFunctionFullNameFromMangledName( result[size] = 0; // 0-terminated string { - #if SWIFT_CASTING_SUPPORTS_MUTEX - StaticScopedWriteLock guard(MangledToPrettyFunctionNameCacheLock); - #endif + LazyMutex::ScopedLock guard(MangledToPrettyFunctionNameCacheLock); cache.insert({mangledName, {result, size}}); return TypeNamePair{result, size}; diff --git a/stdlib/public/runtime/CygwinPort.cpp b/stdlib/public/runtime/CygwinPort.cpp deleted file mode 100644 index e3f7ba9c065fa..0000000000000 --- a/stdlib/public/runtime/CygwinPort.cpp +++ /dev/null @@ -1,49 +0,0 @@ -//===--- CygwinPort.cpp - Functions for Cygwin port -----------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Implementations Cygwin specific functions needed for running Swift. -// -//===----------------------------------------------------------------------===// - -#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(_MSC_VER) -#include "Private.h" -#include "swift/Runtime/Debug.h" -#include -#include -#include -#include -#include - -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include - -using namespace swift; - -static std::mutex swiftOnceMutex; - -void swift::_swift_once_f(uintptr_t *predicate, void *context, - void (*function)(void *)) { - // FIXME: This implementation does a global lock, which is much worse than - // what we have on other platforms. Each swift_once should synchronize on the - // token. - swiftOnceMutex.lock(); - if (*predicate == 0) { - *predicate = 1ul; - swiftOnceMutex.unlock(); - - function(context); - } else - swiftOnceMutex.unlock(); -} -#endif // (defined(_WIN32) || defined(__CYGWIN__)) && !defined(_MSC_VER) diff --git a/stdlib/public/runtime/EnvironmentVariables.cpp b/stdlib/public/runtime/EnvironmentVariables.cpp index 9094a7f1a6629..7f1d09025a2f9 100644 --- a/stdlib/public/runtime/EnvironmentVariables.cpp +++ b/stdlib/public/runtime/EnvironmentVariables.cpp @@ -150,7 +150,7 @@ void printHelp(const char *extra) { #include "EnvironmentVariables.def" // Initialization code. -OnceToken_t swift::runtime::environment::initializeToken; +swift::once_t swift::runtime::environment::initializeToken; #if SWIFT_STDLIB_HAS_ENVIRON && (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__linux__)) extern "C" char **environ; diff --git a/stdlib/public/runtime/Errors.cpp b/stdlib/public/runtime/Errors.cpp index 1129f20cfeba3..ba386a5fc8020 100644 --- a/stdlib/public/runtime/Errors.cpp +++ b/stdlib/public/runtime/Errors.cpp @@ -15,6 +15,10 @@ //===----------------------------------------------------------------------===// #if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include + #include #endif @@ -29,10 +33,11 @@ #include #include "ImageInspection.h" +#include "swift/Demangling/Demangle.h" #include "swift/Runtime/Debug.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/Portability.h" -#include "swift/Demangling/Demangle.h" +#include "swift/Threading/Errors.h" +#include "swift/Threading/Mutex.h" #include "llvm/ADT/StringRef.h" #if defined(_MSC_VER) @@ -460,3 +465,13 @@ void swift::swift_abortDisabledUnicodeSupport() { "Unicode normalization data is disabled on this platform"); } + +/// Halt because of a problem in the threading library +SWIFT_ATTRIBUTE_NORETURN +SWIFT_FORMAT(1, 2) +void swift::threading::fatal(const char *msg, ...) { + va_list val; + va_start(val, msg); + + swift::fatalErrorv(0, msg, val); +} diff --git a/stdlib/public/runtime/Exclusivity.cpp b/stdlib/public/runtime/Exclusivity.cpp index 5e0cf63fd114f..44cc94f490bf3 100644 --- a/stdlib/public/runtime/Exclusivity.cpp +++ b/stdlib/public/runtime/Exclusivity.cpp @@ -31,7 +31,7 @@ #include "swift/Runtime/Debug.h" #include "swift/Runtime/EnvironmentVariables.h" #include "swift/Runtime/Metadata.h" -#include "swift/Runtime/ThreadLocalStorage.h" +#include "swift/Threading/ThreadLocalStorage.h" #include #include #include diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index 510ec6e6256f8..42d88e5ca2bf4 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -41,7 +41,6 @@ # include # include # include "swift/Runtime/ObjCBridge.h" -# include "swift/Runtime/Once.h" # include #endif #include "Leaks.h" @@ -162,13 +161,6 @@ struct InitStaticObjectContext { HeapMetadata const *metadata; }; -// Callback for swift_once. -static void initStaticObjectWithContext(void *OpaqueCtx) { - InitStaticObjectContext *Ctx = (InitStaticObjectContext *)OpaqueCtx; - Ctx->object->metadata = Ctx->metadata; - Ctx->object->refCounts.initImmortal(); -} - // TODO: We could generate inline code for the fast-path, i.e. the metadata // pointer is already set. That would be a performance/codesize tradeoff. HeapObject * @@ -182,7 +174,14 @@ swift::swift_initStaticObject(HeapMetadata const *metadata, // refcount to 1 while another thread already incremented it - and would // decrement it to 0 afterwards. InitStaticObjectContext Ctx = { object, metadata }; - swift_once(token, initStaticObjectWithContext, &Ctx); + swift::once( + *token, + [](void *OpaqueCtx) { + InitStaticObjectContext *Ctx = (InitStaticObjectContext *)OpaqueCtx; + Ctx->object->metadata = Ctx->metadata; + Ctx->object->refCounts.initImmortal(); + }, + &Ctx); return object; } @@ -350,7 +349,7 @@ static HeapObject *_swift_retain_(HeapObject *object) { } HeapObject *swift::swift_retain(HeapObject *object) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE return swift_nonatomic_retain(object); #else CALL_IMPL(swift_retain, (object)); @@ -377,7 +376,7 @@ static HeapObject *_swift_retain_n_(HeapObject *object, uint32_t n) { } HeapObject *swift::swift_retain_n(HeapObject *object, uint32_t n) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE return swift_nonatomic_retain_n(object, n); #else CALL_IMPL(swift_retain_n, (object, n)); @@ -403,7 +402,7 @@ static void _swift_release_(HeapObject *object) { } void swift::swift_release(HeapObject *object) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE swift_nonatomic_release(object); #else CALL_IMPL(swift_release, (object)); @@ -428,7 +427,7 @@ static void _swift_release_n_(HeapObject *object, uint32_t n) { } void swift::swift_release_n(HeapObject *object, uint32_t n) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE swift_nonatomic_release_n(object, n); #else CALL_IMPL(swift_release_n, (object, n)); @@ -460,7 +459,7 @@ size_t swift::swift_weakRetainCount(HeapObject *object) { } HeapObject *swift::swift_unownedRetain(HeapObject *object) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE return static_cast(swift_nonatomic_unownedRetain(object)); #else SWIFT_RT_TRACK_INVOCATION(object, swift_unownedRetain); @@ -473,7 +472,7 @@ HeapObject *swift::swift_unownedRetain(HeapObject *object) { } void swift::swift_unownedRelease(HeapObject *object) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE swift_nonatomic_unownedRelease(object); #else SWIFT_RT_TRACK_INVOCATION(object, swift_unownedRelease); @@ -520,7 +519,7 @@ void swift::swift_nonatomic_unownedRelease(HeapObject *object) { } HeapObject *swift::swift_unownedRetain_n(HeapObject *object, int n) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE return swift_nonatomic_unownedRetain_n(object, n); #else SWIFT_RT_TRACK_INVOCATION(object, swift_unownedRetain_n); @@ -533,7 +532,7 @@ HeapObject *swift::swift_unownedRetain_n(HeapObject *object, int n) { } void swift::swift_unownedRelease_n(HeapObject *object, int n) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE swift_nonatomic_unownedRelease_n(object, n); #else SWIFT_RT_TRACK_INVOCATION(object, swift_unownedRelease_n); @@ -583,7 +582,7 @@ static HeapObject *_swift_tryRetain_(HeapObject *object) { if (!isValidPointerForNativeRetain(object)) return nullptr; -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE if (object->refCounts.tryIncrementNonAtomic()) return object; else return nullptr; #else @@ -612,7 +611,7 @@ void swift::swift_setDeallocating(HeapObject *object) { } HeapObject *swift::swift_unownedRetainStrong(HeapObject *object) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE return swift_nonatomic_unownedRetainStrong(object); #else SWIFT_RT_TRACK_INVOCATION(object, swift_unownedRetainStrong); @@ -640,7 +639,7 @@ HeapObject *swift::swift_nonatomic_unownedRetainStrong(HeapObject *object) { } void swift::swift_unownedRetainStrongAndRelease(HeapObject *object) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE swift_nonatomic_unownedRetainStrongAndRelease(object); #else SWIFT_RT_TRACK_INVOCATION(object, swift_unownedRetainStrongAndRelease); diff --git a/stdlib/public/runtime/ImageInspectionCOFF.cpp b/stdlib/public/runtime/ImageInspectionCOFF.cpp index f952dc8b3dd22..53bf6c70e8e4a 100644 --- a/stdlib/public/runtime/ImageInspectionCOFF.cpp +++ b/stdlib/public/runtime/ImageInspectionCOFF.cpp @@ -23,7 +23,7 @@ #include #endif -#include "swift/Runtime/Mutex.h" +#include "swift/Threading/Mutex.h" using namespace swift; @@ -73,7 +73,7 @@ int swift::lookupSymbol(const void *address, SymbolInfo *info) { } #if defined(_WIN32) -static StaticMutex mutex; +static LazyMutex mutex; static bool isDbgHelpInitialized = false; void swift::_swift_withWin32DbgHelpLibrary( diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index c1a7a04350b4e..c9215ca634681 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -14,21 +14,28 @@ // //===----------------------------------------------------------------------===// -#include "swift/Runtime/Metadata.h" +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +// Avoid defining macro max(), min() which conflict with std::max(), std::min() +#define NOMINMAX +#include +#endif + #include "MetadataCache.h" +#include "swift/ABI/TypeIdentity.h" #include "swift/Basic/Lazy.h" #include "swift/Basic/Range.h" #include "swift/Basic/STLExtras.h" #include "swift/Demangling/Demangler.h" -#include "swift/ABI/TypeIdentity.h" #include "swift/Runtime/Casting.h" #include "swift/Runtime/EnvironmentVariables.h" #include "swift/Runtime/ExistentialContainer.h" #include "swift/Runtime/HeapObject.h" -#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/Metadata.h" #include "swift/Runtime/Once.h" #include "swift/Runtime/Portability.h" #include "swift/Strings.h" +#include "swift/Threading/Mutex.h" #include "llvm/ADT/StringExtras.h" #include #include @@ -37,12 +44,6 @@ #include #include #include -#if defined(_WIN32) -#define WIN32_LEAN_AND_MEAN -// Avoid defining macro max(), min() which conflict with std::max(), std::min() -#define NOMINMAX -#include -#endif #if SWIFT_PTRAUTH #include #endif @@ -786,8 +787,8 @@ _cacheCanonicalSpecializedMetadata(const TypeContextDescriptor *description) { static void cacheCanonicalSpecializedMetadata(const TypeContextDescriptor *description, swift_once_t *token) { - swift_once( - token, + swift::once( + *token, [](void *uncastDescription) { auto *description = (const TypeContextDescriptor *)uncastDescription; _cacheCanonicalSpecializedMetadata(description); @@ -3148,11 +3149,14 @@ _swift_initClassMetadataImpl(ClassMetadata *self, self->Superclass = getRootSuperclass(); // Register our custom implementation of class_getImageName. - static swift_once_t onceToken; - swift_once(&onceToken, [](void *unused) { - (void)unused; - setUpObjCRuntimeGetImageNameFromClass(); - }, nullptr); + static swift::once_t onceToken; + swift::once( + onceToken, + [](void *unused) { + (void)unused; + setUpObjCRuntimeGetImageNameFromClass(); + }, + nullptr); #endif // Copy field offsets, generic arguments and (if necessary) vtable entries @@ -6282,8 +6286,8 @@ void *MetadataAllocator::Allocate(size_t size, size_t alignment) { assert(alignment <= alignof(void*)); assert(size % alignof(void*) == 0); - static OnceToken_t getenvToken; - SWIFT_ONCE_F(getenvToken, checkAllocatorDebugEnvironmentVariables, nullptr); + static swift::once_t getenvToken; + swift::once(getenvToken, checkAllocatorDebugEnvironmentVariables); // If the size is larger than the maximum, just do a normal heap allocation. if (size > PoolRange::MaxPoolAllocationSize) { diff --git a/stdlib/public/runtime/MetadataCache.h b/stdlib/public/runtime/MetadataCache.h index e854ba3f669ef..67fc9583b746b 100644 --- a/stdlib/public/runtime/MetadataCache.h +++ b/stdlib/public/runtime/MetadataCache.h @@ -12,13 +12,16 @@ #ifndef SWIFT_RUNTIME_METADATACACHE_H #define SWIFT_RUNTIME_METADATACACHE_H -#include "llvm/ADT/Hashing.h" -#include "llvm/ADT/STLExtras.h" +#include "swift/Runtime/AtomicWaitQueue.h" #include "swift/Runtime/Concurrent.h" #include "swift/Runtime/Metadata.h" -#include "swift/Runtime/Mutex.h" -#include "swift/Runtime/AtomicWaitQueue.h" +#include "swift/Threading/Mutex.h" + #include "../SwiftShims/Visibility.h" + +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/STLExtras.h" + #include #include @@ -169,8 +172,7 @@ class LockingConcurrentMapStorage { // TargetGenericMetadataInstantiationCache::PrivateData. On 32-bit archs, that // space is not large enough to accommodate a Mutex along with everything // else. There, use a SmallMutex to squeeze into the available space. - using MutexTy = - std::conditional_t; + using MutexTy = std::conditional_t; StableAddressConcurrentReadableHashMap, MutexTy> Map; diff --git a/stdlib/public/runtime/MetadataImpl.h b/stdlib/public/runtime/MetadataImpl.h index 211d2b93512e8..1365e24d6f783 100644 --- a/stdlib/public/runtime/MetadataImpl.h +++ b/stdlib/public/runtime/MetadataImpl.h @@ -137,7 +137,7 @@ template struct RetainableBoxBase { static constexpr size_t stride = sizeof(T); static constexpr bool isPOD = false; static constexpr bool isBitwiseTakable = true; -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE static constexpr bool isAtomic = false; #else static constexpr bool isAtomic = true; diff --git a/stdlib/public/runtime/MetadataLookup.cpp b/stdlib/public/runtime/MetadataLookup.cpp index aab427dc0bc58..2bf5ea412a219 100644 --- a/stdlib/public/runtime/MetadataLookup.cpp +++ b/stdlib/public/runtime/MetadataLookup.cpp @@ -14,32 +14,32 @@ // //===----------------------------------------------------------------------===// +#include "../CompatibilityOverride/CompatibilityOverride.h" +#include "ImageInspection.h" +#include "Private.h" +#include "swift/ABI/TypeIdentity.h" #include "swift/Basic/Lazy.h" #include "swift/Demangling/Demangler.h" #include "swift/Demangling/TypeDecoder.h" #include "swift/Reflection/Records.h" -#include "swift/ABI/TypeIdentity.h" #include "swift/Runtime/Casting.h" #include "swift/Runtime/Concurrent.h" #include "swift/Runtime/Debug.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" -#include "swift/Runtime/Mutex.h" #include "swift/Strings.h" +#include "swift/Threading/Mutex.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/StringExtras.h" -#include "Private.h" -#include "../CompatibilityOverride/CompatibilityOverride.h" -#include "ImageInspection.h" +#include #include -#include #include #include -#include +#include using namespace swift; using namespace Demangle; diff --git a/stdlib/public/runtime/MutexPThread.cpp b/stdlib/public/runtime/MutexPThread.cpp deleted file mode 100644 index 6c27207092224..0000000000000 --- a/stdlib/public/runtime/MutexPThread.cpp +++ /dev/null @@ -1,142 +0,0 @@ -//===--- MutexPThread.cpp - Supports Mutex.h using PThreads ---------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Mutex and Read/Write lock implementations using pthreads. -// -// Darwin shares the pthreads implementation for read/write locks, but -// uses inline implementations for os_unfair_lock. -// -//===----------------------------------------------------------------------===// - -#if __has_include() -#include -#endif - -#if defined(_POSIX_THREADS) && !defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) - -// Notes: swift::fatalError is not shared between libswiftCore and libswift_Concurrency -// and libswift_Concurrency uses swift_Concurrency_fatalError instead. -#ifndef SWIFT_FATAL_ERROR -#define SWIFT_FATAL_ERROR swift::fatalError -#endif - -#include "swift/Runtime/Mutex.h" - -#include "swift/Runtime/Debug.h" -#include -#include - -using namespace swift; - -#define reportError(PThreadFunction) \ - do { \ - int errorcode = PThreadFunction; \ - if (errorcode != 0) { \ - SWIFT_FATAL_ERROR(/* flags = */ 0, "'%s' failed with error '%s'(%d)\n", \ - #PThreadFunction, errorName(errorcode), errorcode); \ - } \ - } while (false) - -#define returnTrueOrReportError(PThreadFunction, returnFalseOnEBUSY) \ - do { \ - int errorcode = PThreadFunction; \ - if (errorcode == 0) \ - return true; \ - if (returnFalseOnEBUSY && errorcode == EBUSY) \ - return false; \ - SWIFT_FATAL_ERROR(/* flags = */ 0, "'%s' failed with error '%s'(%d)\n", \ - #PThreadFunction, errorName(errorcode), errorcode); \ - } while (false) - -static const char *errorName(int errorcode) { - switch (errorcode) { - case EINVAL: - return "EINVAL"; - case EPERM: - return "EPERM"; - case EDEADLK: - return "EDEADLK"; - case ENOMEM: - return "ENOMEM"; - case EAGAIN: - return "EAGAIN"; - case EBUSY: - return "EBUSY"; - default: - return ""; - } -} - -#if !HAS_OS_UNFAIR_LOCK - -void MutexPlatformHelper::init(pthread_mutex_t &mutex, bool checked) { - pthread_mutexattr_t attr; - int kind = (checked ? PTHREAD_MUTEX_ERRORCHECK : PTHREAD_MUTEX_NORMAL); - reportError(pthread_mutexattr_init(&attr)); - reportError(pthread_mutexattr_settype(&attr, kind)); - reportError(pthread_mutex_init(&mutex, &attr)); - reportError(pthread_mutexattr_destroy(&attr)); -} - -void MutexPlatformHelper::destroy(pthread_mutex_t &mutex) { - reportError(pthread_mutex_destroy(&mutex)); -} - -void MutexPlatformHelper::lock(pthread_mutex_t &mutex) { - reportError(pthread_mutex_lock(&mutex)); -} - -void MutexPlatformHelper::unlock(pthread_mutex_t &mutex) { - reportError(pthread_mutex_unlock(&mutex)); -} - -bool MutexPlatformHelper::try_lock(pthread_mutex_t &mutex) { - returnTrueOrReportError(pthread_mutex_trylock(&mutex), - /* returnFalseOnEBUSY = */ true); -} - -#endif - -void ReadWriteLockPlatformHelper::init(pthread_rwlock_t &rwlock) { - reportError(pthread_rwlock_init(&rwlock, nullptr)); -} - -void ReadWriteLockPlatformHelper::destroy(pthread_rwlock_t &rwlock) { - reportError(pthread_rwlock_destroy(&rwlock)); -} - -void ReadWriteLockPlatformHelper::readLock(pthread_rwlock_t &rwlock) { - reportError(pthread_rwlock_rdlock(&rwlock)); -} - -bool ReadWriteLockPlatformHelper::try_readLock(pthread_rwlock_t &rwlock) { - returnTrueOrReportError(pthread_rwlock_tryrdlock(&rwlock), - /* returnFalseOnEBUSY = */ true); -} - -void ReadWriteLockPlatformHelper::writeLock(pthread_rwlock_t &rwlock) { - reportError(pthread_rwlock_wrlock(&rwlock)); -} - -bool ReadWriteLockPlatformHelper::try_writeLock(pthread_rwlock_t &rwlock) { - returnTrueOrReportError(pthread_rwlock_trywrlock(&rwlock), - /* returnFalseOnEBUSY = */ true); -} - -void ReadWriteLockPlatformHelper::readUnlock(pthread_rwlock_t &rwlock) { - reportError(pthread_rwlock_unlock(&rwlock)); -} - -void ReadWriteLockPlatformHelper::writeUnlock(pthread_rwlock_t &rwlock) { - reportError(pthread_rwlock_unlock(&rwlock)); -} -#endif diff --git a/stdlib/public/runtime/MutexWin32.cpp b/stdlib/public/runtime/MutexWin32.cpp deleted file mode 100644 index 1ce30fd9d6f08..0000000000000 --- a/stdlib/public/runtime/MutexWin32.cpp +++ /dev/null @@ -1,31 +0,0 @@ -//===--- MutexWin32.cpp - -------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Mutex, Read/Write lock, and Scoped lock implementations -// using Windows Slim Reader/Writer Locks. -// -//===----------------------------------------------------------------------===// - -#if defined(_WIN32) - -// Notes: swift::fatalError is not shared between libswiftCore and libswift_Concurrency -// and libswift_Concurrency uses swift_Concurrency_fatalError instead. -#ifndef SWIFT_FATAL_ERROR -#define SWIFT_FATAL_ERROR swift::fatalError -#endif - -#include "swift/Runtime/Mutex.h" -#include "swift/Runtime/Debug.h" - -using namespace swift; - -#endif diff --git a/stdlib/public/runtime/Once.cpp b/stdlib/public/runtime/Once.cpp index 1460b0603ef17..7286b53b03cb7 100644 --- a/stdlib/public/runtime/Once.cpp +++ b/stdlib/public/runtime/Once.cpp @@ -14,18 +14,14 @@ // //===----------------------------------------------------------------------===// +#include "swift/Threading/Once.h" #include "Private.h" -#include "swift/Runtime/Once.h" #include "swift/Runtime/Debug.h" #include using namespace swift; -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - -// No dependencies on single-threaded environments. - -#elif defined(__APPLE__) +#if SWIFT_THREADING_DARWIN // On macOS and iOS, swift_once is implemented using GCD. // The compiler emits an inline check matching the barrier-free inline fast @@ -34,12 +30,8 @@ using namespace swift; #include static_assert(std::is_same::value, "swift_once_t and dispatch_once_t must stay in sync"); -#else - -// On non-Darwin platforms we do not assume any barrier-free inline path -// and SwiftTargetInfo.OnceDonePredicateValue is unset in the compiler. -#endif +#endif // SWIFT_THREADING_DARWIN // The compiler generates the swift_once_t values as word-sized zero-initialized // variables, so we want to make sure swift_once_t isn't larger than the @@ -52,16 +44,5 @@ static_assert(sizeof(swift_once_t) <= sizeof(void*), /// extent of type swift_once_t. void swift::swift_once(swift_once_t *predicate, void (*fn)(void *), void *context) { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - if (! *predicate) { - *predicate = true; - fn(context); - } -#elif defined(__APPLE__) - dispatch_once_f(predicate, context, fn); -#elif defined(__CYGWIN__) - _swift_once_f(predicate, context, fn); -#else - std::call_once(*predicate, [fn, context]() { fn(context); }); -#endif + swift::once(*predicate, fn, context); } diff --git a/stdlib/public/runtime/RuntimeInvocationsTracking.cpp b/stdlib/public/runtime/RuntimeInvocationsTracking.cpp index 8971902e6146d..bf5812b4ba100 100644 --- a/stdlib/public/runtime/RuntimeInvocationsTracking.cpp +++ b/stdlib/public/runtime/RuntimeInvocationsTracking.cpp @@ -15,11 +15,13 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/DenseMap.h" +#include + #include "RuntimeInvocationsTracking.h" #include "swift/Basic/Lazy.h" #include "swift/Runtime/HeapObject.h" -#include "swift/Runtime/Mutex.h" +#include "swift/Threading/Mutex.h" +#include "llvm/ADT/DenseMap.h" #if defined(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS) @@ -32,7 +34,7 @@ namespace swift { // functions. struct RuntimeFunctionCountersState { #define FUNCTION_TO_TRACK(RT_FUNCTION) \ - uint32_t SWIFT_RT_FUNCTION_INVOCATION_COUNTER_NAME(RT_FUNCTION) = 0; + std::uint32_t SWIFT_RT_FUNCTION_INVOCATION_COUNTER_NAME(RT_FUNCTION) = 0; // Provide one counter per runtime function being tracked. #include "RuntimeInvocationsTracking.def" }; @@ -49,7 +51,7 @@ static bool UpdateGlobalRuntimeFunctionCounters = false; /// Global set of counters tracking the total number of runtime invocations. struct RuntimeFunctionCountersStateSentinel { RuntimeFunctionCountersState State; - StaticReadWriteLock Lock; + LazyMutex Lock; }; static RuntimeFunctionCountersStateSentinel RuntimeGlobalFunctionCountersState; @@ -57,7 +59,7 @@ static RuntimeFunctionCountersStateSentinel RuntimeGlobalFunctionCountersState; /// them. struct RuntimeObjectCacheSentinel { llvm::DenseMap Cache; - StaticReadWriteLock Lock; + Mutex Lock; }; static Lazy RuntimeObjectStateCache; @@ -73,7 +75,7 @@ static const char *RuntimeFunctionNames[] { /// Define an enum where each enumerator corresponds to a runtime function being /// tracked. Their order is the same as the order of the counters in the /// RuntimeObjectState structure. -enum RuntimeFunctionNamesIDs : uint32_t { +enum RuntimeFunctionNamesIDs : std::uint32_t { /// Defines names of enum cases for each function being tracked. #define FUNCTION_TO_TRACK(RT_FUNCTION) RT_FUNCTION_ID(RT_FUNCTION), #include "RuntimeInvocationsTracking.def" @@ -87,10 +89,10 @@ static RuntimeFunctionCountersUpdateHandler /// The offsets of the runtime function counters being tracked inside the /// RuntimeObjectState structure. The array is indexed by /// the enumerators from RuntimeFunctionNamesIDs. -static uint16_t RuntimeFunctionCountersOffsets[] = { +static std::uint16_t RuntimeFunctionCountersOffsets[] = { /// Define offset for each function being tracked. #define FUNCTION_TO_TRACK(RT_FUNCTION) \ - (sizeof(uint16_t) * (unsigned)RT_FUNCTION_ID(RT_FUNCTION)), + (sizeof(std::uint16_t) * (unsigned)RT_FUNCTION_ID(RT_FUNCTION)), #include "RuntimeInvocationsTracking.def" }; @@ -101,7 +103,7 @@ static uint16_t RuntimeFunctionCountersOffsets[] = { void SWIFT_RT_TRACK_INVOCATION_NAME(RT_FUNCTION)(HeapObject * object) { \ /* Update global counters. */ \ if (UpdateGlobalRuntimeFunctionCounters) { \ - StaticScopedWriteLock lock(RuntimeGlobalFunctionCountersState.Lock); \ + LazyMutex::ScopedLock lock(RuntimeGlobalFunctionCountersState.Lock); \ RuntimeGlobalFunctionCountersState.State \ .SWIFT_RT_FUNCTION_INVOCATION_COUNTER_NAME(RT_FUNCTION)++; \ if (GlobalRuntimeFunctionCountersUpdateHandler) { \ @@ -117,7 +119,7 @@ static uint16_t RuntimeFunctionCountersOffsets[] = { /* Update per object counters. */ \ if (UpdatePerObjectRuntimeFunctionCounters && object) { \ auto &theSentinel = RuntimeObjectStateCache.get(); \ - StaticScopedWriteLock lock(theSentinel.Lock); \ + Mutex::ScopedLock lock(theSentinel.Lock); \ theSentinel.Cache[object].SWIFT_RT_FUNCTION_INVOCATION_COUNTER_NAME( \ RT_FUNCTION)++; \ /* TODO: Remember the order/history of operations? */ \ @@ -131,7 +133,7 @@ static uint16_t RuntimeFunctionCountersOffsets[] = { void _swift_getObjectRuntimeFunctionCounters( HeapObject *object, RuntimeFunctionCountersState *result) { auto &theSentinel = RuntimeObjectStateCache.get(); - StaticScopedReadLock lock(theSentinel.Lock); + Mutex::ScopedLock lock(theSentinel.Lock); *result = theSentinel.Cache[object]; } @@ -140,7 +142,7 @@ void _swift_getObjectRuntimeFunctionCounters( void _swift_setObjectRuntimeFunctionCounters( HeapObject *object, RuntimeFunctionCountersState *state) { auto &theSentinel = RuntimeObjectStateCache.get(); - StaticScopedWriteLock lock(theSentinel.Lock); + Mutex::ScopedLock lock(theSentinel.Lock); theSentinel.Cache[object] = *state; } @@ -148,14 +150,14 @@ void _swift_setObjectRuntimeFunctionCounters( /// each runtime function of interest. void _swift_getGlobalRuntimeFunctionCounters( RuntimeFunctionCountersState *result) { - StaticScopedReadLock lock(RuntimeGlobalFunctionCountersState.Lock); + LazyMutex::ScopedLock lock(RuntimeGlobalFunctionCountersState.Lock); *result = RuntimeGlobalFunctionCountersState.State; } /// Set the global runtime state of function pointers from a provided state. void _swift_setGlobalRuntimeFunctionCounters( RuntimeFunctionCountersState *state) { - StaticScopedWriteLock lock(RuntimeGlobalFunctionCountersState.Lock); + LazyMutex::ScopedLock lock(RuntimeGlobalFunctionCountersState.Lock); RuntimeGlobalFunctionCountersState.State = *state; } @@ -169,17 +171,17 @@ const char **_swift_getRuntimeFunctionNames() { /// Return the offsets of the runtime function counters being tracked. /// Their order is the same as the order of the counters in the /// RuntimeObjectState structure. -const uint16_t *_swift_getRuntimeFunctionCountersOffsets() { +const std::uint16_t *_swift_getRuntimeFunctionCountersOffsets() { return RuntimeFunctionCountersOffsets; } /// Return the number of runtime functions being tracked. -uint64_t _swift_getNumRuntimeFunctionCounters() { +std::uint64_t _swift_getNumRuntimeFunctionCounters() { return ID_LastRuntimeFunctionName; } static void _swift_dumpRuntimeCounters(RuntimeFunctionCountersState *State) { - uint32_t tmp; + std::uint32_t tmp; /// Define how to dump the counter for a given runtime function. #define FUNCTION_TO_TRACK(RT_FUNCTION) \ tmp = State->SWIFT_RT_FUNCTION_INVOCATION_COUNTER_NAME(RT_FUNCTION); \ @@ -192,7 +194,7 @@ static void _swift_dumpRuntimeCounters(RuntimeFunctionCountersState *State) { /// Dump all per-object runtime function pointers. void _swift_dumpObjectsRuntimeFunctionPointers() { auto &theSentinel = RuntimeObjectStateCache.get(); - StaticScopedReadLock lock(theSentinel.Lock); + Mutex::ScopedLock lock(theSentinel.Lock); for (auto &Pair : theSentinel.Cache) { printf("\n\nRuntime counters for object at address %p:\n", Pair.getFirst()); _swift_dumpRuntimeCounters(&Pair.getSecond()); diff --git a/stdlib/public/runtime/SwiftTLSContext.cpp b/stdlib/public/runtime/SwiftTLSContext.cpp index 59657e89df1a5..8a627566cb58d 100644 --- a/stdlib/public/runtime/SwiftTLSContext.cpp +++ b/stdlib/public/runtime/SwiftTLSContext.cpp @@ -11,81 +11,57 @@ //===----------------------------------------------------------------------===// #include "SwiftTLSContext.h" -#include "swift/Basic/Lazy.h" -#include "swift/Runtime/Once.h" -#include "swift/Runtime/ThreadLocalStorage.h" + +#include "swift/Threading/Once.h" +#include "swift/Threading/ThreadLocalStorage.h" using namespace swift; using namespace swift::runtime; -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - SwiftTLSContext &SwiftTLSContext::get() { - static SwiftTLSContext TLSContext; - return TLSContext; -} -#elif SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC -// Use the reserved TSD key if possible. +#if SWIFT_THREADING_USE_RESERVED_TLS_KEYS -SwiftTLSContext &SwiftTLSContext::get() { - SwiftTLSContext *ctx = static_cast( - SWIFT_THREAD_GETSPECIFIC(SWIFT_RUNTIME_TLS_KEY)); + // If we have reserved keys, use those + SwiftTLSContext *ctx = + static_cast(swift::tls_get(swift::tls_key::runtime)); if (ctx) return *ctx; - static OnceToken_t setupToken; - SWIFT_ONCE_F( - setupToken, - [](void *) { - pthread_key_init_np(SWIFT_RUNTIME_TLS_KEY, [](void *pointer) { - delete static_cast(pointer); - }); - }, - nullptr); + static swift::once_t token; + swift::tls_init_once(token, swift::tls_key::runtime, [](void *pointer) { + delete static_cast(pointer); + }); ctx = new SwiftTLSContext(); - SWIFT_THREAD_SETSPECIFIC(SWIFT_RUNTIME_TLS_KEY, ctx); + swift::tls_set(swift::tls_key::runtime, ctx); return *ctx; -} - -#elif __has_feature(cxx_thread_local) -// Second choice is direct language support for thread-locals. - -namespace { - -static thread_local SwiftTLSContext TLSContext; -} // anonymous namespace +#elif defined(SWIFT_THREAD_LOCAL) -SwiftTLSContext &SwiftTLSContext::get() { return TLSContext; } + // If we have the thread local attribute, use that + // (note that this happens for the no-threads case too) + static SWIFT_THREAD_LOCAL SwiftTLSContext TLSContext; + return TLSContext; #else -// Use the platform thread-local data API. -static __swift_thread_key_t createSwiftThreadKey() { - __swift_thread_key_t key; - int result = SWIFT_THREAD_KEY_CREATE(&key, [](void *pointer) { + // Otherwise, allocate ourselves a key and use that + static swift::tls_key_t runtimeKey; + static swift::once_t token; + + swift::tls_alloc_once(token, runtimeKey, [](void *pointer) { delete static_cast(pointer); }); - if (result != 0) { - fatalError(0, "couldn't create thread key for exclusivity: %s\n", - strerror(result)); - } - return key; -} - -static SwiftTLSContext &getTLSContext() { - static __swift_thread_key_t key = createSwiftThreadKey(); - SwiftTLSContext *ctx = - static_cast(SWIFT_THREAD_GETSPECIFIC(key)); - if (!ctx) { - ctx = new SwiftTLSContext(); - SWIFT_THREAD_SETSPECIFIC(key, ctx); - } + static_cast(swift::tls_get(runtimeKey)); + if (ctx) + return *ctx; + + ctx = new SwiftTLSContext(); + swift::tls_set(runtimeKey, ctx); return *ctx; -} #endif +} diff --git a/stdlib/public/stubs/OptionalBridgingHelper.mm b/stdlib/public/stubs/OptionalBridgingHelper.mm index 4d44fad1e8cf7..ae4e400d50372 100644 --- a/stdlib/public/stubs/OptionalBridgingHelper.mm +++ b/stdlib/public/stubs/OptionalBridgingHelper.mm @@ -15,12 +15,12 @@ #if SWIFT_OBJC_INTEROP #include "swift/Basic/Lazy.h" #include "swift/Runtime/Metadata.h" -#include "swift/Runtime/Mutex.h" #include "swift/Runtime/ObjCBridge.h" #include "swift/Runtime/Portability.h" -#include -#import +#include "swift/Threading/Mutex.h" #import +#import +#include using namespace swift; @@ -59,7 +59,7 @@ - (id)description { struct SwiftNullSentinelCache { std::vector Cache; - StaticReadWriteLock Lock; + Mutex Lock; }; static Lazy Sentinels; @@ -73,7 +73,7 @@ static id getSentinelForDepth(unsigned depth) { auto &theSentinels = Sentinels.get(); unsigned depthIndex = depth - 2; { - StaticScopedReadLock lock(theSentinels.Lock); + Mutex::ScopedLock lock(theSentinels.Lock); const auto &cache = theSentinels.Cache; if (depthIndex < cache.size()) { id cached = cache[depthIndex]; @@ -83,7 +83,7 @@ static id getSentinelForDepth(unsigned depth) { } // Make one if we need to. { - StaticScopedWriteLock lock(theSentinels.Lock); + Mutex::ScopedLock lock(theSentinels.Lock); if (depthIndex >= theSentinels.Cache.size()) theSentinels.Cache.resize(depthIndex + 1); auto &cached = theSentinels.Cache[depthIndex]; diff --git a/stdlib/public/stubs/Random.cpp b/stdlib/public/stubs/Random.cpp index dab27c0d8b430..4a44682761656 100644 --- a/stdlib/public/stubs/Random.cpp +++ b/stdlib/public/stubs/Random.cpp @@ -41,9 +41,9 @@ #include -#include "swift/Runtime/Debug.h" -#include "swift/Runtime/Mutex.h" #include "SwiftShims/Random.h" +#include "swift/Runtime/Debug.h" +#include "swift/Threading/Mutex.h" #include // required for std::min @@ -108,7 +108,8 @@ void swift_stdlib_random(void *buf, __swift_size_t nbytes) { WHILE_EINTR(open("/dev/urandom", O_RDONLY | O_CLOEXEC, 0)); if (fd != -1) { - static StaticMutex mutex; + // ###FIXME: Why is this locked? None of the others are. + static LazyMutex mutex; mutex.withLock([&] { actual_nbytes = WHILE_EINTR(read(fd, buf, nbytes)); }); diff --git a/stdlib/public/stubs/Stubs.cpp b/stdlib/public/stubs/Stubs.cpp index 0cc0efa2576dd..fe734ea08cdfb 100644 --- a/stdlib/public/stubs/Stubs.cpp +++ b/stdlib/public/stubs/Stubs.cpp @@ -64,14 +64,12 @@ #include #endif -#if defined(__FreeBSD__) || defined(__OpenBSD__) -#include -#endif - #include "swift/Runtime/Debug.h" #include "swift/Runtime/SwiftDtoa.h" #include "swift/Basic/Lazy.h" +#include "swift/Threading/Thread.h" + #include "SwiftShims/LibcShims.h" #include "SwiftShims/RuntimeShims.h" #include "SwiftShims/RuntimeStubs.h" @@ -484,7 +482,7 @@ int _swift_stdlib_putc_stderr(int C) { } size_t _swift_stdlib_getHardwareConcurrency() { -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifdef SWIFT_THREADING_NONE return 1; #else return std::thread::hardware_concurrency(); @@ -537,66 +535,11 @@ __swift_bool swift_stdlib_isStackAllocationSafe(__swift_size_t byteCount, __swift_bool _swift_stdlib_getCurrentStackBounds(__swift_uintptr_t *outBegin, __swift_uintptr_t *outEnd) { -#if defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) - // This platform does not support threads, so the API we'd call to get stack - // bounds (i.e. libpthread) is not going to be usable. - return false; - -#elif defined(__APPLE__) - pthread_t thread = pthread_self(); - // On Apple platforms, the stack grows down, so that the end of the stack - // comes before the beginning on the number line, and an address on the stack - // will be LESS than the start of the stack and GREATER than the end. - void *end = pthread_get_stackaddr_np(thread); - if (!end) { - return false; - } - *outEnd = (uintptr_t)end; - *outBegin = *outEnd - pthread_get_stacksize_np(thread); - return true; - -#elif defined(_WIN32) && (_WIN32_WINNT >= 0x0602) - ULONG_PTR lowLimit = 0; - ULONG_PTR highLimit = 0; - GetCurrentThreadStackLimits(&lowLimit, &highLimit); - *outBegin = lowLimit; - *outEnd = highLimit; - return true; - -#elif defined(__OpenBSD__) - stack_t sinfo; - if (pthread_stackseg_np(pthread_self(), &sinfo) != 0) { + llvm::Optional bounds = + swift::Thread::stackBounds(); + if (!bounds) return false; - } - - *outBegin = (uintptr_t)sinfo.ss_sp - sinfo.ss_size; - *outEnd = (uintptr_t)sinfo.ss_sp; + *outBegin = (uintptr_t)bounds->low; + *outEnd = (uintptr_t)bounds->high; return true; -#elif defined(__FreeBSD__) || defined(__ANDROID__) || defined(__linux__) - pthread_attr_t attr; - -#if defined(__FreeBSD__) - if (0 != pthread_attr_init(&attr) || 0 != pthread_attr_get_np(pthread_self(), &attr)) { - return false; - } -#else - if (0 != pthread_getattr_np(pthread_self(), &attr)) { - return false; - } -#endif - - void *begin = nullptr; - size_t size = 0; - bool success = (0 == pthread_attr_getstack(&attr, &begin, &size)); - - *outBegin = (uintptr_t)begin; - *outEnd = *outBegin + size; - - pthread_attr_destroy(&attr); - return success; - -#else - // FIXME: implement on this platform - return false; -#endif } diff --git a/stdlib/public/stubs/ThreadLocalStorage.cpp b/stdlib/public/stubs/ThreadLocalStorage.cpp index 20c70fc922559..d357263fc9dea 100644 --- a/stdlib/public/stubs/ThreadLocalStorage.cpp +++ b/stdlib/public/stubs/ThreadLocalStorage.cpp @@ -13,9 +13,8 @@ #include #include "SwiftShims/ThreadLocalStorage.h" -#include "swift/Basic/Lazy.h" #include "swift/Runtime/Debug.h" -#include "swift/Runtime/ThreadLocalStorage.h" +#include "swift/Threading/ThreadLocalStorage.h" SWIFT_CC(swift) SWIFT_RUNTIME_STDLIB_API void _stdlib_destroyTLS(void *); @@ -23,97 +22,46 @@ void _stdlib_destroyTLS(void *); SWIFT_CC(swift) SWIFT_RUNTIME_STDLIB_API void *_stdlib_createTLS(void); -#if !SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC || (defined(_WIN32) && !defined(__CYGWIN__)) - -static void -#if defined(_M_IX86) -__stdcall -#endif -destroyTLS_CCAdjustmentThunk(void *ptr) { - _stdlib_destroyTLS(ptr); -} - -#endif - -#if defined(_WIN32) && !defined(__CYGWIN__) - -typedef -#if defined(_M_IX86) -__stdcall -#endif -void (*__swift_thread_key_destructor)(void *); - -static inline int -_stdlib_thread_key_create(__swift_thread_key_t * _Nonnull key, - __swift_thread_key_destructor _Nullable destructor) { - *key = FlsAlloc(destroyTLS_CCAdjustmentThunk); - if (*key == FLS_OUT_OF_INDEXES) - return GetLastError(); - return 0; -} - -#endif - -#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME - SWIFT_RUNTIME_STDLIB_INTERNAL void * _swift_stdlib_threadLocalStorageGet(void) { - static void *value; - if (!value) { - value = _stdlib_createTLS(); - } + +#if SWIFT_THREADING_NONE + + // If there's no threading, we can just keep a static variable. + static void *value = _stdlib_createTLS(); return value; -} -#elif SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC +#elif SWIFT_THREADING_USE_RESERVED_TLS_KEYS -SWIFT_RUNTIME_STDLIB_INTERNAL -void * -_swift_stdlib_threadLocalStorageGet(void) { - void *value = SWIFT_THREAD_GETSPECIFIC(SWIFT_STDLIB_TLS_KEY); + // If we have reserved keys, use those + void *value = swift::tls_get(swift::tls_key::stdlib); if (value) return value; - - static swift::OnceToken_t token; - SWIFT_ONCE_F(token, [](void *) { - int result = pthread_key_init_np(SWIFT_STDLIB_TLS_KEY, [](void *pointer) { - _stdlib_destroyTLS(pointer); - }); - if (result != 0) - swift::fatalError(0, "couldn't create pthread key for stdlib TLS: %s\n", - std::strerror(result)); - }, nullptr); - + + static swift::once_t token; + swift::tls_init_once(token, swift::tls_key::stdlib, + [](void *pointer) { _stdlib_destroyTLS(pointer); }); + value = _stdlib_createTLS(); - SWIFT_THREAD_SETSPECIFIC(SWIFT_STDLIB_TLS_KEY, value); + swift::tls_set(swift::tls_key::stdlib, value); return value; -} -#else +#else // Threaded, but not using reserved keys -SWIFT_RUNTIME_STDLIB_INTERNAL -void * -_swift_stdlib_threadLocalStorageGet(void) { - static swift::OnceToken_t token; - static __swift_thread_key_t key; + // Register a key and use it + static swift::tls_key_t key; + static swift::once_t token; - SWIFT_ONCE_F(token, [](void *) { - int result = SWIFT_THREAD_KEY_CREATE(&key, destroyTLS_CCAdjustmentThunk); - if (result != 0) - swift::fatalError(0, "couldn't create pthread key for stdlib TLS: %s\n", - std::strerror(result)); - }, nullptr); + swift::tls_alloc_once(token, key, + [](void *pointer) { _stdlib_destroyTLS(pointer); }); - void *value = SWIFT_THREAD_GETSPECIFIC(key); + void *value = swift::tls_get(key); if (!value) { value = _stdlib_createTLS(); - int result = SWIFT_THREAD_SETSPECIFIC(key, value); - if (result != 0) - swift::fatalError(0, "pthread_setspecific failed: %s\n", - std::strerror(result)); + swift::tls_set(key, value); } return value; -} #endif +} diff --git a/stdlib/toolchain/Compatibility50/CMakeLists.txt b/stdlib/toolchain/Compatibility50/CMakeLists.txt index 16824da43b5bf..43cc1efc1ba4e 100644 --- a/stdlib/toolchain/Compatibility50/CMakeLists.txt +++ b/stdlib/toolchain/Compatibility50/CMakeLists.txt @@ -8,6 +8,7 @@ add_swift_target_library("${library_name}" STATIC C_COMPILE_FLAGS ${CXX_COMPILE_FLAGS} LINK_FLAGS ${CXX_LINK_FLAGS} + INCORPORATE_OBJECT_LIBRARIES swiftThreading SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} DEPLOYMENT_VERSION_OSX ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_OSX} DEPLOYMENT_VERSION_IOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_IOS} diff --git a/stdlib/toolchain/Compatibility50/ProtocolConformance.cpp b/stdlib/toolchain/Compatibility50/ProtocolConformance.cpp index a48e801a5a43a..81a52a8dd278d 100644 --- a/stdlib/toolchain/Compatibility50/ProtocolConformance.cpp +++ b/stdlib/toolchain/Compatibility50/ProtocolConformance.cpp @@ -17,9 +17,9 @@ // //===----------------------------------------------------------------------===// -#include "Overrides.h" #include "../../public/runtime/Private.h" -#include "swift/Basic/Lazy.h" +#include "Overrides.h" +#include "swift/Threading/Once.h" #include #include #include @@ -91,8 +91,8 @@ swift::swift50override_conformsToProtocol(const Metadata *type, ConformsToProtocol_t *original_conformsToProtocol) { // Register our add image callback if necessary. - static OnceToken_t token; - SWIFT_ONCE_F(token, registerAddImageCallback, nullptr); + static swift::once_t token; + swift::once(token, registerAddImageCallback, nullptr); // The implementation of swift_conformsToProtocol in Swift 5.0 would return // a false negative answer when asking whether a subclass conforms using diff --git a/stdlib/toolchain/Compatibility51/CMakeLists.txt b/stdlib/toolchain/Compatibility51/CMakeLists.txt index aeea7ae896874..90d11bea1f6f8 100644 --- a/stdlib/toolchain/Compatibility51/CMakeLists.txt +++ b/stdlib/toolchain/Compatibility51/CMakeLists.txt @@ -9,6 +9,7 @@ add_swift_target_library("${library_name}" STATIC C_COMPILE_FLAGS ${CXX_COMPILE_FLAGS} LINK_FLAGS ${CXX_LINK_FLAGS} + INCORPORATE_OBJECT_LIBRARIES swiftThreading SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} DEPLOYMENT_VERSION_OSX ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_OSX} DEPLOYMENT_VERSION_IOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_IOS} diff --git a/stdlib/toolchain/CompatibilityConcurrency/CMakeLists.txt b/stdlib/toolchain/CompatibilityConcurrency/CMakeLists.txt index 6f08bc91dd550..33227ebc88e0d 100644 --- a/stdlib/toolchain/CompatibilityConcurrency/CMakeLists.txt +++ b/stdlib/toolchain/CompatibilityConcurrency/CMakeLists.txt @@ -7,6 +7,7 @@ add_swift_target_library("${library_name}" STATIC C_COMPILE_FLAGS ${CXX_COMPILE_FLAGS} LINK_FLAGS ${CXX_LINK_FLAGS} + INCORPORATE_OBJECT_LIBRARIES swiftThreading SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} DEPLOYMENT_VERSION_OSX ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_OSX} DEPLOYMENT_VERSION_IOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_IOS} diff --git a/stdlib/toolchain/CompatibilityDynamicReplacements/CMakeLists.txt b/stdlib/toolchain/CompatibilityDynamicReplacements/CMakeLists.txt index db63a04992504..b8322103dafa5 100644 --- a/stdlib/toolchain/CompatibilityDynamicReplacements/CMakeLists.txt +++ b/stdlib/toolchain/CompatibilityDynamicReplacements/CMakeLists.txt @@ -7,6 +7,7 @@ add_swift_target_library("${library_name}" STATIC C_COMPILE_FLAGS ${CXX_COMPILE_FLAGS} LINK_FLAGS ${CXX_LINK_FLAGS} + INCORPORATE_OBJECT_LIBRARIES swiftThreading SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} DEPLOYMENT_VERSION_OSX ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_OSX} DEPLOYMENT_VERSION_IOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_IOS} diff --git a/stdlib/toolchain/CompatibilityDynamicReplacements/DynamicReplaceable.cpp b/stdlib/toolchain/CompatibilityDynamicReplacements/DynamicReplaceable.cpp index c3467e2bf901e..63785be88af64 100644 --- a/stdlib/toolchain/CompatibilityDynamicReplacements/DynamicReplaceable.cpp +++ b/stdlib/toolchain/CompatibilityDynamicReplacements/DynamicReplaceable.cpp @@ -19,11 +19,21 @@ #include "swift/Runtime/Exclusivity.h" #include "swift/Runtime/FunctionReplacement.h" -#include "swift/Runtime/Once.h" -#include "swift/Runtime/ThreadLocalStorage.h" +#include "swift/Threading/ThreadLocalStorage.h" using namespace swift; +namespace { + +// Some threading libraries will need a global constructor here; that is +// unavoidable in the general case, so turn off the warning just for this line. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wglobal-constructors" +SWIFT_THREAD_LOCAL_TYPE(uintptr_t, tls_key::compatibility50) compat50Key; +#pragma clang diagnostic pop + +} // namespace + __attribute__((visibility("hidden"), weak)) extern "C" char *swift_getFunctionReplacement50(char **ReplFnPtr, char *CurrFn) { // Call the current implementation if it is available. @@ -40,12 +50,11 @@ extern "C" char *swift_getFunctionReplacement50(char **ReplFnPtr, char *CurrFn) if (RawReplFn == CurrFn) return nullptr; - auto origKey = - (uintptr_t)SWIFT_THREAD_GETSPECIFIC(SWIFT_COMPATIBILITY_50_TLS_KEY); + auto origKey = compat50Key.get(); if ((origKey & 0x1) != 0) { auto mask = ((uintptr_t)-1) < 1; auto resetKey = origKey & mask; - SWIFT_THREAD_SETSPECIFIC(SWIFT_COMPATIBILITY_50_TLS_KEY, (void *)resetKey); + compat50Key.set(resetKey); return nullptr; } return ReplFn; @@ -58,10 +67,9 @@ extern "C" char *swift_getOrigOfReplaceable50(char **OrigFnPtr) { return swift_getOrigOfReplaceable(OrigFnPtr); char *OrigFn = *OrigFnPtr; - auto origKey = - (uintptr_t)SWIFT_THREAD_GETSPECIFIC(SWIFT_COMPATIBILITY_50_TLS_KEY); + auto origKey = compat50Key.get(); auto newKey = origKey | 0x1; - SWIFT_THREAD_SETSPECIFIC(SWIFT_COMPATIBILITY_50_TLS_KEY, (void *)newKey); + compat50Key.set(newKey); return OrigFn; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index eaf28db082459..30883669dcc20 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") include(SwiftTestUtils) +include(Threading) function(swift_configure_lit_site_cfg source_path destination_path installed_name) if (CMAKE_CFG_INTDIR STREQUAL ".") @@ -199,7 +200,7 @@ normalize_boolean_spelling(SWIFT_ENABLE_MACCATALYST) normalize_boolean_spelling(SWIFT_RUN_TESTS_WITH_HOST_COMPILER) normalize_boolean_spelling(SWIFT_RUNTIME_ENABLE_LEAK_CHECKER) normalize_boolean_spelling(SWIFT_OPTIMIZED) -normalize_boolean_spelling(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +normalize_boolean_spelling(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY) normalize_boolean_spelling(SWIFT_ENABLE_REFLECTION) normalize_boolean_spelling(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS) normalize_boolean_spelling(SWIFT_STDLIB_ENABLE_DEBUG_PRECONDITIONS_IN_RELEASE) @@ -394,6 +395,14 @@ foreach(SDK ${SWIFT_SDKS}) list(APPEND LIT_ARGS "--param" "string_processing") endif() + if("${SWIFT_SDK_${SDK}_THREADING_PACKAGE}") + list(APPEND LIT_ARGS "--param" "threading=${SWIFT_SDK_${SDK}_THREADING_PACKAGE}") + else() + threading_package_default("${SDK}" SWIFT_THREADING_PACKAGE) + list(APPEND LIT_ARGS "--param" + "threading=${SWIFT_THREADING_PACKAGE}") + endif() + foreach(test_subset ${TEST_SUBSETS}) set(directories) set(dependencies ${test_dependencies}) diff --git a/test/Concurrency/Runtime/async_task_locals_copy_to_sync.swift b/test/Concurrency/Runtime/async_task_locals_copy_to_sync.swift index 839dc833e2bc6..99f81b654839d 100644 --- a/test/Concurrency/Runtime/async_task_locals_copy_to_sync.swift +++ b/test/Concurrency/Runtime/async_task_locals_copy_to_sync.swift @@ -8,7 +8,7 @@ // REQUIRES: concurrency_runtime // UNSUPPORTED: back_deployment_runtime // Disable on cooperative executor because it can't dispatch jobs before the end of main function -// UNSUPPORTED: single_threaded_runtime +// UNSUPPORTED: single_threaded_concurrency // REQUIRES: rdar80824152 import Dispatch diff --git a/test/Concurrency/Runtime/cancellation_handler.swift b/test/Concurrency/Runtime/cancellation_handler.swift index 4c99261e2ded6..6a096c54a743e 100644 --- a/test/Concurrency/Runtime/cancellation_handler.swift +++ b/test/Concurrency/Runtime/cancellation_handler.swift @@ -5,7 +5,7 @@ // rdar://76038845 // REQUIRES: concurrency_runtime // UNSUPPORTED: back_deployment_runtime -// UNSUPPORTED: single_threaded_runtime +// UNSUPPORTED: single_threaded_concurrency // for sleep #if canImport(Darwin) diff --git a/test/Concurrency/Runtime/data_race_detection.swift b/test/Concurrency/Runtime/data_race_detection.swift index 81153db0a178e..8eb43a94e3d8e 100644 --- a/test/Concurrency/Runtime/data_race_detection.swift +++ b/test/Concurrency/Runtime/data_race_detection.swift @@ -8,7 +8,7 @@ // rdar://76038845 // REQUIRES: concurrency_runtime // UNSUPPORTED: back_deployment_runtime -// UNSUPPORTED: single_threaded_runtime +// UNSUPPORTED: single_threaded_concurrency import _Concurrency import Dispatch diff --git a/test/Concurrency/Runtime/mainactor.swift b/test/Concurrency/Runtime/mainactor.swift index 5fcfb1d5b61c5..08c96067adbb6 100644 --- a/test/Concurrency/Runtime/mainactor.swift +++ b/test/Concurrency/Runtime/mainactor.swift @@ -7,7 +7,7 @@ // rdar://76038845 // REQUIRES: concurrency_runtime // UNSUPPORTED: back_deployment_runtime -// UNSUPPORTED: single_threaded_runtime +// UNSUPPORTED: single_threaded_concurrency import Dispatch diff --git a/test/IRGen/prespecialized-metadata/class-class-flags-run.swift b/test/IRGen/prespecialized-metadata/class-class-flags-run.swift index 265b146b060b0..5d29e157eccc9 100644 --- a/test/IRGen/prespecialized-metadata/class-class-flags-run.swift +++ b/test/IRGen/prespecialized-metadata/class-class-flags-run.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx // RUN: %target-build-swift -v %mcp_opt %s %t/isPrespecialized.o -import-objc-header %S/Inputs/isPrespecialized.h -Xfrontend -prespecialize-generic-metadata -target %module-target-future -lc++ -L %clang-include-dir/../lib/swift/macosx -sdk %sdk -o %t/main // RUN: %target-codesign %t/main diff --git a/test/IRGen/prespecialized-metadata/enum-extradata-run.swift b/test/IRGen/prespecialized-metadata/enum-extradata-run.swift index 14b4cc981df34..8e59bc10ab3b8 100644 --- a/test/IRGen/prespecialized-metadata/enum-extradata-run.swift +++ b/test/IRGen/prespecialized-metadata/enum-extradata-run.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/extraDataFields.cpp -o %t/extraDataFields.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/extraDataFields.cpp -o %t/extraDataFields.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx // RUN: %target-build-swift -c %S/Inputs/enum-extra-data-fields.swift -emit-library -emit-module -enable-library-evolution -module-name ExtraDataFieldsNoTrailingFlags -target %module-target-future -Xfrontend -disable-generic-metadata-prespecialization -emit-module-path %t/ExtraDataFieldsNoTrailingFlags.swiftmodule -o %t/%target-library-name(ExtraDataFieldsNoTrailingFlags) // RUN: %target-build-swift -c %S/Inputs/enum-extra-data-fields.swift -emit-library -emit-module -enable-library-evolution -module-name ExtraDataFieldsTrailingFlags -target %module-target-future -Xfrontend -prespecialize-generic-metadata -emit-module-path %t/ExtraDataFieldsTrailingFlags.swiftmodule -o %t/%target-library-name(ExtraDataFieldsTrailingFlags) diff --git a/test/IRGen/prespecialized-metadata/enum-trailing-flags-run.swift b/test/IRGen/prespecialized-metadata/enum-trailing-flags-run.swift index 227b8f31f23ec..dfa893285f840 100644 --- a/test/IRGen/prespecialized-metadata/enum-trailing-flags-run.swift +++ b/test/IRGen/prespecialized-metadata/enum-trailing-flags-run.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx // RUN: %target-build-swift -v %mcp_opt %s %t/isPrespecialized.o -import-objc-header %S/Inputs/isPrespecialized.h -Xfrontend -prespecialize-generic-metadata -target %module-target-future -lc++ -L %clang-include-dir/../lib/swift/macosx -sdk %sdk -o %t/main // RUN: %target-codesign %t/main diff --git a/test/IRGen/prespecialized-metadata/struct-extradata-run.swift b/test/IRGen/prespecialized-metadata/struct-extradata-run.swift index 6125ef79efc65..9f9d876172f26 100644 --- a/test/IRGen/prespecialized-metadata/struct-extradata-run.swift +++ b/test/IRGen/prespecialized-metadata/struct-extradata-run.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/extraDataFields.cpp -o %t/extraDataFields.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx -I %swift_obj_root/include +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/extraDataFields.cpp -o %t/extraDataFields.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx -I %swift_obj_root/include // RUN: %target-build-swift -c %S/Inputs/struct-extra-data-fields.swift -emit-library -emit-module -module-name ExtraDataFieldsNoTrailingFlags -target %module-target-future -Xfrontend -disable-generic-metadata-prespecialization -emit-module-path %t/ExtraDataFieldsNoTrailingFlags.swiftmodule -o %t/%target-library-name(ExtraDataFieldsNoTrailingFlags) // RUN: %target-build-swift -c %S/Inputs/struct-extra-data-fields.swift -emit-library -emit-module -module-name ExtraDataFieldsTrailingFlags -target %module-target-future -Xfrontend -prespecialize-generic-metadata -emit-module-path %t/ExtraDataFieldsTrailingFlags.swiftmodule -o %t/%target-library-name(ExtraDataFieldsTrailingFlags) diff --git a/test/IRGen/prespecialized-metadata/struct-multi-conformance.swift b/test/IRGen/prespecialized-metadata/struct-multi-conformance.swift index 2eadbb46f97ca..97153c7681e19 100644 --- a/test/IRGen/prespecialized-metadata/struct-multi-conformance.swift +++ b/test/IRGen/prespecialized-metadata/struct-multi-conformance.swift @@ -5,7 +5,7 @@ // RUN: %target-build-swift -c %s -DCONFORMANCE_2 -emit-library -emit-module -module-name Conformance2 -target %module-target-future -Xfrontend -prespecialize-generic-metadata -emit-module-path %t/Conformance2.swiftmodule -o %t/%target-library-name(Conformance2) -lBase -I %t -L %t // RUN: %target-build-swift -c %s -DGENERIC -emit-library -emit-module -module-name Generic -target %module-target-future -Xfrontend -prespecialize-generic-metadata -emit-module-path %t/Generic.swiftmodule -o %t/%target-library-name(Generic) -lBase -lConformance1 -I %t -L %t // RUN: %target-build-swift -c %s -DERASE -emit-library -emit-module -module-name Erase -target %module-target-future -Xfrontend -prespecialize-generic-metadata -emit-module-path %t/Erase.swiftmodule -o %t/%target-library-name(Erase) -lBase -lConformance2 -I %t -L %t -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx // RUN: %target-build-swift %s %S/Inputs/main.swift %S/Inputs/consume-logging-metadata-value.swift %t/isPrespecialized.o -import-objc-header %S/Inputs/isPrespecialized.h -DMAIN -target %module-target-future -Xfrontend -prespecialize-generic-metadata -lBase -lConformance1 -lConformance2 -lGeneric -lErase -lc++ -I %t -L %t -L %clang-include-dir/../lib/swift/macosx -o %t/main // RUN: %target-codesign %t/main diff --git a/test/IRGen/prespecialized-metadata/struct-outmodule-run.swift b/test/IRGen/prespecialized-metadata/struct-outmodule-run.swift index d91573301b7ed..d63c93316d8a9 100644 --- a/test/IRGen/prespecialized-metadata/struct-outmodule-run.swift +++ b/test/IRGen/prespecialized-metadata/struct-outmodule-run.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx // RUN: %target-build-swift %S/Inputs/struct-public-nonfrozen-1argument.swift -emit-module -emit-library -module-name Module -Xfrontend -prespecialize-generic-metadata -target %module-target-future -emit-module-path %t/Module.swiftmodule -o %t/%target-library-name(Module) diff --git a/test/IRGen/prespecialized-metadata/struct-trailing-flags-run.swift b/test/IRGen/prespecialized-metadata/struct-trailing-flags-run.swift index a896e21db6e9f..5eaa447ce7763 100644 --- a/test/IRGen/prespecialized-metadata/struct-trailing-flags-run.swift +++ b/test/IRGen/prespecialized-metadata/struct-trailing-flags-run.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %clang -c -v %target-cc-options -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx +// RUN: %clang -c -v %target-cc-options %target-threading-opt -g -O0 -isysroot %sdk %S/Inputs/isPrespecialized.cpp -o %t/isPrespecialized.o -I %clang-include-dir -I %swift_src_root/include/ -I %llvm_src_root/include -I %llvm_obj_root/include -L %clang-include-dir/../lib/swift/macosx // RUN: %target-build-swift -v %mcp_opt %s %t/isPrespecialized.o -import-objc-header %S/Inputs/isPrespecialized.h -Xfrontend -prespecialize-generic-metadata -target %module-target-future -lc++ -L %clang-include-dir/../lib/swift/macosx -sdk %sdk -o %t/main // RUN: %target-codesign %t/main diff --git a/test/Interpreter/enforce_exclusive_access.swift b/test/Interpreter/enforce_exclusive_access.swift index 066052419262c..4fa3022bb9a0f 100644 --- a/test/Interpreter/enforce_exclusive_access.swift +++ b/test/Interpreter/enforce_exclusive_access.swift @@ -4,7 +4,7 @@ // RUN: %target-codesign %t/a.out // RUN: %target-run %t/a.out // REQUIRES: executable_test -// UNSUPPORTED: single_threaded_runtime +// REQUIRES: thread_safe_runtime // UNSUPPORTED: use_os_stdlib diff --git a/test/TypeRoundTrip/round-trip.swift b/test/TypeRoundTrip/round-trip.swift index ecac9c6150bc1..b9fd055c7cb39 100644 --- a/test/TypeRoundTrip/round-trip.swift +++ b/test/TypeRoundTrip/round-trip.swift @@ -1,5 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %target-clang -std=c++14 %target-rtti-opt %target-pic-opt %target-msvc-runtime-opt -DSWIFT_INLINE_NAMESPACE=__runtime -g -c %S/Inputs/RoundTrip/RoundTrip.cpp -I%swift_obj_root/include -I%swift_src_root/include -I%swift_src_root/stdlib/include -I%swift_src_root/stdlib/public/runtime -I%llvm_src_root/include -I%llvm_obj_root/include -o %t/RoundTrip.o +// RUN: %target-clang -std=c++14 %target-threading-opt %target-rtti-opt %target-pic-opt %target-msvc-runtime-opt -DSWIFT_INLINE_NAMESPACE=__runtime -g -c %S/Inputs/RoundTrip/RoundTrip.cpp -I%swift_obj_root/include -I%swift_src_root/include -I%swift_src_root/stdlib/include -I%swift_src_root/stdlib/public/runtime -I%llvm_src_root/include -I%llvm_obj_root/include -o %t/RoundTrip.o // RUN: %target-build-swift -g -static -emit-module-path %t/RoundTrip.swiftmodule -emit-module -emit-library -module-name RoundTrip -o %t/%target-static-library-name(RoundTrip) %S/Inputs/RoundTrip/RoundTrip.swift %t/RoundTrip.o // RUN: echo "// AUTOGENERATED" > %t/all-tests.swift // RUN: for module in %S/Inputs/testcases/*.swift; do modname=$(basename $module .swift); echo "import $modname" >> %t/all-tests.swift; done diff --git a/test/lit.cfg b/test/lit.cfg index 9ccf987f95eb9..6db7a6af29dc5 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -448,6 +448,14 @@ if string_processing is not None: config.available_features.add('lld_lto') +threading = lit_config.params.get('threading', 'none') +config.available_features.add('threading_{}'.format(threading)) +if threading != "none": + config.available_features.add('thread_safe_runtime') + +config.substitutions.append(('%target-threading-opt', + '-DSWIFT_THREADING_{0}=1'.format(threading.upper()))) + test_options = os.environ.get('SWIFT_TEST_OPTIONS') if test_options: config.swift_test_options += ' ' diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in index 31402817a8913..1299d45c51534 100644 --- a/test/lit.site.cfg.in +++ b/test/lit.site.cfg.in @@ -96,8 +96,13 @@ else: if "@SWIFT_OPTIMIZED@" == "TRUE": config.available_features.add("optimized_stdlib") -if "@SWIFT_STDLIB_SINGLE_THREADED_RUNTIME@" == "TRUE": - config.available_features.add("single_threaded_runtime") +if "@SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY@" == "TRUE": + config.available_features.add("single_threaded_concurrency") + +if "@SWIFT_THREADING_PACKAGE@" != "none": + # This is not called "threading" because we might want that later + config.available_features.add("thread_safe_runtime") + config.available_features.add("threading_@SWIFT_THREADING_PACKAGE@") if "@SWIFT_ENABLE_REFLECTION@" == "TRUE": config.available_features.add("reflection") diff --git a/unittests/Basic/CMakeLists.txt b/unittests/Basic/CMakeLists.txt index bb25992c61c60..547751e6cac8f 100644 --- a/unittests/Basic/CMakeLists.txt +++ b/unittests/Basic/CMakeLists.txt @@ -44,6 +44,11 @@ add_dependencies(SwiftBasicTests "${gyb_dependency_targets}") target_link_libraries(SwiftBasicTests PRIVATE swiftBasic + swiftThreading clangBasic LLVMTestingSupport ) + +if(SWIFT_HOST_VARIANT STREQUAL windows) + target_link_libraries(SwiftBasicTests PRIVATE Synchronization) +endif() diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index cd2e6bdac1b55..be3c50083a30f 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -22,6 +22,7 @@ if(SWIFT_INCLUDE_TOOLS) if(SWIFT_BUILD_SYNTAXPARSERLIB) add_subdirectory(SyntaxParser) endif() + add_subdirectory(Threading) if(SWIFT_BUILD_SDK_OVERLAY) # Runtime tests depend on symbols in StdlibUnittest. diff --git a/unittests/Threading/CMakeLists.txt b/unittests/Threading/CMakeLists.txt new file mode 100644 index 0000000000000..b82f2bce677ff --- /dev/null +++ b/unittests/Threading/CMakeLists.txt @@ -0,0 +1,19 @@ +if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND + ("${SWIFT_HOST_VARIANT_ARCH}" STREQUAL "${SWIFT_PRIMARY_VARIANT_ARCH}")) + add_swift_unittest(SwiftThreadingTests + Mutex.cpp + Once.cpp + LinuxUlock.cpp + Fatal.cpp + ) + + target_link_libraries(SwiftThreadingTests + PRIVATE + swiftThreading${SWIFT_PRIMARY_VARIANT_SUFFIX} + swiftCore${SWIFT_PRIMARY_VARIANT_SUFFIX} + ) + + if(SWIFT_HOST_VARIANT STREQUAL windows) + target_link_libraries(SwiftThreadingTests PRIVATE Synchronization) + endif() +endif() diff --git a/unittests/Threading/Fatal.cpp b/unittests/Threading/Fatal.cpp new file mode 100644 index 0000000000000..d4944b10211bd --- /dev/null +++ b/unittests/Threading/Fatal.cpp @@ -0,0 +1,29 @@ +//===---Fatal.cpp - Stub for the threading tests --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Threading/Errors.h" + +#include +#include +#include + +SWIFT_ATTRIBUTE_NORETURN +SWIFT_FORMAT(1, 2) +void swift::threading::fatal(const char *msg, ...) { + va_list val; + + va_start(val, msg); + std::vfprintf(stderr, msg, val); + va_end(val); + + std::abort(); +} diff --git a/unittests/Threading/LinuxUlock.cpp b/unittests/Threading/LinuxUlock.cpp new file mode 100644 index 0000000000000..d8fabe3a520cd --- /dev/null +++ b/unittests/Threading/LinuxUlock.cpp @@ -0,0 +1,58 @@ +//===--- LinuxULock.cpp - Tests the Linux ulock implementation ------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if SWIFT_THREADING_LINUX + +#include "swift/Threading/Impl/Linux/ulock.h" +#include "gtest/gtest.h" + +#include "LockingHelpers.h" + +using namespace swift; +using namespace swift::threading_impl; + +// ----------------------------------------------------------------------------- + +// Use the existing locking tests to check threaded operation +class UlockMutex { +private: + linux::ulock_t lock_; + +public: + UlockMutex() : lock_(0) {} + + void lock() { linux::ulock_lock(&lock_); } + void unlock() { linux::ulock_unlock(&lock_); } + bool try_lock() { return linux::ulock_trylock(&lock_); } +}; + +TEST(LinuxUlockTest, SingleThreaded) { + UlockMutex mutex; + basicLockable(mutex); +} + +TEST(LinuxUlockTest, SingleThreadedTryLock) { + UlockMutex mutex; + tryLockable(mutex); +} + +TEST(LinuxULockTest, BasicLockableThreaded) { + UlockMutex mutex; + basicLockableThreaded(mutex); +} + +TEST(LinuxULockTest, LockableThreaded) { + UlockMutex mutex; + lockableThreaded(mutex); +} + +#endif // SWIFT_THREADING_LINUX diff --git a/unittests/Threading/LockingHelpers.h b/unittests/Threading/LockingHelpers.h new file mode 100644 index 0000000000000..151fa7a3ec715 --- /dev/null +++ b/unittests/Threading/LockingHelpers.h @@ -0,0 +1,157 @@ +//===--- LockingHelpers.h - Utilities to help test locks ------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef LOCKING_HELPERS_H +#define LOCKING_HELPERS_H + +#include "ThreadingHelpers.h" + +// Test that a Mutex object can be locked and unlocked from a single thread +template +void basicLockable(M &mutex) { + // We can lock, unlock, lock and unlock an unlocked lock + mutex.lock(); + mutex.unlock(); + mutex.lock(); + mutex.unlock(); +} + +// Test that a Mutex object's try_lock() method works. +template +void tryLockable(M &mutex) { + bool ret; + + // We can lock an unlocked lock + ret = mutex.try_lock(); + ASSERT_TRUE(ret); + + // We cannot lock a locked lock + ret = mutex.try_lock(); + ASSERT_FALSE(ret); + + mutex.unlock(); +} + +// Test that a Mutex object can be locked and unlocked +template +void basicLockableThreaded(M &mutex) { + int count1 = 0; + int count2 = 0; + + threadedExecute(10, [&](int) { + for (int j = 0; j < 50; ++j) { + mutex.lock(); + auto count = count2; + count1++; + count2 = count + 1; + mutex.unlock(); + } + }); + + ASSERT_EQ(count1, 500); + ASSERT_EQ(count2, 500); +} + +// More extensive tests +template +void lockableThreaded(M &mutex) { + mutex.lock(); + threadedExecute(5, [&](int) { ASSERT_FALSE(mutex.try_lock()); }); + mutex.unlock(); + threadedExecute(1, [&](int) { + ASSERT_TRUE(mutex.try_lock()); + mutex.unlock(); + }); + + int count1 = 0; + int count2 = 0; + threadedExecute(10, [&](int) { + for (int j = 0; j < 50; ++j) { + if (mutex.try_lock()) { + auto count = count2; + count1++; + count2 = count + 1; + mutex.unlock(); + } else { + j--; + } + } + }); + + ASSERT_EQ(count1, 500); + ASSERT_EQ(count2, 500); +} + +// Test a scoped lock implementation +template +void scopedLockThreaded(M &mutex) { + int count1 = 0; + int count2 = 0; + + threadedExecute(10, [&](int) { + for (int j = 0; j < 50; ++j) { + SL guard(mutex); + auto count = count2; + count1++; + count2 = count + 1; + } + }); + + ASSERT_EQ(count1, 500); + ASSERT_EQ(count2, 500); +} + +// Test a scoped unlock nested in a scoped lock +template +void scopedUnlockUnderScopedLockThreaded(M &mutex) { + int count1 = 0; + int count2 = 0; + int badCount = 0; + + threadedExecute(10, [&](int) { + for (int j = 0; j < 50; ++j) { + SL guard(mutex); + { + SU unguard(mutex); + badCount++; + } + auto count = count2; + count1++; + count2 = count + 1; + } + }); + + ASSERT_EQ(count1, 500); + ASSERT_EQ(count2, 500); +} + +// Test a critical section +template +void criticalSectionThreaded(M &mutex) { + int count1 = 0; + int count2 = 0; + + threadedExecute(10, [&](int) { + for (int j = 0; j < 50; ++j) { + mutex.withLock([&] { + auto count = count2; + count1++; + count2 = count + 1; + }); + } + }); + + ASSERT_EQ(count1, 500); + ASSERT_EQ(count2, 500); +} + +#endif diff --git a/unittests/Threading/Mutex.cpp b/unittests/Threading/Mutex.cpp new file mode 100644 index 0000000000000..d5e5dbd45d2f4 --- /dev/null +++ b/unittests/Threading/Mutex.cpp @@ -0,0 +1,142 @@ +//===--- Mutex.cpp - Mutex Tests ------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Threading/Mutex.h" +#include "gtest/gtest.h" +#include +#include +#include +#include + +#include "LockingHelpers.h" + +using namespace swift; + +// ----------------------------------------------------------------------------- + +TEST(MutexTest, BasicLockable) { + Mutex mutex(/* checked = */ true); + basicLockable(mutex); +} + +TEST(LazyMutexTest, BasicLockable) { + static LazyMutex mutex; + basicLockable(mutex); +} + +TEST(LazyUnsafeMutexTest, BasicLockable) { + static LazyUnsafeMutex mutex; + basicLockable(mutex); +} + +TEST(SmallMutex, BasicLockable) { + SmallMutex mutex; + basicLockable(mutex); +} + +TEST(MutexTest, TryLockable) { + Mutex mutex(/* checked = */ true); + tryLockable(mutex); +} + +TEST(LazyMutexTest, TryLockable) { + static LazyMutex mutex; + tryLockable(mutex); +} + +TEST(LazyUnsafeMutexTest, TryLockable) { + static LazyUnsafeMutex mutex; + tryLockable(mutex); +} + +TEST(SmallMutex, TryLockable) { + SmallMutex mutex; + tryLockable(mutex); +} + +TEST(MutexTest, BasicLockableThreaded) { + Mutex mutex(/* checked = */ true); + basicLockableThreaded(mutex); +} + +TEST(LazyMutexTest, BasicLockableThreaded) { + static LazyMutex mutex; + basicLockableThreaded(mutex); +} + +TEST(LazyUnsafeMutexTest, BasicLockableThreaded) { + static LazyUnsafeMutex mutex; + basicLockableThreaded(mutex); +} + +TEST(SmallMutex, BasicLockableThreaded) { + SmallMutex mutex; + basicLockableThreaded(mutex); +} + +TEST(MutexTest, LockableThreaded) { + Mutex mutex(/* checked = */ true); + lockableThreaded(mutex); +} + +TEST(LazyMutexTest, LockableThreaded) { + static LazyMutex Mutex; + lockableThreaded(Mutex); +} + +TEST(SmallMutexTest, LockableThreaded) { + SmallMutex Mutex; + lockableThreaded(Mutex); +} + +TEST(MutexTest, ScopedLockThreaded) { + Mutex mutex(/* checked = */ true); + scopedLockThreaded(mutex); +} + +TEST(LazyMutexTest, ScopedLockThreaded) { + static LazyMutex Mutex; + scopedLockThreaded(Mutex); +} + +TEST(SmallMutexTest, ScopedLockThreaded) { + SmallMutex mutex(/* checked = */ true); + scopedLockThreaded>(mutex); +} + +TEST(MutexTest, ScopedUnlockUnderScopedLockThreaded) { + Mutex mutex(/* checked = */ true); + scopedUnlockUnderScopedLockThreaded( + mutex); +} + +TEST(LazyMutexTest, ScopedUnlockUnderScopedLockThreaded) { + static LazyMutex Mutex; + scopedUnlockUnderScopedLockThreaded(Mutex); +} + +TEST(SmallMutexTest, ScopedUnlockUnderScopedLockThreaded) { + SmallMutex mutex(/* checked = */ true); + scopedUnlockUnderScopedLockThreaded(mutex); +} + +TEST(MutexTest, CriticalSectionThreaded) { + Mutex mutex(/* checked = */ true); + criticalSectionThreaded(mutex); +} + +TEST(LazyMutexTest, CriticalSectionThreaded) { + static LazyMutex Mutex; + criticalSectionThreaded(Mutex); +} diff --git a/unittests/Threading/Once.cpp b/unittests/Threading/Once.cpp new file mode 100644 index 0000000000000..ecfd30fa5d523 --- /dev/null +++ b/unittests/Threading/Once.cpp @@ -0,0 +1,88 @@ +//===--- Once.cpp - Tests the swift::once() implementation ----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Threading/Once.h" +#include "gtest/gtest.h" + +#include + +#include "ThreadingHelpers.h" + +using namespace swift; + +// Check that swift::once calls the function, with the correct argument +TEST(OnceTest, once_calls_function) { + static swift::once_t predicate; + bool wasCalled = false; + + swift::once( + predicate, + [](void *ctx) { + bool *pWasCalled = static_cast(ctx); + *pWasCalled = true; + }, + &wasCalled); + + ASSERT_TRUE(wasCalled); +} + +// Check that calling swift::once twice only calls the function once +TEST(OnceTest, once_calls_only_once) { + static swift::once_t predicate; + unsigned callCount = 0; + + void (*fn)(void *) = [](void *ctx) { + unsigned *pCallCount = static_cast(ctx); + ++*pCallCount; + }; + + swift::once(predicate, fn, &callCount); + swift::once(predicate, fn, &callCount); + + ASSERT_EQ(1u, callCount); +} + +// Check that swift::once works when threaded +TEST(OnceTest, once_threaded) { + void (*fn)(void *) = [](void *ctx) { + unsigned *pCallCount = static_cast(ctx); + ++*pCallCount; + }; + + for (unsigned tries = 0; tries < 1000; ++tries) { + swift::once_t predicate; + unsigned callCount = 0; + + // We're being naughty here; swift::once_t is supposed to be global/static, + // but since we know what we're doing, this should be OK. + std::memset(&predicate, 0, sizeof(predicate)); + + threadedExecute(16, [&](int) { swift::once(predicate, fn, &callCount); }); + + ASSERT_EQ(1u, callCount); + } +} + +// Check that swift::once works with a C++ lambda +TEST(OnceTest, once_lambda) { + static swift::once_t predicate; + unsigned callCount = 0; + + auto fn = [&callCount]() { + ++callCount; + }; + + swift::once(predicate, fn); + swift::once(predicate, fn); + + ASSERT_EQ(1u, callCount); +} diff --git a/unittests/Threading/ThreadingHelpers.h b/unittests/Threading/ThreadingHelpers.h new file mode 100644 index 0000000000000..08e0e2d9a227f --- /dev/null +++ b/unittests/Threading/ThreadingHelpers.h @@ -0,0 +1,134 @@ +//===--- ThreadingHelpers.h - Utilities to help test threading ------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef THREADING_HELPERS_H +#define THREADING_HELPERS_H + +#include + +// When true many of the threaded tests log activity to help triage issues. +static bool trace = false; + +template +void threadedExecute(int threadCount, ThreadBody threadBody, + AfterSpinRelease afterSpinRelease) { + + std::vector threads; + + // Block the threads we are about to create. + std::atomic spinWait(true); + std::atomic readyCount(0); + std::atomic activeCount(0); + + for (int i = 0; i < threadCount; ++i) { + threads.push_back(std::thread([&, i] { + readyCount++; + + while (spinWait) { + std::this_thread::sleep_for(std::chrono::microseconds(10)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + activeCount++; + + threadBody(i); + })); + } + + while (readyCount < threadCount) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + // Allow our threads to fight for the lock. + spinWait = false; + + while (activeCount < threadCount) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + afterSpinRelease(); + + // Wait until all of our threads have finished. + for (auto &thread : threads) { + thread.join(); + } +} + +template +void threadedExecute(int threadCount, ThreadBody threadBody) { + threadedExecute(threadCount, threadBody, [] {}); +} + +template +void threadedExecute(M &mutex, C &condition, bool &doneCondition, + ConsumerBody consumerBody, ProducerBody producerBody) { + + std::vector producers; + std::vector consumers; + + // Block the threads we are about to create. + std::atomic spinWait(true); + + for (int i = 1; i <= 8; ++i) { + consumers.push_back(std::thread([&, i] { + while (spinWait) { + std::this_thread::sleep_for(std::chrono::microseconds(10)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + consumerBody(i); + + if (trace) + printf("### Consumer[%d] thread exiting.\n", i); + })); + } + + for (int i = 1; i <= 5; ++i) { + producers.push_back(std::thread([&, i] { + while (spinWait) { + std::this_thread::sleep_for(std::chrono::microseconds(10)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + producerBody(i); + + if (trace) + printf("### Producer[%d] thread exiting.\n", i); + })); + } + + // Poor mans attempt to get as many threads ready as possible before + // dropping spinWait, it doesn't have to be perfect. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Allow our threads to fight for the lock. + spinWait = false; + + // Wait until all of our producer threads have finished. + for (auto &thread : producers) { + thread.join(); + } + + // Inform consumers that producers are done. + mutex.withLockThenNotifyAll(condition, [&] { + if (trace) + printf("### Informing consumers we are done.\n"); + doneCondition = true; + }); + + // Wait for consumers to finish. + for (auto &thread : consumers) { + thread.join(); + } +} + +#endif diff --git a/unittests/runtime/CMakeLists.txt b/unittests/runtime/CMakeLists.txt index 323a3ec49ce8c..79df463c0edb0 100644 --- a/unittests/runtime/CMakeLists.txt +++ b/unittests/runtime/CMakeLists.txt @@ -64,7 +64,7 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND ${EXECINFO_LIBRARY} ) elseif(SWIFT_HOST_VARIANT STREQUAL windows) - list(APPEND PLATFORM_TARGET_LINK_LIBRARIES DbgHelp) + list(APPEND PLATFORM_TARGET_LINK_LIBRARIES DbgHelp;Synchronization) endif() if(SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY) @@ -84,10 +84,6 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND endif() endif() - if(NOT SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) - list(APPEND PLATFORM_SOURCES Mutex.cpp) - endif() - if(SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED) # list(APPEND PLATFORM_SOURCES # DistributedActor.cpp @@ -102,8 +98,7 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND weak.mm Refcounting.mm Actor.cpp - TaskStatus.cpp - Mutex.cpp) + TaskStatus.cpp) add_swift_unittest(SwiftRuntimeTests Array.cpp @@ -148,6 +143,7 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND target_link_libraries(SwiftRuntimeTests PRIVATE swiftCore${SWIFT_PRIMARY_VARIANT_SUFFIX} + swiftThreading${SWIFT_PRIMARY_VARIANT_SUFFIX} ${PLATFORM_TARGET_LINK_LIBRARIES} ) endif() diff --git a/unittests/runtime/Concurrent.cpp b/unittests/runtime/Concurrent.cpp index f3b3ea9308b1e..a72d2d16b2eb1 100644 --- a/unittests/runtime/Concurrent.cpp +++ b/unittests/runtime/Concurrent.cpp @@ -45,7 +45,7 @@ TEST(ConcurrentReadableArrayTest, SingleThreaded) { check(); } -#ifndef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#ifndef SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY TEST(ConcurrentReadableArrayTest, MultiThreaded) { const int insertCount = 100000; @@ -544,4 +544,4 @@ TEST(ConcurrentReadableHashMapTest, MultiThreaded4) { runTest(16, 1); runTest(16, 8); } -#endif // !SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +#endif // !SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY diff --git a/unittests/runtime/LongTests/CMakeLists.txt b/unittests/runtime/LongTests/CMakeLists.txt index 0f8268ea6941e..592e532f8cf53 100644 --- a/unittests/runtime/LongTests/CMakeLists.txt +++ b/unittests/runtime/LongTests/CMakeLists.txt @@ -31,7 +31,7 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND ${EXECINFO_LIBRARY} ) elseif(SWIFT_HOST_VARIANT STREQUAL windows) - list(APPEND PLATFORM_TARGET_LINK_LIBRARIES DbgHelp) + list(APPEND PLATFORM_TARGET_LINK_LIBRARIES DbgHelp;Synchronization) endif() add_swift_unittest(SwiftRuntimeLongTests @@ -60,6 +60,7 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND target_link_libraries(SwiftRuntimeLongTests PRIVATE swiftCore${SWIFT_PRIMARY_VARIANT_SUFFIX} + swiftThreading${SWIFT_PRIMARY_VARIANT_SUFFIX} ${PLATFORM_TARGET_LINK_LIBRARIES} ${swift_runtime_test_extra_libraries} ) diff --git a/unittests/runtime/Mutex.cpp b/unittests/runtime/Mutex.cpp deleted file mode 100644 index f3344184be0a8..0000000000000 --- a/unittests/runtime/Mutex.cpp +++ /dev/null @@ -1,757 +0,0 @@ -//===--- Mutex.cpp - Mutex and ReadWriteLock Tests ------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#include "swift/Runtime/Mutex.h" -#include "gtest/gtest.h" -#include -#include -#include -#include - -#include "ThreadingHelpers.h" - -using namespace swift; - -// ----------------------------------------------------------------------------- - -template void basicLockableThreaded(M &mutex) { - int count1 = 0; - int count2 = 0; - - threadedExecute(10, [&](int) { - for (int j = 0; j < 50; ++j) { - mutex.lock(); - auto count = count2; - count1++; - count2 = count + 1; - mutex.unlock(); - } - }); - - ASSERT_EQ(count1, 500); - ASSERT_EQ(count2, 500); -} - -TEST(MutexTest, BasicLockableThreaded) { - Mutex mutex(/* checked = */ true); - basicLockableThreaded(mutex); -} - -TEST(StaticMutexTest, BasicLockableThreaded) { - static StaticMutex mutex; - basicLockableThreaded(mutex); -} - -TEST(StaticUnsafeMutexTest, BasicLockableThreaded) { - static StaticUnsafeMutex mutex; - basicLockableThreaded(mutex); -} - -TEST(SmallMutex, BasicLockableThreaded) { - SmallMutex mutex; - basicLockableThreaded(mutex); -} - -template void lockableThreaded(M &mutex) { - mutex.lock(); - threadedExecute(5, [&](int) { ASSERT_FALSE(mutex.try_lock()); }); - mutex.unlock(); - threadedExecute(1, [&](int) { - ASSERT_TRUE(mutex.try_lock()); - mutex.unlock(); - }); - - int count1 = 0; - int count2 = 0; - threadedExecute(10, [&](int) { - for (int j = 0; j < 50; ++j) { - if (mutex.try_lock()) { - auto count = count2; - count1++; - count2 = count + 1; - mutex.unlock(); - } else { - j--; - } - } - }); - - ASSERT_EQ(count1, 500); - ASSERT_EQ(count2, 500); -} - -TEST(MutexTest, LockableThreaded) { - Mutex mutex(/* checked = */ true); - lockableThreaded(mutex); -} - -TEST(StaticMutexTest, LockableThreaded) { - static StaticMutex Mutex; - lockableThreaded(Mutex); -} - -TEST(SmallMutexTest, LockableThreaded) { - SmallMutex Mutex; - lockableThreaded(Mutex); -} - -template void scopedLockThreaded(M &mutex) { - int count1 = 0; - int count2 = 0; - - threadedExecute(10, [&](int) { - for (int j = 0; j < 50; ++j) { - SL guard(mutex); - auto count = count2; - count1++; - count2 = count + 1; - } - }); - - ASSERT_EQ(count1, 500); - ASSERT_EQ(count2, 500); -} - -TEST(MutexTest, ScopedLockThreaded) { - Mutex mutex(/* checked = */ true); - scopedLockThreaded(mutex); -} - -TEST(StaticMutexTest, ScopedLockThreaded) { - static StaticMutex Mutex; - scopedLockThreaded(Mutex); -} - -TEST(SmallMutexTest, ScopedLockThreaded) { - SmallMutex mutex(/* checked = */ true); - scopedLockThreaded>(mutex); -} - -template -void scopedUnlockUnderScopedLockThreaded(M &mutex) { - int count1 = 0; - int count2 = 0; - int badCount = 0; - - threadedExecute(10, [&](int) { - for (int j = 0; j < 50; ++j) { - SL guard(mutex); - { - SU unguard(mutex); - badCount++; - } - auto count = count2; - count1++; - count2 = count + 1; - } - }); - - ASSERT_EQ(count1, 500); - ASSERT_EQ(count2, 500); -} - -TEST(MutexTest, ScopedUnlockUnderScopedLockThreaded) { - Mutex mutex(/* checked = */ true); - scopedUnlockUnderScopedLockThreaded( - mutex); -} - -TEST(StaticMutexTest, ScopedUnlockUnderScopedLockThreaded) { - static StaticMutex Mutex; - scopedUnlockUnderScopedLockThreaded(Mutex); -} - -TEST(SmallMutexTest, ScopedUnlockUnderScopedLockThreaded) { - SmallMutex mutex(/* checked = */ true); - scopedUnlockUnderScopedLockThreaded(mutex); -} - -template void criticalSectionThreaded(M &mutex) { - int count1 = 0; - int count2 = 0; - - threadedExecute(10, [&](int) { - for (int j = 0; j < 50; ++j) { - mutex.withLock([&] { - auto count = count2; - count1++; - count2 = count + 1; - }); - } - }); - - ASSERT_EQ(count1, 500); - ASSERT_EQ(count2, 500); -} - -TEST(MutexTest, CriticalSectionThreaded) { - Mutex mutex(/* checked = */ true); - criticalSectionThreaded(mutex); -} - -TEST(StaticMutexTest, CriticalSectionThreaded) { - static StaticMutex Mutex; - criticalSectionThreaded(Mutex); -} - -template -void scopedReadThreaded(RW &lock) { - const int threadCount = 10; - - std::set writerHistory; - std::vector> readerHistory; - readerHistory.assign(threadCount, std::set()); - - int protectedValue = 0; - writerHistory.insert(protectedValue); - - threadedExecute(threadCount, - [&](int index) { - if (Locking) { - for (int i = 0; i < 50; ++i) { - { - SRL guard(lock); - readerHistory[index].insert(protectedValue); - } - std::this_thread::yield(); - } - } else { - lock.readLock(); - for (int i = 0; i < 50; ++i) { - readerHistory[index].insert(protectedValue); - - { - SRL unguard(lock); - std::this_thread::yield(); - } - } - lock.readUnlock(); - } - }, - [&] { - for (int i = 0; i < 25; ++i) { - lock.writeLock(); - protectedValue += i; - writerHistory.insert(protectedValue); - lock.writeUnlock(); - } - }); - - for (auto &history : readerHistory) { - for (auto value : history) { - ASSERT_EQ(writerHistory.count(value), 1U); - } - } -} - -TEST(ReadWriteLockTest, ScopedReadLockThreaded) { - ReadWriteLock lock; - scopedReadThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ScopedReadLockThreaded) { - static StaticReadWriteLock lock; - scopedReadThreaded(lock); -} - -TEST(ReadWriteLockTest, ScopedReadUnlockThreaded) { - ReadWriteLock lock; - scopedReadThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ScopedReadUnlockThreaded) { - static StaticReadWriteLock lock; - scopedReadThreaded(lock); -} - -template -void scopedWriteLockThreaded(RW &lock) { - const int threadCount = 10; - - std::set readerHistory; - std::vector> writerHistory; - writerHistory.assign(threadCount, std::set()); - - int protectedValue = 0; - readerHistory.insert(protectedValue); - - threadedExecute(threadCount, - [&](int index) { - if (Locking) { - for (int i = 0; i < 20; ++i) { - { - SWL guard(lock); - protectedValue += index * i; - writerHistory[index].insert(protectedValue); - } - std::this_thread::yield(); - } - } else { - lock.writeLock(); - for (int i = 0; i < 20; ++i) { - protectedValue += index * i; - writerHistory[index].insert(protectedValue); - { - SWL unguard(lock); - std::this_thread::yield(); - } - } - lock.writeUnlock(); - } - }, - [&] { - for (int i = 0; i < 100; ++i) { - lock.readLock(); - readerHistory.insert(protectedValue); - lock.readUnlock(); - } - }); - - std::set mergedHistory; - for (auto &history : writerHistory) { - mergedHistory.insert(history.begin(), history.end()); - } - - for (auto value : readerHistory) { - ASSERT_EQ(mergedHistory.count(value), 1U); - } -} - -TEST(ReadWriteLockTest, ScopedWriteLockThreaded) { - ReadWriteLock lock; - scopedWriteLockThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ScopedWriteLockThreaded) { - static StaticReadWriteLock lock; - scopedWriteLockThreaded(lock); -} - -TEST(ReadWriteLockTest, ScopedWriteUnlockThreaded) { - ReadWriteLock lock; - scopedWriteLockThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ScopedWriteUnlockThreaded) { - static StaticReadWriteLock lock; - scopedWriteLockThreaded(lock); -} - -template void readLockWhileReadLockedThreaded(RW &lock) { - lock.readLock(); - - const int threadCount = 10; - - std::atomic results[threadCount] = {}; - - std::atomic done(false); - threadedExecute(threadCount, - [&](int index) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - lock.withReadLock([&] { - results[index] = true; - std::this_thread::sleep_for( - std::chrono::milliseconds(5)); - }); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - }); - - lock.readUnlock(); - - for (auto &result : results) { - ASSERT_TRUE(result); - } -} - -TEST(ReadWriteLockTest, ReadLockWhileReadLockedThreaded) { - ReadWriteLock lock; - readLockWhileReadLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ReadLockWhileReadLockedThreaded) { - static StaticReadWriteLock lock; - readLockWhileReadLockedThreaded(lock); -} - -template void readLockWhileWriteLockedThreaded(RW &lock) { - lock.writeLock(); - - const int threadCount = 10; - - std::atomic results[threadCount] = {}; - - std::atomic done(false); - threadedExecute(threadCount, - [&](int index) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - lock.withReadLock([&] { - results[index] += 1; - std::this_thread::sleep_for( - std::chrono::milliseconds(5)); - }); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - lock.writeUnlock(); - }); - - for (auto &result : results) { - ASSERT_EQ(result, 1); - } -} - -TEST(ReadWriteLockTest, ReadLockWhileWriteLockedThreaded) { - ReadWriteLock lock; - readLockWhileWriteLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ReadLockWhileWriteLockedThreaded) { - static StaticReadWriteLock lock; - readLockWhileWriteLockedThreaded(lock); -} - -template void writeLockWhileReadLockedThreaded(RW &lock) { - lock.readLock(); - - const int threadCount = 10; - - std::atomic results[threadCount] = {}; - - std::atomic done(false); - threadedExecute(threadCount, - [&](int index) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - lock.withWriteLock([&] { - results[index] += 1; - std::this_thread::sleep_for( - std::chrono::milliseconds(5)); - }); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - lock.readUnlock(); - }); - - for (auto &result : results) { - ASSERT_EQ(result, 1); - } -} - -TEST(ReadWriteLockTest, WriteLockWhileReadLockedThreaded) { - ReadWriteLock lock; - writeLockWhileReadLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, WriteLockWhileReadLockedThreaded) { - static StaticReadWriteLock lock; - writeLockWhileReadLockedThreaded(lock); -} - -template void writeLockWhileWriteLockedThreaded(RW &lock) { - lock.writeLock(); - - const int threadCount = 10; - - std::atomic results[threadCount] = {}; - - std::atomic done(false); - threadedExecute(threadCount, - [&](int index) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - lock.withWriteLock([&] { - results[index] += 1; - std::this_thread::sleep_for( - std::chrono::milliseconds(5)); - }); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - lock.writeUnlock(); - }); - - for (auto &result : results) { - ASSERT_EQ(result, 1); - } -} - -TEST(ReadWriteLockTest, WriteLockWhileWriteLockedThreaded) { - ReadWriteLock lock; - writeLockWhileWriteLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, WriteLockWhileWriteLockedThreaded) { - static StaticReadWriteLock lock; - writeLockWhileWriteLockedThreaded(lock); -} - -template void tryReadLockWhileWriteLockedThreaded(RW &lock) { - lock.writeLock(); - - std::atomic done(false); - threadedExecute(10, - [&](int) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - ASSERT_FALSE(lock.try_readLock()); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - }); - - lock.writeUnlock(); -} - -TEST(ReadWriteLockTest, TryReadLockWhileWriteLockedThreaded) { - ReadWriteLock lock; - tryReadLockWhileWriteLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, TryReadLockWhileWriteLockedThreaded) { - static StaticReadWriteLock lock; - tryReadLockWhileWriteLockedThreaded(lock); -} - -template void tryReadLockWhileReadLockedThreaded(RW &lock) { - lock.readLock(); - - const int threadCount = 10; - - std::atomic results[threadCount] = {}; - - std::atomic done(false); - threadedExecute(threadCount, - [&](int index) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - ASSERT_TRUE(lock.try_readLock()); - results[index] = true; - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - lock.readUnlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - }); - - lock.readUnlock(); - - for (auto &result : results) { - ASSERT_TRUE(result); - } -} - -TEST(ReadWriteLockTest, TryReadLockWhileReadLockedThreaded) { - ReadWriteLock lock; - tryReadLockWhileReadLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, TryReadLockWhileReadLockedThreaded) { - static StaticReadWriteLock lock; - tryReadLockWhileReadLockedThreaded(lock); -} - -template void tryWriteLockWhileWriteLockedThreaded(RW &lock) { - lock.writeLock(); - - std::atomic done(false); - threadedExecute(10, - [&](int) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - ASSERT_FALSE(lock.try_writeLock()); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - }); - - lock.writeUnlock(); -} - -TEST(ReadWriteLockTest, TryWriteLockWhileWriteLockedThreaded) { - ReadWriteLock lock; - tryWriteLockWhileWriteLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, TryWriteLockWhileWriteLockedThreaded) { - static StaticReadWriteLock lock; - tryWriteLockWhileWriteLockedThreaded(lock); -} - -template void tryWriteLockWhileReadLockedThreaded(RW &lock) { - lock.readLock(); - - std::atomic done(false); - threadedExecute(10, - [&](int) { - // Always perform at least one iteration of this loop to - // avoid spurious failures if this thread is slow to run. - do { - ASSERT_FALSE(lock.try_writeLock()); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (!done); - }, - [&] { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - done = true; - }); - - lock.readUnlock(); -} - -TEST(ReadWriteLockTest, TryWriteLockWhileReadLockedThreaded) { - ReadWriteLock lock; - tryWriteLockWhileReadLockedThreaded(lock); -} - -TEST(StaticReadWriteLockTest, TryWriteLockWhileReadLockedThreaded) { - static StaticReadWriteLock lock; - tryWriteLockWhileReadLockedThreaded(lock); -} - -template void readWriteLockCacheExampleThreaded(RW &lock) { - std::map cache; - std::vector workers; - std::vector> workerHistory; - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution<> dis(0, UINT8_MAX); - - workerHistory.push_back(std::set()); - for (int i = 0; i < 16; i++) { - uint8_t key = dis(gen); - cache[key] = 0; - workerHistory[0].insert(key); - - if (trace) - printf("WarmUp create for key = %d, value = %d.\n", key, 0); - } - - // Block the threads we are about to create. - const int threadCount = 20; - std::atomic spinWait(true); - std::atomic readyCount(0); - - for (int i = 1; i <= threadCount; ++i) { - workerHistory.push_back(std::set()); - workers.push_back(std::thread([&, i] { - readyCount++; - - // Block ourself until we are released to start working. - while (spinWait) { - std::this_thread::sleep_for(std::chrono::microseconds(10)); - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - for (int j = 0; j < 50; j++) { - uint8_t key = dis(gen); - bool found = false; - - auto cacheLookupSection = [&] { - auto value = cache.find(key); - if (value == cache.end()) { - if (trace) - printf("Worker[%d] miss for key = %d.\n", i, key); - found = false; // cache miss, need to grab write lock - } - if (trace) - printf("Worker[%d] HIT for key = %d, value = %d.\n", i, key, - value->second); - found = true; // cache hit, no need to grab write lock - }; - - lock.withReadLock(cacheLookupSection); - if (found) { - continue; - } - - lock.withWriteLock([&] { - cacheLookupSection(); - if (!found) { - if (trace) - printf("Worker[%d] create for key = %d, value = %d.\n", i, key, - i); - cache[key] = i; - workerHistory[i].insert(key); - } - }); - } - - if (trace) - printf("### Worker[%d] thread exiting.\n", i); - })); - } - - while (readyCount < threadCount) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - // Allow our threads to fight for the lock. - spinWait = false; - - // Wait until all of our workers threads have finished. - for (auto &thread : workers) { - thread.join(); - } - - for (auto &entry : cache) { - if (trace) - printf("### Cache dump key = %d, value = %d.\n", entry.first, - entry.second); - ASSERT_EQ(workerHistory[entry.second].count(entry.first), 1U); - } -} - -TEST(ReadWriteLockTest, ReadWriteLockCacheExampleThreaded) { - ReadWriteLock lock; - readWriteLockCacheExampleThreaded(lock); -} - -TEST(StaticReadWriteLockTest, ReadWriteLockCacheExampleThreaded) { - static StaticReadWriteLock lock; - readWriteLockCacheExampleThreaded(lock); -} diff --git a/utils/build-presets.ini b/utils/build-presets.ini index ea44792dbd615..07b9365b10323 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -1205,7 +1205,8 @@ mixin-preset=buildbot_incremental_linux_base llvm-targets-to-build=X86;ARM;AArch64;WebAssembly # Ensure single-thread-mode is not broken because it's used # by stdlib for wasm32 in SwiftWasm fork. -swift-stdlib-single-threaded-runtime=1 +swift-stdlib-single-threaded-concurrency=1 +swift-threading-package=none #===------------------------------------------------------------------------===# # OS X Package Builders @@ -2508,7 +2509,7 @@ swift-stdlib-has-stdin=0 swift-stdlib-has-environ=0 swift-stdlib-has-locale=0 swift-runtime-static-image-inspection=1 -swift-stdlib-single-threaded-runtime=1 +swift-stdlib-single-threaded-concurrency=1 swift-stdlib-concurrency-tracing=0 swift-stdlib-os-versioning=0 swift-stdlib-has-commandline=0 @@ -2520,6 +2521,7 @@ swift-stdlib-enable-vector-types=0 swift-stdlib-experimental-hermetic-seal-at-link=1 swift-stdlib-disable-instantiation-caches=1 swift-stdlib-has-type-printing=0 +swift-threading-package=none build-swift-stdlib-unicode-data=0 build-swift-stdlib-static-print=1 darwin-crash-reporter-client=0 diff --git a/utils/build-script-impl b/utils/build-script-impl index b2b91d551e20f..3a47fc49eb60c 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -206,7 +206,8 @@ KNOWN_SETTINGS=( swift-stdlib-has-dladdr "1" "whether to build stdlib assuming the runtime environment provides dladdr API" swift-stdlib-supports-backtrace-reporting "" "whether to build stdlib assuming the runtime environment provides the backtrace(3) API, if not set defaults to true on all platforms except for Cygwin, Haiku and wasm" swift-runtime-static-image-inspection "0" "whether to build stdlib assuming the runtime environment only supports a single runtime image with Swift code" - swift-stdlib-single-threaded-runtime "0" "whether to build stdlib as a single-threaded runtime only" + swift-threading-package "" "which threading package to use for Swift; valid values are empty string (default based on platform), 'pthreads', 'darwin', 'linux', 'win32', 'c11', 'none'" + swift-stdlib-single-threaded-concurrency "0" "build Swift concurrency in single-threaded mode" swift-stdlib-concurrency-tracing "" "whether to enable tracing signposts for concurrency; default is 1 on Darwin platforms, 0 otherwise" swift-stdlib-os-versioning "1" "whether to build stdlib with availability based on OS versions (Darwin only)" swift-stdlib-has-commandline "1" "whether to build stdlib with the CommandLine enum and support for argv/argc" @@ -2004,7 +2005,7 @@ for host in "${ALL_HOSTS[@]}"; do -DSWIFT_ENABLE_DISPATCH:BOOL=$(true_false "${SWIFT_ENABLE_DISPATCH}") -DSWIFT_IMPLICIT_CONCURRENCY_IMPORT:BOOL=$(true_false "${SWIFT_IMPLICIT_CONCURRENCY_IMPORT}") -DSWIFT_STDLIB_SUPPORT_BACK_DEPLOYMENT:BOOL=$(true_false "${SWIFT_STDLIB_SUPPORT_BACK_DEPLOYMENT}") - -DSWIFT_STDLIB_SINGLE_THREADED_RUNTIME:BOOL=$(true_false "${SWIFT_STDLIB_SINGLE_THREADED_RUNTIME}") + -DSWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY:BOOL=$(true_false "${SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY}") -DSWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS:BOOL=$(true_false "${SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS}") -DSWIFT_STDLIB_HAS_DLADDR:BOOL=$(true_false "${SWIFT_STDLIB_HAS_DLADDR}") -DSWIFT_RUNTIME_STATIC_IMAGE_INSPECTION:BOOL=$(true_false "${SWIFT_RUNTIME_STATIC_IMAGE_INSPECTION}") @@ -2226,6 +2227,21 @@ for host in "${ALL_HOSTS[@]}"; do ) fi + if [[ "${SWIFT_THREADING_PACKAGE}" ]] ; then + case "${SWIFT_THREADING_PACKAGE}" in + "" | pthreads | darwin | linux | win32 | c11 | none) ;; + *) + echo "build-script: unknown threading package ${SWIFT_THREADING_PACKAGE}; must be one of 'pthreads', 'darwin', 'linux', 'win32', 'c11', 'none', or empty for platform default" >&2 + exit 1 + ;; + esac + + cmake_options=( + "${cmake_options[@]}" + -DSWIFT_THREADING_PACKAGE:STRING="${SWIFT_THREADING_PACKAGE}" + ) + fi + build_targets=(all "${SWIFT_STDLIB_TARGETS[@]}") if [[ $(true_false "${build_perf_testsuite_this_time}") == "TRUE" ]]; then native_swift_tools_path="$(build_directory_bin ${LOCAL_HOST} swift)" diff --git a/validation-test/lit.site.cfg.in b/validation-test/lit.site.cfg.in index 6e4718d9bff30..ee541f7f53657 100644 --- a/validation-test/lit.site.cfg.in +++ b/validation-test/lit.site.cfg.in @@ -94,8 +94,13 @@ if "@SWIFT_OPTIMIZED@" == "TRUE": if "@SWIFT_ENABLE_SOURCEKIT_TESTS@" == "TRUE": config.available_features.add('sourcekit') -if "@SWIFT_STDLIB_SINGLE_THREADED_RUNTIME@" == "TRUE": - config.available_features.add("single_threaded_runtime") +if "@SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY@" == "TRUE": + config.available_features.add("single_threaded_concurrency") + +if "@SWIFT_THREADING_PACKAGE@" != "none": + # This is not called "threading" because we might want that later + config.available_features.add("thread_safe_runtime") + config.available_features.add("threading_@SWIFT_THREADING_PACKAGE@") if "@SWIFT_ENABLE_REFLECTION@" == "TRUE": config.available_features.add("reflection")