diff --git a/src/library_pthread.js b/src/library_pthread.js
index 4bc001b758b7a..4670ccecc0617 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -1024,8 +1024,8 @@ var LibraryPThread = {
return 0;
},
- pthread_detach__sig: 'vi',
- pthread_detach: function(thread) {
+ {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_detach__sig: 'vi',
+ {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_detach: function(thread) {
if (!thread) {
err('pthread_detach attempted on a null thread pointer!');
return ERRNO_CODES.ESRCH;
@@ -1043,9 +1043,13 @@ var LibraryPThread = {
return wasDetached ? ERRNO_CODES.EINVAL : 0;
},
- // C11 threads function.
+ // C11 thread version.
// TODO: remove this in favor or compiling musl/src/thread/pthread_detach.c
+#if USE_LSAN
+ thrd_detach: 'emscripten_builtin_pthread_detach',
+#else
thrd_detach: 'pthread_detach',
+#endif
pthread_exit__deps: ['exit'],
pthread_exit: function(status) {
diff --git a/system/lib/compiler-rt/include/sanitizer/asan_interface.h b/system/lib/compiler-rt/include/sanitizer/asan_interface.h
index 6af93aad6512f..792ef9cfaa32b 100644
--- a/system/lib/compiler-rt/include/sanitizer/asan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/asan_interface.h
@@ -188,8 +188,8 @@ const char *__asan_get_report_description(void);
/// \param addr Address to locate.
/// \param name Buffer to store the variable's name.
/// \param name_size Size in bytes of the variable's name buffer.
-/// \param region_address [out] Address of the region.
-/// \param region_size [out] Size of the region in bytes.
+/// \param[out] region_address Address of the region.
+/// \param[out] region_size Size of the region in bytes.
///
/// \returns Returns the category of the given pointer as a constant string.
const char *__asan_locate_address(void *addr, char *name, size_t name_size,
@@ -204,7 +204,7 @@ const char *__asan_locate_address(void *addr, char *name, size_t name_size,
/// \param addr A heap address.
/// \param trace A buffer to store the stack trace.
/// \param size Size in bytes of the trace buffer.
-/// \param thread_id [out] The thread ID of the address.
+/// \param[out] thread_id The thread ID of the address.
///
/// \returns Returns the number of stored frames or 0 on error.
size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size,
@@ -219,7 +219,7 @@ size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size,
/// \param addr A heap address.
/// \param trace A buffer to store the stack trace.
/// \param size Size in bytes of the trace buffer.
-/// \param thread_id [out] The thread ID of the address.
+/// \param[out] thread_id The thread ID of the address.
///
/// \returns Returns the number of stored frames or 0 on error.
size_t __asan_get_free_stack(void *addr, void **trace, size_t size,
@@ -228,8 +228,8 @@ size_t __asan_get_free_stack(void *addr, void **trace, size_t size,
/// Gets the current shadow memory mapping (useful for calling from the
/// debugger).
///
-/// \param shadow_scale [out] Shadow scale value.
-/// \param shadow_offset [out] Offset value.
+/// \param[out] shadow_scale Shadow scale value.
+/// \param[out] shadow_offset Offset value.
void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset);
/// This is an internal function that is called to report an error. However,
@@ -302,8 +302,8 @@ void *__asan_get_current_fake_stack(void);
///
/// \param fake_stack An opaque handler to a fake stack.
/// \param addr Address to test.
-/// \param beg [out] Beginning of fake frame.
-/// \param end [out] End of fake frame.
+/// \param[out] beg Beginning of fake frame.
+/// \param[out] end End of fake frame.
/// \returns Stack address or NULL.
void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg,
void **end);
diff --git a/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h b/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h
index f979c6a8f63b1..cd69285b8d4af 100644
--- a/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h
+++ b/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h
@@ -43,6 +43,9 @@ void __sanitizer_set_report_path(const char *path);
// Tell the tools to write their reports to the provided file descriptor
// (casted to void *).
void __sanitizer_set_report_fd(void *fd);
+// Get the current full report file path, if a path was specified by
+// an earlier call to __sanitizer_set_report_path. Returns null otherwise.
+const char *__sanitizer_get_report_path();
// Notify the tools that the sandbox is going to be turned on. The reserved
// parameter will be used in the future to hold a structure with functions
@@ -320,7 +323,7 @@ void __sanitizer_print_memory_profile(size_t top_percent,
/// signal callback runs during the switch, it will not benefit from stack
/// use-after-return detection.
///
-/// \param fake_stack_save [out] Fake stack save location.
+/// \param[out] fake_stack_save Fake stack save location.
/// \param bottom Bottom address of stack.
/// \param size Size of stack in bytes.
void __sanitizer_start_switch_fiber(void **fake_stack_save,
@@ -335,8 +338,8 @@ void __sanitizer_start_switch_fiber(void **fake_stack_save,
/// __sanitizer_start_switch_fiber().
///
/// \param fake_stack_save Fake stack save location.
-/// \param bottom_old [out] Bottom address of old stack.
-/// \param size_old [out] Size of old stack in bytes.
+/// \param[out] bottom_old Bottom address of old stack.
+/// \param[out] size_old Size of old stack in bytes.
void __sanitizer_finish_switch_fiber(void *fake_stack_save,
const void **bottom_old,
size_t *size_old);
diff --git a/system/lib/compiler-rt/include/sanitizer/dfsan_interface.h b/system/lib/compiler-rt/include/sanitizer/dfsan_interface.h
index 81546e5df71af..18b2c81a6023c 100644
--- a/system/lib/compiler-rt/include/sanitizer/dfsan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/dfsan_interface.h
@@ -80,9 +80,11 @@ dfsan_label dfsan_has_label_with_desc(dfsan_label label, const char *desc);
size_t dfsan_get_label_count(void);
/// Flushes the DFSan shadow, i.e. forgets about all labels currently associated
-/// with the application memory. Will work only if there are no other
-/// threads executing DFSan-instrumented code concurrently.
-/// Use this call to start over the taint tracking within the same procces.
+/// with the application memory. Use this call to start over the taint tracking
+/// within the same process.
+///
+/// Note: If another thread is working with tainted data during the flush, that
+/// taint could still be written to shadow after the flush.
void dfsan_flush(void);
/// Sets a callback to be invoked on calls to write(). The callback is invoked
diff --git a/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h b/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h
index 4c9ad13aa0cb1..14035c05c6353 100644
--- a/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h
@@ -73,6 +73,9 @@ extern "C" {
* accessed through the pointer in x, or -1 if the whole range is good. */
intptr_t __hwasan_test_shadow(const volatile void *x, size_t size);
+ /* Sets the callback function to be called during HWASan error reporting. */
+ void __hwasan_set_error_report_callback(void (*callback)(const char *));
+
int __sanitizer_posix_memalign(void **memptr, size_t alignment, size_t size);
void * __sanitizer_memalign(size_t alignment, size_t size);
void * __sanitizer_aligned_alloc(size_t alignment, size_t size);
diff --git a/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h b/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h
index a1794b71af503..56eae3d40f968 100644
--- a/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h
+++ b/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h
@@ -1845,6 +1845,10 @@
#define __sanitizer_syscall_post_rt_sigaction(res, signum, act, oldact, sz) \
__sanitizer_syscall_post_impl_rt_sigaction(res, (long)signum, (long)act, \
(long)oldact, (long)sz)
+#define __sanitizer_syscall_pre_sigaltstack(ss, oss) \
+ __sanitizer_syscall_pre_impl_sigaltstack((long)ss, (long)oss)
+#define __sanitizer_syscall_post_sigaltstack(res, ss, oss) \
+ __sanitizer_syscall_post_impl_sigaltstack(res, (long)ss, (long)oss)
// And now a few syscalls we don't handle yet.
#define __sanitizer_syscall_pre_afs_syscall(...)
@@ -1912,7 +1916,6 @@
#define __sanitizer_syscall_pre_setreuid32(...)
#define __sanitizer_syscall_pre_set_thread_area(...)
#define __sanitizer_syscall_pre_setuid32(...)
-#define __sanitizer_syscall_pre_sigaltstack(...)
#define __sanitizer_syscall_pre_sigreturn(...)
#define __sanitizer_syscall_pre_sigsuspend(...)
#define __sanitizer_syscall_pre_stty(...)
@@ -1992,7 +1995,6 @@
#define __sanitizer_syscall_post_setreuid32(res, ...)
#define __sanitizer_syscall_post_set_thread_area(res, ...)
#define __sanitizer_syscall_post_setuid32(res, ...)
-#define __sanitizer_syscall_post_sigaltstack(res, ...)
#define __sanitizer_syscall_post_sigreturn(res, ...)
#define __sanitizer_syscall_post_sigsuspend(res, ...)
#define __sanitizer_syscall_post_stty(res, ...)
@@ -3075,6 +3077,8 @@ void __sanitizer_syscall_pre_impl_rt_sigaction(long signum, long act,
long oldact, long sz);
void __sanitizer_syscall_post_impl_rt_sigaction(long res, long signum, long act,
long oldact, long sz);
+void __sanitizer_syscall_pre_impl_sigaltstack(long ss, long oss);
+void __sanitizer_syscall_post_impl_sigaltstack(long res, long ss, long oss);
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/system/lib/compiler-rt/include/sanitizer/memprof_interface.h b/system/lib/compiler-rt/include/sanitizer/memprof_interface.h
new file mode 100644
index 0000000000000..76031de4014c0
--- /dev/null
+++ b/system/lib/compiler-rt/include/sanitizer/memprof_interface.h
@@ -0,0 +1,65 @@
+//===-- sanitizer/memprof_interface.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler (MemProf).
+//
+// Public interface header.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_MEMPROF_INTERFACE_H
+#define SANITIZER_MEMPROF_INTERFACE_H
+
+#include
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/// Records access to a memory region ([addr, addr+size)).
+///
+/// This memory must be previously allocated by your program.
+///
+/// \param addr Start of memory region.
+/// \param size Size of memory region.
+void __memprof_record_access_range(void const volatile *addr, size_t size);
+
+/// Records access to a memory address addr.
+///
+/// This memory must be previously allocated by your program.
+///
+/// \param addr Accessed memory address
+void __memprof_record_access(void const volatile *addr);
+
+/// User-provided callback on MemProf errors.
+///
+/// You can provide a function that would be called immediately when MemProf
+/// detects an error. This is useful in cases when MemProf detects an error but
+/// your program crashes before the MemProf report is printed.
+void __memprof_on_error(void);
+
+/// Prints accumulated statistics to stderr (useful for calling from the
+/// debugger).
+void __memprof_print_accumulated_stats(void);
+
+/// User-provided default option settings.
+///
+/// You can provide your own implementation of this function to return a string
+/// containing MemProf runtime options (for example,
+/// verbosity=1:print_stats=1).
+///
+/// \returns Default options string.
+const char *__memprof_default_options(void);
+
+/// Prints the memory profile to the current profile file.
+///
+/// \returns 0 on success.
+int __memprof_profile_dump(void);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SANITIZER_MEMPROF_INTERFACE_H
diff --git a/system/lib/compiler-rt/include/sanitizer/msan_interface.h b/system/lib/compiler-rt/include/sanitizer/msan_interface.h
index d40c556a46d93..eeb39fbed8b49 100644
--- a/system/lib/compiler-rt/include/sanitizer/msan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/msan_interface.h
@@ -114,6 +114,9 @@ extern "C" {
call to __msan_scoped_disable_interceptor_checks. */
void __msan_scoped_enable_interceptor_checks(void);
+ void __msan_start_switch_fiber(const void *bottom, size_t size);
+ void __msan_finish_switch_fiber(const void **bottom_old, size_t *size_old);
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/system/lib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h b/system/lib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h
index 370da0ea72ed8..f661152ccbac7 100644
--- a/system/lib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h
+++ b/system/lib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h
@@ -20,8 +20,8 @@
// DO NOT EDIT! THIS FILE HAS BEEN GENERATED!
//
// Generated with: generate_netbsd_syscalls.awk
-// Generated date: 2019-12-24
-// Generated from: syscalls.master,v 1.296 2019/09/22 22:59:39 christos Exp
+// Generated date: 2020-09-10
+// Generated from: syscalls.master,v 1.306 2020/08/14 00:53:16 riastradh Exp
//
//===----------------------------------------------------------------------===//
#ifndef SANITIZER_NETBSD_SYSCALL_HOOKS_H
@@ -474,7 +474,12 @@
__sanitizer_syscall_pre_impl_dup2((long long)(from), (long long)(to))
#define __sanitizer_syscall_post_dup2(res, from, to) \
__sanitizer_syscall_post_impl_dup2(res, (long long)(from), (long long)(to))
-/* syscall 91 has been skipped */
+#define __sanitizer_syscall_pre_getrandom(buf, buflen, flags) \
+ __sanitizer_syscall_pre_impl_getrandom( \
+ (long long)(buf), (long long)(buflen), (long long)(flags))
+#define __sanitizer_syscall_post_getrandom(res, buf, buflen, flags) \
+ __sanitizer_syscall_post_impl_getrandom( \
+ res, (long long)(buf), (long long)(buflen), (long long)(flags))
#define __sanitizer_syscall_pre_fcntl(fd, cmd, arg) \
__sanitizer_syscall_pre_impl_fcntl((long long)(fd), (long long)(cmd), \
(long long)(arg))
@@ -849,9 +854,31 @@
#define __sanitizer_syscall_post_sysarch(res, op, parms) \
__sanitizer_syscall_post_impl_sysarch(res, (long long)(op), \
(long long)(parms))
-/* syscall 166 has been skipped */
-/* syscall 167 has been skipped */
-/* syscall 168 has been skipped */
+#define __sanitizer_syscall_pre___futex(uaddr, op, val, timeout, uaddr2, val2, \
+ val3) \
+ __sanitizer_syscall_pre_impl___futex((long long)(uaddr), (long long)(op), \
+ (long long)(val), (long long)(timeout), \
+ (long long)(uaddr2), (long long)(val2), \
+ (long long)(val3))
+#define __sanitizer_syscall_post___futex(res, uaddr, op, val, timeout, uaddr2, \
+ val2, val3) \
+ __sanitizer_syscall_post_impl___futex( \
+ res, (long long)(uaddr), (long long)(op), (long long)(val), \
+ (long long)(timeout), (long long)(uaddr2), (long long)(val2), \
+ (long long)(val3))
+#define __sanitizer_syscall_pre___futex_set_robust_list(head, len) \
+ __sanitizer_syscall_pre_impl___futex_set_robust_list((long long)(head), \
+ (long long)(len))
+#define __sanitizer_syscall_post___futex_set_robust_list(res, head, len) \
+ __sanitizer_syscall_post_impl___futex_set_robust_list( \
+ res, (long long)(head), (long long)(len))
+#define __sanitizer_syscall_pre___futex_get_robust_list(lwpid, headp, lenp) \
+ __sanitizer_syscall_pre_impl___futex_get_robust_list( \
+ (long long)(lwpid), (long long)(headp), (long long)(lenp))
+#define __sanitizer_syscall_post___futex_get_robust_list(res, lwpid, headp, \
+ lenp) \
+ __sanitizer_syscall_post_impl___futex_get_robust_list( \
+ res, (long long)(lwpid), (long long)(headp), (long long)(lenp))
#if !defined(_LP64)
#define __sanitizer_syscall_pre_compat_10_osemsys(which, a2, a3, a4, a5) \
__sanitizer_syscall_pre_impl_compat_10_osemsys( \
@@ -2731,6 +2758,83 @@
__sanitizer_syscall_post_impl___fhstatvfs190( \
res, (long long)(fhp), (long long)(fh_size), (long long)(buf), \
(long long)(flags))
+#define __sanitizer_syscall_pre___acl_get_link(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_get_link( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_get_link(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_get_link( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_set_link(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_set_link( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_set_link(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_set_link( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_delete_link(path, type) \
+ __sanitizer_syscall_pre_impl___acl_delete_link((long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_post___acl_delete_link(res, path, type) \
+ __sanitizer_syscall_post_impl___acl_delete_link(res, (long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_pre___acl_aclcheck_link(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_aclcheck_link( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_aclcheck_link(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_aclcheck_link( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_get_file(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_get_file( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_get_file(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_get_file( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_set_file(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_set_file( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_set_file(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_set_file( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_get_fd(filedes, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_get_fd( \
+ (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_get_fd(res, filedes, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_get_fd( \
+ res, (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_set_fd(filedes, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_set_fd( \
+ (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_set_fd(res, filedes, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_set_fd( \
+ res, (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_delete_file(path, type) \
+ __sanitizer_syscall_pre_impl___acl_delete_file((long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_post___acl_delete_file(res, path, type) \
+ __sanitizer_syscall_post_impl___acl_delete_file(res, (long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_pre___acl_delete_fd(filedes, type) \
+ __sanitizer_syscall_pre_impl___acl_delete_fd((long long)(filedes), \
+ (long long)(type))
+#define __sanitizer_syscall_post___acl_delete_fd(res, filedes, type) \
+ __sanitizer_syscall_post_impl___acl_delete_fd(res, (long long)(filedes), \
+ (long long)(type))
+#define __sanitizer_syscall_pre___acl_aclcheck_file(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_aclcheck_file( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_aclcheck_file(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_aclcheck_file( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_aclcheck_fd(filedes, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_aclcheck_fd( \
+ (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_aclcheck_fd(res, filedes, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_aclcheck_fd( \
+ res, (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre_lpathconf(path, name) \
+ __sanitizer_syscall_pre_impl_lpathconf((long long)(path), (long long)(name))
+#define __sanitizer_syscall_post_lpathconf(res, path, name) \
+ __sanitizer_syscall_post_impl_lpathconf(res, (long long)(path), \
+ (long long)(name))
/* Compat with older releases */
#define __sanitizer_syscall_pre_getvfsstat \
@@ -3088,7 +3192,10 @@ void __sanitizer_syscall_post_impl_compat_43_ogetdtablesize(long long res);
void __sanitizer_syscall_pre_impl_dup2(long long from, long long to);
void __sanitizer_syscall_post_impl_dup2(long long res, long long from,
long long to);
-/* syscall 91 has been skipped */
+void __sanitizer_syscall_pre_impl_getrandom(long long buf, long long buflen,
+ long long flags);
+void __sanitizer_syscall_post_impl_getrandom(long long res, long long buf,
+ long long buflen, long long flags);
void __sanitizer_syscall_pre_impl_fcntl(long long fd, long long cmd,
long long arg);
void __sanitizer_syscall_post_impl_fcntl(long long res, long long fd,
@@ -3380,9 +3487,26 @@ void __sanitizer_syscall_post_impl_compat_09_ouname(long long res,
void __sanitizer_syscall_pre_impl_sysarch(long long op, long long parms);
void __sanitizer_syscall_post_impl_sysarch(long long res, long long op,
long long parms);
-/* syscall 166 has been skipped */
-/* syscall 167 has been skipped */
-/* syscall 168 has been skipped */
+void __sanitizer_syscall_pre_impl___futex(long long uaddr, long long op,
+ long long val, long long timeout,
+ long long uaddr2, long long val2,
+ long long val3);
+void __sanitizer_syscall_post_impl___futex(long long res, long long uaddr,
+ long long op, long long val,
+ long long timeout, long long uaddr2,
+ long long val2, long long val3);
+void __sanitizer_syscall_pre_impl___futex_set_robust_list(long long head,
+ long long len);
+void __sanitizer_syscall_post_impl___futex_set_robust_list(long long res,
+ long long head,
+ long long len);
+void __sanitizer_syscall_pre_impl___futex_get_robust_list(long long lwpid,
+ long long headp,
+ long long lenp);
+void __sanitizer_syscall_post_impl___futex_get_robust_list(long long res,
+ long long lwpid,
+ long long headp,
+ long long lenp);
#if !defined(_LP64)
void __sanitizer_syscall_pre_impl_compat_10_osemsys(long long which,
long long a2, long long a3,
@@ -4802,6 +4926,75 @@ void __sanitizer_syscall_post_impl___fhstatvfs190(long long res, long long fhp,
long long fh_size,
long long buf,
long long flags);
+void __sanitizer_syscall_pre_impl___acl_get_link(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_get_link(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_set_link(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_set_link(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_delete_link(long long path,
+ long long type);
+void __sanitizer_syscall_post_impl___acl_delete_link(long long res,
+ long long path,
+ long long type);
+void __sanitizer_syscall_pre_impl___acl_aclcheck_link(long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_aclcheck_link(long long res,
+ long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_get_file(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_get_file(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_set_file(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_set_file(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_get_fd(long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_post_impl___acl_get_fd(long long res,
+ long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_pre_impl___acl_set_fd(long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_post_impl___acl_set_fd(long long res,
+ long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_pre_impl___acl_delete_file(long long path,
+ long long type);
+void __sanitizer_syscall_post_impl___acl_delete_file(long long res,
+ long long path,
+ long long type);
+void __sanitizer_syscall_pre_impl___acl_delete_fd(long long filedes,
+ long long type);
+void __sanitizer_syscall_post_impl___acl_delete_fd(long long res,
+ long long filedes,
+ long long type);
+void __sanitizer_syscall_pre_impl___acl_aclcheck_file(long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_aclcheck_file(long long res,
+ long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_aclcheck_fd(long long filedes,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_aclcheck_fd(long long res,
+ long long filedes,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl_lpathconf(long long path, long long name);
+void __sanitizer_syscall_post_impl_lpathconf(long long res, long long path,
+ long long name);
#ifdef __cplusplus
} // extern "C"
diff --git a/system/lib/compiler-rt/include/sanitizer/tsan_interface.h b/system/lib/compiler-rt/include/sanitizer/tsan_interface.h
index 011b23350cac3..96b8ad58541cb 100644
--- a/system/lib/compiler-rt/include/sanitizer/tsan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/tsan_interface.h
@@ -38,34 +38,34 @@ void __tsan_release(void *addr);
// Mutex has static storage duration and no-op constructor and destructor.
// This effectively makes tsan ignore destroy annotation.
-const unsigned __tsan_mutex_linker_init = 1 << 0;
+static const unsigned __tsan_mutex_linker_init = 1 << 0;
// Mutex is write reentrant.
-const unsigned __tsan_mutex_write_reentrant = 1 << 1;
+static const unsigned __tsan_mutex_write_reentrant = 1 << 1;
// Mutex is read reentrant.
-const unsigned __tsan_mutex_read_reentrant = 1 << 2;
+static const unsigned __tsan_mutex_read_reentrant = 1 << 2;
// Mutex does not have static storage duration, and must not be used after
// its destructor runs. The opposite of __tsan_mutex_linker_init.
// If this flag is passed to __tsan_mutex_destroy, then the destruction
// is ignored unless this flag was previously set on the mutex.
-const unsigned __tsan_mutex_not_static = 1 << 8;
+static const unsigned __tsan_mutex_not_static = 1 << 8;
// Mutex operation flags:
// Denotes read lock operation.
-const unsigned __tsan_mutex_read_lock = 1 << 3;
+static const unsigned __tsan_mutex_read_lock = 1 << 3;
// Denotes try lock operation.
-const unsigned __tsan_mutex_try_lock = 1 << 4;
+static const unsigned __tsan_mutex_try_lock = 1 << 4;
// Denotes that a try lock operation has failed to acquire the mutex.
-const unsigned __tsan_mutex_try_lock_failed = 1 << 5;
+static const unsigned __tsan_mutex_try_lock_failed = 1 << 5;
// Denotes that the lock operation acquires multiple recursion levels.
// Number of levels is passed in recursion parameter.
// This is useful for annotation of e.g. Java builtin monitors,
// for which wait operation releases all recursive acquisitions of the mutex.
-const unsigned __tsan_mutex_recursive_lock = 1 << 6;
+static const unsigned __tsan_mutex_recursive_lock = 1 << 6;
// Denotes that the unlock operation releases all recursion levels.
// Number of released levels is returned and later must be passed to
// the corresponding __tsan_mutex_post_lock annotation.
-const unsigned __tsan_mutex_recursive_unlock = 1 << 7;
+static const unsigned __tsan_mutex_recursive_unlock = 1 << 7;
// Annotate creation of a mutex.
// Supported flags: mutex creation flags.
@@ -152,7 +152,7 @@ void __tsan_set_fiber_name(void *fiber, const char *name);
// Flags for __tsan_switch_to_fiber:
// Do not establish a happens-before relation between fibers
-const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0;
+static const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0;
#ifdef __cplusplus
} // extern "C"
diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp
index 65c51fbafdd0c..cd97b37652f8b 100644
--- a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp
@@ -15,20 +15,21 @@
//===----------------------------------------------------------------------===//
#include "asan_allocator.h"
+
#include "asan_mapping.h"
#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "asan_thread.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_list.h"
-#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_quarantine.h"
-#include "lsan/lsan_common.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
namespace __asan {
@@ -50,6 +51,22 @@ static u32 RZSize2Log(u32 rz_size) {
static AsanAllocator &get_allocator();
+static void AtomicContextStore(volatile atomic_uint64_t *atomic_context,
+ u32 tid, u32 stack) {
+ u64 context = tid;
+ context <<= 32;
+ context += stack;
+ atomic_store(atomic_context, context, memory_order_relaxed);
+}
+
+static void AtomicContextLoad(const volatile atomic_uint64_t *atomic_context,
+ u32 &tid, u32 &stack) {
+ u64 context = atomic_load(atomic_context, memory_order_relaxed);
+ stack = context;
+ context >>= 32;
+ tid = context;
+}
+
// The memory chunk allocated from the underlying allocator looks like this:
// L L L L L L H H U U U U U U R R
// L -- left redzone words (0 or more bytes)
@@ -67,32 +84,59 @@ static AsanAllocator &get_allocator();
// ---------------------|
// M -- magic value kAllocBegMagic
// B -- address of ChunkHeader pointing to the first 'H'
-static const uptr kAllocBegMagic = 0xCC6E96B9;
-
-struct ChunkHeader {
- // 1-st 8 bytes.
- u32 chunk_state : 8; // Must be first.
- u32 alloc_tid : 24;
-
- u32 free_tid : 24;
- u32 from_memalign : 1;
- u32 alloc_type : 2;
- u32 rz_log : 3;
- u32 lsan_tag : 2;
- // 2-nd 8 bytes
- // This field is used for small sizes. For large sizes it is equal to
- // SizeClassMap::kMaxSize and the actual size is stored in the
- // SecondaryAllocator's metadata.
- u32 user_requested_size : 29;
+
+class ChunkHeader {
+ public:
+ atomic_uint8_t chunk_state;
+ u8 alloc_type : 2;
+ u8 lsan_tag : 2;
+
// align < 8 -> 0
// else -> log2(min(align, 512)) - 2
- u32 user_requested_alignment_log : 3;
- u32 alloc_context_id;
+ u8 user_requested_alignment_log : 3;
+
+ private:
+ u16 user_requested_size_hi;
+ u32 user_requested_size_lo;
+ atomic_uint64_t alloc_context_id;
+
+ public:
+ uptr UsedSize() const {
+ uptr R = user_requested_size_lo;
+ if (sizeof(uptr) > sizeof(user_requested_size_lo))
+ R += (uptr)user_requested_size_hi << (8 * sizeof(user_requested_size_lo));
+ return R;
+ }
+
+ void SetUsedSize(uptr size) {
+ user_requested_size_lo = size;
+ if (sizeof(uptr) > sizeof(user_requested_size_lo)) {
+ size >>= (8 * sizeof(user_requested_size_lo));
+ user_requested_size_hi = size;
+ CHECK_EQ(user_requested_size_hi, size);
+ }
+ }
+
+ void SetAllocContext(u32 tid, u32 stack) {
+ AtomicContextStore(&alloc_context_id, tid, stack);
+ }
+
+ void GetAllocContext(u32 &tid, u32 &stack) const {
+ AtomicContextLoad(&alloc_context_id, tid, stack);
+ }
};
-struct ChunkBase : ChunkHeader {
- // Header2, intersects with user memory.
- u32 free_context_id;
+class ChunkBase : public ChunkHeader {
+ atomic_uint64_t free_context_id;
+
+ public:
+ void SetFreeContext(u32 tid, u32 stack) {
+ AtomicContextStore(&free_context_id, tid, stack);
+ }
+
+ void GetFreeContext(u32 &tid, u32 &stack) const {
+ AtomicContextLoad(&free_context_id, tid, stack);
+ }
};
static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
@@ -100,35 +144,50 @@ static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize;
COMPILER_CHECK(kChunkHeaderSize == 16);
COMPILER_CHECK(kChunkHeader2Size <= 16);
-// Every chunk of memory allocated by this allocator can be in one of 3 states:
-// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated.
-// CHUNK_ALLOCATED: the chunk is allocated and not yet freed.
-// CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone.
enum {
- CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it.
- CHUNK_ALLOCATED = 2,
- CHUNK_QUARANTINE = 3
+ // Either just allocated by underlying allocator, but AsanChunk is not yet
+ // ready, or almost returned to undelying allocator and AsanChunk is already
+ // meaningless.
+ CHUNK_INVALID = 0,
+ // The chunk is allocated and not yet freed.
+ CHUNK_ALLOCATED = 2,
+ // The chunk was freed and put into quarantine zone.
+ CHUNK_QUARANTINE = 3,
};
-struct AsanChunk: ChunkBase {
+class AsanChunk : public ChunkBase {
+ public:
uptr Beg() { return reinterpret_cast(this) + kChunkHeaderSize; }
- uptr UsedSize(bool locked_version = false) {
- if (user_requested_size != SizeClassMap::kMaxSize)
- return user_requested_size;
- return *reinterpret_cast(
- get_allocator().GetMetaData(AllocBeg(locked_version)));
+ bool AddrIsInside(uptr addr) {
+ return (addr >= Beg()) && (addr < Beg() + UsedSize());
}
- void *AllocBeg(bool locked_version = false) {
- if (from_memalign) {
- if (locked_version)
- return get_allocator().GetBlockBeginFastLocked(
- reinterpret_cast(this));
- return get_allocator().GetBlockBegin(reinterpret_cast(this));
- }
- return reinterpret_cast(Beg() - RZLog2Size(rz_log));
+};
+
+class LargeChunkHeader {
+ static constexpr uptr kAllocBegMagic =
+ FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
+ atomic_uintptr_t magic;
+ AsanChunk *chunk_header;
+
+ public:
+ AsanChunk *Get() const {
+ return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic
+ ? chunk_header
+ : nullptr;
}
- bool AddrIsInside(uptr addr, bool locked_version = false) {
- return (addr >= Beg()) && (addr < Beg() + UsedSize(locked_version));
+
+ void Set(AsanChunk *p) {
+ if (p) {
+ chunk_header = p;
+ atomic_store(&magic, kAllocBegMagic, memory_order_release);
+ return;
+ }
+
+ uptr old = kAllocBegMagic;
+ if (!atomic_compare_exchange_strong(&magic, &old, 0,
+ memory_order_release)) {
+ CHECK_EQ(old, kAllocBegMagic);
+ }
}
};
@@ -139,23 +198,23 @@ struct QuarantineCallback {
}
void Recycle(AsanChunk *m) {
- CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE);
- atomic_store((atomic_uint8_t*)m, CHUNK_AVAILABLE, memory_order_relaxed);
- CHECK_NE(m->alloc_tid, kInvalidTid);
- CHECK_NE(m->free_tid, kInvalidTid);
- PoisonShadow(m->Beg(),
- RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
- kAsanHeapLeftRedzoneMagic);
- void *p = reinterpret_cast(m->AllocBeg());
+ void *p = get_allocator().GetBlockBegin(m);
if (p != m) {
- uptr *alloc_magic = reinterpret_cast(p);
- CHECK_EQ(alloc_magic[0], kAllocBegMagic);
// Clear the magic value, as allocator internals may overwrite the
// contents of deallocated chunk, confusing GetAsanChunk lookup.
- alloc_magic[0] = 0;
- CHECK_EQ(alloc_magic[1], reinterpret_cast(m));
+ reinterpret_cast(p)->Set(nullptr);
}
+ u8 old_chunk_state = CHUNK_QUARANTINE;
+ if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state,
+ CHUNK_INVALID, memory_order_acquire)) {
+ CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
+ }
+
+ PoisonShadow(m->Beg(),
+ RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
+ kAsanHeapLeftRedzoneMagic);
+
// Statistics.
AsanStats &thread_stats = GetCurrentThreadStats();
thread_stats.real_frees++;
@@ -299,23 +358,26 @@ struct Allocator {
// This could be a user-facing chunk (with redzones), or some internal
// housekeeping chunk, like TransferBatch. Start by assuming the former.
AsanChunk *ac = GetAsanChunk((void *)chunk);
- uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)ac);
- uptr beg = ac->Beg();
- uptr end = ac->Beg() + ac->UsedSize(true);
- uptr chunk_end = chunk + allocated_size;
- if (chunk < beg && beg < end && end <= chunk_end &&
- ac->chunk_state == CHUNK_ALLOCATED) {
- // Looks like a valid AsanChunk in use, poison redzones only.
- PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
- uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
- FastPoisonShadowPartialRightRedzone(
- end_aligned_down, end - end_aligned_down,
- chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
- } else {
- // This is either not an AsanChunk or freed or quarantined AsanChunk.
- // In either case, poison everything.
- PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
+ uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)chunk);
+ if (ac && atomic_load(&ac->chunk_state, memory_order_acquire) ==
+ CHUNK_ALLOCATED) {
+ uptr beg = ac->Beg();
+ uptr end = ac->Beg() + ac->UsedSize();
+ uptr chunk_end = chunk + allocated_size;
+ if (chunk < beg && beg < end && end <= chunk_end) {
+ // Looks like a valid AsanChunk in use, poison redzones only.
+ PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
+ uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
+ FastPoisonShadowPartialRightRedzone(
+ end_aligned_down, end - end_aligned_down,
+ chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
+ return;
+ }
}
+
+ // This is either not an AsanChunk or freed or quarantined AsanChunk.
+ // In either case, poison everything.
+ PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
}
void ReInitialize(const AllocatorOptions &options) {
@@ -348,17 +410,18 @@ struct Allocator {
// -------------------- Helper methods. -------------------------
uptr ComputeRZLog(uptr user_requested_size) {
- u32 rz_log =
- user_requested_size <= 64 - 16 ? 0 :
- user_requested_size <= 128 - 32 ? 1 :
- user_requested_size <= 512 - 64 ? 2 :
- user_requested_size <= 4096 - 128 ? 3 :
- user_requested_size <= (1 << 14) - 256 ? 4 :
- user_requested_size <= (1 << 15) - 512 ? 5 :
- user_requested_size <= (1 << 16) - 1024 ? 6 : 7;
- u32 min_rz = atomic_load(&min_redzone, memory_order_acquire);
- u32 max_rz = atomic_load(&max_redzone, memory_order_acquire);
- return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz));
+ u32 rz_log = user_requested_size <= 64 - 16 ? 0
+ : user_requested_size <= 128 - 32 ? 1
+ : user_requested_size <= 512 - 64 ? 2
+ : user_requested_size <= 4096 - 128 ? 3
+ : user_requested_size <= (1 << 14) - 256 ? 4
+ : user_requested_size <= (1 << 15) - 512 ? 5
+ : user_requested_size <= (1 << 16) - 1024 ? 6
+ : 7;
+ u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader)));
+ u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire));
+ u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire));
+ return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log));
}
static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) {
@@ -378,16 +441,23 @@ struct Allocator {
// We have an address between two chunks, and we want to report just one.
AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk,
AsanChunk *right_chunk) {
+ if (!left_chunk)
+ return right_chunk;
+ if (!right_chunk)
+ return left_chunk;
// Prefer an allocated chunk over freed chunk and freed chunk
// over available chunk.
- if (left_chunk->chunk_state != right_chunk->chunk_state) {
- if (left_chunk->chunk_state == CHUNK_ALLOCATED)
+ u8 left_state = atomic_load(&left_chunk->chunk_state, memory_order_relaxed);
+ u8 right_state =
+ atomic_load(&right_chunk->chunk_state, memory_order_relaxed);
+ if (left_state != right_state) {
+ if (left_state == CHUNK_ALLOCATED)
return left_chunk;
- if (right_chunk->chunk_state == CHUNK_ALLOCATED)
+ if (right_state == CHUNK_ALLOCATED)
return right_chunk;
- if (left_chunk->chunk_state == CHUNK_QUARANTINE)
+ if (left_state == CHUNK_QUARANTINE)
return left_chunk;
- if (right_chunk->chunk_state == CHUNK_QUARANTINE)
+ if (right_state == CHUNK_QUARANTINE)
return right_chunk;
}
// Same chunk_state: choose based on offset.
@@ -402,10 +472,11 @@ struct Allocator {
bool UpdateAllocationStack(uptr addr, BufferedStackTrace *stack) {
AsanChunk *m = GetAsanChunkByAddr(addr);
if (!m) return false;
- if (m->chunk_state != CHUNK_ALLOCATED) return false;
+ if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED)
+ return false;
if (m->Beg() != addr) return false;
- atomic_store((atomic_uint32_t *)&m->alloc_context_id, StackDepotPut(*stack),
- memory_order_relaxed);
+ AsanThread *t = GetCurrentThread();
+ m->SetAllocContext(t ? t->tid() : 0, StackDepotPut(*stack));
return true;
}
@@ -442,13 +513,10 @@ struct Allocator {
uptr needed_size = rounded_size + rz_size;
if (alignment > min_alignment)
needed_size += alignment;
- bool using_primary_allocator = true;
// If we are allocating from the secondary allocator, there will be no
// automatic right redzone, so add the right redzone manually.
- if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) {
+ if (!PrimaryAllocator::CanAllocate(needed_size, alignment))
needed_size += rz_size;
- using_primary_allocator = false;
- }
CHECK(IsAligned(needed_size, min_alignment));
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
size > max_user_defined_malloc_size) {
@@ -490,8 +558,7 @@ struct Allocator {
uptr alloc_beg = reinterpret_cast(allocated);
uptr alloc_end = alloc_beg + needed_size;
- uptr beg_plus_redzone = alloc_beg + rz_size;
- uptr user_beg = beg_plus_redzone;
+ uptr user_beg = alloc_beg + rz_size;
if (!IsAligned(user_beg, alignment))
user_beg = RoundUpTo(user_beg, alignment);
uptr user_end = user_beg + size;
@@ -499,31 +566,11 @@ struct Allocator {
uptr chunk_beg = user_beg - kChunkHeaderSize;
AsanChunk *m = reinterpret_cast(chunk_beg);
m->alloc_type = alloc_type;
- m->rz_log = rz_log;
- u32 alloc_tid = t ? t->tid() : 0;
- m->alloc_tid = alloc_tid;
- CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield?
- m->free_tid = kInvalidTid;
- m->from_memalign = user_beg != beg_plus_redzone;
- if (alloc_beg != chunk_beg) {
- CHECK_LE(alloc_beg+ 2 * sizeof(uptr), chunk_beg);
- reinterpret_cast(alloc_beg)[0] = kAllocBegMagic;
- reinterpret_cast(alloc_beg)[1] = chunk_beg;
- }
- if (using_primary_allocator) {
- CHECK(size);
- m->user_requested_size = size;
- CHECK(allocator.FromPrimary(allocated));
- } else {
- CHECK(!allocator.FromPrimary(allocated));
- m->user_requested_size = SizeClassMap::kMaxSize;
- uptr *meta = reinterpret_cast(allocator.GetMetaData(allocated));
- meta[0] = size;
- meta[1] = chunk_beg;
- }
+ CHECK(size);
+ m->SetUsedSize(size);
m->user_requested_alignment_log = user_requested_alignment_log;
- m->alloc_context_id = StackDepotPut(*stack);
+ m->SetAllocContext(t ? t->tid() : 0, StackDepotPut(*stack));
uptr size_rounded_down_to_granularity =
RoundDownTo(size, SHADOW_GRANULARITY);
@@ -556,7 +603,11 @@ struct Allocator {
: __lsan::kDirectlyLeaked;
#endif
// Must be the last mutation of metadata in this function.
- atomic_store((atomic_uint8_t *)m, CHUNK_ALLOCATED, memory_order_release);
+ atomic_store(&m->chunk_state, CHUNK_ALLOCATED, memory_order_release);
+ if (alloc_beg != chunk_beg) {
+ CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
+ reinterpret_cast(alloc_beg)->Set(m);
+ }
ASAN_MALLOC_HOOK(res, size);
return res;
}
@@ -564,10 +615,10 @@ struct Allocator {
// Set quarantine flag if chunk is allocated, issue ASan error report on
// available and quarantined chunks. Return true on success, false otherwise.
bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk *m, void *ptr,
- BufferedStackTrace *stack) {
+ BufferedStackTrace *stack) {
u8 old_chunk_state = CHUNK_ALLOCATED;
// Flip the chunk_state atomically to avoid race on double-free.
- if (!atomic_compare_exchange_strong((atomic_uint8_t *)m, &old_chunk_state,
+ if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state,
CHUNK_QUARANTINE,
memory_order_acquire)) {
ReportInvalidFree(ptr, old_chunk_state, stack);
@@ -575,19 +626,18 @@ struct Allocator {
return false;
}
CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state);
+ // It was a user data.
+ m->SetFreeContext(kInvalidTid, 0);
return true;
}
// Expects the chunk to already be marked as quarantined by using
// AtomicallySetQuarantineFlagIfAllocated.
void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) {
- CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE);
- CHECK_GE(m->alloc_tid, 0);
- if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area.
- CHECK_EQ(m->free_tid, kInvalidTid);
+ CHECK_EQ(atomic_load(&m->chunk_state, memory_order_relaxed),
+ CHUNK_QUARANTINE);
AsanThread *t = GetCurrentThread();
- m->free_tid = t ? t->tid() : 0;
- m->free_context_id = StackDepotPut(*stack);
+ m->SetFreeContext(t ? t->tid() : 0, StackDepotPut(*stack));
Flags &fl = *flags();
if (fl.max_free_fill_size > 0) {
@@ -676,7 +726,7 @@ struct Allocator {
void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true);
if (new_ptr) {
- u8 chunk_state = m->chunk_state;
+ u8 chunk_state = atomic_load(&m->chunk_state, memory_order_acquire);
if (chunk_state != CHUNK_ALLOCATED)
ReportInvalidFree(old_ptr, chunk_state, stack);
CHECK_NE(REAL(memcpy), nullptr);
@@ -719,17 +769,24 @@ struct Allocator {
// -------------------------- Chunk lookup ----------------------
// Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
+ // Returns nullptr if AsanChunk is not yet initialized just after
+ // get_allocator().Allocate(), or is being destroyed just before
+ // get_allocator().Deallocate().
AsanChunk *GetAsanChunk(void *alloc_beg) {
- if (!alloc_beg) return nullptr;
- if (!allocator.FromPrimary(alloc_beg)) {
- uptr *meta = reinterpret_cast(allocator.GetMetaData(alloc_beg));
- AsanChunk *m = reinterpret_cast(meta[1]);
- return m;
+ if (!alloc_beg)
+ return nullptr;
+ AsanChunk *p = reinterpret_cast(alloc_beg)->Get();
+ if (!p) {
+ if (!allocator.FromPrimary(alloc_beg))
+ return nullptr;
+ p = reinterpret_cast(alloc_beg);
}
- uptr *alloc_magic = reinterpret_cast(alloc_beg);
- if (alloc_magic[0] == kAllocBegMagic)
- return reinterpret_cast(alloc_magic[1]);
- return reinterpret_cast(alloc_beg);
+ u8 state = atomic_load(&p->chunk_state, memory_order_relaxed);
+ // It does not guaranty that Chunk is initialized, but it's
+ // definitely not for any other value.
+ if (state == CHUNK_ALLOCATED || state == CHUNK_QUARANTINE)
+ return p;
+ return nullptr;
}
AsanChunk *GetAsanChunkByAddr(uptr p) {
@@ -747,16 +804,16 @@ struct Allocator {
uptr AllocationSize(uptr p) {
AsanChunk *m = GetAsanChunkByAddr(p);
if (!m) return 0;
- if (m->chunk_state != CHUNK_ALLOCATED) return 0;
+ if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED)
+ return 0;
if (m->Beg() != p) return 0;
return m->UsedSize();
}
AsanChunkView FindHeapChunkByAddress(uptr addr) {
AsanChunk *m1 = GetAsanChunkByAddr(addr);
- if (!m1) return AsanChunkView(m1);
sptr offset = 0;
- if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) {
+ if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) {
// The address is in the chunk's left redzone, so maybe it is actually
// a right buffer overflow from the other chunk to the left.
// Search a bit to the left to see if there is another chunk.
@@ -813,13 +870,16 @@ static AsanAllocator &get_allocator() {
}
bool AsanChunkView::IsValid() const {
- return chunk_ && chunk_->chunk_state != CHUNK_AVAILABLE;
+ return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) !=
+ CHUNK_INVALID;
}
bool AsanChunkView::IsAllocated() const {
- return chunk_ && chunk_->chunk_state == CHUNK_ALLOCATED;
+ return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) ==
+ CHUNK_ALLOCATED;
}
bool AsanChunkView::IsQuarantined() const {
- return chunk_ && chunk_->chunk_state == CHUNK_QUARANTINE;
+ return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) ==
+ CHUNK_QUARANTINE;
}
uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
@@ -827,8 +887,23 @@ uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); }
u32 AsanChunkView::UserRequestedAlignment() const {
return Allocator::ComputeUserAlignment(chunk_->user_requested_alignment_log);
}
-uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; }
-uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; }
+
+uptr AsanChunkView::AllocTid() const {
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetAllocContext(tid, stack);
+ return tid;
+}
+
+uptr AsanChunkView::FreeTid() const {
+ if (!IsQuarantined())
+ return kInvalidTid;
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetFreeContext(tid, stack);
+ return tid;
+}
+
AllocType AsanChunkView::GetAllocType() const {
return (AllocType)chunk_->alloc_type;
}
@@ -840,8 +915,21 @@ static StackTrace GetStackTraceFromId(u32 id) {
return res;
}
-u32 AsanChunkView::GetAllocStackId() const { return chunk_->alloc_context_id; }
-u32 AsanChunkView::GetFreeStackId() const { return chunk_->free_context_id; }
+u32 AsanChunkView::GetAllocStackId() const {
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetAllocContext(tid, stack);
+ return stack;
+}
+
+u32 AsanChunkView::GetFreeStackId() const {
+ if (!IsQuarantined())
+ return 0;
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetFreeContext(tid, stack);
+ return stack;
+}
StackTrace AsanChunkView::GetAllocStack() const {
return GetStackTraceFromId(GetAllocStackId());
@@ -1005,7 +1093,7 @@ void AsanSoftRssLimitExceededCallback(bool limit_exceeded) {
instance.SetRssLimitExceeded(limit_exceeded);
}
-} // namespace __asan
+} // namespace __asan
// --- Implementation of LSan-specific functions --- {{{1
namespace __lsan {
@@ -1022,34 +1110,36 @@ void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
*end = *begin + sizeof(__asan::get_allocator());
}
-uptr PointsIntoChunk(void* p) {
+uptr PointsIntoChunk(void *p) {
uptr addr = reinterpret_cast(p);
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(addr);
- if (!m) return 0;
- uptr chunk = m->Beg();
- if (m->chunk_state != __asan::CHUNK_ALLOCATED)
+ if (!m || atomic_load(&m->chunk_state, memory_order_acquire) !=
+ __asan::CHUNK_ALLOCATED)
return 0;
- if (m->AddrIsInside(addr, /*locked_version=*/true))
+ uptr chunk = m->Beg();
+ if (m->AddrIsInside(addr))
return chunk;
- if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(/*locked_version*/ true),
- addr))
+ if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(), addr))
return chunk;
return 0;
}
uptr GetUserBegin(uptr chunk) {
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk);
- CHECK(m);
- return m->Beg();
+ return m ? m->Beg() : 0;
}
LsanMetadata::LsanMetadata(uptr chunk) {
- metadata_ = reinterpret_cast(chunk - __asan::kChunkHeaderSize);
+ metadata_ = chunk ? reinterpret_cast(chunk - __asan::kChunkHeaderSize)
+ : nullptr;
}
bool LsanMetadata::allocated() const {
+ if (!metadata_)
+ return false;
__asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
- return m->chunk_state == __asan::CHUNK_ALLOCATED;
+ return atomic_load(&m->chunk_state, memory_order_relaxed) ==
+ __asan::CHUNK_ALLOCATED;
}
ChunkTag LsanMetadata::tag() const {
@@ -1064,12 +1154,15 @@ void LsanMetadata::set_tag(ChunkTag value) {
uptr LsanMetadata::requested_size() const {
__asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
- return m->UsedSize(/*locked_version=*/true);
+ return m->UsedSize();
}
u32 LsanMetadata::stack_trace_id() const {
__asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
- return m->alloc_context_id;
+ u32 tid = 0;
+ u32 stack = 0;
+ m->GetAllocContext(tid, stack);
+ return stack;
}
void ForEachChunk(ForEachChunkCallback callback, void *arg) {
@@ -1079,16 +1172,45 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) {
IgnoreObjectResult IgnoreObjectLocked(const void *p) {
uptr addr = reinterpret_cast(p);
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr);
- if (!m) return kIgnoreObjectInvalid;
- if ((m->chunk_state == __asan::CHUNK_ALLOCATED) && m->AddrIsInside(addr)) {
- if (m->lsan_tag == kIgnored)
- return kIgnoreObjectAlreadyIgnored;
- m->lsan_tag = __lsan::kIgnored;
- return kIgnoreObjectSuccess;
- } else {
+ if (!m ||
+ (atomic_load(&m->chunk_state, memory_order_acquire) !=
+ __asan::CHUNK_ALLOCATED) ||
+ !m->AddrIsInside(addr)) {
return kIgnoreObjectInvalid;
}
+ if (m->lsan_tag == kIgnored)
+ return kIgnoreObjectAlreadyIgnored;
+ m->lsan_tag = __lsan::kIgnored;
+ return kIgnoreObjectSuccess;
+}
+
+void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
+ // Look for the arg pointer of threads that have been created or are running.
+ // This is necessary to prevent false positive leaks due to the AsanThread
+ // holding the only live reference to a heap object. This can happen because
+ // the `pthread_create()` interceptor doesn't wait for the child thread to
+ // start before returning and thus loosing the the only live reference to the
+ // heap object on the stack.
+
+ __asan::AsanThreadContext *atctx =
+ reinterpret_cast<__asan::AsanThreadContext *>(tctx);
+ __asan::AsanThread *asan_thread = atctx->thread;
+
+ // Note ThreadStatusRunning is required because there is a small window where
+ // the thread status switches to `ThreadStatusRunning` but the `arg` pointer
+ // still isn't on the stack yet.
+ if (atctx->status != ThreadStatusCreated &&
+ atctx->status != ThreadStatusRunning)
+ return;
+
+ uptr thread_arg = reinterpret_cast(asan_thread->get_arg());
+ if (!thread_arg)
+ return;
+
+ auto ptrsVec = reinterpret_cast *>(ptrs);
+ ptrsVec->push_back(thread_arg);
}
+
} // namespace __lsan
// ---------------------- Interface ---------------- {{{1
diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.h b/system/lib/compiler-rt/lib/asan/asan_allocator.h
index b37d8ef4e8d29..2963e979b55c0 100644
--- a/system/lib/compiler-rt/lib/asan/asan_allocator.h
+++ b/system/lib/compiler-rt/lib/asan/asan_allocator.h
@@ -15,10 +15,11 @@
#define ASAN_ALLOCATOR_H
#include "asan_flags.h"
-#include "asan_internal.h"
#include "asan_interceptors.h"
+#include "asan_internal.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_list.h"
+#include "sanitizer_common/sanitizer_platform.h"
namespace __asan {
@@ -28,7 +29,7 @@ enum AllocType {
FROM_NEW_BR = 3 // Memory block came from operator new [ ]
};
-struct AsanChunk;
+class AsanChunk;
struct AllocatorOptions {
u32 quarantine_size_mb;
@@ -132,6 +133,10 @@ typedef DefaultSizeClassMap SizeClassMap;
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
typedef VeryCompactSizeClassMap SizeClassMap;
+#elif SANITIZER_RISCV64
+const uptr kAllocatorSpace = ~(uptr)0;
+const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
+typedef VeryDenseSizeClassMap SizeClassMap;
# elif defined(__aarch64__)
// AArch64/SANITIZER_CAN_USE_ALLOCATOR64 is only for 42-bit VMA
// so no need to different values for different VMA.
@@ -171,7 +176,7 @@ template
struct AP32 {
static const uptr kSpaceBeg = 0;
static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
- static const uptr kMetadataSize = 16;
+ static const uptr kMetadataSize = 0;
typedef __asan::SizeClassMap SizeClassMap;
static const uptr kRegionSizeLog = 20;
using AddressSpaceView = AddressSpaceViewTy;
diff --git a/system/lib/compiler-rt/lib/asan/asan_emscripten.cpp b/system/lib/compiler-rt/lib/asan/asan_emscripten.cpp
index 5c33e7e3b7f50..cb11a93ac754a 100644
--- a/system/lib/compiler-rt/lib/asan/asan_emscripten.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_emscripten.cpp
@@ -33,6 +33,8 @@ void *AsanDoesNotSupportStaticLinkage() {
void InitializeAsanInterceptors() {}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {}
+
// We can use a plain thread_local variable for TSD.
static thread_local void *per_thread;
@@ -65,7 +67,7 @@ static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
internal_sched_yield();
emscripten_builtin_free(param);
SetCurrentThread(t);
- return t->ThreadStart(GetTid(), nullptr);
+ return t->ThreadStart(GetTid());
}
INTERCEPTOR(int, pthread_create, void *thread,
diff --git a/system/lib/compiler-rt/lib/asan/asan_flags.cpp b/system/lib/compiler-rt/lib/asan/asan_flags.cpp
index a09535f8269e0..197dfb23661af 100644
--- a/system/lib/compiler-rt/lib/asan/asan_flags.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_flags.cpp
@@ -32,10 +32,6 @@ namespace __asan {
Flags asan_flags_dont_use_directly; // use via flags().
-static const char *MaybeCallAsanDefaultOptions() {
- return (&__asan_default_options) ? __asan_default_options() : "";
-}
-
static const char *MaybeUseAsanDefaultOptionsCompileDefinition() {
#ifdef ASAN_DEFAULT_OPTIONS
return SANITIZER_STRINGIFY(ASAN_DEFAULT_OPTIONS);
@@ -118,14 +114,14 @@ void InitializeFlags() {
asan_parser.ParseString(asan_compile_def);
// Override from user-specified string.
- const char *asan_default_options = MaybeCallAsanDefaultOptions();
+ const char *asan_default_options = __asan_default_options();
asan_parser.ParseString(asan_default_options);
#if CAN_SANITIZE_UB
- const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ const char *ubsan_default_options = __ubsan_default_options();
ubsan_parser.ParseString(ubsan_default_options);
#endif
#if CAN_SANITIZE_LEAKS
- const char *lsan_default_options = __lsan::MaybeCallLsanDefaultOptions();
+ const char *lsan_default_options = __lsan_default_options();
lsan_parser.ParseString(lsan_default_options);
#endif
diff --git a/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp b/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp
index f8b2d5f26979d..6c61344f87cfe 100644
--- a/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp
@@ -62,6 +62,8 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
UNIMPLEMENTED();
}
+bool PlatformUnpoisonStacks() { return false; }
+
// We can use a plain thread_local variable for TSD.
static thread_local void *per_thread;
@@ -89,8 +91,7 @@ struct AsanThread::InitOptions {
// Shared setup between thread creation and startup for the initial thread.
static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
uptr user_id, bool detached,
- const char *name, uptr stack_bottom,
- uptr stack_size) {
+ const char *name) {
// In lieu of AsanThread::Create.
AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__);
@@ -99,12 +100,6 @@ static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
asanThreadRegistry().SetThreadName(tid, name);
- // On other systems, AsanThread::Init() is called from the new
- // thread itself. But on Fuchsia we already know the stack address
- // range beforehand, so we can do most of the setup right now.
- const AsanThread::InitOptions options = {stack_bottom, stack_size};
- thread->Init(&options);
-
return thread;
}
@@ -133,9 +128,16 @@ AsanThread *CreateMainThread() {
_zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name,
sizeof(name)) == ZX_OK
? name
- : nullptr,
- __sanitizer::MainThreadStackBase, __sanitizer::MainThreadStackSize);
+ : nullptr);
+ // We need to set the current thread before calling AsanThread::Init() below,
+ // since it reads the thread ID.
SetCurrentThread(t);
+ DCHECK_EQ(t->tid(), 0);
+
+ const AsanThread::InitOptions options = {__sanitizer::MainThreadStackBase,
+ __sanitizer::MainThreadStackSize};
+ t->Init(&options);
+
return t;
}
@@ -151,8 +153,15 @@ static void *BeforeThreadCreateHook(uptr user_id, bool detached,
GET_STACK_TRACE_THREAD;
u32 parent_tid = GetCurrentTidOrInvalid();
- return CreateAsanThread(&stack, parent_tid, user_id, detached, name,
- stack_bottom, stack_size);
+ AsanThread *thread =
+ CreateAsanThread(&stack, parent_tid, user_id, detached, name);
+
+ // On other systems, AsanThread::Init() is called from the new
+ // thread itself. But on Fuchsia we already know the stack address
+ // range beforehand, so we can do most of the setup right now.
+ const AsanThread::InitOptions options = {stack_bottom, stack_size};
+ thread->Init(&options);
+ return thread;
}
// This is called after creating a new thread (in the creating thread),
@@ -196,6 +205,10 @@ bool HandleDlopenInit() {
return false;
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ __sanitizer_fill_shadow(p, size, 0, 0);
+}
+
} // namespace __asan
// These are declared (in extern "C") by .
diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp
index 9002c7fe4dbd9..6f124e9770373 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -189,20 +189,11 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
#include "sanitizer_common/sanitizer_common_syscalls.inc"
#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
-struct ThreadStartParam {
- atomic_uintptr_t t;
- atomic_uintptr_t is_registered;
-};
-
#if ASAN_INTERCEPT_PTHREAD_CREATE
static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
- ThreadStartParam *param = reinterpret_cast(arg);
- AsanThread *t = nullptr;
- while ((t = reinterpret_cast(
- atomic_load(¶m->t, memory_order_acquire))) == nullptr)
- internal_sched_yield();
+ AsanThread *t = (AsanThread *)arg;
SetCurrentThread(t);
- return t->ThreadStart(GetTid(), ¶m->is_registered);
+ return t->ThreadStart(GetTid());
}
INTERCEPTOR(int, pthread_create, void *thread,
@@ -215,9 +206,11 @@ INTERCEPTOR(int, pthread_create, void *thread,
int detached = 0;
if (attr)
REAL(pthread_attr_getdetachstate)(attr, &detached);
- ThreadStartParam param;
- atomic_store(¶m.t, 0, memory_order_relaxed);
- atomic_store(¶m.is_registered, 0, memory_order_relaxed);
+
+ u32 current_tid = GetCurrentTidOrInvalid();
+ AsanThread *t =
+ AsanThread::Create(start_routine, arg, current_tid, &stack, detached);
+
int result;
{
// Ignore all allocations made by pthread_create: thread stack/TLS may be
@@ -227,21 +220,13 @@ INTERCEPTOR(int, pthread_create, void *thread,
#if CAN_SANITIZE_LEAKS
__lsan::ScopedInterceptorDisabler disabler;
#endif
- result = REAL(pthread_create)(thread, attr, asan_thread_start, ¶m);
+ result = REAL(pthread_create)(thread, attr, asan_thread_start, t);
}
- if (result == 0) {
- u32 current_tid = GetCurrentTidOrInvalid();
- AsanThread *t =
- AsanThread::Create(start_routine, arg, current_tid, &stack, detached);
- atomic_store(¶m.t, reinterpret_cast(t), memory_order_release);
- // Wait until the AsanThread object is initialized and the ThreadRegistry
- // entry is in "started" state. One reason for this is that after this
- // interceptor exits, the child thread's stack may be the only thing holding
- // the |arg| pointer. This may cause LSan to report a leak if leak checking
- // happens at a point when the interceptor has already exited, but the stack
- // range for the child thread is not yet known.
- while (atomic_load(¶m.is_registered, memory_order_acquire) == 0)
- internal_sched_yield();
+ if (result != 0) {
+ // If the thread didn't start delete the AsanThread to avoid leaking it.
+ // Note AsanThreadContexts never get destroyed so the AsanThreadContext
+ // that was just created for the AsanThread is wasted.
+ t->Destroy();
}
return result;
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.h b/system/lib/compiler-rt/lib/asan/asan_interceptors.h
index 344a64bd83d33..45cdb80b1b640 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interceptors.h
+++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.h
@@ -13,9 +13,10 @@
#ifndef ASAN_INTERCEPTORS_H
#define ASAN_INTERCEPTORS_H
-#include "asan_internal.h"
#include "asan_interceptors_memintrinsics.h"
+#include "asan_internal.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_platform.h"
#include "sanitizer_common/sanitizer_platform_interceptors.h"
namespace __asan {
@@ -59,7 +60,7 @@ void InitializePlatformInterceptors();
# define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 0
#endif
-#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_SOLARIS
+#if SANITIZER_GLIBC || SANITIZER_SOLARIS
# define ASAN_INTERCEPT_SWAPCONTEXT 1
#else
# define ASAN_INTERCEPT_SWAPCONTEXT 0
@@ -71,7 +72,7 @@ void InitializePlatformInterceptors();
# define ASAN_INTERCEPT_SIGLONGJMP 0
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
# define ASAN_INTERCEPT___LONGJMP_CHK 1
#else
# define ASAN_INTERCEPT___LONGJMP_CHK 0
@@ -105,14 +106,15 @@ void InitializePlatformInterceptors();
# define ASAN_INTERCEPT_ATEXIT 0
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
# define ASAN_INTERCEPT___STRDUP 1
#else
# define ASAN_INTERCEPT___STRDUP 0
#endif
-#if SANITIZER_LINUX && (defined(__arm__) || defined(__aarch64__) || \
- defined(__i386__) || defined(__x86_64__))
+#if SANITIZER_LINUX && \
+ (defined(__arm__) || defined(__aarch64__) || defined(__i386__) || \
+ defined(__x86_64__) || SANITIZER_RISCV64)
# define ASAN_INTERCEPT_VFORK 1
#else
# define ASAN_INTERCEPT_VFORK 0
@@ -132,10 +134,10 @@ DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen)
DECLARE_REAL(char*, strstr, const char *s1, const char *s2)
#if !SANITIZER_MAC
-#define ASAN_INTERCEPT_FUNC(name) \
- do { \
- if (!INTERCEPT_FUNCTION(name)) \
- VReport(1, "AddressSanitizer: failed to intercept '%s'\n'", #name); \
+#define ASAN_INTERCEPT_FUNC(name) \
+ do { \
+ if (!INTERCEPT_FUNCTION(name)) \
+ VReport(1, "AddressSanitizer: failed to intercept '%s'\n", #name); \
} while (0)
#define ASAN_INTERCEPT_FUNC_VER(name, ver) \
do { \
diff --git a/system/lib/compiler-rt/lib/asan/asan_interface_internal.h b/system/lib/compiler-rt/lib/asan/asan_interface_internal.h
index f14cbbcb76a35..3e6e660288746 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interface_internal.h
+++ b/system/lib/compiler-rt/lib/asan/asan_interface_internal.h
@@ -173,8 +173,8 @@ extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE void __asan_print_accumulated_stats();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- const char* __asan_default_options();
+ SANITIZER_INTERFACE_ATTRIBUTE
+ const char *__asan_default_options();
SANITIZER_INTERFACE_ATTRIBUTE
extern uptr __asan_shadow_memory_dynamic_address;
diff --git a/system/lib/compiler-rt/lib/asan/asan_internal.h b/system/lib/compiler-rt/lib/asan/asan_internal.h
index 72a4c3f22ff1c..cfb54927c6cf4 100644
--- a/system/lib/compiler-rt/lib/asan/asan_internal.h
+++ b/system/lib/compiler-rt/lib/asan/asan_internal.h
@@ -83,6 +83,16 @@ void *AsanDoesNotSupportStaticLinkage();
void AsanCheckDynamicRTPrereqs();
void AsanCheckIncompatibleRT();
+// Unpoisons platform-specific stacks.
+// Returns true if all stacks have been unpoisoned.
+bool PlatformUnpoisonStacks();
+
+// asan_rtl.cpp
+// Unpoison a region containing a stack.
+// Performs a sanity check and warns if the bounds don't look right.
+// The warning contains the type string to identify the stack type.
+void UnpoisonStack(uptr bottom, uptr top, const char *type);
+
// asan_thread.cpp
AsanThread *CreateMainThread();
@@ -108,8 +118,6 @@ void AppendToErrorMessageBuffer(const char *buffer);
void *AsanDlSymNext(const char *sym);
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name);
-
// Returns `true` iff most of ASan init process should be skipped due to the
// ASan library being loaded via `dlopen()`. Platforms may perform any
// `dlopen()` specific initialization inside this function.
diff --git a/system/lib/compiler-rt/lib/asan/asan_linux.cpp b/system/lib/compiler-rt/lib/asan/asan_linux.cpp
index ce5e873dc5180..4bcbe5d02e334 100644
--- a/system/lib/compiler-rt/lib/asan/asan_linux.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_linux.cpp
@@ -55,6 +55,7 @@ extern Elf_Dyn _DYNAMIC;
#else
#include
#include
+extern ElfW(Dyn) _DYNAMIC[];
#endif
// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
@@ -84,28 +85,15 @@ bool IsSystemHeapAddress (uptr addr) { return false; }
void *AsanDoesNotSupportStaticLinkage() {
// This will fail to link with -static.
- return &_DYNAMIC; // defined in link.h
-}
-
-static void UnmapFromTo(uptr from, uptr to) {
- CHECK(to >= from);
- if (to == from) return;
- uptr res = internal_munmap(reinterpret_cast(from), to - from);
- if (UNLIKELY(internal_iserror(res))) {
- Report(
- "ERROR: AddresSanitizer failed to unmap 0x%zx (%zd) bytes at address "
- "%p\n",
- to - from, to - from, from);
- CHECK("unable to unmap" && 0);
- }
+ return &_DYNAMIC;
}
#if ASAN_PREMAP_SHADOW
-uptr FindPremappedShadowStart() {
+uptr FindPremappedShadowStart(uptr shadow_size_bytes) {
uptr granularity = GetMmapGranularity();
uptr shadow_start = reinterpret_cast(&__asan_shadow);
uptr premap_shadow_size = PremapShadowSize();
- uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity);
+ uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
// We may have mapped too much. Release extra memory.
UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
return shadow_start;
@@ -113,31 +101,26 @@ uptr FindPremappedShadowStart() {
#endif
uptr FindDynamicShadowStart() {
+ uptr shadow_size_bytes = MemToShadowSize(kHighMemEnd);
#if ASAN_PREMAP_SHADOW
if (!PremapShadowFailed())
- return FindPremappedShadowStart();
+ return FindPremappedShadowStart(shadow_size_bytes);
#endif
- uptr granularity = GetMmapGranularity();
- uptr alignment = granularity * 8;
- uptr left_padding = granularity;
- uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity);
- uptr map_size = shadow_size + left_padding + alignment;
-
- uptr map_start = (uptr)MmapNoAccess(map_size);
- CHECK_NE(map_start, ~(uptr)0);
-
- uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
- UnmapFromTo(map_start, shadow_start - left_padding);
- UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
-
- return shadow_start;
+ return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
UNIMPLEMENTED();
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
#if SANITIZER_ANDROID
// FIXME: should we do anything for Android?
void AsanCheckDynamicRTPrereqs() {}
diff --git a/system/lib/compiler-rt/lib/asan/asan_mac.cpp b/system/lib/compiler-rt/lib/asan/asan_mac.cpp
index a8d3f5d3473c4..c6950547f089f 100644
--- a/system/lib/compiler-rt/lib/asan/asan_mac.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_mac.cpp
@@ -55,46 +55,8 @@ void *AsanDoesNotSupportStaticLinkage() {
}
uptr FindDynamicShadowStart() {
- uptr granularity = GetMmapGranularity();
- uptr alignment = 8 * granularity;
- uptr left_padding = granularity;
- uptr space_size = kHighShadowEnd + left_padding;
-
- uptr largest_gap_found = 0;
- uptr max_occupied_addr = 0;
- VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
- uptr shadow_start =
- FindAvailableMemoryRange(space_size, alignment, granularity,
- &largest_gap_found, &max_occupied_addr);
- // If the shadow doesn't fit, restrict the address space to make it fit.
- if (shadow_start == 0) {
- VReport(
- 2,
- "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
- largest_gap_found, max_occupied_addr);
- uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
- if (new_max_vm < max_occupied_addr) {
- Report("Unable to find a memory range for dynamic shadow.\n");
- Report(
- "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
- "new_max_vm = %p\n",
- space_size, largest_gap_found, max_occupied_addr, new_max_vm);
- CHECK(0 && "cannot place shadow");
- }
- RestrictMemoryToMaxAddress(new_max_vm);
- kHighMemEnd = new_max_vm - 1;
- space_size = kHighShadowEnd + left_padding;
- VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
- shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
- nullptr, nullptr);
- if (shadow_start == 0) {
- Report("Unable to find a memory range after restricting VM.\n");
- CHECK(0 && "cannot place shadow after restricting vm");
- }
- }
- CHECK_NE((uptr)0, shadow_start);
- CHECK(IsAligned(shadow_start, alignment));
- return shadow_start;
+ return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
// No-op. Mac does not support static linkage anyway.
@@ -127,6 +89,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
op(globals, size / sizeof(__asan_global));
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
UNIMPLEMENTED();
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp b/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 20719cd51f959..7d7b3631a187c 100644
--- a/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -35,7 +35,7 @@ static uptr last_dlsym_alloc_size_in_words;
static const uptr kDlsymAllocPoolSize = SANITIZER_RTEMS ? 4096 : 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-static INLINE bool IsInDlsymAllocPool(const void *ptr) {
+static inline bool IsInDlsymAllocPool(const void *ptr) {
uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
}
@@ -96,12 +96,12 @@ bool IsFromLocalPool(const void *ptr) {
}
#endif
-static INLINE bool MaybeInDlsym() {
+static inline bool MaybeInDlsym() {
// Fuchsia doesn't use dlsym-based interceptors.
return !SANITIZER_FUCHSIA && asan_init_is_running;
}
-static INLINE bool UseLocalPool() {
+static inline bool UseLocalPool() {
return EarlyMalloc() || MaybeInDlsym();
}
@@ -121,19 +121,19 @@ static void *ReallocFromLocalPool(void *ptr, uptr size) {
}
INTERCEPTOR(void, free, void *ptr) {
- GET_STACK_TRACE_FREE;
if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
DeallocateFromLocalPool(ptr);
return;
}
+ GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
- GET_STACK_TRACE_FREE;
if (UNLIKELY(IsInDlsymAllocPool(ptr)))
return;
+ GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE
diff --git a/system/lib/compiler-rt/lib/asan/asan_malloc_local.h b/system/lib/compiler-rt/lib/asan/asan_malloc_local.h
index 3f784b90c739c..e2c9be0379f2f 100644
--- a/system/lib/compiler-rt/lib/asan/asan_malloc_local.h
+++ b/system/lib/compiler-rt/lib/asan/asan_malloc_local.h
@@ -17,7 +17,7 @@
#include "sanitizer_common/sanitizer_platform.h"
#include "asan_internal.h"
-static INLINE bool EarlyMalloc() {
+static inline bool EarlyMalloc() {
return SANITIZER_RTEMS &&
(!__asan::asan_inited || __asan::asan_init_is_running);
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_mapping.h b/system/lib/compiler-rt/lib/asan/asan_mapping.h
index 67c5448729a3d..aa499376df64f 100644
--- a/system/lib/compiler-rt/lib/asan/asan_mapping.h
+++ b/system/lib/compiler-rt/lib/asan/asan_mapping.h
@@ -79,6 +79,20 @@
// || `[0x1000000000, 0x11ffffffff]` || lowshadow ||
// || `[0x0000000000, 0x0fffffffff]` || lowmem ||
//
+// RISC-V has only 38 bits for task size
+// Low mem size is set with kRiscv64_ShadowOffset64 in
+// compiler-rt/lib/asan/asan_allocator.h and in
+// llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp with
+// kRiscv64_ShadowOffset64, High mem top border is set with
+// GetMaxVirtualAddress() in
+// compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+// Default Linux/RISCV64 Sv39/Sv48 mapping:
+// || `[0x000820000000, 0x003fffffffff]` || HighMem ||
+// || `[0x000124000000, 0x00081fffffff]` || HighShadow ||
+// || `[0x000024000000, 0x000123ffffff]` || ShadowGap ||
+// || `[0x000020000000, 0x000023ffffff]` || LowShadow ||
+// || `[0x000000000000, 0x00001fffffff]` || LowMem ||
+//
// Default Linux/AArch64 (42-bit VMA) mapping:
// || `[0x10000000000, 0x3ffffffffff]` || highmem ||
// || `[0x0a000000000, 0x0ffffffffff]` || highshadow ||
@@ -161,6 +175,7 @@ static const u64 kDefaultShadowOffset64 = 1ULL << 44;
static const u64 kDefaultShort64bitShadowOffset =
0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G.
static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
+static const u64 kRiscv64_ShadowOffset64 = 0x20000000;
static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
@@ -206,6 +221,10 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL;
#else
# if SANITIZER_IOS
# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+# elif SANITIZER_MAC && defined(__aarch64__)
+# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#elif SANITIZER_RISCV64
+#define SHADOW_OFFSET kRiscv64_ShadowOffset64
# elif defined(__aarch64__)
# define SHADOW_OFFSET kAArch64_ShadowOffset64
# elif defined(__powerpc64__)
@@ -355,6 +374,8 @@ static inline bool AddrIsInShadowGap(uptr a) {
namespace __asan {
+static inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; }
+
static inline bool AddrIsInMem(uptr a) {
PROFILE_ASAN_MAPPING();
return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a) ||
diff --git a/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp b/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp
index a3152eab85409..15be546f62b23 100644
--- a/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -62,14 +62,6 @@ struct ShadowSegmentEndpoint {
}
};
-void FlushUnneededASanShadowMemory(uptr p, uptr size) {
- // Since asan's mapping is compacting, the shadow chunk may be
- // not page-aligned, so we only flush the page-aligned portion.
-#if !SANITIZER_EMSCRIPTEN
- ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
-#endif
-}
-
void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) {
uptr end = ptr + size;
if (Verbosity()) {
diff --git a/system/lib/compiler-rt/lib/asan/asan_posix.cpp b/system/lib/compiler-rt/lib/asan/asan_posix.cpp
index 85ebd181ebf2f..6e49520498158 100644
--- a/system/lib/compiler-rt/lib/asan/asan_posix.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_posix.cpp
@@ -17,6 +17,7 @@
#include "asan_internal.h"
#include "asan_interceptors.h"
#include "asan_mapping.h"
+#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "sanitizer_common/sanitizer_libc.h"
@@ -24,6 +25,7 @@
#include "sanitizer_common/sanitizer_procmaps.h"
#include
+#include
#include
#include
#include
@@ -37,6 +39,36 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
ReportDeadlySignal(sig);
}
+bool PlatformUnpoisonStacks() {
+#if SANITIZER_EMSCRIPTEN
+ return false;
+#else
+ stack_t signal_stack;
+ CHECK_EQ(0, sigaltstack(nullptr, &signal_stack));
+ uptr sigalt_bottom = (uptr)signal_stack.ss_sp;
+ uptr sigalt_top = (uptr)((char *)signal_stack.ss_sp + signal_stack.ss_size);
+ // If we're executing on the signal alternate stack AND the Linux flag
+ // SS_AUTODISARM was used, then we cannot get the signal alternate stack
+ // bounds from sigaltstack -- sigaltstack's output looks just as if no
+ // alternate stack has ever been set up.
+ // We're always unpoisoning the signal alternate stack to support jumping
+ // between the default stack and signal alternate stack.
+ if (signal_stack.ss_flags != SS_DISABLE)
+ UnpoisonStack(sigalt_bottom, sigalt_top, "sigalt");
+
+ if (signal_stack.ss_flags != SS_ONSTACK)
+ return false;
+
+ // Since we're on the signal altnerate stack, we cannot find the DEFAULT
+ // stack bottom using a local variable.
+ uptr default_bottom, tls_addr, tls_size, stack_size;
+ GetThreadStackAndTls(/*main=*/false, &default_bottom, &stack_size, &tls_addr,
+ &tls_size);
+ UnpoisonStack(default_bottom, default_bottom + stack_size, "default");
+ return true;
+#endif
+}
+
// ---------------------- TSD ---------------- {{{1
#if SANITIZER_NETBSD && !ASAN_DYNAMIC
diff --git a/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp b/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp
index 7835e99748ffa..666bb9b34bd39 100644
--- a/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp
@@ -32,22 +32,8 @@ uptr PremapShadowSize() {
// Returns an address aligned to 8 pages, such that one page on the left and
// PremapShadowSize() bytes on the right of it are mapped r/o.
uptr PremapShadow() {
- uptr granularity = GetMmapGranularity();
- uptr alignment = granularity * 8;
- uptr left_padding = granularity;
- uptr shadow_size = PremapShadowSize();
- uptr map_size = shadow_size + left_padding + alignment;
-
- uptr map_start = (uptr)MmapNoAccess(map_size);
- CHECK_NE(map_start, ~(uptr)0);
-
- uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
- uptr shadow_end = shadow_start + shadow_size;
- internal_munmap(reinterpret_cast(map_start),
- shadow_start - left_padding - map_start);
- internal_munmap(reinterpret_cast(shadow_end),
- map_start + map_size - shadow_end);
- return shadow_start;
+ return MapDynamicShadow(PremapShadowSize(), /*mmap_alignment_scale*/ 3,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
bool PremapShadowFailed() {
diff --git a/system/lib/compiler-rt/lib/asan/asan_report.cpp b/system/lib/compiler-rt/lib/asan/asan_report.cpp
index 2e6ce436d0306..03f1ed2b01866 100644
--- a/system/lib/compiler-rt/lib/asan/asan_report.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_report.cpp
@@ -151,7 +151,8 @@ class ScopedInErrorReport {
if (common_flags()->print_cmdline)
PrintCmdline();
- if (common_flags()->print_module_map == 2) PrintModuleMap();
+ if (common_flags()->print_module_map == 2)
+ DumpProcessMap();
// Copy the message buffer so that we could start logging without holding a
// lock that gets aquired during printing.
@@ -160,6 +161,9 @@ class ScopedInErrorReport {
BlockingMutexLock l(&error_message_buf_mutex);
internal_memcpy(buffer_copy.data(),
error_message_buffer, kErrorMessageBufferSize);
+ // Clear error_message_buffer so that if we find other errors
+ // we don't re-log this error.
+ error_message_buffer_pos = 0;
}
LogFullErrorReport(buffer_copy.data());
@@ -408,7 +412,7 @@ static bool IsInvalidPointerPair(uptr a1, uptr a2) {
return false;
}
-static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
+static inline void CheckForInvalidPointerPair(void *p1, void *p2) {
switch (flags()->detect_invalid_pointer_pairs) {
case 0:
return;
diff --git a/system/lib/compiler-rt/lib/asan/asan_rtems.cpp b/system/lib/compiler-rt/lib/asan/asan_rtems.cpp
index ecd568c5981bb..ea0b4ad9db681 100644
--- a/system/lib/compiler-rt/lib/asan/asan_rtems.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_rtems.cpp
@@ -50,6 +50,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
UNIMPLEMENTED();
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
void AsanCheckDynamicRTPrereqs() {}
void AsanCheckIncompatibleRT() {}
void InitializeAsanInterceptors() {}
@@ -64,6 +70,8 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
UNIMPLEMENTED();
}
+bool PlatformUnpoisonStacks() { return false; }
+
void EarlyInit() {
// Provide early initialization of shadow memory so that
// instrumented code running before full initialzation will not
diff --git a/system/lib/compiler-rt/lib/asan/asan_rtl.cpp b/system/lib/compiler-rt/lib/asan/asan_rtl.cpp
index 5665654e942fb..3acaa74e3f5ea 100644
--- a/system/lib/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_rtl.cpp
@@ -45,7 +45,8 @@ static void AsanDie() {
// Don't die twice - run a busy loop.
while (1) { }
}
- if (common_flags()->print_module_map >= 1) PrintModuleMap();
+ if (common_flags()->print_module_map >= 1)
+ DumpProcessMap();
if (flags()->sleep_before_dying) {
Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying);
SleepForSeconds(flags()->sleep_before_dying);
@@ -321,7 +322,7 @@ static void InitializeHighMemEnd() {
kHighMemEnd = GetMaxUserVirtualAddress();
// Increase kHighMemEnd to make sure it's properly
// aligned together with kHighMemBeg:
- kHighMemEnd |= SHADOW_GRANULARITY * GetMmapGranularity() - 1;
+ kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
#endif // !ASAN_FIXED_MAPPING
CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
#endif // !SANITIZER_MYRIAD2
@@ -555,22 +556,33 @@ class AsanInitializer {
static AsanInitializer asan_initializer;
#endif // ASAN_DYNAMIC
-} // namespace __asan
-
-// ---------------------- Interface ---------------- {{{1
-using namespace __asan;
-
-void NOINLINE __asan_handle_no_return() {
- if (asan_init_is_running)
+void UnpoisonStack(uptr bottom, uptr top, const char *type) {
+ static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M
+ if (top - bottom > kMaxExpectedCleanupSize) {
+ static bool reported_warning = false;
+ if (reported_warning)
+ return;
+ reported_warning = true;
+ Report(
+ "WARNING: ASan is ignoring requested __asan_handle_no_return: "
+ "stack type: %s top: %p; bottom %p; size: %p (%zd)\n"
+ "False positive error reports may follow\n"
+ "For details see "
+ "https://github.com/google/sanitizers/issues/189\n",
+ type, top, bottom, top - bottom, top - bottom);
return;
+ }
+ PoisonShadow(bottom, top - bottom, 0);
+}
- int local_stack;
- AsanThread *curr_thread = GetCurrentThread();
- uptr PageSize = GetPageSizeCached();
- uptr top, bottom;
- if (curr_thread) {
+static void UnpoisonDefaultStack() {
+ uptr bottom, top;
+
+ if (AsanThread *curr_thread = GetCurrentThread()) {
+ int local_stack;
+ const uptr page_size = GetPageSizeCached();
top = curr_thread->stack_top();
- bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1);
+ bottom = ((uptr)&local_stack - page_size) & ~(page_size - 1);
} else if (SANITIZER_RTEMS) {
// Give up On RTEMS.
return;
@@ -582,25 +594,31 @@ void NOINLINE __asan_handle_no_return() {
&tls_size);
top = bottom + stack_size;
}
- static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M
- if (top - bottom > kMaxExpectedCleanupSize) {
- static bool reported_warning = false;
- if (reported_warning)
- return;
- reported_warning = true;
- Report("WARNING: ASan is ignoring requested __asan_handle_no_return: "
- "stack top: %p; bottom %p; size: %p (%zd)\n"
- "False positive error reports may follow\n"
- "For details see "
- "https://github.com/google/sanitizers/issues/189\n",
- top, bottom, top - bottom, top - bottom);
- return;
- }
- PoisonShadow(bottom, top - bottom, 0);
+
+ UnpoisonStack(bottom, top, "default");
+}
+
+static void UnpoisonFakeStack() {
+ AsanThread *curr_thread = GetCurrentThread();
if (curr_thread && curr_thread->has_fake_stack())
curr_thread->fake_stack()->HandleNoReturn();
}
+} // namespace __asan
+
+// ---------------------- Interface ---------------- {{{1
+using namespace __asan;
+
+void NOINLINE __asan_handle_no_return() {
+ if (asan_init_is_running)
+ return;
+
+ if (!PlatformUnpoisonStacks())
+ UnpoisonDefaultStack();
+
+ UnpoisonFakeStack();
+}
+
extern "C" void *__asan_extra_spill_area() {
AsanThread *t = GetCurrentThread();
CHECK(t);
diff --git a/system/lib/compiler-rt/lib/asan/asan_shadow_setup.cpp b/system/lib/compiler-rt/lib/asan/asan_shadow_setup.cpp
index e17da6a3696a9..9b9e7ae3fe15f 100644
--- a/system/lib/compiler-rt/lib/asan/asan_shadow_setup.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_shadow_setup.cpp
@@ -22,24 +22,6 @@
namespace __asan {
-// ---------------------- mmap -------------------- {{{1
-// Reserve memory range [beg, end].
-// We need to use inclusive range because end+1 may not be representable.
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
- CHECK_EQ((beg % GetMmapGranularity()), 0);
- CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
- uptr size = end - beg + 1;
- DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
- if (!MmapFixedSuperNoReserve(beg, size, name)) {
- Report(
- "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
- "Perhaps you're using ulimit -v\n",
- size);
- Abort();
- }
- if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
-}
-
static void ProtectGap(uptr addr, uptr size) {
if (!flags()->protect_shadow_gap) {
// The shadow gap is unprotected, so there is a chance that someone
@@ -57,30 +39,13 @@ static void ProtectGap(uptr addr, uptr size) {
"unprotected gap shadow");
return;
}
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res) return;
- // A few pages at the start of the address space can not be protected.
- // But we really want to protect as much as possible, to prevent this memory
- // being returned as a result of a non-FIXED mmap().
- if (addr == kZeroBaseShadowStart) {
- uptr step = GetMmapGranularity();
- while (size > step && addr < kZeroBaseMaxShadowStart) {
- addr += step;
- size -= step;
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res) return;
- }
- }
-
- Report(
- "ERROR: Failed to protect the shadow gap. "
- "ASan cannot proceed correctly. ABORTING.\n");
- DumpProcessMap();
- Die();
+ __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
+ kZeroBaseMaxShadowStart);
}
static void MaybeReportLinuxPIEBug() {
-#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__aarch64__))
+#if SANITIZER_LINUX && \
+ (defined(__x86_64__) || defined(__aarch64__) || SANITIZER_RISCV64)
Report("This might be related to ELF_ET_DYN_BASE change in Linux 4.12.\n");
Report(
"See https://github.com/google/sanitizers/issues/856 for possible "
@@ -99,8 +64,6 @@ void InitializeShadowMemory() {
// |kDefaultShadowSentinel|.
bool full_shadow_is_available = false;
if (shadow_start == kDefaultShadowSentinel) {
- __asan_shadow_memory_dynamic_address = 0;
- CHECK_EQ(0, kLowShadowBeg);
shadow_start = FindDynamicShadowStart();
if (SANITIZER_LINUX) full_shadow_is_available = true;
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_stack.h b/system/lib/compiler-rt/lib/asan/asan_stack.h
index 4089d3d7340ee..47ca85a164435 100644
--- a/system/lib/compiler-rt/lib/asan/asan_stack.h
+++ b/system/lib/compiler-rt/lib/asan/asan_stack.h
@@ -51,11 +51,6 @@ u32 GetMallocContextSize();
stack.Unwind(pc, bp, nullptr, \
common_flags()->fast_unwind_on_fatal)
-#define GET_STACK_TRACE_SIGNAL(sig) \
- BufferedStackTrace stack; \
- stack.Unwind((sig).pc, (sig).bp, (sig).context, \
- common_flags()->fast_unwind_on_fatal)
-
#define GET_STACK_TRACE_FATAL_HERE \
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.cpp b/system/lib/compiler-rt/lib/asan/asan_thread.cpp
index b573b7f2817b1..a15571b0e4258 100644
--- a/system/lib/compiler-rt/lib/asan/asan_thread.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_thread.cpp
@@ -190,7 +190,7 @@ uptr AsanThread::stack_size() {
return bounds.top - bounds.bottom;
}
-// We want to create the FakeStack lazyly on the first use, but not eralier
+// We want to create the FakeStack lazily on the first use, but not earlier
// than the stack size is known and the procedure has to be async-signal safe.
FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
uptr stack_size = this->stack_size();
@@ -213,6 +213,7 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
stack_size_log =
Max(stack_size_log, static_cast(flags()->min_uar_stack_size_log));
fake_stack_ = FakeStack::Create(stack_size_log);
+ DCHECK_EQ(GetCurrentThread(), this);
SetTLSFakeStack(fake_stack_);
return fake_stack_;
}
@@ -220,6 +221,7 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
}
void AsanThread::Init(const InitOptions *options) {
+ DCHECK_NE(tid(), ThreadRegistry::kUnknownTid);
next_stack_top_ = next_stack_bottom_ = 0;
atomic_store(&stack_switching_, false, memory_order_release);
CHECK_EQ(this->stack_size(), 0U);
@@ -231,8 +233,17 @@ void AsanThread::Init(const InitOptions *options) {
}
ClearShadowForThreadStackAndTLS();
fake_stack_ = nullptr;
- if (__asan_option_detect_stack_use_after_return)
+ if (__asan_option_detect_stack_use_after_return &&
+ tid() == GetCurrentTidOrInvalid()) {
+ // AsyncSignalSafeLazyInitFakeStack makes use of threadlocals and must be
+ // called from the context of the thread it is initializing, not its parent.
+ // Most platforms call AsanThread::Init on the newly-spawned thread, but
+ // Fuchsia calls this function from the parent thread. To support that
+ // approach, we avoid calling AsyncSignalSafeLazyInitFakeStack here; it will
+ // be called by the new thread when it first attempts to access the fake
+ // stack.
AsyncSignalSafeLazyInitFakeStack();
+ }
int local = 0;
VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
(void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
@@ -244,12 +255,9 @@ void AsanThread::Init(const InitOptions *options) {
// SetThreadStackAndTls.
#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
-thread_return_t AsanThread::ThreadStart(
- tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) {
+thread_return_t AsanThread::ThreadStart(tid_t os_id) {
Init();
asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr);
- if (signal_thread_is_registered)
- atomic_store(signal_thread_is_registered, 1, memory_order_release);
#if !SANITIZER_EMSCRIPTEN
if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
@@ -281,8 +289,7 @@ AsanThread *CreateMainThread() {
/* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0,
/* stack */ nullptr, /* detached */ true);
SetCurrentThread(main_thread);
- main_thread->ThreadStart(internal_getpid(),
- /* signal_thread_is_registered */ nullptr);
+ main_thread->ThreadStart(internal_getpid());
return main_thread;
}
@@ -370,7 +377,9 @@ uptr AsanThread::GetStackVariableShadowStart(uptr addr) {
bottom = stack_bottom();
} else if (has_fake_stack()) {
bottom = fake_stack()->AddrIsInFakeStack(addr);
- CHECK(bottom);
+ if (bottom == 0) {
+ return 0;
+ }
} else {
return 0;
}
@@ -484,6 +493,8 @@ bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
return true;
}
+void GetAllThreadAllocatorCachesLocked(InternalMmapVector *caches) {}
+
void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
void *arg) {
__asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.h b/system/lib/compiler-rt/lib/asan/asan_thread.h
index c503f507059dd..c33955eee367b 100644
--- a/system/lib/compiler-rt/lib/asan/asan_thread.h
+++ b/system/lib/compiler-rt/lib/asan/asan_thread.h
@@ -35,7 +35,7 @@ class AsanThread;
// These objects are created for every thread and are never deleted,
// so we can find them by tid even if the thread is long dead.
-class AsanThreadContext : public ThreadContextBase {
+class AsanThreadContext final : public ThreadContextBase {
public:
explicit AsanThreadContext(int tid)
: ThreadContextBase(tid), announced(false),
@@ -69,8 +69,7 @@ class AsanThread {
struct InitOptions;
void Init(const InitOptions *options = nullptr);
- thread_return_t ThreadStart(tid_t os_id,
- atomic_uintptr_t *signal_thread_is_registered);
+ thread_return_t ThreadStart(tid_t os_id);
uptr stack_top();
uptr stack_bottom();
@@ -132,6 +131,8 @@ class AsanThread {
void *extra_spill_area() { return &extra_spill_area_; }
+ void *get_arg() { return arg_; }
+
private:
// NOTE: There is no AsanThread constructor. It is allocated
// via mmap() and *must* be valid in zero-initialized state.
diff --git a/system/lib/compiler-rt/lib/asan/asan_win.cpp b/system/lib/compiler-rt/lib/asan/asan_win.cpp
index 417892aaedd8e..1577c83cf9941 100644
--- a/system/lib/compiler-rt/lib/asan/asan_win.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_win.cpp
@@ -134,7 +134,7 @@ INTERCEPTOR(int, _except_handler4, void *a, void *b, void *c, void *d) {
static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
AsanThread *t = (AsanThread *)arg;
SetCurrentThread(t);
- return t->ThreadStart(GetTid(), /* signal_thread_is_registered */ nullptr);
+ return t->ThreadStart(GetTid());
}
INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security,
@@ -191,6 +191,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
UNIMPLEMENTED();
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
// ---------------------- TSD ---------------- {{{
static bool tsd_key_inited = false;
@@ -247,15 +253,8 @@ void *AsanDoesNotSupportStaticLinkage() {
}
uptr FindDynamicShadowStart() {
- uptr granularity = GetMmapGranularity();
- uptr alignment = 8 * granularity;
- uptr left_padding = granularity;
- uptr space_size = kHighShadowEnd + left_padding;
- uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
- granularity, nullptr, nullptr);
- CHECK_NE((uptr)0, shadow_start);
- CHECK(IsAligned(shadow_start, alignment));
- return shadow_start;
+ return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
void AsanCheckDynamicRTPrereqs() {}
@@ -268,6 +267,8 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
void AsanOnDeadlySignal(int, void *siginfo, void *context) { UNIMPLEMENTED(); }
+bool PlatformUnpoisonStacks() { return false; }
+
#if SANITIZER_WINDOWS64
// Exception handler for dealing with shadow memory.
static LONG CALLBACK
diff --git a/system/lib/compiler-rt/lib/builtins/absvsi2.c b/system/lib/compiler-rt/lib/builtins/absvsi2.c
index 44ada169e7e66..9d5de7e8a3f22 100644
--- a/system/lib/compiler-rt/lib/builtins/absvsi2.c
+++ b/system/lib/compiler-rt/lib/builtins/absvsi2.c
@@ -18,7 +18,7 @@
COMPILER_RT_ABI si_int __absvsi2(si_int a) {
const int N = (int)(sizeof(si_int) * CHAR_BIT);
- if (a == (1 << (N - 1)))
+ if (a == ((si_int)1 << (N - 1)))
compilerrt_abort();
const si_int t = a >> (N - 1);
return (a ^ t) - t;
diff --git a/system/lib/compiler-rt/lib/builtins/ashldi3.c b/system/lib/compiler-rt/lib/builtins/ashldi3.c
index 7c81057a22847..04f22228f11db 100644
--- a/system/lib/compiler-rt/lib/builtins/ashldi3.c
+++ b/system/lib/compiler-rt/lib/builtins/ashldi3.c
@@ -16,7 +16,7 @@
// Precondition: 0 <= b < bits_in_dword
-COMPILER_RT_ABI di_int __ashldi3(di_int a, si_int b) {
+COMPILER_RT_ABI di_int __ashldi3(di_int a, int b) {
const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT);
dwords input;
dwords result;
diff --git a/system/lib/compiler-rt/lib/builtins/ashrdi3.c b/system/lib/compiler-rt/lib/builtins/ashrdi3.c
index b9939132205c8..934a5c47fd69b 100644
--- a/system/lib/compiler-rt/lib/builtins/ashrdi3.c
+++ b/system/lib/compiler-rt/lib/builtins/ashrdi3.c
@@ -16,7 +16,7 @@
// Precondition: 0 <= b < bits_in_dword
-COMPILER_RT_ABI di_int __ashrdi3(di_int a, si_int b) {
+COMPILER_RT_ABI di_int __ashrdi3(di_int a, int b) {
const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT);
dwords input;
dwords result;
diff --git a/system/lib/compiler-rt/lib/builtins/assembly.h b/system/lib/compiler-rt/lib/builtins/assembly.h
index f437cb87f60ac..f6ce6a9fccff3 100644
--- a/system/lib/compiler-rt/lib/builtins/assembly.h
+++ b/system/lib/compiler-rt/lib/builtins/assembly.h
@@ -14,8 +14,8 @@
#ifndef COMPILERRT_ASSEMBLY_H
#define COMPILERRT_ASSEMBLY_H
-#if defined(__POWERPC__) || defined(__powerpc__) || defined(__ppc__)
-#define SEPARATOR @
+#if defined(__APPLE__) && defined(__aarch64__)
+#define SEPARATOR %%
#else
#define SEPARATOR ;
#endif
@@ -35,14 +35,14 @@
#define HIDDEN(name) .hidden name
#define LOCAL_LABEL(name) .L_##name
#define FILE_LEVEL_DIRECTIVE
-#if defined(__arm__)
+#if defined(__arm__) || defined(__aarch64__)
#define SYMBOL_IS_FUNC(name) .type name,%function
#else
#define SYMBOL_IS_FUNC(name) .type name,@function
#endif
#define CONST_SECTION .section .rodata
-#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
defined(__linux__)
#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
#else
@@ -65,6 +65,66 @@
#endif
+#if defined(__arm__) || defined(__aarch64__)
+#define FUNC_ALIGN \
+ .text SEPARATOR \
+ .balign 16 SEPARATOR
+#else
+#define FUNC_ALIGN
+#endif
+
+// BTI and PAC gnu property note
+#define NT_GNU_PROPERTY_TYPE_0 5
+#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
+#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
+#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
+
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
+#else
+#define BTI_FLAG 0
+#endif
+
+#if __ARM_FEATURE_PAC_DEFAULT & 3
+#define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC
+#else
+#define PAC_FLAG 0
+#endif
+
+#define GNU_PROPERTY(type, value) \
+ .pushsection .note.gnu.property, "a" SEPARATOR \
+ .p2align 3 SEPARATOR \
+ .word 4 SEPARATOR \
+ .word 16 SEPARATOR \
+ .word NT_GNU_PROPERTY_TYPE_0 SEPARATOR \
+ .asciz "GNU" SEPARATOR \
+ .word type SEPARATOR \
+ .word 4 SEPARATOR \
+ .word value SEPARATOR \
+ .word 0 SEPARATOR \
+ .popsection
+
+#if BTI_FLAG != 0
+#define BTI_C bti c
+#else
+#define BTI_C
+#endif
+
+#if (BTI_FLAG | PAC_FLAG) != 0
+#define GNU_PROPERTY_BTI_PAC \
+ GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG)
+#else
+#define GNU_PROPERTY_BTI_PAC
+#endif
+
+#if defined(__clang__) || defined(__GCC_HAVE_DWARF2_CFI_ASM)
+#define CFI_START .cfi_startproc
+#define CFI_END .cfi_endproc
+#else
+#define CFI_START
+#define CFI_END
+#endif
+
#if defined(__arm__)
// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
@@ -131,8 +191,14 @@
#define DEFINE_CODE_STATE
#endif
-#define GLUE2(a, b) a##b
-#define GLUE(a, b) GLUE2(a, b)
+#define GLUE2_(a, b) a##b
+#define GLUE(a, b) GLUE2_(a, b)
+#define GLUE2(a, b) GLUE2_(a, b)
+#define GLUE3_(a, b, c) a##b##c
+#define GLUE3(a, b, c) GLUE3_(a, b, c)
+#define GLUE4_(a, b, c, d) a##b##c##d
+#define GLUE4(a, b, c, d) GLUE4_(a, b, c, d)
+
#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
#ifdef VISIBILITY_HIDDEN
@@ -177,6 +243,16 @@
DECLARE_FUNC_ENCODING \
name:
+#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name) \
+ DEFINE_CODE_STATE \
+ FUNC_ALIGN \
+ .globl name SEPARATOR \
+ SYMBOL_IS_FUNC(name) SEPARATOR \
+ DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
+ CFI_START SEPARATOR \
+ DECLARE_FUNC_ENCODING \
+ name: SEPARATOR BTI_C
+
#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
.globl SYMBOL_NAME(name) SEPARATOR \
SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
@@ -193,8 +269,13 @@
#ifdef __ELF__
#define END_COMPILERRT_FUNCTION(name) \
.size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
+#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
+ CFI_END SEPARATOR \
+ .size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
#else
#define END_COMPILERRT_FUNCTION(name)
+#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
+ CFI_END
#endif
#endif // COMPILERRT_ASSEMBLY_H
diff --git a/system/lib/compiler-rt/lib/builtins/atomic.c b/system/lib/compiler-rt/lib/builtins/atomic.c
index 32b3a0f9ad239..f48cdc10ccf7a 100644
--- a/system/lib/compiler-rt/lib/builtins/atomic.c
+++ b/system/lib/compiler-rt/lib/builtins/atomic.c
@@ -23,6 +23,7 @@
//
//===----------------------------------------------------------------------===//
+#include
#include
#include
@@ -35,6 +36,8 @@
#pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange)
#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME( \
__atomic_compare_exchange)
+#pragma redefine_extname __atomic_is_lock_free_c SYMBOL_NAME( \
+ __atomic_is_lock_free)
/// Number of locks. This allocates one page on 32-bit platforms, two on
/// 64-bit. This can be specified externally if a different trade between
@@ -119,56 +122,58 @@ static __inline Lock *lock_for_pointer(void *ptr) {
return locks + (hash & SPINLOCK_MASK);
}
-/// Macros for determining whether a size is lock free. Clang can not yet
-/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are
-/// not lock free.
-#define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1)
-#define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2)
-#define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4)
-#define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8)
-#define IS_LOCK_FREE_16 0
+/// Macros for determining whether a size is lock free.
+#define ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(size, p) \
+ (__atomic_always_lock_free(size, p) || \
+ (__atomic_always_lock_free(size, 0) && ((uintptr_t)p % size) == 0))
+#define IS_LOCK_FREE_1(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(1, p)
+#define IS_LOCK_FREE_2(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(2, p)
+#define IS_LOCK_FREE_4(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(4, p)
+#define IS_LOCK_FREE_8(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(8, p)
+#define IS_LOCK_FREE_16(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(16, p)
/// Macro that calls the compiler-generated lock-free versions of functions
/// when they exist.
-#define LOCK_FREE_CASES() \
+#define TRY_LOCK_FREE_CASE(n, type, ptr) \
+ case n: \
+ if (IS_LOCK_FREE_##n(ptr)) { \
+ LOCK_FREE_ACTION(type); \
+ } \
+ break;
+#ifdef __SIZEOF_INT128__
+#define TRY_LOCK_FREE_CASE_16(p) TRY_LOCK_FREE_CASE(16, __uint128_t, p)
+#else
+#define TRY_LOCK_FREE_CASE_16(p) /* __uint128_t not available */
+#endif
+
+#define LOCK_FREE_CASES(ptr) \
do { \
switch (size) { \
- case 1: \
- if (IS_LOCK_FREE_1) { \
- LOCK_FREE_ACTION(uint8_t); \
- } \
- break; \
- case 2: \
- if (IS_LOCK_FREE_2) { \
- LOCK_FREE_ACTION(uint16_t); \
- } \
- break; \
- case 4: \
- if (IS_LOCK_FREE_4) { \
- LOCK_FREE_ACTION(uint32_t); \
- } \
- break; \
- case 8: \
- if (IS_LOCK_FREE_8) { \
- LOCK_FREE_ACTION(uint64_t); \
- } \
- break; \
- case 16: \
- if (IS_LOCK_FREE_16) { \
- /* FIXME: __uint128_t isn't available on 32 bit platforms. \
- LOCK_FREE_ACTION(__uint128_t);*/ \
- } \
+ TRY_LOCK_FREE_CASE(1, uint8_t, ptr) \
+ TRY_LOCK_FREE_CASE(2, uint16_t, ptr) \
+ TRY_LOCK_FREE_CASE(4, uint32_t, ptr) \
+ TRY_LOCK_FREE_CASE(8, uint64_t, ptr) \
+ TRY_LOCK_FREE_CASE_16(ptr) /* __uint128_t may not be supported */ \
+ default: \
break; \
} \
} while (0)
+/// Whether atomic operations for the given size (and alignment) are lock-free.
+bool __atomic_is_lock_free_c(size_t size, void *ptr) {
+#define LOCK_FREE_ACTION(type) return true;
+ LOCK_FREE_CASES(ptr);
+#undef LOCK_FREE_ACTION
+ return false;
+}
+
/// An atomic load operation. This is atomic with respect to the source
/// pointer only.
void __atomic_load_c(int size, void *src, void *dest, int model) {
#define LOCK_FREE_ACTION(type) \
*((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(src);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(src);
lock(l);
@@ -182,7 +187,7 @@ void __atomic_store_c(int size, void *dest, void *src, int model) {
#define LOCK_FREE_ACTION(type) \
__c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(dest);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(dest);
lock(l);
@@ -201,7 +206,7 @@ int __atomic_compare_exchange_c(int size, void *ptr, void *expected,
return __c11_atomic_compare_exchange_strong( \
(_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \
failure)
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(ptr);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
@@ -222,7 +227,7 @@ void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) {
*(type *)old = \
__c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(ptr);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
@@ -252,7 +257,7 @@ void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) {
#define OPTIMISED_CASE(n, lockfree, type) \
type __atomic_load_##n(type *src, int model) { \
- if (lockfree) \
+ if (lockfree(src)) \
return __c11_atomic_load((_Atomic(type) *)src, model); \
Lock *l = lock_for_pointer(src); \
lock(l); \
@@ -265,7 +270,7 @@ OPTIMISED_CASES
#define OPTIMISED_CASE(n, lockfree, type) \
void __atomic_store_##n(type *dest, type val, int model) { \
- if (lockfree) { \
+ if (lockfree(dest)) { \
__c11_atomic_store((_Atomic(type) *)dest, val, model); \
return; \
} \
@@ -280,7 +285,7 @@ OPTIMISED_CASES
#define OPTIMISED_CASE(n, lockfree, type) \
type __atomic_exchange_##n(type *dest, type val, int model) { \
- if (lockfree) \
+ if (lockfree(dest)) \
return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \
Lock *l = lock_for_pointer(dest); \
lock(l); \
@@ -293,9 +298,9 @@ OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type) \
- int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \
- int success, int failure) { \
- if (lockfree) \
+ bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \
+ int success, int failure) { \
+ if (lockfree(ptr)) \
return __c11_atomic_compare_exchange_strong( \
(_Atomic(type) *)ptr, expected, desired, success, failure); \
Lock *l = lock_for_pointer(ptr); \
@@ -303,11 +308,11 @@ OPTIMISED_CASES
if (*ptr == *expected) { \
*ptr = desired; \
unlock(l); \
- return 1; \
+ return true; \
} \
*expected = *ptr; \
unlock(l); \
- return 0; \
+ return false; \
}
OPTIMISED_CASES
#undef OPTIMISED_CASE
@@ -317,7 +322,7 @@ OPTIMISED_CASES
////////////////////////////////////////////////////////////////////////////////
#define ATOMIC_RMW(n, lockfree, type, opname, op) \
type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \
- if (lockfree) \
+ if (lockfree(ptr)) \
return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \
Lock *l = lock_for_pointer(ptr); \
lock(l); \
diff --git a/system/lib/compiler-rt/lib/builtins/clear_cache.c b/system/lib/compiler-rt/lib/builtins/clear_cache.c
index e83e21254e852..5a443ddd4b03a 100644
--- a/system/lib/compiler-rt/lib/builtins/clear_cache.c
+++ b/system/lib/compiler-rt/lib/builtins/clear_cache.c
@@ -33,7 +33,7 @@ uintptr_t GetCurrentProcess(void);
#include
#endif
-#if defined(__OpenBSD__) && defined(__mips__)
+#if defined(__OpenBSD__) && (defined(__arm__) || defined(__mips__))
// clang-format off
#include
#include
@@ -46,6 +46,11 @@ uintptr_t GetCurrentProcess(void);
#include
#endif
+#if defined(__linux__) && defined(__riscv)
+// to get platform-specific syscall definitions
+#include
+#endif
+
// The compiler generates calls to __clear_cache() when creating
// trampoline functions on the stack for use with nested functions.
// It is expected to invalidate the instruction cache for the
@@ -58,7 +63,7 @@ void __clear_cache(void *start, void *end) {
#elif defined(_WIN32) && (defined(__arm__) || defined(__aarch64__))
FlushInstructionCache(GetCurrentProcess(), start, end - start);
#elif defined(__arm__) && !defined(__APPLE__)
-#if defined(__FreeBSD__) || defined(__NetBSD__)
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
struct arm_sync_icache_args arg;
arg.addr = (uintptr_t)start;
@@ -147,10 +152,23 @@ void __clear_cache(void *start, void *end) {
for (uintptr_t dword = start_dword; dword < end_dword; dword += dword_size)
__asm__ volatile("flush %0" : : "r"(dword));
+#elif defined(__riscv) && defined(__linux__)
+ // See: arch/riscv/include/asm/cacheflush.h, arch/riscv/kernel/sys_riscv.c
+ register void *start_reg __asm("a0") = start;
+ const register void *end_reg __asm("a1") = end;
+ // "0" means that we clear cache for all threads (SYS_RISCV_FLUSH_ICACHE_ALL)
+ const register long flags __asm("a2") = 0;
+ const register long syscall_nr __asm("a7") = __NR_riscv_flush_icache;
+ __asm __volatile("ecall"
+ : "=r"(start_reg)
+ : "r"(start_reg), "r"(end_reg), "r"(flags), "r"(syscall_nr));
+ assert(start_reg == 0 && "Cache flush syscall failed.");
#else
#if __APPLE__
// On Darwin, sys_icache_invalidate() provides this functionality
sys_icache_invalidate(start, end - start);
+#elif defined(__ve__)
+ __asm__ volatile("fencec 2");
#else
compilerrt_abort();
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/clzdi2.c b/system/lib/compiler-rt/lib/builtins/clzdi2.c
index a0bacb2ae39ea..12c17982a5cb1 100644
--- a/system/lib/compiler-rt/lib/builtins/clzdi2.c
+++ b/system/lib/compiler-rt/lib/builtins/clzdi2.c
@@ -21,15 +21,15 @@
// ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than
// __clzsi2, leading to infinite recursion.
#define __builtin_clz(a) __clzsi2(a)
-extern si_int __clzsi2(si_int);
+extern int __clzsi2(si_int);
#endif
// Precondition: a != 0
-COMPILER_RT_ABI si_int __clzdi2(di_int a) {
+COMPILER_RT_ABI int __clzdi2(di_int a) {
dwords x;
x.all = a;
const si_int f = -(x.s.high == 0);
- return __builtin_clz((x.s.high & ~f) | (x.s.low & f)) +
+ return clzsi((x.s.high & ~f) | (x.s.low & f)) +
(f & ((si_int)(sizeof(si_int) * CHAR_BIT)));
}
diff --git a/system/lib/compiler-rt/lib/builtins/clzsi2.c b/system/lib/compiler-rt/lib/builtins/clzsi2.c
index 3f9f27f413315..d75f56d937b07 100644
--- a/system/lib/compiler-rt/lib/builtins/clzsi2.c
+++ b/system/lib/compiler-rt/lib/builtins/clzsi2.c
@@ -16,7 +16,7 @@
// Precondition: a != 0
-COMPILER_RT_ABI si_int __clzsi2(si_int a) {
+COMPILER_RT_ABI int __clzsi2(si_int a) {
su_int x = (su_int)a;
si_int t = ((x & 0xFFFF0000) == 0) << 4; // if (x is small) t = 16 else 0
x >>= 16 - t; // x = [0 - 0xFFFF]
diff --git a/system/lib/compiler-rt/lib/builtins/clzti2.c b/system/lib/compiler-rt/lib/builtins/clzti2.c
index 0c787104caa2a..25d30119f271c 100644
--- a/system/lib/compiler-rt/lib/builtins/clzti2.c
+++ b/system/lib/compiler-rt/lib/builtins/clzti2.c
@@ -18,7 +18,7 @@
// Precondition: a != 0
-COMPILER_RT_ABI si_int __clzti2(ti_int a) {
+COMPILER_RT_ABI int __clzti2(ti_int a) {
twords x;
x.all = a;
const di_int f = -(x.s.high == 0);
diff --git a/system/lib/compiler-rt/lib/builtins/cpu_model.c b/system/lib/compiler-rt/lib/builtins/cpu_model.c
index fb619037d398c..51bedd98c3d30 100644
--- a/system/lib/compiler-rt/lib/builtins/cpu_model.c
+++ b/system/lib/compiler-rt/lib/builtins/cpu_model.c
@@ -8,10 +8,21 @@
//
// This file is based on LLVM's lib/Support/Host.cpp.
// It implements the operating system Host concept and builtin
-// __cpu_model for the compiler_rt library, for x86 only.
+// __cpu_model for the compiler_rt library for x86 and
+// __aarch64_have_lse_atomics for AArch64.
//
//===----------------------------------------------------------------------===//
+#if defined(HAVE_INIT_PRIORITY)
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
+#elif __has_attribute(__constructor__)
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
+#else
+// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
+// this runs during initialization.
+#define CONSTRUCTOR_ATTRIBUTE
+#endif
+
#if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || \
defined(_M_X64)) && \
(defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
@@ -57,6 +68,7 @@ enum ProcessorTypes {
INTEL_GOLDMONT,
INTEL_GOLDMONT_PLUS,
INTEL_TREMONT,
+ AMDFAM19H,
CPU_TYPE_MAX
};
@@ -82,6 +94,11 @@ enum ProcessorSubtypes {
INTEL_COREI7_ICELAKE_SERVER,
AMDFAM17H_ZNVER2,
INTEL_COREI7_CASCADELAKE,
+ INTEL_COREI7_TIGERLAKE,
+ INTEL_COREI7_COOPERLAKE,
+ INTEL_COREI7_SAPPHIRERAPIDS,
+ INTEL_COREI7_ALDERLAKE,
+ AMDFAM19H_ZNVER3,
CPU_SUBTYPE_MAX
};
@@ -122,7 +139,9 @@ enum ProcessorFeatures {
FEATURE_VPCLMULQDQ,
FEATURE_AVX512VNNI,
FEATURE_AVX512BITALG,
- FEATURE_AVX512BF16
+ FEATURE_AVX512BF16,
+ FEATURE_AVX512VP2INTERSECT,
+ CPU_FEATURE_MAX
};
// The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max).
@@ -268,13 +287,17 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
}
}
-static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
- unsigned Brand_id,
- unsigned Features,
- unsigned Features2, unsigned *Type,
- unsigned *Subtype) {
- if (Brand_id != 0)
- return;
+static const char *
+getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+ const unsigned *Features,
+ unsigned *Type, unsigned *Subtype) {
+#define testFeature(F) \
+ (Features[F / 32] & (1 << (F % 32))) != 0
+
+ // We select CPU strings to match the code in Host.cpp, but we don't use them
+ // in compiler-rt.
+ const char *CPU = 0;
+
switch (Family) {
case 6:
switch (Model) {
@@ -285,13 +308,17 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
// 0Fh. All processors are manufactured using the 65 nm process.
case 0x16: // Intel Celeron processor model 16h. All processors are
// manufactured using the 65 nm process
+ CPU = "core2";
+ *Type = INTEL_CORE2;
+ break;
case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
// 17h. All processors are manufactured using the 45 nm process.
//
// 45nm: Penryn , Wolfdale, Yorkfield (XE)
case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
// the 45 nm process.
- *Type = INTEL_CORE2; // "penryn"
+ CPU = "penryn";
+ *Type = INTEL_CORE2;
break;
case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
// processors are manufactured using the 45 nm process.
@@ -299,25 +326,29 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
// As found in a Summer 2010 model iMac.
case 0x1f:
case 0x2e: // Nehalem EX
- *Type = INTEL_COREI7; // "nehalem"
+ CPU = "nehalem";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_NEHALEM;
break;
case 0x25: // Intel Core i7, laptop version.
case 0x2c: // Intel Core i7 processor and Intel Xeon processor. All
// processors are manufactured using the 32 nm process.
case 0x2f: // Westmere EX
- *Type = INTEL_COREI7; // "westmere"
+ CPU = "westmere";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_WESTMERE;
break;
case 0x2a: // Intel Core i7 processor. All processors are manufactured
// using the 32 nm process.
case 0x2d:
- *Type = INTEL_COREI7; //"sandybridge"
+ CPU = "sandybridge";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_SANDYBRIDGE;
break;
case 0x3a:
case 0x3e: // Ivy Bridge EP
- *Type = INTEL_COREI7; // "ivybridge"
+ CPU = "ivybridge";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_IVYBRIDGE;
break;
@@ -326,7 +357,8 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
case 0x3f:
case 0x45:
case 0x46:
- *Type = INTEL_COREI7; // "haswell"
+ CPU = "haswell";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_HASWELL;
break;
@@ -335,7 +367,8 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
case 0x47:
case 0x4f:
case 0x56:
- *Type = INTEL_COREI7; // "broadwell"
+ CPU = "broadwell";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_BROADWELL;
break;
@@ -344,37 +377,56 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
case 0x5e: // Skylake desktop
case 0x8e: // Kaby Lake mobile
case 0x9e: // Kaby Lake desktop
- *Type = INTEL_COREI7; // "skylake"
+ case 0xa5: // Comet Lake-H/S
+ case 0xa6: // Comet Lake-U
+ CPU = "skylake";
+ *Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_SKYLAKE;
break;
// Skylake Xeon:
case 0x55:
*Type = INTEL_COREI7;
- if (Features2 & (1 << (FEATURE_AVX512VNNI - 32)))
- *Subtype = INTEL_COREI7_CASCADELAKE; // "cascadelake"
- else
- *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512"
+ if (testFeature(FEATURE_AVX512BF16)) {
+ CPU = "cooperlake";
+ *Subtype = INTEL_COREI7_COOPERLAKE;
+ } else if (testFeature(FEATURE_AVX512VNNI)) {
+ CPU = "cascadelake";
+ *Subtype = INTEL_COREI7_CASCADELAKE;
+ } else {
+ CPU = "skylake-avx512";
+ *Subtype = INTEL_COREI7_SKYLAKE_AVX512;
+ }
break;
// Cannonlake:
case 0x66:
+ CPU = "cannonlake";
*Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_CANNONLAKE; // "cannonlake"
+ *Subtype = INTEL_COREI7_CANNONLAKE;
break;
// Icelake:
case 0x7d:
case 0x7e:
+ CPU = "icelake-client";
*Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_ICELAKE_CLIENT; // "icelake-client"
+ *Subtype = INTEL_COREI7_ICELAKE_CLIENT;
break;
// Icelake Xeon:
case 0x6a:
case 0x6c:
+ CPU = "icelake-server";
*Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_ICELAKE_SERVER; // "icelake-server"
+ *Subtype = INTEL_COREI7_ICELAKE_SERVER;
+ break;
+
+ // Sapphire Rapids:
+ case 0x8f:
+ CPU = "sapphirerapids";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_SAPPHIRERAPIDS;
break;
case 0x1c: // Most 45 nm Intel Atom processors
@@ -382,8 +434,9 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
case 0x27: // 32 nm Atom Medfield
case 0x35: // 32 nm Atom Midview
case 0x36: // 32 nm Atom Midview
+ CPU = "bonnell";
*Type = INTEL_BONNELL;
- break; // "bonnell"
+ break;
// Atom Silvermont codes from the Intel software optimization guide.
case 0x37:
@@ -392,26 +445,32 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
case 0x5a:
case 0x5d:
case 0x4c: // really airmont
+ CPU = "silvermont";
*Type = INTEL_SILVERMONT;
- break; // "silvermont"
+ break;
// Goldmont:
case 0x5c: // Apollo Lake
case 0x5f: // Denverton
+ CPU = "goldmont";
*Type = INTEL_GOLDMONT;
break; // "goldmont"
case 0x7a:
+ CPU = "goldmont-plus";
*Type = INTEL_GOLDMONT_PLUS;
break;
case 0x86:
+ CPU = "tremont";
*Type = INTEL_TREMONT;
break;
case 0x57:
- *Type = INTEL_KNL; // knl
+ CPU = "knl";
+ *Type = INTEL_KNL;
break;
case 0x85:
- *Type = INTEL_KNM; // knm
+ CPU = "knm";
+ *Type = INTEL_KNM;
break;
default: // Unknown family 6 CPU.
@@ -421,17 +480,22 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
default:
break; // Unknown.
}
+
+ return CPU;
}
-static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
- unsigned Features, unsigned Features2,
- unsigned *Type, unsigned *Subtype) {
- // FIXME: this poorly matches the generated SubtargetFeatureKV table. There
- // appears to be no way to generate the wide variety of AMD-specific targets
- // from the information returned from CPUID.
+static const char *
+getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+ const unsigned *Features,
+ unsigned *Type, unsigned *Subtype) {
+ // We select CPU strings to match the code in Host.cpp, but we don't use them
+ // in compiler-rt.
+ const char *CPU = 0;
+
switch (Family) {
case 16:
- *Type = AMDFAM10H; // "amdfam10"
+ CPU = "amdfam10";
+ *Type = AMDFAM10H;
switch (Model) {
case 2:
*Subtype = AMDFAM10H_BARCELONA;
@@ -445,60 +509,70 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
}
break;
case 20:
+ CPU = "btver1";
*Type = AMD_BTVER1;
- break; // "btver1";
+ break;
case 21:
+ CPU = "bdver1";
*Type = AMDFAM15H;
if (Model >= 0x60 && Model <= 0x7f) {
+ CPU = "bdver4";
*Subtype = AMDFAM15H_BDVER4;
- break; // "bdver4"; 60h-7Fh: Excavator
+ break; // 60h-7Fh: Excavator
}
if (Model >= 0x30 && Model <= 0x3f) {
+ CPU = "bdver3";
*Subtype = AMDFAM15H_BDVER3;
- break; // "bdver3"; 30h-3Fh: Steamroller
+ break; // 30h-3Fh: Steamroller
}
if ((Model >= 0x10 && Model <= 0x1f) || Model == 0x02) {
+ CPU = "bdver2";
*Subtype = AMDFAM15H_BDVER2;
- break; // "bdver2"; 02h, 10h-1Fh: Piledriver
+ break; // 02h, 10h-1Fh: Piledriver
}
if (Model <= 0x0f) {
*Subtype = AMDFAM15H_BDVER1;
- break; // "bdver1"; 00h-0Fh: Bulldozer
+ break; // 00h-0Fh: Bulldozer
}
break;
case 22:
+ CPU = "btver2";
*Type = AMD_BTVER2;
- break; // "btver2"
+ break;
case 23:
+ CPU = "znver1";
*Type = AMDFAM17H;
if ((Model >= 0x30 && Model <= 0x3f) || Model == 0x71) {
+ CPU = "znver2";
*Subtype = AMDFAM17H_ZNVER2;
- break; // "znver2"; 30h-3fh, 71h: Zen2
+ break; // 30h-3fh, 71h: Zen2
}
if (Model <= 0x0f) {
*Subtype = AMDFAM17H_ZNVER1;
- break; // "znver1"; 00h-0Fh: Zen1
+ break; // 00h-0Fh: Zen1
+ }
+ break;
+ case 25:
+ CPU = "znver3";
+ *Type = AMDFAM19H;
+ if (Model <= 0x0f) {
+ *Subtype = AMDFAM19H_ZNVER3;
+ break; // 00h-0Fh: Zen3
}
break;
default:
- break; // "generic"
+ break; // Unknown AMD CPU.
}
+
+ return CPU;
}
static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
- unsigned *FeaturesOut,
- unsigned *Features2Out) {
- unsigned Features = 0;
- unsigned Features2 = 0;
+ unsigned *Features) {
unsigned EAX, EBX;
#define setFeature(F) \
- do { \
- if (F < 32) \
- Features |= 1U << (F & 0x1f); \
- else if (F < 64) \
- Features2 |= 1U << ((F - 32) & 0x1f); \
- } while (0)
+ Features[F / 32] |= 1U << (F % 32)
if ((EDX >> 15) & 1)
setFeature(FEATURE_CMOV);
@@ -590,6 +664,8 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
setFeature(FEATURE_AVX5124VNNIW);
if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save)
setFeature(FEATURE_AVX5124FMAPS);
+ if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save)
+ setFeature(FEATURE_AVX512VP2INTERSECT);
bool HasLeaf7Subleaf1 =
MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
@@ -607,22 +683,9 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
setFeature(FEATURE_XOP);
if (HasExtLeaf1 && ((ECX >> 16) & 1))
setFeature(FEATURE_FMA4);
-
- *FeaturesOut = Features;
- *Features2Out = Features2;
#undef setFeature
}
-#if defined(HAVE_INIT_PRIORITY)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
-#elif __has_attribute(__constructor__)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
-#else
-// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
-// this runs during initialization.
-#define CONSTRUCTOR_ATTRIBUTE
-#endif
-
#ifndef _WIN32
__attribute__((visibility("hidden")))
#endif
@@ -641,7 +704,7 @@ struct __processor_model {
#ifndef _WIN32
__attribute__((visibility("hidden")))
#endif
-unsigned int __cpu_features2;
+unsigned int __cpu_features2 = 0;
// A constructor function that is sets __cpu_model and __cpu_features2 with
// the right values. This needs to run only once. This constructor is
@@ -653,40 +716,38 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
unsigned EAX, EBX, ECX, EDX;
unsigned MaxLeaf = 5;
unsigned Vendor;
- unsigned Model, Family, Brand_id;
- unsigned Features = 0;
- unsigned Features2 = 0;
+ unsigned Model, Family;
+ unsigned Features[(CPU_FEATURE_MAX + 31) / 32] = {0};
// This function needs to run just once.
if (__cpu_model.__cpu_vendor)
return 0;
- if (!isCpuIdSupported())
- return -1;
-
- // Assume cpuid insn present. Run in level 0 to get vendor id.
- if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) {
+ if (!isCpuIdSupported() ||
+ getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) {
__cpu_model.__cpu_vendor = VENDOR_OTHER;
return -1;
}
+
getX86CpuIDAndInfo(1, &EAX, &EBX, &ECX, &EDX);
detectX86FamilyModel(EAX, &Family, &Model);
- Brand_id = EBX & 0xff;
// Find available features.
- getAvailableFeatures(ECX, EDX, MaxLeaf, &Features, &Features2);
- __cpu_model.__cpu_features[0] = Features;
- __cpu_features2 = Features2;
+ getAvailableFeatures(ECX, EDX, MaxLeaf, &Features[0]);
+
+ assert((sizeof(Features)/sizeof(Features[0])) == 2);
+ __cpu_model.__cpu_features[0] = Features[0];
+ __cpu_features2 = Features[1];
if (Vendor == SIG_INTEL) {
// Get CPU type.
- getIntelProcessorTypeAndSubtype(Family, Model, Brand_id, Features,
- Features2, &(__cpu_model.__cpu_type),
+ getIntelProcessorTypeAndSubtype(Family, Model, &Features[0],
+ &(__cpu_model.__cpu_type),
&(__cpu_model.__cpu_subtype));
__cpu_model.__cpu_vendor = VENDOR_INTEL;
} else if (Vendor == SIG_AMD) {
// Get CPU type.
- getAMDProcessorTypeAndSubtype(Family, Model, Features, Features2,
+ getAMDProcessorTypeAndSubtype(Family, Model, &Features[0],
&(__cpu_model.__cpu_type),
&(__cpu_model.__cpu_subtype));
__cpu_model.__cpu_vendor = VENDOR_AMD;
@@ -699,5 +760,24 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
return 0;
}
-
+#elif defined(__aarch64__)
+// LSE support detection for out-of-line atomics
+// using HWCAP and Auxiliary vector
+_Bool __aarch64_have_lse_atomics
+ __attribute__((visibility("hidden"), nocommon));
+#if defined(__has_include)
+#if __has_include()
+#include
+#ifndef AT_HWCAP
+#define AT_HWCAP 16
+#endif
+#ifndef HWCAP_ATOMICS
+#define HWCAP_ATOMICS (1 << 8)
#endif
+static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
+ unsigned long hwcap = getauxval(AT_HWCAP);
+ __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0;
+}
+#endif // defined(__has_include)
+#endif // __has_include()
+#endif // defined(__aarch64__)
diff --git a/system/lib/compiler-rt/lib/builtins/ctzdi2.c b/system/lib/compiler-rt/lib/builtins/ctzdi2.c
index 9384aa6055a11..26c908d876ac4 100644
--- a/system/lib/compiler-rt/lib/builtins/ctzdi2.c
+++ b/system/lib/compiler-rt/lib/builtins/ctzdi2.c
@@ -21,15 +21,15 @@
// ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than
// __ctzsi2, leading to infinite recursion.
#define __builtin_ctz(a) __ctzsi2(a)
-extern si_int __ctzsi2(si_int);
+extern int __ctzsi2(si_int);
#endif
// Precondition: a != 0
-COMPILER_RT_ABI si_int __ctzdi2(di_int a) {
+COMPILER_RT_ABI int __ctzdi2(di_int a) {
dwords x;
x.all = a;
const si_int f = -(x.s.low == 0);
- return __builtin_ctz((x.s.high & f) | (x.s.low & ~f)) +
+ return ctzsi((x.s.high & f) | (x.s.low & ~f)) +
(f & ((si_int)(sizeof(si_int) * CHAR_BIT)));
}
diff --git a/system/lib/compiler-rt/lib/builtins/ctzsi2.c b/system/lib/compiler-rt/lib/builtins/ctzsi2.c
index 09c6863b74e3a..ed95c60579339 100644
--- a/system/lib/compiler-rt/lib/builtins/ctzsi2.c
+++ b/system/lib/compiler-rt/lib/builtins/ctzsi2.c
@@ -16,7 +16,7 @@
// Precondition: a != 0
-COMPILER_RT_ABI si_int __ctzsi2(si_int a) {
+COMPILER_RT_ABI int __ctzsi2(si_int a) {
su_int x = (su_int)a;
si_int t = ((x & 0x0000FFFF) == 0)
<< 4; // if (x has no small bits) t = 16 else 0
diff --git a/system/lib/compiler-rt/lib/builtins/ctzti2.c b/system/lib/compiler-rt/lib/builtins/ctzti2.c
index 2a1312c8437de..fb136d0de1c0d 100644
--- a/system/lib/compiler-rt/lib/builtins/ctzti2.c
+++ b/system/lib/compiler-rt/lib/builtins/ctzti2.c
@@ -18,7 +18,7 @@
// Precondition: a != 0
-COMPILER_RT_ABI si_int __ctzti2(ti_int a) {
+COMPILER_RT_ABI int __ctzti2(ti_int a) {
twords x;
x.all = a;
const di_int f = -(x.s.low == 0);
diff --git a/system/lib/compiler-rt/lib/builtins/divdf3.c b/system/lib/compiler-rt/lib/builtins/divdf3.c
index 1dea3b534f5ae..4c11759e0c4a7 100644
--- a/system/lib/compiler-rt/lib/builtins/divdf3.c
+++ b/system/lib/compiler-rt/lib/builtins/divdf3.c
@@ -9,197 +9,16 @@
// This file implements double-precision soft-float division
// with the IEEE-754 default rounding (to nearest, ties to even).
//
-// For simplicity, this implementation currently flushes denormals to zero.
-// It should be a fairly straightforward exercise to implement gradual
-// underflow with correct rounding.
-//
//===----------------------------------------------------------------------===//
#define DOUBLE_PRECISION
-#include "fp_lib.h"
-
-COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) {
-
- const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
- const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
- const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
-
- rep_t aSignificand = toRep(a) & significandMask;
- rep_t bSignificand = toRep(b) & significandMask;
- int scale = 0;
-
- // Detect if a or b is zero, denormal, infinity, or NaN.
- if (aExponent - 1U >= maxExponent - 1U ||
- bExponent - 1U >= maxExponent - 1U) {
-
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
-
- // NaN / anything = qNaN
- if (aAbs > infRep)
- return fromRep(toRep(a) | quietBit);
- // anything / NaN = qNaN
- if (bAbs > infRep)
- return fromRep(toRep(b) | quietBit);
-
- if (aAbs == infRep) {
- // infinity / infinity = NaN
- if (bAbs == infRep)
- return fromRep(qnanRep);
- // infinity / anything else = +/- infinity
- else
- return fromRep(aAbs | quotientSign);
- }
-
- // anything else / infinity = +/- 0
- if (bAbs == infRep)
- return fromRep(quotientSign);
-
- if (!aAbs) {
- // zero / zero = NaN
- if (!bAbs)
- return fromRep(qnanRep);
- // zero / anything else = +/- zero
- else
- return fromRep(quotientSign);
- }
- // anything else / zero = +/- infinity
- if (!bAbs)
- return fromRep(infRep | quotientSign);
-
- // One or both of a or b is denormal. The other (if applicable) is a
- // normal number. Renormalize one or both of a and b, and set scale to
- // include the necessary exponent adjustment.
- if (aAbs < implicitBit)
- scale += normalize(&aSignificand);
- if (bAbs < implicitBit)
- scale -= normalize(&bSignificand);
- }
-
- // Set the implicit significand bit. If we fell through from the
- // denormal path it was already set by normalize( ), but setting it twice
- // won't hurt anything.
- aSignificand |= implicitBit;
- bSignificand |= implicitBit;
- int quotientExponent = aExponent - bExponent + scale;
-
- // Align the significand of b as a Q31 fixed-point number in the range
- // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax
- // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This
- // is accurate to about 3.5 binary digits.
- const uint32_t q31b = bSignificand >> 21;
- uint32_t recip32 = UINT32_C(0x7504f333) - q31b;
- // 0x7504F333 / 2^32 + 1 = 3/4 + 1/sqrt(2)
-
- // Now refine the reciprocal estimate using a Newton-Raphson iteration:
- //
- // x1 = x0 * (2 - x0 * b)
- //
- // This doubles the number of correct binary digits in the approximation
- // with each iteration.
- uint32_t correction32;
- correction32 = -((uint64_t)recip32 * q31b >> 32);
- recip32 = (uint64_t)recip32 * correction32 >> 31;
- correction32 = -((uint64_t)recip32 * q31b >> 32);
- recip32 = (uint64_t)recip32 * correction32 >> 31;
- correction32 = -((uint64_t)recip32 * q31b >> 32);
- recip32 = (uint64_t)recip32 * correction32 >> 31;
-
- // The reciprocal may have overflowed to zero if the upper half of b is
- // exactly 1.0. This would sabatoge the full-width final stage of the
- // computation that follows, so we adjust the reciprocal down by one bit.
- recip32--;
-
- // We need to perform one more iteration to get us to 56 binary digits.
- // The last iteration needs to happen with extra precision.
- const uint32_t q63blo = bSignificand << 11;
- uint64_t correction, reciprocal;
- correction = -((uint64_t)recip32 * q31b + ((uint64_t)recip32 * q63blo >> 32));
- uint32_t cHi = correction >> 32;
- uint32_t cLo = correction;
- reciprocal = (uint64_t)recip32 * cHi + ((uint64_t)recip32 * cLo >> 32);
-
- // Adjust the final 64-bit reciprocal estimate downward to ensure that it is
- // strictly smaller than the infinitely precise exact reciprocal. Because
- // the computation of the Newton-Raphson step is truncating at every step,
- // this adjustment is small; most of the work is already done.
- reciprocal -= 2;
-
- // The numerical reciprocal is accurate to within 2^-56, lies in the
- // interval [0.5, 1.0), and is strictly smaller than the true reciprocal
- // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b
- // in Q53 with the following properties:
- //
- // 1. q < a/b
- // 2. q is in the interval [0.5, 2.0)
- // 3. The error in q is bounded away from 2^-53 (actually, we have a
- // couple of bits to spare, but this is all we need).
-
- // We need a 64 x 64 multiply high to compute q, which isn't a basic
- // operation in C, so we need to be a little bit fussy.
- rep_t quotient, quotientLo;
- wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo);
-
- // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0).
- // In either case, we are going to compute a residual of the form
- //
- // r = a - q*b
- //
- // We know from the construction of q that r satisfies:
- //
- // 0 <= r < ulp(q)*b
- //
- // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we
- // already have the correct result. The exact halfway case cannot occur.
- // We also take this time to right shift quotient if it falls in the [1,2)
- // range and adjust the exponent accordingly.
- rep_t residual;
- if (quotient < (implicitBit << 1)) {
- residual = (aSignificand << 53) - quotient * bSignificand;
- quotientExponent--;
- } else {
- quotient >>= 1;
- residual = (aSignificand << 52) - quotient * bSignificand;
- }
-
- const int writtenExponent = quotientExponent + exponentBias;
- if (writtenExponent >= maxExponent) {
- // If we have overflowed the exponent, return infinity.
- return fromRep(infRep | quotientSign);
- }
+#define NUMBER_OF_HALF_ITERATIONS 3
+#define NUMBER_OF_FULL_ITERATIONS 1
- else if (writtenExponent < 1) {
- if (writtenExponent == 0) {
- // Check whether the rounded result is normal.
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Round.
- absResult += round;
- if (absResult & ~significandMask) {
- // The rounded result is normal; return it.
- return fromRep(absResult | quotientSign);
- }
- }
- // Flush denormals to zero. In the future, it would be nice to add
- // code to round them correctly.
- return fromRep(quotientSign);
- }
+#include "fp_div_impl.inc"
- else {
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Insert the exponent.
- absResult |= (rep_t)writtenExponent << significandBits;
- // Round.
- absResult += round;
- // Insert the sign and return.
- const double result = fromRep(absResult | quotientSign);
- return result;
- }
-}
+COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) { return __divXf3__(a, b); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
diff --git a/system/lib/compiler-rt/lib/builtins/divdi3.c b/system/lib/compiler-rt/lib/builtins/divdi3.c
index ee08d6557783c..d71e138d995c7 100644
--- a/system/lib/compiler-rt/lib/builtins/divdi3.c
+++ b/system/lib/compiler-rt/lib/builtins/divdi3.c
@@ -14,12 +14,9 @@
// Returns: a / b
-COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) {
- const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1;
- di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0
- di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0
- a = (a ^ s_a) - s_a; // negate if s_a == -1
- b = (b ^ s_b) - s_b; // negate if s_b == -1
- s_a ^= s_b; // sign of quotient
- return (__udivmoddi4(a, b, (du_int *)0) ^ s_a) - s_a; // negate if s_a == -1
-}
+#define fixint_t di_int
+#define fixuint_t du_int
+#define COMPUTE_UDIV(a, b) __udivmoddi4((a), (b), (du_int *)0)
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) { return __divXi3(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/divmoddi4.c b/system/lib/compiler-rt/lib/builtins/divmoddi4.c
index 7f333510c0034..e7cbbb1aaa304 100644
--- a/system/lib/compiler-rt/lib/builtins/divmoddi4.c
+++ b/system/lib/compiler-rt/lib/builtins/divmoddi4.c
@@ -15,7 +15,14 @@
// Returns: a / b, *rem = a % b
COMPILER_RT_ABI di_int __divmoddi4(di_int a, di_int b, di_int *rem) {
- di_int d = __divdi3(a, b);
- *rem = a - (d * b);
- return d;
+ const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1;
+ di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0
+ di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0
+ a = (a ^ s_a) - s_a; // negate if s_a == -1
+ b = (b ^ s_b) - s_b; // negate if s_b == -1
+ s_b ^= s_a; // sign of quotient
+ du_int r;
+ di_int q = (__udivmoddi4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1
+ *rem = (r ^ s_a) - s_a; // negate if s_a == -1
+ return q;
}
diff --git a/system/lib/compiler-rt/lib/builtins/divmodsi4.c b/system/lib/compiler-rt/lib/builtins/divmodsi4.c
index 402eed22fe7a0..a85e2993b4e9b 100644
--- a/system/lib/compiler-rt/lib/builtins/divmodsi4.c
+++ b/system/lib/compiler-rt/lib/builtins/divmodsi4.c
@@ -16,7 +16,14 @@
// Returns: a / b, *rem = a % b
COMPILER_RT_ABI si_int __divmodsi4(si_int a, si_int b, si_int *rem) {
- si_int d = __divsi3(a, b);
- *rem = a - (d * b);
- return d;
+ const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1;
+ si_int s_a = a >> bits_in_word_m1; // s_a = a < 0 ? -1 : 0
+ si_int s_b = b >> bits_in_word_m1; // s_b = b < 0 ? -1 : 0
+ a = (a ^ s_a) - s_a; // negate if s_a == -1
+ b = (b ^ s_b) - s_b; // negate if s_b == -1
+ s_b ^= s_a; // sign of quotient
+ su_int r;
+ si_int q = (__udivmodsi4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1
+ *rem = (r ^ s_a) - s_a; // negate if s_a == -1
+ return q;
}
diff --git a/system/lib/compiler-rt/lib/builtins/divmodti4.c b/system/lib/compiler-rt/lib/builtins/divmodti4.c
new file mode 100644
index 0000000000000..b243ba4ef8537
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/divmodti4.c
@@ -0,0 +1,32 @@
+//===-- divmodti4.c - Implement __divmodti4 -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements __divmodti4 for the compiler_rt library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_lib.h"
+
+#ifdef CRT_HAS_128BIT
+
+// Returns: a / b, *rem = a % b
+
+COMPILER_RT_ABI ti_int __divmodti4(ti_int a, ti_int b, ti_int *rem) {
+ const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1;
+ ti_int s_a = a >> bits_in_tword_m1; // s_a = a < 0 ? -1 : 0
+ ti_int s_b = b >> bits_in_tword_m1; // s_b = b < 0 ? -1 : 0
+ a = (a ^ s_a) - s_a; // negate if s_a == -1
+ b = (b ^ s_b) - s_b; // negate if s_b == -1
+ s_b ^= s_a; // sign of quotient
+ tu_int r;
+ ti_int q = (__udivmodti4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1
+ *rem = (r ^ s_a) - s_a; // negate if s_a == -1
+ return q;
+}
+
+#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/divsf3.c b/system/lib/compiler-rt/lib/builtins/divsf3.c
index 593f93b45ac24..5744c015240b6 100644
--- a/system/lib/compiler-rt/lib/builtins/divsf3.c
+++ b/system/lib/compiler-rt/lib/builtins/divsf3.c
@@ -9,181 +9,17 @@
// This file implements single-precision soft-float division
// with the IEEE-754 default rounding (to nearest, ties to even).
//
-// For simplicity, this implementation currently flushes denormals to zero.
-// It should be a fairly straightforward exercise to implement gradual
-// underflow with correct rounding.
-//
//===----------------------------------------------------------------------===//
#define SINGLE_PRECISION
-#include "fp_lib.h"
-
-COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) {
-
- const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
- const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
- const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
-
- rep_t aSignificand = toRep(a) & significandMask;
- rep_t bSignificand = toRep(b) & significandMask;
- int scale = 0;
-
- // Detect if a or b is zero, denormal, infinity, or NaN.
- if (aExponent - 1U >= maxExponent - 1U ||
- bExponent - 1U >= maxExponent - 1U) {
-
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
-
- // NaN / anything = qNaN
- if (aAbs > infRep)
- return fromRep(toRep(a) | quietBit);
- // anything / NaN = qNaN
- if (bAbs > infRep)
- return fromRep(toRep(b) | quietBit);
-
- if (aAbs == infRep) {
- // infinity / infinity = NaN
- if (bAbs == infRep)
- return fromRep(qnanRep);
- // infinity / anything else = +/- infinity
- else
- return fromRep(aAbs | quotientSign);
- }
-
- // anything else / infinity = +/- 0
- if (bAbs == infRep)
- return fromRep(quotientSign);
-
- if (!aAbs) {
- // zero / zero = NaN
- if (!bAbs)
- return fromRep(qnanRep);
- // zero / anything else = +/- zero
- else
- return fromRep(quotientSign);
- }
- // anything else / zero = +/- infinity
- if (!bAbs)
- return fromRep(infRep | quotientSign);
-
- // One or both of a or b is denormal. The other (if applicable) is a
- // normal number. Renormalize one or both of a and b, and set scale to
- // include the necessary exponent adjustment.
- if (aAbs < implicitBit)
- scale += normalize(&aSignificand);
- if (bAbs < implicitBit)
- scale -= normalize(&bSignificand);
- }
-
- // Set the implicit significand bit. If we fell through from the
- // denormal path it was already set by normalize( ), but setting it twice
- // won't hurt anything.
- aSignificand |= implicitBit;
- bSignificand |= implicitBit;
- int quotientExponent = aExponent - bExponent + scale;
- // 0x7504F333 / 2^32 + 1 = 3/4 + 1/sqrt(2)
-
- // Align the significand of b as a Q31 fixed-point number in the range
- // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax
- // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This
- // is accurate to about 3.5 binary digits.
- uint32_t q31b = bSignificand << 8;
- uint32_t reciprocal = UINT32_C(0x7504f333) - q31b;
-
- // Now refine the reciprocal estimate using a Newton-Raphson iteration:
- //
- // x1 = x0 * (2 - x0 * b)
- //
- // This doubles the number of correct binary digits in the approximation
- // with each iteration.
- uint32_t correction;
- correction = -((uint64_t)reciprocal * q31b >> 32);
- reciprocal = (uint64_t)reciprocal * correction >> 31;
- correction = -((uint64_t)reciprocal * q31b >> 32);
- reciprocal = (uint64_t)reciprocal * correction >> 31;
- correction = -((uint64_t)reciprocal * q31b >> 32);
- reciprocal = (uint64_t)reciprocal * correction >> 31;
-
- // Adust the final 32-bit reciprocal estimate downward to ensure that it is
- // strictly smaller than the infinitely precise exact reciprocal. Because
- // the computation of the Newton-Raphson step is truncating at every step,
- // this adjustment is small; most of the work is already done.
- reciprocal -= 2;
-
- // The numerical reciprocal is accurate to within 2^-28, lies in the
- // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller
- // than the true reciprocal of b. Multiplying a by this reciprocal thus
- // gives a numerical q = a/b in Q24 with the following properties:
- //
- // 1. q < a/b
- // 2. q is in the interval [0x1.000000eep-1, 0x1.fffffffcp0)
- // 3. The error in q is at most 2^-24 + 2^-27 -- the 2^24 term comes
- // from the fact that we truncate the product, and the 2^27 term
- // is the error in the reciprocal of b scaled by the maximum
- // possible value of a. As a consequence of this error bound,
- // either q or nextafter(q) is the correctly rounded.
- rep_t quotient = (uint64_t)reciprocal * (aSignificand << 1) >> 32;
-
- // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0).
- // In either case, we are going to compute a residual of the form
- //
- // r = a - q*b
- //
- // We know from the construction of q that r satisfies:
- //
- // 0 <= r < ulp(q)*b
- //
- // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we
- // already have the correct result. The exact halfway case cannot occur.
- // We also take this time to right shift quotient if it falls in the [1,2)
- // range and adjust the exponent accordingly.
- rep_t residual;
- if (quotient < (implicitBit << 1)) {
- residual = (aSignificand << 24) - quotient * bSignificand;
- quotientExponent--;
- } else {
- quotient >>= 1;
- residual = (aSignificand << 23) - quotient * bSignificand;
- }
-
- const int writtenExponent = quotientExponent + exponentBias;
- if (writtenExponent >= maxExponent) {
- // If we have overflowed the exponent, return infinity.
- return fromRep(infRep | quotientSign);
- }
+#define NUMBER_OF_HALF_ITERATIONS 0
+#define NUMBER_OF_FULL_ITERATIONS 3
+#define USE_NATIVE_FULL_ITERATIONS
- else if (writtenExponent < 1) {
- if (writtenExponent == 0) {
- // Check whether the rounded result is normal.
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Round.
- absResult += round;
- if (absResult & ~significandMask) {
- // The rounded result is normal; return it.
- return fromRep(absResult | quotientSign);
- }
- }
- // Flush denormals to zero. In the future, it would be nice to add
- // code to round them correctly.
- return fromRep(quotientSign);
- }
+#include "fp_div_impl.inc"
- else {
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Insert the exponent.
- absResult |= (rep_t)writtenExponent << significandBits;
- // Round.
- absResult += round;
- // Insert the sign and return.
- return fromRep(absResult | quotientSign);
- }
-}
+COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) { return __divXf3__(a, b); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
diff --git a/system/lib/compiler-rt/lib/builtins/divsi3.c b/system/lib/compiler-rt/lib/builtins/divsi3.c
index b97e11119f0eb..f514407477f33 100644
--- a/system/lib/compiler-rt/lib/builtins/divsi3.c
+++ b/system/lib/compiler-rt/lib/builtins/divsi3.c
@@ -14,21 +14,16 @@
// Returns: a / b
-COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b) {
- const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1;
- si_int s_a = a >> bits_in_word_m1; // s_a = a < 0 ? -1 : 0
- si_int s_b = b >> bits_in_word_m1; // s_b = b < 0 ? -1 : 0
- a = (a ^ s_a) - s_a; // negate if s_a == -1
- b = (b ^ s_b) - s_b; // negate if s_b == -1
- s_a ^= s_b; // sign of quotient
- //
- // On CPUs without unsigned hardware division support,
- // this calls __udivsi3 (notice the cast to su_int).
- // On CPUs with unsigned hardware division support,
- // this uses the unsigned division instruction.
- //
- return ((su_int)a / (su_int)b ^ s_a) - s_a; // negate if s_a == -1
-}
+#define fixint_t si_int
+#define fixuint_t su_int
+// On CPUs without unsigned hardware division support,
+// this calls __udivsi3 (notice the cast to su_int).
+// On CPUs with unsigned hardware division support,
+// this uses the unsigned division instruction.
+#define COMPUTE_UDIV(a, b) ((su_int)(a) / (su_int)(b))
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b) { return __divXi3(a, b); }
#if defined(__ARM_EABI__)
COMPILER_RT_ALIAS(__divsi3, __aeabi_idiv)
diff --git a/system/lib/compiler-rt/lib/builtins/divtf3.c b/system/lib/compiler-rt/lib/builtins/divtf3.c
index ce462d4d46c12..5bcc9a8e4aa18 100644
--- a/system/lib/compiler-rt/lib/builtins/divtf3.c
+++ b/system/lib/compiler-rt/lib/builtins/divtf3.c
@@ -9,213 +9,18 @@
// This file implements quad-precision soft-float division
// with the IEEE-754 default rounding (to nearest, ties to even).
//
-// For simplicity, this implementation currently flushes denormals to zero.
-// It should be a fairly straightforward exercise to implement gradual
-// underflow with correct rounding.
-//
//===----------------------------------------------------------------------===//
#define QUAD_PRECISION
#include "fp_lib.h"
#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
-COMPILER_RT_ABI fp_t __divtf3(fp_t a, fp_t b) {
-
- const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
- const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
- const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
-
- rep_t aSignificand = toRep(a) & significandMask;
- rep_t bSignificand = toRep(b) & significandMask;
- int scale = 0;
-
- // Detect if a or b is zero, denormal, infinity, or NaN.
- if (aExponent - 1U >= maxExponent - 1U ||
- bExponent - 1U >= maxExponent - 1U) {
-
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
-
- // NaN / anything = qNaN
- if (aAbs > infRep)
- return fromRep(toRep(a) | quietBit);
- // anything / NaN = qNaN
- if (bAbs > infRep)
- return fromRep(toRep(b) | quietBit);
-
- if (aAbs == infRep) {
- // infinity / infinity = NaN
- if (bAbs == infRep)
- return fromRep(qnanRep);
- // infinity / anything else = +/- infinity
- else
- return fromRep(aAbs | quotientSign);
- }
-
- // anything else / infinity = +/- 0
- if (bAbs == infRep)
- return fromRep(quotientSign);
-
- if (!aAbs) {
- // zero / zero = NaN
- if (!bAbs)
- return fromRep(qnanRep);
- // zero / anything else = +/- zero
- else
- return fromRep(quotientSign);
- }
- // anything else / zero = +/- infinity
- if (!bAbs)
- return fromRep(infRep | quotientSign);
-
- // One or both of a or b is denormal. The other (if applicable) is a
- // normal number. Renormalize one or both of a and b, and set scale to
- // include the necessary exponent adjustment.
- if (aAbs < implicitBit)
- scale += normalize(&aSignificand);
- if (bAbs < implicitBit)
- scale -= normalize(&bSignificand);
- }
-
- // Set the implicit significand bit. If we fell through from the
- // denormal path it was already set by normalize( ), but setting it twice
- // won't hurt anything.
- aSignificand |= implicitBit;
- bSignificand |= implicitBit;
- int quotientExponent = aExponent - bExponent + scale;
-
- // Align the significand of b as a Q63 fixed-point number in the range
- // [1, 2.0) and get a Q64 approximate reciprocal using a small minimax
- // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This
- // is accurate to about 3.5 binary digits.
- const uint64_t q63b = bSignificand >> 49;
- uint64_t recip64 = UINT64_C(0x7504f333F9DE6484) - q63b;
- // 0x7504f333F9DE6484 / 2^64 + 1 = 3/4 + 1/sqrt(2)
-
- // Now refine the reciprocal estimate using a Newton-Raphson iteration:
- //
- // x1 = x0 * (2 - x0 * b)
- //
- // This doubles the number of correct binary digits in the approximation
- // with each iteration.
- uint64_t correction64;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
-
- // The reciprocal may have overflowed to zero if the upper half of b is
- // exactly 1.0. This would sabatoge the full-width final stage of the
- // computation that follows, so we adjust the reciprocal down by one bit.
- recip64--;
-
- // We need to perform one more iteration to get us to 112 binary digits;
- // The last iteration needs to happen with extra precision.
- const uint64_t q127blo = bSignificand << 15;
- rep_t correction, reciprocal;
-
- // NOTE: This operation is equivalent to __multi3, which is not implemented
- // in some architechure
- rep_t r64q63, r64q127, r64cH, r64cL, dummy;
- wideMultiply((rep_t)recip64, (rep_t)q63b, &dummy, &r64q63);
- wideMultiply((rep_t)recip64, (rep_t)q127blo, &dummy, &r64q127);
-
- correction = -(r64q63 + (r64q127 >> 64));
-
- uint64_t cHi = correction >> 64;
- uint64_t cLo = correction;
-
- wideMultiply((rep_t)recip64, (rep_t)cHi, &dummy, &r64cH);
- wideMultiply((rep_t)recip64, (rep_t)cLo, &dummy, &r64cL);
-
- reciprocal = r64cH + (r64cL >> 64);
-
- // Adjust the final 128-bit reciprocal estimate downward to ensure that it
- // is strictly smaller than the infinitely precise exact reciprocal. Because
- // the computation of the Newton-Raphson step is truncating at every step,
- // this adjustment is small; most of the work is already done.
- reciprocal -= 2;
-
- // The numerical reciprocal is accurate to within 2^-112, lies in the
- // interval [0.5, 1.0), and is strictly smaller than the true reciprocal
- // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b
- // in Q127 with the following properties:
- //
- // 1. q < a/b
- // 2. q is in the interval [0.5, 2.0)
- // 3. The error in q is bounded away from 2^-113 (actually, we have a
- // couple of bits to spare, but this is all we need).
-
- // We need a 128 x 128 multiply high to compute q, which isn't a basic
- // operation in C, so we need to be a little bit fussy.
- rep_t quotient, quotientLo;
- wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo);
-
- // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0).
- // In either case, we are going to compute a residual of the form
- //
- // r = a - q*b
- //
- // We know from the construction of q that r satisfies:
- //
- // 0 <= r < ulp(q)*b
- //
- // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we
- // already have the correct result. The exact halfway case cannot occur.
- // We also take this time to right shift quotient if it falls in the [1,2)
- // range and adjust the exponent accordingly.
- rep_t residual;
- rep_t qb;
- if (quotient < (implicitBit << 1)) {
- wideMultiply(quotient, bSignificand, &dummy, &qb);
- residual = (aSignificand << 113) - qb;
- quotientExponent--;
- } else {
- quotient >>= 1;
- wideMultiply(quotient, bSignificand, &dummy, &qb);
- residual = (aSignificand << 112) - qb;
- }
+#define NUMBER_OF_HALF_ITERATIONS 4
+#define NUMBER_OF_FULL_ITERATIONS 1
- const int writtenExponent = quotientExponent + exponentBias;
+#include "fp_div_impl.inc"
- if (writtenExponent >= maxExponent) {
- // If we have overflowed the exponent, return infinity.
- return fromRep(infRep | quotientSign);
- } else if (writtenExponent < 1) {
- if (writtenExponent == 0) {
- // Check whether the rounded result is normal.
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Round.
- absResult += round;
- if (absResult & ~significandMask) {
- // The rounded result is normal; return it.
- return fromRep(absResult | quotientSign);
- }
- }
- // Flush denormals to zero. In the future, it would be nice to add
- // code to round them correctly.
- return fromRep(quotientSign);
- } else {
- const bool round = (residual << 1) >= bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Insert the exponent.
- absResult |= (rep_t)writtenExponent << significandBits;
- // Round.
- absResult += round;
- // Insert the sign and return.
- const fp_t result = fromRep(absResult | quotientSign);
- return result;
- }
-}
+COMPILER_RT_ABI fp_t __divtf3(fp_t a, fp_t b) { return __divXf3__(a, b); }
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/divti3.c b/system/lib/compiler-rt/lib/builtins/divti3.c
index 6d007fe346546..80f2130b590e2 100644
--- a/system/lib/compiler-rt/lib/builtins/divti3.c
+++ b/system/lib/compiler-rt/lib/builtins/divti3.c
@@ -16,14 +16,11 @@
// Returns: a / b
-COMPILER_RT_ABI ti_int __divti3(ti_int a, ti_int b) {
- const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1;
- ti_int s_a = a >> bits_in_tword_m1; // s_a = a < 0 ? -1 : 0
- ti_int s_b = b >> bits_in_tword_m1; // s_b = b < 0 ? -1 : 0
- a = (a ^ s_a) - s_a; // negate if s_a == -1
- b = (b ^ s_b) - s_b; // negate if s_b == -1
- s_a ^= s_b; // sign of quotient
- return (__udivmodti4(a, b, (tu_int *)0) ^ s_a) - s_a; // negate if s_a == -1
-}
+#define fixint_t ti_int
+#define fixuint_t tu_int
+#define COMPUTE_UDIV(a, b) __udivmodti4((a), (b), (tu_int *)0)
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI ti_int __divti3(ti_int a, ti_int b) { return __divXi3(a, b); }
#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/emutls.c b/system/lib/compiler-rt/lib/builtins/emutls.c
index e0aa19155f7d3..98cabd917d6ce 100644
--- a/system/lib/compiler-rt/lib/builtins/emutls.c
+++ b/system/lib/compiler-rt/lib/builtins/emutls.c
@@ -182,9 +182,10 @@ static void emutls_exit(void) {
}
}
-#pragma warning(push)
-#pragma warning(disable : 4100)
static BOOL CALLBACK emutls_init(PINIT_ONCE p0, PVOID p1, PVOID *p2) {
+ (void)p0;
+ (void)p1;
+ (void)p2;
emutls_mutex =
(LPCRITICAL_SECTION)_aligned_malloc(sizeof(CRITICAL_SECTION), 16);
if (!emutls_mutex) {
@@ -251,8 +252,6 @@ static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) {
#endif // __ATOMIC_RELEASE
-#pragma warning(pop)
-
#endif // _WIN32
static size_t emutls_num_object = 0; // number of allocated TLS objects
diff --git a/system/lib/compiler-rt/lib/builtins/extendhfsf2.c b/system/lib/compiler-rt/lib/builtins/extendhfsf2.c
index 7c1a76eb58517..0159ab09d3ebb 100644
--- a/system/lib/compiler-rt/lib/builtins/extendhfsf2.c
+++ b/system/lib/compiler-rt/lib/builtins/extendhfsf2.c
@@ -12,15 +12,15 @@
// Use a forwarding definition and noinline to implement a poor man's alias,
// as there isn't a good cross-platform way of defining one.
-COMPILER_RT_ABI NOINLINE float __extendhfsf2(uint16_t a) {
+COMPILER_RT_ABI NOINLINE float __extendhfsf2(src_t a) {
return __extendXfYf2__(a);
}
-COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { return __extendhfsf2(a); }
+COMPILER_RT_ABI float __gnu_h2f_ieee(src_t a) { return __extendhfsf2(a); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); }
+AEABI_RTABI float __aeabi_h2f(src_t a) { return __extendhfsf2(a); }
#else
COMPILER_RT_ALIAS(__extendhfsf2, __aeabi_h2f)
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/extendhftf2.c b/system/lib/compiler-rt/lib/builtins/extendhftf2.c
new file mode 100644
index 0000000000000..aefe9737d34f9
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/extendhftf2.c
@@ -0,0 +1,23 @@
+//===-- lib/extendhftf2.c - half -> quad conversion ---------------*- C -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define QUAD_PRECISION
+#include "fp_lib.h"
+
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \
+ defined(COMPILER_RT_HAS_FLOAT16)
+#define SRC_HALF
+#define DST_QUAD
+#include "fp_extend_impl.inc"
+
+COMPILER_RT_ABI long double __extendhftf2(_Float16 a) {
+ return __extendXfYf2__(a);
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/builtins/ffsdi2.c b/system/lib/compiler-rt/lib/builtins/ffsdi2.c
index 9c1a242609567..beae5530430ed 100644
--- a/system/lib/compiler-rt/lib/builtins/ffsdi2.c
+++ b/system/lib/compiler-rt/lib/builtins/ffsdi2.c
@@ -15,13 +15,13 @@
// Returns: the index of the least significant 1-bit in a, or
// the value zero if a is zero. The least significant bit is index one.
-COMPILER_RT_ABI si_int __ffsdi2(di_int a) {
+COMPILER_RT_ABI int __ffsdi2(di_int a) {
dwords x;
x.all = a;
if (x.s.low == 0) {
if (x.s.high == 0)
return 0;
- return __builtin_ctz(x.s.high) + (1 + sizeof(si_int) * CHAR_BIT);
+ return ctzsi(x.s.high) + (1 + sizeof(si_int) * CHAR_BIT);
}
- return __builtin_ctz(x.s.low) + 1;
+ return ctzsi(x.s.low) + 1;
}
diff --git a/system/lib/compiler-rt/lib/builtins/ffssi2.c b/system/lib/compiler-rt/lib/builtins/ffssi2.c
index cba1f72fdc61a..ddb52927f8db9 100644
--- a/system/lib/compiler-rt/lib/builtins/ffssi2.c
+++ b/system/lib/compiler-rt/lib/builtins/ffssi2.c
@@ -15,9 +15,9 @@
// Returns: the index of the least significant 1-bit in a, or
// the value zero if a is zero. The least significant bit is index one.
-COMPILER_RT_ABI si_int __ffssi2(si_int a) {
+COMPILER_RT_ABI int __ffssi2(si_int a) {
if (a == 0) {
return 0;
}
- return __builtin_ctz(a) + 1;
+ return ctzsi(a) + 1;
}
diff --git a/system/lib/compiler-rt/lib/builtins/ffsti2.c b/system/lib/compiler-rt/lib/builtins/ffsti2.c
index a2d7ce08ada13..a2177d148a099 100644
--- a/system/lib/compiler-rt/lib/builtins/ffsti2.c
+++ b/system/lib/compiler-rt/lib/builtins/ffsti2.c
@@ -17,7 +17,7 @@
// Returns: the index of the least significant 1-bit in a, or
// the value zero if a is zero. The least significant bit is index one.
-COMPILER_RT_ABI si_int __ffsti2(ti_int a) {
+COMPILER_RT_ABI int __ffsti2(ti_int a) {
twords x;
x.all = a;
if (x.s.low == 0) {
diff --git a/system/lib/compiler-rt/lib/builtins/floatdidf.c b/system/lib/compiler-rt/lib/builtins/floatdidf.c
index 8f887314b9e1b..b2d8f2b44b6d3 100644
--- a/system/lib/compiler-rt/lib/builtins/floatdidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatdidf.c
@@ -87,7 +87,7 @@ COMPILER_RT_ABI double __floatdidf(di_int a) {
}
double_bits fb;
fb.u.s.high = ((su_int)s & 0x80000000) | // sign
- ((e + 1023) << 20) | // exponent
+ ((su_int)(e + 1023) << 20) | // exponent
((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
fb.u.s.low = (su_int)a; // mantissa-low
return fb.f;
diff --git a/system/lib/compiler-rt/lib/builtins/floatdisf.c b/system/lib/compiler-rt/lib/builtins/floatdisf.c
index cd9e0a3b78a50..faaa1bcb3c8ed 100644
--- a/system/lib/compiler-rt/lib/builtins/floatdisf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatdisf.c
@@ -26,7 +26,7 @@ COMPILER_RT_ABI float __floatdisf(di_int a) {
const di_int s = a >> (N - 1);
a = (a ^ s) - s;
int sd = N - __builtin_clzll(a); // number of significant digits
- int e = sd - 1; // exponent
+ si_int e = sd - 1; // exponent
if (sd > FLT_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
diff --git a/system/lib/compiler-rt/lib/builtins/floatsidf.c b/system/lib/compiler-rt/lib/builtins/floatsidf.c
index 2c66167d794dc..28cf32f6388b5 100644
--- a/system/lib/compiler-rt/lib/builtins/floatsidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatsidf.c
@@ -17,7 +17,7 @@
#include "int_lib.h"
-COMPILER_RT_ABI fp_t __floatsidf(int a) {
+COMPILER_RT_ABI fp_t __floatsidf(si_int a) {
const int aWidth = sizeof a * CHAR_BIT;
@@ -33,14 +33,14 @@ COMPILER_RT_ABI fp_t __floatsidf(int a) {
}
// Exponent of (fp_t)a is the width of abs(a).
- const int exponent = (aWidth - 1) - __builtin_clz(a);
+ const int exponent = (aWidth - 1) - clzsi(a);
rep_t result;
// Shift a into the significand field and clear the implicit bit. Extra
// cast to unsigned int is necessary to get the correct behavior for
// the input INT_MIN.
const int shift = significandBits - exponent;
- result = (rep_t)(unsigned int)a << shift ^ implicitBit;
+ result = (rep_t)(su_int)a << shift ^ implicitBit;
// Insert the exponent
result += (rep_t)(exponent + exponentBias) << significandBits;
@@ -50,7 +50,7 @@ COMPILER_RT_ABI fp_t __floatsidf(int a) {
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI fp_t __aeabi_i2d(int a) { return __floatsidf(a); }
+AEABI_RTABI fp_t __aeabi_i2d(si_int a) { return __floatsidf(a); }
#else
COMPILER_RT_ALIAS(__floatsidf, __aeabi_i2d)
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/floatundidf.c b/system/lib/compiler-rt/lib/builtins/floatundidf.c
index e7c6aae5ce382..4c445b118080b 100644
--- a/system/lib/compiler-rt/lib/builtins/floatundidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatundidf.c
@@ -90,7 +90,7 @@ COMPILER_RT_ABI double __floatundidf(du_int a) {
// a is now rounded to DBL_MANT_DIG bits
}
double_bits fb;
- fb.u.s.high = ((e + 1023) << 20) | // exponent
+ fb.u.s.high = ((su_int)(e + 1023) << 20) | // exponent
((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
fb.u.s.low = (su_int)a; // mantissa-low
return fb.f;
diff --git a/system/lib/compiler-rt/lib/builtins/floatundisf.c b/system/lib/compiler-rt/lib/builtins/floatundisf.c
index 87841b761ded5..00d61b0c63107 100644
--- a/system/lib/compiler-rt/lib/builtins/floatundisf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatundisf.c
@@ -24,7 +24,7 @@ COMPILER_RT_ABI float __floatundisf(du_int a) {
return 0.0F;
const unsigned N = sizeof(du_int) * CHAR_BIT;
int sd = N - __builtin_clzll(a); // number of significant digits
- int e = sd - 1; // 8 exponent
+ si_int e = sd - 1; // 8 exponent
if (sd > FLT_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
diff --git a/system/lib/compiler-rt/lib/builtins/floatunsidf.c b/system/lib/compiler-rt/lib/builtins/floatunsidf.c
index 2c01c3041434d..9b3e5fea0e45b 100644
--- a/system/lib/compiler-rt/lib/builtins/floatunsidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatunsidf.c
@@ -17,7 +17,7 @@
#include "int_lib.h"
-COMPILER_RT_ABI fp_t __floatunsidf(unsigned int a) {
+COMPILER_RT_ABI fp_t __floatunsidf(su_int a) {
const int aWidth = sizeof a * CHAR_BIT;
@@ -26,7 +26,7 @@ COMPILER_RT_ABI fp_t __floatunsidf(unsigned int a) {
return fromRep(0);
// Exponent of (fp_t)a is the width of abs(a).
- const int exponent = (aWidth - 1) - __builtin_clz(a);
+ const int exponent = (aWidth - 1) - clzsi(a);
rep_t result;
// Shift a into the significand field and clear the implicit bit.
@@ -40,7 +40,7 @@ COMPILER_RT_ABI fp_t __floatunsidf(unsigned int a) {
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { return __floatunsidf(a); }
+AEABI_RTABI fp_t __aeabi_ui2d(su_int a) { return __floatunsidf(a); }
#else
COMPILER_RT_ALIAS(__floatunsidf, __aeabi_ui2d)
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/fp_div_impl.inc b/system/lib/compiler-rt/lib/builtins/fp_div_impl.inc
new file mode 100644
index 0000000000000..29bcd1920edfb
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/fp_div_impl.inc
@@ -0,0 +1,419 @@
+//===-- fp_div_impl.inc - Floating point division -----------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements soft-float division with the IEEE-754 default
+// rounding (to nearest, ties to even).
+//
+//===----------------------------------------------------------------------===//
+
+#include "fp_lib.h"
+
+// The __divXf3__ function implements Newton-Raphson floating point division.
+// It uses 3 iterations for float32, 4 for float64 and 5 for float128,
+// respectively. Due to number of significant bits being roughly doubled
+// every iteration, the two modes are supported: N full-width iterations (as
+// it is done for float32 by default) and (N-1) half-width iteration plus one
+// final full-width iteration. It is expected that half-width integer
+// operations (w.r.t rep_t size) can be performed faster for some hardware but
+// they require error estimations to be computed separately due to larger
+// computational errors caused by truncating intermediate results.
+
+// Half the bit-size of rep_t
+#define HW (typeWidth / 2)
+// rep_t-sized bitmask with lower half of bits set to ones
+#define loMask (REP_C(-1) >> HW)
+
+#if NUMBER_OF_FULL_ITERATIONS < 1
+#error At least one full iteration is required
+#endif
+
+static __inline fp_t __divXf3__(fp_t a, fp_t b) {
+
+ const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
+ const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
+ const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
+
+ rep_t aSignificand = toRep(a) & significandMask;
+ rep_t bSignificand = toRep(b) & significandMask;
+ int scale = 0;
+
+ // Detect if a or b is zero, denormal, infinity, or NaN.
+ if (aExponent - 1U >= maxExponent - 1U ||
+ bExponent - 1U >= maxExponent - 1U) {
+
+ const rep_t aAbs = toRep(a) & absMask;
+ const rep_t bAbs = toRep(b) & absMask;
+
+ // NaN / anything = qNaN
+ if (aAbs > infRep)
+ return fromRep(toRep(a) | quietBit);
+ // anything / NaN = qNaN
+ if (bAbs > infRep)
+ return fromRep(toRep(b) | quietBit);
+
+ if (aAbs == infRep) {
+ // infinity / infinity = NaN
+ if (bAbs == infRep)
+ return fromRep(qnanRep);
+ // infinity / anything else = +/- infinity
+ else
+ return fromRep(aAbs | quotientSign);
+ }
+
+ // anything else / infinity = +/- 0
+ if (bAbs == infRep)
+ return fromRep(quotientSign);
+
+ if (!aAbs) {
+ // zero / zero = NaN
+ if (!bAbs)
+ return fromRep(qnanRep);
+ // zero / anything else = +/- zero
+ else
+ return fromRep(quotientSign);
+ }
+ // anything else / zero = +/- infinity
+ if (!bAbs)
+ return fromRep(infRep | quotientSign);
+
+ // One or both of a or b is denormal. The other (if applicable) is a
+ // normal number. Renormalize one or both of a and b, and set scale to
+ // include the necessary exponent adjustment.
+ if (aAbs < implicitBit)
+ scale += normalize(&aSignificand);
+ if (bAbs < implicitBit)
+ scale -= normalize(&bSignificand);
+ }
+
+ // Set the implicit significand bit. If we fell through from the
+ // denormal path it was already set by normalize( ), but setting it twice
+ // won't hurt anything.
+ aSignificand |= implicitBit;
+ bSignificand |= implicitBit;
+
+ int writtenExponent = (aExponent - bExponent + scale) + exponentBias;
+
+ const rep_t b_UQ1 = bSignificand << (typeWidth - significandBits - 1);
+
+ // Align the significand of b as a UQ1.(n-1) fixed-point number in the range
+ // [1.0, 2.0) and get a UQ0.n approximate reciprocal using a small minimax
+ // polynomial approximation: x0 = 3/4 + 1/sqrt(2) - b/2.
+ // The max error for this approximation is achieved at endpoints, so
+ // abs(x0(b) - 1/b) <= abs(x0(1) - 1/1) = 3/4 - 1/sqrt(2) = 0.04289...,
+ // which is about 4.5 bits.
+ // The initial approximation is between x0(1.0) = 0.9571... and x0(2.0) = 0.4571...
+
+ // Then, refine the reciprocal estimate using a quadratically converging
+ // Newton-Raphson iteration:
+ // x_{n+1} = x_n * (2 - x_n * b)
+ //
+ // Let b be the original divisor considered "in infinite precision" and
+ // obtained from IEEE754 representation of function argument (with the
+ // implicit bit set). Corresponds to rep_t-sized b_UQ1 represented in
+ // UQ1.(W-1).
+ //
+ // Let b_hw be an infinitely precise number obtained from the highest (HW-1)
+ // bits of divisor significand (with the implicit bit set). Corresponds to
+ // half_rep_t-sized b_UQ1_hw represented in UQ1.(HW-1) that is a **truncated**
+ // version of b_UQ1.
+ //
+ // Let e_n := x_n - 1/b_hw
+ // E_n := x_n - 1/b
+ // abs(E_n) <= abs(e_n) + (1/b_hw - 1/b)
+ // = abs(e_n) + (b - b_hw) / (b*b_hw)
+ // <= abs(e_n) + 2 * 2^-HW
+
+ // rep_t-sized iterations may be slower than the corresponding half-width
+ // variant depending on the handware and whether single/double/quad precision
+ // is selected.
+ // NB: Using half-width iterations increases computation errors due to
+ // rounding, so error estimations have to be computed taking the selected
+ // mode into account!
+#if NUMBER_OF_HALF_ITERATIONS > 0
+ // Starting with (n-1) half-width iterations
+ const half_rep_t b_UQ1_hw = bSignificand >> (significandBits + 1 - HW);
+
+ // C is (3/4 + 1/sqrt(2)) - 1 truncated to W0 fractional bits as UQ0.HW
+ // with W0 being either 16 or 32 and W0 <= HW.
+ // That is, C is the aforementioned 3/4 + 1/sqrt(2) constant (from which
+ // b/2 is subtracted to obtain x0) wrapped to [0, 1) range.
+#if defined(SINGLE_PRECISION)
+ // Use 16-bit initial estimation in case we are using half-width iterations
+ // for float32 division. This is expected to be useful for some 16-bit
+ // targets. Not used by default as it requires performing more work during
+ // rounding and would hardly help on regular 32- or 64-bit targets.
+ const half_rep_t C_hw = HALF_REP_C(0x7504);
+#else
+ // HW is at least 32. Shifting into the highest bits if needed.
+ const half_rep_t C_hw = HALF_REP_C(0x7504F333) << (HW - 32);
+#endif
+
+ // b >= 1, thus an upper bound for 3/4 + 1/sqrt(2) - b/2 is about 0.9572,
+ // so x0 fits to UQ0.HW without wrapping.
+ half_rep_t x_UQ0_hw = C_hw - (b_UQ1_hw /* exact b_hw/2 as UQ0.HW */);
+ // An e_0 error is comprised of errors due to
+ // * x0 being an inherently imprecise first approximation of 1/b_hw
+ // * C_hw being some (irrational) number **truncated** to W0 bits
+ // Please note that e_0 is calculated against the infinitely precise
+ // reciprocal of b_hw (that is, **truncated** version of b).
+ //
+ // e_0 <= 3/4 - 1/sqrt(2) + 2^-W0
+
+ // By construction, 1 <= b < 2
+ // f(x) = x * (2 - b*x) = 2*x - b*x^2
+ // f'(x) = 2 * (1 - b*x)
+ //
+ // On the [0, 1] interval, f(0) = 0,
+ // then it increses until f(1/b) = 1 / b, maximum on (0, 1),
+ // then it decreses to f(1) = 2 - b
+ //
+ // Let g(x) = x - f(x) = b*x^2 - x.
+ // On (0, 1/b), g(x) < 0 <=> f(x) > x
+ // On (1/b, 1], g(x) > 0 <=> f(x) < x
+ //
+ // For half-width iterations, b_hw is used instead of b.
+ REPEAT_N_TIMES(NUMBER_OF_HALF_ITERATIONS, {
+ // corr_UQ1_hw can be **larger** than 2 - b_hw*x by at most 1*Ulp
+ // of corr_UQ1_hw.
+ // "0.0 - (...)" is equivalent to "2.0 - (...)" in UQ1.(HW-1).
+ // On the other hand, corr_UQ1_hw should not overflow from 2.0 to 0.0 provided
+ // no overflow occurred earlier: ((rep_t)x_UQ0_hw * b_UQ1_hw >> HW) is
+ // expected to be strictly positive because b_UQ1_hw has its highest bit set
+ // and x_UQ0_hw should be rather large (it converges to 1/2 < 1/b_hw <= 1).
+ half_rep_t corr_UQ1_hw = 0 - ((rep_t)x_UQ0_hw * b_UQ1_hw >> HW);
+
+ // Now, we should multiply UQ0.HW and UQ1.(HW-1) numbers, naturally
+ // obtaining an UQ1.(HW-1) number and proving its highest bit could be
+ // considered to be 0 to be able to represent it in UQ0.HW.
+ // From the above analysis of f(x), if corr_UQ1_hw would be represented
+ // without any intermediate loss of precision (that is, in twice_rep_t)
+ // x_UQ0_hw could be at most [1.]000... if b_hw is exactly 1.0 and strictly
+ // less otherwise. On the other hand, to obtain [1.]000..., one have to pass
+ // 1/b_hw == 1.0 to f(x), so this cannot occur at all without overflow (due
+ // to 1.0 being not representable as UQ0.HW).
+ // The fact corr_UQ1_hw was virtually round up (due to result of
+ // multiplication being **first** truncated, then negated - to improve
+ // error estimations) can increase x_UQ0_hw by up to 2*Ulp of x_UQ0_hw.
+ x_UQ0_hw = (rep_t)x_UQ0_hw * corr_UQ1_hw >> (HW - 1);
+ // Now, either no overflow occurred or x_UQ0_hw is 0 or 1 in its half_rep_t
+ // representation. In the latter case, x_UQ0_hw will be either 0 or 1 after
+ // any number of iterations, so just subtract 2 from the reciprocal
+ // approximation after last iteration.
+
+ // In infinite precision, with 0 <= eps1, eps2 <= U = 2^-HW:
+ // corr_UQ1_hw = 2 - (1/b_hw + e_n) * b_hw + 2*eps1
+ // = 1 - e_n * b_hw + 2*eps1
+ // x_UQ0_hw = (1/b_hw + e_n) * (1 - e_n*b_hw + 2*eps1) - eps2
+ // = 1/b_hw - e_n + 2*eps1/b_hw + e_n - e_n^2*b_hw + 2*e_n*eps1 - eps2
+ // = 1/b_hw + 2*eps1/b_hw - e_n^2*b_hw + 2*e_n*eps1 - eps2
+ // e_{n+1} = -e_n^2*b_hw + 2*eps1/b_hw + 2*e_n*eps1 - eps2
+ // = 2*e_n*eps1 - (e_n^2*b_hw + eps2) + 2*eps1/b_hw
+ // \------ >0 -------/ \-- >0 ---/
+ // abs(e_{n+1}) <= 2*abs(e_n)*U + max(2*e_n^2 + U, 2 * U)
+ })
+ // For initial half-width iterations, U = 2^-HW
+ // Let abs(e_n) <= u_n * U,
+ // then abs(e_{n+1}) <= 2 * u_n * U^2 + max(2 * u_n^2 * U^2 + U, 2 * U)
+ // u_{n+1} <= 2 * u_n * U + max(2 * u_n^2 * U + 1, 2)
+
+ // Account for possible overflow (see above). For an overflow to occur for the
+ // first time, for "ideal" corr_UQ1_hw (that is, without intermediate
+ // truncation), the result of x_UQ0_hw * corr_UQ1_hw should be either maximum
+ // value representable in UQ0.HW or less by 1. This means that 1/b_hw have to
+ // be not below that value (see g(x) above), so it is safe to decrement just
+ // once after the final iteration. On the other hand, an effective value of
+ // divisor changes after this point (from b_hw to b), so adjust here.
+ x_UQ0_hw -= 1U;
+ rep_t x_UQ0 = (rep_t)x_UQ0_hw << HW;
+ x_UQ0 -= 1U;
+
+#else
+ // C is (3/4 + 1/sqrt(2)) - 1 truncated to 32 fractional bits as UQ0.n
+ const rep_t C = REP_C(0x7504F333) << (typeWidth - 32);
+ rep_t x_UQ0 = C - b_UQ1;
+ // E_0 <= 3/4 - 1/sqrt(2) + 2 * 2^-32
+#endif
+
+ // Error estimations for full-precision iterations are calculated just
+ // as above, but with U := 2^-W and taking extra decrementing into account.
+ // We need at least one such iteration.
+
+#ifdef USE_NATIVE_FULL_ITERATIONS
+ REPEAT_N_TIMES(NUMBER_OF_FULL_ITERATIONS, {
+ rep_t corr_UQ1 = 0 - ((twice_rep_t)x_UQ0 * b_UQ1 >> typeWidth);
+ x_UQ0 = (twice_rep_t)x_UQ0 * corr_UQ1 >> (typeWidth - 1);
+ })
+#else
+#if NUMBER_OF_FULL_ITERATIONS != 1
+#error Only a single emulated full iteration is supported
+#endif
+#if !(NUMBER_OF_HALF_ITERATIONS > 0)
+ // Cannot normally reach here: only one full-width iteration is requested and
+ // the total number of iterations should be at least 3 even for float32.
+#error Check NUMBER_OF_HALF_ITERATIONS, NUMBER_OF_FULL_ITERATIONS and USE_NATIVE_FULL_ITERATIONS.
+#endif
+ // Simulating operations on a twice_rep_t to perform a single final full-width
+ // iteration. Using ad-hoc multiplication implementations to take advantage
+ // of particular structure of operands.
+ rep_t blo = b_UQ1 & loMask;
+ // x_UQ0 = x_UQ0_hw * 2^HW - 1
+ // x_UQ0 * b_UQ1 = (x_UQ0_hw * 2^HW) * (b_UQ1_hw * 2^HW + blo) - b_UQ1
+ //
+ // <--- higher half ---><--- lower half --->
+ // [x_UQ0_hw * b_UQ1_hw]
+ // + [ x_UQ0_hw * blo ]
+ // - [ b_UQ1 ]
+ // = [ result ][.... discarded ...]
+ rep_t corr_UQ1 = 0U - ( (rep_t)x_UQ0_hw * b_UQ1_hw
+ + ((rep_t)x_UQ0_hw * blo >> HW)
+ - REP_C(1)); // account for *possible* carry
+ rep_t lo_corr = corr_UQ1 & loMask;
+ rep_t hi_corr = corr_UQ1 >> HW;
+ // x_UQ0 * corr_UQ1 = (x_UQ0_hw * 2^HW) * (hi_corr * 2^HW + lo_corr) - corr_UQ1
+ x_UQ0 = ((rep_t)x_UQ0_hw * hi_corr << 1)
+ + ((rep_t)x_UQ0_hw * lo_corr >> (HW - 1))
+ - REP_C(2); // 1 to account for the highest bit of corr_UQ1 can be 1
+ // 1 to account for possible carry
+ // Just like the case of half-width iterations but with possibility
+ // of overflowing by one extra Ulp of x_UQ0.
+ x_UQ0 -= 1U;
+ // ... and then traditional fixup by 2 should work
+
+ // On error estimation:
+ // abs(E_{N-1}) <= (u_{N-1} + 2 /* due to conversion e_n -> E_n */) * 2^-HW
+ // + (2^-HW + 2^-W))
+ // abs(E_{N-1}) <= (u_{N-1} + 3.01) * 2^-HW
+
+ // Then like for the half-width iterations:
+ // With 0 <= eps1, eps2 < 2^-W
+ // E_N = 4 * E_{N-1} * eps1 - (E_{N-1}^2 * b + 4 * eps2) + 4 * eps1 / b
+ // abs(E_N) <= 2^-W * [ 4 * abs(E_{N-1}) + max(2 * abs(E_{N-1})^2 * 2^W + 4, 8)) ]
+ // abs(E_N) <= 2^-W * [ 4 * (u_{N-1} + 3.01) * 2^-HW + max(4 + 2 * (u_{N-1} + 3.01)^2, 8) ]
+#endif
+
+ // Finally, account for possible overflow, as explained above.
+ x_UQ0 -= 2U;
+
+ // u_n for different precisions (with N-1 half-width iterations):
+ // W0 is the precision of C
+ // u_0 = (3/4 - 1/sqrt(2) + 2^-W0) * 2^HW
+
+ // Estimated with bc:
+ // define half1(un) { return 2.0 * (un + un^2) / 2.0^hw + 1.0; }
+ // define half2(un) { return 2.0 * un / 2.0^hw + 2.0; }
+ // define full1(un) { return 4.0 * (un + 3.01) / 2.0^hw + 2.0 * (un + 3.01)^2 + 4.0; }
+ // define full2(un) { return 4.0 * (un + 3.01) / 2.0^hw + 8.0; }
+
+ // | f32 (0 + 3) | f32 (2 + 1) | f64 (3 + 1) | f128 (4 + 1)
+ // u_0 | < 184224974 | < 2812.1 | < 184224974 | < 791240234244348797
+ // u_1 | < 15804007 | < 242.7 | < 15804007 | < 67877681371350440
+ // u_2 | < 116308 | < 2.81 | < 116308 | < 499533100252317
+ // u_3 | < 7.31 | | < 7.31 | < 27054456580
+ // u_4 | | | | < 80.4
+ // Final (U_N) | same as u_3 | < 72 | < 218 | < 13920
+
+ // Add 2 to U_N due to final decrement.
+
+#if defined(SINGLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 2 && NUMBER_OF_FULL_ITERATIONS == 1
+#define RECIPROCAL_PRECISION REP_C(74)
+#elif defined(SINGLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 0 && NUMBER_OF_FULL_ITERATIONS == 3
+#define RECIPROCAL_PRECISION REP_C(10)
+#elif defined(DOUBLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 3 && NUMBER_OF_FULL_ITERATIONS == 1
+#define RECIPROCAL_PRECISION REP_C(220)
+#elif defined(QUAD_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 4 && NUMBER_OF_FULL_ITERATIONS == 1
+#define RECIPROCAL_PRECISION REP_C(13922)
+#else
+#error Invalid number of iterations
+#endif
+
+ // Suppose 1/b - P * 2^-W < x < 1/b + P * 2^-W
+ x_UQ0 -= RECIPROCAL_PRECISION;
+ // Now 1/b - (2*P) * 2^-W < x < 1/b
+ // FIXME Is x_UQ0 still >= 0.5?
+
+ rep_t quotient_UQ1, dummy;
+ wideMultiply(x_UQ0, aSignificand << 1, "ient_UQ1, &dummy);
+ // Now, a/b - 4*P * 2^-W < q < a/b for q= in UQ1.(SB+1+W).
+
+ // quotient_UQ1 is in [0.5, 2.0) as UQ1.(SB+1),
+ // adjust it to be in [1.0, 2.0) as UQ1.SB.
+ rep_t residualLo;
+ if (quotient_UQ1 < (implicitBit << 1)) {
+ // Highest bit is 0, so just reinterpret quotient_UQ1 as UQ1.SB,
+ // effectively doubling its value as well as its error estimation.
+ residualLo = (aSignificand << (significandBits + 1)) - quotient_UQ1 * bSignificand;
+ writtenExponent -= 1;
+ aSignificand <<= 1;
+ } else {
+ // Highest bit is 1 (the UQ1.(SB+1) value is in [1, 2)), convert it
+ // to UQ1.SB by right shifting by 1. Least significant bit is omitted.
+ quotient_UQ1 >>= 1;
+ residualLo = (aSignificand << significandBits) - quotient_UQ1 * bSignificand;
+ }
+ // NB: residualLo is calculated above for the normal result case.
+ // It is re-computed on denormal path that is expected to be not so
+ // performance-sensitive.
+
+ // Now, q cannot be greater than a/b and can differ by at most 8*P * 2^-W + 2^-SB
+ // Each NextAfter() increments the floating point value by at least 2^-SB
+ // (more, if exponent was incremented).
+ // Different cases (<---> is of 2^-SB length, * = a/b that is shown as a midpoint):
+ // q
+ // | | * | | | | |
+ // <---> 2^t
+ // | | | | | * | |
+ // q
+ // To require at most one NextAfter(), an error should be less than 1.5 * 2^-SB.
+ // (8*P) * 2^-W + 2^-SB < 1.5 * 2^-SB
+ // (8*P) * 2^-W < 0.5 * 2^-SB
+ // P < 2^(W-4-SB)
+ // Generally, for at most R NextAfter() to be enough,
+ // P < (2*R - 1) * 2^(W-4-SB)
+ // For f32 (0+3): 10 < 32 (OK)
+ // For f32 (2+1): 32 < 74 < 32 * 3, so two NextAfter() are required
+ // For f64: 220 < 256 (OK)
+ // For f128: 4096 * 3 < 13922 < 4096 * 5 (three NextAfter() are required)
+
+ // If we have overflowed the exponent, return infinity
+ if (writtenExponent >= maxExponent)
+ return fromRep(infRep | quotientSign);
+
+ // Now, quotient_UQ1_SB <= the correctly-rounded result
+ // and may need taking NextAfter() up to 3 times (see error estimates above)
+ // r = a - b * q
+ rep_t absResult;
+ if (writtenExponent > 0) {
+ // Clear the implicit bit
+ absResult = quotient_UQ1 & significandMask;
+ // Insert the exponent
+ absResult |= (rep_t)writtenExponent << significandBits;
+ residualLo <<= 1;
+ } else {
+ // Prevent shift amount from being negative
+ if (significandBits + writtenExponent < 0)
+ return fromRep(quotientSign);
+
+ absResult = quotient_UQ1 >> (-writtenExponent + 1);
+
+ // multiplied by two to prevent shift amount to be negative
+ residualLo = (aSignificand << (significandBits + writtenExponent)) - (absResult * bSignificand << 1);
+ }
+
+ // Round
+ residualLo += absResult & 1; // tie to even
+ // The above line conditionally turns the below LT comparison into LTE
+ absResult += residualLo > bSignificand;
+#if defined(QUAD_PRECISION) || (defined(SINGLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS > 0)
+ // Do not round Infinity to NaN
+ absResult += absResult < infRep && residualLo > (2 + 1) * bSignificand;
+#endif
+#if defined(QUAD_PRECISION)
+ absResult += absResult < infRep && residualLo > (4 + 1) * bSignificand;
+#endif
+ return fromRep(absResult | quotientSign);
+}
diff --git a/system/lib/compiler-rt/lib/builtins/fp_extend.h b/system/lib/compiler-rt/lib/builtins/fp_extend.h
index d2083c4267225..aad4436730dd4 100644
--- a/system/lib/compiler-rt/lib/builtins/fp_extend.h
+++ b/system/lib/compiler-rt/lib/builtins/fp_extend.h
@@ -21,7 +21,7 @@ typedef float src_t;
typedef uint32_t src_rep_t;
#define SRC_REP_C UINT32_C
static const int srcSigBits = 23;
-#define src_rep_t_clz __builtin_clz
+#define src_rep_t_clz clzsi
#elif defined SRC_DOUBLE
typedef double src_t;
@@ -40,7 +40,11 @@ static __inline int src_rep_t_clz(src_rep_t a) {
}
#elif defined SRC_HALF
+#ifdef COMPILER_RT_HAS_FLOAT16
+typedef _Float16 src_t;
+#else
typedef uint16_t src_t;
+#endif
typedef uint16_t src_rep_t;
#define SRC_REP_C UINT16_C
static const int srcSigBits = 10;
diff --git a/system/lib/compiler-rt/lib/builtins/fp_lib.h b/system/lib/compiler-rt/lib/builtins/fp_lib.h
index e2a906681c46d..f22feafa4e694 100644
--- a/system/lib/compiler-rt/lib/builtins/fp_lib.h
+++ b/system/lib/compiler-rt/lib/builtins/fp_lib.h
@@ -40,13 +40,16 @@
#if defined SINGLE_PRECISION
+typedef uint16_t half_rep_t;
typedef uint32_t rep_t;
+typedef uint64_t twice_rep_t;
typedef int32_t srep_t;
typedef float fp_t;
+#define HALF_REP_C UINT16_C
#define REP_C UINT32_C
#define significandBits 23
-static __inline int rep_clz(rep_t a) { return __builtin_clz(a); }
+static __inline int rep_clz(rep_t a) { return clzsi(a); }
// 32x32 --> 64 bit multiply
static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
@@ -58,9 +61,11 @@ COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b);
#elif defined DOUBLE_PRECISION
+typedef uint32_t half_rep_t;
typedef uint64_t rep_t;
typedef int64_t srep_t;
typedef double fp_t;
+#define HALF_REP_C UINT32_C
#define REP_C UINT64_C
#define significandBits 52
@@ -69,9 +74,9 @@ static __inline int rep_clz(rep_t a) {
return __builtin_clzl(a);
#else
if (a & REP_C(0xffffffff00000000))
- return __builtin_clz(a >> 32);
+ return clzsi(a >> 32);
else
- return 32 + __builtin_clz(a & REP_C(0xffffffff));
+ return 32 + clzsi(a & REP_C(0xffffffff));
#endif
}
@@ -102,9 +107,11 @@ COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b);
#elif defined QUAD_PRECISION
#if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__)
#define CRT_LDBL_128BIT
+typedef uint64_t half_rep_t;
typedef __uint128_t rep_t;
typedef __int128_t srep_t;
typedef long double fp_t;
+#define HALF_REP_C UINT64_C
#define REP_C (__uint128_t)
// Note: Since there is no explicit way to tell compiler the constant is a
// 128-bit integer, we let the constant be casted to 128-bit integer
diff --git a/system/lib/compiler-rt/lib/builtins/fp_mode.h b/system/lib/compiler-rt/lib/builtins/fp_mode.h
index 51bec0431a402..4ba682c384f2d 100644
--- a/system/lib/compiler-rt/lib/builtins/fp_mode.h
+++ b/system/lib/compiler-rt/lib/builtins/fp_mode.h
@@ -23,7 +23,7 @@ typedef enum {
FE_TOWARDZERO
} FE_ROUND_MODE;
-FE_ROUND_MODE __fe_getround();
-int __fe_raise_inexact();
+FE_ROUND_MODE __fe_getround(void);
+int __fe_raise_inexact(void);
#endif // FP_MODE_H
diff --git a/system/lib/compiler-rt/lib/builtins/fp_trunc.h b/system/lib/compiler-rt/lib/builtins/fp_trunc.h
index aca4c9b6e6777..00595edd5e018 100644
--- a/system/lib/compiler-rt/lib/builtins/fp_trunc.h
+++ b/system/lib/compiler-rt/lib/builtins/fp_trunc.h
@@ -50,7 +50,11 @@ typedef uint32_t dst_rep_t;
static const int dstSigBits = 23;
#elif defined DST_HALF
+#ifdef COMPILER_RT_HAS_FLOAT16
+typedef _Float16 dst_t;
+#else
typedef uint16_t dst_t;
+#endif
typedef uint16_t dst_rep_t;
#define DST_REP_C UINT16_C
static const int dstSigBits = 10;
diff --git a/system/lib/compiler-rt/lib/builtins/int_div_impl.inc b/system/lib/compiler-rt/lib/builtins/int_div_impl.inc
new file mode 100644
index 0000000000000..dc1f97cbeae53
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/int_div_impl.inc
@@ -0,0 +1,95 @@
+//===-- int_div_impl.inc - Integer division ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helpers used by __udivsi3, __umodsi3, __udivdi3, and __umodsi3.
+//
+//===----------------------------------------------------------------------===//
+
+#define clz(a) (sizeof(a) == sizeof(unsigned long long) ? __builtin_clzll(a) : clzsi(a))
+
+// Adapted from Figure 3-40 of The PowerPC Compiler Writer's Guide
+static __inline fixuint_t __udivXi3(fixuint_t n, fixuint_t d) {
+ const unsigned N = sizeof(fixuint_t) * CHAR_BIT;
+ // d == 0 cases are unspecified.
+ unsigned sr = (d ? clz(d) : N) - (n ? clz(n) : N);
+ // 0 <= sr <= N - 1 or sr is very large.
+ if (sr > N - 1) // n < d
+ return 0;
+ if (sr == N - 1) // d == 1
+ return n;
+ ++sr;
+ // 1 <= sr <= N - 1. Shifts do not trigger UB.
+ fixuint_t r = n >> sr;
+ n <<= N - sr;
+ fixuint_t carry = 0;
+ for (; sr > 0; --sr) {
+ r = (r << 1) | (n >> (N - 1));
+ n = (n << 1) | carry;
+ // Branch-less version of:
+ // carry = 0;
+ // if (r >= d) r -= d, carry = 1;
+ const fixint_t s = (fixint_t)(d - r - 1) >> (N - 1);
+ carry = s & 1;
+ r -= d & s;
+ }
+ n = (n << 1) | carry;
+ return n;
+}
+
+// Mostly identical to __udivXi3 but the return values are different.
+static __inline fixuint_t __umodXi3(fixuint_t n, fixuint_t d) {
+ const unsigned N = sizeof(fixuint_t) * CHAR_BIT;
+ // d == 0 cases are unspecified.
+ unsigned sr = (d ? clz(d) : N) - (n ? clz(n) : N);
+ // 0 <= sr <= N - 1 or sr is very large.
+ if (sr > N - 1) // n < d
+ return n;
+ if (sr == N - 1) // d == 1
+ return 0;
+ ++sr;
+ // 1 <= sr <= N - 1. Shifts do not trigger UB.
+ fixuint_t r = n >> sr;
+ n <<= N - sr;
+ fixuint_t carry = 0;
+ for (; sr > 0; --sr) {
+ r = (r << 1) | (n >> (N - 1));
+ n = (n << 1) | carry;
+ // Branch-less version of:
+ // carry = 0;
+ // if (r >= d) r -= d, carry = 1;
+ const fixint_t s = (fixint_t)(d - r - 1) >> (N - 1);
+ carry = s & 1;
+ r -= d & s;
+ }
+ return r;
+}
+
+#ifdef COMPUTE_UDIV
+static __inline fixint_t __divXi3(fixint_t a, fixint_t b) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT) - 1;
+ fixint_t s_a = a >> N; // s_a = a < 0 ? -1 : 0
+ fixint_t s_b = b >> N; // s_b = b < 0 ? -1 : 0
+ fixuint_t a_u = (fixuint_t)(a ^ s_a) + (-s_a); // negate if s_a == -1
+ fixuint_t b_u = (fixuint_t)(b ^ s_b) + (-s_b); // negate if s_b == -1
+ s_a ^= s_b; // sign of quotient
+ return (COMPUTE_UDIV(a_u, b_u) ^ s_a) + (-s_a); // negate if s_a == -1
+}
+#endif // COMPUTE_UDIV
+
+#ifdef ASSIGN_UMOD
+static __inline fixint_t __modXi3(fixint_t a, fixint_t b) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT) - 1;
+ fixint_t s = b >> N; // s = b < 0 ? -1 : 0
+ fixuint_t b_u = (fixuint_t)(b ^ s) + (-s); // negate if s == -1
+ s = a >> N; // s = a < 0 ? -1 : 0
+ fixuint_t a_u = (fixuint_t)(a ^ s) + (-s); // negate if s == -1
+ fixuint_t res;
+ ASSIGN_UMOD(res, a_u, b_u);
+ return (res ^ s) + (-s); // negate if s == -1
+}
+#endif // ASSIGN_UMOD
diff --git a/system/lib/compiler-rt/lib/builtins/int_lib.h b/system/lib/compiler-rt/lib/builtins/int_lib.h
index 3092f68c084a6..991c4a99ea6e2 100644
--- a/system/lib/compiler-rt/lib/builtins/int_lib.h
+++ b/system/lib/compiler-rt/lib/builtins/int_lib.h
@@ -48,12 +48,20 @@
#define XSTR(a) STR(a)
#define SYMBOL_NAME(name) XSTR(__USER_LABEL_PREFIX__) #name
-#if defined(__ELF__) || defined(__MINGW32__) || defined(__wasm__)
+#if defined(__ELF__) || defined(__MINGW32__) || defined(__wasm__) || \
+ defined(_AIX)
#define COMPILER_RT_ALIAS(name, aliasname) \
COMPILER_RT_ABI __typeof(name) aliasname __attribute__((__alias__(#name)));
#elif defined(__APPLE__)
+#if defined(VISIBILITY_HIDDEN)
+#define COMPILER_RT_ALIAS_VISIBILITY(name) \
+ __asm__(".private_extern " SYMBOL_NAME(name));
+#else
+#define COMPILER_RT_ALIAS_VISIBILITY(name)
+#endif
#define COMPILER_RT_ALIAS(name, aliasname) \
__asm__(".globl " SYMBOL_NAME(aliasname)); \
+ COMPILER_RT_ALIAS_VISIBILITY(aliasname) \
__asm__(SYMBOL_NAME(aliasname) " = " SYMBOL_NAME(name)); \
COMPILER_RT_ABI __typeof(name) aliasname;
#elif defined(_WIN32)
@@ -84,8 +92,8 @@
// Include internal utility function declarations.
#include "int_util.h"
-COMPILER_RT_ABI si_int __paritysi2(si_int a);
-COMPILER_RT_ABI si_int __paritydi2(di_int a);
+COMPILER_RT_ABI int __paritysi2(si_int a);
+COMPILER_RT_ABI int __paritydi2(di_int a);
COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b);
COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b);
@@ -94,7 +102,7 @@ COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d);
COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem);
COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem);
#ifdef CRT_HAS_128BIT
-COMPILER_RT_ABI si_int __clzti2(ti_int a);
+COMPILER_RT_ABI int __clzti2(ti_int a);
COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem);
#endif
@@ -102,14 +110,14 @@ COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem);
#if defined(_MSC_VER) && !defined(__clang__)
#include
-uint32_t __inline __builtin_ctz(uint32_t value) {
+int __inline __builtin_ctz(uint32_t value) {
unsigned long trailing_zero = 0;
if (_BitScanForward(&trailing_zero, value))
return trailing_zero;
return 32;
}
-uint32_t __inline __builtin_clz(uint32_t value) {
+int __inline __builtin_clz(uint32_t value) {
unsigned long leading_zero = 0;
if (_BitScanReverse(&leading_zero, value))
return 31 - leading_zero;
@@ -117,14 +125,14 @@ uint32_t __inline __builtin_clz(uint32_t value) {
}
#if defined(_M_ARM) || defined(_M_X64)
-uint32_t __inline __builtin_clzll(uint64_t value) {
+int __inline __builtin_clzll(uint64_t value) {
unsigned long leading_zero = 0;
if (_BitScanReverse64(&leading_zero, value))
return 63 - leading_zero;
return 64;
}
#else
-uint32_t __inline __builtin_clzll(uint64_t value) {
+int __inline __builtin_clzll(uint64_t value) {
if (value == 0)
return 64;
uint32_t msh = (uint32_t)(value >> 32);
diff --git a/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc b/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc
new file mode 100644
index 0000000000000..567d8b9e6e603
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc
@@ -0,0 +1,49 @@
+//===-- int_mulo_impl.inc - Implement __mulo[sdt]i4 ---------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper used by __mulosi4, __mulodi4 and __muloti4.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_lib.h"
+
+// Returns: a * b
+
+// Effects: sets *overflow to 1 if a * b overflows
+
+static __inline fixint_t __muloXi4(fixint_t a, fixint_t b, int *overflow) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
+ const fixint_t MIN = (fixint_t)1 << (N - 1);
+ const fixint_t MAX = ~MIN;
+ *overflow = 0;
+ fixint_t result = a * b;
+ if (a == MIN) {
+ if (b != 0 && b != 1)
+ *overflow = 1;
+ return result;
+ }
+ if (b == MIN) {
+ if (a != 0 && a != 1)
+ *overflow = 1;
+ return result;
+ }
+ fixint_t sa = a >> (N - 1);
+ fixint_t abs_a = (a ^ sa) - sa;
+ fixint_t sb = b >> (N - 1);
+ fixint_t abs_b = (b ^ sb) - sb;
+ if (abs_a < 2 || abs_b < 2)
+ return result;
+ if (sa == sb) {
+ if (abs_a > MAX / abs_b)
+ *overflow = 1;
+ } else {
+ if (abs_a > MIN / -abs_b)
+ *overflow = 1;
+ }
+ return result;
+}
diff --git a/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc b/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc
new file mode 100644
index 0000000000000..1e920716ec499
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc
@@ -0,0 +1,47 @@
+//===-- int_mulv_impl.inc - Implement __mulv[sdt]i3 ---------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper used by __mulvsi3, __mulvdi3 and __mulvti3.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_lib.h"
+
+// Returns: a * b
+
+// Effects: aborts if a * b overflows
+
+static __inline fixint_t __mulvXi3(fixint_t a, fixint_t b) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
+ const fixint_t MIN = (fixint_t)1 << (N - 1);
+ const fixint_t MAX = ~MIN;
+ if (a == MIN) {
+ if (b == 0 || b == 1)
+ return a * b;
+ compilerrt_abort();
+ }
+ if (b == MIN) {
+ if (a == 0 || a == 1)
+ return a * b;
+ compilerrt_abort();
+ }
+ fixint_t sa = a >> (N - 1);
+ fixint_t abs_a = (a ^ sa) - sa;
+ fixint_t sb = b >> (N - 1);
+ fixint_t abs_b = (b ^ sb) - sb;
+ if (abs_a < 2 || abs_b < 2)
+ return a * b;
+ if (sa == sb) {
+ if (abs_a > MAX / abs_b)
+ compilerrt_abort();
+ } else {
+ if (abs_a > MIN / -abs_b)
+ compilerrt_abort();
+ }
+ return a * b;
+}
diff --git a/system/lib/compiler-rt/lib/builtins/int_types.h b/system/lib/compiler-rt/lib/builtins/int_types.h
index f89220d543509..705355a4840d4 100644
--- a/system/lib/compiler-rt/lib/builtins/int_types.h
+++ b/system/lib/compiler-rt/lib/builtins/int_types.h
@@ -22,11 +22,20 @@
#ifdef si_int
#undef si_int
#endif
-typedef int si_int;
-typedef unsigned su_int;
+typedef int32_t si_int;
+typedef uint32_t su_int;
+#if UINT_MAX == 0xFFFFFFFF
+#define clzsi __builtin_clz
+#define ctzsi __builtin_ctz
+#elif ULONG_MAX == 0xFFFFFFFF
+#define clzsi __builtin_clzl
+#define ctzsi __builtin_ctzl
+#else
+#error could not determine appropriate clzsi macro for this system
+#endif
-typedef long long di_int;
-typedef unsigned long long du_int;
+typedef int64_t di_int;
+typedef uint64_t du_int;
typedef union {
di_int all;
@@ -135,9 +144,12 @@ typedef struct {
// Check if the target supports 80 bit extended precision long doubles.
// Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC
// still makes it 80 bits. Clang will match whatever compiler it is trying to
-// be compatible with.
-#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \
- defined(__m68k__) || defined(__ia64__)
+// be compatible with. On 32-bit x86 Android, long double is 64 bits, while on
+// x86_64 Android, long double is 128 bits.
+#if (defined(__i386__) || defined(__x86_64__)) && \
+ !(defined(_MSC_VER) || defined(__ANDROID__))
+#define HAS_80_BIT_LONG_DOUBLE 1
+#elif defined(__m68k__) || defined(__ia64__)
#define HAS_80_BIT_LONG_DOUBLE 1
#else
#define HAS_80_BIT_LONG_DOUBLE 0
diff --git a/system/lib/compiler-rt/lib/builtins/int_util.h b/system/lib/compiler-rt/lib/builtins/int_util.h
index 5fbdfb57c1e74..c372c2edc6371 100644
--- a/system/lib/compiler-rt/lib/builtins/int_util.h
+++ b/system/lib/compiler-rt/lib/builtins/int_util.h
@@ -28,4 +28,20 @@ NORETURN void __compilerrt_abort_impl(const char *file, int line,
#define COMPILE_TIME_ASSERT2(expr, cnt) \
typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED
+// Force unrolling the code specified to be repeated N times.
+#define REPEAT_0_TIMES(code_to_repeat) /* do nothing */
+#define REPEAT_1_TIMES(code_to_repeat) code_to_repeat
+#define REPEAT_2_TIMES(code_to_repeat) \
+ REPEAT_1_TIMES(code_to_repeat) \
+ code_to_repeat
+#define REPEAT_3_TIMES(code_to_repeat) \
+ REPEAT_2_TIMES(code_to_repeat) \
+ code_to_repeat
+#define REPEAT_4_TIMES(code_to_repeat) \
+ REPEAT_3_TIMES(code_to_repeat) \
+ code_to_repeat
+
+#define REPEAT_N_TIMES_(N, code_to_repeat) REPEAT_##N##_TIMES(code_to_repeat)
+#define REPEAT_N_TIMES(N, code_to_repeat) REPEAT_N_TIMES_(N, code_to_repeat)
+
#endif // INT_UTIL_H
diff --git a/system/lib/compiler-rt/lib/builtins/lshrdi3.c b/system/lib/compiler-rt/lib/builtins/lshrdi3.c
index 97e08e1e9ba0b..6072152583ac5 100644
--- a/system/lib/compiler-rt/lib/builtins/lshrdi3.c
+++ b/system/lib/compiler-rt/lib/builtins/lshrdi3.c
@@ -16,7 +16,7 @@
// Precondition: 0 <= b < bits_in_dword
-COMPILER_RT_ABI di_int __lshrdi3(di_int a, si_int b) {
+COMPILER_RT_ABI di_int __lshrdi3(di_int a, int b) {
const int bits_in_word = (int)(sizeof(si_int) * CHAR_BIT);
udwords input;
udwords result;
diff --git a/system/lib/compiler-rt/lib/builtins/moddi3.c b/system/lib/compiler-rt/lib/builtins/moddi3.c
index 92b0996077c64..15cf80b995557 100644
--- a/system/lib/compiler-rt/lib/builtins/moddi3.c
+++ b/system/lib/compiler-rt/lib/builtins/moddi3.c
@@ -14,13 +14,9 @@
// Returns: a % b
-COMPILER_RT_ABI di_int __moddi3(di_int a, di_int b) {
- const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1;
- di_int s = b >> bits_in_dword_m1; // s = b < 0 ? -1 : 0
- b = (b ^ s) - s; // negate if s == -1
- s = a >> bits_in_dword_m1; // s = a < 0 ? -1 : 0
- a = (a ^ s) - s; // negate if s == -1
- du_int r;
- __udivmoddi4(a, b, &r);
- return ((di_int)r ^ s) - s; // negate if s == -1
-}
+#define fixint_t di_int
+#define fixuint_t du_int
+#define ASSIGN_UMOD(res, a, b) __udivmoddi4((a), (b), &(res))
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI di_int __moddi3(di_int a, di_int b) { return __modXi3(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/modti3.c b/system/lib/compiler-rt/lib/builtins/modti3.c
index d11fe220b7691..7c10cfd39027b 100644
--- a/system/lib/compiler-rt/lib/builtins/modti3.c
+++ b/system/lib/compiler-rt/lib/builtins/modti3.c
@@ -16,15 +16,11 @@
// Returns: a % b
-COMPILER_RT_ABI ti_int __modti3(ti_int a, ti_int b) {
- const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1;
- ti_int s = b >> bits_in_tword_m1; // s = b < 0 ? -1 : 0
- b = (b ^ s) - s; // negate if s == -1
- s = a >> bits_in_tword_m1; // s = a < 0 ? -1 : 0
- a = (a ^ s) - s; // negate if s == -1
- tu_int r;
- __udivmodti4(a, b, &r);
- return ((ti_int)r ^ s) - s; // negate if s == -1
-}
+#define fixint_t ti_int
+#define fixuint_t tu_int
+#define ASSIGN_UMOD(res, a, b) __udivmodti4((a), (b), &(res))
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI ti_int __modti3(ti_int a, ti_int b) { return __modXi3(a, b); }
#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/mulodi4.c b/system/lib/compiler-rt/lib/builtins/mulodi4.c
index 23f5571ac4689..7209676a327e4 100644
--- a/system/lib/compiler-rt/lib/builtins/mulodi4.c
+++ b/system/lib/compiler-rt/lib/builtins/mulodi4.c
@@ -10,40 +10,13 @@
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t di_int
+#include "int_mulo_impl.inc"
// Returns: a * b
// Effects: sets *overflow to 1 if a * b overflows
COMPILER_RT_ABI di_int __mulodi4(di_int a, di_int b, int *overflow) {
- const int N = (int)(sizeof(di_int) * CHAR_BIT);
- const di_int MIN = (di_int)1 << (N - 1);
- const di_int MAX = ~MIN;
- *overflow = 0;
- di_int result = a * b;
- if (a == MIN) {
- if (b != 0 && b != 1)
- *overflow = 1;
- return result;
- }
- if (b == MIN) {
- if (a != 0 && a != 1)
- *overflow = 1;
- return result;
- }
- di_int sa = a >> (N - 1);
- di_int abs_a = (a ^ sa) - sa;
- di_int sb = b >> (N - 1);
- di_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return result;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- *overflow = 1;
- } else {
- if (abs_a > MIN / -abs_b)
- *overflow = 1;
- }
- return result;
+ return __muloXi4(a, b, overflow);
}
diff --git a/system/lib/compiler-rt/lib/builtins/mulosi4.c b/system/lib/compiler-rt/lib/builtins/mulosi4.c
index fea4311296f8b..4e03c24455d67 100644
--- a/system/lib/compiler-rt/lib/builtins/mulosi4.c
+++ b/system/lib/compiler-rt/lib/builtins/mulosi4.c
@@ -10,40 +10,13 @@
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t si_int
+#include "int_mulo_impl.inc"
// Returns: a * b
// Effects: sets *overflow to 1 if a * b overflows
COMPILER_RT_ABI si_int __mulosi4(si_int a, si_int b, int *overflow) {
- const int N = (int)(sizeof(si_int) * CHAR_BIT);
- const si_int MIN = (si_int)1 << (N - 1);
- const si_int MAX = ~MIN;
- *overflow = 0;
- si_int result = a * b;
- if (a == MIN) {
- if (b != 0 && b != 1)
- *overflow = 1;
- return result;
- }
- if (b == MIN) {
- if (a != 0 && a != 1)
- *overflow = 1;
- return result;
- }
- si_int sa = a >> (N - 1);
- si_int abs_a = (a ^ sa) - sa;
- si_int sb = b >> (N - 1);
- si_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return result;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- *overflow = 1;
- } else {
- if (abs_a > MIN / -abs_b)
- *overflow = 1;
- }
- return result;
+ return __muloXi4(a, b, overflow);
}
diff --git a/system/lib/compiler-rt/lib/builtins/muloti4.c b/system/lib/compiler-rt/lib/builtins/muloti4.c
index 9bdd5b649908b..9a7aa85b022bf 100644
--- a/system/lib/compiler-rt/lib/builtins/muloti4.c
+++ b/system/lib/compiler-rt/lib/builtins/muloti4.c
@@ -18,36 +18,11 @@
// Effects: sets *overflow to 1 if a * b overflows
+#define fixint_t ti_int
+#include "int_mulo_impl.inc"
+
COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) {
- const int N = (int)(sizeof(ti_int) * CHAR_BIT);
- const ti_int MIN = (ti_int)1 << (N - 1);
- const ti_int MAX = ~MIN;
- *overflow = 0;
- ti_int result = a * b;
- if (a == MIN) {
- if (b != 0 && b != 1)
- *overflow = 1;
- return result;
- }
- if (b == MIN) {
- if (a != 0 && a != 1)
- *overflow = 1;
- return result;
- }
- ti_int sa = a >> (N - 1);
- ti_int abs_a = (a ^ sa) - sa;
- ti_int sb = b >> (N - 1);
- ti_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return result;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- *overflow = 1;
- } else {
- if (abs_a > MIN / -abs_b)
- *overflow = 1;
- }
- return result;
+ return __muloXi4(a, b, overflow);
}
#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/mulvdi3.c b/system/lib/compiler-rt/lib/builtins/mulvdi3.c
index cecc97ccf22ec..1d672c6dc155d 100644
--- a/system/lib/compiler-rt/lib/builtins/mulvdi3.c
+++ b/system/lib/compiler-rt/lib/builtins/mulvdi3.c
@@ -10,38 +10,11 @@
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t di_int
+#include "int_mulv_impl.inc"
// Returns: a * b
// Effects: aborts if a * b overflows
-COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) {
- const int N = (int)(sizeof(di_int) * CHAR_BIT);
- const di_int MIN = (di_int)1 << (N - 1);
- const di_int MAX = ~MIN;
- if (a == MIN) {
- if (b == 0 || b == 1)
- return a * b;
- compilerrt_abort();
- }
- if (b == MIN) {
- if (a == 0 || a == 1)
- return a * b;
- compilerrt_abort();
- }
- di_int sa = a >> (N - 1);
- di_int abs_a = (a ^ sa) - sa;
- di_int sb = b >> (N - 1);
- di_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return a * b;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- compilerrt_abort();
- } else {
- if (abs_a > MIN / -abs_b)
- compilerrt_abort();
- }
- return a * b;
-}
+COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) { return __mulvXi3(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/mulvsi3.c b/system/lib/compiler-rt/lib/builtins/mulvsi3.c
index 0d6b18ad01a40..00b2e50eeca91 100644
--- a/system/lib/compiler-rt/lib/builtins/mulvsi3.c
+++ b/system/lib/compiler-rt/lib/builtins/mulvsi3.c
@@ -10,38 +10,11 @@
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t si_int
+#include "int_mulv_impl.inc"
// Returns: a * b
// Effects: aborts if a * b overflows
-COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) {
- const int N = (int)(sizeof(si_int) * CHAR_BIT);
- const si_int MIN = (si_int)1 << (N - 1);
- const si_int MAX = ~MIN;
- if (a == MIN) {
- if (b == 0 || b == 1)
- return a * b;
- compilerrt_abort();
- }
- if (b == MIN) {
- if (a == 0 || a == 1)
- return a * b;
- compilerrt_abort();
- }
- si_int sa = a >> (N - 1);
- si_int abs_a = (a ^ sa) - sa;
- si_int sb = b >> (N - 1);
- si_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return a * b;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- compilerrt_abort();
- } else {
- if (abs_a > MIN / -abs_b)
- compilerrt_abort();
- }
- return a * b;
-}
+COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) { return __mulvXi3(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/mulvti3.c b/system/lib/compiler-rt/lib/builtins/mulvti3.c
index 03963a0ca694f..ba355149f9a76 100644
--- a/system/lib/compiler-rt/lib/builtins/mulvti3.c
+++ b/system/lib/compiler-rt/lib/builtins/mulvti3.c
@@ -18,34 +18,9 @@
// Effects: aborts if a * b overflows
-COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) {
- const int N = (int)(sizeof(ti_int) * CHAR_BIT);
- const ti_int MIN = (ti_int)1 << (N - 1);
- const ti_int MAX = ~MIN;
- if (a == MIN) {
- if (b == 0 || b == 1)
- return a * b;
- compilerrt_abort();
- }
- if (b == MIN) {
- if (a == 0 || a == 1)
- return a * b;
- compilerrt_abort();
- }
- ti_int sa = a >> (N - 1);
- ti_int abs_a = (a ^ sa) - sa;
- ti_int sb = b >> (N - 1);
- ti_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return a * b;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- compilerrt_abort();
- } else {
- if (abs_a > MIN / -abs_b)
- compilerrt_abort();
- }
- return a * b;
-}
+#define fixint_t ti_int
+#include "int_mulv_impl.inc"
+
+COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { return __mulvXi3(a, b); }
#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/os_version_check.c b/system/lib/compiler-rt/lib/builtins/os_version_check.c
index 3794b979434cc..d7194b99ae54c 100644
--- a/system/lib/compiler-rt/lib/builtins/os_version_check.c
+++ b/system/lib/compiler-rt/lib/builtins/os_version_check.c
@@ -24,6 +24,20 @@
// These three variables hold the host's OS version.
static int32_t GlobalMajor, GlobalMinor, GlobalSubminor;
static dispatch_once_t DispatchOnceCounter;
+static dispatch_once_t CompatibilityDispatchOnceCounter;
+
+// _availability_version_check darwin API support.
+typedef uint32_t dyld_platform_t;
+
+typedef struct {
+ dyld_platform_t platform;
+ uint32_t version;
+} dyld_build_version_t;
+
+typedef bool (*AvailabilityVersionCheckFuncTy)(uint32_t count,
+ dyld_build_version_t versions[]);
+
+static AvailabilityVersionCheckFuncTy AvailabilityVersionCheck;
// We can't include directly from here, so
// just forward declare everything that we need from it.
@@ -72,9 +86,25 @@ typedef Boolean (*CFStringGetCStringFuncTy)(CFStringRef, char *, CFIndex,
CFStringEncoding);
typedef void (*CFReleaseFuncTy)(CFTypeRef);
-// Find and parse the SystemVersion.plist file.
-static void parseSystemVersionPList(void *Unused) {
- (void)Unused;
+static void _initializeAvailabilityCheck(bool LoadPlist) {
+ if (AvailabilityVersionCheck && !LoadPlist) {
+ // New API is supported and we're not being asked to load the plist,
+ // exit early!
+ return;
+ }
+
+ // Use the new API if it's is available.
+ AvailabilityVersionCheck = (AvailabilityVersionCheckFuncTy)dlsym(
+ RTLD_DEFAULT, "_availability_version_check");
+
+ if (AvailabilityVersionCheck && !LoadPlist) {
+ // New API is supported and we're not being asked to load the plist,
+ // exit early!
+ return;
+ }
+ // Still load the PLIST to ensure that the existing calls to
+ // __isOSVersionAtLeast still work even with new compiler-rt and old OSes.
+
// Load CoreFoundation dynamically
const void *NullAllocator = dlsym(RTLD_DEFAULT, "kCFAllocatorNull");
if (!NullAllocator)
@@ -201,9 +231,24 @@ static void parseSystemVersionPList(void *Unused) {
fclose(PropertyList);
}
+// Find and parse the SystemVersion.plist file.
+static void compatibilityInitializeAvailabilityCheck(void *Unused) {
+ (void)Unused;
+ _initializeAvailabilityCheck(/*LoadPlist=*/true);
+}
+
+static void initializeAvailabilityCheck(void *Unused) {
+ (void)Unused;
+ _initializeAvailabilityCheck(/*LoadPlist=*/false);
+}
+
+// This old API entry point is no longer used by Clang for Darwin. We still need
+// to keep it around to ensure that object files that reference it are still
+// usable when linked with new compiler-rt.
int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
// Populate the global version variables, if they haven't already.
- dispatch_once_f(&DispatchOnceCounter, NULL, parseSystemVersionPList);
+ dispatch_once_f(&CompatibilityDispatchOnceCounter, NULL,
+ compatibilityInitializeAvailabilityCheck);
if (Major < GlobalMajor)
return 1;
@@ -216,6 +261,61 @@ int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
return Subminor <= GlobalSubminor;
}
+static inline uint32_t ConstructVersion(uint32_t Major, uint32_t Minor,
+ uint32_t Subminor) {
+ return ((Major & 0xffff) << 16) | ((Minor & 0xff) << 8) | (Subminor & 0xff);
+}
+
+int32_t __isPlatformVersionAtLeast(uint32_t Platform, uint32_t Major,
+ uint32_t Minor, uint32_t Subminor) {
+ dispatch_once_f(&DispatchOnceCounter, NULL, initializeAvailabilityCheck);
+
+ if (!AvailabilityVersionCheck) {
+ return __isOSVersionAtLeast(Major, Minor, Subminor);
+ }
+ dyld_build_version_t Versions[] = {
+ {Platform, ConstructVersion(Major, Minor, Subminor)}};
+ return AvailabilityVersionCheck(1, Versions);
+}
+
+#elif __ANDROID__
+
+#include
+#include
+#include
+#include
+
+static int SdkVersion;
+static int IsPreRelease;
+
+static void readSystemProperties(void) {
+ char buf[PROP_VALUE_MAX];
+
+ if (__system_property_get("ro.build.version.sdk", buf) == 0) {
+ // When the system property doesn't exist, defaults to future API level.
+ SdkVersion = __ANDROID_API_FUTURE__;
+ } else {
+ SdkVersion = atoi(buf);
+ }
+
+ if (__system_property_get("ro.build.version.codename", buf) == 0) {
+ IsPreRelease = 1;
+ } else {
+ IsPreRelease = strcmp(buf, "REL") != 0;
+ }
+ return;
+}
+
+int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
+ (int32_t) Minor;
+ (int32_t) Subminor;
+ static pthread_once_t once = PTHREAD_ONCE_INIT;
+ pthread_once(&once, readSystemProperties);
+
+ return SdkVersion >= Major ||
+ (IsPreRelease && Major == __ANDROID_API_FUTURE__);
+}
+
#else
// Silence an empty translation unit warning.
diff --git a/system/lib/compiler-rt/lib/builtins/paritydi2.c b/system/lib/compiler-rt/lib/builtins/paritydi2.c
index dd9d45e63ea47..350dceb8cef59 100644
--- a/system/lib/compiler-rt/lib/builtins/paritydi2.c
+++ b/system/lib/compiler-rt/lib/builtins/paritydi2.c
@@ -14,8 +14,12 @@
// Returns: 1 if number of bits is odd else returns 0
-COMPILER_RT_ABI si_int __paritydi2(di_int a) {
+COMPILER_RT_ABI int __paritydi2(di_int a) {
dwords x;
x.all = a;
- return __paritysi2(x.s.high ^ x.s.low);
+ su_int x2 = x.s.high ^ x.s.low;
+ x2 ^= x2 >> 16;
+ x2 ^= x2 >> 8;
+ x2 ^= x2 >> 4;
+ return (0x6996 >> (x2 & 0xF)) & 1;
}
diff --git a/system/lib/compiler-rt/lib/builtins/paritysi2.c b/system/lib/compiler-rt/lib/builtins/paritysi2.c
index 3efa961f2f85f..a4b84e0806328 100644
--- a/system/lib/compiler-rt/lib/builtins/paritysi2.c
+++ b/system/lib/compiler-rt/lib/builtins/paritysi2.c
@@ -14,7 +14,7 @@
// Returns: 1 if number of bits is odd else returns 0
-COMPILER_RT_ABI si_int __paritysi2(si_int a) {
+COMPILER_RT_ABI int __paritysi2(si_int a) {
su_int x = (su_int)a;
x ^= x >> 16;
x ^= x >> 8;
diff --git a/system/lib/compiler-rt/lib/builtins/parityti2.c b/system/lib/compiler-rt/lib/builtins/parityti2.c
index f3942ba8378c7..011c8dd455620 100644
--- a/system/lib/compiler-rt/lib/builtins/parityti2.c
+++ b/system/lib/compiler-rt/lib/builtins/parityti2.c
@@ -16,10 +16,16 @@
// Returns: 1 if number of bits is odd else returns 0
-COMPILER_RT_ABI si_int __parityti2(ti_int a) {
+COMPILER_RT_ABI int __parityti2(ti_int a) {
twords x;
+ dwords x2;
x.all = a;
- return __paritydi2(x.s.high ^ x.s.low);
+ x2.all = x.s.high ^ x.s.low;
+ su_int x3 = x2.s.high ^ x2.s.low;
+ x3 ^= x3 >> 16;
+ x3 ^= x3 >> 8;
+ x3 ^= x3 >> 4;
+ return (0x6996 >> (x3 & 0xF)) & 1;
}
#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/popcountdi2.c b/system/lib/compiler-rt/lib/builtins/popcountdi2.c
index 9bbc39c6608a1..20dd0b0239efc 100644
--- a/system/lib/compiler-rt/lib/builtins/popcountdi2.c
+++ b/system/lib/compiler-rt/lib/builtins/popcountdi2.c
@@ -14,7 +14,7 @@
// Returns: count of 1 bits
-COMPILER_RT_ABI si_int __popcountdi2(di_int a) {
+COMPILER_RT_ABI int __popcountdi2(di_int a) {
du_int x2 = (du_int)a;
x2 = x2 - ((x2 >> 1) & 0x5555555555555555uLL);
// Every 2 bits holds the sum of every pair of bits (32)
diff --git a/system/lib/compiler-rt/lib/builtins/popcountsi2.c b/system/lib/compiler-rt/lib/builtins/popcountsi2.c
index 75e592a778d98..4d346c45d9cee 100644
--- a/system/lib/compiler-rt/lib/builtins/popcountsi2.c
+++ b/system/lib/compiler-rt/lib/builtins/popcountsi2.c
@@ -14,7 +14,7 @@
// Returns: count of 1 bits
-COMPILER_RT_ABI si_int __popcountsi2(si_int a) {
+COMPILER_RT_ABI int __popcountsi2(si_int a) {
su_int x = (su_int)a;
x = x - ((x >> 1) & 0x55555555);
// Every 2 bits holds the sum of every pair of bits
diff --git a/system/lib/compiler-rt/lib/builtins/popcountti2.c b/system/lib/compiler-rt/lib/builtins/popcountti2.c
index 853fd722309ee..79cbb2fb34c00 100644
--- a/system/lib/compiler-rt/lib/builtins/popcountti2.c
+++ b/system/lib/compiler-rt/lib/builtins/popcountti2.c
@@ -17,7 +17,7 @@
// Returns: count of 1 bits
-COMPILER_RT_ABI si_int __popcountti2(ti_int a) {
+COMPILER_RT_ABI int __popcountti2(ti_int a) {
tu_int x3 = (tu_int)a;
x3 = x3 - ((x3 >> 1) &
(((tu_int)0x5555555555555555uLL << 64) | 0x5555555555555555uLL));
diff --git a/system/lib/compiler-rt/lib/builtins/powidf2.c b/system/lib/compiler-rt/lib/builtins/powidf2.c
index 9697588484e74..81058af508297 100644
--- a/system/lib/compiler-rt/lib/builtins/powidf2.c
+++ b/system/lib/compiler-rt/lib/builtins/powidf2.c
@@ -14,7 +14,7 @@
// Returns: a ^ b
-COMPILER_RT_ABI double __powidf2(double a, si_int b) {
+COMPILER_RT_ABI double __powidf2(double a, int b) {
const int recip = b < 0;
double r = 1;
while (1) {
diff --git a/system/lib/compiler-rt/lib/builtins/powisf2.c b/system/lib/compiler-rt/lib/builtins/powisf2.c
index 469402348825c..d0ab26167bbd2 100644
--- a/system/lib/compiler-rt/lib/builtins/powisf2.c
+++ b/system/lib/compiler-rt/lib/builtins/powisf2.c
@@ -14,7 +14,7 @@
// Returns: a ^ b
-COMPILER_RT_ABI float __powisf2(float a, si_int b) {
+COMPILER_RT_ABI float __powisf2(float a, int b) {
const int recip = b < 0;
float r = 1;
while (1) {
diff --git a/system/lib/compiler-rt/lib/builtins/powitf2.c b/system/lib/compiler-rt/lib/builtins/powitf2.c
index 443da04b29b19..8e639a03a3c4b 100644
--- a/system/lib/compiler-rt/lib/builtins/powitf2.c
+++ b/system/lib/compiler-rt/lib/builtins/powitf2.c
@@ -10,13 +10,14 @@
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define QUAD_PRECISION
+#include "fp_lib.h"
-#if defined _ARCH_PPC || defined __wasm__
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
// Returns: a ^ b
-COMPILER_RT_ABI long double __powitf2(long double a, si_int b) {
+COMPILER_RT_ABI long double __powitf2(long double a, int b) {
const int recip = b < 0;
long double r = 1;
while (1) {
diff --git a/system/lib/compiler-rt/lib/builtins/powixf2.c b/system/lib/compiler-rt/lib/builtins/powixf2.c
index b7b52095afa17..3edfe9fd7af59 100644
--- a/system/lib/compiler-rt/lib/builtins/powixf2.c
+++ b/system/lib/compiler-rt/lib/builtins/powixf2.c
@@ -16,7 +16,7 @@
// Returns: a ^ b
-COMPILER_RT_ABI long double __powixf2(long double a, si_int b) {
+COMPILER_RT_ABI long double __powixf2(long double a, int b) {
const int recip = b < 0;
long double r = 1;
while (1) {
diff --git a/system/lib/compiler-rt/lib/builtins/truncdfhf2.c b/system/lib/compiler-rt/lib/builtins/truncdfhf2.c
index 90c418a4387ff..24c6e62f715f5 100644
--- a/system/lib/compiler-rt/lib/builtins/truncdfhf2.c
+++ b/system/lib/compiler-rt/lib/builtins/truncdfhf2.c
@@ -10,11 +10,11 @@
#define DST_HALF
#include "fp_trunc_impl.inc"
-COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { return __truncXfYf2__(a); }
+COMPILER_RT_ABI dst_t __truncdfhf2(double a) { return __truncXfYf2__(a); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); }
+AEABI_RTABI dst_t __aeabi_d2h(double a) { return __truncdfhf2(a); }
#else
COMPILER_RT_ALIAS(__truncdfhf2, __aeabi_d2h)
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/truncsfhf2.c b/system/lib/compiler-rt/lib/builtins/truncsfhf2.c
index 1f17194c38e59..379e7cb6f7845 100644
--- a/system/lib/compiler-rt/lib/builtins/truncsfhf2.c
+++ b/system/lib/compiler-rt/lib/builtins/truncsfhf2.c
@@ -12,15 +12,15 @@
// Use a forwarding definition and noinline to implement a poor man's alias,
// as there isn't a good cross-platform way of defining one.
-COMPILER_RT_ABI NOINLINE uint16_t __truncsfhf2(float a) {
+COMPILER_RT_ABI NOINLINE dst_t __truncsfhf2(float a) {
return __truncXfYf2__(a);
}
-COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); }
+COMPILER_RT_ABI dst_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); }
+AEABI_RTABI dst_t __aeabi_f2h(float a) { return __truncsfhf2(a); }
#else
COMPILER_RT_ALIAS(__truncsfhf2, __aeabi_f2h)
#endif
diff --git a/system/lib/compiler-rt/lib/builtins/trunctfhf2.c b/system/lib/compiler-rt/lib/builtins/trunctfhf2.c
new file mode 100644
index 0000000000000..e3a2309d954bc
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/trunctfhf2.c
@@ -0,0 +1,23 @@
+//===-- lib/trunctfhf2.c - quad -> half conversion ----------------*- C -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define QUAD_PRECISION
+#include "fp_lib.h"
+
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \
+ defined(COMPILER_RT_HAS_FLOAT16)
+#define SRC_QUAD
+#define DST_HALF
+#include "fp_trunc_impl.inc"
+
+COMPILER_RT_ABI _Float16 __trunctfhf2(long double a) {
+ return __truncXfYf2__(a);
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/builtins/udivdi3.c b/system/lib/compiler-rt/lib/builtins/udivdi3.c
index a23139ec947f6..74319cbe71c3b 100644
--- a/system/lib/compiler-rt/lib/builtins/udivdi3.c
+++ b/system/lib/compiler-rt/lib/builtins/udivdi3.c
@@ -12,8 +12,12 @@
#include "int_lib.h"
+typedef du_int fixuint_t;
+typedef di_int fixint_t;
+#include "int_div_impl.inc"
+
// Returns: a / b
COMPILER_RT_ABI du_int __udivdi3(du_int a, du_int b) {
- return __udivmoddi4(a, b, 0);
+ return __udivXi3(a, b);
}
diff --git a/system/lib/compiler-rt/lib/builtins/udivmoddi4.c b/system/lib/compiler-rt/lib/builtins/udivmoddi4.c
index 5b297c32d7901..10b41df28f840 100644
--- a/system/lib/compiler-rt/lib/builtins/udivmoddi4.c
+++ b/system/lib/compiler-rt/lib/builtins/udivmoddi4.c
@@ -87,7 +87,7 @@ COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) {
// K K
// ---
// K 0
- sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high);
+ sr = clzsi(d.s.high) - clzsi(n.s.high);
// 0 <= sr <= n_uword_bits - 2 or sr large
if (sr > n_uword_bits - 2) {
if (rem)
@@ -120,7 +120,7 @@ COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) {
// K X
// ---
// 0 K
- sr = 1 + n_uword_bits + __builtin_clz(d.s.low) - __builtin_clz(n.s.high);
+ sr = 1 + n_uword_bits + clzsi(d.s.low) - clzsi(n.s.high);
// 2 <= sr <= n_udword_bits - 1
// q.all = n.all << (n_udword_bits - sr);
// r.all = n.all >> sr;
@@ -145,7 +145,7 @@ COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) {
// K X
// ---
// K K
- sr = __builtin_clz(d.s.high) - __builtin_clz(n.s.high);
+ sr = clzsi(d.s.high) - clzsi(n.s.high);
// 0 <= sr <= n_uword_bits - 1 or sr large
if (sr > n_uword_bits - 1) {
if (rem)
diff --git a/system/lib/compiler-rt/lib/builtins/udivmodti4.c b/system/lib/compiler-rt/lib/builtins/udivmodti4.c
index dd14a8b579ca5..55def37c9e1fe 100644
--- a/system/lib/compiler-rt/lib/builtins/udivmodti4.c
+++ b/system/lib/compiler-rt/lib/builtins/udivmodti4.c
@@ -14,182 +14,145 @@
#ifdef CRT_HAS_128BIT
+// Returns the 128 bit division result by 64 bit. Result must fit in 64 bits.
+// Remainder stored in r.
+// Taken and adjusted from libdivide libdivide_128_div_64_to_64 division
+// fallback. For a correctness proof see the reference for this algorithm
+// in Knuth, Volume 2, section 4.3.1, Algorithm D.
+UNUSED
+static inline du_int udiv128by64to64default(du_int u1, du_int u0, du_int v,
+ du_int *r) {
+ const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT;
+ const du_int b = (1ULL << (n_udword_bits / 2)); // Number base (32 bits)
+ du_int un1, un0; // Norm. dividend LSD's
+ du_int vn1, vn0; // Norm. divisor digits
+ du_int q1, q0; // Quotient digits
+ du_int un64, un21, un10; // Dividend digit pairs
+ du_int rhat; // A remainder
+ si_int s; // Shift amount for normalization
+
+ s = __builtin_clzll(v);
+ if (s > 0) {
+ // Normalize the divisor.
+ v = v << s;
+ un64 = (u1 << s) | (u0 >> (n_udword_bits - s));
+ un10 = u0 << s; // Shift dividend left
+ } else {
+ // Avoid undefined behavior of (u0 >> 64).
+ un64 = u1;
+ un10 = u0;
+ }
+
+ // Break divisor up into two 32-bit digits.
+ vn1 = v >> (n_udword_bits / 2);
+ vn0 = v & 0xFFFFFFFF;
+
+ // Break right half of dividend into two digits.
+ un1 = un10 >> (n_udword_bits / 2);
+ un0 = un10 & 0xFFFFFFFF;
+
+ // Compute the first quotient digit, q1.
+ q1 = un64 / vn1;
+ rhat = un64 - q1 * vn1;
+
+ // q1 has at most error 2. No more than 2 iterations.
+ while (q1 >= b || q1 * vn0 > b * rhat + un1) {
+ q1 = q1 - 1;
+ rhat = rhat + vn1;
+ if (rhat >= b)
+ break;
+ }
+
+ un21 = un64 * b + un1 - q1 * v;
+
+ // Compute the second quotient digit.
+ q0 = un21 / vn1;
+ rhat = un21 - q0 * vn1;
+
+ // q0 has at most error 2. No more than 2 iterations.
+ while (q0 >= b || q0 * vn0 > b * rhat + un0) {
+ q0 = q0 - 1;
+ rhat = rhat + vn1;
+ if (rhat >= b)
+ break;
+ }
+
+ *r = (un21 * b + un0 - q0 * v) >> s;
+ return q1 * b + q0;
+}
+
+static inline du_int udiv128by64to64(du_int u1, du_int u0, du_int v,
+ du_int *r) {
+#if defined(__x86_64__)
+ du_int result;
+ __asm__("divq %[v]"
+ : "=a"(result), "=d"(*r)
+ : [ v ] "r"(v), "a"(u0), "d"(u1));
+ return result;
+#else
+ return udiv128by64to64default(u1, u0, v, r);
+#endif
+}
+
// Effects: if rem != 0, *rem = a % b
// Returns: a / b
-// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide
-
COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem) {
- const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT;
const unsigned n_utword_bits = sizeof(tu_int) * CHAR_BIT;
- utwords n;
- n.all = a;
- utwords d;
- d.all = b;
- utwords q;
- utwords r;
- unsigned sr;
- // special cases, X is unknown, K != 0
- if (n.s.high == 0) {
- if (d.s.high == 0) {
- // 0 X
- // ---
- // 0 X
- if (rem)
- *rem = n.s.low % d.s.low;
- return n.s.low / d.s.low;
- }
- // 0 X
- // ---
- // K X
+ utwords dividend;
+ dividend.all = a;
+ utwords divisor;
+ divisor.all = b;
+ utwords quotient;
+ utwords remainder;
+ if (divisor.all > dividend.all) {
if (rem)
- *rem = n.s.low;
+ *rem = dividend.all;
return 0;
}
- // n.s.high != 0
- if (d.s.low == 0) {
- if (d.s.high == 0) {
- // K X
- // ---
- // 0 0
- if (rem)
- *rem = n.s.high % d.s.low;
- return n.s.high / d.s.low;
- }
- // d.s.high != 0
- if (n.s.low == 0) {
- // K 0
- // ---
- // K 0
- if (rem) {
- r.s.high = n.s.high % d.s.high;
- r.s.low = 0;
- *rem = r.all;
- }
- return n.s.high / d.s.high;
- }
- // K K
- // ---
- // K 0
- if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ {
- if (rem) {
- r.s.low = n.s.low;
- r.s.high = n.s.high & (d.s.high - 1);
- *rem = r.all;
- }
- return n.s.high >> __builtin_ctzll(d.s.high);
- }
- // K K
- // ---
- // K 0
- sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high);
- // 0 <= sr <= n_udword_bits - 2 or sr large
- if (sr > n_udword_bits - 2) {
- if (rem)
- *rem = n.all;
- return 0;
- }
- ++sr;
- // 1 <= sr <= n_udword_bits - 1
- // q.all = n.all << (n_utword_bits - sr);
- q.s.low = 0;
- q.s.high = n.s.low << (n_udword_bits - sr);
- // r.all = n.all >> sr;
- r.s.high = n.s.high >> sr;
- r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
- } else /* d.s.low != 0 */ {
- if (d.s.high == 0) {
- // K X
- // ---
- // 0 K
- if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ {
- if (rem)
- *rem = n.s.low & (d.s.low - 1);
- if (d.s.low == 1)
- return n.all;
- sr = __builtin_ctzll(d.s.low);
- q.s.high = n.s.high >> sr;
- q.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
- return q.all;
- }
- // K X
- // ---
- // 0 K
- sr = 1 + n_udword_bits + __builtin_clzll(d.s.low) -
- __builtin_clzll(n.s.high);
- // 2 <= sr <= n_utword_bits - 1
- // q.all = n.all << (n_utword_bits - sr);
- // r.all = n.all >> sr;
- if (sr == n_udword_bits) {
- q.s.low = 0;
- q.s.high = n.s.low;
- r.s.high = 0;
- r.s.low = n.s.high;
- } else if (sr < n_udword_bits) /* 2 <= sr <= n_udword_bits - 1 */ {
- q.s.low = 0;
- q.s.high = n.s.low << (n_udword_bits - sr);
- r.s.high = n.s.high >> sr;
- r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
- } else /* n_udword_bits + 1 <= sr <= n_utword_bits - 1 */ {
- q.s.low = n.s.low << (n_utword_bits - sr);
- q.s.high = (n.s.high << (n_utword_bits - sr)) |
- (n.s.low >> (sr - n_udword_bits));
- r.s.high = 0;
- r.s.low = n.s.high >> (sr - n_udword_bits);
- }
+ // When the divisor fits in 64 bits, we can use an optimized path.
+ if (divisor.s.high == 0) {
+ remainder.s.high = 0;
+ if (dividend.s.high < divisor.s.low) {
+ // The result fits in 64 bits.
+ quotient.s.low = udiv128by64to64(dividend.s.high, dividend.s.low,
+ divisor.s.low, &remainder.s.low);
+ quotient.s.high = 0;
} else {
- // K X
- // ---
- // K K
- sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high);
- // 0 <= sr <= n_udword_bits - 1 or sr large
- if (sr > n_udword_bits - 1) {
- if (rem)
- *rem = n.all;
- return 0;
- }
- ++sr;
- // 1 <= sr <= n_udword_bits
- // q.all = n.all << (n_utword_bits - sr);
- // r.all = n.all >> sr;
- q.s.low = 0;
- if (sr == n_udword_bits) {
- q.s.high = n.s.low;
- r.s.high = 0;
- r.s.low = n.s.high;
- } else {
- r.s.high = n.s.high >> sr;
- r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
- q.s.high = n.s.low << (n_udword_bits - sr);
- }
+ // First, divide with the high part to get the remainder in dividend.s.high.
+ // After that dividend.s.high < divisor.s.low.
+ quotient.s.high = dividend.s.high / divisor.s.low;
+ dividend.s.high = dividend.s.high % divisor.s.low;
+ quotient.s.low = udiv128by64to64(dividend.s.high, dividend.s.low,
+ divisor.s.low, &remainder.s.low);
}
+ if (rem)
+ *rem = remainder.all;
+ return quotient.all;
}
- // Not a special case
- // q and r are initialized with:
- // q.all = n.all << (n_utword_bits - sr);
- // r.all = n.all >> sr;
- // 1 <= sr <= n_utword_bits - 1
- su_int carry = 0;
- for (; sr > 0; --sr) {
- // r:q = ((r:q) << 1) | carry
- r.s.high = (r.s.high << 1) | (r.s.low >> (n_udword_bits - 1));
- r.s.low = (r.s.low << 1) | (q.s.high >> (n_udword_bits - 1));
- q.s.high = (q.s.high << 1) | (q.s.low >> (n_udword_bits - 1));
- q.s.low = (q.s.low << 1) | carry;
- // carry = 0;
- // if (r.all >= d.all)
+ // 0 <= shift <= 63.
+ si_int shift =
+ __builtin_clzll(divisor.s.high) - __builtin_clzll(dividend.s.high);
+ divisor.all <<= shift;
+ quotient.s.high = 0;
+ quotient.s.low = 0;
+ for (; shift >= 0; --shift) {
+ quotient.s.low <<= 1;
+ // Branch free version of.
+ // if (dividend.all >= divisor.all)
// {
- // r.all -= d.all;
- // carry = 1;
+ // dividend.all -= divisor.all;
+ // carry = 1;
// }
- const ti_int s = (ti_int)(d.all - r.all - 1) >> (n_utword_bits - 1);
- carry = s & 1;
- r.all -= d.all & s;
+ const ti_int s =
+ (ti_int)(divisor.all - dividend.all - 1) >> (n_utword_bits - 1);
+ quotient.s.low |= s & 1;
+ dividend.all -= divisor.all & s;
+ divisor.all >>= 1;
}
- q.all = (q.all << 1) | carry;
if (rem)
- *rem = r.all;
- return q.all;
+ *rem = dividend.all;
+ return quotient.all;
}
#endif // CRT_HAS_128BIT
diff --git a/system/lib/compiler-rt/lib/builtins/udivsi3.c b/system/lib/compiler-rt/lib/builtins/udivsi3.c
index 18cc96c1b2e08..3894e1597552f 100644
--- a/system/lib/compiler-rt/lib/builtins/udivsi3.c
+++ b/system/lib/compiler-rt/lib/builtins/udivsi3.c
@@ -12,49 +12,14 @@
#include "int_lib.h"
-// Returns: a / b
+typedef su_int fixuint_t;
+typedef si_int fixint_t;
+#include "int_div_impl.inc"
-// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide
+// Returns: a / b
-// This function should not call __divsi3!
-COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d) {
- const unsigned n_uword_bits = sizeof(su_int) * CHAR_BIT;
- su_int q;
- su_int r;
- unsigned sr;
- // special cases
- if (d == 0)
- return 0; // ?!
- if (n == 0)
- return 0;
- sr = __builtin_clz(d) - __builtin_clz(n);
- // 0 <= sr <= n_uword_bits - 1 or sr large
- if (sr > n_uword_bits - 1) // d > r
- return 0;
- if (sr == n_uword_bits - 1) // d == 1
- return n;
- ++sr;
- // 1 <= sr <= n_uword_bits - 1
- // Not a special case
- q = n << (n_uword_bits - sr);
- r = n >> sr;
- su_int carry = 0;
- for (; sr > 0; --sr) {
- // r:q = ((r:q) << 1) | carry
- r = (r << 1) | (q >> (n_uword_bits - 1));
- q = (q << 1) | carry;
- // carry = 0;
- // if (r.all >= d.all)
- // {
- // r.all -= d.all;
- // carry = 1;
- // }
- const si_int s = (si_int)(d - r - 1) >> (n_uword_bits - 1);
- carry = s & 1;
- r -= d & s;
- }
- q = (q << 1) | carry;
- return q;
+COMPILER_RT_ABI su_int __udivsi3(su_int a, su_int b) {
+ return __udivXi3(a, b);
}
#if defined(__ARM_EABI__)
diff --git a/system/lib/compiler-rt/lib/builtins/umoddi3.c b/system/lib/compiler-rt/lib/builtins/umoddi3.c
index 965cf8fc01bd2..e672da96ef629 100644
--- a/system/lib/compiler-rt/lib/builtins/umoddi3.c
+++ b/system/lib/compiler-rt/lib/builtins/umoddi3.c
@@ -12,10 +12,12 @@
#include "int_lib.h"
+typedef du_int fixuint_t;
+typedef di_int fixint_t;
+#include "int_div_impl.inc"
+
// Returns: a % b
COMPILER_RT_ABI du_int __umoddi3(du_int a, du_int b) {
- du_int r;
- __udivmoddi4(a, b, &r);
- return r;
+ return __umodXi3(a, b);
}
diff --git a/system/lib/compiler-rt/lib/builtins/umodsi3.c b/system/lib/compiler-rt/lib/builtins/umodsi3.c
index ce9abcd94ef77..5383aea656a99 100644
--- a/system/lib/compiler-rt/lib/builtins/umodsi3.c
+++ b/system/lib/compiler-rt/lib/builtins/umodsi3.c
@@ -12,8 +12,12 @@
#include "int_lib.h"
+typedef su_int fixuint_t;
+typedef si_int fixint_t;
+#include "int_div_impl.inc"
+
// Returns: a % b
COMPILER_RT_ABI su_int __umodsi3(su_int a, su_int b) {
- return a - __udivsi3(a, b) * b;
+ return __umodXi3(a, b);
}
diff --git a/system/lib/compiler-rt/lib/interception/interception.h b/system/lib/compiler-rt/lib/interception/interception.h
index aa2986b1c92fa..c69a63963798d 100644
--- a/system/lib/compiler-rt/lib/interception/interception.h
+++ b/system/lib/compiler-rt/lib/interception/interception.h
@@ -17,7 +17,7 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
- !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_WINDOWS && \
+ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && \
!SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS && \
!SANITIZER_EMSCRIPTEN
# error "Interception doesn't work on this operating system."
@@ -290,7 +290,7 @@ typedef unsigned long uptr; // NOLINT
#define INCLUDED_FROM_INTERCEPTION_LIB
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
+ SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
# include "interception_linux.h"
# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.cpp b/system/lib/compiler-rt/lib/interception/interception_linux.cpp
index 950cd5126538b..5111a87f0a6c9 100644
--- a/system/lib/compiler-rt/lib/interception/interception_linux.cpp
+++ b/system/lib/compiler-rt/lib/interception/interception_linux.cpp
@@ -14,7 +14,7 @@
#include "interception.h"
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
#include // for dlsym() and dlvsym()
@@ -63,8 +63,8 @@ bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func,
return addr && (func == wrapper);
}
-// Android and Solaris do not have dlvsym
-#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
+// dlvsym is a GNU extension supported by some other platforms.
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
static void *GetFuncAddr(const char *name, const char *ver) {
return dlvsym(RTLD_NEXT, name, ver);
}
@@ -75,9 +75,9 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
*ptr_to_real = (uptr)addr;
return addr && (func == wrapper);
}
-#endif // !SANITIZER_ANDROID
+#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
} // namespace __interception
#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
- // SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ // SANITIZER_SOLARIS
diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.h b/system/lib/compiler-rt/lib/interception/interception_linux.h
index 5a98b57d01d7b..6d3957e570267 100644
--- a/system/lib/compiler-rt/lib/interception/interception_linux.h
+++ b/system/lib/compiler-rt/lib/interception/interception_linux.h
@@ -12,7 +12,7 @@
//===----------------------------------------------------------------------===//
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
+ SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
# error "interception_linux.h should be included from interception library only"
@@ -35,9 +35,8 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
(::__interception::uptr) & (func), \
(::__interception::uptr) & WRAP(func))
-// Android, Solaris, OpenBSD and emscripten do not have dlvsym
-#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD && \
- !SANITIZER_EMSCRIPTEN
+// dlvsym is a GNU extension supported by some other platforms.
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
::__interception::InterceptFunction( \
#func, symver, \
@@ -47,8 +46,8 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
#else
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
-#endif // !SANITIZER_ANDROID && !SANITIZER_SOLARIS
+#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
#endif // INTERCEPTION_LINUX_H
#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
- // SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ // SANITIZER_SOLARIS
diff --git a/system/lib/compiler-rt/lib/interception/interception_win.cpp b/system/lib/compiler-rt/lib/interception/interception_win.cpp
index 1a1c327e61240..98bc756ae53aa 100644
--- a/system/lib/compiler-rt/lib/interception/interception_win.cpp
+++ b/system/lib/compiler-rt/lib/interception/interception_win.cpp
@@ -136,7 +136,7 @@ namespace __interception {
static const int kAddressLength = FIRST_32_SECOND_64(4, 8);
static const int kJumpInstructionLength = 5;
static const int kShortJumpInstructionLength = 2;
-static const int kIndirectJumpInstructionLength = 6;
+UNUSED static const int kIndirectJumpInstructionLength = 6;
static const int kBranchLength =
FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
static const int kDirectBranchLength = kBranchLength + kAddressLength;
@@ -165,7 +165,7 @@ static uptr GetMmapGranularity() {
return si.dwAllocationGranularity;
}
-static uptr RoundUpTo(uptr size, uptr boundary) {
+UNUSED static uptr RoundUpTo(uptr size, uptr boundary) {
return (size + boundary - 1) & ~(boundary - 1);
}
@@ -309,7 +309,7 @@ struct TrampolineMemoryRegion {
uptr max_size;
};
-static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
+UNUSED static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
static const int kMaxTrampolineRegion = 1024;
static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
diff --git a/system/lib/compiler-rt/lib/lsan/lsan.cpp b/system/lib/compiler-rt/lib/lsan/lsan.cpp
index 8e3f0294eea04..e1e33c068e1fe 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan.cpp
@@ -15,7 +15,6 @@
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
-#include "sanitizer_common/sanitizer_stacktrace.h"
#include "lsan_allocator.h"
#include "lsan_common.h"
#include "lsan_thread.h"
@@ -83,7 +82,7 @@ static void InitializeFlags() {
RegisterCommonFlags(&parser);
// Override from user-specified string.
- const char *lsan_default_options = MaybeCallLsanDefaultOptions();
+ const char *lsan_default_options = __lsan_default_options();
parser.ParseString(lsan_default_options);
#if SANITIZER_EMSCRIPTEN
char *options = (char*) EM_ASM_INT({
@@ -102,7 +101,7 @@ static void InitializeFlags() {
StackTrace::snapshot_stack = false;
#endif // SANITIZER_EMSCRIPTEN
- SetVerbosity(common_flags()->verbosity);
+ InitializeCommonFlags();
if (Verbosity()) ReportUnrecognizedFlags();
@@ -111,17 +110,6 @@ static void InitializeFlags() {
__sanitizer_set_report_path(common_flags()->log_path);
}
-static void OnStackUnwind(const SignalContext &sig, const void *,
- BufferedStackTrace *stack) {
- stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
- common_flags()->fast_unwind_on_fatal);
-}
-
-static void LsanOnDeadlySignal(int signo, void *siginfo, void *context) {
- HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind,
- nullptr);
-}
-
extern "C" void __lsan_init() {
CHECK(!lsan_init_is_running);
if (lsan_inited)
@@ -141,10 +129,7 @@ extern "C" void __lsan_init() {
// Emscripten does not have signals
InstallDeadlySignalHandlers(LsanOnDeadlySignal);
#endif
- u32 tid = ThreadCreate(0, 0, true);
- CHECK_EQ(tid, 0);
- ThreadStart(tid, GetTid());
- SetCurrentThread(tid);
+ InitializeMainThread();
if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit)
Atexit(DoLeakCheck);
diff --git a/system/lib/compiler-rt/lib/lsan/lsan.h b/system/lib/compiler-rt/lib/lsan/lsan.h
index 9904ada4bb3bd..1e82ad72f0058 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan.h
@@ -12,6 +12,11 @@
//===----------------------------------------------------------------------===//
#include "lsan_thread.h"
+#if SANITIZER_POSIX
+#include "lsan_posix.h"
+#elif SANITIZER_FUCHSIA
+#include "lsan_fuchsia.h"
+#endif
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
@@ -33,6 +38,7 @@ namespace __lsan {
void InitializeInterceptors();
void ReplaceSystemMalloc();
+void LsanOnDeadlySignal(int signo, void *siginfo, void *context);
#define ENSURE_LSAN_INITED do { \
CHECK(!lsan_init_is_running); \
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp
index 845526a8771a1..5dff571e4a059 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp
@@ -309,6 +309,16 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p) {
return kIgnoreObjectInvalid;
}
}
+
+void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
+ // This function can be used to treat memory reachable from `tctx` as live.
+ // This is useful for threads that have been created but not yet started.
+
+ // This is currently a no-op because the LSan `pthread_create()` interceptor
+ // blocks until the child thread starts which keeps the thread's `arg` pointer
+ // live.
+}
+
} // namespace __lsan
using namespace __lsan;
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h
index 3df5f77bd3bcf..050a9143bc014 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h
@@ -72,10 +72,16 @@ struct AP32 {
template
using PrimaryAllocatorASVT = SizeClassAllocator32>;
using PrimaryAllocator = PrimaryAllocatorASVT;
-#elif defined(__x86_64__) || defined(__powerpc64__)
-# if defined(__powerpc64__)
+#elif defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__)
+# if SANITIZER_FUCHSIA
+const uptr kAllocatorSpace = ~(uptr)0;
+const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
+# elif defined(__powerpc64__)
const uptr kAllocatorSpace = 0xa0000000000ULL;
const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
+#elif defined(__s390x__)
+const uptr kAllocatorSpace = 0x40000000000ULL;
+const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
# else
const uptr kAllocatorSpace = 0x600000000000ULL;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp
index fdd53733241b5..50a2e1952ecd6 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp
@@ -1,4 +1,4 @@
-//=-- lsan_common.cc ------------------------------------------------------===//
+//=-- lsan_common.cpp -----------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -38,6 +38,7 @@ BlockingMutex global_mutex(LINKER_INITIALIZED);
Flags lsan_flags;
+
void DisableCounterUnderflow() {
if (common_flags()->detect_leaks) {
Report("Unmatched call to __lsan_enable().\n");
@@ -68,35 +69,67 @@ void RegisterLsanFlags(FlagParser *parser, Flags *f) {
if (flags()->log_threads) Report(__VA_ARGS__); \
} while (0)
-ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
-static SuppressionContext *suppression_ctx = nullptr;
+class LeakSuppressionContext {
+ bool parsed = false;
+ SuppressionContext context;
+ bool suppressed_stacks_sorted = true;
+ InternalMmapVector suppressed_stacks;
+
+ Suppression *GetSuppressionForAddr(uptr addr);
+ void LazyInit();
+
+ public:
+ LeakSuppressionContext(const char *supprression_types[],
+ int suppression_types_num)
+ : context(supprression_types, suppression_types_num) {}
+
+ Suppression *GetSuppressionForStack(u32 stack_trace_id);
+
+ const InternalMmapVector &GetSortedSuppressedStacks() {
+ if (!suppressed_stacks_sorted) {
+ suppressed_stacks_sorted = true;
+ SortAndDedup(suppressed_stacks);
+ }
+ return suppressed_stacks;
+ }
+ void PrintMatchedSuppressions();
+};
+
+ALIGNED(64) static char suppression_placeholder[sizeof(LeakSuppressionContext)];
+static LeakSuppressionContext *suppression_ctx = nullptr;
static const char kSuppressionLeak[] = "leak";
static const char *kSuppressionTypes[] = { kSuppressionLeak };
static const char kStdSuppressions[] =
#if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
- // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
- // definition.
- "leak:*pthread_exit*\n"
+ // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+ // definition.
+ "leak:*pthread_exit*\n"
#endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
#if SANITIZER_MAC
- // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
- "leak:*_os_trace*\n"
+ // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
+ "leak:*_os_trace*\n"
#endif
- // TLS leak in some glibc versions, described in
- // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
- "leak:*tls_get_addr*\n";
+ // TLS leak in some glibc versions, described in
+ // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
+ "leak:*tls_get_addr*\n";
void InitializeSuppressions() {
CHECK_EQ(nullptr, suppression_ctx);
suppression_ctx = new (suppression_placeholder) // NOLINT
- SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
- suppression_ctx->ParseFromFile(flags()->suppressions);
- if (&__lsan_default_suppressions)
- suppression_ctx->Parse(__lsan_default_suppressions());
- suppression_ctx->Parse(kStdSuppressions);
+ LeakSuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
}
-static SuppressionContext *GetSuppressionContext() {
+void LeakSuppressionContext::LazyInit() {
+ if (!parsed) {
+ parsed = true;
+ context.ParseFromFile(flags()->suppressions);
+ if (&__lsan_default_suppressions)
+ context.Parse(__lsan_default_suppressions());
+ context.Parse(kStdSuppressions);
+ }
+}
+
+static LeakSuppressionContext *GetSuppressionContext() {
CHECK(suppression_ctx);
return suppression_ctx;
}
@@ -111,10 +144,6 @@ void InitializeRootRegions() {
root_regions = new (placeholder) InternalMmapVector();
}
-const char *MaybeCallLsanDefaultOptions() {
- return (&__lsan_default_options) ? __lsan_default_options() : "";
-}
-
void InitCommonLsan() {
InitializeRootRegions();
if (common_flags()->detect_leaks) {
@@ -232,14 +261,45 @@ void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) {
ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable);
}
+#if SANITIZER_FUCHSIA
+
+// Fuchsia handles all threads together with its own callback.
+static void ProcessThreads(SuspendedThreadsList const &, Frontier *) {}
+
+#else
+
+#if SANITIZER_ANDROID
+// FIXME: Move this out into *libcdep.cpp
+extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_iterate_dynamic_tls(
+ pid_t, void (*cb)(void *, void *, uptr, void *), void *);
+#endif
+
+static void ProcessThreadRegistry(Frontier *frontier) {
+ InternalMmapVector ptrs;
+ GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ GetAdditionalThreadContextPtrs, &ptrs);
+
+ for (uptr i = 0; i < ptrs.size(); ++i) {
+ void *ptr = reinterpret_cast(ptrs[i]);
+ uptr chunk = PointsIntoChunk(ptr);
+ if (!chunk)
+ continue;
+ LsanMetadata m(chunk);
+ if (!m.allocated())
+ continue;
+
+ // Mark as reachable and add to frontier.
+ LOG_POINTERS("Treating pointer %p from ThreadContext as reachable\n", ptr);
+ m.set_tag(kReachable);
+ frontier->push_back(chunk);
+ }
+}
+
#if !SANITIZER_EMSCRIPTEN
// Scans thread data (stacks and TLS) for heap pointers.
-void ProcessThreads(SuspendedThreadsList const &suspended_threads,
- Frontier *frontier) {
- InternalMmapVector registers(suspended_threads.RegisterCount());
- uptr registers_begin = reinterpret_cast(registers.data());
- uptr registers_end =
- reinterpret_cast(registers.data() + registers.size());
+static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
+ Frontier *frontier) {
+ InternalMmapVector registers;
for (uptr i = 0; i < suspended_threads.ThreadCount(); i++) {
tid_t os_id = static_cast(suspended_threads.GetThreadID(i));
LOG_THREADS("Processing thread %d.\n", os_id);
@@ -256,7 +316,7 @@ void ProcessThreads(SuspendedThreadsList const &suspended_threads,
}
uptr sp;
PtraceRegistersStatus have_registers =
- suspended_threads.GetRegistersAndSP(i, registers.data(), &sp);
+ suspended_threads.GetRegistersAndSP(i, ®isters, &sp);
if (have_registers != REGISTERS_AVAILABLE) {
Report("Unable to get registers from thread %d.\n", os_id);
// If unable to get SP, consider the entire stack to be reachable unless
@@ -265,9 +325,13 @@ void ProcessThreads(SuspendedThreadsList const &suspended_threads,
sp = stack_begin;
}
- if (flags()->use_registers && have_registers)
+ if (flags()->use_registers && have_registers) {
+ uptr registers_begin = reinterpret_cast(registers.data());
+ uptr registers_end =
+ reinterpret_cast(registers.data() + registers.size());
ScanRangeForPointers(registers_begin, registers_end, frontier,
"REGISTERS", kReachable);
+ }
if (flags()->use_stacks) {
LOG_THREADS("Stack at %p-%p (SP = %p).\n", stack_begin, stack_end, sp);
@@ -311,26 +375,46 @@ void ProcessThreads(SuspendedThreadsList const &suspended_threads,
kReachable);
}
}
+#if SANITIZER_ANDROID
+ auto *cb = +[](void *dtls_begin, void *dtls_end, uptr /*dso_idd*/,
+ void *arg) -> void {
+ ScanRangeForPointers(reinterpret_cast(dtls_begin),
+ reinterpret_cast(dtls_end),
+ reinterpret_cast(arg), "DTLS",
+ kReachable);
+ };
+
+ // FIXME: There might be a race-condition here (and in Bionic) if the
+ // thread is suspended in the middle of updating its DTLS. IOWs, we
+ // could scan already freed memory. (probably fine for now)
+ __libc_iterate_dynamic_tls(os_id, cb, frontier);
+#else
if (dtls && !DTLSInDestruction(dtls)) {
- for (uptr j = 0; j < dtls->dtv_size; ++j) {
- uptr dtls_beg = dtls->dtv[j].beg;
- uptr dtls_end = dtls_beg + dtls->dtv[j].size;
+ ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) {
+ uptr dtls_beg = dtv.beg;
+ uptr dtls_end = dtls_beg + dtv.size;
if (dtls_beg < dtls_end) {
- LOG_THREADS("DTLS %zu at %p-%p.\n", j, dtls_beg, dtls_end);
+ LOG_THREADS("DTLS %zu at %p-%p.\n", id, dtls_beg, dtls_end);
ScanRangeForPointers(dtls_beg, dtls_end, frontier, "DTLS",
kReachable);
}
- }
+ });
} else {
// We are handling a thread with DTLS under destruction. Log about
// this and continue.
LOG_THREADS("Thread %d has DTLS under destruction.\n", os_id);
}
+#endif
}
}
+
+ // Add pointers reachable from ThreadContexts
+ ProcessThreadRegistry(frontier);
}
#endif // !SANITIZER_EMSCRIPTEN
+#endif // SANITIZER_FUCHSIA
+
void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
uptr region_begin, uptr region_end, bool is_readable) {
uptr intersection_begin = Max(root_region.begin, region_begin);
@@ -393,6 +477,24 @@ static void MarkIndirectlyLeakedCb(uptr chunk, void *arg) {
}
}
+static void IgnoredSuppressedCb(uptr chunk, void *arg) {
+ CHECK(arg);
+ chunk = GetUserBegin(chunk);
+ LsanMetadata m(chunk);
+ if (!m.allocated() || m.tag() == kIgnored)
+ return;
+
+ const InternalMmapVector &suppressed =
+ *static_cast *>(arg);
+ uptr idx = InternalLowerBound(suppressed, m.stack_trace_id());
+ if (idx >= suppressed.size() || m.stack_trace_id() != suppressed[idx])
+ return;
+
+ LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", chunk,
+ chunk + m.requested_size(), m.requested_size());
+ m.set_tag(kIgnored);
+}
+
// ForEachChunk callback. If chunk is marked as ignored, adds its address to
// frontier.
static void CollectIgnoredCb(uptr chunk, void *arg) {
@@ -476,25 +578,29 @@ void ProcessPC(Frontier *frontier) {
}
// Sets the appropriate tag on each chunk.
-static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads) {
- // Holds the flood fill frontier.
- Frontier frontier;
-
- ForEachChunk(CollectIgnoredCb, &frontier);
- ProcessGlobalRegions(&frontier);
- ProcessThreads(suspended_threads, &frontier);
- ProcessRootRegions(&frontier);
- FloodFillTag(&frontier, kReachable);
+static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
+ Frontier *frontier) {
+ const InternalMmapVector &suppressed_stacks =
+ GetSuppressionContext()->GetSortedSuppressedStacks();
+ if (!suppressed_stacks.empty()) {
+ ForEachChunk(IgnoredSuppressedCb,
+ const_cast *>(&suppressed_stacks));
+ }
+ ForEachChunk(CollectIgnoredCb, frontier);
+ ProcessGlobalRegions(frontier);
+ ProcessThreads(suspended_threads, frontier);
+ ProcessRootRegions(frontier);
+ FloodFillTag(frontier, kReachable);
- CHECK_EQ(0, frontier.size());
- ProcessPC(&frontier);
+ CHECK_EQ(0, frontier->size());
+ ProcessPC(frontier);
// The check here is relatively expensive, so we do this in a separate flood
// fill. That way we can skip the check for chunks that are reachable
// otherwise.
LOG_POINTERS("Processing platform-specific allocations.\n");
- ProcessPlatformSpecificAllocations(&frontier);
- FloodFillTag(&frontier, kReachable);
+ ProcessPlatformSpecificAllocations(frontier);
+ FloodFillTag(frontier, kReachable);
// Iterate over leaked chunks and mark those that are reachable from other
// leaked chunks.
@@ -539,38 +645,42 @@ static void CollectLeaksCb(uptr chunk, void *arg) {
}
}
-static void PrintMatchedSuppressions() {
+void LeakSuppressionContext::PrintMatchedSuppressions() {
InternalMmapVector matched;
- GetSuppressionContext()->GetMatched(&matched);
+ context.GetMatched(&matched);
if (!matched.size())
return;
const char *line = "-----------------------------------------------------";
Printf("%s\n", line);
Printf("Suppressions used:\n");
Printf(" count bytes template\n");
- for (uptr i = 0; i < matched.size(); i++)
- Printf("%7zu %10zu %s\n", static_cast(atomic_load_relaxed(
- &matched[i]->hit_count)), matched[i]->weight, matched[i]->templ);
+ for (uptr i = 0; i < matched.size(); i++) {
+ Printf("%7zu %10zu %s\n",
+ static_cast(atomic_load_relaxed(&matched[i]->hit_count)),
+ matched[i]->weight, matched[i]->templ);
+ }
Printf("%s\n\n", line);
}
-struct CheckForLeaksParam {
- bool success;
- LeakReport leak_report;
-};
-
static void ReportIfNotSuspended(ThreadContextBase *tctx, void *arg) {
const InternalMmapVector &suspended_threads =
*(const InternalMmapVector *)arg;
if (tctx->status == ThreadStatusRunning) {
- uptr i = InternalLowerBound(suspended_threads, 0, suspended_threads.size(),
- tctx->os_id, CompareLess());
+ uptr i = InternalLowerBound(suspended_threads, tctx->os_id);
if (i >= suspended_threads.size() || suspended_threads[i] != tctx->os_id)
Report("Running thread %d was not suspended. False leaks are possible.\n",
tctx->os_id);
}
}
+#if SANITIZER_FUCHSIA
+
+// Fuchsia provides a libc interface that guarantees all threads are
+// covered, and SuspendedThreadList is never really used.
+static void ReportUnsuspendedThreads(const SuspendedThreadsList &) {}
+
+#else // !SANITIZER_FUCHSIA
+
static void ReportUnsuspendedThreads(
const SuspendedThreadsList &suspended_threads) {
InternalMmapVector threads(suspended_threads.ThreadCount());
@@ -583,6 +693,8 @@ static void ReportUnsuspendedThreads(
&ReportIfNotSuspended, &threads);
}
+#endif // !SANITIZER_FUCHSIA
+
static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
void *arg) {
CheckForLeaksParam *param = reinterpret_cast(arg);
@@ -591,7 +703,7 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
#if !SANITIZER_EMSCRIPTEN
ReportUnsuspendedThreads(suspended_threads);
#endif
- ClassifyAllChunks(suspended_threads);
+ ClassifyAllChunks(suspended_threads, ¶m->frontier);
ForEachChunk(CollectLeaksCb, ¶m->leak_report);
// Clean up for subsequent leak checks. This assumes we did not overwrite any
// kIgnored tags.
@@ -599,44 +711,68 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
param->success = true;
}
-static bool CheckForLeaks() {
- if (&__lsan_is_turned_off && __lsan_is_turned_off())
- return false;
- EnsureMainThreadIDIsCorrect();
- CheckForLeaksParam param;
- param.success = false;
- LockStuffAndStopTheWorld(CheckForLeaksCallback, ¶m);
-
- if (!param.success) {
- Report("LeakSanitizer has encountered a fatal error.\n");
- Report(
- "HINT: For debugging, try setting environment variable "
- "LSAN_OPTIONS=verbosity=1:log_threads=1\n");
- Report(
- "HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)\n");
- Die();
- }
- param.leak_report.ApplySuppressions();
- uptr unsuppressed_count = param.leak_report.UnsuppressedLeakCount();
- if (unsuppressed_count > 0) {
+static bool PrintResults(LeakReport &report) {
+ uptr unsuppressed_count = report.UnsuppressedLeakCount();
+ if (unsuppressed_count) {
Decorator d;
- Printf("\n"
- "================================================================="
- "\n");
+ Printf(
+ "\n"
+ "================================================================="
+ "\n");
Printf("%s", d.Error());
Report("ERROR: LeakSanitizer: detected memory leaks\n");
Printf("%s", d.Default());
- param.leak_report.ReportTopLeaks(flags()->max_leaks);
+ report.ReportTopLeaks(flags()->max_leaks);
}
if (common_flags()->print_suppressions)
- PrintMatchedSuppressions();
+ GetSuppressionContext()->PrintMatchedSuppressions();
if (unsuppressed_count > 0) {
- param.leak_report.PrintSummary();
+ report.PrintSummary();
return true;
}
return false;
}
+static bool CheckForLeaks() {
+ if (&__lsan_is_turned_off && __lsan_is_turned_off())
+ return false;
+ // Inside LockStuffAndStopTheWorld we can't run symbolizer, so we can't match
+ // suppressions. However if a stack id was previously suppressed, it should be
+ // suppressed in future checks as well.
+ for (int i = 0;; ++i) {
+ EnsureMainThreadIDIsCorrect();
+ CheckForLeaksParam param;
+ LockStuffAndStopTheWorld(CheckForLeaksCallback, ¶m);
+ if (!param.success) {
+ Report("LeakSanitizer has encountered a fatal error.\n");
+ Report(
+ "HINT: For debugging, try setting environment variable "
+ "LSAN_OPTIONS=verbosity=1:log_threads=1\n");
+ Report(
+ "HINT: LeakSanitizer does not work under ptrace (strace, gdb, "
+ "etc)\n");
+ Die();
+ }
+ // No new suppressions stacks, so rerun will not help and we can report.
+ if (!param.leak_report.ApplySuppressions())
+ return PrintResults(param.leak_report);
+
+ // No indirect leaks to report, so we are done here.
+ if (!param.leak_report.IndirectUnsuppressedLeakCount())
+ return PrintResults(param.leak_report);
+
+ if (i >= 8) {
+ Report("WARNING: LeakSanitizer gave up on indirect leaks suppression.\n");
+ return PrintResults(param.leak_report);
+ }
+
+ // We found a new previously unseen suppressed call stack. Rerun to make
+ // sure it does not hold indirect leaks.
+ VReport(1, "Rerun with %zu suppressed stacks.",
+ GetSuppressionContext()->GetSortedSuppressedStacks().size());
+ }
+}
+
static bool has_reported_leaks = false;
bool HasReportedLeaks() { return has_reported_leaks; }
@@ -657,21 +793,20 @@ static int DoRecoverableLeakCheck() {
void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
-static Suppression *GetSuppressionForAddr(uptr addr) {
+Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
Suppression *s = nullptr;
// Suppress by module name.
- SuppressionContext *suppressions = GetSuppressionContext();
if (const char *module_name =
Symbolizer::GetOrInit()->GetModuleNameForPc(addr))
- if (suppressions->Match(module_name, kSuppressionLeak, &s))
+ if (context.Match(module_name, kSuppressionLeak, &s))
return s;
// Suppress by file or function name.
SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
- if (suppressions->Match(cur->info.function, kSuppressionLeak, &s) ||
- suppressions->Match(cur->info.file, kSuppressionLeak, &s)) {
+ if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
+ context.Match(cur->info.file, kSuppressionLeak, &s)) {
break;
}
}
@@ -679,7 +814,9 @@ static Suppression *GetSuppressionForAddr(uptr addr) {
return s;
}
-static Suppression *GetSuppressionForStack(u32 stack_trace_id) {
+Suppression *LeakSuppressionContext::GetSuppressionForStack(
+ u32 stack_trace_id) {
+ LazyInit();
StackTrace stack = StackDepotGet(stack_trace_id);
for (uptr i = 0; i < stack.size; i++) {
#if SANITIZER_EMSCRIPTEN
@@ -691,7 +828,11 @@ static Suppression *GetSuppressionForStack(u32 stack_trace_id) {
Suppression *s = GetSuppressionForAddr(
StackTrace::GetPreviousInstructionPc(stack.trace[i]));
#endif
- if (s) return s;
+ if (s) {
+ suppressed_stacks_sorted = false;
+ suppressed_stacks.push_back(stack_trace_id);
+ return s;
+ }
}
return nullptr;
}
@@ -802,16 +943,21 @@ void LeakReport::PrintSummary() {
ReportErrorSummary(summary.data());
}
-void LeakReport::ApplySuppressions() {
+uptr LeakReport::ApplySuppressions() {
+ LeakSuppressionContext *suppressions = GetSuppressionContext();
+ uptr new_suppressions = false;
for (uptr i = 0; i < leaks_.size(); i++) {
- Suppression *s = GetSuppressionForStack(leaks_[i].stack_trace_id);
+ Suppression *s =
+ suppressions->GetSuppressionForStack(leaks_[i].stack_trace_id);
if (s) {
s->weight += leaks_[i].total_size;
atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) +
leaks_[i].hit_count);
leaks_[i].is_suppressed = true;
+ ++new_suppressions;
}
}
+ return new_suppressions;
}
uptr LeakReport::UnsuppressedLeakCount() {
@@ -821,6 +967,14 @@ uptr LeakReport::UnsuppressedLeakCount() {
return result;
}
+uptr LeakReport::IndirectUnsuppressedLeakCount() {
+ uptr result = 0;
+ for (uptr i = 0; i < leaks_.size(); i++)
+ if (!leaks_[i].is_suppressed && !leaks_[i].is_directly_leaked)
+ result++;
+ return result;
+}
+
} // namespace __lsan
#else // CAN_SANITIZE_LEAKS
namespace __lsan {
@@ -923,12 +1077,11 @@ int __lsan_do_recoverable_leak_check() {
return 0;
}
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char * __lsan_default_options() {
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_options, void) {
return "";
}
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
int __lsan_is_turned_off() {
return 0;
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.h b/system/lib/compiler-rt/lib/lsan/lsan_common.h
index 6901e18829aeb..809e7e57a9f19 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common.h
@@ -29,18 +29,19 @@
// To enable LeakSanitizer on a new architecture, one needs to implement the
// internal_clone function as well as (probably) adjust the TLS machinery for
// the new architecture inside the sanitizer library.
-#if (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC) && \
- (SANITIZER_WORDSIZE == 64) && \
- (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
- defined(__powerpc64__))
+// Exclude leak-detection on arm32 for Android because `__aeabi_read_tp`
+// is missing. This caused a link error.
+#if SANITIZER_ANDROID && (__ANDROID_API__ < 28 || defined(__arm__))
+#define CAN_SANITIZE_LEAKS 0
+#elif (SANITIZER_LINUX || SANITIZER_MAC) && (SANITIZER_WORDSIZE == 64) && \
+ (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
+ defined(__powerpc64__) || defined(__s390x__))
#define CAN_SANITIZE_LEAKS 1
-#elif defined(__i386__) && \
- (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC)
+#elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_MAC)
#define CAN_SANITIZE_LEAKS 1
-#elif defined(__arm__) && \
- SANITIZER_LINUX && !SANITIZER_ANDROID
+#elif defined(__arm__) && SANITIZER_LINUX
#define CAN_SANITIZE_LEAKS 1
-#elif SANITIZER_EMSCRIPTEN
+#elif SANITIZER_NETBSD || SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN
#define CAN_SANITIZE_LEAKS 1
#else
#define CAN_SANITIZE_LEAKS 0
@@ -49,6 +50,7 @@
namespace __sanitizer {
class FlagParser;
class ThreadRegistry;
+class ThreadContextBase;
struct DTLS;
}
@@ -102,8 +104,9 @@ class LeakReport {
ChunkTag tag);
void ReportTopLeaks(uptr max_leaks);
void PrintSummary();
- void ApplySuppressions();
+ uptr ApplySuppressions();
uptr UnsuppressedLeakCount();
+ uptr IndirectUnsuppressedLeakCount();
private:
void PrintReportForLeak(uptr index);
@@ -126,12 +129,25 @@ struct RootRegion {
uptr size;
};
+// LockStuffAndStopTheWorld can start to use Scan* calls to collect into
+// this Frontier vector before the StopTheWorldCallback actually runs.
+// This is used when the OS has a unified callback API for suspending
+// threads and enumerating roots.
+struct CheckForLeaksParam {
+ Frontier frontier;
+ LeakReport leak_report;
+ bool success = false;
+};
+
InternalMmapVector const *GetRootRegions();
void ScanRootRegion(Frontier *frontier, RootRegion const ®ion,
uptr region_begin, uptr region_end, bool is_readable);
+void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg);
+void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs);
// Run stoptheworld while holding any platform-specific locks, as well as the
// allocator and thread registry locks.
-void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void* argument);
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
+ CheckForLeaksParam* argument);
void ScanRangeForPointers(uptr begin, uptr end,
Frontier *frontier,
@@ -211,6 +227,7 @@ ThreadRegistry *GetThreadRegistryLocked();
bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
uptr *cache_end, DTLS **dtls);
+void GetAllThreadAllocatorCachesLocked(InternalMmapVector *caches);
void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
void *arg);
// Scans thread data (stacks and TLS) for heap pointers.
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp
index de1efe6bc264f..a60dea00e88c8 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp
@@ -102,7 +102,8 @@ void HandleLeaks() {
if (common_flags()->exitcode) Die();
}
-void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void *argument) {
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
+ CheckForLeaksParam *argument) {
// Currently, on Emscripten this does nothing and just calls the callback.
// This works fine on a single-threaded environment.
LockThreadRegistry();
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp
new file mode 100644
index 0000000000000..2d35fa5b1cffd
--- /dev/null
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp
@@ -0,0 +1,165 @@
+//=-- lsan_common_fuchsia.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer.
+// Implementation of common leak checking functionality. Fuchsia-specific code.
+//
+//===---------------------------------------------------------------------===//
+
+#include "lsan_common.h"
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if CAN_SANITIZE_LEAKS && SANITIZER_FUCHSIA
+#include
+
+#include "lsan_allocator.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_stoptheworld_fuchsia.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
+
+// Ensure that the Zircon system ABI is linked in.
+#pragma comment(lib, "zircon")
+
+namespace __lsan {
+
+void InitializePlatformSpecificModules() {}
+
+LoadedModule *GetLinker() { return nullptr; }
+
+__attribute__((tls_model("initial-exec"))) THREADLOCAL int disable_counter;
+bool DisabledInThisThread() { return disable_counter > 0; }
+void DisableInThisThread() { disable_counter++; }
+void EnableInThisThread() {
+ if (disable_counter == 0) {
+ DisableCounterUnderflow();
+ }
+ disable_counter--;
+}
+
+// There is nothing left to do after the globals callbacks.
+void ProcessGlobalRegions(Frontier *frontier) {}
+
+// Nothing to do here.
+void ProcessPlatformSpecificAllocations(Frontier *frontier) {}
+
+// On Fuchsia, we can intercept _Exit gracefully, and return a failing exit
+// code if required at that point. Calling Die() here is undefined
+// behavior and causes rare race conditions.
+void HandleLeaks() {}
+
+int ExitHook(int status) {
+ return status == 0 && HasReportedLeaks() ? common_flags()->exitcode : status;
+}
+
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
+ CheckForLeaksParam *argument) {
+ LockThreadRegistry();
+ LockAllocator();
+
+ struct Params {
+ InternalMmapVector allocator_caches;
+ StopTheWorldCallback callback;
+ CheckForLeaksParam *argument;
+ } params = {{}, callback, argument};
+
+ // Callback from libc for globals (data/bss modulo relro), when enabled.
+ auto globals = +[](void *chunk, size_t size, void *data) {
+ auto params = static_cast(data);
+ uptr begin = reinterpret_cast(chunk);
+ uptr end = begin + size;
+ ScanGlobalRange(begin, end, ¶ms->argument->frontier);
+ };
+
+ // Callback from libc for thread stacks.
+ auto stacks = +[](void *chunk, size_t size, void *data) {
+ auto params = static_cast(data);
+ uptr begin = reinterpret_cast(chunk);
+ uptr end = begin + size;
+ ScanRangeForPointers(begin, end, ¶ms->argument->frontier, "STACK",
+ kReachable);
+ };
+
+ // Callback from libc for thread registers.
+ auto registers = +[](void *chunk, size_t size, void *data) {
+ auto params = static_cast(data);
+ uptr begin = reinterpret_cast(chunk);
+ uptr end = begin + size;
+ ScanRangeForPointers(begin, end, ¶ms->argument->frontier, "REGISTERS",
+ kReachable);
+ };
+
+ if (flags()->use_tls) {
+ // Collect the allocator cache range from each thread so these
+ // can all be excluded from the reported TLS ranges.
+ GetAllThreadAllocatorCachesLocked(¶ms.allocator_caches);
+ __sanitizer::Sort(params.allocator_caches.data(),
+ params.allocator_caches.size());
+ }
+
+ // Callback from libc for TLS regions. This includes thread_local
+ // variables as well as C11 tss_set and POSIX pthread_setspecific.
+ auto tls = +[](void *chunk, size_t size, void *data) {
+ auto params = static_cast(data);
+ uptr begin = reinterpret_cast(chunk);
+ uptr end = begin + size;
+ auto i = __sanitizer::InternalLowerBound(params->allocator_caches, begin);
+ if (i < params->allocator_caches.size() &&
+ params->allocator_caches[i] >= begin &&
+ end - params->allocator_caches[i] <= sizeof(AllocatorCache)) {
+ // Split the range in two and omit the allocator cache within.
+ ScanRangeForPointers(begin, params->allocator_caches[i],
+ ¶ms->argument->frontier, "TLS", kReachable);
+ uptr begin2 = params->allocator_caches[i] + sizeof(AllocatorCache);
+ ScanRangeForPointers(begin2, end, ¶ms->argument->frontier, "TLS",
+ kReachable);
+ } else {
+ ScanRangeForPointers(begin, end, ¶ms->argument->frontier, "TLS",
+ kReachable);
+ }
+ };
+
+ // This stops the world and then makes callbacks for various memory regions.
+ // The final callback is the last thing before the world starts up again.
+ __sanitizer_memory_snapshot(
+ flags()->use_globals ? globals : nullptr,
+ flags()->use_stacks ? stacks : nullptr,
+ flags()->use_registers ? registers : nullptr,
+ flags()->use_tls ? tls : nullptr,
+ [](zx_status_t, void *data) {
+ auto params = static_cast(data);
+
+ // We don't use the thread registry at all for enumerating the threads
+ // and their stacks, registers, and TLS regions. So use it separately
+ // just for the allocator cache, and to call ForEachExtraStackRange,
+ // which ASan needs.
+ if (flags()->use_stacks) {
+ GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ [](ThreadContextBase *tctx, void *arg) {
+ ForEachExtraStackRange(tctx->os_id, ForEachExtraStackRangeCb,
+ arg);
+ },
+ ¶ms->argument->frontier);
+ }
+
+ params->callback(SuspendedThreadsListFuchsia(), params->argument);
+ },
+ ¶ms);
+
+ UnlockAllocator();
+ UnlockThreadRegistry();
+}
+
+} // namespace __lsan
+
+// This is declared (in extern "C") by .
+// _Exit calls this directly to intercept and change the status value.
+int __sanitizer_process_exit_hook(int status) {
+ return __lsan::ExitHook(status);
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_linux.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_linux.cpp
index a9693e02edd1c..896f0479cb0a8 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common_linux.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common_linux.cpp
@@ -93,6 +93,11 @@ static int ProcessGlobalRegionsCallback(struct dl_phdr_info *info, size_t size,
return 0;
}
+#if SANITIZER_ANDROID && __ANDROID_API__ < 21
+extern "C" __attribute__((weak)) int dl_iterate_phdr(
+ int (*)(struct dl_phdr_info *, size_t, void *), void *);
+#endif
+
// Scans global variables for heap pointers.
void ProcessGlobalRegions(Frontier *frontier) {
if (!flags()->use_globals) return;
@@ -134,7 +139,8 @@ static int LockStuffAndStopTheWorldCallback(struct dl_phdr_info *info,
// while holding the libdl lock in the parent thread, we can safely reenter it
// in the tracer. The solution is to run stoptheworld from a dl_iterate_phdr()
// callback in the parent thread.
-void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void *argument) {
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
+ CheckForLeaksParam *argument) {
DoStopTheWorldParam param = {callback, argument};
dl_iterate_phdr(LockStuffAndStopTheWorldCallback, ¶m);
}
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp
index c1804e93c11db..8516a176eb467 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp
@@ -193,7 +193,8 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
// causes rare race conditions.
void HandleLeaks() {}
-void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void *argument) {
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
+ CheckForLeaksParam *argument) {
LockThreadRegistry();
LockAllocator();
StopTheWorld(callback, argument);
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp
new file mode 100644
index 0000000000000..40e65c6fb7293
--- /dev/null
+++ b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp
@@ -0,0 +1,123 @@
+//=-- lsan_fuchsia.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer.
+// Standalone LSan RTL code specific to Fuchsia.
+//
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if SANITIZER_FUCHSIA
+#include
+
+#include "lsan.h"
+#include "lsan_allocator.h"
+
+using namespace __lsan;
+
+namespace __lsan {
+
+void LsanOnDeadlySignal(int signo, void *siginfo, void *context) {}
+
+ThreadContext::ThreadContext(int tid) : ThreadContextLsanBase(tid) {}
+
+struct OnCreatedArgs {
+ uptr stack_begin, stack_end;
+};
+
+// On Fuchsia, the stack bounds of a new thread are available before
+// the thread itself has started running.
+void ThreadContext::OnCreated(void *arg) {
+ // Stack bounds passed through from __sanitizer_before_thread_create_hook
+ // or InitializeMainThread.
+ auto args = reinterpret_cast(arg);
+ stack_begin_ = args->stack_begin;
+ stack_end_ = args->stack_end;
+}
+
+struct OnStartedArgs {
+ uptr cache_begin, cache_end;
+};
+
+void ThreadContext::OnStarted(void *arg) {
+ auto args = reinterpret_cast(arg);
+ cache_begin_ = args->cache_begin;
+ cache_end_ = args->cache_end;
+}
+
+void ThreadStart(u32 tid) {
+ OnStartedArgs args;
+ GetAllocatorCacheRange(&args.cache_begin, &args.cache_end);
+ CHECK_EQ(args.cache_end - args.cache_begin, sizeof(AllocatorCache));
+ ThreadContextLsanBase::ThreadStart(tid, GetTid(), ThreadType::Regular, &args);
+}
+
+void InitializeMainThread() {
+ OnCreatedArgs args;
+ __sanitizer::GetThreadStackTopAndBottom(true, &args.stack_end,
+ &args.stack_begin);
+ u32 tid = ThreadCreate(0, GetThreadSelf(), true, &args);
+ CHECK_EQ(tid, 0);
+ ThreadStart(tid);
+}
+
+void GetAllThreadAllocatorCachesLocked(InternalMmapVector *caches) {
+ GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ [](ThreadContextBase *tctx, void *arg) {
+ auto ctx = static_cast(tctx);
+ static_cast(arg)->push_back(ctx->cache_begin());
+ },
+ caches);
+}
+
+} // namespace __lsan
+
+// These are declared (in extern "C") by .
+// The system runtime will call our definitions directly.
+
+// This is called before each thread creation is attempted. So, in
+// its first call, the calling thread is the initial and sole thread.
+void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
+ const char *name, void *stack_base,
+ size_t stack_size) {
+ uptr user_id = reinterpret_cast(thread);
+ ENSURE_LSAN_INITED;
+ EnsureMainThreadIDIsCorrect();
+ OnCreatedArgs args;
+ args.stack_begin = reinterpret_cast(stack_base);
+ args.stack_end = args.stack_begin + stack_size;
+ u32 parent_tid = GetCurrentThread();
+ u32 tid = ThreadCreate(parent_tid, user_id, detached, &args);
+ return reinterpret_cast(static_cast(tid));
+}
+
+// This is called after creating a new thread (in the creating thread),
+// with the pointer returned by __sanitizer_before_thread_create_hook (above).
+void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) {
+ u32 tid = static_cast(reinterpret_cast(hook));
+ // On success, there is nothing to do here.
+ if (error != thrd_success) {
+ // Clean up the thread registry for the thread creation that didn't happen.
+ GetThreadRegistryLocked()->FinishThread(tid);
+ }
+}
+
+// This is called in the newly-created thread before it runs anything else,
+// with the pointer returned by __sanitizer_before_thread_create_hook (above).
+void __sanitizer_thread_start_hook(void *hook, thrd_t self) {
+ u32 tid = static_cast(reinterpret_cast(hook));
+ ThreadStart(tid);
+}
+
+// Each thread runs this just before it exits,
+// with the pointer returned by BeforeThreadCreateHook (above).
+// All per-thread destructors have already been called.
+void __sanitizer_thread_exit_hook(void *hook, thrd_t self) { ThreadFinish(); }
+
+#endif // SANITIZER_FUCHSIA
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.h b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.h
new file mode 100644
index 0000000000000..e730d8f25f21e
--- /dev/null
+++ b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.h
@@ -0,0 +1,35 @@
+//=-- lsan_fuchsia.h ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer.
+// Standalone LSan RTL code specific to Fuchsia.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LSAN_FUCHSIA_H
+#define LSAN_FUCHSIA_H
+
+#include "lsan_thread.h"
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if !SANITIZER_FUCHSIA
+#error "lsan_fuchsia.h is used only on Fuchsia systems (SANITIZER_FUCHSIA)"
+#endif
+
+namespace __lsan {
+
+class ThreadContext final : public ThreadContextLsanBase {
+ public:
+ explicit ThreadContext(int tid);
+ void OnCreated(void *arg) override;
+ void OnStarted(void *arg) override;
+};
+
+} // namespace __lsan
+
+#endif // LSAN_FUCHSIA_H
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp
index b27f324bc558f..e90bcb50d0f3a 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -22,7 +22,9 @@
#include "sanitizer_common/sanitizer_platform_interceptors.h"
#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#if SANITIZER_POSIX
#include "sanitizer_common/sanitizer_posix.h"
+#endif
#include "sanitizer_common/sanitizer_tls_get_addr.h"
#include "lsan.h"
#include "lsan_allocator.h"
@@ -61,6 +63,9 @@ INTERCEPTOR(void, free, void *p) {
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
+ // This hack is not required for Fuchsia because there are no dlsym calls
+ // involved in setting up interceptors.
+#if !SANITIZER_FUCHSIA
if (lsan_init_is_running) {
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
const uptr kCallocPoolSize = 1024;
@@ -72,6 +77,7 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
CHECK(allocated < kCallocPoolSize);
return mem;
}
+#endif // !SANITIZER_FUCHSIA
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_calloc(nmemb, size, stack);
@@ -100,7 +106,7 @@ INTERCEPTOR(void*, valloc, uptr size) {
GET_STACK_TRACE_MALLOC;
return lsan_valloc(size, stack);
}
-#endif
+#endif // !SANITIZER_MAC
#if SANITIZER_INTERCEPT_MEMALIGN
INTERCEPTOR(void*, memalign, uptr alignment, uptr size) {
@@ -109,7 +115,11 @@ INTERCEPTOR(void*, memalign, uptr alignment, uptr size) {
return lsan_memalign(alignment, size, stack);
}
#define LSAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
+#else
+#define LSAN_MAYBE_INTERCEPT_MEMALIGN
+#endif // SANITIZER_INTERCEPT_MEMALIGN
+#if SANITIZER_INTERCEPT___LIBC_MEMALIGN
INTERCEPTOR(void *, __libc_memalign, uptr alignment, uptr size) {
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
@@ -119,9 +129,8 @@ INTERCEPTOR(void *, __libc_memalign, uptr alignment, uptr size) {
}
#define LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN INTERCEPT_FUNCTION(__libc_memalign)
#else
-#define LSAN_MAYBE_INTERCEPT_MEMALIGN
#define LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN
-#endif // SANITIZER_INTERCEPT_MEMALIGN
+#endif // SANITIZER_INTERCEPT___LIBC_MEMALIGN
#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
INTERCEPTOR(void*, aligned_alloc, uptr alignment, uptr size) {
@@ -307,7 +316,7 @@ INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
///// Thread initialization and finalization. /////
-#if !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+#if !SANITIZER_NETBSD && !SANITIZER_FREEBSD && !SANITIZER_FUCHSIA
static unsigned g_thread_finalize_key;
static void thread_finalize(void *v) {
@@ -389,6 +398,7 @@ extern "C" {
int emscripten_builtin_pthread_create(void *thread, void *attr,
void *(*callback)(void *), void *arg);
int emscripten_builtin_pthread_join(void *th, void **ret);
+ int emscripten_builtin_pthread_detach(void *th);
void *emscripten_builtin_malloc(size_t size);
void emscripten_builtin_free(void *);
}
@@ -404,6 +414,8 @@ INTERCEPTOR(char *, strerror, int errnum) {
#define LSAN_MAYBE_INTERCEPT_STRERROR
#endif
+#if SANITIZER_POSIX
+
struct ThreadParam {
void *(*callback)(void *arg);
void *param;
@@ -426,7 +438,6 @@ extern "C" void *__lsan_thread_start_func(void *arg) {
int tid = 0;
while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
internal_sched_yield();
- SetCurrentThread(tid);
ThreadStart(tid, GetTid());
#if SANITIZER_EMSCRIPTEN
emscripten_builtin_free(p);
@@ -498,27 +509,30 @@ INTERCEPTOR(int, pthread_join, void *th, void **ret) {
return res;
}
+INTERCEPTOR(int, pthread_detach, void *th) {
+ ENSURE_LSAN_INITED;
+ int tid = ThreadTid((uptr)th);
+ int res = REAL(pthread_detach)(th);
+ if (res == 0)
+ ThreadDetach(tid);
+ return res;
+}
+
#if !SANITIZER_EMSCRIPTEN
INTERCEPTOR(void, _exit, int status) {
if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
REAL(_exit)(status);
}
-#endif
-
-#if SANITIZER_EMSCRIPTEN
-namespace __lsan {
-void InitializeInterceptors() {}
-
-} // namespace __lsan
-
-#else
#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+#endif
namespace __lsan {
void InitializeInterceptors() {
+ // Fuchsia doesn't use interceptors that require any setup.
+#if !SANITIZER_FUCHSIA && !SANITIZER_EMSCRIPTEN
InitializeSignalInterceptors();
INTERCEPT_FUNCTION(malloc);
@@ -536,6 +550,7 @@ void InitializeInterceptors() {
LSAN_MAYBE_INTERCEPT_MALLINFO;
LSAN_MAYBE_INTERCEPT_MALLOPT;
INTERCEPT_FUNCTION(pthread_create);
+ INTERCEPT_FUNCTION(pthread_detach);
INTERCEPT_FUNCTION(pthread_join);
INTERCEPT_FUNCTION(_exit);
@@ -554,6 +569,8 @@ void InitializeInterceptors() {
Die();
}
#endif
+
+#endif // !SANITIZER_FUCHSIA
}
} // namespace __lsan
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp b/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp
index 691ce42313687..63ae2c1a945b8 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp
@@ -6,13 +6,13 @@
//
//===----------------------------------------------------------------------===//
//
-// This file is a part of LeakSanitizer. Linux/NetBSD-specific code.
+// This file is a part of LeakSanitizer. Linux/NetBSD/Fuchsia-specific code.
//
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_LINUX || SANITIZER_EMSCRIPTEN
+#if SANITIZER_LINUX || SANITIZER_NETBSD || SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN
#include "lsan_allocator.h"
@@ -29,4 +29,4 @@ void ReplaceSystemMalloc() {}
} // namespace __lsan
-#endif // SANITIZER_LINUX
+#endif // SANITIZER_LINUX || SANITIZER_NETBSD || SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp b/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp
new file mode 100644
index 0000000000000..8e05915dd1b99
--- /dev/null
+++ b/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp
@@ -0,0 +1,96 @@
+//=-- lsan_posix.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer.
+// Standalone LSan RTL code common to POSIX-like systems.
+//
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if SANITIZER_POSIX
+#include "lsan.h"
+#include "lsan_allocator.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+namespace __lsan {
+
+ThreadContext::ThreadContext(int tid) : ThreadContextLsanBase(tid) {}
+
+struct OnStartedArgs {
+ uptr stack_begin;
+ uptr stack_end;
+ uptr cache_begin;
+ uptr cache_end;
+ uptr tls_begin;
+ uptr tls_end;
+ DTLS *dtls;
+};
+
+void ThreadContext::OnStarted(void *arg) {
+ auto args = reinterpret_cast(arg);
+ stack_begin_ = args->stack_begin;
+ stack_end_ = args->stack_end;
+ tls_begin_ = args->tls_begin;
+ tls_end_ = args->tls_end;
+ cache_begin_ = args->cache_begin;
+ cache_end_ = args->cache_end;
+ dtls_ = args->dtls;
+}
+
+void ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type) {
+ OnStartedArgs args;
+ uptr stack_size = 0;
+ uptr tls_size = 0;
+ GetThreadStackAndTls(tid == 0, &args.stack_begin, &stack_size,
+ &args.tls_begin, &tls_size);
+ args.stack_end = args.stack_begin + stack_size;
+ args.tls_end = args.tls_begin + tls_size;
+ GetAllocatorCacheRange(&args.cache_begin, &args.cache_end);
+ args.dtls = DTLS_Get();
+ ThreadContextLsanBase::ThreadStart(tid, os_id, thread_type, &args);
+}
+
+bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
+ uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
+ uptr *cache_end, DTLS **dtls) {
+ ThreadContext *context = static_cast(
+ GetThreadRegistryLocked()->FindThreadContextByOsIDLocked(os_id));
+ if (!context)
+ return false;
+ *stack_begin = context->stack_begin();
+ *stack_end = context->stack_end();
+ *tls_begin = context->tls_begin();
+ *tls_end = context->tls_end();
+ *cache_begin = context->cache_begin();
+ *cache_end = context->cache_end();
+ *dtls = context->dtls();
+ return true;
+}
+
+void InitializeMainThread() {
+ u32 tid = ThreadCreate(0, 0, true);
+ CHECK_EQ(tid, 0);
+ ThreadStart(tid, GetTid());
+}
+
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+void LsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind,
+ nullptr);
+}
+
+} // namespace __lsan
+
+#endif // SANITIZER_POSIX
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_posix.h b/system/lib/compiler-rt/lib/lsan/lsan_posix.h
new file mode 100644
index 0000000000000..b1265f233f363
--- /dev/null
+++ b/system/lib/compiler-rt/lib/lsan/lsan_posix.h
@@ -0,0 +1,49 @@
+//=-- lsan_posix.h -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of LeakSanitizer.
+// Standalone LSan RTL code common to POSIX-like systems.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LSAN_POSIX_H
+#define LSAN_POSIX_H
+
+#include "lsan_thread.h"
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if !SANITIZER_POSIX
+#error "lsan_posix.h is used only on POSIX-like systems (SANITIZER_POSIX)"
+#endif
+
+namespace __sanitizer {
+struct DTLS;
+}
+
+namespace __lsan {
+
+class ThreadContext final : public ThreadContextLsanBase {
+ public:
+ explicit ThreadContext(int tid);
+ void OnStarted(void *arg) override;
+ uptr tls_begin() { return tls_begin_; }
+ uptr tls_end() { return tls_end_; }
+ DTLS *dtls() { return dtls_; }
+
+ private:
+ uptr tls_begin_ = 0;
+ uptr tls_end_ = 0;
+ DTLS *dtls_ = nullptr;
+};
+
+void ThreadStart(u32 tid, tid_t os_id,
+ ThreadType thread_type = ThreadType::Regular);
+
+} // namespace __lsan
+
+#endif // LSAN_POSIX_H
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp b/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp
index 84e7ce61b9756..371a1f29dfe0b 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp
@@ -13,12 +13,13 @@
#include "lsan_thread.h"
+#include "lsan.h"
+#include "lsan_allocator.h"
+#include "lsan_common.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
-#include "lsan_allocator.h"
-#include "lsan_common.h"
namespace __lsan {
@@ -26,7 +27,7 @@ static ThreadRegistry *thread_registry;
static ThreadContextBase *CreateThreadContext(u32 tid) {
void *mem = MmapOrDie(sizeof(ThreadContext), "ThreadContext");
- return new(mem) ThreadContext(tid);
+ return new (mem) ThreadContext(tid);
}
static const uptr kMaxThreads = 1 << 13;
@@ -34,59 +35,26 @@ static const uptr kThreadQuarantineSize = 64;
void InitializeThreadRegistry() {
static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
- thread_registry = new(thread_registry_placeholder)
- ThreadRegistry(CreateThreadContext, kMaxThreads, kThreadQuarantineSize);
+ thread_registry = new (thread_registry_placeholder)
+ ThreadRegistry(CreateThreadContext, kMaxThreads, kThreadQuarantineSize);
}
-ThreadContext::ThreadContext(int tid)
- : ThreadContextBase(tid),
- stack_begin_(0),
- stack_end_(0),
- cache_begin_(0),
- cache_end_(0),
- tls_begin_(0),
- tls_end_(0),
- dtls_(nullptr) {}
-
-struct OnStartedArgs {
- uptr stack_begin, stack_end,
- cache_begin, cache_end,
- tls_begin, tls_end;
- DTLS *dtls;
-};
-
-void ThreadContext::OnStarted(void *arg) {
- OnStartedArgs *args = reinterpret_cast(arg);
- stack_begin_ = args->stack_begin;
- stack_end_ = args->stack_end;
- tls_begin_ = args->tls_begin;
- tls_end_ = args->tls_end;
- cache_begin_ = args->cache_begin;
- cache_end_ = args->cache_end;
- dtls_ = args->dtls;
-}
+ThreadContextLsanBase::ThreadContextLsanBase(int tid)
+ : ThreadContextBase(tid) {}
-void ThreadContext::OnFinished() {
+void ThreadContextLsanBase::OnFinished() {
AllocatorThreadFinish();
DTLS_Destroy();
}
-u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached) {
- return thread_registry->CreateThread(user_id, detached, parent_tid,
- /* arg */ nullptr);
+u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached, void *arg) {
+ return thread_registry->CreateThread(user_id, detached, parent_tid, arg);
}
-void ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type) {
- OnStartedArgs args;
- uptr stack_size = 0;
- uptr tls_size = 0;
- GetThreadStackAndTls(tid == 0, &args.stack_begin, &stack_size,
- &args.tls_begin, &tls_size);
- args.stack_end = args.stack_begin + stack_size;
- args.tls_end = args.tls_begin + tls_size;
- GetAllocatorCacheRange(&args.cache_begin, &args.cache_end);
- args.dtls = DTLS_Get();
- thread_registry->StartThread(tid, os_id, thread_type, &args);
+void ThreadContextLsanBase::ThreadStart(u32 tid, tid_t os_id,
+ ThreadType thread_type, void *arg) {
+ thread_registry->StartThread(tid, os_id, thread_type, arg);
+ SetCurrentThread(tid);
}
void ThreadFinish() {
@@ -95,7 +63,8 @@ void ThreadFinish() {
}
ThreadContext *CurrentThreadContext() {
- if (!thread_registry) return nullptr;
+ if (!thread_registry)
+ return nullptr;
if (GetCurrentThread() == kInvalidTid)
return nullptr;
// No lock needed when getting current thread.
@@ -111,12 +80,17 @@ static bool FindThreadByUid(ThreadContextBase *tctx, void *arg) {
}
u32 ThreadTid(uptr uid) {
- return thread_registry->FindThread(FindThreadByUid, (void*)uid);
+ return thread_registry->FindThread(FindThreadByUid, (void *)uid);
+}
+
+void ThreadDetach(u32 tid) {
+ CHECK_NE(tid, kInvalidTid);
+ thread_registry->DetachThread(tid, /* arg */ nullptr);
}
void ThreadJoin(u32 tid) {
CHECK_NE(tid, kInvalidTid);
- thread_registry->JoinThread(tid, /* arg */nullptr);
+ thread_registry->JoinThread(tid, /* arg */ nullptr);
}
void EnsureMainThreadIDIsCorrect() {
@@ -126,37 +100,16 @@ void EnsureMainThreadIDIsCorrect() {
///// Interface to the common LSan module. /////
-bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
- uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
- uptr *cache_end, DTLS **dtls) {
- ThreadContext *context = static_cast(
- thread_registry->FindThreadContextByOsIDLocked(os_id));
- if (!context) return false;
- *stack_begin = context->stack_begin();
- *stack_end = context->stack_end();
- *tls_begin = context->tls_begin();
- *tls_end = context->tls_end();
- *cache_begin = context->cache_begin();
- *cache_end = context->cache_end();
- *dtls = context->dtls();
- return true;
-}
-
void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
- void *arg) {
-}
+ void *arg) {}
-void LockThreadRegistry() {
- thread_registry->Lock();
-}
+void LockThreadRegistry() { thread_registry->Lock(); }
-void UnlockThreadRegistry() {
- thread_registry->Unlock();
-}
+void UnlockThreadRegistry() { thread_registry->Unlock(); }
ThreadRegistry *GetThreadRegistryLocked() {
thread_registry->CheckLocked();
return thread_registry;
}
-} // namespace __lsan
+} // namespace __lsan
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_thread.h b/system/lib/compiler-rt/lib/lsan/lsan_thread.h
index b869d066d9d8b..36643753d0190 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_thread.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan_thread.h
@@ -16,38 +16,38 @@
#include "sanitizer_common/sanitizer_thread_registry.h"
-namespace __sanitizer {
-struct DTLS;
-}
-
namespace __lsan {
-class ThreadContext : public ThreadContextBase {
+class ThreadContextLsanBase : public ThreadContextBase {
public:
- explicit ThreadContext(int tid);
- void OnStarted(void *arg) override;
+ explicit ThreadContextLsanBase(int tid);
void OnFinished() override;
uptr stack_begin() { return stack_begin_; }
uptr stack_end() { return stack_end_; }
- uptr tls_begin() { return tls_begin_; }
- uptr tls_end() { return tls_end_; }
uptr cache_begin() { return cache_begin_; }
uptr cache_end() { return cache_end_; }
- DTLS *dtls() { return dtls_; }
- private:
- uptr stack_begin_, stack_end_,
- cache_begin_, cache_end_,
- tls_begin_, tls_end_;
- DTLS *dtls_;
+ // The argument is passed on to the subclass's OnStarted member function.
+ static void ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type,
+ void *onstarted_arg);
+
+ protected:
+ ~ThreadContextLsanBase() {}
+ uptr stack_begin_ = 0;
+ uptr stack_end_ = 0;
+ uptr cache_begin_ = 0;
+ uptr cache_end_ = 0;
};
+// This subclass of ThreadContextLsanBase is declared in an OS-specific header.
+class ThreadContext;
+
void InitializeThreadRegistry();
+void InitializeMainThread();
-void ThreadStart(u32 tid, tid_t os_id,
- ThreadType thread_type = ThreadType::Regular);
+u32 ThreadCreate(u32 tid, uptr uid, bool detached, void *arg = nullptr);
void ThreadFinish();
-u32 ThreadCreate(u32 tid, uptr uid, bool detached);
+void ThreadDetach(u32 tid);
void ThreadJoin(u32 tid);
u32 ThreadTid(uptr uid);
@@ -55,6 +55,7 @@ u32 GetCurrentThread();
void SetCurrentThread(u32 tid);
ThreadContext *CurrentThreadContext();
void EnsureMainThreadIDIsCorrect();
+
} // namespace __lsan
#endif // LSAN_THREAD_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/.clang-tidy b/system/lib/compiler-rt/lib/sanitizer_common/.clang-tidy
deleted file mode 100644
index 6c71abff0d382..0000000000000
--- a/system/lib/compiler-rt/lib/sanitizer_common/.clang-tidy
+++ /dev/null
@@ -1,16 +0,0 @@
-Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,readability-identifier-naming'
-CheckOptions:
- - key: readability-identifier-naming.ClassCase
- value: CamelCase
- - key: readability-identifier-naming.EnumCase
- value: CamelCase
- - key: readability-identifier-naming.FunctionCase
- value: CamelCase
- - key: readability-identifier-naming.UnionCase
- value: CamelCase
- - key: readability-identifier-naming.GlobalConstantCase
- value: CamelCase
- - key: readability-identifier-naming.GlobalConstantPrefix
- value: "k"
- - key: readability-identifier-naming.VariableCase
- value: lower_case
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
index 906d4af7f5eed..3157b35ffaf80 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
@@ -25,7 +25,7 @@ const char *PrimaryAllocatorName = "SizeClassAllocator";
const char *SecondaryAllocatorName = "LargeMmapAllocator";
// ThreadSanitizer for Go uses libc malloc/free.
-#if SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
+#if defined(SANITIZER_USE_MALLOC)
# if SANITIZER_LINUX && !SANITIZER_ANDROID
extern "C" void *__libc_malloc(uptr size);
# if !SANITIZER_GO
@@ -137,8 +137,14 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
#endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
+namespace {
const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
+struct BlockHeader {
+ u64 magic;
+};
+} // namespace
+
static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
SetAllocatorOutOfMemory();
Report("FATAL: %s: internal allocator is out of memory trying to allocate "
@@ -147,27 +153,28 @@ static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
}
void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
- if (size + sizeof(u64) < size)
+ uptr s = size + sizeof(BlockHeader);
+ if (s < size)
return nullptr;
- void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment);
+ BlockHeader *p = (BlockHeader *)RawInternalAlloc(s, cache, alignment);
if (UNLIKELY(!p))
- ReportInternalAllocatorOutOfMemory(size + sizeof(u64));
- ((u64*)p)[0] = kBlockMagic;
- return (char*)p + sizeof(u64);
+ ReportInternalAllocatorOutOfMemory(s);
+ p->magic = kBlockMagic;
+ return p + 1;
}
void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
if (!addr)
return InternalAlloc(size, cache);
- if (size + sizeof(u64) < size)
+ uptr s = size + sizeof(BlockHeader);
+ if (s < size)
return nullptr;
- addr = (char*)addr - sizeof(u64);
- size = size + sizeof(u64);
- CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
- void *p = RawInternalRealloc(addr, size, cache);
+ BlockHeader *p = (BlockHeader *)addr - 1;
+ CHECK_EQ(kBlockMagic, p->magic);
+ p = (BlockHeader *)RawInternalRealloc(p, s, cache);
if (UNLIKELY(!p))
- ReportInternalAllocatorOutOfMemory(size);
- return (char*)p + sizeof(u64);
+ ReportInternalAllocatorOutOfMemory(s);
+ return p + 1;
}
void *InternalReallocArray(void *addr, uptr count, uptr size,
@@ -198,10 +205,10 @@ void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
void InternalFree(void *addr, InternalAllocatorCache *cache) {
if (!addr)
return;
- addr = (char*)addr - sizeof(u64);
- CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
- ((u64*)addr)[0] = 0;
- RawInternalFree(addr, cache);
+ BlockHeader *p = (BlockHeader *)addr - 1;
+ CHECK_EQ(kBlockMagic, p->magic);
+ p->magic = 0;
+ RawInternalFree(p, cache);
}
// LowLevelAllocator
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
index 23d589888d3b6..5ec47416fe0c9 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
@@ -52,14 +52,14 @@ struct NoOpMapUnmapCallback {
// Callback type for iterating over chunks.
typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
-INLINE u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
+inline u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
return (*state = *state * 1103515245 + 12345) >> 16;
}
-INLINE u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
+inline u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
template
-INLINE void RandomShuffle(T *a, u32 n, u32 *rand_state) {
+inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
if (n <= 1) return;
u32 state = *rand_state;
for (u32 i = n - 1; i > 0; i--)
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h
index fc426f0e74f48..1cc3992c4c9fa 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h
@@ -27,7 +27,7 @@ namespace __sanitizer {
void SetErrnoToENOMEM();
// A common errno setting logic shared by almost all sanitizer allocator APIs.
-INLINE void *SetErrnoOnNull(void *ptr) {
+inline void *SetErrnoOnNull(void *ptr) {
if (UNLIKELY(!ptr))
SetErrnoToENOMEM();
return ptr;
@@ -41,7 +41,7 @@ INLINE void *SetErrnoOnNull(void *ptr) {
// two and that the size is a multiple of alignment for POSIX implementation,
// and a bit relaxed requirement for non-POSIX ones, that the size is a multiple
// of alignment.
-INLINE bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
+inline bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
#if SANITIZER_POSIX
return alignment != 0 && IsPowerOfTwo(alignment) &&
(size & (alignment - 1)) == 0;
@@ -52,13 +52,13 @@ INLINE bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
// Checks posix_memalign() parameters, verifies that alignment is a power of two
// and a multiple of sizeof(void *).
-INLINE bool CheckPosixMemalignAlignment(uptr alignment) {
+inline bool CheckPosixMemalignAlignment(uptr alignment) {
return alignment != 0 && IsPowerOfTwo(alignment) &&
(alignment % sizeof(void *)) == 0;
}
// Returns true if calloc(size, n) call overflows on size*n calculation.
-INLINE bool CheckForCallocOverflow(uptr size, uptr n) {
+inline bool CheckForCallocOverflow(uptr size, uptr n) {
if (!size)
return false;
uptr max = (uptr)-1L;
@@ -67,7 +67,7 @@ INLINE bool CheckForCallocOverflow(uptr size, uptr n) {
// Returns true if the size passed to pvalloc overflows when rounded to the next
// multiple of page_size.
-INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) {
+inline bool CheckForPvallocOverflow(uptr size, uptr page_size) {
return RoundUpTo(size, page_size) < size;
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
index 3b1838b3985ac..b90dabbf77692 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
@@ -153,6 +153,7 @@ class SizeClassAllocator32 {
}
void *GetMetaData(const void *p) {
+ CHECK(kMetadataSize);
CHECK(PointerIsMine(p));
uptr mem = reinterpret_cast(p);
uptr beg = ComputeRegionBeg(mem);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 90603280e7c95..0a18b0c58ef79 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -72,11 +72,15 @@ class SizeClassAllocator64 {
void Init(s32 release_to_os_interval_ms) {
uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
if (kUsingConstantSpaceBeg) {
+ CHECK(IsAligned(kSpaceBeg, SizeClassMap::kMaxSize));
CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize,
PrimaryAllocatorName, kSpaceBeg));
} else {
- NonConstSpaceBeg = address_range.Init(TotalSpaceSize,
- PrimaryAllocatorName);
+ // Combined allocator expects that an 2^N allocation is always aligned to
+ // 2^N. For this to work, the start of the space needs to be aligned as
+ // high as the largest size class (which also needs to be a power of 2).
+ NonConstSpaceBeg = address_range.InitAligned(
+ TotalSpaceSize, SizeClassMap::kMaxSize, PrimaryAllocatorName);
CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
}
SetReleaseToOSIntervalMs(release_to_os_interval_ms);
@@ -182,13 +186,13 @@ class SizeClassAllocator64 {
void *GetBlockBegin(const void *p) {
uptr class_id = GetSizeClass(p);
+ if (class_id >= kNumClasses) return nullptr;
uptr size = ClassIdToSize(class_id);
if (!size) return nullptr;
uptr chunk_idx = GetChunkIdx((uptr)p, size);
uptr reg_beg = GetRegionBegin(p);
uptr beg = chunk_idx * size;
uptr next_beg = beg + size;
- if (class_id >= kNumClasses) return nullptr;
const RegionInfo *region = AddressSpaceView::Load(GetRegionInfo(class_id));
if (region->mapped_user >= next_beg)
return reinterpret_cast(reg_beg + beg);
@@ -203,6 +207,7 @@ class SizeClassAllocator64 {
static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); }
void *GetMetaData(const void *p) {
+ CHECK(kMetadataSize);
uptr class_id = GetSizeClass(p);
uptr size = ClassIdToSize(class_id);
uptr chunk_idx = GetChunkIdx(reinterpret_cast(p), size);
@@ -220,7 +225,7 @@ class SizeClassAllocator64 {
// Test-only.
void TestOnlyUnmap() {
- UnmapWithCallbackOrDie(SpaceBeg(), kSpaceSize + AdditionalSize());
+ UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());
}
static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.cpp
index d74e08010d5de..1c6520819ef93 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.cpp
@@ -134,4 +134,12 @@ void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
Die();
}
+void NORETURN ReportRssLimitExceeded(const StackTrace *stack) {
+ {
+ ScopedAllocatorErrorReport report("rss-limit-exceeded", stack);
+ Report("ERROR: %s: allocator exceeded the RSS limit\n", SanitizerToolName);
+ }
+ Die();
+}
+
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.h
index 0653c365c1cd4..6e4e6b1354916 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_report.h
@@ -33,6 +33,7 @@ void NORETURN ReportInvalidPosixMemalignAlignment(uptr alignment,
void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
const StackTrace *stack);
void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack);
+void NORETURN ReportRssLimitExceeded(const StackTrace *stack);
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h
index 1d128f55de05a..61fb98742373a 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h
@@ -18,8 +18,8 @@
// (currently, 32 bits and internal allocator).
class LargeMmapAllocatorPtrArrayStatic {
public:
- INLINE void *Init() { return &p_[0]; }
- INLINE void EnsureSpace(uptr n) { CHECK_LT(n, kMaxNumChunks); }
+ inline void *Init() { return &p_[0]; }
+ inline void EnsureSpace(uptr n) { CHECK_LT(n, kMaxNumChunks); }
private:
static const int kMaxNumChunks = 1 << 15;
uptr p_[kMaxNumChunks];
@@ -31,14 +31,14 @@ class LargeMmapAllocatorPtrArrayStatic {
// same functionality in Fuchsia case, which does not support MAP_NORESERVE.
class LargeMmapAllocatorPtrArrayDynamic {
public:
- INLINE void *Init() {
+ inline void *Init() {
uptr p = address_range_.Init(kMaxNumChunks * sizeof(uptr),
SecondaryAllocatorName);
CHECK(p);
return reinterpret_cast(p);
}
- INLINE void EnsureSpace(uptr n) {
+ inline void EnsureSpace(uptr n) {
CHECK_LT(n, kMaxNumChunks);
DCHECK(n <= n_reserved_);
if (UNLIKELY(n == n_reserved_)) {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h
index a798a0cf25d9c..46f06957228c9 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h
@@ -72,12 +72,12 @@ namespace __sanitizer {
// Clutter-reducing helpers.
template
-INLINE typename T::Type atomic_load_relaxed(const volatile T *a) {
+inline typename T::Type atomic_load_relaxed(const volatile T *a) {
return atomic_load(a, memory_order_relaxed);
}
template
-INLINE void atomic_store_relaxed(volatile T *a, typename T::Type v) {
+inline void atomic_store_relaxed(volatile T *a, typename T::Type v) {
atomic_store(a, v, memory_order_relaxed);
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h
index c40461ebc3bf6..fc13ca52dda74 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h
@@ -34,16 +34,16 @@ namespace __sanitizer {
// See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
// for mappings of the memory model to different processors.
-INLINE void atomic_signal_fence(memory_order) {
+inline void atomic_signal_fence(memory_order) {
__asm__ __volatile__("" ::: "memory");
}
-INLINE void atomic_thread_fence(memory_order) {
+inline void atomic_thread_fence(memory_order) {
__sync_synchronize();
}
template
-INLINE typename T::Type atomic_fetch_add(volatile T *a,
+inline typename T::Type atomic_fetch_add(volatile T *a,
typename T::Type v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
@@ -51,7 +51,7 @@ INLINE typename T::Type atomic_fetch_add(volatile T *a,
}
template
-INLINE typename T::Type atomic_fetch_sub(volatile T *a,
+inline typename T::Type atomic_fetch_sub(volatile T *a,
typename T::Type v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
@@ -59,7 +59,7 @@ INLINE typename T::Type atomic_fetch_sub(volatile T *a,
}
template
-INLINE typename T::Type atomic_exchange(volatile T *a,
+inline typename T::Type atomic_exchange(volatile T *a,
typename T::Type v, memory_order mo) {
DCHECK(!((uptr)a % sizeof(*a)));
if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
@@ -71,7 +71,7 @@ INLINE typename T::Type atomic_exchange(volatile T *a,
}
template
-INLINE bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
+inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
typedef typename T::Type Type;
@@ -84,7 +84,7 @@ INLINE bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
}
template
-INLINE bool atomic_compare_exchange_weak(volatile T *a,
+inline bool atomic_compare_exchange_weak(volatile T *a,
typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h
index d369aeb9935c6..59155e9883ebe 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h
@@ -37,7 +37,7 @@ static struct {
} __attribute__((aligned(32))) lock = {0, {0}};
template <>
-INLINE atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
+inline atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
atomic_uint64_t::Type val,
memory_order mo) {
DCHECK(mo &
@@ -55,14 +55,14 @@ INLINE atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
}
template <>
-INLINE atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
+inline atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
atomic_uint64_t::Type val,
memory_order mo) {
return atomic_fetch_add(ptr, -val, mo);
}
template <>
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
atomic_uint64_t::Type *cmp,
atomic_uint64_t::Type xchg,
memory_order mo) {
@@ -87,7 +87,7 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
}
template <>
-INLINE atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
+inline atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
memory_order mo) {
DCHECK(mo &
(memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
@@ -100,7 +100,7 @@ INLINE atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
}
template <>
-INLINE void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
+inline void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
memory_order mo) {
DCHECK(mo &
(memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h
index b8685a8542676..4a39889e534a0 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h
@@ -17,12 +17,12 @@
namespace __sanitizer {
-INLINE void proc_yield(int cnt) {
+inline void proc_yield(int cnt) {
__asm__ __volatile__("" ::: "memory");
}
template
-INLINE typename T::Type atomic_load(
+inline typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
@@ -50,17 +50,14 @@ INLINE typename T::Type atomic_load(
__sync_synchronize();
}
} else {
- // 64-bit load on 32-bit platform.
- // Gross, but simple and reliable.
- // Assume that it is not in read-only memory.
- v = __sync_fetch_and_add(
- const_cast(&a->val_dont_use), 0);
+ __atomic_load(const_cast(&a->val_dont_use), &v,
+ __ATOMIC_SEQ_CST);
}
return v;
}
template
-INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
@@ -79,16 +76,7 @@ INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
__sync_synchronize();
}
} else {
- // 64-bit store on 32-bit platform.
- // Gross, but simple and reliable.
- typename T::Type cmp = a->val_dont_use;
- typename T::Type cur;
- for (;;) {
- cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v);
- if (cur == cmp || cur == v)
- break;
- cmp = cur;
- }
+ __atomic_store(&a->val_dont_use, &v, __ATOMIC_SEQ_CST);
}
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h
index f2ce553baa7a1..51597b4927412 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h
@@ -16,7 +16,7 @@
namespace __sanitizer {
-INLINE void proc_yield(int cnt) {
+inline void proc_yield(int cnt) {
__asm__ __volatile__("" ::: "memory");
for (int i = 0; i < cnt; i++)
__asm__ __volatile__("pause");
@@ -24,7 +24,7 @@ INLINE void proc_yield(int cnt) {
}
template
-INLINE typename T::Type atomic_load(
+inline typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
@@ -70,7 +70,7 @@ INLINE typename T::Type atomic_load(
}
template
-INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h
index 6a7c5465dcbbc..31317adcdfc99 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h
@@ -54,21 +54,21 @@ extern "C" long long _InterlockedExchangeAdd64(long long volatile *Addend,
namespace __sanitizer {
-INLINE void atomic_signal_fence(memory_order) {
+inline void atomic_signal_fence(memory_order) {
_ReadWriteBarrier();
}
-INLINE void atomic_thread_fence(memory_order) {
+inline void atomic_thread_fence(memory_order) {
_mm_mfence();
}
-INLINE void proc_yield(int cnt) {
+inline void proc_yield(int cnt) {
for (int i = 0; i < cnt; i++)
_mm_pause();
}
template
-INLINE typename T::Type atomic_load(
+inline typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
@@ -86,7 +86,7 @@ INLINE typename T::Type atomic_load(
}
template
-INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
@@ -102,7 +102,7 @@ INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
atomic_thread_fence(memory_order_seq_cst);
}
-INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
+inline u32 atomic_fetch_add(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
@@ -110,7 +110,7 @@ INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
(long)v);
}
-INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
+inline uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
uptr v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
@@ -123,7 +123,7 @@ INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
#endif
}
-INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
+inline u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
@@ -131,7 +131,7 @@ INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
-(long)v);
}
-INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
+inline uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
uptr v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
@@ -144,28 +144,28 @@ INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
#endif
}
-INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
+inline u8 atomic_exchange(volatile atomic_uint8_t *a,
u8 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
}
-INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
+inline u16 atomic_exchange(volatile atomic_uint16_t *a,
u16 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
}
-INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
+inline u32 atomic_exchange(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
u8 *cmp,
u8 xchgv,
memory_order mo) {
@@ -191,7 +191,7 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
uptr *cmp,
uptr xchg,
memory_order mo) {
@@ -204,7 +204,7 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
u16 *cmp,
u16 xchg,
memory_order mo) {
@@ -217,7 +217,7 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
u32 *cmp,
u32 xchg,
memory_order mo) {
@@ -230,7 +230,7 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
u64 *cmp,
u64 xchg,
memory_order mo) {
@@ -244,7 +244,7 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
}
template
-INLINE bool atomic_compare_exchange_weak(volatile T *a,
+inline bool atomic_compare_exchange_weak(volatile T *a,
typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
index f5f9f49d8cffc..87efda5bd3721 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
@@ -274,6 +274,7 @@ uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len) {
return name_len;
}
+#if !SANITIZER_GO
void PrintCmdline() {
char **argv = GetArgv();
if (!argv) return;
@@ -282,6 +283,7 @@ void PrintCmdline() {
Printf("%s ", argv[i]);
Printf("\n\n");
}
+#endif
// Malloc hooks.
static const int kMaxMallocFreeHooks = 5;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 3b52172c483c7..a6532eee164d8 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -53,25 +53,25 @@ const u64 kExternalPCBit = 1ULL << 60;
extern const char *SanitizerToolName; // Can be changed by the tool.
extern atomic_uint32_t current_verbosity;
-INLINE void SetVerbosity(int verbosity) {
+inline void SetVerbosity(int verbosity) {
atomic_store(¤t_verbosity, verbosity, memory_order_relaxed);
}
-INLINE int Verbosity() {
+inline int Verbosity() {
return atomic_load(¤t_verbosity, memory_order_relaxed);
}
#if SANITIZER_ANDROID
-INLINE uptr GetPageSize() {
+inline uptr GetPageSize() {
// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
return 4096;
}
-INLINE uptr GetPageSizeCached() {
+inline uptr GetPageSizeCached() {
return 4096;
}
#else
uptr GetPageSize();
extern uptr PageSizeCached;
-INLINE uptr GetPageSizeCached() {
+inline uptr GetPageSizeCached() {
if (!PageSizeCached)
PageSizeCached = GetPageSize();
return PageSizeCached;
@@ -91,7 +91,7 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
// Memory management
void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false);
-INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
+inline void *MmapOrDieQuietly(uptr size, const char *mem_type) {
return MmapOrDie(size, mem_type, /*raw_report*/ true);
}
void UnmapOrDie(void *addr, uptr size);
@@ -121,6 +121,31 @@ bool MprotectReadOnly(uptr addr, uptr size);
void MprotectMallocZones(void *addr, int prot);
+#if SANITIZER_LINUX
+// Unmap memory. Currently only used on Linux.
+void UnmapFromTo(uptr from, uptr to);
+#endif
+
+// Maps shadow_size_bytes of shadow memory and returns shadow address. It will
+// be aligned to the mmap granularity * 2^shadow_scale, or to
+// 2^min_shadow_base_alignment if that is larger. The returned address will
+// have max(2^min_shadow_base_alignment, mmap granularity) on the left, and
+// shadow_size_bytes bytes on the right, which on linux is mapped no access.
+// The high_mem_end may be updated if the original shadow size doesn't fit.
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment, uptr &high_mem_end);
+
+// Reserve memory range [beg, end]. If madvise_shadow is true then apply
+// madvise (e.g. hugepages, core dumping) requested by options.
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
+ bool madvise_shadow = true);
+
+// Protect size bytes of memory starting at addr. Also try to protect
+// several pages at the start of the address space as specified by
+// zero_base_shadow_start, at most up to the size or zero_base_max_shadow_start.
+void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
+ uptr zero_base_max_shadow_start);
+
// Find an available address space.
uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
uptr *largest_gap_found, uptr *max_occupied_addr);
@@ -143,6 +168,7 @@ void RunFreeHooks(const void *ptr);
class ReservedAddressRange {
public:
uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0);
+ uptr InitAligned(uptr size, uptr align, const char *name = nullptr);
uptr Map(uptr fixed_addr, uptr size, const char *name = nullptr);
uptr MapOrDie(uptr fixed_addr, uptr size, const char *name = nullptr);
void Unmap(uptr addr, uptr size);
@@ -228,7 +254,6 @@ void UpdateProcessName();
void CacheBinaryName();
void DisableCoreDumperIfNecessary();
void DumpProcessMap();
-void PrintModuleMap();
const char *GetEnv(const char *name);
bool SetEnv(const char *name, const char *value);
@@ -348,7 +373,7 @@ unsigned char _BitScanReverse64(unsigned long *index, unsigned __int64 mask);
}
#endif
-INLINE uptr MostSignificantSetBitIndex(uptr x) {
+inline uptr MostSignificantSetBitIndex(uptr x) {
CHECK_NE(x, 0U);
unsigned long up;
#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__)
@@ -365,7 +390,7 @@ INLINE uptr MostSignificantSetBitIndex(uptr x) {
return up;
}
-INLINE uptr LeastSignificantSetBitIndex(uptr x) {
+inline uptr LeastSignificantSetBitIndex(uptr x) {
CHECK_NE(x, 0U);
unsigned long up;
#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__)
@@ -382,11 +407,11 @@ INLINE uptr LeastSignificantSetBitIndex(uptr x) {
return up;
}
-INLINE bool IsPowerOfTwo(uptr x) {
+inline bool IsPowerOfTwo(uptr x) {
return (x & (x - 1)) == 0;
}
-INLINE uptr RoundUpToPowerOfTwo(uptr size) {
+inline uptr RoundUpToPowerOfTwo(uptr size) {
CHECK(size);
if (IsPowerOfTwo(size)) return size;
@@ -396,20 +421,20 @@ INLINE uptr RoundUpToPowerOfTwo(uptr size) {
return 1ULL << (up + 1);
}
-INLINE uptr RoundUpTo(uptr size, uptr boundary) {
+inline uptr RoundUpTo(uptr size, uptr boundary) {
RAW_CHECK(IsPowerOfTwo(boundary));
return (size + boundary - 1) & ~(boundary - 1);
}
-INLINE uptr RoundDownTo(uptr x, uptr boundary) {
+inline uptr RoundDownTo(uptr x, uptr boundary) {
return x & ~(boundary - 1);
}
-INLINE bool IsAligned(uptr a, uptr alignment) {
+inline bool IsAligned(uptr a, uptr alignment) {
return (a & (alignment - 1)) == 0;
}
-INLINE uptr Log2(uptr x) {
+inline uptr Log2(uptr x) {
CHECK(IsPowerOfTwo(x));
return LeastSignificantSetBitIndex(x);
}
@@ -425,14 +450,14 @@ template void Swap(T& a, T& b) {
}
// Char handling
-INLINE bool IsSpace(int c) {
+inline bool IsSpace(int c) {
return (c == ' ') || (c == '\n') || (c == '\t') ||
(c == '\f') || (c == '\r') || (c == '\v');
}
-INLINE bool IsDigit(int c) {
+inline bool IsDigit(int c) {
return (c >= '0') && (c <= '9');
}
-INLINE int ToLower(int c) {
+inline int ToLower(int c) {
return (c >= 'A' && c <= 'Z') ? (c + 'a' - 'A') : c;
}
@@ -442,6 +467,7 @@ INLINE int ToLower(int c) {
template
class InternalMmapVectorNoCtor {
public:
+ using value_type = T;
void Initialize(uptr initial_capacity) {
capacity_bytes_ = 0;
size_ = 0;
@@ -626,9 +652,13 @@ void Sort(T *v, uptr size, Compare comp = {}) {
// Works like std::lower_bound: finds the first element that is not less
// than the val.
-template
-uptr InternalLowerBound(const Container &v, uptr first, uptr last,
- const Value &val, Compare comp) {
+template >
+uptr InternalLowerBound(const Container &v,
+ const typename Container::value_type &val,
+ Compare comp = {}) {
+ uptr first = 0;
+ uptr last = v.size();
while (last > first) {
uptr mid = (first + last) / 2;
if (comp(v[mid], val))
@@ -648,9 +678,31 @@ enum ModuleArch {
kModuleArchARMV7,
kModuleArchARMV7S,
kModuleArchARMV7K,
- kModuleArchARM64
+ kModuleArchARM64,
+ kModuleArchRISCV64
};
+// Sorts and removes duplicates from the container.
+template >
+void SortAndDedup(Container &v, Compare comp = {}) {
+ Sort(v.data(), v.size(), comp);
+ uptr size = v.size();
+ if (size < 2)
+ return;
+ uptr last = 0;
+ for (uptr i = 1; i < size; ++i) {
+ if (comp(v[last], v[i])) {
+ ++last;
+ if (last != i)
+ v[last] = v[i];
+ } else {
+ CHECK(!comp(v[i], v[last]));
+ }
+ }
+ v.resize(last + 1);
+}
+
// Opens the file 'file_name" and reads up to 'max_len' bytes.
// The resulting buffer is mmaped and stored in '*buff'.
// Returns true if file was successfully opened and read.
@@ -692,6 +744,8 @@ inline const char *ModuleArchToString(ModuleArch arch) {
return "armv7k";
case kModuleArchARM64:
return "arm64";
+ case kModuleArchRISCV64:
+ return "riscv64";
}
CHECK(0 && "Invalid module arch");
return "";
@@ -814,15 +868,15 @@ void WriteToSyslog(const char *buffer);
#if SANITIZER_MAC || SANITIZER_WIN_TRACE
void LogFullErrorReport(const char *buffer);
#else
-INLINE void LogFullErrorReport(const char *buffer) {}
+inline void LogFullErrorReport(const char *buffer) {}
#endif
#if SANITIZER_LINUX || SANITIZER_MAC
void WriteOneLineToSyslog(const char *s);
void LogMessageOnPrintf(const char *str);
#else
-INLINE void WriteOneLineToSyslog(const char *s) {}
-INLINE void LogMessageOnPrintf(const char *str) {}
+inline void WriteOneLineToSyslog(const char *s) {}
+inline void LogMessageOnPrintf(const char *str) {}
#endif
#if SANITIZER_LINUX || SANITIZER_WIN_TRACE
@@ -830,21 +884,21 @@ INLINE void LogMessageOnPrintf(const char *str) {}
void AndroidLogInit();
void SetAbortMessage(const char *);
#else
-INLINE void AndroidLogInit() {}
+inline void AndroidLogInit() {}
// FIXME: MacOS implementation could use CRSetCrashLogMessage.
-INLINE void SetAbortMessage(const char *) {}
+inline void SetAbortMessage(const char *) {}
#endif
#if SANITIZER_ANDROID
void SanitizerInitializeUnwinder();
AndroidApiLevel AndroidGetApiLevel();
#else
-INLINE void AndroidLogWrite(const char *buffer_unused) {}
-INLINE void SanitizerInitializeUnwinder() {}
-INLINE AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; }
+inline void AndroidLogWrite(const char *buffer_unused) {}
+inline void SanitizerInitializeUnwinder() {}
+inline AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; }
#endif
-INLINE uptr GetPthreadDestructorIterations() {
+inline uptr GetPthreadDestructorIterations() {
#if SANITIZER_ANDROID
return (AndroidGetApiLevel() == ANDROID_LOLLIPOP_MR1) ? 8 : 4;
#elif SANITIZER_POSIX
@@ -855,7 +909,7 @@ INLINE uptr GetPthreadDestructorIterations() {
#endif
}
-void *internal_start_thread(void(*func)(void*), void *arg);
+void *internal_start_thread(void *(*func)(void*), void *arg);
void internal_join_thread(void *th);
void MaybeStartBackgroudThread();
@@ -950,7 +1004,7 @@ RunOnDestruction at_scope_exit(Fn fn) {
#if SANITIZER_LINUX && SANITIZER_S390_64
void AvoidCVE_2016_2143();
#else
-INLINE void AvoidCVE_2016_2143() {}
+inline void AvoidCVE_2016_2143() {}
#endif
struct StackDepotStats {
@@ -971,12 +1025,26 @@ bool GetRandom(void *buffer, uptr length, bool blocking = true);
// Returns the number of logical processors on the system.
u32 GetNumberOfCPUs();
extern u32 NumberOfCPUsCached;
-INLINE u32 GetNumberOfCPUsCached() {
+inline u32 GetNumberOfCPUsCached() {
if (!NumberOfCPUsCached)
NumberOfCPUsCached = GetNumberOfCPUs();
return NumberOfCPUsCached;
}
+template
+class ArrayRef {
+ public:
+ ArrayRef() {}
+ ArrayRef(T *begin, T *end) : begin_(begin), end_(end) {}
+
+ T *begin() { return begin_; }
+ T *end() { return end_; }
+
+ private:
+ T *begin_ = nullptr;
+ T *end_ = nullptr;
+};
+
} // namespace __sanitizer
inline void *operator new(__sanitizer::operator_new_size_type size,
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 2a4ab7e67a5ce..d4b9ea5f7f067 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -113,6 +113,7 @@
#define setitimer __setitimer50
#define setlocale __setlocale50
#define shmctl __shmctl50
+#define sigaltstack __sigaltstack14
#define sigemptyset __sigemptyset14
#define sigfillset __sigfillset14
#define sigpending __sigpending14
@@ -133,11 +134,7 @@ extern const short *_tolower_tab_;
// Platform-specific options.
#if SANITIZER_MAC
-namespace __sanitizer {
-bool PlatformHasDifferentMemcpyAndMemmove();
-}
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE \
- (__sanitizer::PlatformHasDifferentMemcpyAndMemmove())
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
#elif SANITIZER_WINDOWS64
#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
#else
@@ -448,8 +445,10 @@ INTERCEPTOR(int, strcmp, const char *s1, const char *s2) {
c2 = (unsigned char)s2[i];
if (c1 != c2 || c1 == '\0') break;
}
- COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1);
- COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1);
+ if (common_flags()->intercept_strcmp) {
+ COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1);
+ COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1);
+ }
int result = CharCmpX(c1, c2);
CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, GET_CALLER_PC(), s1,
s2, result);
@@ -1865,7 +1864,7 @@ UNUSED static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,
REAL(strlen)(pwd->pw_gecos) + 1);
#endif
-#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD
if (pwd->pw_class)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,
REAL(strlen)(pwd->pw_class) + 1);
@@ -2202,6 +2201,24 @@ INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
#define INIT_CLOCK_GETTIME
#endif
+#if SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID
+INTERCEPTOR(int, clock_getcpuclockid, pid_t pid,
+ __sanitizer_clockid_t *clockid) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, clock_getcpuclockid, pid, clockid);
+ int res = REAL(clock_getcpuclockid)(pid, clockid);
+ if (!res && clockid) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, clockid, sizeof *clockid);
+ }
+ return res;
+}
+
+#define INIT_CLOCK_GETCPUCLOCKID \
+ COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid);
+#else
+#define INIT_CLOCK_GETCPUCLOCKID
+#endif
+
#if SANITIZER_INTERCEPT_GETITIMER
INTERCEPTOR(int, getitimer, int which, void *curr_value) {
void *ctx;
@@ -3095,6 +3112,34 @@ INTERCEPTOR(int, sendmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
#define INIT_SENDMMSG
#endif
+#if SANITIZER_INTERCEPT_SYSMSG
+INTERCEPTOR(int, msgsnd, int msqid, const void *msgp, SIZE_T msgsz,
+ int msgflg) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, msgsnd, msqid, msgp, msgsz, msgflg);
+ if (msgp)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, msgp, sizeof(long) + msgsz);
+ int res = REAL(msgsnd)(msqid, msgp, msgsz, msgflg);
+ return res;
+}
+
+INTERCEPTOR(SSIZE_T, msgrcv, int msqid, void *msgp, SIZE_T msgsz,
+ long msgtyp, int msgflg) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, msgrcv, msqid, msgp, msgsz, msgtyp, msgflg);
+ SSIZE_T len = REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);
+ if (len != -1)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msgp, sizeof(long) + len);
+ return len;
+}
+
+#define INIT_SYSMSG \
+ COMMON_INTERCEPT_FUNCTION(msgsnd); \
+ COMMON_INTERCEPT_FUNCTION(msgrcv);
+#else
+#define INIT_SYSMSG
+#endif
+
#if SANITIZER_INTERCEPT_GETPEERNAME
INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) {
void *ctx;
@@ -3705,7 +3750,7 @@ INTERCEPTOR(char *, strerror, int errnum) {
// static storage.
#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \
SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD || \
- SANITIZER_FREEBSD || SANITIZER_OPENBSD
+ SANITIZER_FREEBSD
// POSIX version. Spec is not clear on whether buf is NULL-terminated.
// At least on OSX, buf contents are valid even when the call fails.
INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) {
@@ -4042,6 +4087,41 @@ INTERCEPTOR(int, sigfillset, __sanitizer_sigset_t *set) {
#define INIT_SIGSETOPS
#endif
+#if SANITIZER_INTERCEPT_SIGSET_LOGICOPS
+INTERCEPTOR(int, sigandset, __sanitizer_sigset_t *dst,
+ __sanitizer_sigset_t *src1, __sanitizer_sigset_t *src2) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, sigandset, dst, src1, src2);
+ if (src1)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src1, sizeof(*src1));
+ if (src2)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src2, sizeof(*src2));
+ int res = REAL(sigandset)(dst, src1, src2);
+ if (!res && dst)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst));
+ return res;
+}
+
+INTERCEPTOR(int, sigorset, __sanitizer_sigset_t *dst,
+ __sanitizer_sigset_t *src1, __sanitizer_sigset_t *src2) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, sigorset, dst, src1, src2);
+ if (src1)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src1, sizeof(*src1));
+ if (src2)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src2, sizeof(*src2));
+ int res = REAL(sigorset)(dst, src1, src2);
+ if (!res && dst)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst));
+ return res;
+}
+#define INIT_SIGSET_LOGICOPS \
+ COMMON_INTERCEPT_FUNCTION(sigandset); \
+ COMMON_INTERCEPT_FUNCTION(sigorset);
+#else
+#define INIT_SIGSET_LOGICOPS
+#endif
+
#if SANITIZER_INTERCEPT_SIGPENDING
INTERCEPTOR(int, sigpending, __sanitizer_sigset_t *set) {
void *ctx;
@@ -4795,6 +4875,34 @@ INTERCEPTOR(char *, tmpnam_r, char *s) {
#define INIT_TMPNAM_R
#endif
+#if SANITIZER_INTERCEPT_PTSNAME
+INTERCEPTOR(char *, ptsname, int fd) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, ptsname, fd);
+ char *res = REAL(ptsname)(fd);
+ if (res != nullptr)
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ return res;
+}
+#define INIT_PTSNAME COMMON_INTERCEPT_FUNCTION(ptsname);
+#else
+#define INIT_PTSNAME
+#endif
+
+#if SANITIZER_INTERCEPT_PTSNAME_R
+INTERCEPTOR(int, ptsname_r, int fd, char *name, SIZE_T namesize) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, ptsname_r, fd, name, namesize);
+ int res = REAL(ptsname_r)(fd, name, namesize);
+ if (res == 0)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ return res;
+}
+#define INIT_PTSNAME_R COMMON_INTERCEPT_FUNCTION(ptsname_r);
+#else
+#define INIT_PTSNAME_R
+#endif
+
#if SANITIZER_INTERCEPT_TTYNAME
INTERCEPTOR(char *, ttyname, int fd) {
void *ctx;
@@ -5766,6 +5874,79 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR *xdrs, char **p,
#define INIT_XDR
#endif // SANITIZER_INTERCEPT_XDR
+#if SANITIZER_INTERCEPT_XDRREC
+typedef int (*xdrrec_cb)(char*, char*, int);
+struct XdrRecWrapper {
+ char *handle;
+ xdrrec_cb rd, wr;
+};
+typedef AddrHashMap XdrRecWrapMap;
+static XdrRecWrapMap *xdrrec_wrap_map;
+
+static int xdrrec_wr_wrap(char *handle, char *buf, int count) {
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(buf, count);
+ XdrRecWrapper *wrap = (XdrRecWrapper *)handle;
+ return wrap->wr(wrap->handle, buf, count);
+}
+
+static int xdrrec_rd_wrap(char *handle, char *buf, int count) {
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+ XdrRecWrapper *wrap = (XdrRecWrapper *)handle;
+ return wrap->rd(wrap->handle, buf, count);
+}
+
+// This doesn't apply to the solaris version as it has a different function
+// signature.
+INTERCEPTOR(void, xdrrec_create, __sanitizer_XDR *xdr, unsigned sndsize,
+ unsigned rcvsize, char *handle, int (*rd)(char*, char*, int),
+ int (*wr)(char*, char*, int)) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, xdrrec_create, xdr, sndsize, rcvsize,
+ handle, rd, wr);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, &xdr->x_op, sizeof xdr->x_op);
+
+ // We can't allocate a wrapper on the stack, as the handle is used outside
+ // this stack frame. So we put it on the heap, and keep track of it with
+ // the HashMap (keyed by x_private). When we later need to xdr_destroy,
+ // we can index the map, free the wrapper, and then clean the map entry.
+ XdrRecWrapper *wrap_data =
+ (XdrRecWrapper *)InternalAlloc(sizeof(XdrRecWrapper));
+ wrap_data->handle = handle;
+ wrap_data->rd = rd;
+ wrap_data->wr = wr;
+ if (wr)
+ wr = xdrrec_wr_wrap;
+ if (rd)
+ rd = xdrrec_rd_wrap;
+ handle = (char *)wrap_data;
+
+ REAL(xdrrec_create)(xdr, sndsize, rcvsize, handle, rd, wr);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdr, sizeof *xdr);
+
+ XdrRecWrapMap::Handle wrap(xdrrec_wrap_map, xdr->x_private, false, true);
+ *wrap = wrap_data;
+}
+
+// We have to intercept this to be able to free wrapper memory;
+// otherwise it's not necessary.
+INTERCEPTOR(void, xdr_destroy, __sanitizer_XDR *xdr) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, xdr_destroy, xdr);
+
+ XdrRecWrapMap::Handle wrap(xdrrec_wrap_map, xdr->x_private, true);
+ InternalFree(*wrap);
+ REAL(xdr_destroy)(xdr);
+}
+#define INIT_XDRREC_LINUX \
+ static u64 xdrrec_wrap_mem[sizeof(XdrRecWrapMap) / sizeof(u64) + 1]; \
+ xdrrec_wrap_map = new ((void *)&xdrrec_wrap_mem) XdrRecWrapMap(); \
+ COMMON_INTERCEPT_FUNCTION(xdrrec_create); \
+ COMMON_INTERCEPT_FUNCTION(xdr_destroy);
+#else
+#define INIT_XDRREC_LINUX
+#endif
+
#if SANITIZER_INTERCEPT_TSEARCH
INTERCEPTOR(void *, tsearch, void *key, void **rootp,
int (*compar)(const void *, const void *)) {
@@ -5797,6 +5978,9 @@ void unpoison_file(__sanitizer_FILE *fp) {
if (fp->_IO_read_base && fp->_IO_read_base < fp->_IO_read_end)
COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_read_base,
fp->_IO_read_end - fp->_IO_read_base);
+ if (fp->_IO_write_base && fp->_IO_write_base < fp->_IO_write_end)
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_write_base,
+ fp->_IO_write_end - fp->_IO_write_base);
#endif
#endif // SANITIZER_HAS_STRUCT_FILE
}
@@ -6023,6 +6207,8 @@ INTERCEPTOR(void, _obstack_newchunk, __sanitizer_obstack *obstack, int length) {
INTERCEPTOR(int, fflush, __sanitizer_FILE *fp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fflush, fp);
+ if (fp)
+ unpoison_file(fp);
int res = REAL(fflush)(fp);
// FIXME: handle fp == NULL
if (fp) {
@@ -6042,6 +6228,8 @@ INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) {
COMMON_INTERCEPTOR_ENTER(ctx, fclose, fp);
COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
const FileMetadata *m = GetInterceptorMetadata(fp);
+ if (fp)
+ unpoison_file(fp);
int res = REAL(fclose)(fp);
if (m) {
COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size);
@@ -6432,12 +6620,11 @@ INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags,
if (srcaddr) srcaddr_sz = *addrlen;
(void)srcaddr_sz; // prevent "set but not used" warning
SSIZE_T res = REAL(recvfrom)(fd, buf, len, flags, srcaddr, addrlen);
- if (res > 0) {
+ if (res > 0)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
- if (srcaddr)
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(srcaddr,
- Min((SIZE_T)*addrlen, srcaddr_sz));
- }
+ if (res >= 0 && srcaddr)
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(srcaddr,
+ Min((SIZE_T)*addrlen, srcaddr_sz));
return res;
}
#define INIT_RECV_RECVFROM \
@@ -7275,23 +7462,26 @@ INTERCEPTOR(int, setttyentpath, char *path) {
#endif
#if SANITIZER_INTERCEPT_PROTOENT
-INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
- struct __sanitizer_protoent *p = REAL(getprotoent)();
- if (p) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
- SIZE_T pp_size = 1; // One handles the trailing \0
+ SIZE_T pp_size = 1; // One handles the trailing \0
- for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
+ for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
- pp_size * sizeof(char **));
- }
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
+ pp_size * sizeof(char **));
+}
+
+INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
+ struct __sanitizer_protoent *p = REAL(getprotoent)();
+ if (p)
+ write_protoent(ctx, p);
return p;
}
@@ -7301,19 +7491,8 @@ INTERCEPTOR(struct __sanitizer_protoent *, getprotobyname, const char *name) {
if (name)
COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
struct __sanitizer_protoent *p = REAL(getprotobyname)(name);
- if (p) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
-
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
-
- SIZE_T pp_size = 1; // One handles the trailing \0
-
- for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
-
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
- pp_size * sizeof(char **));
- }
+ if (p)
+ write_protoent(ctx, p);
return p;
}
@@ -7321,19 +7500,8 @@ INTERCEPTOR(struct __sanitizer_protoent *, getprotobynumber, int proto) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getprotobynumber, proto);
struct __sanitizer_protoent *p = REAL(getprotobynumber)(proto);
- if (p) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
-
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
-
- SIZE_T pp_size = 1; // One handles the trailing \0
-
- for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
-
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
- pp_size * sizeof(char **));
- }
+ if (p)
+ write_protoent(ctx, p);
return p;
}
#define INIT_PROTOENT \
@@ -7344,6 +7512,58 @@ INTERCEPTOR(struct __sanitizer_protoent *, getprotobynumber, int proto) {
#define INIT_PROTOENT
#endif
+#if SANITIZER_INTERCEPT_PROTOENT_R
+INTERCEPTOR(int, getprotoent_r, struct __sanitizer_protoent *result_buf,
+ char *buf, SIZE_T buflen, struct __sanitizer_protoent **result) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, getprotoent_r, result_buf, buf, buflen,
+ result);
+ int res = REAL(getprotoent_r)(result_buf, buf, buflen, result);
+
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+ if (!res && *result)
+ write_protoent(ctx, *result);
+ return res;
+}
+
+INTERCEPTOR(int, getprotobyname_r, const char *name,
+ struct __sanitizer_protoent *result_buf, char *buf, SIZE_T buflen,
+ struct __sanitizer_protoent **result) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,
+ buflen, result);
+ if (name)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);
+
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+ if (!res && *result)
+ write_protoent(ctx, *result);
+ return res;
+}
+
+INTERCEPTOR(int, getprotobynumber_r, int num,
+ struct __sanitizer_protoent *result_buf, char *buf,
+ SIZE_T buflen, struct __sanitizer_protoent **result) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, getprotobynumber_r, num, result_buf, buf,
+ buflen, result);
+ int res = REAL(getprotobynumber_r)(num, result_buf, buf, buflen, result);
+
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+ if (!res && *result)
+ write_protoent(ctx, *result);
+ return res;
+}
+
+#define INIT_PROTOENT_R \
+ COMMON_INTERCEPT_FUNCTION(getprotoent_r); \
+ COMMON_INTERCEPT_FUNCTION(getprotobyname_r); \
+ COMMON_INTERCEPT_FUNCTION(getprotobynumber_r);
+#else
+#define INIT_PROTOENT_R
+#endif
+
#if SANITIZER_INTERCEPT_NETENT
INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
void *ctx;
@@ -9680,12 +9900,25 @@ INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
}
}
qsort_compar_f old_compar = qsort_compar;
- qsort_compar = compar;
SIZE_T old_size = qsort_size;
- qsort_size = size;
+ // Handle qsort() implementations that recurse using an
+ // interposable function call:
+ bool already_wrapped = compar == wrapped_qsort_compar;
+ if (already_wrapped) {
+ // This case should only happen if the qsort() implementation calls itself
+ // using a preemptible function call (e.g. the FreeBSD libc version).
+ // Check that the size and comparator arguments are as expected.
+ CHECK_NE(compar, qsort_compar);
+ CHECK_EQ(qsort_size, size);
+ } else {
+ qsort_compar = compar;
+ qsort_size = size;
+ }
REAL(qsort)(base, nmemb, size, wrapped_qsort_compar);
- qsort_compar = old_compar;
- qsort_size = old_size;
+ if (!already_wrapped) {
+ qsort_compar = old_compar;
+ qsort_size = old_size;
+ }
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
}
#define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
@@ -9718,12 +9951,25 @@ INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
}
}
qsort_r_compar_f old_compar = qsort_r_compar;
- qsort_r_compar = compar;
SIZE_T old_size = qsort_r_size;
- qsort_r_size = size;
+ // Handle qsort_r() implementations that recurse using an
+ // interposable function call:
+ bool already_wrapped = compar == wrapped_qsort_r_compar;
+ if (already_wrapped) {
+ // This case should only happen if the qsort() implementation calls itself
+ // using a preemptible function call (e.g. the FreeBSD libc version).
+ // Check that the size and comparator arguments are as expected.
+ CHECK_NE(compar, qsort_r_compar);
+ CHECK_EQ(qsort_r_size, size);
+ } else {
+ qsort_r_compar = compar;
+ qsort_r_size = size;
+ }
REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, arg);
- qsort_r_compar = old_compar;
- qsort_r_size = old_size;
+ if (!already_wrapped) {
+ qsort_r_compar = old_compar;
+ qsort_r_size = old_size;
+ }
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
}
#define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
@@ -9731,6 +9977,59 @@ INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
#define INIT_QSORT_R
#endif
+#if SANITIZER_INTERCEPT_SIGALTSTACK
+INTERCEPTOR(int, sigaltstack, void *ss, void *oss) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, sigaltstack, ss, oss);
+ int r = REAL(sigaltstack)(ss, oss);
+ if (r == 0 && oss != nullptr) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oss, struct_stack_t_sz);
+ }
+ return r;
+}
+#define INIT_SIGALTSTACK COMMON_INTERCEPT_FUNCTION(sigaltstack)
+#else
+#define INIT_SIGALTSTACK
+#endif
+
+#if SANITIZER_INTERCEPT_UNAME
+INTERCEPTOR(int, uname, struct utsname *utsname) {
+#if SANITIZER_LINUX
+ if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+ return internal_uname(utsname);
+#endif
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, uname, utsname);
+ int res = REAL(uname)(utsname);
+ if (!res)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, utsname,
+ __sanitizer::struct_utsname_sz);
+ return res;
+}
+#define INIT_UNAME COMMON_INTERCEPT_FUNCTION(uname)
+#else
+#define INIT_UNAME
+#endif
+
+#if SANITIZER_INTERCEPT___XUNAME
+// FreeBSD's define uname() as
+// static __inline int uname(struct utsname *name) {
+// return __xuname(SYS_NMLN, (void*)name);
+// }
+INTERCEPTOR(int, __xuname, int size, void *utsname) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, __xuname, size, utsname);
+ int res = REAL(__xuname)(size, utsname);
+ if (!res)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, utsname,
+ __sanitizer::struct_utsname_sz);
+ return res;
+}
+#define INIT___XUNAME COMMON_INTERCEPT_FUNCTION(__xuname)
+#else
+#define INIT___XUNAME
+#endif
+
#include "sanitizer_common_interceptors_netbsd_compat.inc"
static void InitializeCommonInterceptors() {
@@ -9804,6 +10103,7 @@ static void InitializeCommonInterceptors() {
INIT_FGETGRENT_R;
INIT_SETPWENT;
INIT_CLOCK_GETTIME;
+ INIT_CLOCK_GETCPUCLOCKID;
INIT_GETITIMER;
INIT_TIME;
INIT_GLOB;
@@ -9830,6 +10130,7 @@ static void InitializeCommonInterceptors() {
INIT_SENDMSG;
INIT_RECVMMSG;
INIT_SENDMMSG;
+ INIT_SYSMSG;
INIT_GETPEERNAME;
INIT_IOCTL;
INIT_INET_ATON;
@@ -9866,6 +10167,7 @@ static void InitializeCommonInterceptors() {
INIT_SIGWAITINFO;
INIT_SIGTIMEDWAIT;
INIT_SIGSETOPS;
+ INIT_SIGSET_LOGICOPS;
INIT_SIGPENDING;
INIT_SIGPROCMASK;
INIT_PTHREAD_SIGMASK;
@@ -9907,6 +10209,8 @@ static void InitializeCommonInterceptors() {
INIT_PTHREAD_BARRIERATTR_GETPSHARED;
INIT_TMPNAM;
INIT_TMPNAM_R;
+ INIT_PTSNAME;
+ INIT_PTSNAME_R;
INIT_TTYNAME;
INIT_TTYNAME_R;
INIT_TEMPNAM;
@@ -9936,6 +10240,7 @@ static void InitializeCommonInterceptors() {
INIT_BZERO;
INIT_FTIME;
INIT_XDR;
+ INIT_XDRREC_LINUX;
INIT_TSEARCH;
INIT_LIBIO_INTERNALS;
INIT_FOPEN;
@@ -9993,6 +10298,7 @@ static void InitializeCommonInterceptors() {
INIT_STRMODE;
INIT_TTYENT;
INIT_PROTOENT;
+ INIT_PROTOENT_R;
INIT_NETENT;
INIT_GETMNTINFO;
INIT_MI_VECTOR_HASH;
@@ -10036,6 +10342,9 @@ static void InitializeCommonInterceptors() {
INIT_GETENTROPY;
INIT_QSORT;
INIT_QSORT_R;
+ INIT_SIGALTSTACK;
+ INIT_UNAME;
+ INIT___XUNAME;
INIT___PRINTF_CHK;
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc
index bbbedda8fbe21..082398ba960af 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc
@@ -340,6 +340,12 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
size = 0;
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
+ // For %ms/%mc, write the allocated output buffer as well.
+ if (dir.allocate) {
+ char *buf = *(char **)argp;
+ if (buf)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
+ }
}
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
index 490a04b2181b0..7f181258eab52 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
@@ -330,13 +330,17 @@ static void ioctl_table_fill() {
_(SOUND_PCM_WRITE_CHANNELS, WRITE, sizeof(int));
_(SOUND_PCM_WRITE_FILTER, WRITE, sizeof(int));
_(TCFLSH, NONE, 0);
+#if SANITIZER_GLIBC
_(TCGETA, WRITE, struct_termio_sz);
+#endif
_(TCGETS, WRITE, struct_termios_sz);
_(TCSBRK, NONE, 0);
_(TCSBRKP, NONE, 0);
+#if SANITIZER_GLIBC
_(TCSETA, READ, struct_termio_sz);
_(TCSETAF, READ, struct_termio_sz);
_(TCSETAW, READ, struct_termio_sz);
+#endif
_(TCSETS, READ, struct_termios_sz);
_(TCSETSF, READ, struct_termios_sz);
_(TCSETSW, READ, struct_termios_sz);
@@ -364,7 +368,7 @@ static void ioctl_table_fill() {
_(VT_WAITACTIVE, NONE, 0);
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
// _(SIOCDEVPLIP, WRITE, struct_ifreq_sz); // the same as EQL_ENSLAVE
_(CYGETDEFTHRESH, WRITE, sizeof(int));
_(CYGETDEFTIMEOUT, WRITE, sizeof(int));
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
index c78b6e10b6895..932e5478616d5 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
@@ -13,6 +13,7 @@ INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
INTERFACE_FUNCTION(__sanitizer_set_death_callback)
INTERFACE_FUNCTION(__sanitizer_set_report_path)
INTERFACE_FUNCTION(__sanitizer_set_report_fd)
+INTERFACE_FUNCTION(__sanitizer_get_report_path)
INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
index 27d6a177760e4..047c5a17ea6e7 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -30,7 +30,7 @@ SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
return nullptr;
}
-void BackgroundThread(void *arg) {
+void *BackgroundThread(void *arg) {
const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
const bool heap_profile = common_flags()->heap_profile;
@@ -129,6 +129,69 @@ void SetSandboxingCallback(void (*f)()) {
sandboxing_callback = f;
}
+uptr ReservedAddressRange::InitAligned(uptr size, uptr align,
+ const char *name) {
+ CHECK(IsPowerOfTwo(align));
+ if (align <= GetPageSizeCached())
+ return Init(size, name);
+ uptr start = Init(size + align, name);
+ start += align - (start & (align - 1));
+ return start;
+}
+
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+
+// Reserve memory range [beg, end].
+// We need to use inclusive range because end+1 may not be representable.
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
+ bool madvise_shadow) {
+ CHECK_EQ((beg % GetMmapGranularity()), 0);
+ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+ uptr size = end - beg + 1;
+ DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
+ if (madvise_shadow ? !MmapFixedSuperNoReserve(beg, size, name)
+ : !MmapFixedNoReserve(beg, size, name)) {
+ Report(
+ "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+ "Perhaps you're using ulimit -v\n",
+ size);
+ Abort();
+ }
+ if (madvise_shadow && common_flags()->use_madv_dontdump)
+ DontDumpShadowMemory(beg, size);
+}
+
+void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
+ uptr zero_base_max_shadow_start) {
+ if (!size)
+ return;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res)
+ return;
+ // A few pages at the start of the address space can not be protected.
+ // But we really want to protect as much as possible, to prevent this memory
+ // being returned as a result of a non-FIXED mmap().
+ if (addr == zero_base_shadow_start) {
+ uptr step = GetMmapGranularity();
+ while (size > step && addr < zero_base_max_shadow_start) {
+ addr += step;
+ size -= step;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res)
+ return;
+ }
+ }
+
+ Report(
+ "ERROR: Failed to protect the shadow gap. "
+ "%s cannot proceed correctly. ABORTING.\n",
+ SanitizerToolName);
+ DumpProcessMap();
+ Die();
+}
+
+#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+
} // namespace __sanitizer
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_nolibc.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_nolibc.cpp
index 3b278e017eb7c..487a634a16523 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_nolibc.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_nolibc.cpp
@@ -10,9 +10,10 @@
// libc in no-libcdep sources.
//===----------------------------------------------------------------------===//
-#include "sanitizer_platform.h"
#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
#include "sanitizer_libc.h"
+#include "sanitizer_platform.h"
namespace __sanitizer {
@@ -29,6 +30,7 @@ void SleepForSeconds(int seconds) { internal_sleep(seconds); }
#if !SANITIZER_WINDOWS && !SANITIZER_MAC
void ListOfModules::init() {}
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
#endif
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
index 31ff48cfd2cfc..1b89d6e176840 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
@@ -2294,9 +2294,10 @@ PRE_SYSCALL(ni_syscall)() {}
POST_SYSCALL(ni_syscall)(long res) {}
PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__))
+#if !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+ SANITIZER_RISCV64)
if (data) {
if (request == ptrace_setregs) {
PRE_READ((void *)data, struct_user_regs_struct_sz);
@@ -2315,9 +2316,10 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
}
POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__))
+#if !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+ SANITIZER_RISCV64)
if (res >= 0 && data) {
// Note that this is different from the interceptor in
// sanitizer_common_interceptors.inc.
@@ -2885,6 +2887,23 @@ POST_SYSCALL(getrandom)(long res, void *buf, uptr count, long flags) {
POST_WRITE(buf, res);
}
}
+
+PRE_SYSCALL(sigaltstack)(const void *ss, void *oss) {
+ if (ss != nullptr) {
+ PRE_READ(ss, struct_stack_t_sz);
+ }
+ if (oss != nullptr) {
+ PRE_WRITE(oss, struct_stack_t_sz);
+ }
+}
+
+POST_SYSCALL(sigaltstack)(long res, void *ss, void *oss) {
+ if (res == 0) {
+ if (oss != nullptr) {
+ POST_WRITE(oss, struct_stack_t_sz);
+ }
+ }
+}
} // extern "C"
#undef PRE_SYSCALL
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cpp
index f18cee66b8431..a52db08433e3b 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cpp
@@ -27,15 +27,15 @@
#include "sanitizer_platform.h"
#if SANITIZER_FUCHSIA
+#include
+#include
+#include
+
#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_symbolizer_fuchsia.h"
-#include
-#include
-#include
-
using namespace __sanitizer;
namespace __sancov {
@@ -82,7 +82,8 @@ class TracePcGuardController final {
void TracePcGuard(u32 *guard, uptr pc) {
atomic_uint32_t *guard_ptr = reinterpret_cast(guard);
u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed);
- if (idx > 0) array_[idx] = pc;
+ if (idx > 0)
+ array_[idx] = pc;
}
void Dump() {
@@ -140,6 +141,10 @@ class TracePcGuardController final {
internal_getpid());
_zx_object_set_property(vmo_, ZX_PROP_NAME, vmo_name_,
internal_strlen(vmo_name_));
+ uint64_t size = DataSize();
+ status = _zx_object_set_property(vmo_, ZX_PROP_VMO_CONTENT_SIZE, &size,
+ sizeof(size));
+ CHECK_EQ(status, ZX_OK);
// Map the largest possible view we might need into the VMO. Later
// we might need to increase the VMO's size before we can use larger
@@ -172,6 +177,10 @@ class TracePcGuardController final {
zx_status_t status = _zx_vmo_set_size(vmo_, DataSize());
CHECK_EQ(status, ZX_OK);
+ uint64_t size = DataSize();
+ status = _zx_object_set_property(vmo_, ZX_PROP_VMO_CONTENT_SIZE, &size,
+ sizeof(size));
+ CHECK_EQ(status, ZX_OK);
return first_index;
}
@@ -204,13 +213,15 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(const uptr *pcs,
}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *guard) {
- if (!*guard) return;
+ if (!*guard)
+ return;
__sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1);
}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init,
u32 *start, u32 *end) {
- if (start == end || *start) return;
+ if (start == end || *start)
+ return;
__sancov::pc_guard_controller.InitTracePcGuard(start, end);
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc
index 7beeff7e8af80..d7ab0c3d98c15 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc
@@ -29,4 +29,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init)
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
index 6a75792f92624..73ebeb5fa14ad 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
@@ -207,6 +207,7 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_bool_flag_init, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {}
} // extern "C"
// Weak definition for code instrumented with -fsanitize-coverage=stack-depth
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector1.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector1.cpp
index d4a325bea4b2d..2c924f5d39639 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector1.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector1.cpp
@@ -32,7 +32,7 @@ struct DDLogicalThread {
bool report_pending;
};
-struct DD : public DDetector {
+struct DD final : public DDetector {
SpinMutex mtx;
DeadlockDetector dd;
DDFlags flags;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector2.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector2.cpp
index 4026739d4e515..e3f8e1b127623 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector2.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector2.cpp
@@ -80,7 +80,7 @@ struct Mutex {
Link link[kMaxLink];
};
-struct DD : public DDetector {
+struct DD final : public DDetector {
explicit DD(const DDFlags *flags);
DDPhysicalThread* CreatePhysicalThread();
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h
index a4722b080ebd1..7f461c98bade4 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h
@@ -66,6 +66,9 @@ struct DDCallback {
virtual u32 Unwind() { return 0; }
virtual int UniqueTid() { return 0; }
+
+ protected:
+ ~DDCallback() {}
};
struct DDetector {
@@ -85,6 +88,9 @@ struct DDetector {
virtual void MutexDestroy(DDCallback *cb, DDMutex *m) {}
virtual DDReport *GetReport(DDCallback *cb) { return nullptr; }
+
+ protected:
+ ~DDetector() {}
};
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cpp
index 6cdfa28a95afd..785ebf97169d8 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cpp
@@ -124,12 +124,16 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
#endif
}
+class SuspendedThreadsListEmscripten final : public SuspendedThreadsList {};
+
void StopTheWorld(StopTheWorldCallback callback, void *argument) {
// TODO: have some workable alternative, since we can't just fork and suspend
// the parent process. This does not matter when single thread.
- callback(SuspendedThreadsList(), argument);
+ callback(SuspendedThreadsListEmscripten(), argument);
}
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
+
} // namespace __sanitizer
#endif
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h
index 584e66e4a861a..94f16b6e87358 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h
@@ -23,7 +23,7 @@
#if SANITIZER_FREEBSD || SANITIZER_MAC
# define __errno_location __error
-#elif SANITIZER_ANDROID || SANITIZER_NETBSD || SANITIZER_OPENBSD || \
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD || \
SANITIZER_RTEMS
# define __errno_location __errno
#elif SANITIZER_SOLARIS
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h
index 7f797c2c4c15c..650cf99128d78 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h
@@ -24,9 +24,10 @@
namespace __sanitizer {
-#define errno_ENOMEM __WASI_ERRNO_NOMEM
-#define errno_EBUSY __WASI_ERRNO_BUSY
-#define errno_EINVAL __WASI_ERRNO_INVAL
+#define errno_ENOMEM __WASI_ERRNO_NOMEM
+#define errno_EBUSY __WASI_ERRNO_BUSY
+#define errno_EINVAL __WASI_ERRNO_INVAL
+#define errno_ENAMETOOLONG __WASI_ERRNO_NAMETOOLONG
// Those might not present or their value differ on different platforms.
extern const int errno_EOWNERDEAD;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.cpp
index 79930d7942501..7c64b53e9b11d 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.cpp
@@ -58,40 +58,49 @@ void ReportFile::ReopenIfNecessary() {
} else {
internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
}
- fd = OpenFile(full_path, WrOnly);
+ error_t err;
+ fd = OpenFile(full_path, WrOnly, &err);
if (fd == kInvalidFd) {
const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
+ char errmsg[100];
+ internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)", err);
+ WriteToFile(kStderrFd, errmsg, internal_strlen(errmsg));
Die();
}
fd_pid = pid;
}
void ReportFile::SetReportPath(const char *path) {
- if (!path)
- return;
- uptr len = internal_strlen(path);
- if (len > sizeof(path_prefix) - 100) {
- Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
- path[0], path[1], path[2], path[3],
- path[4], path[5], path[6], path[7]);
- Die();
+ if (path) {
+ uptr len = internal_strlen(path);
+ if (len > sizeof(path_prefix) - 100) {
+ Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n", path[0], path[1],
+ path[2], path[3], path[4], path[5], path[6], path[7]);
+ Die();
+ }
}
SpinMutexLock l(mu);
if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
CloseFile(fd);
fd = kInvalidFd;
- if (internal_strcmp(path, "stdout") == 0) {
- fd = kStdoutFd;
- } else if (internal_strcmp(path, "stderr") == 0) {
+ if (!path || internal_strcmp(path, "stderr") == 0) {
fd = kStderrFd;
+ } else if (internal_strcmp(path, "stdout") == 0) {
+ fd = kStdoutFd;
} else {
internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
}
}
+const char *ReportFile::GetReportPath() {
+ SpinMutexLock l(mu);
+ ReopenIfNecessary();
+ return full_path;
+}
+
bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
uptr *read_len, uptr max_len, error_t *errno_p) {
*buff = nullptr;
@@ -210,6 +219,10 @@ void __sanitizer_set_report_fd(void *fd) {
report_file.fd = (fd_t)reinterpret_cast(fd);
report_file.fd_pid = internal_getpid();
}
+
+const char *__sanitizer_get_report_path() {
+ return report_file.GetReportPath();
+}
} // extern "C"
#endif // !SANITIZER_FUCHSIA
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
index 4a78a0e0ac881..08671ab67d0f5 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
@@ -26,6 +26,7 @@ struct ReportFile {
void Write(const char *buffer, uptr length);
bool SupportsColors();
void SetReportPath(const char *path);
+ const char *GetReportPath();
// Don't use fields directly. They are only declared public to allow
// aggregate initialization.
@@ -87,8 +88,8 @@ bool IsAbsolutePath(const char *path);
// The child process will close all fds after STDERR_FILENO
// before passing control to a program.
pid_t StartSubprocess(const char *filename, const char *const argv[],
- fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd,
- fd_t stderr_fd = kInvalidFd);
+ const char *const envp[], fd_t stdin_fd = kInvalidFd,
+ fd_t stdout_fd = kInvalidFd, fd_t stderr_fd = kInvalidFd);
// Checks if specified process is still running
bool IsProcessRunning(pid_t pid);
// Waits for the process to finish and returns its exit code.
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
index fac5dff346333..acc71ccd89eea 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
@@ -42,7 +42,7 @@ class FlagHandlerBase {
};
template
-class FlagHandler : public FlagHandlerBase {
+class FlagHandler final : public FlagHandlerBase {
T *t_;
public:
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cpp
index 684ee1e0b9995..21048be730416 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cpp
@@ -13,9 +13,10 @@
#include "sanitizer_flags.h"
#include "sanitizer_common.h"
+#include "sanitizer_flag_parser.h"
#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
#include "sanitizer_list.h"
-#include "sanitizer_flag_parser.h"
namespace __sanitizer {
@@ -72,7 +73,7 @@ void SubstituteForFlagValue(const char *s, char *out, uptr out_size) {
*out = '\0';
}
-class FlagHandlerInclude : public FlagHandlerBase {
+class FlagHandlerInclude final : public FlagHandlerBase {
FlagParser *parser_;
bool ignore_missing_;
const char *original_path_;
@@ -91,7 +92,7 @@ class FlagHandlerInclude : public FlagHandlerBase {
}
return parser_->ParseFile(value, ignore_missing_);
}
- bool Format(char *buffer, uptr size) {
+ bool Format(char *buffer, uptr size) override {
// Note `original_path_` isn't actually what's parsed due to `%`
// substitutions. Printing the substituted path would require holding onto
// mmap'ed memory.
@@ -124,6 +125,8 @@ void InitializeCommonFlags(CommonFlags *cf) {
// need to record coverage to generate coverage report.
cf->coverage |= cf->html_cov_report;
SetVerbosity(cf->verbosity);
+
+ InitializePlatformCommonFlags(cf);
}
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h
index 8f5e987da3ffd..5b59e5801bf91 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h
@@ -62,6 +62,10 @@ void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf);
// and perform initializations common to all sanitizers (e.g. setting
// verbosity).
void InitializeCommonFlags(CommonFlags *cf = &common_flags_dont_use);
+
+// Platform specific flags initialization.
+void InitializePlatformCommonFlags(CommonFlags *cf);
+
} // namespace __sanitizer
#endif // SANITIZER_FLAGS_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
index 065258a5a6e13..cfb5822645f13 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
@@ -40,16 +40,21 @@ COMMON_FLAG(bool, fast_unwind_on_check, false,
COMMON_FLAG(bool, fast_unwind_on_fatal, false,
"If available, use the fast frame-pointer-based unwinder on fatal "
"errors.")
-COMMON_FLAG(bool, fast_unwind_on_malloc, true,
+// ARM thumb/thumb2 frame pointer is inconsistent on GCC and Clang [1]
+// and fast-unwider is also unreliable with mixing arm and thumb code [2].
+// [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92172
+// [2] https://bugs.llvm.org/show_bug.cgi?id=44158
+COMMON_FLAG(bool, fast_unwind_on_malloc,
+ !(SANITIZER_LINUX && !SANITIZER_ANDROID && SANITIZER_ARM),
"If available, use the fast frame-pointer-based unwinder on "
"malloc/free.")
COMMON_FLAG(bool, handle_ioctl, false, "Intercept and handle ioctl requests.")
COMMON_FLAG(int, malloc_context_size, 1,
"Max number of stack frames kept for each allocation/deallocation.")
COMMON_FLAG(
- const char *, log_path, "stderr",
+ const char *, log_path, nullptr,
"Write logs to \"log_path.pid\". The special values are \"stdout\" and "
- "\"stderr\". The default is \"stderr\".")
+ "\"stderr\". If unspecified, defaults to \"stderr\".")
COMMON_FLAG(
bool, log_exe_name, false,
"Mention name of executable when reporting error and "
@@ -77,8 +82,9 @@ COMMON_FLAG(bool, print_summary, true,
"If false, disable printing error summaries in addition to error "
"reports.")
COMMON_FLAG(int, print_module_map, 0,
- "OS X only (0 - don't print, 1 - print only once before process "
- "exits, 2 - print after each report).")
+ "Print the process module map where supported (0 - don't print, "
+ "1 - print only once before process exits, 2 - print after each "
+ "report).")
COMMON_FLAG(bool, check_printf, true, "Check printf arguments.")
#define COMMON_FLAG_HANDLE_SIGNAL_HELP(signal) \
"Controls custom tool's " #signal " handler (0 - do not registers the " \
@@ -195,6 +201,9 @@ COMMON_FLAG(bool, intercept_strtok, true,
COMMON_FLAG(bool, intercept_strpbrk, true,
"If set, uses custom wrappers for strpbrk function "
"to find more errors.")
+COMMON_FLAG(
+ bool, intercept_strcmp, true,
+ "If set, uses custom wrappers for strcmp functions to find more errors.")
COMMON_FLAG(bool, intercept_strlen, true,
"If set, uses custom wrappers for strlen and strnlen functions "
"to find more errors.")
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_freebsd.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_freebsd.h
index 64cb21f1c3d2b..82b227eab6dab 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_freebsd.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_freebsd.h
@@ -19,11 +19,11 @@
// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
// 32-bit mode.
#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
-# include
-# if __FreeBSD_version <= 902001 // v9.2
-# include
-# include
-# include
+#include
+#if __FreeBSD_version <= 902001 // v9.2
+#include
+#include
+#include
namespace __sanitizer {
@@ -68,8 +68,8 @@ typedef struct __xmcontext {
} xmcontext_t;
typedef struct __xucontext {
- sigset_t uc_sigmask;
- xmcontext_t uc_mcontext;
+ sigset_t uc_sigmask;
+ xmcontext_t uc_mcontext;
struct __ucontext *uc_link;
stack_t uc_stack;
@@ -122,15 +122,16 @@ struct xdl_phdr_info {
void *dlpi_tls_data;
};
-typedef int (*__xdl_iterate_hdr_callback)(struct xdl_phdr_info*, size_t, void*);
-typedef int xdl_iterate_phdr_t(__xdl_iterate_hdr_callback, void*);
+typedef int (*__xdl_iterate_hdr_callback)(struct xdl_phdr_info *, size_t,
+ void *);
+typedef int xdl_iterate_phdr_t(__xdl_iterate_hdr_callback, void *);
#define xdl_iterate_phdr(callback, param) \
- (((xdl_iterate_phdr_t*) dl_iterate_phdr)((callback), (param)))
+ (((xdl_iterate_phdr_t *)dl_iterate_phdr)((callback), (param)))
} // namespace __sanitizer
-# endif // __FreeBSD_version <= 902001
+#endif // __FreeBSD_version <= 902001
#endif // SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
#endif // SANITIZER_FREEBSD_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
index 6e2c6137f0ce7..5ad20d0d7da62 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
@@ -14,10 +14,6 @@
#include "sanitizer_fuchsia.h"
#if SANITIZER_FUCHSIA
-#include "sanitizer_common.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_mutex.h"
-
#include
#include
#include
@@ -25,6 +21,11 @@
#include
#include
#include
+#include
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
namespace __sanitizer {
@@ -47,8 +48,10 @@ unsigned int internal_sleep(unsigned int seconds) {
}
u64 NanoTime() {
+ zx_handle_t utc_clock = _zx_utc_reference_get();
+ CHECK_NE(utc_clock, ZX_HANDLE_INVALID);
zx_time_t time;
- zx_status_t status = _zx_clock_get(ZX_CLOCK_UTC, &time);
+ zx_status_t status = _zx_clock_read(utc_clock, &time);
CHECK_EQ(status, ZX_OK);
return time;
}
@@ -66,6 +69,10 @@ uptr internal_getpid() {
return pid;
}
+int internal_dlinfo(void *handle, int request, void *p) {
+ UNIMPLEMENTED();
+}
+
uptr GetThreadSelf() { return reinterpret_cast(thrd_current()); }
tid_t GetTid() { return GetThreadSelf(); }
@@ -101,8 +108,6 @@ void SetAlternateSignalStack() {}
void UnsetAlternateSignalStack() {}
void InitTlsSize() {}
-void PrintModuleMap() {}
-
bool SignalContext::IsStackOverflow() const { return false; }
void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
const char *SignalContext::Describe() const { UNIMPLEMENTED(); }
@@ -500,6 +505,8 @@ u32 GetNumberOfCPUs() {
uptr GetRSS() { UNIMPLEMENTED(); }
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
+
} // namespace __sanitizer
using namespace __sanitizer;
@@ -522,6 +529,10 @@ void __sanitizer_set_report_path(const char *path) {
void __sanitizer_set_report_fd(void *fd) {
UNREACHABLE("not available on Fuchsia");
}
+
+const char *__sanitizer_get_report_path() {
+ UNREACHABLE("not available on Fuchsia");
+}
} // extern "C"
#endif // SANITIZER_FUCHSIA
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h
index 5a2ad32b4113b..96f9cde7ef191 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h
@@ -18,12 +18,18 @@
#include "sanitizer_common.h"
#include
+#include
namespace __sanitizer {
extern uptr MainThreadStackBase, MainThreadStackSize;
extern sanitizer_shadow_bounds_t ShadowBounds;
+struct MemoryMappingLayoutData {
+ InternalMmapVector data;
+ size_t current; // Current index into the vector.
+};
+
} // namespace __sanitizer
#endif // SANITIZER_FUCHSIA
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
index 86ad3a5e2c2aa..38439e44f611e 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
@@ -21,8 +21,9 @@
#if SANITIZER_LINUX || SANITIZER_FUCHSIA
-# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
- SANITIZER_FUCHSIA
+# if (__GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
+ SANITIZER_FUCHSIA) && \
+ !SANITIZER_GO
# define SANITIZER_USE_GETAUXVAL 1
# else
# define SANITIZER_USE_GETAUXVAL 0
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
index d0cc4da9755f4..576807ea3a6a1 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
@@ -446,9 +446,6 @@ static void ioctl_table_fill() {
_(STICIO_STOPQ, NONE, 0);
/* Entries from file: dev/usb/ukyopon.h */
_(UKYOPON_IDENTIFY, WRITE, struct_ukyopon_identify_sz);
- /* Entries from file: dev/usb/urio.h */
- _(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);
- _(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);
/* Entries from file: dev/usb/usb.h */
_(USB_REQUEST, READWRITE, struct_usb_ctl_request_sz);
_(USB_SETDEBUG, READ, sizeof(int));
@@ -1405,6 +1402,9 @@ static void ioctl_table_fill() {
/* Entries from file: dev/filemon/filemon.h (compat <= 9.99.26) */
_(FILEMON_SET_FD, READWRITE, sizeof(int));
_(FILEMON_SET_PID, READWRITE, sizeof(int));
+ /* Entries from file: dev/usb/urio.h (compat <= 9.99.43) */
+ _(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);
+ _(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);
#undef _
} // NOLINT
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
index c110eff130f24..0b001c1c48300 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
@@ -28,6 +28,10 @@ extern "C" {
// (casted to void *).
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_set_report_fd(void *fd);
+ // Get the current full report file path, if a path was specified by
+ // an earlier call to __sanitizer_set_report_path. Returns null otherwise.
+ SANITIZER_INTERFACE_ATTRIBUTE
+ const char *__sanitizer_get_report_path();
typedef struct {
int coverage_sandboxed;
@@ -109,8 +113,10 @@ extern "C" {
__sanitizer::u32*);
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_cov_8bit_counters_init();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_pcs_init();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+ __sanitizer_cov_bool_flag_init();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+ __sanitizer_cov_pcs_init();
} // extern "C"
#endif // SANITIZER_INTERFACE_INTERNAL_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
index 4e4609c27cb66..9e6f406029e0d 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -39,7 +39,7 @@
// TLS is handled differently on different platforms
#if SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_FREEBSD || SANITIZER_OPENBSD
+ SANITIZER_FREEBSD
# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
__attribute__((tls_model("initial-exec"))) thread_local
#else
@@ -104,8 +104,7 @@
//
// FIXME: do we have anything like this on Mac?
#ifndef SANITIZER_CAN_USE_PREINIT_ARRAY
-#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD || \
- SANITIZER_FUCHSIA || SANITIZER_NETBSD) && !defined(PIC)
+#if (SANITIZER_LINUX || SANITIZER_FUCHSIA || SANITIZER_NETBSD) && !defined(PIC)
#define SANITIZER_CAN_USE_PREINIT_ARRAY 1
// Before Solaris 11.4, .preinit_array is fully supported only with GNU ld.
// FIXME: Check for those conditions.
@@ -170,7 +169,7 @@ typedef int pid_t;
#endif
#if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_MAC || \
+ SANITIZER_MAC || \
(SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
(SANITIZER_LINUX && defined(__x86_64__))
typedef u64 OFF_T;
@@ -182,8 +181,7 @@ typedef u64 OFF64_T;
#if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC
typedef uptr operator_new_size_type;
#else
-# if SANITIZER_OPENBSD || defined(__s390__) && !defined(__s390x__) || \
- SANITIZER_EMSCRIPTEN
+# if defined(__s390__) && !defined(__s390x__) || SANITIZER_EMSCRIPTEN
// Special case: 31-bit s390 has unsigned long as size_t.
typedef unsigned long operator_new_size_type;
# else
@@ -197,9 +195,6 @@ typedef u64 tid_t;
// This header should NOT include any other headers to avoid portability issues.
// Common defs.
-#ifndef INLINE
-#define INLINE inline
-#endif
#define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE
#define SANITIZER_WEAK_DEFAULT_IMPL \
extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE
@@ -334,14 +329,10 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
#define UNIMPLEMENTED() UNREACHABLE("unimplemented")
-#define COMPILER_CHECK(pred) IMPL_COMPILER_ASSERT(pred, __LINE__)
+#define COMPILER_CHECK(pred) static_assert(pred, "")
#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
-#define IMPL_PASTE(a, b) a##b
-#define IMPL_COMPILER_ASSERT(pred, line) \
- typedef char IMPL_PASTE(assertion_failed_##_, line)[2*(int)(pred)-1]
-
// Limits for integral types. We have to redefine it in case we don't
// have stdint.h (like in Visual Studio 9).
#undef __INT64_C
@@ -456,5 +447,8 @@ using namespace __sanitizer;
namespace __hwasan {
using namespace __sanitizer;
}
+namespace __memprof {
+using namespace __sanitizer;
+}
#endif // SANITIZER_DEFS_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h
index 3d5db35d68ba5..ec0a6ded009b0 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h
@@ -72,6 +72,8 @@ unsigned int internal_sleep(unsigned int seconds);
uptr internal_getpid();
uptr internal_getppid();
+int internal_dlinfo(void *handle, int request, void *p);
+
// Threading
uptr internal_sched_yield();
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
index eb9bb765013da..9ea19bc21fa3c 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
@@ -9,7 +9,7 @@
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
- SANITIZER_NETBSD || SANITIZER_OPENBSD
+ SANITIZER_NETBSD
#include "sanitizer_libignore.h"
#include "sanitizer_flags.h"
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index dc5bbb6bc6cc6..d1738e0c2aecf 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -14,7 +14,7 @@
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
+ SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
@@ -26,7 +26,7 @@
#include "sanitizer_placement_new.h"
#include "sanitizer_procmaps.h"
-#if SANITIZER_LINUX
+#if SANITIZER_LINUX && !SANITIZER_GO
#include
#endif
@@ -38,6 +38,14 @@
#include
#include
#define stat kernel_stat
+#if SANITIZER_GO
+#undef st_atime
+#undef st_mtime
+#undef st_ctime
+#define st_atime st_atim
+#define st_mtime st_mtim
+#define st_ctime st_ctim
+#endif
#include
#undef stat
#endif
@@ -59,13 +67,7 @@
#include
#include
#include
-#if !SANITIZER_OPENBSD
#include
-#endif
-#if SANITIZER_OPENBSD
-#include
-#include
-#endif
#include
#if SANITIZER_LINUX
@@ -137,7 +139,7 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
#endif
// Note : FreeBSD had implemented both
-// Linux and OpenBSD apis, available from
+// Linux apis, available from
// future 12.x version most likely
#if SANITIZER_LINUX && defined(__NR_getrandom)
# if !defined(GRND_NONBLOCK)
@@ -148,20 +150,18 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
# define SANITIZER_USE_GETRANDOM 0
#endif // SANITIZER_LINUX && defined(__NR_getrandom)
-#if SANITIZER_OPENBSD
-# define SANITIZER_USE_GETENTROPY 1
+#if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000
+# define SANITIZER_USE_GETENTROPY 1
#else
-# if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000
-# define SANITIZER_USE_GETENTROPY 1
-# else
-# define SANITIZER_USE_GETENTROPY 0
-# endif
-#endif // SANITIZER_USE_GETENTROPY
+# define SANITIZER_USE_GETENTROPY 0
+#endif
namespace __sanitizer {
#if SANITIZER_LINUX && defined(__x86_64__)
#include "sanitizer_syscall_linux_x86_64.inc"
+#elif SANITIZER_LINUX && SANITIZER_RISCV64
+#include "sanitizer_syscall_linux_riscv64.inc"
#elif SANITIZER_LINUX && defined(__aarch64__)
#include "sanitizer_syscall_linux_aarch64.inc"
#elif SANITIZER_LINUX && defined(__arm__)
@@ -172,7 +172,7 @@ namespace __sanitizer {
// --------------- sanitizer_libc.h
#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
-#if !SANITIZER_S390 && !SANITIZER_OPENBSD && !SANITIZER_EMSCRIPTEN
+#if !SANITIZER_S390 && !SANITIZER_EMSCRIPTEN
uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
u64 offset) {
#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
@@ -185,20 +185,22 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
offset / 4096);
#endif
}
-#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD && !SANITIZER_NETBSD
+#endif // !SANITIZER_S390 && !SANITIZER_NETBSD
-#if !SANITIZER_OPENBSD && !SANITIZER_EMSCRIPTEN
+#if !SANITIZER_EMSCRIPTEN
uptr internal_munmap(void *addr, uptr length) {
return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
}
-#endif
-#if !SANITIZER_OPENBSD
int internal_mprotect(void *addr, uptr length, int prot) {
return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
}
#endif
+int internal_madvise(uptr addr, uptr length, int advice) {
+ return internal_syscall(SYSCALL(madvise), addr, length, advice);
+}
+
uptr internal_close(fd_t fd) {
#if SANITIZER_EMSCRIPTEN
return __wasi_fd_close(fd);
@@ -291,9 +293,11 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) {
// Undefine compatibility macros from
// so that they would not clash with the kernel_stat
// st_[a|m|c]time fields
+#if !SANITIZER_GO
#undef st_atime
#undef st_mtime
#undef st_ctime
+#endif
#if defined(SANITIZER_ANDROID)
// Bionic sys/stat.h defines additional macros
// for compatibility with the old NDKs and
@@ -336,7 +340,7 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
#endif
uptr internal_stat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
@@ -360,7 +364,7 @@ uptr internal_stat(const char *path, void *buf) {
}
uptr internal_lstat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
AT_SYMLINK_NOFOLLOW);
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
@@ -385,9 +389,8 @@ uptr internal_lstat(const char *path, void *buf) {
}
uptr internal_fstat(fd_t fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD || \
- SANITIZER_LINUX_USES_64BIT_SYSCALLS
-#if SANITIZER_MIPS64 && !SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+#if SANITIZER_MIPS64
// For mips64, fstat syscall fills buffer in the format of kernel_stat
struct kernel_stat kbuf;
int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
@@ -427,16 +430,13 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) {
#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
bufsize);
-#elif SANITIZER_OPENBSD
- return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
- bufsize);
#else
return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);
#endif
}
uptr internal_unlink(const char *path) {
-#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
#else
return internal_syscall(SYSCALL(unlink), (uptr)path);
@@ -447,7 +447,7 @@ uptr internal_rename(const char *oldpath, const char *newpath) {
#if defined(__riscv)
return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
(uptr)newpath, 0);
-#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
(uptr)newpath);
#else
@@ -463,16 +463,6 @@ uptr internal_sched_yield() {
#endif
}
-void internal__exit(int exitcode) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
- internal_syscall(SYSCALL(exit), exitcode);
-#else
- internal_syscall(SYSCALL(exit_group), exitcode);
-#endif
- Die(); // Unreachable.
-}
-
-
#if !SANITIZER_EMSCRIPTEN
unsigned int internal_sleep(unsigned int seconds) {
struct timespec ts;
@@ -491,6 +481,17 @@ uptr internal_execve(const char *filename, char *const argv[],
#endif // !SANITIZER_EMSCRIPTEN
#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+#if !SANITIZER_NETBSD
+void internal__exit(int exitcode) {
+#if SANITIZER_FREEBSD || SANITIZER_SOLARIS
+ internal_syscall(SYSCALL(exit), exitcode);
+#else
+ internal_syscall(SYSCALL(exit_group), exitcode);
+#endif
+ Die(); // Unreachable.
+}
+#endif // !SANITIZER_NETBSD
+
// ----------------- sanitizer_common.h
bool FileExists(const char *filename) {
if (ShouldMockFailureToOpen(filename))
@@ -512,8 +513,6 @@ tid_t GetTid() {
long Tid;
thr_self(&Tid);
return Tid;
-#elif SANITIZER_OPENBSD
- return internal_syscall(SYSCALL(getthrid));
#elif SANITIZER_SOLARIS
return thr_self();
#elif SANITIZER_EMSCRIPTEN
@@ -529,9 +528,6 @@ int TgKill(pid_t pid, tid_t tid, int sig) {
return internal_syscall(SYSCALL(tgkill), pid, tid, sig);
#elif SANITIZER_FREEBSD
return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig);
-#elif SANITIZER_OPENBSD
- (void)pid;
- return internal_syscall(SYSCALL(thrkill), tid, sig, nullptr);
#elif SANITIZER_SOLARIS
(void)pid;
return thr_kill(tid, sig);
@@ -542,7 +538,7 @@ int TgKill(pid_t pid, tid_t tid, int sig) {
#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD && !SANITIZER_EMSCRIPTEN
u64 NanoTime() {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
timeval tv;
#else
kernel_timeval tv;
@@ -569,8 +565,7 @@ uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
// 'environ' array (on some others) and does not use libc. This function
// should be called first inside __asan_init.
const char *GetEnv(const char *name) {
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \
- SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
if (::environ != 0) {
uptr NameLen = internal_strlen(name);
for (char **Env = ::environ; *Env != 0; Env++) {
@@ -608,14 +603,13 @@ const char *GetEnv(const char *name) {
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_GO
extern "C" {
SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
}
#endif
-#if !SANITIZER_GO && !SANITIZER_FREEBSD && !SANITIZER_NETBSD && \
- !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
static void ReadNullSepFileToArray(const char *path, char ***arr,
int arr_size) {
char *buff;
@@ -640,7 +634,7 @@ static void ReadNullSepFileToArray(const char *path, char ***arr,
}
#endif
-#if !SANITIZER_OPENBSD && !SANITIZER_EMSCRIPTEN
+#if !SANITIZER_EMSCRIPTEN
static void GetArgsAndEnv(char ***argv, char ***envp) {
#if SANITIZER_FREEBSD
// On FreeBSD, retrieving the argument and environment arrays is done via the
@@ -660,16 +654,21 @@ static void GetArgsAndEnv(char ***argv, char ***envp) {
#else // SANITIZER_FREEBSD
#if !SANITIZER_GO
if (&__libc_stack_end) {
-#endif // !SANITIZER_GO
uptr* stack_end = (uptr*)__libc_stack_end;
- int argc = *stack_end;
+ // Normally argc can be obtained from *stack_end, however, on ARM glibc's
+ // _start clobbers it:
+ // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75
+ // Do not special-case ARM and infer argc from argv everywhere.
+ int argc = 0;
+ while (stack_end[argc + 1]) argc++;
*argv = (char**)(stack_end + 1);
*envp = (char**)(stack_end + argc + 2);
-#if !SANITIZER_GO
} else {
+#endif // !SANITIZER_GO
static const int kMaxArgv = 2000, kMaxEnvp = 2000;
ReadNullSepFileToArray("/proc/self/cmdline", argv, kMaxArgv);
ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp);
+#if !SANITIZER_GO
}
#endif // !SANITIZER_GO
#endif // SANITIZER_FREEBSD
@@ -687,7 +686,7 @@ char **GetEnviron() {
return envp;
}
-#endif // !SANITIZER_OPENBSD && !SANITIZER_EMSCRIPTEN
+#endif // !SANITIZER_EMSCRIPTEN
#if !SANITIZER_SOLARIS
enum MutexState {
@@ -748,19 +747,9 @@ void BlockingMutex::CheckLocked() {
// 32-bit syscall here.
#if SANITIZER_NETBSD
// Not used
-#elif SANITIZER_OPENBSD
-// struct dirent is different for Linux and us. At this moment, we use only
-// d_fileno (Linux call this d_ino), d_reclen, and d_name.
-struct linux_dirent {
- u64 d_ino; // d_fileno
- u16 d_reclen;
- u16 d_namlen; // not used
- u8 d_type; // not used
- char d_name[NAME_MAX + 1];
-};
#else
struct linux_dirent {
-#if SANITIZER_X32 || defined(__aarch64__)
+#if SANITIZER_X32 || defined(__aarch64__) || SANITIZER_RISCV64
u64 d_ino;
u64 d_off;
#else
@@ -768,7 +757,7 @@ struct linux_dirent {
unsigned long d_off;
#endif
unsigned short d_reclen;
-#ifdef __aarch64__
+#if defined(__aarch64__) || SANITIZER_RISCV64
unsigned char d_type;
#endif
char d_name[256];
@@ -797,6 +786,14 @@ uptr internal_getppid() {
return internal_syscall(SYSCALL(getppid));
}
+int internal_dlinfo(void *handle, int request, void *p) {
+#if SANITIZER_FREEBSD
+ return dlinfo(handle, request, p);
+#else
+ UNIMPLEMENTED();
+#endif
+}
+
uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
@@ -839,28 +836,39 @@ int internal_fork() {
#endif
}
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
int internal_sysctl(const int *name, unsigned int namelen, void *oldp,
uptr *oldlenp, const void *newp, uptr newlen) {
-#if SANITIZER_OPENBSD
- return sysctl(name, namelen, oldp, (size_t *)oldlenp, (void *)newp,
- (size_t)newlen);
-#else
return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp,
(size_t *)oldlenp, newp, (size_t)newlen);
-#endif
}
-#if SANITIZER_FREEBSD
int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
const void *newp, uptr newlen) {
- static decltype(sysctlbyname) *real = nullptr;
- if (!real)
- real = (decltype(sysctlbyname) *)dlsym(RTLD_NEXT, "sysctlbyname");
- CHECK(real);
- return real(sname, oldp, (size_t *)oldlenp, newp, (size_t)newlen);
-}
+ // Note: this function can be called during startup, so we need to avoid
+ // calling any interceptable functions. On FreeBSD >= 1300045 sysctlbyname()
+ // is a real syscall, but for older versions it calls sysctlnametomib()
+ // followed by sysctl(). To avoid calling the intercepted version and
+ // asserting if this happens during startup, call the real sysctlnametomib()
+ // followed by internal_sysctl() if the syscall is not available.
+#ifdef SYS___sysctlbyname
+ return internal_syscall(SYSCALL(__sysctlbyname), sname,
+ internal_strlen(sname), oldp, (size_t *)oldlenp, newp,
+ (size_t)newlen);
+#else
+ static decltype(sysctlnametomib) *real_sysctlnametomib = nullptr;
+ if (!real_sysctlnametomib)
+ real_sysctlnametomib =
+ (decltype(sysctlnametomib) *)dlsym(RTLD_NEXT, "sysctlnametomib");
+ CHECK(real_sysctlnametomib);
+
+ int oid[CTL_MAXNAME];
+ size_t len = CTL_MAXNAME;
+ if (real_sysctlnametomib(sname, oid, &len) == -1)
+ return (-1);
+ return internal_sysctl(oid, len, oldp, oldlenp, newp, newlen);
#endif
+}
#endif
#if SANITIZER_LINUX
@@ -914,16 +922,15 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) {
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
#elif SANITIZER_EMSCRIPTEN
return NULL;
#else
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
__sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset;
- return internal_syscall(SYSCALL(rt_sigprocmask), (uptr)how,
- (uptr)&k_set->sig[0], (uptr)&k_oldset->sig[0],
- sizeof(__sanitizer_kernel_sigset_t));
+ return internal_syscall(SYSCALL(rt_sigprocmask), (uptr)how, (uptr)k_set,
+ (uptr)k_oldset, sizeof(__sanitizer_kernel_sigset_t));
#endif
}
@@ -1080,9 +1087,8 @@ static uptr GetKernelAreaSize() {
// is modified (e.g. under schroot) so check this as well.
struct utsname uname_info;
int pers = personality(0xffffffffUL);
- if (!(pers & PER_MASK)
- && uname(&uname_info) == 0
- && internal_strstr(uname_info.machine, "64"))
+ if (!(pers & PER_MASK) && internal_uname(&uname_info) == 0 &&
+ internal_strstr(uname_info.machine, "64"))
return 0;
#endif // SANITIZER_ANDROID
@@ -1095,7 +1101,7 @@ static uptr GetKernelAreaSize() {
#endif // SANITIZER_WORDSIZE == 32
uptr GetMaxVirtualAddress() {
-#if (SANITIZER_NETBSD || SANITIZER_OPENBSD) && defined(__x86_64__)
+#if SANITIZER_NETBSD && defined(__x86_64__)
return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
#elif SANITIZER_WORDSIZE == 64
# if defined(__powerpc64__) || defined(__aarch64__)
@@ -1107,6 +1113,8 @@ uptr GetMaxVirtualAddress() {
// This should (does) work for both PowerPC64 Endian modes.
// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
+#elif SANITIZER_RISCV64
+ return (1ULL << 38) - 1;
# elif defined(__mips64)
return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
# elif defined(__s390x__)
@@ -1137,7 +1145,8 @@ uptr GetMaxUserVirtualAddress() {
#if !SANITIZER_ANDROID
uptr GetPageSize() {
-#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__))
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) && \
+ defined(EXEC_PAGESIZE)
return EXEC_PAGESIZE;
#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
// Use sysctl as sysconf can trigger interceptors internally.
@@ -1157,7 +1166,6 @@ uptr GetPageSize() {
extern "C" uptr emscripten_get_module_name(char *buf, uptr buf_len);
#endif
-#if !SANITIZER_OPENBSD
uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
#if SANITIZER_SOLARIS
const char *default_module_name = getexecname();
@@ -1196,7 +1204,6 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
return module_name_len;
#endif
}
-#endif // !SANITIZER_OPENBSD
uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
#if SANITIZER_LINUX
@@ -1229,10 +1236,10 @@ bool LibraryNameIs(const char *full_name, const char *base_name) {
// Call cb for each region mapped by map.
void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)) {
CHECK_NE(map, nullptr);
-#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD
typedef ElfW(Phdr) Elf_Phdr;
typedef ElfW(Ehdr) Elf_Ehdr;
-#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#endif // !SANITIZER_FREEBSD
char *base = (char *)map->l_addr;
Elf_Ehdr *ehdr = (Elf_Ehdr *)base;
char *phdrs = base + ehdr->e_phoff;
@@ -1404,6 +1411,55 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
: "memory", "$29" );
return res;
}
+#elif SANITIZER_RISCV64
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+ int *parent_tidptr, void *newtls, int *child_tidptr) {
+ long long res;
+ if (!fn || !child_stack)
+ return -EINVAL;
+ CHECK_EQ(0, (uptr)child_stack % 16);
+ child_stack = (char *)child_stack - 2 * sizeof(unsigned long long);
+ ((unsigned long long *)child_stack)[0] = (uptr)fn;
+ ((unsigned long long *)child_stack)[1] = (uptr)arg;
+
+ register int (*__fn)(void *) __asm__("a0") = fn;
+ register void *__stack __asm__("a1") = child_stack;
+ register int __flags __asm__("a2") = flags;
+ register void *__arg __asm__("a3") = arg;
+ register int *__ptid __asm__("a4") = parent_tidptr;
+ register void *__tls __asm__("a5") = newtls;
+ register int *__ctid __asm__("a6") = child_tidptr;
+
+ __asm__ __volatile__(
+ "mv a0,a2\n" /* flags */
+ "mv a2,a4\n" /* ptid */
+ "mv a3,a5\n" /* tls */
+ "mv a4,a6\n" /* ctid */
+ "addi a7, zero, %9\n" /* clone */
+
+ "ecall\n"
+
+ /* if (%r0 != 0)
+ * return %r0;
+ */
+ "bnez a0, 1f\n"
+
+ /* In the child, now. Call "fn(arg)". */
+ "ld a0, 8(sp)\n"
+ "ld a1, 16(sp)\n"
+ "jalr a1\n"
+
+ /* Call _exit(%r0). */
+ "addi a7, zero, %10\n"
+ "ecall\n"
+ "1:\n"
+
+ : "=r"(res)
+ : "i"(-EINVAL), "r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg),
+ "r"(__ptid), "r"(__tls), "r"(__ctid), "i"(__NR_clone), "i"(__NR_exit)
+ : "ra", "memory");
+ return res;
+}
#elif defined(__aarch64__)
uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
int *parent_tidptr, void *newtls, int *child_tidptr) {
@@ -1697,6 +1753,12 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
}
#endif // defined(__x86_64__) && SANITIZER_LINUX
+#if SANITIZER_LINUX
+int internal_uname(struct utsname *buf) {
+ return internal_syscall(SYSCALL(uname), buf);
+}
+#endif
+
#if SANITIZER_ANDROID
#if __ANDROID_API__ < 21
extern "C" __attribute__((weak)) int dl_iterate_phdr(
@@ -1779,7 +1841,7 @@ HandleSignalMode GetHandleSignalMode(int signum) {
}
#if !SANITIZER_GO
-void *internal_start_thread(void(*func)(void *arg), void *arg) {
+void *internal_start_thread(void *(*func)(void *arg), void *arg) {
// Start the thread with signals blocked, otherwise it can steal user signals.
__sanitizer_sigset_t set, old;
internal_sigfillset(&set);
@@ -1790,7 +1852,7 @@ void *internal_start_thread(void(*func)(void *arg), void *arg) {
#endif
internal_sigprocmask(SIG_SETMASK, &set, &old);
void *th;
- real_pthread_create(&th, nullptr, (void*(*)(void *arg))func, arg);
+ real_pthread_create(&th, nullptr, func, arg);
internal_sigprocmask(SIG_SETMASK, &old, nullptr);
return th;
}
@@ -1799,7 +1861,7 @@ void internal_join_thread(void *th) {
real_pthread_join(th, nullptr);
}
#else
-void *internal_start_thread(void (*func)(void *), void *arg) { return 0; }
+void *internal_start_thread(void *(*func)(void *), void *arg) { return 0; }
void internal_join_thread(void *th) {}
#endif
@@ -1827,11 +1889,7 @@ static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) {
}
#endif
-#if SANITIZER_OPENBSD
-using Context = sigcontext;
-#else
using Context = ucontext_t;
-#endif
SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
Context *ucontext = (Context *)context;
@@ -1841,8 +1899,6 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
uptr err = ucontext->uc_mcontext.mc_err;
#elif SANITIZER_NETBSD
uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
-#elif SANITIZER_OPENBSD
- uptr err = ucontext->sc_err;
#elif SANITIZER_SOLARIS && defined(__i386__)
const int Err = 13;
uptr err = ucontext->uc_mcontext.gregs[Err];
@@ -1924,6 +1980,105 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
#endif
u32 instr = *(u32 *)pc;
return (instr >> 21) & 1 ? WRITE: READ;
+#elif defined(__riscv)
+ unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
+ unsigned faulty_instruction = *(uint16_t *)pc;
+
+#if defined(__riscv_compressed)
+ if ((faulty_instruction & 0x3) != 0x3) { // it's a compressed instruction
+ // set op_bits to the instruction bits [1, 0, 15, 14, 13]
+ unsigned op_bits =
+ ((faulty_instruction & 0x3) << 3) | (faulty_instruction >> 13);
+ unsigned rd = faulty_instruction & 0xF80; // bits 7-11, inclusive
+ switch (op_bits) {
+ case 0b10'010: // c.lwsp (rd != x0)
+#if __riscv_xlen == 64
+ case 0b10'011: // c.ldsp (rd != x0)
+#endif
+ return rd ? SignalContext::READ : SignalContext::UNKNOWN;
+ case 0b00'010: // c.lw
+#if __riscv_flen >= 32 && __riscv_xlen == 32
+ case 0b10'011: // c.flwsp
+#endif
+#if __riscv_flen >= 32 || __riscv_xlen == 64
+ case 0b00'011: // c.flw / c.ld
+#endif
+#if __riscv_flen == 64
+ case 0b00'001: // c.fld
+ case 0b10'001: // c.fldsp
+#endif
+ return SignalContext::READ;
+ case 0b00'110: // c.sw
+ case 0b10'110: // c.swsp
+#if __riscv_flen >= 32 || __riscv_xlen == 64
+ case 0b00'111: // c.fsw / c.sd
+ case 0b10'111: // c.fswsp / c.sdsp
+#endif
+#if __riscv_flen == 64
+ case 0b00'101: // c.fsd
+ case 0b10'101: // c.fsdsp
+#endif
+ return SignalContext::WRITE;
+ default:
+ return SignalContext::UNKNOWN;
+ }
+ }
+#endif
+
+ unsigned opcode = faulty_instruction & 0x7f; // lower 7 bits
+ unsigned funct3 = (faulty_instruction >> 12) & 0x7; // bits 12-14, inclusive
+ switch (opcode) {
+ case 0b0000011: // loads
+ switch (funct3) {
+ case 0b000: // lb
+ case 0b001: // lh
+ case 0b010: // lw
+#if __riscv_xlen == 64
+ case 0b011: // ld
+#endif
+ case 0b100: // lbu
+ case 0b101: // lhu
+ return SignalContext::READ;
+ default:
+ return SignalContext::UNKNOWN;
+ }
+ case 0b0100011: // stores
+ switch (funct3) {
+ case 0b000: // sb
+ case 0b001: // sh
+ case 0b010: // sw
+#if __riscv_xlen == 64
+ case 0b011: // sd
+#endif
+ return SignalContext::WRITE;
+ default:
+ return SignalContext::UNKNOWN;
+ }
+#if __riscv_flen >= 32
+ case 0b0000111: // floating-point loads
+ switch (funct3) {
+ case 0b010: // flw
+#if __riscv_flen == 64
+ case 0b011: // fld
+#endif
+ return SignalContext::READ;
+ default:
+ return SignalContext::UNKNOWN;
+ }
+ case 0b0100111: // floating-point stores
+ switch (funct3) {
+ case 0b010: // fsw
+#if __riscv_flen == 64
+ case 0b011: // fsd
+#endif
+ return SignalContext::WRITE;
+ default:
+ return SignalContext::UNKNOWN;
+ }
+#endif
+ default:
+ return SignalContext::UNKNOWN;
+ }
#else
(void)ucontext;
return UNKNOWN; // FIXME: Implement.
@@ -1969,11 +2124,6 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
*pc = ucontext->uc_mcontext.mc_rip;
*bp = ucontext->uc_mcontext.mc_rbp;
*sp = ucontext->uc_mcontext.mc_rsp;
-#elif SANITIZER_OPENBSD
- sigcontext *ucontext = (sigcontext *)context;
- *pc = ucontext->sc_rip;
- *bp = ucontext->sc_rbp;
- *sp = ucontext->sc_rsp;
# else
ucontext_t *ucontext = (ucontext_t*)context;
*pc = ucontext->uc_mcontext.gregs[REG_RIP];
@@ -1986,11 +2136,6 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
*pc = ucontext->uc_mcontext.mc_eip;
*bp = ucontext->uc_mcontext.mc_ebp;
*sp = ucontext->uc_mcontext.mc_esp;
-#elif SANITIZER_OPENBSD
- sigcontext *ucontext = (sigcontext *)context;
- *pc = ucontext->sc_eip;
- *bp = ucontext->sc_ebp;
- *sp = ucontext->sc_esp;
# else
ucontext_t *ucontext = (ucontext_t*)context;
# if SANITIZER_SOLARIS
@@ -2002,13 +2147,13 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
# ifndef REG_EBP
# define REG_EBP 6 // REG_FP
# endif
-# ifndef REG_ESP
-# define REG_ESP 17 // REG_SP
+# ifndef REG_UESP
+# define REG_UESP 17 // REG_SP
# endif
# endif
*pc = ucontext->uc_mcontext.gregs[REG_EIP];
*bp = ucontext->uc_mcontext.gregs[REG_EBP];
- *sp = ucontext->uc_mcontext.gregs[REG_ESP];
+ *sp = ucontext->uc_mcontext.gregs[REG_UESP];
# endif
#elif defined(__powerpc__) || defined(__powerpc64__)
ucontext_t *ucontext = (ucontext_t*)context;
@@ -2087,7 +2232,9 @@ void CheckASLR() {
}
if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_ASLR)) {
- Printf("This sanitizer is not compatible with enabled ASLR\n");
+ Printf("This sanitizer is not compatible with enabled ASLR.\n"
+ "To disable ASLR, please run \"paxctl +a %s\" and try again.\n",
+ GetArgv()[0]);
Die();
}
#elif SANITIZER_PPC64V2
@@ -2159,14 +2306,12 @@ void CheckMPROTECT() {
#endif
}
-void PrintModuleMap() { }
-
void CheckNoDeepBind(const char *filename, int flag) {
#ifdef RTLD_DEEPBIND
if (flag & RTLD_DEEPBIND) {
Report(
"You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
- " which is incompatibe with sanitizer runtime "
+ " which is incompatible with sanitizer runtime "
"(see https://github.com/google/sanitizers/issues/611 for details"
"). If you want to run %s library under sanitizers please remove "
"RTLD_DEEPBIND from dlopen flags.\n",
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
index d73b539c3cdb8..bb432f75aa86a 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
@@ -14,17 +14,17 @@
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
+ SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform_limits_freebsd.h"
#include "sanitizer_platform_limits_netbsd.h"
-#include "sanitizer_platform_limits_openbsd.h"
#include "sanitizer_platform_limits_posix.h"
#include "sanitizer_platform_limits_solaris.h"
#include "sanitizer_posix.h"
struct link_map; // Opaque type returned by dlopen().
+struct utsname;
namespace __sanitizer {
// Dirent structure for getdents(). Note that this structure is different from
@@ -59,12 +59,13 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);
// internal_sigaction instead.
int internal_sigaction_norestorer(int signum, const void *act, void *oldact);
void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
-#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \
- || defined(__powerpc64__) || defined(__s390__) || defined(__i386__) \
- || defined(__arm__)
+#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \
+ defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \
+ defined(__arm__) || SANITIZER_RISCV64
uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
int *parent_tidptr, void *newtls, int *child_tidptr);
#endif
+int internal_uname(struct utsname *buf);
#elif SANITIZER_FREEBSD
void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
#elif SANITIZER_NETBSD
@@ -107,7 +108,7 @@ void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
// Releases memory pages entirely within the [beg, end] address range.
// The pages no longer count toward RSS; reads are guaranteed to return 0.
// Requires (but does not verify!) that pages are MAP_PRIVATE.
-INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
+inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
// man madvise on Linux promises zero-fill for anonymous private pages.
// Testing shows the same behaviour for private (but not anonymous) mappings
// of shm_open() files, as long as the underlying file is untouched.
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index edbe8402808a8..f20b9001c2c20 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -13,8 +13,8 @@
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_allocator_internal.h"
#include "sanitizer_atomic.h"
@@ -28,6 +28,10 @@
#include "sanitizer_placement_new.h"
#include "sanitizer_procmaps.h"
+#if SANITIZER_NETBSD
+#define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
+#endif
+
#include // for dlsym()
#include
#include
@@ -35,6 +39,10 @@
#include
#include
+#if !defined(ElfW)
+#define ElfW(type) Elf_##type
+#endif
+
#if SANITIZER_FREEBSD
#include
#include
@@ -42,11 +50,6 @@
#define pthread_getattr_np pthread_attr_get_np
#endif
-#if SANITIZER_OPENBSD
-#include
-#include
-#endif
-
#if SANITIZER_NETBSD
#include
#include
@@ -134,18 +137,13 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
CHECK_EQ(thr_stksegment(&ss), 0);
stacksize = ss.ss_size;
stackaddr = (char *)ss.ss_sp - stacksize;
-#elif SANITIZER_OPENBSD
- stack_t sattr;
- CHECK_EQ(pthread_stackseg_np(pthread_self(), &sattr), 0);
- stackaddr = sattr.ss_sp;
- stacksize = sattr.ss_size;
#else // !SANITIZER_SOLARIS
pthread_attr_t attr;
pthread_attr_init(&attr);
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
pthread_attr_destroy(&attr);
-#endif // SANITIZER_SOLARIS
+#endif // SANITIZER_SOLARIS
*stack_top = (uptr)stackaddr + stacksize;
*stack_bottom = (uptr)stackaddr;
@@ -185,20 +183,19 @@ __attribute__((unused)) static bool GetLibcVersion(int *major, int *minor,
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \
- !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_SOLARIS
+#if SANITIZER_GLIBC && !SANITIZER_GO
static uptr g_tls_size;
#ifdef __i386__
-# define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
+#define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
#else
-# define CHECK_GET_TLS_STATIC_INFO_VERSION 0
+#define CHECK_GET_TLS_STATIC_INFO_VERSION 0
#endif
#if CHECK_GET_TLS_STATIC_INFO_VERSION
-# define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
+#define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
#else
-# define DL_INTERNAL_FUNCTION
+#define DL_INTERNAL_FUNCTION
#endif
namespace {
@@ -258,12 +255,11 @@ void InitTlsSize() {
}
#else
void InitTlsSize() { }
-#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
- // !SANITIZER_NETBSD && !SANITIZER_SOLARIS
+#endif // SANITIZER_GLIBC && !SANITIZER_GO
-#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) || \
- defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) || \
- defined(__arm__)) && \
+#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) || \
+ defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) || \
+ defined(__arm__) || SANITIZER_RISCV64) && \
SANITIZER_LINUX && !SANITIZER_ANDROID
// sizeof(struct pthread) from glibc.
static atomic_uintptr_t thread_descriptor_size;
@@ -297,12 +293,29 @@ uptr ThreadDescriptorSize() {
val = FIRST_32_SECOND_64(1168, 2288);
else if (minor <= 14)
val = FIRST_32_SECOND_64(1168, 2304);
- else
+ else if (minor < 32) // Unknown version
val = FIRST_32_SECOND_64(1216, 2304);
+ else // minor == 32
+ val = FIRST_32_SECOND_64(1344, 2496);
}
#elif defined(__mips__)
// TODO(sagarthakur): add more values as per different glibc versions.
val = FIRST_32_SECOND_64(1152, 1776);
+#elif SANITIZER_RISCV64
+ int major;
+ int minor;
+ int patch;
+ if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
+ // TODO: consider adding an optional runtime check for an unknown (untested)
+ // glibc version
+ if (minor <= 28) // WARNING: the highest tested version is 2.29
+ val = 1772; // no guarantees for this one
+ else if (minor <= 31)
+ val = 1772; // tested against glibc 2.29, 2.31
+ else
+ val = 1936; // tested against glibc 2.32
+ }
+
#elif defined(__aarch64__)
// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
val = 1776;
@@ -323,15 +336,17 @@ uptr ThreadSelfOffset() {
return kThreadSelfOffset;
}
-#if defined(__mips__) || defined(__powerpc64__)
+#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
// head structure. It lies before the static tls blocks.
static uptr TlsPreTcbSize() {
-# if defined(__mips__)
+#if defined(__mips__)
const uptr kTcbHead = 16; // sizeof (tcbhead_t)
-# elif defined(__powerpc64__)
+#elif defined(__powerpc64__)
const uptr kTcbHead = 88; // sizeof (tcbhead_t)
-# endif
+#elif SANITIZER_RISCV64
+ const uptr kTcbHead = 16; // sizeof (tcbhead_t)
+#endif
const uptr kTlsAlign = 16;
const uptr kTlsPreTcbSize =
RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
@@ -341,11 +356,11 @@ static uptr TlsPreTcbSize() {
uptr ThreadSelf() {
uptr descr_addr;
-# if defined(__i386__)
+#if defined(__i386__)
asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
-# elif defined(__x86_64__)
+#elif defined(__x86_64__)
asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
-# elif defined(__mips__)
+#elif defined(__mips__)
// MIPS uses TLS variant I. The thread pointer (in hardware register $29)
// points to the end of the TCB + 0x7000. The pthread_descr structure is
// immediately in front of the TCB. TlsPreTcbSize() includes the size of the
@@ -357,12 +372,16 @@ uptr ThreadSelf() {
rdhwr %0,$29;\
.set pop" : "=r" (thread_pointer));
descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
-# elif defined(__aarch64__) || defined(__arm__)
+#elif defined(__aarch64__) || defined(__arm__)
descr_addr = reinterpret_cast(__builtin_thread_pointer()) -
ThreadDescriptorSize();
-# elif defined(__s390__)
+#elif SANITIZER_RISCV64
+ // https://github.com/riscv/riscv-elf-psabi-doc/issues/53
+ uptr thread_pointer = reinterpret_cast(__builtin_thread_pointer());
+ descr_addr = thread_pointer - TlsPreTcbSize();
+#elif defined(__s390__)
descr_addr = reinterpret_cast(__builtin_thread_pointer());
-# elif defined(__powerpc64__)
+#elif defined(__powerpc64__)
// PPC64LE uses TLS variant I. The thread pointer (in GPR 13)
// points to the end of the TCB + 0x7000. The pthread_descr structure is
// immediately in front of the TCB. TlsPreTcbSize() includes the size of the
@@ -371,9 +390,9 @@ uptr ThreadSelf() {
uptr thread_pointer;
asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset));
descr_addr = thread_pointer - TlsPreTcbSize();
-# else
-# error "unsupported CPU arch"
-# endif
+#else
+#error "unsupported CPU arch"
+#endif
return descr_addr;
}
#endif // (x86_64 || i386 || MIPS) && SANITIZER_LINUX
@@ -381,15 +400,15 @@ uptr ThreadSelf() {
#if SANITIZER_FREEBSD
static void **ThreadSelfSegbase() {
void **segbase = 0;
-# if defined(__i386__)
+#if defined(__i386__)
// sysarch(I386_GET_GSBASE, segbase);
__asm __volatile("mov %%gs:0, %0" : "=r" (segbase));
-# elif defined(__x86_64__)
+#elif defined(__x86_64__)
// sysarch(AMD64_GET_FSBASE, segbase);
__asm __volatile("movq %%fs:0, %0" : "=r" (segbase));
-# else
-# error "unsupported CPU arch"
-# endif
+#else
+#error "unsupported CPU arch"
+#endif
return segbase;
}
@@ -400,7 +419,13 @@ uptr ThreadSelf() {
#if SANITIZER_NETBSD
static struct tls_tcb * ThreadSelfTlsTcb() {
- return (struct tls_tcb *)_lwp_getprivate();
+ struct tls_tcb *tcb = nullptr;
+#ifdef __HAVE___LWP_GETTCB_FAST
+ tcb = (struct tls_tcb *)__lwp_gettcb_fast();
+#elif defined(__HAVE___LWP_GETPRIVATE_FAST)
+ tcb = (struct tls_tcb *)__lwp_getprivate_fast();
+#endif
+ return tcb;
}
uptr ThreadSelf() {
@@ -421,22 +446,40 @@ int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) {
}
#endif // SANITIZER_NETBSD
+#if SANITIZER_ANDROID
+// Bionic provides this API since S.
+extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,
+ void **);
+#endif
+
#if !SANITIZER_GO
static void GetTls(uptr *addr, uptr *size) {
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-# if defined(__x86_64__) || defined(__i386__) || defined(__s390__)
+#if SANITIZER_ANDROID
+ if (&__libc_get_static_tls_bounds) {
+ void *start_addr;
+ void *end_addr;
+ __libc_get_static_tls_bounds(&start_addr, &end_addr);
+ *addr = reinterpret_cast(start_addr);
+ *size =
+ reinterpret_cast(end_addr) - reinterpret_cast(start_addr);
+ } else {
+ *addr = 0;
+ *size = 0;
+ }
+#elif SANITIZER_LINUX
+#if defined(__x86_64__) || defined(__i386__) || defined(__s390__)
*addr = ThreadSelf();
*size = GetTlsSize();
*addr -= *size;
*addr += ThreadDescriptorSize();
-# elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) \
- || defined(__arm__)
+#elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) || \
+ defined(__arm__) || SANITIZER_RISCV64
*addr = ThreadSelf();
*size = GetTlsSize();
-# else
+#else
*addr = 0;
*size = 0;
-# endif
+#endif
#elif SANITIZER_FREEBSD
void** segbase = ThreadSelfSegbase();
*addr = 0;
@@ -464,34 +507,32 @@ static void GetTls(uptr *addr, uptr *size) {
*addr = (uptr)tcb->tcb_dtv[1];
}
}
-#elif SANITIZER_OPENBSD
- *addr = 0;
- *size = 0;
-#elif SANITIZER_ANDROID
- *addr = 0;
- *size = 0;
#elif SANITIZER_SOLARIS
// FIXME
*addr = 0;
*size = 0;
#else
-# error "Unknown OS"
+#error "Unknown OS"
#endif
}
#endif
#if !SANITIZER_GO
uptr GetTlsSize() {
-#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
uptr addr, size;
GetTls(&addr, &size);
return size;
-#elif defined(__mips__) || defined(__powerpc64__)
+#elif SANITIZER_GLIBC
+#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
#else
return g_tls_size;
#endif
+#else
+ return 0;
+#endif
}
#endif
@@ -520,13 +561,13 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD
typedef ElfW(Phdr) Elf_Phdr;
-#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
+#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
#define Elf_Phdr XElf32_Phdr
#define dl_phdr_info xdl_phdr_info
#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
-#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#endif // !SANITIZER_FREEBSD
struct DlIteratePhdrData {
InternalMmapVectorNoCtor *modules;
@@ -646,7 +687,7 @@ uptr GetRSS() {
// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
// they allocate memory.
u32 GetNumberOfCPUs() {
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
u32 ncpu;
int req[2];
uptr len = sizeof(ncpu);
@@ -701,7 +742,7 @@ u32 GetNumberOfCPUs() {
#if SANITIZER_LINUX
-# if SANITIZER_ANDROID
+#if SANITIZER_ANDROID
static atomic_uint8_t android_log_initialized;
void AndroidLogInit() {
@@ -745,7 +786,7 @@ void SetAbortMessage(const char *str) {
if (&android_set_abort_message)
android_set_abort_message(str);
}
-# else
+#else
void AndroidLogInit() {}
static bool ShouldLogAfterPrintf() { return true; }
@@ -753,7 +794,7 @@ static bool ShouldLogAfterPrintf() { return true; }
void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
void SetAbortMessage(const char *str) {}
-# endif // SANITIZER_ANDROID
+#endif // SANITIZER_ANDROID
void LogMessageOnPrintf(const char *str) {
if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
@@ -768,7 +809,7 @@ void LogMessageOnPrintf(const char *str) {
// initialized after the vDSO function pointers, so if it exists, is not null
// and is not empty, we can use clock_gettime.
extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
-INLINE bool CanUseVDSO() {
+inline bool CanUseVDSO() {
// Bionic is safe, it checks for the vDSO function pointers to be initialized.
if (SANITIZER_ANDROID)
return true;
@@ -803,7 +844,6 @@ u64 MonotonicNanoTime() {
}
#endif // SANITIZER_LINUX && !SANITIZER_GO
-#if !SANITIZER_OPENBSD
void ReExec() {
const char *pathname = "/proc/self/exe";
@@ -835,7 +875,48 @@ void ReExec() {
Printf("execve failed, errno %d\n", rverrno);
Die();
}
-#endif // !SANITIZER_OPENBSD
+
+void UnmapFromTo(uptr from, uptr to) {
+ if (to == from)
+ return;
+ CHECK(to >= from);
+ uptr res = internal_munmap(reinterpret_cast(from), to - from);
+ if (UNLIKELY(internal_iserror(res))) {
+ Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
+ SanitizerToolName, to - from, to - from, (void *)from);
+ CHECK("unable to unmap" && 0);
+ }
+}
+
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment,
+ UNUSED uptr &high_mem_end) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr alignment =
+ Max(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
+ const uptr left_padding =
+ Max(granularity, 1ULL << min_shadow_base_alignment);
+
+ const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
+ const uptr map_size = shadow_size + left_padding + alignment;
+
+ const uptr map_start = (uptr)MmapNoAccess(map_size);
+ CHECK_NE(map_start, ~(uptr)0);
+
+ const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+
+ UnmapFromTo(map_start, shadow_start - left_padding);
+ UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
+
+ return shadow_start;
+}
+
+void InitializePlatformCommonFlags(CommonFlags *cf) {
+#if SANITIZER_ANDROID
+ if (&__libc_get_static_tls_bounds == nullptr)
+ cf->detect_leaks = false;
+#endif
+}
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp
index 9e3b4f13a4365..bb2f5b5f9f7df 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp
@@ -15,14 +15,15 @@
#if SANITIZER_LINUX && SANITIZER_S390
-#include "sanitizer_libc.h"
-#include "sanitizer_linux.h"
-
+#include
#include
#include
#include
#include
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+
namespace __sanitizer {
// --------------- sanitizer_libc.h
@@ -123,7 +124,7 @@ static bool FixedCVE_2016_2143() {
struct utsname buf;
unsigned int major, minor, patch = 0;
// This should never fail, but just in case...
- if (uname(&buf))
+ if (internal_uname(&buf))
return false;
const char *ptr = buf.release;
major = internal_simple_strtoll(ptr, &ptr, 10);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index ea4bd02aa92e4..2b53d7d730d7e 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -27,9 +27,9 @@
#include "sanitizer_flags.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
-#include "sanitizer_placement_new.h"
#include "sanitizer_platform_limits_posix.h"
#include "sanitizer_procmaps.h"
+#include "sanitizer_ptrauth.h"
#if !SANITIZER_IOS
#include // for _NSGetEnviron
@@ -137,6 +137,10 @@ int internal_mprotect(void *addr, uptr length, int prot) {
return mprotect(addr, length, prot);
}
+int internal_madvise(uptr addr, uptr length, int advice) {
+ return madvise((void *)addr, length, advice);
+}
+
uptr internal_close(fd_t fd) {
return close(fd);
}
@@ -208,6 +212,10 @@ uptr internal_getpid() {
return getpid();
}
+int internal_dlinfo(void *handle, int request, void *p) {
+ UNIMPLEMENTED();
+}
+
int internal_sigaction(int signum, const void *act, void *oldact) {
return sigaction(signum,
(const struct sigaction *)act, (struct sigaction *)oldact);
@@ -242,7 +250,8 @@ int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
(size_t)newlen);
}
-static fd_t internal_spawn_impl(const char *argv[], pid_t *pid) {
+static fd_t internal_spawn_impl(const char *argv[], const char *envp[],
+ pid_t *pid) {
fd_t master_fd = kInvalidFd;
fd_t slave_fd = kInvalidFd;
@@ -298,8 +307,8 @@ static fd_t internal_spawn_impl(const char *argv[], pid_t *pid) {
// posix_spawn
char **argv_casted = const_cast(argv);
- char **env = GetEnviron();
- res = posix_spawn(pid, argv[0], &acts, &attrs, argv_casted, env);
+ char **envp_casted = const_cast(envp);
+ res = posix_spawn(pid, argv[0], &acts, &attrs, argv_casted, envp_casted);
if (res != 0) return kInvalidFd;
// Disable echo in the new terminal, disable CR.
@@ -316,7 +325,7 @@ static fd_t internal_spawn_impl(const char *argv[], pid_t *pid) {
return fd;
}
-fd_t internal_spawn(const char *argv[], pid_t *pid) {
+fd_t internal_spawn(const char *argv[], const char *envp[], pid_t *pid) {
// The client program may close its stdin and/or stdout and/or stderr thus
// allowing open/posix_openpt to reuse file descriptors 0, 1 or 2. In this
// case the communication is broken if either the parent or the child tries to
@@ -331,7 +340,7 @@ fd_t internal_spawn(const char *argv[], pid_t *pid) {
break;
}
- fd_t fd = internal_spawn_impl(argv, pid);
+ fd_t fd = internal_spawn_impl(argv, envp, pid);
for (; count > 0; count--) {
internal_close(low_fds[count]);
@@ -382,7 +391,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
// pthread_get_stacksize_np() returns an incorrect stack size for the main
// thread on Mavericks. See
// https://github.com/google/sanitizers/issues/261
- if ((GetMacosVersion() >= MACOS_VERSION_MAVERICKS) && at_initialization &&
+ if ((GetMacosAlignedVersion() >= MacosVersion(10, 9)) && at_initialization &&
stacksize == (1 << 19)) {
struct rlimit rl;
CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
@@ -601,68 +610,132 @@ HandleSignalMode GetHandleSignalMode(int signum) {
return result;
}
-MacosVersion cached_macos_version = MACOS_VERSION_UNINITIALIZED;
+// Offset example:
+// XNU 17 -- macOS 10.13 -- iOS 11 -- tvOS 11 -- watchOS 4
+constexpr u16 GetOSMajorKernelOffset() {
+ if (TARGET_OS_OSX) return 4;
+ if (TARGET_OS_IOS || TARGET_OS_TV) return 6;
+ if (TARGET_OS_WATCH) return 13;
+}
+
+using VersStr = char[64];
+
+static uptr ApproximateOSVersionViaKernelVersion(VersStr vers) {
+ u16 kernel_major = GetDarwinKernelVersion().major;
+ u16 offset = GetOSMajorKernelOffset();
+ CHECK_GE(kernel_major, offset);
+ u16 os_major = kernel_major - offset;
+
+ const char *format = "%d.0";
+ if (TARGET_OS_OSX) {
+ if (os_major >= 16) { // macOS 11+
+ os_major -= 5;
+ } else { // macOS 10.15 and below
+ format = "10.%d";
+ }
+ }
+ return internal_snprintf(vers, sizeof(VersStr), format, os_major);
+}
-MacosVersion GetMacosVersionInternal() {
- int mib[2] = { CTL_KERN, KERN_OSRELEASE };
- char version[100];
- uptr len = 0, maxlen = sizeof(version) / sizeof(version[0]);
- for (uptr i = 0; i < maxlen; i++) version[i] = '\0';
- // Get the version length.
- CHECK_NE(internal_sysctl(mib, 2, 0, &len, 0, 0), -1);
- CHECK_LT(len, maxlen);
- CHECK_NE(internal_sysctl(mib, 2, version, &len, 0, 0), -1);
+static void GetOSVersion(VersStr vers) {
+ uptr len = sizeof(VersStr);
+ if (SANITIZER_IOSSIM) {
+ const char *vers_env = GetEnv("SIMULATOR_RUNTIME_VERSION");
+ if (!vers_env) {
+ Report("ERROR: Running in simulator but SIMULATOR_RUNTIME_VERSION env "
+ "var is not set.\n");
+ Die();
+ }
+ len = internal_strlcpy(vers, vers_env, len);
+ } else {
+ int res =
+ internal_sysctlbyname("kern.osproductversion", vers, &len, nullptr, 0);
+
+ // XNU 17 (macOS 10.13) and below do not provide the sysctl
+ // `kern.osproductversion` entry (res != 0).
+ bool no_os_version = res != 0;
+
+ // For launchd, sanitizer initialization runs before sysctl is setup
+ // (res == 0 && len != strlen(vers), vers is not a valid version). However,
+ // the kernel version `kern.osrelease` is available.
+ bool launchd = (res == 0 && internal_strlen(vers) < 3);
+ if (launchd) CHECK_EQ(internal_getpid(), 1);
+
+ if (no_os_version || launchd) {
+ len = ApproximateOSVersionViaKernelVersion(vers);
+ }
+ }
+ CHECK_LT(len, sizeof(VersStr));
+}
- // Expect .(.)
- CHECK_GE(len, 3);
- const char *p = version;
- int major = internal_simple_strtoll(p, &p, /*base=*/10);
- if (*p != '.') return MACOS_VERSION_UNKNOWN;
+void ParseVersion(const char *vers, u16 *major, u16 *minor) {
+ // Format: .[.]\0
+ CHECK_GE(internal_strlen(vers), 3);
+ const char *p = vers;
+ *major = internal_simple_strtoll(p, &p, /*base=*/10);
+ CHECK_EQ(*p, '.');
p += 1;
- int minor = internal_simple_strtoll(p, &p, /*base=*/10);
- if (*p != '.') return MACOS_VERSION_UNKNOWN;
-
- switch (major) {
- case 9: return MACOS_VERSION_LEOPARD;
- case 10: return MACOS_VERSION_SNOW_LEOPARD;
- case 11: return MACOS_VERSION_LION;
- case 12: return MACOS_VERSION_MOUNTAIN_LION;
- case 13: return MACOS_VERSION_MAVERICKS;
- case 14: return MACOS_VERSION_YOSEMITE;
- case 15: return MACOS_VERSION_EL_CAPITAN;
- case 16: return MACOS_VERSION_SIERRA;
- case 17:
- // Not a typo, 17.5 Darwin Kernel Version maps to High Sierra 10.13.4.
- if (minor >= 5)
- return MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4;
- return MACOS_VERSION_HIGH_SIERRA;
- case 18: return MACOS_VERSION_MOJAVE;
- case 19: return MACOS_VERSION_CATALINA;
- default:
- if (major < 9) return MACOS_VERSION_UNKNOWN;
- return MACOS_VERSION_UNKNOWN_NEWER;
+ *minor = internal_simple_strtoll(p, &p, /*base=*/10);
+}
+
+// Aligned versions example:
+// macOS 10.15 -- iOS 13 -- tvOS 13 -- watchOS 6
+static void MapToMacos(u16 *major, u16 *minor) {
+ if (TARGET_OS_OSX)
+ return;
+
+ if (TARGET_OS_IOS || TARGET_OS_TV)
+ *major += 2;
+ else if (TARGET_OS_WATCH)
+ *major += 9;
+ else
+ UNREACHABLE("unsupported platform");
+
+ if (*major >= 16) { // macOS 11+
+ *major -= 5;
+ } else { // macOS 10.15 and below
+ *minor = *major;
+ *major = 10;
}
}
-MacosVersion GetMacosVersion() {
- atomic_uint32_t *cache =
- reinterpret_cast(&cached_macos_version);
- MacosVersion result =
- static_cast(atomic_load(cache, memory_order_acquire));
- if (result == MACOS_VERSION_UNINITIALIZED) {
- result = GetMacosVersionInternal();
- atomic_store(cache, result, memory_order_release);
+static MacosVersion GetMacosAlignedVersionInternal() {
+ VersStr vers = {};
+ GetOSVersion(vers);
+
+ u16 major, minor;
+ ParseVersion(vers, &major, &minor);
+ MapToMacos(&major, &minor);
+
+ return MacosVersion(major, minor);
+}
+
+static_assert(sizeof(MacosVersion) == sizeof(atomic_uint32_t::Type),
+ "MacosVersion cache size");
+static atomic_uint32_t cached_macos_version;
+
+MacosVersion GetMacosAlignedVersion() {
+ atomic_uint32_t::Type result =
+ atomic_load(&cached_macos_version, memory_order_acquire);
+ if (!result) {
+ MacosVersion version = GetMacosAlignedVersionInternal();
+ result = *reinterpret_cast(&version);
+ atomic_store(&cached_macos_version, result, memory_order_release);
}
- return result;
+ return *reinterpret_cast(&result);
}
-bool PlatformHasDifferentMemcpyAndMemmove() {
- // On OS X 10.7 memcpy() and memmove() are both resolved
- // into memmove$VARIANT$sse42.
- // See also https://github.com/google/sanitizers/issues/34.
- // TODO(glider): need to check dynamically that memcpy() and memmove() are
- // actually the same function.
- return GetMacosVersion() == MACOS_VERSION_SNOW_LEOPARD;
+DarwinKernelVersion GetDarwinKernelVersion() {
+ VersStr vers = {};
+ uptr len = sizeof(VersStr);
+ int res = internal_sysctlbyname("kern.osrelease", vers, &len, nullptr, 0);
+ CHECK_EQ(res, 0);
+ CHECK_LT(len, sizeof(VersStr));
+
+ u16 major, minor;
+ ParseVersion(vers, &major, &minor);
+
+ return DarwinKernelVersion(major, minor);
}
uptr GetRSS() {
@@ -677,13 +750,13 @@ uptr GetRSS() {
return info.resident_size;
}
-void *internal_start_thread(void(*func)(void *arg), void *arg) {
+void *internal_start_thread(void *(*func)(void *arg), void *arg) {
// Start the thread with signals blocked, otherwise it can steal user signals.
__sanitizer_sigset_t set, old;
internal_sigfillset(&set);
internal_sigprocmask(SIG_SETMASK, &set, &old);
pthread_t th;
- pthread_create(&th, 0, (void*(*)(void *arg))func, arg);
+ pthread_create(&th, 0, func, arg);
internal_sigprocmask(SIG_SETMASK, &old, 0);
return th;
}
@@ -711,7 +784,7 @@ void LogFullErrorReport(const char *buffer) {
#if !SANITIZER_GO
// Log with os_trace. This will make it into the crash log.
#if SANITIZER_OS_TRACE
- if (GetMacosVersion() >= MACOS_VERSION_YOSEMITE) {
+ if (GetMacosAlignedVersion() >= MacosVersion(10, 10)) {
// os_trace requires the message (format parameter) to be a string literal.
if (internal_strncmp(SanitizerToolName, "AddressSanitizer",
sizeof("AddressSanitizer") - 1) == 0)
@@ -760,16 +833,24 @@ bool SignalContext::IsTrueFaultingAddress() const {
return si->si_signo == SIGSEGV && si->si_code != 0;
}
+#if defined(__aarch64__) && defined(arm_thread_state64_get_sp)
+ #define AARCH64_GET_REG(r) \
+ (uptr)ptrauth_strip( \
+ (void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)
+#else
+ #define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r
+#endif
+
static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
ucontext_t *ucontext = (ucontext_t*)context;
# if defined(__aarch64__)
- *pc = ucontext->uc_mcontext->__ss.__pc;
+ *pc = AARCH64_GET_REG(pc);
# if defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0
- *bp = ucontext->uc_mcontext->__ss.__fp;
+ *bp = AARCH64_GET_REG(fp);
# else
- *bp = ucontext->uc_mcontext->__ss.__lr;
+ *bp = AARCH64_GET_REG(lr);
# endif
- *sp = ucontext->uc_mcontext->__ss.__sp;
+ *sp = AARCH64_GET_REG(sp);
# elif defined(__x86_64__)
*pc = ucontext->uc_mcontext->__ss.__rip;
*bp = ucontext->uc_mcontext->__ss.__rbp;
@@ -787,16 +868,34 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
# endif
}
-void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
+void SignalContext::InitPcSpBp() {
+ addr = (uptr)ptrauth_strip((void *)addr, 0);
+ GetPcSpBp(context, &pc, &sp, &bp);
+}
+
+// ASan/TSan use mmap in a way that creates “deallocation gaps” which triggers
+// EXC_GUARD exceptions on macOS 10.15+ (XNU 19.0+).
+static void DisableMmapExcGuardExceptions() {
+ using task_exc_guard_behavior_t = uint32_t;
+ using task_set_exc_guard_behavior_t =
+ kern_return_t(task_t task, task_exc_guard_behavior_t behavior);
+ auto *set_behavior = (task_set_exc_guard_behavior_t *)dlsym(
+ RTLD_DEFAULT, "task_set_exc_guard_behavior");
+ if (set_behavior == nullptr) return;
+ const task_exc_guard_behavior_t task_exc_guard_none = 0;
+ set_behavior(mach_task_self(), task_exc_guard_none);
+}
void InitializePlatformEarly() {
- // Only use xnu_fast_mmap when on x86_64 and the OS supports it.
+ // Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
use_xnu_fast_mmap =
#if defined(__x86_64__)
- GetMacosVersion() >= MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4;
+ GetDarwinKernelVersion() >= DarwinKernelVersion(17, 5);
#else
false;
#endif
+ if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))
+ DisableMmapExcGuardExceptions();
}
#if !SANITIZER_GO
@@ -837,20 +936,10 @@ bool ReexecDisabled() {
return false;
}
-extern "C" SANITIZER_WEAK_ATTRIBUTE double dyldVersionNumber;
-static const double kMinDyldVersionWithAutoInterposition = 360.0;
-
-bool DyldNeedsEnvVariable() {
- // Although sanitizer support was added to LLVM on OS X 10.7+, GCC users
- // still may want use them on older systems. On older Darwin platforms, dyld
- // doesn't export dyldVersionNumber symbol and we simply return true.
- if (!&dyldVersionNumber) return true;
+static bool DyldNeedsEnvVariable() {
// If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
- // DYLD_INSERT_LIBRARIES is not set. However, checking OS version via
- // GetMacosVersion() doesn't work for the simulator. Let's instead check
- // `dyldVersionNumber`, which is exported by dyld, against a known version
- // number from the first OS release where this appeared.
- return dyldVersionNumber < kMinDyldVersionWithAutoInterposition;
+ // DYLD_INSERT_LIBRARIES is not set.
+ return GetMacosAlignedVersion() < MacosVersion(10, 11);
}
void MaybeReexec() {
@@ -996,7 +1085,7 @@ char **GetArgv() {
return *_NSGetArgv();
}
-#if SANITIZER_IOS
+#if SANITIZER_IOS && !SANITIZER_IOSSIM
// The task_vm_info struct is normally provided by the macOS SDK, but we need
// fields only available in 10.12+. Declare the struct manually to be able to
// build against older SDKs.
@@ -1063,6 +1152,53 @@ uptr GetMaxVirtualAddress() {
return GetMaxUserVirtualAddress();
}
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment, uptr &high_mem_end) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr alignment =
+ Max(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
+ const uptr left_padding =
+ Max(granularity, 1ULL << min_shadow_base_alignment);
+
+ uptr space_size = shadow_size_bytes + left_padding;
+
+ uptr largest_gap_found = 0;
+ uptr max_occupied_addr = 0;
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ uptr shadow_start =
+ FindAvailableMemoryRange(space_size, alignment, granularity,
+ &largest_gap_found, &max_occupied_addr);
+ // If the shadow doesn't fit, restrict the address space to make it fit.
+ if (shadow_start == 0) {
+ VReport(
+ 2,
+ "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
+ largest_gap_found, max_occupied_addr);
+ uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);
+ if (new_max_vm < max_occupied_addr) {
+ Report("Unable to find a memory range for dynamic shadow.\n");
+ Report(
+ "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
+ "new_max_vm = %p\n",
+ space_size, largest_gap_found, max_occupied_addr, new_max_vm);
+ CHECK(0 && "cannot place shadow");
+ }
+ RestrictMemoryToMaxAddress(new_max_vm);
+ high_mem_end = new_max_vm - 1;
+ space_size = (high_mem_end >> shadow_scale) + left_padding;
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
+ nullptr, nullptr);
+ if (shadow_start == 0) {
+ Report("Unable to find a memory range after restricting VM.\n");
+ CHECK(0 && "cannot place shadow after restricting vm");
+ }
+ }
+ CHECK_NE((uptr)0, shadow_start);
+ CHECK(IsAligned(shadow_start, alignment));
+ return shadow_start;
+}
+
uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
uptr *largest_gap_found,
uptr *max_occupied_addr) {
@@ -1123,6 +1259,8 @@ void SignalContext::DumpAllRegisters(void *context) {
ucontext_t *ucontext = (ucontext_t*)context;
# define DUMPREG64(r) \
Printf("%s = 0x%016llx ", #r, ucontext->uc_mcontext->__ss.__ ## r);
+# define DUMPREGA64(r) \
+ Printf(" %s = 0x%016llx ", #r, AARCH64_GET_REG(r));
# define DUMPREG32(r) \
Printf("%s = 0x%08x ", #r, ucontext->uc_mcontext->__ss.__ ## r);
# define DUMPREG_(r) Printf(" "); DUMPREG(r);
@@ -1148,7 +1286,7 @@ void SignalContext::DumpAllRegisters(void *context) {
DUMPREG(x[16]); DUMPREG(x[17]); DUMPREG(x[18]); DUMPREG(x[19]); Printf("\n");
DUMPREG(x[20]); DUMPREG(x[21]); DUMPREG(x[22]); DUMPREG(x[23]); Printf("\n");
DUMPREG(x[24]); DUMPREG(x[25]); DUMPREG(x[26]); DUMPREG(x[27]); Printf("\n");
- DUMPREG(x[28]); DUMPREG___(fp); DUMPREG___(lr); DUMPREG___(sp); Printf("\n");
+ DUMPREG(x[28]); DUMPREGA64(fp); DUMPREGA64(lr); DUMPREGA64(sp); Printf("\n");
# elif defined(__arm__)
# define DUMPREG(r) DUMPREG32(r)
DUMPREG_(r[0]); DUMPREG_(r[1]); DUMPREG_(r[2]); DUMPREG_(r[3]); Printf("\n");
@@ -1181,7 +1319,7 @@ void FormatUUID(char *out, uptr size, const u8 *uuid) {
uuid[12], uuid[13], uuid[14], uuid[15]);
}
-void PrintModuleMap() {
+void DumpProcessMap() {
Printf("Process module map:\n");
MemoryMappingLayout memory_mapping(false);
InternalMmapVector modules;
@@ -1214,6 +1352,8 @@ u32 GetNumberOfCPUs() {
return (u32)sysconf(_SC_NPROCESSORS_ONLN);
}
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
+
} // namespace __sanitizer
#endif // SANITIZER_MAC
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
index 2257883084ea9..023071e4f11de 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
@@ -30,25 +30,33 @@ struct MemoryMappingLayoutData {
bool current_instrumented;
};
-enum MacosVersion {
- MACOS_VERSION_UNINITIALIZED = 0,
- MACOS_VERSION_UNKNOWN,
- MACOS_VERSION_LEOPARD,
- MACOS_VERSION_SNOW_LEOPARD,
- MACOS_VERSION_LION,
- MACOS_VERSION_MOUNTAIN_LION,
- MACOS_VERSION_MAVERICKS,
- MACOS_VERSION_YOSEMITE,
- MACOS_VERSION_EL_CAPITAN,
- MACOS_VERSION_SIERRA,
- MACOS_VERSION_HIGH_SIERRA,
- MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4,
- MACOS_VERSION_MOJAVE,
- MACOS_VERSION_CATALINA,
- MACOS_VERSION_UNKNOWN_NEWER
+template
+struct VersionBase {
+ u16 major;
+ u16 minor;
+
+ VersionBase(u16 major, u16 minor) : major(major), minor(minor) {}
+
+ bool operator==(const VersionType &other) const {
+ return major == other.major && minor == other.minor;
+ }
+ bool operator>=(const VersionType &other) const {
+ return major > other.major ||
+ (major == other.major && minor >= other.minor);
+ }
+ bool operator<(const VersionType &other) const { return !(*this >= other); }
+};
+
+struct MacosVersion : VersionBase {
+ MacosVersion(u16 major, u16 minor) : VersionBase(major, minor) {}
+};
+
+struct DarwinKernelVersion : VersionBase {
+ DarwinKernelVersion(u16 major, u16 minor) : VersionBase(major, minor) {}
};
-MacosVersion GetMacosVersion();
+MacosVersion GetMacosAlignedVersion();
+DarwinKernelVersion GetDarwinKernelVersion();
char **GetEnviron();
@@ -67,7 +75,7 @@ asm(".desc ___crashreporter_info__, 0x10");
namespace __sanitizer {
static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);
-INLINE void CRAppendCrashLogMessage(const char *msg) {
+inline void CRAppendCrashLogMessage(const char *msg) {
BlockingMutexLock l(&crashreporter_info_mutex);
internal_strlcat(__crashreporter_info_buff__, msg,
sizeof(__crashreporter_info_buff__)); }
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
index 11adbe5c25b4b..647bcdfe105e6 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
@@ -61,12 +61,10 @@ INTERCEPTOR(malloc_zone_t *, malloc_create_zone,
malloc_zone_t *new_zone = (malloc_zone_t *)p;
internal_memcpy(new_zone, &sanitizer_zone, sizeof(sanitizer_zone));
new_zone->zone_name = NULL; // The name will be changed anyway.
- if (GetMacosVersion() >= MACOS_VERSION_LION) {
- // Prevent the client app from overwriting the zone contents.
- // Library functions that need to modify the zone will set PROT_WRITE on it.
- // This matches the behavior of malloc_create_zone() on OSX 10.7 and higher.
- mprotect(new_zone, allocated_size, PROT_READ);
- }
+ // Prevent the client app from overwriting the zone contents.
+ // Library functions that need to modify the zone will set PROT_WRITE on it.
+ // This matches the behavior of malloc_create_zone() on OSX 10.7 and higher.
+ mprotect(new_zone, allocated_size, PROT_READ);
// We're explicitly *NOT* registering the zone.
return new_zone;
}
@@ -75,11 +73,9 @@ INTERCEPTOR(void, malloc_destroy_zone, malloc_zone_t *zone) {
COMMON_MALLOC_ENTER();
// We don't need to do anything here. We're not registering new zones, so we
// don't to unregister. Just un-mprotect and free() the zone.
- if (GetMacosVersion() >= MACOS_VERSION_LION) {
- uptr page_size = GetPageSizeCached();
- uptr allocated_size = RoundUpTo(sizeof(sanitizer_zone), page_size);
- mprotect(zone, allocated_size, PROT_READ | PROT_WRITE);
- }
+ uptr page_size = GetPageSizeCached();
+ uptr allocated_size = RoundUpTo(sizeof(sanitizer_zone), page_size);
+ mprotect(zone, allocated_size, PROT_READ | PROT_WRITE);
if (zone->zone_name) {
COMMON_MALLOC_FREE((void *)zone->zone_name);
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp
index 49a951e04b374..98ac7365da051 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp
@@ -110,6 +110,11 @@ int internal_mprotect(void *addr, uptr length, int prot) {
return _REAL(mprotect, addr, length, prot);
}
+int internal_madvise(uptr addr, uptr length, int advice) {
+ DEFINE__REAL(int, madvise, void *a, uptr b, int c);
+ return _REAL(madvise, (void *)addr, length, advice);
+}
+
uptr internal_close(fd_t fd) {
CHECK(&_sys_close);
return _sys_close(fd);
@@ -265,6 +270,11 @@ uptr internal_getppid() {
return _REAL(getppid);
}
+int internal_dlinfo(void *handle, int request, void *p) {
+ DEFINE__REAL(int, dlinfo, void *a, int b, void *c);
+ return _REAL(dlinfo, handle, request, p);
+}
+
uptr internal_getdents(fd_t fd, void *dirp, unsigned int count) {
DEFINE__REAL(int, __getdents30, int a, void *b, size_t c);
return _REAL(__getdents30, fd, dirp, count);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cpp
index ed2d8edeb7a2c..e69de29bb2d1d 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cpp
@@ -1,115 +0,0 @@
-//===-- sanitizer_openbsd.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is shared between various sanitizers' runtime libraries and
-// implements Solaris-specific functions.
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_platform.h"
-#if SANITIZER_OPENBSD
-
-#include
-
-#include "sanitizer_common.h"
-#include "sanitizer_flags.h"
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_placement_new.h"
-#include "sanitizer_platform_limits_posix.h"
-#include "sanitizer_procmaps.h"
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include