diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index 3770a05230f..73a0213dd02 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -38,6 +38,158 @@ # xargs -n1 | sort | uniq -d; # done +# 20190304: new libc++ import which bumps version from 7.0.1 to 8.0.0. +OLD_FILES+=usr/include/c++/v1/experimental/dynarray +# 20190304: new clang import which bumps version from 7.0.1 to 8.0.0. +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/allocator_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/asan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/common_interface_defs.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/coverage_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/dfsan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/esan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/hwasan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/linux_syscall_hooks.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/lsan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/msan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/netbsd_syscall_hooks.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/scudo_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/tsan_interface.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/tsan_interface_atomic.h +OLD_DIRS+=usr/lib/clang/7.0.1/include/sanitizer +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_builtin_vars.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_cmath.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_complex_builtins.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_device_functions.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_intrinsics.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_libdevice_declares.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_math_forward_declares.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_runtime_wrapper.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__stddef_max_align_t.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__wmmintrin_aes.h +OLD_FILES+=usr/lib/clang/7.0.1/include/__wmmintrin_pclmul.h +OLD_FILES+=usr/lib/clang/7.0.1/include/adxintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/altivec.h +OLD_FILES+=usr/lib/clang/7.0.1/include/ammintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/arm64intr.h +OLD_FILES+=usr/lib/clang/7.0.1/include/arm_acle.h +OLD_FILES+=usr/lib/clang/7.0.1/include/arm_fp16.h +OLD_FILES+=usr/lib/clang/7.0.1/include/arm_neon.h +OLD_FILES+=usr/lib/clang/7.0.1/include/armintr.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx2intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512bitalgintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512bwintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512cdintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512dqintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512erintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512fintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512ifmaintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512ifmavlintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512pfintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmi2intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmiintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmivlintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlbitalgintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlbwintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlcdintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vldqintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlvbmi2intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlvnniintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vnniintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vpopcntdqintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vpopcntdqvlintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/avxintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/bmi2intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/bmiintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/cetintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/cldemoteintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/clflushoptintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/clwbintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/clzerointrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/cpuid.h +OLD_FILES+=usr/lib/clang/7.0.1/include/emmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/f16cintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/fma4intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/fmaintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/fxsrintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/gfniintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/htmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/htmxlintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/ia32intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/immintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/invpcidintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/lwpintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/lzcntintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/mm3dnow.h +OLD_FILES+=usr/lib/clang/7.0.1/include/mm_malloc.h +OLD_FILES+=usr/lib/clang/7.0.1/include/mmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/module.modulemap +OLD_FILES+=usr/lib/clang/7.0.1/include/movdirintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/msa.h +OLD_FILES+=usr/lib/clang/7.0.1/include/mwaitxintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/nmmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/opencl-c.h +OLD_FILES+=usr/lib/clang/7.0.1/include/pconfigintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/pkuintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/pmmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/popcntintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/prfchwintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/ptwriteintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/rdseedintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/rtmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/s390intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/sgxintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/shaintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/smmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/tbmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/tmmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/vadefs.h +OLD_FILES+=usr/lib/clang/7.0.1/include/vaesintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/vecintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/vpclmulqdqintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/waitpkgintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/wbnoinvdintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/wmmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/x86intrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xmmintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xopintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xsavecintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xsaveintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xsaveoptintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xsavesintrin.h +OLD_FILES+=usr/lib/clang/7.0.1/include/xtestintrin.h +OLD_DIRS+=usr/lib/clang/7.0.1/include +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-i386.so +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-x86_64.so +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.msan-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-arm.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-armhf.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.safestack-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats_client-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a +OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a +OLD_DIRS+=usr/lib/clang/7.0.1/lib/freebsd +OLD_DIRS+=usr/lib/clang/7.0.1/lib +OLD_DIRS+=usr/lib/clang/7.0.1 # 20190227: rename seq.h to seqc.h OLD_FILES+=usr/include/sys/seq.h # 20190222: libifconfig made INTERNALLIB diff --git a/UPDATING b/UPDATING index 9f55df6cc24..c84bbc79f09 100644 --- a/UPDATING +++ b/UPDATING @@ -31,6 +31,12 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 13.x IS SLOW: disable the most expensive debugging functionality run "ln -s 'abort:false,junk:false' /etc/malloc.conf".) +20190304: + Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to + 8.0.0. Please see the 20141231 entry below for information about + prerequisites and upgrading, if you are not already using clang 3.5.0 + or higher. + 20190219: drm and drm2 have been removed from the tree. Please see https://wiki.freebsd.org/Graphics for the latest information on diff --git a/contrib/compiler-rt/LICENSE.TXT b/contrib/compiler-rt/LICENSE.TXT index 0134694e4e5..1c94ad5d891 100644 --- a/contrib/compiler-rt/LICENSE.TXT +++ b/contrib/compiler-rt/LICENSE.TXT @@ -14,7 +14,7 @@ Full text of the relevant licenses is included below. University of Illinois/NCSA Open Source License -Copyright (c) 2009-2018 by the contributors listed in CREDITS.TXT +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT All rights reserved. diff --git a/contrib/compiler-rt/include/sanitizer/allocator_interface.h b/contrib/compiler-rt/include/sanitizer/allocator_interface.h index 89f328301db..e44c4a13ed6 100644 --- a/contrib/compiler-rt/include/sanitizer/allocator_interface.h +++ b/contrib/compiler-rt/include/sanitizer/allocator_interface.h @@ -82,7 +82,6 @@ extern "C" { Currently available with ASan only. */ void __sanitizer_purge_allocator(void); - #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h index d11cb1addc2..bf015eb237c 100644 --- a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h +++ b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h @@ -124,6 +124,12 @@ extern "C" { // Symbolizes the supplied 'pc' using the format string 'fmt'. // Outputs at most 'out_buf_size' bytes into 'out_buf'. + // If 'out_buf' is not empty then output is zero or more non empty C strings + // followed by single empty C string. Multiple strings can be returned if PC + // corresponds to inlined function. Inlined frames are printed in the order + // from "most-inlined" to the "least-inlined", so the last frame should be the + // not inlined function. + // Inlined frames can be removed with 'symbolize_inline_frames=0'. // The format syntax is described in // lib/sanitizer_common/sanitizer_stacktrace_printer.h. void __sanitizer_symbolize_pc(void *pc, const char *fmt, char *out_buf, diff --git a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h index 0c306cf279e..1affd875a11 100644 --- a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h @@ -19,6 +19,12 @@ #ifdef __cplusplus extern "C" { #endif + // Initialize shadow but not the rest of the runtime. + // Does not call libc unless there is an error. + // Can be called multiple times, or not at all (in which case shadow will + // be initialized in compiler-inserted __hwasan_init() call). + void __hwasan_shadow_init(void); + // This function may be optionally provided by user and should return // a string containing HWASan runtime options. See asan_flags.h for details. const char* __hwasan_default_options(void); @@ -26,6 +32,51 @@ extern "C" { void __hwasan_enable_allocator_tagging(void); void __hwasan_disable_allocator_tagging(void); + // Mark region of memory with the given tag. Both address and size need to be + // 16-byte aligned. + void __hwasan_tag_memory(const volatile void *p, unsigned char tag, + size_t size); + + /// Set pointer tag. Previous tag is lost. + void *__hwasan_tag_pointer(const volatile void *p, unsigned char tag); + + // Set memory tag from the current SP address to the given address to zero. + // This is meant to annotate longjmp and other non-local jumps. + // This function needs to know the (almost) exact destination frame address; + // clearing shadow for the entire thread stack like __asan_handle_no_return + // does would cause false reports. + void __hwasan_handle_longjmp(const void *sp_dst); + + // Libc hook for thread creation. Should be called in the child thread before + // any instrumented code. + void __hwasan_thread_enter(); + + // Libc hook for thread destruction. No instrumented code should run after + // this call. + void __hwasan_thread_exit(); + + // Print shadow and origin for the memory range to stderr in a human-readable + // format. + void __hwasan_print_shadow(const volatile void *x, size_t size); + + // Print one-line report about the memory usage of the current process. + void __hwasan_print_memory_usage(); + + int __sanitizer_posix_memalign(void **memptr, size_t alignment, size_t size); + void * __sanitizer_memalign(size_t alignment, size_t size); + void * __sanitizer_aligned_alloc(size_t alignment, size_t size); + void * __sanitizer___libc_memalign(size_t alignment, size_t size); + void * __sanitizer_valloc(size_t size); + void * __sanitizer_pvalloc(size_t size); + void __sanitizer_free(void *ptr); + void __sanitizer_cfree(void *ptr); + size_t __sanitizer_malloc_usable_size(const void *ptr); + struct mallinfo __sanitizer_mallinfo(); + int __sanitizer_mallopt(int cmd, int value); + void __sanitizer_malloc_stats(void); + void * __sanitizer_calloc(size_t nmemb, size_t size); + void * __sanitizer_realloc(void *ptr, size_t size); + void * __sanitizer_malloc(size_t size); #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h b/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h index 4c6c6a88af2..b69f3d2be39 100644 --- a/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h +++ b/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h @@ -21,8 +21,8 @@ // DO NOT EDIT! THIS FILE HAS BEEN GENERATED! // // Generated with: generate_netbsd_syscalls.awk -// Generated date: 2018-03-03 -// Generated from: syscalls.master,v 1.291 2018/01/06 16:41:23 kamil Exp +// Generated date: 2018-10-30 +// Generated from: syscalls.master,v 1.293 2018/07/31 13:00:13 rjs Exp // //===----------------------------------------------------------------------===// #ifndef SANITIZER_NETBSD_SYSCALL_HOOKS_H @@ -986,7 +986,15 @@ #define __sanitizer_syscall_post_fpathconf(res, fd, name) \ __sanitizer_syscall_post_impl_fpathconf(res, (long long)(fd), \ (long long)(name)) -/* syscall 193 has been skipped */ +#define __sanitizer_syscall_pre_getsockopt2(s, level, name, val, avalsize) \ + __sanitizer_syscall_pre_impl_getsockopt2( \ + (long long)(s), (long long)(level), (long long)(name), (long long)(val), \ + (long long)(avalsize)) +#define __sanitizer_syscall_post_getsockopt2(res, s, level, name, val, \ + avalsize) \ + __sanitizer_syscall_post_impl_getsockopt2( \ + res, (long long)(s), (long long)(level), (long long)(name), \ + (long long)(val), (long long)(avalsize)) #define __sanitizer_syscall_pre_getrlimit(which, rlp) \ __sanitizer_syscall_pre_impl_getrlimit((long long)(which), (long long)(rlp)) #define __sanitizer_syscall_post_getrlimit(res, which, rlp) \ @@ -1752,18 +1760,8 @@ __sanitizer_syscall_post_impl___sigaction_sigtramp( \ res, (long long)(signum), (long long)(nsa), (long long)(osa), \ (long long)(tramp), (long long)(vers)) -#define __sanitizer_syscall_pre_pmc_get_info(ctr, op, args) \ - __sanitizer_syscall_pre_impl_pmc_get_info((long long)(ctr), (long long)(op), \ - (long long)(args)) -#define __sanitizer_syscall_post_pmc_get_info(res, ctr, op, args) \ - __sanitizer_syscall_post_impl_pmc_get_info( \ - res, (long long)(ctr), (long long)(op), (long long)(args)) -#define __sanitizer_syscall_pre_pmc_control(ctr, op, args) \ - __sanitizer_syscall_pre_impl_pmc_control((long long)(ctr), (long long)(op), \ - (long long)(args)) -#define __sanitizer_syscall_post_pmc_control(res, ctr, op, args) \ - __sanitizer_syscall_post_impl_pmc_control( \ - res, (long long)(ctr), (long long)(op), (long long)(args)) +/* syscall 341 has been skipped */ +/* syscall 342 has been skipped */ #define __sanitizer_syscall_pre_rasctl(addr, len, op) \ __sanitizer_syscall_pre_impl_rasctl((long long)(addr), (long long)(len), \ (long long)(op)) @@ -3444,7 +3442,13 @@ void __sanitizer_syscall_post_impl_pathconf(long long res, long long path, void __sanitizer_syscall_pre_impl_fpathconf(long long fd, long long name); void __sanitizer_syscall_post_impl_fpathconf(long long res, long long fd, long long name); -/* syscall 193 has been skipped */ +void __sanitizer_syscall_pre_impl_getsockopt2(long long s, long long level, + long long name, long long val, + long long avalsize); +void __sanitizer_syscall_post_impl_getsockopt2(long long res, long long s, + long long level, long long name, + long long val, + long long avalsize); void __sanitizer_syscall_pre_impl_getrlimit(long long which, long long rlp); void __sanitizer_syscall_post_impl_getrlimit(long long res, long long which, long long rlp); @@ -4001,14 +4005,8 @@ void __sanitizer_syscall_pre_impl___sigaction_sigtramp(long long signum, void __sanitizer_syscall_post_impl___sigaction_sigtramp( long long res, long long signum, long long nsa, long long osa, long long tramp, long long vers); -void __sanitizer_syscall_pre_impl_pmc_get_info(long long ctr, long long op, - long long args); -void __sanitizer_syscall_post_impl_pmc_get_info(long long res, long long ctr, - long long op, long long args); -void __sanitizer_syscall_pre_impl_pmc_control(long long ctr, long long op, - long long args); -void __sanitizer_syscall_post_impl_pmc_control(long long res, long long ctr, - long long op, long long args); +/* syscall 341 has been skipped */ +/* syscall 342 has been skipped */ void __sanitizer_syscall_pre_impl_rasctl(long long addr, long long len, long long op); void __sanitizer_syscall_post_impl_rasctl(long long res, long long addr, diff --git a/contrib/compiler-rt/include/xray/xray_log_interface.h b/contrib/compiler-rt/include/xray/xray_log_interface.h index 5f8b3a4a06a..399467860e7 100644 --- a/contrib/compiler-rt/include/xray/xray_log_interface.h +++ b/contrib/compiler-rt/include/xray/xray_log_interface.h @@ -158,8 +158,8 @@ struct XRayLogImpl { /// The log initialization routine provided by the implementation, always /// provided with the following parameters: /// - /// - buffer size - /// - maximum number of buffers + /// - buffer size (unused) + /// - maximum number of buffers (unused) /// - a pointer to an argument struct that the implementation MUST handle /// - the size of the argument struct /// @@ -355,25 +355,4 @@ XRayLogFlushStatus __xray_log_process_buffers(void (*Processor)(const char *, } // extern "C" -namespace __xray { - -/// DEPRECATED: Use __xray_log_init_mode(...) instead, and provide flag -/// configuration strings to set the options instead. -/// Options used by the LLVM XRay FDR logging implementation. -struct FDRLoggingOptions { - bool ReportErrors = false; - int Fd = -1; -}; - -/// DEPRECATED: Use __xray_log_init_mode(...) instead, and provide flag -/// configuration strings to set the options instead. -/// Options used by the LLVM XRay Basic (Naive) logging implementation. -struct BasicLoggingOptions { - int DurationFilterMicros = 0; - size_t MaxStackDepth = 0; - size_t ThreadBufferSize = 0; -}; - -} // namespace __xray - #endif // XRAY_XRAY_LOG_INTERFACE_H diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.h b/contrib/compiler-rt/lib/asan/asan_allocator.h index 93d6f29c5bf..c9b37dc7a6d 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.h +++ b/contrib/compiler-rt/lib/asan/asan_allocator.h @@ -148,6 +148,7 @@ const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. typedef DefaultSizeClassMap SizeClassMap; # endif +template struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; static const uptr kSpaceSize = kAllocatorSize; @@ -155,37 +156,57 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. typedef __asan::SizeClassMap SizeClassMap; typedef AsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; }; -typedef SizeClassAllocator64 PrimaryAllocator; +template +using PrimaryAllocatorASVT = SizeClassAllocator64>; +using PrimaryAllocator = PrimaryAllocatorASVT; #else // Fallback to SizeClassAllocator32. static const uptr kRegionSizeLog = 20; static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; # if SANITIZER_WORDSIZE == 32 -typedef FlatByteMap ByteMap; +template +using ByteMapASVT = FlatByteMap; # elif SANITIZER_WORDSIZE == 64 -typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; +template +using ByteMapASVT = + TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; # endif typedef CompactSizeClassMap SizeClassMap; +template struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 16; typedef __asan::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __asan::kRegionSizeLog; - typedef __asan::ByteMap ByteMap; + using AddressSpaceView = AddressSpaceViewTy; + using ByteMap = __asan::ByteMapASVT; typedef AsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; -typedef SizeClassAllocator32 PrimaryAllocator; +template +using PrimaryAllocatorASVT = SizeClassAllocator32 >; +using PrimaryAllocator = PrimaryAllocatorASVT; #endif // SANITIZER_CAN_USE_ALLOCATOR64 static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; -typedef SizeClassAllocatorLocalCache AllocatorCache; -typedef LargeMmapAllocator SecondaryAllocator; -typedef CombinedAllocator AsanAllocator; +template +using AllocatorCacheASVT = + SizeClassAllocatorLocalCache>; +using AllocatorCache = AllocatorCacheASVT; +template +using SecondaryAllocatorASVT = + LargeMmapAllocator; +template +using AsanAllocatorASVT = + CombinedAllocator, + AllocatorCacheASVT, + SecondaryAllocatorASVT>; +using AsanAllocator = AsanAllocatorASVT; struct AsanThreadLocalMallocStorage { uptr quarantine_cache[16]; diff --git a/contrib/compiler-rt/lib/asan/asan_errors.cc b/contrib/compiler-rt/lib/asan/asan_errors.cc index 33d0613f79f..0ecd30dcacf 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.cc +++ b/contrib/compiler-rt/lib/asan/asan_errors.cc @@ -125,9 +125,8 @@ void ErrorAllocTypeMismatch::Print() { Decorator d; Printf("%s", d.Error()); Report("ERROR: AddressSanitizer: %s (%s vs %s) on %p\n", - scariness.GetDescription(), - alloc_names[alloc_type], dealloc_names[dealloc_type], - addr_description.addr); + scariness.GetDescription(), alloc_names[alloc_type], + dealloc_names[dealloc_type], addr_description.Address()); Printf("%s", d.Default()); CHECK_GT(dealloc_stack->size, 0); scariness.Print(); diff --git a/contrib/compiler-rt/lib/asan/asan_errors.h b/contrib/compiler-rt/lib/asan/asan_errors.h index 574197ebff8..7ddd7e94e0f 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.h +++ b/contrib/compiler-rt/lib/asan/asan_errors.h @@ -110,8 +110,8 @@ struct ErrorFreeNotMalloced : ErrorBase { struct ErrorAllocTypeMismatch : ErrorBase { const BufferedStackTrace *dealloc_stack; - HeapAddressDescription addr_description; AllocType alloc_type, dealloc_type; + AddressDescription addr_description; ErrorAllocTypeMismatch() = default; // (*) ErrorAllocTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr, @@ -119,9 +119,8 @@ struct ErrorAllocTypeMismatch : ErrorBase { : ErrorBase(tid, 10, "alloc-dealloc-mismatch"), dealloc_stack(stack), alloc_type(alloc_type_), - dealloc_type(dealloc_type_) { - GetHeapAddressInformation(addr, 1, &addr_description); - }; + dealloc_type(dealloc_type_), + addr_description(addr, 1, false) {} void Print(); }; diff --git a/contrib/compiler-rt/lib/asan/asan_flags.inc b/contrib/compiler-rt/lib/asan/asan_flags.inc index 4af94c56fca..a9c97d53b31 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.inc +++ b/contrib/compiler-rt/lib/asan/asan_flags.inc @@ -152,8 +152,6 @@ ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.") ASAN_FLAG(bool, halt_on_error, true, "Crash the program after printing the first error report " "(WARNING: USE AT YOUR OWN RISK!)") -ASAN_FLAG(bool, use_odr_indicator, false, - "Use special ODR indicator symbol for ODR violation detection") ASAN_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true, "realloc(p, 0) is equivalent to free(p) by default (Same as the " "POSIX standard). If set to false, realloc(p, 0) will return a " diff --git a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc index 0b5bff4f565..34399c92310 100644 --- a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc +++ b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc @@ -190,6 +190,13 @@ static void ThreadExitHook(void *hook, uptr os_id) { AsanThread::TSDDtor(per_thread); } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + } // namespace __asan // These are declared (in extern "C") by . diff --git a/contrib/compiler-rt/lib/asan/asan_globals.cc b/contrib/compiler-rt/lib/asan/asan_globals.cc index 898f7f40d31..146234ac6c6 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals.cc @@ -83,9 +83,11 @@ static bool IsAddressNearGlobal(uptr addr, const __asan_global &g) { } static void ReportGlobal(const Global &g, const char *prefix) { - Report("%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu\n", - prefix, &g, (void *)g.beg, g.size, g.size_with_redzone, g.name, - g.module_name, g.has_dynamic_init); + Report( + "%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu " + "odr_indicator=%p\n", + prefix, &g, (void *)g.beg, g.size, g.size_with_redzone, g.name, + g.module_name, g.has_dynamic_init, (void *)g.odr_indicator); if (g.location) { Report(" location (%p): name=%s[%p], %d %d\n", g.location, g.location->filename, g.location->filename, g.location->line_no, @@ -133,6 +135,9 @@ enum GlobalSymbolState { // this method in case compiler instruments global variables through their // local aliases. static void CheckODRViolationViaIndicator(const Global *g) { + // Instrumentation requests to skip ODR check. + if (g->odr_indicator == UINTPTR_MAX) + return; u8 *odr_indicator = reinterpret_cast(g->odr_indicator); if (*odr_indicator == UNREGISTERED) { *odr_indicator = REGISTERED; @@ -183,9 +188,7 @@ static void CheckODRViolationViaPoisoning(const Global *g) { // This routine chooses between two different methods of ODR violation // detection. static inline bool UseODRIndicator(const Global *g) { - // Use ODR indicator method iff use_odr_indicator flag is set and - // indicator symbol address is not 0. - return flags()->use_odr_indicator && g->odr_indicator > 0; + return g->odr_indicator > 0; } // Register a global variable. @@ -248,7 +251,7 @@ static void UnregisterGlobal(const Global *g) { // implementation. It might not be worth doing anyway. // Release ODR indicator. - if (UseODRIndicator(g)) { + if (UseODRIndicator(g) && g->odr_indicator != UINTPTR_MAX) { u8 *odr_indicator = reinterpret_cast(g->odr_indicator); *odr_indicator = UNREGISTERED; } diff --git a/contrib/compiler-rt/lib/asan/asan_globals_win.cc b/contrib/compiler-rt/lib/asan/asan_globals_win.cc index 29ab5ebf16d..0e75992bfc8 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals_win.cc @@ -29,7 +29,7 @@ static void call_on_globals(void (*hook)(__asan_global *, uptr)) { __asan_global *end = &__asan_globals_end; uptr bytediff = (uptr)end - (uptr)start; if (bytediff % sizeof(__asan_global) != 0) { -#ifdef SANITIZER_DLL_THUNK +#if defined(SANITIZER_DLL_THUNK) || defined(SANITIZER_DYNAMIC_RUNTIME_THUNK) __debugbreak(); #else CHECK("corrupt asan global array"); diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h index 654878cd15f..57869497c7d 100644 --- a/contrib/compiler-rt/lib/asan/asan_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_internal.h @@ -111,6 +111,11 @@ void *AsanDlSymNext(const char *sym); void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name); +// Returns `true` iff most of ASan init process should be skipped due to the +// ASan library being loaded via `dlopen()`. Platforms may perform any +// `dlopen()` specific initialization inside this function. +bool HandleDlopenInit(); + // Add convenient macro for interface functions that may be represented as // weak hooks. #define ASAN_MALLOC_HOOK(ptr, size) \ diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc index 625f32d408d..a150b1955d6 100644 --- a/contrib/compiler-rt/lib/asan/asan_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_linux.cc @@ -248,6 +248,13 @@ void *AsanDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + } // namespace __asan #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc index 76bdff999d6..0a534fe2116 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc @@ -209,7 +209,7 @@ INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { } INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; + return 0; } #endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc b/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc index 733ba2d86e1..27281f1bc83 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc @@ -61,4 +61,25 @@ using namespace __asan; #include "sanitizer_common/sanitizer_malloc_mac.inc" +namespace COMMON_MALLOC_NAMESPACE { +bool HandleDlopenInit() { + static_assert(SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be true"); + // We have no reliable way of knowing how we are being loaded + // so make it a requirement on Apple platforms to set this environment + // variable to indicate that we want to perform initialization via + // dlopen(). + auto init_str = GetEnv("APPLE_ASAN_INIT_FOR_DLOPEN"); + if (!init_str) + return false; + if (internal_strncmp(init_str, "1", 1) != 0) + return false; + // When we are loaded via `dlopen()` path we still initialize the malloc zone + // so Symbolication clients (e.g. `leaks`) that load the ASan allocator can + // find an initialized malloc zone. + InitMallocZoneFields(); + return true; +} +} // namespace COMMON_MALLOC_NAMESPACE + #endif diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_win.cc b/contrib/compiler-rt/lib/asan/asan_malloc_win.cc index efa05824397..88793643193 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_win.cc @@ -14,8 +14,17 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_WINDOWS -#define WIN32_LEAN_AND_MEAN -#include +// Intentionally not including windows.h here, to avoid the risk of +// pulling in conflicting declarations of these functions. (With mingw-w64, +// there's a risk of windows.h pulling in stdint.h.) +typedef int BOOL; +typedef void *HANDLE; +typedef const void *LPCVOID; +typedef void *LPVOID; + +#define HEAP_ZERO_MEMORY 0x00000008 +#define HEAP_REALLOC_IN_PLACE_ONLY 0x00000010 + #include "asan_allocator.h" #include "asan_interceptors.h" @@ -125,12 +134,17 @@ void *_recalloc_base(void *p, size_t n, size_t elem_size) { } ALLOCATION_FUNCTION_ATTRIBUTE -size_t _msize(const void *ptr) { +size_t _msize(void *ptr) { GET_CURRENT_PC_BP_SP; (void)sp; return asan_malloc_usable_size(ptr, pc, bp); } +ALLOCATION_FUNCTION_ATTRIBUTE +size_t _msize_base(void *ptr) { + return _msize(ptr); +} + ALLOCATION_FUNCTION_ATTRIBUTE void *_expand(void *memblock, size_t size) { // _expand is used in realloc-like functions to resize the buffer if possible. @@ -226,6 +240,7 @@ void ReplaceSystemMalloc() { TryToOverrideFunction("_recalloc_base", (uptr)_recalloc); TryToOverrideFunction("_recalloc_crt", (uptr)_recalloc); TryToOverrideFunction("_msize", (uptr)_msize); + TryToOverrideFunction("_msize_base", (uptr)_msize); TryToOverrideFunction("_expand", (uptr)_expand); TryToOverrideFunction("_expand_base", (uptr)_expand); diff --git a/contrib/compiler-rt/lib/asan/asan_new_delete.cc b/contrib/compiler-rt/lib/asan/asan_new_delete.cc index 30efd61a968..e6053c1fe87 100644 --- a/contrib/compiler-rt/lib/asan/asan_new_delete.cc +++ b/contrib/compiler-rt/lib/asan/asan_new_delete.cc @@ -26,7 +26,7 @@ // anyway by passing extra -export flags to the linker, which is exactly that // dllexport would normally do. We need to export them in order to make the // VS2015 dynamic CRT (MD) work. -#if SANITIZER_WINDOWS +#if SANITIZER_WINDOWS && defined(_MSC_VER) #define CXX_OPERATOR_ATTRIBUTE #define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:" sym)) #ifdef _WIN64 diff --git a/contrib/compiler-rt/lib/asan/asan_posix.cc b/contrib/compiler-rt/lib/asan/asan_posix.cc index 17c28b0aea2..ca99c04b3ad 100644 --- a/contrib/compiler-rt/lib/asan/asan_posix.cc +++ b/contrib/compiler-rt/lib/asan/asan_posix.cc @@ -40,6 +40,51 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { // ---------------------- TSD ---------------- {{{1 +#if SANITIZER_NETBSD || SANITIZER_FREEBSD +// Thread Static Data cannot be used in early init on NetBSD and FreeBSD. +// Reuse the Asan TSD API for compatibility with existing code +// with an alternative implementation. + +static void (*tsd_destructor)(void *tsd) = nullptr; + +struct tsd_key { + tsd_key() : key(nullptr) {} + ~tsd_key() { + CHECK(tsd_destructor); + if (key) + (*tsd_destructor)(key); + } + void *key; +}; + +static thread_local struct tsd_key key; + +void AsanTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_destructor); + tsd_destructor = destructor; +} + +void *AsanTSDGet() { + CHECK(tsd_destructor); + return key.key; +} + +void AsanTSDSet(void *tsd) { + CHECK(tsd_destructor); + CHECK(tsd); + CHECK(!key.key); + key.key = tsd; +} + +void PlatformTSDDtor(void *tsd) { + CHECK(tsd_destructor); + CHECK_EQ(key.key, tsd); + key.key = nullptr; + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + AsanThread::TSDDtor(tsd); +} +#else static pthread_key_t tsd_key; static bool tsd_key_inited = false; void AsanTSDInit(void (*destructor)(void *tsd)) { @@ -67,6 +112,7 @@ void PlatformTSDDtor(void *tsd) { } AsanThread::TSDDtor(tsd); } +#endif } // namespace __asan #endif // SANITIZER_POSIX diff --git a/contrib/compiler-rt/lib/asan/asan_report.h b/contrib/compiler-rt/lib/asan/asan_report.h index f7153d4810d..b0c167dda75 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.h +++ b/contrib/compiler-rt/lib/asan/asan_report.h @@ -12,6 +12,9 @@ // ASan-private header for error reporting functions. //===----------------------------------------------------------------------===// +#ifndef ASAN_REPORT_H +#define ASAN_REPORT_H + #include "asan_allocator.h" #include "asan_internal.h" #include "asan_thread.h" @@ -92,3 +95,4 @@ void ReportMacCfReallocUnknown(uptr addr, uptr zone_ptr, BufferedStackTrace *stack); } // namespace __asan +#endif // ASAN_REPORT_H diff --git a/contrib/compiler-rt/lib/asan/asan_rtems.cc b/contrib/compiler-rt/lib/asan/asan_rtems.cc index a4af940057e..b48cc6a75d7 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtems.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtems.cc @@ -213,6 +213,12 @@ static void HandleExit() { } } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} } // namespace __asan // These are declared (in extern "C") by . diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc index 4cff736f213..13344f3b888 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtl.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc @@ -383,6 +383,19 @@ void PrintAddressSpaceLayout() { kHighShadowBeg > kMidMemEnd); } +#if defined(__thumb__) && defined(__linux__) +#define START_BACKGROUND_THREAD_IN_ASAN_INTERNAL +#endif + +#ifndef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL +static bool UNUSED __local_asan_dyninit = [] { + MaybeStartBackgroudThread(); + SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); + + return false; +}(); +#endif + static void AsanInitInternal() { if (LIKELY(asan_inited)) return; SanitizerToolName = "AddressSanitizer"; @@ -396,6 +409,14 @@ static void AsanInitInternal() { // initialization steps look at flags(). InitializeFlags(); + // Stop performing init at this point if we are being loaded via + // dlopen() and the platform supports it. + if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { + asan_init_is_running = false; + VReport(1, "AddressSanitizer init is being performed for dlopen().\n"); + return; + } + AsanCheckIncompatibleRT(); AsanCheckDynamicRTPrereqs(); AvoidCVE_2016_2143(); @@ -420,6 +441,8 @@ static void AsanInitInternal() { __asan_option_detect_stack_use_after_return = flags()->detect_stack_use_after_return; + __sanitizer::InitializePlatformEarly(); + // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec(); @@ -447,8 +470,10 @@ static void AsanInitInternal() { allocator_options.SetFrom(flags(), common_flags()); InitializeAllocator(allocator_options); +#ifdef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL MaybeStartBackgroudThread(); SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); +#endif // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited // should be set to 1 prior to initializing the threads. diff --git a/contrib/compiler-rt/lib/asan/asan_thread.cc b/contrib/compiler-rt/lib/asan/asan_thread.cc index faf423d305b..0895e4ce0d9 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.cc +++ b/contrib/compiler-rt/lib/asan/asan_thread.cc @@ -223,9 +223,11 @@ void AsanThread::Init(const InitOptions *options) { atomic_store(&stack_switching_, false, memory_order_release); CHECK_EQ(this->stack_size(), 0U); SetThreadStackAndTls(options); - CHECK_GT(this->stack_size(), 0U); - CHECK(AddrIsInMem(stack_bottom_)); - CHECK(AddrIsInMem(stack_top_ - 1)); + if (stack_top_ != stack_bottom_) { + CHECK_GT(this->stack_size(), 0U); + CHECK(AddrIsInMem(stack_bottom_)); + CHECK(AddrIsInMem(stack_top_ - 1)); + } ClearShadowForThreadStackAndTLS(); fake_stack_ = nullptr; if (__asan_option_detect_stack_use_after_return) @@ -289,20 +291,23 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) { DCHECK_EQ(options, nullptr); uptr tls_size = 0; uptr stack_size = 0; - GetThreadStackAndTls(tid() == 0, const_cast(&stack_bottom_), - const_cast(&stack_size), &tls_begin_, &tls_size); + GetThreadStackAndTls(tid() == 0, &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); stack_top_ = stack_bottom_ + stack_size; tls_end_ = tls_begin_ + tls_size; dtls_ = DTLS_Get(); - int local; - CHECK(AddrIsInStack((uptr)&local)); + if (stack_top_ != stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + } } #endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS void AsanThread::ClearShadowForThreadStackAndTLS() { - PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); + if (stack_top_ != stack_bottom_) + PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) { uptr tls_begin_aligned = RoundDownTo(tls_begin_, SHADOW_GRANULARITY); uptr tls_end_aligned = RoundUpTo(tls_end_, SHADOW_GRANULARITY); @@ -314,6 +319,9 @@ void AsanThread::ClearShadowForThreadStackAndTLS() { bool AsanThread::GetStackFrameAccessByAddr(uptr addr, StackFrameAccess *access) { + if (stack_top_ == stack_bottom_) + return false; + uptr bottom = 0; if (AddrIsInStack(addr)) { bottom = stack_bottom(); diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc index 67125d38ad4..068f4a5d247 100644 --- a/contrib/compiler-rt/lib/asan/asan_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_win.cc @@ -159,6 +159,14 @@ INTERCEPTOR_WINAPI(DWORD, CreateThread, namespace __asan { void InitializePlatformInterceptors() { + // The interceptors were not designed to be removable, so we have to keep this + // module alive for the life of the process. + HMODULE pinned; + CHECK(GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_PIN, + (LPCWSTR)&InitializePlatformInterceptors, + &pinned)); + ASAN_INTERCEPT_FUNC(CreateThread); ASAN_INTERCEPT_FUNC(SetUnhandledExceptionFilter); @@ -314,6 +322,13 @@ int __asan_set_seh_filter() { return 0; } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + #if !ASAN_DYNAMIC // The CRT runs initializers in this order: // - C initializers, from XIA to XIZ diff --git a/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc b/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc index c6a313d2402..df593ab92de 100644 --- a/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc +++ b/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc @@ -48,6 +48,7 @@ INTERCEPT_WRAP_W_WWW(_recalloc) INTERCEPT_WRAP_W_WWW(_recalloc_base) INTERCEPT_WRAP_W_W(_msize) +INTERCEPT_WRAP_W_W(_msize_base) INTERCEPT_WRAP_W_W(_expand) INTERCEPT_WRAP_W_W(_expand_dbg) diff --git a/contrib/compiler-rt/lib/builtins/arm/addsf3.S b/contrib/compiler-rt/lib/builtins/arm/addsf3.S index 362b5c147ea..74723cbeff7 100644 --- a/contrib/compiler-rt/lib/builtins/arm/addsf3.S +++ b/contrib/compiler-rt/lib/builtins/arm/addsf3.S @@ -178,7 +178,7 @@ LOCAL_LABEL(do_substraction): push {r0, r1, r2, r3} movs r0, r4 - bl __clzsi2 + bl SYMBOL_NAME(__clzsi2) movs r5, r0 pop {r0, r1, r2, r3} // shift = rep_clz(aSignificand) - rep_clz(implicitBit << 3); diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S index 87dd03dce94..adc2d55d90f 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S @@ -55,7 +55,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) mov ip, #APSR_C msr APSR_nzcvq, ip #else - msr CPSR_f, #APSR_C + msr APSR_nzcvq, #APSR_C #endif JMP(lr) #endif @@ -115,11 +115,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple) movne ip, #(APSR_C) 1: -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) msr APSR_nzcvq, ip -#else - msr CPSR_f, ip -#endif pop {r0-r3} POP_PC() #endif diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S index c5fee6b6a08..4b1de997687 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S @@ -55,7 +55,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) mov ip, #APSR_C msr APSR_nzcvq, ip #else - msr CPSR_f, #APSR_C + msr APSR_nzcvq, #APSR_C #endif JMP(lr) #endif @@ -115,11 +115,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple) movne ip, #(APSR_C) 1: -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) msr APSR_nzcvq, ip -#else - msr CPSR_f, ip -#endif pop {r0-r3} POP_PC() #endif diff --git a/contrib/compiler-rt/lib/builtins/clzdi2.c b/contrib/compiler-rt/lib/builtins/clzdi2.c index b56d98f5c01..1819e6be436 100644 --- a/contrib/compiler-rt/lib/builtins/clzdi2.c +++ b/contrib/compiler-rt/lib/builtins/clzdi2.c @@ -16,8 +16,13 @@ /* Returns: the number of leading 0-bits */ -#if !defined(__clang__) && (defined(__sparc64__) || defined(__mips64) || defined(__riscv__)) -/* gcc resolves __builtin_clz -> __clzdi2 leading to infinite recursion */ +#if !defined(__clang__) && \ + ((defined(__sparc__) && defined(__arch64__)) || \ + defined(__mips64) || \ + (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) +/* On 64-bit architectures with neither a native clz instruction nor a native + * ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than + * __clzsi2, leading to infinite recursion. */ #define __builtin_clz(a) __clzsi2(a) extern si_int __clzsi2(si_int); #endif diff --git a/contrib/compiler-rt/lib/builtins/cpu_model.c b/contrib/compiler-rt/lib/builtins/cpu_model.c index 43b913390fc..fb2b899fc70 100644 --- a/contrib/compiler-rt/lib/builtins/cpu_model.c +++ b/contrib/compiler-rt/lib/builtins/cpu_model.c @@ -55,6 +55,9 @@ enum ProcessorTypes { AMD_BTVER2, AMDFAM17H, INTEL_KNM, + INTEL_GOLDMONT, + INTEL_GOLDMONT_PLUS, + INTEL_TREMONT, CPU_TYPE_MAX }; @@ -76,6 +79,8 @@ enum ProcessorSubtypes { INTEL_COREI7_SKYLAKE, INTEL_COREI7_SKYLAKE_AVX512, INTEL_COREI7_CANNONLAKE, + INTEL_COREI7_ICELAKE_CLIENT, + INTEL_COREI7_ICELAKE_SERVER, CPU_SUBTYPE_MAX }; @@ -110,7 +115,12 @@ enum ProcessorFeatures { FEATURE_AVX512IFMA, FEATURE_AVX5124VNNIW, FEATURE_AVX5124FMAPS, - FEATURE_AVX512VPOPCNTDQ + FEATURE_AVX512VPOPCNTDQ, + FEATURE_AVX512VBMI2, + FEATURE_GFNI, + FEATURE_VPCLMULQDQ, + FEATURE_AVX512VNNI, + FEATURE_AVX512BITALG }; // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max). @@ -364,6 +374,14 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, case 0x4c: // really airmont *Type = INTEL_SILVERMONT; break; // "silvermont" + // Goldmont: + case 0x5c: // Apollo Lake + case 0x5f: // Denverton + *Type = INTEL_GOLDMONT; + break; // "goldmont" + case 0x7a: + *Type = INTEL_GOLDMONT_PLUS; + break; case 0x57: *Type = INTEL_KNL; // knl @@ -438,35 +456,45 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, - unsigned *FeaturesOut) { + unsigned *FeaturesOut, + unsigned *Features2Out) { unsigned Features = 0; + unsigned Features2 = 0; unsigned EAX, EBX; +#define setFeature(F) \ + do { \ + if (F < 32) \ + Features |= 1U << (F & 0x1f); \ + else if (F < 64) \ + Features2 |= 1U << ((F - 32) & 0x1f); \ + } while (0) + if ((EDX >> 15) & 1) - Features |= 1 << FEATURE_CMOV; + setFeature(FEATURE_CMOV); if ((EDX >> 23) & 1) - Features |= 1 << FEATURE_MMX; + setFeature(FEATURE_MMX); if ((EDX >> 25) & 1) - Features |= 1 << FEATURE_SSE; + setFeature(FEATURE_SSE); if ((EDX >> 26) & 1) - Features |= 1 << FEATURE_SSE2; + setFeature(FEATURE_SSE2); if ((ECX >> 0) & 1) - Features |= 1 << FEATURE_SSE3; + setFeature(FEATURE_SSE3); if ((ECX >> 1) & 1) - Features |= 1 << FEATURE_PCLMUL; + setFeature(FEATURE_PCLMUL); if ((ECX >> 9) & 1) - Features |= 1 << FEATURE_SSSE3; + setFeature(FEATURE_SSSE3); if ((ECX >> 12) & 1) - Features |= 1 << FEATURE_FMA; + setFeature(FEATURE_FMA); if ((ECX >> 19) & 1) - Features |= 1 << FEATURE_SSE4_1; + setFeature(FEATURE_SSE4_1); if ((ECX >> 20) & 1) - Features |= 1 << FEATURE_SSE4_2; + setFeature(FEATURE_SSE4_2); if ((ECX >> 23) & 1) - Features |= 1 << FEATURE_POPCNT; + setFeature(FEATURE_POPCNT); if ((ECX >> 25) & 1) - Features |= 1 << FEATURE_AES; + setFeature(FEATURE_AES); // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV // indicates that the AVX registers will be saved and restored on context @@ -477,43 +505,53 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0); if (HasAVX) - Features |= 1 << FEATURE_AVX; + setFeature(FEATURE_AVX); bool HasLeaf7 = MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); if (HasLeaf7 && ((EBX >> 3) & 1)) - Features |= 1 << FEATURE_BMI; + setFeature(FEATURE_BMI); if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX) - Features |= 1 << FEATURE_AVX2; + setFeature(FEATURE_AVX2); if (HasLeaf7 && ((EBX >> 9) & 1)) - Features |= 1 << FEATURE_BMI2; + setFeature(FEATURE_BMI2); if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512F; + setFeature(FEATURE_AVX512F); if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512DQ; + setFeature(FEATURE_AVX512DQ); if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512IFMA; + setFeature(FEATURE_AVX512IFMA); if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512PF; + setFeature(FEATURE_AVX512PF); if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512ER; + setFeature(FEATURE_AVX512ER); if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512CD; + setFeature(FEATURE_AVX512CD); if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512BW; + setFeature(FEATURE_AVX512BW); if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512VL; + setFeature(FEATURE_AVX512VL); if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512VBMI; + setFeature(FEATURE_AVX512VBMI); + if (HasLeaf7 && ((ECX >> 6) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VBMI2); + if (HasLeaf7 && ((ECX >> 8) & 1)) + setFeature(FEATURE_GFNI); + if (HasLeaf7 && ((ECX >> 10) & 1) && HasAVX) + setFeature(FEATURE_VPCLMULQDQ); + if (HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VNNI); + if (HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512BITALG); if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512VPOPCNTDQ; + setFeature(FEATURE_AVX512VPOPCNTDQ); if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX5124VNNIW; + setFeature(FEATURE_AVX5124VNNIW); if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX5124FMAPS; + setFeature(FEATURE_AVX5124FMAPS); unsigned MaxExtLevel; getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX); @@ -521,13 +559,15 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 && !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); if (HasExtLeaf1 && ((ECX >> 6) & 1)) - Features |= 1 << FEATURE_SSE4_A; + setFeature(FEATURE_SSE4_A); if (HasExtLeaf1 && ((ECX >> 11) & 1)) - Features |= 1 << FEATURE_XOP; + setFeature(FEATURE_XOP); if (HasExtLeaf1 && ((ECX >> 16) & 1)) - Features |= 1 << FEATURE_FMA4; + setFeature(FEATURE_FMA4); *FeaturesOut = Features; + *Features2Out = Features2; +#undef setFeature } #if defined(HAVE_INIT_PRIORITY) @@ -548,8 +588,9 @@ struct __processor_model { unsigned int __cpu_subtype; unsigned int __cpu_features[1]; } __cpu_model = {0, 0, 0, {0}}; +unsigned int __cpu_features2; -/* A constructor function that is sets __cpu_model and __cpu_features with +/* A constructor function that is sets __cpu_model and __cpu_features2 with the right values. This needs to run only once. This constructor is given the highest priority and it should run before constructors without the priority set. However, it still runs after ifunc initializers and @@ -562,6 +603,7 @@ __cpu_indicator_init(void) { unsigned Vendor; unsigned Model, Family, Brand_id; unsigned Features = 0; + unsigned Features2 = 0; /* This function needs to run just once. */ if (__cpu_model.__cpu_vendor) @@ -580,8 +622,9 @@ __cpu_indicator_init(void) { Brand_id = EBX & 0xff; /* Find available features. */ - getAvailableFeatures(ECX, EDX, MaxLeaf, &Features); + getAvailableFeatures(ECX, EDX, MaxLeaf, &Features, &Features2); __cpu_model.__cpu_features[0] = Features; + __cpu_features2 = Features2; if (Vendor == SIG_INTEL) { /* Get CPU type. */ diff --git a/contrib/compiler-rt/lib/builtins/ctzdi2.c b/contrib/compiler-rt/lib/builtins/ctzdi2.c index eecde29718d..ef6d7fea136 100644 --- a/contrib/compiler-rt/lib/builtins/ctzdi2.c +++ b/contrib/compiler-rt/lib/builtins/ctzdi2.c @@ -16,8 +16,13 @@ /* Returns: the number of trailing 0-bits */ -#if !defined(__clang__) && (defined(__sparc64__) || defined(__mips64) || defined(__riscv__)) -/* gcc resolves __builtin_ctz -> __ctzdi2 leading to infinite recursion */ +#if !defined(__clang__) && \ + ((defined(__sparc__) && defined(__arch64__)) || \ + defined(__mips64) || \ + (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) +/* On 64-bit architectures with neither a native clz instruction nor a native + * ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than + * __ctzsi2, leading to infinite recursion. */ #define __builtin_ctz(a) __ctzsi2(a) extern si_int __ctzsi2(si_int); #endif diff --git a/contrib/compiler-rt/lib/builtins/divdc3.c b/contrib/compiler-rt/lib/builtins/divdc3.c index 3c88390b5e7..392d6ecacde 100644 --- a/contrib/compiler-rt/lib/builtins/divdc3.c +++ b/contrib/compiler-rt/lib/builtins/divdc3.c @@ -12,6 +12,8 @@ * ===----------------------------------------------------------------------=== */ +#define DOUBLE_PRECISION +#include "fp_lib.h" #include "int_lib.h" #include "int_math.h" @@ -21,7 +23,7 @@ COMPILER_RT_ABI Dcomplex __divdc3(double __a, double __b, double __c, double __d) { int __ilogbw = 0; - double __logbw = crt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); + double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); if (crt_isfinite(__logbw)) { __ilogbw = (int)__logbw; diff --git a/contrib/compiler-rt/lib/builtins/divdf3.c b/contrib/compiler-rt/lib/builtins/divdf3.c index 04a4dc5571c..411c82ebb87 100644 --- a/contrib/compiler-rt/lib/builtins/divdf3.c +++ b/contrib/compiler-rt/lib/builtins/divdf3.c @@ -21,36 +21,36 @@ COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) { - + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - + rep_t aSignificand = toRep(a) & significandMask; rep_t bSignificand = toRep(b) & significandMask; int scale = 0; - + // Detect if a or b is zero, denormal, infinity, or NaN. if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - + const rep_t aAbs = toRep(a) & absMask; const rep_t bAbs = toRep(b) & absMask; - + // NaN / anything = qNaN if (aAbs > infRep) return fromRep(toRep(a) | quietBit); // anything / NaN = qNaN if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - + if (aAbs == infRep) { // infinity / infinity = NaN if (bAbs == infRep) return fromRep(qnanRep); // infinity / anything else = +/- infinity else return fromRep(aAbs | quotientSign); } - + // anything else / infinity = +/- 0 if (bAbs == infRep) return fromRep(quotientSign); - + if (!aAbs) { // zero / zero = NaN if (!bAbs) return fromRep(qnanRep); @@ -59,28 +59,28 @@ __divdf3(fp_t a, fp_t b) { } // anything else / zero = +/- infinity if (!bAbs) return fromRep(infRep | quotientSign); - + // one or both of a or b is denormal, the other (if applicable) is a // normal number. Renormalize one or both of a and b, and set scale to // include the necessary exponent adjustment. if (aAbs < implicitBit) scale += normalize(&aSignificand); if (bAbs < implicitBit) scale -= normalize(&bSignificand); } - + // Or in the implicit significand bit. (If we fell through from the // denormal path it was already set by normalize( ), but setting it twice // won't hurt anything.) aSignificand |= implicitBit; bSignificand |= implicitBit; int quotientExponent = aExponent - bExponent + scale; - + // Align the significand of b as a Q31 fixed-point number in the range // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This // is accurate to about 3.5 binary digits. const uint32_t q31b = bSignificand >> 21; uint32_t recip32 = UINT32_C(0x7504f333) - q31b; - + // Now refine the reciprocal estimate using a Newton-Raphson iteration: // // x1 = x0 * (2 - x0 * b) @@ -95,13 +95,13 @@ __divdf3(fp_t a, fp_t b) { recip32 = (uint64_t)recip32 * correction32 >> 31; correction32 = -((uint64_t)recip32 * q31b >> 32); recip32 = (uint64_t)recip32 * correction32 >> 31; - + // recip32 might have overflowed to exactly zero in the preceding // computation if the high word of b is exactly 1.0. This would sabotage // the full-width final stage of the computation that follows, so we adjust // recip32 downward by one bit. recip32--; - + // We need to perform one more iteration to get us to 56 binary digits; // The last iteration needs to happen with extra precision. const uint32_t q63blo = bSignificand << 11; @@ -110,14 +110,14 @@ __divdf3(fp_t a, fp_t b) { uint32_t cHi = correction >> 32; uint32_t cLo = correction; reciprocal = (uint64_t)recip32*cHi + ((uint64_t)recip32*cLo >> 32); - + // We already adjusted the 32-bit estimate, now we need to adjust the final // 64-bit reciprocal estimate downward to ensure that it is strictly smaller // than the infinitely precise exact reciprocal. Because the computation // of the Newton-Raphson step is truncating at every step, this adjustment // is small; most of the work is already done. reciprocal -= 2; - + // The numerical reciprocal is accurate to within 2^-56, lies in the // interval [0.5, 1.0), and is strictly smaller than the true reciprocal // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b @@ -127,12 +127,12 @@ __divdf3(fp_t a, fp_t b) { // 2. q is in the interval [0.5, 2.0) // 3. the error in q is bounded away from 2^-53 (actually, we have a // couple of bits to spare, but this is all we need). - + // We need a 64 x 64 multiply high to compute q, which isn't a basic // operation in C, so we need to be a little bit fussy. rep_t quotient, quotientLo; wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); - + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). // In either case, we are going to compute a residual of the form // @@ -141,7 +141,7 @@ __divdf3(fp_t a, fp_t b) { // We know from the construction of q that r satisfies: // // 0 <= r < ulp(q)*b - // + // // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we // already have the correct result. The exact halfway case cannot occur. // We also take this time to right shift quotient if it falls in the [1,2) @@ -154,20 +154,20 @@ __divdf3(fp_t a, fp_t b) { quotient >>= 1; residual = (aSignificand << 52) - quotient * bSignificand; } - + const int writtenExponent = quotientExponent + exponentBias; - + if (writtenExponent >= maxExponent) { // If we have overflowed the exponent, return infinity. return fromRep(infRep | quotientSign); } - + else if (writtenExponent < 1) { // Flush denormals to zero. In the future, it would be nice to add // code to round them correctly. return fromRep(quotientSign); } - + else { const bool round = (residual << 1) > bSignificand; // Clear the implicit bit diff --git a/contrib/compiler-rt/lib/builtins/divsc3.c b/contrib/compiler-rt/lib/builtins/divsc3.c index 42a48315e66..0d18a256c3d 100644 --- a/contrib/compiler-rt/lib/builtins/divsc3.c +++ b/contrib/compiler-rt/lib/builtins/divsc3.c @@ -12,6 +12,8 @@ *===----------------------------------------------------------------------=== */ +#define SINGLE_PRECISION +#include "fp_lib.h" #include "int_lib.h" #include "int_math.h" @@ -21,7 +23,8 @@ COMPILER_RT_ABI Fcomplex __divsc3(float __a, float __b, float __c, float __d) { int __ilogbw = 0; - float __logbw = crt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); + float __logbw = + __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); if (crt_isfinite(__logbw)) { __ilogbw = (int)__logbw; diff --git a/contrib/compiler-rt/lib/builtins/divsf3.c b/contrib/compiler-rt/lib/builtins/divsf3.c index 65294d70fc6..a74917fd1de 100644 --- a/contrib/compiler-rt/lib/builtins/divsf3.c +++ b/contrib/compiler-rt/lib/builtins/divsf3.c @@ -21,36 +21,36 @@ COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) { - + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - + rep_t aSignificand = toRep(a) & significandMask; rep_t bSignificand = toRep(b) & significandMask; int scale = 0; - + // Detect if a or b is zero, denormal, infinity, or NaN. if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - + const rep_t aAbs = toRep(a) & absMask; const rep_t bAbs = toRep(b) & absMask; - + // NaN / anything = qNaN if (aAbs > infRep) return fromRep(toRep(a) | quietBit); // anything / NaN = qNaN if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - + if (aAbs == infRep) { // infinity / infinity = NaN if (bAbs == infRep) return fromRep(qnanRep); // infinity / anything else = +/- infinity else return fromRep(aAbs | quotientSign); } - + // anything else / infinity = +/- 0 if (bAbs == infRep) return fromRep(quotientSign); - + if (!aAbs) { // zero / zero = NaN if (!bAbs) return fromRep(qnanRep); @@ -59,28 +59,28 @@ __divsf3(fp_t a, fp_t b) { } // anything else / zero = +/- infinity if (!bAbs) return fromRep(infRep | quotientSign); - + // one or both of a or b is denormal, the other (if applicable) is a // normal number. Renormalize one or both of a and b, and set scale to // include the necessary exponent adjustment. if (aAbs < implicitBit) scale += normalize(&aSignificand); if (bAbs < implicitBit) scale -= normalize(&bSignificand); } - + // Or in the implicit significand bit. (If we fell through from the // denormal path it was already set by normalize( ), but setting it twice // won't hurt anything.) aSignificand |= implicitBit; bSignificand |= implicitBit; int quotientExponent = aExponent - bExponent + scale; - + // Align the significand of b as a Q31 fixed-point number in the range // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This // is accurate to about 3.5 binary digits. uint32_t q31b = bSignificand << 8; uint32_t reciprocal = UINT32_C(0x7504f333) - q31b; - + // Now refine the reciprocal estimate using a Newton-Raphson iteration: // // x1 = x0 * (2 - x0 * b) @@ -95,7 +95,7 @@ __divsf3(fp_t a, fp_t b) { reciprocal = (uint64_t)reciprocal * correction >> 31; correction = -((uint64_t)reciprocal * q31b >> 32); reciprocal = (uint64_t)reciprocal * correction >> 31; - + // Exhaustive testing shows that the error in reciprocal after three steps // is in the interval [-0x1.f58108p-31, 0x1.d0e48cp-29], in line with our // expectations. We bump the reciprocal by a tiny value to force the error @@ -103,7 +103,7 @@ __divsf3(fp_t a, fp_t b) { // be specific). This also causes 1/1 to give a sensible approximation // instead of zero (due to overflow). reciprocal -= 2; - + // The numerical reciprocal is accurate to within 2^-28, lies in the // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller // than the true reciprocal of b. Multiplying a by this reciprocal thus @@ -115,9 +115,9 @@ __divsf3(fp_t a, fp_t b) { // from the fact that we truncate the product, and the 2^27 term // is the error in the reciprocal of b scaled by the maximum // possible value of a. As a consequence of this error bound, - // either q or nextafter(q) is the correctly rounded + // either q or nextafter(q) is the correctly rounded rep_t quotient = (uint64_t)reciprocal*(aSignificand << 1) >> 32; - + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). // In either case, we are going to compute a residual of the form // @@ -126,7 +126,7 @@ __divsf3(fp_t a, fp_t b) { // We know from the construction of q that r satisfies: // // 0 <= r < ulp(q)*b - // + // // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we // already have the correct result. The exact halfway case cannot occur. // We also take this time to right shift quotient if it falls in the [1,2) @@ -141,18 +141,18 @@ __divsf3(fp_t a, fp_t b) { } const int writtenExponent = quotientExponent + exponentBias; - + if (writtenExponent >= maxExponent) { // If we have overflowed the exponent, return infinity. return fromRep(infRep | quotientSign); } - + else if (writtenExponent < 1) { // Flush denormals to zero. In the future, it would be nice to add // code to round them correctly. return fromRep(quotientSign); } - + else { const bool round = (residual << 1) > bSignificand; // Clear the implicit bit diff --git a/contrib/compiler-rt/lib/builtins/divtc3.c b/contrib/compiler-rt/lib/builtins/divtc3.c index 16e538ba4a3..e5ea00d841e 100644 --- a/contrib/compiler-rt/lib/builtins/divtc3.c +++ b/contrib/compiler-rt/lib/builtins/divtc3.c @@ -12,6 +12,8 @@ *===----------------------------------------------------------------------=== */ +#define QUAD_PRECISION +#include "fp_lib.h" #include "int_lib.h" #include "int_math.h" @@ -21,7 +23,8 @@ COMPILER_RT_ABI Lcomplex __divtc3(long double __a, long double __b, long double __c, long double __d) { int __ilogbw = 0; - long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + long double __logbw = + __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); if (crt_isfinite(__logbw)) { __ilogbw = (int)__logbw; diff --git a/contrib/compiler-rt/lib/builtins/emutls.c b/contrib/compiler-rt/lib/builtins/emutls.c index 07d436e267d..ef95a1c260c 100644 --- a/contrib/compiler-rt/lib/builtins/emutls.c +++ b/contrib/compiler-rt/lib/builtins/emutls.c @@ -42,6 +42,7 @@ static void emutls_shutdown(emutls_address_array *array); static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_key_t emutls_pthread_key; +static bool emutls_key_created = false; typedef unsigned int gcc_word __attribute__((mode(word))); typedef unsigned int gcc_pointer __attribute__((mode(pointer))); @@ -109,6 +110,7 @@ static void emutls_key_destructor(void* ptr) { static __inline void emutls_init(void) { if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) abort(); + emutls_key_created = true; } static __inline void emutls_init_once(void) { @@ -390,3 +392,14 @@ void* __emutls_get_address(__emutls_control* control) { array->data[index] = emutls_allocate_object(control); return array->data[index]; } + +#ifdef __BIONIC__ +/* Called by Bionic on dlclose to delete the emutls pthread key. */ +__attribute__((visibility("hidden"))) +void __emutls_unregister_key(void) { + if (emutls_key_created) { + pthread_key_delete(emutls_pthread_key); + emutls_key_created = false; + } +} +#endif diff --git a/contrib/compiler-rt/lib/builtins/fp_lib.h b/contrib/compiler-rt/lib/builtins/fp_lib.h index 223fb980aae..a0e19ab6a8f 100644 --- a/contrib/compiler-rt/lib/builtins/fp_lib.h +++ b/contrib/compiler-rt/lib/builtins/fp_lib.h @@ -25,6 +25,7 @@ #include #include #include "int_lib.h" +#include "int_math.h" // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in // 32-bit mode. @@ -265,6 +266,62 @@ static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int *hi = 0; } } + +// Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids +// pulling in a libm dependency from compiler-rt, but is not meant to replace +// it (i.e. code calling logb() should get the one from libm, not this), hence +// the __compiler_rt prefix. +static __inline fp_t __compiler_rt_logbX(fp_t x) { + rep_t rep = toRep(x); + int exp = (rep & exponentMask) >> significandBits; + + // Abnormal cases: + // 1) +/- inf returns +inf; NaN returns NaN + // 2) 0.0 returns -inf + if (exp == maxExponent) { + if (((rep & signBit) == 0) || (x != x)) { + return x; // NaN or +inf: return x + } else { + return -x; // -inf: return -x + } + } else if (x == 0.0) { + // 0.0: return -inf + return fromRep(infRep | signBit); + } + + if (exp != 0) { + // Normal number + return exp - exponentBias; // Unbias exponent + } else { + // Subnormal number; normalize and repeat + rep &= absMask; + const int shift = 1 - normalize(&rep); + exp = (rep & exponentMask) >> significandBits; + return exp - exponentBias - shift; // Unbias exponent + } +} +#endif + +#if defined(SINGLE_PRECISION) +static __inline fp_t __compiler_rt_logbf(fp_t x) { + return __compiler_rt_logbX(x); +} +#elif defined(DOUBLE_PRECISION) +static __inline fp_t __compiler_rt_logb(fp_t x) { + return __compiler_rt_logbX(x); +} +#elif defined(QUAD_PRECISION) + #if defined(CRT_LDBL_128BIT) +static __inline fp_t __compiler_rt_logbl(fp_t x) { + return __compiler_rt_logbX(x); +} + #else +// The generic implementation only works for ieee754 floating point. For other +// floating point types, continue to rely on the libm implementation for now. +static __inline long double __compiler_rt_logbl(long double x) { + return crt_logbl(x); +} + #endif #endif #endif // FP_LIB_HEADER diff --git a/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c b/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c index 0bc76562456..68581ef1643 100644 --- a/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c +++ b/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c @@ -206,8 +206,8 @@ __gcc_personality_v0(int version, _Unwind_Action actions, if ( lsda == (uint8_t*) 0 ) return continueUnwind(exceptionObject, context); - uintptr_t pc = _Unwind_GetIP(context)-1; - uintptr_t funcStart = _Unwind_GetRegionStart(context); + uintptr_t pc = (uintptr_t)_Unwind_GetIP(context)-1; + uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); uintptr_t pcOffset = pc - funcStart; /* Parse LSDA header. */ @@ -249,4 +249,3 @@ __gcc_personality_v0(int version, _Unwind_Action actions, /* No landing pad found, continue unwinding. */ return continueUnwind(exceptionObject, context); } - diff --git a/contrib/compiler-rt/lib/builtins/int_lib.h b/contrib/compiler-rt/lib/builtins/int_lib.h index 93394cebf7e..e07867d5580 100644 --- a/contrib/compiler-rt/lib/builtins/int_lib.h +++ b/contrib/compiler-rt/lib/builtins/int_lib.h @@ -45,7 +45,7 @@ #define AEABI_RTABI __attribute__((__pcs__("aapcs"))) -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) #define ALWAYS_INLINE __forceinline #define NOINLINE __declspec(noinline) #define NORETURN __declspec(noreturn) diff --git a/contrib/compiler-rt/lib/builtins/int_math.h b/contrib/compiler-rt/lib/builtins/int_math.h index fc81fb7f022..aa3d0721a8a 100644 --- a/contrib/compiler-rt/lib/builtins/int_math.h +++ b/contrib/compiler-rt/lib/builtins/int_math.h @@ -92,12 +92,8 @@ #endif #if defined(_MSC_VER) && !defined(__clang__) -#define crt_logb(x) logb((x)) -#define crt_logbf(x) logbf((x)) #define crt_logbl(x) logbl((x)) #else -#define crt_logb(x) __builtin_logb((x)) -#define crt_logbf(x) __builtin_logbf((x)) #define crt_logbl(x) __builtin_logbl((x)) #endif diff --git a/contrib/compiler-rt/lib/builtins/int_types.h b/contrib/compiler-rt/lib/builtins/int_types.h index 19002c68bd9..4c91b75bac6 100644 --- a/contrib/compiler-rt/lib/builtins/int_types.h +++ b/contrib/compiler-rt/lib/builtins/int_types.h @@ -60,10 +60,19 @@ typedef union }s; } udwords; -#if (defined(__LP64__) || defined(__wasm__) || defined(__mips64)) || defined(__riscv) +#if defined(__LP64__) || defined(__wasm__) || defined(__mips64) || \ + defined(__riscv) || defined(_WIN64) #define CRT_HAS_128BIT #endif +/* MSVC doesn't have a working 128bit integer type. Users should really compile + * compiler-rt with clang, but if they happen to be doing a standalone build for + * asan or something else, disable the 128 bit parts so things sort of work. + */ +#if defined(_MSC_VER) && !defined(__clang__) +#undef CRT_HAS_128BIT +#endif + #ifdef CRT_HAS_128BIT typedef int ti_int __attribute__ ((mode (TI))); typedef unsigned tu_int __attribute__ ((mode (TI))); @@ -139,6 +148,18 @@ typedef struct #endif /* _YUGA_LITTLE_ENDIAN */ } uqwords; +/* Check if the target supports 80 bit extended precision long doubles. + * Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC + * still makes it 80 bits. Clang will match whatever compiler it is trying to + * be compatible with. + */ +#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \ + defined(__m68k__) || defined(__ia64__) +#define HAS_80_BIT_LONG_DOUBLE 1 +#else +#define HAS_80_BIT_LONG_DOUBLE 0 +#endif + #ifndef _STANDALONE typedef union { diff --git a/contrib/compiler-rt/lib/builtins/int_util.c b/contrib/compiler-rt/lib/builtins/int_util.c index de87410dbca..752f2015580 100644 --- a/contrib/compiler-rt/lib/builtins/int_util.c +++ b/contrib/compiler-rt/lib/builtins/int_util.c @@ -27,7 +27,7 @@ NORETURN extern void panic(const char *, ...); #ifndef _WIN32 __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { panic("%s:%d: abort in %s", file, line, function); } @@ -41,7 +41,7 @@ NORETURN extern void __assert_rtn(const char *func, const char *file, int line, __attribute__((weak)) __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { __assert_rtn(function, file, line, "libcompiler_rt abort"); } @@ -51,7 +51,7 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) { __attribute__((weak)) __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { __builtin_trap(); } @@ -64,7 +64,7 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) { __attribute__((weak)) __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { abort(); } diff --git a/contrib/compiler-rt/lib/builtins/int_util.h b/contrib/compiler-rt/lib/builtins/int_util.h index a7b20ed6624..c3c87381ad8 100644 --- a/contrib/compiler-rt/lib/builtins/int_util.h +++ b/contrib/compiler-rt/lib/builtins/int_util.h @@ -20,10 +20,10 @@ #define INT_UTIL_H /** \brief Trigger a program abort (or panic for kernel code). */ -#define compilerrt_abort() compilerrt_abort_impl(__FILE__, __LINE__, __func__) +#define compilerrt_abort() __compilerrt_abort_impl(__FILE__, __LINE__, __func__) -NORETURN void compilerrt_abort_impl(const char *file, int line, - const char *function); +NORETURN void __compilerrt_abort_impl(const char *file, int line, + const char *function); #define COMPILE_TIME_ASSERT(expr) COMPILE_TIME_ASSERT1(expr, __COUNTER__) #define COMPILE_TIME_ASSERT1(expr, cnt) COMPILE_TIME_ASSERT2(expr, cnt) diff --git a/contrib/compiler-rt/lib/builtins/os_version_check.c b/contrib/compiler-rt/lib/builtins/os_version_check.c index 772e33333c0..e0d40edc7e3 100644 --- a/contrib/compiler-rt/lib/builtins/os_version_check.c +++ b/contrib/compiler-rt/lib/builtins/os_version_check.c @@ -15,7 +15,6 @@ #ifdef __APPLE__ -#include #include #include #include @@ -28,6 +27,33 @@ static int32_t GlobalMajor, GlobalMinor, GlobalSubminor; static dispatch_once_t DispatchOnceCounter; +/* We can't include directly from here, so + * just forward declare everything that we need from it. */ + +typedef const void *CFDataRef, *CFAllocatorRef, *CFPropertyListRef, + *CFStringRef, *CFDictionaryRef, *CFTypeRef, *CFErrorRef; + +#if __LLP64__ +typedef unsigned long long CFTypeID; +typedef unsigned long long CFOptionFlags; +typedef signed long long CFIndex; +#else +typedef unsigned long CFTypeID; +typedef unsigned long CFOptionFlags; +typedef signed long CFIndex; +#endif + +typedef unsigned char UInt8; +typedef _Bool Boolean; +typedef CFIndex CFPropertyListFormat; +typedef uint32_t CFStringEncoding; + +/* kCFStringEncodingASCII analog. */ +#define CF_STRING_ENCODING_ASCII 0x0600 +/* kCFStringEncodingUTF8 analog. */ +#define CF_STRING_ENCODING_UTF8 0x08000100 +#define CF_PROPERTY_LIST_IMMUTABLE 0 + typedef CFDataRef (*CFDataCreateWithBytesNoCopyFuncTy)(CFAllocatorRef, const UInt8 *, CFIndex, CFAllocatorRef); @@ -55,8 +81,7 @@ static void parseSystemVersionPList(void *Unused) { const void *NullAllocator = dlsym(RTLD_DEFAULT, "kCFAllocatorNull"); if (!NullAllocator) return; - const CFAllocatorRef kCFAllocatorNull = - *(const CFAllocatorRef *)NullAllocator; + const CFAllocatorRef AllocatorNull = *(const CFAllocatorRef *)NullAllocator; CFDataCreateWithBytesNoCopyFuncTy CFDataCreateWithBytesNoCopyFunc = (CFDataCreateWithBytesNoCopyFuncTy)dlsym(RTLD_DEFAULT, "CFDataCreateWithBytesNoCopy"); @@ -140,21 +165,21 @@ static void parseSystemVersionPList(void *Unused) { /* Get the file buffer into CF's format. We pass in a null allocator here * * because we free PListBuf ourselves */ FileContentsRef = (*CFDataCreateWithBytesNoCopyFunc)( - NULL, PListBuf, (CFIndex)NumRead, kCFAllocatorNull); + NULL, PListBuf, (CFIndex)NumRead, AllocatorNull); if (!FileContentsRef) goto Fail; if (CFPropertyListCreateWithDataFunc) PListRef = (*CFPropertyListCreateWithDataFunc)( - NULL, FileContentsRef, kCFPropertyListImmutable, NULL, NULL); + NULL, FileContentsRef, CF_PROPERTY_LIST_IMMUTABLE, NULL, NULL); else PListRef = (*CFPropertyListCreateFromXMLDataFunc)( - NULL, FileContentsRef, kCFPropertyListImmutable, NULL); + NULL, FileContentsRef, CF_PROPERTY_LIST_IMMUTABLE, NULL); if (!PListRef) goto Fail; CFStringRef ProductVersion = (*CFStringCreateWithCStringNoCopyFunc)( - NULL, "ProductVersion", kCFStringEncodingASCII, kCFAllocatorNull); + NULL, "ProductVersion", CF_STRING_ENCODING_ASCII, AllocatorNull); if (!ProductVersion) goto Fail; CFTypeRef OpaqueValue = (*CFDictionaryGetValueFunc)(PListRef, ProductVersion); @@ -165,7 +190,7 @@ static void parseSystemVersionPList(void *Unused) { char VersionStr[32]; if (!(*CFStringGetCStringFunc)((CFStringRef)OpaqueValue, VersionStr, - sizeof(VersionStr), kCFStringEncodingUTF8)) + sizeof(VersionStr), CF_STRING_ENCODING_UTF8)) goto Fail; sscanf(VersionStr, "%d.%d.%d", &GlobalMajor, &GlobalMinor, &GlobalSubminor); diff --git a/contrib/compiler-rt/lib/builtins/ppc/divtc3.c b/contrib/compiler-rt/lib/builtins/ppc/divtc3.c index 8ec41c528ab..ef532b84114 100644 --- a/contrib/compiler-rt/lib/builtins/ppc/divtc3.c +++ b/contrib/compiler-rt/lib/builtins/ppc/divtc3.c @@ -4,6 +4,11 @@ #include "DD.h" #include "../int_math.h" +// Use DOUBLE_PRECISION because the soft-fp method we use is logb (on the upper +// half of the long doubles), even though this file defines complex division for +// 128-bit floats. +#define DOUBLE_PRECISION +#include "../fp_lib.h" #if !defined(CRT_INFINITY) && defined(HUGE_VAL) #define CRT_INFINITY HUGE_VAL @@ -21,9 +26,10 @@ __divtc3(long double a, long double b, long double c, long double d) DD dDD = { .ld = d }; int ilogbw = 0; - const double logbw = crt_logb(crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi) )); - - if (crt_isfinite(logbw)) + const double logbw = __compiler_rt_logb( + crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi))); + + if (crt_isfinite(logbw)) { ilogbw = (int)logbw; diff --git a/contrib/compiler-rt/lib/builtins/ppc/fixunstfti.c b/contrib/compiler-rt/lib/builtins/ppc/fixunstfti.c new file mode 100644 index 00000000000..fa21084cb57 --- /dev/null +++ b/contrib/compiler-rt/lib/builtins/ppc/fixunstfti.c @@ -0,0 +1,106 @@ +//===-- lib/builtins/ppc/fixunstfti.c - Convert long double->int128 *-C -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements converting the 128bit IBM/PowerPC long double (double- +// double) data type to an unsigned 128 bit integer. +// +//===----------------------------------------------------------------------===// + +#include "../int_math.h" +#define BIAS 1023 + +/* Convert long double into an unsigned 128-bit integer. */ +__uint128_t __fixunstfti(long double input) { + + /* If we are trying to convert a NaN, return the NaN bit pattern. */ + if (crt_isnan(input)) { + return ((__uint128_t)0x7FF8000000000000ll) << 64 | + (__uint128_t)0x0000000000000000ll; + } + + __uint128_t result, hiResult, loResult; + int hiExponent, loExponent, shift; + /* The long double representation, with the high and low portions of + * the long double, and the corresponding bit patterns of each double. */ + union { + long double ld; + double d[2]; /* [0] is the high double, [1] is the low double. */ + unsigned long long ull[2]; /* High and low doubles as 64-bit integers. */ + } ldUnion; + + /* If the long double is less than 1.0 or negative, + * return 0.0. */ + if (input < 1.0) + return 0.0; + + /* Retrieve the 64-bit patterns of high and low doubles. + * Compute the unbiased exponent of both high and low doubles by + * removing the signs, isolating the exponent, and subtracting + * the bias from it. */ + ldUnion.ld = input; + hiExponent = ((ldUnion.ull[0] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; + loExponent = ((ldUnion.ull[1] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; + + /* Convert each double into int64; they will be added to the int128 result. + * CASE 1: High or low double fits in int64 + * - Convert the each double normally into int64. + * + * CASE 2: High or low double does not fit in int64 + * - Scale the double to fit within a 64-bit integer + * - Calculate the shift (amount to scale the double by in the int128) + * - Clear all the bits of the exponent (with 0x800FFFFFFFFFFFFF) + * - Add BIAS+53 (0x4350000000000000) to exponent to correct the value + * - Scale (move) the double to the correct place in the int128 + * (Move it by 2^53 places) + * + * Note: If the high double is assumed to be positive, an unsigned conversion + * from long double to 64-bit integer is needed. The low double can be either + * positive or negative, so a signed conversion is needed to retain the result + * of the low double and to ensure it does not simply get converted to 0. */ + + /* CASE 1 - High double fits in int64. */ + if (hiExponent < 63) { + hiResult = (unsigned long long)ldUnion.d[0]; + } else if (hiExponent < 128) { + /* CASE 2 - High double does not fit in int64, scale and convert it. */ + shift = hiExponent - 54; + ldUnion.ull[0] &= 0x800FFFFFFFFFFFFFll; + ldUnion.ull[0] |= 0x4350000000000000ll; + hiResult = (unsigned long long)ldUnion.d[0]; + hiResult <<= shift; + } else { + /* Detect cases for overflow. When the exponent of the high + * double is greater than 128 bits and when the long double + * input is positive, return the max 128-bit integer. + * For negative inputs with exponents > 128, return 1, like gcc. */ + if (ldUnion.d[0] > 0) { + return ((__uint128_t)0xFFFFFFFFFFFFFFFFll) << 64 | + (__uint128_t)0xFFFFFFFFFFFFFFFFll; + } else { + return ((__uint128_t)0x0000000000000000ll) << 64 | + (__uint128_t)0x0000000000000001ll; + } + } + + /* CASE 1 - Low double fits in int64. */ + if (loExponent < 63) { + loResult = (long long)ldUnion.d[1]; + } else { + /* CASE 2 - Low double does not fit in int64, scale and convert it. */ + shift = loExponent - 54; + ldUnion.ull[1] &= 0x800FFFFFFFFFFFFFll; + ldUnion.ull[1] |= 0x4350000000000000ll; + loResult = (long long)ldUnion.d[1]; + loResult <<= shift; + } + + /* Add the high and low doublewords together to form a 128 bit integer. */ + result = loResult + hiResult; + return result; +} diff --git a/contrib/compiler-rt/lib/builtins/ppc/floattitf.c b/contrib/compiler-rt/lib/builtins/ppc/floattitf.c new file mode 100644 index 00000000000..b8e297b6b8d --- /dev/null +++ b/contrib/compiler-rt/lib/builtins/ppc/floattitf.c @@ -0,0 +1,48 @@ +//===-- lib/builtins/ppc/floattitf.c - Convert int128->long double -*-C -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements converting a signed 128 bit integer to a 128bit IBM / +// PowerPC long double (double-double) value. +// +//===----------------------------------------------------------------------===// + +#include + +/* Conversions from signed and unsigned 64-bit int to long double. */ +long double __floatditf(int64_t); +long double __floatunditf(uint64_t); + +/* Convert a signed 128-bit integer to long double. + * This uses the following property: Let hi and lo be 64-bits each, + * and let signed_val_k() and unsigned_val_k() be the value of the + * argument interpreted as a signed or unsigned k-bit integer. Then, + * + * signed_val_128(hi,lo) = signed_val_64(hi) * 2^64 + unsigned_val_64(lo) + * = (long double)hi * 2^64 + (long double)lo, + * + * where (long double)hi and (long double)lo are signed and + * unsigned 64-bit integer to long double conversions, respectively. + */ +long double __floattitf(__int128_t arg) { + /* Split the int128 argument into 64-bit high and low int64 parts. */ + int64_t ArgHiPart = (int64_t)(arg >> 64); + uint64_t ArgLoPart = (uint64_t)arg; + + /* Convert each 64-bit part into long double. The high part + * must be a signed conversion and the low part an unsigned conversion + * to ensure the correct result. */ + long double ConvertedHiPart = __floatditf(ArgHiPart); + long double ConvertedLoPart = __floatunditf(ArgLoPart); + + /* The low bit of ArgHiPart corresponds to the 2^64 bit in arg. + * Multiply the high part by 2^64 to undo the right shift by 64-bits + * done in the splitting. Then, add to the low part to obtain the + * final result. */ + return ((ConvertedHiPart * 0x1.0p64) + ConvertedLoPart); +} diff --git a/contrib/compiler-rt/lib/cfi/cfi.cc b/contrib/compiler-rt/lib/cfi/cfi.cc index a2f127f93cd..b0a94375967 100644 --- a/contrib/compiler-rt/lib/cfi/cfi.cc +++ b/contrib/compiler-rt/lib/cfi/cfi.cc @@ -13,15 +13,33 @@ #include #include + +#include "sanitizer_common/sanitizer_common.h" +#if SANITIZER_FREEBSD +#include +#endif #include #include +#include #include +#if SANITIZER_LINUX typedef ElfW(Phdr) Elf_Phdr; typedef ElfW(Ehdr) Elf_Ehdr; +typedef ElfW(Addr) Elf_Addr; +typedef ElfW(Sym) Elf_Sym; +typedef ElfW(Dyn) Elf_Dyn; +#elif SANITIZER_FREEBSD +#if SANITIZER_WORDSIZE == 64 +#define ElfW64_Dyn Elf_Dyn +#define ElfW64_Sym Elf_Sym +#else +#define ElfW32_Dyn Elf_Dyn +#define ElfW32_Sym Elf_Sym +#endif +#endif #include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flag_parser.h" #include "ubsan/ubsan_init.h" #include "ubsan/ubsan_flags.h" @@ -154,15 +172,25 @@ void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) { *s = sv; } -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD void ShadowBuilder::Install() { MprotectReadOnly(shadow_, GetShadowSize()); uptr main_shadow = GetShadow(); if (main_shadow) { // Update. +#if SANITIZER_LINUX void *res = mremap((void *)shadow_, GetShadowSize(), GetShadowSize(), MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow); CHECK(res != MAP_FAILED); +#elif SANITIZER_NETBSD + void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow, + GetShadowSize(), MAP_FIXED); + CHECK(res != MAP_FAILED); +#else + void *res = MmapFixedOrDie(shadow_, GetShadowSize()); + CHECK(res != MAP_FAILED); + ::memcpy(&shadow_, &main_shadow, GetShadowSize()); +#endif } else { // Initial setup. CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached()); @@ -183,17 +211,17 @@ void ShadowBuilder::Install() { // dlopen(RTLD_NOLOAD | RTLD_LAZY) // dlsym("__cfi_check"). uptr find_cfi_check_in_dso(dl_phdr_info *info) { - const ElfW(Dyn) *dynamic = nullptr; + const Elf_Dyn *dynamic = nullptr; for (int i = 0; i < info->dlpi_phnum; ++i) { if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) { dynamic = - (const ElfW(Dyn) *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); + (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); break; } } if (!dynamic) return 0; uptr strtab = 0, symtab = 0, strsz = 0; - for (const ElfW(Dyn) *p = dynamic; p->d_tag != PT_NULL; ++p) { + for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) { if (p->d_tag == DT_SYMTAB) symtab = p->d_un.d_ptr; else if (p->d_tag == DT_STRTAB) @@ -227,7 +255,7 @@ uptr find_cfi_check_in_dso(dl_phdr_info *info) { return 0; } - for (const ElfW(Sym) *p = (const ElfW(Sym) *)symtab; (ElfW(Addr))p < strtab; + for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab; ++p) { // There is no reliable way to find the end of the symbol table. In // lld-produces files, there are other sections between symtab and strtab. diff --git a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt index 3d73508f570..4a0f03949ca 100644 --- a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt +++ b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt @@ -1,13 +1,11 @@ [cfi-unrelated-cast] # The specification of std::get_temporary_buffer mandates a cast to -# uninitialized T* (libstdc++, libc++, MSVC stdlib). +# uninitialized T* (libstdc++, MSVC stdlib). fun:_ZSt20get_temporary_buffer* -fun:_ZNSt3__120get_temporary_buffer* fun:*get_temporary_buffer@.*@std@@* -# STL address-of magic (libstdc++, libc++). +# STL address-of magic (libstdc++). fun:*__addressof* -fun:_ZNSt3__19addressof* # Windows C++ stdlib headers that contain bad unrelated casts. src:*xmemory0 diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.cc b/contrib/compiler-rt/lib/dfsan/dfsan.cc index d4dbebc43a7..585bdceac76 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan.cc +++ b/contrib/compiler-rt/lib/dfsan/dfsan.cc @@ -423,7 +423,7 @@ static void dfsan_fini() { static void dfsan_init(int argc, char **argv, char **envp) { InitializeFlags(); - InitializePlatformEarly(); + ::InitializePlatformEarly(); if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr())) Die(); diff --git a/contrib/compiler-rt/lib/esan/esan_interceptors.cpp b/contrib/compiler-rt/lib/esan/esan_interceptors.cpp index 0c596f1cf37..833faa2cd5b 100644 --- a/contrib/compiler-rt/lib/esan/esan_interceptors.cpp +++ b/contrib/compiler-rt/lib/esan/esan_interceptors.cpp @@ -327,7 +327,7 @@ INTERCEPTOR(int, rmdir, char *path) { // Signal-related interceptors //===----------------------------------------------------------------------===// -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD typedef void (*signal_handler_t)(int); INTERCEPTOR(signal_handler_t, signal, int signum, signal_handler_t handler) { void *ctx; @@ -344,7 +344,7 @@ INTERCEPTOR(signal_handler_t, signal, int signum, signal_handler_t handler) { #define ESAN_MAYBE_INTERCEPT_SIGNAL #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, struct sigaction *oldact) INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, @@ -363,7 +363,11 @@ int real_sigaction(int signum, const void *act, void *oldact) { if (REAL(sigaction) == nullptr) { // With an instrumented allocator, this is called during interceptor init // and we need a raw syscall solution. +#if SANITIZER_LINUX return internal_sigaction_syscall(signum, act, oldact); +#else + return internal_sigaction(signum, act, oldact); +#endif } return REAL(sigaction)(signum, (const struct sigaction *)act, (struct sigaction *)oldact); @@ -376,7 +380,7 @@ int real_sigaction(int signum, const void *act, void *oldact) { #define ESAN_MAYBE_INTERCEPT_SIGACTION #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { void *ctx; diff --git a/contrib/compiler-rt/lib/esan/esan_shadow.h b/contrib/compiler-rt/lib/esan/esan_shadow.h index 72a919a190d..b76a9ccbd9e 100644 --- a/contrib/compiler-rt/lib/esan/esan_shadow.h +++ b/contrib/compiler-rt/lib/esan/esan_shadow.h @@ -30,7 +30,7 @@ struct ApplicationRegion { bool ShadowMergedWithPrev; }; -#if SANITIZER_LINUX && defined(__x86_64__) +#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && defined(__x86_64__) // Linux x86_64 // // Application memory falls into these 5 regions (ignoring the corner case diff --git a/contrib/compiler-rt/lib/esan/esan_sideline.h b/contrib/compiler-rt/lib/esan/esan_sideline.h index 04aff22f4cf..74551fbde85 100644 --- a/contrib/compiler-rt/lib/esan/esan_sideline.h +++ b/contrib/compiler-rt/lib/esan/esan_sideline.h @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_freebsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" namespace __esan { diff --git a/contrib/compiler-rt/lib/esan/esan_sideline_bsd.cpp b/contrib/compiler-rt/lib/esan/esan_sideline_bsd.cpp new file mode 100644 index 00000000000..3134d377673 --- /dev/null +++ b/contrib/compiler-rt/lib/esan/esan_sideline_bsd.cpp @@ -0,0 +1,35 @@ +//===-- esan_sideline_bsd.cpp -----------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of EfficiencySanitizer, a family of performance tuners. +// +// Support for a separate or "sideline" tool thread on FreeBSD. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_FREEBSD + +#include "esan_sideline.h" + +namespace __esan { + +static SidelineThread *TheThread; + +bool SidelineThread::launchThread(SidelineFunc takeSample, void *Arg, + u32 FreqMilliSec) { + return true; +} + +bool SidelineThread::joinThread() { + return true; +} + +} // namespace __esan + +#endif // SANITIZER_FREEBSD diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltins.h b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltins.h new file mode 100644 index 00000000000..a80938d9a54 --- /dev/null +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltins.h @@ -0,0 +1,36 @@ +//===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos around builtin functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_H +#define LLVM_FUZZER_BUILTINS_H + +#include "FuzzerDefs.h" + +#if !LIBFUZZER_MSVC +#include + +#define GET_CALLER_PC() __builtin_return_address(0) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } +inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } +inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } + +inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } +inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } +inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } + +} // namespace fuzzer + +#endif // !LIBFUZZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_H diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h new file mode 100644 index 00000000000..67dd57ff9a9 --- /dev/null +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h @@ -0,0 +1,59 @@ +//===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos that use intrinsics instead of builtin functions +// which cannot be compiled by MSVC. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_MSVC_H +#define LLVM_FUZZER_BUILTINS_MSVC_H + +#include "FuzzerDefs.h" + +#if LIBFUZZER_MSVC +#if !defined(_M_ARM) && !defined(_M_X64) +#error "_BitScanReverse64 unavailable on this platform so MSVC is unsupported." +#endif +#include +#include +#include + +// __builtin_return_address() cannot be compiled with MSVC. Use the equivalent +// from +#define GET_CALLER_PC() reinterpret_cast(_ReturnAddress()) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +// Use alternatives to __builtin functions from and on +// Windows since the builtins are not supported by MSVC. +inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); } +inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); } +inline uint64_t Bswap(uint64_t x) { return _byteswap_uint64(x); } + +// The functions below were mostly copied from +// compiler-rt/lib/builtins/int_lib.h which defines the __builtin functions used +// outside of Windows. +inline uint32_t Clzll(uint64_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; + return 64; +} + +inline uint32_t Clz(uint32_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; + return 32; +} + +inline int Popcountll(unsigned long long X) { return __popcnt64(X); } + +} // namespace fuzzer + +#endif // LIBFUZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_MSVC_H diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h b/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h index 255f571ecf3..9d258a228f8 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h @@ -81,7 +81,7 @@ public: } // Like hasArgument, but checks for "-[Flag]=...". - bool hasFlag(const std::string &Flag) { + bool hasFlag(const std::string &Flag) const { std::string Arg("-" + Flag + "="); auto IsMatch = [&](const std::string &Other) { return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; @@ -92,7 +92,7 @@ public: // Returns the value of the first instance of a given flag, or an empty string // if the flag isn't present. Ignores any occurrences after // "-ignore_remaining_args=1", if present. - std::string getFlagValue(const std::string &Flag) { + std::string getFlagValue(const std::string &Flag) const { std::string Arg("-" + Flag + "="); auto IsMatch = [&](const std::string &Other) { return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h b/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h index 8ad14656cff..f844c07c757 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h @@ -238,12 +238,6 @@ class InputCorpus { return false; } - bool IsFeatureNew(size_t Idx, uint32_t NewSize, bool Shrink) { - assert(NewSize); - uint32_t OldSize = GetFeature(Idx % kFeatureSetSize); - return OldSize == 0 || (Shrink && OldSize > NewSize); - } - size_t NumFeatures() const { return NumAddedFeatures; } size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h b/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h index a35c7a181b7..c3dccbcd86f 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h @@ -82,6 +82,13 @@ #error "Support for your platform has not been implemented" #endif +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC compiler is being used. +#define LIBFUZZER_MSVC 1 +#else +#define LIBFUZZER_MSVC 0 +#endif + #ifndef __has_attribute # define __has_attribute(x) 0 #endif @@ -129,8 +136,15 @@ #if LIBFUZZER_WINDOWS #define ATTRIBUTE_INTERFACE __declspec(dllexport) +// This is used for __sancov_lowest_stack which is needed for +// -fsanitize-coverage=stack-depth. That feature is not yet available on +// Windows, so make the symbol static to avoid linking errors. +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + __attribute__((tls_model("initial-exec"))) thread_local static #else #define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local #endif namespace fuzzer { @@ -176,11 +190,6 @@ typedef int (*UserCallback)(const uint8_t *Data, size_t Size); int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); -inline uint8_t Bswap(uint8_t x) { return x; } -inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } -inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } -inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } - uint8_t *ExtraCountersBegin(); uint8_t *ExtraCountersEnd(); void ClearExtraCounters(); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp index 783474a39e1..ff2a639ac4a 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -615,13 +615,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; - Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; - Options.PrintUnstableStats = Flags.print_unstable_stats; - if (Flags.handle_unstable == TracePC::MinUnstable || - Flags.handle_unstable == TracePC::ZeroUnstable) - Options.HandleUnstable = Flags.handle_unstable; Options.DumpCoverage = Flags.dump_coverage; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsymWin.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsymWin.cpp deleted file mode 100644 index 321b3ec5d41..00000000000 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsymWin.cpp +++ /dev/null @@ -1,62 +0,0 @@ -//===- FuzzerExtFunctionsDlsymWin.cpp - Interface to external functions ---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using dynamic loading for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" -#include "Windows.h" - -// This must be included after Windows.h. -#include "Psapi.h" - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { - HMODULE Modules[1024]; - DWORD BytesNeeded; - HANDLE CurrentProcess = GetCurrentProcess(); - - if (!EnumProcessModules(CurrentProcess, Modules, sizeof(Modules), - &BytesNeeded)) { - Printf("EnumProcessModules failed (error: %d).\n", GetLastError()); - exit(1); - } - - if (sizeof(Modules) < BytesNeeded) { - Printf("Error: the array is not big enough to hold all loaded modules.\n"); - exit(1); - } - - for (size_t i = 0; i < (BytesNeeded / sizeof(HMODULE)); i++) - { - FARPROC Fn; -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr) { \ - Fn = GetProcAddress(Modules[i], #NAME); \ - if (Fn == nullptr) \ - Fn = GetProcAddress(Modules[i], #NAME "__dll"); \ - this->NAME = (decltype(ExternalFunctions::NAME)) Fn; \ - } -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC - } - -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr && WARN) \ - Printf("WARNING: Failed to find function \"%s\".\n", #NAME); -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp index a4e56fc27b8..6a6ef4932f4 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp @@ -22,7 +22,7 @@ extern "C" { // Declare these symbols as weak to allow them to be optionally defined. #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - __attribute__((weak)) RETURN_TYPE NAME FUNC_SIG + __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG #include "FuzzerExtFunctions.def" diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeakAlias.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeakAlias.cpp deleted file mode 100644 index e10f7b4dcac..00000000000 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeakAlias.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- FuzzerExtFunctionsWeakAlias.cpp - Interface to external functions --===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using weak aliases. Works for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" - -using namespace fuzzer; - -extern "C" { -// Declare these symbols as weak to allow them to be optionally defined. -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - RETURN_TYPE NAME##Def FUNC_SIG { \ - Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ - exit(1); \ - } \ - RETURN_TYPE NAME FUNC_SIG __attribute__((weak, alias(#NAME "Def"))); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -template -static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { - if (Fun == FunDef) { - if (WarnIfMissing) - Printf("WARNING: Failed to find function \"%s\".\n", FnName); - return nullptr; - } - return Fun; -} - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp new file mode 100644 index 00000000000..b01871439ca --- /dev/null +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp @@ -0,0 +1,83 @@ +//=== FuzzerExtWindows.cpp - Interface to external functions --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Implementation of FuzzerExtFunctions for Windows. Uses alternatename when +// compiled with MSVC. Uses weak aliases when compiled with clang. Unfortunately +// the method each compiler supports is not supported by the other. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +using namespace fuzzer; + +// Intermediate macro to ensure the parameter is expanded before stringified. +#define STRINGIFY_(A) #A +#define STRINGIFY(A) STRINGIFY_(A) + +#if LIBFUZZER_MSVC +// Copied from compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif + +// Declare external functions as having alternativenames, so that we can +// determine if they are not defined. +#define EXTERNAL_FUNC(Name, Default) \ + __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY( \ + Name) "=" WIN_SYM_PREFIX STRINGIFY(Default))) +#else +// Declare external functions as weak to allow them to default to a specified +// function if not defined explicitly. We must use weak symbols because clang's +// support for alternatename is not 100%, see +// https://bugs.llvm.org/show_bug.cgi?id=40218 for more details. +#define EXTERNAL_FUNC(Name, Default) \ + __attribute__((weak, alias(STRINGIFY(Default)))) +#endif // LIBFUZZER_MSVC + +extern "C" { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE NAME##Def FUNC_SIG { \ + Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ + exit(1); \ + } \ + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG; + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +template +static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { + if (Fun == FunDef) { + if (WarnIfMissing) + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + return nullptr; + } + return Fun; +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def b/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def index ba04bc25fd4..91281c979c5 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -17,7 +17,7 @@ FUZZER_FLAG_INT(runs, -1, FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " "If 0, libFuzzer tries to guess a good value based on the corpus " "and reports it. ") -FUZZER_FLAG_INT(len_control, 1000, "Try generating small inputs first, " +FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "then try larger inputs over time. Specifies the rate at which the length " "limit is increased (smaller == faster). If 0, immediately try inputs with " "size up to max_len.") @@ -110,15 +110,6 @@ FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated." " If 1, dump coverage information as a" " .sancov file at exit.") -FUZZER_FLAG_INT(handle_unstable, 0, "Experimental." - " Executes every input 3 times in total if a unique feature" - " is found during the first execution." - " If 1, we only use the minimum hit count from the 3 runs" - " to determine whether an input is interesting." - " If 2, we disregard edges that are found unstable for" - " feature collection.") -FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental." - " If 1, print unstable statistics at exit.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") @@ -162,4 +153,3 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_server) FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") -FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp index f3ead0ec535..c4c31e82471 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp @@ -31,7 +31,7 @@ long GetEpoch(const std::string &Path) { } Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { - std::ifstream T(Path); + std::ifstream T(Path, std::ios::binary); if (ExitOnError && !T) { Printf("No such directory: %s; exiting\n", Path.c_str()); exit(1); @@ -51,7 +51,7 @@ Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { } std::string FileToString(const std::string &Path) { - std::ifstream T(Path); + std::ifstream T(Path, std::ios::binary); return std::string((std::istreambuf_iterator(T)), std::istreambuf_iterator()); } @@ -100,14 +100,6 @@ std::string DirPlusFile(const std::string &DirPath, return DirPath + GetSeparator() + FileName; } -std::string Basename(const std::string &Path, char Separator) { - size_t Pos = Path.rfind(Separator); - if (Pos == std::string::npos) - return Path; - assert(Pos < Path.size()); - return Path.substr(Pos + 1); -} - void DupAndCloseStderr() { int OutputFd = DuplicateFile(2); if (OutputFd > 0) { diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h index 6d7757435b7..b4a68190e78 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h @@ -68,7 +68,7 @@ void GetSizedFilesFromDir(const std::string &Dir, Vector *V); char GetSeparator(); // Similar to the basename utility: returns the file name w/o the dir prefix. -std::string Basename(const std::string &Path, char Separator = GetSeparator()); +std::string Basename(const std::string &Path); FILE* OpenFile(int Fd, const char *Mode); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp index 17e884d3c4c..401b4cbbf74 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp @@ -46,6 +46,13 @@ size_t FileSize(const std::string &Path) { return St.st_size; } +std::string Basename(const std::string &Path) { + size_t Pos = Path.rfind(GetSeparator()); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector *V, bool TopDir) { auto E = GetEpoch(Dir); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp index 74853646b21..75dcaf72a94 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp @@ -72,6 +72,26 @@ bool IsFile(const std::string &Path) { return IsFile(Path, Att); } +std::string Basename(const std::string &Path) { + size_t Pos = Path.find_last_of("/\\"); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +size_t FileSize(const std::string &Path) { + WIN32_FILE_ATTRIBUTE_DATA attr; + if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), GetLastError()); + return 0; + } + ULARGE_INTEGER size; + size.HighPart = attr.nFileSizeHigh; + size.LowPart = attr.nFileSizeLow; + return size.QuadPart; +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector *V, bool TopDir) { auto E = GetEpoch(Dir); @@ -91,7 +111,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, { if (GetLastError() == ERROR_FILE_NOT_FOUND) return; - Printf("No such directory: %s; exiting\n", Dir.c_str()); + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); exit(1); } diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h b/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h index bfc898248ad..a7fdc89cba5 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h @@ -67,7 +67,6 @@ public: static void StaticGracefulExitCallback(); void ExecuteCallback(const uint8_t *Data, size_t Size); - void CheckForUnstableCounters(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index 4bc88365a0b..a32a3072300 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -275,7 +275,8 @@ NO_SANITIZE_MEMORY void Fuzzer::AlarmCallback() { assert(Options.UnitTimeoutSec > 0); // In Windows Alarm callback is executed by a different thread. -#if !LIBFUZZER_WINDOWS + // NetBSD's current behavior needs this change too. +#if !LIBFUZZER_WINDOWS && !LIBFUZZER_NETBSD if (!InFuzzingThread()) return; #endif @@ -354,13 +355,10 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { void Fuzzer::PrintFinalStats() { if (Options.PrintCoverage) TPC.PrintCoverage(); - if (Options.PrintUnstableStats) - TPC.PrintUnstableStats(); if (Options.DumpCoverage) TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); - if (Options.PrintMutationStats) MD.PrintMutationStats(); if (!Options.PrintFinalStats) return; size_t ExecPerSec = execPerSec(); @@ -449,34 +447,6 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { } } -void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) { - auto CBSetupAndRun = [&]() { - ScopedEnableMsanInterceptorChecks S; - UnitStartTime = system_clock::now(); - TPC.ResetMaps(); - RunningUserCallback = true; - CB(Data, Size); - RunningUserCallback = false; - UnitStopTime = system_clock::now(); - }; - - // Copy original run counters into our unstable counters - TPC.InitializeUnstableCounters(); - - // First Rerun - CBSetupAndRun(); - TPC.UpdateUnstableCounters(Options.HandleUnstable); - - // Second Rerun - CBSetupAndRun(); - TPC.UpdateUnstableCounters(Options.HandleUnstable); - - // Move minimum hit counts back to ModuleInline8bitCounters - if (Options.HandleUnstable == TracePC::MinUnstable || - Options.HandleUnstable == TracePC::ZeroUnstable) - TPC.ApplyUnstableCounters(); -} - bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, InputInfo *II, bool *FoundUniqFeatures) { if (!Size) @@ -487,17 +457,6 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - bool NewFeaturesUnstable = false; - - if (Options.HandleUnstable || Options.PrintUnstableStats) { - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.IsFeatureNew(Feature, Size, Options.Shrink)) - NewFeaturesUnstable = true; - }); - if (NewFeaturesUnstable) - CheckForUnstableCounters(Data, Size); - } - TPC.CollectFeatures([&](size_t Feature) { if (Corpus.AddFeature(Feature, Size, Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); @@ -506,12 +465,10 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, II->UniqFeatureSet.end(), Feature)) FoundUniqFeaturesOfII++; }); - if (FoundUniqFeatures) *FoundUniqFeatures = FoundUniqFeaturesOfII; PrintPulseAndReportSlowInput(Data, Size); size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; - if (NumNewFeatures) { TPC.UpdateObservedPCs(); Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp index ff076cca683..142b2b0b001 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp @@ -30,36 +30,34 @@ MutationDispatcher::MutationDispatcher(Random &Rand, DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0}, + {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes", 0, 0}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0, - 0}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0, - 0}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0}, + "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict", 0, 0}, + "ManualDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict", 0, 0}, + "PersAutoDict"}, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0}); + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0}); + {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); } static char RandCh(Random &Rand) { @@ -466,7 +464,6 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } - RecordUsefulMutations(); } void MutationDispatcher::PrintRecommendedDictionary() { @@ -487,7 +484,8 @@ void MutationDispatcher::PrintRecommendedDictionary() { void MutationDispatcher::PrintMutationSequence() { Printf("MS: %zd ", CurrentMutatorSequence.size()); - for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name); + for (auto M : CurrentMutatorSequence) + Printf("%s-", M.Name); if (!CurrentDictionaryEntrySequence.empty()) { Printf(" DE: "); for (auto DE : CurrentDictionaryEntrySequence) { @@ -515,13 +513,12 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // in which case they will return 0. // Try several times before returning un-mutated data. for (int Iter = 0; Iter < 100; Iter++) { - auto M = &Mutators[Rand(Mutators.size())]; - size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize); + auto M = Mutators[Rand(Mutators.size())]; + size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); if (NewSize && NewSize <= MaxSize) { if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); - M->TotalCount++; return NewSize; } } @@ -562,21 +559,4 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) { {W, std::numeric_limits::max()}); } -void MutationDispatcher::RecordUsefulMutations() { - for (auto M : CurrentMutatorSequence) M->UsefulCount++; -} - -void MutationDispatcher::PrintMutationStats() { - Printf("\nstat::mutation_usefulness: "); - for (size_t i = 0; i < Mutators.size(); i++) { - double UsefulPercentage = - Mutators[i].TotalCount - ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount - : 0; - Printf("%.3f", UsefulPercentage); - if (i < Mutators.size() - 1) Printf(","); - } - Printf("\n"); -} - } // namespace fuzzer diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h index 828ecc13d86..a51c7fb44d4 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h @@ -93,16 +93,10 @@ public: Random &GetRand() { return Rand; } - void PrintMutationStats(); - - void RecordUsefulMutations(); - private: struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); const char *Name; - uint64_t UsefulCount; - uint64_t TotalCount; }; size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, @@ -141,7 +135,6 @@ public: Dictionary PersistentAutoDictionary; Vector CurrentDictionaryEntrySequence; - Vector CurrentMutatorSequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; @@ -156,6 +149,7 @@ public: Vector Mutators; Vector DefaultMutators; + Vector CurrentMutatorSequence; }; } // namespace fuzzer diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h b/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h index ce39c0876cd..ab90df82a63 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h @@ -52,11 +52,8 @@ struct FuzzingOptions { bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; - bool PrintMutationStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; - bool PrintUnstableStats = false; - int HandleUnstable = 0; bool DumpCoverage = false; bool DetectLeaks = true; int PurgeAllocatorIntervalSec = 1; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp index 29ffc8e34fc..80b33105bb2 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp @@ -13,6 +13,8 @@ //===----------------------------------------------------------------------===// #include "FuzzerTracePC.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerDictionary.h" @@ -32,8 +34,7 @@ ATTRIBUTE_INTERFACE uintptr_t __sancov_trace_pc_pcs[fuzzer::TracePC::kNumPCs]; // Used by -fsanitize-coverage=stack-depth to track stack depth -ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) -thread_local uintptr_t __sancov_lowest_stack; +ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; namespace fuzzer { @@ -57,49 +58,6 @@ size_t TracePC::GetTotalPCCoverage() { return Res; } -template -void TracePC::IterateInline8bitCounters(CallBack CB) const { - if (NumInline8bitCounters && NumInline8bitCounters == NumPCsInPCTables) { - size_t CounterIdx = 0; - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - assert(Size == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++, CounterIdx++) - CB(i, j, CounterIdx); - } - } -} - -// Initializes unstable counters by copying Inline8bitCounters to unstable -// counters. -void TracePC::InitializeUnstableCounters() { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j]; - }); -} - -// Compares the current counters with counters from previous runs -// and records differences as unstable edges. -void TracePC::UpdateUnstableCounters(int UnstableMode) { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) { - UnstableCounters[UnstableIdx].IsUnstable = true; - if (UnstableMode == ZeroUnstable) - UnstableCounters[UnstableIdx].Counter = 0; - else if (UnstableMode == MinUnstable) - UnstableCounters[UnstableIdx].Counter = std::min( - ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter); - } - }); -} - -// Moves the minimum hit counts to ModuleCounters. -void TracePC::ApplyUnstableCounters() { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter; - }); -} void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { if (Start == Stop) return; @@ -185,11 +143,42 @@ void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) { ValueProfileMap.AddValueModPrime(Idx); } +/// \return the address of the previous instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.h` +inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { +#if defined(__arm__) + // T32 (Thumb) branch instructions might be 16 or 32 bit long, + // so we return (pc-2) in that case in order to be safe. + // For A32 mode we return (pc-4) because all instructions are 32 bit long. + return (PC - 3) & (~1); +#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) + // PCs are always 4 byte aligned. + return PC - 4; +#elif defined(__sparc__) || defined(__mips__) + return PC - 8; +#else + return PC - 1; +#endif +} + +/// \return the address of the next instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cc` +inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { +#if defined(__mips__) + return PC + 8; +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) + return PC + 4; +#else + return PC + 1; +#endif +} + void TracePC::UpdateObservedPCs() { Vector CoveredFuncs; auto ObservePC = [&](uintptr_t PC) { if (ObservedPCs.insert(PC).second && DoPrintNewPCs) { - PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", PC + 1); + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", GetNextInstructionPc(PC)); Printf("\n"); } }; @@ -203,10 +192,15 @@ void TracePC::UpdateObservedPCs() { if (NumPCsInPCTables) { if (NumInline8bitCounters == NumPCsInPCTables) { - IterateInline8bitCounters([&](int i, int j, int CounterIdx) { - if (ModuleCounters[i].Start[j]) - Observe(ModulePCTable[i].Start[j]); - }); + for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { + uint8_t *Beg = ModuleCounters[i].Start; + size_t Size = ModuleCounters[i].Stop - Beg; + assert(Size == + (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); + for (size_t j = 0; j < Size; j++) + if (Beg[j]) + Observe(ModulePCTable[i].Start[j]); + } } else if (NumGuards == NumPCsInPCTables) { size_t GuardIdx = 1; for (size_t i = 0; i < NumModules; i++) { @@ -224,22 +218,11 @@ void TracePC::UpdateObservedPCs() { for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; i++) { Printf("\tNEW_FUNC[%zd/%zd]: ", i + 1, CoveredFuncs.size()); - PrintPC("%p %F %L", "%p", CoveredFuncs[i] + 1); + PrintPC("%p %F %L", "%p", GetNextInstructionPc(CoveredFuncs[i])); Printf("\n"); } } -inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC - 1; -} - -inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC + 1; -} static std::string GetModuleName(uintptr_t PC) { char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? @@ -349,15 +332,6 @@ void TracePC::DumpCoverage() { } } -void TracePC::PrintUnstableStats() { - size_t count = 0; - for (size_t i = 0; i < NumInline8bitCounters; i++) - if (UnstableCounters[i].IsUnstable) - count++; - Printf("stat::stability_rate: %.2f\n", - 100 - static_cast(count * 100) / NumInline8bitCounters); -} - // Value profile. // We keep track of various values that affect control flow. // These values are inserted into a bit-set-based hash map. @@ -401,20 +375,14 @@ ATTRIBUTE_TARGET_POPCNT ALWAYS_INLINE ATTRIBUTE_NO_SANITIZE_ALL void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { uint64_t ArgXor = Arg1 ^ Arg2; - uint64_t ArgDistance = __builtin_popcountll(ArgXor) + 1; // [1,65] - uintptr_t Idx = ((PC & 4095) + 1) * ArgDistance; if (sizeof(T) == 4) TORC4.Insert(ArgXor, Arg1, Arg2); else if (sizeof(T) == 8) TORC8.Insert(ArgXor, Arg1, Arg2); - // TODO: remove these flags and instead use all metrics at once. - if (UseValueProfileMask & 1) - ValueProfileMap.AddValue(Idx); - if (UseValueProfileMask & 2) - ValueProfileMap.AddValue( - PC * 64 + (Arg1 == Arg2 ? 0 : __builtin_clzll(Arg1 - Arg2) + 1)); - if (UseValueProfileMask & 4) // alternative way to use the hamming distance - ValueProfileMap.AddValue(PC * 64 + ArgDistance); + uint64_t HammingDistance = Popcountll(ArgXor); // [0,64] + uint64_t AbsoluteDistance = (Arg1 == Arg2 ? 0 : Clzll(Arg1 - Arg2) + 1); + ValueProfileMap.AddValue(PC * 128 + HammingDistance); + ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); } static size_t InternalStrnlen(const char *S, size_t MaxLen) { @@ -455,7 +423,7 @@ extern "C" { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); uint32_t Idx = *Guard; __sancov_trace_pc_pcs[Idx] = PC; __sancov_trace_pc_guard_8bit_counters[Idx]++; @@ -466,7 +434,7 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc() { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); uintptr_t Idx = PC & (((uintptr_t)1 << fuzzer::TracePC::kTracePcBits) - 1); __sancov_trace_pc_pcs[Idx] = PC; __sancov_trace_pc_guard_8bit_counters[Idx]++; @@ -491,7 +459,7 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_indir(uintptr_t Callee) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCallerCallee(PC, Callee); } @@ -499,7 +467,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp8(uint64_t Arg1, uint64_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -510,7 +478,7 @@ ATTRIBUTE_TARGET_POPCNT // the behaviour of __sanitizer_cov_trace_cmp[1248] ones. This, however, // should be changed later to make full use of instrumentation. void __sanitizer_cov_trace_const_cmp8(uint64_t Arg1, uint64_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -518,7 +486,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -526,7 +494,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -534,7 +502,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp2(uint16_t Arg1, uint16_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -542,7 +510,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp2(uint16_t Arg1, uint16_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -550,7 +518,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp1(uint8_t Arg1, uint8_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -558,7 +526,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp1(uint8_t Arg1, uint8_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -572,7 +540,7 @@ void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { // Skip the most common and the most boring case. if (Vals[N - 1] < 256 && Val < 256) return; - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); size_t i; uint64_t Token = 0; for (i = 0; i < N; i++) { @@ -593,7 +561,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_div4(uint32_t Val) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Val, (uint32_t)0); } @@ -601,7 +569,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_div8(uint64_t Val) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Val, (uint64_t)0); } @@ -609,7 +577,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_gep(uintptr_t Idx) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); } diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h index 097ba69bdc0..46d6c24887f 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h @@ -74,11 +74,6 @@ class TracePC { // How many bits of PC are used from __sanitizer_cov_trace_pc. static const size_t kTracePcBits = 18; - enum HandleUnstableOptions { - MinUnstable = 1, - ZeroUnstable = 2, - }; - void HandleInit(uint32_t *Start, uint32_t *Stop); void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); @@ -109,7 +104,6 @@ class TracePC { void PrintCoverage(); void DumpCoverage(); - void PrintUnstableStats(); template void IterateCoveredFunctions(CallBack CB); @@ -142,18 +136,7 @@ class TracePC { void SetFocusFunction(const std::string &FuncName); bool ObservedFocusFunction(); - void InitializeUnstableCounters(); - void UpdateUnstableCounters(int UnstableMode); - void ApplyUnstableCounters(); - private: - struct UnstableEdge { - uint8_t Counter; - bool IsUnstable; - }; - - UnstableEdge UnstableCounters[kNumPCs]; - bool UseCounters = false; uint32_t UseValueProfileMask = false; bool DoPrintNewPCs = false; @@ -185,9 +168,6 @@ private: Set ObservedPCs; std::unordered_map ObservedFuncs; // PC => Counter. - template - void IterateInline8bitCounters(Callback CB) const; - std::pair FocusFunction = {-1, -1}; // Module and PC IDs. ValueBitMap ValueProfileMap; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h b/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h index 8c5c57c3ab8..d2f1d5de426 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h @@ -12,8 +12,10 @@ #ifndef LLVM_FUZZER_UTIL_H #define LLVM_FUZZER_UTIL_H -#include "FuzzerDefs.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" #include "FuzzerCommand.h" +#include "FuzzerDefs.h" namespace fuzzer { @@ -84,7 +86,7 @@ std::string SearchRegexCmd(const std::string &Regex); size_t SimpleFastHash(const uint8_t *Data, size_t Size); -inline uint32_t Log(uint32_t X) { return 32 - __builtin_clz(X) - 1; } +inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } } // namespace fuzzer diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp index cd2bb7438e9..cd48fefef35 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp @@ -49,9 +49,6 @@ void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); namespace { -// TODO(phosek): remove this and replace it with ZX_TIME_INFINITE -#define ZX_TIME_INFINITE_OLD INT64_MAX - // A magic value for the Zircon exception port, chosen to spell 'FUZZING' // when interpreted as a byte sequence on little-endian platforms. const uint64_t kFuzzingCrash = 0x474e495a5a5546; @@ -237,7 +234,7 @@ void CrashHandler(zx_handle_t *Event) { "_zx_object_signal"); zx_port_packet_t Packet; - ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE_OLD, &Packet), + ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE, &Packet), "_zx_port_wait"); // At this point, we want to get the state of the crashing thread, but @@ -315,8 +312,8 @@ void SetSignalHandler(const FuzzingOptions &Options) { ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); std::thread T(CrashHandler, &Event); - zx_status_t Status = _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, - ZX_TIME_INFINITE_OLD, nullptr); + zx_status_t Status = + _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); _zx_handle_close(Event); ExitOnErr(Status, "_zx_object_wait_one"); @@ -378,19 +375,28 @@ int ExecuteCommand(const Command &Cmd) { Argv[i] = Args[i].c_str(); Argv[Argc] = nullptr; - // Determine stdout + // Determine output. On Fuchsia, the fuzzer is typically run as a component + // that lacks a mutable working directory. Fortunately, when this is the case + // a mutable output directory must be specified using "-artifact_prefix=...", + // so write the log file(s) there. int FdOut = STDOUT_FILENO; - if (Cmd.hasOutputFile()) { - auto Filename = Cmd.getOutputFile(); - FdOut = open(Filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + std::string Path; + if (Cmd.hasFlag("artifact_prefix")) + Path = Cmd.getFlagValue("artifact_prefix") + "/" + Cmd.getOutputFile(); + else + Path = Cmd.getOutputFile(); + FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); if (FdOut == -1) { - Printf("libFuzzer: failed to open %s: %s\n", Filename.c_str(), + Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), strerror(errno)); return ZX_ERR_IO; } } - auto CloseFdOut = at_scope_exit([&]() { close(FdOut); } ); + auto CloseFdOut = at_scope_exit([FdOut]() { + if (FdOut != STDOUT_FILENO) + close(FdOut); + }); // Determine stderr int FdErr = STDERR_FILENO; @@ -440,7 +446,7 @@ int ExecuteCommand(const Command &Cmd) { // Now join the process and return the exit status. if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED, - ZX_TIME_INFINITE_OLD, nullptr)) != ZX_OK) { + ZX_TIME_INFINITE, nullptr)) != ZX_OK) { Printf("libFuzzer: failed to join '%s': %s\n", Argv[0], _zx_status_get_string(rc)); return rc; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp index 8227e778ea0..393b4768be7 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp @@ -24,7 +24,7 @@ #include // This must be included after windows.h. -#include +#include namespace fuzzer { @@ -179,7 +179,9 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, } std::string DisassembleCmd(const std::string &FileName) { - if (ExecuteCommand("dumpbin /summary > nul") == 0) + Vector command_vector; + command_vector.push_back("dumpbin /summary > nul"); + if (ExecuteCommand(Command(command_vector)) == 0) return "dumpbin /disasm " + FileName; Printf("libFuzzer: couldn't find tool to disassemble (dumpbin)\n"); exit(1); diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.cc b/contrib/compiler-rt/lib/hwasan/hwasan.cc index 7dab8249e3f..e2bfea5e422 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan.cc @@ -13,19 +13,20 @@ //===----------------------------------------------------------------------===// #include "hwasan.h" -#include "hwasan_mapping.h" +#include "hwasan_checks.h" #include "hwasan_poisoning.h" #include "hwasan_report.h" #include "hwasan_thread.h" +#include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_symbolizer.h" -#include "sanitizer_common/sanitizer_stackdepot.h" #include "ubsan/ubsan_flags.h" #include "ubsan/ubsan_init.h" @@ -36,17 +37,17 @@ using namespace __sanitizer; namespace __hwasan { void EnterSymbolizer() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); CHECK(t); t->EnterSymbolizer(); } void ExitSymbolizer() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); CHECK(t); t->LeaveSymbolizer(); } bool IsInSymbolizer() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); return t && t->InSymbolizer(); } @@ -57,6 +58,7 @@ Flags *flags() { } int hwasan_inited = 0; +int hwasan_shadow_inited = 0; bool hwasan_init_is_running; int hwasan_report_count = 0; @@ -86,7 +88,18 @@ static void InitializeFlags() { cf.check_printf = false; cf.intercept_tls_get_addr = true; cf.exitcode = 99; + // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; + +#if SANITIZER_ANDROID + // Let platform handle other signals. It is better at reporting them then we + // are. + cf.handle_segv = kHandleSignalNo; + cf.handle_sigbus = kHandleSignalNo; + cf.handle_abort = kHandleSignalNo; + cf.handle_sigill = kHandleSignalNo; + cf.handle_sigfpe = kHandleSignalNo; +#endif OverrideCommonFlags(cf); } @@ -119,7 +132,8 @@ static void InitializeFlags() { #if HWASAN_CONTAINS_UBSAN ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); #endif - VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : ""); + VPrintf(1, "HWASAN_OPTIONS: %s\n", + hwasan_options ? hwasan_options : ""); InitializeCommonFlags(); @@ -130,8 +144,13 @@ static void InitializeFlags() { void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, void *context, bool request_fast_unwind) { - HwasanThread *t = GetCurrentThread(); - if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { + Thread *t = GetCurrentThread(); + if (!t) { + // the thread is still being created. + stack->size = 0; + return; + } + if (!StackTrace::WillUseFastUnwind(request_fast_unwind)) { // Block reports from our interceptors during _Unwind_Backtrace. SymbolizerScope sym_scope; return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); @@ -140,11 +159,6 @@ void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, request_fast_unwind); } -void PrintWarning(uptr pc, uptr bp) { - GET_FATAL_STACK_TRACE_PC_BP(pc, bp); - ReportInvalidAccess(&stack, 0); -} - static void HWAsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, @@ -153,6 +167,84 @@ static void HWAsanCheckFailed(const char *file, int line, const char *cond, Die(); } +static constexpr uptr kMemoryUsageBufferSize = 4096; + +static void HwasanFormatMemoryUsage(InternalScopedString &s) { + HwasanThreadList &thread_list = hwasanThreadList(); + auto thread_stats = thread_list.GetThreadStats(); + auto *sds = StackDepotGetStats(); + AllocatorStatCounters asc; + GetAllocatorStats(asc); + s.append( + "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd" + " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd" + " heap: %zd", + internal_getpid(), GetRSS(), thread_stats.n_live_threads, + thread_stats.total_stack_size, + thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(), + sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]); +} + +#if SANITIZER_ANDROID +static char *memory_usage_buffer = nullptr; + +#define PR_SET_VMA 0x53564d41 +#define PR_SET_VMA_ANON_NAME 0 + +static void InitMemoryUsage() { + memory_usage_buffer = + (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); + CHECK(memory_usage_buffer); + memory_usage_buffer[0] = '\0'; + CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, + (uptr)memory_usage_buffer, kMemoryUsageBufferSize, + (uptr)memory_usage_buffer) == 0); +} + +void UpdateMemoryUsage() { + if (!flags()->export_memory_stats) + return; + if (!memory_usage_buffer) + InitMemoryUsage(); + InternalScopedString s(kMemoryUsageBufferSize); + HwasanFormatMemoryUsage(s); + internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1); + memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0'; +} +#else +void UpdateMemoryUsage() {} +#endif + +struct FrameDescription { + uptr PC; + const char *Descr; +}; + +struct FrameDescriptionArray { + FrameDescription *beg, *end; +}; + +static InternalMmapVectorNoCtor AllFrames; + +void InitFrameDescriptors(uptr b, uptr e) { + FrameDescription *beg = reinterpret_cast(b); + FrameDescription *end = reinterpret_cast(e); + if (beg == end) + return; + AllFrames.push_back({beg, end}); + if (Verbosity()) + for (FrameDescription *frame_descr = beg; frame_descr < end; frame_descr++) + Printf("Frame: %p %s\n", frame_descr->PC, frame_descr->Descr); +} + +const char *GetStackFrameDescr(uptr pc) { + for (uptr i = 0, n = AllFrames.size(); i < n; i++) + for (auto p = AllFrames[i].beg; p < AllFrames[i].end; p++) + if (p->PC == pc) + return p->Descr; + return nullptr; +} + } // namespace __hwasan // Interface. @@ -161,6 +253,20 @@ using namespace __hwasan; uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. +void __hwasan_shadow_init() { + if (hwasan_shadow_inited) return; + if (!InitShadow()) { + Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); + DumpProcessMap(); + Die(); + } + hwasan_shadow_inited = 1; +} + +void __hwasan_init_frames(uptr beg, uptr end) { + InitFrameDescriptors(beg, end); +} + void __hwasan_init() { CHECK(!hwasan_init_is_running); if (hwasan_inited) return; @@ -177,18 +283,20 @@ void __hwasan_init() { __sanitizer_set_report_path(common_flags()->log_path); + AndroidTestTlsSlot(); + DisableCoreDumperIfNecessary(); - if (!InitShadow()) { - Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); - if (HWASAN_FIXED_MAPPING) { - Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); - Printf("FATAL: Disabling ASLR is known to cause this error.\n"); - Printf("FATAL: If running under GDB, try " - "'set disable-randomization off'.\n"); - } - DumpProcessMap(); - Die(); - } + + __hwasan_shadow_init(); + + InitThreads(); + hwasanThreadList().CreateCurrentThread(); + + MadviseShadow(); + + SetPrintfAndReportCallback(AppendToErrorMessageBuffer); + // This may call libc -> needs initialized shadow. + AndroidLogInit(); InitializeInterceptors(); InstallDeadlySignalHandlers(HwasanOnDeadlySignal); @@ -198,14 +306,11 @@ void __hwasan_init() { InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); - HwasanTSDInit(HwasanTSDDtor); + HwasanTSDInit(); + HwasanTSDThreadInit(); HwasanAllocatorInit(); - HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr); - SetCurrentThread(main_thread); - main_thread->ThreadStart(); - #if HWASAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -216,9 +321,14 @@ void __hwasan_init() { hwasan_inited = 1; } -void __hwasan_print_shadow(const void *x, uptr size) { - // FIXME: - Printf("FIXME: __hwasan_print_shadow unimplemented\n"); +void __hwasan_print_shadow(const void *p, uptr sz) { + uptr ptr_raw = UntagAddr(reinterpret_cast(p)); + uptr shadow_first = MemToShadow(ptr_raw); + uptr shadow_last = MemToShadow(ptr_raw + sz - 1); + Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw, + ptr_raw + sz, GetTagFromPointer((uptr)p)); + for (uptr s = shadow_first; s <= shadow_last; ++s) + Printf(" %zx: %x\n", ShadowToMem(s), *(tag_t *)s); } sptr __hwasan_test_shadow(const void *p, uptr sz) { @@ -227,12 +337,12 @@ sptr __hwasan_test_shadow(const void *p, uptr sz) { tag_t ptr_tag = GetTagFromPointer((uptr)p); if (ptr_tag == 0) return -1; - uptr ptr_raw = GetAddressFromPointer((uptr)p); - uptr shadow_first = MEM_TO_SHADOW(ptr_raw); - uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1); + uptr ptr_raw = UntagAddr(reinterpret_cast(p)); + uptr shadow_first = MemToShadow(ptr_raw); + uptr shadow_last = MemToShadow(ptr_raw + sz - 1); for (uptr s = shadow_first; s <= shadow_last; ++s) if (*(tag_t*)s != ptr_tag) - return SHADOW_TO_MEM(s) - ptr_raw; + return ShadowToMem(s) - ptr_raw; return -1; } @@ -255,63 +365,6 @@ void __sanitizer_unaligned_store64(uu64 *p, u64 x) { *p = x; } -template -__attribute__((always_inline)) -static void SigTrap(uptr p) { -#if defined(__aarch64__) - (void)p; - // 0x900 is added to do not interfere with the kernel use of lower values of - // brk immediate. - // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. - asm("brk %0\n\t" ::"n"(0x900 + X)); -#elif defined(__x86_64__) - // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes - // total. The pointer is passed via rdi. - // 0x40 is added as a safeguard, to help distinguish our trap from others and - // to avoid 0 offsets in the command (otherwise it'll be reduced to a - // different nop command, the three bytes one). - asm volatile( - "int3\n" - "nopl %c0(%%rax)\n" - :: "n"(0x40 + X), "D"(p)); -#else - // FIXME: not always sigill. - __builtin_trap(); -#endif - // __builtin_unreachable(); -} - -enum class ErrorAction { Abort, Recover }; -enum class AccessType { Load, Store }; - -template -__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { - tag_t ptr_tag = GetTagFromPointer(p); - uptr ptr_raw = p & ~kAddressTagMask; - tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw); - if (UNLIKELY(ptr_tag != mem_tag)) { - SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + LogSize>(p); - if (EA == ErrorAction::Abort) __builtin_unreachable(); - } -} - -template -__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, - uptr sz) { - CHECK_NE(0, sz); - tag_t ptr_tag = GetTagFromPointer(p); - uptr ptr_raw = p & ~kAddressTagMask; - tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw); - tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1); - for (tag_t *t = shadow_first; t <= shadow_last; ++t) - if (UNLIKELY(ptr_tag != *t)) { - SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + 0xf>(p); - if (EA == ErrorAction::Abort) __builtin_unreachable(); - } -} - void __hwasan_loadN(uptr p, uptr sz) { CheckAddressSized(p, sz); } @@ -392,10 +445,38 @@ void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { TagMemoryAligned(p, sz, tag); } +uptr __hwasan_tag_pointer(uptr p, u8 tag) { + return AddTagToPointer(p, tag); +} + +void __hwasan_handle_longjmp(const void *sp_dst) { + uptr dst = (uptr)sp_dst; + // HWASan does not support tagged SP. + CHECK(GetTagFromPointer(dst) == 0); + + uptr sp = (uptr)__builtin_frame_address(0); + static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M + if (dst < sp || dst - sp > kMaxExpectedCleanupSize) { + Report( + "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: " + "stack top: %p; target %p; distance: %p (%zd)\n" + "False positive error reports may follow\n", + (void *)sp, (void *)dst, dst - sp); + return; + } + TagMemory(sp, dst - sp, 0); +} + +void __hwasan_print_memory_usage() { + InternalScopedString s(kMemoryUsageBufferSize); + HwasanFormatMemoryUsage(s); + Printf("%s\n", s.data()); +} + static const u8 kFallbackTag = 0xBB; u8 __hwasan_generate_tag() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); if (!t) return kFallbackTag; return t->GenerateRandomTag(); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.h b/contrib/compiler-rt/lib/hwasan/hwasan.h index 47d1d057a0d..ce9e904c5c6 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan.h @@ -30,6 +30,10 @@ # define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB #endif +#ifndef HWASAN_WITH_INTERCEPTORS +#define HWASAN_WITH_INTERCEPTORS 0 +#endif + typedef u8 tag_t; // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address @@ -37,16 +41,21 @@ typedef u8 tag_t; const unsigned kAddressTagShift = 56; const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; +// Minimal alignment of the shadow base address. Determines the space available +// for threads and stack histories. This is an ABI constant. +const unsigned kShadowBaseAlignment = 32; + static inline tag_t GetTagFromPointer(uptr p) { return p >> kAddressTagShift; } -static inline uptr GetAddressFromPointer(uptr p) { - return p & ~kAddressTagMask; +static inline uptr UntagAddr(uptr tagged_addr) { + return tagged_addr & ~kAddressTagMask; } -static inline void * GetAddressFromPointer(const void *p) { - return (void *)((uptr)p & ~kAddressTagMask); +static inline void *UntagPtr(const void *tagged_ptr) { + return reinterpret_cast( + UntagAddr(reinterpret_cast(tagged_ptr))); } static inline uptr AddTagToPointer(uptr p, tag_t tag) { @@ -61,12 +70,13 @@ extern int hwasan_report_count; bool ProtectRange(uptr beg, uptr end); bool InitShadow(); +void InitThreads(); +void MadviseShadow(); char *GetProcSelfMaps(); void InitializeInterceptors(); void HwasanAllocatorInit(); void HwasanAllocatorThreadFinish(); -void HwasanDeallocate(StackTrace *stack, void *ptr); void *hwasan_malloc(uptr size, StackTrace *stack); void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); @@ -77,11 +87,13 @@ void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack); int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, StackTrace *stack); +void hwasan_free(void *ptr, StackTrace *stack); void InstallTrapHandler(); void InstallAtExitHandler(); const char *GetStackOriginDescr(u32 id, uptr *pc); +const char *GetStackFrameDescr(uptr pc); void EnterSymbolizer(); void ExitSymbolizer(); @@ -92,8 +104,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void PrintWarning(uptr pc, uptr bp); - void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, void *context, bool request_fast_unwind); @@ -135,13 +145,17 @@ class ScopedThreadLocalStateBackup { u64 va_arg_overflow_size_tls; }; -void HwasanTSDInit(void (*destructor)(void *tsd)); -void *HwasanTSDGet(); -void HwasanTSDSet(void *tsd); -void HwasanTSDDtor(void *tsd); +void HwasanTSDInit(); +void HwasanTSDThreadInit(); void HwasanOnDeadlySignal(int signo, void *info, void *context); +void UpdateMemoryUsage(); + +void AppendToErrorMessageBuffer(const char *buffer); + +void AndroidTestTlsSlot(); + } // namespace __hwasan #define HWASAN_MALLOC_HOOK(ptr, size) \ diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc index c2b9b0b6958..8487ed7e1e5 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc @@ -12,10 +12,6 @@ // HWAddressSanitizer allocator. //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_allocator.h" -#include "sanitizer_common/sanitizer_allocator_checks.h" -#include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -23,30 +19,53 @@ #include "hwasan_allocator.h" #include "hwasan_mapping.h" #include "hwasan_thread.h" -#include "hwasan_poisoning.h" +#include "hwasan_report.h" + +#if HWASAN_WITH_INTERCEPTORS +DEFINE_REAL(void *, realloc, void *ptr, uptr size) +DEFINE_REAL(void, free, void *ptr) +#endif namespace __hwasan { -enum { - CHUNK_INVALID = 0, - CHUNK_FREE = 1, - CHUNK_ALLOCATED = 2 +static Allocator allocator; +static AllocatorCache fallback_allocator_cache; +static SpinMutex fallback_mutex; +static atomic_uint8_t hwasan_allocator_tagging_enabled; + +static const tag_t kFallbackAllocTag = 0xBB; +static const tag_t kFallbackFreeTag = 0xBC; + +enum RightAlignMode { + kRightAlignNever, + kRightAlignSometimes, + kRightAlignAlways }; -struct Metadata { - u64 state : 2; - u64 requested_size : 62; - u32 alloc_context_id; - u32 free_context_id; -}; +// These two variables are initialized from flags()->malloc_align_right +// in HwasanAllocatorInit and are never changed afterwards. +static RightAlignMode right_align_mode = kRightAlignNever; +static bool right_align_8 = false; + +// Initialized in HwasanAllocatorInit, an never changed. +static ALIGNED(16) u8 tail_magic[kShadowAlignment]; -bool HwasanChunkView::IsValid() const { - return metadata_ && metadata_->state != CHUNK_INVALID; -} bool HwasanChunkView::IsAllocated() const { - return metadata_ && metadata_->state == CHUNK_ALLOCATED; + return metadata_ && metadata_->alloc_context_id && metadata_->requested_size; } + +// Aligns the 'addr' right to the granule boundary. +static uptr AlignRight(uptr addr, uptr requested_size) { + uptr tail_size = requested_size % kShadowAlignment; + if (!tail_size) return addr; + if (right_align_8) + return tail_size > 8 ? addr : addr + 8; + return addr + kShadowAlignment - tail_size; +} + uptr HwasanChunkView::Beg() const { + if (metadata_ && metadata_->right_aligned) + return AlignRight(block_, metadata_->requested_size); return block_; } uptr HwasanChunkView::End() const { @@ -58,88 +77,79 @@ uptr HwasanChunkView::UsedSize() const { u32 HwasanChunkView::GetAllocStackId() const { return metadata_->alloc_context_id; } -u32 HwasanChunkView::GetFreeStackId() const { - return metadata_->free_context_id; + +uptr HwasanChunkView::ActualSize() const { + return allocator.GetActuallyAllocatedSize(reinterpret_cast(block_)); } -struct HwasanMapUnmapCallback { - void OnMap(uptr p, uptr size) const {} - void OnUnmap(uptr p, uptr size) const { - // We are about to unmap a chunk of user memory. - // It can return as user-requested mmap() or another thread stack. - // Make it accessible with zero-tagged pointer. - TagMemory(p, size, 0); - } -}; +bool HwasanChunkView::FromSmallHeap() const { + return allocator.FromPrimary(reinterpret_cast(block_)); +} -#if !defined(__aarch64__) && !defined(__x86_64__) -#error Unsupported platform -#endif - -static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G -static const uptr kRegionSizeLog = 20; -static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; - -struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = sizeof(Metadata); - typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog; - typedef __hwasan::ByteMap ByteMap; - typedef HwasanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; -}; -typedef SizeClassAllocator32 PrimaryAllocator; -typedef SizeClassAllocatorLocalCache AllocatorCache; -typedef LargeMmapAllocator SecondaryAllocator; -typedef CombinedAllocator Allocator; - -static Allocator allocator; -static AllocatorCache fallback_allocator_cache; -static SpinMutex fallback_mutex; -static atomic_uint8_t hwasan_allocator_tagging_enabled; - -static const tag_t kFallbackAllocTag = 0xBB; -static const tag_t kFallbackFreeTag = 0xBC; +void GetAllocatorStats(AllocatorStatCounters s) { + allocator.GetStats(s); +} void HwasanAllocatorInit() { atomic_store_relaxed(&hwasan_allocator_tagging_enabled, !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); allocator.Init(common_flags()->allocator_release_to_os_interval_ms); + switch (flags()->malloc_align_right) { + case 0: break; + case 1: + right_align_mode = kRightAlignSometimes; + right_align_8 = false; + break; + case 2: + right_align_mode = kRightAlignAlways; + right_align_8 = false; + break; + case 8: + right_align_mode = kRightAlignSometimes; + right_align_8 = true; + break; + case 9: + right_align_mode = kRightAlignAlways; + right_align_8 = true; + break; + default: + Report("ERROR: unsupported value of malloc_align_right flag: %d\n", + flags()->malloc_align_right); + Die(); + } + for (uptr i = 0; i < kShadowAlignment; i++) + tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); } -AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) { - CHECK(ms); - CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); - return reinterpret_cast(ms->allocator_cache); +void AllocatorSwallowThreadLocalCache(AllocatorCache *cache) { + allocator.SwallowCache(cache); } -void HwasanThreadLocalMallocStorage::CommitBack() { - allocator.SwallowCache(GetAllocatorCache(this)); +static uptr TaggedSize(uptr size) { + if (!size) size = 1; + uptr new_size = RoundUpTo(size, kShadowAlignment); + CHECK_GE(new_size, size); + return new_size; } -static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, - bool zeroise) { - alignment = Max(alignment, kShadowAlignment); - size = RoundUpTo(size, kShadowAlignment); - - if (size > kMaxAllowedMallocSize) { +static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, + bool zeroise) { + if (orig_size > kMaxAllowedMallocSize) { if (AllocatorMayReturnNull()) { Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n", - size); + orig_size); return nullptr; } - ReportAllocationSizeTooBig(size, kMaxAllowedMallocSize, stack); + ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack); } - HwasanThread *t = GetCurrentThread(); + + alignment = Max(alignment, kShadowAlignment); + uptr size = TaggedSize(orig_size); + Thread *t = GetCurrentThread(); void *allocated; if (t) { - AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); - allocated = allocator.Allocate(cache, size, alignment); + allocated = allocator.Allocate(t->allocator_cache(), size, alignment); } else { SpinMutexLock l(&fallback_mutex); AllocatorCache *cache = &fallback_allocator_cache; @@ -153,11 +163,18 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, } Metadata *meta = reinterpret_cast(allocator.GetMetaData(allocated)); - meta->state = CHUNK_ALLOCATED; - meta->requested_size = size; + meta->requested_size = static_cast(orig_size); meta->alloc_context_id = StackDepotPut(*stack); - if (zeroise) + meta->right_aligned = false; + if (zeroise) { internal_memset(allocated, 0, size); + } else if (flags()->max_malloc_fill_size > 0) { + uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size); + internal_memset(allocated, flags()->malloc_fill_byte, fill_size); + } + if (!right_align_mode) + internal_memcpy(reinterpret_cast(allocated) + orig_size, tail_magic, + size - orig_size); void *user_ptr = allocated; if (flags()->tag_in_malloc && @@ -165,74 +182,101 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, user_ptr = (void *)TagMemoryAligned( (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag); + if ((orig_size % kShadowAlignment) && (alignment <= kShadowAlignment) && + right_align_mode) { + uptr as_uptr = reinterpret_cast(user_ptr); + if (right_align_mode == kRightAlignAlways || + GetTagFromPointer(as_uptr) & 1) { // use a tag bit as a random bit. + user_ptr = reinterpret_cast(AlignRight(as_uptr, orig_size)); + meta->right_aligned = 1; + } + } + HWASAN_MALLOC_HOOK(user_ptr, size); return user_ptr; } -void HwasanDeallocate(StackTrace *stack, void *user_ptr) { - CHECK(user_ptr); - HWASAN_FREE_HOOK(user_ptr); +static bool PointerAndMemoryTagsMatch(void *tagged_ptr) { + CHECK(tagged_ptr); + tag_t ptr_tag = GetTagFromPointer(reinterpret_cast(tagged_ptr)); + tag_t mem_tag = *reinterpret_cast( + MemToShadow(reinterpret_cast(UntagPtr(tagged_ptr)))); + return ptr_tag == mem_tag; +} + +static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { + CHECK(tagged_ptr); + HWASAN_FREE_HOOK(tagged_ptr); + + if (!PointerAndMemoryTagsMatch(tagged_ptr)) + ReportInvalidFree(stack, reinterpret_cast(tagged_ptr)); + + void *untagged_ptr = UntagPtr(tagged_ptr); + void *aligned_ptr = reinterpret_cast( + RoundDownTo(reinterpret_cast(untagged_ptr), kShadowAlignment)); + Metadata *meta = + reinterpret_cast(allocator.GetMetaData(aligned_ptr)); + uptr orig_size = meta->requested_size; + u32 free_context_id = StackDepotPut(*stack); + u32 alloc_context_id = meta->alloc_context_id; + + // Check tail magic. + uptr tagged_size = TaggedSize(orig_size); + if (flags()->free_checks_tail_magic && !right_align_mode && orig_size) { + uptr tail_size = tagged_size - orig_size; + CHECK_LT(tail_size, kShadowAlignment); + void *tail_beg = reinterpret_cast( + reinterpret_cast(aligned_ptr) + orig_size); + if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size)) + ReportTailOverwritten(stack, reinterpret_cast(tagged_ptr), + orig_size, tail_size, tail_magic); + } - void *p = GetAddressFromPointer(user_ptr); - Metadata *meta = reinterpret_cast(allocator.GetMetaData(p)); - uptr size = meta->requested_size; - meta->state = CHUNK_FREE; meta->requested_size = 0; - meta->free_context_id = StackDepotPut(*stack); + meta->alloc_context_id = 0; // This memory will not be reused by anyone else, so we are free to keep it // poisoned. - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); + if (flags()->max_free_fill_size > 0) { + uptr fill_size = + Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); + internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); + } if (flags()->tag_in_free && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - TagMemoryAligned((uptr)p, size, + TagMemoryAligned(reinterpret_cast(aligned_ptr), TaggedSize(orig_size), t ? t->GenerateRandomTag() : kFallbackFreeTag); if (t) { - AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); - allocator.Deallocate(cache, p); + allocator.Deallocate(t->allocator_cache(), aligned_ptr); + if (auto *ha = t->heap_allocations()) + ha->push({reinterpret_cast(tagged_ptr), alloc_context_id, + free_context_id, static_cast(orig_size)}); } else { SpinMutexLock l(&fallback_mutex); AllocatorCache *cache = &fallback_allocator_cache; - allocator.Deallocate(cache, p); + allocator.Deallocate(cache, aligned_ptr); } } -void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size, - uptr alignment) { - alignment = Max(alignment, kShadowAlignment); - new_size = RoundUpTo(new_size, kShadowAlignment); +static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old, + uptr new_size, uptr alignment) { + if (!PointerAndMemoryTagsMatch(tagged_ptr_old)) + ReportInvalidFree(stack, reinterpret_cast(tagged_ptr_old)); - void *old_p = GetAddressFromPointer(user_old_p); - Metadata *meta = reinterpret_cast(allocator.GetMetaData(old_p)); - uptr old_size = meta->requested_size; - uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); - if (new_size <= actually_allocated_size) { - // We are not reallocating here. - // FIXME: update stack trace for the allocation? - meta->requested_size = new_size; - if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - return user_old_p; - if (flags()->retag_in_realloc) { - HwasanThread *t = GetCurrentThread(); - return (void *)TagMemoryAligned( - (uptr)old_p, new_size, - t ? t->GenerateRandomTag() : kFallbackAllocTag); - } - if (new_size > old_size) { - tag_t tag = GetTagFromPointer((uptr)user_old_p); - TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag); - } - return user_old_p; + void *tagged_ptr_new = + HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); + if (tagged_ptr_old && tagged_ptr_new) { + void *untagged_ptr_old = UntagPtr(tagged_ptr_old); + Metadata *meta = + reinterpret_cast(allocator.GetMetaData(untagged_ptr_old)); + internal_memcpy(UntagPtr(tagged_ptr_new), untagged_ptr_old, + Min(new_size, static_cast(meta->requested_size))); + HwasanDeallocate(stack, tagged_ptr_old); } - uptr memcpy_size = Min(new_size, old_size); - void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); - if (new_p) { - internal_memcpy(new_p, old_p, memcpy_size); - HwasanDeallocate(stack, old_p); - } - return new_p; + return tagged_ptr_new; } -void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) { +static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) { if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { if (AllocatorMayReturnNull()) return nullptr; @@ -250,12 +294,18 @@ HwasanChunkView FindHeapChunkByAddress(uptr address) { return HwasanChunkView(reinterpret_cast(block), metadata); } -static uptr AllocationSize(const void *user_ptr) { - const void *p = GetAddressFromPointer(user_ptr); - if (!p) return 0; - const void *beg = allocator.GetBlockBegin(p); - if (beg != p) return 0; - Metadata *b = (Metadata *)allocator.GetMetaData(p); +static uptr AllocationSize(const void *tagged_ptr) { + const void *untagged_ptr = UntagPtr(tagged_ptr); + if (!untagged_ptr) return 0; + const void *beg = allocator.GetBlockBegin(untagged_ptr); + Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr); + if (b->right_aligned) { + if (beg != reinterpret_cast(RoundDownTo( + reinterpret_cast(untagged_ptr), kShadowAlignment))) + return 0; + } else { + if (beg != untagged_ptr) return 0; + } return b->requested_size; } @@ -270,6 +320,14 @@ void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { if (!ptr) return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + +#if HWASAN_WITH_INTERCEPTORS + // A tag of 0 means that this is a system allocator allocation, so we must use + // the system allocator to realloc it. + if (!flags()->disable_allocator_tagging && GetTagFromPointer((uptr)ptr) == 0) + return REAL(realloc)(ptr, size); +#endif + if (size == 0) { HwasanDeallocate(stack, ptr); return nullptr; @@ -331,6 +389,17 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, return 0; } +void hwasan_free(void *ptr, StackTrace *stack) { +#if HWASAN_WITH_INTERCEPTORS + // A tag of 0 means that this is a system allocator allocation, so we must use + // the system allocator to free it. + if (!flags()->disable_allocator_tagging && GetTagFromPointer((uptr)ptr) == 0) + return REAL(free)(ptr); +#endif + + return HwasanDeallocate(stack, ptr); +} + } // namespace __hwasan using namespace __hwasan; @@ -340,6 +409,15 @@ void __hwasan_enable_allocator_tagging() { } void __hwasan_disable_allocator_tagging() { +#if HWASAN_WITH_INTERCEPTORS + // Allocator tagging must be enabled for the system allocator fallback to work + // correctly. This means that we can't disable it at runtime if it was enabled + // at startup since that might result in our deallocations going to the system + // allocator. If tagging was disabled at startup we avoid this problem by + // disabling the fallback altogether. + CHECK(flags()->disable_allocator_tagging); +#endif + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h index d025112e977..6ab722fa6be 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -1,4 +1,4 @@ -//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===// +//===-- hwasan_allocator.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,35 +14,73 @@ #ifndef HWASAN_ALLOCATOR_H #define HWASAN_ALLOCATOR_H +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_ring_buffer.h" +#include "hwasan_poisoning.h" + +#if !defined(__aarch64__) && !defined(__x86_64__) +#error Unsupported platform +#endif + +#if HWASAN_WITH_INTERCEPTORS +DECLARE_REAL(void *, realloc, void *ptr, uptr size) +DECLARE_REAL(void, free, void *ptr) +#endif namespace __hwasan { -struct HwasanThreadLocalMallocStorage { - uptr quarantine_cache[16]; - // Allocator cache contains atomic_uint64_t which must be 8-byte aligned. - ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque. - void CommitBack(); - - private: - // These objects are allocated via mmap() and are zero-initialized. - HwasanThreadLocalMallocStorage() {} +struct Metadata { + u32 requested_size : 31; // sizes are < 2G. + u32 right_aligned : 1; + u32 alloc_context_id; }; -struct Metadata; +struct HwasanMapUnmapCallback { + void OnMap(uptr p, uptr size) const { UpdateMemoryUsage(); } + void OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // It can return as user-requested mmap() or another thread stack. + // Make it accessible with zero-tagged pointer. + TagMemory(p, size, 0); + } +}; + +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP64 { + static const uptr kSpaceBeg = ~0ULL; + static const uptr kSpaceSize = 0x2000000000ULL; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap; + using AddressSpaceView = LocalAddressSpaceView; + typedef HwasanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator64 PrimaryAllocator; +typedef SizeClassAllocatorLocalCache AllocatorCache; +typedef LargeMmapAllocator SecondaryAllocator; +typedef CombinedAllocator Allocator; + +void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); class HwasanChunkView { public: HwasanChunkView() : block_(0), metadata_(nullptr) {} HwasanChunkView(uptr block, Metadata *metadata) : block_(block), metadata_(metadata) {} - bool IsValid() const; // Checks if it points to a valid allocated chunk bool IsAllocated() const; // Checks if the memory is currently allocated uptr Beg() const; // First byte of user memory uptr End() const; // Last byte of user memory uptr UsedSize() const; // Size requested by the user + uptr ActualSize() const; // Size allocated by the allocator. u32 GetAllocStackId() const; - u32 GetFreeStackId() const; + bool FromSmallHeap() const; private: uptr block_; Metadata *const metadata_; @@ -50,6 +88,21 @@ class HwasanChunkView { HwasanChunkView FindHeapChunkByAddress(uptr address); +// Information about one (de)allocation that happened in the past. +// These are recorded in a thread-local ring buffer. +// TODO: this is currently 24 bytes (20 bytes + alignment). +// Compress it to 16 bytes or extend it to be more useful. +struct HeapAllocationRecord { + uptr tagged_addr; + u32 alloc_context_id; + u32 free_context_id; + u32 requested_size; +}; + +typedef RingBuffer HeapAllocationsRingBuffer; + +void GetAllocatorStats(AllocatorStatCounters s); + } // namespace __hwasan #endif // HWASAN_ALLOCATOR_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_checks.h b/contrib/compiler-rt/lib/hwasan/hwasan_checks.h new file mode 100644 index 00000000000..688b5e2bed8 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_checks.h @@ -0,0 +1,80 @@ +//===-- hwasan_checks.h -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_CHECKS_H +#define HWASAN_CHECKS_H + +#include "hwasan_mapping.h" + +namespace __hwasan { +template +__attribute__((always_inline)) static void SigTrap(uptr p) { +#if defined(__aarch64__) + (void)p; + // 0x900 is added to do not interfere with the kernel use of lower values of + // brk immediate. + // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. + asm("brk %0\n\t" ::"n"(0x900 + X)); +#elif defined(__x86_64__) + // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes + // total. The pointer is passed via rdi. + // 0x40 is added as a safeguard, to help distinguish our trap from others and + // to avoid 0 offsets in the command (otherwise it'll be reduced to a + // different nop command, the three bytes one). + asm volatile( + "int3\n" + "nopl %c0(%%rax)\n" ::"n"(0x40 + X), + "D"(p)); +#else + // FIXME: not always sigill. + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + +enum class ErrorAction { Abort, Recover }; +enum class AccessType { Load, Store }; + +template +__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw); + if (UNLIKELY(ptr_tag != mem_tag)) { + SigTrap<0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + LogSize>(p); + if (EA == ErrorAction::Abort) + __builtin_unreachable(); + } +} + +template +__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, + uptr sz) { + if (sz == 0) + return; + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t *shadow_first = (tag_t *)MemToShadow(ptr_raw); + tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz - 1); + for (tag_t *t = shadow_first; t <= shadow_last; ++t) + if (UNLIKELY(ptr_tag != *t)) { + SigTrap<0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + 0xf>(p); + if (EA == ErrorAction::Abort) + __builtin_unreachable(); + } +} +} // end namespace __hwasan + +#endif // HWASAN_CHECKS_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc b/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc index 17338003aa6..87670f50828 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc @@ -13,6 +13,7 @@ /// //===----------------------------------------------------------------------===// +#include "hwasan.h" #include "hwasan_dynamic_shadow.h" #include "hwasan_mapping.h" #include "sanitizer_common/sanitizer_common.h" @@ -35,12 +36,16 @@ static void UnmapFromTo(uptr from, uptr to) { } } -// Returns an address aligned to 8 pages, such that one page on the left and -// shadow_size_bytes bytes on the right of it are mapped r/o. +// Returns an address aligned to kShadowBaseAlignment, such that +// 2**kShadowBaseAlingment on the left and shadow_size_bytes bytes on the right +// of it are mapped no access. static uptr MapDynamicShadow(uptr shadow_size_bytes) { const uptr granularity = GetMmapGranularity(); - const uptr alignment = granularity * SHADOW_GRANULARITY; - const uptr left_padding = granularity; + const uptr min_alignment = granularity << kShadowScale; + const uptr alignment = 1ULL << kShadowBaseAlignment; + CHECK_GE(alignment, min_alignment); + + const uptr left_padding = 1ULL << kShadowBaseAlignment; const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity); const uptr map_size = shadow_size + left_padding + alignment; @@ -58,8 +63,7 @@ static uptr MapDynamicShadow(uptr shadow_size_bytes) { } // namespace __hwasan -#if HWASAN_PREMAP_SHADOW - +#if SANITIZER_ANDROID extern "C" { INTERFACE_ATTRIBUTE void __hwasan_shadow(); @@ -117,16 +121,22 @@ void __hwasan_shadow(); } // extern "C" -#endif // HWASAN_PREMAP_SHADOW - namespace __hwasan { uptr FindDynamicShadowStart(uptr shadow_size_bytes) { -#if HWASAN_PREMAP_SHADOW if (IsPremapShadowAvailable()) return FindPremappedShadowStart(shadow_size_bytes); -#endif return MapDynamicShadow(shadow_size_bytes); } } // namespace __hwasan +#else +namespace __hwasan { + +uptr FindDynamicShadowStart(uptr shadow_size_bytes) { + return MapDynamicShadow(shadow_size_bytes); +} + +} // namespace __hwasan +# +#endif // SANITIZER_ANDROID diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h index 16d60c4d8ba..492d5bb98c2 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h @@ -1,4 +1,4 @@ -//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===// +//===-- hwasan_flags.h ------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc index c45781168d6..b450ab9503f 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -17,9 +17,10 @@ // HWASAN_FLAG(Type, Name, DefaultValue, Description) // See COMMON_FLAG in sanitizer_flags.inc for more details. +HWASAN_FLAG(bool, verbose_threads, false, + "inform on thread creation/destruction") HWASAN_FLAG(bool, tag_in_malloc, true, "") HWASAN_FLAG(bool, tag_in_free, true, "") -HWASAN_FLAG(bool, retag_in_realloc, true, "") HWASAN_FLAG(bool, print_stats, false, "") HWASAN_FLAG(bool, halt_on_error, true, "") HWASAN_FLAG(bool, atexit, false, "") @@ -31,3 +32,57 @@ HWASAN_FLAG(bool, disable_allocator_tagging, false, "") // If false, use simple increment of a thread local counter to generate new // tags. HWASAN_FLAG(bool, random_tags, true, "") + +HWASAN_FLAG( + int, max_malloc_fill_size, 0x1000, // By default, fill only the first 4K. + "HWASan allocator flag. max_malloc_fill_size is the maximal amount of " + "bytes that will be filled with malloc_fill_byte on malloc.") + +// Rules for malloc alignment on aarch64: +// * If the size is 16-aligned, then malloc should return 16-aligned memory. +// * Otherwise, malloc should return 8-alignment memory. +// So, +// * If the size is 16-aligned, we don't need to do anything. +// * Otherwise we don't have to obey 16-alignment, just the 8-alignment. +// * We may want to break the 8-alignment rule to catch more buffer overflows +// but this will break valid code in some rare cases, like this: +// struct Foo { +// // accessed via atomic instructions that require 8-alignment. +// std::atomic atomic_stuff; +// ... +// char vla[1]; // the actual size of vla could be anything. +// } +// Which means that the safe values for malloc_align_right are 0, 8, 9, +// and the values 1 and 2 may require changes in otherwise valid code. + +HWASAN_FLAG( + int, malloc_align_right, 0, // off by default + "HWASan allocator flag. " + "0 (default): allocations are always aligned left to 16-byte boundary; " + "1: allocations are sometimes aligned right to 1-byte boundary (risky); " + "2: allocations are always aligned right to 1-byte boundary (risky); " + "8: allocations are sometimes aligned right to 8-byte boundary; " + "9: allocations are always aligned right to 8-byte boundary." + ) +HWASAN_FLAG(bool, free_checks_tail_magic, 1, + "If set, free() will check the magic values " + "to the right of the allocated object " + "if the allocation size is not a divident of the granule size") +HWASAN_FLAG( + int, max_free_fill_size, 0, + "HWASan allocator flag. max_free_fill_size is the maximal amount of " + "bytes that will be filled with free_fill_byte during free.") +HWASAN_FLAG(int, malloc_fill_byte, 0xbe, + "Value used to fill the newly allocated memory.") +HWASAN_FLAG(int, free_fill_byte, 0x55, + "Value used to fill deallocated memory.") +HWASAN_FLAG(int, heap_history_size, 1023, + "The number of heap (de)allocations remembered per thread. " + "Affects the quality of heap-related reports, but not the ability " + "to find bugs.") +HWASAN_FLAG(bool, export_memory_stats, true, + "Export up-to-date memory stats through /proc") +HWASAN_FLAG(int, stack_history_size, 1024, + "The number of stack frames remembered per thread. " + "Affects the quality of stack-related reports, but not the ability " + "to find bugs.") diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc index 66aab95db56..fb0dcb8905c 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc @@ -1,4 +1,4 @@ -//===-- hwasan_interceptors.cc ----------------------------------------------===// +//===-- hwasan_interceptors.cc --------------------------------------------===// // // The LLVM Compiler Infrastructure // @@ -17,6 +17,7 @@ #include "interception/interception.h" #include "hwasan.h" +#include "hwasan_allocator.h" #include "hwasan_mapping.h" #include "hwasan_thread.h" #include "hwasan_poisoning.h" @@ -44,24 +45,19 @@ using __sanitizer::atomic_load; using __sanitizer::atomic_store; using __sanitizer::atomic_uintptr_t; -DECLARE_REAL(SIZE_T, strlen, const char *s) -DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen) -DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n) -DECLARE_REAL(void *, memset, void *dest, int c, uptr n) - bool IsInInterceptorScope() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); return t && t->InInterceptorScope(); } struct InterceptorScope { InterceptorScope() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); if (t) t->EnterInterceptorScope(); } ~InterceptorScope() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); if (t) t->LeaveInterceptorScope(); } @@ -92,66 +88,24 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { } while (0) - -#define HWASAN_READ_RANGE(ctx, offset, size) \ - CHECK_UNPOISONED(offset, size) -#define HWASAN_WRITE_RANGE(ctx, offset, size) \ - CHECK_UNPOISONED(offset, size) - - - -// Check that [x, x+n) range is unpoisoned. -#define CHECK_UNPOISONED_0(x, n) \ - do { \ - sptr __offset = __hwasan_test_shadow(x, n); \ - if (__hwasan::IsInSymbolizer()) break; \ - if (__offset >= 0) { \ - GET_CALLER_PC_BP_SP; \ - (void)sp; \ - ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \ - __hwasan::PrintWarning(pc, bp); \ - if (__hwasan::flags()->halt_on_error) { \ - Printf("Exiting\n"); \ - Die(); \ - } \ - } \ - } while (0) - -// Check that [x, x+n) range is unpoisoned unless we are in a nested -// interceptor. -#define CHECK_UNPOISONED(x, n) \ - do { \ - if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \ - } while (0) - -#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \ - CHECK_UNPOISONED((x), \ - common_flags()->strict_string_checks ? (len) + 1 : (n) ) - - -INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { +int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; CHECK_NE(memptr, 0); int res = hwasan_posix_memalign(memptr, alignment, size, &stack); return res; } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) { +void * __sanitizer_memalign(uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_memalign(alignment, size, &stack); } -#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign) -#else -#define HWASAN_MAYBE_INTERCEPT_MEMALIGN -#endif -INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) { +void * __sanitizer_aligned_alloc(uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_aligned_alloc(alignment, size, &stack); } -INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { +void * __sanitizer___libc_memalign(uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; void *ptr = hwasan_memalign(alignment, size, &stack); if (ptr) @@ -159,80 +113,47 @@ INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { return ptr; } -INTERCEPTOR(void *, valloc, SIZE_T size) { +void * __sanitizer_valloc(uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_valloc(size, &stack); } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void *, pvalloc, SIZE_T size) { +void * __sanitizer_pvalloc(uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_pvalloc(size, &stack); } -#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc) -#else -#define HWASAN_MAYBE_INTERCEPT_PVALLOC -#endif -INTERCEPTOR(void, free, void *ptr) { +void __sanitizer_free(void *ptr) { GET_MALLOC_STACK_TRACE; if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - HwasanDeallocate(&stack, ptr); + hwasan_free(ptr, &stack); } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void, cfree, void *ptr) { +void __sanitizer_cfree(void *ptr) { GET_MALLOC_STACK_TRACE; if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - HwasanDeallocate(&stack, ptr); + hwasan_free(ptr, &stack); } -#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) -#else -#define HWASAN_MAYBE_INTERCEPT_CFREE -#endif -INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { +uptr __sanitizer_malloc_usable_size(const void *ptr) { return __sanitizer_get_allocated_size(ptr); } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -// This function actually returns a struct by value, but we can't unpoison a -// temporary! The following is equivalent on all supported platforms but -// aarch64 (which uses a different register for sret value). We have a test -// to confirm that. -INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { -#ifdef __aarch64__ - uptr r8; - asm volatile("mov %0,x8" : "=r" (r8)); - sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); -#endif - REAL(memset)(sret, 0, sizeof(*sret)); +struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { + __sanitizer_struct_mallinfo sret; + internal_memset(&sret, 0, sizeof(sret)); + return sret; } -#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) -#else -#define HWASAN_MAYBE_INTERCEPT_MALLINFO -#endif -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; +int __sanitizer_mallopt(int cmd, int value) { + return 0; } -#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) -#else -#define HWASAN_MAYBE_INTERCEPT_MALLOPT -#endif -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void, malloc_stats, void) { +void __sanitizer_malloc_stats(void) { // FIXME: implement, but don't call REAL(malloc_stats)! } -#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats) -#else -#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS -#endif - -INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { +void * __sanitizer_calloc(uptr nmemb, uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_inited)) // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. @@ -240,7 +161,7 @@ INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { return hwasan_calloc(nmemb, size, &stack); } -INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { +void * __sanitizer_realloc(void *ptr, uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(IsInDlsymAllocPool(ptr))) { uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; @@ -258,7 +179,7 @@ INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { return hwasan_realloc(ptr, size, &stack); } -INTERCEPTOR(void *, malloc, SIZE_T size) { +void * __sanitizer_malloc(uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_init_is_running)) ENSURE_HWASAN_INITED(); @@ -268,48 +189,44 @@ INTERCEPTOR(void *, malloc, SIZE_T size) { return hwasan_malloc(size, &stack); } -template -static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T sz, int prot, - int flags, int fd, OFF64_T off) { - if (addr && !MEM_IS_APP(addr)) { - if (flags & map_fixed) { - errno = errno_EINVAL; - return (void *)-1; - } else { - addr = nullptr; - } - } - return real_mmap(addr, sz, prot, flags, fd, off); -} +#if HWASAN_WITH_INTERCEPTORS +#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ + ALIAS("__sanitizer_" #FN); \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ + ARGS) ALIAS("__sanitizer_" #FN) -extern "C" int pthread_attr_init(void *attr); -extern "C" int pthread_attr_destroy(void *attr); +INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, + SIZE_T size); +INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, free, void *ptr); +INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); +INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); +INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); -static void *HwasanThreadStartFunc(void *arg) { - HwasanThread *t = (HwasanThread *)arg; - SetCurrentThread(t); - return t->ThreadStart(); -} +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, cfree, void *ptr); +INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); +INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); +INTERCEPTOR_ALIAS(void, malloc_stats, void); +#endif +#endif // HWASAN_WITH_INTERCEPTORS -INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), - void * param) { - ENSURE_HWASAN_INITED(); // for GetTlsSize() - __sanitizer_pthread_attr_t myattr; - if (!attr) { - pthread_attr_init(&myattr); - attr = &myattr; - } - AdjustStackSize(attr); - - HwasanThread *t = HwasanThread::Create(callback, param); - - int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t); - - if (attr == &myattr) - pthread_attr_destroy(&myattr); +#if HWASAN_WITH_INTERCEPTORS && !defined(__aarch64__) +INTERCEPTOR(int, pthread_create, void *th, void *attr, + void *(*callback)(void *), void *param) { + ScopedTaggingDisabler disabler; + int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr), + callback, param); return res; } +#endif static void BeforeFork() { StackDepotLockAll(); @@ -341,137 +258,21 @@ int OnExit() { } // namespace __hwasan -// A version of CHECK_UNPOISONED using a saved scope value. Used in common -// interceptors. -#define CHECK_UNPOISONED_CTX(ctx, x, n) \ - do { \ - if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \ - CHECK_UNPOISONED_0(x, n); \ - } while (0) - -#define HWASAN_INTERCEPT_FUNC(name) \ - do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ - VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \ - } while (0) - -#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \ - do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ - VReport( \ - 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ - } while (0) - -#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name) -#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ - HWASAN_INTERCEPT_FUNC_VER(name, ver) -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - CHECK_UNPOISONED_CTX(ctx, ptr, size) -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - CHECK_UNPOISONED_CTX(ctx, ptr, size) -#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \ - HWASAN_WRITE_RANGE(ctx, ptr, size) -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \ - ENSURE_HWASAN_INITED(); \ - HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \ - ctx = (void *)&hwasan_ctx; \ - (void)ctx; \ - InterceptorScope interceptor_scope; -#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ - do { \ - } while (false) // FIXME -#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ - do { \ - } while (false) // FIXME -#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) -#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() - -#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ - if (HwasanThread *t = GetCurrentThread()) { \ - *begin = t->tls_begin(); \ - *end = t->tls_end(); \ - } else { \ - *begin = *end = 0; \ - } - -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ - { \ - COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ - if (common_flags()->intercept_intrin && \ - MEM_IS_APP(GetAddressFromPointer(dst))) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - return REAL(memset)(dst, v, size); \ - } - -#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, fd, \ - offset) \ - do { \ - return mmap_interceptor(REAL(mmap), addr, length, prot, flags, fd, \ - offset); \ - } while (false) - -#include "sanitizer_common/sanitizer_platform_interceptors.h" -#include "sanitizer_common/sanitizer_common_interceptors.inc" -#include "sanitizer_common/sanitizer_signal_interceptors.inc" - -#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) -#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#include "sanitizer_common/sanitizer_common_syscalls.inc" -#include "sanitizer_common/sanitizer_syscalls_netbsd.inc" - - - namespace __hwasan { void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); - InitializeCommonInterceptors(); - InitializeSignalInterceptors(); - INTERCEPT_FUNCTION(posix_memalign); - HWASAN_MAYBE_INTERCEPT_MEMALIGN; - INTERCEPT_FUNCTION(__libc_memalign); - INTERCEPT_FUNCTION(valloc); - HWASAN_MAYBE_INTERCEPT_PVALLOC; - INTERCEPT_FUNCTION(malloc); - INTERCEPT_FUNCTION(calloc); + INTERCEPT_FUNCTION(fork); + +#if HWASAN_WITH_INTERCEPTORS +#if !defined(__aarch64__) + INTERCEPT_FUNCTION(pthread_create); +#endif INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); - HWASAN_MAYBE_INTERCEPT_CFREE; - INTERCEPT_FUNCTION(malloc_usable_size); - HWASAN_MAYBE_INTERCEPT_MALLINFO; - HWASAN_MAYBE_INTERCEPT_MALLOPT; - HWASAN_MAYBE_INTERCEPT_MALLOC_STATS; - INTERCEPT_FUNCTION(pthread_create); - INTERCEPT_FUNCTION(fork); +#endif inited = 1; } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h index b4e5c80904d..d3b2087d03c 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -1,4 +1,4 @@ -//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===// +//===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,9 +16,13 @@ #define HWASAN_INTERFACE_INTERNAL_H #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_shadow_init(); + SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_init(); @@ -32,6 +36,9 @@ using __sanitizer::u32; using __sanitizer::u16; using __sanitizer::u8; +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_init_frames(uptr, uptr); + SANITIZER_INTERFACE_ATTRIBUTE extern uptr __hwasan_shadow_memory_dynamic_address; @@ -90,6 +97,9 @@ void __hwasan_store16_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_memory(uptr p, u8 tag, uptr sz); +SANITIZER_INTERFACE_ATTRIBUTE +uptr __hwasan_tag_pointer(uptr p, u8 tag); + SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); @@ -104,6 +114,9 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_print_shadow(const void *x, uptr size); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_handle_longjmp(const void *sp_dst); + SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); @@ -128,6 +141,66 @@ void __hwasan_enable_allocator_tagging(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_disable_allocator_tagging(); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_thread_enter(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_thread_exit(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_print_memory_usage(); + +SANITIZER_INTERFACE_ATTRIBUTE +int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_memalign(uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_aligned_alloc(uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer___libc_memalign(uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_valloc(uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_pvalloc(uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_free(void *ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_cfree(void *ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_malloc_usable_size(const void *ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +__hwasan::__sanitizer_struct_mallinfo __sanitizer_mallinfo(); + +SANITIZER_INTERFACE_ATTRIBUTE +int __sanitizer_mallopt(int cmd, int value); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_malloc_stats(void); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_calloc(uptr nmemb, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_realloc(void *ptr, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_malloc(uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memcpy(void *dst, const void *src, uptr size); +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memset(void *s, int c, uptr n); +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memmove(void *dest, const void *src, uptr n); } // extern "C" #endif // HWASAN_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc index 5ab98dca594..5b0a8b4ac98 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc @@ -22,7 +22,9 @@ #include "hwasan_mapping.h" #include "hwasan_report.h" #include "hwasan_thread.h" +#include "hwasan_thread_list.h" +#include #include #include #include @@ -37,6 +39,11 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" +#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +SANITIZER_INTERFACE_ATTRIBUTE +THREADLOCAL uptr __hwasan_tls; +#endif + namespace __hwasan { static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { @@ -51,8 +58,6 @@ static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { size); Abort(); } - if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); - if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); } static void ProtectGap(uptr addr, uptr size) { @@ -103,55 +108,31 @@ static void PrintAddressSpaceLayout() { else CHECK_EQ(kHighShadowEnd + 1, kHighMemStart); PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow"); - if (SHADOW_OFFSET) { - if (kLowShadowEnd + 1 < kHighShadowStart) - PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); - else - CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); - PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); - if (kLowMemEnd + 1 < kLowShadowStart) - PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); - else - CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); - PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); - CHECK_EQ(0, kLowMemStart); - } else { - if (kLowMemEnd + 1 < kHighShadowStart) - PrintRange(kLowMemEnd + 1, kHighShadowStart - 1, "ShadowGap"); - else - CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); - PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); - CHECK_EQ(kLowShadowEnd + 1, kLowMemStart); - PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); - PrintRange(0, kLowShadowStart - 1, "ShadowGap"); - } + if (kLowShadowEnd + 1 < kHighShadowStart) + PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); + else + CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); + PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); + if (kLowMemEnd + 1 < kLowShadowStart) + PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); + else + CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); + PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); + CHECK_EQ(0, kLowMemStart); } static uptr GetHighMemEnd() { // HighMem covers the upper part of the address space. uptr max_address = GetMaxUserVirtualAddress(); - if (SHADOW_OFFSET) - // Adjust max address to make sure that kHighMemEnd and kHighMemStart are - // properly aligned: - max_address |= SHADOW_GRANULARITY * GetMmapGranularity() - 1; + // Adjust max address to make sure that kHighMemEnd and kHighMemStart are + // properly aligned: + max_address |= (GetMmapGranularity() << kShadowScale) - 1; return max_address; } static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { - // Set the shadow memory address to uninitialized. - __hwasan_shadow_memory_dynamic_address = kDefaultShadowSentinel; - uptr shadow_start = SHADOW_OFFSET; - // Detect if a dynamic shadow address must be used and find the available - // location when necessary. When dynamic address is used, the macro - // kLowShadowBeg expands to __hwasan_shadow_memory_dynamic_address which - // was just set to kDefaultShadowSentinel. - if (shadow_start == kDefaultShadowSentinel) { - __hwasan_shadow_memory_dynamic_address = 0; - CHECK_EQ(0, SHADOW_OFFSET); - shadow_start = FindDynamicShadowStart(shadow_size_bytes); - } - // Update the shadow memory address (potentially) used by instrumentation. - __hwasan_shadow_memory_dynamic_address = shadow_start; + __hwasan_shadow_memory_dynamic_address = + FindDynamicShadowStart(shadow_size_bytes); } bool InitShadow() { @@ -159,29 +140,23 @@ bool InitShadow() { kHighMemEnd = GetHighMemEnd(); // Determine shadow memory base offset. - InitializeShadowBaseAddress(MEM_TO_SHADOW_SIZE(kHighMemEnd)); + InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd)); // Place the low memory first. - if (SHADOW_OFFSET) { - kLowMemEnd = SHADOW_OFFSET - 1; - kLowMemStart = 0; - } else { - // LowMem covers as much of the first 4GB as possible. - kLowMemEnd = (1UL << 32) - 1; - kLowMemStart = MEM_TO_SHADOW(kLowMemEnd) + 1; - } + kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1; + kLowMemStart = 0; // Define the low shadow based on the already placed low memory. - kLowShadowEnd = MEM_TO_SHADOW(kLowMemEnd); - kLowShadowStart = SHADOW_OFFSET ? SHADOW_OFFSET : MEM_TO_SHADOW(kLowMemStart); + kLowShadowEnd = MemToShadow(kLowMemEnd); + kLowShadowStart = __hwasan_shadow_memory_dynamic_address; // High shadow takes whatever memory is left up there (making sure it is not // interfering with low memory in the fixed case). - kHighShadowEnd = MEM_TO_SHADOW(kHighMemEnd); - kHighShadowStart = Max(kLowMemEnd, MEM_TO_SHADOW(kHighShadowEnd)) + 1; + kHighShadowEnd = MemToShadow(kHighMemEnd); + kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1; // High memory starts where allocated shadow allows. - kHighMemStart = SHADOW_TO_MEM(kHighShadowStart); + kHighMemStart = ShadowToMem(kHighShadowStart); // Check the sanity of the defined memory ranges (there might be gaps). CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); @@ -190,10 +165,7 @@ bool InitShadow() { CHECK_GT(kHighShadowStart, kLowMemEnd); CHECK_GT(kLowMemEnd, kLowMemStart); CHECK_GT(kLowShadowEnd, kLowShadowStart); - if (SHADOW_OFFSET) - CHECK_GT(kLowShadowStart, kLowMemEnd); - else - CHECK_GT(kLowMemEnd, kLowShadowStart); + CHECK_GT(kLowShadowStart, kLowMemEnd); if (Verbosity()) PrintAddressSpaceLayout(); @@ -204,21 +176,43 @@ bool InitShadow() { // Protect all the gaps. ProtectGap(0, Min(kLowMemStart, kLowShadowStart)); - if (SHADOW_OFFSET) { - if (kLowMemEnd + 1 < kLowShadowStart) - ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); - if (kLowShadowEnd + 1 < kHighShadowStart) - ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); - } else { - if (kLowMemEnd + 1 < kHighShadowStart) - ProtectGap(kLowMemEnd + 1, kHighShadowStart - kLowMemEnd - 1); - } + if (kLowMemEnd + 1 < kLowShadowStart) + ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); + if (kLowShadowEnd + 1 < kHighShadowStart) + ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); if (kHighShadowEnd + 1 < kHighMemStart) ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1); return true; } +void InitThreads() { + CHECK(__hwasan_shadow_memory_dynamic_address); + uptr guard_page_size = GetMmapGranularity(); + uptr thread_space_start = + __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment); + uptr thread_space_end = + __hwasan_shadow_memory_dynamic_address - guard_page_size; + ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1, + "hwasan threads"); + ProtectGap(thread_space_end, + __hwasan_shadow_memory_dynamic_address - thread_space_end); + InitThreadList(thread_space_start, thread_space_end - thread_space_start); +} + +static void MadviseShadowRegion(uptr beg, uptr end) { + uptr size = end - beg + 1; + if (common_flags()->no_huge_pages_for_shadow) + NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(beg, size); +} + +void MadviseShadow() { + MadviseShadowRegion(kLowShadowStart, kLowShadowEnd); + MadviseShadowRegion(kHighShadowStart, kHighShadowEnd); +} + bool MemIsApp(uptr p) { CHECK(GetTagFromPointer(p) == 0); return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); @@ -240,37 +234,81 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 +extern "C" void __hwasan_thread_enter() { + hwasanThreadList().CreateCurrentThread(); +} + +extern "C" void __hwasan_thread_exit() { + Thread *t = GetCurrentThread(); + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + if (t) + hwasanThreadList().ReleaseThread(t); +} + +#if HWASAN_WITH_INTERCEPTORS static pthread_key_t tsd_key; static bool tsd_key_inited = false; -void HwasanTSDInit(void (*destructor)(void *tsd)) { - CHECK(!tsd_key_inited); - tsd_key_inited = true; - CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); -} - -HwasanThread *GetCurrentThread() { - return (HwasanThread*)pthread_getspecific(tsd_key); -} - -void SetCurrentThread(HwasanThread *t) { - // Make sure that HwasanTSDDtor gets called at the end. - CHECK(tsd_key_inited); - // Make sure we do not reset the current HwasanThread. - CHECK_EQ(0, pthread_getspecific(tsd_key)); - pthread_setspecific(tsd_key, (void *)t); +void HwasanTSDThreadInit() { + if (tsd_key_inited) + CHECK_EQ(0, pthread_setspecific(tsd_key, + (void *)GetPthreadDestructorIterations())); } void HwasanTSDDtor(void *tsd) { - HwasanThread *t = (HwasanThread*)tsd; - if (t->destructor_iterations_ > 1) { - t->destructor_iterations_--; - CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); + uptr iterations = (uptr)tsd; + if (iterations > 1) { + CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1))); return; } - // Make sure that signal handler can not see a stale current thread pointer. - atomic_signal_fence(memory_order_seq_cst); - HwasanThread::TSDDtor(tsd); + __hwasan_thread_exit(); +} + +void HwasanTSDInit() { + CHECK(!tsd_key_inited); + tsd_key_inited = true; + CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor)); +} +#else +void HwasanTSDInit() {} +void HwasanTSDThreadInit() {} +#endif + +#if SANITIZER_ANDROID +uptr *GetCurrentThreadLongPtr() { + return (uptr *)get_android_tls_ptr(); +} +#else +uptr *GetCurrentThreadLongPtr() { + return &__hwasan_tls; +} +#endif + +#if SANITIZER_ANDROID +void AndroidTestTlsSlot() { + uptr kMagicValue = 0x010203040A0B0C0D; + *(uptr *)get_android_tls_ptr() = kMagicValue; + dlerror(); + if (*(uptr *)get_android_tls_ptr() != kMagicValue) { + Printf( + "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used " + "for dlerror().\n"); + Die(); + } +} +#else +void AndroidTestTlsSlot() {} +#endif + +Thread *GetCurrentThread() { + uptr *ThreadLong = GetCurrentThreadLongPtr(); +#if HWASAN_WITH_INTERCEPTORS + if (!*ThreadLong) + __hwasan_thread_enter(); +#endif + auto *R = (StackAllocationsRingBuffer *)ThreadLong; + return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next())); } struct AccessInfo { @@ -340,14 +378,13 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { BufferedStackTrace *stack = stack_buffer.data(); stack->Reset(); SignalContext sig{info, uc}; - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc, - common_flags()->fast_unwind_on_fatal); - - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store); + GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), + sig.bp, uc, common_flags()->fast_unwind_on_fatal); ++hwasan_report_count; - if (flags()->halt_on_error || !ai.recover) - Die(); + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); #if defined(__aarch64__) uc->uc_mcontext.pc += 4; @@ -360,8 +397,8 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, - common_flags()->fast_unwind_on_fatal); + GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), + sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } void HwasanOnDeadlySignal(int signo, void *info, void *context) { diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h b/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h index 650a5aefcb2..e5e23dc6033 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h @@ -16,68 +16,41 @@ #define HWASAN_MAPPING_H #include "sanitizer_common/sanitizer_internal_defs.h" +#include "hwasan_interface_internal.h" -// Typical mapping on Linux/x86_64 with fixed shadow mapping: -// || [0x080000000000, 0x7fffffffffff] || HighMem || -// || [0x008000000000, 0x07ffffffffff] || HighShadow || -// || [0x000100000000, 0x007fffffffff] || ShadowGap || -// || [0x000010000000, 0x0000ffffffff] || LowMem || -// || [0x000001000000, 0x00000fffffff] || LowShadow || -// || [0x000000000000, 0x000000ffffff] || ShadowGap || -// -// and with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]: +// Typical mapping on Linux/x86_64: +// with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]: // || [0x7f0d59f40000, 0x7fffffffffff] || HighMem || // || [0x7efe2f934000, 0x7f0d59f3ffff] || HighShadow || // || [0x7e7e2f934000, 0x7efe2f933fff] || ShadowGap || // || [0x770d59f40000, 0x7e7e2f933fff] || LowShadow || // || [0x000000000000, 0x770d59f3ffff] || LowMem || -// Typical mapping on Android/AArch64 (39-bit VMA): -// || [0x001000000000, 0x007fffffffff] || HighMem || -// || [0x000800000000, 0x000fffffffff] || ShadowGap || -// || [0x000100000000, 0x0007ffffffff] || HighShadow || -// || [0x000010000000, 0x0000ffffffff] || LowMem || -// || [0x000001000000, 0x00000fffffff] || LowShadow || -// || [0x000000000000, 0x000000ffffff] || ShadowGap || -// -// and with dynamic shadow mapped: [0x007477480000, 0x007c77480000]: +// Typical mapping on Android/AArch64 +// with dynamic shadow mapped: [0x007477480000, 0x007c77480000]: // || [0x007c77480000, 0x007fffffffff] || HighMem || // || [0x007c3ebc8000, 0x007c7747ffff] || HighShadow || // || [0x007bbebc8000, 0x007c3ebc7fff] || ShadowGap || // || [0x007477480000, 0x007bbebc7fff] || LowShadow || // || [0x000000000000, 0x00747747ffff] || LowMem || -static constexpr __sanitizer::u64 kDefaultShadowSentinel = ~(__sanitizer::u64)0; - // Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th). -constexpr __sanitizer::uptr kShadowScale = 4; -constexpr __sanitizer::uptr kShadowAlignment = 1ULL << kShadowScale; - -#if SANITIZER_ANDROID -# define HWASAN_FIXED_MAPPING 0 -#else -# define HWASAN_FIXED_MAPPING 1 -#endif - -#if HWASAN_FIXED_MAPPING -# define SHADOW_OFFSET (0) -# define HWASAN_PREMAP_SHADOW 0 -#else -# define SHADOW_OFFSET (__hwasan_shadow_memory_dynamic_address) -# define HWASAN_PREMAP_SHADOW 1 -#endif - -#define SHADOW_GRANULARITY (1ULL << kShadowScale) - -#define MEM_TO_SHADOW(mem) (((uptr)(mem) >> kShadowScale) + SHADOW_OFFSET) -#define SHADOW_TO_MEM(shadow) (((uptr)(shadow) - SHADOW_OFFSET) << kShadowScale) - -#define MEM_TO_SHADOW_SIZE(size) ((uptr)(size) >> kShadowScale) - -#define MEM_IS_APP(mem) MemIsApp((uptr)(mem)) +constexpr uptr kShadowScale = 4; +constexpr uptr kShadowAlignment = 1ULL << kShadowScale; namespace __hwasan { +inline uptr MemToShadow(uptr untagged_addr) { + return (untagged_addr >> kShadowScale) + + __hwasan_shadow_memory_dynamic_address; +} +inline uptr ShadowToMem(uptr shadow_addr) { + return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale; +} +inline uptr MemToShadowSize(uptr size) { + return size >> kShadowScale; +} + bool MemIsApp(uptr p); } // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_memintrinsics.cc b/contrib/compiler-rt/lib/hwasan/hwasan_memintrinsics.cc new file mode 100644 index 00000000000..9cb844e45c4 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_memintrinsics.cc @@ -0,0 +1,45 @@ +//===-- hwasan_memintrinsics.cc ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file is a part of HWAddressSanitizer and contains HWASAN versions of +/// memset, memcpy and memmove +/// +//===----------------------------------------------------------------------===// + +#include +#include "hwasan.h" +#include "hwasan_checks.h" +#include "hwasan_flags.h" +#include "hwasan_interface_internal.h" +#include "sanitizer_common/sanitizer_libc.h" + +using namespace __hwasan; + +void *__hwasan_memset(void *block, int c, uptr size) { + CheckAddressSized( + reinterpret_cast(block), size); + return memset(UntagPtr(block), c, size); +} + +void *__hwasan_memcpy(void *to, const void *from, uptr size) { + CheckAddressSized( + reinterpret_cast(to), size); + CheckAddressSized( + reinterpret_cast(from), size); + return memcpy(UntagPtr(to), UntagPtr(from), size); +} + +void *__hwasan_memmove(void *to, const void *from, uptr size) { + CheckAddressSized( + reinterpret_cast(to), size); + CheckAddressSized( + reinterpret_cast(from), size); + return memmove(UntagPtr(to), UntagPtr(from), size); +} diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc index 63ca74edd48..f2e8faf5da7 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc @@ -51,7 +51,7 @@ void *operator new[](size_t size, std::nothrow_t const&) { #define OPERATOR_DELETE_BODY \ GET_MALLOC_STACK_TRACE; \ - if (ptr) HwasanDeallocate(&stack, ptr) + if (ptr) hwasan_free(ptr, &stack) INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc index b99d8ed0be7..9c8e16b12ad 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc @@ -1,4 +1,4 @@ -//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===// +//===-- hwasan_poisoning.cc -------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -22,8 +22,8 @@ namespace __hwasan { uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { CHECK(IsAligned(p, kShadowAlignment)); CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MEM_TO_SHADOW(p); - uptr shadow_size = MEM_TO_SHADOW_SIZE(size); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); internal_memset((void *)shadow_start, tag, shadow_size); return AddTagToPointer(p, tag); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h index b44a91f975f..0dbf9d8ed4e 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h @@ -1,4 +1,4 @@ -//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===// +//===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc index 16e9016ea35..ea3e4096dae 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc @@ -15,18 +15,64 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_thread.h" +#include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_mutex.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_stacktrace_printer.h" #include "sanitizer_common/sanitizer_symbolizer.h" using namespace __sanitizer; namespace __hwasan { +class ScopedReport { + public: + ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { + BlockingMutexLock lock(&error_message_lock_); + error_message_ptr_ = fatal ? &error_message_ : nullptr; + } + + ~ScopedReport() { + BlockingMutexLock lock(&error_message_lock_); + if (fatal) { + SetAbortMessage(error_message_.data()); + Die(); + } + error_message_ptr_ = nullptr; + } + + static void MaybeAppendToErrorMessage(const char *msg) { + BlockingMutexLock lock(&error_message_lock_); + if (!error_message_ptr_) + return; + uptr len = internal_strlen(msg); + uptr old_size = error_message_ptr_->size(); + error_message_ptr_->resize(old_size + len); + // overwrite old trailing '\0', keep new trailing '\0' untouched. + internal_memcpy(&(*error_message_ptr_)[old_size - 1], msg, len); + } + private: + ScopedErrorReportLock error_report_lock_; + InternalMmapVector error_message_; + bool fatal; + + static InternalMmapVector *error_message_ptr_; + static BlockingMutex error_message_lock_; +}; + +InternalMmapVector *ScopedReport::error_message_ptr_; +BlockingMutex ScopedReport::error_message_lock_; + +// If there is an active ScopedReport, append to its error message. +void AppendToErrorMessageBuffer(const char *buffer) { + ScopedReport::MaybeAppendToErrorMessage(buffer); +} + static StackTrace GetStackTraceFromId(u32 id) { CHECK(id); StackTrace res = StackDepotGet(id); @@ -34,100 +80,344 @@ static StackTrace GetStackTraceFromId(u32 id) { return res; } +// A RAII object that holds a copy of the current thread stack ring buffer. +// The actual stack buffer may change while we are iterating over it (for +// example, Printf may call syslog() which can itself be built with hwasan). +class SavedStackAllocations { + public: + SavedStackAllocations(StackAllocationsRingBuffer *rb) { + uptr size = rb->size() * sizeof(uptr); + void *storage = + MmapAlignedOrDieOnFatalError(size, size * 2, "saved stack allocations"); + new (&rb_) StackAllocationsRingBuffer(*rb, storage); + } + + ~SavedStackAllocations() { + StackAllocationsRingBuffer *rb = get(); + UnmapOrDie(rb->StartOfStorage(), rb->size() * sizeof(uptr)); + } + + StackAllocationsRingBuffer *get() { + return (StackAllocationsRingBuffer *)&rb_; + } + + private: + uptr rb_; +}; + class Decorator: public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() { } + const char *Access() { return Blue(); } const char *Allocation() const { return Magenta(); } const char *Origin() const { return Magenta(); } const char *Name() const { return Green(); } + const char *Location() { return Green(); } + const char *Thread() { return Green(); } }; -struct HeapAddressDescription { - uptr addr; - u32 alloc_stack_id; - u32 free_stack_id; - - void Print() const { - Decorator d; - if (free_stack_id) { - Printf("%sfreed here:%s\n", d.Allocation(), d.Default()); - GetStackTraceFromId(free_stack_id).Print(); - Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default()); - } else { - Printf("%sallocated here:%s\n", d.Allocation(), d.Default()); +// Returns the index of the rb element that matches tagged_addr (plus one), +// or zero if found nothing. +uptr FindHeapAllocation(HeapAllocationsRingBuffer *rb, + uptr tagged_addr, + HeapAllocationRecord *har) { + if (!rb) return 0; + for (uptr i = 0, size = rb->size(); i < size; i++) { + auto h = (*rb)[i]; + if (h.tagged_addr <= tagged_addr && + h.tagged_addr + h.requested_size > tagged_addr) { + *har = h; + return i + 1; } - GetStackTraceFromId(alloc_stack_id).Print(); } -}; - -bool GetHeapAddressInformation(uptr addr, uptr access_size, - HeapAddressDescription *description) { - HwasanChunkView chunk = FindHeapChunkByAddress(addr); - if (!chunk.IsValid()) - return false; - description->addr = addr; - description->alloc_stack_id = chunk.GetAllocStackId(); - description->free_stack_id = chunk.GetFreeStackId(); - return true; + return 0; } -void PrintAddressDescription(uptr addr, uptr access_size) { - HeapAddressDescription heap_description; - if (GetHeapAddressInformation(addr, access_size, &heap_description)) { - heap_description.Print(); - return; - } - // We exhausted our possibilities. Bail out. - Printf("HWAddressSanitizer can not describe address in more detail.\n"); -} - -void ReportInvalidAccess(StackTrace *stack, u32 origin) { - ScopedErrorReportLock l; - +void PrintAddressDescription( + uptr tagged_addr, uptr access_size, + StackAllocationsRingBuffer *current_stack_allocations) { Decorator d; - Printf("%s", d.Warning()); - Report("WARNING: HWAddressSanitizer: invalid access\n"); - Printf("%s", d.Default()); - stack->Print(); - ReportErrorSummary("invalid-access", stack); + int num_descriptions_printed = 0; + uptr untagged_addr = UntagAddr(tagged_addr); + + // Print some very basic information about the address, if it's a heap. + HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); + if (uptr beg = chunk.Beg()) { + uptr size = chunk.ActualSize(); + Printf("%s[%p,%p) is a %s %s heap chunk; " + "size: %zd offset: %zd\n%s", + d.Location(), + beg, beg + size, + chunk.FromSmallHeap() ? "small" : "large", + chunk.IsAllocated() ? "allocated" : "unallocated", + size, untagged_addr - beg, + d.Default()); + } + + // Check if this looks like a heap buffer overflow by scanning + // the shadow left and right and looking for the first adjacent + // object with a different memory tag. If that tag matches addr_tag, + // check the allocator if it has a live chunk there. + tag_t addr_tag = GetTagFromPointer(tagged_addr); + tag_t *tag_ptr = reinterpret_cast(MemToShadow(untagged_addr)); + if (*tag_ptr != addr_tag) { // should be true usually. + tag_t *left = tag_ptr, *right = tag_ptr; + // scan left. + for (int i = 0; i < 1000 && *left == *tag_ptr; i++, left--){} + // scan right. + for (int i = 0; i < 1000 && *right == *tag_ptr; i++, right++){} + // Chose the object that has addr_tag and that is closer to addr. + tag_t *candidate = nullptr; + if (*right == addr_tag && *left == addr_tag) + candidate = right - tag_ptr < tag_ptr - left ? right : left; + else if (*right == addr_tag) + candidate = right; + else if (*left == addr_tag) + candidate = left; + + if (candidate) { + uptr mem = ShadowToMem(reinterpret_cast(candidate)); + HwasanChunkView chunk = FindHeapChunkByAddress(mem); + if (chunk.IsAllocated()) { + Printf("%s", d.Location()); + Printf( + "%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", + untagged_addr, + candidate == left ? untagged_addr - chunk.End() + : chunk.Beg() - untagged_addr, + candidate == right ? "left" : "right", chunk.UsedSize(), + chunk.Beg(), chunk.End()); + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + num_descriptions_printed++; + } + } + } + + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { + // Scan all threads' ring buffers to find if it's a heap-use-after-free. + HeapAllocationRecord har; + if (uptr D = FindHeapAllocation(t->heap_allocations(), tagged_addr, &har)) { + Printf("%s", d.Location()); + Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n", + untagged_addr, untagged_addr - UntagAddr(har.tagged_addr), + har.requested_size, UntagAddr(har.tagged_addr), + UntagAddr(har.tagged_addr) + har.requested_size); + Printf("%s", d.Allocation()); + Printf("freed by thread T%zd here:\n", t->unique_id()); + Printf("%s", d.Default()); + GetStackTraceFromId(har.free_context_id).Print(); + + Printf("%s", d.Allocation()); + Printf("previously allocated here:\n", t); + Printf("%s", d.Default()); + GetStackTraceFromId(har.alloc_context_id).Print(); + + // Print a developer note: the index of this heap object + // in the thread's deallocation ring buffer. + Printf("hwasan_dev_note_heap_rb_distance: %zd %zd\n", D, + flags()->heap_history_size); + + t->Announce(); + num_descriptions_printed++; + } + + // Very basic check for stack memory. + if (t->AddrIsInStack(untagged_addr)) { + Printf("%s", d.Location()); + Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, + t->unique_id()); + Printf("%s", d.Default()); + t->Announce(); + + // Temporary report section, needs to be improved. + Printf("Previously allocated frames:\n"); + auto *sa = (t == GetCurrentThread() && current_stack_allocations) + ? current_stack_allocations + : t->stack_allocations(); + uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); + InternalScopedString frame_desc(GetPageSizeCached() * 2); + for (uptr i = 0; i < frames; i++) { + uptr record = (*sa)[i]; + if (!record) + break; + uptr sp = (record >> 48) << 4; + uptr pc_mask = (1ULL << 48) - 1; + uptr pc = record & pc_mask; + if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { + frame_desc.append(" sp: 0x%zx pc: %p ", sp, pc); + RenderFrame(&frame_desc, "in %f %s:%l\n", 0, frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + frame->ClearAll(); + if (auto Descr = GetStackFrameDescr(pc)) + frame_desc.append(" %s\n", Descr); + } + Printf("%s", frame_desc.data()); + frame_desc.clear(); + } + + num_descriptions_printed++; + } + }); + + // Print the remaining threads, as an extra information, 1 line per thread. + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); }); + + if (!num_descriptions_printed) + // We exhausted our possibilities. Bail out. + Printf("HWAddressSanitizer can not describe address in more detail.\n"); } void ReportStats() {} -void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, - uptr size, uptr offset) { - ScopedErrorReportLock l; +static void PrintTagsAroundAddr(tag_t *tag_ptr) { + Printf( + "Memory tags around the buggy address (one tag corresponds to %zd " + "bytes):\n", kShadowAlignment); - Decorator d; - Printf("%s", d.Warning()); - Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n", - d.Warning(), d.Name(), what, d.Warning(), offset, start, size, - d.Default()); - PrintAddressDescription((uptr)start + offset, 1); - // if (__sanitizer::Verbosity()) - // DescribeMemoryRange(start, size); + const uptr row_len = 16; // better be power of two. + const uptr num_rows = 17; + tag_t *center_row_beg = reinterpret_cast( + RoundDownTo(reinterpret_cast(tag_ptr), row_len)); + tag_t *beg_row = center_row_beg - row_len * (num_rows / 2); + tag_t *end_row = center_row_beg + row_len * (num_rows / 2); + InternalScopedString s(GetPageSizeCached() * 8); + for (tag_t *row = beg_row; row < end_row; row += row_len) { + s.append("%s", row == center_row_beg ? "=>" : " "); + for (uptr i = 0; i < row_len; i++) { + s.append("%s", row + i == tag_ptr ? "[" : " "); + s.append("%02x", row[i]); + s.append("%s", row + i == tag_ptr ? "]" : " "); + } + s.append("%s\n", row == center_row_beg ? "<=" : " "); + } + Printf("%s", s.data()); } -void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store) { - ScopedErrorReportLock l; +void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { + ScopedReport R(flags()->halt_on_error); + uptr untagged_addr = UntagAddr(tagged_addr); + tag_t ptr_tag = GetTagFromPointer(tagged_addr); + tag_t *tag_ptr = reinterpret_cast(MemToShadow(untagged_addr)); + tag_t mem_tag = *tag_ptr; Decorator d; - Printf("%s", d.Warning()); - uptr address = GetAddressFromPointer(addr); - Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size, - address); - - tag_t ptr_tag = GetTagFromPointer(addr); - tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address); - Printf("pointer tag 0x%x\nmemory tag 0x%x\n", ptr_tag, mem_tag); + Printf("%s", d.Error()); + uptr pc = stack->size ? stack->trace[0] : 0; + const char *bug_type = "invalid-free"; + Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type, + untagged_addr, pc); + Printf("%s", d.Access()); + Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, mem_tag); Printf("%s", d.Default()); stack->Print(); - PrintAddressDescription(address, access_size); + PrintAddressDescription(tagged_addr, 0, nullptr); - ReportErrorSummary("tag-mismatch", stack); + PrintTagsAroundAddr(tag_ptr); + + ReportErrorSummary(bug_type, stack); +} + +void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, + uptr tail_size, const u8 *expected) { + ScopedReport R(flags()->halt_on_error); + Decorator d; + uptr untagged_addr = UntagAddr(tagged_addr); + Printf("%s", d.Error()); + const char *bug_type = "alocation-tail-overwritten"; + Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName, + bug_type, untagged_addr, untagged_addr + orig_size, orig_size); + Printf("\n%s", d.Default()); + stack->Print(); + HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); + if (chunk.Beg()) { + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + } + + InternalScopedString s(GetPageSizeCached() * 8); + CHECK_GT(tail_size, 0U); + CHECK_LT(tail_size, kShadowAlignment); + u8 *tail = reinterpret_cast(untagged_addr + orig_size); + s.append("Tail contains: "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) + s.append(".. "); + for (uptr i = 0; i < tail_size; i++) + s.append("%02x ", tail[i]); + s.append("\n"); + s.append("Expected: "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) + s.append(".. "); + for (uptr i = 0; i < tail_size; i++) + s.append("%02x ", expected[i]); + s.append("\n"); + s.append(" "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) + s.append(" "); + for (uptr i = 0; i < tail_size; i++) + s.append("%s ", expected[i] != tail[i] ? "^^" : " "); + + s.append("\nThis error occurs when a buffer overflow overwrites memory\n" + "to the right of a heap object, but within the %zd-byte granule, e.g.\n" + " char *x = new char[20];\n" + " x[25] = 42;\n" + "By default %s does not detect such bugs at the time of write,\n" + "but can detect them at the time of free/delete.\n" + "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0;\n" + "To enable checking at the time of access, set " + "HWASAN_OPTIONS=malloc_align_right to non-zero\n\n", + kShadowAlignment, SanitizerToolName); + Printf("%s", s.data()); + GetCurrentThread()->Announce(); + + tag_t *tag_ptr = reinterpret_cast(MemToShadow(untagged_addr)); + PrintTagsAroundAddr(tag_ptr); + + ReportErrorSummary(bug_type, stack); +} + +void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, + bool is_store, bool fatal) { + ScopedReport R(fatal); + SavedStackAllocations current_stack_allocations( + GetCurrentThread()->stack_allocations()); + + Decorator d; + Printf("%s", d.Error()); + uptr untagged_addr = UntagAddr(tagged_addr); + // TODO: when possible, try to print heap-use-after-free, etc. + const char *bug_type = "tag-mismatch"; + uptr pc = stack->size ? stack->trace[0] : 0; + Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type, + untagged_addr, pc); + + Thread *t = GetCurrentThread(); + + tag_t ptr_tag = GetTagFromPointer(tagged_addr); + tag_t *tag_ptr = reinterpret_cast(MemToShadow(untagged_addr)); + tag_t mem_tag = *tag_ptr; + Printf("%s", d.Access()); + Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n", + is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, + mem_tag, t->unique_id()); + Printf("%s", d.Default()); + + stack->Print(); + + PrintAddressDescription(tagged_addr, access_size, + current_stack_allocations.get()); + t->Announce(); + + PrintTagsAroundAddr(tag_ptr); + + ReportErrorSummary(bug_type, stack); } } // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.h b/contrib/compiler-rt/lib/hwasan/hwasan_report.h index bb33f1a8730..10fb20cc5be 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_report.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.h @@ -21,12 +21,12 @@ namespace __hwasan { -void ReportInvalidAccess(StackTrace *stack, u32 origin); void ReportStats(); -void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, - uptr size, uptr offset); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store); + bool is_store, bool fatal); +void ReportInvalidFree(StackTrace *stack, uptr addr); +void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, + uptr tail_size, const u8 *expected); void ReportAtExitStatistics(); diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc index b50c0fc76be..631c2813eb0 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc @@ -5,8 +5,11 @@ #include "hwasan_poisoning.h" #include "hwasan_interface_internal.h" +#include "sanitizer_common/sanitizer_file.h" +#include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" + namespace __hwasan { static u32 RandomSeed() { @@ -22,69 +25,70 @@ static u32 RandomSeed() { return seed; } -HwasanThread *HwasanThread::Create(thread_callback_t start_routine, - void *arg) { - uptr PageSize = GetPageSizeCached(); - uptr size = RoundUpTo(sizeof(HwasanThread), PageSize); - HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__); - thread->start_routine_ = start_routine; - thread->arg_ = arg; - thread->destructor_iterations_ = GetPthreadDestructorIterations(); - thread->random_state_ = flags()->random_tags ? RandomSeed() : 0; +void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { + static u64 unique_id; + unique_id_ = unique_id++; + random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; + if (auto sz = flags()->heap_history_size) + heap_allocations_ = HeapAllocationsRingBuffer::New(sz); - return thread; -} + HwasanTSDThreadInit(); // Only needed with interceptors. + uptr *ThreadLong = GetCurrentThreadLongPtr(); + // The following implicitly sets (this) as the current thread. + stack_allocations_ = new (ThreadLong) + StackAllocationsRingBuffer((void *)stack_buffer_start, stack_buffer_size); + // Check that it worked. + CHECK_EQ(GetCurrentThread(), this); -void HwasanThread::SetThreadStackAndTls() { - uptr tls_size = 0; - uptr stack_size = 0; - GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, - &tls_begin_, &tls_size); + // ScopedTaggingDisable needs GetCurrentThread to be set up. + ScopedTaggingDisabler disabler; + + uptr tls_size; + uptr stack_size; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); stack_top_ = stack_bottom_ + stack_size; tls_end_ = tls_begin_ + tls_size; - int local; - CHECK(AddrIsInStack((uptr)&local)); + if (stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + CHECK(MemIsApp(stack_bottom_)); + CHECK(MemIsApp(stack_top_ - 1)); + } + + if (flags()->verbose_threads) { + if (IsMainThread()) { + Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", + sizeof(Thread), heap_allocations_->SizeInBytes(), + stack_allocations_->size() * sizeof(uptr)); + } + Print("Creating : "); + } } -void HwasanThread::Init() { - SetThreadStackAndTls(); - CHECK(MEM_IS_APP(stack_bottom_)); - CHECK(MEM_IS_APP(stack_top_ - 1)); -} - -void HwasanThread::TSDDtor(void *tsd) { - HwasanThread *t = (HwasanThread*)tsd; - t->Destroy(); -} - -void HwasanThread::ClearShadowForThreadStackAndTLS() { - TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); +void Thread::ClearShadowForThreadStackAndTLS() { + if (stack_top_ != stack_bottom_) + TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) TagMemory(tls_begin_, tls_end_ - tls_begin_, 0); } -void HwasanThread::Destroy() { - malloc_storage().CommitBack(); +void Thread::Destroy() { + if (flags()->verbose_threads) + Print("Destroying: "); + AllocatorSwallowThreadLocalCache(allocator_cache()); ClearShadowForThreadStackAndTLS(); - uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached()); - UnmapOrDie(this, size); + if (heap_allocations_) + heap_allocations_->Delete(); DTLS_Destroy(); } -thread_return_t HwasanThread::ThreadStart() { - Init(); - - if (!start_routine_) { - // start_routine_ == 0 if we're on the main thread or on one of the - // OS X libdispatch worker threads. But nobody is supposed to call - // ThreadStart() for the worker threads. - return 0; - } - - thread_return_t res = start_routine_(arg_); - - return res; +void Thread::Print(const char *Prefix) { + Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix, + unique_id_, this, stack_bottom(), stack_top(), + stack_top() - stack_bottom(), + tls_begin(), tls_end()); } static u32 xorshift(u32 state) { @@ -95,7 +99,8 @@ static u32 xorshift(u32 state) { } // Generate a (pseudo-)random non-zero tag. -tag_t HwasanThread::GenerateRandomTag() { +tag_t Thread::GenerateRandomTag() { + if (tagging_disabled_) return 0; tag_t tag; do { if (flags()->random_tags) { diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h index 1e482adeac8..4830473f4aa 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h @@ -1,4 +1,4 @@ -//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===// +//===-- hwasan_thread.h -----------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,23 +16,23 @@ #include "hwasan_allocator.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_ring_buffer.h" namespace __hwasan { -class HwasanThread { - public: - static HwasanThread *Create(thread_callback_t start_routine, void *arg); - static void TSDDtor(void *tsd); - void Destroy(); +typedef __sanitizer::CompactRingBuffer StackAllocationsRingBuffer; - void Init(); // Should be called from the thread itself. - thread_return_t ThreadStart(); +class Thread { + public: + void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + void Destroy(); uptr stack_top() { return stack_top_; } uptr stack_bottom() { return stack_bottom_; } + uptr stack_size() { return stack_top() - stack_bottom(); } uptr tls_begin() { return tls_begin_; } uptr tls_end() { return tls_end_; } - bool IsMainThread() { return start_routine_ == nullptr; } + bool IsMainThread() { return unique_id_ == 0; } bool AddrIsInStack(uptr addr) { return addr >= stack_bottom_ && addr < stack_top_; @@ -50,19 +50,28 @@ class HwasanThread { void EnterInterceptorScope() { in_interceptor_scope_++; } void LeaveInterceptorScope() { in_interceptor_scope_--; } - HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } + AllocatorCache *allocator_cache() { return &allocator_cache_; } + HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; } + StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; } tag_t GenerateRandomTag(); - int destructor_iterations_; + void DisableTagging() { tagging_disabled_++; } + void EnableTagging() { tagging_disabled_--; } + bool TaggingIsDisabled() const { return tagging_disabled_; } + + u64 unique_id() const { return unique_id_; } + void Announce() { + if (announced_) return; + announced_ = true; + Print("Thread: "); + } private: - // NOTE: There is no HwasanThread constructor. It is allocated + // NOTE: There is no Thread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. - void SetThreadStackAndTls(); void ClearShadowForThreadStackAndTLS(); - thread_callback_t start_routine_; - void *arg_; + void Print(const char *prefix); uptr stack_top_; uptr stack_bottom_; uptr tls_begin_; @@ -75,11 +84,30 @@ class HwasanThread { u32 random_state_; u32 random_buffer_; - HwasanThreadLocalMallocStorage malloc_storage_; + AllocatorCache allocator_cache_; + HeapAllocationsRingBuffer *heap_allocations_; + StackAllocationsRingBuffer *stack_allocations_; + + static void InsertIntoThreadList(Thread *t); + static void RemoveFromThreadList(Thread *t); + Thread *next_; // All live threads form a linked list. + + u64 unique_id_; // counting from zero. + + u32 tagging_disabled_; // if non-zero, malloc uses zero tag in this thread. + + bool announced_; + + friend struct ThreadListHead; }; -HwasanThread *GetCurrentThread(); -void SetCurrentThread(HwasanThread *t); +Thread *GetCurrentThread(); +uptr *GetCurrentThreadLongPtr(); + +struct ScopedTaggingDisabler { + ScopedTaggingDisabler() { GetCurrentThread()->DisableTagging(); } + ~ScopedTaggingDisabler() { GetCurrentThread()->EnableTagging(); } +}; } // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.cc new file mode 100644 index 00000000000..a31eee84ed9 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.cc @@ -0,0 +1,15 @@ +#include "hwasan_thread_list.h" + +namespace __hwasan { +static ALIGNED(16) char thread_list_placeholder[sizeof(HwasanThreadList)]; +static HwasanThreadList *hwasan_thread_list; + +HwasanThreadList &hwasanThreadList() { return *hwasan_thread_list; } + +void InitThreadList(uptr storage, uptr size) { + CHECK(hwasan_thread_list == nullptr); + hwasan_thread_list = + new (thread_list_placeholder) HwasanThreadList(storage, size); +} + +} // namespace diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.h new file mode 100644 index 00000000000..53747b51fd6 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.h @@ -0,0 +1,200 @@ +//===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +// HwasanThreadList is a registry for live threads, as well as an allocator for +// HwasanThread objects and their stack history ring buffers. There are +// constraints on memory layout of the shadow region and CompactRingBuffer that +// are part of the ABI contract between compiler-rt and llvm. +// +// * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment. +// * All stack ring buffers are located within (2**kShadowBaseAlignment) +// sized region below and adjacent to the shadow region. +// * Each ring buffer has a size of (2**N)*4096 where N is in [0, 8), and is +// aligned to twice its size. The value of N can be different for each buffer. +// +// These constrains guarantee that, given an address A of any element of the +// ring buffer, +// A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1) +// is the address of the next element of that ring buffer (with wrap-around). +// And, with K = kShadowBaseAlignment, +// S = (A | ((1 << K) - 1)) + 1 +// (align up to kShadowBaseAlignment) is the start of the shadow region. +// +// These calculations are used in compiler instrumentation to update the ring +// buffer and obtain the base address of shadow using only two inputs: address +// of the current element of the ring buffer, and N (i.e. size of the ring +// buffer). Since the value of N is very limited, we pack both inputs into a +// single thread-local word as +// (1 << (N + 56)) | A +// See the implementation of class CompactRingBuffer, which is what is stored in +// said thread-local word. +// +// Note the unusual way of aligning up the address of the shadow: +// (A | ((1 << K) - 1)) + 1 +// It is only correct if A is not already equal to the shadow base address, but +// it saves 2 instructions on AArch64. + +#include "hwasan.h" +#include "hwasan_allocator.h" +#include "hwasan_flags.h" +#include "hwasan_thread.h" + +#include "sanitizer_common/sanitizer_placement_new.h" + +namespace __hwasan { + +static uptr RingBufferSize() { + uptr desired_bytes = flags()->stack_history_size * sizeof(uptr); + // FIXME: increase the limit to 8 once this bug is fixed: + // https://bugs.llvm.org/show_bug.cgi?id=39030 + for (int shift = 1; shift < 7; ++shift) { + uptr size = 4096 * (1ULL << shift); + if (size >= desired_bytes) + return size; + } + Printf("stack history size too large: %d\n", flags()->stack_history_size); + CHECK(0); + return 0; +} + +struct ThreadListHead { + Thread *list_; + + ThreadListHead() : list_(nullptr) {} + + void Push(Thread *t) { + t->next_ = list_; + list_ = t; + } + + Thread *Pop() { + Thread *t = list_; + if (t) + list_ = t->next_; + return t; + } + + void Remove(Thread *t) { + Thread **cur = &list_; + while (*cur != t) cur = &(*cur)->next_; + CHECK(*cur && "thread not found"); + *cur = (*cur)->next_; + } + + template + void ForEach(CB cb) { + Thread *t = list_; + while (t) { + cb(t); + t = t->next_; + } + } +}; + +struct ThreadStats { + uptr n_live_threads; + uptr total_stack_size; +}; + +class HwasanThreadList { + public: + HwasanThreadList(uptr storage, uptr size) + : free_space_(storage), + free_space_end_(storage + size), + ring_buffer_size_(RingBufferSize()) {} + + Thread *CreateCurrentThread() { + Thread *t; + { + SpinMutexLock l(&list_mutex_); + t = free_list_.Pop(); + if (t) + internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_); + else + t = AllocThread(); + live_list_.Push(t); + } + t->Init((uptr)(t + 1), ring_buffer_size_); + AddThreadStats(t); + return t; + } + + void ReleaseThread(Thread *t) { + // FIXME: madvise away the ring buffer? + RemoveThreadStats(t); + t->Destroy(); + SpinMutexLock l(&list_mutex_); + live_list_.Remove(t); + free_list_.Push(t); + } + + Thread *GetThreadByBufferAddress(uptr p) { + uptr align = ring_buffer_size_ * 2; + return (Thread *)(RoundDownTo(p, align) - sizeof(Thread)); + } + + uptr MemoryUsedPerThread() { + uptr res = sizeof(Thread) + ring_buffer_size_; + if (auto sz = flags()->heap_history_size) + res += HeapAllocationsRingBuffer::SizeInBytes(sz); + return res; + } + + template + void VisitAllLiveThreads(CB cb) { + SpinMutexLock l(&list_mutex_); + live_list_.ForEach(cb); + } + + void AddThreadStats(Thread *t) { + SpinMutexLock l(&stats_mutex_); + stats_.n_live_threads++; + stats_.total_stack_size += t->stack_size(); + } + + void RemoveThreadStats(Thread *t) { + SpinMutexLock l(&stats_mutex_); + stats_.n_live_threads--; + stats_.total_stack_size -= t->stack_size(); + } + + ThreadStats GetThreadStats() { + SpinMutexLock l(&stats_mutex_); + return stats_; + } + + private: + Thread *AllocThread() { + uptr align = ring_buffer_size_ * 2; + uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align); + free_space_ = ring_buffer_start + ring_buffer_size_; + CHECK(free_space_ <= free_space_end_ && "out of thread memory"); + return (Thread *)(ring_buffer_start - sizeof(Thread)); + } + + uptr free_space_; + uptr free_space_end_; + uptr ring_buffer_size_; + + ThreadListHead free_list_; + ThreadListHead live_list_; + SpinMutex list_mutex_; + + ThreadStats stats_; + SpinMutex stats_mutex_; +}; + +void InitThreadList(uptr storage, uptr size); +HwasanThreadList &hwasanThreadList(); + +} // namespace diff --git a/contrib/compiler-rt/lib/interception/interception.h b/contrib/compiler-rt/lib/interception/interception.h index ddd6ec20979..87b2365fd76 100644 --- a/contrib/compiler-rt/lib/interception/interception.h +++ b/contrib/compiler-rt/lib/interception/interception.h @@ -29,6 +29,7 @@ typedef __sanitizer::uptr SIZE_T; typedef __sanitizer::sptr SSIZE_T; typedef __sanitizer::sptr PTRDIFF_T; typedef __sanitizer::s64 INTMAX_T; +typedef __sanitizer::u64 UINTMAX_T; typedef __sanitizer::OFF_T OFF_T; typedef __sanitizer::OFF64_T OFF64_T; @@ -169,7 +170,7 @@ const interpose_substitution substitution_##func_name[] \ #elif !SANITIZER_MAC # define PTR_TO_REAL(x) real_##x # define REAL(x) __interception::PTR_TO_REAL(x) -# define FUNC_TYPE(x) x##_f +# define FUNC_TYPE(x) x##_type # define DECLARE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ diff --git a/contrib/compiler-rt/lib/interception/interception_linux.h b/contrib/compiler-rt/lib/interception/interception_linux.h index 942c25609cc..765a186e582 100644 --- a/contrib/compiler-rt/lib/interception/interception_linux.h +++ b/contrib/compiler-rt/lib/interception/interception_linux.h @@ -38,7 +38,7 @@ void *GetFuncAddrVer(const char *func_name, const char *ver); // Android, Solaris and OpenBSD do not have dlvsym #if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ - (::__interception::real_##func = (func##_f)( \ + (::__interception::real_##func = (func##_type)( \ unsigned long)::__interception::GetFuncAddrVer(#func, symver)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ diff --git a/contrib/compiler-rt/lib/interception/interception_win.cc b/contrib/compiler-rt/lib/interception/interception_win.cc index bd4ad7274dd..cd13827e585 100644 --- a/contrib/compiler-rt/lib/interception/interception_win.cc +++ b/contrib/compiler-rt/lib/interception/interception_win.cc @@ -223,8 +223,8 @@ static bool IsMemoryPadding(uptr address, uptr size) { return true; } -static const u8 kHintNop9Bytes[] = { - 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 +static const u8 kHintNop8Bytes[] = { + 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; template @@ -239,8 +239,8 @@ static bool FunctionHasPrefix(uptr address, const T &pattern) { static bool FunctionHasPadding(uptr address, uptr size) { if (IsMemoryPadding(address - size, size)) return true; - if (size <= sizeof(kHintNop9Bytes) && - FunctionHasPrefix(address, kHintNop9Bytes)) + if (size <= sizeof(kHintNop8Bytes) && + FunctionHasPrefix(address, kHintNop8Bytes)) return true; return false; } diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.cc b/contrib/compiler-rt/lib/lsan/lsan_allocator.cc index c58c3548002..1b338bd5973 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.cc @@ -34,9 +34,6 @@ static const uptr kMaxAllowedMallocSize = 4UL << 30; #else static const uptr kMaxAllowedMallocSize = 8UL << 30; #endif -typedef LargeMmapAllocator<> SecondaryAllocator; -typedef CombinedAllocator Allocator; static Allocator allocator; diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.h b/contrib/compiler-rt/lib/lsan/lsan_allocator.h index 7c70bb6d976..4c4e02fc090 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.h +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.h @@ -54,19 +54,25 @@ struct ChunkMetadata { defined(__arm__) static const uptr kRegionSizeLog = 20; static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; +template +using ByteMapASVT = + TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; +template struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = sizeof(ChunkMetadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __lsan::kRegionSizeLog; - typedef __lsan::ByteMap ByteMap; + using AddressSpaceView = AddressSpaceViewTy; + using ByteMap = __lsan::ByteMapASVT; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; -typedef SizeClassAllocator32 PrimaryAllocator; +template +using PrimaryAllocatorASVT = SizeClassAllocator32>; +using PrimaryAllocator = PrimaryAllocatorASVT; #elif defined(__x86_64__) || defined(__powerpc64__) # if defined(__powerpc64__) const uptr kAllocatorSpace = 0xa0000000000ULL; @@ -75,6 +81,7 @@ const uptr kAllocatorSize = 0x20000000000ULL; // 2T. const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. # endif +template struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; static const uptr kSpaceSize = kAllocatorSize; @@ -82,11 +89,30 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. typedef DefaultSizeClassMap SizeClassMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; }; -typedef SizeClassAllocator64 PrimaryAllocator; +template +using PrimaryAllocatorASVT = SizeClassAllocator64>; +using PrimaryAllocator = PrimaryAllocatorASVT; #endif -typedef SizeClassAllocatorLocalCache AllocatorCache; + +template +using AllocatorCacheASVT = + SizeClassAllocatorLocalCache>; +using AllocatorCache = AllocatorCacheASVT; + +template +using SecondaryAllocatorASVT = + LargeMmapAllocator; + +template +using AllocatorASVT = + CombinedAllocator, + AllocatorCacheASVT, + SecondaryAllocatorASVT>; +using Allocator = AllocatorASVT; AllocatorCache *GetAllocatorCache(); diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc index 2508c1dbd87..a355cea96c4 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" +#include "sanitizer_common/sanitizer_libc.h" #include "lsan_common.h" #if CAN_SANITIZE_LEAKS && SANITIZER_MAC @@ -116,7 +117,8 @@ static const char *kSkippedSecNames[] = { // Scans global variables for heap pointers. void ProcessGlobalRegions(Frontier *frontier) { - for (auto name : kSkippedSecNames) CHECK(ARRAY_SIZE(name) < kMaxSegName); + for (auto name : kSkippedSecNames) + CHECK(internal_strnlen(name, kMaxSegName + 1) <= kMaxSegName); MemoryMappingLayout memory_mapping(false); InternalMmapVector modules; @@ -142,12 +144,6 @@ void ProcessGlobalRegions(Frontier *frontier) { } void ProcessPlatformSpecificAllocations(Frontier *frontier) { - mach_port_name_t port; - if (task_for_pid(mach_task_self(), internal_getpid(), &port) - != KERN_SUCCESS) { - return; - } - unsigned depth = 1; vm_size_t size = 0; vm_address_t address = 0; @@ -158,7 +154,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { while (err == KERN_SUCCESS) { struct vm_region_submap_info_64 info; - err = vm_region_recurse_64(port, &address, &size, &depth, + err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth, (vm_region_info_t)&info, &count); uptr end_address = address + size; diff --git a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc index fde52e49616..a9bd2ba4231 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc @@ -153,7 +153,7 @@ INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { #define LSAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; + return 0; } #define LSAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) #else diff --git a/contrib/compiler-rt/lib/msan/msan.cc b/contrib/compiler-rt/lib/msan/msan.cc index 06bcbdf8869..ba2d5d59330 100644 --- a/contrib/compiler-rt/lib/msan/msan.cc +++ b/contrib/compiler-rt/lib/msan/msan.cc @@ -58,6 +58,10 @@ THREADLOCAL u32 __msan_retval_origin_tls; SANITIZER_INTERFACE_ATTRIBUTE ALIGNED(16) THREADLOCAL u64 __msan_va_arg_tls[kMsanParamTlsSize / sizeof(u64)]; +SANITIZER_INTERFACE_ATTRIBUTE +ALIGNED(16) +THREADLOCAL u32 __msan_va_arg_origin_tls[kMsanParamTlsSize / sizeof(u32)]; + SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64 __msan_va_arg_overflow_size_tls; @@ -277,6 +281,8 @@ void ScopedThreadLocalStateBackup::Restore() { internal_memset(__msan_param_tls, 0, sizeof(__msan_param_tls)); internal_memset(__msan_retval_tls, 0, sizeof(__msan_retval_tls)); internal_memset(__msan_va_arg_tls, 0, sizeof(__msan_va_arg_tls)); + internal_memset(__msan_va_arg_origin_tls, 0, + sizeof(__msan_va_arg_origin_tls)); if (__msan_get_track_origins()) { internal_memset(&__msan_retval_origin_tls, 0, diff --git a/contrib/compiler-rt/lib/msan/msan_allocator.cc b/contrib/compiler-rt/lib/msan/msan_allocator.cc index 36f0497a9d8..053ab028039 100644 --- a/contrib/compiler-rt/lib/msan/msan_allocator.cc +++ b/contrib/compiler-rt/lib/msan/msan_allocator.cc @@ -57,7 +57,8 @@ struct MsanMapUnmapCallback { static const uptr kMetadataSize = sizeof(Metadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - typedef __msan::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __msan::ByteMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -78,6 +79,7 @@ struct MsanMapUnmapCallback { typedef DefaultSizeClassMap SizeClassMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64 PrimaryAllocator; @@ -92,6 +94,7 @@ struct MsanMapUnmapCallback { typedef DefaultSizeClassMap SizeClassMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64 PrimaryAllocator; @@ -107,7 +110,8 @@ struct MsanMapUnmapCallback { static const uptr kMetadataSize = sizeof(Metadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - typedef __msan::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __msan::ByteMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc index b3429bcf06b..497f943a8a0 100644 --- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc +++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc @@ -34,11 +34,13 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" +#include "sanitizer_common/sanitizer_vector.h" #if SANITIZER_NETBSD #define fstat __fstat50 #define gettimeofday __gettimeofday50 #define getrusage __getrusage50 +#define tzset __tzset50 #endif #include @@ -249,11 +251,11 @@ INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { // temporary! The following is equivalent on all supported platforms but // aarch64 (which uses a different register for sret value). We have a test // to confirm that. -INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { +INTERCEPTOR(void, mallinfo, __sanitizer_struct_mallinfo *sret) { #ifdef __aarch64__ uptr r8; asm volatile("mov %0,x8" : "=r" (r8)); - sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); + sret = reinterpret_cast<__sanitizer_struct_mallinfo*>(r8); #endif REAL(memset)(sret, 0, sizeof(*sret)); __msan_unpoison(sret, sizeof(*sret)); @@ -265,7 +267,7 @@ INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; + return 0; } #define MSAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) #else @@ -1071,6 +1073,7 @@ extern char *tzname[2]; INTERCEPTOR(void, tzset, int fake) { ENSURE_MSAN_INITED(); + InterceptorScope interceptor_scope; REAL(tzset)(fake); if (tzname[0]) __msan_unpoison(tzname[0], REAL(strlen)(tzname[0]) + 1); @@ -1084,23 +1087,80 @@ struct MSanAtExitRecord { void *arg; }; -void MSanAtExitWrapper(void *arg) { +struct InterceptorContext { + BlockingMutex atexit_mu; + Vector AtExitStack; + + InterceptorContext() + : AtExitStack() { + } +}; + +static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; +InterceptorContext *interceptor_ctx() { + return reinterpret_cast(&interceptor_placeholder[0]); +} + +void MSanAtExitWrapper() { + MSanAtExitRecord *r; + { + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + uptr element = interceptor_ctx()->AtExitStack.Size() - 1; + r = interceptor_ctx()->AtExitStack[element]; + interceptor_ctx()->AtExitStack.PopBack(); + } + + UnpoisonParam(1); + ((void(*)())r->func)(); + InternalFree(r); +} + +void MSanCxaAtExitWrapper(void *arg) { UnpoisonParam(1); MSanAtExitRecord *r = (MSanAtExitRecord *)arg; r->func(r->arg); InternalFree(r); } +static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso); + // Unpoison argument shadow for C++ module destructors. INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, void *dso_handle) { if (msan_init_is_running) return REAL(__cxa_atexit)(func, arg, dso_handle); + return setup_at_exit_wrapper((void(*)())func, arg, dso_handle); +} + +// Unpoison argument shadow for C++ module destructors. +INTERCEPTOR(int, atexit, void (*func)()) { + // Avoid calling real atexit as it is unrechable on at least on Linux. + if (msan_init_is_running) + return REAL(__cxa_atexit)((void (*)(void *a))func, 0, 0); + return setup_at_exit_wrapper((void(*)())func, 0, 0); +} + +static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso) { ENSURE_MSAN_INITED(); MSanAtExitRecord *r = (MSanAtExitRecord *)InternalAlloc(sizeof(MSanAtExitRecord)); - r->func = func; + r->func = (void(*)(void *a))f; r->arg = arg; - return REAL(__cxa_atexit)(MSanAtExitWrapper, r, dso_handle); + int res; + if (!dso) { + // NetBSD does not preserve the 2nd argument if dso is equal to 0 + // Store ctx in a local stack-like structure + + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + res = REAL(__cxa_atexit)((void (*)(void *a))MSanAtExitWrapper, 0, 0); + if (!res) { + interceptor_ctx()->AtExitStack.PushBack(r); + } + } else { + res = REAL(__cxa_atexit)(MSanCxaAtExitWrapper, r, dso); + } + return res; } static void BeforeFork() { @@ -1520,6 +1580,9 @@ namespace __msan { void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); + + new(interceptor_ctx()) InterceptorContext(); + InitializeCommonInterceptors(); InitializeSignalInterceptors(); @@ -1629,6 +1692,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(tzset); + INTERCEPT_FUNCTION(atexit); INTERCEPT_FUNCTION(__cxa_atexit); INTERCEPT_FUNCTION(shmat); INTERCEPT_FUNCTION(fork); diff --git a/contrib/compiler-rt/lib/msan/msan_linux.cc b/contrib/compiler-rt/lib/msan/msan_linux.cc index 385a650c4af..0b0208884d2 100644 --- a/contrib/compiler-rt/lib/msan/msan_linux.cc +++ b/contrib/compiler-rt/lib/msan/msan_linux.cc @@ -175,6 +175,51 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 +#if SANITIZER_NETBSD || SANITIZER_FREEBSD +// Thread Static Data cannot be used in early init on NetBSD and FreeBSD. +// Reuse the MSan TSD API for compatibility with existing code +// with an alternative implementation. + +static void (*tsd_destructor)(void *tsd) = nullptr; + +struct tsd_key { + tsd_key() : key(nullptr) {} + ~tsd_key() { + CHECK(tsd_destructor); + if (key) + (*tsd_destructor)(key); + } + MsanThread *key; +}; + +static thread_local struct tsd_key key; + +void MsanTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_destructor); + tsd_destructor = destructor; +} + +MsanThread *GetCurrentThread() { + CHECK(tsd_destructor); + return key.key; +} + +void SetCurrentThread(MsanThread *tsd) { + CHECK(tsd_destructor); + CHECK(tsd); + CHECK(!key.key); + key.key = tsd; +} + +void MsanTSDDtor(void *tsd) { + CHECK(tsd_destructor); + CHECK_EQ(key.key, tsd); + key.key = nullptr; + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + MsanThread::TSDDtor(tsd); +} +#else static pthread_key_t tsd_key; static bool tsd_key_inited = false; @@ -211,6 +256,7 @@ void MsanTSDDtor(void *tsd) { atomic_signal_fence(memory_order_seq_cst); MsanThread::TSDDtor(tsd); } +#endif } // namespace __msan diff --git a/contrib/compiler-rt/lib/profile/GCDAProfiling.c b/contrib/compiler-rt/lib/profile/GCDAProfiling.c index cbca365510b..0665a680cf0 100644 --- a/contrib/compiler-rt/lib/profile/GCDAProfiling.c +++ b/contrib/compiler-rt/lib/profile/GCDAProfiling.c @@ -29,6 +29,8 @@ #include #if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include #include "WindowsMMap.h" #else #include @@ -86,6 +88,9 @@ static uint64_t cur_buffer_size = 0; static uint64_t cur_pos = 0; static uint64_t file_size = 0; static int new_file = 0; +#if defined(_WIN32) +static HANDLE mmap_handle = NULL; +#endif static int fd = -1; typedef void (*fn_ptr)(); @@ -255,6 +260,28 @@ static int map_file() { if (file_size == 0) return -1; +#if defined(_WIN32) + HANDLE mmap_fd; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + + mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); + if (mmap_handle == NULL) { + fprintf(stderr, "profiling: %s: cannot create file mapping: %d\n", filename, + GetLastError()); + return -1; + } + + write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); + if (write_buffer == NULL) { + fprintf(stderr, "profiling: %s: cannot map: %d\n", filename, + GetLastError()); + CloseHandle(mmap_handle); + return -1; + } +#else write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, fd, 0); if (write_buffer == (void *)-1) { @@ -263,10 +290,30 @@ static int map_file() { strerror(errnum)); return -1; } +#endif + return 0; } static void unmap_file() { +#if defined(_WIN32) + if (!FlushViewOfFile(write_buffer, file_size)) { + fprintf(stderr, "profiling: %s: cannot flush mapped view: %d\n", filename, + GetLastError()); + } + + if (!UnmapViewOfFile(write_buffer)) { + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %d\n", filename, + GetLastError()); + } + + if (!CloseHandle(mmap_handle)) { + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %d\n", filename, + GetLastError()); + } + + mmap_handle = NULL; +#else if (msync(write_buffer, file_size, MS_SYNC) == -1) { int errnum = errno; fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename, @@ -277,6 +324,8 @@ static void unmap_file() { * is written and we don't care. */ (void)munmap(write_buffer, file_size); +#endif + write_buffer = NULL; file_size = 0; } diff --git a/contrib/compiler-rt/lib/profile/InstrProfData.inc b/contrib/compiler-rt/lib/profile/InstrProfData.inc index eb4a792ce82..454620ed997 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfData.inc +++ b/contrib/compiler-rt/lib/profile/InstrProfData.inc @@ -308,14 +308,14 @@ typedef struct ValueProfRecord { #ifdef __cplusplus /*! - * \brief Return the number of value sites. + * Return the number of value sites. */ uint32_t getNumValueSites() const { return NumValueSites; } /*! - * \brief Read data from this record and save it to Record. + * Read data from this record and save it to Record. */ void deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap); + InstrProfSymtab *SymTab); /* * In-place byte swap: * Do byte swap for this instance. \c Old is the original order before @@ -393,7 +393,7 @@ typedef struct ValueProfData { * Read data from this data and save it to \c Record. */ void deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap); + InstrProfSymtab *SymTab); void operator delete(void *ptr) { ::operator delete(ptr); } #endif } ValueProfData; @@ -458,7 +458,7 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #endif /*! - * \brief Return the \c ValueProfRecord header size including the + * Return the \c ValueProfRecord header size including the * padding bytes. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE @@ -471,7 +471,7 @@ uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { } /*! - * \brief Return the total size of the value profile record including the + * Return the total size of the value profile record including the * header and the value data. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE @@ -482,7 +482,7 @@ uint32_t getValueProfRecordSize(uint32_t NumValueSites, } /*! - * \brief Return the pointer to the start of value data array. + * Return the pointer to the start of value data array. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { @@ -491,7 +491,7 @@ InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { } /*! - * \brief Return the total number of value data for \c This record. + * Return the total number of value data for \c This record. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { @@ -503,7 +503,7 @@ uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { } /*! - * \brief Use this method to advance to the next \c This \c ValueProfRecord. + * Use this method to advance to the next \c This \c ValueProfRecord. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { @@ -514,7 +514,7 @@ ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { } /*! - * \brief Return the first \c ValueProfRecord instance. + * Return the first \c ValueProfRecord instance. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index a517821a2fb..3764df1d807 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -8,7 +8,7 @@ \*===----------------------------------------------------------------------===*/ #if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ - (defined(__sun__) && defined(__svr4__)) + (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) #include diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index a339abc7f88..7c2f14cfce1 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -8,7 +8,7 @@ \*===----------------------------------------------------------------------===*/ #if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ - !(defined(__sun__) && defined(__svr4__)) + !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) #include diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c index 674a4860987..c7b01a57064 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c @@ -105,8 +105,7 @@ static int allocateValueProfileCounters(__llvm_profile_data *Data) { return 1; } -static ValueProfNode *allocateOneNode(__llvm_profile_data *Data, uint32_t Index, - uint64_t Value) { +static ValueProfNode *allocateOneNode(void) { ValueProfNode *Node; if (!hasStaticCounters) @@ -205,7 +204,7 @@ instrumentTargetValueImpl(uint64_t TargetValue, void *Data, return; } - CurVNode = allocateOneNode(PData, CounterIndex, TargetValue); + CurVNode = allocateOneNode(); if (!CurVNode) return; CurVNode->Value = TargetValue; diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.c b/contrib/compiler-rt/lib/profile/WindowsMMap.c index dc87a888ae7..41cc67f41f1 100644 --- a/contrib/compiler-rt/lib/profile/WindowsMMap.c +++ b/contrib/compiler-rt/lib/profile/WindowsMMap.c @@ -24,14 +24,6 @@ #include "InstrProfiling.h" -#ifdef __USE_FILE_OFFSET64 -# define DWORD_HI(x) (x >> 32) -# define DWORD_LO(x) ((x) & 0xffffffff) -#else -# define DWORD_HI(x) (0) -# define DWORD_LO(x) (x) -#endif - COMPILER_RT_VISIBILITY void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) { diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.h b/contrib/compiler-rt/lib/profile/WindowsMMap.h index 271619aea09..51a130b3197 100644 --- a/contrib/compiler-rt/lib/profile/WindowsMMap.h +++ b/contrib/compiler-rt/lib/profile/WindowsMMap.h @@ -12,7 +12,7 @@ #if defined(_WIN32) -#include +#include #include #include @@ -45,6 +45,14 @@ #define LOCK_NB 4 /* don't block when locking */ #define LOCK_UN 8 /* unlock */ +#ifdef __USE_FILE_OFFSET64 +# define DWORD_HI(x) (x >> 32) +# define DWORD_LO(x) ((x) & 0xffffffff) +#else +# define DWORD_HI(x) (0) +# define DWORD_LO(x) (x) +#endif + void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset); diff --git a/contrib/compiler-rt/lib/safestack/safestack.cc b/contrib/compiler-rt/lib/safestack/safestack.cc index 8af93624b99..e68208015fb 100644 --- a/contrib/compiler-rt/lib/safestack/safestack.cc +++ b/contrib/compiler-rt/lib/safestack/safestack.cc @@ -14,11 +14,13 @@ // //===----------------------------------------------------------------------===// +#include #include #include #include #include #include +#include #include #include #if !defined(__NetBSD__) @@ -115,14 +117,6 @@ static inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { unsafe_stack_guard = guard; } -static void unsafe_stack_free() { - if (unsafe_stack_start) { - UnmapOrDie((char *)unsafe_stack_start - unsafe_stack_guard, - unsafe_stack_size + unsafe_stack_guard); - } - unsafe_stack_start = nullptr; -} - /// Thread data for the cleanup handler static pthread_key_t thread_cleanup_key; @@ -149,26 +143,73 @@ static void *thread_start(void *arg) { tinfo->unsafe_stack_guard); // Make sure out thread-specific destructor will be called - // FIXME: we can do this only any other specific key is set by - // intercepting the pthread_setspecific function itself pthread_setspecific(thread_cleanup_key, (void *)1); return start_routine(start_routine_arg); } -/// Thread-specific data destructor +/// Linked list used to store exiting threads stack/thread information. +struct thread_stack_ll { + struct thread_stack_ll *next; + void *stack_base; + size_t size; + pid_t pid; + tid_t tid; +}; + +/// Linked list of unsafe stacks for threads that are exiting. We delay +/// unmapping them until the thread exits. +static thread_stack_ll *thread_stacks = nullptr; +static pthread_mutex_t thread_stacks_mutex = PTHREAD_MUTEX_INITIALIZER; + +/// Thread-specific data destructor. We want to free the unsafe stack only after +/// this thread is terminated. libc can call functions in safestack-instrumented +/// code (like free) after thread-specific data destructors have run. static void thread_cleanup_handler(void *_iter) { - // We want to free the unsafe stack only after all other destructors - // have already run. We force this function to be called multiple times. - // User destructors that might run more then PTHREAD_DESTRUCTOR_ITERATIONS-1 - // times might still end up executing after the unsafe stack is deallocated. - size_t iter = (size_t)_iter; - if (iter < PTHREAD_DESTRUCTOR_ITERATIONS) { - pthread_setspecific(thread_cleanup_key, (void *)(iter + 1)); - } else { - // This is the last iteration - unsafe_stack_free(); + CHECK_NE(unsafe_stack_start, nullptr); + pthread_setspecific(thread_cleanup_key, NULL); + + pthread_mutex_lock(&thread_stacks_mutex); + // Temporary list to hold the previous threads stacks so we don't hold the + // thread_stacks_mutex for long. + thread_stack_ll *temp_stacks = thread_stacks; + thread_stacks = nullptr; + pthread_mutex_unlock(&thread_stacks_mutex); + + pid_t pid = getpid(); + tid_t tid = GetTid(); + + // Free stacks for dead threads + thread_stack_ll **stackp = &temp_stacks; + while (*stackp) { + thread_stack_ll *stack = *stackp; + int error; + if (stack->pid != pid || + (internal_iserror(TgKill(stack->pid, stack->tid, 0), &error) && + error == ESRCH)) { + UnmapOrDie(stack->stack_base, stack->size); + *stackp = stack->next; + free(stack); + } else + stackp = &stack->next; } + + thread_stack_ll *cur_stack = + (thread_stack_ll *)malloc(sizeof(thread_stack_ll)); + cur_stack->stack_base = (char *)unsafe_stack_start - unsafe_stack_guard; + cur_stack->size = unsafe_stack_size + unsafe_stack_guard; + cur_stack->pid = pid; + cur_stack->tid = tid; + + pthread_mutex_lock(&thread_stacks_mutex); + // Merge thread_stacks with the current thread's stack and any remaining + // temp_stacks + *stackp = thread_stacks; + cur_stack->next = temp_stacks; + thread_stacks = cur_stack; + pthread_mutex_unlock(&thread_stacks_mutex); + + unsafe_stack_start = nullptr; } static void EnsureInterceptorsInitialized(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index 9655a2264f3..88017160acf 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -14,13 +14,15 @@ #ifndef SANITIZER_ALLOCATOR_H #define SANITIZER_ALLOCATOR_H -#include "sanitizer_internal_defs.h" #include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_lfstack.h" #include "sanitizer_libc.h" #include "sanitizer_list.h" +#include "sanitizer_local_address_space_view.h" #include "sanitizer_mutex.h" -#include "sanitizer_lfstack.h" #include "sanitizer_procmaps.h" +#include "sanitizer_type_traits.h" namespace __sanitizer { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h index 7df3e4097bf..ef26941fe60 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h @@ -15,9 +15,10 @@ #endif // Maps integers in rage [0, kSize) to u8 values. -template +template class FlatByteMap { public: + using AddressSpaceView = AddressSpaceViewTy; void Init() { internal_memset(map_, 0, sizeof(map_)); } @@ -41,9 +42,12 @@ class FlatByteMap { // to kSize2-byte arrays. The secondary arrays are mmaped on demand. // Each value is initially zero and can be set to something else only once. // Setting and getting values from multiple threads is safe w/o extra locking. -template +template class TwoLevelByteMap { public: + using AddressSpaceView = AddressSpaceViewTy; void Init() { internal_memset(map1_, 0, sizeof(map1_)); mu_.Init(); @@ -73,7 +77,8 @@ class TwoLevelByteMap { CHECK_LT(idx, kSize1 * kSize2); u8 *map2 = Get(idx / kSize2); if (!map2) return 0; - return map2[idx % kSize2]; + auto value_ptr = AddressSpaceView::Load(&map2[idx % kSize2]); + return *value_ptr; } private: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h index 1f874d60b92..fcc4469c98c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -21,19 +21,28 @@ // PrimaryAllocator is used via a local AllocatorCache. // SecondaryAllocator can allocate anything, but is not efficient. template // NOLINT + class SecondaryAllocator, + typename AddressSpaceViewTy = LocalAddressSpaceView> // NOLINT class CombinedAllocator { public: + using AddressSpaceView = AddressSpaceViewTy; + static_assert(is_same::value, + "PrimaryAllocator is using wrong AddressSpaceView"); + static_assert(is_same::value, + "SecondaryAllocator is using wrong AddressSpaceView"); + void InitLinkerInitialized(s32 release_to_os_interval_ms) { + stats_.InitLinkerInitialized(); primary_.Init(release_to_os_interval_ms); secondary_.InitLinkerInitialized(); - stats_.InitLinkerInitialized(); } void Init(s32 release_to_os_interval_ms) { + stats_.Init(); primary_.Init(release_to_os_interval_ms); secondary_.Init(); - stats_.Init(); } void *Allocate(AllocatorCache *cache, uptr size, uptr alignment) { @@ -194,4 +203,3 @@ class CombinedAllocator { SecondaryAllocator secondary_; AllocatorGlobalStats stats_; }; - diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h index c0c03d3f434..30fc7042b6b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -37,7 +37,8 @@ struct AP32 { static const uptr kMetadataSize = 0; typedef InternalSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = kInternalAllocatorRegionSizeLog; - typedef __sanitizer::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __sanitizer::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h index 67970e95b31..abaac3d1a70 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -48,6 +48,7 @@ struct SizeClassAllocator32FlagMasks { // Bit masks. template class SizeClassAllocator32 { public: + using AddressSpaceView = typename Params::AddressSpaceView; static const uptr kSpaceBeg = Params::kSpaceBeg; static const u64 kSpaceSize = Params::kSpaceSize; static const uptr kMetadataSize = Params::kMetadataSize; @@ -56,6 +57,10 @@ class SizeClassAllocator32 { typedef typename Params::ByteMap ByteMap; typedef typename Params::MapUnmapCallback MapUnmapCallback; + static_assert( + is_same::value, + "AddressSpaceView type mismatch"); + static const bool kRandomShuffleChunks = Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks; static const bool kUseSeparateSizeClassForBatch = Params::kFlags & diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h index 6acb4f8bc56..b063bf0d302 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -46,6 +46,7 @@ struct SizeClassAllocator64FlagMasks { // Bit masks. template class SizeClassAllocator64 { public: + using AddressSpaceView = typename Params::AddressSpaceView; static const uptr kSpaceBeg = Params::kSpaceBeg; static const uptr kSpaceSize = Params::kSpaceSize; static const uptr kMetadataSize = Params::kMetadataSize; @@ -294,8 +295,10 @@ class SizeClassAllocator64 { RegionInfo *region = GetRegionInfo(class_id); uptr chunk_size = ClassIdToSize(class_id); uptr region_beg = SpaceBeg() + class_id * kRegionSize; + uptr region_allocated_user_size = + AddressSpaceView::Load(region)->allocated_user; for (uptr chunk = region_beg; - chunk < region_beg + region->allocated_user; + chunk < region_beg + region_allocated_user_size; chunk += chunk_size) { // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk)); callback(chunk, arg); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h index ab680b5e2d1..0c8505c34c8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -68,9 +68,11 @@ typedef LargeMmapAllocatorPtrArrayDynamic DefaultLargeMmapAllocatorPtrArray; // The main purpose of this allocator is to cover large and rare allocation // sizes not covered by more efficient allocators (e.g. SizeClassAllocator64). template + class PtrArrayT = DefaultLargeMmapAllocatorPtrArray, + class AddressSpaceViewTy = LocalAddressSpaceView> class LargeMmapAllocator { public: + using AddressSpaceView = AddressSpaceViewTy; void InitLinkerInitialized() { page_size_ = GetPageSizeCached(); chunks_ = reinterpret_cast(ptr_array_.Init()); @@ -202,9 +204,10 @@ class LargeMmapAllocator { void EnsureSortedChunks() { if (chunks_sorted_) return; - Sort(reinterpret_cast(chunks_), n_chunks_); + Header **chunks = AddressSpaceView::LoadWritable(chunks_, n_chunks_); + Sort(reinterpret_cast(chunks), n_chunks_); for (uptr i = 0; i < n_chunks_; i++) - chunks_[i]->chunk_idx = i; + AddressSpaceView::LoadWritable(chunks[i])->chunk_idx = i; chunks_sorted_ = true; } @@ -272,12 +275,13 @@ class LargeMmapAllocator { // The allocator must be locked when calling this function. void ForEachChunk(ForEachChunkCallback callback, void *arg) { EnsureSortedChunks(); // Avoid doing the sort while iterating. + const Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_); for (uptr i = 0; i < n_chunks_; i++) { - auto t = chunks_[i]; + const Header *t = chunks[i]; callback(reinterpret_cast(GetUser(t)), arg); // Consistency check: verify that the array did not change. - CHECK_EQ(chunks_[i], t); - CHECK_EQ(chunks_[i]->chunk_idx, i); + CHECK_EQ(chunks[i], t); + CHECK_EQ(AddressSpaceView::Load(chunks[i])->chunk_idx, i); } } @@ -297,7 +301,7 @@ class LargeMmapAllocator { return GetHeader(reinterpret_cast(p)); } - void *GetUser(Header *h) { + void *GetUser(const Header *h) { CHECK(IsAligned((uptr)h, page_size_)); return reinterpret_cast(reinterpret_cast(h) + page_size_); } @@ -316,4 +320,3 @@ class LargeMmapAllocator { } stats; StaticSpinMutex mutex_; }; - diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h index 77ab4fb544a..1c05fb8ff06 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -232,3 +232,11 @@ class SizeClassMap { typedef SizeClassMap<3, 4, 8, 17, 128, 16> DefaultSizeClassMap; typedef SizeClassMap<3, 4, 8, 17, 64, 14> CompactSizeClassMap; typedef SizeClassMap<2, 5, 9, 16, 64, 14> VeryCompactSizeClassMap; + +// The following SizeClassMap only holds a way small number of cached entries, +// allowing for denser per-class arrays, smaller memory footprint and usually +// better performances in threaded environments. +typedef SizeClassMap<3, 4, 8, 17, 8, 10> DenseSizeClassMap; +// Similar to VeryCompact map above, this one has a small number of different +// size classes, and also reduced thread-local caches. +typedef SizeClassMap<2, 5, 9, 16, 8, 10> VeryDenseSizeClassMap; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h index 38feb29287f..195533ea232 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h @@ -61,8 +61,7 @@ INLINE typename T::Type atomic_load( "emms;" // Empty mmx state/Reset FP regs : "=m" (v) : "m" (a->val_dont_use) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)", + : // mark the mmx registers as clobbered #ifdef __MMX__ "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", #endif // #ifdef __MMX__ @@ -100,8 +99,7 @@ INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { "emms;" // Empty mmx state/Reset FP regs : "=m" (a->val_dont_use) : "m" (v) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)", + : // mark the mmx registers as clobbered #ifdef __MMX__ "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", #endif // #ifdef __MMX__ diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc index 7d72b0cfe4c..6868961902c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc @@ -338,11 +338,6 @@ int __sanitizer_acquire_crash_state() { return !atomic_exchange(&in_crash_state, 1, memory_order_relaxed); } -SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_set_death_callback(void (*callback)(void)) { - SetUserDieCallback(callback); -} - SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_install_malloc_and_free_hooks(void (*malloc_hook)(const void *, uptr), diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 3b999edfbe5..d0aebd99412 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -61,6 +61,15 @@ INLINE int Verbosity() { return atomic_load(¤t_verbosity, memory_order_relaxed); } +#if SANITIZER_ANDROID +INLINE uptr GetPageSize() { +// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. + return 4096; +} +INLINE uptr GetPageSizeCached() { + return 4096; +} +#else uptr GetPageSize(); extern uptr PageSizeCached; INLINE uptr GetPageSizeCached() { @@ -68,11 +77,13 @@ INLINE uptr GetPageSizeCached() { PageSizeCached = GetPageSize(); return PageSizeCached; } +#endif uptr GetMmapGranularity(); uptr GetMaxVirtualAddress(); uptr GetMaxUserVirtualAddress(); // Threads tid_t GetTid(); +int TgKill(pid_t pid, tid_t tid, int sig); uptr GetThreadSelf(); void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, uptr *stack_bottom); @@ -222,7 +233,9 @@ bool SetEnv(const char *name, const char *value); u32 GetUid(); void ReExec(); void CheckASLR(); +void CheckMPROTECT(); char **GetArgv(); +char **GetEnviron(); void PrintCmdline(); bool StackSizeIsUnlimited(); uptr GetStackSizeLimitInBytes(); @@ -896,6 +909,7 @@ struct SignalContext { bool IsMemoryAccess() const; }; +void InitializePlatformEarly(); void MaybeReexec(); template diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 801d6a0bdea..50f7837747e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -77,7 +77,15 @@ #define ctime __ctime50 #define ctime_r __ctime_r50 #define devname __devname50 +#define fgetpos __fgetpos50 +#define fsetpos __fsetpos50 +#define fts_children __fts_children60 +#define fts_close __fts_close60 +#define fts_open __fts_open60 +#define fts_read __fts_read60 +#define fts_set __fts_set60 #define getitimer __getitimer50 +#define getmntinfo __getmntinfo13 #define getpwent __getpwent50 #define getpwnam __getpwnam50 #define getpwnam_r __getpwnam_r50 @@ -87,6 +95,7 @@ #define getutxent __getutxent50 #define getutxid __getutxid50 #define getutxline __getutxline50 +#define pututxline __pututxline50 #define glob __glob30 #define gmtime __gmtime50 #define gmtime_r __gmtime_r50 @@ -109,6 +118,7 @@ #define stat __stat50 #define time __time50 #define times __times13 +#define unvis __unvis50 #define wait3 __wait350 #define wait4 __wait450 extern const unsigned short *_ctype_tab_; @@ -1810,7 +1820,10 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) { #if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS || \ SANITIZER_INTERCEPT_GETPWENT || SANITIZER_INTERCEPT_FGETPWENT || \ - SANITIZER_INTERCEPT_GETPWENT_R || SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS + SANITIZER_INTERCEPT_GETPWENT_R || \ + SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS || \ + SANITIZER_INTERCEPT_FGETPWENT_R || \ + SANITIZER_INTERCEPT_FGETGRENT_R static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { if (pwd) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd)); @@ -2034,21 +2047,6 @@ INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf, if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } -INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, - SIZE_T buflen, __sanitizer_passwd **pwbufp) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); - if (!res) { - if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); - } - if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); - return res; -} INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, __sanitizer_group **pwbufp) { void *ctx; @@ -2064,6 +2062,36 @@ INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } +#define INIT_GETPWENT_R \ + COMMON_INTERCEPT_FUNCTION(getpwent_r); \ + COMMON_INTERCEPT_FUNCTION(getgrent_r); +#else +#define INIT_GETPWENT_R +#endif + +#if SANITIZER_INTERCEPT_FGETPWENT_R +INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, + SIZE_T buflen, __sanitizer_passwd **pwbufp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp); + // FIXME: under ASan the call below may write to freed memory and corrupt + // its metadata. See + // https://github.com/google/sanitizers/issues/321. + int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); + if (!res) { + if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); + } + if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); + return res; +} +#define INIT_FGETPWENT_R \ + COMMON_INTERCEPT_FUNCTION(fgetpwent_r); +#else +#define INIT_FGETPWENT_R +#endif + +#if SANITIZER_INTERCEPT_FGETGRENT_R INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, __sanitizer_group **pwbufp) { void *ctx; @@ -2079,13 +2107,10 @@ INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } -#define INIT_GETPWENT_R \ - COMMON_INTERCEPT_FUNCTION(getpwent_r); \ - COMMON_INTERCEPT_FUNCTION(fgetpwent_r); \ - COMMON_INTERCEPT_FUNCTION(getgrent_r); \ +#define INIT_FGETGRENT_R \ COMMON_INTERCEPT_FUNCTION(fgetgrent_r); #else -#define INIT_GETPWENT_R +#define INIT_FGETGRENT_R #endif #if SANITIZER_INTERCEPT_SETPWENT @@ -2149,6 +2174,8 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) { namespace __sanitizer { extern "C" { int real_clock_gettime(u32 clk_id, void *tp) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_clock_gettime(clk_id, tp); return REAL(clock_gettime)(clk_id, tp); } } // extern "C" @@ -3345,14 +3372,14 @@ INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { return res; } -INTERCEPTOR(INTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { +INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. char *real_endptr; - INTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); + UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); return res; } @@ -4254,11 +4281,16 @@ INTERCEPTOR(int, statvfs, char *path, void *buf) { INTERCEPTOR(int, fstatvfs, int fd, void *buf) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs, fd, buf); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatvfs)(fd, buf); - if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + if (!res) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + if (fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } return res; } #define INIT_STATVFS \ @@ -4819,6 +4851,14 @@ INTERCEPTOR(float, remquof, float x, float y, int *quo) { if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; } +#define INIT_REMQUO \ + COMMON_INTERCEPT_FUNCTION(remquo); \ + COMMON_INTERCEPT_FUNCTION(remquof); +#else +#define INIT_REMQUO +#endif + +#if SANITIZER_INTERCEPT_REMQUOL INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, remquol, x, y, quo); @@ -4829,12 +4869,10 @@ INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; } -#define INIT_REMQUO \ - COMMON_INTERCEPT_FUNCTION(remquo); \ - COMMON_INTERCEPT_FUNCTION(remquof); \ +#define INIT_REMQUOL \ COMMON_INTERCEPT_FUNCTION_LDBL(remquol); #else -#define INIT_REMQUO +#define INIT_REMQUOL #endif #if SANITIZER_INTERCEPT_LGAMMA @@ -4853,6 +4891,14 @@ INTERCEPTOR(float, lgammaf, float x) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam)); return res; } +#define INIT_LGAMMA \ + COMMON_INTERCEPT_FUNCTION(lgamma); \ + COMMON_INTERCEPT_FUNCTION(lgammaf); +#else +#define INIT_LGAMMA +#endif + +#if SANITIZER_INTERCEPT_LGAMMAL INTERCEPTOR(long double, lgammal, long double x) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, lgammal, x); @@ -4860,12 +4906,10 @@ INTERCEPTOR(long double, lgammal, long double x) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam)); return res; } -#define INIT_LGAMMA \ - COMMON_INTERCEPT_FUNCTION(lgamma); \ - COMMON_INTERCEPT_FUNCTION(lgammaf); \ +#define INIT_LGAMMAL \ COMMON_INTERCEPT_FUNCTION_LDBL(lgammal); #else -#define INIT_LGAMMA +#define INIT_LGAMMAL #endif #if SANITIZER_INTERCEPT_LGAMMA_R @@ -5651,9 +5695,15 @@ INTERCEPTOR(void *, tsearch, void *key, void **rootp, void unpoison_file(__sanitizer_FILE *fp) { #if SANITIZER_HAS_STRUCT_FILE COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp, sizeof(*fp)); +#if SANITIZER_NETBSD + if (fp->_bf._base && fp->_bf._size > 0) + COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_bf._base, + fp->_bf._size); +#else if (fp->_IO_read_base && fp->_IO_read_base < fp->_IO_read_end) COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_read_base, fp->_IO_read_end - fp->_IO_read_base); +#endif #endif // SANITIZER_HAS_STRUCT_FILE } #endif @@ -6523,10 +6573,21 @@ INTERCEPTOR(void *, getutxline, void *ut) { COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz); return res; } +INTERCEPTOR(void *, pututxline, const void *ut) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pututxline, ut); + if (ut) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ut, __sanitizer::struct_utmpx_sz); + void *res = REAL(pututxline)(ut); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer::struct_utmpx_sz); + return res; +} #define INIT_UTMPX \ COMMON_INTERCEPT_FUNCTION(getutxent); \ COMMON_INTERCEPT_FUNCTION(getutxid); \ - COMMON_INTERCEPT_FUNCTION(getutxline); + COMMON_INTERCEPT_FUNCTION(getutxline); \ + COMMON_INTERCEPT_FUNCTION(pututxline); #else #define INIT_UTMPX #endif @@ -7018,12 +7079,19 @@ INTERCEPTOR(char *, devname, u64 dev, u32 type) { #endif #if SANITIZER_INTERCEPT_DEVNAME_R -INTERCEPTOR(int, devname_r, u64 dev, u32 type, char *path, uptr len) { +#if SANITIZER_NETBSD +#define DEVNAME_R_RETTYPE int +#define DEVNAME_R_SUCCESS(x) (!(x)) +#else +#define DEVNAME_R_RETTYPE char* +#define DEVNAME_R_SUCCESS(x) (x) +#endif +INTERCEPTOR(DEVNAME_R_RETTYPE, devname_r, u64 dev, u32 type, char *path, + uptr len) { void *ctx; - int res; COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len); - res = REAL(devname_r)(dev, type, path, len); - if (!res) + DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len); + if (DEVNAME_R_SUCCESS(res)) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, REAL(strlen)(path) + 1); return res; } @@ -7235,9 +7303,2174 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) { #define INIT_NETENT #endif +#if SANITIZER_INTERCEPT_GETMNTINFO +INTERCEPTOR(int, getmntinfo, void **mntbufp, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getmntinfo, mntbufp, flags); + int cnt = REAL(getmntinfo)(mntbufp, flags); + if (cnt > 0 && mntbufp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mntbufp, sizeof(void *)); + if (*mntbufp) +#if SANITIZER_NETBSD + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statvfs_sz); +#else + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statfs_sz); +#endif + } + return cnt; +} +#define INIT_GETMNTINFO COMMON_INTERCEPT_FUNCTION(getmntinfo) +#else +#define INIT_GETMNTINFO +#endif + +#if SANITIZER_INTERCEPT_MI_VECTOR_HASH +INTERCEPTOR(void, mi_vector_hash, const void *key, SIZE_T len, u32 seed, + u32 hashes[3]) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, mi_vector_hash, key, len, seed, hashes); + if (key) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, len); + REAL(mi_vector_hash)(key, len, seed, hashes); + if (hashes) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hashes, sizeof(hashes[0]) * 3); +} +#define INIT_MI_VECTOR_HASH COMMON_INTERCEPT_FUNCTION(mi_vector_hash) +#else +#define INIT_MI_VECTOR_HASH +#endif + +#if SANITIZER_INTERCEPT_SETVBUF +INTERCEPTOR(int, setvbuf, __sanitizer_FILE *stream, char *buf, int mode, + SIZE_T size) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setvbuf, stream, buf, mode, size); + int ret = REAL(setvbuf)(stream, buf, mode, size); + if (buf) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size); + if (stream) + unpoison_file(stream); + return ret; +} + +INTERCEPTOR(void, setbuf, __sanitizer_FILE *stream, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setbuf, stream, buf); + REAL(setbuf)(stream, buf); + if (buf) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz); + } + if (stream) + unpoison_file(stream); +} + +INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, int mode) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode); + REAL(setbuffer)(stream, buf, mode); + if (buf) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz); + } + if (stream) + unpoison_file(stream); +} + +INTERCEPTOR(void, setlinebuf, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setlinebuf, stream); + REAL(setlinebuf)(stream); + if (stream) + unpoison_file(stream); +} +#define INIT_SETVBUF COMMON_INTERCEPT_FUNCTION(setvbuf); \ + COMMON_INTERCEPT_FUNCTION(setbuf); \ + COMMON_INTERCEPT_FUNCTION(setbuffer); \ + COMMON_INTERCEPT_FUNCTION(setlinebuf) +#else +#define INIT_SETVBUF +#endif + +#if SANITIZER_INTERCEPT_GETVFSSTAT +INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getvfsstat, buf, bufsize, flags); + int ret = REAL(getvfsstat)(buf, bufsize, flags); + if (buf && ret > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, ret * struct_statvfs_sz); + return ret; +} +#define INIT_GETVFSSTAT COMMON_INTERCEPT_FUNCTION(getvfsstat) +#else +#define INIT_GETVFSSTAT +#endif + +#if SANITIZER_INTERCEPT_REGEX +INTERCEPTOR(int, regcomp, void *preg, const char *pattern, int cflags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags); + if (pattern) + COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, REAL(strlen)(pattern) + 1); + int res = REAL(regcomp)(preg, pattern, cflags); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz); + return res; +} +INTERCEPTOR(int, regexec, const void *preg, const char *string, SIZE_T nmatch, + struct __sanitizer_regmatch *pmatch[], int eflags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regexec, preg, string, nmatch, pmatch, eflags); + if (preg) + COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz); + if (string) + COMMON_INTERCEPTOR_READ_RANGE(ctx, string, REAL(strlen)(string) + 1); + int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags); + if (!res && pmatch) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz); + return res; +} +INTERCEPTOR(SIZE_T, regerror, int errcode, const void *preg, char *errbuf, + SIZE_T errbuf_size) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regerror, errcode, preg, errbuf, errbuf_size); + if (preg) + COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz); + SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size); + if (errbuf) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, REAL(strlen)(errbuf) + 1); + return res; +} +INTERCEPTOR(void, regfree, const void *preg) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regfree, preg); + if (preg) + COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz); + REAL(regfree)(preg); +} +#define INIT_REGEX \ + COMMON_INTERCEPT_FUNCTION(regcomp); \ + COMMON_INTERCEPT_FUNCTION(regexec); \ + COMMON_INTERCEPT_FUNCTION(regerror); \ + COMMON_INTERCEPT_FUNCTION(regfree); +#else +#define INIT_REGEX +#endif + +#if SANITIZER_INTERCEPT_REGEXSUB +INTERCEPTOR(SSIZE_T, regnsub, char *buf, SIZE_T bufsiz, const char *sub, + const struct __sanitizer_regmatch *rm, const char *str) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str); + if (sub) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1); + // The implementation demands and hardcodes 10 elements + if (rm) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz); + if (str) + COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1); + SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str); + if (res > 0 && buf) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1); + return res; +} +INTERCEPTOR(SSIZE_T, regasub, char **buf, const char *sub, + const struct __sanitizer_regmatch *rm, const char *sstr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr); + if (sub) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1); + // Hardcode 10 elements as this is hardcoded size + if (rm) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz); + if (sstr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, REAL(strlen)(sstr) + 1); + SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr); + if (res > 0 && buf) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *buf, REAL(strlen)(*buf) + 1); + } + return res; +} + +#define INIT_REGEXSUB \ + COMMON_INTERCEPT_FUNCTION(regnsub); \ + COMMON_INTERCEPT_FUNCTION(regasub); +#else +#define INIT_REGEXSUB +#endif + +#if SANITIZER_INTERCEPT_FTS +INTERCEPTOR(void *, fts_open, char *const *path_argv, int options, + int (*compar)(void **, void **)) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_open, path_argv, options, compar); + if (path_argv) { + for (char *const *pa = path_argv; ; ++pa) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **)); + if (!*pa) + break; + COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1); + } + } + // TODO(kamil): handle compar callback + void *fts = REAL(fts_open)(path_argv, options, compar); + if (fts) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, fts, struct_FTS_sz); + return fts; +} + +INTERCEPTOR(void *, fts_read, void *ftsp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_read, ftsp); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + void *ftsent = REAL(fts_read)(ftsp); + if (ftsent) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ftsent, struct_FTSENT_sz); + return ftsent; +} + +INTERCEPTOR(void *, fts_children, void *ftsp, int options) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_children, ftsp, options); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + void *ftsent = REAL(fts_children)(ftsp, options); + if (ftsent) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ftsent, struct_FTSENT_sz); + return ftsent; +} + +INTERCEPTOR(int, fts_set, void *ftsp, void *f, int options) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_set, ftsp, f, options); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + if (f) + COMMON_INTERCEPTOR_READ_RANGE(ctx, f, struct_FTSENT_sz); + return REAL(fts_set)(ftsp, f, options); +} + +INTERCEPTOR(int, fts_close, void *ftsp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_close, ftsp); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + return REAL(fts_close)(ftsp); +} +#define INIT_FTS \ + COMMON_INTERCEPT_FUNCTION(fts_open); \ + COMMON_INTERCEPT_FUNCTION(fts_read); \ + COMMON_INTERCEPT_FUNCTION(fts_children); \ + COMMON_INTERCEPT_FUNCTION(fts_set); \ + COMMON_INTERCEPT_FUNCTION(fts_close); +#else +#define INIT_FTS +#endif + +#if SANITIZER_INTERCEPT_SYSCTL +INTERCEPTOR(int, sysctl, int *name, unsigned int namelen, void *oldp, + SIZE_T *oldlenp, void *newp, SIZE_T newlen) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_sysctl(name, namelen, oldp, oldlenp, newp, newlen); + COMMON_INTERCEPTOR_ENTER(ctx, sysctl, name, namelen, oldp, oldlenp, newp, + newlen); + if (name) + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, namelen * sizeof(*name)); + if (oldlenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (newp && newlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, newp, newlen); + int res = REAL(sysctl)(name, namelen, oldp, oldlenp, newp, newlen); + if (!res) { + if (oldlenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (oldp) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldp, *oldlenp); + } + } + return res; +} + +INTERCEPTOR(int, sysctlbyname, char *sname, void *oldp, SIZE_T *oldlenp, + void *newp, SIZE_T newlen) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_sysctlbyname(sname, oldp, oldlenp, newp, newlen); + COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp, + newlen); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + if (oldlenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (newp && newlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, newp, newlen); + int res = REAL(sysctlbyname)(sname, oldp, oldlenp, newp, newlen); + if (!res) { + if (oldlenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (oldp) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldp, *oldlenp); + } + } + return res; +} + +INTERCEPTOR(int, sysctlnametomib, const char *sname, int *name, + SIZE_T *namelenp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + if (namelenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp)); + int res = REAL(sysctlnametomib)(sname, name, namelenp); + if (!res) { + if (namelenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelenp, sizeof(*namelenp)); + if (name) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, *namelenp * sizeof(*name)); + } + } + return res; +} + +#define INIT_SYSCTL \ + COMMON_INTERCEPT_FUNCTION(sysctl); \ + COMMON_INTERCEPT_FUNCTION(sysctlbyname); \ + COMMON_INTERCEPT_FUNCTION(sysctlnametomib); +#else +#define INIT_SYSCTL +#endif + +#if SANITIZER_INTERCEPT_ASYSCTL +INTERCEPTOR(void *, asysctl, const int *name, SIZE_T namelen, SIZE_T *len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, asysctl, name, namelen, len); + if (name) + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, sizeof(*name) * namelen); + void *res = REAL(asysctl)(name, namelen, len); + if (res && len) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, *len); + } + return res; +} + +INTERCEPTOR(void *, asysctlbyname, const char *sname, SIZE_T *len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + void *res = REAL(asysctlbyname)(sname, len); + if (res && len) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, *len); + } + return res; +} +#define INIT_ASYSCTL \ + COMMON_INTERCEPT_FUNCTION(asysctl); \ + COMMON_INTERCEPT_FUNCTION(asysctlbyname); +#else +#define INIT_ASYSCTL +#endif + +#if SANITIZER_INTERCEPT_SYSCTLGETMIBINFO +INTERCEPTOR(int, sysctlgetmibinfo, char *sname, int *name, + unsigned int *namelenp, char *cname, SIZE_T *csz, void **rnode, + int v) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname, + csz, rnode, v); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + if (namelenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp)); + if (csz) + COMMON_INTERCEPTOR_READ_RANGE(ctx, csz, sizeof(*csz)); + // Skip rnode, it's rarely used and not trivial to sanitize + // It's also used mostly internally + int res = REAL(sysctlgetmibinfo)(sname, name, namelenp, cname, csz, rnode, v); + if (!res) { + if (namelenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelenp, sizeof(*namelenp)); + if (name) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, *namelenp * sizeof(*name)); + } + if (csz) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, csz, sizeof(*csz)); + if (cname) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cname, *csz); + } + } + return res; +} +#define INIT_SYSCTLGETMIBINFO \ + COMMON_INTERCEPT_FUNCTION(sysctlgetmibinfo); +#else +#define INIT_SYSCTLGETMIBINFO +#endif + +#if SANITIZER_INTERCEPT_NL_LANGINFO +INTERCEPTOR(char *, nl_langinfo, long item) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item); + char *ret = REAL(nl_langinfo)(item); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1); + return ret; +} +#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo) +#else +#define INIT_NL_LANGINFO +#endif + +#if SANITIZER_INTERCEPT_MODCTL +INTERCEPTOR(int, modctl, int operation, void *argp) { + void *ctx; + int ret; + COMMON_INTERCEPTOR_ENTER(ctx, modctl, operation, argp); + + if (operation == modctl_load) { + if (argp) { + __sanitizer_modctl_load_t *ml = (__sanitizer_modctl_load_t *)argp; + COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml)); + if (ml->ml_filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename, + REAL(strlen)(ml->ml_filename) + 1); + if (ml->ml_props) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen); + } + ret = REAL(modctl)(operation, argp); + } else if (operation == modctl_unload) { + if (argp) { + const char *name = (const char *)argp; + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); + } + ret = REAL(modctl)(operation, argp); + } else if (operation == modctl_stat) { + uptr iov_len; + struct __sanitizer_iovec *iov = (struct __sanitizer_iovec *)argp; + if (iov) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, iov, sizeof(*iov)); + iov_len = iov->iov_len; + } + ret = REAL(modctl)(operation, argp); + if (iov) + COMMON_INTERCEPTOR_WRITE_RANGE( + ctx, iov->iov_base, Min(iov_len, iov->iov_len)); + } else if (operation == modctl_exists) + ret = REAL(modctl)(operation, argp); + else + ret = REAL(modctl)(operation, argp); + + return ret; +} +#define INIT_MODCTL COMMON_INTERCEPT_FUNCTION(modctl) +#else +#define INIT_MODCTL +#endif + +#if SANITIZER_INTERCEPT_STRTONUM +INTERCEPTOR(long long, strtonum, const char *nptr, long long minval, + long long maxval, const char **errstr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtonum, nptr, minval, maxval, errstr); + + // TODO(kamil): Implement strtoll as a common inteceptor + char *real_endptr; + long long ret = (long long)REAL(strtoimax)(nptr, &real_endptr, 10); + StrtolFixAndCheck(ctx, nptr, nullptr, real_endptr, 10); + + ret = REAL(strtonum)(nptr, minval, maxval, errstr); + if (errstr) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *)); + if (*errstr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *errstr, REAL(strlen)(*errstr) + 1); + } + return ret; +} +#define INIT_STRTONUM COMMON_INTERCEPT_FUNCTION(strtonum) +#else +#define INIT_STRTONUM +#endif + +#if SANITIZER_INTERCEPT_FPARSELN +INTERCEPTOR(char *, fparseln, __sanitizer_FILE *stream, SIZE_T *len, + SIZE_T *lineno, const char delim[3], int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fparseln, stream, len, lineno, delim, flags); + if (lineno) + COMMON_INTERCEPTOR_READ_RANGE(ctx, lineno, sizeof(*lineno)); + if (delim) + COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3); + char *ret = REAL(fparseln)(stream, len, lineno, delim, flags); + if (ret) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1); + if (len) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len)); + if (lineno) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineno, sizeof(*lineno)); + } + return ret; +} +#define INIT_FPARSELN COMMON_INTERCEPT_FUNCTION(fparseln) +#else +#define INIT_FPARSELN +#endif + +#if SANITIZER_INTERCEPT_STATVFS1 +INTERCEPTOR(int, statvfs1, const char *path, void *buf, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags); + if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + int res = REAL(statvfs1)(path, buf, flags); + if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + return res; +} +INTERCEPTOR(int, fstatvfs1, int fd, void *buf, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs1, fd, buf, flags); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + int res = REAL(fstatvfs1)(fd, buf, flags); + if (!res) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + if (fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return res; +} +#define INIT_STATVFS1 \ + COMMON_INTERCEPT_FUNCTION(statvfs1); \ + COMMON_INTERCEPT_FUNCTION(fstatvfs1); +#else +#define INIT_STATVFS1 +#endif + +#if SANITIZER_INTERCEPT_STRTOI +INTERCEPTOR(INTMAX_T, strtoi, const char *nptr, char **endptr, int base, + INTMAX_T low, INTMAX_T high, int *rstatus) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtoi, nptr, endptr, base, low, high, rstatus); + char *real_endptr; + INTMAX_T ret = REAL(strtoi)(nptr, &real_endptr, base, low, high, rstatus); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + if (rstatus) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rstatus, sizeof(*rstatus)); + return ret; +} + +INTERCEPTOR(UINTMAX_T, strtou, const char *nptr, char **endptr, int base, + UINTMAX_T low, UINTMAX_T high, int *rstatus) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtou, nptr, endptr, base, low, high, rstatus); + char *real_endptr; + UINTMAX_T ret = REAL(strtou)(nptr, &real_endptr, base, low, high, rstatus); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + if (rstatus) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rstatus, sizeof(*rstatus)); + return ret; +} +#define INIT_STRTOI \ + COMMON_INTERCEPT_FUNCTION(strtoi); \ + COMMON_INTERCEPT_FUNCTION(strtou) +#else +#define INIT_STRTOI +#endif + +#if SANITIZER_INTERCEPT_CAPSICUM +#define CAP_RIGHTS_INIT_INTERCEPTOR(cap_rights_init, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_init, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + __sanitizer_cap_rights_t *ret = \ + REAL(cap_rights_init)(rights, ##__VA_ARGS__); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); \ + return ret; \ + } + +#define CAP_RIGHTS_SET_INTERCEPTOR(cap_rights_set, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_set, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + __sanitizer_cap_rights_t *ret = \ + REAL(cap_rights_set)(rights, ##__VA_ARGS__); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); \ + return ret; \ + } + +#define CAP_RIGHTS_CLEAR_INTERCEPTOR(cap_rights_clear, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_clear, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + __sanitizer_cap_rights_t *ret = \ + REAL(cap_rights_clear)(rights, ##__VA_ARGS__); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); \ + return ret; \ + } + +#define CAP_RIGHTS_IS_SET_INTERCEPTOR(cap_rights_is_set, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_is_set, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + return REAL(cap_rights_is_set)(rights, ##__VA_ARGS__); \ + } + +INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_init, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_INIT_INTERCEPTOR(cap_rights_init, rights); +} + +INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_set, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_SET_INTERCEPTOR(cap_rights_set, rights); +} + +INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_clear, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_CLEAR_INTERCEPTOR(cap_rights_clear, rights); +} + +INTERCEPTOR(bool, cap_rights_is_set, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_IS_SET_INTERCEPTOR(cap_rights_is_set, rights); +} + +INTERCEPTOR(int, cap_rights_limit, int fd, + const __sanitizer_cap_rights_t *rights) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_limit, fd, rights); + if (rights) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); + + return REAL(cap_rights_limit)(fd, rights); +} + +INTERCEPTOR(int, cap_rights_get, int fd, __sanitizer_cap_rights_t *rights) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_get, fd, rights); + int ret = REAL(cap_rights_get)(fd, rights); + if (!ret && rights) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rights, sizeof(*rights)); + + return ret; +} + +INTERCEPTOR(bool, cap_rights_is_valid, const __sanitizer_cap_rights_t *rights) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_is_valid, rights); + if (rights) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); + + return REAL(cap_rights_is_valid(rights)); +} + +INTERCEPTOR(__sanitizer_cap_rights *, cap_rights_merge, + __sanitizer_cap_rights *dst, const __sanitizer_cap_rights *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_merge, dst, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src)); + + __sanitizer_cap_rights *ret = REAL(cap_rights_merge)(dst, src); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst)); + + return ret; +} + +INTERCEPTOR(__sanitizer_cap_rights *, cap_rights_remove, + __sanitizer_cap_rights *dst, const __sanitizer_cap_rights *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_remove, dst, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src)); + + __sanitizer_cap_rights *ret = REAL(cap_rights_remove)(dst, src); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst)); + + return ret; +} + +INTERCEPTOR(bool, cap_rights_contains, const __sanitizer_cap_rights *big, + const __sanitizer_cap_rights *little) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_contains, big, little); + if (little) + COMMON_INTERCEPTOR_READ_RANGE(ctx, little, sizeof(*little)); + if (big) + COMMON_INTERCEPTOR_READ_RANGE(ctx, big, sizeof(*big)); + + return REAL(cap_rights_contains)(big, little); +} + +INTERCEPTOR(int, cap_ioctls_limit, int fd, const uptr *cmds, SIZE_T ncmds) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_ioctls_limit, fd, cmds, ncmds); + if (cmds) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cmds, sizeof(*cmds) * ncmds); + + return REAL(cap_ioctls_limit)(fd, cmds, ncmds); +} + +INTERCEPTOR(int, cap_ioctls_get, int fd, uptr *cmds, SIZE_T maxcmds) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_ioctls_get, fd, cmds, maxcmds); + int ret = REAL(cap_ioctls_get)(fd, cmds, maxcmds); + if (!ret && cmds) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cmds, sizeof(*cmds) * maxcmds); + + return ret; +} +#define INIT_CAPSICUM \ + COMMON_INTERCEPT_FUNCTION(cap_rights_init); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_set); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_clear); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_is_set); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_get); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_limit); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_contains); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_remove); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_merge); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_is_valid); \ + COMMON_INTERCEPT_FUNCTION(cap_ioctls_get); \ + COMMON_INTERCEPT_FUNCTION(cap_ioctls_limit) +#else +#define INIT_CAPSICUM +#endif + +#if SANITIZER_INTERCEPT_SHA1 +INTERCEPTOR(void, SHA1Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Init, context); + REAL(SHA1Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA1_CTX_sz); +} +INTERCEPTOR(void, SHA1Update, void *context, const u8 *data, unsigned len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz); + REAL(SHA1Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA1_CTX_sz); +} +INTERCEPTOR(void, SHA1Final, u8 digest[20], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz); + REAL(SHA1Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(u8) * 20); +} +INTERCEPTOR(void, SHA1Transform, u32 state[5], u8 buffer[64]) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Transform, state, buffer); + if (state) + COMMON_INTERCEPTOR_READ_RANGE(ctx, state, sizeof(u32) * 5); + if (buffer) + COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, sizeof(u8) * 64); + REAL(SHA1Transform)(state, buffer); + if (state) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, state, sizeof(u32) * 5); +} +INTERCEPTOR(char *, SHA1End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz); + char *ret = REAL(SHA1End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +INTERCEPTOR(char *, SHA1File, char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(SHA1File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +INTERCEPTOR(char *, SHA1FileChunk, char *filename, char *buf, OFF_T offset, + OFF_T length) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +INTERCEPTOR(char *, SHA1Data, u8 *data, SIZE_T len, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Data, data, len, buf); + if (data) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(SHA1Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +#define INIT_SHA1 \ + COMMON_INTERCEPT_FUNCTION(SHA1Init); \ + COMMON_INTERCEPT_FUNCTION(SHA1Update); \ + COMMON_INTERCEPT_FUNCTION(SHA1Final); \ + COMMON_INTERCEPT_FUNCTION(SHA1Transform); \ + COMMON_INTERCEPT_FUNCTION(SHA1End); \ + COMMON_INTERCEPT_FUNCTION(SHA1File); \ + COMMON_INTERCEPT_FUNCTION(SHA1FileChunk); \ + COMMON_INTERCEPT_FUNCTION(SHA1Data) +#else +#define INIT_SHA1 +#endif + +#if SANITIZER_INTERCEPT_MD4 +INTERCEPTOR(void, MD4Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Init, context); + REAL(MD4Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD4_CTX_sz); +} + +INTERCEPTOR(void, MD4Update, void *context, const unsigned char *data, + unsigned int len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz); + REAL(MD4Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD4_CTX_sz); +} + +INTERCEPTOR(void, MD4Final, unsigned char digest[16], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz); + REAL(MD4Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); +} + +INTERCEPTOR(char *, MD4End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz); + char *ret = REAL(MD4End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length); + return ret; +} + +INTERCEPTOR(char *, MD4File, const char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(MD4File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length); + return ret; +} + +INTERCEPTOR(char *, MD4Data, const unsigned char *data, unsigned int len, + char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(MD4Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length); + return ret; +} + +#define INIT_MD4 \ + COMMON_INTERCEPT_FUNCTION(MD4Init); \ + COMMON_INTERCEPT_FUNCTION(MD4Update); \ + COMMON_INTERCEPT_FUNCTION(MD4Final); \ + COMMON_INTERCEPT_FUNCTION(MD4End); \ + COMMON_INTERCEPT_FUNCTION(MD4File); \ + COMMON_INTERCEPT_FUNCTION(MD4Data) +#else +#define INIT_MD4 +#endif + +#if SANITIZER_INTERCEPT_RMD160 +INTERCEPTOR(void, RMD160Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Init, context); + REAL(RMD160Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, RMD160_CTX_sz); +} +INTERCEPTOR(void, RMD160Update, void *context, const u8 *data, unsigned len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz); + REAL(RMD160Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, RMD160_CTX_sz); +} +INTERCEPTOR(void, RMD160Final, u8 digest[20], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz); + REAL(RMD160Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(u8) * 20); +} +INTERCEPTOR(void, RMD160Transform, u32 state[5], u16 buffer[16]) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Transform, state, buffer); + if (state) + COMMON_INTERCEPTOR_READ_RANGE(ctx, state, sizeof(u32) * 5); + if (buffer) + COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, sizeof(u32) * 16); + REAL(RMD160Transform)(state, buffer); + if (state) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, state, sizeof(u32) * 5); +} +INTERCEPTOR(char *, RMD160End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz); + char *ret = REAL(RMD160End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +INTERCEPTOR(char *, RMD160File, char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(RMD160File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +INTERCEPTOR(char *, RMD160FileChunk, char *filename, char *buf, OFF_T offset, + OFF_T length) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +INTERCEPTOR(char *, RMD160Data, u8 *data, SIZE_T len, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(RMD160Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +#define INIT_RMD160 \ + COMMON_INTERCEPT_FUNCTION(RMD160Init); \ + COMMON_INTERCEPT_FUNCTION(RMD160Update); \ + COMMON_INTERCEPT_FUNCTION(RMD160Final); \ + COMMON_INTERCEPT_FUNCTION(RMD160Transform); \ + COMMON_INTERCEPT_FUNCTION(RMD160End); \ + COMMON_INTERCEPT_FUNCTION(RMD160File); \ + COMMON_INTERCEPT_FUNCTION(RMD160FileChunk); \ + COMMON_INTERCEPT_FUNCTION(RMD160Data) +#else +#define INIT_RMD160 +#endif + +#if SANITIZER_INTERCEPT_MD5 +INTERCEPTOR(void, MD5Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Init, context); + REAL(MD5Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz); +} + +INTERCEPTOR(void, MD5Update, void *context, const unsigned char *data, + unsigned int len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); + REAL(MD5Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz); +} + +INTERCEPTOR(void, MD5Final, unsigned char digest[16], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); + REAL(MD5Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); +} + +INTERCEPTOR(char *, MD5End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); + char *ret = REAL(MD5End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); + return ret; +} + +INTERCEPTOR(char *, MD5File, const char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(MD5File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); + return ret; +} + +INTERCEPTOR(char *, MD5Data, const unsigned char *data, unsigned int len, + char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(MD5Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); + return ret; +} + +#define INIT_MD5 \ + COMMON_INTERCEPT_FUNCTION(MD5Init); \ + COMMON_INTERCEPT_FUNCTION(MD5Update); \ + COMMON_INTERCEPT_FUNCTION(MD5Final); \ + COMMON_INTERCEPT_FUNCTION(MD5End); \ + COMMON_INTERCEPT_FUNCTION(MD5File); \ + COMMON_INTERCEPT_FUNCTION(MD5Data) +#else +#define INIT_MD5 +#endif + +#if SANITIZER_INTERCEPT_FSEEK +INTERCEPTOR(int, fseek, __sanitizer_FILE *stream, long int offset, int whence) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fseek, stream, offset, whence); + return REAL(fseek)(stream, offset, whence); +} +INTERCEPTOR(int, fseeko, __sanitizer_FILE *stream, OFF_T offset, int whence) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fseeko, stream, offset, whence); + return REAL(fseeko)(stream, offset, whence); +} +INTERCEPTOR(long int, ftell, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ftell, stream); + return REAL(ftell)(stream); +} +INTERCEPTOR(OFF_T, ftello, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ftello, stream); + return REAL(ftello)(stream); +} +INTERCEPTOR(void, rewind, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, rewind, stream); + return REAL(rewind)(stream); +} +INTERCEPTOR(int, fgetpos, __sanitizer_FILE *stream, void *pos) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fgetpos, stream, pos); + int ret = REAL(fgetpos)(stream, pos); + if (pos && !ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pos, fpos_t_sz); + return ret; +} +INTERCEPTOR(int, fsetpos, __sanitizer_FILE *stream, const void *pos) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fsetpos, stream, pos); + if (pos) + COMMON_INTERCEPTOR_READ_RANGE(ctx, pos, fpos_t_sz); + return REAL(fsetpos)(stream, pos); +} +#define INIT_FSEEK \ + COMMON_INTERCEPT_FUNCTION(fseek); \ + COMMON_INTERCEPT_FUNCTION(fseeko); \ + COMMON_INTERCEPT_FUNCTION(ftell); \ + COMMON_INTERCEPT_FUNCTION(ftello); \ + COMMON_INTERCEPT_FUNCTION(rewind); \ + COMMON_INTERCEPT_FUNCTION(fgetpos); \ + COMMON_INTERCEPT_FUNCTION(fsetpos) +#else +#define INIT_FSEEK +#endif + +#if SANITIZER_INTERCEPT_MD2 +INTERCEPTOR(void, MD2Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Init, context); + REAL(MD2Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD2_CTX_sz); +} + +INTERCEPTOR(void, MD2Update, void *context, const unsigned char *data, + unsigned int len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz); + REAL(MD2Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD2_CTX_sz); +} + +INTERCEPTOR(void, MD2Final, unsigned char digest[16], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz); + REAL(MD2Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); +} + +INTERCEPTOR(char *, MD2End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz); + char *ret = REAL(MD2End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length); + return ret; +} + +INTERCEPTOR(char *, MD2File, const char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(MD2File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length); + return ret; +} + +INTERCEPTOR(char *, MD2Data, const unsigned char *data, unsigned int len, + char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(MD2Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length); + return ret; +} + +#define INIT_MD2 \ + COMMON_INTERCEPT_FUNCTION(MD2Init); \ + COMMON_INTERCEPT_FUNCTION(MD2Update); \ + COMMON_INTERCEPT_FUNCTION(MD2Final); \ + COMMON_INTERCEPT_FUNCTION(MD2End); \ + COMMON_INTERCEPT_FUNCTION(MD2File); \ + COMMON_INTERCEPT_FUNCTION(MD2Data) +#else +#define INIT_MD2 +#endif + +#if SANITIZER_INTERCEPT_SHA2 +#define SHA2_INTERCEPTORS(LEN, SHA2_STATE_T) \ + INTERCEPTOR(void, SHA##LEN##_Init, void *context) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Init, context); \ + REAL(SHA##LEN##_Init)(context); \ + if (context) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + } \ + INTERCEPTOR(void, SHA##LEN##_Update, void *context, \ + const u8 *data, SIZE_T len) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Update, context, data, len); \ + if (data && len > 0) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \ + if (context) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + REAL(SHA##LEN##_Update)(context, data, len); \ + if (context) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + } \ + INTERCEPTOR(void, SHA##LEN##_Final, u8 digest[LEN/8], \ + void *context) { \ + void *ctx; \ + CHECK_EQ(SHA##LEN##_digest_length, LEN/8); \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Final, digest, context); \ + if (context) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + REAL(SHA##LEN##_Final)(digest, context); \ + if (digest) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, \ + sizeof(digest[0]) * \ + SHA##LEN##_digest_length); \ + } \ + INTERCEPTOR(char *, SHA##LEN##_End, void *context, char *buf) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_End, context, buf); \ + if (context) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + char *ret = REAL(SHA##LEN##_End)(context, buf); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } \ + INTERCEPTOR(char *, SHA##LEN##_File, const char *filename, char *buf) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \ + if (filename) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\ + char *ret = REAL(SHA##LEN##_File)(filename, buf); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } \ + INTERCEPTOR(char *, SHA##LEN##_FileChunk, const char *filename, char *buf, \ + OFF_T offset, OFF_T length) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \ + length); \ + if (filename) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\ + char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } \ + INTERCEPTOR(char *, SHA##LEN##_Data, u8 *data, SIZE_T len, char *buf) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Data, data, len, buf); \ + if (data && len > 0) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \ + char *ret = REAL(SHA##LEN##_Data)(data, len, buf); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } + +SHA2_INTERCEPTORS(224, u32); +SHA2_INTERCEPTORS(256, u32); +SHA2_INTERCEPTORS(384, u64); +SHA2_INTERCEPTORS(512, u64); + +#define INIT_SHA2_INTECEPTORS(LEN) \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Update); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Final); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_End); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_File); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_FileChunk); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Data) + +#define INIT_SHA2 \ + INIT_SHA2_INTECEPTORS(224); \ + INIT_SHA2_INTECEPTORS(256); \ + INIT_SHA2_INTECEPTORS(384); \ + INIT_SHA2_INTECEPTORS(512) +#undef SHA2_INTERCEPTORS +#else +#define INIT_SHA2 +#endif + +#if SANITIZER_INTERCEPT_VIS +INTERCEPTOR(char *, vis, char *dst, int c, int flag, int nextc) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, vis, dst, c, flag, nextc); + char *end = REAL(vis)(dst, c, flag, nextc); + // dst is NULL terminated and end points to the NULL char + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1); + return end; +} +INTERCEPTOR(char *, nvis, char *dst, SIZE_T dlen, int c, int flag, int nextc) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, nvis, dst, dlen, c, flag, nextc); + char *end = REAL(nvis)(dst, dlen, c, flag, nextc); + // nvis cannot make sure the dst is NULL terminated + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1); + return end; +} +INTERCEPTOR(int, strvis, char *dst, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int len = REAL(strvis)(dst, src, flag); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, stravis, char **dst, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int len = REAL(stravis)(dst, src, flag); + if (dst) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *)); + if (*dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *dst, len + 1); + } + return len; +} +INTERCEPTOR(int, strnvis, char *dst, SIZE_T dlen, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int len = REAL(strnvis)(dst, dlen, src, flag); + // The interface will be valid even if there is no space for NULL char + if (dst && len > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, strvisx, char *dst, const char *src, SIZE_T len, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strvisx, dst, src, len, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + int ret = REAL(strvisx)(dst, src, len, flag); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strnvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len, + int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnvisx, dst, dlen, src, len, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + int ret = REAL(strnvisx)(dst, dlen, src, len, flag); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strenvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len, + int flag, int *cerr_ptr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strenvisx, dst, dlen, src, len, flag, cerr_ptr); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + // FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold + // according to the implementation + if (cerr_ptr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cerr_ptr, sizeof(int)); + int ret = REAL(strenvisx)(dst, dlen, src, len, flag, cerr_ptr); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + if (cerr_ptr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cerr_ptr, sizeof(int)); + return ret; +} +INTERCEPTOR(char *, svis, char *dst, int c, int flag, int nextc, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + char *end = REAL(svis)(dst, c, flag, nextc, extra); + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1); + return end; +} +INTERCEPTOR(char *, snvis, char *dst, SIZE_T dlen, int c, int flag, int nextc, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra); + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, + Min((SIZE_T)(end - dst + 1), dlen)); + return end; +} +INTERCEPTOR(int, strsvis, char *dst, const char *src, int flag, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int len = REAL(strsvis)(dst, src, flag, extra); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, strsnvis, char *dst, SIZE_T dlen, const char *src, int flag, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int len = REAL(strsnvis)(dst, dlen, src, flag, extra); + // The interface will be valid even if there is no space for NULL char + if (dst && len >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, strsvisx, char *dst, const char *src, SIZE_T len, int flag, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsvisx, dst, src, len, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int ret = REAL(strsvisx)(dst, src, len, flag, extra); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strsnvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len, + int flag, const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsnvisx, dst, dlen, src, len, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strsenvisx, char *dst, SIZE_T dlen, const char *src, + SIZE_T len, int flag, const char *extra, int *cerr_ptr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsenvisx, dst, dlen, src, len, flag, extra, + cerr_ptr); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + // FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold + // according to the implementation + if (cerr_ptr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cerr_ptr, sizeof(int)); + int ret = REAL(strsenvisx)(dst, dlen, src, len, flag, extra, cerr_ptr); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + if (cerr_ptr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cerr_ptr, sizeof(int)); + return ret; +} +INTERCEPTOR(int, unvis, char *cp, int c, int *astate, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, unvis, cp, c, astate, flag); + if (astate) + COMMON_INTERCEPTOR_READ_RANGE(ctx, astate, sizeof(*astate)); + int ret = REAL(unvis)(cp, c, astate, flag); + if (ret == unvis_valid || ret == unvis_validpush) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cp, sizeof(*cp)); + } + return ret; +} +INTERCEPTOR(int, strunvis, char *dst, const char *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strunvis)(dst, src); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strnunvis, char *dst, SIZE_T dlen, const char *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strnunvis)(dst, dlen, src); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strunvisx, char *dst, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strunvisx)(dst, src, flag); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strnunvisx, char *dst, SIZE_T dlen, const char *src, + int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strnunvisx)(dst, dlen, src, flag); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +#define INIT_VIS \ + COMMON_INTERCEPT_FUNCTION(vis); \ + COMMON_INTERCEPT_FUNCTION(nvis); \ + COMMON_INTERCEPT_FUNCTION(strvis); \ + COMMON_INTERCEPT_FUNCTION(stravis); \ + COMMON_INTERCEPT_FUNCTION(strnvis); \ + COMMON_INTERCEPT_FUNCTION(strvisx); \ + COMMON_INTERCEPT_FUNCTION(strnvisx); \ + COMMON_INTERCEPT_FUNCTION(strenvisx); \ + COMMON_INTERCEPT_FUNCTION(svis); \ + COMMON_INTERCEPT_FUNCTION(snvis); \ + COMMON_INTERCEPT_FUNCTION(strsvis); \ + COMMON_INTERCEPT_FUNCTION(strsnvis); \ + COMMON_INTERCEPT_FUNCTION(strsvisx); \ + COMMON_INTERCEPT_FUNCTION(strsnvisx); \ + COMMON_INTERCEPT_FUNCTION(strsenvisx); \ + COMMON_INTERCEPT_FUNCTION(unvis); \ + COMMON_INTERCEPT_FUNCTION(strunvis); \ + COMMON_INTERCEPT_FUNCTION(strnunvis); \ + COMMON_INTERCEPT_FUNCTION(strunvisx); \ + COMMON_INTERCEPT_FUNCTION(strnunvisx) +#else +#define INIT_VIS +#endif + +#if SANITIZER_INTERCEPT_CDB +INTERCEPTOR(struct __sanitizer_cdbr *, cdbr_open, const char *path, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags); + if (path) + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags); + if (cdbr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr)); + return cdbr; +} + +INTERCEPTOR(struct __sanitizer_cdbr *, cdbr_open_mem, void *base, SIZE_T size, + int flags, void (*unmap)(void *, void *, SIZE_T), void *cookie) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open_mem, base, size, flags, unmap, + cookie); + if (base && size) + COMMON_INTERCEPTOR_READ_RANGE(ctx, base, size); + struct __sanitizer_cdbr *cdbr = + REAL(cdbr_open_mem)(base, size, flags, unmap, cookie); + if (cdbr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr)); + return cdbr; +} + +INTERCEPTOR(u32, cdbr_entries, struct __sanitizer_cdbr *cdbr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_entries, cdbr); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + return REAL(cdbr_entries)(cdbr); +} + +INTERCEPTOR(int, cdbr_get, struct __sanitizer_cdbr *cdbr, u32 index, + const void **data, SIZE_T *datalen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_get, cdbr, index, data, datalen); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + int ret = REAL(cdbr_get)(cdbr, index, data, datalen); + if (!ret) { + if (data) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(*data)); + if (datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datalen, sizeof(*datalen)); + if (data && datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *data, *datalen); + } + return ret; +} + +INTERCEPTOR(int, cdbr_find, struct __sanitizer_cdbr *cdbr, const void *key, + SIZE_T keylen, const void **data, SIZE_T *datalen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_find, cdbr, key, keylen, data, datalen); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + if (key) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen); + int ret = REAL(cdbr_find)(cdbr, key, keylen, data, datalen); + if (!ret) { + if (data) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(*data)); + if (datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datalen, sizeof(*datalen)); + if (data && datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *data, *datalen); + } + return ret; +} + +INTERCEPTOR(void, cdbr_close, struct __sanitizer_cdbr *cdbr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_close, cdbr); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + REAL(cdbr_close)(cdbr); +} + +INTERCEPTOR(struct __sanitizer_cdbw *, cdbw_open) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_open); + struct __sanitizer_cdbw *ret = REAL(cdbw_open)(); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); + return ret; +} + +INTERCEPTOR(int, cdbw_put, struct __sanitizer_cdbw *cdbw, const void *key, + SIZE_T keylen, const void *data, SIZE_T datalen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put, cdbw, key, keylen, data, datalen); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (data && datalen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, datalen); + if (key && keylen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen); + int ret = REAL(cdbw_put)(cdbw, key, keylen, data, datalen); + if (!ret && cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + return ret; +} + +INTERCEPTOR(int, cdbw_put_data, struct __sanitizer_cdbw *cdbw, const void *data, + SIZE_T datalen, u32 *index) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put_data, cdbw, data, datalen, index); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (data && datalen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, datalen); + int ret = REAL(cdbw_put_data)(cdbw, data, datalen, index); + if (!ret) { + if (index) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, index, sizeof(*index)); + if (cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + } + return ret; +} + +INTERCEPTOR(int, cdbw_put_key, struct __sanitizer_cdbw *cdbw, const void *key, + SIZE_T keylen, u32 index) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put_key, cdbw, key, keylen, index); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (key && keylen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen); + int ret = REAL(cdbw_put_key)(cdbw, key, keylen, index); + if (!ret && cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + return ret; +} + +INTERCEPTOR(int, cdbw_output, struct __sanitizer_cdbw *cdbw, int output, + const char descr[16], u32 (*seedgen)(void)) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_output, cdbw, output, descr, seedgen); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, output); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (descr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, descr, internal_strnlen(descr, 16)); + if (seedgen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)seedgen, sizeof(seedgen)); + int ret = REAL(cdbw_output)(cdbw, output, descr, seedgen); + if (!ret) { + if (cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (output >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, output); + } + return ret; +} + +INTERCEPTOR(void, cdbw_close, struct __sanitizer_cdbw *cdbw) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_close, cdbw); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + REAL(cdbw_close)(cdbw); +} + +#define INIT_CDB \ + COMMON_INTERCEPT_FUNCTION(cdbr_open); \ + COMMON_INTERCEPT_FUNCTION(cdbr_open_mem); \ + COMMON_INTERCEPT_FUNCTION(cdbr_entries); \ + COMMON_INTERCEPT_FUNCTION(cdbr_get); \ + COMMON_INTERCEPT_FUNCTION(cdbr_find); \ + COMMON_INTERCEPT_FUNCTION(cdbr_close); \ + COMMON_INTERCEPT_FUNCTION(cdbw_open); \ + COMMON_INTERCEPT_FUNCTION(cdbw_put); \ + COMMON_INTERCEPT_FUNCTION(cdbw_put_data); \ + COMMON_INTERCEPT_FUNCTION(cdbw_put_key); \ + COMMON_INTERCEPT_FUNCTION(cdbw_output); \ + COMMON_INTERCEPT_FUNCTION(cdbw_close) +#else +#define INIT_CDB +#endif + +#if SANITIZER_INTERCEPT_GETFSENT +INTERCEPTOR(void *, getfsent) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getfsent); + void *ret = REAL(getfsent)(); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz); + return ret; +} + +INTERCEPTOR(void *, getfsspec, const char *spec) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec); + if (spec) + COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, REAL(strlen)(spec) + 1); + void *ret = REAL(getfsspec)(spec); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz); + return ret; +} + +INTERCEPTOR(void *, getfsfile, const char *file) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file); + if (file) + COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1); + void *ret = REAL(getfsfile)(file); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz); + return ret; +} + +#define INIT_GETFSENT \ + COMMON_INTERCEPT_FUNCTION(getfsent); \ + COMMON_INTERCEPT_FUNCTION(getfsspec); \ + COMMON_INTERCEPT_FUNCTION(getfsfile); +#else +#define INIT_GETFSENT +#endif + +#if SANITIZER_INTERCEPT_ARC4RANDOM +INTERCEPTOR(void, arc4random_buf, void *buf, SIZE_T len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, arc4random_buf, buf, len); + REAL(arc4random_buf)(buf, len); + if (buf && len) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, len); +} + +INTERCEPTOR(void, arc4random_addrandom, u8 *dat, int datlen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, arc4random_addrandom, dat, datlen); + if (dat && datlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, dat, datlen); + REAL(arc4random_addrandom)(dat, datlen); +} + +#define INIT_ARC4RANDOM \ + COMMON_INTERCEPT_FUNCTION(arc4random_buf); \ + COMMON_INTERCEPT_FUNCTION(arc4random_addrandom); +#else +#define INIT_ARC4RANDOM +#endif + +#if SANITIZER_INTERCEPT_POPEN +INTERCEPTOR(__sanitizer_FILE *, popen, const char *command, const char *type) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type); + if (command) + COMMON_INTERCEPTOR_READ_RANGE(ctx, command, REAL(strlen)(command) + 1); + if (type) + COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1); + __sanitizer_FILE *res = REAL(popen)(command, type); + COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr); + if (res) unpoison_file(res); + return res; +} +#define INIT_POPEN COMMON_INTERCEPT_FUNCTION(popen) +#else +#define INIT_POPEN +#endif + +#if SANITIZER_INTERCEPT_POPENVE +INTERCEPTOR(__sanitizer_FILE *, popenve, const char *path, + char *const *argv, char *const *envp, const char *type) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type); + if (path) + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + if (argv) { + for (char *const *pa = argv; ; ++pa) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **)); + if (!*pa) + break; + COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1); + } + } + if (envp) { + for (char *const *pa = envp; ; ++pa) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **)); + if (!*pa) + break; + COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1); + } + } + if (type) + COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1); + __sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type); + COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr); + if (res) unpoison_file(res); + return res; +} +#define INIT_POPENVE COMMON_INTERCEPT_FUNCTION(popenve) +#else +#define INIT_POPENVE +#endif + +#if SANITIZER_INTERCEPT_PCLOSE +INTERCEPTOR(int, pclose, __sanitizer_FILE *fp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pclose, fp); + COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp); + const FileMetadata *m = GetInterceptorMetadata(fp); + int res = REAL(pclose)(fp); + if (m) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size); + DeleteInterceptorMetadata(fp); + } + return res; +} +#define INIT_PCLOSE COMMON_INTERCEPT_FUNCTION(pclose); +#else +#define INIT_PCLOSE +#endif + +#if SANITIZER_INTERCEPT_FUNOPEN +typedef int (*funopen_readfn)(void *cookie, char *buf, int len); +typedef int (*funopen_writefn)(void *cookie, const char *buf, int len); +typedef OFF_T (*funopen_seekfn)(void *cookie, OFF_T offset, int whence); +typedef int (*funopen_closefn)(void *cookie); + +struct WrappedFunopenCookie { + void *real_cookie; + funopen_readfn real_read; + funopen_writefn real_write; + funopen_seekfn real_seek; + funopen_closefn real_close; +}; + +static int wrapped_funopen_read(void *cookie, char *buf, int len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_readfn real_read = wrapped_cookie->real_read; + return real_read(wrapped_cookie->real_cookie, buf, len); +} + +static int wrapped_funopen_write(void *cookie, const char *buf, int len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_writefn real_write = wrapped_cookie->real_write; + return real_write(wrapped_cookie->real_cookie, buf, len); +} + +static OFF_T wrapped_funopen_seek(void *cookie, OFF_T offset, int whence) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_seekfn real_seek = wrapped_cookie->real_seek; + return real_seek(wrapped_cookie->real_cookie, offset, whence); +} + +static int wrapped_funopen_close(void *cookie) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(1); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_closefn real_close = wrapped_cookie->real_close; + int res = real_close(wrapped_cookie->real_cookie); + InternalFree(wrapped_cookie); + return res; +} + +INTERCEPTOR(__sanitizer_FILE *, funopen, void *cookie, funopen_readfn readfn, + funopen_writefn writefn, funopen_seekfn seekfn, + funopen_closefn closefn) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, funopen, cookie, readfn, writefn, seekfn, + closefn); + + WrappedFunopenCookie *wrapped_cookie = + (WrappedFunopenCookie *)InternalAlloc(sizeof(WrappedFunopenCookie)); + wrapped_cookie->real_cookie = cookie; + wrapped_cookie->real_read = readfn; + wrapped_cookie->real_write = writefn; + wrapped_cookie->real_seek = seekfn; + wrapped_cookie->real_close = closefn; + + __sanitizer_FILE *res = + REAL(funopen)(wrapped_cookie, + readfn ? wrapped_funopen_read : nullptr, + writefn ? wrapped_funopen_write : nullptr, + seekfn ? wrapped_funopen_seek : nullptr, + closefn ? wrapped_funopen_close : nullptr); + if (res) + unpoison_file(res); + return res; +} +#define INIT_FUNOPEN COMMON_INTERCEPT_FUNCTION(funopen) +#else +#define INIT_FUNOPEN +#endif + +#if SANITIZER_INTERCEPT_FUNOPEN2 +typedef SSIZE_T (*funopen2_readfn)(void *cookie, void *buf, SIZE_T len); +typedef SSIZE_T (*funopen2_writefn)(void *cookie, const void *buf, SIZE_T len); +typedef OFF_T (*funopen2_seekfn)(void *cookie, OFF_T offset, int whence); +typedef int (*funopen2_flushfn)(void *cookie); +typedef int (*funopen2_closefn)(void *cookie); + +struct WrappedFunopen2Cookie { + void *real_cookie; + funopen2_readfn real_read; + funopen2_writefn real_write; + funopen2_seekfn real_seek; + funopen2_flushfn real_flush; + funopen2_closefn real_close; +}; + +static SSIZE_T wrapped_funopen2_read(void *cookie, void *buf, SIZE_T len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_readfn real_read = wrapped_cookie->real_read; + return real_read(wrapped_cookie->real_cookie, buf, len); +} + +static SSIZE_T wrapped_funopen2_write(void *cookie, const void *buf, + SIZE_T len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_writefn real_write = wrapped_cookie->real_write; + return real_write(wrapped_cookie->real_cookie, buf, len); +} + +static OFF_T wrapped_funopen2_seek(void *cookie, OFF_T offset, int whence) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_seekfn real_seek = wrapped_cookie->real_seek; + return real_seek(wrapped_cookie->real_cookie, offset, whence); +} + +static int wrapped_funopen2_flush(void *cookie) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(1); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_flushfn real_flush = wrapped_cookie->real_flush; + return real_flush(wrapped_cookie->real_cookie); +} + +static int wrapped_funopen2_close(void *cookie) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(1); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_closefn real_close = wrapped_cookie->real_close; + int res = real_close(wrapped_cookie->real_cookie); + InternalFree(wrapped_cookie); + return res; +} + +INTERCEPTOR(__sanitizer_FILE *, funopen2, void *cookie, funopen2_readfn readfn, + funopen2_writefn writefn, funopen2_seekfn seekfn, + funopen2_flushfn flushfn, funopen2_closefn closefn) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, funopen2, cookie, readfn, writefn, seekfn, + flushfn, closefn); + + WrappedFunopen2Cookie *wrapped_cookie = + (WrappedFunopen2Cookie *)InternalAlloc(sizeof(WrappedFunopen2Cookie)); + wrapped_cookie->real_cookie = cookie; + wrapped_cookie->real_read = readfn; + wrapped_cookie->real_write = writefn; + wrapped_cookie->real_seek = seekfn; + wrapped_cookie->real_flush = flushfn; + wrapped_cookie->real_close = closefn; + + __sanitizer_FILE *res = + REAL(funopen2)(wrapped_cookie, + readfn ? wrapped_funopen2_read : nullptr, + writefn ? wrapped_funopen2_write : nullptr, + seekfn ? wrapped_funopen2_seek : nullptr, + flushfn ? wrapped_funopen2_flush : nullptr, + closefn ? wrapped_funopen2_close : nullptr); + if (res) + unpoison_file(res); + return res; +} +#define INIT_FUNOPEN2 COMMON_INTERCEPT_FUNCTION(funopen2) +#else +#define INIT_FUNOPEN2 +#endif + +#if SANITIZER_INTERCEPT_FDEVNAME +INTERCEPTOR(char *, fdevname, int fd) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fdevname, fd); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + char *name = REAL(fdevname)(fd); + if (name) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1); + if (fd > 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return name; +} + +INTERCEPTOR(char *, fdevname_r, int fd, char *buf, SIZE_T len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fdevname_r, fd, buf, len); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + char *name = REAL(fdevname_r)(fd, buf, len); + if (name && buf && len > 0) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1); + if (fd > 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return name; +} + +#define INIT_FDEVNAME \ + COMMON_INTERCEPT_FUNCTION(fdevname); \ + COMMON_INTERCEPT_FUNCTION(fdevname_r); +#else +#define INIT_FDEVNAME +#endif + +#if SANITIZER_INTERCEPT_GETUSERSHELL +INTERCEPTOR(char *, getusershell) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getusershell); + char *res = REAL(getusershell)(); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); + return res; +} + +#define INIT_GETUSERSHELL COMMON_INTERCEPT_FUNCTION(getusershell); +#else +#define INIT_GETUSERSHELL +#endif + +#if SANITIZER_INTERCEPT_SL_INIT +INTERCEPTOR(void *, sl_init) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_init); + void *res = REAL(sl_init)(); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer::struct_StringList_sz); + return res; +} + +INTERCEPTOR(int, sl_add, void *sl, char *item) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_add, sl, item); + if (sl) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + if (item) + COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1); + int res = REAL(sl_add)(sl, item); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + return res; +} + +INTERCEPTOR(char *, sl_find, void *sl, const char *item) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_find, sl, item); + if (sl) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + if (item) + COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1); + char *res = REAL(sl_find)(sl, item); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); + return res; +} + +INTERCEPTOR(void, sl_free, void *sl, int freeall) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_free, sl, freeall); + if (sl) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + REAL(sl_free)(sl, freeall); +} + +#define INIT_SL_INIT \ + COMMON_INTERCEPT_FUNCTION(sl_init); \ + COMMON_INTERCEPT_FUNCTION(sl_add); \ + COMMON_INTERCEPT_FUNCTION(sl_find); \ + COMMON_INTERCEPT_FUNCTION(sl_free); +#else +#define INIT_SL_INIT +#endif + static void InitializeCommonInterceptors() { static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1]; - interceptor_metadata_map = new((void *)&metadata_mem) MetadataHashMap(); + interceptor_metadata_map = + new ((void *)&metadata_mem) MetadataHashMap(); // NOLINT INIT_MMAP; INIT_MMAP64; @@ -7299,6 +9532,8 @@ static void InitializeCommonInterceptors() { INIT_GETPWENT; INIT_FGETPWENT; INIT_GETPWENT_R; + INIT_FGETPWENT_R; + INIT_FGETGRENT_R; INIT_SETPWENT; INIT_CLOCK_GETTIME; INIT_GETITIMER; @@ -7408,7 +9643,9 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_GETNAME_NP; INIT_SINCOS; INIT_REMQUO; + INIT_REMQUOL; INIT_LGAMMA; + INIT_LGAMMAL; INIT_LGAMMA_R; INIT_LGAMMAL_R; INIT_DRAND48_R; @@ -7484,6 +9721,42 @@ static void InitializeCommonInterceptors() { INIT_TTYENT; INIT_PROTOENT; INIT_NETENT; + INIT_GETMNTINFO; + INIT_MI_VECTOR_HASH; + INIT_SETVBUF; + INIT_GETVFSSTAT; + INIT_REGEX; + INIT_REGEXSUB; + INIT_FTS; + INIT_SYSCTL; + INIT_ASYSCTL; + INIT_SYSCTLGETMIBINFO; + INIT_NL_LANGINFO; + INIT_MODCTL; + INIT_STRTONUM; + INIT_FPARSELN; + INIT_STATVFS1; + INIT_STRTOI; + INIT_CAPSICUM; + INIT_SHA1; + INIT_MD4; + INIT_RMD160; + INIT_MD5; + INIT_FSEEK; + INIT_MD2; + INIT_SHA2; + INIT_VIS; + INIT_CDB; + INIT_GETFSENT; + INIT_ARC4RANDOM; + INIT_POPEN; + INIT_POPENVE; + INIT_PCLOSE; + INIT_FUNOPEN; + INIT_FUNOPEN2; + INIT_FDEVNAME; + INIT_GETUSERSHELL; + INIT_SL_INIT; INIT___PRINTF_CHK; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc index 796a054858b..1c0995b2d27 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -25,7 +25,7 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) { SoftRssLimitExceededCallback = Callback; } -#if SANITIZER_LINUX && !SANITIZER_GO +#if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO // Weak default implementation for when sanitizer_stackdepot is not linked in. SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() { return nullptr; @@ -100,18 +100,21 @@ void WriteToSyslog(const char *msg) { // Print one line at a time. // syslog, at least on Android, has an implicit message length limit. - do { - q = internal_strchr(p, '\n'); - if (q) - *q = '\0'; + while ((q = internal_strchr(p, '\n'))) { + *q = '\0'; + WriteOneLineToSyslog(p); + p = q + 1; + } + // Print remaining characters, if there are any. + // Note that this will add an extra newline at the end. + // FIXME: buffer extra output. This would need a thread-local buffer, which + // on Android requires plugging into the tools (ex. ASan's) Thread class. + if (*p) WriteOneLineToSyslog(p); - if (q) - p = q + 1; - } while (q); } void MaybeStartBackgroudThread() { -#if SANITIZER_LINUX && \ +#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \ !SANITIZER_GO // Need to implement/test on other platforms. // Start the background thread if one of the rss limits is given. if (!common_flags()->hard_rss_limit_mb && diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc index b4ffcca5c30..9ff8798a573 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc @@ -31,6 +31,7 @@ #include "sanitizer_atomic.h" #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_symbolizer_fuchsia.h" #include #include @@ -101,7 +102,7 @@ class TracePcGuardController final { // uses the `dumpfile` symbolizer markup element to highlight the // dump. See the explanation for this in: // https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md - Printf("SanitizerCoverage: {{{dumpfile:%s:%s}}} with up to %u PCs\n", + Printf("SanitizerCoverage: " FORMAT_DUMPFILE " with up to %u PCs\n", kSancovSinkName, vmo_name_, next_index_ - 1); } } @@ -146,9 +147,9 @@ class TracePcGuardController final { // indices, but we'll never move the mapping address so we don't have // any multi-thread synchronization issues with that. uintptr_t mapping; - status = _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo_, 0, MappingSize, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, - &mapping); + status = + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, + 0, vmo_, 0, MappingSize, &mapping); CHECK_EQ(status, ZX_OK); // Hereafter other threads are free to start storing into diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc index 4b0bbf1ed1e..108f76effee 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc @@ -7,16 +7,57 @@ // //===----------------------------------------------------------------------===// // -// This file defines delimiters for Sanitizer Coverage's section. +// This file defines delimiters for Sanitizer Coverage's section. It contains +// Windows specific tricks to coax the linker into giving us the start and stop +// addresses of a section, as ELF linkers can do, to get the size of certain +// arrays. According to https://msdn.microsoft.com/en-us/library/7977wcck.aspx +// sections with the same name before "$" are sorted alphabetically by the +// string that comes after "$" and merged into one section. We take advantage +// of this by putting data we want the size of into the middle (M) of a section, +// by using the letter "M" after "$". We get the start of this data (ie: +// __start_section_name) by making the start variable come at the start of the +// section (using the letter A after "$"). We do the same to get the end of the +// data by using the letter "Z" after "$" to make the end variable come after +// the data. Note that because of our technique the address of the start +// variable is actually the address of data that comes before our middle +// section. We also need to prevent the linker from adding any padding. Each +// technique we use for this is explained in the comments below. //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" #if SANITIZER_WINDOWS #include -#pragma section(".SCOV$A", read, write) // NOLINT -#pragma section(".SCOV$Z", read, write) // NOLINT extern "C" { -__declspec(allocate(".SCOV$A")) uint32_t __start___sancov_guards = 0; -__declspec(allocate(".SCOV$Z")) uint32_t __stop___sancov_guards = 0; +// The Guard array and counter array should both be merged into the .data +// section to reduce the number of PE sections However, because PCTable is +// constant it should be merged with the .rdata section. +#pragma section(".SCOV$GA", read, write) // NOLINT +// Use align(1) to avoid adding any padding that will mess up clients trying to +// determine the start and end of the array. +__declspec(allocate(".SCOV$GA")) __declspec(align(1)) uint64_t + __start___sancov_guards = 0; +#pragma section(".SCOV$GZ", read, write) // NOLINT +__declspec(allocate(".SCOV$GZ")) __declspec(align(1)) uint64_t + __stop___sancov_guards = 0; + +#pragma section(".SCOV$CA", read, write) // NOLINT +__declspec(allocate(".SCOV$CA")) __declspec(align(1)) uint64_t + __start___sancov_cntrs = 0; +#pragma section(".SCOV$CZ", read, write) // NOLINT +__declspec(allocate(".SCOV$CZ")) __declspec(align(1)) uint64_t + __stop___sancov_cntrs = 0; + +#pragma comment(linker, "/MERGE:.SCOV=.data") + +// Use uint64_t so there won't be any issues if the linker tries to word align +// the pc array. +#pragma section(".SCOVP$A", read) // NOLINT +__declspec(allocate(".SCOVP$A")) __declspec(align(1)) uint64_t + __start___sancov_pcs = 0; +#pragma section(".SCOVP$Z", read) // NOLINT +__declspec(allocate(".SCOVP$Z")) __declspec(align(1)) uint64_t + __stop___sancov_pcs = 0; + +#pragma comment(linker, "/MERGE:.SCOVP=.rdata") } -#endif // SANITIZER_WINDOWS +#endif // SANITIZER_WINDOWS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h index 9a12ab7342c..52e6ef8fb1a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h @@ -66,9 +66,6 @@ bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written = nullptr, error_t *error_p = nullptr); -bool RenameFile(const char *oldpath, const char *newpath, - error_t *error_p = nullptr); - // Scoped file handle closer. struct FileCloser { explicit FileCloser(fd_t fd) : fd(fd) {} diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc index cfe8af89342..b047741b908 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc @@ -103,7 +103,7 @@ COMMON_FLAG(bool, allow_user_segv_handler, true, "handle_*=1 will be upgraded to handle_*=2.") COMMON_FLAG(bool, use_sigaltstack, true, "If set, uses alternate stack for signal handling.") -COMMON_FLAG(bool, detect_deadlocks, false, +COMMON_FLAG(bool, detect_deadlocks, true, "If set, deadlock detection is enabled.") COMMON_FLAG( uptr, clear_shadow_mmap_threshold, 64 * 1024, @@ -243,3 +243,6 @@ COMMON_FLAG(bool, dump_registers, true, COMMON_FLAG(bool, detect_write_exec, false, "If true, triggers warning when writable-executable pages requests " "are being made") +COMMON_FLAG(bool, test_only_emulate_no_memorymap, false, + "TEST ONLY fail to read memory mappings to emulate sanitized " + "\"init\"") diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc index 391620690f3..2c259b6cd7d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -29,9 +29,6 @@ namespace __sanitizer { -// TODO(phosek): remove this and replace it with ZX_TIME_INFINITE -#define ZX_TIME_INFINITE_OLD INT64_MAX - void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); } uptr internal_sched_yield() { @@ -89,8 +86,10 @@ void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) { *stack_top = *stack_bottom + size; } +void InitializePlatformEarly() {} void MaybeReexec() {} void CheckASLR() {} +void CheckMPROTECT() {} void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {} void DisableCoreDumperIfNecessary() {} void InstallDeadlySignalHandlers(SignalHandlerType handler) {} @@ -122,8 +121,9 @@ void BlockingMutex::Lock() { if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked) return; while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { - zx_status_t status = _zx_futex_wait(reinterpret_cast(m), - MtxSleeping, ZX_TIME_INFINITE_OLD); + zx_status_t status = + _zx_futex_wait(reinterpret_cast(m), MtxSleeping, + ZX_HANDLE_INVALID, ZX_TIME_INFINITE); if (status != ZX_ERR_BAD_STATE) // Normal race. CHECK_EQ(status, ZX_OK); } @@ -175,8 +175,8 @@ static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? uintptr_t addr; status = - _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo, 0, size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + vmo, 0, size, &addr); _zx_handle_close(vmo); if (status != ZX_OK) { @@ -210,10 +210,10 @@ uptr ReservedAddressRange::Init(uptr init_size, const char *name, uintptr_t base; zx_handle_t vmar; zx_status_t status = - _zx_vmar_allocate_old(_zx_vmar_root_self(), 0, init_size, - ZX_VM_FLAG_CAN_MAP_READ | ZX_VM_FLAG_CAN_MAP_WRITE | - ZX_VM_FLAG_CAN_MAP_SPECIFIC, - &vmar, &base); + _zx_vmar_allocate( + _zx_vmar_root_self(), + ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, + 0, init_size, &vmar, &base); if (status != ZX_OK) ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status); base_ = reinterpret_cast(base); @@ -239,10 +239,9 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, DCHECK_GE(base + size_, map_size + offset); uintptr_t addr; - status = _zx_vmar_map_old( - vmar, offset, vmo, 0, map_size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_SPECIFIC, - &addr); + status = + _zx_vmar_map(vmar, ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC, + offset, vmo, 0, map_size, &addr); _zx_handle_close(vmo); if (status != ZX_OK) { if (status != ZX_ERR_NO_MEMORY || die_for_nomem) { @@ -281,14 +280,22 @@ void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) { void ReservedAddressRange::Unmap(uptr addr, uptr size) { CHECK_LE(size, size_); - if (addr == reinterpret_cast(base_)) - // If we unmap the whole range, just null out the base. - base_ = (size == size_) ? nullptr : reinterpret_cast(addr + size); - else + const zx_handle_t vmar = static_cast(os_handle_); + if (addr == reinterpret_cast(base_)) { + if (size == size_) { + // Destroying the vmar effectively unmaps the whole mapping. + _zx_vmar_destroy(vmar); + _zx_handle_close(vmar); + os_handle_ = static_cast(ZX_HANDLE_INVALID); + DecreaseTotalMmap(size); + return; + } + } else { CHECK_EQ(addr + size, reinterpret_cast(base_) + size_); - size_ -= size; - UnmapOrDieVmar(reinterpret_cast(addr), size, - static_cast(os_handle_)); + } + // Partial unmapping does not affect the fact that the initial range is still + // reserved, and the resulting unmapped memory can't be reused. + UnmapOrDieVmar(reinterpret_cast(addr), size, vmar); } // This should never be called. @@ -321,8 +328,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, size_t map_size = size + alignment; uintptr_t addr; status = - _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo, 0, map_size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + vmo, 0, map_size, &addr); if (status == ZX_OK) { uintptr_t map_addr = addr; uintptr_t map_end = map_addr + map_size; @@ -334,11 +341,10 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, sizeof(info), NULL, NULL); if (status == ZX_OK) { uintptr_t new_addr; - status = _zx_vmar_map_old(_zx_vmar_root_self(), addr - info.base, vmo, - 0, size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | - ZX_VM_FLAG_SPECIFIC_OVERWRITE, - &new_addr); + status = _zx_vmar_map( + _zx_vmar_root_self(), + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC_OVERWRITE, + addr - info.base, vmo, 0, size, &new_addr); if (status == ZX_OK) CHECK_EQ(new_addr, addr); } } @@ -398,8 +404,8 @@ bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, if (vmo_size < max_len) max_len = vmo_size; size_t map_size = RoundUpTo(max_len, PAGE_SIZE); uintptr_t addr; - status = _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo, 0, map_size, - ZX_VM_FLAG_PERM_READ, &addr); + status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0, + map_size, &addr); if (status == ZX_OK) { *buff = reinterpret_cast(addr); *buff_size = map_size; @@ -448,6 +454,7 @@ char **StoredArgv; char **StoredEnviron; char **GetArgv() { return StoredArgv; } +char **GetEnviron() { return StoredEnviron; } const char *GetEnv(const char *name) { if (StoredEnviron) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc index 9a2a9f12068..bddc26d2001 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc @@ -25,7 +25,7 @@ struct ioctl_desc { const char *name; }; -const unsigned ioctl_table_max = 1198; +const unsigned ioctl_table_max = 1200; static ioctl_desc ioctl_table[ioctl_table_max]; static unsigned ioctl_table_size = 0; @@ -288,6 +288,8 @@ static void ioctl_table_fill() { _(MLX_GET_CINFO, WRITE, struct_mlx_cinfo_sz); /* Entries from file: dev/ic/nvmeio.h */ _(NVME_PASSTHROUGH_CMD, READWRITE, struct_nvme_pt_command_sz); + /* Entries from file: dev/ic/qemufwcfgio.h */ + _(FWCFGIO_SET_INDEX, READ, sizeof(u16)); /* Entries from file: dev/ir/irdaio.h */ _(IRDA_RESET_PARAMS, NONE, 0); _(IRDA_SET_PARAMS, READ, struct_irda_params_sz); @@ -296,9 +298,6 @@ static void ioctl_table_fill() { _(IRFRAMETTY_GET_DEVICE, WRITE, sizeof(unsigned int)); _(IRFRAMETTY_GET_DONGLE, WRITE, sizeof(unsigned int)); _(IRFRAMETTY_SET_DONGLE, READ, sizeof(unsigned int)); - /* Entries from file: dev/isa/satlinkio.h */ - _(SATIORESET, NONE, 0); - _(SATIOGID, WRITE, struct_satlink_id_sz); /* Entries from file: dev/isa/isvio.h */ _(ISV_CMD, READWRITE, struct_isv_cmd_sz); /* Entries from file: dev/isa/wtreg.h */ @@ -647,6 +646,26 @@ static void ioctl_table_fill() { _(SPKRTUNE, NONE, 0); _(SPKRGETVOL, WRITE, sizeof(unsigned int)); _(SPKRSETVOL, READ, sizeof(unsigned int)); +#if 0 /* WIP */ + /* Entries from file: dev/nvmm/nvmm_ioctl.h */ + _(NVMM_IOC_CAPABILITY, WRITE, struct_nvmm_ioc_capability_sz); + _(NVMM_IOC_MACHINE_CREATE, READWRITE, struct_nvmm_ioc_machine_create_sz); + _(NVMM_IOC_MACHINE_DESTROY, READ, struct_nvmm_ioc_machine_destroy_sz); + _(NVMM_IOC_MACHINE_CONFIGURE, READ, struct_nvmm_ioc_machine_configure_sz); + _(NVMM_IOC_VCPU_CREATE, READ, struct_nvmm_ioc_vcpu_create_sz); + _(NVMM_IOC_VCPU_DESTROY, READ, struct_nvmm_ioc_vcpu_destroy_sz); + _(NVMM_IOC_VCPU_SETSTATE, READ, struct_nvmm_ioc_vcpu_setstate_sz); + _(NVMM_IOC_VCPU_GETSTATE, READ, struct_nvmm_ioc_vcpu_getstate_sz); + _(NVMM_IOC_VCPU_INJECT, READ, struct_nvmm_ioc_vcpu_inject_sz); + _(NVMM_IOC_VCPU_RUN, READWRITE, struct_nvmm_ioc_vcpu_run_sz); + _(NVMM_IOC_GPA_MAP, READ, struct_nvmm_ioc_gpa_map_sz); + _(NVMM_IOC_GPA_UNMAP, READ, struct_nvmm_ioc_gpa_unmap_sz); + _(NVMM_IOC_HVA_MAP, READ, struct_nvmm_ioc_hva_map_sz); + _(NVMM_IOC_HVA_UNMAP, READ, struct_nvmm_ioc_hva_unmap_sz); +#endif + /* Entries from file: fs/autofs/autofs_ioctl.h */ + _(AUTOFSREQUEST, WRITE, struct_autofs_daemon_request_sz); + _(AUTOFSDONE, READ, struct_autofs_daemon_done_sz); /* Entries from file: net/bpf.h */ _(BIOCGBLEN, WRITE, sizeof(unsigned int)); _(BIOCSBLEN, READWRITE, sizeof(unsigned int)); @@ -666,20 +685,12 @@ static void ioctl_table_fill() { _(BIOCSHDRCMPLT, READ, sizeof(unsigned int)); _(BIOCSDLT, READ, sizeof(unsigned int)); _(BIOCGDLTLIST, READWRITE, struct_bpf_dltlist_sz); - _(BIOCGSEESENT, WRITE, sizeof(unsigned int)); - _(BIOCSSEESENT, READ, sizeof(unsigned int)); + _(BIOCGDIRECTION, WRITE, sizeof(unsigned int)); + _(BIOCSDIRECTION, READ, sizeof(unsigned int)); _(BIOCSRTIMEOUT, READ, struct_timeval_sz); _(BIOCGRTIMEOUT, WRITE, struct_timeval_sz); _(BIOCGFEEDBACK, WRITE, sizeof(unsigned int)); _(BIOCSFEEDBACK, READ, sizeof(unsigned int)); - /* Entries from file: net/if_atm.h */ - _(SIOCRAWATM, READWRITE, sizeof(int)); - _(SIOCATMENA, READWRITE, struct_atm_pseudoioctl_sz); - _(SIOCATMDIS, READWRITE, struct_atm_pseudoioctl_sz); - _(SIOCSPVCTX, READWRITE, struct_pvctxreq_sz); - _(SIOCGPVCTX, READWRITE, struct_pvctxreq_sz); - _(SIOCSPVCSIF, READWRITE, struct_ifreq_sz); - _(SIOCGPVCSIF, READWRITE, struct_ifreq_sz); /* Entries from file: net/if_gre.h */ _(GRESADDRS, READ, struct_ifreq_sz); _(GRESADDRD, READ, struct_ifreq_sz); @@ -715,12 +726,12 @@ static void ioctl_table_fill() { /* Entries from file: net/npf.h */ _(IOC_NPF_VERSION, WRITE, sizeof(int)); _(IOC_NPF_SWITCH, READ, sizeof(int)); - _(IOC_NPF_LOAD, READWRITE, struct_plistref_sz); + _(IOC_NPF_LOAD, READWRITE, struct_nvlist_ref_sz); _(IOC_NPF_TABLE, READ, struct_npf_ioctl_table_sz); _(IOC_NPF_STATS, READ, sizeof(uptr)); - _(IOC_NPF_SAVE, WRITE, struct_plistref_sz); - _(IOC_NPF_RULE, READWRITE, struct_plistref_sz); - _(IOC_NPF_CONN_LOOKUP, READWRITE, struct_plistref_sz); + _(IOC_NPF_SAVE, WRITE, struct_nvlist_ref_sz); + _(IOC_NPF_RULE, READWRITE, struct_nvlist_ref_sz); + _(IOC_NPF_CONN_LOOKUP, READWRITE, struct_nvlist_ref_sz); /* Entries from file: net/if_pppoe.h */ _(PPPOESETPARMS, READ, struct_pppoediscparms_sz); _(PPPOEGETPARMS, READWRITE, struct_pppoediscparms_sz); @@ -843,6 +854,9 @@ static void ioctl_table_fill() { _(SIOCGNATS, READWRITE, struct_ipfobj_sz); _(SIOCGNATL, READWRITE, struct_ipfobj_sz); _(SIOCPURGENAT, READWRITE, struct_ipfobj_sz); + /* Entries from file: netinet/sctp_uio.h */ + _(SIOCCONNECTX, READWRITE, struct_sctp_connectx_addrs_sz); + _(SIOCCONNECTXDEL, READWRITE, struct_sctp_connectx_addrs_sz); /* Entries from file: netinet6/in6_var.h */ _(SIOCSIFINFO_FLAGS, READWRITE, struct_in6_ndireq_sz); _(SIOCAADDRCTL_POLICY, READ, struct_in6_addrpolicy_sz); @@ -1002,6 +1016,8 @@ static void ioctl_table_fill() { /* Entries from file: sys/filio.h */ _(FIOCLEX, NONE, 0); _(FIONCLEX, NONE, 0); + _(FIOSEEKDATA, READWRITE, sizeof(uptr)); + _(FIOSEEKHOLE, READWRITE, sizeof(uptr)); _(FIONREAD, WRITE, sizeof(int)); _(FIONBIO, READ, sizeof(int)); _(FIOASYNC, READ, sizeof(int)); @@ -1095,7 +1111,6 @@ static void ioctl_table_fill() { /* Entries from file: sys/power.h */ _(POWER_EVENT_RECVDICT, READWRITE, struct_plistref_sz); _(POWER_IOC_GET_TYPE, WRITE, struct_power_type_sz); - _(POWER_IOC_GET_TYPE_WITH_LOSSAGE, WRITE, sizeof(uptr)); /* Entries from file: sys/radioio.h */ _(RIOCGINFO, WRITE, struct_radio_info_sz); _(RIOCSINFO, READWRITE, struct_radio_info_sz); @@ -1133,6 +1148,7 @@ static void ioctl_table_fill() { _(SIOCATMARK, WRITE, sizeof(int)); _(SIOCSPGRP, READ, sizeof(int)); _(SIOCGPGRP, WRITE, sizeof(int)); + _(SIOCPEELOFF, READWRITE, sizeof(int)); _(SIOCADDRT, READ, struct_ortentry_sz); _(SIOCDELRT, READ, struct_ortentry_sz); _(SIOCSIFADDR, READ, struct_ifreq_sz); @@ -1190,6 +1206,10 @@ static void ioctl_table_fill() { _(SIOCSLINKSTR, READ, struct_ifdrv_sz); _(SIOCGETHERCAP, READWRITE, struct_eccapreq_sz); _(SIOCGIFINDEX, READWRITE, struct_ifreq_sz); + _(SIOCSETHERCAP, READ, struct_eccapreq_sz); + _(SIOCGUMBINFO, READWRITE, struct_ifreq_sz); + _(SIOCSUMBPARAM, READ, struct_ifreq_sz); + _(SIOCGUMBPARAM, READWRITE, struct_ifreq_sz); _(SIOCSETPFSYNC, READ, struct_ifreq_sz); _(SIOCGETPFSYNC, READWRITE, struct_ifreq_sz); /* Entries from file: sys/timepps.h */ diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index f8a405ba6e4..14258d617e6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -105,8 +105,8 @@ // // FIXME: do we have anything like this on Mac? #ifndef SANITIZER_CAN_USE_PREINIT_ARRAY -#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD) && \ - !defined(PIC) +#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD || \ + SANITIZER_FUCHSIA) && !defined(PIC) #define SANITIZER_CAN_USE_PREINIT_ARRAY 1 // Before Solaris 11.4, .preinit_array is fully supported only with GNU ld. // FIXME: Check for those conditions. @@ -172,6 +172,7 @@ typedef int pid_t; #if SANITIZER_FREEBSD || SANITIZER_NETBSD || \ SANITIZER_OPENBSD || SANITIZER_MAC || \ + (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \ (SANITIZER_LINUX && defined(__x86_64__)) typedef u64 OFF_T; #else @@ -196,7 +197,9 @@ typedef u64 tid_t; // This header should NOT include any other headers to avoid portability issues. // Common defs. +#ifndef INLINE #define INLINE inline +#endif #define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE #define SANITIZER_WEAK_DEFAULT_IMPL \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE @@ -275,8 +278,6 @@ typedef thread_return_t (THREAD_CALLING_CONV *thread_callback_t)(void* arg); // NOTE: Functions below must be defined in each run-time. void NORETURN Die(); -// FIXME: No, this shouldn't be in the sanitizer interface. -SANITIZER_INTERFACE_ATTRIBUTE void NORETURN CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); @@ -431,6 +432,7 @@ namespace __scudo { using namespace __sanitizer; } // NOLINT namespace __ubsan { using namespace __sanitizer; } // NOLINT namespace __xray { using namespace __sanitizer; } // NOLINT namespace __interception { using namespace __sanitizer; } // NOLINT +namespace __hwasan { using namespace __sanitizer; } // NOLINT #endif // SANITIZER_DEFS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc index 4b462bfe972..4032cb10461 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc @@ -73,6 +73,18 @@ void *internal_memmove(void *dest, const void *src, uptr n) { } void *internal_memset(void* s, int c, uptr n) { + // Optimize for the most performance-critical case: + if ((reinterpret_cast(s) % 16) == 0 && (n % 16) == 0) { + u64 *p = reinterpret_cast(s); + u64 *e = p + n / 8; + u64 v = c; + v |= v << 8; + v |= v << 16; + v |= v << 32; + for (; p < e; p += 2) + p[0] = p[1] = v; + return s; + } // The next line prevents Clang from making a call to memset() instead of the // loop below. // FIXME: building the runtime with -ffreestanding is a better idea. However diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index 96d6c1eff42..48795674c54 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -31,10 +31,6 @@ #include #endif -#if SANITIZER_NETBSD -#include -#endif - // For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat' // format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To // access stat from asm/stat.h, without conflicting with definition in @@ -55,6 +51,7 @@ #include #include #include +#include #if !SANITIZER_SOLARIS #include #endif @@ -68,6 +65,7 @@ #endif #if SANITIZER_OPENBSD #include +#include #endif #include @@ -116,6 +114,9 @@ struct kernel_timeval { // is broken on some linux distributions. const int FUTEX_WAIT = 0; const int FUTEX_WAKE = 1; +const int FUTEX_PRIVATE_FLAG = 128; +const int FUTEX_WAIT_PRIVATE = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; +const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; #endif // SANITIZER_LINUX // Are we using 32-bit or 64-bit Linux syscalls? @@ -149,7 +150,11 @@ extern void internal_sigreturn(); #if SANITIZER_OPENBSD # define SANITIZER_USE_GETENTROPY 1 #else -# define SANITIZER_USE_GETENTROPY 0 +# if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000 +# define SANITIZER_USE_GETENTROPY 1 +# else +# define SANITIZER_USE_GETENTROPY 0 +# endif #endif // SANITIZER_USE_GETENTROPY namespace __sanitizer { @@ -165,14 +170,11 @@ namespace __sanitizer { #endif // --------------- sanitizer_libc.h -#if !SANITIZER_SOLARIS +#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD #if !SANITIZER_S390 && !SANITIZER_OPENBSD uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, OFF_T offset) { -#if SANITIZER_NETBSD - return internal_syscall64(SYSCALL(mmap), addr, length, prot, flags, fd, - (long)0, offset); -#elif SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd, offset); #else @@ -186,11 +188,11 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, #if !SANITIZER_OPENBSD uptr internal_munmap(void *addr, uptr length) { - return internal_syscall_ptr(SYSCALL(munmap), (uptr)addr, length); + return internal_syscall(SYSCALL(munmap), (uptr)addr, length); } int internal_mprotect(void *addr, uptr length, int prot) { - return internal_syscall_ptr(SYSCALL(mprotect), (uptr)addr, length, prot); + return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot); } #endif @@ -202,7 +204,7 @@ uptr internal_open(const char *filename, int flags) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags); #else - return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags); + return internal_syscall(SYSCALL(open), (uptr)filename, flags); #endif } @@ -211,32 +213,28 @@ uptr internal_open(const char *filename, int flags, u32 mode) { return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags, mode); #else - return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags, mode); + return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode); #endif } uptr internal_read(fd_t fd, void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(read), fd, (uptr)buf, - count)); + HANDLE_EINTR(res, + (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf, count)); return res; } uptr internal_write(fd_t fd, const void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(write), fd, (uptr)buf, - count)); + HANDLE_EINTR(res, + (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf, count)); return res; } uptr internal_ftruncate(fd_t fd, uptr size) { sptr res; -#if SANITIZER_NETBSD - HANDLE_EINTR(res, internal_syscall64(SYSCALL(ftruncate), fd, 0, (s64)size)); -#else HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd, (OFF_T)size)); -#endif return res; } @@ -308,9 +306,8 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { #endif uptr internal_stat(const char *path, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD - return internal_syscall_ptr(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, - 0); +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD + return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); @@ -333,9 +330,7 @@ uptr internal_stat(const char *path, void *buf) { } uptr internal_lstat(const char *path, void *buf) { -#if SANITIZER_NETBSD - return internal_syscall_ptr(SYSCALL(lstat), path, buf); -#elif SANITIZER_FREEBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS @@ -360,16 +355,16 @@ uptr internal_lstat(const char *path, void *buf) { } uptr internal_fstat(fd_t fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \ +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD || \ SANITIZER_LINUX_USES_64BIT_SYSCALLS -#if SANITIZER_MIPS64 && !SANITIZER_NETBSD && !SANITIZER_OPENBSD +#if SANITIZER_MIPS64 && !SANITIZER_OPENBSD // For mips64, fstat syscall fills buffer in the format of kernel_stat struct kernel_stat kbuf; int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; # else - return internal_syscall_ptr(SYSCALL(fstat), fd, (uptr)buf); + return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); # endif #else struct stat64 buf64; @@ -386,6 +381,10 @@ uptr internal_filesize(fd_t fd) { return (uptr)st.st_size; } +uptr internal_dup(int oldfd) { + return internal_syscall(SYSCALL(dup), oldfd); +} + uptr internal_dup2(int oldfd, int newfd) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0); @@ -399,10 +398,10 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, bufsize); #elif SANITIZER_OPENBSD - return internal_syscall_ptr(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, - (uptr)buf, bufsize); + return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, + bufsize); #else - return internal_syscall_ptr(SYSCALL(readlink), path, buf, bufsize); + return internal_syscall(SYSCALL(readlink), path, buf, bufsize); #endif } @@ -410,7 +409,7 @@ uptr internal_unlink(const char *path) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0); #else - return internal_syscall_ptr(SYSCALL(unlink), (uptr)path); + return internal_syscall(SYSCALL(unlink), (uptr)path); #endif } @@ -419,7 +418,7 @@ uptr internal_rename(const char *oldpath, const char *newpath) { return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath); #else - return internal_syscall_ptr(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); + return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); #endif } @@ -428,7 +427,7 @@ uptr internal_sched_yield() { } void internal__exit(int exitcode) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD internal_syscall(SYSCALL(exit), exitcode); #else internal_syscall(SYSCALL(exit_group), exitcode); @@ -438,22 +437,24 @@ void internal__exit(int exitcode) { unsigned int internal_sleep(unsigned int seconds) { struct timespec ts; - ts.tv_sec = 1; + ts.tv_sec = seconds; ts.tv_nsec = 0; - int res = internal_syscall_ptr(SYSCALL(nanosleep), &ts, &ts); + int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts); if (res) return ts.tv_sec; return 0; } uptr internal_execve(const char *filename, char *const argv[], char *const envp[]) { - return internal_syscall_ptr(SYSCALL(execve), (uptr)filename, (uptr)argv, - (uptr)envp); + return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv, + (uptr)envp); } -#endif // !SANITIZER_SOLARIS +#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD // ----------------- sanitizer_common.h bool FileExists(const char *filename) { + if (ShouldMockFailureToOpen(filename)) + return false; struct stat st; #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS if (internal_syscall(SYSCALL(newfstatat), AT_FDCWD, filename, &st, 0)) @@ -465,6 +466,7 @@ bool FileExists(const char *filename) { return S_ISREG(st.st_mode); } +#if !SANITIZER_NETBSD tid_t GetTid() { #if SANITIZER_FREEBSD long Tid; @@ -472,8 +474,6 @@ tid_t GetTid() { return Tid; #elif SANITIZER_OPENBSD return internal_syscall(SYSCALL(getthrid)); -#elif SANITIZER_NETBSD - return _lwp_self(); #elif SANITIZER_SOLARIS return thr_self(); #else @@ -481,28 +481,43 @@ tid_t GetTid() { #endif } -#if !SANITIZER_SOLARIS +int TgKill(pid_t pid, tid_t tid, int sig) { +#if SANITIZER_LINUX + return internal_syscall(SYSCALL(tgkill), pid, tid, sig); +#elif SANITIZER_FREEBSD + return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig); +#elif SANITIZER_OPENBSD + (void)pid; + return internal_syscall(SYSCALL(thrkill), tid, sig, nullptr); +#elif SANITIZER_SOLARIS + (void)pid; + return thr_kill(tid, sig); +#endif +} +#endif + +#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD u64 NanoTime() { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD timeval tv; #else kernel_timeval tv; #endif internal_memset(&tv, 0, sizeof(tv)); - internal_syscall_ptr(SYSCALL(gettimeofday), &tv, 0); + internal_syscall(SYSCALL(gettimeofday), &tv, 0); return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000; } uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { - return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp); + return internal_syscall(SYSCALL(clock_gettime), clk_id, tp); } -#endif // !SANITIZER_SOLARIS +#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD // Like getenv, but reads env directly from /proc (on Linux) or parses the // 'environ' array (on some others) and does not use libc. This function // should be called first inside __asan_init. const char *GetEnv(const char *name) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \ +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \ SANITIZER_SOLARIS if (::environ != 0) { uptr NameLen = internal_strlen(name); @@ -580,8 +595,8 @@ static void GetArgsAndEnv(char ***argv, char ***envp) { // kern.ps_strings sysctl, which returns a pointer to a structure containing // this information. See also . ps_strings *pss; - size_t sz = sizeof(pss); - if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { + uptr sz = sizeof(pss); + if (internal_sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { Printf("sysctl kern.ps_strings failed\n"); Die(); } @@ -614,33 +629,13 @@ char **GetArgv() { return argv; } -void ReExec() { +char **GetEnviron() { char **argv, **envp; - const char *pathname = "/proc/self/exe"; - -#if SANITIZER_NETBSD - static const int name[] = { - CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME, - }; - char path[400]; - size_t len; - - len = sizeof(path); - if (sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) - pathname = path; -#elif SANITIZER_SOLARIS - pathname = getexecname(); - CHECK_NE(pathname, NULL); -#endif - GetArgsAndEnv(&argv, &envp); - uptr rv = internal_execve(pathname, argv, envp); - int rverrno; - CHECK_EQ(internal_iserror(rv, &rverrno), true); - Printf("execve failed, errno %d\n", rverrno); - Die(); + return envp; } -#endif + +#endif // !SANITIZER_OPENBSD #if !SANITIZER_SOLARIS enum MutexState { @@ -664,7 +659,8 @@ void BlockingMutex::Lock() { #elif SANITIZER_NETBSD sched_yield(); /* No userspace futex-like synchronization */ #else - internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT, MtxSleeping, 0, 0, 0); + internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT_PRIVATE, MtxSleeping, + 0, 0, 0); #endif } } @@ -679,7 +675,7 @@ void BlockingMutex::Unlock() { #elif SANITIZER_NETBSD /* No userspace futex-like synchronization */ #else - internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE, 1, 0, 0, 0); + internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0); #endif } } @@ -694,7 +690,9 @@ void BlockingMutex::CheckLocked() { // The actual size of this structure is specified by d_reclen. // Note that getdents64 uses a different structure format. We only provide the // 32-bit syscall here. -#if SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_NETBSD +// Not used +#elif SANITIZER_OPENBSD // struct dirent is different for Linux and us. At this moment, we use only // d_fileno (Linux call this d_ino), d_reclen, and d_name. struct linux_dirent { @@ -721,27 +719,15 @@ struct linux_dirent { }; #endif -#if !SANITIZER_SOLARIS +#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD // Syscall wrappers. uptr internal_ptrace(int request, int pid, void *addr, void *data) { -#if SANITIZER_NETBSD - // XXX We need additional work for ptrace: - // - for request, we use PT_FOO whereas Linux uses PTRACE_FOO - // - data is int for us, but void * for Linux - // - Linux sometimes uses data in the case where we use addr instead - // At this moment, this function is used only within - // "#if SANITIZER_LINUX && defined(__x86_64__)" block in - // sanitizer_stoptheworld_linux_libcdep.cc. - return internal_syscall_ptr(SYSCALL(ptrace), request, pid, (uptr)addr, - (uptr)data); -#else return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr, (uptr)data); -#endif } uptr internal_waitpid(int pid, int *status, int options) { - return internal_syscall_ptr(SYSCALL(wait4), pid, (uptr)status, options, + return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options, 0 /* rusage */); } @@ -759,16 +745,12 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) { #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count); #else - return internal_syscall_ptr(SYSCALL(getdents), fd, (uptr)dirp, count); + return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count); #endif } uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { -#if SANITIZER_NETBSD - return internal_syscall64(SYSCALL(lseek), fd, 0, offset, whence); -#else return internal_syscall(SYSCALL(lseek), fd, offset, whence); -#endif } #if SANITIZER_LINUX @@ -778,7 +760,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { #endif uptr internal_sigaltstack(const void *ss, void *oss) { - return internal_syscall_ptr(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); + return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); } int internal_fork() { @@ -789,6 +771,26 @@ int internal_fork() { #endif } +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen) { +#if SANITIZER_OPENBSD + return sysctl(name, namelen, oldp, (size_t *)oldlenp, (void *)newp, + (size_t)newlen); +#else + return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp, + (size_t *)oldlenp, newp, (size_t)newlen); +#endif +} + +#if SANITIZER_FREEBSD +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + return sysctlbyname(sname, oldp, (size_t *)oldlenp, newp, (size_t)newlen); +} +#endif +#endif + #if SANITIZER_LINUX #define SA_RESTORER 0x04000000 // Doesn't set sa_restorer if the caller did not set it, so use with caution @@ -858,8 +860,8 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact) { uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD - return internal_syscall_ptr(SYSCALL(sigprocmask), how, set, oldset); +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD + return internal_syscall(SYSCALL(sigprocmask), how, set, oldset); #else __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; __sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset; @@ -897,9 +899,20 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { const uptr bit = signum % (sizeof(k_set->sig[0]) * 8); return k_set->sig[idx] & (1 << bit); } -#endif // SANITIZER_LINUX +#elif SANITIZER_FREEBSD +void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { + sigset_t *rset = reinterpret_cast(set); + sigdelset(rset, signum); +} + +bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { + sigset_t *rset = reinterpret_cast(set); + return sigismember(rset, signum); +} +#endif #endif // !SANITIZER_SOLARIS +#if !SANITIZER_NETBSD // ThreadLister implementation. ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) { char task_directory_path[80]; @@ -986,6 +999,7 @@ ThreadLister::~ThreadLister() { if (!internal_iserror(descriptor_)) internal_close(descriptor_); } +#endif #if SANITIZER_WORDSIZE == 32 // Take care of unusable kernel area in top gigabyte. @@ -996,6 +1010,8 @@ static uptr GetKernelAreaSize() { // Firstly check if there are writable segments // mapped to top gigabyte (e.g. stack). MemoryMappingLayout proc_maps(/*cache_enabled*/true); + if (proc_maps.Error()) + return 0; MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0; @@ -1061,18 +1077,25 @@ uptr GetMaxUserVirtualAddress() { return addr; } +#if !SANITIZER_ANDROID uptr GetPageSize() { -// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. -#if SANITIZER_ANDROID - return 4096; -#elif SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) return EXEC_PAGESIZE; #elif SANITIZER_USE_GETAUXVAL return getauxval(AT_PAGESZ); +#elif SANITIZER_FREEBSD || SANITIZER_NETBSD +// Use sysctl as sysconf can trigger interceptors internally. + int pz = 0; + uptr pzl = sizeof(pz); + int mib[2] = {CTL_HW, HW_PAGESIZE}; + int rv = internal_sysctl(mib, 2, &pz, &pzl, nullptr, 0); + CHECK_EQ(rv, 0); + return (uptr)pz; #else return sysconf(_SC_PAGESIZE); // EXEC_PAGESIZE may not be trustworthy. #endif } +#endif // !SANITIZER_ANDROID #if !SANITIZER_OPENBSD uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { @@ -1088,8 +1111,9 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME}; #endif const char *default_module_name = "kern.proc.pathname"; - size_t Size = buf_len; - bool IsErr = (sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0); + uptr Size = buf_len; + bool IsErr = + (internal_sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0); int readlink_error = IsErr ? errno : 0; uptr module_name_len = Size; #else @@ -1239,7 +1263,7 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, "d"(parent_tidptr), "r"(r8), "r"(r10) - : "rsp", "memory", "r11", "rcx"); + : "memory", "r11", "rcx"); return res; } #elif defined(__mips__) @@ -1631,6 +1655,16 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size, static atomic_uint32_t android_api_level; +static AndroidApiLevel AndroidDetectApiLevelStatic() { +#if __ANDROID_API__ <= 19 + return ANDROID_KITKAT; +#elif __ANDROID_API__ <= 22 + return ANDROID_LOLLIPOP_MR1; +#else + return ANDROID_POST_LOLLIPOP; +#endif +} + static AndroidApiLevel AndroidDetectApiLevel() { if (!&dl_iterate_phdr) return ANDROID_KITKAT; // K or lower @@ -1643,11 +1677,14 @@ static AndroidApiLevel AndroidDetectApiLevel() { // interesting to detect. } +extern "C" __attribute__((weak)) void* _DYNAMIC; + AndroidApiLevel AndroidGetApiLevel() { AndroidApiLevel level = (AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed); if (level) return level; - level = AndroidDetectApiLevel(); + level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic() + : AndroidDetectApiLevel(); atomic_store(&android_api_level, level, memory_order_relaxed); return level; } @@ -1905,14 +1942,14 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #elif defined(__sparc__) ucontext_t *ucontext = (ucontext_t*)context; uptr *stk_ptr; -# if defined (__sparcv9) +# if defined(__sparcv9) || defined (__arch64__) # ifndef MC_PC # define MC_PC REG_PC # endif # ifndef MC_O6 # define MC_O6 REG_O6 # endif -# ifdef SANITIZER_SOLARIS +# if SANITIZER_SOLARIS # define mc_gregs gregs # endif *pc = ucontext->uc_mcontext.mc_gregs[MC_PC]; @@ -1946,6 +1983,10 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } +void InitializePlatformEarly() { + // Do nothing. +} + void MaybeReexec() { // No need to re-exec on Linux. } @@ -1954,13 +1995,13 @@ void CheckASLR() { #if SANITIZER_NETBSD int mib[3]; int paxflags; - size_t len = sizeof(paxflags); + uptr len = sizeof(paxflags); mib[0] = CTL_PROC; mib[1] = internal_getpid(); mib[2] = PROC_PID_PAXFLAGS; - if (UNLIKELY(sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) { + if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) { Printf("sysctl failed\n"); Die(); } @@ -1969,6 +2010,41 @@ void CheckASLR() { Printf("This sanitizer is not compatible with enabled ASLR\n"); Die(); } +#elif SANITIZER_PPC64V2 + // Disable ASLR for Linux PPC64LE. + int old_personality = personality(0xffffffff); + if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { + VReport(1, "WARNING: Program is being run with address space layout " + "randomization (ASLR) enabled which prevents the thread and " + "memory sanitizers from working on powerpc64le.\n" + "ASLR will be disabled and the program re-executed.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + ReExec(); + } +#else + // Do nothing +#endif +} + +void CheckMPROTECT() { +#if SANITIZER_NETBSD + int mib[3]; + int paxflags; + uptr len = sizeof(paxflags); + + mib[0] = CTL_PROC; + mib[1] = internal_getpid(); + mib[2] = PROC_PID_PAXFLAGS; + + if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) { + Printf("sysctl failed\n"); + Die(); + } + + if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_MPROTECT)) { + Printf("This sanitizer is not compatible with enabled MPROTECT\n"); + Die(); + } #else // Do nothing #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 975d6541d88..c309e33f81b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -18,6 +18,7 @@ SANITIZER_OPENBSD || SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_freebsd.h" #include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_openbsd.h" #include "sanitizer_platform_limits_posix.h" @@ -69,6 +70,8 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum); uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr); #endif +#elif SANITIZER_FREEBSD +void internal_sigdelset(__sanitizer_sigset_t *set, int signum); #endif // SANITIZER_LINUX // This class reads thread IDs from /proc//task using only syscalls. @@ -131,13 +134,13 @@ void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)); #error "Unsupported architecture." #endif -// The Android Bionic team has allocated a TLS slot for TSan starting with N, -// given that Android currently doesn't support ELF TLS. It is used to store -// Sanitizers thread specific data. -static const int TLS_SLOT_TSAN = 8; +// The Android Bionic team has allocated a TLS slot for sanitizers starting +// with Q, given that Android currently doesn't support ELF TLS. It is used to +// store sanitizer thread specific data. +static const int TLS_SLOT_SANITIZER = 6; ALWAYS_INLINE uptr *get_android_tls_ptr() { - return reinterpret_cast(&__get_tls()[TLS_SLOT_TSAN]); + return reinterpret_cast(&__get_tls()[TLS_SLOT_SANITIZER]); } #endif // SANITIZER_ANDROID diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 4962ff8322d..6ce47ec620d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -23,6 +23,7 @@ #include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_freebsd.h" +#include "sanitizer_getauxval.h" #include "sanitizer_linux.h" #include "sanitizer_placement_new.h" #include "sanitizer_procmaps.h" @@ -99,6 +100,10 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, // Find the mapping that contains a stack variable. MemoryMappingLayout proc_maps(/*cache_enabled*/true); + if (proc_maps.Error()) { + *stack_top = *stack_bottom = 0; + return; + } MemoryMappedSegment segment; uptr prev_end = 0; while (proc_maps.Next(&segment)) { @@ -652,10 +657,10 @@ u32 GetNumberOfCPUs() { #if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD u32 ncpu; int req[2]; - size_t len = sizeof(ncpu); + uptr len = sizeof(ncpu); req[0] = CTL_HW; req[1] = HW_NCPU; - CHECK_EQ(sysctl(req, 2, &ncpu, &len, NULL, 0), 0); + CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0); return ncpu; #elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__) // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't @@ -806,6 +811,40 @@ u64 MonotonicNanoTime() { } #endif // SANITIZER_LINUX && !SANITIZER_GO +#if !SANITIZER_OPENBSD +void ReExec() { + const char *pathname = "/proc/self/exe"; + +#if SANITIZER_NETBSD + static const int name[] = { + CTL_KERN, + KERN_PROC_ARGS, + -1, + KERN_PROC_PATHNAME, + }; + char path[400]; + uptr len; + + len = sizeof(path); + if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) + pathname = path; +#elif SANITIZER_SOLARIS + pathname = getexecname(); + CHECK_NE(pathname, NULL); +#elif SANITIZER_USE_GETAUXVAL + // Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that + // rely on that will fail to load shared libraries. Query AT_EXECFN instead. + pathname = reinterpret_cast(getauxval(AT_EXECFN)); +#endif + + uptr rv = internal_execve(pathname, GetArgv(), GetEnviron()); + int rverrno; + CHECK_EQ(internal_iserror(rv, &rverrno), true); + Printf("execve failed, errno %d\n", rverrno); + Die(); +} +#endif // !SANITIZER_OPENBSD + } // namespace __sanitizer #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h new file mode 100644 index 00000000000..ec1847abc53 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h @@ -0,0 +1,77 @@ +//===-- sanitizer_local_address_space_view.h --------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// `LocalAddressSpaceView` provides the local (i.e. target and current address +// space are the same) implementation of the `AddressSpaveView` interface which +// provides a simple interface to load memory from another process (i.e. +// out-of-process) +// +// The `AddressSpaceView` interface requires that the type can be used as a +// template parameter to objects that wish to be able to operate in an +// out-of-process manner. In normal usage, objects are in-process and are thus +// instantiated with the `LocalAddressSpaceView` type. This type is used to +// load any pointers in instance methods. This implementation is effectively +// a no-op. When an object is to be used in an out-of-process manner it is +// instansiated with the `RemoteAddressSpaceView` type. +// +// By making `AddressSpaceView` a template parameter of an object, it can +// change its implementation at compile time which has no run time overhead. +// This also allows unifying in-process and out-of-process code which avoids +// code duplication. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_LOCAL_ADDRES_SPACE_VIEW_H +#define SANITIZER_LOCAL_ADDRES_SPACE_VIEW_H + +namespace __sanitizer { +struct LocalAddressSpaceView { + // Load memory `sizeof(T) * num_elements` bytes of memory from the target + // process (always local for this implementation) starting at address + // `target_address`. The local copy of this memory is returned as a pointer. + // The caller should not write to this memory. The behaviour when doing so is + // undefined. Callers should use `LoadWritable()` to get access to memory + // that is writable. + // + // The lifetime of loaded memory is implementation defined. + template + static const T *Load(const T *target_address, uptr num_elements = 1) { + // The target address space is the local address space so + // nothing needs to be copied. Just return the pointer. + return target_address; + } + + // Load memory `sizeof(T) * num_elements` bytes of memory from the target + // process (always local for this implementation) starting at address + // `target_address`. The local copy of this memory is returned as a pointer. + // The memory returned may be written to. + // + // Writes made to the returned memory will be visible in the memory returned + // by subsequent `Load()` or `LoadWritable()` calls provided the + // `target_address` parameter is the same. It is not guaranteed that the + // memory returned by previous calls to `Load()` will contain any performed + // writes. If two or more overlapping regions of memory are loaded via + // separate calls to `LoadWritable()`, it is implementation defined whether + // writes made to the region returned by one call are visible in the regions + // returned by other calls. + // + // Given the above it is recommended to load the largest possible object + // that requires modification (e.g. a class) rather than individual fields + // from a class to avoid issues with overlapping writable regions. + // + // The lifetime of loaded memory is implementation defined. + template + static T *LoadWritable(T *target_address, uptr num_elements = 1) { + // The target address space is the local address space so + // nothing needs to be copied. Just return the pointer. + return target_address; + } +}; +} // namespace __sanitizer + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc index 180d7c199ae..8954a7a8807 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -108,9 +108,20 @@ extern "C" int __munmap(void *, size_t) SANITIZER_WEAK_ATTRIBUTE; #define VM_MEMORY_SANITIZER 99 #endif +// XNU on Darwin provides a mmap flag that optimizes allocation/deallocation of +// giant memory regions (i.e. shadow memory regions). +#define kXnuFastMmapFd 0x4 +static size_t kXnuFastMmapThreshold = 2 << 30; // 2 GB +static bool use_xnu_fast_mmap = false; + uptr internal_mmap(void *addr, size_t length, int prot, int flags, int fd, u64 offset) { - if (fd == -1) fd = VM_MAKE_TAG(VM_MEMORY_SANITIZER); + if (fd == -1) { + fd = VM_MAKE_TAG(VM_MEMORY_SANITIZER); + if (length >= kXnuFastMmapThreshold) { + if (use_xnu_fast_mmap) fd |= kXnuFastMmapFd; + } + } if (&__mmap) return (uptr)__mmap(addr, length, prot, flags, fd, offset); return (uptr)mmap(addr, length, prot, flags, fd, offset); } @@ -163,6 +174,10 @@ uptr internal_filesize(fd_t fd) { return (uptr)st.st_size; } +uptr internal_dup(int oldfd) { + return dup(oldfd); +} + uptr internal_dup2(int oldfd, int newfd) { return dup2(oldfd, newfd); } @@ -213,6 +228,18 @@ int internal_fork() { return fork(); } +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen) { + return sysctl(const_cast(name), namelen, oldp, (size_t *)oldlenp, + const_cast(newp), (size_t)newlen); +} + +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + return sysctlbyname(sname, oldp, (size_t *)oldlenp, const_cast(newp), + (size_t)newlen); +} + int internal_forkpty(int *amaster) { int master, slave; if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1; @@ -255,6 +282,8 @@ uptr internal_waitpid(int pid, int *status, int options) { // ----------------- sanitizer_common.h bool FileExists(const char *filename) { + if (ShouldMockFailureToOpen(filename)) + return false; struct stat st; if (stat(filename, &st)) return false; @@ -350,6 +379,10 @@ void CheckASLR() { // Do nothing } +void CheckMPROTECT() { + // Do nothing +} + uptr GetPageSize() { return sysconf(_SC_PAGESIZE); } @@ -499,26 +532,38 @@ MacosVersion GetMacosVersionInternal() { uptr len = 0, maxlen = sizeof(version) / sizeof(version[0]); for (uptr i = 0; i < maxlen; i++) version[i] = '\0'; // Get the version length. - CHECK_NE(sysctl(mib, 2, 0, &len, 0, 0), -1); + CHECK_NE(internal_sysctl(mib, 2, 0, &len, 0, 0), -1); CHECK_LT(len, maxlen); - CHECK_NE(sysctl(mib, 2, version, &len, 0, 0), -1); - switch (version[0]) { - case '9': return MACOS_VERSION_LEOPARD; - case '1': { - switch (version[1]) { - case '0': return MACOS_VERSION_SNOW_LEOPARD; - case '1': return MACOS_VERSION_LION; - case '2': return MACOS_VERSION_MOUNTAIN_LION; - case '3': return MACOS_VERSION_MAVERICKS; - case '4': return MACOS_VERSION_YOSEMITE; - default: - if (IsDigit(version[1])) - return MACOS_VERSION_UNKNOWN_NEWER; - else - return MACOS_VERSION_UNKNOWN; - } - } - default: return MACOS_VERSION_UNKNOWN; + CHECK_NE(internal_sysctl(mib, 2, version, &len, 0, 0), -1); + + // Expect .(.) + CHECK_GE(len, 3); + const char *p = version; + int major = internal_simple_strtoll(p, &p, /*base=*/10); + if (*p != '.') return MACOS_VERSION_UNKNOWN; + p += 1; + int minor = internal_simple_strtoll(p, &p, /*base=*/10); + if (*p != '.') return MACOS_VERSION_UNKNOWN; + + switch (major) { + case 9: return MACOS_VERSION_LEOPARD; + case 10: return MACOS_VERSION_SNOW_LEOPARD; + case 11: return MACOS_VERSION_LION; + case 12: return MACOS_VERSION_MOUNTAIN_LION; + case 13: return MACOS_VERSION_MAVERICKS; + case 14: return MACOS_VERSION_YOSEMITE; + case 15: return MACOS_VERSION_EL_CAPITAN; + case 16: return MACOS_VERSION_SIERRA; + case 17: + // Not a typo, 17.5 Darwin Kernel Version maps to High Sierra 10.13.4. + if (minor >= 5) + return MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; + return MACOS_VERSION_HIGH_SIERRA; + case 18: + return MACOS_VERSION_MOJAVE; + default: + if (major < 9) return MACOS_VERSION_UNKNOWN; + return MACOS_VERSION_UNKNOWN_NEWER; } } @@ -661,6 +706,16 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } +void InitializePlatformEarly() { + // Only use xnu_fast_mmap when on x86_64 and the OS supports it. + use_xnu_fast_mmap = +#if defined(__x86_64__) + GetMacosVersion() >= MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; +#else + false; +#endif +} + #if !SANITIZER_GO static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; @@ -890,10 +945,10 @@ struct __sanitizer_task_vm_info { (sizeof(__sanitizer_task_vm_info) / sizeof(natural_t))) uptr GetTaskInfoMaxAddress() { - __sanitizer_task_vm_info vm_info = {}; + __sanitizer_task_vm_info vm_info = {} /* zero initialize */; mach_msg_type_number_t count = __SANITIZER_TASK_VM_INFO_COUNT; int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count); - if (err == 0) { + if (err == 0 && vm_info.max_address != 0) { return vm_info.max_address - 1; } else { // xnu cannot provide vm address limit @@ -1060,14 +1115,16 @@ void CheckNoDeepBind(const char *filename, int flag) { // Do nothing. } -// FIXME: implement on this platform. bool GetRandom(void *buffer, uptr length, bool blocking) { - UNIMPLEMENTED(); + if (!buffer || !length || length > 256) + return false; + // arc4random never fails. + arc4random_buf(buffer, length); + return true; } -// FIXME: implement on this platform. u32 GetNumberOfCPUs() { - UNIMPLEMENTED(); + return (u32)sysconf(_SC_NPROCESSORS_ONLN); } } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index e022a2c0363..52825f8cf81 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -40,6 +40,11 @@ enum MacosVersion { MACOS_VERSION_MOUNTAIN_LION, MACOS_VERSION_MAVERICKS, MACOS_VERSION_YOSEMITE, + MACOS_VERSION_EL_CAPITAN, + MACOS_VERSION_SIERRA, + MACOS_VERSION_HIGH_SIERRA, + MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4, + MACOS_VERSION_MOJAVE, MACOS_VERSION_UNKNOWN_NEWER }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc index e69d6f94b0e..44c914cea9f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -30,9 +30,27 @@ // https://github.com/gperftools/gperftools. namespace __sanitizer { + extern malloc_zone_t sanitizer_zone; + +struct sanitizer_malloc_introspection_t : public malloc_introspection_t { + // IMPORTANT: Do not change the order, alignment, or types of these fields to + // maintain binary compatibility. You should only add fields to this struct. + + // Used to track changes to the allocator that will affect + // zone enumeration. + u64 allocator_enumeration_version; +}; + +u64 GetMallocZoneAllocatorEnumerationVersion() { + // This represents the current allocator ABI version. + // This field should be incremented every time the Allocator + // ABI changes in a way that breaks allocator enumeration. + return 0; } +} // namespace __sanitizer + INTERCEPTOR(malloc_zone_t *, malloc_create_zone, vm_size_t start_size, unsigned zone_flags) { COMMON_MALLOC_ENTER(); @@ -247,6 +265,13 @@ void *__sanitizer_mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { return p; } +// This public API exists purely for testing purposes. +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +malloc_zone_t* __sanitizer_mz_default_zone() { + return &sanitizer_zone; +} + // This function is currently unused, and we build with -Werror. #if 0 void __sanitizer_mz_free_definite_size( @@ -302,8 +327,8 @@ boolean_t mi_zone_locked(malloc_zone_t *zone) { namespace COMMON_MALLOC_NAMESPACE { -void ReplaceSystemMalloc() { - static malloc_introspection_t sanitizer_zone_introspection; +void InitMallocZoneFields() { + static sanitizer_malloc_introspection_t sanitizer_zone_introspection; // Ok to use internal_memset, these places are not performance-critical. internal_memset(&sanitizer_zone_introspection, 0, sizeof(sanitizer_zone_introspection)); @@ -318,6 +343,10 @@ void ReplaceSystemMalloc() { sanitizer_zone_introspection.statistics = &mi_statistics; sanitizer_zone_introspection.zone_locked = &mi_zone_locked; + // Set current allocator enumeration version. + sanitizer_zone_introspection.allocator_enumeration_version = + GetMallocZoneAllocatorEnumerationVersion(); + internal_memset(&sanitizer_zone, 0, sizeof(malloc_zone_t)); // Use version 6 for OSX >= 10.6. @@ -335,6 +364,10 @@ void ReplaceSystemMalloc() { sanitizer_zone.free_definite_size = 0; sanitizer_zone.memalign = &__sanitizer_mz_memalign; sanitizer_zone.introspect = &sanitizer_zone_introspection; +} + +void ReplaceSystemMalloc() { + InitMallocZoneFields(); // Register the zone. malloc_zone_register(&sanitizer_zone); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cc new file mode 100644 index 00000000000..80d0855efab --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cc @@ -0,0 +1,335 @@ +//===-- sanitizer_netbsd.cc -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between Sanitizer run-time libraries and implements +// NetBSD-specific functions from sanitizer_libc.h. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_NETBSD + +#include "sanitizer_common.h" +#include "sanitizer_flags.h" +#include "sanitizer_getauxval.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_linux.h" +#include "sanitizer_mutex.h" +#include "sanitizer_placement_new.h" +#include "sanitizer_procmaps.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" void *__mmap(void *, size_t, int, int, int, int, + off_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int __sysctl(const int *, unsigned int, void *, size_t *, + const void *, size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_close(int) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_open(const char *, int, ...) SANITIZER_WEAK_ATTRIBUTE; +extern "C" ssize_t _sys_read(int, void *, size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" ssize_t _sys_write(int, const void *, + size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int __ftruncate(int, int, off_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" ssize_t _sys_readlink(const char *, char *, + size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_sched_yield() SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys___nanosleep50(const void *, + void *) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_execve(const char *, char *const[], + char *const[]) SANITIZER_WEAK_ATTRIBUTE; +extern "C" off_t __lseek(int, int, off_t, int) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int __fork() SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys___sigprocmask14(int, const void *, + void *) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys___wait450(int wpid, int *, int, + void *) SANITIZER_WEAK_ATTRIBUTE; + +namespace __sanitizer { + +static void *GetRealLibcAddress(const char *symbol) { + void *real = dlsym(RTLD_NEXT, symbol); + if (!real) + real = dlsym(RTLD_DEFAULT, symbol); + if (!real) { + Printf("GetRealLibcAddress failed for symbol=%s", symbol); + Die(); + } + return real; +} + +#define _REAL(func, ...) real##_##func(__VA_ARGS__) +#define DEFINE__REAL(ret_type, func, ...) \ + static ret_type (*real_##func)(__VA_ARGS__) = NULL; \ + if (!real_##func) { \ + real_##func = (ret_type(*)(__VA_ARGS__))GetRealLibcAddress(#func); \ + } \ + CHECK(real_##func); + +// --------------- sanitizer_libc.h +uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + CHECK(&__mmap); + return (uptr)__mmap(addr, length, prot, flags, fd, 0, offset); +} + +uptr internal_munmap(void *addr, uptr length) { + DEFINE__REAL(int, munmap, void *a, uptr b); + return _REAL(munmap, addr, length); +} + +int internal_mprotect(void *addr, uptr length, int prot) { + DEFINE__REAL(int, mprotect, void *a, uptr b, int c); + return _REAL(mprotect, addr, length, prot); +} + +uptr internal_close(fd_t fd) { + CHECK(&_sys_close); + return _sys_close(fd); +} + +uptr internal_open(const char *filename, int flags) { + CHECK(&_sys_open); + return _sys_open(filename, flags); +} + +uptr internal_open(const char *filename, int flags, u32 mode) { + CHECK(&_sys_open); + return _sys_open(filename, flags, mode); +} + +uptr internal_read(fd_t fd, void *buf, uptr count) { + sptr res; + CHECK(&_sys_read); + HANDLE_EINTR(res, (sptr)_sys_read(fd, buf, (size_t)count)); + return res; +} + +uptr internal_write(fd_t fd, const void *buf, uptr count) { + sptr res; + CHECK(&_sys_write); + HANDLE_EINTR(res, (sptr)_sys_write(fd, buf, count)); + return res; +} + +uptr internal_ftruncate(fd_t fd, uptr size) { + sptr res; + CHECK(&__ftruncate); + HANDLE_EINTR(res, __ftruncate(fd, 0, (s64)size)); + return res; +} + +uptr internal_stat(const char *path, void *buf) { + DEFINE__REAL(int, __stat50, const char *a, void *b); + return _REAL(__stat50, path, buf); +} + +uptr internal_lstat(const char *path, void *buf) { + DEFINE__REAL(int, __lstat50, const char *a, void *b); + return _REAL(__lstat50, path, buf); +} + +uptr internal_fstat(fd_t fd, void *buf) { + DEFINE__REAL(int, __fstat50, int a, void *b); + return _REAL(__fstat50, fd, buf); +} + +uptr internal_filesize(fd_t fd) { + struct stat st; + if (internal_fstat(fd, &st)) + return -1; + return (uptr)st.st_size; +} + +uptr internal_dup(int oldfd) { + DEFINE__REAL(int, dup, int a); + return _REAL(dup, oldfd); +} + +uptr internal_dup2(int oldfd, int newfd) { + DEFINE__REAL(int, dup2, int a, int b); + return _REAL(dup2, oldfd, newfd); +} + +uptr internal_readlink(const char *path, char *buf, uptr bufsize) { + CHECK(&_sys_readlink); + return (uptr)_sys_readlink(path, buf, bufsize); +} + +uptr internal_unlink(const char *path) { + DEFINE__REAL(int, unlink, const char *a); + return _REAL(unlink, path); +} + +uptr internal_rename(const char *oldpath, const char *newpath) { + DEFINE__REAL(int, rename, const char *a, const char *b); + return _REAL(rename, oldpath, newpath); +} + +uptr internal_sched_yield() { + CHECK(&_sys_sched_yield); + return _sys_sched_yield(); +} + +void internal__exit(int exitcode) { + DEFINE__REAL(void, _exit, int a); + _REAL(_exit, exitcode); + Die(); // Unreachable. +} + +unsigned int internal_sleep(unsigned int seconds) { + struct timespec ts; + ts.tv_sec = seconds; + ts.tv_nsec = 0; + CHECK(&_sys___nanosleep50); + int res = _sys___nanosleep50(&ts, &ts); + if (res) + return ts.tv_sec; + return 0; +} + +uptr internal_execve(const char *filename, char *const argv[], + char *const envp[]) { + CHECK(&_sys_execve); + return _sys_execve(filename, argv, envp); +} + +tid_t GetTid() { + DEFINE__REAL(int, _lwp_self); + return _REAL(_lwp_self); +} + +int TgKill(pid_t pid, tid_t tid, int sig) { + DEFINE__REAL(int, _lwp_kill, int a, int b); + (void)pid; + return _REAL(_lwp_kill, tid, sig); +} + +u64 NanoTime() { + timeval tv; + DEFINE__REAL(int, __gettimeofday50, void *a, void *b); + internal_memset(&tv, 0, sizeof(tv)); + _REAL(__gettimeofday50, &tv, 0); + return (u64)tv.tv_sec * 1000 * 1000 * 1000 + tv.tv_usec * 1000; +} + +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { + DEFINE__REAL(int, __clock_gettime50, __sanitizer_clockid_t a, void *b); + return _REAL(__clock_gettime50, clk_id, tp); +} + +uptr internal_ptrace(int request, int pid, void *addr, void *data) { + Printf("internal_ptrace not implemented for NetBSD"); + Die(); + return 0; +} + +uptr internal_waitpid(int pid, int *status, int options) { + CHECK(&_sys___wait450); + return _sys___wait450(pid, status, options, 0 /* rusage */); +} + +uptr internal_getpid() { + DEFINE__REAL(int, getpid); + return _REAL(getpid); +} + +uptr internal_getppid() { + DEFINE__REAL(int, getppid); + return _REAL(getppid); +} + +uptr internal_getdents(fd_t fd, void *dirp, unsigned int count) { + DEFINE__REAL(int, __getdents30, int a, void *b, size_t c); + return _REAL(__getdents30, fd, dirp, count); +} + +uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { + CHECK(&__lseek); + return __lseek(fd, 0, offset, whence); +} + +uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { + Printf("internal_prctl not implemented for NetBSD"); + Die(); + return 0; +} + +uptr internal_sigaltstack(const void *ss, void *oss) { + DEFINE__REAL(int, __sigaltstack14, const void *a, void *b); + return _REAL(__sigaltstack14, ss, oss); +} + +int internal_fork() { + CHECK(&__fork); + return __fork(); +} + +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen) { + CHECK(&__sysctl); + return __sysctl(name, namelen, oldp, (size_t *)oldlenp, newp, (size_t)newlen); +} + +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + DEFINE__REAL(int, sysctlbyname, const char *a, void *b, size_t *c, + const void *d, size_t e); + return _REAL(sysctlbyname, sname, oldp, (size_t *)oldlenp, newp, + (size_t)newlen); +} + +uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + CHECK(&_sys___sigprocmask14); + return _sys___sigprocmask14(how, set, oldset); +} + +void internal_sigfillset(__sanitizer_sigset_t *set) { + DEFINE__REAL(int, __sigfillset14, const void *a); + (void)_REAL(__sigfillset14, set); +} + +void internal_sigemptyset(__sanitizer_sigset_t *set) { + DEFINE__REAL(int, __sigemptyset14, const void *a); + (void)_REAL(__sigemptyset14, set); +} + +uptr intrnal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + Printf("internal_clone not implemented for NetBSD"); + Die(); + return 0; +} + +} // namespace __sanitizer + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc index 2aea7cb1487..f0d071e51f0 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc @@ -51,12 +51,19 @@ int internal_mprotect(void *addr, uptr length, int prot) { return mprotect(addr, length, prot); } +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + Printf("internal_sysctlbyname not implemented for OpenBSD"); + Die(); + return 0; +} + uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { // On OpenBSD we cannot get the full path struct kinfo_proc kp; - size_t kl; + uptr kl; const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()}; - if (sysctl(Mib, ARRAY_SIZE(Mib), &kp, &kl, NULL, 0) != -1) + if (internal_sysctl(Mib, ARRAY_SIZE(Mib), &kp, &kl, NULL, 0) != -1) return internal_snprintf(buf, (KI_MAXCOMLEN < buf_len ? KI_MAXCOMLEN : buf_len), "%s", kp.p_comm); @@ -64,23 +71,23 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { } static void GetArgsAndEnv(char ***argv, char ***envp) { - size_t nargv; - size_t nenv; + uptr nargv; + uptr nenv; int argvmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ARGV}; int envmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ENV}; - if (sysctl(argvmib, 4, NULL, &nargv, NULL, 0) == -1) { + if (internal_sysctl(argvmib, 4, NULL, &nargv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_NARGV failed\n"); Die(); } - if (sysctl(envmib, 4, NULL, &nenv, NULL, 0) == -1) { + if (internal_sysctl(envmib, 4, NULL, &nenv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_NENV failed\n"); Die(); } - if (sysctl(argvmib, 4, &argv, &nargv, NULL, 0) == -1) { + if (internal_sysctl(argvmib, 4, &argv, &nargv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_ARGV failed\n"); Die(); } - if (sysctl(envmib, 4, &envp, &nenv, NULL, 0) == -1) { + if (internal_sysctl(envmib, 4, &envp, &nenv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_ENV failed\n"); Die(); } @@ -92,6 +99,12 @@ char **GetArgv() { return argv; } +char **GetEnviron() { + char **argv, **envp; + GetArgsAndEnv(&argv, &envp); + return envp; +} + void ReExec() { UNIMPLEMENTED(); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index d81e25580d9..82bb1af7746 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -235,7 +235,12 @@ #if defined(__mips__) # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) #elif defined(__aarch64__) -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +# if SANITIZER_MAC +// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36) +# else +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +# endif #else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) #endif @@ -270,12 +275,6 @@ # define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) #endif -// Assume obsolete RPC headers are available by default -#if !defined(HAVE_RPC_XDR_H) && !defined(HAVE_TIRPC_RPC_XDR_H) -# define HAVE_RPC_XDR_H (SANITIZER_LINUX && !SANITIZER_ANDROID) -# define HAVE_TIRPC_RPC_XDR_H 0 -#endif - /// \macro MSC_PREREQ /// \brief Is the compiler MSVC of at least the specified version? /// The common \param version values to check for are: @@ -337,4 +336,13 @@ #define SANITIZER_SYMBOLIZER_MARKUP 0 #endif +// Enable ability to support sanitizer initialization that is +// compatible with the sanitizer library being loaded via +// `dlopen()`. +#if SANITIZER_MAC +#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1 +#else +#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0 +#endif + #endif // SANITIZER_PLATFORM_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index f95539a73c6..4d146651b78 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -33,8 +33,9 @@ #endif #if SI_POSIX +# include "sanitizer_platform_limits_freebsd.h" # include "sanitizer_platform_limits_netbsd.h" -#include "sanitizer_platform_limits_openbsd.h" +# include "sanitizer_platform_limits_openbsd.h" # include "sanitizer_platform_limits_posix.h" # include "sanitizer_platform_limits_solaris.h" #endif @@ -207,9 +208,13 @@ #define SANITIZER_INTERCEPT_GETPWENT \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \ SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETGRENT_R \ + (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_GETPWENT_R \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETPWENT_R \ + (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_SETPWENT \ (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_CLOCK_GETTIME \ @@ -356,7 +361,9 @@ #define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX #define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS #define SANITIZER_INTERCEPT_REMQUO SI_POSIX +#define SANITIZER_INTERCEPT_REMQUOL (SI_POSIX && !SI_NETBSD) #define SANITIZER_INTERCEPT_LGAMMA SI_POSIX +#define SANITIZER_INTERCEPT_LGAMMAL (SI_POSIX && !SI_NETBSD) #define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS) #define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID @@ -380,7 +387,7 @@ #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP \ - (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_TLS_GET_ADDR \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) @@ -510,5 +517,43 @@ #define SANITIZER_INTERCEPT_TTYENT SI_NETBSD #define SANITIZER_INTERCEPT_PROTOENT SI_NETBSD #define SANITIZER_INTERCEPT_NETENT SI_NETBSD +#define SANITIZER_INTERCEPT_SETVBUF (SI_NETBSD || SI_FREEBSD || \ + SI_LINUX || SI_MAC) +#define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC) +#define SANITIZER_INTERCEPT_MI_VECTOR_HASH SI_NETBSD +#define SANITIZER_INTERCEPT_GETVFSSTAT SI_NETBSD +#define SANITIZER_INTERCEPT_REGEX (SI_NETBSD || SI_FREEBSD || SI_LINUX) +#define SANITIZER_INTERCEPT_REGEXSUB SI_NETBSD +#define SANITIZER_INTERCEPT_FTS (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_SYSCTL (SI_NETBSD || SI_FREEBSD || SI_MAC) +#define SANITIZER_INTERCEPT_ASYSCTL SI_NETBSD +#define SANITIZER_INTERCEPT_SYSCTLGETMIBINFO SI_NETBSD +#define SANITIZER_INTERCEPT_NL_LANGINFO (SI_NETBSD || SI_FREEBSD || SI_MAC) +#define SANITIZER_INTERCEPT_MODCTL SI_NETBSD +#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD +#define SANITIZER_INTERCEPT_STRTONUM (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_FPARSELN SI_NETBSD +#define SANITIZER_INTERCEPT_STATVFS1 SI_NETBSD +#define SANITIZER_INTERCEPT_STRTOI SI_NETBSD +#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD +#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD +#define SANITIZER_INTERCEPT_MD4 SI_NETBSD +#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD +#define SANITIZER_INTERCEPT_MD5 SI_NETBSD +#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_MD2 SI_NETBSD +#define SANITIZER_INTERCEPT_SHA2 SI_NETBSD +#define SANITIZER_INTERCEPT_CDB SI_NETBSD +#define SANITIZER_INTERCEPT_VIS (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_POPEN SI_POSIX +#define SANITIZER_INTERCEPT_POPENVE SI_NETBSD +#define SANITIZER_INTERCEPT_PCLOSE SI_POSIX +#define SANITIZER_INTERCEPT_FUNOPEN (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_FUNOPEN2 SI_NETBSD +#define SANITIZER_INTERCEPT_GETFSENT (SI_FREEBSD || SI_NETBSD || SI_MAC) +#define SANITIZER_INTERCEPT_ARC4RANDOM (SI_FREEBSD || SI_NETBSD) +#define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD +#define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_POSIX) +#define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD) #endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc new file mode 100644 index 00000000000..377a62cab6c --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc @@ -0,0 +1,526 @@ +//===-- sanitizer_platform_limits_freebsd.cc ------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific FreeBSD data structures. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_FREEBSD + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define _KERNEL // to declare 'shminfo' structure +# include +#undef _KERNEL + +#undef INLINE // to avoid clashes with sanitizers' definitions + +#undef IOC_DIRMASK + +# include +# include +# include + +#include +#include +#include + +// Include these after system headers to avoid name clashes and ambiguities. +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_freebsd.h" + +namespace __sanitizer { + unsigned struct_cap_rights_sz = sizeof(cap_rights_t); + unsigned struct_utsname_sz = sizeof(struct utsname); + unsigned struct_stat_sz = sizeof(struct stat); + unsigned struct_rusage_sz = sizeof(struct rusage); + unsigned struct_tm_sz = sizeof(struct tm); + unsigned struct_passwd_sz = sizeof(struct passwd); + unsigned struct_group_sz = sizeof(struct group); + unsigned siginfo_t_sz = sizeof(siginfo_t); + unsigned struct_sigaction_sz = sizeof(struct sigaction); + unsigned struct_itimerval_sz = sizeof(struct itimerval); + unsigned pthread_t_sz = sizeof(pthread_t); + unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); + unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); + unsigned pid_t_sz = sizeof(pid_t); + unsigned timeval_sz = sizeof(timeval); + unsigned uid_t_sz = sizeof(uid_t); + unsigned gid_t_sz = sizeof(gid_t); + unsigned fpos_t_sz = sizeof(fpos_t); + unsigned mbstate_t_sz = sizeof(mbstate_t); + unsigned sigset_t_sz = sizeof(sigset_t); + unsigned struct_timezone_sz = sizeof(struct timezone); + unsigned struct_tms_sz = sizeof(struct tms); + unsigned struct_sigevent_sz = sizeof(struct sigevent); + unsigned struct_sched_param_sz = sizeof(struct sched_param); + unsigned struct_statfs_sz = sizeof(struct statfs); + unsigned struct_sockaddr_sz = sizeof(struct sockaddr); + unsigned ucontext_t_sz = sizeof(ucontext_t); + unsigned struct_rlimit_sz = sizeof(struct rlimit); + unsigned struct_timespec_sz = sizeof(struct timespec); + unsigned struct_utimbuf_sz = sizeof(struct utimbuf); + unsigned struct_itimerspec_sz = sizeof(struct itimerspec); + unsigned struct_timeb_sz = sizeof(struct timeb); + unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); + unsigned struct_mq_attr_sz = sizeof(struct mq_attr); + unsigned struct_statvfs_sz = sizeof(struct statvfs); + unsigned struct_shminfo_sz = sizeof(struct shminfo); + unsigned struct_shm_info_sz = sizeof(struct shm_info); + unsigned struct_regmatch_sz = sizeof(regmatch_t); + unsigned struct_regex_sz = sizeof(regex_t); + unsigned struct_fstab_sz = sizeof(struct fstab); + unsigned struct_FTS_sz = sizeof(FTS); + unsigned struct_FTSENT_sz = sizeof(FTSENT); + unsigned struct_StringList_sz = sizeof(StringList); + + const uptr sig_ign = (uptr)SIG_IGN; + const uptr sig_dfl = (uptr)SIG_DFL; + const uptr sig_err = (uptr)SIG_ERR; + const uptr sa_siginfo = (uptr)SA_SIGINFO; + + int shmctl_ipc_stat = (int)IPC_STAT; + int shmctl_ipc_info = (int)IPC_INFO; + int shmctl_shm_info = (int)SHM_INFO; + int shmctl_shm_stat = (int)SHM_STAT; + unsigned struct_utmpx_sz = sizeof(struct utmpx); + + int map_fixed = MAP_FIXED; + + int af_inet = (int)AF_INET; + int af_inet6 = (int)AF_INET6; + + uptr __sanitizer_in_addr_sz(int af) { + if (af == AF_INET) + return sizeof(struct in_addr); + else if (af == AF_INET6) + return sizeof(struct in6_addr); + else + return 0; + } + + unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); + int glob_nomatch = GLOB_NOMATCH; + int glob_altdirfunc = GLOB_ALTDIRFUNC; + + unsigned path_max = PATH_MAX; + + // ioctl arguments + unsigned struct_ifreq_sz = sizeof(struct ifreq); + unsigned struct_termios_sz = sizeof(struct termios); + unsigned struct_winsize_sz = sizeof(struct winsize); +#if SOUND_VERSION >= 0x040000 + unsigned struct_copr_buffer_sz = 0; + unsigned struct_copr_debug_buf_sz = 0; + unsigned struct_copr_msg_sz = 0; +#else + unsigned struct_copr_buffer_sz = sizeof(struct copr_buffer); + unsigned struct_copr_debug_buf_sz = sizeof(struct copr_debug_buf); + unsigned struct_copr_msg_sz = sizeof(struct copr_msg); +#endif + unsigned struct_midi_info_sz = sizeof(struct midi_info); + unsigned struct_mtget_sz = sizeof(struct mtget); + unsigned struct_mtop_sz = sizeof(struct mtop); + unsigned struct_sbi_instrument_sz = sizeof(struct sbi_instrument); + unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec); + unsigned struct_synth_info_sz = sizeof(struct synth_info); + unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); + unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); + unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); + unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + const unsigned long __sanitizer_bufsiz = BUFSIZ; + + const unsigned IOCTL_NOT_PRESENT = 0; + + unsigned IOCTL_FIOASYNC = FIOASYNC; + unsigned IOCTL_FIOCLEX = FIOCLEX; + unsigned IOCTL_FIOGETOWN = FIOGETOWN; + unsigned IOCTL_FIONBIO = FIONBIO; + unsigned IOCTL_FIONCLEX = FIONCLEX; + unsigned IOCTL_FIOSETOWN = FIOSETOWN; + unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; + unsigned IOCTL_SIOCATMARK = SIOCATMARK; + unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; + unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR; + unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR; + unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF; + unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR; + unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS; + unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC; + unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU; + unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK; + unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; + unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; + unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR; + unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR; + unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS; + unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC; + unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU; + unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; + unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; + unsigned IOCTL_TIOCCONS = TIOCCONS; + unsigned IOCTL_TIOCEXCL = TIOCEXCL; + unsigned IOCTL_TIOCGETD = TIOCGETD; + unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; + unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ; + unsigned IOCTL_TIOCMBIC = TIOCMBIC; + unsigned IOCTL_TIOCMBIS = TIOCMBIS; + unsigned IOCTL_TIOCMGET = TIOCMGET; + unsigned IOCTL_TIOCMSET = TIOCMSET; + unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; + unsigned IOCTL_TIOCNXCL = TIOCNXCL; + unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; + unsigned IOCTL_TIOCPKT = TIOCPKT; + unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; + unsigned IOCTL_TIOCSETD = TIOCSETD; + unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; + unsigned IOCTL_TIOCSTI = TIOCSTI; + unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; + unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; + unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; + unsigned IOCTL_MTIOCGET = MTIOCGET; + unsigned IOCTL_MTIOCTOP = MTIOCTOP; + unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE; + unsigned IOCTL_SNDCTL_DSP_GETFMTS = SNDCTL_DSP_GETFMTS; + unsigned IOCTL_SNDCTL_DSP_NONBLOCK = SNDCTL_DSP_NONBLOCK; + unsigned IOCTL_SNDCTL_DSP_POST = SNDCTL_DSP_POST; + unsigned IOCTL_SNDCTL_DSP_RESET = SNDCTL_DSP_RESET; + unsigned IOCTL_SNDCTL_DSP_SETFMT = SNDCTL_DSP_SETFMT; + unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT = SNDCTL_DSP_SETFRAGMENT; + unsigned IOCTL_SNDCTL_DSP_SPEED = SNDCTL_DSP_SPEED; + unsigned IOCTL_SNDCTL_DSP_STEREO = SNDCTL_DSP_STEREO; + unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE = SNDCTL_DSP_SUBDIVIDE; + unsigned IOCTL_SNDCTL_DSP_SYNC = SNDCTL_DSP_SYNC; + unsigned IOCTL_SNDCTL_FM_4OP_ENABLE = SNDCTL_FM_4OP_ENABLE; + unsigned IOCTL_SNDCTL_FM_LOAD_INSTR = SNDCTL_FM_LOAD_INSTR; + unsigned IOCTL_SNDCTL_MIDI_INFO = SNDCTL_MIDI_INFO; + unsigned IOCTL_SNDCTL_MIDI_PRETIME = SNDCTL_MIDI_PRETIME; + unsigned IOCTL_SNDCTL_SEQ_CTRLRATE = SNDCTL_SEQ_CTRLRATE; + unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT = SNDCTL_SEQ_GETINCOUNT; + unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT = SNDCTL_SEQ_GETOUTCOUNT; + unsigned IOCTL_SNDCTL_SEQ_NRMIDIS = SNDCTL_SEQ_NRMIDIS; + unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS = SNDCTL_SEQ_NRSYNTHS; + unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND = SNDCTL_SEQ_OUTOFBAND; + unsigned IOCTL_SNDCTL_SEQ_PANIC = SNDCTL_SEQ_PANIC; + unsigned IOCTL_SNDCTL_SEQ_PERCMODE = SNDCTL_SEQ_PERCMODE; + unsigned IOCTL_SNDCTL_SEQ_RESET = SNDCTL_SEQ_RESET; + unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES = SNDCTL_SEQ_RESETSAMPLES; + unsigned IOCTL_SNDCTL_SEQ_SYNC = SNDCTL_SEQ_SYNC; + unsigned IOCTL_SNDCTL_SEQ_TESTMIDI = SNDCTL_SEQ_TESTMIDI; + unsigned IOCTL_SNDCTL_SEQ_THRESHOLD = SNDCTL_SEQ_THRESHOLD; + unsigned IOCTL_SNDCTL_SYNTH_INFO = SNDCTL_SYNTH_INFO; + unsigned IOCTL_SNDCTL_SYNTH_MEMAVL = SNDCTL_SYNTH_MEMAVL; + unsigned IOCTL_SNDCTL_TMR_CONTINUE = SNDCTL_TMR_CONTINUE; + unsigned IOCTL_SNDCTL_TMR_METRONOME = SNDCTL_TMR_METRONOME; + unsigned IOCTL_SNDCTL_TMR_SELECT = SNDCTL_TMR_SELECT; + unsigned IOCTL_SNDCTL_TMR_SOURCE = SNDCTL_TMR_SOURCE; + unsigned IOCTL_SNDCTL_TMR_START = SNDCTL_TMR_START; + unsigned IOCTL_SNDCTL_TMR_STOP = SNDCTL_TMR_STOP; + unsigned IOCTL_SNDCTL_TMR_TEMPO = SNDCTL_TMR_TEMPO; + unsigned IOCTL_SNDCTL_TMR_TIMEBASE = SNDCTL_TMR_TIMEBASE; + unsigned IOCTL_SOUND_MIXER_READ_ALTPCM = SOUND_MIXER_READ_ALTPCM; + unsigned IOCTL_SOUND_MIXER_READ_BASS = SOUND_MIXER_READ_BASS; + unsigned IOCTL_SOUND_MIXER_READ_CAPS = SOUND_MIXER_READ_CAPS; + unsigned IOCTL_SOUND_MIXER_READ_CD = SOUND_MIXER_READ_CD; + unsigned IOCTL_SOUND_MIXER_READ_DEVMASK = SOUND_MIXER_READ_DEVMASK; + unsigned IOCTL_SOUND_MIXER_READ_ENHANCE = SOUND_MIXER_READ_ENHANCE; + unsigned IOCTL_SOUND_MIXER_READ_IGAIN = SOUND_MIXER_READ_IGAIN; + unsigned IOCTL_SOUND_MIXER_READ_IMIX = SOUND_MIXER_READ_IMIX; + unsigned IOCTL_SOUND_MIXER_READ_LINE = SOUND_MIXER_READ_LINE; + unsigned IOCTL_SOUND_MIXER_READ_LINE1 = SOUND_MIXER_READ_LINE1; + unsigned IOCTL_SOUND_MIXER_READ_LINE2 = SOUND_MIXER_READ_LINE2; + unsigned IOCTL_SOUND_MIXER_READ_LINE3 = SOUND_MIXER_READ_LINE3; + unsigned IOCTL_SOUND_MIXER_READ_LOUD = SOUND_MIXER_READ_LOUD; + unsigned IOCTL_SOUND_MIXER_READ_MIC = SOUND_MIXER_READ_MIC; + unsigned IOCTL_SOUND_MIXER_READ_MUTE = SOUND_MIXER_READ_MUTE; + unsigned IOCTL_SOUND_MIXER_READ_OGAIN = SOUND_MIXER_READ_OGAIN; + unsigned IOCTL_SOUND_MIXER_READ_PCM = SOUND_MIXER_READ_PCM; + unsigned IOCTL_SOUND_MIXER_READ_RECLEV = SOUND_MIXER_READ_RECLEV; + unsigned IOCTL_SOUND_MIXER_READ_RECMASK = SOUND_MIXER_READ_RECMASK; + unsigned IOCTL_SOUND_MIXER_READ_RECSRC = SOUND_MIXER_READ_RECSRC; + unsigned IOCTL_SOUND_MIXER_READ_SPEAKER = SOUND_MIXER_READ_SPEAKER; + unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS = SOUND_MIXER_READ_STEREODEVS; + unsigned IOCTL_SOUND_MIXER_READ_SYNTH = SOUND_MIXER_READ_SYNTH; + unsigned IOCTL_SOUND_MIXER_READ_TREBLE = SOUND_MIXER_READ_TREBLE; + unsigned IOCTL_SOUND_MIXER_READ_VOLUME = SOUND_MIXER_READ_VOLUME; + unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM = SOUND_MIXER_WRITE_ALTPCM; + unsigned IOCTL_SOUND_MIXER_WRITE_BASS = SOUND_MIXER_WRITE_BASS; + unsigned IOCTL_SOUND_MIXER_WRITE_CD = SOUND_MIXER_WRITE_CD; + unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE = SOUND_MIXER_WRITE_ENHANCE; + unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN = SOUND_MIXER_WRITE_IGAIN; + unsigned IOCTL_SOUND_MIXER_WRITE_IMIX = SOUND_MIXER_WRITE_IMIX; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE = SOUND_MIXER_WRITE_LINE; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE1 = SOUND_MIXER_WRITE_LINE1; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE2 = SOUND_MIXER_WRITE_LINE2; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE3 = SOUND_MIXER_WRITE_LINE3; + unsigned IOCTL_SOUND_MIXER_WRITE_LOUD = SOUND_MIXER_WRITE_LOUD; + unsigned IOCTL_SOUND_MIXER_WRITE_MIC = SOUND_MIXER_WRITE_MIC; + unsigned IOCTL_SOUND_MIXER_WRITE_MUTE = SOUND_MIXER_WRITE_MUTE; + unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN = SOUND_MIXER_WRITE_OGAIN; + unsigned IOCTL_SOUND_MIXER_WRITE_PCM = SOUND_MIXER_WRITE_PCM; + unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV = SOUND_MIXER_WRITE_RECLEV; + unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC = SOUND_MIXER_WRITE_RECSRC; + unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER = SOUND_MIXER_WRITE_SPEAKER; + unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH = SOUND_MIXER_WRITE_SYNTH; + unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE = SOUND_MIXER_WRITE_TREBLE; + unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME = SOUND_MIXER_WRITE_VOLUME; + unsigned IOCTL_VT_ACTIVATE = VT_ACTIVATE; + unsigned IOCTL_VT_GETMODE = VT_GETMODE; + unsigned IOCTL_VT_OPENQRY = VT_OPENQRY; + unsigned IOCTL_VT_RELDISP = VT_RELDISP; + unsigned IOCTL_VT_SETMODE = VT_SETMODE; + unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE; + unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP; + unsigned IOCTL_KDDISABIO = KDDISABIO; + unsigned IOCTL_KDENABIO = KDENABIO; + unsigned IOCTL_KDGETLED = KDGETLED; + unsigned IOCTL_KDGETMODE = KDGETMODE; + unsigned IOCTL_KDGKBMODE = KDGKBMODE; + unsigned IOCTL_KDGKBTYPE = KDGKBTYPE; + unsigned IOCTL_KDMKTONE = KDMKTONE; + unsigned IOCTL_KDSETLED = KDSETLED; + unsigned IOCTL_KDSETMODE = KDSETMODE; + unsigned IOCTL_KDSKBMODE = KDSKBMODE; + unsigned IOCTL_KIOCSOUND = KIOCSOUND; + unsigned IOCTL_PIO_SCRNMAP = PIO_SCRNMAP; + unsigned IOCTL_SNDCTL_DSP_GETISPACE = SNDCTL_DSP_GETISPACE; + + const int si_SEGV_MAPERR = SEGV_MAPERR; + const int si_SEGV_ACCERR = SEGV_ACCERR; + const int unvis_valid = UNVIS_VALID; + const int unvis_validpush = UNVIS_VALIDPUSH; +} // namespace __sanitizer + +using namespace __sanitizer; + +COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); + +COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); +CHECK_TYPE_SIZE(pthread_key_t); + +// There are more undocumented fields in dl_phdr_info that we are not interested +// in. +COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info)); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); + +CHECK_TYPE_SIZE(glob_t); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv); +CHECK_SIZE_AND_OFFSET(glob_t, gl_offs); +CHECK_SIZE_AND_OFFSET(glob_t, gl_flags); +CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat); +CHECK_SIZE_AND_OFFSET(glob_t, gl_stat); + +CHECK_TYPE_SIZE(addrinfo); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_family); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); + +CHECK_TYPE_SIZE(hostent); +CHECK_SIZE_AND_OFFSET(hostent, h_name); +CHECK_SIZE_AND_OFFSET(hostent, h_aliases); +CHECK_SIZE_AND_OFFSET(hostent, h_addrtype); +CHECK_SIZE_AND_OFFSET(hostent, h_length); +CHECK_SIZE_AND_OFFSET(hostent, h_addr_list); + +CHECK_TYPE_SIZE(iovec); +CHECK_SIZE_AND_OFFSET(iovec, iov_base); +CHECK_SIZE_AND_OFFSET(iovec, iov_len); + +CHECK_TYPE_SIZE(msghdr); +CHECK_SIZE_AND_OFFSET(msghdr, msg_name); +CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iov); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_control); +CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_flags); + +CHECK_TYPE_SIZE(cmsghdr); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); + +COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); +CHECK_SIZE_AND_OFFSET(dirent, d_ino); +CHECK_SIZE_AND_OFFSET(dirent, d_reclen); + +CHECK_TYPE_SIZE(ifconf); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_len); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu); + +CHECK_TYPE_SIZE(pollfd); +CHECK_SIZE_AND_OFFSET(pollfd, fd); +CHECK_SIZE_AND_OFFSET(pollfd, events); +CHECK_SIZE_AND_OFFSET(pollfd, revents); + +CHECK_TYPE_SIZE(nfds_t); + +CHECK_TYPE_SIZE(sigset_t); + +COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); +// Can't write checks for sa_handler and sa_sigaction due to them being +// preprocessor macros. +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); + +CHECK_TYPE_SIZE(wordexp_t); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); + +CHECK_TYPE_SIZE(tm); +CHECK_SIZE_AND_OFFSET(tm, tm_sec); +CHECK_SIZE_AND_OFFSET(tm, tm_min); +CHECK_SIZE_AND_OFFSET(tm, tm_hour); +CHECK_SIZE_AND_OFFSET(tm, tm_mday); +CHECK_SIZE_AND_OFFSET(tm, tm_mon); +CHECK_SIZE_AND_OFFSET(tm, tm_year); +CHECK_SIZE_AND_OFFSET(tm, tm_wday); +CHECK_SIZE_AND_OFFSET(tm, tm_yday); +CHECK_SIZE_AND_OFFSET(tm, tm_isdst); +CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff); +CHECK_SIZE_AND_OFFSET(tm, tm_zone); + +CHECK_TYPE_SIZE(ether_addr); + +CHECK_TYPE_SIZE(ipc_perm); +CHECK_SIZE_AND_OFFSET(ipc_perm, key); +CHECK_SIZE_AND_OFFSET(ipc_perm, seq); +CHECK_SIZE_AND_OFFSET(ipc_perm, uid); +CHECK_SIZE_AND_OFFSET(ipc_perm, gid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); + +CHECK_TYPE_SIZE(shmid_ds); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch); + +CHECK_TYPE_SIZE(clock_t); + +CHECK_TYPE_SIZE(ifaddrs); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); +#undef ifa_dstaddr +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); + +CHECK_TYPE_SIZE(timeb); +CHECK_SIZE_AND_OFFSET(timeb, time); +CHECK_SIZE_AND_OFFSET(timeb, millitm); +CHECK_SIZE_AND_OFFSET(timeb, timezone); +CHECK_SIZE_AND_OFFSET(timeb, dstflag); + +CHECK_TYPE_SIZE(passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_name); +CHECK_SIZE_AND_OFFSET(passwd, pw_passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_uid); +CHECK_SIZE_AND_OFFSET(passwd, pw_gid); +CHECK_SIZE_AND_OFFSET(passwd, pw_dir); +CHECK_SIZE_AND_OFFSET(passwd, pw_shell); + +CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); + +CHECK_TYPE_SIZE(group); +CHECK_SIZE_AND_OFFSET(group, gr_name); +CHECK_SIZE_AND_OFFSET(group, gr_passwd); +CHECK_SIZE_AND_OFFSET(group, gr_gid); +CHECK_SIZE_AND_OFFSET(group, gr_mem); + +#if HAVE_RPC_XDR_H +CHECK_TYPE_SIZE(XDR); +CHECK_SIZE_AND_OFFSET(XDR, x_op); +CHECK_SIZE_AND_OFFSET(XDR, x_ops); +CHECK_SIZE_AND_OFFSET(XDR, x_public); +CHECK_SIZE_AND_OFFSET(XDR, x_private); +CHECK_SIZE_AND_OFFSET(XDR, x_base); +CHECK_SIZE_AND_OFFSET(XDR, x_handy); +COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE); +COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE); +COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE); +#endif + +CHECK_TYPE_SIZE(sem_t); + +COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t)); +#endif // SANITIZER_FREEBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h new file mode 100644 index 00000000000..588bead2ac1 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -0,0 +1,657 @@ +//===-- sanitizer_platform_limits_freebsd.h -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific FreeBSD data structures. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_PLATFORM_LIMITS_FREEBSD_H +#define SANITIZER_PLATFORM_LIMITS_FREEBSD_H + +#if SANITIZER_FREEBSD + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +#include "sanitizer_platform_limits_posix.h" + +// FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that +// incorporates the map structure. +# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 560))) +// Get sys/_types.h, because that tells us whether 64-bit inodes are +// used in struct dirent below. +#include + +namespace __sanitizer { + extern unsigned struct_utsname_sz; + extern unsigned struct_stat_sz; +#if defined(__powerpc64__) + const unsigned struct___old_kernel_stat_sz = 0; +#else + const unsigned struct___old_kernel_stat_sz = 32; +#endif + extern unsigned struct_rusage_sz; + extern unsigned siginfo_t_sz; + extern unsigned struct_itimerval_sz; + extern unsigned pthread_t_sz; + extern unsigned pthread_mutex_t_sz; + extern unsigned pthread_cond_t_sz; + extern unsigned pid_t_sz; + extern unsigned timeval_sz; + extern unsigned uid_t_sz; + extern unsigned gid_t_sz; + extern unsigned fpos_t_sz; + extern unsigned mbstate_t_sz; + extern unsigned struct_timezone_sz; + extern unsigned struct_tms_sz; + extern unsigned struct_itimerspec_sz; + extern unsigned struct_sigevent_sz; + extern unsigned struct_sched_param_sz; + extern unsigned struct_statfs64_sz; + extern unsigned struct_statfs_sz; + extern unsigned struct_sockaddr_sz; + extern unsigned ucontext_t_sz; + extern unsigned struct_rlimit_sz; + extern unsigned struct_utimbuf_sz; + extern unsigned struct_timespec_sz; + extern unsigned struct_regmatch_sz; + extern unsigned struct_regex_sz; + extern unsigned struct_FTS_sz; + extern unsigned struct_FTSENT_sz; + extern const int unvis_valid; + extern const int unvis_validpush; + + struct __sanitizer_iocb { + u64 aio_data; + u32 aio_key_or_aio_reserved1; // Simply crazy. + u32 aio_reserved1_or_aio_key; // Luckily, we don't need these. + u16 aio_lio_opcode; + s16 aio_reqprio; + u32 aio_fildes; + u64 aio_buf; + u64 aio_nbytes; + s64 aio_offset; + u64 aio_reserved2; + u64 aio_reserved3; + }; + + struct __sanitizer_io_event { + u64 data; + u64 obj; + u64 res; + u64 res2; + }; + + const unsigned iocb_cmd_pread = 0; + const unsigned iocb_cmd_pwrite = 1; + const unsigned iocb_cmd_preadv = 7; + const unsigned iocb_cmd_pwritev = 8; + + struct __sanitizer___sysctl_args { + int *name; + int nlen; + void *oldval; + uptr *oldlenp; + void *newval; + uptr newlen; + unsigned long ___unused[4]; + }; + + struct __sanitizer_ipc_perm { + unsigned int cuid; + unsigned int cgid; + unsigned int uid; + unsigned int gid; + unsigned short mode; + unsigned short seq; + long key; + }; + + struct __sanitizer_shmid_ds { + __sanitizer_ipc_perm shm_perm; + unsigned long shm_segsz; + unsigned int shm_lpid; + unsigned int shm_cpid; + int shm_nattch; + unsigned long shm_atime; + unsigned long shm_dtime; + unsigned long shm_ctime; + }; + + extern unsigned struct_msqid_ds_sz; + extern unsigned struct_mq_attr_sz; + extern unsigned struct_timeb_sz; + extern unsigned struct_statvfs_sz; + + struct __sanitizer_iovec { + void *iov_base; + uptr iov_len; + }; + + struct __sanitizer_ifaddrs { + struct __sanitizer_ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + void *ifa_addr; // (struct sockaddr *) + void *ifa_netmask; // (struct sockaddr *) +# undef ifa_dstaddr + void *ifa_dstaddr; // (struct sockaddr *) + void *ifa_data; + }; + + typedef unsigned __sanitizer_pthread_key_t; + + struct __sanitizer_passwd { + char *pw_name; + char *pw_passwd; + int pw_uid; + int pw_gid; + long pw_change; + char *pw_class; + char *pw_gecos; + char *pw_dir; + char *pw_shell; + long pw_expire; + int pw_fields; + }; + + struct __sanitizer_group { + char *gr_name; + char *gr_passwd; + int gr_gid; + char **gr_mem; + }; + +#if defined(__LP64___) + typedef long long __sanitizer_time_t; +#else + typedef long __sanitizer_time_t; +#endif + + typedef long __sanitizer_suseconds_t; + + struct __sanitizer_timeval { + __sanitizer_time_t tv_sec; + __sanitizer_suseconds_t tv_usec; + }; + + struct __sanitizer_itimerval { + struct __sanitizer_timeval it_interval; + struct __sanitizer_timeval it_value; + }; + + struct __sanitizer_timeb { + __sanitizer_time_t time; + unsigned short millitm; + short timezone; + short dstflag; + }; + + struct __sanitizer_ether_addr { + u8 octet[6]; + }; + + struct __sanitizer_tm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; + long int tm_gmtoff; + const char *tm_zone; + }; + + struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + unsigned msg_iovlen; + void *msg_control; + unsigned msg_controllen; + int msg_flags; + }; + + struct __sanitizer_cmsghdr { + unsigned cmsg_len; + int cmsg_level; + int cmsg_type; + }; + + struct __sanitizer_dirent { +#if defined(__INO64) + unsigned long long d_fileno; + unsigned long long d_off; +#else + unsigned int d_fileno; +#endif + unsigned short d_reclen; + // more fields that we don't care about + }; + +// 'clock_t' is 32 bits wide on x64 FreeBSD + typedef int __sanitizer_clock_t; + typedef int __sanitizer_clockid_t; + +#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__)\ + || defined(__mips__) + typedef unsigned __sanitizer___kernel_uid_t; + typedef unsigned __sanitizer___kernel_gid_t; +#else + typedef unsigned short __sanitizer___kernel_uid_t; + typedef unsigned short __sanitizer___kernel_gid_t; +#endif + typedef long long __sanitizer___kernel_off_t; + +#if defined(__powerpc__) || defined(__mips__) + typedef unsigned int __sanitizer___kernel_old_uid_t; + typedef unsigned int __sanitizer___kernel_old_gid_t; +#else + typedef unsigned short __sanitizer___kernel_old_uid_t; + typedef unsigned short __sanitizer___kernel_old_gid_t; +#endif + + typedef long long __sanitizer___kernel_loff_t; + typedef struct { + unsigned long fds_bits[1024 / (8 * sizeof(long))]; + } __sanitizer___kernel_fd_set; + + // This thing depends on the platform. We are only interested in the upper + // limit. Verified with a compiler assert in .cc. + const int pthread_attr_t_max_sz = 128; + union __sanitizer_pthread_attr_t { + char size[pthread_attr_t_max_sz]; // NOLINT + void *align; + }; + + const unsigned old_sigset_t_sz = sizeof(unsigned long); + + struct __sanitizer_sigset_t { + // uint32_t * 4 + unsigned int __bits[4]; + }; + + typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; + + struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; + }; + + using __sanitizer_sighandler_ptr = void (*)(int sig); + using __sanitizer_sigactionhandler_ptr = + void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx); + + struct __sanitizer_sigaction { + union { + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; + }; + int sa_flags; + __sanitizer_sigset_t sa_mask; + }; + + struct __sanitizer_sem_t { + u32 data[4]; + }; + + extern const uptr sig_ign; + extern const uptr sig_dfl; + extern const uptr sig_err; + extern const uptr sa_siginfo; + + extern int af_inet; + extern int af_inet6; + uptr __sanitizer_in_addr_sz(int af); + + struct __sanitizer_dl_phdr_info { + uptr dlpi_addr; + const char *dlpi_name; + const void *dlpi_phdr; + short dlpi_phnum; + }; + + extern unsigned struct_ElfW_Phdr_sz; + + struct __sanitizer_addrinfo { + int ai_flags; + int ai_family; + int ai_socktype; + int ai_protocol; + unsigned ai_addrlen; + char *ai_canonname; + void *ai_addr; + struct __sanitizer_addrinfo *ai_next; + }; + + struct __sanitizer_hostent { + char *h_name; + char **h_aliases; + int h_addrtype; + int h_length; + char **h_addr_list; + }; + + struct __sanitizer_pollfd { + int fd; + short events; + short revents; + }; + + typedef unsigned __sanitizer_nfds_t; + + struct __sanitizer_glob_t { + uptr gl_pathc; + uptr gl_matchc; + uptr gl_offs; + int gl_flags; + char **gl_pathv; + int (*gl_errfunc)(const char*, int); + void (*gl_closedir)(void *dirp); + struct dirent *(*gl_readdir)(void *dirp); + void *(*gl_opendir)(const char*); + int (*gl_lstat)(const char*, void* /* struct stat* */); + int (*gl_stat)(const char*, void* /* struct stat* */); + }; + + extern int glob_nomatch; + extern int glob_altdirfunc; + + extern unsigned path_max; + + struct __sanitizer_wordexp_t { + uptr we_wordc; + char **we_wordv; + uptr we_offs; + char *we_strings; + uptr we_nbytes; + }; + + typedef void __sanitizer_FILE; + + extern unsigned struct_shminfo_sz; + extern unsigned struct_shm_info_sz; + extern int shmctl_ipc_stat; + extern int shmctl_ipc_info; + extern int shmctl_shm_info; + extern int shmctl_shm_stat; + + extern unsigned struct_utmpx_sz; + + extern int map_fixed; + + // ioctl arguments + struct __sanitizer_ifconf { + int ifc_len; + union { + void *ifcu_req; + } ifc_ifcu; + }; + +#define IOC_NRBITS 8 +#define IOC_TYPEBITS 8 +#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) +#define IOC_SIZEBITS 13 +#define IOC_DIRBITS 3 +#define IOC_NONE 1U +#define IOC_WRITE 4U +#define IOC_READ 2U +#else +#define IOC_SIZEBITS 14 +#define IOC_DIRBITS 2 +#define IOC_NONE 0U +#define IOC_WRITE 1U +#define IOC_READ 2U +#endif +#define IOC_NRMASK ((1 << IOC_NRBITS) - 1) +#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) +#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) +#if defined(IOC_DIRMASK) +#undef IOC_DIRMASK +#endif +#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) +#define IOC_NRSHIFT 0 +#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) +#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) +#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) +#define EVIOC_EV_MAX 0x1f +#define EVIOC_ABS_MAX 0x3f + +#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) +#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) +#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) +#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) + + extern unsigned struct_ifreq_sz; + extern unsigned struct_termios_sz; + extern unsigned struct_winsize_sz; + + extern unsigned struct_copr_buffer_sz; + extern unsigned struct_copr_debug_buf_sz; + extern unsigned struct_copr_msg_sz; + extern unsigned struct_midi_info_sz; + extern unsigned struct_mtget_sz; + extern unsigned struct_mtop_sz; + extern unsigned struct_rtentry_sz; + extern unsigned struct_sbi_instrument_sz; + extern unsigned struct_seq_event_rec_sz; + extern unsigned struct_synth_info_sz; + extern unsigned struct_vt_mode_sz; + + extern const unsigned long __sanitizer_bufsiz; + extern unsigned struct_audio_buf_info_sz; + extern unsigned struct_ppp_stats_sz; + extern unsigned struct_sioc_sg_req_sz; + extern unsigned struct_sioc_vif_req_sz; + + // ioctl request identifiers + + // A special value to mark ioctls that are not present on the target platform, + // when it can not be determined without including any system headers. + extern const unsigned IOCTL_NOT_PRESENT; + + extern unsigned IOCTL_FIOASYNC; + extern unsigned IOCTL_FIOCLEX; + extern unsigned IOCTL_FIOGETOWN; + extern unsigned IOCTL_FIONBIO; + extern unsigned IOCTL_FIONCLEX; + extern unsigned IOCTL_FIOSETOWN; + extern unsigned IOCTL_SIOCADDMULTI; + extern unsigned IOCTL_SIOCATMARK; + extern unsigned IOCTL_SIOCDELMULTI; + extern unsigned IOCTL_SIOCGIFADDR; + extern unsigned IOCTL_SIOCGIFBRDADDR; + extern unsigned IOCTL_SIOCGIFCONF; + extern unsigned IOCTL_SIOCGIFDSTADDR; + extern unsigned IOCTL_SIOCGIFFLAGS; + extern unsigned IOCTL_SIOCGIFMETRIC; + extern unsigned IOCTL_SIOCGIFMTU; + extern unsigned IOCTL_SIOCGIFNETMASK; + extern unsigned IOCTL_SIOCGPGRP; + extern unsigned IOCTL_SIOCSIFADDR; + extern unsigned IOCTL_SIOCSIFBRDADDR; + extern unsigned IOCTL_SIOCSIFDSTADDR; + extern unsigned IOCTL_SIOCSIFFLAGS; + extern unsigned IOCTL_SIOCSIFMETRIC; + extern unsigned IOCTL_SIOCSIFMTU; + extern unsigned IOCTL_SIOCSIFNETMASK; + extern unsigned IOCTL_SIOCSPGRP; + extern unsigned IOCTL_TIOCCONS; + extern unsigned IOCTL_TIOCEXCL; + extern unsigned IOCTL_TIOCGETD; + extern unsigned IOCTL_TIOCGPGRP; + extern unsigned IOCTL_TIOCGWINSZ; + extern unsigned IOCTL_TIOCMBIC; + extern unsigned IOCTL_TIOCMBIS; + extern unsigned IOCTL_TIOCMGET; + extern unsigned IOCTL_TIOCMSET; + extern unsigned IOCTL_TIOCNOTTY; + extern unsigned IOCTL_TIOCNXCL; + extern unsigned IOCTL_TIOCOUTQ; + extern unsigned IOCTL_TIOCPKT; + extern unsigned IOCTL_TIOCSCTTY; + extern unsigned IOCTL_TIOCSETD; + extern unsigned IOCTL_TIOCSPGRP; + extern unsigned IOCTL_TIOCSTI; + extern unsigned IOCTL_TIOCSWINSZ; + extern unsigned IOCTL_SIOCGETSGCNT; + extern unsigned IOCTL_SIOCGETVIFCNT; + extern unsigned IOCTL_MTIOCGET; + extern unsigned IOCTL_MTIOCTOP; + extern unsigned IOCTL_SIOCADDRT; + extern unsigned IOCTL_SIOCDELRT; + extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE; + extern unsigned IOCTL_SNDCTL_DSP_GETFMTS; + extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK; + extern unsigned IOCTL_SNDCTL_DSP_POST; + extern unsigned IOCTL_SNDCTL_DSP_RESET; + extern unsigned IOCTL_SNDCTL_DSP_SETFMT; + extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT; + extern unsigned IOCTL_SNDCTL_DSP_SPEED; + extern unsigned IOCTL_SNDCTL_DSP_STEREO; + extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE; + extern unsigned IOCTL_SNDCTL_DSP_SYNC; + extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE; + extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR; + extern unsigned IOCTL_SNDCTL_MIDI_INFO; + extern unsigned IOCTL_SNDCTL_MIDI_PRETIME; + extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE; + extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT; + extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT; + extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS; + extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS; + extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND; + extern unsigned IOCTL_SNDCTL_SEQ_PANIC; + extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE; + extern unsigned IOCTL_SNDCTL_SEQ_RESET; + extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES; + extern unsigned IOCTL_SNDCTL_SEQ_SYNC; + extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI; + extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD; + extern unsigned IOCTL_SNDCTL_SYNTH_INFO; + extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL; + extern unsigned IOCTL_SNDCTL_TMR_CONTINUE; + extern unsigned IOCTL_SNDCTL_TMR_METRONOME; + extern unsigned IOCTL_SNDCTL_TMR_SELECT; + extern unsigned IOCTL_SNDCTL_TMR_SOURCE; + extern unsigned IOCTL_SNDCTL_TMR_START; + extern unsigned IOCTL_SNDCTL_TMR_STOP; + extern unsigned IOCTL_SNDCTL_TMR_TEMPO; + extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE; + extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM; + extern unsigned IOCTL_SOUND_MIXER_READ_BASS; + extern unsigned IOCTL_SOUND_MIXER_READ_CAPS; + extern unsigned IOCTL_SOUND_MIXER_READ_CD; + extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK; + extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE; + extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN; + extern unsigned IOCTL_SOUND_MIXER_READ_IMIX; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE1; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE2; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE3; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE; + extern unsigned IOCTL_SOUND_MIXER_READ_LOUD; + extern unsigned IOCTL_SOUND_MIXER_READ_MIC; + extern unsigned IOCTL_SOUND_MIXER_READ_MUTE; + extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN; + extern unsigned IOCTL_SOUND_MIXER_READ_PCM; + extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV; + extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK; + extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC; + extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER; + extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS; + extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH; + extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE; + extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME; + extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM; + extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS; + extern unsigned IOCTL_SOUND_MIXER_WRITE_CD; + extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN; + extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD; + extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC; + extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN; + extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM; + extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV; + extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC; + extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER; + extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH; + extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME; + extern unsigned IOCTL_SOUND_PCM_READ_BITS; + extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS; + extern unsigned IOCTL_SOUND_PCM_READ_FILTER; + extern unsigned IOCTL_SOUND_PCM_READ_RATE; + extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS; + extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER; + extern unsigned IOCTL_VT_ACTIVATE; + extern unsigned IOCTL_VT_GETMODE; + extern unsigned IOCTL_VT_OPENQRY; + extern unsigned IOCTL_VT_RELDISP; + extern unsigned IOCTL_VT_SETMODE; + extern unsigned IOCTL_VT_WAITACTIVE; + extern unsigned IOCTL_GIO_SCRNMAP; + extern unsigned IOCTL_KDDISABIO; + extern unsigned IOCTL_KDENABIO; + extern unsigned IOCTL_KDGETLED; + extern unsigned IOCTL_KDGETMODE; + extern unsigned IOCTL_KDGKBMODE; + extern unsigned IOCTL_KDGKBTYPE; + extern unsigned IOCTL_KDMKTONE; + extern unsigned IOCTL_KDSETLED; + extern unsigned IOCTL_KDSETMODE; + extern unsigned IOCTL_KDSKBMODE; + + extern const int si_SEGV_MAPERR; + extern const int si_SEGV_ACCERR; + + struct __sanitizer_cap_rights { + u64 cr_rights[2]; + }; + + typedef struct __sanitizer_cap_rights __sanitizer_cap_rights_t; + extern unsigned struct_cap_rights_sz; + + extern unsigned struct_fstab_sz; + extern unsigned struct_StringList_sz; +} // namespace __sanitizer + +#define CHECK_TYPE_SIZE(TYPE) \ + COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) + +#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ + offsetof(CLASS, MEMBER)) + +// For sigaction, which is a function and struct at the same time, +// and thus requires explicit "struct" in sizeof() expression. +#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((struct CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ + offsetof(struct CLASS, MEMBER)) + +#define SIGACTION_SYMNAME sigaction + +#endif + +#endif // SANITIZER_FREEBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc index 3c6bb2090fa..c112e044b1d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc @@ -15,129 +15,21 @@ #include "sanitizer_platform.h" #if SANITIZER_NETBSD + +#define _KMEMUSER +#define RAY_DO_SIGLEV + +// clang-format off #include #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include #include #include -#define RAY_DO_SIGLEV -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include +#include #include #include #include @@ -168,11 +60,11 @@ #include #include #include -//#include #include #include #include #include +#include #include #include #include @@ -180,6 +72,8 @@ #include #include #include +#include +#include #include #include #include @@ -195,6 +89,122 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -202,8 +212,15 @@ #include #include #include +#include #include #include +#include +#include +#include +#include +#include +// clang-format on // Include these after system headers to avoid name clashes and ambiguities. #include "sanitizer_internal_defs.h" @@ -238,6 +255,11 @@ unsigned struct_rlimit_sz = sizeof(struct rlimit); unsigned struct_timespec_sz = sizeof(struct timespec); unsigned struct_sembuf_sz = sizeof(struct sembuf); unsigned struct_kevent_sz = sizeof(struct kevent); +unsigned struct_FTS_sz = sizeof(FTS); +unsigned struct_FTSENT_sz = sizeof(FTSENT); +unsigned struct_regex_sz = sizeof(regex_t); +unsigned struct_regmatch_sz = sizeof(regmatch_t); +unsigned struct_fstab_sz = sizeof(struct fstab); unsigned struct_utimbuf_sz = sizeof(struct utimbuf); unsigned struct_itimerspec_sz = sizeof(struct itimerspec); unsigned struct_timex_sz = sizeof(struct timex); @@ -251,6 +273,8 @@ const uptr sig_dfl = (uptr)SIG_DFL; const uptr sig_err = (uptr)SIG_ERR; const uptr sa_siginfo = (uptr)SA_SIGINFO; +const unsigned long __sanitizer_bufsiz = BUFSIZ; + int ptrace_pt_io = PT_IO; int ptrace_pt_lwpinfo = PT_LWPINFO; int ptrace_pt_set_event_mask = PT_SET_EVENT_MASK; @@ -339,6 +363,14 @@ unsigned path_max = PATH_MAX; int struct_ttyent_sz = sizeof(struct ttyent); +struct __sanitizer_nvlist_ref_t { + void *buf; + uptr len; + int flags; +}; + +typedef __sanitizer_nvlist_ref_t nvlist_ref_t; + // ioctl arguments unsigned struct_altqreq_sz = sizeof(altqreq); unsigned struct_amr_user_ioctl_sz = sizeof(amr_user_ioctl); @@ -350,7 +382,6 @@ unsigned struct_atabusiodetach_args_sz = sizeof(atabusiodetach_args); unsigned struct_atabusioscan_args_sz = sizeof(atabusioscan_args); unsigned struct_ath_diag_sz = sizeof(ath_diag); unsigned struct_atm_flowmap_sz = sizeof(atm_flowmap); -unsigned struct_atm_pseudoioctl_sz = sizeof(atm_pseudoioctl); unsigned struct_audio_buf_info_sz = sizeof(audio_buf_info); unsigned struct_audio_device_sz = sizeof(audio_device); unsigned struct_audio_encoding_sz = sizeof(audio_encoding); @@ -596,7 +627,6 @@ unsigned struct_priq_delete_filter_sz = sizeof(priq_delete_filter); unsigned struct_priq_interface_sz = sizeof(priq_interface); unsigned struct_priq_modify_class_sz = sizeof(priq_modify_class); unsigned struct_ptmget_sz = sizeof(ptmget); -unsigned struct_pvctxreq_sz = sizeof(pvctxreq); unsigned struct_radio_info_sz = sizeof(radio_info); unsigned struct_red_conf_sz = sizeof(red_conf); unsigned struct_red_interface_sz = sizeof(red_interface); @@ -608,7 +638,6 @@ unsigned struct_rf_recon_req_sz = sizeof(rf_recon_req); unsigned struct_rio_conf_sz = sizeof(rio_conf); unsigned struct_rio_interface_sz = sizeof(rio_interface); unsigned struct_rio_stats_sz = sizeof(rio_stats); -unsigned struct_satlink_id_sz = sizeof(satlink_id); unsigned struct_scan_io_sz = sizeof(scan_io); unsigned struct_scbusaccel_args_sz = sizeof(scbusaccel_args); unsigned struct_scbusiodetach_args_sz = sizeof(scbusiodetach_args); @@ -656,6 +685,9 @@ unsigned struct_usb_config_desc_sz = sizeof(usb_config_desc); unsigned struct_usb_ctl_report_desc_sz = sizeof(usb_ctl_report_desc); unsigned struct_usb_ctl_report_sz = sizeof(usb_ctl_report); unsigned struct_usb_ctl_request_sz = sizeof(usb_ctl_request); +unsigned struct_autofs_daemon_request_sz = sizeof(autofs_daemon_request); +unsigned struct_autofs_daemon_done_sz = sizeof(autofs_daemon_done); +unsigned struct_sctp_connectx_addrs_sz = sizeof(sctp_connectx_addrs); unsigned struct_usb_device_info_old_sz = sizeof(usb_device_info_old); unsigned struct_usb_device_info_sz = sizeof(usb_device_info); unsigned struct_usb_device_stats_sz = sizeof(usb_device_stats); @@ -797,6 +829,8 @@ unsigned struct_RF_SparetWait_sz = sizeof(RF_SparetWait_t); unsigned struct_RF_ComponentLabel_sz = sizeof(RF_ComponentLabel_t); unsigned struct_RF_SingleComponent_sz = sizeof(RF_SingleComponent_t); unsigned struct_RF_ProgressInfo_sz = sizeof(RF_ProgressInfo_t); +unsigned struct_nvlist_ref_sz = sizeof(struct __sanitizer_nvlist_ref_t); +unsigned struct_StringList_sz = sizeof(StringList); const unsigned IOCTL_NOT_PRESENT = 0; @@ -1061,6 +1095,7 @@ unsigned IOCTL_MLX_REBUILDSTAT = MLX_REBUILDSTAT; unsigned IOCTL_MLX_GET_SYSDRIVE = MLX_GET_SYSDRIVE; unsigned IOCTL_MLX_GET_CINFO = MLX_GET_CINFO; unsigned IOCTL_NVME_PASSTHROUGH_CMD = NVME_PASSTHROUGH_CMD; +unsigned IOCTL_FWCFGIO_SET_INDEX = FWCFGIO_SET_INDEX; unsigned IOCTL_IRDA_RESET_PARAMS = IRDA_RESET_PARAMS; unsigned IOCTL_IRDA_SET_PARAMS = IRDA_SET_PARAMS; unsigned IOCTL_IRDA_GET_SPEEDMASK = IRDA_GET_SPEEDMASK; @@ -1068,9 +1103,6 @@ unsigned IOCTL_IRDA_GET_TURNAROUNDMASK = IRDA_GET_TURNAROUNDMASK; unsigned IOCTL_IRFRAMETTY_GET_DEVICE = IRFRAMETTY_GET_DEVICE; unsigned IOCTL_IRFRAMETTY_GET_DONGLE = IRFRAMETTY_GET_DONGLE; unsigned IOCTL_IRFRAMETTY_SET_DONGLE = IRFRAMETTY_SET_DONGLE; -unsigned IOCTL_SATIORESET = SATIORESET; -unsigned IOCTL_SATIOGID = SATIOGID; -unsigned IOCTL_SATIOSBUFSIZE = SATIOSBUFSIZE; unsigned IOCTL_ISV_CMD = ISV_CMD; unsigned IOCTL_WTQICMD = WTQICMD; unsigned IOCTL_ISCSI_GET_VERSION = ISCSI_GET_VERSION; @@ -1390,6 +1422,24 @@ unsigned IOCTL_SPKRTONE = SPKRTONE; unsigned IOCTL_SPKRTUNE = SPKRTUNE; unsigned IOCTL_SPKRGETVOL = SPKRGETVOL; unsigned IOCTL_SPKRSETVOL = SPKRSETVOL; +#if 0 /* interfaces are WIP */ +unsigned IOCTL_NVMM_IOC_CAPABILITY = NVMM_IOC_CAPABILITY; +unsigned IOCTL_NVMM_IOC_MACHINE_CREATE = NVMM_IOC_MACHINE_CREATE; +unsigned IOCTL_NVMM_IOC_MACHINE_DESTROY = NVMM_IOC_MACHINE_DESTROY; +unsigned IOCTL_NVMM_IOC_MACHINE_CONFIGURE = NVMM_IOC_MACHINE_CONFIGURE; +unsigned IOCTL_NVMM_IOC_VCPU_CREATE = NVMM_IOC_VCPU_CREATE; +unsigned IOCTL_NVMM_IOC_VCPU_DESTROY = NVMM_IOC_VCPU_DESTROY; +unsigned IOCTL_NVMM_IOC_VCPU_SETSTATE = NVMM_IOC_VCPU_SETSTATE; +unsigned IOCTL_NVMM_IOC_VCPU_GETSTATE = NVMM_IOC_VCPU_GETSTATE; +unsigned IOCTL_NVMM_IOC_VCPU_INJECT = NVMM_IOC_VCPU_INJECT; +unsigned IOCTL_NVMM_IOC_VCPU_RUN = NVMM_IOC_VCPU_RUN; +unsigned IOCTL_NVMM_IOC_GPA_MAP = NVMM_IOC_GPA_MAP; +unsigned IOCTL_NVMM_IOC_GPA_UNMAP = NVMM_IOC_GPA_UNMAP; +unsigned IOCTL_NVMM_IOC_HVA_MAP = NVMM_IOC_HVA_MAP; +unsigned IOCTL_NVMM_IOC_HVA_UNMAP = NVMM_IOC_HVA_UNMAP; +#endif +unsigned IOCTL_AUTOFSREQUEST = AUTOFSREQUEST; +unsigned IOCTL_AUTOFSDONE = AUTOFSDONE; unsigned IOCTL_BIOCGBLEN = BIOCGBLEN; unsigned IOCTL_BIOCSBLEN = BIOCSBLEN; unsigned IOCTL_BIOCSETF = BIOCSETF; @@ -1408,19 +1458,12 @@ unsigned IOCTL_BIOCGHDRCMPLT = BIOCGHDRCMPLT; unsigned IOCTL_BIOCSHDRCMPLT = BIOCSHDRCMPLT; unsigned IOCTL_BIOCSDLT = BIOCSDLT; unsigned IOCTL_BIOCGDLTLIST = BIOCGDLTLIST; -unsigned IOCTL_BIOCGSEESENT = BIOCGSEESENT; -unsigned IOCTL_BIOCSSEESENT = BIOCSSEESENT; +unsigned IOCTL_BIOCGDIRECTION = BIOCGDIRECTION; +unsigned IOCTL_BIOCSDIRECTION = BIOCSDIRECTION; unsigned IOCTL_BIOCSRTIMEOUT = BIOCSRTIMEOUT; unsigned IOCTL_BIOCGRTIMEOUT = BIOCGRTIMEOUT; unsigned IOCTL_BIOCGFEEDBACK = BIOCGFEEDBACK; unsigned IOCTL_BIOCSFEEDBACK = BIOCSFEEDBACK; -unsigned IOCTL_SIOCRAWATM = SIOCRAWATM; -unsigned IOCTL_SIOCATMENA = SIOCATMENA; -unsigned IOCTL_SIOCATMDIS = SIOCATMDIS; -unsigned IOCTL_SIOCSPVCTX = SIOCSPVCTX; -unsigned IOCTL_SIOCGPVCTX = SIOCGPVCTX; -unsigned IOCTL_SIOCSPVCSIF = SIOCSPVCSIF; -unsigned IOCTL_SIOCGPVCSIF = SIOCGPVCSIF; unsigned IOCTL_GRESADDRS = GRESADDRS; unsigned IOCTL_GRESADDRD = GRESADDRD; unsigned IOCTL_GREGADDRS = GREGADDRS; @@ -1575,6 +1618,8 @@ unsigned IOCTL_SIOCRMNAT = SIOCRMNAT; unsigned IOCTL_SIOCGNATS = SIOCGNATS; unsigned IOCTL_SIOCGNATL = SIOCGNATL; unsigned IOCTL_SIOCPURGENAT = SIOCPURGENAT; +unsigned IOCTL_SIOCCONNECTX = SIOCCONNECTX; +unsigned IOCTL_SIOCCONNECTXDEL = SIOCCONNECTXDEL; unsigned IOCTL_SIOCSIFINFO_FLAGS = SIOCSIFINFO_FLAGS; unsigned IOCTL_SIOCAADDRCTL_POLICY = SIOCAADDRCTL_POLICY; unsigned IOCTL_SIOCDADDRCTL_POLICY = SIOCDADDRCTL_POLICY; @@ -1719,6 +1764,8 @@ unsigned IOCTL_FDIOCGETFORMAT = FDIOCGETFORMAT; unsigned IOCTL_FDIOCFORMAT_TRACK = FDIOCFORMAT_TRACK; unsigned IOCTL_FIOCLEX = FIOCLEX; unsigned IOCTL_FIONCLEX = FIONCLEX; +unsigned IOCTL_FIOSEEKDATA = FIOSEEKDATA; +unsigned IOCTL_FIOSEEKHOLE = FIOSEEKHOLE; unsigned IOCTL_FIONREAD = FIONREAD; unsigned IOCTL_FIONBIO = FIONBIO; unsigned IOCTL_FIOASYNC = FIOASYNC; @@ -1804,8 +1851,6 @@ unsigned IOCTL_MTIOCSLOCATE = MTIOCSLOCATE; unsigned IOCTL_MTIOCHLOCATE = MTIOCHLOCATE; unsigned IOCTL_POWER_EVENT_RECVDICT = POWER_EVENT_RECVDICT; unsigned IOCTL_POWER_IOC_GET_TYPE = POWER_IOC_GET_TYPE; -unsigned IOCTL_POWER_IOC_GET_TYPE_WITH_LOSSAGE = - POWER_IOC_GET_TYPE_WITH_LOSSAGE; unsigned IOCTL_RIOCGINFO = RIOCGINFO; unsigned IOCTL_RIOCSINFO = RIOCSINFO; unsigned IOCTL_RIOCSSRCH = RIOCSSRCH; @@ -1840,6 +1885,7 @@ unsigned IOCTL_SIOCGLOWAT = SIOCGLOWAT; unsigned IOCTL_SIOCATMARK = SIOCATMARK; unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; +unsigned IOCTL_SIOCPEELOFF = SIOCPEELOFF; unsigned IOCTL_SIOCADDRT = SIOCADDRT; unsigned IOCTL_SIOCDELRT = SIOCDELRT; unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; @@ -1897,6 +1943,10 @@ unsigned IOCTL_SIOCGLINKSTR = SIOCGLINKSTR; unsigned IOCTL_SIOCSLINKSTR = SIOCSLINKSTR; unsigned IOCTL_SIOCGETHERCAP = SIOCGETHERCAP; unsigned IOCTL_SIOCGIFINDEX = SIOCGIFINDEX; +unsigned IOCTL_SIOCSETHERCAP = SIOCSETHERCAP; +unsigned IOCTL_SIOCGUMBINFO = SIOCGUMBINFO; +unsigned IOCTL_SIOCSUMBPARAM = SIOCSUMBPARAM; +unsigned IOCTL_SIOCGUMBPARAM = SIOCGUMBPARAM; unsigned IOCTL_SIOCSETPFSYNC = SIOCSETPFSYNC; unsigned IOCTL_SIOCGETPFSYNC = SIOCGETPFSYNC; unsigned IOCTL_PPS_IOC_CREATE = PPS_IOC_CREATE; @@ -2063,6 +2113,44 @@ unsigned IOCTL_SNDCTL_DSP_SILENCE = SNDCTL_DSP_SILENCE; const int si_SEGV_MAPERR = SEGV_MAPERR; const int si_SEGV_ACCERR = SEGV_ACCERR; + +const int modctl_load = MODCTL_LOAD; +const int modctl_unload = MODCTL_UNLOAD; +const int modctl_stat = MODCTL_STAT; +const int modctl_exists = MODCTL_EXISTS; + +const unsigned SHA1_CTX_sz = sizeof(SHA1_CTX); +const unsigned SHA1_return_length = SHA1_DIGEST_STRING_LENGTH; + +const unsigned MD4_CTX_sz = sizeof(MD4_CTX); +const unsigned MD4_return_length = MD4_DIGEST_STRING_LENGTH; + +const unsigned RMD160_CTX_sz = sizeof(RMD160_CTX); +const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH; + +const unsigned MD5_CTX_sz = sizeof(MD5_CTX); +const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH; + +const unsigned fpos_t_sz = sizeof(fpos_t); + +const unsigned MD2_CTX_sz = sizeof(MD2_CTX); +const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH; + +#define SHA2_CONST(LEN) \ + const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX); \ + const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \ + const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH; \ + const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH + +SHA2_CONST(224); +SHA2_CONST(256); +SHA2_CONST(384); +SHA2_CONST(512); + +#undef SHA2_CONST + +const int unvis_valid = UNVIS_VALID; +const int unvis_validpush = UNVIS_VALIDPUSH; } // namespace __sanitizer using namespace __sanitizer; @@ -2153,6 +2241,29 @@ CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); +COMPILER_CHECK(sizeof(__sanitizer_FILE) <= sizeof(FILE)); +CHECK_SIZE_AND_OFFSET(FILE, _p); +CHECK_SIZE_AND_OFFSET(FILE, _r); +CHECK_SIZE_AND_OFFSET(FILE, _w); +CHECK_SIZE_AND_OFFSET(FILE, _flags); +CHECK_SIZE_AND_OFFSET(FILE, _file); +CHECK_SIZE_AND_OFFSET(FILE, _bf); +CHECK_SIZE_AND_OFFSET(FILE, _lbfsize); +CHECK_SIZE_AND_OFFSET(FILE, _cookie); +CHECK_SIZE_AND_OFFSET(FILE, _close); +CHECK_SIZE_AND_OFFSET(FILE, _read); +CHECK_SIZE_AND_OFFSET(FILE, _seek); +CHECK_SIZE_AND_OFFSET(FILE, _write); +CHECK_SIZE_AND_OFFSET(FILE, _ext); +CHECK_SIZE_AND_OFFSET(FILE, _up); +CHECK_SIZE_AND_OFFSET(FILE, _ur); +CHECK_SIZE_AND_OFFSET(FILE, _ubuf); +CHECK_SIZE_AND_OFFSET(FILE, _nbuf); +CHECK_SIZE_AND_OFFSET(FILE, _flush); +CHECK_SIZE_AND_OFFSET(FILE, _lb_unused); +CHECK_SIZE_AND_OFFSET(FILE, _blksize); +CHECK_SIZE_AND_OFFSET(FILE, _offset); + CHECK_TYPE_SIZE(tm); CHECK_SIZE_AND_OFFSET(tm, tm_sec); CHECK_SIZE_AND_OFFSET(tm, tm_min); @@ -2224,4 +2335,10 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd); CHECK_SIZE_AND_OFFSET(group, gr_gid); CHECK_SIZE_AND_OFFSET(group, gr_mem); +CHECK_TYPE_SIZE(modctl_load_t); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_filename); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_flags); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_props); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_propslen); + #endif // SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h index 1718e3b1e8e..594cfa6c0d4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -25,10 +25,10 @@ #if defined(__x86_64__) #define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ - _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312) + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 264) #elif defined(__i386__) #define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ - _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164) + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 136) #endif namespace __sanitizer { @@ -60,6 +60,29 @@ extern unsigned struct_timespec_sz; extern unsigned struct_sembuf_sz; extern unsigned struct_kevent_sz; +extern unsigned struct_FTS_sz; +extern unsigned struct_FTSENT_sz; + +extern unsigned struct_regex_sz; +extern unsigned struct_regmatch_sz; + +extern unsigned struct_fstab_sz; + +struct __sanitizer_regmatch { + OFF_T rm_so; + OFF_T rm_eo; +}; + +typedef struct __sanitizer_modctl_load { + const char *ml_filename; + int ml_flags; + const char *ml_props; + uptr ml_propslen; +} __sanitizer_modctl_load_t; +extern const int modctl_load; +extern const int modctl_unload; +extern const int modctl_stat; +extern const int modctl_exists; union __sanitizer_sigval { int sival_int; @@ -425,8 +448,36 @@ struct __sanitizer_wordexp_t { uptr we_nbytes; }; -typedef char __sanitizer_FILE; -#define SANITIZER_HAS_STRUCT_FILE 0 +struct __sanitizer_FILE { + unsigned char *_p; + int _r; + int _w; + unsigned short _flags; + short _file; + struct { + unsigned char *_base; + int _size; + } _bf; + int _lbfsize; + void *_cookie; + int (*_close)(void *ptr); + u64 (*_read)(void *, void *, uptr); + u64 (*_seek)(void *, u64, int); + uptr (*_write)(void *, const void *, uptr); + struct { + unsigned char *_base; + int _size; + } _ext; + unsigned char *_up; + int _ur; + unsigned char _ubuf[3]; + unsigned char _nbuf[1]; + int (*_flush)(void *ptr); + char _lb_unused[sizeof(uptr)]; + int _blksize; + u64 _offset; +}; +#define SANITIZER_HAS_STRUCT_FILE 1 extern int shmctl_ipc_stat; @@ -460,6 +511,8 @@ struct __sanitizer_ttyent { char *ty_class; }; +extern const unsigned long __sanitizer_bufsiz; + #define IOC_NRBITS 8 #define IOC_TYPEBITS 8 #define IOC_SIZEBITS 14 @@ -496,7 +549,6 @@ extern unsigned struct_atabusiodetach_args_sz; extern unsigned struct_atabusioscan_args_sz; extern unsigned struct_ath_diag_sz; extern unsigned struct_atm_flowmap_sz; -extern unsigned struct_atm_pseudoioctl_sz; extern unsigned struct_audio_buf_info_sz; extern unsigned struct_audio_device_sz; extern unsigned struct_audio_encoding_sz; @@ -751,7 +803,6 @@ extern unsigned struct_rf_recon_req_sz; extern unsigned struct_rio_conf_sz; extern unsigned struct_rio_interface_sz; extern unsigned struct_rio_stats_sz; -extern unsigned struct_satlink_id_sz; extern unsigned struct_scan_io_sz; extern unsigned struct_scbusaccel_args_sz; extern unsigned struct_scbusiodetach_args_sz; @@ -799,6 +850,9 @@ extern unsigned struct_usb_config_desc_sz; extern unsigned struct_usb_ctl_report_desc_sz; extern unsigned struct_usb_ctl_report_sz; extern unsigned struct_usb_ctl_request_sz; +extern unsigned struct_autofs_daemon_request_sz; +extern unsigned struct_autofs_daemon_done_sz; +extern unsigned struct_sctp_connectx_addrs_sz; extern unsigned struct_usb_device_info_old_sz; extern unsigned struct_usb_device_info_sz; extern unsigned struct_usb_device_stats_sz; @@ -933,6 +987,8 @@ extern unsigned struct_RF_SparetWait_sz; extern unsigned struct_RF_ComponentLabel_sz; extern unsigned struct_RF_SingleComponent_sz; extern unsigned struct_RF_ProgressInfo_sz; +extern unsigned struct_nvlist_ref_sz; +extern unsigned struct_StringList_sz; // A special value to mark ioctls that are not present on the target platform, @@ -1201,6 +1257,7 @@ extern unsigned IOCTL_MLX_REBUILDSTAT; extern unsigned IOCTL_MLX_GET_SYSDRIVE; extern unsigned IOCTL_MLX_GET_CINFO; extern unsigned IOCTL_NVME_PASSTHROUGH_CMD; +extern unsigned IOCTL_FWCFGIO_SET_INDEX; extern unsigned IOCTL_IRDA_RESET_PARAMS; extern unsigned IOCTL_IRDA_SET_PARAMS; extern unsigned IOCTL_IRDA_GET_SPEEDMASK; @@ -1208,9 +1265,6 @@ extern unsigned IOCTL_IRDA_GET_TURNAROUNDMASK; extern unsigned IOCTL_IRFRAMETTY_GET_DEVICE; extern unsigned IOCTL_IRFRAMETTY_GET_DONGLE; extern unsigned IOCTL_IRFRAMETTY_SET_DONGLE; -extern unsigned IOCTL_SATIORESET; -extern unsigned IOCTL_SATIOGID; -extern unsigned IOCTL_SATIOSBUFSIZE; extern unsigned IOCTL_ISV_CMD; extern unsigned IOCTL_WTQICMD; extern unsigned IOCTL_ISCSI_GET_VERSION; @@ -1522,6 +1576,24 @@ extern unsigned IOCTL_SPKRTONE; extern unsigned IOCTL_SPKRTUNE; extern unsigned IOCTL_SPKRGETVOL; extern unsigned IOCTL_SPKRSETVOL; +#if 0 /* interfaces are WIP */ +extern unsigned IOCTL_NVMM_IOC_CAPABILITY; +extern unsigned IOCTL_NVMM_IOC_MACHINE_CREATE; +extern unsigned IOCTL_NVMM_IOC_MACHINE_DESTROY; +extern unsigned IOCTL_NVMM_IOC_MACHINE_CONFIGURE; +extern unsigned IOCTL_NVMM_IOC_VCPU_CREATE; +extern unsigned IOCTL_NVMM_IOC_VCPU_DESTROY; +extern unsigned IOCTL_NVMM_IOC_VCPU_SETSTATE; +extern unsigned IOCTL_NVMM_IOC_VCPU_GETSTATE; +extern unsigned IOCTL_NVMM_IOC_VCPU_INJECT; +extern unsigned IOCTL_NVMM_IOC_VCPU_RUN; +extern unsigned IOCTL_NVMM_IOC_GPA_MAP; +extern unsigned IOCTL_NVMM_IOC_GPA_UNMAP; +extern unsigned IOCTL_NVMM_IOC_HVA_MAP; +extern unsigned IOCTL_NVMM_IOC_HVA_UNMAP; +#endif +extern unsigned IOCTL_AUTOFSREQUEST; +extern unsigned IOCTL_AUTOFSDONE; extern unsigned IOCTL_BIOCGBLEN; extern unsigned IOCTL_BIOCSBLEN; extern unsigned IOCTL_BIOCSETF; @@ -1540,19 +1612,12 @@ extern unsigned IOCTL_BIOCGHDRCMPLT; extern unsigned IOCTL_BIOCSHDRCMPLT; extern unsigned IOCTL_BIOCSDLT; extern unsigned IOCTL_BIOCGDLTLIST; -extern unsigned IOCTL_BIOCGSEESENT; -extern unsigned IOCTL_BIOCSSEESENT; +extern unsigned IOCTL_BIOCGDIRECTION; +extern unsigned IOCTL_BIOCSDIRECTION; extern unsigned IOCTL_BIOCSRTIMEOUT; extern unsigned IOCTL_BIOCGRTIMEOUT; extern unsigned IOCTL_BIOCGFEEDBACK; extern unsigned IOCTL_BIOCSFEEDBACK; -extern unsigned IOCTL_SIOCRAWATM; -extern unsigned IOCTL_SIOCATMENA; -extern unsigned IOCTL_SIOCATMDIS; -extern unsigned IOCTL_SIOCSPVCTX; -extern unsigned IOCTL_SIOCGPVCTX; -extern unsigned IOCTL_SIOCSPVCSIF; -extern unsigned IOCTL_SIOCGPVCSIF; extern unsigned IOCTL_GRESADDRS; extern unsigned IOCTL_GRESADDRD; extern unsigned IOCTL_GREGADDRS; @@ -1707,6 +1772,8 @@ extern unsigned IOCTL_SIOCRMNAT; extern unsigned IOCTL_SIOCGNATS; extern unsigned IOCTL_SIOCGNATL; extern unsigned IOCTL_SIOCPURGENAT; +extern unsigned IOCTL_SIOCCONNECTX; +extern unsigned IOCTL_SIOCCONNECTXDEL; extern unsigned IOCTL_SIOCSIFINFO_FLAGS; extern unsigned IOCTL_SIOCAADDRCTL_POLICY; extern unsigned IOCTL_SIOCDADDRCTL_POLICY; @@ -1851,6 +1918,8 @@ extern unsigned IOCTL_FDIOCGETFORMAT; extern unsigned IOCTL_FDIOCFORMAT_TRACK; extern unsigned IOCTL_FIOCLEX; extern unsigned IOCTL_FIONCLEX; +extern unsigned IOCTL_FIOSEEKDATA; +extern unsigned IOCTL_FIOSEEKHOLE; extern unsigned IOCTL_FIONREAD; extern unsigned IOCTL_FIONBIO; extern unsigned IOCTL_FIOASYNC; @@ -1936,7 +2005,6 @@ extern unsigned IOCTL_MTIOCSLOCATE; extern unsigned IOCTL_MTIOCHLOCATE; extern unsigned IOCTL_POWER_EVENT_RECVDICT; extern unsigned IOCTL_POWER_IOC_GET_TYPE; -extern unsigned IOCTL_POWER_IOC_GET_TYPE_WITH_LOSSAGE; extern unsigned IOCTL_RIOCGINFO; extern unsigned IOCTL_RIOCSINFO; extern unsigned IOCTL_RIOCSSRCH; @@ -1971,6 +2039,7 @@ extern unsigned IOCTL_SIOCGLOWAT; extern unsigned IOCTL_SIOCATMARK; extern unsigned IOCTL_SIOCSPGRP; extern unsigned IOCTL_SIOCGPGRP; +extern unsigned IOCTL_SIOCPEELOFF; extern unsigned IOCTL_SIOCADDRT; extern unsigned IOCTL_SIOCDELRT; extern unsigned IOCTL_SIOCSIFADDR; @@ -2028,6 +2097,10 @@ extern unsigned IOCTL_SIOCGLINKSTR; extern unsigned IOCTL_SIOCSLINKSTR; extern unsigned IOCTL_SIOCGETHERCAP; extern unsigned IOCTL_SIOCGIFINDEX; +extern unsigned IOCTL_SIOCSETHERCAP; +extern unsigned IOCTL_SIOCGUMBINFO; +extern unsigned IOCTL_SIOCSUMBPARAM; +extern unsigned IOCTL_SIOCGUMBPARAM; extern unsigned IOCTL_SIOCSETPFSYNC; extern unsigned IOCTL_SIOCGETPFSYNC; extern unsigned IOCTL_PPS_IOC_CREATE; @@ -2194,6 +2267,74 @@ extern unsigned IOCTL_SNDCTL_DSP_SILENCE; extern const int si_SEGV_MAPERR; extern const int si_SEGV_ACCERR; + +extern const unsigned SHA1_CTX_sz; +extern const unsigned SHA1_return_length; + +extern const unsigned MD4_CTX_sz; +extern const unsigned MD4_return_length; + +extern const unsigned RMD160_CTX_sz; +extern const unsigned RMD160_return_length; + +extern const unsigned MD5_CTX_sz; +extern const unsigned MD5_return_length; + +extern const unsigned fpos_t_sz; + +extern const unsigned MD2_CTX_sz; +extern const unsigned MD2_return_length; + +#define SHA2_EXTERN(LEN) \ + extern const unsigned SHA##LEN##_CTX_sz; \ + extern const unsigned SHA##LEN##_return_length; \ + extern const unsigned SHA##LEN##_block_length; \ + extern const unsigned SHA##LEN##_digest_length + +SHA2_EXTERN(224); +SHA2_EXTERN(256); +SHA2_EXTERN(384); +SHA2_EXTERN(512); + +#undef SHA2_EXTERN + +extern const int unvis_valid; +extern const int unvis_validpush; + +struct __sanitizer_cdbr { + void (*unmap)(void *, void *, uptr); + void *cookie; + u8 *mmap_base; + uptr mmap_size; + + u8 *hash_base; + u8 *offset_base; + u8 *data_base; + + u32 data_size; + u32 entries; + u32 entries_index; + u32 seed; + + u8 offset_size; + u8 index_size; + + u32 entries_m; + u32 entries_index_m; + u8 entries_s1, entries_s2; + u8 entries_index_s1, entries_index_s2; +}; + +struct __sanitizer_cdbw { + uptr data_counter; + uptr data_allocated; + uptr data_size; + uptr *data_len; + void **data_ptr; + uptr hash_size; + void *hash; + uptr key_counter; +}; } // namespace __sanitizer #define CHECK_TYPE_SIZE(TYPE) \ diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index 54da635d71f..ecc69bcea79 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -14,7 +14,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_MAC // Tests in this file assume that off_t-dependent data structures match the // libc ABI. For example, struct dirent here is what readdir() function (as // exported from libc) returns, and not the user-facing "dirent", which @@ -45,7 +45,8 @@ #include #include #include -#if !SANITIZER_MAC && !SANITIZER_FREEBSD +#include +#if !SANITIZER_MAC #include #endif @@ -54,6 +55,7 @@ #endif #if !SANITIZER_ANDROID +#include #include #include #include @@ -78,43 +80,11 @@ #include #endif -#if SANITIZER_FREEBSD -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -#define _KERNEL // to declare 'shminfo' structure -# include -#undef _KERNEL - -#undef INLINE // to avoid clashes with sanitizers' definitions -#endif - -#if SANITIZER_FREEBSD || SANITIZER_IOS +#if SANITIZER_IOS #undef IOC_DIRMASK #endif -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX # include # include # if defined(__mips64) || defined(__aarch64__) || defined(__arm__) @@ -146,8 +116,6 @@ typedef struct user_fpregs elf_fpregset_t; #include #if HAVE_RPC_XDR_H # include -#elif HAVE_TIRPC_RPC_XDR_H -# include #endif #include #include @@ -198,9 +166,9 @@ typedef struct user_fpregs elf_fpregset_t; namespace __sanitizer { unsigned struct_utsname_sz = sizeof(struct utsname); unsigned struct_stat_sz = sizeof(struct stat); -#if !SANITIZER_IOS && !SANITIZER_FREEBSD +#if !SANITIZER_IOS unsigned struct_stat64_sz = sizeof(struct stat64); -#endif // !SANITIZER_IOS && !SANITIZER_FREEBSD +#endif // !SANITIZER_IOS unsigned struct_rusage_sz = sizeof(struct rusage); unsigned struct_tm_sz = sizeof(struct tm); unsigned struct_passwd_sz = sizeof(struct passwd); @@ -221,13 +189,15 @@ namespace __sanitizer { unsigned struct_tms_sz = sizeof(struct tms); unsigned struct_sigevent_sz = sizeof(struct sigevent); unsigned struct_sched_param_sz = sizeof(struct sched_param); - + unsigned struct_regex_sz = sizeof(regex_t); + unsigned struct_regmatch_sz = sizeof(regmatch_t); #if SANITIZER_MAC && !SANITIZER_IOS unsigned struct_statfs64_sz = sizeof(struct statfs64); #endif // SANITIZER_MAC && !SANITIZER_IOS #if !SANITIZER_ANDROID + unsigned struct_fstab_sz = sizeof(struct fstab); unsigned struct_statfs_sz = sizeof(struct statfs); unsigned struct_sockaddr_sz = sizeof(struct sockaddr); unsigned ucontext_t_sz = sizeof(ucontext_t); @@ -244,12 +214,12 @@ namespace __sanitizer { unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname); #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX unsigned struct_rlimit_sz = sizeof(struct rlimit); unsigned struct_timespec_sz = sizeof(struct timespec); unsigned struct_utimbuf_sz = sizeof(struct utimbuf); unsigned struct_itimerspec_sz = sizeof(struct itimerspec); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID // Use pre-computed size of struct ustat to avoid which @@ -269,12 +239,12 @@ namespace __sanitizer { unsigned struct_statvfs64_sz = sizeof(struct statvfs64); #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_timex_sz = sizeof(struct timex); unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); unsigned struct_mq_attr_sz = sizeof(struct mq_attr); unsigned struct_statvfs_sz = sizeof(struct statvfs); -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID const uptr sig_ign = (uptr)SIG_IGN; const uptr sig_dfl = (uptr)SIG_DFL; @@ -286,7 +256,7 @@ namespace __sanitizer { #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_shminfo_sz = sizeof(struct shminfo); unsigned struct_shm_info_sz = sizeof(struct shm_info); int shmctl_ipc_stat = (int)IPC_STAT; @@ -322,7 +292,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr)); unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID int glob_nomatch = GLOB_NOMATCH; int glob_altdirfunc = GLOB_ALTDIRFUNC; #endif @@ -447,7 +417,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_vt_stat_sz = sizeof(struct vt_stat); #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX #if SOUND_VERSION >= 0x040000 unsigned struct_copr_buffer_sz = 0; unsigned struct_copr_debug_buf_sz = 0; @@ -464,7 +434,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec); unsigned struct_synth_info_sz = sizeof(struct synth_info); unsigned struct_vt_mode_sz = sizeof(struct vt_mode); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct); @@ -491,7 +461,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_unimapinit_sz = sizeof(struct unimapinit); #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID @@ -501,6 +471,8 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); #endif + const unsigned long __sanitizer_bufsiz = BUFSIZ; + const unsigned IOCTL_NOT_PRESENT = 0; unsigned IOCTL_FIOASYNC = FIOASYNC; @@ -547,7 +519,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; unsigned IOCTL_TIOCSTI = TIOCSTI; unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; -#if ((SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID) +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; #endif @@ -737,9 +709,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_VT_RESIZE = VT_RESIZE; unsigned IOCTL_VT_RESIZEX = VT_RESIZEX; unsigned IOCTL_VT_SENDSIG = VT_SENDSIG; -#endif // SANITIZER_LINUX - -#if SANITIZER_LINUX || SANITIZER_FREEBSD unsigned IOCTL_MTIOCGET = MTIOCGET; unsigned IOCTL_MTIOCTOP = MTIOCTOP; unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE; @@ -832,7 +801,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_VT_RELDISP = VT_RELDISP; unsigned IOCTL_VT_SETMODE = VT_SETMODE; unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_CYGETDEFTHRESH = CYGETDEFTHRESH; @@ -925,7 +894,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_TIOCSSERIAL = TIOCSSERIAL; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP; unsigned IOCTL_KDDISABIO = KDDISABIO; unsigned IOCTL_KDENABIO = KDENABIO; @@ -1207,7 +1176,7 @@ CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); #endif #if SANITIZER_LINUX -COMPILER_CHECK(sizeof(__sanitizer_mallinfo) == sizeof(struct mallinfo)); +COMPILER_CHECK(sizeof(__sanitizer_struct_mallinfo) == sizeof(struct mallinfo)); #endif #if !SANITIZER_ANDROID @@ -1243,7 +1212,7 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd); CHECK_SIZE_AND_OFFSET(group, gr_gid); CHECK_SIZE_AND_OFFSET(group, gr_mem); -#if HAVE_RPC_XDR_H || HAVE_TIRPC_RPC_XDR_H +#if HAVE_RPC_XDR_H CHECK_TYPE_SIZE(XDR); CHECK_SIZE_AND_OFFSET(XDR, x_op); CHECK_SIZE_AND_OFFSET(XDR, x_ops); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 476b9be736a..bc4ddce0a81 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -15,22 +15,12 @@ #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H -#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_MAC #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD -// FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that -// incorporates the map structure. -# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ - ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 560))) -// Get sys/_types.h, because that tells us whether 64-bit inodes are -// used in struct dirent below. -#include -#else # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) ((link_map*)(handle)) -#endif // !SANITIZER_FREEBSD #ifndef __GLIBC_PREREQ #define __GLIBC_PREREQ(x, y) 0 @@ -39,7 +29,7 @@ namespace __sanitizer { extern unsigned struct_utsname_sz; extern unsigned struct_stat_sz; -#if !SANITIZER_FREEBSD && !SANITIZER_IOS +#if !SANITIZER_IOS extern unsigned struct_stat64_sz; #endif extern unsigned struct_rusage_sz; @@ -59,8 +49,11 @@ namespace __sanitizer { extern unsigned struct_sigevent_sz; extern unsigned struct_sched_param_sz; extern unsigned struct_statfs64_sz; + extern unsigned struct_regex_sz; + extern unsigned struct_regmatch_sz; #if !SANITIZER_ANDROID + extern unsigned struct_fstab_sz; extern unsigned struct_statfs_sz; extern unsigned struct_sockaddr_sz; extern unsigned ucontext_t_sz; @@ -127,7 +120,7 @@ namespace __sanitizer { const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long); #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX #if defined(__powerpc64__) || defined(__riscv) || defined(__s390__) const unsigned struct___old_kernel_stat_sz = 0; @@ -184,20 +177,18 @@ namespace __sanitizer { int data; #elif SANITIZER_LINUX uptr data[4]; -#elif SANITIZER_FREEBSD - u32 data[4]; #endif }; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_ANDROID - struct __sanitizer_mallinfo { + struct __sanitizer_struct_mallinfo { uptr v[10]; }; #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID - struct __sanitizer_mallinfo { + struct __sanitizer_struct_mallinfo { int v[10]; }; @@ -310,35 +301,14 @@ namespace __sanitizer { #endif #endif }; -#elif SANITIZER_FREEBSD - struct __sanitizer_ipc_perm { - unsigned int cuid; - unsigned int cgid; - unsigned int uid; - unsigned int gid; - unsigned short mode; - unsigned short seq; - long key; - }; - - struct __sanitizer_shmid_ds { - __sanitizer_ipc_perm shm_perm; - unsigned long shm_segsz; - unsigned int shm_lpid; - unsigned int shm_cpid; - int shm_nattch; - unsigned long shm_atime; - unsigned long shm_dtime; - unsigned long shm_ctime; - }; #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_msqid_ds_sz; extern unsigned struct_mq_attr_sz; extern unsigned struct_timex_sz; extern unsigned struct_statvfs_sz; -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_iovec { void *iov_base; @@ -388,7 +358,7 @@ namespace __sanitizer { char *pw_passwd; int pw_uid; int pw_gid; -#if SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_MAC long pw_change; char *pw_class; #endif @@ -397,11 +367,8 @@ namespace __sanitizer { #endif char *pw_dir; char *pw_shell; -#if SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_MAC long pw_expire; -#endif -#if SANITIZER_FREEBSD - int pw_fields; #endif }; @@ -472,7 +439,7 @@ namespace __sanitizer { }; #endif -#if SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_MAC struct __sanitizer_msghdr { void *msg_name; unsigned msg_namelen; @@ -518,17 +485,6 @@ namespace __sanitizer { unsigned short d_reclen; // more fields that we don't care about }; -#elif SANITIZER_FREEBSD - struct __sanitizer_dirent { -#if defined(__INO64) - unsigned long long d_fileno; - unsigned long long d_off; -#else - unsigned int d_fileno; -#endif - unsigned short d_reclen; - // more fields that we don't care about - }; #elif SANITIZER_ANDROID || defined(__x86_64__) struct __sanitizer_dirent { unsigned long long d_ino; @@ -554,20 +510,17 @@ namespace __sanitizer { }; #endif -// 'clock_t' is 32 bits wide on x64 FreeBSD -#if SANITIZER_FREEBSD - typedef int __sanitizer_clock_t; -#elif defined(__x86_64__) && !defined(_LP64) +#if defined(__x86_64__) && !defined(_LP64) typedef long long __sanitizer_clock_t; #else typedef long __sanitizer_clock_t; #endif -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX typedef int __sanitizer_clockid_t; #endif -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX #if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__)\ || defined(__mips__) typedef unsigned __sanitizer___kernel_uid_t; @@ -617,11 +570,6 @@ namespace __sanitizer { // The size is determined by looking at sizeof of real sigset_t on linux. uptr val[128 / sizeof(uptr)]; }; -#elif SANITIZER_FREEBSD - struct __sanitizer_sigset_t { - // uint32_t * 4 - unsigned int __bits[4]; - }; #endif struct __sanitizer_siginfo { @@ -711,9 +659,7 @@ namespace __sanitizer { }; #endif // !SANITIZER_ANDROID -#if SANITIZER_FREEBSD - typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; -#elif defined(__mips__) +#if defined(__mips__) struct __sanitizer_kernel_sigset_t { uptr sig[2]; }; @@ -759,7 +705,7 @@ namespace __sanitizer { extern int af_inet6; uptr __sanitizer_in_addr_sz(int af); -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX struct __sanitizer_dl_phdr_info { uptr dlpi_addr; const char *dlpi_name; @@ -775,7 +721,7 @@ namespace __sanitizer { int ai_family; int ai_socktype; int ai_protocol; -#if SANITIZER_ANDROID || SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_ANDROID || SANITIZER_MAC unsigned ai_addrlen; char *ai_canonname; void *ai_addr; @@ -801,7 +747,7 @@ namespace __sanitizer { short revents; }; -#if SANITIZER_ANDROID || SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_ANDROID || SANITIZER_MAC typedef unsigned __sanitizer_nfds_t; #else typedef unsigned long __sanitizer_nfds_t; @@ -821,23 +767,9 @@ namespace __sanitizer { int (*gl_lstat)(const char *, void *); int (*gl_stat)(const char *, void *); }; -# elif SANITIZER_FREEBSD - struct __sanitizer_glob_t { - uptr gl_pathc; - uptr gl_matchc; - uptr gl_offs; - int gl_flags; - char **gl_pathv; - int (*gl_errfunc)(const char*, int); - void (*gl_closedir)(void *dirp); - struct dirent *(*gl_readdir)(void *dirp); - void *(*gl_opendir)(const char*); - int (*gl_lstat)(const char*, void* /* struct stat* */); - int (*gl_stat)(const char*, void* /* struct stat* */); - }; -# endif // SANITIZER_FREEBSD +# endif // SANITIZER_LINUX -# if SANITIZER_LINUX || SANITIZER_FREEBSD +# if SANITIZER_LINUX extern int glob_nomatch; extern int glob_altdirfunc; # endif @@ -849,10 +781,6 @@ namespace __sanitizer { uptr we_wordc; char **we_wordv; uptr we_offs; -#if SANITIZER_FREEBSD - char *we_strings; - uptr we_nbytes; -#endif }; #if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -906,7 +834,7 @@ namespace __sanitizer { extern int ptrace_geteventmsg; #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_shminfo_sz; extern unsigned struct_shm_info_sz; extern int shmctl_ipc_stat; @@ -1043,7 +971,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned struct_vt_stat_sz; #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX extern unsigned struct_copr_buffer_sz; extern unsigned struct_copr_debug_buf_sz; extern unsigned struct_copr_msg_sz; @@ -1055,7 +983,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned struct_seq_event_rec_sz; extern unsigned struct_synth_info_sz; extern unsigned struct_vt_mode_sz; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_ax25_parms_struct_sz; @@ -1077,7 +1005,9 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned struct_unimapinit_sz; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID + extern const unsigned long __sanitizer_bufsiz; + +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_audio_buf_info_sz; extern unsigned struct_ppp_stats_sz; #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID @@ -1137,7 +1067,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_TIOCSPGRP; extern unsigned IOCTL_TIOCSTI; extern unsigned IOCTL_TIOCSWINSZ; -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned IOCTL_SIOCGETSGCNT; extern unsigned IOCTL_SIOCGETVIFCNT; #endif @@ -1299,8 +1229,6 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_VT_RESIZE; extern unsigned IOCTL_VT_RESIZEX; extern unsigned IOCTL_VT_SENDSIG; -#endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD extern unsigned IOCTL_MTIOCGET; extern unsigned IOCTL_MTIOCTOP; extern unsigned IOCTL_SIOCADDRT; @@ -1401,7 +1329,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_VT_RELDISP; extern unsigned IOCTL_VT_SETMODE; extern unsigned IOCTL_VT_WAITACTIVE; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned IOCTL_CYGETDEFTHRESH; @@ -1488,9 +1416,6 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_TIOCSERGETMULTI; extern unsigned IOCTL_TIOCSERSETMULTI; extern unsigned IOCTL_TIOCSSERIAL; -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID - -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned IOCTL_GIO_SCRNMAP; extern unsigned IOCTL_KDDISABIO; extern unsigned IOCTL_KDENABIO; @@ -1529,6 +1454,6 @@ struct __sanitizer_cookie_io_functions_t { #define SIGACTION_SYMNAME sigaction -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC +#endif // SANITIZER_LINUX || SANITIZER_MAC #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h index c0aa4cc1b17..08bd24cd23e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -210,8 +210,7 @@ struct __sanitizer_cmsghdr { int cmsg_type; }; -#if SANITIZER_SOLARIS32 && 0 -// FIXME: need to deal with large file and non-large file cases +#if SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64) struct __sanitizer_dirent { unsigned long long d_ino; long long d_off; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc index f7dfc86f58c..b965fb0da1a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc @@ -18,6 +18,7 @@ #include "sanitizer_common.h" #include "sanitizer_file.h" +#include "sanitizer_flags.h" #include "sanitizer_libc.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" @@ -157,6 +158,8 @@ void MprotectMallocZones(void *addr, int prot) {} #endif fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) { + if (ShouldMockFailureToOpen(filename)) + return kInvalidFd; int flags; switch (mode) { case RdOnly: flags = O_RDONLY; break; @@ -166,7 +169,7 @@ fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) { fd_t res = internal_open(filename, flags, 0660); if (internal_iserror(res, errno_p)) return kInvalidFd; - return res; + return ReserveStandardFds(res); } void CloseFile(fd_t fd) { @@ -193,11 +196,6 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, return true; } -bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p) { - uptr res = internal_rename(oldpath, newpath); - return !internal_iserror(res, error_p); -} - void *MapFileToMemory(const char *file_name, uptr *buff_size) { fd_t fd = OpenFile(file_name, RdOnly); CHECK(fd != kInvalidFd); @@ -235,6 +233,8 @@ static inline bool IntervalsAreSeparate(uptr start1, uptr end1, // memory). bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { MemoryMappingLayout proc_maps(/*cache_enabled*/true); + if (proc_maps.Error()) + return true; // and hope for the best MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { if (segment.start == segment.end) continue; // Empty range. @@ -274,13 +274,8 @@ bool IsAbsolutePath(const char *path) { void ReportFile::Write(const char *buffer, uptr length) { SpinMutexLock l(mu); - static const char *kWriteError = - "ReportFile::Write() can't output requested buffer!\n"; ReopenIfNecessary(); - if (length != internal_write(fd, buffer, length)) { - internal_write(fd, kWriteError, internal_strlen(kWriteError)); - Die(); - } + internal_write(fd, buffer, length); } bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) { @@ -328,6 +323,27 @@ const char *SignalContext::Describe() const { return "UNKNOWN SIGNAL"; } +fd_t ReserveStandardFds(fd_t fd) { + CHECK_GE(fd, 0); + if (fd > 2) + return fd; + bool used[3]; + internal_memset(used, 0, sizeof(used)); + while (fd <= 2) { + used[fd] = true; + fd = internal_dup(fd); + } + for (int i = 0; i <= 2; ++i) + if (used[i]) + internal_close(i); + return fd; +} + +bool ShouldMockFailureToOpen(const char *path) { + return common_flags()->test_only_emulate_no_memorymap && + internal_strncmp(path, "/proc/", 6) == 0; +} + } // namespace __sanitizer #endif // SANITIZER_POSIX diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h index da447002b66..04a76445eab 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h @@ -16,6 +16,7 @@ // ----------- ATTENTION ------------- // This header should NOT include any other headers from sanitizer runtime. #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_freebsd.h" #include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_openbsd.h" #include "sanitizer_platform_limits_posix.h" @@ -48,6 +49,7 @@ uptr internal_filesize(fd_t fd); // -1 on error. uptr internal_stat(const char *path, void *buf); uptr internal_lstat(const char *path, void *buf); uptr internal_fstat(fd_t fd, void *buf); +uptr internal_dup(int oldfd); uptr internal_dup2(int oldfd, int newfd); uptr internal_readlink(const char *path, char *buf, uptr bufsize); uptr internal_unlink(const char *path); @@ -60,6 +62,11 @@ uptr internal_waitpid(int pid, int *status, int options); int internal_fork(); int internal_forkpty(int *amaster); +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen); +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen); + // These functions call appropriate pthread_ functions directly, bypassing // the interceptor. They are weak and may not be present in some tools. SANITIZER_WEAK_ATTRIBUTE @@ -93,6 +100,11 @@ uptr internal_execve(const char *filename, char *const argv[], bool IsStateDetached(int state); +// Move the fd out of {0, 1, 2} range. +fd_t ReserveStandardFds(fd_t fd); + +bool ShouldMockFailureToOpen(const char *path); + } // namespace __sanitizer #endif // SANITIZER_POSIX_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc index 266e9bdba05..3006e60d89f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -94,10 +94,12 @@ static rlim_t getlim(int res) { } static void setlim(int res, rlim_t lim) { - // The following magic is to prevent clang from replacing it with memset. - volatile struct rlimit rlim; + struct rlimit rlim; + if (getrlimit(res, const_cast(&rlim))) { + Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno); + Die(); + } rlim.rlim_cur = lim; - rlim.rlim_max = lim; if (setrlimit(res, const_cast(&rlim))) { Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno); Die(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h index 9fde040a11a..acb7104f33e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h @@ -70,6 +70,7 @@ class MemoryMappingLayout { explicit MemoryMappingLayout(bool cache_enabled); ~MemoryMappingLayout(); bool Next(MemoryMappedSegment *segment); + bool Error() const; void Reset(); // In some cases, e.g. when running under a sandbox on Linux, ASan is unable // to obtain the memory mappings. It should fall back to pre-cached data diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc index e41dc987dcd..362a424d737 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc @@ -67,8 +67,8 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { #endif }; - size_t Size = 0; - int Err = sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0); + uptr Size = 0; + int Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0); CHECK_EQ(Err, 0); CHECK_GT(Size, 0); @@ -76,7 +76,7 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { size_t MmapedSize = Size * 4 / 3; void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()"); Size = MmapedSize; - Err = sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0); + Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0); CHECK_EQ(Err, 0); proc_maps->data = (char *)VmMap; #else @@ -88,7 +88,7 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { if (Size > 0x10000) Size = 0x10000; Size = (Size / sizeof(struct kinfo_vmentry)) * sizeof(struct kinfo_vmentry); - Err = sysctl(Mib, ARRAY_SIZE(Mib), Mem, &Size, NULL, 0); + Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), Mem, &Size, NULL, 0); CHECK_EQ(Err, 0); MmapedSize = Size; proc_maps->data = Mem; @@ -99,6 +99,7 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + CHECK(!Error()); // can not fail char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; if (data_.current >= last) return false; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc index 1f2b431c7cc..17d61b6c5e8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -80,12 +80,14 @@ MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { ReadProcMaps(&data_.proc_self_maps); if (cache_enabled && data_.proc_self_maps.mmaped_size == 0) LoadFromCache(); - CHECK_GT(data_.proc_self_maps.mmaped_size, 0); - CHECK_GT(data_.proc_self_maps.len, 0); Reset(); } +bool MemoryMappingLayout::Error() const { + return data_.current == nullptr; +} + MemoryMappingLayout::~MemoryMappingLayout() { // Only unmap the buffer if it is different from the cached one. Otherwise // it will be unmapped when the cache is refreshed. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc index 633e9393cce..cf9cb25ba06 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -31,6 +31,7 @@ static bool IsOneOf(char c, char c1, char c2) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + if (Error()) return false; // simulate empty maps char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; if (data_.current >= last) return false; char *next_line = diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 0167ab18ba1..267c960b55d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -88,6 +88,10 @@ MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { MemoryMappingLayout::~MemoryMappingLayout() { } +bool MemoryMappingLayout::Error() const { + return false; +} + // More information about Mach-O headers can be found in mach-o/loader.h // Each Mach-O image has a header (mach_header or mach_header_64) starting with // a magic number, and a list of linker load commands directly following the @@ -140,12 +144,6 @@ void MemoryMappingLayout::LoadFromCache() { // early in the process, when dyld is one of the only images loaded, // so it will be hit after only a few iterations. static mach_header *get_dyld_image_header() { - mach_port_name_t port; - if (task_for_pid(mach_task_self(), internal_getpid(), &port) != - KERN_SUCCESS) { - return nullptr; - } - unsigned depth = 1; vm_size_t size = 0; vm_address_t address = 0; @@ -154,7 +152,7 @@ static mach_header *get_dyld_image_header() { while (true) { struct vm_region_submap_info_64 info; - err = vm_region_recurse_64(port, &address, &size, &depth, + err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth, (vm_region_info_t)&info, &count); if (err != KERN_SUCCESS) return nullptr; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc index bfe83170f4e..49bb46c31b0 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc @@ -13,17 +13,24 @@ #include "sanitizer_common.h" #include "sanitizer_procmaps.h" +// Before Solaris 11.4, doesn't work in a largefile environment. +#undef _FILE_OFFSET_BITS #include #include namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, &proc_maps->mmaped_size, - &proc_maps->len); + if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, + &proc_maps->mmaped_size, &proc_maps->len)) { + proc_maps->data = nullptr; + proc_maps->mmaped_size = 0; + proc_maps->len = 0; + } } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + if (Error()) return false; // simulate empty maps char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; if (data_.current >= last) return false; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_ring_buffer.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_ring_buffer.h new file mode 100644 index 00000000000..d15f27fd4a8 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_ring_buffer.h @@ -0,0 +1,162 @@ +//===-- sanitizer_ring_buffer.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Simple ring buffer. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_RING_BUFFER_H +#define SANITIZER_RING_BUFFER_H + +#include "sanitizer_common.h" + +namespace __sanitizer { +// RingBuffer: fixed-size ring buffer optimized for speed of push(). +// T should be a POD type and sizeof(T) should be divisible by sizeof(void*). +// At creation, all elements are zero. +template +class RingBuffer { + public: + COMPILER_CHECK(sizeof(T) % sizeof(void *) == 0); + static RingBuffer *New(uptr Size) { + void *Ptr = MmapOrDie(SizeInBytes(Size), "RingBuffer"); + RingBuffer *RB = reinterpret_cast(Ptr); + uptr End = reinterpret_cast(Ptr) + SizeInBytes(Size); + RB->last_ = RB->next_ = reinterpret_cast(End - sizeof(T)); + return RB; + } + void Delete() { + UnmapOrDie(this, SizeInBytes(size())); + } + uptr size() const { + return last_ + 1 - + reinterpret_cast(reinterpret_cast(this) + + 2 * sizeof(T *)); + } + + static uptr SizeInBytes(uptr Size) { + return Size * sizeof(T) + 2 * sizeof(T*); + } + + uptr SizeInBytes() { return SizeInBytes(size()); } + + void push(T t) { + *next_ = t; + next_--; + // The condition below works only if sizeof(T) is divisible by sizeof(T*). + if (next_ <= reinterpret_cast(&next_)) + next_ = last_; + } + + T operator[](uptr Idx) const { + CHECK_LT(Idx, size()); + sptr IdxNext = Idx + 1; + if (IdxNext > last_ - next_) + IdxNext -= size(); + return next_[IdxNext]; + } + + private: + RingBuffer() {} + ~RingBuffer() {} + RingBuffer(const RingBuffer&) = delete; + + // Data layout: + // LNDDDDDDDD + // D: data elements. + // L: last_, always points to the last data element. + // N: next_, initially equals to last_, is decremented on every push, + // wraps around if it's less or equal than its own address. + T *last_; + T *next_; + T data_[1]; // flexible array. +}; + +// A ring buffer with externally provided storage that encodes its state in 8 +// bytes. Has significant constraints on size and alignment of storage. +// See a comment in hwasan/hwasan_thread_list.h for the motivation behind this. +#if SANITIZER_WORDSIZE == 64 +template +class CompactRingBuffer { + // Top byte of long_ stores the buffer size in pages. + // Lower bytes store the address of the next buffer element. + static constexpr int kPageSizeBits = 12; + static constexpr int kSizeShift = 56; + static constexpr uptr kNextMask = (1ULL << kSizeShift) - 1; + + uptr GetStorageSize() const { return (long_ >> kSizeShift) << kPageSizeBits; } + + void Init(void *storage, uptr size) { + CHECK_EQ(sizeof(CompactRingBuffer), sizeof(void *)); + CHECK(IsPowerOfTwo(size)); + CHECK_GE(size, 1 << kPageSizeBits); + CHECK_LE(size, 128 << kPageSizeBits); + CHECK_EQ(size % 4096, 0); + CHECK_EQ(size % sizeof(T), 0); + CHECK_EQ((uptr)storage % (size * 2), 0); + long_ = (uptr)storage | ((size >> kPageSizeBits) << kSizeShift); + } + + void SetNext(const T *next) { + long_ = (long_ & ~kNextMask) | (uptr)next; + } + + public: + CompactRingBuffer(void *storage, uptr size) { + Init(storage, size); + } + + // A copy constructor of sorts. + CompactRingBuffer(const CompactRingBuffer &other, void *storage) { + uptr size = other.GetStorageSize(); + internal_memcpy(storage, other.StartOfStorage(), size); + Init(storage, size); + uptr Idx = other.Next() - (const T *)other.StartOfStorage(); + SetNext((const T *)storage + Idx); + } + + T *Next() const { return (T *)(long_ & kNextMask); } + + void *StartOfStorage() const { + return (void *)((uptr)Next() & ~(GetStorageSize() - 1)); + } + + void *EndOfStorage() const { + return (void *)((uptr)StartOfStorage() + GetStorageSize()); + } + + uptr size() const { return GetStorageSize() / sizeof(T); } + + void push(T t) { + T *next = Next(); + *next = t; + next++; + next = (T *)((uptr)next & ~GetStorageSize()); + SetNext(next); + } + + T operator[](uptr Idx) const { + CHECK_LT(Idx, size()); + const T *Begin = (const T *)StartOfStorage(); + sptr StorageIdx = Next() - Begin; + StorageIdx -= (sptr)(Idx + 1); + if (StorageIdx < 0) + StorageIdx += size(); + return Begin[StorageIdx]; + } + + public: + ~CompactRingBuffer() {} + CompactRingBuffer(const CompactRingBuffer &) = delete; + + uptr long_; +}; +#endif +} // namespace __sanitizer + +#endif // SANITIZER_RING_BUFFER_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc index 4be367911e3..76aebe41903 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc @@ -95,8 +95,10 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, *tls_addr = *tls_size = 0; } +void InitializePlatformEarly() {} void MaybeReexec() {} void CheckASLR() {} +void CheckMPROTECT() {} void DisableCoreDumperIfNecessary() {} void InstallDeadlySignalHandlers(SignalHandlerType handler) {} void SetAlternateSignalStack() {} @@ -226,11 +228,6 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, return true; } -bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p) { - uptr res = rename(oldpath, newpath); - return !internal_iserror(res, error_p); -} - void ReleaseMemoryPagesToOS(uptr beg, uptr end) {} void DumpProcessMap() {} @@ -240,6 +237,7 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { } char **GetArgv() { return nullptr; } +char **GetEnviron() { return nullptr; } const char *GetEnv(const char *name) { return getenv(name); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc index a5db22994e0..cc0201c7a37 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc @@ -48,10 +48,21 @@ namespace __sanitizer { DECLARE__REAL(ret_type, func, __VA_ARGS__); \ ret_type internal_ ## func(__VA_ARGS__) +#if !defined(_LP64) && _FILE_OFFSET_BITS == 64 +#define _REAL64(func) _ ## func ## 64 +#else +#define _REAL64(func) _REAL(func) +#endif +#define DECLARE__REAL64(ret_type, func, ...) \ + extern "C" ret_type _REAL64(func)(__VA_ARGS__) +#define DECLARE__REAL_AND_INTERNAL64(ret_type, func, ...) \ + DECLARE__REAL64(ret_type, func, __VA_ARGS__); \ + ret_type internal_ ## func(__VA_ARGS__) + // ---------------------- sanitizer_libc.h -DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length, - int prot, int flags, int fd, OFF_T offset) { - return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset); +DECLARE__REAL_AND_INTERNAL64(uptr, mmap, void *addr, uptr /*size_t*/ length, + int prot, int flags, int fd, OFF_T offset) { + return (uptr)_REAL64(mmap)(addr, length, prot, flags, fd, offset); } DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) { @@ -66,19 +77,19 @@ DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) { return _REAL(close)(fd); } -extern "C" int _REAL(open)(const char *, int, ...); +extern "C" int _REAL64(open)(const char *, int, ...); uptr internal_open(const char *filename, int flags) { - return _REAL(open)(filename, flags); + return _REAL64(open)(filename, flags); } uptr internal_open(const char *filename, int flags, u32 mode) { - return _REAL(open)(filename, flags, mode); + return _REAL64(open)(filename, flags, mode); } uptr OpenFile(const char *filename, bool write) { - return internal_open(filename, - write ? O_WRONLY | O_CREAT : O_RDONLY, 0660); + return ReserveStandardFds( + internal_open(filename, write ? O_WRONLY | O_CREAT : O_RDONLY, 0660)); } DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) { @@ -94,16 +105,16 @@ DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) { return ftruncate(fd, size); } -DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) { - return _REAL(stat)(path, (struct stat *)buf); +DECLARE__REAL_AND_INTERNAL64(uptr, stat, const char *path, void *buf) { + return _REAL64(stat)(path, (struct stat *)buf); } -DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) { - return _REAL(lstat)(path, (struct stat *)buf); +DECLARE__REAL_AND_INTERNAL64(uptr, lstat, const char *path, void *buf) { + return _REAL64(lstat)(path, (struct stat *)buf); } -DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) { - return _REAL(fstat)(fd, (struct stat *)buf); +DECLARE__REAL_AND_INTERNAL64(uptr, fstat, fd_t fd, void *buf) { + return _REAL64(fstat)(fd, (struct stat *)buf); } uptr internal_filesize(fd_t fd) { @@ -153,13 +164,13 @@ DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) { } // FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *. -DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp, - unsigned int count) { - return _REAL(getdents)(fd, dirp, count); +DECLARE__REAL_AND_INTERNAL64(uptr, getdents, fd_t fd, struct linux_dirent *dirp, + unsigned int count) { + return _REAL64(getdents)(fd, dirp, count); } -DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) { - return _REAL(lseek)(fd, offset, whence); +DECLARE__REAL_AND_INTERNAL64(uptr, lseek, fd_t fd, OFF_T offset, int whence) { + return _REAL64(lseek)(fd, offset, whence); } // FIXME: This might be wrong: _sigfillset doesn't take a diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc index 3bd5b677a1f..6aab9848522 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc @@ -26,7 +26,7 @@ struct StackDepotNode { u32 tag; uptr stack[1]; // [size] - static const u32 kTabSizeLog = 20; + static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20; // Lower kTabSizeLog bits are equal for all items in one bucket. // We use these bits to store the per-stack use counter. static const u32 kUseCountBits = kTabSizeLog; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h index cb7345002a4..e22ed2e38e5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h @@ -32,7 +32,7 @@ struct StackDepotHandle { void inc_use_count_unsafe(); }; -const int kStackDepotMaxUseCount = 1U << 20; +const int kStackDepotMaxUseCount = 1U << (SANITIZER_ANDROID ? 16 : 20); StackDepotStats *StackDepotGetStats(); u32 StackDepotPut(StackTrace stack); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index 562d2e9f737..450a40a9069 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -19,7 +19,7 @@ namespace __sanitizer { static const u32 kStackTraceMax = 256; -#if SANITIZER_LINUX && (defined(__sparc__) || defined(__mips__)) +#if defined(__sparc__) || (SANITIZER_LINUX && defined(__mips__)) # define SANITIZER_CAN_FAST_UNWIND 0 #elif SANITIZER_WINDOWS # define SANITIZER_CAN_FAST_UNWIND 0 diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc index 747a4a70172..c87b18e1b69 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -114,11 +114,25 @@ void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf, return; } InternalScopedString frame_desc(GetPageSizeCached()); - RenderFrame(&frame_desc, fmt, 0, frame->info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); - internal_strncpy(out_buf, frame_desc.data(), out_buf_size); - out_buf[out_buf_size - 1] = 0; + uptr frame_num = 0; + // Reserve one byte for the final 0. + char *out_end = out_buf + out_buf_size - 1; + for (SymbolizedStack *cur = frame; cur && out_buf < out_end; + cur = cur->next) { + frame_desc.clear(); + RenderFrame(&frame_desc, fmt, frame_num++, cur->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + if (!frame_desc.length()) + continue; + // Reserve one byte for the terminating 0. + uptr n = out_end - out_buf - 1; + internal_strncpy(out_buf, frame_desc.data(), n); + out_buf += __sanitizer::Min(n, frame_desc.length()); + *out_buf++ = 0; + } + CHECK(out_buf <= out_end); + *out_buf = 0; } SANITIZER_INTERFACE_ATTRIBUTE diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index ac0731d46a6..f2b33743342 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -98,6 +98,8 @@ static const char *DemangleFunctionName(const char *function) { return "pthread_equal"; if (!internal_strcmp(function, "__libc_thr_curcpu")) return "pthread_curcpu_np"; + if (!internal_strcmp(function, "__libc_thr_sigsetmask")) + return "pthread_sigmask"; #endif return function; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc index 9f9920ece80..f41a3cefb35 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc @@ -15,7 +15,7 @@ // This file is ported to Sparc v8, but it should be easy to port to // Sparc v9. -#if defined(__sparcv8__) +#if defined(__sparcv8__) || defined(__sparcv8) || defined(__sparc_v8__) #include "sanitizer_common.h" #include "sanitizer_stacktrace.h" @@ -55,4 +55,5 @@ void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, } // namespace __sanitizer -#endif // !defined(__sparcv8__) +#endif // !defined(__sparcv8__) && !defined(__sparcv8) && + // !defined(__sparc_v8__) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h index 3c1c864c061..b241b9dbc3c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h @@ -35,6 +35,9 @@ constexpr const char *kFormatData = "{{{data:%p}}}"; // One frame in a backtrace (printed on a line by itself). constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}"; +// Dump trigger element. +#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}" + } // namespace __sanitizer #endif // SANITIZER_SYMBOLIZER_FUCHSIA_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc index e4ed1b4deee..ddfb26397f4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -11,46 +11,21 @@ // //===----------------------------------------------------------------------===// -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || \ - SANITIZER_OPENBSD || SANITIZER_SOLARIS +// NetBSD uses libc calls directly +#if !SANITIZER_NETBSD + +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_OPENBSD || SANITIZER_SOLARIS # define SYSCALL(name) SYS_ ## name #else # define SYSCALL(name) __NR_ ## name #endif -#if SANITIZER_NETBSD -// We use 3 kinds of internal_syscall's for different types of retval in order -// to address differences in calling conventions (e.g. registers to place the -// return value in). -// - internal_syscall for 32-bit length (int, pid_t) -// - internal_syscall64 for 64-bit length (off_t) -// - internal_syscall_ptr for pointer and (s)size_t -# define internal_syscall syscall -# define internal_syscall64 __syscall -// Handle syscall renames manually -# define SYS_stat SYS___stat50 -# define SYS_lstat SYS___lstat50 -# define SYS_fstat SYS___fstat50 -# define SYS_gettimeofday SYS___gettimeofday50 -# define SYS_wait4 SYS___wait450 -# define SYS_getdents SYS___getdents30 -# define SYS_sigaltstack SYS___sigaltstack14 -# define SYS_sigprocmask SYS___sigprocmask14 -# define SYS_nanosleep SYS___nanosleep50 -# define SYS_clock_gettime SYS___clock_gettime50 -# if SANITIZER_WORDSIZE == 64 -# define internal_syscall_ptr __syscall -# else -# define internal_syscall_ptr syscall -# endif -#elif defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) +#if defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) # define internal_syscall __syscall -# define internal_syscall64 __syscall -# define internal_syscall_ptr __syscall # else # define internal_syscall syscall -# define internal_syscall64 syscall -# define internal_syscall_ptr syscall +#endif + #endif bool internal_iserror(uptr retval, int *rverrno) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc index 1f05ed9b6c1..7ab1d764144 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -127,9 +127,6 @@ static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, #define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) -#define internal_syscall_ptr internal_syscall -#define internal_syscall64 internal_syscall - // Helper function used to avoid cobbler errno. bool internal_iserror(uptr retval, int *rverrno) { if (retval >= (uptr)-4095) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc index a3fdb9e60d5..b4fd0962a9f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc @@ -127,9 +127,6 @@ static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, #define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) -#define internal_syscall_ptr internal_syscall -#define internal_syscall64 internal_syscall - // Helper function used to avoid cobbler errno. bool internal_iserror(uptr retval, int *rverrno) { if (retval >= (uptr)-4095) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc index 327aaa80a67..9853a6a675d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc @@ -20,9 +20,6 @@ static uptr internal_syscall(u64 nr) { return retval; } -#define internal_syscall_ptr internal_syscall -#define internal_syscall64 internal_syscall - template static uptr internal_syscall(u64 nr, T1 arg1) { u64 retval; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc index 4f766100813..75aea2760d2 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc @@ -43,8 +43,8 @@ // DO NOT EDIT! THIS FILE HAS BEEN GENERATED! // // Generated with: generate_netbsd_syscalls.awk -// Generated date: 2018-03-03 -// Generated from: syscalls.master,v 1.291 2018/01/06 16:41:23 kamil Exp +// Generated date: 2018-10-30 +// Generated from: syscalls.master,v 1.293 2018/07/31 13:00:13 rjs Exp // //===----------------------------------------------------------------------===// @@ -1454,7 +1454,15 @@ PRE_SYSCALL(fpathconf)(long long fd_, long long name_) { /* Nothing to do */ } POST_SYSCALL(fpathconf)(long long res, long long fd_, long long name_) { /* Nothing to do */ } -/* syscall 193 has been skipped */ +PRE_SYSCALL(getsockopt2) +(long long s_, long long level_, long long name_, void *val_, void *avalsize_) { + /* TODO */ +} +POST_SYSCALL(getsockopt2) +(long long res, long long s_, long long level_, long long name_, void *val_, + void *avalsize_) { + /* TODO */ +} PRE_SYSCALL(getrlimit)(long long which_, void *rlp_) { PRE_WRITE(rlp_, struct_rlimit_sz); } @@ -2341,20 +2349,8 @@ POST_SYSCALL(__sigaction_sigtramp) PRE_READ(nsa_, sizeof(__sanitizer_sigaction)); } } -PRE_SYSCALL(pmc_get_info)(long long ctr_, long long op_, void *args_) { - /* TODO */ -} -POST_SYSCALL(pmc_get_info) -(long long res, long long ctr_, long long op_, void *args_) { - /* TODO */ -} -PRE_SYSCALL(pmc_control)(long long ctr_, long long op_, void *args_) { - /* TODO */ -} -POST_SYSCALL(pmc_control) -(long long res, long long ctr_, long long op_, void *args_) { - /* TODO */ -} +/* syscall 341 has been skipped */ +/* syscall 342 has been skipped */ PRE_SYSCALL(rasctl)(void *addr_, long long len_, long long op_) { /* Nothing to do */ } @@ -3695,18 +3691,18 @@ POST_SYSCALL(recvmmsg) PRE_SYSCALL(sendmmsg) (long long s_, void *mmsg_, long long vlen_, long long flags_) { struct __sanitizer_mmsghdr *mmsg = (struct __sanitizer_mmsghdr *)mmsg_; - unsigned int vlen = (vlen_ > 1024 ? 1024 : vlen_); if (mmsg) { - PRE_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * vlen); + PRE_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * + (vlen_ > 1024 ? 1024 : vlen_)); } } POST_SYSCALL(sendmmsg) (long long res, long long s_, void *mmsg_, long long vlen_, long long flags_) { struct __sanitizer_mmsghdr *mmsg = (struct __sanitizer_mmsghdr *)mmsg_; - unsigned int vlen = (vlen_ > 1024 ? 1024 : vlen_); if (res >= 0) { if (mmsg) { - POST_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * vlen); + POST_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * + (vlen_ > 1024 ? 1024 : vlen_)); } } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc index 8243fc05d02..35e4403adfb 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc @@ -84,3 +84,12 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, } } // namespace __sanitizer + +using namespace __sanitizer; // NOLINT + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_set_death_callback(void (*callback)(void)) { + SetUserDieCallback(callback); +} +} // extern "C" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc index d9fd6549ba4..eb35cb6c583 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc @@ -338,4 +338,15 @@ ThreadContextBase *ThreadRegistry::QuarantinePop() { return tctx; } +void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) { + BlockingMutexLock l(&mtx_); + CHECK_LT(tid, n_contexts_); + ThreadContextBase *tctx = threads_[tid]; + CHECK_NE(tctx, 0); + CHECK_NE(tctx->status, ThreadStatusInvalid); + CHECK_NE(tctx->status, ThreadStatusDead); + CHECK_EQ(tctx->user_id, 0); + tctx->user_id = user_id; +} + } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h index b203be2f4db..30dc603febd 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h @@ -122,6 +122,7 @@ class ThreadRegistry { void JoinThread(u32 tid, void *arg); void FinishThread(u32 tid); void StartThread(u32 tid, tid_t os_id, bool workerthread, void *arg); + void SetThreadUserId(u32 tid, uptr user_id); private: const ThreadContextFactory context_factory_; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cc new file mode 100644 index 00000000000..27fec6e1f49 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cc @@ -0,0 +1,21 @@ +//===-- sanitizer_type_traits.cc --------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implements a subset of C++ type traits. This is so we can avoid depending +// on system C++ headers. +// +//===----------------------------------------------------------------------===// +#include "sanitizer_type_traits.h" + +namespace __sanitizer { + +const bool true_type::value; +const bool false_type::value; + +} // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h new file mode 100644 index 00000000000..4495f2c34b7 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h @@ -0,0 +1,44 @@ +//===-- sanitizer_type_traits.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implements a subset of C++ type traits. This is so we can avoid depending +// on system C++ headers. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_TYPE_TRAITS_H +#define SANITIZER_TYPE_TRAITS_H + +namespace __sanitizer { + +struct true_type { + static const bool value = true; +}; + +struct false_type { + static const bool value = false; +}; + +// is_same +// +// Type trait to compare if types are the same. +// E.g. +// +// ``` +// is_same::value - True +// is_same::value - False +// ``` +template +struct is_same : public false_type {}; + +template +struct is_same : public true_type {}; + +} // namespace __sanitizer + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index 9e12c417c71..c7a5ec86fc7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -97,7 +97,7 @@ uptr Unwind_GetIP(struct _Unwind_Context *ctx) { // Clear the Thumb bit. return val & ~(uptr)1; #else - return _Unwind_GetIP(ctx); + return (uptr)_Unwind_GetIP(ctx); #endif } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc index 38e567d9a73..9a574dd23de 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -29,6 +29,10 @@ #include "sanitizer_placement_new.h" #include "sanitizer_win_defs.h" +#if defined(PSAPI_VERSION) && PSAPI_VERSION == 1 +#pragma comment(lib, "psapi") +#endif + // A macro to tell the compiler that this part of the code cannot be reached, // if the compiler supports this feature. Since we're using this in // code that is called when terminating the process, the expansion of the @@ -733,10 +737,6 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, } } -bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p) { - UNIMPLEMENTED(); -} - uptr internal_sched_yield() { Sleep(0); return 0; @@ -1004,6 +1004,10 @@ void CheckVMASize() { // Do nothing. } +void InitializePlatformEarly() { + // Do nothing. +} + void MaybeReexec() { // No need to re-exec on Windows. } @@ -1012,11 +1016,20 @@ void CheckASLR() { // Do nothing } +void CheckMPROTECT() { + // Do nothing +} + char **GetArgv() { // FIXME: Actually implement this function. return 0; } +char **GetEnviron() { + // FIXME: Actually implement this function. + return 0; +} + pid_t StartSubprocess(const char *program, const char *const argv[], fd_t stdin_fd, fd_t stdout_fd, fd_t stderr_fd) { // FIXME: implement on this platform diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h index 077ff9ccc8d..10fc2d021ee 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h @@ -17,17 +17,27 @@ #if SANITIZER_WINDOWS #ifndef WINAPI -#ifdef _M_IX86 +#if defined(_M_IX86) || defined(__i386__) #define WINAPI __stdcall #else #define WINAPI #endif #endif -#if defined(_WIN64) -#define WIN_SYM_PREFIX -#else +#if defined(_M_IX86) || defined(__i386__) #define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif + +// For MinGW, the /export: directives contain undecorated symbols, contrary to +// link/lld-link. The GNU linker doesn't support /alternatename and /include +// though, thus lld-link in MinGW mode interprets them in the same way as +// in the default mode. +#ifdef __MINGW32__ +#define WIN_EXPORT_PREFIX +#else +#define WIN_EXPORT_PREFIX WIN_SYM_PREFIX #endif // Intermediate macro to ensure the parameter is expanded before stringified. @@ -62,8 +72,8 @@ __pragma(comment(linker, "/include:" WIN_SYM_PREFIX STRINGIFY(Name))) #define WIN_EXPORT(ExportedName, Name) \ - __pragma(comment(linker, "/export:" WIN_SYM_PREFIX STRINGIFY(ExportedName) \ - "=" WIN_SYM_PREFIX STRINGIFY(Name))) + __pragma(comment(linker, "/export:" WIN_EXPORT_PREFIX STRINGIFY(ExportedName)\ + "=" WIN_EXPORT_PREFIX STRINGIFY(Name))) // We cannot define weak functions on Windows, but we can use WIN_WEAK_ALIAS() // which defines an alias to a default implementation, and only works when diff --git a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt index f77648d7818..beee0acf484 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt +++ b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt @@ -49,6 +49,7 @@ dup U dup2 U environ U execv U +execve U exit U fclose U fflush U @@ -65,6 +66,7 @@ getcwd U getenv U getpagesize U getpid U +getrlimit U gettimeofday U ioctl U isalpha U @@ -101,6 +103,7 @@ rand U readlink U realloc U remove U +setrlimit U setvbuf U sigfillset U sigprocmask U diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp index 4a11bf5fcc2..fb04fb281c4 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -129,16 +129,9 @@ namespace Chunk { computeChecksum(Ptr, &NewUnpackedHeader)); } - // Nulls out a chunk header. When returning the chunk to the backend, there - // is no need to store a valid ChunkAvailable header, as this would be - // computationally expensive. Zeroing out serves the same purpose by making - // the header invalid. In the extremely rare event where 0 would be a valid - // checksum for the chunk, the state of the chunk is ChunkAvailable anyway. + // Ensure that ChunkAvailable is 0, so that if a 0 checksum is ever valid + // for a fully nulled out header, its state will be available anyway. COMPILER_CHECK(ChunkAvailable == 0); - static INLINE void eraseHeader(void *Ptr) { - const PackedHeader NullPackedHeader = 0; - atomic_store_relaxed(getAtomicHeader(Ptr), NullPackedHeader); - } // Loads and unpacks the header, verifying the checksum in the process. static INLINE @@ -185,7 +178,9 @@ struct QuarantineCallback { Chunk::loadHeader(Ptr, &Header); if (UNLIKELY(Header.State != ChunkQuarantine)) dieWithMessage("invalid chunk state when recycling address %p\n", Ptr); - Chunk::eraseHeader(Ptr); + UnpackedHeader NewHeader = Header; + NewHeader.State = ChunkAvailable; + Chunk::compareExchangeHeader(Ptr, &NewHeader, &Header); void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header); if (Header.ClassId) getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId); @@ -264,7 +259,8 @@ struct Allocator { Quarantine.Init( static_cast(getFlags()->QuarantineSizeKb) << 10, static_cast(getFlags()->ThreadLocalQuarantineSizeKb) << 10); - QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize; + QuarantineChunksUpToSize = (Quarantine.GetCacheSize() == 0) ? 0 : + getFlags()->QuarantineChunksUpToSize; DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch; DeleteSizeMismatch = getFlags()->DeleteSizeMismatch; ZeroContents = getFlags()->ZeroContents; @@ -389,10 +385,11 @@ struct Allocator { // quarantine chunk size threshold. void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header, uptr Size) { - const bool BypassQuarantine = (Quarantine.GetCacheSize() == 0) || - (Size > QuarantineChunksUpToSize); + const bool BypassQuarantine = !Size || (Size > QuarantineChunksUpToSize); if (BypassQuarantine) { - Chunk::eraseHeader(Ptr); + UnpackedHeader NewHeader = *Header; + NewHeader.State = ChunkAvailable; + Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); void *BackendPtr = Chunk::getBackendPtr(Ptr, Header); if (Header->ClassId) { bool UnlockRequired; @@ -675,7 +672,7 @@ void *scudoValloc(uptr Size) { } void *scudoPvalloc(uptr Size) { - uptr PageSize = GetPageSizeCached(); + const uptr PageSize = GetPageSizeCached(); if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) { errno = ENOMEM; if (Instance.canReturnNull()) diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.h b/contrib/compiler-rt/lib/scudo/scudo_allocator.h index 0002b4a44b7..814bb08ab01 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.h @@ -81,6 +81,7 @@ struct AP64 { typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator64FlagMasks::kRandomShuffleChunks; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64 PrimaryT; #else @@ -96,7 +97,8 @@ struct AP32 { static const uptr kMetadataSize = 0; typedef __scudo::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = RegionSizeLog; - typedef __scudo::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __scudo::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator32FlagMasks::kRandomShuffleChunks | diff --git a/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp b/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp index 91a77b36582..eef776809bd 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp @@ -79,7 +79,7 @@ INTERCEPTOR_ATTRIBUTE size_t malloc_usable_size(void *ptr) { #if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) { - return -1; + return 0; } #endif } // extern "C" diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc index 9e5465d3755..067aeef97c0 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc @@ -20,31 +20,37 @@ using namespace __tsan; static const char *ReportTypeDescription(ReportType typ) { - if (typ == ReportTypeRace) return "data-race"; - if (typ == ReportTypeVptrRace) return "data-race-vptr"; - if (typ == ReportTypeUseAfterFree) return "heap-use-after-free"; - if (typ == ReportTypeVptrUseAfterFree) return "heap-use-after-free-vptr"; - if (typ == ReportTypeExternalRace) return "external-race"; - if (typ == ReportTypeThreadLeak) return "thread-leak"; - if (typ == ReportTypeMutexDestroyLocked) return "locked-mutex-destroy"; - if (typ == ReportTypeMutexDoubleLock) return "mutex-double-lock"; - if (typ == ReportTypeMutexInvalidAccess) return "mutex-invalid-access"; - if (typ == ReportTypeMutexBadUnlock) return "mutex-bad-unlock"; - if (typ == ReportTypeMutexBadReadLock) return "mutex-bad-read-lock"; - if (typ == ReportTypeMutexBadReadUnlock) return "mutex-bad-read-unlock"; - if (typ == ReportTypeSignalUnsafe) return "signal-unsafe-call"; - if (typ == ReportTypeErrnoInSignal) return "errno-in-signal-handler"; - if (typ == ReportTypeDeadlock) return "lock-order-inversion"; - return ""; + switch (typ) { + case ReportTypeRace: return "data-race"; + case ReportTypeVptrRace: return "data-race-vptr"; + case ReportTypeUseAfterFree: return "heap-use-after-free"; + case ReportTypeVptrUseAfterFree: return "heap-use-after-free-vptr"; + case ReportTypeExternalRace: return "external-race"; + case ReportTypeThreadLeak: return "thread-leak"; + case ReportTypeMutexDestroyLocked: return "locked-mutex-destroy"; + case ReportTypeMutexDoubleLock: return "mutex-double-lock"; + case ReportTypeMutexInvalidAccess: return "mutex-invalid-access"; + case ReportTypeMutexBadUnlock: return "mutex-bad-unlock"; + case ReportTypeMutexBadReadLock: return "mutex-bad-read-lock"; + case ReportTypeMutexBadReadUnlock: return "mutex-bad-read-unlock"; + case ReportTypeSignalUnsafe: return "signal-unsafe-call"; + case ReportTypeErrnoInSignal: return "errno-in-signal-handler"; + case ReportTypeDeadlock: return "lock-order-inversion"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); } static const char *ReportLocationTypeDescription(ReportLocationType typ) { - if (typ == ReportLocationGlobal) return "global"; - if (typ == ReportLocationHeap) return "heap"; - if (typ == ReportLocationStack) return "stack"; - if (typ == ReportLocationTLS) return "tls"; - if (typ == ReportLocationFD) return "fd"; - return ""; + switch (typ) { + case ReportLocationGlobal: return "global"; + case ReportLocationHeap: return "heap"; + case ReportLocationStack: return "stack"; + case ReportLocationTLS: return "tls"; + case ReportLocationFD: return "fd"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); } static void CopyTrace(SymbolizedStack *first_frame, void **trace, diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc index 89e22a13278..877fc8b819f 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc @@ -61,8 +61,7 @@ void InitializeFlags(Flags *f, const char *env) { // Does not work as expected for Go: runtime handles SIGABRT and crashes. cf.abort_on_error = false; // Go does not have mutexes. - } else { - cf.detect_deadlocks = true; + cf.detect_deadlocks = false; } cf.print_suppressions = false; cf.stack_trace_format = " #%n %f %S %M"; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc index e9b3e35f07e..d3b678fdd15 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc @@ -77,8 +77,6 @@ TSAN_FLAG(int, io_sync, 1, TSAN_FLAG(bool, die_after_fork, true, "Die after multi-threaded fork if the child creates new threads.") TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") -TSAN_FLAG(bool, ignore_interceptors_accesses, false, - "Ignore reads and writes from all interceptors.") TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false, "Interceptors should only detect races when called from instrumented " "modules.") diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 901997b3e85..9e49dfe5d58 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -42,17 +42,15 @@ using namespace __tsan; // NOLINT #if SANITIZER_NETBSD #define dirfd(dirp) (*(int *)(dirp)) -#define fileno_unlocked fileno +#define fileno_unlocked(fp) \ + (((__sanitizer_FILE*)fp)->_file == -1 ? -1 : \ + (int)(unsigned short)(((__sanitizer_FILE*)fp)->_file)) // NOLINT -#if _LP64 -#define __sF_size 152 -#else -#define __sF_size 88 -#endif - -#define stdout ((char*)&__sF + (__sF_size * 1)) -#define stderr ((char*)&__sF + (__sF_size * 2)) +#define stdout ((__sanitizer_FILE*)&__sF[1]) +#define stderr ((__sanitizer_FILE*)&__sF[2]) +#define nanosleep __nanosleep50 +#define vfork __vfork14 #endif #if SANITIZER_ANDROID @@ -94,8 +92,8 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); -extern "C" int fileno_unlocked(void *stream); #if !SANITIZER_NETBSD +extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #endif #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD @@ -226,6 +224,16 @@ void InitializeLibIgnore() { libignore()->OnLibraryLoaded(0); } +// The following two hooks can be used by for cooperative scheduling when +// locking. +#ifdef TSAN_EXTERNAL_HOOKS +void OnPotentiallyBlockingRegionBegin(); +void OnPotentiallyBlockingRegionEnd(); +#else +SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} +SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} +#endif + } // namespace __tsan static ThreadSignalContext *SigCtx(ThreadState *thr) { @@ -246,8 +254,7 @@ ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, if (!thr_->ignore_interceptors) FuncEntry(thr, pc); DPrintf("#%d: intercept %s()\n", thr_->tid, fname); ignoring_ = - !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || - libignore()->IsIgnored(pc, &in_ignored_lib_)); + !thr_->in_ignored_lib && libignore()->IsIgnored(pc, &in_ignored_lib_); EnableIgnores(); } @@ -508,7 +515,8 @@ static void LongJmp(ThreadState *thr, uptr *env) { uptr mangled_sp = env[6]; #elif SANITIZER_MAC # ifdef __aarch64__ - uptr mangled_sp = env[13]; + uptr mangled_sp = + (GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? env[12] : env[13]; # else uptr mangled_sp = env[2]; # endif @@ -863,6 +871,8 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { // Used in thread-safe function static initialization. STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); + OnPotentiallyBlockingRegionBegin(); + auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); if (cmp == 0) { @@ -1041,6 +1051,35 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { + SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); + int tid = ThreadTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = REAL(pthread_tryjoin_np)(th, ret); + ThreadIgnoreEnd(thr, pc); + if (res == 0) + ThreadJoin(thr, pc, tid); + else + ThreadNotJoined(thr, pc, tid, (uptr)th); + return res; +} + +TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, + const struct timespec *abstime) { + SCOPED_TSAN_INTERCEPTOR(pthread_timedjoin_np, th, ret, abstime); + int tid = ThreadTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); + ThreadIgnoreEnd(thr, pc); + if (res == 0) + ThreadJoin(thr, pc, tid); + else + ThreadNotJoined(thr, pc, tid, (uptr)th); + return res; +} +#endif + // Problem: // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). // pthread_cond_t has different size in the different versions. @@ -2210,7 +2249,8 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, (void) ctx; #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ - Acquire(thr, pc, File2addr(path)); \ + if (path) \ + Acquire(thr, pc, File2addr(path)); \ if (file) { \ int fd = fileno_unlocked(file); \ if (fd >= 0) FdFileCreate(thr, pc, fd); \ @@ -2559,6 +2599,8 @@ TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()) +TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b, + void *c) namespace __tsan { @@ -2635,6 +2677,10 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + #if SANITIZER_LINUX + TSAN_INTERCEPT(pthread_tryjoin_np); + TSAN_INTERCEPT(pthread_timedjoin_np); + #endif TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); @@ -2768,6 +2814,7 @@ void InitializeInterceptors() { TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask); FdInit(); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index 959a39465e3..763b46b88c8 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -56,9 +56,13 @@ LibIgnore *libignore(); # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \ TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ ALIAS(WRAPPER_NAME(pthread_##func)); +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \ + TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func2)); #else # define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) #endif #endif // TSAN_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc index b58e6b7071c..579c4d0c0b8 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -19,8 +19,10 @@ #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_interface_ann.h" +#include "sanitizer_common/sanitizer_addrhashmap.h" #include +#include #if defined(__has_include) && __has_include() #include @@ -294,41 +296,62 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { #endif // #if defined(__has_include) && __has_include() -// Is the Obj-C object a tagged pointer (i.e. isn't really a valid pointer and -// contains data in the pointers bits instead)? -static bool IsTaggedObjCPointer(void *obj) { +// Determines whether the Obj-C object pointer is a tagged pointer. Tagged +// pointers encode the object data directly in their pointer bits and do not +// have an associated memory allocation. The Obj-C runtime uses tagged pointers +// to transparently optimize small objects. +static bool IsTaggedObjCPointer(id obj) { const uptr kPossibleTaggedBits = 0x8000000000000001ull; return ((uptr)obj & kPossibleTaggedBits) != 0; } -// Return an address on which we can synchronize (Acquire and Release) for a -// Obj-C tagged pointer (which is not a valid pointer). Ideally should be a -// derived address from 'obj', but for now just return the same global address. -// TODO(kubamracek): Return different address for different pointers. -static uptr SyncAddressForTaggedPointer(void *obj) { - (void)obj; - static u64 addr; - return (uptr)&addr; +// Returns an address which can be used to inform TSan about synchronization +// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid +// address in the process space. We do a small allocation here to obtain a +// stable address (the array backing the hash map can change). The memory is +// never free'd (leaked) and allocation and locking are slow, but this code only +// runs for @synchronized with tagged pointers, which is very rare. +static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { + typedef AddrHashMap Map; + static Map Addresses; + Map::Handle h(&Addresses, addr); + if (h.created()) { + ThreadIgnoreBegin(thr, pc); + *h = (uptr) user_alloc(thr, pc, /*size=*/1); + ThreadIgnoreEnd(thr, pc); + } + return *h; } -// Address on which we can synchronize for an Objective-C object. Supports -// tagged pointers. -static uptr SyncAddressForObjCObject(void *obj) { - if (IsTaggedObjCPointer(obj)) return SyncAddressForTaggedPointer(obj); +// Returns an address on which we can synchronize given an Obj-C object pointer. +// For normal object pointers, this is just the address of the object in memory. +// Tagged pointers are not backed by an actual memory allocation, so we need to +// synthesize a valid address. +static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { + if (IsTaggedObjCPointer(obj)) + return GetOrCreateSyncAddress((uptr)obj, thr, pc); return (uptr)obj; } -TSAN_INTERCEPTOR(int, objc_sync_enter, void *obj) { +TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); + if (!obj) return REAL(objc_sync_enter)(obj); + uptr addr = SyncAddressForObjCObject(obj, thr, pc); + MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); int result = REAL(objc_sync_enter)(obj); - if (obj) Acquire(thr, pc, SyncAddressForObjCObject(obj)); + CHECK_EQ(result, OBJC_SYNC_SUCCESS); + MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); return result; } -TSAN_INTERCEPTOR(int, objc_sync_exit, void *obj) { - SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); - if (obj) Release(thr, pc, SyncAddressForObjCObject(obj)); - return REAL(objc_sync_exit)(obj); +TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { + SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); + if (!obj) return REAL(objc_sync_exit)(obj); + uptr addr = SyncAddressForObjCObject(obj, thr, pc); + MutexUnlock(thr, pc, addr); + int result = REAL(objc_sync_exit)(obj); + if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); + return result; } // On macOS, libc++ is always linked dynamically, so intercepting works the diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc index d6c1ca6622f..df22888b3f1 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc @@ -185,11 +185,8 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ - SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ - dispatch_block_t heap_block = Block_copy(block); \ - SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ tsan_block_context_t new_context = { \ - q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \ + q, block, &invoke_block, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h index 6b3e6bac27c..8303c241809 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -458,6 +458,32 @@ struct Mapping47 { static const uptr kAppMemEnd = 0x00e000000000ull; }; +#elif SANITIZER_GO && defined(__aarch64__) + +/* Go on linux/aarch64 (48-bit VMA) +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 3000 0000 0000: shadow +3000 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 8000 0000 0000: - +*/ + +struct Mapping { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x00e000000000ull; +}; + // Indicates the runtime will define the memory regions at runtime. #define TSAN_RUNTIME_VMA 1 @@ -523,7 +549,7 @@ uptr MappingImpl(void) { template uptr MappingArchImpl(void) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return MappingImpl(); case 42: return MappingImpl(); @@ -680,7 +706,7 @@ bool IsAppMemImpl(uptr mem) { ALWAYS_INLINE bool IsAppMem(uptr mem) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return IsAppMemImpl(mem); case 42: return IsAppMemImpl(mem); @@ -711,7 +737,7 @@ bool IsShadowMemImpl(uptr mem) { ALWAYS_INLINE bool IsShadowMem(uptr mem) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return IsShadowMemImpl(mem); case 42: return IsShadowMemImpl(mem); @@ -742,7 +768,7 @@ bool IsMetaMemImpl(uptr mem) { ALWAYS_INLINE bool IsMetaMem(uptr mem) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return IsMetaMemImpl(mem); case 42: return IsMetaMemImpl(mem); @@ -783,7 +809,7 @@ uptr MemToShadowImpl(uptr x) { ALWAYS_INLINE uptr MemToShadow(uptr x) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return MemToShadowImpl(x); case 42: return MemToShadowImpl(x); @@ -826,7 +852,7 @@ u32 *MemToMetaImpl(uptr x) { ALWAYS_INLINE u32 *MemToMeta(uptr x) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return MemToMetaImpl(x); case 42: return MemToMetaImpl(x); @@ -882,7 +908,7 @@ uptr ShadowToMemImpl(uptr s) { ALWAYS_INLINE uptr ShadowToMem(uptr s) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return ShadowToMemImpl(s); case 42: return ShadowToMemImpl(s); @@ -921,7 +947,7 @@ uptr GetThreadTraceImpl(int tid) { ALWAYS_INLINE uptr GetThreadTrace(int tid) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return GetThreadTraceImpl(tid); case 42: return GetThreadTraceImpl(tid); @@ -955,7 +981,7 @@ uptr GetThreadTraceHeaderImpl(int tid) { ALWAYS_INLINE uptr GetThreadTraceHeader(int tid) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return GetThreadTraceHeaderImpl(tid); case 42: return GetThreadTraceHeaderImpl(tid); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc index de989b78088..d2ce6070906 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc @@ -212,11 +212,19 @@ void InitializePlatformEarly() { vmaSize = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); #if defined(__aarch64__) +# if !SANITIZER_GO if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) { Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize); Die(); } +#else + if (vmaSize != 48) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 48\n", vmaSize); + Die(); + } +#endif #elif defined(__powerpc64__) # if !SANITIZER_GO if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc index f8d7324b763..7e3a47387b6 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc @@ -240,6 +240,9 @@ void InitializePlatformEarly() { #endif } +static const uptr kPthreadSetjmpXorKeySlot = 0x7; +extern "C" uptr __tsan_darwin_setjmp_xor_key = 0; + void InitializePlatform() { DisableCoreDumperIfNecessary(); #if !SANITIZER_GO @@ -251,6 +254,11 @@ void InitializePlatform() { prev_pthread_introspection_hook = pthread_introspection_hook_install(&my_pthread_introspection_hook); #endif + + if (GetMacosVersion() >= MACOS_VERSION_MOJAVE) { + __tsan_darwin_setjmp_xor_key = + (uptr)pthread_getspecific(kPthreadSetjmpXorKeySlot); + } } #if !SANITIZER_GO diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc index af47076968d..629c3e9337a 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc @@ -77,39 +77,42 @@ const char *thread_name(char *buf, int tid) { } static const char *ReportTypeString(ReportType typ, uptr tag) { - if (typ == ReportTypeRace) - return "data race"; - if (typ == ReportTypeVptrRace) - return "data race on vptr (ctor/dtor vs virtual call)"; - if (typ == ReportTypeUseAfterFree) - return "heap-use-after-free"; - if (typ == ReportTypeVptrUseAfterFree) - return "heap-use-after-free (virtual call vs free)"; - if (typ == ReportTypeExternalRace) { - const char *str = GetReportHeaderFromTag(tag); - return str ? str : "race on external object"; + switch (typ) { + case ReportTypeRace: + return "data race"; + case ReportTypeVptrRace: + return "data race on vptr (ctor/dtor vs virtual call)"; + case ReportTypeUseAfterFree: + return "heap-use-after-free"; + case ReportTypeVptrUseAfterFree: + return "heap-use-after-free (virtual call vs free)"; + case ReportTypeExternalRace: { + const char *str = GetReportHeaderFromTag(tag); + return str ? str : "race on external object"; + } + case ReportTypeThreadLeak: + return "thread leak"; + case ReportTypeMutexDestroyLocked: + return "destroy of a locked mutex"; + case ReportTypeMutexDoubleLock: + return "double lock of a mutex"; + case ReportTypeMutexInvalidAccess: + return "use of an invalid mutex (e.g. uninitialized or destroyed)"; + case ReportTypeMutexBadUnlock: + return "unlock of an unlocked mutex (or by a wrong thread)"; + case ReportTypeMutexBadReadLock: + return "read lock of a write locked mutex"; + case ReportTypeMutexBadReadUnlock: + return "read unlock of a write locked mutex"; + case ReportTypeSignalUnsafe: + return "signal-unsafe call inside of a signal"; + case ReportTypeErrnoInSignal: + return "signal handler spoils errno"; + case ReportTypeDeadlock: + return "lock-order-inversion (potential deadlock)"; + // No default case so compiler warns us if we miss one } - if (typ == ReportTypeThreadLeak) - return "thread leak"; - if (typ == ReportTypeMutexDestroyLocked) - return "destroy of a locked mutex"; - if (typ == ReportTypeMutexDoubleLock) - return "double lock of a mutex"; - if (typ == ReportTypeMutexInvalidAccess) - return "use of an invalid mutex (e.g. uninitialized or destroyed)"; - if (typ == ReportTypeMutexBadUnlock) - return "unlock of an unlocked mutex (or by a wrong thread)"; - if (typ == ReportTypeMutexBadReadLock) - return "read lock of a write locked mutex"; - if (typ == ReportTypeMutexBadReadUnlock) - return "read unlock of a write locked mutex"; - if (typ == ReportTypeSignalUnsafe) - return "signal-unsafe call inside of a signal"; - if (typ == ReportTypeErrnoInSignal) - return "signal handler spoils errno"; - if (typ == ReportTypeDeadlock) - return "lock-order-inversion (potential deadlock)"; - return ""; + UNREACHABLE("missing case"); } #if SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index 5841222927b..f038e9682d8 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -359,7 +359,9 @@ void Initialize(ThreadState *thr) { CheckASLR(); InitializeFlags(&ctx->flags, options); AvoidCVE_2016_2143(); - InitializePlatformEarly(); + __sanitizer::InitializePlatformEarly(); + __tsan::InitializePlatformEarly(); + #if !SANITIZER_GO // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec(); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h index 523b69aaa6b..3410be29450 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -59,15 +59,16 @@ struct MapUnmapCallback; static const uptr kAllocatorRegionSizeLog = 20; static const uptr kAllocatorNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog; -typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, - MapUnmapCallback> ByteMap; +using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, + LocalAddressSpaceView, MapUnmapCallback>; struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = kAllocatorRegionSizeLog; - typedef __tsan::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __tsan::ByteMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -80,6 +81,7 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. typedef DefaultSizeClassMap SizeClassMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64 PrimaryAllocator; #endif @@ -772,6 +774,7 @@ void ThreadFinalize(ThreadState *thr); void ThreadSetName(ThreadState *thr, const char *name); int ThreadCount(ThreadState *thr); void ProcessPendingSignals(ThreadState *thr); +void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid); Processor *ProcCreate(); void ProcDestroy(Processor *proc); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S index f1e4dbb50fb..31999cf346a 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -120,8 +120,10 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): add x0, x29, 32 eor x1, x2, x0 #else + adrp x2, ___tsan_darwin_setjmp_xor_key@page + ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] add x0, x29, 32 - mov x1, x0 + eor x1, x2, x0 #endif // call tsan interceptor @@ -178,8 +180,10 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): add x0, x29, 32 eor x1, x2, x0 #else + adrp x2, ___tsan_darwin_setjmp_xor_key@page + ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] add x0, x29, 32 - mov x1, x0 + eor x1, x2, x0 #endif // call tsan interceptor @@ -238,8 +242,10 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): add x0, x29, 32 eor x1, x2, x0 #else + adrp x2, ___tsan_darwin_setjmp_xor_key@page + ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] add x0, x29, 32 - mov x1, x0 + eor x1, x2, x0 #endif // call tsan interceptor diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S index 8af61bf0e89..34ef51c2a72 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S @@ -196,6 +196,7 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): #elif defined(__APPLE__) lea 16(%rsp), %rdi mov %rdi, %rsi + xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi #elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi @@ -244,6 +245,7 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): #elif defined(__APPLE__) lea 16(%rsp), %rdi mov %rdi, %rsi + xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi #elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi @@ -299,6 +301,7 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): #elif defined(__APPLE__) lea 32(%rsp), %rdi mov %rdi, %rsi + xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi #elif defined(__linux__) lea 32(%rsp), %rdi mov %rdi, %rsi diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc index e4d65b9a909..766a0f5a505 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc @@ -312,6 +312,12 @@ void ThreadDetach(ThreadState *thr, uptr pc, int tid) { ctx->thread_registry->DetachThread(tid, thr); } +void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid) { + CHECK_GT(tid, 0); + CHECK_LT(tid, kMaxTid); + ctx->thread_registry->SetThreadUserId(tid, uid); +} + void ThreadSetName(ThreadState *thr, const char *name) { ctx->thread_registry->SetThreadName(thr->tid, name); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc index be38f331ad9..6df074118a1 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc @@ -66,38 +66,30 @@ SuppressionContext *Suppressions() { } static const char *conv(ReportType typ) { - if (typ == ReportTypeRace) - return kSuppressionRace; - else if (typ == ReportTypeVptrRace) - return kSuppressionRace; - else if (typ == ReportTypeUseAfterFree) - return kSuppressionRace; - else if (typ == ReportTypeVptrUseAfterFree) - return kSuppressionRace; - else if (typ == ReportTypeExternalRace) - return kSuppressionRace; - else if (typ == ReportTypeThreadLeak) - return kSuppressionThread; - else if (typ == ReportTypeMutexDestroyLocked) - return kSuppressionMutex; - else if (typ == ReportTypeMutexDoubleLock) - return kSuppressionMutex; - else if (typ == ReportTypeMutexInvalidAccess) - return kSuppressionMutex; - else if (typ == ReportTypeMutexBadUnlock) - return kSuppressionMutex; - else if (typ == ReportTypeMutexBadReadLock) - return kSuppressionMutex; - else if (typ == ReportTypeMutexBadReadUnlock) - return kSuppressionMutex; - else if (typ == ReportTypeSignalUnsafe) - return kSuppressionSignal; - else if (typ == ReportTypeErrnoInSignal) - return kSuppressionNone; - else if (typ == ReportTypeDeadlock) - return kSuppressionDeadlock; - Printf("ThreadSanitizer: unknown report type %d\n", typ); - Die(); + switch (typ) { + case ReportTypeRace: + case ReportTypeVptrRace: + case ReportTypeUseAfterFree: + case ReportTypeVptrUseAfterFree: + case ReportTypeExternalRace: + return kSuppressionRace; + case ReportTypeThreadLeak: + return kSuppressionThread; + case ReportTypeMutexDestroyLocked: + case ReportTypeMutexDoubleLock: + case ReportTypeMutexInvalidAccess: + case ReportTypeMutexBadUnlock: + case ReportTypeMutexBadReadLock: + case ReportTypeMutexBadReadUnlock: + return kSuppressionMutex; + case ReportTypeSignalUnsafe: + case ReportTypeErrnoInSignal: + return kSuppressionSignal; + case ReportTypeDeadlock: + return kSuppressionDeadlock; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); } static uptr IsSuppressed(const char *stype, const AddressInfo &info, diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc index 5a7bdec2df1..ea82f89e129 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc @@ -21,6 +21,7 @@ UBSAN_CHECK(GenericUB, "undefined-behavior", "undefined") UBSAN_CHECK(NullPointerUse, "null-pointer-use", "null") UBSAN_CHECK(PointerOverflow, "pointer-overflow", "pointer-overflow") UBSAN_CHECK(MisalignedPointerUse, "misaligned-pointer-use", "alignment") +UBSAN_CHECK(AlignmentAssumption, "alignment-assumption", "alignment") UBSAN_CHECK(InsufficientObjectSize, "insufficient-object-size", "object-size") UBSAN_CHECK(SignedIntegerOverflow, "signed-integer-overflow", "signed-integer-overflow") @@ -30,8 +31,18 @@ UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero", "integer-divide-by-zero") UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero") UBSAN_CHECK(InvalidBuiltin, "invalid-builtin-use", "invalid-builtin-use") -UBSAN_CHECK(ImplicitIntegerTruncation, "implicit-integer-truncation", - "implicit-integer-truncation") +UBSAN_CHECK(ImplicitUnsignedIntegerTruncation, + "implicit-unsigned-integer-truncation", + "implicit-unsigned-integer-truncation") +UBSAN_CHECK(ImplicitSignedIntegerTruncation, + "implicit-signed-integer-truncation", + "implicit-signed-integer-truncation") +UBSAN_CHECK(ImplicitIntegerSignChange, + "implicit-integer-sign-change", + "implicit-integer-sign-change") +UBSAN_CHECK(ImplicitSignedIntegerTruncationOrSignChange, + "implicit-signed-integer-truncation-or-sign-change", + "implicit-signed-integer-truncation,implicit-integer-sign-change") UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base") UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent") UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds") diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc index 1638a054e8f..e75a4c44e62 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc @@ -25,5 +25,5 @@ UBSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") UBSAN_FLAG(bool, report_error_type, false, "Print specific error type instead of 'undefined-behavior' in summary.") UBSAN_FLAG(bool, silence_unsigned_overflow, false, - "Do not print error reports for unsigned integer overflow. " - "Used to provide fuzzing signal without blowing up logs.") + "Do not print non-fatal error reports for unsigned integer overflow. " + "Used to provide fuzzing signal without blowing up logs.") diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc index e72a32cf32c..11e09b0ff0f 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc @@ -106,6 +106,62 @@ void __ubsan::__ubsan_handle_type_mismatch_v1_abort(TypeMismatchData *Data, Die(); } +static void handleAlignmentAssumptionImpl(AlignmentAssumptionData *Data, + ValueHandle Pointer, + ValueHandle Alignment, + ValueHandle Offset, + ReportOptions Opts) { + Location Loc = Data->Loc.acquire(); + SourceLocation AssumptionLoc = Data->AssumptionLoc.acquire(); + + ErrorType ET = ErrorType::AlignmentAssumption; + + if (ignoreReport(Loc.getSourceLocation(), Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + uptr RealPointer = Pointer - Offset; + uptr LSB = LeastSignificantSetBitIndex(RealPointer); + uptr ActualAlignment = uptr(1) << LSB; + + uptr Mask = Alignment - 1; + uptr MisAlignmentOffset = RealPointer & Mask; + + if (!Offset) { + Diag(Loc, DL_Error, ET, + "assumption of %0 byte alignment for pointer of type %1 failed") + << Alignment << Data->Type; + } else { + Diag(Loc, DL_Error, ET, + "assumption of %0 byte alignment (with offset of %1 byte) for pointer " + "of type %2 failed") + << Alignment << Offset << Data->Type; + } + + if (!AssumptionLoc.isInvalid()) + Diag(AssumptionLoc, DL_Note, ET, "alignment assumption was specified here"); + + Diag(RealPointer, DL_Note, ET, + "%0address is %1 aligned, misalignment offset is %2 bytes") + << (Offset ? "offset " : "") << ActualAlignment << MisAlignmentOffset; +} + +void __ubsan::__ubsan_handle_alignment_assumption(AlignmentAssumptionData *Data, + ValueHandle Pointer, + ValueHandle Alignment, + ValueHandle Offset) { + GET_REPORT_OPTIONS(false); + handleAlignmentAssumptionImpl(Data, Pointer, Alignment, Offset, Opts); +} +void __ubsan::__ubsan_handle_alignment_assumption_abort( + AlignmentAssumptionData *Data, ValueHandle Pointer, ValueHandle Alignment, + ValueHandle Offset) { + GET_REPORT_OPTIONS(true); + handleAlignmentAssumptionImpl(Data, Pointer, Alignment, Offset, Opts); + Die(); +} + /// \brief Common diagnostic emission for various forms of integer overflow. template static void handleIntegerOverflowImpl(OverflowData *Data, ValueHandle LHS, @@ -119,7 +175,9 @@ static void handleIntegerOverflowImpl(OverflowData *Data, ValueHandle LHS, if (ignoreReport(Loc, Opts, ET)) return; - if (!IsSigned && flags()->silence_unsigned_overflow) + // If this is an unsigned overflow in non-fatal mode, potentially ignore it. + if (!IsSigned && !Opts.FromUnrecoverableHandler && + flags()->silence_unsigned_overflow) return; ScopedReport R(Opts, Loc, ET); @@ -457,18 +515,41 @@ static void handleImplicitConversion(ImplicitConversionData *Data, SourceLocation Loc = Data->Loc.acquire(); ErrorType ET = ErrorType::GenericUB; + const TypeDescriptor &SrcTy = Data->FromType; + const TypeDescriptor &DstTy = Data->ToType; + + bool SrcSigned = SrcTy.isSignedIntegerTy(); + bool DstSigned = DstTy.isSignedIntegerTy(); + switch (Data->Kind) { - case ICCK_IntegerTruncation: - ET = ErrorType::ImplicitIntegerTruncation; + case ICCK_IntegerTruncation: { // Legacy, no longer used. + // Let's figure out what it should be as per the new types, and upgrade. + // If both types are unsigned, then it's an unsigned truncation. + // Else, it is a signed truncation. + if (!SrcSigned && !DstSigned) { + ET = ErrorType::ImplicitUnsignedIntegerTruncation; + } else { + ET = ErrorType::ImplicitSignedIntegerTruncation; + } + break; + } + case ICCK_UnsignedIntegerTruncation: + ET = ErrorType::ImplicitUnsignedIntegerTruncation; + break; + case ICCK_SignedIntegerTruncation: + ET = ErrorType::ImplicitSignedIntegerTruncation; + break; + case ICCK_IntegerSignChange: + ET = ErrorType::ImplicitIntegerSignChange; + break; + case ICCK_SignedIntegerTruncationOrSignChange: + ET = ErrorType::ImplicitSignedIntegerTruncationOrSignChange; break; } if (ignoreReport(Loc, Opts, ET)) return; - const TypeDescriptor &SrcTy = Data->FromType; - const TypeDescriptor &DstTy = Data->ToType; - ScopedReport R(Opts, Loc, ET); // FIXME: is it possible to dump the values as hex with fixed width? @@ -477,8 +558,8 @@ static void handleImplicitConversion(ImplicitConversionData *Data, "implicit conversion from type %0 of value %1 (%2-bit, %3signed) to " "type %4 changed the value to %5 (%6-bit, %7signed)") << SrcTy << Value(SrcTy, Src) << SrcTy.getIntegerBitWidth() - << (SrcTy.isSignedIntegerTy() ? "" : "un") << DstTy << Value(DstTy, Dst) - << DstTy.getIntegerBitWidth() << (DstTy.isSignedIntegerTy() ? "" : "un"); + << (SrcSigned ? "" : "un") << DstTy << Value(DstTy, Dst) + << DstTy.getIntegerBitWidth() << (DstSigned ? "" : "un"); } void __ubsan::__ubsan_handle_implicit_conversion(ImplicitConversionData *Data, diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h index 07644b7ea15..2bf9ff4320e 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h @@ -39,6 +39,17 @@ struct TypeMismatchData { /// type. RECOVERABLE(type_mismatch_v1, TypeMismatchData *Data, ValueHandle Pointer) +struct AlignmentAssumptionData { + SourceLocation Loc; + SourceLocation AssumptionLoc; + const TypeDescriptor &Type; +}; + +/// \brief Handle a runtime alignment assumption check failure, +/// caused by a misaligned pointer. +RECOVERABLE(alignment_assumption, AlignmentAssumptionData *Data, + ValueHandle Pointer, ValueHandle Alignment, ValueHandle Offset) + struct OverflowData { SourceLocation Loc; const TypeDescriptor &Type; @@ -125,7 +136,11 @@ RECOVERABLE(load_invalid_value, InvalidValueData *Data, ValueHandle Val) /// Known implicit conversion check kinds. /// Keep in sync with the enum of the same name in CGExprScalar.cpp enum ImplicitConversionCheckKind : unsigned char { - ICCK_IntegerTruncation = 0, + ICCK_IntegerTruncation = 0, // Legacy, was only used by clang 7. + ICCK_UnsignedIntegerTruncation = 1, + ICCK_SignedIntegerTruncation = 2, + ICCK_IntegerSignChange = 3, + ICCK_SignedIntegerTruncationOrSignChange = 4, }; struct ImplicitConversionData { diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc index 0be6010ad27..81e06345dcf 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc @@ -10,6 +10,8 @@ //===----------------------------------------------------------------------===// INTERFACE_FUNCTION(__ubsan_handle_add_overflow) INTERFACE_FUNCTION(__ubsan_handle_add_overflow_abort) +INTERFACE_FUNCTION(__ubsan_handle_alignment_assumption) +INTERFACE_FUNCTION(__ubsan_handle_alignment_assumption_abort) INTERFACE_FUNCTION(__ubsan_handle_builtin_unreachable) INTERFACE_FUNCTION(__ubsan_handle_cfi_bad_type) INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail) diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc index e8fc3a8499b..ed62ddd0fa3 100644 --- a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc @@ -95,6 +95,7 @@ void NORETURN CheckFailed(const char *file, int, const char *cond, u64, u64) { HANDLER_NORECOVER(name, msg) HANDLER(type_mismatch, "type-mismatch") +HANDLER(alignment_assumption, "alignment-assumption") HANDLER(add_overflow, "add-overflow") HANDLER(sub_overflow, "sub-overflow") HANDLER(mul_overflow, "mul-overflow") diff --git a/contrib/compiler-rt/lib/xray/xray_allocator.h b/contrib/compiler-rt/lib/xray/xray_allocator.h index 8244815284a..907c54542a5 100644 --- a/contrib/compiler-rt/lib/xray/xray_allocator.h +++ b/contrib/compiler-rt/lib/xray/xray_allocator.h @@ -19,19 +19,132 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_mutex.h" +#if SANITIZER_FUCHSIA +#include +#include +#include +#else #include "sanitizer_common/sanitizer_posix.h" +#endif +#include "xray_defs.h" #include "xray_utils.h" -#include #include #include - -#ifndef MAP_NORESERVE -// no-op on NetBSD (at least), unsupported flag on FreeBSD basically because unneeded -#define MAP_NORESERVE 0 -#endif +#include namespace __xray { +// We implement our own memory allocation routine which will bypass the +// internal allocator. This allows us to manage the memory directly, using +// mmap'ed memory to back the allocators. +template T *allocate() XRAY_NEVER_INSTRUMENT { + uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + zx_handle_t Vmo; + zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", + sizeof(T), _zx_status_get_string(Status)); + return nullptr; + } + uintptr_t B; + Status = + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + Vmo, 0, sizeof(T), &B); + _zx_handle_close(Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", sizeof(T), + _zx_status_get_string(Status)); + return nullptr; + } + return reinterpret_cast(B); +#else + uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int ErrNo = 0; + if (UNLIKELY(internal_iserror(B, &ErrNo))) { + if (Verbosity()) + Report( + "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", + RoundedSize, B); + return nullptr; + } +#endif + return reinterpret_cast(B); +} + +template void deallocate(T *B) XRAY_NEVER_INSTRUMENT { + if (B == nullptr) + return; + uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast(B), + RoundedSize); +#else + internal_munmap(B, RoundedSize); +#endif +} + +template +T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT { + uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + zx_handle_t Vmo; + zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", S, + _zx_status_get_string(Status)); + return nullptr; + } + uintptr_t B; + Status = _zx_vmar_map(_zx_vmar_root_self(), + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, Vmo, 0, S, &B); + _zx_handle_close(Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", S, + _zx_status_get_string(Status)); + return nullptr; + } +#else + uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int ErrNo = 0; + if (UNLIKELY(internal_iserror(B, &ErrNo))) { + if (Verbosity()) + Report( + "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", + RoundedSize, B); + return nullptr; + } +#endif + return reinterpret_cast(B); +} + +template void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT { + if (B == nullptr) + return; + uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast(B), + RoundedSize); +#else + internal_munmap(B, RoundedSize); +#endif +} + +template +T *initArray(size_t N, U &&... Us) XRAY_NEVER_INSTRUMENT { + auto A = allocateBuffer(N); + if (A != nullptr) + while (N > 0) + new (A + (--N)) T(std::forward(Us)...); + return A; +} + /// The Allocator type hands out fixed-sized chunks of memory that are /// cache-line aligned and sized. This is useful for placement of /// performance-sensitive data in memory that's frequently accessed. The @@ -58,20 +171,18 @@ template struct Allocator { }; private: - const size_t MaxMemory{0}; - void *BackingStore = nullptr; - void *AlignedNextBlock = nullptr; + size_t MaxMemory{0}; + unsigned char *BackingStore = nullptr; + unsigned char *AlignedNextBlock = nullptr; size_t AllocatedBlocks = 0; + bool Owned; SpinMutex Mutex{}; - void *Alloc() { + void *Alloc() XRAY_NEVER_INSTRUMENT { SpinMutexLock Lock(&Mutex); if (UNLIKELY(BackingStore == nullptr)) { - BackingStore = reinterpret_cast( - internal_mmap(NULL, MaxMemory, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, 0, 0)); - if (BackingStore == MAP_FAILED) { - BackingStore = nullptr; + BackingStore = allocateBuffer(MaxMemory); + if (BackingStore == nullptr) { if (Verbosity()) Report("XRay Profiling: Failed to allocate memory for allocator.\n"); return nullptr; @@ -84,7 +195,7 @@ private: auto AlignedNextBlockNum = nearest_boundary( reinterpret_cast(AlignedNextBlock), kCacheLineSize); if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) { - munmap(BackingStore, MaxMemory); + deallocateBuffer(BackingStore, MaxMemory); AlignedNextBlock = BackingStore = nullptr; if (Verbosity()) Report("XRay Profiling: Cannot obtain enough memory from " @@ -92,34 +203,83 @@ private: return nullptr; } - AlignedNextBlock = reinterpret_cast(AlignedNextBlockNum); + AlignedNextBlock = reinterpret_cast(AlignedNextBlockNum); // Assert that AlignedNextBlock is cache-line aligned. DCHECK_EQ(reinterpret_cast(AlignedNextBlock) % kCacheLineSize, 0); } - if ((AllocatedBlocks * Block::Size) >= MaxMemory) + if (((AllocatedBlocks + 1) * Block::Size) > MaxMemory) return nullptr; // Align the pointer we'd like to return to an appropriate alignment, then // advance the pointer from where to start allocations. void *Result = AlignedNextBlock; - AlignedNextBlock = reinterpret_cast( - reinterpret_cast(AlignedNextBlock) + N); + AlignedNextBlock = + reinterpret_cast(AlignedNextBlock) + Block::Size; ++AllocatedBlocks; return Result; } public: - explicit Allocator(size_t M) - : MaxMemory(nearest_boundary(M, kCacheLineSize)) {} + explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT + : MaxMemory(RoundUpTo(M, kCacheLineSize)), + BackingStore(nullptr), + AlignedNextBlock(nullptr), + AllocatedBlocks(0), + Owned(true), + Mutex() {} - Block Allocate() { return {Alloc()}; } + explicit Allocator(void *P, size_t M) XRAY_NEVER_INSTRUMENT + : MaxMemory(M), + BackingStore(reinterpret_cast(P)), + AlignedNextBlock(reinterpret_cast(P)), + AllocatedBlocks(0), + Owned(false), + Mutex() {} - ~Allocator() NOEXCEPT { - if (BackingStore != nullptr) { - internal_munmap(BackingStore, MaxMemory); + Allocator(const Allocator &) = delete; + Allocator &operator=(const Allocator &) = delete; + + Allocator(Allocator &&O) XRAY_NEVER_INSTRUMENT { + SpinMutexLock L0(&Mutex); + SpinMutexLock L1(&O.Mutex); + MaxMemory = O.MaxMemory; + O.MaxMemory = 0; + BackingStore = O.BackingStore; + O.BackingStore = nullptr; + AlignedNextBlock = O.AlignedNextBlock; + O.AlignedNextBlock = nullptr; + AllocatedBlocks = O.AllocatedBlocks; + O.AllocatedBlocks = 0; + Owned = O.Owned; + O.Owned = false; + } + + Allocator &operator=(Allocator &&O) XRAY_NEVER_INSTRUMENT { + SpinMutexLock L0(&Mutex); + SpinMutexLock L1(&O.Mutex); + MaxMemory = O.MaxMemory; + O.MaxMemory = 0; + if (BackingStore != nullptr) + deallocateBuffer(BackingStore, MaxMemory); + BackingStore = O.BackingStore; + O.BackingStore = nullptr; + AlignedNextBlock = O.AlignedNextBlock; + O.AlignedNextBlock = nullptr; + AllocatedBlocks = O.AllocatedBlocks; + O.AllocatedBlocks = 0; + Owned = O.Owned; + O.Owned = false; + return *this; + } + + Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; } + + ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT { + if (Owned && BackingStore != nullptr) { + deallocateBuffer(BackingStore, MaxMemory); } } }; diff --git a/contrib/compiler-rt/lib/xray/xray_basic_logging.cc b/contrib/compiler-rt/lib/xray/xray_basic_logging.cc index 585ca641cd0..ae1cc0ba79d 100644 --- a/contrib/compiler-rt/lib/xray/xray_basic_logging.cc +++ b/contrib/compiler-rt/lib/xray/xray_basic_logging.cc @@ -19,7 +19,9 @@ #include #include #include +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC #include +#endif #include #include #include @@ -38,8 +40,9 @@ namespace __xray { -SpinMutex LogMutex; +static SpinMutex LogMutex; +namespace { // We use elements of this type to record the entry TSC of every function ID we // see as we're tracing a particular thread's execution. struct alignas(16) StackEntry { @@ -52,21 +55,28 @@ struct alignas(16) StackEntry { static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry"); -struct alignas(64) ThreadLocalData { +struct XRAY_TLS_ALIGNAS(64) ThreadLocalData { void *InMemoryBuffer = nullptr; size_t BufferSize = 0; size_t BufferOffset = 0; void *ShadowStack = nullptr; size_t StackSize = 0; size_t StackEntries = 0; - int Fd = -1; + __xray::LogWriter *LogWriter = nullptr; }; +struct BasicLoggingOptions { + int DurationFilterMicros = 0; + size_t MaxStackDepth = 0; + size_t ThreadBufferSize = 0; +}; +} // namespace + static pthread_key_t PThreadKey; static atomic_uint8_t BasicInitialized{0}; -BasicLoggingOptions GlobalOptions; +struct BasicLoggingOptions GlobalOptions; thread_local atomic_uint8_t Guard{0}; @@ -75,10 +85,10 @@ static atomic_uint64_t ThresholdTicks{0}; static atomic_uint64_t TicksPerSec{0}; static atomic_uint64_t CycleFrequency{NanosecondsPerSecond}; -static int openLogFile() XRAY_NEVER_INSTRUMENT { - int F = getLogFD(); - if (F == -1) - return -1; +static LogWriter *getLog() XRAY_NEVER_INSTRUMENT { + LogWriter* LW = LogWriter::Open(); + if (LW == nullptr) + return LW; static pthread_once_t DetectOnce = PTHREAD_ONCE_INIT; pthread_once(&DetectOnce, +[] { @@ -100,16 +110,16 @@ static int openLogFile() XRAY_NEVER_INSTRUMENT { // before setting the values in the header. Header.ConstantTSC = 1; Header.NonstopTSC = 1; - retryingWriteAll(F, reinterpret_cast(&Header), - reinterpret_cast(&Header) + sizeof(Header)); - return F; + LW->WriteAll(reinterpret_cast(&Header), + reinterpret_cast(&Header) + sizeof(Header)); + return LW; } -static int getGlobalFd() XRAY_NEVER_INSTRUMENT { +static LogWriter *getGlobalLog() XRAY_NEVER_INSTRUMENT { static pthread_once_t OnceInit = PTHREAD_ONCE_INIT; - static int Fd = 0; - pthread_once(&OnceInit, +[] { Fd = openLogFile(); }); - return Fd; + static LogWriter *LW = nullptr; + pthread_once(&OnceInit, +[] { LW = getLog(); }); + return LW; } static ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { @@ -121,7 +131,7 @@ static ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { return false; } pthread_setspecific(PThreadKey, &TLD); - TLD.Fd = getGlobalFd(); + TLD.LogWriter = getGlobalLog(); TLD.InMemoryBuffer = reinterpret_cast( InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize, nullptr, alignof(XRayRecord))); @@ -149,8 +159,8 @@ template void InMemoryRawLog(int32_t FuncId, XRayEntryType Type, RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { auto &TLD = getThreadLocalData(); - int Fd = getGlobalFd(); - if (Fd == -1) + LogWriter *LW = getGlobalLog(); + if (LW == nullptr) return; // Use a simple recursion guard, to handle cases where we're already logging @@ -234,9 +244,9 @@ void InMemoryRawLog(int32_t FuncId, XRayEntryType Type, auto FirstEntry = reinterpret_cast(TLD.InMemoryBuffer); internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R)); if (++TLD.BufferOffset == TLD.BufferSize) { - SpinMutexLock L(&LogMutex); - retryingWriteAll(Fd, reinterpret_cast(FirstEntry), - reinterpret_cast(FirstEntry + TLD.BufferOffset)); + SpinMutexLock Lock(&LogMutex); + LW->WriteAll(reinterpret_cast(FirstEntry), + reinterpret_cast(FirstEntry + TLD.BufferOffset)); TLD.BufferOffset = 0; TLD.StackEntries = 0; } @@ -249,17 +259,17 @@ void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1, auto FirstEntry = reinterpret_cast(TLD.InMemoryBuffer); const auto &BuffLen = TLD.BufferSize; - int Fd = getGlobalFd(); - if (Fd == -1) + LogWriter *LW = getGlobalLog(); + if (LW == nullptr) return; // First we check whether there's enough space to write the data consecutively // in the thread-local buffer. If not, we first flush the buffer before // attempting to write the two records that must be consecutive. if (TLD.BufferOffset + 2 > BuffLen) { - SpinMutexLock L(&LogMutex); - retryingWriteAll(Fd, reinterpret_cast(FirstEntry), - reinterpret_cast(FirstEntry + TLD.BufferOffset)); + SpinMutexLock Lock(&LogMutex); + LW->WriteAll(reinterpret_cast(FirstEntry), + reinterpret_cast(FirstEntry + TLD.BufferOffset)); TLD.BufferOffset = 0; TLD.StackEntries = 0; } @@ -280,9 +290,9 @@ void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1, R.Arg = Arg1; internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R)); if (++TLD.BufferOffset == BuffLen) { - SpinMutexLock L(&LogMutex); - retryingWriteAll(Fd, reinterpret_cast(FirstEntry), - reinterpret_cast(FirstEntry + TLD.BufferOffset)); + SpinMutexLock Lock(&LogMutex); + LW->WriteAll(reinterpret_cast(FirstEntry), + reinterpret_cast(FirstEntry + TLD.BufferOffset)); TLD.BufferOffset = 0; TLD.StackEntries = 0; } @@ -339,29 +349,29 @@ static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT { Report("Cleaned up log for TID: %d\n", GetTid()); }); - if (TLD.Fd == -1 || TLD.BufferOffset == 0) { + if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) { if (Verbosity()) - Report("Skipping buffer for TID: %d; Fd = %d; Offset = %llu\n", GetTid(), - TLD.Fd, TLD.BufferOffset); + Report("Skipping buffer for TID: %d; Offset = %llu\n", GetTid(), + TLD.BufferOffset); return; } { SpinMutexLock L(&LogMutex); - retryingWriteAll(TLD.Fd, reinterpret_cast(TLD.InMemoryBuffer), - reinterpret_cast(TLD.InMemoryBuffer) + - (sizeof(XRayRecord) * TLD.BufferOffset)); + TLD.LogWriter->WriteAll(reinterpret_cast(TLD.InMemoryBuffer), + reinterpret_cast(TLD.InMemoryBuffer) + + (sizeof(XRayRecord) * TLD.BufferOffset)); } // Because this thread's exit could be the last one trying to write to // the file and that we're not able to close out the file properly, we // sync instead and hope that the pending writes are flushed as the // thread exits. - fsync(TLD.Fd); + TLD.LogWriter->Flush(); } -XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, - void *Options, +XRayLogInitStatus basicLoggingInit(UNUSED size_t BufferSize, + UNUSED size_t BufferMax, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { uint8_t Expected = 0; if (!atomic_compare_exchange_strong(&BasicInitialized, &Expected, 1, @@ -385,43 +395,32 @@ XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, "using emulation instead.\n"); }); - if (BufferSize == 0 && BufferMax == 0 && Options != nullptr) { - FlagParser P; - BasicFlags F; - F.setDefaults(); - registerXRayBasicFlags(&P, &F); - P.ParseString(useCompilerDefinedBasicFlags()); - auto *EnvOpts = GetEnv("XRAY_BASIC_OPTIONS"); - if (EnvOpts == nullptr) - EnvOpts = ""; + FlagParser P; + BasicFlags F; + F.setDefaults(); + registerXRayBasicFlags(&P, &F); + P.ParseString(useCompilerDefinedBasicFlags()); + auto *EnvOpts = GetEnv("XRAY_BASIC_OPTIONS"); + if (EnvOpts == nullptr) + EnvOpts = ""; - P.ParseString(EnvOpts); + P.ParseString(EnvOpts); - // If XRAY_BASIC_OPTIONS was not defined, then we use the deprecated options - // set through XRAY_OPTIONS instead. - if (internal_strlen(EnvOpts) == 0) { - F.func_duration_threshold_us = - flags()->xray_naive_log_func_duration_threshold_us; - F.max_stack_depth = flags()->xray_naive_log_max_stack_depth; - F.thread_buffer_size = flags()->xray_naive_log_thread_buffer_size; - } - - P.ParseString(static_cast(Options)); - GlobalOptions.ThreadBufferSize = F.thread_buffer_size; - GlobalOptions.DurationFilterMicros = F.func_duration_threshold_us; - GlobalOptions.MaxStackDepth = F.max_stack_depth; - *basicFlags() = F; - } else if (OptionsSize != sizeof(BasicLoggingOptions)) { - Report("Invalid options size, potential ABI mismatch; expected %d got %d", - sizeof(BasicLoggingOptions), OptionsSize); - return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; - } else { - if (Verbosity()) - Report("XRay Basic: struct-based init is deprecated, please use " - "string-based configuration instead.\n"); - GlobalOptions = *reinterpret_cast(Options); + // If XRAY_BASIC_OPTIONS was not defined, then we use the deprecated options + // set through XRAY_OPTIONS instead. + if (internal_strlen(EnvOpts) == 0) { + F.func_duration_threshold_us = + flags()->xray_naive_log_func_duration_threshold_us; + F.max_stack_depth = flags()->xray_naive_log_max_stack_depth; + F.thread_buffer_size = flags()->xray_naive_log_thread_buffer_size; } + P.ParseString(static_cast(Options)); + GlobalOptions.ThreadBufferSize = F.thread_buffer_size; + GlobalOptions.DurationFilterMicros = F.func_duration_threshold_us; + GlobalOptions.MaxStackDepth = F.max_stack_depth; + *basicFlags() = F; + atomic_store(&ThresholdTicks, atomic_load(&TicksPerSec, memory_order_acquire) * GlobalOptions.DurationFilterMicros / 1000000, diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc index 3ce72890078..7d0e5a1f323 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc @@ -13,141 +13,206 @@ // //===----------------------------------------------------------------------===// #include "xray_buffer_queue.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#if !SANITIZER_FUCHSIA #include "sanitizer_common/sanitizer_posix.h" +#endif +#include "xray_allocator.h" +#include "xray_defs.h" #include #include -#ifndef MAP_NORESERVE -// no-op on NetBSD (at least), unsupported flag on FreeBSD -#define MAP_NORESERVE 0 -#endif - using namespace __xray; -using namespace __sanitizer; -template static T *allocRaw(size_t N) { - // TODO: Report errors? - // We use MAP_NORESERVE on platforms where it's supported to ensure that the - // pages we're allocating for XRay never end up in pages that can be swapped - // in/out. We're doing this because for FDR mode, we want to ensure that - // writes to the buffers stay resident in memory to prevent XRay itself from - // causing swapping/thrashing. - // - // In the case when XRay pages cannot be swapped in/out or there's not enough - // RAM to back these pages, we're willing to cause a segmentation fault - // instead of introducing latency in the measurement. We assume here that - // there are enough pages that are swappable in/out outside of the buffers - // being used by FDR mode (which are bounded and configurable anyway) to allow - // us to keep using always-resident memory. - // - // TODO: Make this configurable? - void *A = reinterpret_cast( - internal_mmap(NULL, N * sizeof(T), PROT_WRITE | PROT_READ, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0)); - return (A == MAP_FAILED) ? nullptr : reinterpret_cast(A); +namespace { + +BufferQueue::ControlBlock *allocControlBlock(size_t Size, size_t Count) { + auto B = + allocateBuffer((sizeof(BufferQueue::ControlBlock) - 1) + (Size * Count)); + return B == nullptr ? nullptr + : reinterpret_cast(B); } -template static void deallocRaw(T *ptr, size_t N) { - // TODO: Report errors? - if (ptr != nullptr) - internal_munmap(ptr, N); +void deallocControlBlock(BufferQueue::ControlBlock *C, size_t Size, + size_t Count) { + deallocateBuffer(reinterpret_cast(C), + (sizeof(BufferQueue::ControlBlock) - 1) + (Size * Count)); } -template static T *initArray(size_t N) { - auto A = allocRaw(N); - if (A != nullptr) - while (N > 0) - new (A + (--N)) T(); - return A; -} - -BufferQueue::BufferQueue(size_t B, size_t N, bool &Success) - : BufferSize(B), Buffers(initArray(N)), - BufferCount(N), Finalizing{0}, OwnedBuffers(initArray(N)), - Next(Buffers), First(Buffers), LiveBuffers(0) { - if (Buffers == nullptr) { - Success = false; +void decRefCount(BufferQueue::ControlBlock *C, size_t Size, size_t Count) { + if (C == nullptr) return; - } - if (OwnedBuffers == nullptr) { - // Clean up the buffers we've already allocated. - for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B) - B->~BufferRep(); - deallocRaw(Buffers, N); - Success = false; + if (atomic_fetch_sub(&C->RefCount, 1, memory_order_acq_rel) == 1) + deallocControlBlock(C, Size, Count); +} + +void incRefCount(BufferQueue::ControlBlock *C) { + if (C == nullptr) return; + atomic_fetch_add(&C->RefCount, 1, memory_order_acq_rel); +} + +// We use a struct to ensure that we are allocating one atomic_uint64_t per +// cache line. This allows us to not worry about false-sharing among atomic +// objects being updated (constantly) by different threads. +struct ExtentsPadded { + union { + atomic_uint64_t Extents; + unsigned char Storage[kCacheLineSize]; }; +}; - for (size_t i = 0; i < N; ++i) { +constexpr size_t kExtentsSize = sizeof(ExtentsPadded); + +} // namespace + +BufferQueue::ErrorCode BufferQueue::init(size_t BS, size_t BC) { + SpinMutexLock Guard(&Mutex); + + if (!finalizing()) + return BufferQueue::ErrorCode::AlreadyInitialized; + + cleanupBuffers(); + + bool Success = false; + BufferSize = BS; + BufferCount = BC; + + BackingStore = allocControlBlock(BufferSize, BufferCount); + if (BackingStore == nullptr) + return BufferQueue::ErrorCode::NotEnoughMemory; + + auto CleanupBackingStore = at_scope_exit([&, this] { + if (Success) + return; + deallocControlBlock(BackingStore, BufferSize, BufferCount); + BackingStore = nullptr; + }); + + // Initialize enough atomic_uint64_t instances, each + ExtentsBackingStore = allocControlBlock(kExtentsSize, BufferCount); + if (ExtentsBackingStore == nullptr) + return BufferQueue::ErrorCode::NotEnoughMemory; + + auto CleanupExtentsBackingStore = at_scope_exit([&, this] { + if (Success) + return; + deallocControlBlock(ExtentsBackingStore, kExtentsSize, BufferCount); + ExtentsBackingStore = nullptr; + }); + + Buffers = initArray(BufferCount); + if (Buffers == nullptr) + return BufferQueue::ErrorCode::NotEnoughMemory; + + // At this point we increment the generation number to associate the buffers + // to the new generation. + atomic_fetch_add(&Generation, 1, memory_order_acq_rel); + + // First, we initialize the refcount in the ControlBlock, which we treat as + // being at the start of the BackingStore pointer. + atomic_store(&BackingStore->RefCount, 1, memory_order_release); + atomic_store(&ExtentsBackingStore->RefCount, 1, memory_order_release); + + // Then we initialise the individual buffers that sub-divide the whole backing + // store. Each buffer will start at the `Data` member of the ControlBlock, and + // will be offsets from these locations. + for (size_t i = 0; i < BufferCount; ++i) { auto &T = Buffers[i]; - void *Tmp = allocRaw(BufferSize); - if (Tmp == nullptr) { - Success = false; - return; - } - auto *Extents = allocRaw(1); - if (Extents == nullptr) { - Success = false; - return; - } auto &Buf = T.Buff; - Buf.Data = Tmp; - Buf.Size = B; - Buf.Extents = Extents; - OwnedBuffers[i] = Tmp; + auto *E = reinterpret_cast(&ExtentsBackingStore->Data + + (kExtentsSize * i)); + Buf.Extents = &E->Extents; + atomic_store(Buf.Extents, 0, memory_order_release); + Buf.Generation = generation(); + Buf.Data = &BackingStore->Data + (BufferSize * i); + Buf.Size = BufferSize; + Buf.BackingStore = BackingStore; + Buf.ExtentsBackingStore = ExtentsBackingStore; + Buf.Count = BufferCount; + T.Used = false; } + + Next = Buffers; + First = Buffers; + LiveBuffers = 0; + atomic_store(&Finalizing, 0, memory_order_release); Success = true; + return BufferQueue::ErrorCode::Ok; +} + +BufferQueue::BufferQueue(size_t B, size_t N, + bool &Success) XRAY_NEVER_INSTRUMENT + : BufferSize(B), + BufferCount(N), + Mutex(), + Finalizing{1}, + BackingStore(nullptr), + ExtentsBackingStore(nullptr), + Buffers(nullptr), + Next(Buffers), + First(Buffers), + LiveBuffers(0), + Generation{0} { + Success = init(B, N) == BufferQueue::ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) { if (atomic_load(&Finalizing, memory_order_acquire)) return ErrorCode::QueueFinalizing; - SpinMutexLock Guard(&Mutex); - if (LiveBuffers == BufferCount) - return ErrorCode::NotEnoughMemory; - auto &T = *Next; - auto &B = T.Buff; - Buf = B; - T.Used = true; - ++LiveBuffers; - - if (++Next == (Buffers + BufferCount)) - Next = Buffers; + BufferRep *B = nullptr; + { + SpinMutexLock Guard(&Mutex); + if (LiveBuffers == BufferCount) + return ErrorCode::NotEnoughMemory; + B = Next++; + if (Next == (Buffers + BufferCount)) + Next = Buffers; + ++LiveBuffers; + } + incRefCount(BackingStore); + incRefCount(ExtentsBackingStore); + Buf = B->Buff; + Buf.Generation = generation(); + B->Used = true; return ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) { - // Blitz through the buffers array to find the buffer. - bool Found = false; - for (auto I = OwnedBuffers, E = OwnedBuffers + BufferCount; I != E; ++I) { - if (*I == Buf.Data) { - Found = true; - break; + // Check whether the buffer being referred to is within the bounds of the + // backing store's range. + BufferRep *B = nullptr; + { + SpinMutexLock Guard(&Mutex); + if (Buf.Generation != generation() || LiveBuffers == 0) { + Buf = {}; + decRefCount(Buf.BackingStore, Buf.Size, Buf.Count); + decRefCount(Buf.ExtentsBackingStore, kExtentsSize, Buf.Count); + return BufferQueue::ErrorCode::Ok; } + + if (Buf.Data < &BackingStore->Data || + Buf.Data > &BackingStore->Data + (BufferCount * BufferSize)) + return BufferQueue::ErrorCode::UnrecognizedBuffer; + + --LiveBuffers; + B = First++; + if (First == (Buffers + BufferCount)) + First = Buffers; } - if (!Found) - return ErrorCode::UnrecognizedBuffer; - - SpinMutexLock Guard(&Mutex); - - // This points to a semantic bug, we really ought to not be releasing more - // buffers than we actually get. - if (LiveBuffers == 0) - return ErrorCode::NotEnoughMemory; // Now that the buffer has been released, we mark it as "used". - First->Buff = Buf; - First->Used = true; - Buf.Data = nullptr; - Buf.Size = 0; - --LiveBuffers; - if (++First == (Buffers + BufferCount)) - First = Buffers; - + B->Buff = Buf; + B->Used = true; + decRefCount(Buf.BackingStore, Buf.Size, Buf.Count); + decRefCount(Buf.ExtentsBackingStore, kExtentsSize, Buf.Count); + atomic_store(B->Buff.Extents, atomic_load(Buf.Extents, memory_order_acquire), + memory_order_release); + Buf = {}; return ErrorCode::Ok; } @@ -157,15 +222,17 @@ BufferQueue::ErrorCode BufferQueue::finalize() { return ErrorCode::Ok; } -BufferQueue::~BufferQueue() { - for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { - auto &T = *I; - auto &Buf = T.Buff; - deallocRaw(Buf.Data, Buf.Size); - deallocRaw(Buf.Extents, 1); - } +void BufferQueue::cleanupBuffers() { for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B) B->~BufferRep(); - deallocRaw(Buffers, BufferCount); - deallocRaw(OwnedBuffers, BufferCount); + deallocateBuffer(Buffers, BufferCount); + decRefCount(BackingStore, BufferSize, BufferCount); + decRefCount(ExtentsBackingStore, kExtentsSize, BufferCount); + BackingStore = nullptr; + ExtentsBackingStore = nullptr; + Buffers = nullptr; + BufferCount = 0; + BufferSize = 0; } + +BufferQueue::~BufferQueue() { cleanupBuffers(); } diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h index e76fa7983c9..ef2b433f9a3 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h @@ -18,25 +18,51 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_mutex.h" +#include "xray_defs.h" #include +#include namespace __xray { /// BufferQueue implements a circular queue of fixed sized buffers (much like a -/// freelist) but is concerned mostly with making it really quick to initialise, -/// finalise, and get/return buffers to the queue. This is one key component of -/// the "flight data recorder" (FDR) mode to support ongoing XRay function call +/// freelist) but is concerned with making it quick to initialise, finalise, and +/// get from or return buffers to the queue. This is one key component of the +/// "flight data recorder" (FDR) mode to support ongoing XRay function call /// trace collection. class BufferQueue { public: - struct alignas(64) BufferExtents { - atomic_uint64_t Size; + /// ControlBlock represents the memory layout of how we interpret the backing + /// store for all buffers and extents managed by a BufferQueue instance. The + /// ControlBlock has the reference count as the first member, sized according + /// to platform-specific cache-line size. We never use the Buffer member of + /// the union, which is only there for compiler-supported alignment and + /// sizing. + /// + /// This ensures that the `Data` member will be placed at least kCacheLineSize + /// bytes from the beginning of the structure. + struct ControlBlock { + union { + atomic_uint64_t RefCount; + char Buffer[kCacheLineSize]; + }; + + /// We need to make this size 1, to conform to the C++ rules for array data + /// members. Typically, we want to subtract this 1 byte for sizing + /// information. + char Data[1]; }; struct Buffer { + atomic_uint64_t *Extents = nullptr; + uint64_t Generation{0}; void *Data = nullptr; size_t Size = 0; - BufferExtents *Extents; + + private: + friend class BufferQueue; + ControlBlock *BackingStore = nullptr; + ControlBlock *ExtentsBackingStore = nullptr; + size_t Count = 0; }; struct BufferRep { @@ -76,8 +102,10 @@ private: T *operator->() const { return &(Buffers[Offset].Buff); } - Iterator(BufferRep *Root, size_t O, size_t M) - : Buffers(Root), Offset(O), Max(M) { + Iterator(BufferRep *Root, size_t O, size_t M) XRAY_NEVER_INSTRUMENT + : Buffers(Root), + Offset(O), + Max(M) { // We want to advance to the first Offset where the 'Used' property is // true, or to the end of the list/queue. while (!Buffers[Offset].Used && Offset != Max) { @@ -107,16 +135,20 @@ private: // Size of each individual Buffer. size_t BufferSize; - BufferRep *Buffers; - // Amount of pre-allocated buffers. size_t BufferCount; SpinMutex Mutex; atomic_uint8_t Finalizing; - // Pointers to buffers managed/owned by the BufferQueue. - void **OwnedBuffers; + // The collocated ControlBlock and buffer storage. + ControlBlock *BackingStore; + + // The collocated ControlBlock and extents storage. + ControlBlock *ExtentsBackingStore; + + // A dynamically allocated array of BufferRep instances. + BufferRep *Buffers; // Pointer to the next buffer to be handed out. BufferRep *Next; @@ -128,6 +160,13 @@ private: // Count of buffers that have been handed out through 'getBuffer'. size_t LiveBuffers; + // We use a generation number to identify buffers and which generation they're + // associated with. + atomic_uint64_t Generation; + + /// Releases references to the buffers backed by the current buffer queue. + void cleanupBuffers(); + public: enum class ErrorCode : unsigned { Ok, @@ -135,6 +174,7 @@ public: QueueFinalizing, UnrecognizedBuffer, AlreadyFinalized, + AlreadyInitialized, }; static const char *getErrorString(ErrorCode E) { @@ -149,6 +189,8 @@ public: return "buffer being returned not owned by buffer queue"; case ErrorCode::AlreadyFinalized: return "queue already finalized"; + case ErrorCode::AlreadyInitialized: + return "queue already initialized"; } return "unknown error"; } @@ -179,10 +221,23 @@ public: /// the buffer being released. ErrorCode releaseBuffer(Buffer &Buf); + /// Initializes the buffer queue, starting a new generation. We can re-set the + /// size of buffers with |BS| along with the buffer count with |BC|. + /// + /// Returns: + /// - ErrorCode::Ok when we successfully initialize the buffer. This + /// requires that the buffer queue is previously finalized. + /// - ErrorCode::AlreadyInitialized when the buffer queue is not finalized. + ErrorCode init(size_t BS, size_t BC); + bool finalizing() const { return atomic_load(&Finalizing, memory_order_acquire); } + uint64_t generation() const { + return atomic_load(&Generation, memory_order_acquire); + } + /// Returns the configured size of the buffers in the buffer queue. size_t ConfiguredBufferSize() const { return BufferSize; } @@ -198,7 +253,7 @@ public: /// Applies the provided function F to each Buffer in the queue, only if the /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a /// releaseBuffer(...) operation). - template void apply(F Fn) { + template void apply(F Fn) XRAY_NEVER_INSTRUMENT { SpinMutexLock G(&Mutex); for (auto I = begin(), E = end(); I != E; ++I) Fn(*I); diff --git a/contrib/compiler-rt/lib/xray/xray_defs.h b/contrib/compiler-rt/lib/xray/xray_defs.h index e5c37c0665d..c009bcc879f 100644 --- a/contrib/compiler-rt/lib/xray/xray_defs.h +++ b/contrib/compiler-rt/lib/xray/xray_defs.h @@ -19,4 +19,14 @@ #define XRAY_NEVER_INSTRUMENT #endif +#if SANITIZER_NETBSD +// NetBSD: thread_local is not aligned properly, and the code relying +// on it segfaults +#define XRAY_TLS_ALIGNAS(x) +#define XRAY_HAS_TLS_ALIGNAS 0 +#else +#define XRAY_TLS_ALIGNAS(x) alignas(x) +#define XRAY_HAS_TLS_ALIGNAS 1 +#endif + #endif // XRAY_XRAY_DEFS_H diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_controller.h b/contrib/compiler-rt/lib/xray/xray_fdr_controller.h new file mode 100644 index 00000000000..d44d0309b37 --- /dev/null +++ b/contrib/compiler-rt/lib/xray/xray_fdr_controller.h @@ -0,0 +1,373 @@ +//===-- xray_fdr_controller.h ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a function call tracing system. +// +//===----------------------------------------------------------------------===// +#ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ +#define COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ + +#include +#include + +#include "xray/xray_interface.h" +#include "xray/xray_records.h" +#include "xray_buffer_queue.h" +#include "xray_fdr_log_writer.h" + +namespace __xray { + +template class FDRController { + BufferQueue *BQ; + BufferQueue::Buffer &B; + FDRLogWriter &W; + int (*WallClockReader)(clockid_t, struct timespec *) = 0; + uint64_t CycleThreshold = 0; + + uint64_t LastFunctionEntryTSC = 0; + uint64_t LatestTSC = 0; + uint16_t LatestCPU = 0; + tid_t TId = 0; + pid_t PId = 0; + bool First = true; + + uint32_t UndoableFunctionEnters = 0; + uint32_t UndoableTailExits = 0; + + bool finalized() const XRAY_NEVER_INSTRUMENT { + return BQ == nullptr || BQ->finalizing(); + } + + bool hasSpace(size_t S) XRAY_NEVER_INSTRUMENT { + return B.Data != nullptr && B.Generation == BQ->generation() && + W.getNextRecord() + S <= reinterpret_cast(B.Data) + B.Size; + } + + constexpr int32_t mask(int32_t FuncId) const XRAY_NEVER_INSTRUMENT { + return FuncId & ((1 << 29) - 1); + } + + bool getNewBuffer() XRAY_NEVER_INSTRUMENT { + if (BQ->getBuffer(B) != BufferQueue::ErrorCode::Ok) + return false; + + W.resetRecord(); + DCHECK_EQ(W.getNextRecord(), B.Data); + LatestTSC = 0; + LatestCPU = 0; + First = true; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + atomic_store(B.Extents, 0, memory_order_release); + return true; + } + + bool setupNewBuffer() XRAY_NEVER_INSTRUMENT { + if (finalized()) + return false; + + DCHECK(hasSpace(sizeof(MetadataRecord) * 3)); + TId = GetTid(); + PId = internal_getpid(); + struct timespec TS { + 0, 0 + }; + WallClockReader(CLOCK_MONOTONIC, &TS); + + MetadataRecord Metadata[] = { + // Write out a MetadataRecord to signify that this is the start of a new + // buffer, associated with a particular thread, with a new CPU. For the + // data, we have 15 bytes to squeeze as much information as we can. At + // this point we only write down the following bytes: + // - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8 + // bytes) + createMetadataRecord( + static_cast(TId)), + + // Also write the WalltimeMarker record. We only really need microsecond + // precision here, and enforce across platforms that we need 64-bit + // seconds and 32-bit microseconds encoded in the Metadata record. + createMetadataRecord( + static_cast(TS.tv_sec), + static_cast(TS.tv_nsec / 1000)), + + // Also write the Pid record. + createMetadataRecord( + static_cast(PId)), + }; + + if (finalized()) + return false; + return W.writeMetadataRecords(Metadata); + } + + bool prepareBuffer(size_t S) XRAY_NEVER_INSTRUMENT { + if (finalized()) + return returnBuffer(); + + if (UNLIKELY(!hasSpace(S))) { + if (!returnBuffer()) + return false; + if (!getNewBuffer()) + return false; + if (!setupNewBuffer()) + return false; + } + + if (First) { + First = false; + W.resetRecord(); + atomic_store(B.Extents, 0, memory_order_release); + return setupNewBuffer(); + } + + return true; + } + + bool returnBuffer() XRAY_NEVER_INSTRUMENT { + if (BQ == nullptr) + return false; + + First = true; + if (finalized()) { + BQ->releaseBuffer(B); // ignore result. + return false; + } + + return BQ->releaseBuffer(B) == BufferQueue::ErrorCode::Ok; + } + + enum class PreambleResult { NoChange, WroteMetadata, InvalidBuffer }; + PreambleResult recordPreamble(uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (UNLIKELY(LatestCPU != CPU || LatestTSC == 0)) { + // We update our internal tracking state for the Latest TSC and CPU we've + // seen, then write out the appropriate metadata and function records. + LatestTSC = TSC; + LatestCPU = CPU; + + if (B.Generation != BQ->generation()) + return PreambleResult::InvalidBuffer; + + W.writeMetadata(CPU, TSC); + return PreambleResult::WroteMetadata; + } + + DCHECK_EQ(LatestCPU, CPU); + + if (UNLIKELY(LatestTSC > TSC || + TSC - LatestTSC > + uint64_t{std::numeric_limits::max()})) { + // Either the TSC has wrapped around from the last TSC we've seen or the + // delta is too large to fit in a 32-bit signed integer, so we write a + // wrap-around record. + LatestTSC = TSC; + + if (B.Generation != BQ->generation()) + return PreambleResult::InvalidBuffer; + + W.writeMetadata(TSC); + return PreambleResult::WroteMetadata; + } + + return PreambleResult::NoChange; + } + + bool rewindRecords(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + // Undo one enter record, because at this point we are either at the state + // of: + // - We are exiting a function that we recently entered. + // - We are exiting a function that was the result of a sequence of tail + // exits, and we can check whether the tail exits can be re-wound. + // + FunctionRecord F; + W.undoWrites(sizeof(FunctionRecord)); + if (B.Generation != BQ->generation()) + return false; + internal_memcpy(&F, W.getNextRecord(), sizeof(FunctionRecord)); + + DCHECK(F.RecordKind == + uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && + "Expected to find function entry recording when rewinding."); + DCHECK_EQ(F.FuncId, FuncId & ~(0x0F << 28)); + + LatestTSC -= F.TSCDelta; + if (--UndoableFunctionEnters != 0) { + LastFunctionEntryTSC -= F.TSCDelta; + return true; + } + + LastFunctionEntryTSC = 0; + auto RewindingTSC = LatestTSC; + auto RewindingRecordPtr = W.getNextRecord() - sizeof(FunctionRecord); + while (UndoableTailExits) { + if (B.Generation != BQ->generation()) + return false; + internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord)); + DCHECK_EQ(F.RecordKind, + uint8_t(FunctionRecord::RecordKinds::FunctionTailExit)); + RewindingTSC -= F.TSCDelta; + RewindingRecordPtr -= sizeof(FunctionRecord); + if (B.Generation != BQ->generation()) + return false; + internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord)); + + // This tail call exceeded the threshold duration. It will not be erased. + if ((TSC - RewindingTSC) >= CycleThreshold) { + UndoableTailExits = 0; + return true; + } + + --UndoableTailExits; + W.undoWrites(sizeof(FunctionRecord) * 2); + LatestTSC = RewindingTSC; + } + return true; + } + +public: + template + FDRController(BufferQueue *BQ, BufferQueue::Buffer &B, FDRLogWriter &W, + WallClockFunc R, uint64_t C) XRAY_NEVER_INSTRUMENT + : BQ(BQ), + B(B), + W(W), + WallClockReader(R), + CycleThreshold(C) {} + + bool functionEnter(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) + return returnBuffer(); + + auto PreambleStatus = recordPreamble(TSC, CPU); + if (PreambleStatus == PreambleResult::InvalidBuffer) + return returnBuffer(); + + if (PreambleStatus == PreambleResult::WroteMetadata) { + UndoableFunctionEnters = 1; + UndoableTailExits = 0; + } else { + ++UndoableFunctionEnters; + } + + auto Delta = TSC - LatestTSC; + LastFunctionEntryTSC = TSC; + LatestTSC = TSC; + return W.writeFunction(FDRLogWriter::FunctionRecordKind::Enter, + mask(FuncId), Delta); + } + + bool functionTailExit(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (finalized()) + return returnBuffer(); + + if (!prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) + return returnBuffer(); + + auto PreambleStatus = recordPreamble(TSC, CPU); + if (PreambleStatus == PreambleResult::InvalidBuffer) + return returnBuffer(); + + if (PreambleStatus == PreambleResult::NoChange && + UndoableFunctionEnters != 0 && + TSC - LastFunctionEntryTSC < CycleThreshold) + return rewindRecords(FuncId, TSC, CPU); + + UndoableTailExits = UndoableFunctionEnters ? UndoableTailExits + 1 : 0; + UndoableFunctionEnters = 0; + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + return W.writeFunction(FDRLogWriter::FunctionRecordKind::TailExit, + mask(FuncId), Delta); + } + + bool functionEnterArg(int32_t FuncId, uint64_t TSC, uint16_t CPU, + uint64_t Arg) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer((2 * sizeof(MetadataRecord)) + sizeof(FunctionRecord)) || + recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) + return returnBuffer(); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + LastFunctionEntryTSC = 0; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + + return W.writeFunctionWithArg(FDRLogWriter::FunctionRecordKind::EnterArg, + mask(FuncId), Delta, Arg); + } + + bool functionExit(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) + return returnBuffer(); + + auto PreambleStatus = recordPreamble(TSC, CPU); + if (PreambleStatus == PreambleResult::InvalidBuffer) + return returnBuffer(); + + if (PreambleStatus == PreambleResult::NoChange && + UndoableFunctionEnters != 0 && + TSC - LastFunctionEntryTSC < CycleThreshold) + return rewindRecords(FuncId, TSC, CPU); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + return W.writeFunction(FDRLogWriter::FunctionRecordKind::Exit, mask(FuncId), + Delta); + } + + bool customEvent(uint64_t TSC, uint16_t CPU, const void *Event, + int32_t EventSize) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) || + recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) + return returnBuffer(); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + return W.writeCustomEvent(Delta, Event, EventSize); + } + + bool typedEvent(uint64_t TSC, uint16_t CPU, uint16_t EventType, + const void *Event, int32_t EventSize) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) || + recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) + return returnBuffer(); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + return W.writeTypedEvent(Delta, EventType, Event, EventSize); + } + + bool flush() XRAY_NEVER_INSTRUMENT { + if (finalized()) { + returnBuffer(); // ignore result. + return true; + } + return returnBuffer(); + } +}; + +} // namespace __xray + +#endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h index 87096d4fc29..e7b1ee562e1 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h @@ -12,6 +12,9 @@ //===----------------------------------------------------------------------===// #ifndef XRAY_XRAY_FDR_LOG_RECORDS_H #define XRAY_XRAY_FDR_LOG_RECORDS_H +#include + +namespace __xray { enum class RecordType : uint8_t { Function, Metadata }; @@ -68,4 +71,6 @@ struct alignas(8) FunctionRecord { static_assert(sizeof(FunctionRecord) == 8, "Wrong size for FunctionRecord."); +} // namespace __xray + #endif // XRAY_XRAY_FDR_LOG_RECORDS_H diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_writer.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_writer.h new file mode 100644 index 00000000000..7712e137776 --- /dev/null +++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_writer.h @@ -0,0 +1,232 @@ +//===-- xray_fdr_log_writer.h ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a function call tracing system. +// +//===----------------------------------------------------------------------===// +#ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_LOG_WRITER_H_ +#define COMPILER_RT_LIB_XRAY_XRAY_FDR_LOG_WRITER_H_ + +#include "xray_buffer_queue.h" +#include "xray_fdr_log_records.h" +#include +#include +#include +#include + +namespace __xray { + +template struct SerializerImpl { + template ::type>::value, + int>::type = 0> static void serializeTo(char *Buffer, + Tuple &&T) { + auto P = reinterpret_cast(&std::get(T)); + constexpr auto Size = sizeof(std::get(T)); + internal_memcpy(Buffer, P, Size); + SerializerImpl::serializeTo(Buffer + Size, + std::forward(T)); + } + + template = std::tuple_size::type>::value, + int>::type = 0> + static void serializeTo(char *, Tuple &&) {} +}; + +using Serializer = SerializerImpl<0>; + +template struct AggregateSizesImpl { + static constexpr size_t value = + sizeof(typename std::tuple_element::type) + + AggregateSizesImpl::value; +}; + +template struct AggregateSizesImpl { + static constexpr size_t value = + sizeof(typename std::tuple_element<0, Tuple>::type); +}; + +template struct AggregateSizes { + static constexpr size_t value = + AggregateSizesImpl::value - 1>::value; +}; + +template +MetadataRecord createMetadataRecord(DataTypes &&... Ds) { + static_assert(AggregateSizes>::value <= + sizeof(MetadataRecord) - 1, + "Metadata payload longer than metadata buffer!"); + MetadataRecord R; + R.Type = 1; + R.RecordKind = static_cast(Kind); + Serializer::serializeTo(R.Data, + std::make_tuple(std::forward(Ds)...)); + return R; +} + +class FDRLogWriter { + BufferQueue::Buffer &Buffer; + char *NextRecord = nullptr; + + template void writeRecord(const T &R) { + internal_memcpy(NextRecord, reinterpret_cast(&R), sizeof(T)); + NextRecord += sizeof(T); + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, sizeof(T), memory_order_acq_rel); + } + +public: + explicit FDRLogWriter(BufferQueue::Buffer &B, char *P) + : Buffer(B), NextRecord(P) { + DCHECK_NE(Buffer.Data, nullptr); + DCHECK_NE(NextRecord, nullptr); + } + + explicit FDRLogWriter(BufferQueue::Buffer &B) + : FDRLogWriter(B, static_cast(B.Data)) {} + + template + bool writeMetadata(Data &&... Ds) { + // TODO: Check boundary conditions: + // 1) Buffer is full, and cannot handle one metadata record. + // 2) Buffer queue is finalising. + writeRecord(createMetadataRecord(std::forward(Ds)...)); + return true; + } + + template size_t writeMetadataRecords(MetadataRecord (&Recs)[N]) { + constexpr auto Size = sizeof(MetadataRecord) * N; + internal_memcpy(NextRecord, reinterpret_cast(Recs), Size); + NextRecord += Size; + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, Size, memory_order_acq_rel); + return Size; + } + + enum class FunctionRecordKind : uint8_t { + Enter = 0x00, + Exit = 0x01, + TailExit = 0x02, + EnterArg = 0x03, + }; + + bool writeFunction(FunctionRecordKind Kind, int32_t FuncId, int32_t Delta) { + FunctionRecord R; + R.Type = 0; + R.RecordKind = uint8_t(Kind); + R.FuncId = FuncId; + R.TSCDelta = Delta; + writeRecord(R); + return true; + } + + bool writeFunctionWithArg(FunctionRecordKind Kind, int32_t FuncId, + int32_t Delta, uint64_t Arg) { + // We need to write the function with arg into the buffer, and then + // atomically update the buffer extents. This ensures that any reads + // synchronised on the buffer extents record will always see the writes + // that happen before the atomic update. + FunctionRecord R; + R.Type = 0; + R.RecordKind = uint8_t(Kind); + R.FuncId = FuncId; + R.TSCDelta = Delta; + MetadataRecord A = + createMetadataRecord(Arg); + NextRecord = reinterpret_cast(internal_memcpy( + NextRecord, reinterpret_cast(&R), sizeof(R))) + + sizeof(R); + NextRecord = reinterpret_cast(internal_memcpy( + NextRecord, reinterpret_cast(&A), sizeof(A))) + + sizeof(A); + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, sizeof(R) + sizeof(A), + memory_order_acq_rel); + return true; + } + + bool writeCustomEvent(int32_t Delta, const void *Event, int32_t EventSize) { + // We write the metadata record and the custom event data into the buffer + // first, before we atomically update the extents for the buffer. This + // allows us to ensure that any threads reading the extents of the buffer + // will only ever see the full metadata and custom event payload accounted + // (no partial writes accounted). + MetadataRecord R = + createMetadataRecord( + EventSize, Delta); + NextRecord = reinterpret_cast(internal_memcpy( + NextRecord, reinterpret_cast(&R), sizeof(R))) + + sizeof(R); + NextRecord = reinterpret_cast( + internal_memcpy(NextRecord, Event, EventSize)) + + EventSize; + + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, sizeof(R) + EventSize, + memory_order_acq_rel); + return true; + } + + bool writeTypedEvent(int32_t Delta, uint16_t EventType, const void *Event, + int32_t EventSize) { + // We do something similar when writing out typed events, see + // writeCustomEvent(...) above for details. + MetadataRecord R = + createMetadataRecord( + EventSize, Delta, EventType); + NextRecord = reinterpret_cast(internal_memcpy( + NextRecord, reinterpret_cast(&R), sizeof(R))) + + sizeof(R); + NextRecord = reinterpret_cast( + internal_memcpy(NextRecord, Event, EventSize)) + + EventSize; + + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, EventSize, memory_order_acq_rel); + return true; + } + + char *getNextRecord() const { return NextRecord; } + + void resetRecord() { + NextRecord = reinterpret_cast(Buffer.Data); + atomic_store(Buffer.Extents, 0, memory_order_release); + } + + void undoWrites(size_t B) { + DCHECK_GE(NextRecord - B, reinterpret_cast(Buffer.Data)); + NextRecord -= B; + atomic_fetch_sub(Buffer.Extents, B, memory_order_acq_rel); + } + +}; // namespace __xray + +} // namespace __xray + +#endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_LOG_WRITER_H_ diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc index 6cb2dfa0c65..1eda26df7a8 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -30,9 +29,12 @@ #include "sanitizer_common/sanitizer_common.h" #include "xray/xray_interface.h" #include "xray/xray_records.h" +#include "xray_allocator.h" #include "xray_buffer_queue.h" #include "xray_defs.h" +#include "xray_fdr_controller.h" #include "xray_fdr_flags.h" +#include "xray_fdr_log_writer.h" #include "xray_flags.h" #include "xray_recursion_guard.h" #include "xray_tsc.h" @@ -40,56 +42,54 @@ namespace __xray { -atomic_sint32_t LoggingStatus = {XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; +static atomic_sint32_t LoggingStatus = { + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; + +namespace { // Group together thread-local-data in a struct, then hide it behind a function // call so that it can be initialized on first use instead of as a global. We // force the alignment to 64-bytes for x86 cache line alignment, as this // structure is used in the hot path of implementation. -struct alignas(64) ThreadLocalData { - BufferQueue::Buffer Buffer; - char *RecordPtr = nullptr; - // The number of FunctionEntry records immediately preceding RecordPtr. - uint8_t NumConsecutiveFnEnters = 0; - - // The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit - // records preceding RecordPtr. - uint8_t NumTailCalls = 0; - - // We use a thread_local variable to keep track of which CPUs we've already - // run, and the TSC times for these CPUs. This allows us to stop repeating the - // CPU field in the function records. - // - // We assume that we'll support only 65536 CPUs for x86_64. - uint16_t CurrentCPU = std::numeric_limits::max(); - uint64_t LastTSC = 0; - uint64_t LastFunctionEntryTSC = 0; - - // Make sure a thread that's ever called handleArg0 has a thread-local - // live reference to the buffer queue for this particular instance of - // FDRLogging, and that we're going to clean it up when the thread exits. +struct XRAY_TLS_ALIGNAS(64) ThreadLocalData { + BufferQueue::Buffer Buffer{}; BufferQueue *BQ = nullptr; + + using LogWriterStorage = + typename std::aligned_storage::type; + + LogWriterStorage LWStorage; + FDRLogWriter *Writer = nullptr; + + using ControllerStorage = + typename std::aligned_storage), + alignof(FDRController<>)>::type; + ControllerStorage CStorage; + FDRController<> *Controller = nullptr; }; +} // namespace + static_assert(std::is_trivially_destructible::value, "ThreadLocalData must be trivially destructible"); -static constexpr auto MetadataRecSize = sizeof(MetadataRecord); -static constexpr auto FunctionRecSize = sizeof(FunctionRecord); - // Use a global pthread key to identify thread-local data for logging. static pthread_key_t Key; // Global BufferQueue. +static std::aligned_storage::type BufferQueueStorage; static BufferQueue *BQ = nullptr; +// Global thresholds for function durations. +static atomic_uint64_t ThresholdTicks{0}; + +// Global for ticks per second. +static atomic_uint64_t TicksPerSec{0}; + static atomic_sint32_t LogFlushStatus = { XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; -static FDRLoggingOptions FDROptions; - -static SpinMutex FDROptionsMutex; - // This function will initialize the thread-local data structure used by the FDR // logging implementation and return a reference to it. The implementation // details require a bit of care to maintain. @@ -124,8 +124,10 @@ static SpinMutex FDROptionsMutex; // critical section, calling a function that might be XRay instrumented (and // thus in turn calling into malloc by virtue of registration of the // thread_local's destructor). +#if XRAY_HAS_TLS_ALIGNAS static_assert(alignof(ThreadLocalData) >= 64, "ThreadLocalData must be cache line aligned."); +#endif static ThreadLocalData &getThreadLocalData() { thread_local typename std::aligned_storage< sizeof(ThreadLocalData), alignof(ThreadLocalData)>::type TLDStorage{}; @@ -138,559 +140,36 @@ static ThreadLocalData &getThreadLocalData() { return *reinterpret_cast(&TLDStorage); } -static void writeNewBufferPreamble(tid_t Tid, timespec TS, - pid_t Pid) XRAY_NEVER_INSTRUMENT { - static constexpr int InitRecordsCount = 3; - auto &TLD = getThreadLocalData(); - MetadataRecord Metadata[InitRecordsCount]; - { - // Write out a MetadataRecord to signify that this is the start of a new - // buffer, associated with a particular thread, with a new CPU. For the - // data, we have 15 bytes to squeeze as much information as we can. At this - // point we only write down the following bytes: - // - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8 bytes) - auto &NewBuffer = Metadata[0]; - NewBuffer.Type = uint8_t(RecordType::Metadata); - NewBuffer.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewBuffer); - int32_t tid = static_cast(Tid); - internal_memcpy(&NewBuffer.Data, &tid, sizeof(tid)); - } - - // Also write the WalltimeMarker record. - { - static_assert(sizeof(time_t) <= 8, "time_t needs to be at most 8 bytes"); - auto &WalltimeMarker = Metadata[1]; - WalltimeMarker.Type = uint8_t(RecordType::Metadata); - WalltimeMarker.RecordKind = - uint8_t(MetadataRecord::RecordKinds::WalltimeMarker); - - // We only really need microsecond precision here, and enforce across - // platforms that we need 64-bit seconds and 32-bit microseconds encoded in - // the Metadata record. - int32_t Micros = TS.tv_nsec / 1000; - int64_t Seconds = TS.tv_sec; - internal_memcpy(WalltimeMarker.Data, &Seconds, sizeof(Seconds)); - internal_memcpy(WalltimeMarker.Data + sizeof(Seconds), &Micros, - sizeof(Micros)); - } - - // Also write the Pid record. - { - // Write out a MetadataRecord that contains the current pid - auto &PidMetadata = Metadata[2]; - PidMetadata.Type = uint8_t(RecordType::Metadata); - PidMetadata.RecordKind = uint8_t(MetadataRecord::RecordKinds::Pid); - int32_t pid = static_cast(Pid); - internal_memcpy(&PidMetadata.Data, &pid, sizeof(pid)); - } - - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - if (TLD.BQ == nullptr || TLD.BQ->finalizing()) - return; - internal_memcpy(TLD.RecordPtr, Metadata, sizeof(Metadata)); - TLD.RecordPtr += sizeof(Metadata); - // Since we write out the extents as the first metadata record of the - // buffer, we need to write out the extents including the extents record. - atomic_store(&TLD.Buffer.Extents->Size, sizeof(Metadata), - memory_order_release); -} - -static void setupNewBuffer(int (*wall_clock_reader)( - clockid_t, struct timespec *)) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - auto &B = TLD.Buffer; - TLD.RecordPtr = static_cast(B.Data); - tid_t Tid = GetTid(); - timespec TS{0, 0}; - pid_t Pid = internal_getpid(); - // This is typically clock_gettime, but callers have injection ability. - wall_clock_reader(CLOCK_MONOTONIC, &TS); - writeNewBufferPreamble(Tid, TS, Pid); - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; -} - -static void incrementExtents(size_t Add) { - auto &TLD = getThreadLocalData(); - atomic_fetch_add(&TLD.Buffer.Extents->Size, Add, memory_order_acq_rel); -} - -static void decrementExtents(size_t Subtract) { - auto &TLD = getThreadLocalData(); - atomic_fetch_sub(&TLD.Buffer.Extents->Size, Subtract, memory_order_acq_rel); -} - -static void writeNewCPUIdMetadata(uint16_t CPU, - uint64_t TSC) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - MetadataRecord NewCPUId; - NewCPUId.Type = uint8_t(RecordType::Metadata); - NewCPUId.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewCPUId); - - // The data for the New CPU will contain the following bytes: - // - CPU ID (uint16_t, 2 bytes) - // - Full TSC (uint64_t, 8 bytes) - // Total = 10 bytes. - internal_memcpy(&NewCPUId.Data, &CPU, sizeof(CPU)); - internal_memcpy(&NewCPUId.Data[sizeof(CPU)], &TSC, sizeof(TSC)); - internal_memcpy(TLD.RecordPtr, &NewCPUId, sizeof(MetadataRecord)); - TLD.RecordPtr += sizeof(MetadataRecord); - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - incrementExtents(sizeof(MetadataRecord)); -} - -static void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - MetadataRecord TSCWrap; - TSCWrap.Type = uint8_t(RecordType::Metadata); - TSCWrap.RecordKind = uint8_t(MetadataRecord::RecordKinds::TSCWrap); - - // The data for the TSCWrap record contains the following bytes: - // - Full TSC (uint64_t, 8 bytes) - // Total = 8 bytes. - internal_memcpy(&TSCWrap.Data, &TSC, sizeof(TSC)); - internal_memcpy(TLD.RecordPtr, &TSCWrap, sizeof(MetadataRecord)); - TLD.RecordPtr += sizeof(MetadataRecord); - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - incrementExtents(sizeof(MetadataRecord)); -} - -// Call Argument metadata records store the arguments to a function in the -// order of their appearance; holes are not supported by the buffer format. -static void writeCallArgumentMetadata(uint64_t A) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - MetadataRecord CallArg; - CallArg.Type = uint8_t(RecordType::Metadata); - CallArg.RecordKind = uint8_t(MetadataRecord::RecordKinds::CallArgument); - - internal_memcpy(CallArg.Data, &A, sizeof(A)); - internal_memcpy(TLD.RecordPtr, &CallArg, sizeof(MetadataRecord)); - TLD.RecordPtr += sizeof(MetadataRecord); - incrementExtents(sizeof(MetadataRecord)); -} - -static void writeFunctionRecord(int FuncId, uint32_t TSCDelta, - XRayEntryType EntryType) XRAY_NEVER_INSTRUMENT { - FunctionRecord FuncRecord; - FuncRecord.Type = uint8_t(RecordType::Function); - // Only take 28 bits of the function id. - FuncRecord.FuncId = FuncId & ~(0x0F << 28); - FuncRecord.TSCDelta = TSCDelta; - - auto &TLD = getThreadLocalData(); - switch (EntryType) { - case XRayEntryType::ENTRY: - ++TLD.NumConsecutiveFnEnters; - FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); - break; - case XRayEntryType::LOG_ARGS_ENTRY: - // We should not rewind functions with logged args. - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); - break; - case XRayEntryType::EXIT: - // If we've decided to log the function exit, we will never erase the log - // before it. - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionExit); - break; - case XRayEntryType::TAIL: - // If we just entered the function we're tail exiting from or erased every - // invocation since then, this function entry tail pair is a candidate to - // be erased when the child function exits. - if (TLD.NumConsecutiveFnEnters > 0) { - ++TLD.NumTailCalls; - TLD.NumConsecutiveFnEnters = 0; - } else { - // We will never be able to erase this tail call since we have logged - // something in between the function entry and tail exit. - TLD.NumTailCalls = 0; - TLD.NumConsecutiveFnEnters = 0; - } - FuncRecord.RecordKind = - uint8_t(FunctionRecord::RecordKinds::FunctionTailExit); - break; - case XRayEntryType::CUSTOM_EVENT: { - // This is a bug in patching, so we'll report it once and move on. - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay custom event call as a function; " - "func id = %d\n", - FuncId); - return; - } - case XRayEntryType::TYPED_EVENT: { - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay typed event call as a function; " - "func id = %d\n", - FuncId); - return; - } - } - - internal_memcpy(TLD.RecordPtr, &FuncRecord, sizeof(FunctionRecord)); - TLD.RecordPtr += sizeof(FunctionRecord); - incrementExtents(sizeof(FunctionRecord)); -} - -static atomic_uint64_t TicksPerSec{0}; -static atomic_uint64_t ThresholdTicks{0}; - -// Re-point the thread local pointer into this thread's Buffer before the recent -// "Function Entry" record and any "Tail Call Exit" records after that. -static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, - uint64_t &LastFunctionEntryTSC, int32_t FuncId) { - auto &TLD = getThreadLocalData(); - TLD.RecordPtr -= FunctionRecSize; - decrementExtents(FunctionRecSize); - FunctionRecord FuncRecord; - internal_memcpy(&FuncRecord, TLD.RecordPtr, FunctionRecSize); - DCHECK(FuncRecord.RecordKind == - uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && - "Expected to find function entry recording when rewinding."); - DCHECK(FuncRecord.FuncId == (FuncId & ~(0x0F << 28)) && - "Expected matching function id when rewinding Exit"); - --TLD.NumConsecutiveFnEnters; - LastTSC -= FuncRecord.TSCDelta; - - // We unwound one call. Update the state and return without writing a log. - if (TLD.NumConsecutiveFnEnters != 0) { - LastFunctionEntryTSC -= FuncRecord.TSCDelta; - return; - } - - // Otherwise we've rewound the stack of all function entries, we might be - // able to rewind further by erasing tail call functions that are being - // exited from via this exit. - LastFunctionEntryTSC = 0; - auto RewindingTSC = LastTSC; - auto RewindingRecordPtr = TLD.RecordPtr - FunctionRecSize; - while (TLD.NumTailCalls > 0) { - // Rewind the TSC back over the TAIL EXIT record. - FunctionRecord ExpectedTailExit; - internal_memcpy(&ExpectedTailExit, RewindingRecordPtr, FunctionRecSize); - - DCHECK(ExpectedTailExit.RecordKind == - uint8_t(FunctionRecord::RecordKinds::FunctionTailExit) && - "Expected to find tail exit when rewinding."); - RewindingRecordPtr -= FunctionRecSize; - RewindingTSC -= ExpectedTailExit.TSCDelta; - FunctionRecord ExpectedFunctionEntry; - internal_memcpy(&ExpectedFunctionEntry, RewindingRecordPtr, - FunctionRecSize); - DCHECK(ExpectedFunctionEntry.RecordKind == - uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && - "Expected to find function entry when rewinding tail call."); - DCHECK(ExpectedFunctionEntry.FuncId == ExpectedTailExit.FuncId && - "Expected funcids to match when rewinding tail call."); - - // This tail call exceeded the threshold duration. It will not be erased. - if ((TSC - RewindingTSC) >= atomic_load_relaxed(&ThresholdTicks)) { - TLD.NumTailCalls = 0; - return; - } - - // We can erase a tail exit pair that we're exiting through since - // its duration is under threshold. - --TLD.NumTailCalls; - RewindingRecordPtr -= FunctionRecSize; - RewindingTSC -= ExpectedFunctionEntry.TSCDelta; - TLD.RecordPtr -= 2 * FunctionRecSize; - LastTSC = RewindingTSC; - decrementExtents(2 * FunctionRecSize); - } -} - -static bool releaseThreadLocalBuffer(BufferQueue &BQArg) { - auto &TLD = getThreadLocalData(); - auto EC = BQArg.releaseBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) { - Report("Failed to release buffer at %p; error=%s\n", TLD.Buffer.Data, - BufferQueue::getErrorString(EC)); - return false; - } - return true; -} - -static bool prepareBuffer(uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, - struct timespec *), - size_t MaxSize) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - char *BufferStart = static_cast(TLD.Buffer.Data); - if ((TLD.RecordPtr + MaxSize) > (BufferStart + TLD.Buffer.Size)) { - if (!releaseThreadLocalBuffer(*TLD.BQ)) - return false; - auto EC = TLD.BQ->getBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) { - Report("Failed to prepare a buffer; error = '%s'\n", - BufferQueue::getErrorString(EC)); - return false; - } - setupNewBuffer(wall_clock_reader); - - // Always write the CPU metadata as the first record in the buffer. - writeNewCPUIdMetadata(CPU, TSC); - } - return true; -} - -static bool -isLogInitializedAndReady(BufferQueue *LBQ, uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, struct timespec *)) - XRAY_NEVER_INSTRUMENT { - // Bail out right away if logging is not initialized yet. - // We should take the opportunity to release the buffer though. - auto Status = atomic_load(&LoggingStatus, memory_order_acquire); - auto &TLD = getThreadLocalData(); - if (Status != XRayLogInitStatus::XRAY_LOG_INITIALIZED) { - if (TLD.RecordPtr != nullptr && - (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || - Status == XRayLogInitStatus::XRAY_LOG_FINALIZED)) { - if (!releaseThreadLocalBuffer(*LBQ)) - return false; - TLD.RecordPtr = nullptr; - return false; - } - return false; - } - - if (atomic_load(&LoggingStatus, memory_order_acquire) != - XRayLogInitStatus::XRAY_LOG_INITIALIZED || - LBQ->finalizing()) { - if (!releaseThreadLocalBuffer(*LBQ)) - return false; - TLD.RecordPtr = nullptr; - } - - if (TLD.Buffer.Data == nullptr) { - auto EC = LBQ->getBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) { - auto LS = atomic_load(&LoggingStatus, memory_order_acquire); - if (LS != XRayLogInitStatus::XRAY_LOG_FINALIZING && - LS != XRayLogInitStatus::XRAY_LOG_FINALIZED) - Report("Failed to acquire a buffer; error = '%s'\n", - BufferQueue::getErrorString(EC)); - return false; - } - - setupNewBuffer(wall_clock_reader); - - // Always write the CPU metadata as the first record in the buffer. - writeNewCPUIdMetadata(CPU, TSC); - } - - if (TLD.CurrentCPU == std::numeric_limits::max()) { - // This means this is the first CPU this thread has ever run on. We set - // the current CPU and record this as the first TSC we've seen. - TLD.CurrentCPU = CPU; - writeNewCPUIdMetadata(CPU, TSC); - } - - return true; -} - -// Compute the TSC difference between the time of measurement and the previous -// event. There are a few interesting situations we need to account for: -// -// - The thread has migrated to a different CPU. If this is the case, then -// we write down the following records: -// -// 1. A 'NewCPUId' Metadata record. -// 2. A FunctionRecord with a 0 for the TSCDelta field. -// -// - The TSC delta is greater than the 32 bits we can store in a -// FunctionRecord. In this case we write down the following records: -// -// 1. A 'TSCWrap' Metadata record. -// 2. A FunctionRecord with a 0 for the TSCDelta field. -// -// - The TSC delta is representable within the 32 bits we can store in a -// FunctionRecord. In this case we write down just a FunctionRecord with -// the correct TSC delta. -static uint32_t writeCurrentCPUTSC(ThreadLocalData &TLD, uint64_t TSC, - uint8_t CPU) { - if (CPU != TLD.CurrentCPU) { - // We've moved to a new CPU. - writeNewCPUIdMetadata(CPU, TSC); - return 0; - } - // If the delta is greater than the range for a uint32_t, then we write out - // the TSC wrap metadata entry with the full TSC, and the TSC for the - // function record be 0. - uint64_t Delta = TSC - TLD.LastTSC; - if (Delta <= std::numeric_limits::max()) - return Delta; - - writeTSCWrapMetadata(TSC); - return 0; -} - -static void endBufferIfFull() XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - auto BufferStart = static_cast(TLD.Buffer.Data); - if ((TLD.RecordPtr + MetadataRecSize) - BufferStart <= - ptrdiff_t{MetadataRecSize}) { - if (!releaseThreadLocalBuffer(*TLD.BQ)) - return; - TLD.RecordPtr = nullptr; - } -} - -thread_local atomic_uint8_t Running{0}; - -/// Here's where the meat of the processing happens. The writer captures -/// function entry, exit and tail exit points with a time and will create -/// TSCWrap, NewCPUId and Function records as necessary. The writer might -/// walk backward through its buffer and erase trivial functions to avoid -/// polluting the log and may use the buffer queue to obtain or release a -/// buffer. -static void processFunctionHook(int32_t FuncId, XRayEntryType Entry, - uint64_t TSC, unsigned char CPU, uint64_t Arg1, - int (*wall_clock_reader)(clockid_t, - struct timespec *)) - XRAY_NEVER_INSTRUMENT { - __asm volatile("# LLVM-MCA-BEGIN processFunctionHook"); - // Prevent signal handler recursion, so in case we're already in a log writing - // mode and the signal handler comes in (and is also instrumented) then we - // don't want to be clobbering potentially partial writes already happening in - // the thread. We use a simple thread_local latch to only allow one on-going - // handleArg0 to happen at any given time. - RecursionGuard Guard{Running}; - if (!Guard) { - DCHECK(atomic_load_relaxed(&Running) && "RecursionGuard is buggy!"); - return; - } - - auto &TLD = getThreadLocalData(); - - if (TLD.BQ == nullptr) - TLD.BQ = BQ; - - if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, wall_clock_reader)) - return; - - // Before we go setting up writing new function entries, we need to be really - // careful about the pointer math we're doing. This means we need to ensure - // that the record we are about to write is going to fit into the buffer, - // without overflowing the buffer. - // - // To do this properly, we use the following assumptions: - // - // - The least number of bytes we will ever write is 8 - // (sizeof(FunctionRecord)) only if the delta between the previous entry - // and this entry is within 32 bits. - // - The most number of bytes we will ever write is 8 + 16 + 16 = 40. - // This is computed by: - // - // MaxSize = sizeof(FunctionRecord) + 2 * sizeof(MetadataRecord) - // - // These arise in the following cases: - // - // 1. When the delta between the TSC we get and the previous TSC for the - // same CPU is outside of the uint32_t range, we end up having to - // write a MetadataRecord to indicate a "tsc wrap" before the actual - // FunctionRecord. - // 2. When we learn that we've moved CPUs, we need to write a - // MetadataRecord to indicate a "cpu change", and thus write out the - // current TSC for that CPU before writing out the actual - // FunctionRecord. - // 3. When we learn about a new CPU ID, we need to write down a "new cpu - // id" MetadataRecord before writing out the actual FunctionRecord. - // 4. The second MetadataRecord is the optional function call argument. - // - // So the math we need to do is to determine whether writing 40 bytes past the - // current pointer exceeds the buffer's maximum size. If we don't have enough - // space to write 40 bytes in the buffer, we need get a new Buffer, set it up - // properly before doing any further writing. - size_t MaxSize = FunctionRecSize + 2 * MetadataRecSize; - if (!prepareBuffer(TSC, CPU, wall_clock_reader, MaxSize)) { - TLD.BQ = nullptr; - return; - } - - // By this point, we are now ready to write up to 40 bytes (explained above). - DCHECK((TLD.RecordPtr + MaxSize) - static_cast(TLD.Buffer.Data) >= - static_cast(MetadataRecSize) && - "Misconfigured BufferQueue provided; Buffer size not large enough."); - - auto RecordTSCDelta = writeCurrentCPUTSC(TLD, TSC, CPU); - TLD.LastTSC = TSC; - TLD.CurrentCPU = CPU; - switch (Entry) { - case XRayEntryType::ENTRY: - case XRayEntryType::LOG_ARGS_ENTRY: - // Update the thread local state for the next invocation. - TLD.LastFunctionEntryTSC = TSC; - break; - case XRayEntryType::TAIL: - case XRayEntryType::EXIT: - // Break out and write the exit record if we can't erase any functions. - if (TLD.NumConsecutiveFnEnters == 0 || - (TSC - TLD.LastFunctionEntryTSC) >= - atomic_load_relaxed(&ThresholdTicks)) - break; - rewindRecentCall(TSC, TLD.LastTSC, TLD.LastFunctionEntryTSC, FuncId); - return; // without writing log. - case XRayEntryType::CUSTOM_EVENT: { - // This is a bug in patching, so we'll report it once and move on. - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay custom event call as a function; " - "func id = %d\n", - FuncId); - return; - } - case XRayEntryType::TYPED_EVENT: { - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay typed event call as a function; " - "func id = %d\n", - FuncId); - return; - } - } - - writeFunctionRecord(FuncId, RecordTSCDelta, Entry); - if (Entry == XRayEntryType::LOG_ARGS_ENTRY) - writeCallArgumentMetadata(Arg1); - - // If we've exhausted the buffer by this time, we then release the buffer to - // make sure that other threads may start using this buffer. - endBufferIfFull(); - __asm volatile("# LLVM-MCA-END"); -} - static XRayFileHeader &fdrCommonHeaderInfo() { static std::aligned_storage::type HStorage; static pthread_once_t OnceInit = PTHREAD_ONCE_INIT; static bool TSCSupported = true; static uint64_t CycleFrequency = NanosecondsPerSecond; - pthread_once(&OnceInit, +[] { - XRayFileHeader &H = reinterpret_cast(HStorage); - // Version 2 of the log writes the extents of the buffer, instead of - // relying on an end-of-buffer record. - // Version 3 includes PID metadata record - H.Version = 3; - H.Type = FileTypes::FDR_LOG; + pthread_once( + &OnceInit, +[] { + XRayFileHeader &H = reinterpret_cast(HStorage); + // Version 2 of the log writes the extents of the buffer, instead of + // relying on an end-of-buffer record. + // Version 3 includes PID metadata record. + // Version 4 includes CPU data in the custom event records. + // Version 5 uses relative deltas for custom and typed event records, + // and removes the CPU data in custom event records (similar to how + // function records use deltas instead of full TSCs and rely on other + // metadata records for TSC wraparound and CPU migration). + H.Version = 5; + H.Type = FileTypes::FDR_LOG; - // Test for required CPU features and cache the cycle frequency - TSCSupported = probeRequiredCPUFeatures(); - if (TSCSupported) - CycleFrequency = getTSCFrequency(); - H.CycleFrequency = CycleFrequency; + // Test for required CPU features and cache the cycle frequency + TSCSupported = probeRequiredCPUFeatures(); + if (TSCSupported) + CycleFrequency = getTSCFrequency(); + H.CycleFrequency = CycleFrequency; - // FIXME: Actually check whether we have 'constant_tsc' and - // 'nonstop_tsc' before setting the values in the header. - H.ConstantTSC = 1; - H.NonstopTSC = 1; - }); + // FIXME: Actually check whether we have 'constant_tsc' and + // 'nonstop_tsc' before setting the values in the header. + H.ConstantTSC = 1; + H.NonstopTSC = 1; + }); return reinterpret_cast(HStorage); } @@ -728,9 +207,11 @@ XRayBuffer fdrIterator(const XRayBuffer B) { // buffers to expect). static std::aligned_storage::type HeaderStorage; static pthread_once_t HeaderOnce = PTHREAD_ONCE_INIT; - pthread_once(&HeaderOnce, +[] { - reinterpret_cast(HeaderStorage) = fdrCommonHeaderInfo(); - }); + pthread_once( + &HeaderOnce, +[] { + reinterpret_cast(HeaderStorage) = + fdrCommonHeaderInfo(); + }); // We use a convenience alias for code referring to Header from here on out. auto &Header = reinterpret_cast(HeaderStorage); @@ -741,7 +222,8 @@ XRayBuffer fdrIterator(const XRayBuffer B) { static BufferQueue::const_iterator It{}; static BufferQueue::const_iterator End{}; - static void *CurrentBuffer{nullptr}; + static uint8_t *CurrentBuffer{nullptr}; + static size_t SerializedBufferSize = 0; if (B.Data == static_cast(&Header) && B.Size == sizeof(Header)) { // From this point on, we provide raw access to the raw buffer we're getting // from the BufferQueue. We're relying on the iterators from the current @@ -751,7 +233,7 @@ XRayBuffer fdrIterator(const XRayBuffer B) { } if (CurrentBuffer != nullptr) { - InternalFree(CurrentBuffer); + deallocateBuffer(CurrentBuffer, SerializedBufferSize); CurrentBuffer = nullptr; } @@ -762,9 +244,16 @@ XRayBuffer fdrIterator(const XRayBuffer B) { // out to disk. The difference here would be that we still write "empty" // buffers, or at least go through the iterators faithfully to let the // handlers see the empty buffers in the queue. - auto BufferSize = atomic_load(&It->Extents->Size, memory_order_acquire); - auto SerializedBufferSize = BufferSize + sizeof(MetadataRecord); - CurrentBuffer = InternalAlloc(SerializedBufferSize); + // + // We need this atomic fence here to ensure that writes happening to the + // buffer have been committed before we load the extents atomically. Because + // the buffer is not explicitly synchronised across threads, we rely on the + // fence ordering to ensure that writes we expect to have been completed + // before the fence are fully committed before we read the extents. + atomic_thread_fence(memory_order_acquire); + auto BufferSize = atomic_load(It->Extents, memory_order_acquire); + SerializedBufferSize = BufferSize + sizeof(MetadataRecord); + CurrentBuffer = allocateBuffer(SerializedBufferSize); if (CurrentBuffer == nullptr) return {nullptr, 0}; @@ -827,14 +316,9 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { }); auto CleanupBuffers = at_scope_exit([] { - if (BQ != nullptr) { - auto &TLD = getThreadLocalData(); - if (TLD.RecordPtr != nullptr && TLD.BQ != nullptr) - releaseThreadLocalBuffer(*TLD.BQ); - BQ->~BufferQueue(); - InternalFree(BQ); - BQ = nullptr; - } + auto &TLD = getThreadLocalData(); + if (TLD.Controller != nullptr) + TLD.Controller->flush(); }); if (fdrFlags()->no_file_flush) { @@ -855,16 +339,8 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { // (fixed-sized) and let the tools reading the buffers deal with the data // afterwards. // - int Fd = -1; - { - // FIXME: Remove this section of the code, when we remove the struct-based - // configuration API. - SpinMutexLock Guard(&FDROptionsMutex); - Fd = FDROptions.Fd; - } - if (Fd == -1) - Fd = getLogFD(); - if (Fd == -1) { + LogWriter *LW = LogWriter::Open(); + if (LW == nullptr) { auto Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; atomic_store(&LogFlushStatus, Result, memory_order_release); return Result; @@ -872,8 +348,15 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { XRayFileHeader Header = fdrCommonHeaderInfo(); Header.FdrData = FdrAdditionalHeaderData{BQ->ConfiguredBufferSize()}; - retryingWriteAll(Fd, reinterpret_cast(&Header), - reinterpret_cast(&Header) + sizeof(Header)); + LW->WriteAll(reinterpret_cast(&Header), + reinterpret_cast(&Header) + sizeof(Header)); + + // Release the current thread's buffer before we attempt to write out all the + // buffers. This ensures that in case we had only a single thread going, that + // we are able to capture the data nonetheless. + auto &TLD = getThreadLocalData(); + if (TLD.Controller != nullptr) + TLD.Controller->flush(); BQ->apply([&](const BufferQueue::Buffer &B) { // Starting at version 2 of the FDR logging implementation, we only write @@ -882,18 +365,18 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { // still use a Metadata record, but fill in the extents instead for the // data. MetadataRecord ExtentsRecord; - auto BufferExtents = atomic_load(&B.Extents->Size, memory_order_acquire); + auto BufferExtents = atomic_load(B.Extents, memory_order_acquire); DCHECK(BufferExtents <= B.Size); ExtentsRecord.Type = uint8_t(RecordType::Metadata); ExtentsRecord.RecordKind = uint8_t(MetadataRecord::RecordKinds::BufferExtents); internal_memcpy(ExtentsRecord.Data, &BufferExtents, sizeof(BufferExtents)); if (BufferExtents > 0) { - retryingWriteAll(Fd, reinterpret_cast(&ExtentsRecord), - reinterpret_cast(&ExtentsRecord) + - sizeof(MetadataRecord)); - retryingWriteAll(Fd, reinterpret_cast(B.Data), - reinterpret_cast(B.Data) + BufferExtents); + LW->WriteAll(reinterpret_cast(&ExtentsRecord), + reinterpret_cast(&ExtentsRecord) + + sizeof(MetadataRecord)); + LW->WriteAll(reinterpret_cast(B.Data), + reinterpret_cast(B.Data) + BufferExtents); } }); @@ -914,7 +397,12 @@ XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT { // Do special things to make the log finalize itself, and not allow any more // operations to be performed until re-initialized. - BQ->finalize(); + if (BQ == nullptr) { + if (Verbosity()) + Report("Attempting to finalize an uninitialized global buffer!\n"); + } else { + BQ->finalize(); + } atomic_store(&LoggingStatus, XRayLogInitStatus::XRAY_LOG_FINALIZED, memory_order_release); @@ -935,7 +423,8 @@ static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT { // Test once for required CPU features static pthread_once_t OnceProbe = PTHREAD_ONCE_INIT; static bool TSCSupported = true; - pthread_once(&OnceProbe, +[] { TSCSupported = probeRequiredCPUFeatures(); }); + pthread_once( + &OnceProbe, +[] { TSCSupported = probeRequiredCPUFeatures(); }); if (TSCSupported) { Result.TSC = __xray::readTSC(Result.CPU); @@ -953,16 +442,115 @@ static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT { return Result; } +thread_local atomic_uint8_t Running{0}; + +static bool setupTLD(ThreadLocalData &TLD) XRAY_NEVER_INSTRUMENT { + // Check if we're finalizing, before proceeding. + { + auto Status = atomic_load(&LoggingStatus, memory_order_acquire); + if (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || + Status == XRayLogInitStatus::XRAY_LOG_FINALIZED) { + if (TLD.Controller != nullptr) { + TLD.Controller->flush(); + TLD.Controller = nullptr; + } + return false; + } + } + + if (UNLIKELY(TLD.Controller == nullptr)) { + // Set up the TLD buffer queue. + if (UNLIKELY(BQ == nullptr)) + return false; + TLD.BQ = BQ; + + // Check that we have a valid buffer. + if (TLD.Buffer.Generation != BQ->generation() && + TLD.BQ->releaseBuffer(TLD.Buffer) != BufferQueue::ErrorCode::Ok) + return false; + + // Set up a buffer, before setting up the log writer. Bail out on failure. + if (TLD.BQ->getBuffer(TLD.Buffer) != BufferQueue::ErrorCode::Ok) + return false; + + // Set up the Log Writer for this thread. + if (UNLIKELY(TLD.Writer == nullptr)) { + auto *LWStorage = reinterpret_cast(&TLD.LWStorage); + new (LWStorage) FDRLogWriter(TLD.Buffer); + TLD.Writer = LWStorage; + } else { + TLD.Writer->resetRecord(); + } + + auto *CStorage = reinterpret_cast *>(&TLD.CStorage); + new (CStorage) + FDRController<>(TLD.BQ, TLD.Buffer, *TLD.Writer, clock_gettime, + atomic_load_relaxed(&ThresholdTicks)); + TLD.Controller = CStorage; + } + + DCHECK_NE(TLD.Controller, nullptr); + return true; +} + void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { auto TC = getTimestamp(); - processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, 0, clock_gettime); + auto &TSC = TC.TSC; + auto &CPU = TC.CPU; + RecursionGuard Guard{Running}; + if (!Guard) + return; + + auto &TLD = getThreadLocalData(); + if (!setupTLD(TLD)) + return; + + switch (Entry) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: + TLD.Controller->functionEnter(FuncId, TSC, CPU); + return; + case XRayEntryType::EXIT: + TLD.Controller->functionExit(FuncId, TSC, CPU); + return; + case XRayEntryType::TAIL: + TLD.Controller->functionTailExit(FuncId, TSC, CPU); + return; + case XRayEntryType::CUSTOM_EVENT: + case XRayEntryType::TYPED_EVENT: + break; + } } void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, uint64_t Arg) XRAY_NEVER_INSTRUMENT { auto TC = getTimestamp(); - processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, Arg, clock_gettime); + auto &TSC = TC.TSC; + auto &CPU = TC.CPU; + RecursionGuard Guard{Running}; + if (!Guard) + return; + + auto &TLD = getThreadLocalData(); + if (!setupTLD(TLD)) + return; + + switch (Entry) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: + TLD.Controller->functionEnterArg(FuncId, TSC, CPU, Arg); + return; + case XRayEntryType::EXIT: + TLD.Controller->functionExit(FuncId, TSC, CPU); + return; + case XRayEntryType::TAIL: + TLD.Controller->functionTailExit(FuncId, TSC, CPU); + return; + case XRayEntryType::CUSTOM_EVENT: + case XRayEntryType::TYPED_EVENT: + break; + } } void fdrLoggingHandleCustomEvent(void *Event, @@ -973,40 +561,25 @@ void fdrLoggingHandleCustomEvent(void *Event, RecursionGuard Guard{Running}; if (!Guard) return; - if (EventSize > std::numeric_limits::max()) { + + // Complain when we ever get at least one custom event that's larger than what + // we can possibly support. + if (EventSize > + static_cast(std::numeric_limits::max())) { static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { Report("Event size too large.\n"); }); + pthread_once( + &Once, +[] { + Report("Custom event size too large; truncating to %d.\n", + std::numeric_limits::max()); + }); } - int32_t ReducedEventSize = static_cast(EventSize); + auto &TLD = getThreadLocalData(); - if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime)) + if (!setupTLD(TLD)) return; - // Here we need to prepare the log to handle: - // - The metadata record we're going to write. (16 bytes) - // - The additional data we're going to write. Currently, that's the size - // of the event we're going to dump into the log as free-form bytes. - if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) { - TLD.BQ = nullptr; - return; - } - - // Write the custom event metadata record, which consists of the following - // information: - // - 8 bytes (64-bits) for the full TSC when the event started. - // - 4 bytes (32-bits) for the length of the data. - MetadataRecord CustomEvent; - CustomEvent.Type = uint8_t(RecordType::Metadata); - CustomEvent.RecordKind = - uint8_t(MetadataRecord::RecordKinds::CustomEventMarker); - constexpr auto TSCSize = sizeof(TC.TSC); - internal_memcpy(&CustomEvent.Data, &ReducedEventSize, sizeof(int32_t)); - internal_memcpy(&CustomEvent.Data[sizeof(int32_t)], &TSC, TSCSize); - internal_memcpy(TLD.RecordPtr, &CustomEvent, sizeof(CustomEvent)); - TLD.RecordPtr += sizeof(CustomEvent); - internal_memcpy(TLD.RecordPtr, Event, ReducedEventSize); - incrementExtents(MetadataRecSize + EventSize); - endBufferIfFull(); + int32_t ReducedEventSize = static_cast(EventSize); + TLD.Controller->customEvent(TSC, CPU, Event, ReducedEventSize); } void fdrLoggingHandleTypedEvent( @@ -1018,50 +591,28 @@ void fdrLoggingHandleTypedEvent( RecursionGuard Guard{Running}; if (!Guard) return; - if (EventSize > std::numeric_limits::max()) { + + // Complain when we ever get at least one typed event that's larger than what + // we can possibly support. + if (EventSize > + static_cast(std::numeric_limits::max())) { static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { Report("Event size too large.\n"); }); + pthread_once( + &Once, +[] { + Report("Typed event size too large; truncating to %d.\n", + std::numeric_limits::max()); + }); } - int32_t ReducedEventSize = static_cast(EventSize); + auto &TLD = getThreadLocalData(); - if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime)) + if (!setupTLD(TLD)) return; - // Here we need to prepare the log to handle: - // - The metadata record we're going to write. (16 bytes) - // - The additional data we're going to write. Currently, that's the size - // of the event we're going to dump into the log as free-form bytes. - if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) { - TLD.BQ = nullptr; - return; - } - // Write the custom event metadata record, which consists of the following - // information: - // - 8 bytes (64-bits) for the full TSC when the event started. - // - 4 bytes (32-bits) for the length of the data. - // - 2 bytes (16-bits) for the event type. 3 bytes remain since one of the - // bytes has the record type (Metadata Record) and kind (TypedEvent). - // We'll log the error if the event type is greater than 2 bytes. - // Event types are generated sequentially, so 2^16 is enough. - MetadataRecord TypedEvent; - TypedEvent.Type = uint8_t(RecordType::Metadata); - TypedEvent.RecordKind = - uint8_t(MetadataRecord::RecordKinds::TypedEventMarker); - constexpr auto TSCSize = sizeof(TC.TSC); - internal_memcpy(&TypedEvent.Data, &ReducedEventSize, sizeof(int32_t)); - internal_memcpy(&TypedEvent.Data[sizeof(int32_t)], &TSC, TSCSize); - internal_memcpy(&TypedEvent.Data[sizeof(int32_t) + TSCSize], &EventType, - sizeof(EventType)); - internal_memcpy(TLD.RecordPtr, &TypedEvent, sizeof(TypedEvent)); - - TLD.RecordPtr += sizeof(TypedEvent); - internal_memcpy(TLD.RecordPtr, Event, ReducedEventSize); - incrementExtents(MetadataRecSize + EventSize); - endBufferIfFull(); + int32_t ReducedEventSize = static_cast(EventSize); + TLD.Controller->typedEvent(TSC, CPU, EventType, Event, ReducedEventSize); } -XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax, - void *Options, +XRayLogInitStatus fdrLoggingInit(size_t, size_t, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { if (Options == nullptr) return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; @@ -1075,107 +626,81 @@ XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax, return static_cast(CurrentStatus); } - // Because of __xray_log_init_mode(...) which guarantees that this will be - // called with BufferSize == 0 and BufferMax == 0 we parse the configuration - // provided in the Options pointer as a string instead. - if (BufferSize == 0 && BufferMax == 0) { - if (Verbosity()) - Report("Initializing FDR mode with options: %s\n", - static_cast(Options)); + if (Verbosity()) + Report("Initializing FDR mode with options: %s\n", + static_cast(Options)); - // TODO: Factor out the flags specific to the FDR mode implementation. For - // now, use the global/single definition of the flags, since the FDR mode - // flags are already defined there. - FlagParser FDRParser; - FDRFlags FDRFlags; - registerXRayFDRFlags(&FDRParser, &FDRFlags); - FDRFlags.setDefaults(); + // TODO: Factor out the flags specific to the FDR mode implementation. For + // now, use the global/single definition of the flags, since the FDR mode + // flags are already defined there. + FlagParser FDRParser; + FDRFlags FDRFlags; + registerXRayFDRFlags(&FDRParser, &FDRFlags); + FDRFlags.setDefaults(); - // Override first from the general XRAY_DEFAULT_OPTIONS compiler-provided - // options until we migrate everyone to use the XRAY_FDR_OPTIONS - // compiler-provided options. - FDRParser.ParseString(useCompilerDefinedFlags()); - FDRParser.ParseString(useCompilerDefinedFDRFlags()); - auto *EnvOpts = GetEnv("XRAY_FDR_OPTIONS"); - if (EnvOpts == nullptr) - EnvOpts = ""; - FDRParser.ParseString(EnvOpts); + // Override first from the general XRAY_DEFAULT_OPTIONS compiler-provided + // options until we migrate everyone to use the XRAY_FDR_OPTIONS + // compiler-provided options. + FDRParser.ParseString(useCompilerDefinedFlags()); + FDRParser.ParseString(useCompilerDefinedFDRFlags()); + auto *EnvOpts = GetEnv("XRAY_FDR_OPTIONS"); + if (EnvOpts == nullptr) + EnvOpts = ""; + FDRParser.ParseString(EnvOpts); - // FIXME: Remove this when we fully remove the deprecated flags. - if (internal_strlen(EnvOpts) == 0) { - FDRFlags.func_duration_threshold_us = - flags()->xray_fdr_log_func_duration_threshold_us; - FDRFlags.grace_period_ms = flags()->xray_fdr_log_grace_period_ms; - } - - // The provided options should always override the compiler-provided and - // environment-variable defined options. - FDRParser.ParseString(static_cast(Options)); - *fdrFlags() = FDRFlags; - BufferSize = FDRFlags.buffer_size; - BufferMax = FDRFlags.buffer_max; - SpinMutexLock Guard(&FDROptionsMutex); - FDROptions.Fd = -1; - FDROptions.ReportErrors = true; - } else if (OptionsSize != sizeof(FDRLoggingOptions)) { - // FIXME: This is deprecated, and should really be removed. - // At this point we use the flag parser specific to the FDR mode - // implementation. - if (Verbosity()) - Report("Cannot initialize FDR logging; wrong size for options: %d\n", - OptionsSize); - return static_cast( - atomic_load(&LoggingStatus, memory_order_acquire)); - } else { - if (Verbosity()) - Report("XRay FDR: struct-based init is deprecated, please use " - "string-based configuration instead.\n"); - SpinMutexLock Guard(&FDROptionsMutex); - internal_memcpy(&FDROptions, Options, OptionsSize); + // FIXME: Remove this when we fully remove the deprecated flags. + if (internal_strlen(EnvOpts) == 0) { + FDRFlags.func_duration_threshold_us = + flags()->xray_fdr_log_func_duration_threshold_us; + FDRFlags.grace_period_ms = flags()->xray_fdr_log_grace_period_ms; } - bool Success = false; - - if (BQ != nullptr) { - BQ->~BufferQueue(); - InternalFree(BQ); - BQ = nullptr; - } + // The provided options should always override the compiler-provided and + // environment-variable defined options. + FDRParser.ParseString(static_cast(Options)); + *fdrFlags() = FDRFlags; + auto BufferSize = FDRFlags.buffer_size; + auto BufferMax = FDRFlags.buffer_max; if (BQ == nullptr) { - BQ = reinterpret_cast( - InternalAlloc(sizeof(BufferQueue), nullptr, 64)); + bool Success = false; + BQ = reinterpret_cast(&BufferQueueStorage); new (BQ) BufferQueue(BufferSize, BufferMax, Success); - } - - if (!Success) { - Report("BufferQueue init failed.\n"); - if (BQ != nullptr) { - BQ->~BufferQueue(); - InternalFree(BQ); - BQ = nullptr; + if (!Success) { + Report("BufferQueue init failed.\n"); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + } else { + if (BQ->init(BufferSize, BufferMax) != BufferQueue::ErrorCode::Ok) { + if (Verbosity()) + Report("Failed to re-initialize global buffer queue. Init failed.\n"); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } - return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } static pthread_once_t OnceInit = PTHREAD_ONCE_INIT; - pthread_once(&OnceInit, +[] { - atomic_store(&TicksPerSec, - probeRequiredCPUFeatures() ? getTSCFrequency() - : __xray::NanosecondsPerSecond, - memory_order_release); - pthread_key_create(&Key, +[](void *TLDPtr) { - if (TLDPtr == nullptr) - return; - auto &TLD = *reinterpret_cast(TLDPtr); - if (TLD.BQ == nullptr) - return; - auto EC = TLD.BQ->releaseBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) - Report("At thread exit, failed to release buffer at %p; error=%s\n", - TLD.Buffer.Data, BufferQueue::getErrorString(EC)); - }); - }); + pthread_once( + &OnceInit, +[] { + atomic_store(&TicksPerSec, + probeRequiredCPUFeatures() ? getTSCFrequency() + : __xray::NanosecondsPerSecond, + memory_order_release); + pthread_key_create( + &Key, +[](void *TLDPtr) { + if (TLDPtr == nullptr) + return; + auto &TLD = *reinterpret_cast(TLDPtr); + if (TLD.BQ == nullptr) + return; + if (TLD.Buffer.Data == nullptr) + return; + auto EC = TLD.BQ->releaseBuffer(TLD.Buffer); + if (EC != BufferQueue::ErrorCode::Ok) + Report("At thread exit, failed to release buffer at %p; " + "error=%s\n", + TLD.Buffer.Data, BufferQueue::getErrorString(EC)); + }); + }); atomic_store(&ThresholdTicks, atomic_load_relaxed(&TicksPerSec) * @@ -1209,11 +734,22 @@ bool fdrLogDynamicInitializer() XRAY_NEVER_INSTRUMENT { }; auto RegistrationResult = __xray_log_register_mode("xray-fdr", Impl); if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && - Verbosity()) + Verbosity()) { Report("Cannot register XRay FDR mode to 'xray-fdr'; error = %d\n", RegistrationResult); - if (flags()->xray_fdr_log || !internal_strcmp(flags()->xray_mode, "xray-fdr")) - __xray_set_log_impl(Impl); + return false; + } + + if (flags()->xray_fdr_log || + !internal_strcmp(flags()->xray_mode, "xray-fdr")) { + auto SelectResult = __xray_log_select_mode("xray-fdr"); + if (SelectResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + Verbosity()) { + Report("Cannot select XRay FDR mode as 'xray-fdr'; error = %d\n", + SelectResult); + return false; + } + } return true; } diff --git a/contrib/compiler-rt/lib/xray/xray_function_call_trie.h b/contrib/compiler-rt/lib/xray/xray_function_call_trie.h index 2acf14aa562..d01ad20e3d7 100644 --- a/contrib/compiler-rt/lib/xray/xray_function_call_trie.h +++ b/contrib/compiler-rt/lib/xray/xray_function_call_trie.h @@ -15,9 +15,11 @@ #ifndef XRAY_FUNCTION_CALL_TRIE_H #define XRAY_FUNCTION_CALL_TRIE_H -#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "xray_buffer_queue.h" +#include "xray_defs.h" #include "xray_profiling_flags.h" #include "xray_segmented_array.h" +#include #include // For placement new. #include @@ -97,9 +99,6 @@ public: struct NodeIdPair { Node *NodePtr; int32_t FId; - - // Constructor for inplace-construction. - NodeIdPair(Node *N, int32_t F) : NodePtr(N), FId(F) {} }; using NodeIdPairArray = Array; @@ -113,17 +112,10 @@ public: struct Node { Node *Parent; NodeIdPairArray Callees; - int64_t CallCount; - int64_t CumulativeLocalTime; // Typically in TSC deltas, not wall-time. + uint64_t CallCount; + uint64_t CumulativeLocalTime; // Typically in TSC deltas, not wall-time. int32_t FId; - // We add a constructor here to allow us to inplace-construct through - // Array<...>'s AppendEmplace. - Node(Node *P, NodeIdPairAllocatorType &A, int64_t CC, int64_t CLT, - int32_t F) - : Parent(P), Callees(A), CallCount(CC), CumulativeLocalTime(CLT), - FId(F) {} - // TODO: Include the compact histogram. }; @@ -131,10 +123,7 @@ private: struct ShadowStackEntry { uint64_t EntryTSC; Node *NodePtr; - - // We add a constructor here to allow us to inplace-construct through - // Array<...>'s AppendEmplace. - ShadowStackEntry(uint64_t T, Node *N) : EntryTSC{T}, NodePtr{N} {} + uint16_t EntryCPU; }; using NodeArray = Array; @@ -149,103 +138,184 @@ public: using RootAllocatorType = RootArray::AllocatorType; using ShadowStackAllocatorType = ShadowStackArray::AllocatorType; + // Use hosted aligned storage members to allow for trivial move and init. + // This also allows us to sidestep the potential-failing allocation issue. + typename std::aligned_storage::type + NodeAllocatorStorage; + typename std::aligned_storage::type + RootAllocatorStorage; + typename std::aligned_storage::type + ShadowStackAllocatorStorage; + typename std::aligned_storage::type + NodeIdPairAllocatorStorage; + NodeAllocatorType *NodeAllocator = nullptr; RootAllocatorType *RootAllocator = nullptr; ShadowStackAllocatorType *ShadowStackAllocator = nullptr; NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr; - Allocators() {} + Allocators() = default; Allocators(const Allocators &) = delete; Allocators &operator=(const Allocators &) = delete; - Allocators(Allocators &&O) - : NodeAllocator(O.NodeAllocator), RootAllocator(O.RootAllocator), - ShadowStackAllocator(O.ShadowStackAllocator), - NodeIdPairAllocator(O.NodeIdPairAllocator) { + struct Buffers { + BufferQueue::Buffer NodeBuffer; + BufferQueue::Buffer RootsBuffer; + BufferQueue::Buffer ShadowStackBuffer; + BufferQueue::Buffer NodeIdPairBuffer; + }; + + explicit Allocators(Buffers &B) XRAY_NEVER_INSTRUMENT { + new (&NodeAllocatorStorage) + NodeAllocatorType(B.NodeBuffer.Data, B.NodeBuffer.Size); + NodeAllocator = + reinterpret_cast(&NodeAllocatorStorage); + + new (&RootAllocatorStorage) + RootAllocatorType(B.RootsBuffer.Data, B.RootsBuffer.Size); + RootAllocator = + reinterpret_cast(&RootAllocatorStorage); + + new (&ShadowStackAllocatorStorage) ShadowStackAllocatorType( + B.ShadowStackBuffer.Data, B.ShadowStackBuffer.Size); + ShadowStackAllocator = reinterpret_cast( + &ShadowStackAllocatorStorage); + + new (&NodeIdPairAllocatorStorage) NodeIdPairAllocatorType( + B.NodeIdPairBuffer.Data, B.NodeIdPairBuffer.Size); + NodeIdPairAllocator = reinterpret_cast( + &NodeIdPairAllocatorStorage); + } + + explicit Allocators(uptr Max) XRAY_NEVER_INSTRUMENT { + new (&NodeAllocatorStorage) NodeAllocatorType(Max); + NodeAllocator = + reinterpret_cast(&NodeAllocatorStorage); + + new (&RootAllocatorStorage) RootAllocatorType(Max); + RootAllocator = + reinterpret_cast(&RootAllocatorStorage); + + new (&ShadowStackAllocatorStorage) ShadowStackAllocatorType(Max); + ShadowStackAllocator = reinterpret_cast( + &ShadowStackAllocatorStorage); + + new (&NodeIdPairAllocatorStorage) NodeIdPairAllocatorType(Max); + NodeIdPairAllocator = reinterpret_cast( + &NodeIdPairAllocatorStorage); + } + + Allocators(Allocators &&O) XRAY_NEVER_INSTRUMENT { + // Here we rely on the safety of memcpy'ing contents of the storage + // members, and then pointing the source pointers to nullptr. + internal_memcpy(&NodeAllocatorStorage, &O.NodeAllocatorStorage, + sizeof(NodeAllocatorType)); + internal_memcpy(&RootAllocatorStorage, &O.RootAllocatorStorage, + sizeof(RootAllocatorType)); + internal_memcpy(&ShadowStackAllocatorStorage, + &O.ShadowStackAllocatorStorage, + sizeof(ShadowStackAllocatorType)); + internal_memcpy(&NodeIdPairAllocatorStorage, + &O.NodeIdPairAllocatorStorage, + sizeof(NodeIdPairAllocatorType)); + + NodeAllocator = + reinterpret_cast(&NodeAllocatorStorage); + RootAllocator = + reinterpret_cast(&RootAllocatorStorage); + ShadowStackAllocator = reinterpret_cast( + &ShadowStackAllocatorStorage); + NodeIdPairAllocator = reinterpret_cast( + &NodeIdPairAllocatorStorage); + O.NodeAllocator = nullptr; O.RootAllocator = nullptr; O.ShadowStackAllocator = nullptr; O.NodeIdPairAllocator = nullptr; } - Allocators &operator=(Allocators &&O) { - { - auto Tmp = O.NodeAllocator; - O.NodeAllocator = this->NodeAllocator; - this->NodeAllocator = Tmp; + Allocators &operator=(Allocators &&O) XRAY_NEVER_INSTRUMENT { + // When moving into an existing instance, we ensure that we clean up the + // current allocators. + if (NodeAllocator) + NodeAllocator->~NodeAllocatorType(); + if (O.NodeAllocator) { + new (&NodeAllocatorStorage) + NodeAllocatorType(std::move(*O.NodeAllocator)); + NodeAllocator = + reinterpret_cast(&NodeAllocatorStorage); + O.NodeAllocator = nullptr; + } else { + NodeAllocator = nullptr; } - { - auto Tmp = O.RootAllocator; - O.RootAllocator = this->RootAllocator; - this->RootAllocator = Tmp; + + if (RootAllocator) + RootAllocator->~RootAllocatorType(); + if (O.RootAllocator) { + new (&RootAllocatorStorage) + RootAllocatorType(std::move(*O.RootAllocator)); + RootAllocator = + reinterpret_cast(&RootAllocatorStorage); + O.RootAllocator = nullptr; + } else { + RootAllocator = nullptr; } - { - auto Tmp = O.ShadowStackAllocator; - O.ShadowStackAllocator = this->ShadowStackAllocator; - this->ShadowStackAllocator = Tmp; + + if (ShadowStackAllocator) + ShadowStackAllocator->~ShadowStackAllocatorType(); + if (O.ShadowStackAllocator) { + new (&ShadowStackAllocatorStorage) + ShadowStackAllocatorType(std::move(*O.ShadowStackAllocator)); + ShadowStackAllocator = reinterpret_cast( + &ShadowStackAllocatorStorage); + O.ShadowStackAllocator = nullptr; + } else { + ShadowStackAllocator = nullptr; } - { - auto Tmp = O.NodeIdPairAllocator; - O.NodeIdPairAllocator = this->NodeIdPairAllocator; - this->NodeIdPairAllocator = Tmp; + + if (NodeIdPairAllocator) + NodeIdPairAllocator->~NodeIdPairAllocatorType(); + if (O.NodeIdPairAllocator) { + new (&NodeIdPairAllocatorStorage) + NodeIdPairAllocatorType(std::move(*O.NodeIdPairAllocator)); + NodeIdPairAllocator = reinterpret_cast( + &NodeIdPairAllocatorStorage); + O.NodeIdPairAllocator = nullptr; + } else { + NodeIdPairAllocator = nullptr; } + return *this; } - ~Allocators() { - // Note that we cannot use delete on these pointers, as they need to be - // returned to the sanitizer_common library's internal memory tracking - // system. - if (NodeAllocator != nullptr) { + ~Allocators() XRAY_NEVER_INSTRUMENT { + if (NodeAllocator != nullptr) NodeAllocator->~NodeAllocatorType(); - InternalFree(NodeAllocator); - NodeAllocator = nullptr; - } - if (RootAllocator != nullptr) { + if (RootAllocator != nullptr) RootAllocator->~RootAllocatorType(); - InternalFree(RootAllocator); - RootAllocator = nullptr; - } - if (ShadowStackAllocator != nullptr) { + if (ShadowStackAllocator != nullptr) ShadowStackAllocator->~ShadowStackAllocatorType(); - InternalFree(ShadowStackAllocator); - ShadowStackAllocator = nullptr; - } - if (NodeIdPairAllocator != nullptr) { + if (NodeIdPairAllocator != nullptr) NodeIdPairAllocator->~NodeIdPairAllocatorType(); - InternalFree(NodeIdPairAllocator); - NodeIdPairAllocator = nullptr; - } } }; - // TODO: Support configuration of options through the arguments. - static Allocators InitAllocators() { + static Allocators InitAllocators() XRAY_NEVER_INSTRUMENT { return InitAllocatorsCustom(profilingFlags()->per_thread_allocator_max); } - static Allocators InitAllocatorsCustom(uptr Max) { - Allocators A; - auto NodeAllocator = reinterpret_cast( - InternalAlloc(sizeof(Allocators::NodeAllocatorType))); - new (NodeAllocator) Allocators::NodeAllocatorType(Max); - A.NodeAllocator = NodeAllocator; + static Allocators InitAllocatorsCustom(uptr Max) XRAY_NEVER_INSTRUMENT { + Allocators A(Max); + return A; + } - auto RootAllocator = reinterpret_cast( - InternalAlloc(sizeof(Allocators::RootAllocatorType))); - new (RootAllocator) Allocators::RootAllocatorType(Max); - A.RootAllocator = RootAllocator; - - auto ShadowStackAllocator = - reinterpret_cast( - InternalAlloc(sizeof(Allocators::ShadowStackAllocatorType))); - new (ShadowStackAllocator) Allocators::ShadowStackAllocatorType(Max); - A.ShadowStackAllocator = ShadowStackAllocator; - - auto NodeIdPairAllocator = reinterpret_cast( - InternalAlloc(sizeof(NodeIdPairAllocatorType))); - new (NodeIdPairAllocator) NodeIdPairAllocatorType(Max); - A.NodeIdPairAllocator = NodeIdPairAllocator; + static Allocators + InitAllocatorsFromBuffers(Allocators::Buffers &Bufs) XRAY_NEVER_INSTRUMENT { + Allocators A(Bufs); return A; } @@ -253,65 +323,135 @@ private: NodeArray Nodes; RootArray Roots; ShadowStackArray ShadowStack; - NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr; + NodeIdPairAllocatorType *NodeIdPairAllocator; + uint32_t OverflowedFunctions; public: - explicit FunctionCallTrie(const Allocators &A) - : Nodes(*A.NodeAllocator), Roots(*A.RootAllocator), + explicit FunctionCallTrie(const Allocators &A) XRAY_NEVER_INSTRUMENT + : Nodes(*A.NodeAllocator), + Roots(*A.RootAllocator), ShadowStack(*A.ShadowStackAllocator), - NodeIdPairAllocator(A.NodeIdPairAllocator) {} + NodeIdPairAllocator(A.NodeIdPairAllocator), + OverflowedFunctions(0) {} - void enterFunction(const int32_t FId, uint64_t TSC) { + FunctionCallTrie() = delete; + FunctionCallTrie(const FunctionCallTrie &) = delete; + FunctionCallTrie &operator=(const FunctionCallTrie &) = delete; + + FunctionCallTrie(FunctionCallTrie &&O) XRAY_NEVER_INSTRUMENT + : Nodes(std::move(O.Nodes)), + Roots(std::move(O.Roots)), + ShadowStack(std::move(O.ShadowStack)), + NodeIdPairAllocator(O.NodeIdPairAllocator), + OverflowedFunctions(O.OverflowedFunctions) {} + + FunctionCallTrie &operator=(FunctionCallTrie &&O) XRAY_NEVER_INSTRUMENT { + Nodes = std::move(O.Nodes); + Roots = std::move(O.Roots); + ShadowStack = std::move(O.ShadowStack); + NodeIdPairAllocator = O.NodeIdPairAllocator; + OverflowedFunctions = O.OverflowedFunctions; + return *this; + } + + ~FunctionCallTrie() XRAY_NEVER_INSTRUMENT {} + + void enterFunction(const int32_t FId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { DCHECK_NE(FId, 0); - // This function primarily deals with ensuring that the ShadowStack is - // consistent and ready for when an exit event is encountered. - if (UNLIKELY(ShadowStack.empty())) { - auto NewRoot = - Nodes.AppendEmplace(nullptr, *NodeIdPairAllocator, 0, 0, FId); - if (UNLIKELY(NewRoot == nullptr)) - return; - Roots.Append(NewRoot); - ShadowStack.AppendEmplace(TSC, NewRoot); + + // If we're already overflowed the function call stack, do not bother + // attempting to record any more function entries. + if (UNLIKELY(OverflowedFunctions)) { + ++OverflowedFunctions; return; } - auto &Top = ShadowStack.back(); - auto TopNode = Top.NodePtr; + // If this is the first function we've encountered, we want to set up the + // node(s) and treat it as a root. + if (UNLIKELY(ShadowStack.empty())) { + auto *NewRoot = Nodes.AppendEmplace( + nullptr, NodeIdPairArray(*NodeIdPairAllocator), 0u, 0u, FId); + if (UNLIKELY(NewRoot == nullptr)) + return; + if (Roots.AppendEmplace(NewRoot) == nullptr) { + Nodes.trim(1); + return; + } + if (ShadowStack.AppendEmplace(TSC, NewRoot, CPU) == nullptr) { + Nodes.trim(1); + Roots.trim(1); + ++OverflowedFunctions; + return; + } + return; + } + + // From this point on, we require that the stack is not empty. + DCHECK(!ShadowStack.empty()); + auto TopNode = ShadowStack.back().NodePtr; DCHECK_NE(TopNode, nullptr); - // If we've seen this callee before, then we just access that node and place - // that on the top of the stack. - auto Callee = TopNode->Callees.find_element( + // If we've seen this callee before, then we access that node and place that + // on the top of the stack. + auto* Callee = TopNode->Callees.find_element( [FId](const NodeIdPair &NR) { return NR.FId == FId; }); if (Callee != nullptr) { CHECK_NE(Callee->NodePtr, nullptr); - ShadowStack.AppendEmplace(TSC, Callee->NodePtr); + if (ShadowStack.AppendEmplace(TSC, Callee->NodePtr, CPU) == nullptr) + ++OverflowedFunctions; return; } // This means we've never seen this stack before, create a new node here. - auto NewNode = - Nodes.AppendEmplace(TopNode, *NodeIdPairAllocator, 0, 0, FId); + auto* NewNode = Nodes.AppendEmplace( + TopNode, NodeIdPairArray(*NodeIdPairAllocator), 0u, 0u, FId); if (UNLIKELY(NewNode == nullptr)) return; DCHECK_NE(NewNode, nullptr); TopNode->Callees.AppendEmplace(NewNode, FId); - ShadowStack.AppendEmplace(TSC, NewNode); - DCHECK_NE(ShadowStack.back().NodePtr, nullptr); + if (ShadowStack.AppendEmplace(TSC, NewNode, CPU) == nullptr) + ++OverflowedFunctions; return; } - void exitFunction(int32_t FId, uint64_t TSC) { + void exitFunction(int32_t FId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + // If we're exiting functions that have "overflowed" or don't fit into the + // stack due to allocator constraints, we then decrement that count first. + if (OverflowedFunctions) { + --OverflowedFunctions; + return; + } + // When we exit a function, we look up the ShadowStack to see whether we've // entered this function before. We do as little processing here as we can, // since most of the hard work would have already been done at function // entry. uint64_t CumulativeTreeTime = 0; + while (!ShadowStack.empty()) { const auto &Top = ShadowStack.back(); auto TopNode = Top.NodePtr; DCHECK_NE(TopNode, nullptr); - auto LocalTime = TSC - Top.EntryTSC; + + // We may encounter overflow on the TSC we're provided, which may end up + // being less than the TSC when we first entered the function. + // + // To get the accurate measurement of cycles, we need to check whether + // we've overflowed (TSC < Top.EntryTSC) and then account the difference + // between the entry TSC and the max for the TSC counter (max of uint64_t) + // then add the value of TSC. We can prove that the maximum delta we will + // get is at most the 64-bit unsigned value, since the difference between + // a TSC of 0 and a Top.EntryTSC of 1 is (numeric_limits::max() + // - 1) + 1. + // + // NOTE: This assumes that TSCs are synchronised across CPUs. + // TODO: Count the number of times we've seen CPU migrations. + uint64_t LocalTime = + Top.EntryTSC > TSC + ? (std::numeric_limits::max() - Top.EntryTSC) + TSC + : TSC - Top.EntryTSC; TopNode->CallCount++; TopNode->CumulativeLocalTime += LocalTime - CumulativeTreeTime; CumulativeTreeTime += LocalTime; @@ -323,7 +463,7 @@ public: } } - const RootArray &getRoots() const { return Roots; } + const RootArray &getRoots() const XRAY_NEVER_INSTRUMENT { return Roots; } // The deepCopyInto operation will update the provided FunctionCallTrie by // re-creating the contents of this particular FunctionCallTrie in the other @@ -338,7 +478,7 @@ public: // synchronisation of both "this" and |O|. // // This function must *not* be called with a non-empty FunctionCallTrie |O|. - void deepCopyInto(FunctionCallTrie &O) const { + void deepCopyInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT { DCHECK(O.getRoots().empty()); // We then push the root into a stack, to use as the parent marker for new @@ -356,18 +496,20 @@ public: for (const auto Root : getRoots()) { // Add a node in O for this root. auto NewRoot = O.Nodes.AppendEmplace( - nullptr, *O.NodeIdPairAllocator, Root->CallCount, + nullptr, NodeIdPairArray(*O.NodeIdPairAllocator), Root->CallCount, Root->CumulativeLocalTime, Root->FId); // Because we cannot allocate more memory we should bail out right away. if (UNLIKELY(NewRoot == nullptr)) return; - O.Roots.Append(NewRoot); + if (UNLIKELY(O.Roots.Append(NewRoot) == nullptr)) + return; // TODO: Figure out what to do if we fail to allocate any more stack // space. Maybe warn or report once? - DFSStack.AppendEmplace(Root, NewRoot); + if (DFSStack.AppendEmplace(Root, NewRoot) == nullptr) + return; while (!DFSStack.empty()) { NodeAndParent NP = DFSStack.back(); DCHECK_NE(NP.Node, nullptr); @@ -375,12 +517,17 @@ public: DFSStack.trim(1); for (const auto Callee : NP.Node->Callees) { auto NewNode = O.Nodes.AppendEmplace( - NP.NewNode, *O.NodeIdPairAllocator, Callee.NodePtr->CallCount, - Callee.NodePtr->CumulativeLocalTime, Callee.FId); + NP.NewNode, NodeIdPairArray(*O.NodeIdPairAllocator), + Callee.NodePtr->CallCount, Callee.NodePtr->CumulativeLocalTime, + Callee.FId); if (UNLIKELY(NewNode == nullptr)) return; - NP.NewNode->Callees.AppendEmplace(NewNode, Callee.FId); - DFSStack.AppendEmplace(Callee.NodePtr, NewNode); + if (UNLIKELY(NP.NewNode->Callees.AppendEmplace(NewNode, Callee.FId) == + nullptr)) + return; + if (UNLIKELY(DFSStack.AppendEmplace(Callee.NodePtr, NewNode) == + nullptr)) + return; } } } @@ -394,7 +541,7 @@ public: // // This function is *not* thread-safe, and may require external // synchronisation of both "this" and |O|. - void mergeInto(FunctionCallTrie &O) const { + void mergeInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT { struct NodeAndTarget { FunctionCallTrie::Node *OrigNode; FunctionCallTrie::Node *TargetNode; @@ -409,8 +556,9 @@ public: auto R = O.Roots.find_element( [&](const Node *Node) { return Node->FId == Root->FId; }); if (R == nullptr) { - TargetRoot = O.Nodes.AppendEmplace(nullptr, *O.NodeIdPairAllocator, 0, - 0, Root->FId); + TargetRoot = O.Nodes.AppendEmplace( + nullptr, NodeIdPairArray(*O.NodeIdPairAllocator), 0u, 0u, + Root->FId); if (UNLIKELY(TargetRoot == nullptr)) return; @@ -419,7 +567,7 @@ public: TargetRoot = *R; } - DFSStack.Append(NodeAndTarget{Root, TargetRoot}); + DFSStack.AppendEmplace(Root, TargetRoot); while (!DFSStack.empty()) { NodeAndTarget NT = DFSStack.back(); DCHECK_NE(NT.OrigNode, nullptr); @@ -435,7 +583,8 @@ public: }); if (TargetCallee == nullptr) { auto NewTargetNode = O.Nodes.AppendEmplace( - NT.TargetNode, *O.NodeIdPairAllocator, 0, 0, Callee.FId); + NT.TargetNode, NodeIdPairArray(*O.NodeIdPairAllocator), 0u, 0u, + Callee.FId); if (UNLIKELY(NewTargetNode == nullptr)) return; diff --git a/contrib/compiler-rt/lib/xray/xray_init.cc b/contrib/compiler-rt/lib/xray/xray_init.cc index b4e06979519..b0922aa8e37 100644 --- a/contrib/compiler-rt/lib/xray/xray_init.cc +++ b/contrib/compiler-rt/lib/xray/xray_init.cc @@ -27,6 +27,15 @@ extern const XRaySledEntry __start_xray_instr_map[] __attribute__((weak)); extern const XRaySledEntry __stop_xray_instr_map[] __attribute__((weak)); extern const XRayFunctionSledIndex __start_xray_fn_idx[] __attribute__((weak)); extern const XRayFunctionSledIndex __stop_xray_fn_idx[] __attribute__((weak)); + +#if SANITIZER_MAC +// HACK: This is a temporary workaround to make XRay build on +// Darwin, but it will probably not work at runtime. +const XRaySledEntry __start_xray_instr_map[] = {}; +extern const XRaySledEntry __stop_xray_instr_map[] = {}; +extern const XRayFunctionSledIndex __start_xray_fn_idx[] = {}; +extern const XRayFunctionSledIndex __stop_xray_fn_idx[] = {}; +#endif } using namespace __xray; @@ -58,6 +67,9 @@ void __xray_init() XRAY_NEVER_INSTRUMENT { if (atomic_load(&XRayInitialized, memory_order_acquire)) return; + // XRAY is not compatible with PaX MPROTECT + CheckMPROTECT(); + if (!atomic_load(&XRayFlagsInitialized, memory_order_acquire)) { initializeFlags(); atomic_store(&XRayFlagsInitialized, true, memory_order_release); @@ -97,8 +109,8 @@ __attribute__((section(".preinit_array"), #else // If we cannot use the .preinit_array section, we should instead use dynamic // initialisation. -static bool UNUSED __local_xray_dyninit = [] { +__attribute__ ((constructor (0))) +static void __local_xray_dyninit() { __xray_init(); - return true; -}(); +} #endif diff --git a/contrib/compiler-rt/lib/xray/xray_interface.cc b/contrib/compiler-rt/lib/xray/xray_interface.cc index 01bf6ddc607..6f7b6615b2c 100644 --- a/contrib/compiler-rt/lib/xray/xray_interface.cc +++ b/contrib/compiler-rt/lib/xray/xray_interface.cc @@ -22,6 +22,13 @@ #include #include +#if SANITIZER_FUCHSIA +#include +#include +#include +#include +#endif + #include "sanitizer_common/sanitizer_addrhashmap.h" #include "sanitizer_common/sanitizer_common.h" @@ -92,22 +99,48 @@ class MProtectHelper { public: explicit MProtectHelper(void *PageAlignedAddr, - std::size_t MProtectLen) XRAY_NEVER_INSTRUMENT + std::size_t MProtectLen, + std::size_t PageSize) XRAY_NEVER_INSTRUMENT : PageAlignedAddr(PageAlignedAddr), MProtectLen(MProtectLen), - MustCleanup(false) {} + MustCleanup(false) { +#if SANITIZER_FUCHSIA + MProtectLen = RoundUpTo(MProtectLen, PageSize); +#endif + } int MakeWriteable() XRAY_NEVER_INSTRUMENT { +#if SANITIZER_FUCHSIA + auto R = __sanitizer_change_code_protection( + reinterpret_cast(PageAlignedAddr), MProtectLen, true); + if (R != ZX_OK) { + Report("XRay: cannot change code protection: %s\n", + _zx_status_get_string(R)); + return -1; + } + MustCleanup = true; + return 0; +#else auto R = mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_WRITE | PROT_EXEC); if (R != -1) MustCleanup = true; return R; +#endif } ~MProtectHelper() XRAY_NEVER_INSTRUMENT { if (MustCleanup) { +#if SANITIZER_FUCHSIA + auto R = __sanitizer_change_code_protection( + reinterpret_cast(PageAlignedAddr), MProtectLen, false); + if (R != ZX_OK) { + Report("XRay: cannot change code protection: %s\n", + _zx_status_get_string(R)); + } +#else mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC); +#endif } } }; @@ -254,7 +287,7 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { reinterpret_cast(MinSled.Address & ~(PageSize - 1)); size_t MProtectLen = (MaxSled.Address - reinterpret_cast(PageAlignedAddr)) + cSledLength; - MProtectHelper Protector(PageAlignedAddr, MProtectLen); + MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); if (Protector.MakeWriteable() == -1) { Report("Failed mprotect: %d\n", errno); return XRayPatchingStatus::FAILED; @@ -319,7 +352,7 @@ XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, reinterpret_cast(MinSled.Address & ~(PageSize - 1)); size_t MProtectLen = (MaxSled.Address - reinterpret_cast(PageAlignedAddr)) + cSledLength; - MProtectHelper Protector(PageAlignedAddr, MProtectLen); + MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); if (Protector.MakeWriteable() == -1) { Report("Failed mprotect: %d\n", errno); return XRayPatchingStatus::FAILED; diff --git a/contrib/compiler-rt/lib/xray/xray_profile_collector.cc b/contrib/compiler-rt/lib/xray/xray_profile_collector.cc index 17a611eeacb..dc3a8206984 100644 --- a/contrib/compiler-rt/lib/xray/xray_profile_collector.cc +++ b/contrib/compiler-rt/lib/xray/xray_profile_collector.cc @@ -13,10 +13,11 @@ // //===----------------------------------------------------------------------===// #include "xray_profile_collector.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_vector.h" +#include "xray_allocator.h" +#include "xray_defs.h" #include "xray_profiling_flags.h" +#include "xray_segmented_array.h" #include #include #include @@ -29,7 +30,7 @@ namespace { SpinMutex GlobalMutex; struct ThreadTrie { tid_t TId; - FunctionCallTrie *Trie; + typename std::aligned_storage::type TrieStorage; }; struct ProfileBuffer { @@ -56,65 +57,91 @@ struct BlockHeader { u64 ThreadId; }; -// These need to be pointers that point to heap/internal-allocator-allocated -// objects because these are accessed even at program exit. -Vector *ThreadTries = nullptr; -Vector *ProfileBuffers = nullptr; -FunctionCallTrie::Allocators *GlobalAllocators = nullptr; +struct ThreadData { + BufferQueue *BQ; + FunctionCallTrie::Allocators::Buffers Buffers; + FunctionCallTrie::Allocators Allocators; + FunctionCallTrie FCT; + tid_t TId; +}; + +using ThreadDataArray = Array; +using ThreadDataAllocator = ThreadDataArray::AllocatorType; + +// We use a separate buffer queue for the backing store for the allocator used +// by the ThreadData array. This lets us host the buffers, allocators, and tries +// associated with a thread by moving the data into the array instead of +// attempting to copy the data to a separately backed set of tries. +static typename std::aligned_storage< + sizeof(BufferQueue), alignof(BufferQueue)>::type BufferQueueStorage; +static BufferQueue *BQ = nullptr; +static BufferQueue::Buffer Buffer; +static typename std::aligned_storage::type + ThreadDataAllocatorStorage; +static typename std::aligned_storage::type + ThreadDataArrayStorage; + +static ThreadDataAllocator *TDAllocator = nullptr; +static ThreadDataArray *TDArray = nullptr; + +using ProfileBufferArray = Array; +using ProfileBufferArrayAllocator = typename ProfileBufferArray::AllocatorType; + +// These need to be global aligned storage to avoid dynamic initialization. We +// need these to be aligned to allow us to placement new objects into the +// storage, and have pointers to those objects be appropriately aligned. +static typename std::aligned_storage::type + ProfileBuffersStorage; +static typename std::aligned_storage::type + ProfileBufferArrayAllocatorStorage; + +static ProfileBufferArrayAllocator *ProfileBuffersAllocator = nullptr; +static ProfileBufferArray *ProfileBuffers = nullptr; + +// Use a global flag to determine whether the collector implementation has been +// initialized. +static atomic_uint8_t CollectorInitialized{0}; } // namespace -void post(const FunctionCallTrie &T, tid_t TId) { - static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { - SpinMutexLock Lock(&GlobalMutex); - GlobalAllocators = reinterpret_cast( - InternalAlloc(sizeof(FunctionCallTrie::Allocators))); - new (GlobalAllocators) FunctionCallTrie::Allocators(); - *GlobalAllocators = FunctionCallTrie::InitAllocatorsCustom( - profilingFlags()->global_allocator_max); - ThreadTries = reinterpret_cast *>( - InternalAlloc(sizeof(Vector))); - new (ThreadTries) Vector(); - ProfileBuffers = reinterpret_cast *>( - InternalAlloc(sizeof(Vector))); - new (ProfileBuffers) Vector(); - }); - DCHECK_NE(GlobalAllocators, nullptr); - DCHECK_NE(ThreadTries, nullptr); - DCHECK_NE(ProfileBuffers, nullptr); +void post(BufferQueue *Q, FunctionCallTrie &&T, + FunctionCallTrie::Allocators &&A, + FunctionCallTrie::Allocators::Buffers &&B, + tid_t TId) XRAY_NEVER_INSTRUMENT { + DCHECK_NE(Q, nullptr); - ThreadTrie *Item = nullptr; - { - SpinMutexLock Lock(&GlobalMutex); - if (GlobalAllocators == nullptr) - return; - - Item = ThreadTries->PushBack(); - Item->TId = TId; - - // Here we're using the internal allocator instead of the managed allocator - // because: - // - // 1) We're not using the segmented array data structure to host - // FunctionCallTrie objects. We're using a Vector (from sanitizer_common) - // which works like a std::vector<...> keeping elements contiguous in - // memory. The segmented array data structure assumes that elements are - // trivially destructible, where FunctionCallTrie isn't. - // - // 2) Using a managed allocator means we need to manage that separately, - // which complicates the nature of this code. To get around that, we're - // using the internal allocator instead, which has its own global state - // and is decoupled from the lifetime management required by the managed - // allocator we have in XRay. - // - Item->Trie = reinterpret_cast(InternalAlloc( - sizeof(FunctionCallTrie), nullptr, alignof(FunctionCallTrie))); - DCHECK_NE(Item->Trie, nullptr); - new (Item->Trie) FunctionCallTrie(*GlobalAllocators); + // Bail out early if the collector has not been initialized. + if (!atomic_load(&CollectorInitialized, memory_order_acquire)) { + T.~FunctionCallTrie(); + A.~Allocators(); + Q->releaseBuffer(B.NodeBuffer); + Q->releaseBuffer(B.RootsBuffer); + Q->releaseBuffer(B.ShadowStackBuffer); + Q->releaseBuffer(B.NodeIdPairBuffer); + B.~Buffers(); + return; } - T.deepCopyInto(*Item->Trie); + { + SpinMutexLock Lock(&GlobalMutex); + DCHECK_NE(TDAllocator, nullptr); + DCHECK_NE(TDArray, nullptr); + + if (TDArray->AppendEmplace(Q, std::move(B), std::move(A), std::move(T), + TId) == nullptr) { + // If we fail to add the data to the array, we should destroy the objects + // handed us. + T.~FunctionCallTrie(); + A.~Allocators(); + Q->releaseBuffer(B.NodeBuffer); + Q->releaseBuffer(B.RootsBuffer); + Q->releaseBuffer(B.ShadowStackBuffer); + Q->releaseBuffer(B.NodeIdPairBuffer); + B.~Buffers(); + } + } } // A PathArray represents the function id's representing a stack trace. In this @@ -127,18 +154,8 @@ struct ProfileRecord { // The Path in this record is the function id's from the leaf to the root of // the function call stack as represented from a FunctionCallTrie. - PathArray *Path = nullptr; - const FunctionCallTrie::Node *Node = nullptr; - - // Constructor for in-place construction. - ProfileRecord(PathAllocator &A, const FunctionCallTrie::Node *N) - : Path([&] { - auto P = - reinterpret_cast(InternalAlloc(sizeof(PathArray))); - new (P) PathArray(A); - return P; - }()), - Node(N) {} + PathArray Path; + const FunctionCallTrie::Node *Node; }; namespace { @@ -147,19 +164,21 @@ using ProfileRecordArray = Array; // Walk a depth-first traversal of each root of the FunctionCallTrie to generate // the path(s) and the data associated with the path. -static void populateRecords(ProfileRecordArray &PRs, - ProfileRecord::PathAllocator &PA, - const FunctionCallTrie &Trie) { +static void +populateRecords(ProfileRecordArray &PRs, ProfileRecord::PathAllocator &PA, + const FunctionCallTrie &Trie) XRAY_NEVER_INSTRUMENT { using StackArray = Array; using StackAllocator = typename StackArray::AllocatorType; StackAllocator StackAlloc(profilingFlags()->stack_allocator_max); StackArray DFSStack(StackAlloc); - for (const auto R : Trie.getRoots()) { + for (const auto *R : Trie.getRoots()) { DFSStack.Append(R); while (!DFSStack.empty()) { - auto Node = DFSStack.back(); + auto *Node = DFSStack.back(); DFSStack.trim(1); - auto Record = PRs.AppendEmplace(PA, Node); + if (Node == nullptr) + continue; + auto Record = PRs.AppendEmplace(PathArray{PA}, Node); if (Record == nullptr) return; DCHECK_NE(Record, nullptr); @@ -167,8 +186,8 @@ static void populateRecords(ProfileRecordArray &PRs, // Traverse the Node's parents and as we're doing so, get the FIds in // the order they appear. for (auto N = Node; N != nullptr; N = N->Parent) - Record->Path->Append(N->FId); - DCHECK(!Record->Path->empty()); + Record->Path.Append(N->FId); + DCHECK(!Record->Path.empty()); for (const auto C : Node->Callees) DFSStack.Append(C.NodePtr); @@ -177,67 +196,89 @@ static void populateRecords(ProfileRecordArray &PRs, } static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header, - const ProfileRecordArray &ProfileRecords) { - auto NextPtr = static_cast( + const ProfileRecordArray &ProfileRecords) + XRAY_NEVER_INSTRUMENT { + auto NextPtr = static_cast( internal_memcpy(Buffer->Data, &Header, sizeof(Header))) + sizeof(Header); for (const auto &Record : ProfileRecords) { // List of IDs follow: - for (const auto FId : *Record.Path) + for (const auto FId : Record.Path) NextPtr = - static_cast(internal_memcpy(NextPtr, &FId, sizeof(FId))) + + static_cast(internal_memcpy(NextPtr, &FId, sizeof(FId))) + sizeof(FId); // Add the sentinel here. constexpr int32_t SentinelFId = 0; - NextPtr = static_cast( + NextPtr = static_cast( internal_memset(NextPtr, SentinelFId, sizeof(SentinelFId))) + sizeof(SentinelFId); // Add the node data here. NextPtr = - static_cast(internal_memcpy(NextPtr, &Record.Node->CallCount, - sizeof(Record.Node->CallCount))) + + static_cast(internal_memcpy( + NextPtr, &Record.Node->CallCount, sizeof(Record.Node->CallCount))) + sizeof(Record.Node->CallCount); - NextPtr = static_cast( + NextPtr = static_cast( internal_memcpy(NextPtr, &Record.Node->CumulativeLocalTime, sizeof(Record.Node->CumulativeLocalTime))) + sizeof(Record.Node->CumulativeLocalTime); } - DCHECK_EQ(NextPtr - static_cast(Buffer->Data), Buffer->Size); + DCHECK_EQ(NextPtr - static_cast(Buffer->Data), Buffer->Size); } } // namespace -void serialize() { +void serialize() XRAY_NEVER_INSTRUMENT { + if (!atomic_load(&CollectorInitialized, memory_order_acquire)) + return; + SpinMutexLock Lock(&GlobalMutex); - // Clear out the global ProfileBuffers. - for (uptr I = 0; I < ProfileBuffers->Size(); ++I) - InternalFree((*ProfileBuffers)[I].Data); - ProfileBuffers->Reset(); + // Clear out the global ProfileBuffers, if it's not empty. + for (auto &B : *ProfileBuffers) + deallocateBuffer(reinterpret_cast(B.Data), B.Size); + ProfileBuffers->trim(ProfileBuffers->size()); - if (ThreadTries->Size() == 0) + DCHECK_NE(TDArray, nullptr); + if (TDArray->empty()) return; // Then repopulate the global ProfileBuffers. - for (u32 I = 0; I < ThreadTries->Size(); ++I) { + u32 I = 0; + auto MaxSize = profilingFlags()->global_allocator_max; + auto ProfileArena = allocateBuffer(MaxSize); + if (ProfileArena == nullptr) + return; + + auto ProfileArenaCleanup = at_scope_exit( + [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(ProfileArena, MaxSize); }); + + auto PathArena = allocateBuffer(profilingFlags()->global_allocator_max); + if (PathArena == nullptr) + return; + + auto PathArenaCleanup = at_scope_exit( + [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(PathArena, MaxSize); }); + + for (const auto &ThreadTrie : *TDArray) { using ProfileRecordAllocator = typename ProfileRecordArray::AllocatorType; - ProfileRecordAllocator PRAlloc(profilingFlags()->global_allocator_max); + ProfileRecordAllocator PRAlloc(ProfileArena, + profilingFlags()->global_allocator_max); ProfileRecord::PathAllocator PathAlloc( - profilingFlags()->global_allocator_max); + PathArena, profilingFlags()->global_allocator_max); ProfileRecordArray ProfileRecords(PRAlloc); // First, we want to compute the amount of space we're going to need. We'll // use a local allocator and an __xray::Array<...> to store the intermediary // data, then compute the size as we're going along. Then we'll allocate the // contiguous space to contain the thread buffer data. - const auto &Trie = *(*ThreadTries)[I].Trie; - if (Trie.getRoots().empty()) + if (ThreadTrie.FCT.getRoots().empty()) continue; - populateRecords(ProfileRecords, PathAlloc, Trie); - DCHECK(!Trie.getRoots().empty()); + + populateRecords(ProfileRecords, PathAlloc, ThreadTrie.FCT); + DCHECK(!ThreadTrie.FCT.getRoots().empty()); DCHECK(!ProfileRecords.empty()); // Go through each record, to compute the sizes. @@ -251,75 +292,103 @@ void serialize() { // + end of record (8 bytes) u32 CumulativeSizes = 0; for (const auto &Record : ProfileRecords) - CumulativeSizes += 20 + (4 * Record.Path->size()); + CumulativeSizes += 20 + (4 * Record.Path.size()); - BlockHeader Header{16 + CumulativeSizes, I, (*ThreadTries)[I].TId}; - auto Buffer = ProfileBuffers->PushBack(); - Buffer->Size = sizeof(Header) + CumulativeSizes; - Buffer->Data = InternalAlloc(Buffer->Size, nullptr, 64); - DCHECK_NE(Buffer->Data, nullptr); - serializeRecords(Buffer, Header, ProfileRecords); - - // Now clean up the ProfileRecords array, one at a time. - for (auto &Record : ProfileRecords) { - Record.Path->~PathArray(); - InternalFree(Record.Path); - } + BlockHeader Header{16 + CumulativeSizes, I++, ThreadTrie.TId}; + auto B = ProfileBuffers->Append({}); + B->Size = sizeof(Header) + CumulativeSizes; + B->Data = allocateBuffer(B->Size); + DCHECK_NE(B->Data, nullptr); + serializeRecords(B, Header, ProfileRecords); } } -void reset() { +void reset() XRAY_NEVER_INSTRUMENT { + atomic_store(&CollectorInitialized, 0, memory_order_release); SpinMutexLock Lock(&GlobalMutex); + if (ProfileBuffers != nullptr) { // Clear out the profile buffers that have been serialized. - for (uptr I = 0; I < ProfileBuffers->Size(); ++I) - InternalFree((*ProfileBuffers)[I].Data); - ProfileBuffers->Reset(); - InternalFree(ProfileBuffers); + for (auto &B : *ProfileBuffers) + deallocateBuffer(reinterpret_cast(B.Data), B.Size); + ProfileBuffers->trim(ProfileBuffers->size()); ProfileBuffers = nullptr; } - if (ThreadTries != nullptr) { - // Clear out the function call tries per thread. - for (uptr I = 0; I < ThreadTries->Size(); ++I) { - auto &T = (*ThreadTries)[I]; - T.Trie->~FunctionCallTrie(); - InternalFree(T.Trie); + if (TDArray != nullptr) { + // Release the resources as required. + for (auto &TD : *TDArray) { + TD.BQ->releaseBuffer(TD.Buffers.NodeBuffer); + TD.BQ->releaseBuffer(TD.Buffers.RootsBuffer); + TD.BQ->releaseBuffer(TD.Buffers.ShadowStackBuffer); + TD.BQ->releaseBuffer(TD.Buffers.NodeIdPairBuffer); } - ThreadTries->Reset(); - InternalFree(ThreadTries); - ThreadTries = nullptr; + // We don't bother destroying the array here because we've already + // potentially freed the backing store for the array. Instead we're going to + // reset the pointer to nullptr, and re-use the storage later instead + // (placement-new'ing into the storage as-is). + TDArray = nullptr; } - // Reset the global allocators. - if (GlobalAllocators != nullptr) { - GlobalAllocators->~Allocators(); - InternalFree(GlobalAllocators); - GlobalAllocators = nullptr; + if (TDAllocator != nullptr) { + TDAllocator->~Allocator(); + TDAllocator = nullptr; } - GlobalAllocators = reinterpret_cast( - InternalAlloc(sizeof(FunctionCallTrie::Allocators))); - new (GlobalAllocators) FunctionCallTrie::Allocators(); - *GlobalAllocators = FunctionCallTrie::InitAllocators(); - ThreadTries = reinterpret_cast *>( - InternalAlloc(sizeof(Vector))); - new (ThreadTries) Vector(); - ProfileBuffers = reinterpret_cast *>( - InternalAlloc(sizeof(Vector))); - new (ProfileBuffers) Vector(); + + if (Buffer.Data != nullptr) { + BQ->releaseBuffer(Buffer); + } + + if (BQ == nullptr) { + bool Success = false; + new (&BufferQueueStorage) + BufferQueue(profilingFlags()->global_allocator_max, 1, Success); + if (!Success) + return; + BQ = reinterpret_cast(&BufferQueueStorage); + } else { + BQ->finalize(); + + if (BQ->init(profilingFlags()->global_allocator_max, 1) != + BufferQueue::ErrorCode::Ok) + return; + } + + if (BQ->getBuffer(Buffer) != BufferQueue::ErrorCode::Ok) + return; + + new (&ProfileBufferArrayAllocatorStorage) + ProfileBufferArrayAllocator(profilingFlags()->global_allocator_max); + ProfileBuffersAllocator = reinterpret_cast( + &ProfileBufferArrayAllocatorStorage); + + new (&ProfileBuffersStorage) ProfileBufferArray(*ProfileBuffersAllocator); + ProfileBuffers = + reinterpret_cast(&ProfileBuffersStorage); + + new (&ThreadDataAllocatorStorage) + ThreadDataAllocator(Buffer.Data, Buffer.Size); + TDAllocator = + reinterpret_cast(&ThreadDataAllocatorStorage); + new (&ThreadDataArrayStorage) ThreadDataArray(*TDAllocator); + TDArray = reinterpret_cast(&ThreadDataArrayStorage); + + atomic_store(&CollectorInitialized, 1, memory_order_release); } -XRayBuffer nextBuffer(XRayBuffer B) { +XRayBuffer nextBuffer(XRayBuffer B) XRAY_NEVER_INSTRUMENT { SpinMutexLock Lock(&GlobalMutex); - if (ProfileBuffers == nullptr || ProfileBuffers->Size() == 0) + if (ProfileBuffers == nullptr || ProfileBuffers->size() == 0) return {nullptr, 0}; static pthread_once_t Once = PTHREAD_ONCE_INIT; static typename std::aligned_storage::type FileHeaderStorage; - pthread_once(&Once, - +[] { new (&FileHeaderStorage) XRayProfilingFileHeader{}; }); + pthread_once( + &Once, +[]() XRAY_NEVER_INSTRUMENT { + new (&FileHeaderStorage) XRayProfilingFileHeader{}; + }); if (UNLIKELY(B.Data == nullptr)) { // The first buffer should always contain the file header information. @@ -336,7 +405,7 @@ XRayBuffer nextBuffer(XRayBuffer B) { BlockHeader Header; internal_memcpy(&Header, B.Data, sizeof(BlockHeader)); auto NextBlock = Header.BlockNum + 1; - if (NextBlock < ProfileBuffers->Size()) + if (NextBlock < ProfileBuffers->size()) return {(*ProfileBuffers)[NextBlock].Data, (*ProfileBuffers)[NextBlock].Size}; return {nullptr, 0}; diff --git a/contrib/compiler-rt/lib/xray/xray_profile_collector.h b/contrib/compiler-rt/lib/xray/xray_profile_collector.h index 335043db952..86c4ce85379 100644 --- a/contrib/compiler-rt/lib/xray/xray_profile_collector.h +++ b/contrib/compiler-rt/lib/xray/xray_profile_collector.h @@ -33,27 +33,13 @@ namespace profileCollectorService { /// Posts the FunctionCallTrie associated with a specific Thread ID. This /// will: /// -/// - Make a copy of the FunctionCallTrie and store that against the Thread -/// ID. This will use the global allocator for the service-managed -/// FunctionCallTrie instances. -/// - Queue up a pointer to the FunctionCallTrie. -/// - If the queue is long enough (longer than some arbitrary threshold) we -/// then pre-calculate a single FunctionCallTrie for the whole process. +/// Moves the collection of FunctionCallTrie, Allocators, and Buffers associated +/// with a thread's data to the queue. This takes ownership of the memory +/// associated with a thread, and manages those exclusively. /// -/// -/// We are making a copy of the FunctionCallTrie because the intent is to have -/// this function be called at thread exit, or soon after the profiling -/// handler is finalized through the XRay APIs. By letting threads each -/// process their own thread-local FunctionCallTrie instances, we're removing -/// the need for synchronisation across threads while we're profiling. -/// However, once we're done profiling, we can then collect copies of these -/// FunctionCallTrie instances and pay the cost of the copy. -/// -/// NOTE: In the future, if this turns out to be more costly than "moving" the -/// FunctionCallTrie instances from the owning thread to the collector -/// service, then we can change the implementation to do it this way (moving) -/// instead. -void post(const FunctionCallTrie &T, tid_t TId); +void post(BufferQueue *Q, FunctionCallTrie &&T, + FunctionCallTrie::Allocators &&A, + FunctionCallTrie::Allocators::Buffers &&B, tid_t TId); /// The serialize will process all FunctionCallTrie instances in memory, and /// turn those into specifically formatted blocks, each describing the diff --git a/contrib/compiler-rt/lib/xray/xray_profiling.cc b/contrib/compiler-rt/lib/xray/xray_profiling.cc index d4b4345d764..4323170cd1b 100644 --- a/contrib/compiler-rt/lib/xray/xray_profiling.cc +++ b/contrib/compiler-rt/lib/xray/xray_profiling.cc @@ -19,7 +19,7 @@ #include "sanitizer_common/sanitizer_flags.h" #include "xray/xray_interface.h" #include "xray/xray_log_interface.h" - +#include "xray_buffer_queue.h" #include "xray_flags.h" #include "xray_profile_collector.h" #include "xray_profiling_flags.h" @@ -32,62 +32,167 @@ namespace __xray { namespace { -atomic_sint32_t ProfilerLogFlushStatus = { +static atomic_sint32_t ProfilerLogFlushStatus = { XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; -atomic_sint32_t ProfilerLogStatus = {XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; +static atomic_sint32_t ProfilerLogStatus = { + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; -SpinMutex ProfilerOptionsMutex; +static SpinMutex ProfilerOptionsMutex; -struct alignas(64) ProfilingData { - FunctionCallTrie::Allocators *Allocators = nullptr; - FunctionCallTrie *FCT = nullptr; +struct ProfilingData { + atomic_uintptr_t Allocators; + atomic_uintptr_t FCT; }; static pthread_key_t ProfilingKey; -thread_local std::aligned_storage::type ThreadStorage{}; -static ProfilingData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { - thread_local auto ThreadOnce = [] { - new (&ThreadStorage) ProfilingData{}; - pthread_setspecific(ProfilingKey, &ThreadStorage); +// We use a global buffer queue, which gets initialized once at initialisation +// time, and gets reset when profiling is "done". +static std::aligned_storage::type + BufferQueueStorage; +static BufferQueue *BQ = nullptr; + +thread_local FunctionCallTrie::Allocators::Buffers ThreadBuffers; +thread_local std::aligned_storage::type + AllocatorsStorage; +thread_local std::aligned_storage::type + FunctionCallTrieStorage; +thread_local ProfilingData TLD{{0}, {0}}; +thread_local atomic_uint8_t ReentranceGuard{0}; + +// We use a separate guard for ensuring that for this thread, if we're already +// cleaning up, that any signal handlers don't attempt to cleanup nor +// initialise. +thread_local atomic_uint8_t TLDInitGuard{0}; + +// We also use a separate latch to signal that the thread is exiting, and +// non-essential work should be ignored (things like recording events, etc.). +thread_local atomic_uint8_t ThreadExitingLatch{0}; + +static ProfilingData *getThreadLocalData() XRAY_NEVER_INSTRUMENT { + thread_local auto ThreadOnce = []() XRAY_NEVER_INSTRUMENT { + pthread_setspecific(ProfilingKey, &TLD); return false; }(); (void)ThreadOnce; - auto &TLD = *reinterpret_cast(&ThreadStorage); + RecursionGuard TLDInit(TLDInitGuard); + if (!TLDInit) + return nullptr; - // We need to check whether the global flag to finalizing/finalized has been - // switched. If it is, then we ought to not actually initialise the data. - auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire); - if (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || - Status == XRayLogInitStatus::XRAY_LOG_FINALIZED) - return TLD; + if (atomic_load_relaxed(&ThreadExitingLatch)) + return nullptr; - // If we're live, then we re-initialize TLD if the pointers are not null. - if (UNLIKELY(TLD.Allocators == nullptr && TLD.FCT == nullptr)) { - TLD.Allocators = reinterpret_cast( - InternalAlloc(sizeof(FunctionCallTrie::Allocators))); - new (TLD.Allocators) FunctionCallTrie::Allocators(); - *TLD.Allocators = FunctionCallTrie::InitAllocators(); - TLD.FCT = reinterpret_cast( - InternalAlloc(sizeof(FunctionCallTrie))); - new (TLD.FCT) FunctionCallTrie(*TLD.Allocators); + uptr Allocators = 0; + if (atomic_compare_exchange_strong(&TLD.Allocators, &Allocators, 1, + memory_order_acq_rel)) { + bool Success = false; + auto AllocatorsUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + atomic_store(&TLD.Allocators, 0, memory_order_release); + }); + + // Acquire a set of buffers for this thread. + if (BQ == nullptr) + return nullptr; + + if (BQ->getBuffer(ThreadBuffers.NodeBuffer) != BufferQueue::ErrorCode::Ok) + return nullptr; + auto NodeBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + BQ->releaseBuffer(ThreadBuffers.NodeBuffer); + }); + + if (BQ->getBuffer(ThreadBuffers.RootsBuffer) != BufferQueue::ErrorCode::Ok) + return nullptr; + auto RootsBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + BQ->releaseBuffer(ThreadBuffers.RootsBuffer); + }); + + if (BQ->getBuffer(ThreadBuffers.ShadowStackBuffer) != + BufferQueue::ErrorCode::Ok) + return nullptr; + auto ShadowStackBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + BQ->releaseBuffer(ThreadBuffers.ShadowStackBuffer); + }); + + if (BQ->getBuffer(ThreadBuffers.NodeIdPairBuffer) != + BufferQueue::ErrorCode::Ok) + return nullptr; + + Success = true; + new (&AllocatorsStorage) FunctionCallTrie::Allocators( + FunctionCallTrie::InitAllocatorsFromBuffers(ThreadBuffers)); + Allocators = reinterpret_cast( + reinterpret_cast(&AllocatorsStorage)); + atomic_store(&TLD.Allocators, Allocators, memory_order_release); } - return TLD; + if (Allocators == 1) + return nullptr; + + uptr FCT = 0; + if (atomic_compare_exchange_strong(&TLD.FCT, &FCT, 1, memory_order_acq_rel)) { + new (&FunctionCallTrieStorage) + FunctionCallTrie(*reinterpret_cast( + atomic_load_relaxed(&TLD.Allocators))); + FCT = reinterpret_cast( + reinterpret_cast(&FunctionCallTrieStorage)); + atomic_store(&TLD.FCT, FCT, memory_order_release); + } + + if (FCT == 1) + return nullptr; + + return &TLD; } static void cleanupTLD() XRAY_NEVER_INSTRUMENT { - auto &TLD = *reinterpret_cast(&ThreadStorage); - if (TLD.Allocators != nullptr && TLD.FCT != nullptr) { - TLD.FCT->~FunctionCallTrie(); - TLD.Allocators->~Allocators(); - InternalFree(TLD.FCT); - InternalFree(TLD.Allocators); - TLD.FCT = nullptr; - TLD.Allocators = nullptr; - } + auto FCT = atomic_exchange(&TLD.FCT, 0, memory_order_acq_rel); + if (FCT == reinterpret_cast(reinterpret_cast( + &FunctionCallTrieStorage))) + reinterpret_cast(FCT)->~FunctionCallTrie(); + + auto Allocators = atomic_exchange(&TLD.Allocators, 0, memory_order_acq_rel); + if (Allocators == + reinterpret_cast( + reinterpret_cast(&AllocatorsStorage))) + reinterpret_cast(Allocators)->~Allocators(); +} + +static void postCurrentThreadFCT(ProfilingData &T) XRAY_NEVER_INSTRUMENT { + RecursionGuard TLDInit(TLDInitGuard); + if (!TLDInit) + return; + + uptr P = atomic_exchange(&T.FCT, 0, memory_order_acq_rel); + if (P != reinterpret_cast( + reinterpret_cast(&FunctionCallTrieStorage))) + return; + + auto FCT = reinterpret_cast(P); + DCHECK_NE(FCT, nullptr); + + uptr A = atomic_exchange(&T.Allocators, 0, memory_order_acq_rel); + if (A != + reinterpret_cast( + reinterpret_cast(&AllocatorsStorage))) + return; + + auto Allocators = reinterpret_cast(A); + DCHECK_NE(Allocators, nullptr); + + // Always move the data into the profile collector. + profileCollectorService::post(BQ, std::move(*FCT), std::move(*Allocators), + std::move(ThreadBuffers), GetTid()); + + // Re-initialize the ThreadBuffers object to a known "default" state. + ThreadBuffers = FunctionCallTrie::Allocators::Buffers{}; } } // namespace @@ -100,9 +205,6 @@ const char *profilingCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT { #endif } -atomic_sint32_t ProfileFlushStatus = { - XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; - XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { if (atomic_load(&ProfilerLogStatus, memory_order_acquire) != XRayLogInitStatus::XRAY_LOG_FINALIZED) { @@ -111,12 +213,23 @@ XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; } - s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; - if (!atomic_compare_exchange_strong(&ProfilerLogFlushStatus, &Result, - XRayLogFlushStatus::XRAY_LOG_FLUSHING, - memory_order_acq_rel)) { + RecursionGuard SignalGuard(ReentranceGuard); + if (!SignalGuard) { if (Verbosity()) - Report("Not flushing profiles, implementation still finalizing.\n"); + Report("Cannot finalize properly inside a signal handler!\n"); + atomic_store(&ProfilerLogFlushStatus, + XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING, + memory_order_release); + return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } + + s32 Previous = atomic_exchange(&ProfilerLogFlushStatus, + XRayLogFlushStatus::XRAY_LOG_FLUSHING, + memory_order_acq_rel); + if (Previous == XRayLogFlushStatus::XRAY_LOG_FLUSHING) { + if (Verbosity()) + Report("Not flushing profiles, implementation still flushing.\n"); + return XRayLogFlushStatus::XRAY_LOG_FLUSHING; } // At this point, we'll create the file that will contain the profile, but @@ -129,49 +242,33 @@ XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { if (Verbosity()) Report("profiling: No data to flush.\n"); } else { - int Fd = getLogFD(); - if (Fd == -1) { + LogWriter *LW = LogWriter::Open(); + if (LW == nullptr) { if (Verbosity()) Report("profiling: Failed to flush to file, dropping data.\n"); } else { // Now for each of the buffers, write out the profile data as we would // see it in memory, verbatim. while (B.Data != nullptr && B.Size != 0) { - retryingWriteAll(Fd, reinterpret_cast(B.Data), - reinterpret_cast(B.Data) + B.Size); + LW->WriteAll(reinterpret_cast(B.Data), + reinterpret_cast(B.Data) + B.Size); B = profileCollectorService::nextBuffer(B); } - // Then we close out the file. - internal_close(Fd); } + LogWriter::Close(LW); } } profileCollectorService::reset(); - // Flush the current thread's local data structures as well. - cleanupTLD(); - - atomic_store(&ProfilerLogStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, + atomic_store(&ProfilerLogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, + memory_order_release); + atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, memory_order_release); return XRayLogFlushStatus::XRAY_LOG_FLUSHED; } -namespace { - -thread_local atomic_uint8_t ReentranceGuard{0}; - -static void postCurrentThreadFCT(ProfilingData &TLD) { - if (TLD.Allocators == nullptr || TLD.FCT == nullptr) - return; - - profileCollectorService::post(*TLD.FCT, GetTid()); - cleanupTLD(); -} - -} // namespace - void profilingHandleArg0(int32_t FuncId, XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { unsigned char CPU; @@ -181,21 +278,29 @@ void profilingHandleArg0(int32_t FuncId, return; auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire); - auto &TLD = getThreadLocalData(); + if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_UNINITIALIZED || + Status == XRayLogInitStatus::XRAY_LOG_INITIALIZING)) + return; + if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED || Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) { postCurrentThreadFCT(TLD); return; } + auto T = getThreadLocalData(); + if (T == nullptr) + return; + + auto FCT = reinterpret_cast(atomic_load_relaxed(&T->FCT)); switch (Entry) { case XRayEntryType::ENTRY: case XRayEntryType::LOG_ARGS_ENTRY: - TLD.FCT->enterFunction(FuncId, TSC); + FCT->enterFunction(FuncId, TSC, CPU); break; case XRayEntryType::EXIT: case XRayEntryType::TAIL: - TLD.FCT->exitFunction(FuncId, TSC); + FCT->exitFunction(FuncId, TSC, CPU); break; default: // FIXME: Handle bugs. @@ -218,12 +323,22 @@ XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT { return static_cast(CurrentStatus); } + // Mark then finalize the current generation of buffers. This allows us to let + // the threads currently holding onto new buffers still use them, but let the + // last reference do the memory cleanup. + DCHECK_NE(BQ, nullptr); + BQ->finalize(); + // Wait a grace period to allow threads to see that we're finalizing. SleepForMillis(profilingFlags()->grace_period_ms); - // We also want to make sure that the current thread's data is cleaned up, - // if we have any. - auto &TLD = getThreadLocalData(); + // If we for some reason are entering this function from an instrumented + // handler, we bail out. + RecursionGuard G(ReentranceGuard); + if (!G) + return static_cast(CurrentStatus); + + // Post the current thread's data if we have any. postCurrentThreadFCT(TLD); // Then we force serialize the log data. @@ -235,19 +350,16 @@ XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT { } XRayLogInitStatus -profilingLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, +profilingLoggingInit(size_t, size_t, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { - if (BufferSize != 0 || BufferMax != 0) { - if (Verbosity()) - Report("__xray_log_init() being used, and is unsupported. Use " - "__xray_log_init_mode(...) instead. Bailing out."); + RecursionGuard G(ReentranceGuard); + if (!G) return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; - } s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZING, - memory_order_release)) { + memory_order_acq_rel)) { if (Verbosity()) Report("Cannot initialize already initialised profiling " "implementation.\n"); @@ -276,35 +388,88 @@ profilingLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, // We need to reset the profile data collection implementation now. profileCollectorService::reset(); + // Then also reset the buffer queue implementation. + if (BQ == nullptr) { + bool Success = false; + new (&BufferQueueStorage) + BufferQueue(profilingFlags()->per_thread_allocator_max, + profilingFlags()->buffers_max, Success); + if (!Success) { + if (Verbosity()) + Report("Failed to initialize preallocated memory buffers!"); + atomic_store(&ProfilerLogStatus, + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, + memory_order_release); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + + // If we've succeded, set the global pointer to the initialised storage. + BQ = reinterpret_cast(&BufferQueueStorage); + } else { + BQ->finalize(); + auto InitStatus = BQ->init(profilingFlags()->per_thread_allocator_max, + profilingFlags()->buffers_max); + + if (InitStatus != BufferQueue::ErrorCode::Ok) { + if (Verbosity()) + Report("Failed to initialize preallocated memory buffers; error: %s", + BufferQueue::getErrorString(InitStatus)); + atomic_store(&ProfilerLogStatus, + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, + memory_order_release); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + + DCHECK(!BQ->finalizing()); + } + // We need to set up the exit handlers. static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { - pthread_key_create(&ProfilingKey, +[](void *P) { - // This is the thread-exit handler. - auto &TLD = *reinterpret_cast(P); - if (TLD.Allocators == nullptr && TLD.FCT == nullptr) - return; + pthread_once( + &Once, +[] { + pthread_key_create( + &ProfilingKey, +[](void *P) XRAY_NEVER_INSTRUMENT { + if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel)) + return; - postCurrentThreadFCT(TLD); - }); + if (P == nullptr) + return; - // We also need to set up an exit handler, so that we can get the profile - // information at exit time. We use the C API to do this, to not rely on C++ - // ABI functions for registering exit handlers. - Atexit(+[] { - // Finalize and flush. - if (profilingFinalize() != XRAY_LOG_FINALIZED) { - cleanupTLD(); - return; - } - if (profilingFlush() != XRAY_LOG_FLUSHED) { - cleanupTLD(); - return; - } - if (Verbosity()) - Report("XRay Profile flushed at exit."); - }); - }); + auto T = reinterpret_cast(P); + if (atomic_load_relaxed(&T->Allocators) == 0) + return; + + { + // If we're somehow executing this while inside a + // non-reentrant-friendly context, we skip attempting to post + // the current thread's data. + RecursionGuard G(ReentranceGuard); + if (!G) + return; + + postCurrentThreadFCT(*T); + } + }); + + // We also need to set up an exit handler, so that we can get the + // profile information at exit time. We use the C API to do this, to not + // rely on C++ ABI functions for registering exit handlers. + Atexit(+[]() XRAY_NEVER_INSTRUMENT { + if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel)) + return; + + auto Cleanup = + at_scope_exit([]() XRAY_NEVER_INSTRUMENT { cleanupTLD(); }); + + // Finalize and flush. + if (profilingFinalize() != XRAY_LOG_FINALIZED || + profilingFlush() != XRAY_LOG_FLUSHED) + return; + + if (Verbosity()) + Report("XRay Profile flushed at exit."); + }); + }); __xray_log_set_buffer_iterator(profileCollectorService::nextBuffer); __xray_set_handler(profilingHandleArg0); diff --git a/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc b/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc index e9230ae6418..ccd70860bf6 100644 --- a/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc +++ b/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc @@ -14,7 +14,7 @@ #error "Define XRAY_FLAG prior to including this file!" #endif -XRAY_FLAG(uptr, per_thread_allocator_max, 2 << 20, +XRAY_FLAG(uptr, per_thread_allocator_max, 16384, "Maximum size of any single per-thread allocator.") XRAY_FLAG(uptr, global_allocator_max, 2 << 24, "Maximum size of the global allocator for profile storage.") @@ -27,3 +27,6 @@ XRAY_FLAG(int, grace_period_ms, 1, XRAY_FLAG(bool, no_flush, false, "Set to true if we want the profiling implementation to not write " "out files.") +XRAY_FLAG(int, buffers_max, 128, + "The number of buffers to pre-allocate used by the profiling " + "implementation.") diff --git a/contrib/compiler-rt/lib/xray/xray_segmented_array.h b/contrib/compiler-rt/lib/xray/xray_segmented_array.h index 11dd794fa52..bc7e9379f63 100644 --- a/contrib/compiler-rt/lib/xray/xray_segmented_array.h +++ b/contrib/compiler-rt/lib/xray/xray_segmented_array.h @@ -32,14 +32,9 @@ namespace __xray { /// is destroyed. When an Array is destroyed, it will destroy elements in the /// backing store but will not free the memory. template class Array { - struct SegmentBase { - SegmentBase *Prev; - SegmentBase *Next; - }; - - // We want each segment of the array to be cache-line aligned, and elements of - // the array be offset from the beginning of the segment. - struct Segment : SegmentBase { + struct Segment { + Segment *Prev; + Segment *Next; char Data[1]; }; @@ -62,98 +57,46 @@ public: // kCacheLineSize-multiple segments, minus the size of two pointers. // // - Request cacheline-multiple sized elements from the allocator. - static constexpr size_t AlignedElementStorageSize = + static constexpr uint64_t AlignedElementStorageSize = sizeof(typename std::aligned_storage::type); - static constexpr size_t SegmentSize = - nearest_boundary(sizeof(Segment) + next_pow2(sizeof(T)), kCacheLineSize); + static constexpr uint64_t SegmentControlBlockSize = sizeof(Segment *) * 2; + + static constexpr uint64_t SegmentSize = nearest_boundary( + SegmentControlBlockSize + next_pow2(sizeof(T)), kCacheLineSize); using AllocatorType = Allocator; - static constexpr size_t ElementsPerSegment = - (SegmentSize - sizeof(Segment)) / next_pow2(sizeof(T)); + static constexpr uint64_t ElementsPerSegment = + (SegmentSize - SegmentControlBlockSize) / next_pow2(sizeof(T)); static_assert(ElementsPerSegment > 0, "Must have at least 1 element per segment."); - static SegmentBase SentinelSegment; + static Segment SentinelSegment; + + using size_type = uint64_t; private: - AllocatorType *Alloc; - SegmentBase *Head = &SentinelSegment; - SegmentBase *Tail = &SentinelSegment; - size_t Size = 0; - - // Here we keep track of segments in the freelist, to allow us to re-use - // segments when elements are trimmed off the end. - SegmentBase *Freelist = &SentinelSegment; - - Segment *NewSegment() { - // We need to handle the case in which enough elements have been trimmed to - // allow us to re-use segments we've allocated before. For this we look into - // the Freelist, to see whether we need to actually allocate new blocks or - // just re-use blocks we've already seen before. - if (Freelist != &SentinelSegment) { - auto *FreeSegment = Freelist; - Freelist = FreeSegment->Next; - FreeSegment->Next = &SentinelSegment; - Freelist->Prev = &SentinelSegment; - return static_cast(FreeSegment); - } - - auto SegmentBlock = Alloc->Allocate(); - if (SegmentBlock.Data == nullptr) - return nullptr; - - // Placement-new the Segment element at the beginning of the SegmentBlock. - auto S = reinterpret_cast(SegmentBlock.Data); - new (S) SegmentBase{&SentinelSegment, &SentinelSegment}; - return S; - } - - Segment *InitHeadAndTail() { - DCHECK_EQ(Head, &SentinelSegment); - DCHECK_EQ(Tail, &SentinelSegment); - auto Segment = NewSegment(); - if (Segment == nullptr) - return nullptr; - DCHECK_EQ(Segment->Next, &SentinelSegment); - DCHECK_EQ(Segment->Prev, &SentinelSegment); - Head = Tail = static_cast(Segment); - return Segment; - } - - Segment *AppendNewSegment() { - auto S = NewSegment(); - if (S == nullptr) - return nullptr; - DCHECK_NE(Tail, &SentinelSegment); - DCHECK_EQ(Tail->Next, &SentinelSegment); - DCHECK_EQ(S->Prev, &SentinelSegment); - DCHECK_EQ(S->Next, &SentinelSegment); - Tail->Next = S; - S->Prev = Tail; - Tail = S; - return static_cast(Tail); - } - // This Iterator models a BidirectionalIterator. template class Iterator { - SegmentBase *S = &SentinelSegment; - size_t Offset = 0; - size_t Size = 0; + Segment *S = &SentinelSegment; + uint64_t Offset = 0; + uint64_t Size = 0; public: - Iterator(SegmentBase *IS, size_t Off, size_t S) - : S(IS), Offset(Off), Size(S) {} - Iterator(const Iterator &) noexcept = default; - Iterator() noexcept = default; - Iterator(Iterator &&) noexcept = default; - Iterator &operator=(const Iterator &) = default; - Iterator &operator=(Iterator &&) = default; - ~Iterator() = default; + Iterator(Segment *IS, uint64_t Off, uint64_t S) XRAY_NEVER_INSTRUMENT + : S(IS), + Offset(Off), + Size(S) {} + Iterator(const Iterator &) NOEXCEPT XRAY_NEVER_INSTRUMENT = default; + Iterator() NOEXCEPT XRAY_NEVER_INSTRUMENT = default; + Iterator(Iterator &&) NOEXCEPT XRAY_NEVER_INSTRUMENT = default; + Iterator &operator=(const Iterator &) XRAY_NEVER_INSTRUMENT = default; + Iterator &operator=(Iterator &&) XRAY_NEVER_INSTRUMENT = default; + ~Iterator() XRAY_NEVER_INSTRUMENT = default; - Iterator &operator++() { + Iterator &operator++() XRAY_NEVER_INSTRUMENT { if (++Offset % ElementsPerSegment || Offset == Size) return *this; @@ -168,7 +111,7 @@ private: return *this; } - Iterator &operator--() { + Iterator &operator--() XRAY_NEVER_INSTRUMENT { DCHECK_NE(S, &SentinelSegment); DCHECK_GT(Offset, 0); @@ -181,107 +124,295 @@ private: return *this; } - Iterator operator++(int) { + Iterator operator++(int) XRAY_NEVER_INSTRUMENT { Iterator Copy(*this); ++(*this); return Copy; } - Iterator operator--(int) { + Iterator operator--(int) XRAY_NEVER_INSTRUMENT { Iterator Copy(*this); --(*this); return Copy; } template - friend bool operator==(const Iterator &L, const Iterator &R) { + friend bool operator==(const Iterator &L, + const Iterator &R) XRAY_NEVER_INSTRUMENT { return L.S == R.S && L.Offset == R.Offset; } template - friend bool operator!=(const Iterator &L, const Iterator &R) { + friend bool operator!=(const Iterator &L, + const Iterator &R) XRAY_NEVER_INSTRUMENT { return !(L == R); } - U &operator*() const { + U &operator*() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(S, &SentinelSegment); auto RelOff = Offset % ElementsPerSegment; // We need to compute the character-aligned pointer, offset from the // segment's Data location to get the element in the position of Offset. - auto Base = static_cast(S)->Data; + auto Base = &S->Data; auto AlignedOffset = Base + (RelOff * AlignedElementStorageSize); return *reinterpret_cast(AlignedOffset); } - U *operator->() const { return &(**this); } + U *operator->() const XRAY_NEVER_INSTRUMENT { return &(**this); } }; + AllocatorType *Alloc; + Segment *Head; + Segment *Tail; + + // Here we keep track of segments in the freelist, to allow us to re-use + // segments when elements are trimmed off the end. + Segment *Freelist; + uint64_t Size; + + // =============================== + // In the following implementation, we work through the algorithms and the + // list operations using the following notation: + // + // - pred(s) is the predecessor (previous node accessor) and succ(s) is + // the successor (next node accessor). + // + // - S is a sentinel segment, which has the following property: + // + // pred(S) == succ(S) == S + // + // - @ is a loop operator, which can imply pred(s) == s if it appears on + // the left of s, or succ(s) == S if it appears on the right of s. + // + // - sL <-> sR : means a bidirectional relation between sL and sR, which + // means: + // + // succ(sL) == sR && pred(SR) == sL + // + // - sL -> sR : implies a unidirectional relation between sL and SR, + // with the following properties: + // + // succ(sL) == sR + // + // sL <- sR : implies a unidirectional relation between sR and sL, + // with the following properties: + // + // pred(sR) == sL + // + // =============================== + + Segment *NewSegment() XRAY_NEVER_INSTRUMENT { + // We need to handle the case in which enough elements have been trimmed to + // allow us to re-use segments we've allocated before. For this we look into + // the Freelist, to see whether we need to actually allocate new blocks or + // just re-use blocks we've already seen before. + if (Freelist != &SentinelSegment) { + // The current state of lists resemble something like this at this point: + // + // Freelist: @S@<-f0->...<->fN->@S@ + // ^ Freelist + // + // We want to perform a splice of `f0` from Freelist to a temporary list, + // which looks like: + // + // Templist: @S@<-f0->@S@ + // ^ FreeSegment + // + // Our algorithm preconditions are: + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + + // Then the algorithm we implement is: + // + // SFS = Freelist + // Freelist = succ(Freelist) + // if (Freelist != S) + // pred(Freelist) = S + // succ(SFS) = S + // pred(SFS) = S + // + auto *FreeSegment = Freelist; + Freelist = Freelist->Next; + + // Note that we need to handle the case where Freelist is now pointing to + // S, which we don't want to be overwriting. + // TODO: Determine whether the cost of the branch is higher than the cost + // of the blind assignment. + if (Freelist != &SentinelSegment) + Freelist->Prev = &SentinelSegment; + + FreeSegment->Next = &SentinelSegment; + FreeSegment->Prev = &SentinelSegment; + + // Our postconditions are: + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + DCHECK_NE(FreeSegment, &SentinelSegment); + return FreeSegment; + } + + auto SegmentBlock = Alloc->Allocate(); + if (SegmentBlock.Data == nullptr) + return nullptr; + + // Placement-new the Segment element at the beginning of the SegmentBlock. + new (SegmentBlock.Data) Segment{&SentinelSegment, &SentinelSegment, {0}}; + auto SB = reinterpret_cast(SegmentBlock.Data); + return SB; + } + + Segment *InitHeadAndTail() XRAY_NEVER_INSTRUMENT { + DCHECK_EQ(Head, &SentinelSegment); + DCHECK_EQ(Tail, &SentinelSegment); + auto S = NewSegment(); + if (S == nullptr) + return nullptr; + DCHECK_EQ(S->Next, &SentinelSegment); + DCHECK_EQ(S->Prev, &SentinelSegment); + DCHECK_NE(S, &SentinelSegment); + Head = S; + Tail = S; + DCHECK_EQ(Head, Tail); + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(Tail->Prev, &SentinelSegment); + return S; + } + + Segment *AppendNewSegment() XRAY_NEVER_INSTRUMENT { + auto S = NewSegment(); + if (S == nullptr) + return nullptr; + DCHECK_NE(Tail, &SentinelSegment); + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(S->Prev, &SentinelSegment); + DCHECK_EQ(S->Next, &SentinelSegment); + S->Prev = Tail; + Tail->Next = S; + Tail = S; + DCHECK_EQ(S, S->Prev->Next); + DCHECK_EQ(Tail->Next, &SentinelSegment); + return S; + } + public: - explicit Array(AllocatorType &A) : Alloc(&A) {} + explicit Array(AllocatorType &A) XRAY_NEVER_INSTRUMENT + : Alloc(&A), + Head(&SentinelSegment), + Tail(&SentinelSegment), + Freelist(&SentinelSegment), + Size(0) {} + + Array() XRAY_NEVER_INSTRUMENT : Alloc(nullptr), + Head(&SentinelSegment), + Tail(&SentinelSegment), + Freelist(&SentinelSegment), + Size(0) {} Array(const Array &) = delete; - Array(Array &&O) NOEXCEPT : Alloc(O.Alloc), - Head(O.Head), - Tail(O.Tail), - Size(O.Size) { + Array &operator=(const Array &) = delete; + + Array(Array &&O) XRAY_NEVER_INSTRUMENT : Alloc(O.Alloc), + Head(O.Head), + Tail(O.Tail), + Freelist(O.Freelist), + Size(O.Size) { + O.Alloc = nullptr; O.Head = &SentinelSegment; O.Tail = &SentinelSegment; O.Size = 0; + O.Freelist = &SentinelSegment; } - bool empty() const { return Size == 0; } + Array &operator=(Array &&O) XRAY_NEVER_INSTRUMENT { + Alloc = O.Alloc; + O.Alloc = nullptr; + Head = O.Head; + O.Head = &SentinelSegment; + Tail = O.Tail; + O.Tail = &SentinelSegment; + Freelist = O.Freelist; + O.Freelist = &SentinelSegment; + Size = O.Size; + O.Size = 0; + return *this; + } - AllocatorType &allocator() const { + ~Array() XRAY_NEVER_INSTRUMENT { + for (auto &E : *this) + (&E)->~T(); + } + + bool empty() const XRAY_NEVER_INSTRUMENT { return Size == 0; } + + AllocatorType &allocator() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(Alloc, nullptr); return *Alloc; } - size_t size() const { return Size; } + uint64_t size() const XRAY_NEVER_INSTRUMENT { return Size; } - T *Append(const T &E) { - if (UNLIKELY(Head == &SentinelSegment)) - if (InitHeadAndTail() == nullptr) + template + T *AppendEmplace(Args &&... args) XRAY_NEVER_INSTRUMENT { + DCHECK((Size == 0 && Head == &SentinelSegment && Head == Tail) || + (Size != 0 && Head != &SentinelSegment && Tail != &SentinelSegment)); + if (UNLIKELY(Head == &SentinelSegment)) { + auto R = InitHeadAndTail(); + if (R == nullptr) return nullptr; + } + + DCHECK_NE(Head, &SentinelSegment); + DCHECK_NE(Tail, &SentinelSegment); auto Offset = Size % ElementsPerSegment; if (UNLIKELY(Size != 0 && Offset == 0)) if (AppendNewSegment() == nullptr) return nullptr; - auto Base = static_cast(Tail)->Data; + DCHECK_NE(Tail, &SentinelSegment); + auto Base = &Tail->Data; auto AlignedOffset = Base + (Offset * AlignedElementStorageSize); - auto Position = reinterpret_cast(AlignedOffset); - *Position = E; + DCHECK_LE(AlignedOffset + sizeof(T), + reinterpret_cast(Base) + SegmentSize); + + // In-place construct at Position. + new (AlignedOffset) T{std::forward(args)...}; ++Size; - return Position; + return reinterpret_cast(AlignedOffset); } - template T *AppendEmplace(Args &&... args) { - if (UNLIKELY(Head == &SentinelSegment)) - if (InitHeadAndTail() == nullptr) - return nullptr; - - auto Offset = Size % ElementsPerSegment; - auto *LatestSegment = Tail; - if (UNLIKELY(Size != 0 && Offset == 0)) { - LatestSegment = AppendNewSegment(); - if (LatestSegment == nullptr) + T *Append(const T &E) XRAY_NEVER_INSTRUMENT { + // FIXME: This is a duplication of AppenEmplace with the copy semantics + // explicitly used, as a work-around to GCC 4.8 not invoking the copy + // constructor with the placement new with braced-init syntax. + DCHECK((Size == 0 && Head == &SentinelSegment && Head == Tail) || + (Size != 0 && Head != &SentinelSegment && Tail != &SentinelSegment)); + if (UNLIKELY(Head == &SentinelSegment)) { + auto R = InitHeadAndTail(); + if (R == nullptr) return nullptr; } + DCHECK_NE(Head, &SentinelSegment); DCHECK_NE(Tail, &SentinelSegment); - auto Base = static_cast(LatestSegment)->Data; + + auto Offset = Size % ElementsPerSegment; + if (UNLIKELY(Size != 0 && Offset == 0)) + if (AppendNewSegment() == nullptr) + return nullptr; + + DCHECK_NE(Tail, &SentinelSegment); + auto Base = &Tail->Data; auto AlignedOffset = Base + (Offset * AlignedElementStorageSize); - auto Position = reinterpret_cast(AlignedOffset); + DCHECK_LE(AlignedOffset + sizeof(T), + reinterpret_cast(Tail) + SegmentSize); // In-place construct at Position. - new (Position) T{std::forward(args)...}; + new (AlignedOffset) T(E); ++Size; - return reinterpret_cast(Position); + return reinterpret_cast(AlignedOffset); } - T &operator[](size_t Offset) const { + T &operator[](uint64_t Offset) const XRAY_NEVER_INSTRUMENT { DCHECK_LE(Offset, Size); // We need to traverse the array enough times to find the element at Offset. auto S = Head; @@ -290,19 +421,19 @@ public: Offset -= ElementsPerSegment; DCHECK_NE(S, &SentinelSegment); } - auto Base = static_cast(S)->Data; + auto Base = &S->Data; auto AlignedOffset = Base + (Offset * AlignedElementStorageSize); auto Position = reinterpret_cast(AlignedOffset); return *reinterpret_cast(Position); } - T &front() const { + T &front() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(Head, &SentinelSegment); DCHECK_NE(Size, 0u); return *begin(); } - T &back() const { + T &back() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(Tail, &SentinelSegment); DCHECK_NE(Size, 0u); auto It = end(); @@ -310,7 +441,8 @@ public: return *It; } - template T *find_element(Predicate P) const { + template + T *find_element(Predicate P) const XRAY_NEVER_INSTRUMENT { if (empty()) return nullptr; @@ -324,51 +456,195 @@ public: /// Remove N Elements from the end. This leaves the blocks behind, and not /// require allocation of new blocks for new elements added after trimming. - void trim(size_t Elements) { - DCHECK_LE(Elements, Size); - DCHECK_GT(Size, 0); + void trim(uint64_t Elements) XRAY_NEVER_INSTRUMENT { auto OldSize = Size; + Elements = Elements > Size ? Size : Elements; Size -= Elements; - DCHECK_NE(Head, &SentinelSegment); - DCHECK_NE(Tail, &SentinelSegment); - - for (auto SegmentsToTrim = (nearest_boundary(OldSize, ElementsPerSegment) - - nearest_boundary(Size, ElementsPerSegment)) / - ElementsPerSegment; - SegmentsToTrim > 0; --SegmentsToTrim) { + // We compute the number of segments we're going to return from the tail by + // counting how many elements have been trimmed. Given the following: + // + // - Each segment has N valid positions, where N > 0 + // - The previous size > current size + // + // To compute the number of segments to return, we need to perform the + // following calculations for the number of segments required given 'x' + // elements: + // + // f(x) = { + // x == 0 : 0 + // , 0 < x <= N : 1 + // , N < x <= max : x / N + (x % N ? 1 : 0) + // } + // + // We can simplify this down to: + // + // f(x) = { + // x == 0 : 0, + // , 0 < x <= max : x / N + (x < N || x % N ? 1 : 0) + // } + // + // And further down to: + // + // f(x) = x ? x / N + (x < N || x % N ? 1 : 0) : 0 + // + // We can then perform the following calculation `s` which counts the number + // of segments we need to remove from the end of the data structure: + // + // s(p, c) = f(p) - f(c) + // + // If we treat p = previous size, and c = current size, and given the + // properties above, the possible range for s(...) is [0..max(typeof(p))/N] + // given that typeof(p) == typeof(c). + auto F = [](uint64_t X) { + return X ? (X / ElementsPerSegment) + + (X < ElementsPerSegment || X % ElementsPerSegment ? 1 : 0) + : 0; + }; + auto PS = F(OldSize); + auto CS = F(Size); + DCHECK_GE(PS, CS); + auto SegmentsToTrim = PS - CS; + for (auto I = 0uL; I < SegmentsToTrim; ++I) { + // Here we place the current tail segment to the freelist. To do this + // appropriately, we need to perform a splice operation on two + // bidirectional linked-lists. In particular, we have the current state of + // the doubly-linked list of segments: + // + // @S@ <- s0 <-> s1 <-> ... <-> sT -> @S@ + // DCHECK_NE(Head, &SentinelSegment); DCHECK_NE(Tail, &SentinelSegment); - // Put the tail into the Freelist. - auto *FreeSegment = Tail; - Tail = Tail->Prev; - if (Tail == &SentinelSegment) - Head = Tail; - else - Tail->Next = &SentinelSegment; - DCHECK_EQ(Tail->Next, &SentinelSegment); - FreeSegment->Next = Freelist; - FreeSegment->Prev = &SentinelSegment; - if (Freelist != &SentinelSegment) - Freelist->Prev = FreeSegment; - Freelist = FreeSegment; + + if (Freelist == &SentinelSegment) { + // Our two lists at this point are in this configuration: + // + // Freelist: (potentially) @S@ + // Mainlist: @S@<-s0<->s1<->...<->sPT<->sT->@S@ + // ^ Head ^ Tail + // + // The end state for us will be this configuration: + // + // Freelist: @S@<-sT->@S@ + // Mainlist: @S@<-s0<->s1<->...<->sPT->@S@ + // ^ Head ^ Tail + // + // The first step for us is to hold a reference to the tail of Mainlist, + // which in our notation is represented by sT. We call this our "free + // segment" which is the segment we are placing on the Freelist. + // + // sF = sT + // + // Then, we also hold a reference to the "pre-tail" element, which we + // call sPT: + // + // sPT = pred(sT) + // + // We want to splice sT into the beginning of the Freelist, which in + // an empty Freelist means placing a segment whose predecessor and + // successor is the sentinel segment. + // + // The splice operation then can be performed in the following + // algorithm: + // + // succ(sPT) = S + // pred(sT) = S + // succ(sT) = Freelist + // Freelist = sT + // Tail = sPT + // + auto SPT = Tail->Prev; + SPT->Next = &SentinelSegment; + Tail->Prev = &SentinelSegment; + Tail->Next = Freelist; + Freelist = Tail; + Tail = SPT; + + // Our post-conditions here are: + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + } else { + // In the other case, where the Freelist is not empty, we perform the + // following transformation instead: + // + // This transforms the current state: + // + // Freelist: @S@<-f0->@S@ + // ^ Freelist + // Mainlist: @S@<-s0<->s1<->...<->sPT<->sT->@S@ + // ^ Head ^ Tail + // + // Into the following: + // + // Freelist: @S@<-sT<->f0->@S@ + // ^ Freelist + // Mainlist: @S@<-s0<->s1<->...<->sPT->@S@ + // ^ Head ^ Tail + // + // The algorithm is: + // + // sFH = Freelist + // sPT = pred(sT) + // pred(SFH) = sT + // succ(sT) = Freelist + // pred(sT) = S + // succ(sPT) = S + // Tail = sPT + // Freelist = sT + // + auto SFH = Freelist; + auto SPT = Tail->Prev; + auto ST = Tail; + SFH->Prev = ST; + ST->Next = Freelist; + ST->Prev = &SentinelSegment; + SPT->Next = &SentinelSegment; + Tail = SPT; + Freelist = ST; + + // Our post-conditions here are: + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + DCHECK_EQ(Freelist->Next->Prev, Freelist); + } } + + // Now in case we've spliced all the segments in the end, we ensure that the + // main list is "empty", or both the head and tail pointing to the sentinel + // segment. + if (Tail == &SentinelSegment) + Head = Tail; + + DCHECK( + (Size == 0 && Head == &SentinelSegment && Tail == &SentinelSegment) || + (Size != 0 && Head != &SentinelSegment && Tail != &SentinelSegment)); + DCHECK( + (Freelist != &SentinelSegment && Freelist->Prev == &SentinelSegment) || + (Freelist == &SentinelSegment && Tail->Next == &SentinelSegment)); } // Provide iterators. - Iterator begin() const { return Iterator(Head, 0, Size); } - Iterator end() const { return Iterator(Tail, Size, Size); } - Iterator cbegin() const { return Iterator(Head, 0, Size); } - Iterator cend() const { return Iterator(Tail, Size, Size); } + Iterator begin() const XRAY_NEVER_INSTRUMENT { + return Iterator(Head, 0, Size); + } + Iterator end() const XRAY_NEVER_INSTRUMENT { + return Iterator(Tail, Size, Size); + } + Iterator cbegin() const XRAY_NEVER_INSTRUMENT { + return Iterator(Head, 0, Size); + } + Iterator cend() const XRAY_NEVER_INSTRUMENT { + return Iterator(Tail, Size, Size); + } }; // We need to have this storage definition out-of-line so that the compiler can // ensure that storage for the SentinelSegment is defined and has a single // address. template -typename Array::SegmentBase Array::SentinelSegment{ - &Array::SentinelSegment, &Array::SentinelSegment}; +typename Array::Segment Array::SentinelSegment{ + &Array::SentinelSegment, &Array::SentinelSegment, {'\0'}}; } // namespace __xray diff --git a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S index 99ad3966ee3..52985ffd19a 100644 --- a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S +++ b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S @@ -19,6 +19,7 @@ .macro SAVE_REGISTERS + pushfq subq $240, %rsp CFI_DEF_CFA_OFFSET(248) movq %rbp, 232(%rsp) @@ -69,6 +70,7 @@ movq 8(%rsp), %r14 movq 0(%rsp), %r15 addq $240, %rsp + popfq CFI_DEF_CFA_OFFSET(8) .endm @@ -89,10 +91,10 @@ .text #if !defined(__APPLE__) .section .text + .file "xray_trampoline_x86.S" #else .section __TEXT,__text #endif - .file "xray_trampoline_x86.S" //===----------------------------------------------------------------------===// diff --git a/contrib/compiler-rt/lib/xray/xray_tsc.h b/contrib/compiler-rt/lib/xray/xray_tsc.h index 4507564e7cd..180d6df188c 100644 --- a/contrib/compiler-rt/lib/xray/xray_tsc.h +++ b/contrib/compiler-rt/lib/xray/xray_tsc.h @@ -13,10 +13,32 @@ #ifndef XRAY_EMULATE_TSC_H #define XRAY_EMULATE_TSC_H +#include "sanitizer_common/sanitizer_common.h" + namespace __xray { static constexpr uint64_t NanosecondsPerSecond = 1000ULL * 1000 * 1000; } +#if SANITIZER_FUCHSIA +#include + +namespace __xray { + +inline bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { return true; } + +ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT { + CPU = 0; + return _zx_ticks_get(); +} + +inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { + return _zx_ticks_per_second(); +} + +} // namespace __xray + +#else // SANITIZER_FUCHSIA + #if defined(__x86_64__) #include "xray_x86_64.inc" #elif defined(__powerpc64__) @@ -64,5 +86,6 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { #else #error Target architecture is not supported. #endif // CPU architecture +#endif // SANITIZER_FUCHSIA #endif // XRAY_EMULATE_TSC_H diff --git a/contrib/compiler-rt/lib/xray/xray_utils.cc b/contrib/compiler-rt/lib/xray/xray_utils.cc index 68f4e8c1094..59ba6c3082b 100644 --- a/contrib/compiler-rt/lib/xray/xray_utils.cc +++ b/contrib/compiler-rt/lib/xray/xray_utils.cc @@ -12,7 +12,9 @@ //===----------------------------------------------------------------------===// #include "xray_utils.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" +#include "xray_allocator.h" #include "xray_defs.h" #include "xray_flags.h" #include @@ -25,13 +27,113 @@ #include #include +#if SANITIZER_FUCHSIA +#include "sanitizer_common/sanitizer_symbolizer_fuchsia.h" + +#include +#include +#include +#include +#include +#endif + namespace __xray { -void printToStdErr(const char *Buffer) XRAY_NEVER_INSTRUMENT { - fprintf(stderr, "%s", Buffer); +#if SANITIZER_FUCHSIA +constexpr const char* ProfileSinkName = "llvm-xray"; + +LogWriter::~LogWriter() { + _zx_handle_close(Vmo); } -void retryingWriteAll(int Fd, const char *Begin, const char *End) XRAY_NEVER_INSTRUMENT { +void LogWriter::WriteAll(const char *Begin, const char *End) XRAY_NEVER_INSTRUMENT { + if (Begin == End) + return; + auto TotalBytes = std::distance(Begin, End); + + const size_t PageSize = flags()->xray_page_size_override > 0 + ? flags()->xray_page_size_override + : GetPageSizeCached(); + if (RoundUpTo(Offset, PageSize) != RoundUpTo(Offset + TotalBytes, PageSize)) { + // Resize the VMO to ensure there's sufficient space for the data. + zx_status_t Status = _zx_vmo_set_size(Vmo, Offset + TotalBytes); + if (Status != ZX_OK) { + Report("Failed to resize VMO: %s\n", _zx_status_get_string(Status)); + return; + } + } + + // Write the data into VMO. + zx_status_t Status = _zx_vmo_write(Vmo, Begin, Offset, TotalBytes); + if (Status != ZX_OK) { + Report("Failed to write: %s\n", _zx_status_get_string(Status)); + return; + } + Offset += TotalBytes; +} + +void LogWriter::Flush() XRAY_NEVER_INSTRUMENT { + // Nothing to do here since WriteAll writes directly into the VMO. +} + +LogWriter *LogWriter::Open() XRAY_NEVER_INSTRUMENT { + // Create VMO to hold the profile data. + zx_handle_t Vmo; + zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); + if (Status != ZX_OK) { + Report("XRay: cannot create VMO: %s\n", _zx_status_get_string(Status)); + return nullptr; + } + + // Get the KOID of the current process to use in the VMO name. + zx_info_handle_basic_t Info; + Status = _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, + sizeof(Info), NULL, NULL); + if (Status != ZX_OK) { + Report("XRay: cannot get basic info about current process handle: %s\n", + _zx_status_get_string(Status)); + return nullptr; + } + + // Give the VMO a name including our process KOID so it's easy to spot. + char VmoName[ZX_MAX_NAME_LEN]; + internal_snprintf(VmoName, sizeof(VmoName), "%s.%zu", ProfileSinkName, + Info.koid); + _zx_object_set_property(Vmo, ZX_PROP_NAME, VmoName, strlen(VmoName)); + + // Duplicate the handle since __sanitizer_publish_data consumes it and + // LogWriter needs to hold onto it. + zx_handle_t Handle; + Status =_zx_handle_duplicate(Vmo, ZX_RIGHT_SAME_RIGHTS, &Handle); + if (Status != ZX_OK) { + Report("XRay: cannot duplicate VMO handle: %s\n", + _zx_status_get_string(Status)); + return nullptr; + } + + // Publish the VMO that receives the logging. Note the VMO's contents can + // grow and change after publication. The contents won't be read out until + // after the process exits. + __sanitizer_publish_data(ProfileSinkName, Handle); + + // Use the dumpfile symbolizer markup element to write the name of the VMO. + Report("XRay: " FORMAT_DUMPFILE "\n", ProfileSinkName, VmoName); + + LogWriter *LW = reinterpret_cast(InternalAlloc(sizeof(LogWriter))); + new (LW) LogWriter(Vmo); + return LW; +} + +void LogWriter::Close(LogWriter *LW) { + LW->~LogWriter(); + InternalFree(LW); +} +#else // SANITIZER_FUCHSIA +LogWriter::~LogWriter() { + internal_close(Fd); +} + +void LogWriter::WriteAll(const char *Begin, const char *End) XRAY_NEVER_INSTRUMENT { if (Begin == End) return; auto TotalBytes = std::distance(Begin, End); @@ -49,50 +151,11 @@ void retryingWriteAll(int Fd, const char *Begin, const char *End) XRAY_NEVER_INS } } -std::pair retryingReadSome(int Fd, char *Begin, - char *End) XRAY_NEVER_INSTRUMENT { - auto BytesToRead = std::distance(Begin, End); - ssize_t BytesRead; - ssize_t TotalBytesRead = 0; - while (BytesToRead && (BytesRead = read(Fd, Begin, BytesToRead))) { - if (BytesRead == -1) { - if (errno == EINTR) - continue; - Report("Read error; errno = %d\n", errno); - return std::make_pair(TotalBytesRead, false); - } - - TotalBytesRead += BytesRead; - BytesToRead -= BytesRead; - Begin += BytesRead; - } - return std::make_pair(TotalBytesRead, true); +void LogWriter::Flush() XRAY_NEVER_INSTRUMENT { + fsync(Fd); } -bool readValueFromFile(const char *Filename, - long long *Value) XRAY_NEVER_INSTRUMENT { - int Fd = open(Filename, O_RDONLY | O_CLOEXEC); - if (Fd == -1) - return false; - static constexpr size_t BufSize = 256; - char Line[BufSize] = {}; - ssize_t BytesRead; - bool Success; - std::tie(BytesRead, Success) = retryingReadSome(Fd, Line, Line + BufSize); - if (!Success) - return false; - close(Fd); - const char *End = nullptr; - long long Tmp = internal_simple_strtoll(Line, &End, 10); - bool Result = false; - if (Line[0] != '\0' && (*End == '\n' || *End == '\0')) { - *Value = Tmp; - Result = true; - } - return Result; -} - -int getLogFD() XRAY_NEVER_INSTRUMENT { +LogWriter *LogWriter::Open() XRAY_NEVER_INSTRUMENT { // Open a temporary file once for the log. char TmpFilename[256] = {}; char TmpWildcardPattern[] = "XXXXXX"; @@ -103,24 +166,31 @@ int getLogFD() XRAY_NEVER_INSTRUMENT { if (LastSlash != nullptr) Progname = LastSlash + 1; - const int HalfLength = sizeof(TmpFilename) / 2 - sizeof(TmpWildcardPattern); int NeededLength = internal_snprintf( - TmpFilename, sizeof(TmpFilename), "%.*s%.*s.%s", HalfLength, - flags()->xray_logfile_base, HalfLength, Progname, TmpWildcardPattern); + TmpFilename, sizeof(TmpFilename), "%s%s.%s", + flags()->xray_logfile_base, Progname, TmpWildcardPattern); if (NeededLength > int(sizeof(TmpFilename))) { Report("XRay log file name too long (%d): %s\n", NeededLength, TmpFilename); - return -1; + return nullptr; } int Fd = mkstemp(TmpFilename); if (Fd == -1) { Report("XRay: Failed opening temporary file '%s'; not logging events.\n", TmpFilename); - return -1; + return nullptr; } if (Verbosity()) Report("XRay: Log file in '%s'\n", TmpFilename); - return Fd; + LogWriter *LW = allocate(); + new (LW) LogWriter(Fd); + return LW; } +void LogWriter::Close(LogWriter *LW) { + LW->~LogWriter(); + deallocate(LW); +} +#endif // SANITIZER_FUCHSIA + } // namespace __xray diff --git a/contrib/compiler-rt/lib/xray/xray_utils.h b/contrib/compiler-rt/lib/xray/xray_utils.h index eafa16e1a9d..60438973fbd 100644 --- a/contrib/compiler-rt/lib/xray/xray_utils.h +++ b/contrib/compiler-rt/lib/xray/xray_utils.h @@ -20,23 +20,40 @@ #include #include +#include "sanitizer_common/sanitizer_common.h" +#if SANITIZER_FUCHSIA +#include +#endif + namespace __xray { -// Default implementation of the reporting interface for sanitizer errors. -void printToStdErr(const char *Buffer); +class LogWriter { +public: +#if SANITIZER_FUCHSIA + LogWriter(zx_handle_t Vmo) : Vmo(Vmo) {} +#else + explicit LogWriter(int Fd) : Fd(Fd) {} +#endif + ~LogWriter(); -// EINTR-safe write routine, provided a file descriptor and a character range. -void retryingWriteAll(int Fd, const char *Begin, const char *End); + // Write a character range into a log. + void WriteAll(const char *Begin, const char *End); -// Reads a long long value from a provided file. -bool readValueFromFile(const char *Filename, long long *Value); + void Flush(); -// EINTR-safe read routine, providing a file descriptor and a character range. -std::pair retryingReadSome(int Fd, char *Begin, char *End); + // Returns a new log instance initialized using the flag-provided values. + static LogWriter *Open(); + // Closes and deallocates the log instance. + static void Close(LogWriter *LogWriter); -// EINTR-safe open routine, uses flag-provided values for initialising a log -// file. -int getLogFD(); +private: +#if SANITIZER_FUCHSIA + zx_handle_t Vmo = ZX_HANDLE_INVALID; + uint64_t Offset = 0; +#else + int Fd = -1; +#endif +}; constexpr size_t gcd(size_t a, size_t b) { return (b == 0) ? a : gcd(b, a % b); diff --git a/contrib/compiler-rt/lib/xray/xray_x86_64.cc b/contrib/compiler-rt/lib/xray/xray_x86_64.cc index 51dc4ce43b1..e63ee1b3bd0 100644 --- a/contrib/compiler-rt/lib/xray/xray_x86_64.cc +++ b/contrib/compiler-rt/lib/xray/xray_x86_64.cc @@ -1,15 +1,20 @@ #include "cpuid.h" #include "sanitizer_common/sanitizer_common.h" +#if !SANITIZER_FUCHSIA +#include "sanitizer_common/sanitizer_posix.h" +#endif #include "xray_defs.h" #include "xray_interface_internal.h" -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC #include #if SANITIZER_OPENBSD #include #include #endif #include +#elif SANITIZER_FUCHSIA +#include #endif #include @@ -81,17 +86,20 @@ uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { } return TSCFrequency == -1 ? 0 : static_cast(TSCFrequency); } -#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { long long TSCFrequency = -1; size_t tscfreqsz = sizeof(TSCFrequency); #if SANITIZER_OPENBSD int Mib[2] = { CTL_MACHDEP, CPU_TSCFREQ }; - if (sysctl(Mib, 2, &TSCFrequency, &tscfreqsz, NULL, 0) != -1) { + if (internal_sysctl(Mib, 2, &TSCFrequency, &tscfreqsz, NULL, 0) != -1) { +#elif SANITIZER_MAC + if (internal_sysctlbyname("machdep.tsc.frequency", &TSCFrequency, + &tscfreqsz, NULL, 0) != -1) { #else - if (sysctlbyname("machdep.tsc_freq", &TSCFrequency, &tscfreqsz, - NULL, 0) != -1) { + if (internal_sysctlbyname("machdep.tsc_freq", &TSCFrequency, &tscfreqsz, + NULL, 0) != -1) { #endif return static_cast(TSCFrequency); } else { @@ -100,7 +108,7 @@ uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { return 0; } -#else +#elif !SANITIZER_FUCHSIA uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { /* Not supported */ return 0; @@ -317,6 +325,7 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId, return false; } +#if !SANITIZER_FUCHSIA // We determine whether the CPU we're running on has the correct features we // need. In x86_64 this will be rdtscp support. bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { @@ -339,5 +348,6 @@ bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { } return true; } +#endif } // namespace __xray diff --git a/contrib/libc++/LICENSE.TXT b/contrib/libc++/LICENSE.TXT index c278f2c9283..190d9394d51 100644 --- a/contrib/libc++/LICENSE.TXT +++ b/contrib/libc++/LICENSE.TXT @@ -14,7 +14,7 @@ Full text of the relevant licenses is included below. University of Illinois/NCSA Open Source License -Copyright (c) 2009-2017 by the contributors listed in CREDITS.TXT +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT All rights reserved. diff --git a/contrib/libc++/include/__bit_reference b/contrib/libc++/include/__bit_reference index 3e4a21d261f..c208af2b4d7 100644 --- a/contrib/libc++/include/__bit_reference +++ b/contrib/libc++/include/__bit_reference @@ -12,6 +12,7 @@ #define _LIBCPP___BIT_REFERENCE #include <__config> +#include #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -254,18 +255,18 @@ __count_bool_true(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_type __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_); __storage_type __dn = _VSTD::min(__clz_f, __n); __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn)); - __r = _VSTD::__pop_count(*__first.__seg_ & __m); + __r = _VSTD::__popcount(*__first.__seg_ & __m); __n -= __dn; ++__first.__seg_; } // do middle whole words for (; __n >= __bits_per_word; ++__first.__seg_, __n -= __bits_per_word) - __r += _VSTD::__pop_count(*__first.__seg_); + __r += _VSTD::__popcount(*__first.__seg_); // do last partial word if (__n > 0) { __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n); - __r += _VSTD::__pop_count(*__first.__seg_ & __m); + __r += _VSTD::__popcount(*__first.__seg_ & __m); } return __r; } @@ -285,18 +286,18 @@ __count_bool_false(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_typ __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_); __storage_type __dn = _VSTD::min(__clz_f, __n); __storage_type __m = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn)); - __r = _VSTD::__pop_count(~*__first.__seg_ & __m); + __r = _VSTD::__popcount(~*__first.__seg_ & __m); __n -= __dn; ++__first.__seg_; } // do middle whole words for (; __n >= __bits_per_word; ++__first.__seg_, __n -= __bits_per_word) - __r += _VSTD::__pop_count(~*__first.__seg_); + __r += _VSTD::__popcount(~*__first.__seg_); // do last partial word if (__n > 0) { __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n); - __r += _VSTD::__pop_count(~*__first.__seg_ & __m); + __r += _VSTD::__popcount(~*__first.__seg_ & __m); } return __r; } diff --git a/contrib/libc++/include/__config b/contrib/libc++/include/__config index 738d891e303..f0bc3483fac 100644 --- a/contrib/libc++/include/__config +++ b/contrib/libc++/include/__config @@ -33,14 +33,10 @@ # define _GNUC_VER_NEW 0 #endif -#define _LIBCPP_VERSION 7000 +#define _LIBCPP_VERSION 8000 #ifndef _LIBCPP_ABI_VERSION -# ifdef __Fuchsia__ -# define _LIBCPP_ABI_VERSION 2 -# else -# define _LIBCPP_ABI_VERSION 1 -# endif +# define _LIBCPP_ABI_VERSION 1 #endif #ifndef _LIBCPP_STD_VER @@ -99,6 +95,8 @@ // Use the smallest possible integer type to represent the index of the variant. // Previously libc++ used "unsigned int" exclusivly. # define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +// Unstable attempt to provide a more optimized std::function +# define _LIBCPP_ABI_OPTIMIZED_FUNCTION #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support @@ -123,7 +121,9 @@ #define _LIBCPP_CONCAT1(_LIBCPP_X,_LIBCPP_Y) _LIBCPP_X##_LIBCPP_Y #define _LIBCPP_CONCAT(_LIBCPP_X,_LIBCPP_Y) _LIBCPP_CONCAT1(_LIBCPP_X,_LIBCPP_Y) -#define _LIBCPP_NAMESPACE _LIBCPP_CONCAT(__,_LIBCPP_ABI_VERSION) +#ifndef _LIBCPP_ABI_NAMESPACE +# define _LIBCPP_ABI_NAMESPACE _LIBCPP_CONCAT(__,_LIBCPP_ABI_VERSION) +#endif #if __cplusplus < 201103L #define _LIBCPP_CXX03_LANG @@ -353,6 +353,18 @@ # endif // __linux__ #endif +#ifndef _LIBCPP_CXX03_LANG +# define _LIBCPP_ALIGNOF(_Tp) alignof(_Tp) +#elif defined(_LIBCPP_COMPILER_CLANG) +# define _LIBCPP_ALIGNOF(_Tp) _Alignof(_Tp) +#else +// This definition is potentially buggy, but it's only taken with GCC in C++03, +// which we barely support anyway. See llvm.org/PR39713 +# define _LIBCPP_ALIGNOF(_Tp) __alignof(_Tp) +#endif + +#define _LIBCPP_PREFERRED_ALIGNOF(_Tp) __alignof(_Tp) + #if defined(_LIBCPP_COMPILER_CLANG) // _LIBCPP_ALTERNATE_STRING_LAYOUT is an old name for @@ -367,7 +379,7 @@ # define _ALIGNAS_TYPE(x) alignas(x) # define _ALIGNAS(x) alignas(x) #else -# define _ALIGNAS_TYPE(x) __attribute__((__aligned__(__alignof(x)))) +# define _ALIGNAS_TYPE(x) __attribute__((__aligned__(_LIBCPP_ALIGNOF(x)))) # define _ALIGNAS(x) __attribute__((__aligned__(x))) #endif @@ -467,16 +479,6 @@ typedef __char32_t char32_t; #define _LIBCPP_IS_LITERAL(T) __is_literal(T) #endif -// Inline namespaces are available in Clang regardless of C++ dialect. -#define _LIBCPP_BEGIN_NAMESPACE_STD namespace std {inline namespace _LIBCPP_NAMESPACE { -#define _LIBCPP_END_NAMESPACE_STD } } -#define _VSTD std::_LIBCPP_NAMESPACE - -namespace std { - inline namespace _LIBCPP_NAMESPACE { - } -} - #if !defined(_LIBCPP_HAS_NO_ASAN) && !__has_feature(address_sanitizer) #define _LIBCPP_HAS_NO_ASAN #endif @@ -496,10 +498,15 @@ namespace std { #define _LIBCPP_ALWAYS_INLINE __attribute__ ((__always_inline__)) +// No apple compilers support ""d and ""y at this time. +#if _LIBCPP_CLANG_VER < 800 || defined(__apple_build_version__) +#define _LIBCPP_HAS_NO_CXX20_CHRONO_LITERALS +#endif + #elif defined(_LIBCPP_COMPILER_GCC) #define _ALIGNAS(x) __attribute__((__aligned__(x))) -#define _ALIGNAS_TYPE(x) __attribute__((__aligned__(__alignof(x)))) +#define _ALIGNAS_TYPE(x) __attribute__((__aligned__(_LIBCPP_ALIGNOF(x)))) #define _LIBCPP_NORETURN __attribute__((noreturn)) @@ -566,15 +573,6 @@ namespace std { #endif // __GXX_EXPERIMENTAL_CXX0X__ -#define _LIBCPP_BEGIN_NAMESPACE_STD namespace std { inline namespace _LIBCPP_NAMESPACE { -#define _LIBCPP_END_NAMESPACE_STD } } -#define _VSTD std::_LIBCPP_NAMESPACE - -namespace std { - inline namespace _LIBCPP_NAMESPACE { - } -} - #if !defined(_LIBCPP_HAS_NO_ASAN) && !defined(__SANITIZE_ADDRESS__) #define _LIBCPP_HAS_NO_ASAN #endif @@ -610,13 +608,6 @@ namespace std { #define _ALIGNAS_TYPE(x) alignas(x) #define _LIBCPP_HAS_NO_VARIADICS -#define _LIBCPP_BEGIN_NAMESPACE_STD namespace std { -#define _LIBCPP_END_NAMESPACE_STD } -#define _VSTD std - -namespace std { -} - #define _LIBCPP_WEAK #define _LIBCPP_HAS_NO_ASAN @@ -628,7 +619,7 @@ namespace std { #elif defined(_LIBCPP_COMPILER_IBM) #define _ALIGNAS(x) __attribute__((__aligned__(x))) -#define _ALIGNAS_TYPE(x) __attribute__((__aligned__(__alignof(x)))) +#define _ALIGNAS_TYPE(x) __attribute__((__aligned__(_LIBCPP_ALIGNOF(x)))) #define _ATTRIBUTE(x) __attribute__((x)) #define _LIBCPP_NORETURN __attribute__((noreturn)) @@ -644,15 +635,6 @@ namespace std { #define __MULTILOCALE_API #endif -#define _LIBCPP_BEGIN_NAMESPACE_STD namespace std {inline namespace _LIBCPP_NAMESPACE { -#define _LIBCPP_END_NAMESPACE_STD } } -#define _VSTD std::_LIBCPP_NAMESPACE - -namespace std { - inline namespace _LIBCPP_NAMESPACE { - } -} - #define _LIBCPP_HAS_NO_ASAN #define _LIBCPP_ALWAYS_INLINE __attribute__ ((__always_inline__)) @@ -661,20 +643,6 @@ namespace std { #endif // _LIBCPP_COMPILER_[CLANG|GCC|MSVC|IBM] -#if _LIBCPP_STD_VER >= 17 -#define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM \ - _LIBCPP_BEGIN_NAMESPACE_STD inline namespace __fs { namespace filesystem { -#else -#define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM \ - _LIBCPP_BEGIN_NAMESPACE_STD namespace __fs { namespace filesystem { -#endif - -#define _LIBCPP_END_NAMESPACE_FILESYSTEM \ - _LIBCPP_END_NAMESPACE_STD } } - -#define _VSTD_FS _VSTD::__fs::filesystem - - #if defined(_LIBCPP_OBJECT_FORMAT_COFF) #ifdef _DLL @@ -688,33 +656,29 @@ namespace std { # define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS # define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS # define _LIBCPP_OVERRIDABLE_FUNC_VIS +# define _LIBCPP_EXPORTED_FROM_ABI #elif defined(_LIBCPP_BUILDING_LIBRARY) # define _LIBCPP_DLL_VIS __declspec(dllexport) # define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS # define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS _LIBCPP_DLL_VIS # define _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_DLL_VIS +# define _LIBCPP_EXPORTED_FROM_ABI __declspec(dllexport) #else # define _LIBCPP_DLL_VIS __declspec(dllimport) # define _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_DLL_VIS # define _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS # define _LIBCPP_OVERRIDABLE_FUNC_VIS +# define _LIBCPP_EXPORTED_FROM_ABI __declspec(dllimport) #endif #define _LIBCPP_TYPE_VIS _LIBCPP_DLL_VIS #define _LIBCPP_FUNC_VIS _LIBCPP_DLL_VIS -#define _LIBCPP_EXTERN_VIS _LIBCPP_DLL_VIS #define _LIBCPP_EXCEPTION_ABI _LIBCPP_DLL_VIS #define _LIBCPP_HIDDEN #define _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS #define _LIBCPP_TEMPLATE_VIS #define _LIBCPP_ENUM_VIS -#if defined(_LIBCPP_COMPILER_MSVC) -# define _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY __forceinline -#else -# define _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY __attribute__ ((__always_inline__)) -#endif - #endif // defined(_LIBCPP_OBJECT_FORMAT_COFF) #ifndef _LIBCPP_HIDDEN @@ -762,8 +726,12 @@ namespace std { # endif #endif -#ifndef _LIBCPP_EXTERN_VIS -#define _LIBCPP_EXTERN_VIS +#ifndef _LIBCPP_EXPORTED_FROM_ABI +# if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS) +# define _LIBCPP_EXPORTED_FROM_ABI __attribute__((__visibility__("default"))) +# else +# define _LIBCPP_EXPORTED_FROM_ABI +# endif #endif #ifndef _LIBCPP_OVERRIDABLE_FUNC_VIS @@ -804,6 +772,15 @@ namespace std { # define _LIBCPP_INTERNAL_LINKAGE _LIBCPP_ALWAYS_INLINE #endif +#if __has_attribute(exclude_from_explicit_instantiation) +# define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION __attribute__ ((__exclude_from_explicit_instantiation__)) +#else + // Try to approximate the effect of exclude_from_explicit_instantiation + // (which is that entities are not assumed to be provided by explicit + // template instantitations in the dylib) by always inlining those entities. +# define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION _LIBCPP_ALWAYS_INLINE +#endif + #ifndef _LIBCPP_HIDE_FROM_ABI_PER_TU # ifndef _LIBCPP_HIDE_FROM_ABI_PER_TU_BY_DEFAULT # define _LIBCPP_HIDE_FROM_ABI_PER_TU 0 @@ -816,21 +793,42 @@ namespace std { # if _LIBCPP_HIDE_FROM_ABI_PER_TU # define _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDDEN _LIBCPP_INTERNAL_LINKAGE # else -# define _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDDEN _LIBCPP_ALWAYS_INLINE +# define _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION # endif #endif -// Just so we can migrate to _LIBCPP_HIDE_FROM_ABI gradually. +#ifdef _LIBCPP_BUILDING_LIBRARY +# if _LIBCPP_ABI_VERSION > 1 +# define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI +# else +# define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 +# endif +#else +# define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI +#endif + +// Just so we can migrate to the new macros gradually. #define _LIBCPP_INLINE_VISIBILITY _LIBCPP_HIDE_FROM_ABI -#ifndef _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY -# if !defined(_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS) -# define _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY __attribute__((__visibility__("default"), __always_inline__)) -# else -# define _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY __attribute__((__always_inline__)) -# endif +// Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect. +#define _LIBCPP_BEGIN_NAMESPACE_STD namespace std { inline namespace _LIBCPP_ABI_NAMESPACE { +#define _LIBCPP_END_NAMESPACE_STD } } +#define _VSTD std::_LIBCPP_ABI_NAMESPACE +_LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_END_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 17 +#define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM \ + _LIBCPP_BEGIN_NAMESPACE_STD inline namespace __fs { namespace filesystem { +#else +#define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM \ + _LIBCPP_BEGIN_NAMESPACE_STD namespace __fs { namespace filesystem { #endif +#define _LIBCPP_END_NAMESPACE_FILESYSTEM \ + _LIBCPP_END_NAMESPACE_STD } } + +#define _VSTD_FS _VSTD::__fs::filesystem + #ifndef _LIBCPP_PREFERRED_OVERLOAD # if __has_attribute(__enable_if__) # define _LIBCPP_PREFERRED_OVERLOAD __attribute__ ((__enable_if__(true, ""))) @@ -991,7 +989,14 @@ template struct __static_assert_check {}; // If we are getting operator new from the MSVC CRT, then allocation overloads // for align_val_t were added in 19.12, aka VS 2017 version 15.3. #if defined(_LIBCPP_MSVCRT) && defined(_MSC_VER) && _MSC_VER < 1912 -#define _LIBCPP_HAS_NO_ALIGNED_ALLOCATION +# define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION +#elif defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_NO_VCRUNTIME) +# define _LIBCPP_DEFER_NEW_TO_VCRUNTIME +# if !defined(__cpp_aligned_new) + // We're defering to Microsoft's STL to provide aligned new et al. We don't + // have it unless the language feature test macro is defined. +# define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION +# endif #endif #if defined(__APPLE__) @@ -999,16 +1004,11 @@ template struct __static_assert_check {}; defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) # define __MAC_OS_X_VERSION_MIN_REQUIRED __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ # endif -# if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) -# if __MAC_OS_X_VERSION_MIN_REQUIRED < 1060 -# define _LIBCPP_HAS_NO_ALIGNED_ALLOCATION -# endif -# endif #endif // defined(__APPLE__) #if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) && \ - !defined(_LIBCPP_BUILDING_LIBRARY) && \ - (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606) + (defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) || \ + (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606)) # define _LIBCPP_HAS_NO_ALIGNED_ALLOCATION #endif @@ -1020,18 +1020,46 @@ template struct __static_assert_check {}; #define _LIBCPP_WCTYPE_IS_MASK #endif -#if _LIBCPP_STD_VER > 11 -# define _LIBCPP_DEPRECATED [[deprecated]] +#if _LIBCPP_STD_VER <= 17 || !defined(__cpp_char8_t) +#define _LIBCPP_NO_HAS_CHAR8_T +#endif + +// Deprecation macros. +// Deprecations warnings are only enabled when _LIBCPP_ENABLE_DEPRECATION_WARNINGS is defined. +#if defined(_LIBCPP_ENABLE_DEPRECATION_WARNINGS) +# if __has_attribute(deprecated) +# define _LIBCPP_DEPRECATED __attribute__ ((deprecated)) +# elif _LIBCPP_STD_VER > 11 +# define _LIBCPP_DEPRECATED [[deprecated]] +# else +# define _LIBCPP_DEPRECATED +# endif #else # define _LIBCPP_DEPRECATED #endif +#if !defined(_LIBCPP_CXX03_LANG) +# define _LIBCPP_DEPRECATED_IN_CXX11 _LIBCPP_DEPRECATED +#else +# define _LIBCPP_DEPRECATED_IN_CXX11 +#endif + +#if _LIBCPP_STD_VER >= 14 +# define _LIBCPP_DEPRECATED_IN_CXX14 _LIBCPP_DEPRECATED +#else +# define _LIBCPP_DEPRECATED_IN_CXX14 +#endif + +#if _LIBCPP_STD_VER >= 17 +# define _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_DEPRECATED +#else +# define _LIBCPP_DEPRECATED_IN_CXX17 +#endif + #if _LIBCPP_STD_VER <= 11 # define _LIBCPP_EXPLICIT_AFTER_CXX11 -# define _LIBCPP_DEPRECATED_AFTER_CXX11 #else # define _LIBCPP_EXPLICIT_AFTER_CXX11 explicit -# define _LIBCPP_DEPRECATED_AFTER_CXX11 [[deprecated]] #endif #if _LIBCPP_STD_VER > 11 && !defined(_LIBCPP_HAS_NO_CXX14_CONSTEXPR) @@ -1052,8 +1080,30 @@ template struct __static_assert_check {}; # define _LIBCPP_CONSTEXPR_AFTER_CXX17 #endif -#if __has_cpp_attribute(nodiscard) && _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_NODISCARD_AFTER_CXX17) -# define _LIBCPP_NODISCARD_AFTER_CXX17 [[nodiscard]] +// The _LIBCPP_NODISCARD_ATTRIBUTE should only be used to define other +// NODISCARD macros to the correct attribute. +#if __has_cpp_attribute(nodiscard) || defined(_LIBCPP_COMPILER_MSVC) +# define _LIBCPP_NODISCARD_ATTRIBUTE [[nodiscard]] +#elif defined(_LIBCPP_COMPILER_CLANG) && !defined(_LIBCPP_CXX03_LANG) +# define _LIBCPP_NODISCARD_ATTRIBUTE [[clang::warn_unused_result]] +#else +// We can't use GCC's [[gnu::warn_unused_result]] and +// __attribute__((warn_unused_result)), because GCC does not silence them via +// (void) cast. +# define _LIBCPP_NODISCARD_ATTRIBUTE +#endif + +// _LIBCPP_NODISCARD_EXT may be used to apply [[nodiscard]] to entities not +// specified as such as an extension. +#if defined(_LIBCPP_ENABLE_NODISCARD) && !defined(_LIBCPP_DISABLE_NODISCARD_EXT) +# define _LIBCPP_NODISCARD_EXT _LIBCPP_NODISCARD_ATTRIBUTE +#else +# define _LIBCPP_NODISCARD_EXT +#endif + +#if !defined(_LIBCPP_DISABLE_NODISCARD_AFTER_CXX17) && \ + (_LIBCPP_STD_VER > 17 || defined(_LIBCPP_ENABLE_NODISCARD)) +# define _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_NODISCARD_ATTRIBUTE #else # define _LIBCPP_NODISCARD_AFTER_CXX17 #endif @@ -1110,6 +1160,7 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( defined(__Fuchsia__) || \ defined(__NetBSD__) || \ defined(__linux__) || \ + defined(__GNU__) || \ defined(__APPLE__) || \ defined(__CloudABI__) || \ defined(__sun__) || \ @@ -1215,8 +1266,12 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( # define _LIBCPP_DIAGNOSE_ERROR(...) #endif -#if __has_attribute(fallthough) || _GNUC_VER >= 700 // Use a function like macro to imply that it must be followed by a semicolon +#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough) +# define _LIBCPP_FALLTHROUGH() [[fallthrough]] +#elif __has_cpp_attribute(clang::fallthrough) +# define _LIBCPP_FALLTHROUGH() [[clang::fallthrough]] +#elif __has_attribute(fallthough) || _GNUC_VER >= 700 # define _LIBCPP_FALLTHROUGH() __attribute__((__fallthrough__)) #else # define _LIBCPP_FALLTHROUGH() ((void)0) @@ -1270,9 +1325,15 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( __attribute__((availability(ios,strict,introduced=10.0))) \ __attribute__((availability(tvos,strict,introduced=10.0))) \ __attribute__((availability(watchos,strict,introduced=3.0))) -# define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS __attribute__((unavailable)) -# define _LIBCPP_AVAILABILITY_BAD_ARRAY_LENGTH __attribute__((unavailable)) -# define _LIBCPP_AVAILABILITY_BAD_ANY_CAST __attribute__((unavailable)) +# define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS \ + __attribute__((availability(macosx,strict,introduced=10.14))) \ + __attribute__((availability(ios,strict,introduced=12.0))) \ + __attribute__((availability(tvos,strict,introduced=12.0))) \ + __attribute__((availability(watchos,strict,introduced=5.0))) +# define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS \ + _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS +# define _LIBCPP_AVAILABILITY_BAD_ANY_CAST \ + _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS # define _LIBCPP_AVAILABILITY_UNCAUGHT_EXCEPTIONS \ __attribute__((availability(macosx,strict,introduced=10.12))) \ __attribute__((availability(ios,strict,introduced=10.0))) \ @@ -1296,8 +1357,8 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( __attribute__((availability(ios,strict,introduced=7.0))) #else # define _LIBCPP_AVAILABILITY_SHARED_MUTEX +# define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS # define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS -# define _LIBCPP_AVAILABILITY_BAD_ARRAY_LENGTH # define _LIBCPP_AVAILABILITY_BAD_ANY_CAST # define _LIBCPP_AVAILABILITY_UNCAUGHT_EXCEPTIONS # define _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE @@ -1309,26 +1370,30 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( // Define availability that depends on _LIBCPP_NO_EXCEPTIONS. #ifdef _LIBCPP_NO_EXCEPTIONS -# define _LIBCPP_AVAILABILITY_DYNARRAY # define _LIBCPP_AVAILABILITY_FUTURE # define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST +# define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS +# define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS #else -# define _LIBCPP_AVAILABILITY_DYNARRAY _LIBCPP_AVAILABILITY_BAD_ARRAY_LENGTH -# define _LIBCPP_AVAILABILITY_FUTURE _LIBCPP_AVAILABILITY_FUTURE_ERROR -# define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST \ - _LIBCPP_AVAILABILITY_BAD_ANY_CAST +# define _LIBCPP_AVAILABILITY_FUTURE _LIBCPP_AVAILABILITY_FUTURE_ERROR +# define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _LIBCPP_AVAILABILITY_BAD_ANY_CAST +# define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS +# define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS #endif -// Availability of stream API in the dylib got dropped and re-added. The -// extern template should effectively be available at: -// availability(macosx,introduced=10.9) -// availability(ios,introduced=7.0) -#if defined(_LIBCPP_USE_AVAILABILITY_APPLE) && \ +// The stream API was dropped and re-added in the dylib shipped on macOS +// and iOS. We can only assume the dylib to provide these definitions for +// macosx >= 10.9 and ios >= 7.0. Otherwise, the definitions are available +// from the headers, but not from the dylib. Explicit instantiation +// declarations for streams exist conditionally to this; if we provide +// an explicit instantiation declaration and we try to deploy to a dylib +// that does not provide those symbols, we'll get a load-time error. +#if !defined(_LIBCPP_BUILDING_LIBRARY) && \ ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1090) || \ (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 70000)) -#define _LIBCPP_AVAILABILITY_NO_STREAMS_EXTERN_TEMPLATE +# define _LIBCPP_DO_NOT_ASSUME_STREAMS_EXPLICIT_INSTANTIATION_IN_DYLIB #endif #if defined(_LIBCPP_COMPILER_IBM) @@ -1371,6 +1436,8 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( # endif // defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_BUILDING_LIBRARY) #endif // _LIBCPP_NO_AUTO_LINK +#define _LIBCPP_UNUSED_VAR(x) ((void)(x)) + #endif // __cplusplus #endif // _LIBCPP_CONFIG diff --git a/contrib/libc++/include/__debug b/contrib/libc++/include/__debug index d01bacdf7ed..a8788f68f8f 100644 --- a/contrib/libc++/include/__debug +++ b/contrib/libc++/include/__debug @@ -74,7 +74,7 @@ typedef void(*__libcpp_debug_function_type)(__libcpp_debug_info const&); /// __libcpp_debug_function - The handler function called when a _LIBCPP_ASSERT /// fails. -extern _LIBCPP_EXTERN_VIS __libcpp_debug_function_type __libcpp_debug_function; +extern _LIBCPP_EXPORTED_FROM_ABI __libcpp_debug_function_type __libcpp_debug_function; /// __libcpp_abort_debug_function - A debug handler that aborts when called. _LIBCPP_NORETURN _LIBCPP_FUNC_VIS diff --git a/contrib/libc++/include/__functional_base b/contrib/libc++/include/__functional_base index 57fdf2b9f66..032be99bf6a 100644 --- a/contrib/libc++/include/__functional_base +++ b/contrib/libc++/include/__functional_base @@ -50,7 +50,7 @@ template #endif struct _LIBCPP_TEMPLATE_VIS less : binary_function<_Tp, _Tp, bool> { - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator()(const _Tp& __x, const _Tp& __y) const {return __x < __y;} }; @@ -59,7 +59,7 @@ struct _LIBCPP_TEMPLATE_VIS less : binary_function<_Tp, _Tp, bool> template <> struct _LIBCPP_TEMPLATE_VIS less { - template + template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY auto operator()(_T1&& __t, _T2&& __u) const _NOEXCEPT_(noexcept(_VSTD::forward<_T1>(__t) < _VSTD::forward<_T2>(__u))) @@ -552,7 +552,7 @@ template struct __is_transparent : false_type {}; template -struct __is_transparent<_Tp, _Up, +struct __is_transparent<_Tp, _Up, typename __void_t::type> : true_type {}; #endif @@ -562,7 +562,7 @@ struct __is_transparent<_Tp, _Up, struct _LIBCPP_TEMPLATE_VIS allocator_arg_t { }; #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY) -extern const allocator_arg_t allocator_arg; +extern _LIBCPP_EXPORTED_FROM_ABI const allocator_arg_t allocator_arg; #else /* _LIBCPP_INLINE_VAR */ constexpr allocator_arg_t allocator_arg = allocator_arg_t(); #endif diff --git a/contrib/libc++/include/__hash_table b/contrib/libc++/include/__hash_table index c77de961be6..6f5b183105e 100644 --- a/contrib/libc++/include/__hash_table +++ b/contrib/libc++/include/__hash_table @@ -35,15 +35,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD template struct __hash_value_type; -template ::value && !__libcpp_is_final<_Hash>::value> -class __unordered_map_hasher; - -template ::value && !__libcpp_is_final<_Pred>::value - > -class __unordered_map_equal; - #ifndef _LIBCPP_CXX03_LANG template struct __is_hash_value_type_imp : false_type {}; @@ -418,7 +409,7 @@ public: _LIBCPP_DEBUG_MODE(__get_db()->__insert_i(this)); } - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_INLINE_VISIBILITY __hash_const_iterator(const __non_const_iterator& __x) _NOEXCEPT : __node_(__x.__node_) { @@ -871,35 +862,32 @@ struct __generic_container_node_destructor<__hash_node<_Tp, _VoidPtr>, _Alloc> }; #endif +template +struct __enforce_unordered_container_requirements { #ifndef _LIBCPP_CXX03_LANG -template -struct __diagnose_hash_table_helper { - static constexpr bool __trigger_diagnostics() - _LIBCPP_DIAGNOSE_WARNING(__check_hash_requirements<_Key, _Hash>::value - && !__invokable<_Hash const&, _Key const&>::value, - "the specified hash functor does not provide a const call operator") - _LIBCPP_DIAGNOSE_WARNING(is_copy_constructible<_Equal>::value - && !__invokable<_Equal const&, _Key const&, _Key const&>::value, - "the specified comparator type does not provide a const call operator") - { static_assert(__check_hash_requirements<_Key, _Hash>::value, - "the specified hash does not meet the Hash requirements"); + "the specified hash does not meet the Hash requirements"); static_assert(is_copy_constructible<_Equal>::value, - "the specified comparator is required to be copy constructible"); - return true; - } + "the specified comparator is required to be copy constructible"); +#endif + typedef int type; }; -template -struct __diagnose_hash_table_helper< - __hash_value_type<_Key, _Value>, - __unordered_map_hasher<_Key, __hash_value_type<_Key, _Value>, _Hash>, - __unordered_map_equal<_Key, __hash_value_type<_Key, _Value>, _Equal>, - _Alloc> -: __diagnose_hash_table_helper<_Key, _Hash, _Equal, _Alloc> -{ -}; -#endif // _LIBCPP_CXX03_LANG +template +#ifndef _LIBCPP_CXX03_LANG + _LIBCPP_DIAGNOSE_WARNING(!__invokable<_Equal const&, _Key const&, _Key const&>::value, + "the specified comparator type does not provide a const call operator") + _LIBCPP_DIAGNOSE_WARNING(!__invokable<_Hash const&, _Key const&>::value, + "the specified hash functor does not provide a const call operator") +#endif +typename __enforce_unordered_container_requirements<_Key, _Hash, _Equal>::type +__diagnose_unordered_container_requirements(int); + +// This dummy overload is used so that the compiler won't emit a spurious +// "no matching function for call to __diagnose_unordered_xxx" diagnostic +// when the overload above causes a hard error. +template +int __diagnose_unordered_container_requirements(void*); template class __hash_table @@ -963,10 +951,6 @@ private: typedef allocator_traits<__pointer_allocator> __pointer_alloc_traits; typedef typename __bucket_list_deleter::pointer __node_pointer_pointer; -#ifndef _LIBCPP_CXX03_LANG - static_assert(__diagnose_hash_table_helper<_Tp, _Hash, _Equal, _Alloc>::__trigger_diagnostics(), ""); -#endif - // --- Member data begin --- __bucket_list __bucket_list_; __compressed_pair<__first_node, __node_allocator> __p1_; @@ -1058,8 +1042,26 @@ public: ); } +private: + _LIBCPP_INLINE_VISIBILITY + __next_pointer __node_insert_multi_prepare(size_t __cp_hash, + value_type& __cp_val); + _LIBCPP_INLINE_VISIBILITY + void __node_insert_multi_perform(__node_pointer __cp, + __next_pointer __pn) _NOEXCEPT; + + _LIBCPP_INLINE_VISIBILITY + __next_pointer __node_insert_unique_prepare(size_t __nd_hash, + value_type& __nd_val); + _LIBCPP_INLINE_VISIBILITY + void __node_insert_unique_perform(__node_pointer __ptr) _NOEXCEPT; + +public: + _LIBCPP_INLINE_VISIBILITY pair __node_insert_unique(__node_pointer __nd); + _LIBCPP_INLINE_VISIBILITY iterator __node_insert_multi(__node_pointer __nd); + _LIBCPP_INLINE_VISIBILITY iterator __node_insert_multi(const_iterator __p, __node_pointer __nd); @@ -1170,6 +1172,9 @@ public: _LIBCPP_INLINE_VISIBILITY iterator __node_handle_insert_unique(const_iterator __hint, _NodeHandle&& __nh); + template + _LIBCPP_INLINE_VISIBILITY + void __node_handle_merge_unique(_Table& __source); template _LIBCPP_INLINE_VISIBILITY @@ -1177,6 +1182,9 @@ public: template _LIBCPP_INLINE_VISIBILITY iterator __node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh); + template + _LIBCPP_INLINE_VISIBILITY + void __node_handle_merge_multi(_Table& __source); template _LIBCPP_INLINE_VISIBILITY @@ -1849,73 +1857,112 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::clear() _NOEXCEPT } } + +// Prepare the container for an insertion of the value __value with the hash +// __hash. This does a lookup into the container to see if __value is already +// present, and performs a rehash if necessary. Returns a pointer to the +// existing element if it exists, otherwise nullptr. +// +// Note that this function does forward exceptions if key_eq() throws, and never +// mutates __value or actually inserts into the map. template -pair::iterator, bool> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __nd) +_LIBCPP_INLINE_VISIBILITY +typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare( + size_t __hash, value_type& __value) { - __nd->__hash_ = hash_function()(__nd->__value_); size_type __bc = bucket_count(); - bool __inserted = false; - __next_pointer __ndptr; - size_t __chash; + if (__bc != 0) { - __chash = __constrain_hash(__nd->__hash_, __bc); - __ndptr = __bucket_list_[__chash]; + size_t __chash = __constrain_hash(__hash, __bc); + __next_pointer __ndptr = __bucket_list_[__chash]; if (__ndptr != nullptr) { for (__ndptr = __ndptr->__next_; __ndptr != nullptr && __constrain_hash(__ndptr->__hash(), __bc) == __chash; __ndptr = __ndptr->__next_) { - if (key_eq()(__ndptr->__upcast()->__value_, __nd->__value_)) - goto __done; + if (key_eq()(__ndptr->__upcast()->__value_, __value)) + return __ndptr; } } } + if (size()+1 > __bc * max_load_factor() || __bc == 0) { - if (size()+1 > __bc * max_load_factor() || __bc == 0) - { - rehash(_VSTD::max(2 * __bc + !__is_hash_power2(__bc), - size_type(ceil(float(size() + 1) / max_load_factor())))); - __bc = bucket_count(); - __chash = __constrain_hash(__nd->__hash_, __bc); - } - // insert_after __bucket_list_[__chash], or __first_node if bucket is null - __next_pointer __pn = __bucket_list_[__chash]; - if (__pn == nullptr) - { - __pn =__p1_.first().__ptr(); - __nd->__next_ = __pn->__next_; - __pn->__next_ = __nd->__ptr(); - // fix up __bucket_list_ - __bucket_list_[__chash] = __pn; - if (__nd->__next_ != nullptr) - __bucket_list_[__constrain_hash(__nd->__next_->__hash(), __bc)] = __nd->__ptr(); - } - else - { - __nd->__next_ = __pn->__next_; - __pn->__next_ = __nd->__ptr(); - } - __ndptr = __nd->__ptr(); - // increment size - ++size(); - __inserted = true; + rehash(_VSTD::max(2 * __bc + !__is_hash_power2(__bc), + size_type(ceil(float(size() + 1) / max_load_factor())))); } -__done: -#if _LIBCPP_DEBUG_LEVEL >= 2 - return pair(iterator(__ndptr, this), __inserted); -#else - return pair(iterator(__ndptr), __inserted); -#endif + return nullptr; +} + +// Insert the node __nd into the container by pushing it into the right bucket, +// and updating size(). Assumes that __nd->__hash is up-to-date, and that +// rehashing has already occurred and that no element with the same key exists +// in the map. +template +_LIBCPP_INLINE_VISIBILITY +void +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_perform( + __node_pointer __nd) _NOEXCEPT +{ + size_type __bc = bucket_count(); + size_t __chash = __constrain_hash(__nd->__hash(), __bc); + // insert_after __bucket_list_[__chash], or __first_node if bucket is null + __next_pointer __pn = __bucket_list_[__chash]; + if (__pn == nullptr) + { + __pn =__p1_.first().__ptr(); + __nd->__next_ = __pn->__next_; + __pn->__next_ = __nd->__ptr(); + // fix up __bucket_list_ + __bucket_list_[__chash] = __pn; + if (__nd->__next_ != nullptr) + __bucket_list_[__constrain_hash(__nd->__next_->__hash(), __bc)] = __nd->__ptr(); + } + else + { + __nd->__next_ = __pn->__next_; + __pn->__next_ = __nd->__ptr(); + } + ++size(); } template -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __cp) +pair::iterator, bool> +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __nd) +{ + __nd->__hash_ = hash_function()(__nd->__value_); + __next_pointer __existing_node = + __node_insert_unique_prepare(__nd->__hash(), __nd->__value_); + + // Insert the node, unless it already exists in the container. + bool __inserted = false; + if (__existing_node == nullptr) + { + __node_insert_unique_perform(__nd); + __existing_node = __nd->__ptr(); + __inserted = true; + } +#if _LIBCPP_DEBUG_LEVEL >= 2 + return pair(iterator(__existing_node, this), __inserted); +#else + return pair(iterator(__existing_node), __inserted); +#endif +} + +// Prepare the container for an insertion of the value __cp_val with the hash +// __cp_hash. This does a lookup into the container to see if __cp_value is +// already present, and performs a rehash if necessary. Returns a pointer to the +// last occurance of __cp_val in the map. +// +// Note that this function does forward exceptions if key_eq() throws, and never +// mutates __value or actually inserts into the map. +template +typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare( + size_t __cp_hash, value_type& __cp_val) { - __cp->__hash_ = hash_function()(__cp->__value_); size_type __bc = bucket_count(); if (size()+1 > __bc * max_load_factor() || __bc == 0) { @@ -1923,8 +1970,44 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __c size_type(ceil(float(size() + 1) / max_load_factor())))); __bc = bucket_count(); } - size_t __chash = __constrain_hash(__cp->__hash_, __bc); + size_t __chash = __constrain_hash(__cp_hash, __bc); __next_pointer __pn = __bucket_list_[__chash]; + if (__pn != nullptr) + { + for (bool __found = false; __pn->__next_ != nullptr && + __constrain_hash(__pn->__next_->__hash(), __bc) == __chash; + __pn = __pn->__next_) + { + // __found key_eq() action + // false false loop + // true true loop + // false true set __found to true + // true false break + if (__found != (__pn->__next_->__hash() == __cp_hash && + key_eq()(__pn->__next_->__upcast()->__value_, __cp_val))) + { + if (!__found) + __found = true; + else + break; + } + } + } + return __pn; +} + +// Insert the node __cp into the container after __pn (which is the last node in +// the bucket that compares equal to __cp). Rehashing, and checking for +// uniqueness has already been performed (in __node_insert_multi_prepare), so +// all we need to do is update the bucket and size(). Assumes that __cp->__hash +// is up-to-date. +template +void +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_perform( + __node_pointer __cp, __next_pointer __pn) _NOEXCEPT +{ + size_type __bc = bucket_count(); + size_t __chash = __constrain_hash(__cp->__hash_, __bc); if (__pn == nullptr) { __pn =__p1_.first().__ptr(); @@ -1938,24 +2021,6 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __c } else { - for (bool __found = false; __pn->__next_ != nullptr && - __constrain_hash(__pn->__next_->__hash(), __bc) == __chash; - __pn = __pn->__next_) - { - // __found key_eq() action - // false false loop - // true true loop - // false true set __found to true - // true false break - if (__found != (__pn->__next_->__hash() == __cp->__hash_ && - key_eq()(__pn->__next_->__upcast()->__value_, __cp->__value_))) - { - if (!__found) - __found = true; - else - break; - } - } __cp->__next_ = __pn->__next_; __pn->__next_ = __cp->__ptr(); if (__cp->__next_ != nullptr) @@ -1966,6 +2031,17 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __c } } ++size(); +} + + +template +typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __cp) +{ + __cp->__hash_ = hash_function()(__cp->__value_); + __next_pointer __pn = __node_insert_multi_prepare(__cp->__hash(), __cp->__value_); + __node_insert_multi_perform(__cp, __pn); + #if _LIBCPP_DEBUG_LEVEL >= 2 return iterator(__cp->__ptr(), this); #else @@ -2216,6 +2292,32 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract( return _NodeHandle(remove(__p).release(), __alloc); } +template +template +_LIBCPP_INLINE_VISIBILITY +void +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_unique( + _Table& __source) +{ + static_assert(is_same<__node, typename _Table::__node>::value, ""); + + for (typename _Table::iterator __it = __source.begin(); + __it != __source.end();) + { + __node_pointer __src_ptr = __it.__node_->__upcast(); + size_t __hash = hash_function()(__src_ptr->__value_); + __next_pointer __existing_node = + __node_insert_unique_prepare(__hash, __src_ptr->__value_); + auto __prev_iter = __it++; + if (__existing_node == nullptr) + { + (void)__source.remove(__prev_iter).release(); + __src_ptr->__hash_ = __hash; + __node_insert_unique_perform(__src_ptr); + } + } +} + template template _LIBCPP_INLINE_VISIBILITY @@ -2244,6 +2346,27 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi( return __result; } +template +template +_LIBCPP_INLINE_VISIBILITY +void +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_multi( + _Table& __source) +{ + static_assert(is_same::value, ""); + + for (typename _Table::iterator __it = __source.begin(); + __it != __source.end();) + { + __node_pointer __src_ptr = __it.__node_->__upcast(); + size_t __src_hash = hash_function()(__src_ptr->__value_); + __next_pointer __pn = + __node_insert_multi_prepare(__src_hash, __src_ptr->__value_); + (void)__source.remove(__it++).release(); + __src_ptr->__hash_ = __src_hash; + __node_insert_multi_perform(__src_ptr, __pn); + } +} #endif // _LIBCPP_STD_VER > 14 template diff --git a/contrib/libc++/include/__libcpp_version b/contrib/libc++/include/__libcpp_version index 2bdc6533bec..e002b3628b3 100644 --- a/contrib/libc++/include/__libcpp_version +++ b/contrib/libc++/include/__libcpp_version @@ -1 +1 @@ -7000 +8000 diff --git a/contrib/libc++/include/__locale b/contrib/libc++/include/__locale index 24042a8e7f9..aa060d2a9f7 100644 --- a/contrib/libc++/include/__locale +++ b/contrib/libc++/include/__locale @@ -1253,13 +1253,13 @@ struct __narrow_to_utf8<8> }; template <> -struct __narrow_to_utf8<16> +struct _LIBCPP_TEMPLATE_VIS __narrow_to_utf8<16> : public codecvt { _LIBCPP_INLINE_VISIBILITY __narrow_to_utf8() : codecvt(1) {} - ~__narrow_to_utf8(); + _LIBCPP_EXPORTED_FROM_ABI ~__narrow_to_utf8(); template _LIBCPP_INLINE_VISIBILITY @@ -1287,13 +1287,13 @@ struct __narrow_to_utf8<16> }; template <> -struct __narrow_to_utf8<32> +struct _LIBCPP_TEMPLATE_VIS __narrow_to_utf8<32> : public codecvt { _LIBCPP_INLINE_VISIBILITY __narrow_to_utf8() : codecvt(1) {} - ~__narrow_to_utf8(); + _LIBCPP_EXPORTED_FROM_ABI ~__narrow_to_utf8(); template _LIBCPP_INLINE_VISIBILITY @@ -1343,13 +1343,13 @@ struct __widen_from_utf8<8> }; template <> -struct __widen_from_utf8<16> +struct _LIBCPP_TEMPLATE_VIS __widen_from_utf8<16> : public codecvt { _LIBCPP_INLINE_VISIBILITY __widen_from_utf8() : codecvt(1) {} - ~__widen_from_utf8(); + _LIBCPP_EXPORTED_FROM_ABI ~__widen_from_utf8(); template _LIBCPP_INLINE_VISIBILITY @@ -1377,13 +1377,13 @@ struct __widen_from_utf8<16> }; template <> -struct __widen_from_utf8<32> +struct _LIBCPP_TEMPLATE_VIS __widen_from_utf8<32> : public codecvt { _LIBCPP_INLINE_VISIBILITY __widen_from_utf8() : codecvt(1) {} - ~__widen_from_utf8(); + _LIBCPP_EXPORTED_FROM_ABI ~__widen_from_utf8(); template _LIBCPP_INLINE_VISIBILITY diff --git a/contrib/libc++/include/__mutex_base b/contrib/libc++/include/__mutex_base index 4659ca9298c..da21a5f8eb6 100644 --- a/contrib/libc++/include/__mutex_base +++ b/contrib/libc++/include/__mutex_base @@ -76,9 +76,9 @@ struct _LIBCPP_TYPE_VIS adopt_lock_t {}; #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY) -extern const defer_lock_t defer_lock; -extern const try_to_lock_t try_to_lock; -extern const adopt_lock_t adopt_lock; +extern _LIBCPP_EXPORTED_FROM_ABI const defer_lock_t defer_lock; +extern _LIBCPP_EXPORTED_FROM_ABI const try_to_lock_t try_to_lock; +extern _LIBCPP_EXPORTED_FROM_ABI const adopt_lock_t adopt_lock; #else diff --git a/contrib/libc++/include/__node_handle b/contrib/libc++/include/__node_handle index 567f8b047a3..a9cf3b7217a 100644 --- a/contrib/libc++/include/__node_handle +++ b/contrib/libc++/include/__node_handle @@ -26,9 +26,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER > 14 -// FIXME: Uncomment this when we support the 'merge' functionality. -// #define __cpp_lib_node_extract 201606L - // Specialized in __tree & __hash_table for their _NodeType. template struct __generic_container_node_destructor; diff --git a/contrib/libc++/include/__sso_allocator b/contrib/libc++/include/__sso_allocator index 40027363a18..8aca0495d75 100644 --- a/contrib/libc++/include/__sso_allocator +++ b/contrib/libc++/include/__sso_allocator @@ -55,14 +55,14 @@ public: __allocated_ = true; return (pointer)&buf_; } - return static_cast(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp))); + return static_cast(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp))); } - _LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) + _LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type __n) { if (__p == (pointer)&buf_) __allocated_ = false; else - _VSTD::__libcpp_deallocate(__p, __alignof(_Tp)); + _VSTD::__libcpp_deallocate(__p, __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)); } _LIBCPP_INLINE_VISIBILITY size_type max_size() const throw() {return size_type(~0) / sizeof(_Tp);} diff --git a/contrib/libc++/include/__string b/contrib/libc++/include/__string index 44c55987f9f..1ddeec7149f 100644 --- a/contrib/libc++/include/__string +++ b/contrib/libc++/include/__string @@ -47,6 +47,7 @@ struct char_traits template <> struct char_traits; template <> struct char_traits; +template <> struct char_traits; // c++20 } // std @@ -389,6 +390,102 @@ char_traits::find(const char_type* __s, size_t __n, const char_type& __ } +#ifndef _LIBCPP_NO_HAS_CHAR8_T + +template <> +struct _LIBCPP_TEMPLATE_VIS char_traits +{ + typedef char8_t char_type; + typedef unsigned int int_type; + typedef streamoff off_type; + typedef u8streampos pos_type; + typedef mbstate_t state_type; + + static inline constexpr void assign(char_type& __c1, const char_type& __c2) noexcept + {__c1 = __c2;} + static inline constexpr bool eq(char_type __c1, char_type __c2) noexcept + {return __c1 == __c2;} + static inline constexpr bool lt(char_type __c1, char_type __c2) noexcept + {return __c1 < __c2;} + + static constexpr + int compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT; + + static constexpr + size_t length(const char_type* __s) _NOEXCEPT; + + _LIBCPP_INLINE_VISIBILITY static constexpr + const char_type* find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT; + + static char_type* move(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT + {return __n == 0 ? __s1 : (char_type*) memmove(__s1, __s2, __n);} + + static char_type* copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT + { + _LIBCPP_ASSERT(__s2 < __s1 || __s2 >= __s1+__n, "char_traits::copy overlapped range"); + return __n == 0 ? __s1 : (char_type*)memcpy(__s1, __s2, __n); + } + + static char_type* assign(char_type* __s, size_t __n, char_type __a) _NOEXCEPT + {return __n == 0 ? __s : (char_type*)memset(__s, to_int_type(__a), __n);} + + static inline constexpr int_type not_eof(int_type __c) noexcept + {return eq_int_type(__c, eof()) ? ~eof() : __c;} + static inline constexpr char_type to_char_type(int_type __c) noexcept + {return char_type(__c);} + static inline constexpr int_type to_int_type(char_type __c) noexcept + {return int_type(__c);} + static inline constexpr bool eq_int_type(int_type __c1, int_type __c2) noexcept + {return __c1 == __c2;} + static inline constexpr int_type eof() noexcept + {return int_type(EOF);} +}; + +// TODO use '__builtin_strlen' if it ever supports char8_t ?? +inline constexpr +size_t +char_traits::length(const char_type* __s) _NOEXCEPT +{ + size_t __len = 0; + for (; !eq(*__s, char_type(0)); ++__s) + ++__len; + return __len; +} + +inline constexpr +int +char_traits::compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT +{ +#if __has_feature(cxx_constexpr_string_builtins) + return __builtin_memcmp(__s1, __s2, __n); +#else + for (; __n; --__n, ++__s1, ++__s2) + { + if (lt(*__s1, *__s2)) + return -1; + if (lt(*__s2, *__s1)) + return 1; + } + return 0; +#endif +} + +// TODO use '__builtin_char_memchr' if it ever supports char8_t ?? +inline constexpr +const char8_t* +char_traits::find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT +{ + for (; __n; --__n) + { + if (eq(*__s, __a)) + return __s; + ++__s; + } + return 0; +} + +#endif // #_LIBCPP_NO_HAS_CHAR8_T + #ifndef _LIBCPP_HAS_NO_UNICODE_CHARS template <> diff --git a/contrib/libc++/include/__threading_support b/contrib/libc++/include/__threading_support index 3c1eff22f53..20249006272 100644 --- a/contrib/libc++/include/__threading_support +++ b/contrib/libc++/include/__threading_support @@ -70,7 +70,7 @@ typedef pthread_t __libcpp_thread_id; typedef pthread_t __libcpp_thread_t; -// Thrad Local Storage +// Thread Local Storage typedef pthread_key_t __libcpp_tls_key; #define _LIBCPP_TLS_DESTRUCTOR_CC diff --git a/contrib/libc++/include/__tree b/contrib/libc++/include/__tree index af9b9616df8..814851085b9 100644 --- a/contrib/libc++/include/__tree +++ b/contrib/libc++/include/__tree @@ -40,10 +40,6 @@ template class __tree_node; template struct __value_type; -template ::value && !__libcpp_is_final<_Compare>::value> -class __map_value_compare; - template class __map_node_destructor; template class _LIBCPP_TEMPLATE_VIS __map_iterator; template class _LIBCPP_TEMPLATE_VIS __map_const_iterator; @@ -857,7 +853,7 @@ public: __tree_iterator operator--(int) {__tree_iterator __t(*this); --(*this); return __t;} - friend _LIBCPP_INLINE_VISIBILITY + friend _LIBCPP_INLINE_VISIBILITY bool operator==(const __tree_iterator& __x, const __tree_iterator& __y) {return __x.__ptr_ == __y.__ptr_;} friend _LIBCPP_INLINE_VISIBILITY @@ -966,24 +962,12 @@ private: }; +template #ifndef _LIBCPP_CXX03_LANG -template -struct __diagnose_tree_helper { - static constexpr bool __trigger_diagnostics() - _LIBCPP_DIAGNOSE_WARNING(!__invokable<_Compare const&, _Tp const&, _Tp const&>::value, - "the specified comparator type does not provide a const call operator") - { return true; } -}; - -template -struct __diagnose_tree_helper< - __value_type<_Key, _Value>, - __map_value_compare<_Key, __value_type<_Key, _Value>, _KeyComp>, - _Alloc -> : __diagnose_tree_helper<_Key, _KeyComp, _Alloc> -{ -}; -#endif // !_LIBCPP_CXX03_LANG + _LIBCPP_DIAGNOSE_WARNING(!std::__invokable<_Compare const&, _Tp const&, _Tp const&>::value, + "the specified comparator type does not provide a const call operator") +#endif +int __diagnose_non_const_comparator(); template class __tree @@ -1341,15 +1325,20 @@ public: #endif // !_LIBCPP_CXX03_LANG + _LIBCPP_INLINE_VISIBILITY pair __node_insert_unique(__node_pointer __nd); + _LIBCPP_INLINE_VISIBILITY iterator __node_insert_unique(const_iterator __p, __node_pointer __nd); + _LIBCPP_INLINE_VISIBILITY iterator __node_insert_multi(__node_pointer __nd); + _LIBCPP_INLINE_VISIBILITY iterator __node_insert_multi(const_iterator __p, __node_pointer __nd); - _LIBCPP_INLINE_VISIBILITY iterator __remove_node_pointer(__node_pointer); + _LIBCPP_INLINE_VISIBILITY iterator + __remove_node_pointer(__node_pointer) _NOEXCEPT; #if _LIBCPP_STD_VER > 14 template @@ -1358,6 +1347,9 @@ public: template _LIBCPP_INLINE_VISIBILITY iterator __node_handle_insert_unique(const_iterator, _NodeHandle&&); + template + _LIBCPP_INLINE_VISIBILITY + void __node_handle_merge_unique(_Tree& __source); template _LIBCPP_INLINE_VISIBILITY @@ -1365,6 +1357,9 @@ public: template _LIBCPP_INLINE_VISIBILITY iterator __node_handle_insert_multi(const_iterator, _NodeHandle&&); + template + _LIBCPP_INLINE_VISIBILITY + void __node_handle_merge_multi(_Tree& __source); template @@ -1384,7 +1379,7 @@ public: void __insert_node_at(__parent_pointer __parent, __node_base_pointer& __child, - __node_base_pointer __new_node); + __node_base_pointer __new_node) _NOEXCEPT; template iterator find(const _Key& __v); @@ -1488,7 +1483,7 @@ private: void __copy_assign_alloc(const __tree& __t, true_type) { if (__node_alloc() != __t.__node_alloc()) - clear(); + clear(); __node_alloc() = __t.__node_alloc(); } _LIBCPP_INLINE_VISIBILITY @@ -1830,7 +1825,7 @@ __tree<_Tp, _Compare, _Allocator>::operator=(__tree&& __t) __node_traits::propagate_on_container_move_assignment::value && is_nothrow_move_assignable::value && is_nothrow_move_assignable<__node_allocator>::value) - + { __move_assign(__t, integral_constant()); @@ -1844,10 +1839,6 @@ __tree<_Tp, _Compare, _Allocator>::~__tree() { static_assert((is_copy_constructible::value), "Comparator must be copy-constructible."); -#ifndef _LIBCPP_CXX03_LANG - static_assert((__diagnose_tree_helper<_Tp, _Compare, _Allocator>:: - __trigger_diagnostics()), ""); -#endif destroy(__root()); } @@ -2129,10 +2120,9 @@ __tree<_Tp, _Compare, _Allocator>::__find_equal(const_iterator __hint, } template -void -__tree<_Tp, _Compare, _Allocator>::__insert_node_at(__parent_pointer __parent, - __node_base_pointer& __child, - __node_base_pointer __new_node) +void __tree<_Tp, _Compare, _Allocator>::__insert_node_at( + __parent_pointer __parent, __node_base_pointer& __child, + __node_base_pointer __new_node) _NOEXCEPT { __new_node->__left_ = nullptr; __new_node->__right_ = nullptr; @@ -2384,7 +2374,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_insert_multi(const_iterator __p, template typename __tree<_Tp, _Compare, _Allocator>::iterator -__tree<_Tp, _Compare, _Allocator>::__remove_node_pointer(__node_pointer __ptr) +__tree<_Tp, _Compare, _Allocator>::__remove_node_pointer(__node_pointer __ptr) _NOEXCEPT { iterator __r(__ptr); ++__r; @@ -2471,6 +2461,30 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_extract(const_iterator __p) return _NodeHandle(__np, __alloc()); } +template +template +_LIBCPP_INLINE_VISIBILITY +void +__tree<_Tp, _Compare, _Allocator>::__node_handle_merge_unique(_Tree& __source) +{ + static_assert(is_same::value, ""); + + for (typename _Tree::iterator __i = __source.begin(); + __i != __source.end();) + { + __node_pointer __src_ptr = __i.__get_np(); + __parent_pointer __parent; + __node_base_pointer& __child = + __find_equal(__parent, _NodeTypes::__get_key(__src_ptr->__value_)); + ++__i; + if (__child != nullptr) + continue; + __source.__remove_node_pointer(__src_ptr); + __insert_node_at(__parent, __child, + static_cast<__node_base_pointer>(__src_ptr)); + } +} + template template _LIBCPP_INLINE_VISIBILITY @@ -2507,6 +2521,28 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi( return iterator(__ptr); } +template +template +_LIBCPP_INLINE_VISIBILITY +void +__tree<_Tp, _Compare, _Allocator>::__node_handle_merge_multi(_Tree& __source) +{ + static_assert(is_same::value, ""); + + for (typename _Tree::iterator __i = __source.begin(); + __i != __source.end();) + { + __node_pointer __src_ptr = __i.__get_np(); + __parent_pointer __parent; + __node_base_pointer& __child = __find_leaf_high( + __parent, _NodeTypes::__get_key(__src_ptr->__value_)); + ++__i; + __source.__remove_node_pointer(__src_ptr); + __insert_node_at(__parent, __child, + static_cast<__node_base_pointer>(__src_ptr)); + } +} + #endif // _LIBCPP_STD_VER > 14 template diff --git a/contrib/libc++/include/__tuple b/contrib/libc++/include/__tuple index 69d6ee96111..3b23d78afa1 100644 --- a/contrib/libc++/include/__tuple +++ b/contrib/libc++/include/__tuple @@ -22,36 +22,36 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template class _LIBCPP_TEMPLATE_VIS tuple_size; +template struct _LIBCPP_TEMPLATE_VIS tuple_size; #if !defined(_LIBCPP_CXX03_LANG) template using __enable_if_tuple_size_imp = _Tp; template -class _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< +struct _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< const _Tp, typename enable_if::value>::type, integral_constant)>>> : public integral_constant::value> {}; template -class _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< +struct _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< volatile _Tp, typename enable_if::value>::type, integral_constant)>>> : public integral_constant::value> {}; template -class _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< +struct _LIBCPP_TEMPLATE_VIS tuple_size<__enable_if_tuple_size_imp< const volatile _Tp, integral_constant)>>> : public integral_constant::value> {}; #else -template class _LIBCPP_TEMPLATE_VIS tuple_size : public tuple_size<_Tp> {}; -template class _LIBCPP_TEMPLATE_VIS tuple_size : public tuple_size<_Tp> {}; -template class _LIBCPP_TEMPLATE_VIS tuple_size : public tuple_size<_Tp> {}; +template struct _LIBCPP_TEMPLATE_VIS tuple_size : public tuple_size<_Tp> {}; +template struct _LIBCPP_TEMPLATE_VIS tuple_size : public tuple_size<_Tp> {}; +template struct _LIBCPP_TEMPLATE_VIS tuple_size : public tuple_size<_Tp> {}; #endif template class _LIBCPP_TEMPLATE_VIS tuple_element; @@ -165,7 +165,7 @@ template class _LIBCPP_TEMPLATE_VIS tuple; template struct __tuple_like > : true_type {}; template -class _LIBCPP_TEMPLATE_VIS tuple_size > +struct _LIBCPP_TEMPLATE_VIS tuple_size > : public integral_constant { }; @@ -291,7 +291,7 @@ public: template -class _LIBCPP_TEMPLATE_VIS tuple_size<__tuple_types<_Tp...> > +struct _LIBCPP_TEMPLATE_VIS tuple_size<__tuple_types<_Tp...> > : public integral_constant { }; diff --git a/contrib/libc++/include/algorithm b/contrib/libc++/include/algorithm index 90f1d246c63..d102899f2df 100644 --- a/contrib/libc++/include/algorithm +++ b/contrib/libc++/include/algorithm @@ -645,13 +645,8 @@ template #include #include #include - -#if defined(__IBMCPP__) -#include "support/ibm/support.h" -#endif -#if defined(_LIBCPP_COMPILER_MSVC) -#include -#endif +#include +#include #include <__debug> @@ -755,6 +750,32 @@ public: bool operator()(const _T1& __x, const _T2& __y) {return __p_(__y, __x);} }; +// Perform division by two quickly for positive integers (llvm.org/PR39129) + +template +_LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR +typename enable_if +< + is_integral<_Integral>::value, + _Integral +>::type +__half_positive(_Integral __value) +{ + return static_cast<_Integral>(static_cast::type>(__value) / 2); +} + +template +_LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR +typename enable_if +< + !is_integral<_Tp>::value, + _Tp +>::type +__half_positive(_Tp __value) +{ + return __value / 2; +} + #ifdef _LIBCPP_DEBUG template @@ -788,135 +809,6 @@ struct __debug_less #endif // _LIBCPP_DEBUG -// Precondition: __x != 0 -inline _LIBCPP_INLINE_VISIBILITY -unsigned __ctz(unsigned __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return static_cast(__builtin_ctz(__x)); -#else - static_assert(sizeof(unsigned) == sizeof(unsigned long), ""); - static_assert(sizeof(unsigned long) == 4, ""); - unsigned long where; - // Search from LSB to MSB for first set bit. - // Returns zero if no set bit is found. - if (_BitScanForward(&where, __x)) - return where; - return 32; -#endif -} - -inline _LIBCPP_INLINE_VISIBILITY -unsigned long __ctz(unsigned long __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return static_cast(__builtin_ctzl(__x)); -#else - static_assert(sizeof(unsigned long) == sizeof(unsigned), ""); - return __ctz(static_cast(__x)); -#endif -} - -inline _LIBCPP_INLINE_VISIBILITY -unsigned long long __ctz(unsigned long long __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return static_cast(__builtin_ctzll(__x)); -#else - unsigned long where; -// Search from LSB to MSB for first set bit. -// Returns zero if no set bit is found. -#if defined(_LIBCPP_HAS_BITSCAN64) - (defined(_M_AMD64) || defined(__x86_64__)) - if (_BitScanForward64(&where, __x)) - return static_cast(where); -#else - // Win32 doesn't have _BitScanForward64 so emulate it with two 32 bit calls. - // Scan the Low Word. - if (_BitScanForward(&where, static_cast(__x))) - return where; - // Scan the High Word. - if (_BitScanForward(&where, static_cast(__x >> 32))) - return where + 32; // Create a bit offset from the LSB. -#endif - return 64; -#endif // _LIBCPP_COMPILER_MSVC -} - -// Precondition: __x != 0 -inline _LIBCPP_INLINE_VISIBILITY -unsigned __clz(unsigned __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return static_cast(__builtin_clz(__x)); -#else - static_assert(sizeof(unsigned) == sizeof(unsigned long), ""); - static_assert(sizeof(unsigned long) == 4, ""); - unsigned long where; - // Search from LSB to MSB for first set bit. - // Returns zero if no set bit is found. - if (_BitScanReverse(&where, __x)) - return 31 - where; - return 32; // Undefined Behavior. -#endif -} - -inline _LIBCPP_INLINE_VISIBILITY -unsigned long __clz(unsigned long __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return static_cast(__builtin_clzl (__x)); -#else - static_assert(sizeof(unsigned) == sizeof(unsigned long), ""); - return __clz(static_cast(__x)); -#endif -} - -inline _LIBCPP_INLINE_VISIBILITY -unsigned long long __clz(unsigned long long __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return static_cast(__builtin_clzll(__x)); -#else - unsigned long where; -// BitScanReverse scans from MSB to LSB for first set bit. -// Returns 0 if no set bit is found. -#if defined(_LIBCPP_HAS_BITSCAN64) - if (_BitScanReverse64(&where, __x)) - return static_cast(63 - where); -#else - // Scan the high 32 bits. - if (_BitScanReverse(&where, static_cast(__x >> 32))) - return 63 - (where + 32); // Create a bit offset from the MSB. - // Scan the low 32 bits. - if (_BitScanReverse(&where, static_cast(__x))) - return 63 - where; -#endif - return 64; // Undefined Behavior. -#endif // _LIBCPP_COMPILER_MSVC -} - -inline _LIBCPP_INLINE_VISIBILITY int __pop_count(unsigned __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return __builtin_popcount (__x); -#else - static_assert(sizeof(unsigned) == 4, ""); - return __popcnt(__x); -#endif -} - -inline _LIBCPP_INLINE_VISIBILITY int __pop_count(unsigned long __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return __builtin_popcountl (__x); -#else - static_assert(sizeof(unsigned long) == 4, ""); - return __popcnt(__x); -#endif -} - -inline _LIBCPP_INLINE_VISIBILITY int __pop_count(unsigned long long __x) { -#ifndef _LIBCPP_COMPILER_MSVC - return __builtin_popcountll(__x); -#else - static_assert(sizeof(unsigned long long) == 8, ""); - return __popcnt64(__x); -#endif -} - // all_of template @@ -2533,6 +2425,8 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 _ForwardIterator min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + static_assert(__is_forward_iterator<_ForwardIterator>::value, + "std::min_element requires a ForwardIterator"); if (__first != __last) { _ForwardIterator __i = __first; @@ -2597,6 +2491,8 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 _ForwardIterator max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + static_assert(__is_forward_iterator<_ForwardIterator>::value, + "std::max_element requires a ForwardIterator"); if (__first != __last) { _ForwardIterator __i = __first; @@ -2683,6 +2579,8 @@ _LIBCPP_CONSTEXPR_AFTER_CXX11 std::pair<_ForwardIterator, _ForwardIterator> minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + static_assert(__is_forward_iterator<_ForwardIterator>::value, + "std::minmax_element requires a ForwardIterator"); std::pair<_ForwardIterator, _ForwardIterator> __result(__first, __first); if (__first != __last) { @@ -3027,10 +2925,11 @@ template template typename uniform_int_distribution<_IntType>::result_type uniform_int_distribution<_IntType>::operator()(_URNG& __g, const param_type& __p) +_LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK { typedef typename conditional::type _UIntType; - const _UIntType _Rp = __p.b() - __p.a() + _UIntType(1); + const _UIntType _Rp = _UIntType(__p.b()) - _UIntType(__p.a()) + _UIntType(1); if (_Rp == 1) return __p.a(); const size_t _Dt = numeric_limits<_UIntType>::digits; @@ -3080,7 +2979,7 @@ public: _LIBCPP_FUNC_VIS __rs_default __rs_get(); template -void +_LIBCPP_DEPRECATED_IN_CXX14 void random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last) { typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; @@ -3101,7 +3000,7 @@ random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last) } template -void +_LIBCPP_DEPRECATED_IN_CXX14 void random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last, #ifndef _LIBCPP_CXX03_LANG _RandomNumberGenerator&& __rand) @@ -3116,7 +3015,8 @@ random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last, for (--__last; __first < __last; ++__first, --__d) { difference_type __i = __rand(__d); - swap(*__first, *(__first + __i)); + if (__i != difference_type(0)) + swap(*__first, *(__first + __i)); } } } @@ -3328,7 +3228,7 @@ partition_point(_ForwardIterator __first, _ForwardIterator __last, _Predicate __ difference_type __len = _VSTD::distance(__first, __last); while (__len != 0) { - difference_type __l2 = __len / 2; + difference_type __l2 = _VSTD::__half_positive(__len); _ForwardIterator __m = __first; _VSTD::advance(__m, __l2); if (__pred(*__m)) @@ -3737,6 +3637,7 @@ __sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, // stable, 4-10 compares, 0-9 swaps template +_LIBCPP_HIDDEN unsigned __sort5(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, _ForwardIterator __x4, _ForwardIterator __x5, _Compare __c) @@ -4195,7 +4096,7 @@ __lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __va difference_type __len = _VSTD::distance(__first, __last); while (__len != 0) { - difference_type __l2 = __len / 2; + difference_type __l2 = _VSTD::__half_positive(__len); _ForwardIterator __m = __first; _VSTD::advance(__m, __l2); if (__comp(*__m, __value_)) @@ -4214,14 +4115,8 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 _ForwardIterator lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp) { -#ifdef _LIBCPP_DEBUG - typedef typename add_lvalue_reference<__debug_less<_Compare> >::type _Comp_ref; - __debug_less<_Compare> __c(__comp); - return __lower_bound<_Comp_ref>(__first, __last, __value_, __c); -#else // _LIBCPP_DEBUG typedef typename add_lvalue_reference<_Compare>::type _Comp_ref; return __lower_bound<_Comp_ref>(__first, __last, __value_, __comp); -#endif // _LIBCPP_DEBUG } template @@ -4243,7 +4138,7 @@ __upper_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __va difference_type __len = _VSTD::distance(__first, __last); while (__len != 0) { - difference_type __l2 = __len / 2; + difference_type __l2 = _VSTD::__half_positive(__len); _ForwardIterator __m = __first; _VSTD::advance(__m, __l2); if (__comp(__value_, *__m)) @@ -4262,14 +4157,8 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 _ForwardIterator upper_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp) { -#ifdef _LIBCPP_DEBUG - typedef typename add_lvalue_reference<__debug_less<_Compare> >::type _Comp_ref; - __debug_less<_Compare> __c(__comp); - return __upper_bound<_Comp_ref>(__first, __last, __value_, __c); -#else // _LIBCPP_DEBUG typedef typename add_lvalue_reference<_Compare>::type _Comp_ref; return __upper_bound<_Comp_ref>(__first, __last, __value_, __comp); -#endif // _LIBCPP_DEBUG } template @@ -4291,7 +4180,7 @@ __equal_range(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __va difference_type __len = _VSTD::distance(__first, __last); while (__len != 0) { - difference_type __l2 = __len / 2; + difference_type __l2 = _VSTD::__half_positive(__len); _ForwardIterator __m = __first; _VSTD::advance(__m, __l2); if (__comp(*__m, __value_)) diff --git a/contrib/libc++/include/any b/contrib/libc++/include/any index 9bd2f53c560..781eee7869c 100644 --- a/contrib/libc++/include/any +++ b/contrib/libc++/include/any @@ -87,13 +87,14 @@ namespace std { #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header #endif namespace std { -class _LIBCPP_EXCEPTION_ABI bad_any_cast : public bad_cast +class _LIBCPP_EXCEPTION_ABI _LIBCPP_AVAILABILITY_BAD_ANY_CAST bad_any_cast : public bad_cast { public: virtual const char* what() const _NOEXCEPT; @@ -105,12 +106,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER > 14 _LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST void __throw_bad_any_cast() { #ifndef _LIBCPP_NO_EXCEPTIONS throw bad_any_cast(); #else - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -576,6 +578,7 @@ any make_any(initializer_list<_Up> __il, _Args&&... __args) { template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any const & __v) { using _RawValueType = __uncvref_t<_ValueType>; @@ -590,6 +593,7 @@ _ValueType any_cast(any const & __v) template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any & __v) { using _RawValueType = __uncvref_t<_ValueType>; @@ -604,6 +608,7 @@ _ValueType any_cast(any & __v) template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any && __v) { using _RawValueType = __uncvref_t<_ValueType>; diff --git a/contrib/libc++/include/array b/contrib/libc++/include/array index 1e6a650198b..56f6887655a 100644 --- a/contrib/libc++/include/array +++ b/contrib/libc++/include/array @@ -91,7 +91,7 @@ template template void swap(array& x, array& y) noexcept(noexcept(x.swap(y))); // C++17 -template class tuple_size; +template struct tuple_size; template class tuple_element; template struct tuple_size>; template struct tuple_element>; @@ -112,6 +112,7 @@ template const T&& get(const array&&) noexce #include #include #include // for _LIBCPP_UNREACHABLE +#include #include <__debug> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -367,7 +368,7 @@ array(_Tp, _Args...) template inline _LIBCPP_INLINE_VISIBILITY -bool +_LIBCPP_CONSTEXPR_AFTER_CXX17 bool operator==(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) { return _VSTD::equal(__x.begin(), __x.end(), __y.begin()); @@ -375,7 +376,7 @@ operator==(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) template inline _LIBCPP_INLINE_VISIBILITY -bool +_LIBCPP_CONSTEXPR_AFTER_CXX17 bool operator!=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) { return !(__x == __y); @@ -383,7 +384,7 @@ operator!=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) template inline _LIBCPP_INLINE_VISIBILITY -bool +_LIBCPP_CONSTEXPR_AFTER_CXX17 bool operator<(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) { return _VSTD::lexicographical_compare(__x.begin(), __x.end(), @@ -392,7 +393,7 @@ operator<(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) template inline _LIBCPP_INLINE_VISIBILITY -bool +_LIBCPP_CONSTEXPR_AFTER_CXX17 bool operator>(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) { return __y < __x; @@ -400,7 +401,7 @@ operator>(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) template inline _LIBCPP_INLINE_VISIBILITY -bool +_LIBCPP_CONSTEXPR_AFTER_CXX17 bool operator<=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) { return !(__y < __x); @@ -408,7 +409,7 @@ operator<=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) template inline _LIBCPP_INLINE_VISIBILITY -bool +_LIBCPP_CONSTEXPR_AFTER_CXX17 bool operator>=(const array<_Tp, _Size>& __x, const array<_Tp, _Size>& __y) { return !(__x < __y); @@ -429,7 +430,7 @@ swap(array<_Tp, _Size>& __x, array<_Tp, _Size>& __y) } template -class _LIBCPP_TEMPLATE_VIS tuple_size > +struct _LIBCPP_TEMPLATE_VIS tuple_size > : public integral_constant {}; template diff --git a/contrib/libc++/include/atomic b/contrib/libc++/include/atomic index 809f78a06d3..d37e7b4b035 100644 --- a/contrib/libc++/include/atomic +++ b/contrib/libc++/include/atomic @@ -544,6 +544,7 @@ void atomic_signal_fence(memory_order m) noexcept; #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -559,10 +560,6 @@ void atomic_signal_fence(memory_order m) noexcept; #error C++ standard library is incompatible with #endif -#if _LIBCPP_STD_VER > 14 -# define __cpp_lib_atomic_is_always_lock_free 201603L -#endif - #define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ __m == memory_order_acquire || \ diff --git a/contrib/libc++/include/bit b/contrib/libc++/include/bit new file mode 100644 index 00000000000..db3812e5b5b --- /dev/null +++ b/contrib/libc++/include/bit @@ -0,0 +1,158 @@ +// -*- C++ -*- +//===------------------------------ bit ----------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// + +#ifndef _LIBCPP_BIT +#define _LIBCPP_BIT + +/* + bit synopsis + +namespace std { + +} // namespace std + +*/ + +#include <__config> +#include + +#if defined(__IBMCPP__) +#include "support/ibm/support.h" +#endif +#if defined(_LIBCPP_COMPILER_MSVC) +#include +#endif + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +#pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#ifndef _LIBCPP_COMPILER_MSVC + +inline _LIBCPP_INLINE_VISIBILITY +int __ctz(unsigned __x) { return __builtin_ctz(__x); } + +inline _LIBCPP_INLINE_VISIBILITY +int __ctz(unsigned long __x) { return __builtin_ctzl(__x); } + +inline _LIBCPP_INLINE_VISIBILITY +int __ctz(unsigned long long __x) { return __builtin_ctzll(__x); } + + +inline _LIBCPP_INLINE_VISIBILITY +int __clz(unsigned __x) { return __builtin_clz(__x); } + +inline _LIBCPP_INLINE_VISIBILITY +int __clz(unsigned long __x) { return __builtin_clzl(__x); } + +inline _LIBCPP_INLINE_VISIBILITY +int __clz(unsigned long long __x) { return __builtin_clzll(__x); } + + +inline _LIBCPP_INLINE_VISIBILITY +int __popcount(unsigned __x) { return __builtin_popcount(__x); } + +inline _LIBCPP_INLINE_VISIBILITY +int __popcount(unsigned long __x) { return __builtin_popcountl(__x); } + +inline _LIBCPP_INLINE_VISIBILITY +int __popcount(unsigned long long __x) { return __builtin_popcountll(__x); } + +#else // _LIBCPP_COMPILER_MSVC + +// Precondition: __x != 0 +inline _LIBCPP_INLINE_VISIBILITY +int __ctz(unsigned __x) { + static_assert(sizeof(unsigned) == sizeof(unsigned long), ""); + static_assert(sizeof(unsigned long) == 4, ""); + unsigned long __where; + if (_BitScanForward(&__where, __x)) + return static_cast(__where); + return 32; +} + +inline _LIBCPP_INLINE_VISIBILITY +int __ctz(unsigned long __x) { + static_assert(sizeof(unsigned long) == sizeof(unsigned), ""); + return __ctz(static_cast(__x)); +} + +inline _LIBCPP_INLINE_VISIBILITY +int __ctz(unsigned long long __x) { + unsigned long __where; +#if defined(_LIBCPP_HAS_BITSCAN64) + (defined(_M_AMD64) || defined(__x86_64__)) + if (_BitScanForward64(&__where, __x)) + return static_cast(__where); +#else + // Win32 doesn't have _BitScanForward64 so emulate it with two 32 bit calls. + if (_BitScanForward(&__where, static_cast(__x))) + return static_cast(__where); + if (_BitScanForward(&__where, static_cast(__x >> 32))) + return static_cast(__where + 32); +#endif + return 64; +} + +// Precondition: __x != 0 +inline _LIBCPP_INLINE_VISIBILITY +int __clz(unsigned __x) { + static_assert(sizeof(unsigned) == sizeof(unsigned long), ""); + static_assert(sizeof(unsigned long) == 4, ""); + unsigned long __where; + if (_BitScanReverse(&__where, __x)) + return static_cast(31 - __where); + return 32; // Undefined Behavior. +} + +inline _LIBCPP_INLINE_VISIBILITY +int __clz(unsigned long __x) { + static_assert(sizeof(unsigned) == sizeof(unsigned long), ""); + return __clz(static_cast(__x)); +} + +inline _LIBCPP_INLINE_VISIBILITY +int __clz(unsigned long long __x) { + unsigned long __where; +#if defined(_LIBCPP_HAS_BITSCAN64) + if (_BitScanReverse64(&__where, __x)) + return static_cast(63 - __where); +#else + // Win32 doesn't have _BitScanReverse64 so emulate it with two 32 bit calls. + if (_BitScanReverse(&__where, static_cast(__x >> 32))) + return static_cast(63 - (__where + 32)); + if (_BitScanReverse(&__where, static_cast(__x))) + return static_cast(63 - __where); +#endif + return 64; // Undefined Behavior. +} + +inline _LIBCPP_INLINE_VISIBILITY int __popcount(unsigned __x) { + static_assert(sizeof(unsigned) == 4, ""); + return __popcnt(__x); +} + +inline _LIBCPP_INLINE_VISIBILITY int __popcount(unsigned long __x) { + static_assert(sizeof(unsigned long) == 4, ""); + return __popcnt(__x); +} + +inline _LIBCPP_INLINE_VISIBILITY int __popcount(unsigned long long __x) { + static_assert(sizeof(unsigned long long) == 8, ""); + return __popcnt64(__x); +} + +#endif // _LIBCPP_COMPILER_MSVC + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_BIT diff --git a/contrib/libc++/include/bitset b/contrib/libc++/include/bitset index cd6c289b4c4..98947e027c1 100644 --- a/contrib/libc++/include/bitset +++ b/contrib/libc++/include/bitset @@ -238,9 +238,9 @@ __bitset<_N_words, _Size>::__init(unsigned long long __v, false_type) _NOEXCEPT size_t __sz = _Size; for (size_t __i = 0; __i < sizeof(__t)/sizeof(__t[0]); ++__i, __v >>= __bits_per_word, __sz -= __bits_per_word ) if ( __sz < __bits_per_word) - __t[__i] = static_cast<__storage_type>(__v) & ( 1ULL << __sz ) - 1; + __t[__i] = static_cast<__storage_type>(__v) & ( 1ULL << __sz ) - 1; else - __t[__i] = static_cast<__storage_type>(__v); + __t[__i] = static_cast<__storage_type>(__v); _VSTD::copy(__t, __t + sizeof(__t)/sizeof(__t[0]), __first_); _VSTD::fill(__first_ + sizeof(__t)/sizeof(__t[0]), __first_ + sizeof(__first_)/sizeof(__first_[0]), @@ -254,7 +254,7 @@ __bitset<_N_words, _Size>::__init(unsigned long long __v, true_type) _NOEXCEPT { __first_[0] = __v; if (_Size < __bits_per_word) - __first_[0] &= ( 1ULL << _Size ) - 1; + __first_[0] &= ( 1ULL << _Size ) - 1; _VSTD::fill(__first_ + 1, __first_ + sizeof(__first_)/sizeof(__first_[0]), __storage_type(0)); } @@ -269,9 +269,9 @@ __bitset<_N_words, _Size>::__bitset(unsigned long long __v) _NOEXCEPT #if __SIZEOF_SIZE_T__ == 8 : __first_{__v} #elif __SIZEOF_SIZE_T__ == 4 - : __first_{static_cast<__storage_type>(__v), - _Size >= 2 * __bits_per_word ? static_cast<__storage_type>(__v >> __bits_per_word) - : static_cast<__storage_type>((__v >> __bits_per_word) & (__storage_type(1) << (_Size - __bits_per_word)) - 1)} + : __first_{static_cast<__storage_type>(__v), + _Size >= 2 * __bits_per_word ? static_cast<__storage_type>(__v >> __bits_per_word) + : static_cast<__storage_type>((__v >> __bits_per_word) & (__storage_type(1) << (_Size - __bits_per_word)) - 1)} #else #error This constructor has not been ported to this platform #endif @@ -991,7 +991,7 @@ inline size_t bitset<_Size>::count() const _NOEXCEPT { - return static_cast(_VSTD::count(base::__make_iter(0), base::__make_iter(_Size), true)); + return static_cast(__count_bool_true(base::__make_iter(0), _Size)); } template diff --git a/contrib/libc++/include/charconv b/contrib/libc++/include/charconv index 7cb790e1bee..064f2e11c3f 100644 --- a/contrib/libc++/include/charconv +++ b/contrib/libc++/include/charconv @@ -87,8 +87,16 @@ namespace std { #pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + _LIBCPP_BEGIN_NAMESPACE_STD +namespace __itoa { +_LIBCPP_FUNC_VIS char* __u64toa(uint64_t __value, char* __buffer); +_LIBCPP_FUNC_VIS char* __u32toa(uint32_t __value, char* __buffer); +} + #if _LIBCPP_STD_VER > 11 enum class _LIBCPP_ENUM_VIS chars_format @@ -147,9 +155,6 @@ static constexpr uint32_t __pow10_32[] = { UINT32_C(1000000000), }; -_LIBCPP_FUNC_VIS char* __u64toa(uint64_t __value, char* __buffer); -_LIBCPP_FUNC_VIS char* __u32toa(uint32_t __value, char* __buffer); - template struct _LIBCPP_HIDDEN __traits_base { @@ -607,4 +612,6 @@ from_chars(const char* __first, const char* __last, _Tp& __value, int __base) _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP_CHARCONV diff --git a/contrib/libc++/include/chrono b/contrib/libc++/include/chrono index f6a6f4b2434..96759f9860e 100644 --- a/contrib/libc++/include/chrono +++ b/contrib/libc++/include/chrono @@ -33,9 +33,9 @@ template struct duration_values { public: - static constexpr Rep zero(); - static constexpr Rep max(); - static constexpr Rep min(); + static constexpr Rep zero(); // noexcept in C++20 + static constexpr Rep max(); // noexcept in C++20 + static constexpr Rep min(); // noexcept in C++20 }; // duration @@ -77,22 +77,24 @@ public: constexpr common_type::type operator+() const; constexpr common_type::type operator-() const; - constexpr duration& operator++(); - constexpr duration operator++(int); - constexpr duration& operator--(); - constexpr duration operator--(int); + constexpr duration& operator++(); // constexpr in C++17 + constexpr duration operator++(int); // constexpr in C++17 + constexpr duration& operator--(); // constexpr in C++17 + constexpr duration operator--(int); // constexpr in C++17 - constexpr duration& operator+=(const duration& d); - constexpr duration& operator-=(const duration& d); + constexpr duration& operator+=(const duration& d); // constexpr in C++17 + constexpr duration& operator-=(const duration& d); // constexpr in C++17 - duration& operator*=(const rep& rhs); - duration& operator/=(const rep& rhs); + duration& operator*=(const rep& rhs); // constexpr in C++17 + duration& operator/=(const rep& rhs); // constexpr in C++17 + duration& operator%=(const rep& rhs); // constexpr in C++17 + duration& operator%=(const duration& rhs); // constexpr in C++17 // special values - static constexpr duration zero(); - static constexpr duration min(); - static constexpr duration max(); + static constexpr duration zero(); // noexcept in C++20 + static constexpr duration min(); // noexcept in C++20 + static constexpr duration max(); // noexcept in C++20 }; typedef duration nanoseconds; @@ -127,13 +129,13 @@ public: // arithmetic - time_point& operator+=(const duration& d); - time_point& operator-=(const duration& d); + time_point& operator+=(const duration& d); // constexpr in C++17 + time_point& operator-=(const duration& d); // constexpr in C++17 // special values - static constexpr time_point min(); - static constexpr time_point max(); + static constexpr time_point min(); // noexcept in C++20 + static constexpr time_point max(); // noexcept in C++20 }; } // chrono @@ -323,7 +325,7 @@ struct clock_time_conversion; template auto clock_cast(const time_point& t); - + // 25.8.2, class last_spec // C++20 struct last_spec; @@ -528,7 +530,7 @@ constexpr year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, const months& dm) noexcept; constexpr year_month_weekday_last operator-(const year_month_weekday_last& ymwdl, const years& dy) noexcept; - + // 25.8.18, civil calendar conventional syntax operators // C++20 constexpr year_month operator/(const year& y, const month& m) noexcept; @@ -609,7 +611,7 @@ constexpr year_month_weekday_last constexpr year_month_weekday_last operator/(const month_weekday_last& mwdl, const year& y) noexcept; constexpr year_month_weekday_last - operator/(const month_weekday_last& mwdl, int y) noexcept; + operator/(const month_weekday_last& mwdl, int y) noexcept; // 25.9, class template time_of_day // C++20 template class time_of_day; @@ -640,7 +642,7 @@ class ambiguous_local_time; // 25.10.4, information classes // C++20 struct sys_info; struct local_info; - + // 25.10.5, class time_zone // C++20 enum class choose {earliest, latest}; class time_zone; @@ -650,7 +652,7 @@ bool operator<(const time_zone& x, const time_zone& y) noexcept; bool operator>(const time_zone& x, const time_zone& y) noexcept; bool operator<=(const time_zone& x, const time_zone& y) noexcept; bool operator>=(const time_zone& x, const time_zone& y) noexcept; - + // 25.10.6, class template zoned_traits // C++20 template struct zoned_traits; @@ -724,7 +726,7 @@ template template basic_string format(const locale& loc, const basic_string& fmt, - const Streamable& s); + const Streamable& s); // 25.12, parsing // C++20 template @@ -746,7 +748,8 @@ unspecified parse(const basic_string& format, Parsable& tp, basic_string& abbrev, minutes& offset); -inline constexpr last_spec last{}; // C++20 +// calendrical constants +inline constexpr last_spec last{}; // C++20 inline constexpr chrono::weekday Sunday{0}; // C++20 inline constexpr chrono::weekday Monday{1}; // C++20 inline constexpr chrono::weekday Tuesday{2}; // C++20 @@ -796,6 +799,7 @@ constexpr chrono::year operator ""y(unsigned lo #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -804,6 +808,11 @@ constexpr chrono::year operator ""y(unsigned lo _LIBCPP_PUSH_MACROS #include <__undef_macros> +#ifndef _LIBCPP_CXX03_LANG +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM +struct _FilesystemClock; +_LIBCPP_END_NAMESPACE_FILESYSTEM +#endif // !_LIBCPP_CXX03_LANG _LIBCPP_BEGIN_NAMESPACE_STD @@ -920,9 +929,9 @@ template struct _LIBCPP_TEMPLATE_VIS duration_values { public: - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR _Rep zero() {return _Rep(0);} - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR _Rep max() {return numeric_limits<_Rep>::max();} - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR _Rep min() {return numeric_limits<_Rep>::lowest();} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR _Rep zero() _NOEXCEPT {return _Rep(0);} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR _Rep max() _NOEXCEPT {return numeric_limits<_Rep>::max();} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR _Rep min() _NOEXCEPT {return numeric_limits<_Rep>::lowest();} }; #if _LIBCPP_STD_VER > 14 @@ -1015,7 +1024,7 @@ class _LIBCPP_TEMPLATE_VIS duration typedef ratio<__mul<__n1, __d2, !value>::value, __mul<__n2, __d1, !value>::value> type; }; - + public: typedef _Rep rep; typedef typename _Period::type period; @@ -1077,9 +1086,9 @@ public: // special values - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR duration zero() {return duration(duration_values::zero());} - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR duration min() {return duration(duration_values::min());} - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR duration max() {return duration(duration_values::max());} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR duration zero() _NOEXCEPT {return duration(duration_values::zero());} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR duration min() _NOEXCEPT {return duration(duration_values::min());} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR duration max() _NOEXCEPT {return duration(duration_values::max());} }; typedef duration nanoseconds; @@ -1088,7 +1097,12 @@ typedef duration milliseconds; typedef duration seconds; typedef duration< long, ratio< 60> > minutes; typedef duration< long, ratio<3600> > hours; - +#if _LIBCPP_STD_VER > 17 +typedef duration< int, ratio_multiply, hours::period>> days; +typedef duration< int, ratio_multiply, days::period>> weeks; +typedef duration< int, ratio_multiply, days::period>> years; +typedef duration< int, ratio_divide>> months; +#endif // Duration == template @@ -1355,13 +1369,13 @@ public: // arithmetic - _LIBCPP_INLINE_VISIBILITY time_point& operator+=(const duration& __d) {__d_ += __d; return *this;} - _LIBCPP_INLINE_VISIBILITY time_point& operator-=(const duration& __d) {__d_ -= __d; return *this;} + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 time_point& operator+=(const duration& __d) {__d_ += __d; return *this;} + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 time_point& operator-=(const duration& __d) {__d_ -= __d; return *this;} // special values - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR time_point min() {return time_point(duration::min());} - _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR time_point max() {return time_point(duration::max());} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR time_point min() _NOEXCEPT {return time_point(duration::min());} + _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT {return time_point(duration::max());} }; } // chrono @@ -1571,12 +1585,1158 @@ typedef steady_clock high_resolution_clock; typedef system_clock high_resolution_clock; #endif +#if _LIBCPP_STD_VER > 17 +// [time.clock.file], type file_clock +using file_clock = _VSTD_FS::_FilesystemClock; + +template +using file_time = time_point; + + +template +using sys_time = time_point; +using sys_seconds = sys_time; +using sys_days = sys_time; + +struct local_t {}; +template +using local_time = time_point; +using local_seconds = local_time; +using local_days = local_time; + + +struct _LIBCPP_TYPE_VIS last_spec { explicit last_spec() = default; }; + +class _LIBCPP_TYPE_VIS day { +private: + unsigned char __d; +public: + day() = default; + explicit inline constexpr day(unsigned __val) noexcept : __d(static_cast(__val)) {} + inline constexpr day& operator++() noexcept { ++__d; return *this; } + inline constexpr day operator++(int) noexcept { day __tmp = *this; ++(*this); return __tmp; } + inline constexpr day& operator--() noexcept { --__d; return *this; } + inline constexpr day operator--(int) noexcept { day __tmp = *this; --(*this); return __tmp; } + constexpr day& operator+=(const days& __dd) noexcept; + constexpr day& operator-=(const days& __dd) noexcept; + explicit inline constexpr operator unsigned() const noexcept { return __d; } + inline constexpr bool ok() const noexcept { return __d >= 1 && __d <= 31; } + }; + + +inline constexpr +bool operator==(const day& __lhs, const day& __rhs) noexcept +{ return static_cast(__lhs) == static_cast(__rhs); } + +inline constexpr +bool operator!=(const day& __lhs, const day& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const day& __lhs, const day& __rhs) noexcept +{ return static_cast(__lhs) < static_cast(__rhs); } + +inline constexpr +bool operator> (const day& __lhs, const day& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const day& __lhs, const day& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const day& __lhs, const day& __rhs) noexcept +{ return !(__lhs < __rhs); } + +inline constexpr +day operator+ (const day& __lhs, const days& __rhs) noexcept +{ return day(static_cast(__lhs) + __rhs.count()); } + +inline constexpr +day operator+ (const days& __lhs, const day& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +day operator- (const day& __lhs, const days& __rhs) noexcept +{ return __lhs + -__rhs; } + +inline constexpr +days operator-(const day& __lhs, const day& __rhs) noexcept +{ return days(static_cast(static_cast(__lhs)) - + static_cast(static_cast(__rhs))); } + +inline constexpr day& day::operator+=(const days& __dd) noexcept +{ *this = *this + __dd; return *this; } + +inline constexpr day& day::operator-=(const days& __dd) noexcept +{ *this = *this - __dd; return *this; } + + +class _LIBCPP_TYPE_VIS month { +private: + unsigned char __m; +public: + month() = default; + explicit inline constexpr month(unsigned __val) noexcept : __m(static_cast(__val)) {} + inline constexpr month& operator++() noexcept { ++__m; return *this; } + inline constexpr month operator++(int) noexcept { month __tmp = *this; ++(*this); return __tmp; } + inline constexpr month& operator--() noexcept { --__m; return *this; } + inline constexpr month operator--(int) noexcept { month __tmp = *this; --(*this); return __tmp; } + constexpr month& operator+=(const months& __m1) noexcept; + constexpr month& operator-=(const months& __m1) noexcept; + explicit inline constexpr operator unsigned() const noexcept { return __m; } + inline constexpr bool ok() const noexcept { return __m >= 1 && __m <= 12; } +}; + + +inline constexpr +bool operator==(const month& __lhs, const month& __rhs) noexcept +{ return static_cast(__lhs) == static_cast(__rhs); } + +inline constexpr +bool operator!=(const month& __lhs, const month& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const month& __lhs, const month& __rhs) noexcept +{ return static_cast(__lhs) < static_cast(__rhs); } + +inline constexpr +bool operator> (const month& __lhs, const month& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const month& __lhs, const month& __rhs) noexcept +{ return !(__rhs < __lhs); } + +inline constexpr +bool operator>=(const month& __lhs, const month& __rhs) noexcept +{ return !(__lhs < __rhs); } + +inline constexpr +month operator+ (const month& __lhs, const months& __rhs) noexcept +{ + auto const __mu = static_cast(static_cast(__lhs)) + (__rhs.count() - 1); + auto const __yr = (__mu >= 0 ? __mu : __mu - 11) / 12; + return month{static_cast(__mu - __yr * 12 + 1)}; +} + +inline constexpr +month operator+ (const months& __lhs, const month& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +month operator- (const month& __lhs, const months& __rhs) noexcept +{ return __lhs + -__rhs; } + +inline constexpr +months operator-(const month& __lhs, const month& __rhs) noexcept +{ + auto const __dm = static_cast(__lhs) - static_cast(__rhs); + return months(__dm <= 11 ? __dm : __dm + 12); +} + +inline constexpr month& month::operator+=(const months& __dm) noexcept +{ *this = *this + __dm; return *this; } + +inline constexpr month& month::operator-=(const months& __dm) noexcept +{ *this = *this - __dm; return *this; } + + +class _LIBCPP_TYPE_VIS year { +private: + short __y; +public: + year() = default; + explicit inline constexpr year(int __val) noexcept : __y(static_cast(__val)) {} + + inline constexpr year& operator++() noexcept { ++__y; return *this; }; + inline constexpr year operator++(int) noexcept { year __tmp = *this; ++(*this); return __tmp; }; + inline constexpr year& operator--() noexcept { --__y; return *this; }; + inline constexpr year operator--(int) noexcept { year __tmp = *this; --(*this); return __tmp; }; + constexpr year& operator+=(const years& __dy) noexcept; + constexpr year& operator-=(const years& __dy) noexcept; + inline constexpr year operator+() const noexcept { return *this; } + inline constexpr year operator-() const noexcept { return year{-__y}; }; + + inline constexpr bool is_leap() const noexcept { return __y % 4 == 0 && (__y % 100 != 0 || __y % 400 == 0); } + explicit inline constexpr operator int() const noexcept { return __y; } + constexpr bool ok() const noexcept; + static inline constexpr year min() noexcept { return year{-32767}; } + static inline constexpr year max() noexcept { return year{ 32767}; } +}; + + +inline constexpr +bool operator==(const year& __lhs, const year& __rhs) noexcept +{ return static_cast(__lhs) == static_cast(__rhs); } + +inline constexpr +bool operator!=(const year& __lhs, const year& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const year& __lhs, const year& __rhs) noexcept +{ return static_cast(__lhs) < static_cast(__rhs); } + +inline constexpr +bool operator> (const year& __lhs, const year& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const year& __lhs, const year& __rhs) noexcept +{ return !(__rhs < __lhs); } + +inline constexpr +bool operator>=(const year& __lhs, const year& __rhs) noexcept +{ return !(__lhs < __rhs); } + +inline constexpr +year operator+ (const year& __lhs, const years& __rhs) noexcept +{ return year(static_cast(__lhs) + __rhs.count()); } + +inline constexpr +year operator+ (const years& __lhs, const year& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year operator- (const year& __lhs, const years& __rhs) noexcept +{ return __lhs + -__rhs; } + +inline constexpr +years operator-(const year& __lhs, const year& __rhs) noexcept +{ return years{static_cast(__lhs) - static_cast(__rhs)}; } + + +inline constexpr year& year::operator+=(const years& __dy) noexcept +{ *this = *this + __dy; return *this; } + +inline constexpr year& year::operator-=(const years& __dy) noexcept +{ *this = *this - __dy; return *this; } + +inline constexpr bool year::ok() const noexcept +{ return static_cast(min()) <= __y && __y <= static_cast(max()); } + +class _LIBCPP_TYPE_VIS weekday_indexed; +class _LIBCPP_TYPE_VIS weekday_last; + +class _LIBCPP_TYPE_VIS weekday { +private: + unsigned char __wd; +public: + weekday() = default; + inline explicit constexpr weekday(unsigned __val) noexcept : __wd(static_cast(__val)) {} + inline constexpr weekday(const sys_days& __sysd) noexcept + : __wd(__weekday_from_days(__sysd.time_since_epoch().count())) {} + inline explicit constexpr weekday(const local_days& __locd) noexcept + : __wd(__weekday_from_days(__locd.time_since_epoch().count())) {} + + inline constexpr weekday& operator++() noexcept { __wd = (__wd == 6 ? 0 : __wd + 1); return *this; } + inline constexpr weekday operator++(int) noexcept { weekday __tmp = *this; ++(*this); return __tmp; } + inline constexpr weekday& operator--() noexcept { __wd = (__wd == 0 ? 6 : __wd - 1); return *this; } + inline constexpr weekday operator--(int) noexcept { weekday __tmp = *this; --(*this); return __tmp; } + constexpr weekday& operator+=(const days& __dd) noexcept; + constexpr weekday& operator-=(const days& __dd) noexcept; + inline explicit constexpr operator unsigned() const noexcept { return __wd; } + inline constexpr bool ok() const noexcept { return __wd <= 6; } + constexpr weekday_indexed operator[](unsigned __index) const noexcept; + constexpr weekday_last operator[](last_spec) const noexcept; + + static constexpr unsigned char __weekday_from_days(int __days) noexcept; +}; + + +// https://howardhinnant.github.io/date_algorithms.html#weekday_from_days +inline constexpr +unsigned char weekday::__weekday_from_days(int __days) noexcept +{ + return static_cast( + static_cast(__days >= -4 ? (__days+4) % 7 : (__days+5) % 7 + 6) + ); +} + +inline constexpr +bool operator==(const weekday& __lhs, const weekday& __rhs) noexcept +{ return static_cast(__lhs) == static_cast(__rhs); } + +inline constexpr +bool operator!=(const weekday& __lhs, const weekday& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const weekday& __lhs, const weekday& __rhs) noexcept +{ return static_cast(__lhs) < static_cast(__rhs); } + +inline constexpr +bool operator> (const weekday& __lhs, const weekday& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const weekday& __lhs, const weekday& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const weekday& __lhs, const weekday& __rhs) noexcept +{ return !(__lhs < __rhs); } + +constexpr weekday operator+(const weekday& __lhs, const days& __rhs) noexcept +{ + auto const __mu = static_cast(static_cast(__lhs)) + __rhs.count(); + auto const __yr = (__mu >= 0 ? __mu : __mu - 6) / 7; + return weekday{static_cast(__mu - __yr * 7)}; +} + +constexpr weekday operator+(const days& __lhs, const weekday& __rhs) noexcept +{ return __rhs + __lhs; } + +constexpr weekday operator-(const weekday& __lhs, const days& __rhs) noexcept +{ return __lhs + -__rhs; } + +constexpr days operator-(const weekday& __lhs, const weekday& __rhs) noexcept +{ + const int __wdu = static_cast(__lhs) - static_cast(__rhs); + const int __wk = (__wdu >= 0 ? __wdu : __wdu-6) / 7; + return days{__wdu - __wk * 7}; +} + +inline constexpr weekday& weekday::operator+=(const days& __dd) noexcept +{ *this = *this + __dd; return *this; } + +inline constexpr weekday& weekday::operator-=(const days& __dd) noexcept +{ *this = *this - __dd; return *this; } + + +class _LIBCPP_TYPE_VIS weekday_indexed { +private: + _VSTD::chrono::weekday __wd; + unsigned char __idx; +public: + weekday_indexed() = default; + inline constexpr weekday_indexed(const _VSTD::chrono::weekday& __wdval, unsigned __idxval) noexcept + : __wd{__wdval}, __idx(__idxval) {} + inline constexpr _VSTD::chrono::weekday weekday() const noexcept { return __wd; } + inline constexpr unsigned index() const noexcept { return __idx; } + inline constexpr bool ok() const noexcept { return __wd.ok() && __idx >= 1 && __idx <= 5; } +}; + +inline constexpr +bool operator==(const weekday_indexed& __lhs, const weekday_indexed& __rhs) noexcept +{ return __lhs.weekday() == __rhs.weekday() && __lhs.index() == __rhs.index(); } + +inline constexpr +bool operator!=(const weekday_indexed& __lhs, const weekday_indexed& __rhs) noexcept +{ return !(__lhs == __rhs); } + + +class _LIBCPP_TYPE_VIS weekday_last { +private: + _VSTD::chrono::weekday __wd; +public: + explicit constexpr weekday_last(const _VSTD::chrono::weekday& __val) noexcept + : __wd{__val} {} + constexpr _VSTD::chrono::weekday weekday() const noexcept { return __wd; } + constexpr bool ok() const noexcept { return __wd.ok(); } +}; + +inline constexpr +bool operator==(const weekday_last& __lhs, const weekday_last& __rhs) noexcept +{ return __lhs.weekday() == __rhs.weekday(); } + +inline constexpr +bool operator!=(const weekday_last& __lhs, const weekday_last& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +weekday_indexed weekday::operator[](unsigned __index) const noexcept { return weekday_indexed{*this, __index}; } + +inline constexpr +weekday_last weekday::operator[](last_spec) const noexcept { return weekday_last{*this}; } + + +inline constexpr last_spec last{}; +inline constexpr weekday Sunday{0}; +inline constexpr weekday Monday{1}; +inline constexpr weekday Tuesday{2}; +inline constexpr weekday Wednesday{3}; +inline constexpr weekday Thursday{4}; +inline constexpr weekday Friday{5}; +inline constexpr weekday Saturday{6}; + +inline constexpr month January{1}; +inline constexpr month February{2}; +inline constexpr month March{3}; +inline constexpr month April{4}; +inline constexpr month May{5}; +inline constexpr month June{6}; +inline constexpr month July{7}; +inline constexpr month August{8}; +inline constexpr month September{9}; +inline constexpr month October{10}; +inline constexpr month November{11}; +inline constexpr month December{12}; + + +class _LIBCPP_TYPE_VIS month_day { +private: + chrono::month __m; + chrono::day __d; +public: + month_day() = default; + constexpr month_day(const chrono::month& __mval, const chrono::day& __dval) noexcept + : __m{__mval}, __d{__dval} {} + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr chrono::day day() const noexcept { return __d; } + constexpr bool ok() const noexcept; +}; + +inline constexpr +bool month_day::ok() const noexcept +{ + if (!__m.ok()) return false; + const unsigned __dval = static_cast(__d); + if (__dval < 1 || __dval > 31) return false; + if (__dval <= 29) return true; +// Now we've got either 30 or 31 + const unsigned __mval = static_cast(__m); + if (__mval == 2) return false; + if (__mval == 4 || __mval == 6 || __mval == 9 || __mval == 11) + return __dval == 30; + return true; +} + +inline constexpr +bool operator==(const month_day& __lhs, const month_day& __rhs) noexcept +{ return __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); } + +inline constexpr +bool operator!=(const month_day& __lhs, const month_day& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +month_day operator/(const month& __lhs, const day& __rhs) noexcept +{ return month_day{__lhs, __rhs}; } + +constexpr +month_day operator/(const day& __lhs, const month& __rhs) noexcept +{ return __rhs / __lhs; } + +inline constexpr +month_day operator/(const month& __lhs, int __rhs) noexcept +{ return __lhs / day(__rhs); } + +constexpr +month_day operator/(int __lhs, const day& __rhs) noexcept +{ return month(__lhs) / __rhs; } + +constexpr +month_day operator/(const day& __lhs, int __rhs) noexcept +{ return month(__rhs) / __lhs; } + + +inline constexpr +bool operator< (const month_day& __lhs, const month_day& __rhs) noexcept +{ return __lhs.month() != __rhs.month() ? __lhs.month() < __rhs.month() : __lhs.day() < __rhs.day(); } + +inline constexpr +bool operator> (const month_day& __lhs, const month_day& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const month_day& __lhs, const month_day& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const month_day& __lhs, const month_day& __rhs) noexcept +{ return !(__lhs < __rhs); } + + + +class _LIBCPP_TYPE_VIS month_day_last { +private: + chrono::month __m; +public: + explicit constexpr month_day_last(const chrono::month& __val) noexcept + : __m{__val} {} + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr bool ok() const noexcept { return __m.ok(); } +}; + +inline constexpr +bool operator==(const month_day_last& __lhs, const month_day_last& __rhs) noexcept +{ return __lhs.month() == __rhs.month(); } + +inline constexpr +bool operator!=(const month_day_last& __lhs, const month_day_last& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const month_day_last& __lhs, const month_day_last& __rhs) noexcept +{ return __lhs.month() < __rhs.month(); } + +inline constexpr +bool operator> (const month_day_last& __lhs, const month_day_last& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const month_day_last& __lhs, const month_day_last& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const month_day_last& __lhs, const month_day_last& __rhs) noexcept +{ return !(__lhs < __rhs); } + +inline constexpr +month_day_last operator/(const month& __lhs, last_spec) noexcept +{ return month_day_last{__lhs}; } + +inline constexpr +month_day_last operator/(last_spec, const month& __rhs) noexcept +{ return month_day_last{__rhs}; } + +inline constexpr +month_day_last operator/(int __lhs, last_spec) noexcept +{ return month_day_last{month(__lhs)}; } + +inline constexpr +month_day_last operator/(last_spec, int __rhs) noexcept +{ return month_day_last{month(__rhs)}; } + + +class _LIBCPP_TYPE_VIS month_weekday { +private: + chrono::month __m; + chrono::weekday_indexed __wdi; +public: + month_weekday() = default; + constexpr month_weekday(const chrono::month& __mval, const chrono::weekday_indexed& __wdival) noexcept + : __m{__mval}, __wdi{__wdival} {} + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi; } + inline constexpr bool ok() const noexcept { return __m.ok() && __wdi.ok(); } +}; + +inline constexpr +bool operator==(const month_weekday& __lhs, const month_weekday& __rhs) noexcept +{ return __lhs.month() == __rhs.month() && __lhs.weekday_indexed() == __rhs.weekday_indexed(); } + +inline constexpr +bool operator!=(const month_weekday& __lhs, const month_weekday& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +month_weekday operator/(const month& __lhs, const weekday_indexed& __rhs) noexcept +{ return month_weekday{__lhs, __rhs}; } + +inline constexpr +month_weekday operator/(int __lhs, const weekday_indexed& __rhs) noexcept +{ return month_weekday{month(__lhs), __rhs}; } + +inline constexpr +month_weekday operator/(const weekday_indexed& __lhs, const month& __rhs) noexcept +{ return month_weekday{__rhs, __lhs}; } + +inline constexpr +month_weekday operator/(const weekday_indexed& __lhs, int __rhs) noexcept +{ return month_weekday{month(__rhs), __lhs}; } + + +class _LIBCPP_TYPE_VIS month_weekday_last { + chrono::month __m; + chrono::weekday_last __wdl; + public: + constexpr month_weekday_last(const chrono::month& __mval, const chrono::weekday_last& __wdlval) noexcept + : __m{__mval}, __wdl{__wdlval} {} + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl; } + inline constexpr bool ok() const noexcept { return __m.ok() && __wdl.ok(); } +}; + +inline constexpr +bool operator==(const month_weekday_last& __lhs, const month_weekday_last& __rhs) noexcept +{ return __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last(); } + +inline constexpr +bool operator!=(const month_weekday_last& __lhs, const month_weekday_last& __rhs) noexcept +{ return !(__lhs == __rhs); } + + +inline constexpr +month_weekday_last operator/(const month& __lhs, const weekday_last& __rhs) noexcept +{ return month_weekday_last{__lhs, __rhs}; } + +inline constexpr +month_weekday_last operator/(int __lhs, const weekday_last& __rhs) noexcept +{ return month_weekday_last{month(__lhs), __rhs}; } + +inline constexpr +month_weekday_last operator/(const weekday_last& __lhs, const month& __rhs) noexcept +{ return month_weekday_last{__rhs, __lhs}; } + +inline constexpr +month_weekday_last operator/(const weekday_last& __lhs, int __rhs) noexcept +{ return month_weekday_last{month(__rhs), __lhs}; } + + +class _LIBCPP_TYPE_VIS year_month { + chrono::year __y; + chrono::month __m; +public: + year_month() = default; + constexpr year_month(const chrono::year& __yval, const chrono::month& __mval) noexcept + : __y{__yval}, __m{__mval} {} + inline constexpr chrono::year year() const noexcept { return __y; } + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr year_month& operator+=(const months& __dm) noexcept { this->__m += __dm; return *this; } + inline constexpr year_month& operator-=(const months& __dm) noexcept { this->__m -= __dm; return *this; } + inline constexpr year_month& operator+=(const years& __dy) noexcept { this->__y += __dy; return *this; } + inline constexpr year_month& operator-=(const years& __dy) noexcept { this->__y -= __dy; return *this; } + inline constexpr bool ok() const noexcept { return __y.ok() && __m.ok(); } +}; + +inline constexpr +year_month operator/(const year& __y, const month& __m) noexcept { return year_month{__y, __m}; } + +inline constexpr +year_month operator/(const year& __y, int __m) noexcept { return year_month{__y, month(__m)}; } + +inline constexpr +bool operator==(const year_month& __lhs, const year_month& __rhs) noexcept +{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month(); } + +inline constexpr +bool operator!=(const year_month& __lhs, const year_month& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const year_month& __lhs, const year_month& __rhs) noexcept +{ return __lhs.year() != __rhs.year() ? __lhs.year() < __rhs.year() : __lhs.month() < __rhs.month(); } + +inline constexpr +bool operator> (const year_month& __lhs, const year_month& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const year_month& __lhs, const year_month& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const year_month& __lhs, const year_month& __rhs) noexcept +{ return !(__lhs < __rhs); } + +constexpr year_month operator+(const year_month& __lhs, const months& __rhs) noexcept +{ + int __dmi = static_cast(static_cast(__lhs.month())) - 1 + __rhs.count(); + const int __dy = (__dmi >= 0 ? __dmi : __dmi-11) / 12; + __dmi = __dmi - __dy * 12 + 1; + return (__lhs.year() + years(__dy)) / month(static_cast(__dmi)); +} + +constexpr year_month operator+(const months& __lhs, const year_month& __rhs) noexcept +{ return __rhs + __lhs; } + +constexpr year_month operator+(const year_month& __lhs, const years& __rhs) noexcept +{ return (__lhs.year() + __rhs) / __lhs.month(); } + +constexpr year_month operator+(const years& __lhs, const year_month& __rhs) noexcept +{ return __rhs + __lhs; } + +constexpr months operator-(const year_month& __lhs, const year_month& __rhs) noexcept +{ return (__lhs.year() - __rhs.year()) + months(static_cast(__lhs.month()) - static_cast(__rhs.month())); } + +constexpr year_month operator-(const year_month& __lhs, const months& __rhs) noexcept +{ return __lhs + -__rhs; } + +constexpr year_month operator-(const year_month& __lhs, const years& __rhs) noexcept +{ return __lhs + -__rhs; } + +class year_month_day_last; + +class _LIBCPP_TYPE_VIS year_month_day { +private: + chrono::year __y; + chrono::month __m; + chrono::day __d; +public: + year_month_day() = default; + inline constexpr year_month_day( + const chrono::year& __yval, const chrono::month& __mval, const chrono::day& __dval) noexcept + : __y{__yval}, __m{__mval}, __d{__dval} {} + constexpr year_month_day(const year_month_day_last& __ymdl) noexcept; + inline constexpr year_month_day(const sys_days& __sysd) noexcept + : year_month_day(__from_days(__sysd.time_since_epoch())) {} + inline explicit constexpr year_month_day(const local_days& __locd) noexcept + : year_month_day(__from_days(__locd.time_since_epoch())) {} + + constexpr year_month_day& operator+=(const months& __dm) noexcept; + constexpr year_month_day& operator-=(const months& __dm) noexcept; + constexpr year_month_day& operator+=(const years& __dy) noexcept; + constexpr year_month_day& operator-=(const years& __dy) noexcept; + + inline constexpr chrono::year year() const noexcept { return __y; } + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr chrono::day day() const noexcept { return __d; } + inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } + inline explicit constexpr operator local_days() const noexcept { return local_days{__to_days()}; } + + constexpr bool ok() const noexcept; + + static constexpr year_month_day __from_days(days __d) noexcept; + constexpr days __to_days() const noexcept; +}; + + +// https://howardhinnant.github.io/date_algorithms.html#civil_from_days +inline constexpr +year_month_day +year_month_day::__from_days(days __d) noexcept +{ + static_assert(std::numeric_limits::digits >= 18, ""); + static_assert(std::numeric_limits::digits >= 20 , ""); + const int __z = __d.count() + 719468; + const int __era = (__z >= 0 ? __z : __z - 146096) / 146097; + const unsigned __doe = static_cast(__z - __era * 146097); // [0, 146096] + const unsigned __yoe = (__doe - __doe/1460 + __doe/36524 - __doe/146096) / 365; // [0, 399] + const int __yr = static_cast(__yoe) + __era * 400; + const unsigned __doy = __doe - (365 * __yoe + __yoe/4 - __yoe/100); // [0, 365] + const unsigned __mp = (5 * __doy + 2)/153; // [0, 11] + const unsigned __dy = __doy - (153 * __mp + 2)/5 + 1; // [1, 31] + const unsigned __mth = __mp + (__mp < 10 ? 3 : -9); // [1, 12] + return year_month_day{chrono::year{__yr + (__mth <= 2)}, chrono::month{__mth}, chrono::day{__dy}}; +} + +// https://howardhinnant.github.io/date_algorithms.html#days_from_civil +inline constexpr days year_month_day::__to_days() const noexcept +{ + static_assert(std::numeric_limits::digits >= 18, ""); + static_assert(std::numeric_limits::digits >= 20 , ""); + + const int __yr = static_cast(__y) - (__m <= February); + const unsigned __mth = static_cast(__m); + const unsigned __dy = static_cast(__d); + + const int __era = (__yr >= 0 ? __yr : __yr - 399) / 400; + const unsigned __yoe = static_cast(__yr - __era * 400); // [0, 399] + const unsigned __doy = (153 * (__mth + (__mth > 2 ? -3 : 9)) + 2) / 5 + __dy-1; // [0, 365] + const unsigned __doe = __yoe * 365 + __yoe/4 - __yoe/100 + __doy; // [0, 146096] + return days{__era * 146097 + static_cast(__doe) - 719468}; +} + +inline constexpr +bool operator==(const year_month_day& __lhs, const year_month_day& __rhs) noexcept +{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); } + +inline constexpr +bool operator!=(const year_month_day& __lhs, const year_month_day& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const year_month_day& __lhs, const year_month_day& __rhs) noexcept +{ + if (__lhs.year() < __rhs.year()) return true; + if (__lhs.year() > __rhs.year()) return false; + if (__lhs.month() < __rhs.month()) return true; + if (__lhs.month() > __rhs.month()) return false; + return __lhs.day() < __rhs.day(); +} + +inline constexpr +bool operator> (const year_month_day& __lhs, const year_month_day& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const year_month_day& __lhs, const year_month_day& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const year_month_day& __lhs, const year_month_day& __rhs) noexcept +{ return !(__lhs < __rhs); } + +inline constexpr +year_month_day operator/(const year_month& __lhs, const day& __rhs) noexcept +{ return year_month_day{__lhs.year(), __lhs.month(), __rhs}; } + +inline constexpr +year_month_day operator/(const year_month& __lhs, int __rhs) noexcept +{ return __lhs / day(__rhs); } + +inline constexpr +year_month_day operator/(const year& __lhs, const month_day& __rhs) noexcept +{ return __lhs / __rhs.month() / __rhs.day(); } + +inline constexpr +year_month_day operator/(int __lhs, const month_day& __rhs) noexcept +{ return year(__lhs) / __rhs; } + +inline constexpr +year_month_day operator/(const month_day& __lhs, const year& __rhs) noexcept +{ return __rhs / __lhs; } + +inline constexpr +year_month_day operator/(const month_day& __lhs, int __rhs) noexcept +{ return year(__rhs) / __lhs; } + + +inline constexpr +year_month_day operator+(const year_month_day& __lhs, const months& __rhs) noexcept +{ return (__lhs.year()/__lhs.month() + __rhs)/__lhs.day(); } + +inline constexpr +year_month_day operator+(const months& __lhs, const year_month_day& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_day operator-(const year_month_day& __lhs, const months& __rhs) noexcept +{ return __lhs + -__rhs; } + +inline constexpr +year_month_day operator+(const year_month_day& __lhs, const years& __rhs) noexcept +{ return (__lhs.year() + __rhs) / __lhs.month() / __lhs.day(); } + +inline constexpr +year_month_day operator+(const years& __lhs, const year_month_day& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_day operator-(const year_month_day& __lhs, const years& __rhs) noexcept +{ return __lhs + -__rhs; } + +inline constexpr year_month_day& year_month_day::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } +inline constexpr year_month_day& year_month_day::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } +inline constexpr year_month_day& year_month_day::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } +inline constexpr year_month_day& year_month_day::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } + +class _LIBCPP_TYPE_VIS year_month_day_last { +private: + chrono::year __y; + chrono::month_day_last __mdl; +public: + constexpr year_month_day_last(const year& __yval, const month_day_last& __mdlval) noexcept + : __y{__yval}, __mdl{__mdlval} {} + + constexpr year_month_day_last& operator+=(const months& __m) noexcept; + constexpr year_month_day_last& operator-=(const months& __m) noexcept; + constexpr year_month_day_last& operator+=(const years& __y) noexcept; + constexpr year_month_day_last& operator-=(const years& __y) noexcept; + + inline constexpr chrono::year year() const noexcept { return __y; } + inline constexpr chrono::month month() const noexcept { return __mdl.month(); } + inline constexpr chrono::month_day_last month_day_last() const noexcept { return __mdl; } + constexpr chrono::day day() const noexcept; + inline constexpr operator sys_days() const noexcept { return sys_days{year()/month()/day()}; } + inline explicit constexpr operator local_days() const noexcept { return local_days{year()/month()/day()}; } + inline constexpr bool ok() const noexcept { return __y.ok() && __mdl.ok(); } +}; + +inline constexpr +chrono::day year_month_day_last::day() const noexcept +{ + constexpr chrono::day __d[] = + { + chrono::day(31), chrono::day(28), chrono::day(31), + chrono::day(30), chrono::day(31), chrono::day(30), + chrono::day(31), chrono::day(31), chrono::day(30), + chrono::day(31), chrono::day(30), chrono::day(31) + }; + return month() != February || !__y.is_leap() ? + __d[static_cast(month()) - 1] : chrono::day{29}; +} + +inline constexpr +bool operator==(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept +{ return __lhs.year() == __rhs.year() && __lhs.month_day_last() == __rhs.month_day_last(); } + +inline constexpr +bool operator!=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +bool operator< (const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept +{ + if (__lhs.year() < __rhs.year()) return true; + if (__lhs.year() > __rhs.year()) return false; + return __lhs.month_day_last() < __rhs.month_day_last(); +} + +inline constexpr +bool operator> (const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept +{ return __rhs < __lhs; } + +inline constexpr +bool operator<=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept +{ return !(__rhs < __lhs);} + +inline constexpr +bool operator>=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept +{ return !(__lhs < __rhs); } + +inline constexpr year_month_day_last operator/(const year_month& __lhs, last_spec) noexcept +{ return year_month_day_last{__lhs.year(), month_day_last{__lhs.month()}}; } + +inline constexpr year_month_day_last operator/(const year& __lhs, const month_day_last& __rhs) noexcept +{ return year_month_day_last{__lhs, __rhs}; } + +inline constexpr year_month_day_last operator/(int __lhs, const month_day_last& __rhs) noexcept +{ return year_month_day_last{year{__lhs}, __rhs}; } + +inline constexpr year_month_day_last operator/(const month_day_last& __lhs, const year& __rhs) noexcept +{ return __rhs / __lhs; } + +inline constexpr year_month_day_last operator/(const month_day_last& __lhs, int __rhs) noexcept +{ return year{__rhs} / __lhs; } + + +inline constexpr +year_month_day_last operator+(const year_month_day_last& __lhs, const months& __rhs) noexcept +{ return (__lhs.year() / __lhs.month() + __rhs) / last; } + +inline constexpr +year_month_day_last operator+(const months& __lhs, const year_month_day_last& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_day_last operator-(const year_month_day_last& __lhs, const months& __rhs) noexcept +{ return __lhs + (-__rhs); } + +inline constexpr +year_month_day_last operator+(const year_month_day_last& __lhs, const years& __rhs) noexcept +{ return year_month_day_last{__lhs.year() + __rhs, __lhs.month_day_last()}; } + +inline constexpr +year_month_day_last operator+(const years& __lhs, const year_month_day_last& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_day_last operator-(const year_month_day_last& __lhs, const years& __rhs) noexcept +{ return __lhs + (-__rhs); } + +inline constexpr year_month_day_last& year_month_day_last::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } +inline constexpr year_month_day_last& year_month_day_last::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } +inline constexpr year_month_day_last& year_month_day_last::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } +inline constexpr year_month_day_last& year_month_day_last::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } + +inline constexpr year_month_day::year_month_day(const year_month_day_last& __ymdl) noexcept + : __y{__ymdl.year()}, __m{__ymdl.month()}, __d{__ymdl.day()} {} + +inline constexpr bool year_month_day::ok() const noexcept +{ + if (!__y.ok() || !__m.ok()) return false; + return chrono::day{1} <= __d && __d <= (__y / __m / last).day(); +} + +class _LIBCPP_TYPE_VIS year_month_weekday { + chrono::year __y; + chrono::month __m; + chrono::weekday_indexed __wdi; +public: + year_month_weekday() = default; + constexpr year_month_weekday(const chrono::year& __yval, const chrono::month& __mval, + const chrono::weekday_indexed& __wdival) noexcept + : __y{__yval}, __m{__mval}, __wdi{__wdival} {} + constexpr year_month_weekday(const sys_days& __sysd) noexcept + : year_month_weekday(__from_days(__sysd.time_since_epoch())) {} + inline explicit constexpr year_month_weekday(const local_days& __locd) noexcept + : year_month_weekday(__from_days(__locd.time_since_epoch())) {} + constexpr year_month_weekday& operator+=(const months& m) noexcept; + constexpr year_month_weekday& operator-=(const months& m) noexcept; + constexpr year_month_weekday& operator+=(const years& y) noexcept; + constexpr year_month_weekday& operator-=(const years& y) noexcept; + + inline constexpr chrono::year year() const noexcept { return __y; } + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr chrono::weekday weekday() const noexcept { return __wdi.weekday(); } + inline constexpr unsigned index() const noexcept { return __wdi.index(); } + inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi; } + + inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } + inline explicit constexpr operator local_days() const noexcept { return local_days{__to_days()}; } + inline constexpr bool ok() const noexcept + { + if (!__y.ok() || !__m.ok() || !__wdi.ok()) return false; + // TODO: make sure it's a valid date + return true; + } + + static constexpr year_month_weekday __from_days(days __d) noexcept; + constexpr days __to_days() const noexcept; +}; + +inline constexpr +year_month_weekday year_month_weekday::__from_days(days __d) noexcept +{ + const sys_days __sysd{__d}; + const chrono::weekday __wd = chrono::weekday(__sysd); + const year_month_day __ymd = year_month_day(__sysd); + return year_month_weekday{__ymd.year(), __ymd.month(), + __wd[(static_cast(__ymd.day())-1)/7+1]}; +} + +inline constexpr +days year_month_weekday::__to_days() const noexcept +{ + const sys_days __sysd = sys_days(__y/__m/1); + return (__sysd + (__wdi.weekday() - chrono::weekday(__sysd) + days{(__wdi.index()-1)*7})) + .time_since_epoch(); +} + +inline constexpr +bool operator==(const year_month_weekday& __lhs, const year_month_weekday& __rhs) noexcept +{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.weekday_indexed() == __rhs.weekday_indexed(); } + +inline constexpr +bool operator!=(const year_month_weekday& __lhs, const year_month_weekday& __rhs) noexcept +{ return !(__lhs == __rhs); } + +inline constexpr +year_month_weekday operator/(const year_month& __lhs, const weekday_indexed& __rhs) noexcept +{ return year_month_weekday{__lhs.year(), __lhs.month(), __rhs}; } + +inline constexpr +year_month_weekday operator/(const year& __lhs, const month_weekday& __rhs) noexcept +{ return year_month_weekday{__lhs, __rhs.month(), __rhs.weekday_indexed()}; } + +inline constexpr +year_month_weekday operator/(int __lhs, const month_weekday& __rhs) noexcept +{ return year(__lhs) / __rhs; } + +inline constexpr +year_month_weekday operator/(const month_weekday& __lhs, const year& __rhs) noexcept +{ return __rhs / __lhs; } + +inline constexpr +year_month_weekday operator/(const month_weekday& __lhs, int __rhs) noexcept +{ return year(__rhs) / __lhs; } + + +inline constexpr +year_month_weekday operator+(const year_month_weekday& __lhs, const months& __rhs) noexcept +{ return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_indexed(); } + +inline constexpr +year_month_weekday operator+(const months& __lhs, const year_month_weekday& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_weekday operator-(const year_month_weekday& __lhs, const months& __rhs) noexcept +{ return __lhs + (-__rhs); } + +inline constexpr +year_month_weekday operator+(const year_month_weekday& __lhs, const years& __rhs) noexcept +{ return year_month_weekday{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_indexed()}; } + +inline constexpr +year_month_weekday operator+(const years& __lhs, const year_month_weekday& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_weekday operator-(const year_month_weekday& __lhs, const years& __rhs) noexcept +{ return __lhs + (-__rhs); } + + +inline constexpr year_month_weekday& year_month_weekday::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } +inline constexpr year_month_weekday& year_month_weekday::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } +inline constexpr year_month_weekday& year_month_weekday::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } +inline constexpr year_month_weekday& year_month_weekday::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } + +class _LIBCPP_TYPE_VIS year_month_weekday_last { +private: + chrono::year __y; + chrono::month __m; + chrono::weekday_last __wdl; +public: + constexpr year_month_weekday_last(const chrono::year& __yval, const chrono::month& __mval, + const chrono::weekday_last& __wdlval) noexcept + : __y{__yval}, __m{__mval}, __wdl{__wdlval} {} + constexpr year_month_weekday_last& operator+=(const months& __dm) noexcept; + constexpr year_month_weekday_last& operator-=(const months& __dm) noexcept; + constexpr year_month_weekday_last& operator+=(const years& __dy) noexcept; + constexpr year_month_weekday_last& operator-=(const years& __dy) noexcept; + + inline constexpr chrono::year year() const noexcept { return __y; } + inline constexpr chrono::month month() const noexcept { return __m; } + inline constexpr chrono::weekday weekday() const noexcept { return __wdl.weekday(); } + inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl; } + inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } + inline explicit constexpr operator local_days() const noexcept { return local_days{__to_days()}; } + inline constexpr bool ok() const noexcept { return __y.ok() && __m.ok() && __wdl.ok(); } + + constexpr days __to_days() const noexcept; + +}; + +inline constexpr +days year_month_weekday_last::__to_days() const noexcept +{ + const sys_days __last = sys_days{__y/__m/last}; + return (__last - (chrono::weekday{__last} - __wdl.weekday())).time_since_epoch(); + +} + +inline constexpr +bool operator==(const year_month_weekday_last& __lhs, const year_month_weekday_last& __rhs) noexcept +{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last(); } + +inline constexpr +bool operator!=(const year_month_weekday_last& __lhs, const year_month_weekday_last& __rhs) noexcept +{ return !(__lhs == __rhs); } + + +inline constexpr +year_month_weekday_last operator/(const year_month& __lhs, const weekday_last& __rhs) noexcept +{ return year_month_weekday_last{__lhs.year(), __lhs.month(), __rhs}; } + +inline constexpr +year_month_weekday_last operator/(const year& __lhs, const month_weekday_last& __rhs) noexcept +{ return year_month_weekday_last{__lhs, __rhs.month(), __rhs.weekday_last()}; } + +inline constexpr +year_month_weekday_last operator/(int __lhs, const month_weekday_last& __rhs) noexcept +{ return year(__lhs) / __rhs; } + +inline constexpr +year_month_weekday_last operator/(const month_weekday_last& __lhs, const year& __rhs) noexcept +{ return __rhs / __lhs; } + +inline constexpr +year_month_weekday_last operator/(const month_weekday_last& __lhs, int __rhs) noexcept +{ return year(__rhs) / __lhs; } + + +inline constexpr +year_month_weekday_last operator+(const year_month_weekday_last& __lhs, const months& __rhs) noexcept +{ return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_last(); } + +inline constexpr +year_month_weekday_last operator+(const months& __lhs, const year_month_weekday_last& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_weekday_last operator-(const year_month_weekday_last& __lhs, const months& __rhs) noexcept +{ return __lhs + (-__rhs); } + +inline constexpr +year_month_weekday_last operator+(const year_month_weekday_last& __lhs, const years& __rhs) noexcept +{ return year_month_weekday_last{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_last()}; } + +inline constexpr +year_month_weekday_last operator+(const years& __lhs, const year_month_weekday_last& __rhs) noexcept +{ return __rhs + __lhs; } + +inline constexpr +year_month_weekday_last operator-(const year_month_weekday_last& __lhs, const years& __rhs) noexcept +{ return __lhs + (-__rhs); } + +inline constexpr year_month_weekday_last& year_month_weekday_last::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } +inline constexpr year_month_weekday_last& year_month_weekday_last::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } +inline constexpr year_month_weekday_last& year_month_weekday_last::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } +inline constexpr year_month_weekday_last& year_month_weekday_last::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } + +#endif // _LIBCPP_STD_VER > 17 } // chrono #if _LIBCPP_STD_VER > 11 // Suffixes for duration literals [time.duration.literals] inline namespace literals -{ +{ inline namespace chrono_literals { @@ -1633,7 +2793,7 @@ inline namespace literals { return chrono::duration (__us); } - + constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) { @@ -1645,6 +2805,17 @@ inline namespace literals return chrono::duration (__ns); } +#if _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_HAS_NO_CXX20_CHRONO_LITERALS) + constexpr chrono::day operator ""d(unsigned long long __d) noexcept + { + return chrono::day(static_cast(__d)); + } + + constexpr chrono::year operator ""y(unsigned long long __y) noexcept + { + return chrono::year(static_cast(__y)); + } +#endif }} namespace chrono { // hoist the literals into namespace std::chrono @@ -1655,6 +2826,40 @@ namespace chrono { // hoist the literals into namespace std::chrono _LIBCPP_END_NAMESPACE_STD +#ifndef _LIBCPP_CXX03_LANG +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM +struct _FilesystemClock { +#if !defined(_LIBCPP_HAS_NO_INT128) + typedef __int128_t rep; + typedef nano period; +#else + typedef long long rep; + typedef nano period; +#endif + + typedef chrono::duration duration; + typedef chrono::time_point<_FilesystemClock> time_point; + + static _LIBCPP_CONSTEXPR_AFTER_CXX11 const bool is_steady = false; + + _LIBCPP_FUNC_VIS static time_point now() noexcept; + + _LIBCPP_INLINE_VISIBILITY + static time_t to_time_t(const time_point& __t) noexcept { + typedef chrono::duration __secs; + return time_t( + chrono::duration_cast<__secs>(__t.time_since_epoch()).count()); + } + + _LIBCPP_INLINE_VISIBILITY + static time_point from_time_t(time_t __t) noexcept { + typedef chrono::duration __secs; + return time_point(__secs(__t)); + } +}; +_LIBCPP_END_NAMESPACE_FILESYSTEM +#endif // !_LIBCPP_CXX03_LANG + _LIBCPP_POP_MACROS #endif // _LIBCPP_CHRONO diff --git a/contrib/libc++/include/cmath b/contrib/libc++/include/cmath index ffb1c46c7b6..f5f62adcfb8 100644 --- a/contrib/libc++/include/cmath +++ b/contrib/libc++/include/cmath @@ -303,6 +303,7 @@ long double truncl(long double x); #include <__config> #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header diff --git a/contrib/libc++/include/complex b/contrib/libc++/include/complex index d692ee31920..8cf6a946d71 100644 --- a/contrib/libc++/include/complex +++ b/contrib/libc++/include/complex @@ -245,6 +245,7 @@ template #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header diff --git a/contrib/libc++/include/cstddef b/contrib/libc++/include/cstddef index adeefdac9be..b4c42b19ddb 100644 --- a/contrib/libc++/include/cstddef +++ b/contrib/libc++/include/cstddef @@ -35,6 +35,7 @@ Types: */ #include <__config> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -50,7 +51,7 @@ using ::ptrdiff_t; using ::size_t; #if defined(__CLANG_MAX_ALIGN_T_DEFINED) || defined(_GCC_MAX_ALIGN_T) || \ - defined(__DEFINED_max_align_t) + defined(__DEFINED_max_align_t) || defined(__NetBSD__) // Re-use the compiler's max_align_t where possible. using ::max_align_t; #else @@ -66,10 +67,10 @@ enum class byte : unsigned char {}; constexpr byte operator| (byte __lhs, byte __rhs) noexcept { - return static_cast( - static_cast( - static_cast(__lhs) | static_cast(__rhs) - )); + return static_cast( + static_cast( + static_cast(__lhs) | static_cast(__rhs) + )); } constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept @@ -77,10 +78,10 @@ constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept constexpr byte operator& (byte __lhs, byte __rhs) noexcept { - return static_cast( - static_cast( - static_cast(__lhs) & static_cast(__rhs) - )); + return static_cast( + static_cast( + static_cast(__lhs) & static_cast(__rhs) + )); } constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept @@ -88,13 +89,13 @@ constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept constexpr byte operator^ (byte __lhs, byte __rhs) noexcept { - return static_cast( - static_cast( - static_cast(__lhs) ^ static_cast(__rhs) - )); + return static_cast( + static_cast( + static_cast(__lhs) ^ static_cast(__rhs) + )); } -constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept +constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept { return __lhs = __lhs ^ __rhs; } constexpr byte operator~ (byte __b) noexcept diff --git a/contrib/libc++/include/deque b/contrib/libc++/include/deque index bfbd3a5ef54..6f7d04be52b 100644 --- a/contrib/libc++/include/deque +++ b/contrib/libc++/include/deque @@ -150,6 +150,11 @@ template void swap(deque& x, deque& y) noexcept(noexcept(x.swap(y))); +template + void erase(deque& c, const U& value); // C++20 +template + void erase_if(deque& c, Predicate pred); // C++20 + } // std */ @@ -161,6 +166,7 @@ template #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -986,7 +992,7 @@ public: #if _LIBCPP_STD_VER >= 14 _NOEXCEPT; #else - _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || + _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable::value); #endif protected: @@ -1155,7 +1161,7 @@ __deque_base<_Tp, _Allocator>::swap(__deque_base& __c) #if _LIBCPP_STD_VER >= 14 _NOEXCEPT #else - _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || + _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable::value) #endif { @@ -2341,7 +2347,7 @@ deque<_Tp, _Allocator>::__add_front_capacity() _Dp(__a, __base::__block_size)); __buf.push_back(__hold.get()); __hold.release(); - + for (typename __base::__map_pointer __i = __base::__map_.begin(); __i != __base::__map_.end(); ++__i) __buf.push_back(*__i); @@ -2603,6 +2609,7 @@ template void deque<_Tp, _Allocator>::pop_back() { + _LIBCPP_ASSERT(!empty(), "deque::pop_back called for empty deque"); allocator_type& __a = __base::__alloc(); size_type __p = __base::size() + __base::__start_ - 1; __alloc_traits::destroy(__a, __to_raw_pointer(*(__base::__map_.begin() + @@ -2853,7 +2860,7 @@ deque<_Tp, _Allocator>::swap(deque& __c) #if _LIBCPP_STD_VER >= 14 _NOEXCEPT #else - _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || + _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable::value) #endif { @@ -2926,6 +2933,19 @@ swap(deque<_Tp, _Allocator>& __x, deque<_Tp, _Allocator>& __y) __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase(deque<_Tp, _Allocator>& __c, const _Up& __v) +{ __c.erase(_VSTD::remove(__c.begin(), __c.end(), __v), __c.end()); } + +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(deque<_Tp, _Allocator>& __c, _Predicate __pred) +{ __c.erase(_VSTD::remove_if(__c.begin(), __c.end(), __pred), __c.end()); } +#endif + + _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS diff --git a/contrib/libc++/include/exception b/contrib/libc++/include/exception index b517486b5a8..fdd83d10c3d 100644 --- a/contrib/libc++/include/exception +++ b/contrib/libc++/include/exception @@ -81,6 +81,7 @@ template void rethrow_if_nested(const E& e); #include #include #include +#include #if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_NO_VCRUNTIME) #include @@ -163,7 +164,7 @@ public: }; template -exception_ptr +_LIBCPP_INLINE_VISIBILITY exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { #ifndef _LIBCPP_NO_EXCEPTIONS @@ -222,7 +223,7 @@ _LIBCPP_NORETURN _LIBCPP_FUNC_VIS void rethrow_exception(exception_ptr p); template void *__GetExceptionInfo(_E); template -exception_ptr +_LIBCPP_INLINE_VISIBILITY exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { return __copy_exception_ptr(_VSTD::addressof(__e), __GetExceptionInfo(__e)); diff --git a/contrib/libc++/include/experimental/any b/contrib/libc++/include/experimental/any index 1dcdd0f25ec..d9c95342589 100644 --- a/contrib/libc++/include/experimental/any +++ b/contrib/libc++/include/experimental/any @@ -1,11 +1,21 @@ // -*- C++ -*- -//===------------------------------ any -----------------------------------===// +//===------------------------------- any ----------------------------------===// // // The LLVM Compiler Infrastructure // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_ANY +#define _LIBCPP_EXPERIMENTAL_ANY -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_ANY diff --git a/contrib/libc++/include/experimental/chrono b/contrib/libc++/include/experimental/chrono index 591cf7160c1..30c7e4a9d5a 100644 --- a/contrib/libc++/include/experimental/chrono +++ b/contrib/libc++/include/experimental/chrono @@ -1,11 +1,21 @@ // -*- C++ -*- -//===------------------------------ chrono ---------------------------------===// +//===---------------------------- chrono ----------------------------------===// // // The LLVM Compiler Infrastructure // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_CHRONO +#define _LIBCPP_EXPERIMENTAL_CHRONO -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_CHRONO diff --git a/contrib/libc++/include/experimental/coroutine b/contrib/libc++/include/experimental/coroutine index 1eb224a535a..7cb39b81b48 100644 --- a/contrib/libc++/include/experimental/coroutine +++ b/contrib/libc++/include/experimental/coroutine @@ -214,7 +214,7 @@ public: _LIBCPP_INLINE_VISIBILITY _Promise& promise() const { return *static_cast<_Promise*>( - __builtin_coro_promise(this->__handle_, __alignof(_Promise), false)); + __builtin_coro_promise(this->__handle_, _LIBCPP_ALIGNOF(_Promise), false)); } public: @@ -254,7 +254,7 @@ public: coroutine_handle __tmp; __tmp.__handle_ = __builtin_coro_promise( _VSTD::addressof(const_cast<_RawPromise&>(__promise)), - __alignof(_Promise), true); + _LIBCPP_ALIGNOF(_Promise), true); return __tmp; } }; @@ -272,7 +272,7 @@ public: _LIBCPP_INLINE_VISIBILITY _Promise& promise() const { return *static_cast<_Promise*>( - __builtin_coro_promise(this->__handle_, __alignof(_Promise), false)); + __builtin_coro_promise(this->__handle_, _LIBCPP_ALIGNOF(_Promise), false)); } _LIBCPP_CONSTEXPR explicit operator bool() const _NOEXCEPT { return true; } diff --git a/contrib/libc++/include/experimental/dynarray b/contrib/libc++/include/experimental/dynarray deleted file mode 100644 index a60c87c3f97..00000000000 --- a/contrib/libc++/include/experimental/dynarray +++ /dev/null @@ -1,305 +0,0 @@ -// -*- C++ -*- -//===-------------------------- dynarray ----------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP_DYNARRAY -#define _LIBCPP_DYNARRAY - -/* - dynarray synopsis - -namespace std { namespace experimental { - -template< typename T > -class dynarray -{ - // types: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef T* pointer; - typedef const T* const_pointer; - typedef implementation-defined iterator; - typedef implementation-defined const_iterator; - typedef reverse_iterator reverse_iterator; - typedef reverse_iterator const_reverse_iterator; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - -public: - // construct/copy/destroy: - explicit dynarray(size_type c); - dynarray(size_type c, const T& v); - dynarray(const dynarray& d); - dynarray(initializer_list); - - template - dynarray(allocator_arg_t, const Alloc& a, size_type c, const Alloc& alloc); - template - dynarray(allocator_arg_t, const Alloc& a, size_type c, const T& v, const Alloc& alloc); - template - dynarray(allocator_arg_t, const Alloc& a, const dynarray& d, const Alloc& alloc); - template - dynarray(allocator_arg_t, const Alloc& a, initializer_list, const Alloc& alloc); - dynarray& operator=(const dynarray&) = delete; - ~dynarray(); - - // iterators: - iterator begin() noexcept; - const_iterator begin() const noexcept; - const_iterator cbegin() const noexcept; - iterator end() noexcept; - const_iterator end() const noexcept; - const_iterator cend() const noexcept; - - reverse_iterator rbegin() noexcept; - const_reverse_iterator rbegin() const noexcept; - const_reverse_iterator crbegin() const noexcept; - reverse_iterator rend() noexcept; - const_reverse_iterator rend() const noexcept; - const_reverse_iterator crend() const noexcept; - - // capacity: - size_type size() const noexcept; - size_type max_size() const noexcept; - bool empty() const noexcept; - - // element access: - reference operator[](size_type n); - const_reference operator[](size_type n) const; - - reference front(); - const_reference front() const; - reference back(); - const_reference back() const; - - const_reference at(size_type n) const; - reference at(size_type n); - - // data access: - T* data() noexcept; - const T* data() const noexcept; - - // mutating member functions: - void fill(const T& v); -}; - -}} // std::experimental - -*/ -#include <__config> -#if _LIBCPP_STD_VER > 11 - -#include <__functional_base> -#include -#include -#include -#include -#include - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -#pragma GCC system_header -#endif - -_LIBCPP_PUSH_MACROS -#include <__undef_macros> - -namespace std { namespace experimental { inline namespace __array_extensions_v1 { - -template -struct _LIBCPP_TEMPLATE_VIS _LIBCPP_AVAILABILITY_DYNARRAY dynarray -{ -public: - // types: - typedef dynarray __self; - typedef _Tp value_type; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef value_type* iterator; - typedef const value_type* const_iterator; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; - -private: - size_t __size_; - value_type * __base_; - _LIBCPP_INLINE_VISIBILITY dynarray () noexcept : __size_(0), __base_(nullptr) {} - - static inline _LIBCPP_INLINE_VISIBILITY - value_type* __allocate(size_t __count) { - if (numeric_limits::max() / sizeof (value_type) <= __count) - __throw_bad_array_length(); - - return static_cast( - _VSTD::__libcpp_allocate(sizeof(value_type) * __count, __alignof(value_type))); - } - - static inline _LIBCPP_INLINE_VISIBILITY - void __deallocate_value(value_type* __ptr ) noexcept { - _VSTD::__libcpp_deallocate(static_cast(__ptr), __alignof(value_type)); - } - -public: - - _LIBCPP_INLINE_VISIBILITY - explicit dynarray(size_type __c); - _LIBCPP_INLINE_VISIBILITY - dynarray(size_type __c, const value_type& __v); - _LIBCPP_INLINE_VISIBILITY - dynarray(const dynarray& __d); - _LIBCPP_INLINE_VISIBILITY - dynarray(initializer_list); - -// We're not implementing these right now. -// Updated with the resolution of LWG issue #2255 -// template -// dynarray(allocator_arg_t, const _Alloc& __alloc, size_type __c); -// template -// dynarray(allocator_arg_t, const _Alloc& __alloc, size_type __c, const value_type& __v); -// template -// dynarray(allocator_arg_t, const _Alloc& __alloc, const dynarray& __d); -// template -// dynarray(allocator_arg_t, const _Alloc& __alloc, initializer_list); - - dynarray& operator=(const dynarray&) = delete; - _LIBCPP_INLINE_VISIBILITY - ~dynarray(); - - // iterators: - inline _LIBCPP_INLINE_VISIBILITY iterator begin() noexcept { return iterator(data()); } - inline _LIBCPP_INLINE_VISIBILITY const_iterator begin() const noexcept { return const_iterator(data()); } - inline _LIBCPP_INLINE_VISIBILITY const_iterator cbegin() const noexcept { return const_iterator(data()); } - inline _LIBCPP_INLINE_VISIBILITY iterator end() noexcept { return iterator(data() + __size_); } - inline _LIBCPP_INLINE_VISIBILITY const_iterator end() const noexcept { return const_iterator(data() + __size_); } - inline _LIBCPP_INLINE_VISIBILITY const_iterator cend() const noexcept { return const_iterator(data() + __size_); } - - inline _LIBCPP_INLINE_VISIBILITY reverse_iterator rbegin() noexcept { return reverse_iterator(end()); } - inline _LIBCPP_INLINE_VISIBILITY const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); } - inline _LIBCPP_INLINE_VISIBILITY const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); } - inline _LIBCPP_INLINE_VISIBILITY reverse_iterator rend() noexcept { return reverse_iterator(begin()); } - inline _LIBCPP_INLINE_VISIBILITY const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); } - inline _LIBCPP_INLINE_VISIBILITY const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); } - - // capacity: - inline _LIBCPP_INLINE_VISIBILITY size_type size() const noexcept { return __size_; } - inline _LIBCPP_INLINE_VISIBILITY size_type max_size() const noexcept { return __size_; } - inline _LIBCPP_INLINE_VISIBILITY bool empty() const noexcept { return __size_ == 0; } - - // element access: - inline _LIBCPP_INLINE_VISIBILITY reference operator[](size_type __n) { return data()[__n]; } - inline _LIBCPP_INLINE_VISIBILITY const_reference operator[](size_type __n) const { return data()[__n]; } - - inline _LIBCPP_INLINE_VISIBILITY reference front() { return data()[0]; } - inline _LIBCPP_INLINE_VISIBILITY const_reference front() const { return data()[0]; } - inline _LIBCPP_INLINE_VISIBILITY reference back() { return data()[__size_-1]; } - inline _LIBCPP_INLINE_VISIBILITY const_reference back() const { return data()[__size_-1]; } - - inline _LIBCPP_INLINE_VISIBILITY const_reference at(size_type __n) const; - inline _LIBCPP_INLINE_VISIBILITY reference at(size_type __n); - - // data access: - inline _LIBCPP_INLINE_VISIBILITY _Tp* data() noexcept { return __base_; } - inline _LIBCPP_INLINE_VISIBILITY const _Tp* data() const noexcept { return __base_; } - - // mutating member functions: - inline _LIBCPP_INLINE_VISIBILITY void fill(const value_type& __v) { fill_n(begin(), __size_, __v); } -}; - -template -inline -dynarray<_Tp>::dynarray(size_type __c) : dynarray () -{ - __base_ = __allocate (__c); - value_type *__data = data (); - for ( __size_ = 0; __size_ < __c; ++__size_, ++__data ) - ::new (__data) value_type; -} - -template -inline -dynarray<_Tp>::dynarray(size_type __c, const value_type& __v) : dynarray () -{ - __base_ = __allocate (__c); - value_type *__data = data (); - for ( __size_ = 0; __size_ < __c; ++__size_, ++__data ) - ::new (__data) value_type (__v); -} - -template -inline -dynarray<_Tp>::dynarray(initializer_list __il) : dynarray () -{ - size_t sz = __il.size(); - __base_ = __allocate (sz); - value_type *__data = data (); - auto src = __il.begin(); - for ( __size_ = 0; __size_ < sz; ++__size_, ++__data, ++src ) - ::new (__data) value_type (*src); -} - -template -inline -dynarray<_Tp>::dynarray(const dynarray& __d) : dynarray () -{ - size_t sz = __d.size(); - __base_ = __allocate (sz); - value_type *__data = data (); - auto src = __d.begin(); - for ( __size_ = 0; __size_ < sz; ++__size_, ++__data, ++src ) - ::new (__data) value_type (*src); -} - -template -inline -dynarray<_Tp>::~dynarray() -{ - value_type *__data = data () + __size_; - for ( size_t i = 0; i < __size_; ++i ) - (--__data)->value_type::~value_type(); - __deallocate_value( __base_ ); -} - -template -inline _LIBCPP_INLINE_VISIBILITY -typename dynarray<_Tp>::reference -dynarray<_Tp>::at(size_type __n) -{ - if (__n >= __size_) - __throw_out_of_range("dynarray::at"); - - return data()[__n]; -} - -template -inline _LIBCPP_INLINE_VISIBILITY -typename dynarray<_Tp>::const_reference -dynarray<_Tp>::at(size_type __n) const -{ - if (__n >= __size_) - __throw_out_of_range("dynarray::at"); - - return data()[__n]; -} - -}}} - - -_LIBCPP_BEGIN_NAMESPACE_STD -template -struct _LIBCPP_TEMPLATE_VIS uses_allocator, _Alloc> : true_type {}; -_LIBCPP_END_NAMESPACE_STD - -_LIBCPP_POP_MACROS - -#endif // if _LIBCPP_STD_VER > 11 -#endif // _LIBCPP_DYNARRAY diff --git a/contrib/libc++/include/experimental/memory_resource b/contrib/libc++/include/experimental/memory_resource index 221ce5b8eac..83781d46203 100644 --- a/contrib/libc++/include/experimental/memory_resource +++ b/contrib/libc++/include/experimental/memory_resource @@ -98,7 +98,7 @@ size_t __aligned_allocation_size(size_t __s, size_t __a) _NOEXCEPT // 8.5, memory.resource class _LIBCPP_TYPE_VIS memory_resource { - static const size_t __max_align = alignof(max_align_t); + static const size_t __max_align = _LIBCPP_ALIGNOF(max_align_t); // 8.5.2, memory.resource.public public: @@ -190,7 +190,7 @@ public: " 'n' exceeds maximum supported size"); } return static_cast<_ValueType*>( - __res_->allocate(__n * sizeof(_ValueType), alignof(_ValueType)) + __res_->allocate(__n * sizeof(_ValueType), _LIBCPP_ALIGNOF(_ValueType)) ); } @@ -198,7 +198,7 @@ public: void deallocate(_ValueType * __p, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT(__n <= __max_size(), "deallocate called for size which exceeds max_size()"); - __res_->deallocate(__p, __n * sizeof(_ValueType), alignof(_ValueType)); + __res_->deallocate(__p, __n * sizeof(_ValueType), _LIBCPP_ALIGNOF(_ValueType)); } template @@ -345,7 +345,7 @@ class _LIBCPP_TEMPLATE_VIS __resource_adaptor_imp && is_same::value && is_same::value, ""); - static const size_t _MaxAlign = alignof(max_align_t); + static const size_t _MaxAlign = _LIBCPP_ALIGNOF(max_align_t); using _Alloc = typename _CTraits::template rebind_alloc< typename aligned_storage<_MaxAlign, _MaxAlign>::type diff --git a/contrib/libc++/include/experimental/numeric b/contrib/libc++/include/experimental/numeric index 14a664011b5..19c65313f0f 100644 --- a/contrib/libc++/include/experimental/numeric +++ b/contrib/libc++/include/experimental/numeric @@ -7,5 +7,15 @@ // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_NUMERIC +#define _LIBCPP_EXPERIMENTAL_NUMERIC -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_NUMERIC diff --git a/contrib/libc++/include/experimental/optional b/contrib/libc++/include/experimental/optional index d68cefdf6c1..6eb4a2618d2 100644 --- a/contrib/libc++/include/experimental/optional +++ b/contrib/libc++/include/experimental/optional @@ -7,5 +7,15 @@ // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_OPTIONAL +#define _LIBCPP_EXPERIMENTAL_OPTIONAL -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_OPTIONAL diff --git a/contrib/libc++/include/experimental/ratio b/contrib/libc++/include/experimental/ratio index 9c2bf2e4624..52c12004dba 100644 --- a/contrib/libc++/include/experimental/ratio +++ b/contrib/libc++/include/experimental/ratio @@ -1,11 +1,21 @@ // -*- C++ -*- -//===------------------------------ ratio ---------------------------------===// +//===----------------------------- ratio ----------------------------------===// // // The LLVM Compiler Infrastructure // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_RATIO +#define _LIBCPP_EXPERIMENTAL_RATIO -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_RATIO diff --git a/contrib/libc++/include/experimental/string_view b/contrib/libc++/include/experimental/string_view index f13bff54d53..100bdfe7273 100644 --- a/contrib/libc++/include/experimental/string_view +++ b/contrib/libc++/include/experimental/string_view @@ -3,9 +3,19 @@ // // The LLVM Compiler Infrastructure // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_STRING_VIEW +#define _LIBCPP_EXPERIMENTAL_STRING_VIEW -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_STRING_VIEW diff --git a/contrib/libc++/include/experimental/system_error b/contrib/libc++/include/experimental/system_error index 7937357fa14..1cf84ee0125 100644 --- a/contrib/libc++/include/experimental/system_error +++ b/contrib/libc++/include/experimental/system_error @@ -7,5 +7,15 @@ // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_SYSTEM_ERROR +#define _LIBCPP_EXPERIMENTAL_SYSTEM_ERROR -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_SYSTEM_ERROR diff --git a/contrib/libc++/include/experimental/tuple b/contrib/libc++/include/experimental/tuple index 1f37a6293ba..6d71bb559b0 100644 --- a/contrib/libc++/include/experimental/tuple +++ b/contrib/libc++/include/experimental/tuple @@ -7,5 +7,15 @@ // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef _LIBCPP_EXPERIMENTAL_TUPLE +#define _LIBCPP_EXPERIMENTAL_TUPLE -#error " has been removed. Use instead." +#include <__config> + +#ifdef _LIBCPP_WARNING +_LIBCPP_WARNING(" has been removed. Use instead.") +#else +# warning " has been removed. Use instead." +#endif + +#endif // _LIBCPP_EXPERIMENTAL_TUPLE diff --git a/contrib/libc++/include/filesystem b/contrib/libc++/include/filesystem index aa1d7180072..af713a06358 100644 --- a/contrib/libc++/include/filesystem +++ b/contrib/libc++/include/filesystem @@ -244,6 +244,7 @@ #include #include // for quoted #include +#include #include <__debug> @@ -256,42 +257,8 @@ _LIBCPP_PUSH_MACROS #ifndef _LIBCPP_CXX03_LANG -#if _LIBCPP_STD_VER >= 17 -#define __cpp_lib_filesystem 201703 -#endif - _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM -struct _FilesystemClock { -#if !defined(_LIBCPP_HAS_NO_INT128) - typedef __int128_t rep; - typedef nano period; -#else - typedef long long rep; - typedef nano period; -#endif - - typedef chrono::duration duration; - typedef chrono::time_point<_FilesystemClock> time_point; - - static _LIBCPP_CONSTEXPR_AFTER_CXX11 const bool is_steady = false; - - _LIBCPP_FUNC_VIS static time_point now() noexcept; - - _LIBCPP_INLINE_VISIBILITY - static time_t to_time_t(const time_point& __t) noexcept { - typedef chrono::duration __secs; - return time_t( - chrono::duration_cast<__secs>(__t.time_since_epoch()).count()); - } - - _LIBCPP_INLINE_VISIBILITY - static time_point from_time_t(time_t __t) noexcept { - typedef chrono::duration __secs; - return time_point(__secs(__t)); - } -}; - typedef chrono::time_point<_FilesystemClock> file_time_type; struct _LIBCPP_TYPE_VIS space_info { @@ -590,7 +557,7 @@ template typename enable_if<__can_convert_char<_ECharT>::value, bool>::type __is_separator(_ECharT __e) { return __e == _ECharT('/'); -}; +} struct _NullSentinal {}; @@ -1184,6 +1151,31 @@ public: return __is; } + friend _LIBCPP_INLINE_VISIBILITY bool operator==(const path& __lhs, const path& __rhs) noexcept { + return __lhs.compare(__rhs) == 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator!=(const path& __lhs, const path& __rhs) noexcept { + return __lhs.compare(__rhs) != 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator<(const path& __lhs, const path& __rhs) noexcept { + return __lhs.compare(__rhs) < 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator<=(const path& __lhs, const path& __rhs) noexcept { + return __lhs.compare(__rhs) <= 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator>(const path& __lhs, const path& __rhs) noexcept { + return __lhs.compare(__rhs) > 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator>=(const path& __lhs, const path& __rhs) noexcept { + return __lhs.compare(__rhs) >= 0; + } + + friend _LIBCPP_INLINE_VISIBILITY path operator/(const path& __lhs, + const path& __rhs) { + path __result(__lhs); + __result /= __rhs; + return __result; + } private: inline _LIBCPP_INLINE_VISIBILITY path& __assign_view(__string_view const& __s) noexcept { @@ -1200,43 +1192,6 @@ inline _LIBCPP_INLINE_VISIBILITY void swap(path& __lhs, path& __rhs) noexcept { _LIBCPP_FUNC_VIS size_t hash_value(const path& __p) noexcept; -inline _LIBCPP_INLINE_VISIBILITY bool operator==(const path& __lhs, - const path& __rhs) noexcept { - return __lhs.compare(__rhs) == 0; -} - -inline _LIBCPP_INLINE_VISIBILITY bool operator!=(const path& __lhs, - const path& __rhs) noexcept { - return __lhs.compare(__rhs) != 0; -} - -inline _LIBCPP_INLINE_VISIBILITY bool operator<(const path& __lhs, - const path& __rhs) noexcept { - return __lhs.compare(__rhs) < 0; -} - -inline _LIBCPP_INLINE_VISIBILITY bool operator<=(const path& __lhs, - const path& __rhs) noexcept { - return __lhs.compare(__rhs) <= 0; -} - -inline _LIBCPP_INLINE_VISIBILITY bool operator>(const path& __lhs, - const path& __rhs) noexcept { - return __lhs.compare(__rhs) > 0; -} - -inline _LIBCPP_INLINE_VISIBILITY bool operator>=(const path& __lhs, - const path& __rhs) noexcept { - return __lhs.compare(__rhs) >= 0; -} - -inline _LIBCPP_INLINE_VISIBILITY path operator/(const path& __lhs, - const path& __rhs) { - path __result(__lhs); - __result /= __rhs; - return __result; -} - template _LIBCPP_INLINE_VISIBILITY typename enable_if<__is_pathable<_Source>::value, path>::type diff --git a/contrib/libc++/include/forward_list b/contrib/libc++/include/forward_list index 571afdc925b..b506acd1ff2 100644 --- a/contrib/libc++/include/forward_list +++ b/contrib/libc++/include/forward_list @@ -167,6 +167,11 @@ template void swap(forward_list& x, forward_list& y) noexcept(noexcept(x.swap(y))); +template + void erase(forward_list& c, const U& value); // C++20 +template + void erase_if(forward_list& c, Predicate pred); // C++20 + } // std */ @@ -177,6 +182,7 @@ template #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -1743,6 +1749,18 @@ swap(forward_list<_Tp, _Alloc>& __x, forward_list<_Tp, _Alloc>& __y) __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(forward_list<_Tp, _Allocator>& __c, _Predicate __pred) +{ __c.remove_if(__pred); } + +template +inline _LIBCPP_INLINE_VISIBILITY +void erase(forward_list<_Tp, _Allocator>& __c, const _Up& __v) +{ _VSTD::erase_if(__c, [&](auto& __elem) { return __elem == __v; }); } +#endif + _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS diff --git a/contrib/libc++/include/fstream b/contrib/libc++/include/fstream index 332b4747c1a..711e484e2dc 100644 --- a/contrib/libc++/include/fstream +++ b/contrib/libc++/include/fstream @@ -702,6 +702,7 @@ basic_filebuf<_CharT, _Traits>::close() __file_ = 0; else __rt = 0; + setbuf(0, 0); } return __rt; } diff --git a/contrib/libc++/include/functional b/contrib/libc++/include/functional index 6b70f731e1c..95491879b0c 100644 --- a/contrib/libc++/include/functional +++ b/contrib/libc++/include/functional @@ -68,6 +68,11 @@ template reference_wrapper cref(const T& t) noexcept; template void cref(const T&& t) = delete; template reference_wrapper cref(reference_wrapper t) noexcept; +template struct unwrap_reference; // since C++20 +template struct unwrap_ref_decay : unwrap_reference> { }; // since C++20 +template using unwrap_reference_t = typename unwrap_reference::type; // since C++20 +template using unwrap_ref_decay_t = typename unwrap_ref_decay::type; // since C++20 + template // in C++14 struct plus : binary_function { @@ -183,7 +188,7 @@ struct bit_xor : unary_function }; template -class unary_negate +class unary_negate // deprecated in C++17 : public unary_function { public: @@ -191,10 +196,11 @@ public: bool operator()(const typename Predicate::argument_type& x) const; }; -template unary_negate not1(const Predicate& pred); +template // deprecated in C++17 +unary_negate not1(const Predicate& pred); template -class binary_negate +class binary_negate // deprecated in C++17 : public binary_function @@ -205,7 +211,8 @@ public: const typename Predicate::second_argument_type& y) const; }; -template binary_negate not2(const Predicate& pred); +template // deprecated in C++17 +binary_negate not2(const Predicate& pred); template unspecified not_fn(F&& f); // C++17 @@ -487,6 +494,7 @@ POLICY: For non-variadic implementations, the number of arguments is limited #include #include #include +#include #include <__functional_base> @@ -980,7 +988,7 @@ struct _LIBCPP_TEMPLATE_VIS bit_not #endif template -class _LIBCPP_TEMPLATE_VIS unary_negate +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 unary_negate : public unary_function { _Predicate __pred_; @@ -994,19 +1002,19 @@ public: }; template -inline _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY unary_negate<_Predicate> not1(const _Predicate& __pred) {return unary_negate<_Predicate>(__pred);} template -class _LIBCPP_TEMPLATE_VIS binary_negate +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 binary_negate : public binary_function { _Predicate __pred_; public: - _LIBCPP_INLINE_VISIBILITY explicit _LIBCPP_CONSTEXPR_AFTER_CXX11 + _LIBCPP_INLINE_VISIBILITY explicit _LIBCPP_CONSTEXPR_AFTER_CXX11 binary_negate(const _Predicate& __pred) : __pred_(__pred) {} _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY @@ -1016,13 +1024,13 @@ public: }; template -inline _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY binary_negate<_Predicate> not2(const _Predicate& __pred) {return binary_negate<_Predicate>(__pred);} #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS) template -class _LIBCPP_TEMPLATE_VIS binder1st +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binder1st : public unary_function { @@ -1042,13 +1050,13 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY binder1st<__Operation> bind1st(const __Operation& __op, const _Tp& __x) {return binder1st<__Operation>(__op, __x);} template -class _LIBCPP_TEMPLATE_VIS binder2nd +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binder2nd : public unary_function { @@ -1068,13 +1076,13 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY binder2nd<__Operation> bind2nd(const __Operation& __op, const _Tp& __x) {return binder2nd<__Operation>(__op, __x);} template -class _LIBCPP_TEMPLATE_VIS pointer_to_unary_function +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 pointer_to_unary_function : public unary_function<_Arg, _Result> { _Result (*__f_)(_Arg); @@ -1086,13 +1094,13 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY pointer_to_unary_function<_Arg,_Result> ptr_fun(_Result (*__f)(_Arg)) {return pointer_to_unary_function<_Arg,_Result>(__f);} template -class _LIBCPP_TEMPLATE_VIS pointer_to_binary_function +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 pointer_to_binary_function : public binary_function<_Arg1, _Arg2, _Result> { _Result (*__f_)(_Arg1, _Arg2); @@ -1104,13 +1112,14 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY pointer_to_binary_function<_Arg1,_Arg2,_Result> ptr_fun(_Result (*__f)(_Arg1,_Arg2)) {return pointer_to_binary_function<_Arg1,_Arg2,_Result>(__f);} template -class _LIBCPP_TEMPLATE_VIS mem_fun_t : public unary_function<_Tp*, _Sp> +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_t + : public unary_function<_Tp*, _Sp> { _Sp (_Tp::*__p_)(); public: @@ -1121,7 +1130,8 @@ public: }; template -class _LIBCPP_TEMPLATE_VIS mem_fun1_t : public binary_function<_Tp*, _Ap, _Sp> +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_t + : public binary_function<_Tp*, _Ap, _Sp> { _Sp (_Tp::*__p_)(_Ap); public: @@ -1132,19 +1142,20 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY mem_fun_t<_Sp,_Tp> mem_fun(_Sp (_Tp::*__f)()) {return mem_fun_t<_Sp,_Tp>(__f);} template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY mem_fun1_t<_Sp,_Tp,_Ap> mem_fun(_Sp (_Tp::*__f)(_Ap)) {return mem_fun1_t<_Sp,_Tp,_Ap>(__f);} template -class _LIBCPP_TEMPLATE_VIS mem_fun_ref_t : public unary_function<_Tp, _Sp> +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_ref_t + : public unary_function<_Tp, _Sp> { _Sp (_Tp::*__p_)(); public: @@ -1155,7 +1166,8 @@ public: }; template -class _LIBCPP_TEMPLATE_VIS mem_fun1_ref_t : public binary_function<_Tp, _Ap, _Sp> +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_ref_t + : public binary_function<_Tp, _Ap, _Sp> { _Sp (_Tp::*__p_)(_Ap); public: @@ -1166,19 +1178,20 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY mem_fun_ref_t<_Sp,_Tp> mem_fun_ref(_Sp (_Tp::*__f)()) {return mem_fun_ref_t<_Sp,_Tp>(__f);} template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY mem_fun1_ref_t<_Sp,_Tp,_Ap> mem_fun_ref(_Sp (_Tp::*__f)(_Ap)) {return mem_fun1_ref_t<_Sp,_Tp,_Ap>(__f);} template -class _LIBCPP_TEMPLATE_VIS const_mem_fun_t : public unary_function +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_t + : public unary_function { _Sp (_Tp::*__p_)() const; public: @@ -1189,7 +1202,8 @@ public: }; template -class _LIBCPP_TEMPLATE_VIS const_mem_fun1_t : public binary_function +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_t + : public binary_function { _Sp (_Tp::*__p_)(_Ap) const; public: @@ -1200,19 +1214,20 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY const_mem_fun_t<_Sp,_Tp> mem_fun(_Sp (_Tp::*__f)() const) {return const_mem_fun_t<_Sp,_Tp>(__f);} template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY const_mem_fun1_t<_Sp,_Tp,_Ap> mem_fun(_Sp (_Tp::*__f)(_Ap) const) {return const_mem_fun1_t<_Sp,_Tp,_Ap>(__f);} template -class _LIBCPP_TEMPLATE_VIS const_mem_fun_ref_t : public unary_function<_Tp, _Sp> +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_ref_t + : public unary_function<_Tp, _Sp> { _Sp (_Tp::*__p_)() const; public: @@ -1223,7 +1238,7 @@ public: }; template -class _LIBCPP_TEMPLATE_VIS const_mem_fun1_ref_t +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_ref_t : public binary_function<_Tp, _Ap, _Sp> { _Sp (_Tp::*__p_)(_Ap) const; @@ -1235,13 +1250,13 @@ public: }; template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY const_mem_fun_ref_t<_Sp,_Tp> mem_fun_ref(_Sp (_Tp::*__f)() const) {return const_mem_fun_ref_t<_Sp,_Tp>(__f);} template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_INLINE_VISIBILITY const_mem_fun1_ref_t<_Sp,_Tp,_Ap> mem_fun_ref(_Sp (_Tp::*__f)(_Ap) const) {return const_mem_fun1_ref_t<_Sp,_Tp,_Ap>(__f);} @@ -1405,7 +1420,7 @@ void __throw_bad_function_call() #ifndef _LIBCPP_NO_EXCEPTIONS throw bad_function_call(); #else - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -1458,6 +1473,81 @@ bool __not_null(function<_Fp> const& __f) { return !!__f; } namespace __function { +// __alloc_func holds a functor and an allocator. + +template class __alloc_func; + +template +class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> +{ + __compressed_pair<_Fp, _Ap> __f_; + + public: + typedef _Fp _Target; + typedef _Ap _Alloc; + + _LIBCPP_INLINE_VISIBILITY + const _Target& __target() const { return __f_.first(); } + + _LIBCPP_INLINE_VISIBILITY + const _Alloc& __allocator() const { return __f_.second(); } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(_Target&& __f) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), + _VSTD::forward_as_tuple()) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(const _Target& __f, const _Alloc& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), + _VSTD::forward_as_tuple(__a)) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(const _Target& __f, _Alloc&& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), + _VSTD::forward_as_tuple(_VSTD::move(__a))) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(_Target&& __f, _Alloc&& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), + _VSTD::forward_as_tuple(_VSTD::move(__a))) + { + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __arg) + { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(__f_.first(), + _VSTD::forward<_ArgTypes>(__arg)...); + } + + _LIBCPP_INLINE_VISIBILITY + __alloc_func* __clone() const + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef + typename __rebind_alloc_helper<__alloc_traits, __alloc_func>::type + _AA; + _AA __a(__f_.second()); + typedef __allocator_destructor<_AA> _Dp; + unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a)); + return __hold.release(); + } + + _LIBCPP_INLINE_VISIBILITY + void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); } +}; + +// __base provides an abstract interface for copyable functors. + template class __base; template @@ -1479,37 +1569,37 @@ public: #endif // _LIBCPP_NO_RTTI }; +// __func implements __base for a given functor type. + template class __func; template class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> { - __compressed_pair<_Fp, _Alloc> __f_; + __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_; public: _LIBCPP_INLINE_VISIBILITY explicit __func(_Fp&& __f) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), - _VSTD::forward_as_tuple()) {} + : __f_(_VSTD::move(__f)) {} + _LIBCPP_INLINE_VISIBILITY explicit __func(const _Fp& __f, const _Alloc& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), - _VSTD::forward_as_tuple(__a)) {} + : __f_(__f, __a) {} _LIBCPP_INLINE_VISIBILITY explicit __func(const _Fp& __f, _Alloc&& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), - _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + : __f_(__f, _VSTD::move(__a)) {} _LIBCPP_INLINE_VISIBILITY explicit __func(_Fp&& __f, _Alloc&& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), - _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + : __f_(_VSTD::move(__f), _VSTD::move(__a)) {} + virtual __base<_Rp(_ArgTypes...)>* __clone() const; virtual void __clone(__base<_Rp(_ArgTypes...)>*) const; virtual void destroy() _NOEXCEPT; virtual void destroy_deallocate() _NOEXCEPT; - virtual _Rp operator()(_ArgTypes&& ... __arg); + virtual _Rp operator()(_ArgTypes&&... __arg); #ifndef _LIBCPP_NO_RTTI virtual const void* target(const type_info&) const _NOEXCEPT; virtual const std::type_info& target_type() const _NOEXCEPT; @@ -1522,10 +1612,10 @@ __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const { typedef allocator_traits<_Alloc> __alloc_traits; typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _Ap; - _Ap __a(__f_.second()); + _Ap __a(__f_.__allocator()); typedef __allocator_destructor<_Ap> _Dp; unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) __func(__f_.first(), _Alloc(__a)); + ::new ((void*)__hold.get()) __func(__f_.__target(), _Alloc(__a)); return __hold.release(); } @@ -1533,14 +1623,14 @@ template void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const { - ::new (__p) __func(__f_.first(), __f_.second()); + ::new (__p) __func(__f_.__target(), __f_.__allocator()); } template void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT { - __f_.~__compressed_pair<_Fp, _Alloc>(); + __f_.destroy(); } template @@ -1549,8 +1639,8 @@ __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT { typedef allocator_traits<_Alloc> __alloc_traits; typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _Ap; - _Ap __a(__f_.second()); - __f_.~__compressed_pair<_Fp, _Alloc>(); + _Ap __a(__f_.__allocator()); + __f_.destroy(); __a.deallocate(this, 1); } @@ -1558,8 +1648,7 @@ template _Rp __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&& ... __arg) { - typedef __invoke_void_return_wrapper<_Rp> _Invoker; - return _Invoker::__call(__f_.first(), _VSTD::forward<_ArgTypes>(__arg)...); + return __f_(_VSTD::forward<_ArgTypes>(__arg)...); } #ifndef _LIBCPP_NO_RTTI @@ -1569,7 +1658,7 @@ const void* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT { if (__ti == typeid(_Fp)) - return &__f_.first(); + return &__f_.__target(); return (const void*)0; } @@ -1582,6 +1671,493 @@ __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT #endif // _LIBCPP_NO_RTTI +// __value_func creates a value-type from a __func. + +template class __value_func; + +template class __value_func<_Rp(_ArgTypes...)> +{ + typename aligned_storage<3 * sizeof(void*)>::type __buf_; + + typedef __base<_Rp(_ArgTypes...)> __func; + __func* __f_; + + _LIBCPP_NO_CFI static __func* __as_base(void* p) + { + return reinterpret_cast<__func*>(p); + } + + public: + _LIBCPP_INLINE_VISIBILITY + __value_func() _NOEXCEPT : __f_(0) {} + + template + _LIBCPP_INLINE_VISIBILITY __value_func(_Fp&& __f, const _Alloc __a) + : __f_(0) + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + _FunAlloc __af(__a); + if (sizeof(_Fun) <= sizeof(__buf_) && + is_nothrow_copy_constructible<_Fp>::value && + is_nothrow_copy_constructible<_FunAlloc>::value) + { + __f_ = + ::new ((void*)&__buf_) _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(_VSTD::move(__f), _Alloc(__a)); + __f_ = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __value_func(const __value_func& __f) + { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + __f_ = __f.__f_->__clone(); + } + + _LIBCPP_INLINE_VISIBILITY + __value_func(__value_func&& __f) _NOEXCEPT + { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__value_func() + { + if ((void*)__f_ == &__buf_) + __f_->destroy(); + else if (__f_) + __f_->destroy_deallocate(); + } + + _LIBCPP_INLINE_VISIBILITY + __value_func& operator=(__value_func&& __f) + { + *this = nullptr; + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __value_func& operator=(nullptr_t) + { + __func* __f = __f_; + __f_ = 0; + if ((void*)__f == &__buf_) + __f->destroy(); + else if (__f) + __f->destroy_deallocate(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + if (__f_ == 0) + __throw_bad_function_call(); + return (*__f_)(_VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__value_func& __f) _NOEXCEPT + { + if (&__f == this) + return; + if ((void*)__f_ == &__buf_ && (void*)__f.__f_ == &__f.__buf_) + { + typename aligned_storage::type __tempbuf; + __func* __t = __as_base(&__tempbuf); + __f_->__clone(__t); + __f_->destroy(); + __f_ = 0; + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = 0; + __f_ = __as_base(&__buf_); + __t->__clone(__as_base(&__f.__buf_)); + __t->destroy(); + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void*)__f_ == &__buf_) + { + __f_->__clone(__as_base(&__f.__buf_)); + __f_->destroy(); + __f_ = __f.__f_; + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = __f_; + __f_ = __as_base(&__buf_); + } + else + _VSTD::swap(__f_, __f.__f_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { return __f_ != 0; } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + if (__f_ == 0) + return typeid(void); + return __f_->target_type(); + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__f_ == 0) + return 0; + return (const _Tp*)__f_->target(typeid(_Tp)); + } +#endif // _LIBCPP_NO_RTTI +}; + +// Storage for a functor object, to be used with __policy to manage copy and +// destruction. +union __policy_storage +{ + mutable char __small[sizeof(void*) * 2]; + void* __large; +}; + +// True if _Fun can safely be held in __policy_storage.__small. +template +struct __use_small_storage + : public _VSTD::integral_constant< + bool, sizeof(_Fun) <= sizeof(__policy_storage) && + _LIBCPP_ALIGNOF(_Fun) <= _LIBCPP_ALIGNOF(__policy_storage) && + _VSTD::is_trivially_copy_constructible<_Fun>::value && + _VSTD::is_trivially_destructible<_Fun>::value> {}; + +// Policy contains information about how to copy, destroy, and move the +// underlying functor. You can think of it as a vtable of sorts. +struct __policy +{ + // Used to copy or destroy __large values. null for trivial objects. + void* (*const __clone)(const void*); + void (*const __destroy)(void*); + + // True if this is the null policy (no value). + const bool __is_null; + + // The target type. May be null if RTTI is disabled. + const std::type_info* const __type_info; + + // Returns a pointer to a static policy object suitable for the functor + // type. + template + _LIBCPP_INLINE_VISIBILITY static const __policy* __create() + { + return __choose_policy<_Fun>(__use_small_storage<_Fun>()); + } + + _LIBCPP_INLINE_VISIBILITY + static const __policy* __create_empty() + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = {nullptr, nullptr, + true, +#ifndef _LIBCPP_NO_RTTI + &typeid(void) +#else + nullptr +#endif + }; + return &__policy_; + } + + private: + template static void* __large_clone(const void* __s) + { + const _Fun* __f = static_cast(__s); + return __f->__clone(); + } + + template static void __large_destroy(void* __s) + { + typedef allocator_traits __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + _Fun* __f = static_cast<_Fun*>(__s); + _FunAlloc __a(__f->__allocator()); + __f->destroy(); + __a.deallocate(__f, 1); + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ false_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + &__large_clone<_Fun>, &__large_destroy<_Fun>, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ true_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + nullptr, nullptr, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } +}; + +// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is +// faster for types that can be passed in registers. +template +using __fast_forward = + typename _VSTD::conditional<_VSTD::is_scalar<_Tp>::value, _Tp, _Tp&&>::type; + +// __policy_invoker calls an instance of __alloc_func held in __policy_storage. + +template struct __policy_invoker; + +template +struct __policy_invoker<_Rp(_ArgTypes...)> +{ + typedef _Rp (*__Call)(const __policy_storage*, + __fast_forward<_ArgTypes>...); + + __Call __call_; + + // Creates an invoker that throws bad_function_call. + _LIBCPP_INLINE_VISIBILITY + __policy_invoker() : __call_(&__call_empty) {} + + // Creates an invoker that calls the given instance of __func. + template + _LIBCPP_INLINE_VISIBILITY static __policy_invoker __create() + { + return __policy_invoker(&__call_impl<_Fun>); + } + + private: + _LIBCPP_INLINE_VISIBILITY + explicit __policy_invoker(__Call __c) : __call_(__c) {} + + static _Rp __call_empty(const __policy_storage*, + __fast_forward<_ArgTypes>...) + { + __throw_bad_function_call(); + } + + template + static _Rp __call_impl(const __policy_storage* __buf, + __fast_forward<_ArgTypes>... __args) + { + _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value + ? &__buf->__small + : __buf->__large); + return (*__f)(_VSTD::forward<_ArgTypes>(__args)...); + } +}; + +// __policy_func uses a __policy and __policy_invoker to create a type-erased, +// copyable functor. + +template class __policy_func; + +template class __policy_func<_Rp(_ArgTypes...)> +{ + // Inline storage for small objects. + __policy_storage __buf_; + + // Calls the value stored in __buf_. This could technically be part of + // policy, but storing it here eliminates a level of indirection inside + // operator(). + typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker; + __invoker __invoker_; + + // The policy that describes how to move / copy / destroy __buf_. Never + // null, even if the function is empty. + const __policy* __policy_; + + public: + _LIBCPP_INLINE_VISIBILITY + __policy_func() : __policy_(__policy::__create_empty()) {} + + template + _LIBCPP_INLINE_VISIBILITY __policy_func(_Fp&& __f, const _Alloc& __a) + : __policy_(__policy::__create_empty()) + { + typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef allocator_traits<_Alloc> __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + + _FunAlloc __af(__a); + if (__use_small_storage<_Fun>()) + { + ::new ((void*)&__buf_.__small) + _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) + _Fun(_VSTD::move(__f), _Alloc(__af)); + __buf_.__large = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(const __policy_func& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__clone) + __buf_.__large = __policy_->__clone(__f.__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(__policy_func&& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__destroy) + { + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__policy_func() + { + if (__policy_->__destroy) + __policy_->__destroy(__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(__policy_func&& __f) + { + *this = nullptr; + __buf_ = __f.__buf_; + __invoker_ = __f.__invoker_; + __policy_ = __f.__policy_; + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(nullptr_t) + { + const __policy* __p = __policy_; + __policy_ = __policy::__create_empty(); + __invoker_ = __invoker(); + if (__p->__destroy) + __p->__destroy(__buf_.__large); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + return __invoker_.__call_(_VSTD::addressof(__buf_), + _VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__policy_func& __f) + { + _VSTD::swap(__invoker_, __f.__invoker_); + _VSTD::swap(__policy_, __f.__policy_); + _VSTD::swap(__buf_, __f.__buf_); + } + + _LIBCPP_INLINE_VISIBILITY + explicit operator bool() const _NOEXCEPT + { + return !__policy_->__is_null; + } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + return *__policy_->__type_info; + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info) + return nullptr; + if (__policy_->__clone) // Out of line storage. + return reinterpret_cast(__buf_.__large); + else + return reinterpret_cast(&__buf_.__small); + } +#endif // _LIBCPP_NO_RTTI +}; + } // __function template @@ -1589,13 +2165,13 @@ class _LIBCPP_TEMPLATE_VIS function<_Rp(_ArgTypes...)> : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>, public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> { - typedef __function::__base<_Rp(_ArgTypes...)> __base; - typename aligned_storage<3*sizeof(void*)>::type __buf_; - __base* __f_; +#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION + typedef __function::__value_func<_Rp(_ArgTypes...)> __func; +#else + typedef __function::__policy_func<_Rp(_ArgTypes...)> __func; +#endif - _LIBCPP_NO_CFI static __base *__as_base(void *p) { - return reinterpret_cast<__base*>(p); - } + __func __f_; template , function>::value>, @@ -1622,9 +2198,9 @@ public: // construct/copy/destroy: _LIBCPP_INLINE_VISIBILITY - function() _NOEXCEPT : __f_(0) {} + function() _NOEXCEPT { } _LIBCPP_INLINE_VISIBILITY - function(nullptr_t) _NOEXCEPT : __f_(0) {} + function(nullptr_t) _NOEXCEPT {} function(const function&); function(function&&) _NOEXCEPT; template> @@ -1633,10 +2209,10 @@ public: #if _LIBCPP_STD_VER <= 14 template _LIBCPP_INLINE_VISIBILITY - function(allocator_arg_t, const _Alloc&) _NOEXCEPT : __f_(0) {} + function(allocator_arg_t, const _Alloc&) _NOEXCEPT {} template _LIBCPP_INLINE_VISIBILITY - function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT : __f_(0) {} + function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT {} template function(allocator_arg_t, const _Alloc&, const function&); template @@ -1665,7 +2241,9 @@ public: // function capacity: _LIBCPP_INLINE_VISIBILITY - _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT {return __f_;} + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { + return static_cast(__f_); + } // deleted overloads close possible hole in the type system template @@ -1685,125 +2263,38 @@ public: }; template -function<_Rp(_ArgTypes...)>::function(const function& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); -} +function<_Rp(_ArgTypes...)>::function(const function& __f) : __f_(__f.__f_) {} #if _LIBCPP_STD_VER <= 14 template template function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - const function& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); -} + const function& __f) : __f_(__f.__f_) {} #endif -template +template function<_Rp(_ArgTypes...)>::function(function&& __f) _NOEXCEPT -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } -} + : __f_(_VSTD::move(__f.__f_)) {} #if _LIBCPP_STD_VER <= 14 template template function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - function&& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } -} + function&& __f) + : __f_(_VSTD::move(__f.__f_)) {} #endif -template +template template function<_Rp(_ArgTypes...)>::function(_Fp __f) - : __f_(0) -{ - if (__function::__not_null(__f)) - { - typedef __function::__func<_Fp, allocator<_Fp>, _Rp(_ArgTypes...)> _FF; - if (sizeof(_FF) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value) - { - __f_ = ::new((void*)&__buf_) _FF(_VSTD::move(__f)); - } - else - { - typedef allocator<_FF> _Ap; - _Ap __a; - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) _FF(_VSTD::move(__f), allocator<_Fp>(__a)); - __f_ = __hold.release(); - } - } -} + : __f_(_VSTD::move(__f), allocator<_Fp>()) {} #if _LIBCPP_STD_VER <= 14 -template +template template -function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a0, _Fp __f) - : __f_(0) -{ - typedef allocator_traits<_Alloc> __alloc_traits; - if (__function::__not_null(__f)) - { - typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _FF; - typedef typename __rebind_alloc_helper<__alloc_traits, _FF>::type _Ap; - _Ap __a(__a0); - if (sizeof(_FF) <= sizeof(__buf_) && - is_nothrow_copy_constructible<_Fp>::value && is_nothrow_copy_constructible<_Ap>::value) - { - __f_ = ::new((void*)&__buf_) _FF(_VSTD::move(__f), _Alloc(__a)); - } - else - { - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) _FF(_VSTD::move(__f), _Alloc(__a)); - __f_ = __hold.release(); - } - } -} +function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, + _Fp __f) + : __f_(_VSTD::move(__f), __a) {} #endif template @@ -1818,19 +2309,7 @@ template function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT { - *this = nullptr; - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } + __f_ = std::move(__f.__f_); return *this; } @@ -1838,12 +2317,7 @@ template function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(nullptr_t) _NOEXCEPT { - __base* __t = __f_; - __f_ = 0; - if ((void *)__t == &__buf_) - __t->destroy(); - else if (__t) - __t->destroy_deallocate(); + __f_ = nullptr; return *this; } @@ -1857,60 +2331,20 @@ function<_Rp(_ArgTypes...)>::operator=(_Fp&& __f) } template -function<_Rp(_ArgTypes...)>::~function() -{ - if ((void *)__f_ == &__buf_) - __f_->destroy(); - else if (__f_) - __f_->destroy_deallocate(); -} +function<_Rp(_ArgTypes...)>::~function() {} template void function<_Rp(_ArgTypes...)>::swap(function& __f) _NOEXCEPT { - if (_VSTD::addressof(__f) == this) - return; - if ((void *)__f_ == &__buf_ && (void *)__f.__f_ == &__f.__buf_) - { - typename aligned_storage::type __tempbuf; - __base* __t = __as_base(&__tempbuf); - __f_->__clone(__t); - __f_->destroy(); - __f_ = 0; - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = 0; - __f_ = __as_base(&__buf_); - __t->__clone(__as_base(&__f.__buf_)); - __t->destroy(); - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void *)__f_ == &__buf_) - { - __f_->__clone(__as_base(&__f.__buf_)); - __f_->destroy(); - __f_ = __f.__f_; - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = __f_; - __f_ = __as_base(&__buf_); - } - else - _VSTD::swap(__f_, __f.__f_); + __f_.swap(__f.__f_); } template _Rp function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const { - if (__f_ == 0) - __throw_bad_function_call(); - return (*__f_)(_VSTD::forward<_ArgTypes>(__arg)...); + return __f_(_VSTD::forward<_ArgTypes>(__arg)...); } #ifndef _LIBCPP_NO_RTTI @@ -1919,9 +2353,7 @@ template const std::type_info& function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT { - if (__f_ == 0) - return typeid(void); - return __f_->target_type(); + return __f_.target_type(); } template @@ -1929,9 +2361,7 @@ template _Tp* function<_Rp(_ArgTypes...)>::target() _NOEXCEPT { - if (__f_ == 0) - return nullptr; - return (_Tp*) const_cast(__f_->target(typeid(_Tp))); + return (_Tp*)(__f_.template target<_Tp>()); } template @@ -1939,9 +2369,7 @@ template const _Tp* function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT { - if (__f_ == 0) - return nullptr; - return (const _Tp*)__f_->target(typeid(_Tp)); + return __f_.template target<_Tp>(); } #endif // _LIBCPP_NO_RTTI @@ -2105,53 +2533,53 @@ __mu(_Ti& __ti, _Uj&) template -struct ____mu_return; +struct __mu_return_impl; template -struct ____mu_return_invokable // false +struct __mu_return_invokable // false { typedef __nat type; }; template -struct ____mu_return_invokable +struct __mu_return_invokable { typedef typename __invoke_of<_Ti&, _Uj...>::type type; }; template -struct ____mu_return<_Ti, false, true, false, tuple<_Uj...> > - : public ____mu_return_invokable<__invokable<_Ti&, _Uj...>::value, _Ti, _Uj...> +struct __mu_return_impl<_Ti, false, true, false, tuple<_Uj...> > + : public __mu_return_invokable<__invokable<_Ti&, _Uj...>::value, _Ti, _Uj...> { }; template -struct ____mu_return<_Ti, false, false, true, _TupleUj> +struct __mu_return_impl<_Ti, false, false, true, _TupleUj> { typedef typename tuple_element::value - 1, _TupleUj>::type&& type; }; template -struct ____mu_return<_Ti, true, false, false, _TupleUj> +struct __mu_return_impl<_Ti, true, false, false, _TupleUj> { typedef typename _Ti::type& type; }; template -struct ____mu_return<_Ti, false, false, false, _TupleUj> +struct __mu_return_impl<_Ti, false, false, false, _TupleUj> { typedef _Ti& type; }; template struct __mu_return - : public ____mu_return<_Ti, - __is_reference_wrapper<_Ti>::value, - is_bind_expression<_Ti>::value, - 0 < is_placeholder<_Ti>::value && - is_placeholder<_Ti>::value <= tuple_size<_TupleUj>::value, - _TupleUj> + : public __mu_return_impl<_Ti, + __is_reference_wrapper<_Ti>::value, + is_bind_expression<_Ti>::value, + 0 < is_placeholder<_Ti>::value && + is_placeholder<_Ti>::value <= tuple_size<_TupleUj>::value, + _TupleUj> { }; @@ -2340,8 +2768,6 @@ bind(_Fp&& __f, _BoundArgs&&... __bound_args) #if _LIBCPP_STD_VER > 14 -#define __cpp_lib_invoke 201411 - template result_of_t<_Fn&&(_Args&&...)> invoke(_Fn&& __f, _Args&&... __args) @@ -2497,7 +2923,7 @@ template> class _LIBCPP_TYPE_VIS default_searcher { public: _LIBCPP_INLINE_VISIBILITY - default_searcher(_ForwardIterator __f, _ForwardIterator __l, + default_searcher(_ForwardIterator __f, _ForwardIterator __l, _BinaryPredicate __p = _BinaryPredicate()) : __first_(__f), __last_(__l), __pred_(__p) {} @@ -2519,6 +2945,26 @@ private: #endif // _LIBCPP_STD_VER > 14 +#if _LIBCPP_STD_VER > 17 +template +using unwrap_reference_t = typename unwrap_reference<_Tp>::type; + +template +using unwrap_ref_decay_t = typename unwrap_ref_decay<_Tp>::type; +#endif // > C++17 + +template +inline void __libcpp_erase_if_container( _Container& __c, _Predicate __pred) +{ + for (typename _Container::iterator __iter = __c.begin(), __last = __c.end(); __iter != __last;) + { + if (__pred(*__iter)) + __iter = __c.erase(__iter); + else + ++__iter; + } +} + _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP_FUNCTIONAL diff --git a/contrib/libc++/include/future b/contrib/libc++/include/future index 0f6d42678ef..b3ffc7e3517 100644 --- a/contrib/libc++/include/future +++ b/contrib/libc++/include/future @@ -556,13 +556,14 @@ public: {return (__state_ & __constructed) || (__exception_ != nullptr);} _LIBCPP_INLINE_VISIBILITY - void __set_future_attached() - { + void __attach_future() { lock_guard __lk(__mut_); + bool __has_future_attached = (__state_ & __future_attached) != 0; + if (__has_future_attached) + __throw_future_error(future_errc::future_already_retrieved); + this->__add_shared(); __state_ |= __future_attached; } - _LIBCPP_INLINE_VISIBILITY - bool __has_future_attached() const {return (__state_ & __future_attached) != 0;} _LIBCPP_INLINE_VISIBILITY void __set_deferred() {__state_ |= deferred;} @@ -1154,10 +1155,7 @@ template future<_Rp>::future(__assoc_state<_Rp>* __state) : __state_(__state) { - if (__state_->__has_future_attached()) - __throw_future_error(future_errc::future_already_retrieved); - __state_->__add_shared(); - __state_->__set_future_attached(); + __state_->__attach_future(); } struct __release_shared_count @@ -1257,10 +1255,7 @@ template future<_Rp&>::future(__assoc_state<_Rp&>* __state) : __state_(__state) { - if (__state_->__has_future_attached()) - __throw_future_error(future_errc::future_already_retrieved); - __state_->__add_shared(); - __state_->__set_future_attached(); + __state_->__attach_future(); } template diff --git a/contrib/libc++/include/iomanip b/contrib/libc++/include/iomanip index a6bee736f45..36c11167a44 100644 --- a/contrib/libc++/include/iomanip +++ b/contrib/libc++/include/iomanip @@ -46,6 +46,7 @@ template #include <__config> #include <__string> #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header diff --git a/contrib/libc++/include/iosfwd b/contrib/libc++/include/iosfwd index d4384859e41..31f1902e5f5 100644 --- a/contrib/libc++/include/iosfwd +++ b/contrib/libc++/include/iosfwd @@ -18,6 +18,12 @@ namespace std { template struct char_traits; +template<> struct char_traits; +template<> struct char_traits; // C++20 +template<> struct char_traits; +template<> struct char_traits; +template<> struct char_traits; + template class allocator; class ios_base; @@ -98,6 +104,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD class _LIBCPP_TYPE_VIS ios_base; template struct _LIBCPP_TEMPLATE_VIS char_traits; +template<> struct char_traits; +#ifndef _LIBCPP_NO_HAS_CHAR8_T +template<> struct char_traits; +#endif +template<> struct char_traits; +template<> struct char_traits; +template<> struct char_traits; + template class _LIBCPP_TEMPLATE_VIS allocator; template > @@ -175,6 +189,9 @@ typedef basic_fstream wfstream; template class _LIBCPP_TEMPLATE_VIS fpos; typedef fpos streampos; typedef fpos wstreampos; +#ifndef _LIBCPP_NO_HAS_CHAR8_T +typedef fpos u8streampos; +#endif #ifndef _LIBCPP_HAS_NO_UNICODE_CHARS typedef fpos u16streampos; typedef fpos u32streampos; diff --git a/contrib/libc++/include/istream b/contrib/libc++/include/istream index 71c162b0d41..30ee4f4b871 100644 --- a/contrib/libc++/include/istream +++ b/contrib/libc++/include/istream @@ -160,6 +160,7 @@ template */ #include <__config> +#include #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -186,7 +187,7 @@ public: typedef typename traits_type::off_type off_type; // 27.7.1.1.1 Constructor/destructor: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_istream(basic_streambuf* __sb) : __gc_(0) { this->init(__sb); } virtual ~basic_istream(); @@ -200,7 +201,7 @@ protected: basic_istream& operator=(basic_istream&& __rhs); #endif - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_istream& __rhs) { _VSTD::swap(__gc_, __rhs.__gc_); basic_ios::swap(__rhs); @@ -216,16 +217,16 @@ public: class _LIBCPP_TEMPLATE_VIS sentry; // 27.7.1.2 Formatted input: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(basic_istream& (*__pf)(basic_istream&)) { return __pf(*this); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(basic_ios& (*__pf)(basic_ios&)) { __pf(*this); return *this; } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(ios_base& (*__pf)(ios_base&)) { __pf(*this); return *this; } @@ -249,7 +250,7 @@ public: streamsize gcount() const {return __gc_;} int_type get(); - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(char_type& __c) { int_type __ch = get(); if (__ch != traits_type::eof()) @@ -257,19 +258,19 @@ public: return *this; } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(char_type* __s, streamsize __n) { return get(__s, __n, this->widen('\n')); } basic_istream& get(char_type* __s, streamsize __n, char_type __dlm); - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(basic_streambuf& __sb) { return get(__sb, this->widen('\n')); } basic_istream& get(basic_streambuf& __sb, char_type __dlm); - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& getline(char_type* __s, streamsize __n) { return getline(__s, __n, this->widen('\n')); } @@ -517,8 +518,9 @@ basic_istream<_CharT, _Traits>::operator>>(int& __n) } template +_LIBCPP_INLINE_VISIBILITY basic_istream<_CharT, _Traits>& -operator>>(basic_istream<_CharT, _Traits>& __is, _CharT* __s) +__input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n) { #ifndef _LIBCPP_NO_EXCEPTIONS try @@ -527,13 +529,10 @@ operator>>(basic_istream<_CharT, _Traits>& __is, _CharT* __s) typename basic_istream<_CharT, _Traits>::sentry __sen(__is); if (__sen) { - streamsize __n = __is.width(); - if (__n <= 0) - __n = numeric_limits::max() / sizeof(_CharT) - 1; - streamsize __c = 0; + auto __s = __p; const ctype<_CharT>& __ct = use_facet >(__is.getloc()); ios_base::iostate __err = ios_base::goodbit; - while (__c < __n-1) + while (__s != __p + (__n-1)) { typename _Traits::int_type __i = __is.rdbuf()->sgetc(); if (_Traits::eq_int_type(__i, _Traits::eof())) @@ -545,12 +544,11 @@ operator>>(basic_istream<_CharT, _Traits>& __is, _CharT* __s) if (__ct.is(__ct.space, __ch)) break; *__s++ = __ch; - ++__c; __is.rdbuf()->sbumpc(); } *__s = _CharT(); __is.width(0); - if (__c == 0) + if (__s == __p) __err |= ios_base::failbit; __is.setstate(__err); } @@ -564,6 +562,48 @@ operator>>(basic_istream<_CharT, _Traits>& __is, _CharT* __s) return __is; } +#if _LIBCPP_STD_VER > 17 + +template +inline _LIBCPP_INLINE_VISIBILITY +basic_istream<_CharT, _Traits>& +operator>>(basic_istream<_CharT, _Traits>& __is, _CharT (&__buf)[_Np]) +{ + auto __n = _Np; + if (__is.width() > 0) + __n = _VSTD::min(size_t(__is.width()), _Np); + return _VSTD::__input_c_string(__is, __buf, __n); +} + +template +inline _LIBCPP_INLINE_VISIBILITY +basic_istream& +operator>>(basic_istream& __is, unsigned char (&__buf)[_Np]) +{ + return __is >> (char(&)[_Np])__buf; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +basic_istream& +operator>>(basic_istream& __is, signed char (&__buf)[_Np]) +{ + return __is >> (char(&)[_Np])__buf; +} + +#else + +template +inline _LIBCPP_INLINE_VISIBILITY +basic_istream<_CharT, _Traits>& +operator>>(basic_istream<_CharT, _Traits>& __is, _CharT* __s) +{ + streamsize __n = __is.width(); + if (__n <= 0) + __n = numeric_limits::max() / sizeof(_CharT) - 1; + return _VSTD::__input_c_string(__is, __s, size_t(__n)); +} + template inline _LIBCPP_INLINE_VISIBILITY basic_istream& @@ -580,6 +620,8 @@ operator>>(basic_istream& __is, signed char* __s) return __is >> (char*)__s; } +#endif // _LIBCPP_STD_VER > 17 + template basic_istream<_CharT, _Traits>& operator>>(basic_istream<_CharT, _Traits>& __is, _CharT& __c) @@ -1238,7 +1280,7 @@ public: typedef typename traits_type::off_type off_type; // constructor/destructor - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_iostream(basic_streambuf* __sb) : basic_istream<_CharT, _Traits>(__sb) {} @@ -1253,7 +1295,7 @@ protected: inline _LIBCPP_INLINE_VISIBILITY basic_iostream& operator=(basic_iostream&& __rhs); #endif - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_iostream& __rhs) { basic_istream::swap(__rhs); } public: @@ -1463,7 +1505,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) return __is; } -#ifndef _LIBCPP_AVAILABILITY_NO_STREAMS_EXTERN_TEMPLATE +#ifndef _LIBCPP_DO_NOT_ASSUME_STREAMS_EXPLICIT_INSTANTIATION_IN_DYLIB _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istream) _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_istream) _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_iostream) diff --git a/contrib/libc++/include/iterator b/contrib/libc++/include/iterator index 9415e0b8396..bda177e11e6 100644 --- a/contrib/libc++/include/iterator +++ b/contrib/libc++/include/iterator @@ -418,6 +418,7 @@ template constexpr const E* data(initializer_list il) noexcept; #include #include #include +#include #ifdef __APPLE__ #include #endif @@ -436,6 +437,23 @@ struct _LIBCPP_TEMPLATE_VIS forward_iterator_tag : public input_iterator_t struct _LIBCPP_TEMPLATE_VIS bidirectional_iterator_tag : public forward_iterator_tag {}; struct _LIBCPP_TEMPLATE_VIS random_access_iterator_tag : public bidirectional_iterator_tag {}; +template +struct __has_iterator_typedefs +{ +private: + struct __two {char __lx; char __lxx;}; + template static __two __test(...); + template static char __test(typename std::__void_t::type* = 0, + typename std::__void_t::type* = 0, + typename std::__void_t::type* = 0, + typename std::__void_t::type* = 0, + typename std::__void_t::type* = 0 + ); +public: + static const bool value = sizeof(__test<_Tp>(0,0,0,0,0)) == 1; +}; + + template struct __has_iterator_category { @@ -478,7 +496,7 @@ struct __iterator_traits<_Iter, true> template struct _LIBCPP_TEMPLATE_VIS iterator_traits - : __iterator_traits<_Iter, __has_iterator_category<_Iter>::value> {}; + : __iterator_traits<_Iter, __has_iterator_typedefs<_Iter>::value> {}; template struct _LIBCPP_TEMPLATE_VIS iterator_traits<_Tp*> diff --git a/contrib/libc++/include/limits b/contrib/libc++/include/limits index f530507f724..5ea9a9e6fdf 100644 --- a/contrib/libc++/include/limits +++ b/contrib/libc++/include/limits @@ -82,6 +82,7 @@ template<> class numeric_limits; template<> class numeric_limits; template<> class numeric_limits; template<> class numeric_limits; +template<> class numeric_limits; // C++20 template<> class numeric_limits; template<> class numeric_limits; @@ -118,6 +119,7 @@ template<> class numeric_limits; _LIBCPP_PUSH_MACROS #include <__undef_macros> +#include _LIBCPP_BEGIN_NAMESPACE_STD diff --git a/contrib/libc++/include/list b/contrib/libc++/include/list index d2e78cd66af..c69e31d93a4 100644 --- a/contrib/libc++/include/list +++ b/contrib/libc++/include/list @@ -169,6 +169,11 @@ template void swap(list& x, list& y) noexcept(noexcept(x.swap(y))); +template + void erase(list& c, const U& value); // C++20 +template + void erase_if(list& c, Predicate pred); // C++20 + } // std */ @@ -181,6 +186,7 @@ template #include #include #include +#include #include <__debug> @@ -1169,7 +1175,7 @@ list<_Tp, _Alloc>::__link_nodes_at_front(__link_pointer __f, __link_pointer __l) base::__end_.__next_ = __f; } -// Link in nodes [__f, __l] at the front of the list +// Link in nodes [__f, __l] at the back of the list template inline void @@ -2208,7 +2214,7 @@ template void list<_Tp, _Alloc>::merge(list& __c, _Comp __comp) { - if (this != &__c) + if (this != _VSTD::addressof(__c)) { iterator __f1 = begin(); iterator __e1 = end(); @@ -2449,6 +2455,18 @@ swap(list<_Tp, _Alloc>& __x, list<_Tp, _Alloc>& __y) __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(list<_Tp, _Allocator>& __c, _Predicate __pred) +{ __c.remove_if(__pred); } + +template +inline _LIBCPP_INLINE_VISIBILITY +void erase(list<_Tp, _Allocator>& __c, const _Up& __v) +{ _VSTD::erase_if(__c, [&](auto& __elem) { return __elem == __v; }); } +#endif + _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS diff --git a/contrib/libc++/include/locale b/contrib/libc++/include/locale index e240799f383..2043892fa2d 100644 --- a/contrib/libc++/include/locale +++ b/contrib/libc++/include/locale @@ -187,6 +187,7 @@ template class messages_byname; #include #include #include +#include #ifndef __APPLE__ #include #endif @@ -727,7 +728,7 @@ locale::id num_get<_CharT, _InputIterator>::id; template -_Tp +_LIBCPP_HIDDEN _Tp __num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) { @@ -762,7 +763,7 @@ __num_get_signed_integral(const char* __a, const char* __a_end, } template -_Tp +_LIBCPP_HIDDEN _Tp __num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) { @@ -2408,6 +2409,23 @@ private: string_type __analyze(char __fmt, const ctype<_CharT>&); }; +#define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT) \ +template <> _LIBCPP_FUNC_VIS time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const; \ +template <> _LIBCPP_FUNC_VIS __time_get_storage<_CharT>::__time_get_storage(const char*); \ +template <> _LIBCPP_FUNC_VIS __time_get_storage<_CharT>::__time_get_storage(const string&); \ +template <> _LIBCPP_FUNC_VIS void __time_get_storage<_CharT>::init(const ctype<_CharT>&); \ +template <> _LIBCPP_FUNC_VIS __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&); \ +extern template _LIBCPP_FUNC_VIS time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const; \ +extern template _LIBCPP_FUNC_VIS __time_get_storage<_CharT>::__time_get_storage(const char*); \ +extern template _LIBCPP_FUNC_VIS __time_get_storage<_CharT>::__time_get_storage(const string&); \ +extern template _LIBCPP_FUNC_VIS void __time_get_storage<_CharT>::init(const ctype<_CharT>&); \ +extern template _LIBCPP_FUNC_VIS __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&); \ +/**/ + +_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char) +_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t) +#undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION + template > class _LIBCPP_TEMPLATE_VIS time_get_byname : public time_get<_CharT, _InputIterator>, @@ -3550,6 +3568,7 @@ messages<_CharT>::do_open(const basic_string& __nm, const locale&) const __cat = static_cast((static_cast(__cat) >> 1)); return __cat; #else // !_LIBCPP_HAS_CATOPEN + _LIBCPP_UNUSED_VAR(__nm); return -1; #endif // _LIBCPP_HAS_CATOPEN } @@ -3573,6 +3592,9 @@ messages<_CharT>::do_get(catalog __c, int __set, int __msgid, __n, __n + strlen(__n)); return __w; #else // !_LIBCPP_HAS_CATOPEN + _LIBCPP_UNUSED_VAR(__c); + _LIBCPP_UNUSED_VAR(__set); + _LIBCPP_UNUSED_VAR(__msgid); return __dflt; #endif // _LIBCPP_HAS_CATOPEN } @@ -3586,6 +3608,8 @@ messages<_CharT>::do_close(catalog __c) const __c <<= 1; nl_catd __cat = (nl_catd)__c; catclose(__cat); +#else // !_LIBCPP_HAS_CATOPEN + _LIBCPP_UNUSED_VAR(__c); #endif // _LIBCPP_HAS_CATOPEN } diff --git a/contrib/libc++/include/map b/contrib/libc++/include/map index 559ec484aca..616bb46cfcc 100644 --- a/contrib/libc++/include/map +++ b/contrib/libc++/include/map @@ -167,6 +167,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(map& source); // C++17 + template + void merge(map&& source); // C++17 + template + void merge(multimap& source); // C++17 + template + void merge(multimap&& source); // C++17 + void swap(map& m) noexcept(allocator_traits::is_always_equal::value && is_nothrow_swappable::value); // C++17 @@ -245,6 +254,10 @@ void swap(map& x, map& y) noexcept(noexcept(x.swap(y))); +template + void erase_if(map& c, Predicate pred); // C++20 + + template , class Allocator = allocator>> class multimap @@ -368,6 +381,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(multimap& source); // C++17 + template + void merge(multimap&& source); // C++17 + template + void merge(map& source); // C++17 + template + void merge(map&& source); // C++17 + void swap(multimap& m) noexcept(allocator_traits::is_always_equal::value && is_nothrow_swappable::value); // C++17 @@ -447,6 +469,9 @@ swap(multimap& x, multimap& y) noexcept(noexcept(x.swap(y))); +template + void erase_if(multimap& c, Predicate pred); // C++20 + } // std */ @@ -460,6 +485,7 @@ swap(multimap& x, #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -467,7 +493,8 @@ swap(multimap& x, _LIBCPP_BEGIN_NAMESPACE_STD -template +template ::value && !__libcpp_is_final<_Compare>::value> class __map_value_compare : private _Compare { @@ -881,6 +908,7 @@ public: typedef value_type& reference; typedef const value_type& const_reference; + static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); static_assert((is_same::value), "Allocator::value_type must be same type as value_type"); @@ -925,6 +953,11 @@ public: typedef __insert_return_type insert_return_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS map; + template + friend class _LIBCPP_TEMPLATE_VIS multimap; + _LIBCPP_INLINE_VISIBILITY map() _NOEXCEPT_( @@ -1299,6 +1332,38 @@ public: { return __tree_.template __node_handle_extract(__it.__i_); } + template + _LIBCPP_INLINE_VISIBILITY + void merge(map& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(map&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multimap& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multimap&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -1556,6 +1621,14 @@ swap(map<_Key, _Tp, _Compare, _Allocator>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(map<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + + template , class _Allocator = allocator > > class _LIBCPP_TEMPLATE_VIS multimap @@ -1570,6 +1643,7 @@ public: typedef value_type& reference; typedef const value_type& const_reference; + static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); static_assert((is_same::value), "Allocator::value_type must be same type as value_type"); @@ -1614,6 +1688,11 @@ public: typedef __map_node_handle node_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS map; + template + friend class _LIBCPP_TEMPLATE_VIS multimap; + _LIBCPP_INLINE_VISIBILITY multimap() _NOEXCEPT_( @@ -1881,10 +1960,42 @@ public: return __tree_.template __node_handle_extract( __it.__i_); } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multimap& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __tree_.__node_handle_merge_multi(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multimap&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __tree_.__node_handle_merge_multi(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(map& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __tree_.__node_handle_merge_multi(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(map&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __tree_.__node_handle_merge_multi(__source.__tree_); + } #endif _LIBCPP_INLINE_VISIBILITY - void clear() {__tree_.clear();} + void clear() _NOEXCEPT {__tree_.clear();} _LIBCPP_INLINE_VISIBILITY void swap(multimap& __m) @@ -2055,6 +2166,13 @@ swap(multimap<_Key, _Tp, _Compare, _Allocator>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(multimap<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP_MAP diff --git a/contrib/libc++/include/memory b/contrib/libc++/include/memory index adfe4f4fbbe..ce2c3576623 100644 --- a/contrib/libc++/include/memory +++ b/contrib/libc++/include/memory @@ -43,7 +43,7 @@ struct pointer_traits template using rebind = U*; - static pointer pointer_to(
) noexcept; + static pointer pointer_to(
) noexcept; // constexpr in C++20 }; template constexpr T* to_address(T* p) noexcept; // C++20 @@ -212,10 +212,10 @@ template template ForwardIterator uninitialized_default_construct_n(ForwardIterator first, Size n); -template struct auto_ptr_ref {}; // removed in C++17 +template struct auto_ptr_ref {}; // deprecated in C++11, removed in C++17 template -class auto_ptr // removed in C++17 +class auto_ptr // deprecated in C++11, removed in C++17 { public: typedef X element_type; @@ -667,6 +667,7 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space); #if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER) # include #endif +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -983,7 +984,7 @@ struct _LIBCPP_TEMPLATE_VIS pointer_traits<_Tp*> private: struct __nat {}; public: - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 static pointer pointer_to(typename conditional::value, __nat, element_type>::type& __r) _NOEXCEPT {return _VSTD::addressof(__r);} @@ -1459,29 +1460,21 @@ struct __has_select_on_container_copy_construction #else // _LIBCPP_CXX03_LANG -#ifndef _LIBCPP_HAS_NO_VARIADICS +template +struct __has_construct : std::false_type {}; -template -struct __has_construct - : false_type -{ -}; +template +struct __has_construct<_Alloc, _Pointer, _Tp, typename __void_t< + decltype(_VSTD::declval<_Alloc>().construct(_VSTD::declval<_Pointer>(), _VSTD::declval<_Tp>())) +>::type> : std::true_type {}; -#else // _LIBCPP_HAS_NO_VARIADICS - -template -struct __has_construct - : false_type -{ -}; - -#endif // _LIBCPP_HAS_NO_VARIADICS +template +struct __has_destroy : false_type {}; template -struct __has_destroy - : false_type -{ -}; +struct __has_destroy<_Alloc, _Pointer, typename __void_t< + decltype(_VSTD::declval<_Alloc>().destroy(_VSTD::declval<_Pointer>())) +>::type> : std::true_type {}; template struct __has_max_size @@ -1509,6 +1502,12 @@ struct __alloc_traits_difference_type<_Alloc, _Ptr, true> typedef typename _Alloc::difference_type type; }; +template +struct __is_default_allocator : false_type {}; + +template +struct __is_default_allocator<_VSTD::allocator<_Tp> > : true_type {}; + template struct _LIBCPP_TEMPLATE_VIS allocator_traits { @@ -1570,9 +1569,10 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits } template _LIBCPP_INLINE_VISIBILITY - static void construct(allocator_type&, _Tp* __p, const _A0& __a0) + static void construct(allocator_type& __a, _Tp* __p, const _A0& __a0) { - ::new ((void*)__p) _Tp(__a0); + __construct(__has_construct(), + __a, __p, __a0); } template _LIBCPP_INLINE_VISIBILITY @@ -1612,7 +1612,7 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits void __construct_forward(allocator_type& __a, _Ptr __begin1, _Ptr __end1, _Ptr& __begin2) { - for (; __begin1 != __end1; ++__begin1, ++__begin2) + for (; __begin1 != __end1; ++__begin1, (void) ++__begin2) construct(__a, _VSTD::__to_raw_pointer(__begin2), _VSTD::move_if_noexcept(*__begin1)); } @@ -1621,7 +1621,7 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits static typename enable_if < - (is_same >::value + (__is_default_allocator::value || !__has_construct::value) && is_trivially_move_constructible<_Tp>::value, void @@ -1646,23 +1646,25 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits construct(__a, _VSTD::__to_raw_pointer(__begin2), *__begin1); } - template + template ::type, + class _RawDestTp = typename remove_const<_DestTp>::type> _LIBCPP_INLINE_VISIBILITY static typename enable_if < - (is_same >::value - || !__has_construct::value) && - is_trivially_move_constructible<_Tp>::value, + is_trivially_move_constructible<_DestTp>::value && + is_same<_RawSourceTp, _RawDestTp>::value && + (__is_default_allocator::value || + !__has_construct::value), void >::type - __construct_range_forward(allocator_type&, _Tp* __begin1, _Tp* __end1, _Tp*& __begin2) + __construct_range_forward(allocator_type&, _SourceTp* __begin1, _SourceTp* __end1, _DestTp*& __begin2) { - typedef typename remove_const<_Tp>::type _Vp; ptrdiff_t _Np = __end1 - __begin1; if (_Np > 0) { - _VSTD::memcpy(const_cast<_Vp*>(__begin2), __begin1, _Np * sizeof(_Tp)); + _VSTD::memcpy(const_cast<_RawDestTp*>(__begin2), __begin1, _Np * sizeof(_DestTp)); __begin2 += _Np; } } @@ -1685,7 +1687,7 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits static typename enable_if < - (is_same >::value + (__is_default_allocator::value || !__has_construct::value) && is_trivially_move_constructible<_Tp>::value, void @@ -1720,6 +1722,19 @@ private: { ::new ((void*)__p) _Tp(_VSTD::forward<_Args>(__args)...); } +#else // _LIBCPP_HAS_NO_VARIADICS + template + _LIBCPP_INLINE_VISIBILITY + static void __construct(true_type, allocator_type& __a, _Tp* __p, + const _A0& __a0) + {__a.construct(__p, __a0);} + template + _LIBCPP_INLINE_VISIBILITY + static void __construct(false_type, allocator_type&, _Tp* __p, + const _A0& __a0) + { + ::new ((void*)__p) _Tp(__a0); + } #endif // _LIBCPP_HAS_NO_VARIADICS template @@ -1782,7 +1797,7 @@ public: _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 allocator() _NOEXCEPT {} - template + template _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 allocator(const allocator<_Up>&) _NOEXCEPT {} @@ -1790,16 +1805,16 @@ public: {return _VSTD::addressof(__x);} _LIBCPP_INLINE_VISIBILITY const_pointer address(const_reference __x) const _NOEXCEPT {return _VSTD::addressof(__x);} - _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY pointer allocate(size_type __n, allocator::const_pointer = 0) { if (__n > max_size()) __throw_length_error("allocator::allocate(size_t n)" " 'n' exceeds maximum supported size"); - return static_cast(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp))); + return static_cast(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp))); } - _LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) _NOEXCEPT - {_VSTD::__libcpp_deallocate((void*)__p, __alignof(_Tp));} + _LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type __n) _NOEXCEPT + {_VSTD::__libcpp_deallocate((void*)__p, __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));} _LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT {return size_type(~0) / sizeof(_Tp);} #if !defined(_LIBCPP_HAS_NO_RVALUE_REFERENCES) && !defined(_LIBCPP_HAS_NO_VARIADICS) @@ -1887,7 +1902,7 @@ public: allocator() _NOEXCEPT {} template - _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 allocator(const allocator<_Up>&) _NOEXCEPT {} _LIBCPP_INLINE_VISIBILITY const_pointer address(const_reference __x) const _NOEXCEPT @@ -1897,10 +1912,10 @@ public: if (__n > max_size()) __throw_length_error("allocator::allocate(size_t n)" " 'n' exceeds maximum supported size"); - return static_cast(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp))); + return static_cast(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp))); } - _LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) _NOEXCEPT - {_VSTD::__libcpp_deallocate((void*) const_cast<_Tp *>(__p), __alignof(_Tp));} + _LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type __n) _NOEXCEPT + {_VSTD::__libcpp_deallocate((void*) const_cast<_Tp *>(__p), __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));} _LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT {return size_type(~0) / sizeof(_Tp);} #if !defined(_LIBCPP_HAS_NO_RVALUE_REFERENCES) && !defined(_LIBCPP_HAS_NO_VARIADICS) @@ -2003,7 +2018,7 @@ public: }; template -_LIBCPP_NO_CFI +_LIBCPP_NODISCARD_EXT _LIBCPP_NO_CFI pair<_Tp*, ptrdiff_t> get_temporary_buffer(ptrdiff_t __n) _NOEXCEPT { @@ -2016,7 +2031,7 @@ get_temporary_buffer(ptrdiff_t __n) _NOEXCEPT while (__n > 0) { #if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) - if (__is_overaligned_for_new(__alignof(_Tp))) + if (__is_overaligned_for_new(_LIBCPP_ALIGNOF(_Tp))) { std::align_val_t __al = std::align_val_t(std::alignment_of<_Tp>::value); @@ -2027,7 +2042,7 @@ get_temporary_buffer(ptrdiff_t __n) _NOEXCEPT __n * sizeof(_Tp), nothrow)); } #else - if (__is_overaligned_for_new(__alignof(_Tp))) + if (__is_overaligned_for_new(_LIBCPP_ALIGNOF(_Tp))) { // Since aligned operator new is unavailable, return an empty // buffer rather than one with invalid alignment. @@ -2051,18 +2066,18 @@ template inline _LIBCPP_INLINE_VISIBILITY void return_temporary_buffer(_Tp* __p) _NOEXCEPT { - _VSTD::__libcpp_deallocate((void*)__p, __alignof(_Tp)); + _VSTD::__libcpp_deallocate_unsized((void*)__p, _LIBCPP_ALIGNOF(_Tp)); } #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) template -struct auto_ptr_ref +struct _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr_ref { _Tp* __ptr_; }; template -class _LIBCPP_TEMPLATE_VIS auto_ptr +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr { private: _Tp* __ptr_; @@ -2106,7 +2121,7 @@ public: }; template <> -class _LIBCPP_TEMPLATE_VIS auto_ptr +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr { public: typedef void element_type; @@ -2130,7 +2145,9 @@ struct __compressed_pair_elem { _LIBCPP_INLINE_VISIBILITY constexpr explicit __compressed_pair_elem(_Up&& __u) - : __value_(_VSTD::forward<_Up>(__u)){}; + : __value_(_VSTD::forward<_Up>(__u)) + { + } template _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 @@ -2167,7 +2184,8 @@ struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp { _LIBCPP_INLINE_VISIBILITY constexpr explicit __compressed_pair_elem(_Up&& __u) - : __value_type(_VSTD::forward<_Up>(__u)){}; + : __value_type(_VSTD::forward<_Up>(__u)) + {} template _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 @@ -5624,15 +5642,18 @@ template struct __temp_value { typedef allocator_traits<_Alloc> _Traits; - typename aligned_storage::type __v; + typename aligned_storage::type __v; _Alloc &__a; _Tp *__addr() { return reinterpret_cast<_Tp *>(addressof(__v)); } _Tp & get() { return *__addr(); } template - __temp_value(_Alloc &__alloc, _Args&& ... __args) : __a(__alloc) - { _Traits::construct(__a, __addr(), _VSTD::forward<_Args>(__args)...); } + _LIBCPP_NO_CFI + __temp_value(_Alloc &__alloc, _Args&& ... __args) : __a(__alloc) { + _Traits::construct(__a, reinterpret_cast<_Tp*>(addressof(__v)), + _VSTD::forward<_Args>(__args)...); + } ~__temp_value() { _Traits::destroy(__a, __addr()); } }; diff --git a/contrib/libc++/include/module.modulemap b/contrib/libc++/include/module.modulemap index 089505586fb..6d88f52113c 100644 --- a/contrib/libc++/include/module.modulemap +++ b/contrib/libc++/include/module.modulemap @@ -228,6 +228,10 @@ module std [system] { header "atomic" export * } + module bit { + header "bit" + export * + } module bitset { header "bitset" export string @@ -520,10 +524,6 @@ module std [system] { header "experimental/deque" export * } - module dynarray { - header "experimental/dynarray" - export * - } module filesystem { header "experimental/filesystem" export * diff --git a/contrib/libc++/include/mutex b/contrib/libc++/include/mutex index 52e39b0fb1f..6d2de2b460e 100644 --- a/contrib/libc++/include/mutex +++ b/contrib/libc++/include/mutex @@ -194,6 +194,7 @@ template #ifndef _LIBCPP_CXX03_LANG #include #endif +#include #include <__threading_support> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -488,7 +489,7 @@ public: }; template -class _LIBCPP_TEMPLATE_VIS scoped_lock<_Mutex> { +class _LIBCPP_TEMPLATE_VIS _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) scoped_lock<_Mutex> { public: typedef _Mutex mutex_type; private: diff --git a/contrib/libc++/include/new b/contrib/libc++/include/new index 5a658c7a2f2..24ffcad5a6b 100644 --- a/contrib/libc++/include/new +++ b/contrib/libc++/include/new @@ -27,12 +27,6 @@ public: virtual const char* what() const noexcept; }; -class bad_array_length : public bad_alloc // FIXME: Not part of C++ -{ -public: - bad_array_length() noexcept; -}; - class bad_array_new_length : public bad_alloc // C++14 { public: @@ -91,6 +85,7 @@ void operator delete[](void* ptr, void*) noexcept; #include #include #include +#include #ifdef _LIBCPP_NO_EXCEPTIONS #include #endif @@ -103,16 +98,23 @@ void operator delete[](void* ptr, void*) noexcept; #pragma GCC system_header #endif -#if !(defined(_LIBCPP_BUILDING_LIBRARY) || _LIBCPP_STD_VER >= 14 || \ - (defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309)) +#if !defined(__cpp_sized_deallocation) || __cpp_sized_deallocation < 201309L +#define _LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION +#endif + +#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14 && \ + defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION) +# define _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION +#endif + +#if defined(_LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION) || \ + defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION) # define _LIBCPP_HAS_NO_SIZED_DEALLOCATION #endif #if !__has_builtin(__builtin_operator_new) || \ - __has_builtin(__builtin_operator_new) < 201802L || \ - defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) || \ - !defined(__cpp_aligned_new) || __cpp_aligned_new < 201606 -#define _LIBCPP_HAS_NO_BUILTIN_ALIGNED_OPERATOR_NEW_DELETE + __has_builtin(__builtin_operator_new) < 201802L +#define _LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE #endif namespace std // purposefully not using versioning namespace @@ -148,29 +150,14 @@ _LIBCPP_FUNC_VIS new_handler get_new_handler() _NOEXCEPT; _LIBCPP_NORETURN _LIBCPP_FUNC_VIS void __throw_bad_alloc(); // not in C++ spec -#if defined(_LIBCPP_BUILDING_LIBRARY) || (_LIBCPP_STD_VER > 11) - -class _LIBCPP_EXCEPTION_ABI _LIBCPP_AVAILABILITY_BAD_ARRAY_LENGTH - bad_array_length : public bad_alloc { -public: - bad_array_length() _NOEXCEPT; - virtual ~bad_array_length() _NOEXCEPT; - virtual const char* what() const _NOEXCEPT; -}; - -#define _LIBCPP_BAD_ARRAY_LENGTH_DEFINED - -#endif // defined(_LIBCPP_BUILDING_LIBRARY) || (_LIBCPP_STD_VER > 11) - -#if !defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME) -#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) || _LIBCPP_STD_VER > 14 +#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) && \ + !defined(_LIBCPP_DEFER_NEW_TO_VCRUNTIME) #ifndef _LIBCPP_CXX03_LANG enum class _LIBCPP_ENUM_VIS align_val_t : size_t { }; #else enum align_val_t { __zero = 0, __max = (size_t)-1 }; #endif #endif -#endif } // std @@ -180,13 +167,13 @@ enum align_val_t { __zero = 0, __max = (size_t)-1 }; #define _THROW_BAD_ALLOC #endif -#if !defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME) +#if !defined(_LIBCPP_DEFER_NEW_TO_VCRUNTIME) _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz) _THROW_BAD_ALLOC; _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t&) _NOEXCEPT; -#ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION +#ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operator delete(void* __p, std::size_t __sz) _NOEXCEPT; #endif @@ -194,16 +181,16 @@ _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[]( _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t&) _NOEXCEPT; -#ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION +#ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operator delete[](void* __p, std::size_t __sz) _NOEXCEPT; #endif -#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION +#ifndef _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC; _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT; -#ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION +#ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operator delete(void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT; #endif @@ -211,7 +198,7 @@ _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[]( _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT; -#ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION +#ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operator delete[](void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT; #endif #endif @@ -221,7 +208,7 @@ _LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY void* operator ne inline _LIBCPP_INLINE_VISIBILITY void operator delete (void*, void*) _NOEXCEPT {} inline _LIBCPP_INLINE_VISIBILITY void operator delete[](void*, void*) _NOEXCEPT {} -#endif // !_LIBCPP_ABI_MICROSOFT || _LIBCPP_NO_VCRUNTIME +#endif // !_LIBCPP_DEFER_NEW_TO_VCRUNTIME _LIBCPP_BEGIN_NAMESPACE_STD @@ -237,7 +224,7 @@ inline _LIBCPP_INLINE_VISIBILITY void *__libcpp_allocate(size_t __size, size_t _ #ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION if (__is_overaligned_for_new(__align)) { const align_val_t __align_val = static_cast(__align); -# ifdef _LIBCPP_HAS_NO_BUILTIN_ALIGNED_OPERATOR_NEW_DELETE +# ifdef _LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE return ::operator new(__size, __align_val); # else return __builtin_operator_new(__size, __align_val); @@ -253,43 +240,98 @@ inline _LIBCPP_INLINE_VISIBILITY void *__libcpp_allocate(size_t __size, size_t _ #endif } -inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __align) { -#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION - if (__is_overaligned_for_new(__align)) { - const align_val_t __align_val = static_cast(__align); -# ifdef _LIBCPP_HAS_NO_BUILTIN_ALIGNED_OPERATOR_NEW_DELETE - return ::operator delete(__ptr, __align_val); -# else - return __builtin_operator_delete(__ptr, __align_val); -# endif +struct _DeallocateCaller { + static inline _LIBCPP_INLINE_VISIBILITY + void __do_deallocate_handle_size_align(void *__ptr, size_t __size, size_t __align) { +#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) + ((void)__align); + return __do_deallocate_handle_size(__ptr, __size); +#else + if (__is_overaligned_for_new(__align)) { + const align_val_t __align_val = static_cast(__align); + return __do_deallocate_handle_size(__ptr, __size, __align_val); + } else { + return __do_deallocate_handle_size(__ptr, __size); + } +#endif } + + static inline _LIBCPP_INLINE_VISIBILITY + void __do_deallocate_handle_align(void *__ptr, size_t __align) { +#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) + ((void)__align); + return __do_call(__ptr); #else - ((void)__align); + if (__is_overaligned_for_new(__align)) { + const align_val_t __align_val = static_cast(__align); + return __do_call(__ptr, __align_val); + } else { + return __do_call(__ptr); + } #endif + } + + private: + static inline void __do_deallocate_handle_size(void *__ptr, size_t __size) { +#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION + ((void)__size); + return __do_call(__ptr); +#else + return __do_call(__ptr, __size); +#endif + } + +#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION + static inline void __do_deallocate_handle_size(void *__ptr, size_t __size, align_val_t __align) { +#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION + ((void)__size); + return __do_call(__ptr, __align); +#else + return __do_call(__ptr, __size, __align); +#endif + } +#endif + +private: + template + static inline void __do_call(void *__ptr, _A1 __a1, _A2 __a2) { +#if defined(_LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE) || \ + defined(_LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE) + return ::operator delete(__ptr, __a1, __a2); +#else + return __builtin_operator_delete(__ptr, __a1, __a2); +#endif + } + + template + static inline void __do_call(void *__ptr, _A1 __a1) { +#if defined(_LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE) || \ + defined(_LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE) + return ::operator delete(__ptr, __a1); +#else + return __builtin_operator_delete(__ptr, __a1); +#endif + } + + static inline void __do_call(void *__ptr) { #ifdef _LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE - return ::operator delete(__ptr); + return ::operator delete(__ptr); #else - return __builtin_operator_delete(__ptr); + return __builtin_operator_delete(__ptr); #endif + } +}; + +inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) { + _DeallocateCaller::__do_deallocate_handle_size_align(__ptr, __size, __align); } -#ifdef _LIBCPP_BAD_ARRAY_LENGTH_DEFINED -_LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY -#ifndef _LIBCPP_NO_EXCEPTIONS -_LIBCPP_AVAILABILITY_BAD_ARRAY_LENGTH -#endif -void __throw_bad_array_length() -{ -#ifndef _LIBCPP_NO_EXCEPTIONS - throw bad_array_length(); -#else - _VSTD::abort(); -#endif +inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate_unsized(void* __ptr, size_t __align) { + _DeallocateCaller::__do_deallocate_handle_align(__ptr, __align); } -#endif template -_LIBCPP_NODISCARD_AFTER_CXX17 inline +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_CONSTEXPR _Tp* __launder(_Tp* __p) _NOEXCEPT { static_assert (!(is_function<_Tp>::value), "can't launder functions" ); diff --git a/contrib/libc++/include/numeric b/contrib/libc++/include/numeric index b33b6a398eb..4e68239d059 100644 --- a/contrib/libc++/include/numeric +++ b/contrib/libc++/include/numeric @@ -104,15 +104,15 @@ template - OutputIterator - transform_inclusive_scan(InputIterator first, InputIterator last, + OutputIterator + transform_inclusive_scan(InputIterator first, InputIterator last, OutputIterator result, BinaryOperation binary_op, UnaryOperation unary_op); // C++17 template - OutputIterator - transform_inclusive_scan(InputIterator first, InputIterator last, + OutputIterator + transform_inclusive_scan(InputIterator first, InputIterator last, OutputIterator result, BinaryOperation binary_op, UnaryOperation unary_op, T init); // C++17 @@ -142,6 +142,7 @@ template #include #include // for numeric_limits #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header diff --git a/contrib/libc++/include/optional b/contrib/libc++/include/optional index a76f8d18976..70422068e28 100644 --- a/contrib/libc++/include/optional +++ b/contrib/libc++/include/optional @@ -105,8 +105,8 @@ namespace std { // 23.6.3.3, assignment optional &operator=(nullopt_t) noexcept; - optional &operator=(const optional &); - optional &operator=(optional &&) noexcept(see below ); + optional &operator=(const optional &); // constexpr in C++20 + optional &operator=(optional &&) noexcept(see below); // constexpr in C++20 template optional &operator=(U &&); template optional &operator=(const optional &); template optional &operator=(optional &&); @@ -156,6 +156,7 @@ template #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -168,7 +169,7 @@ _LIBCPP_PUSH_MACROS namespace std // purposefully not using versioning namespace { -class _LIBCPP_EXCEPTION_ABI bad_optional_access +class _LIBCPP_EXCEPTION_ABI _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS bad_optional_access : public exception { public: @@ -185,6 +186,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS void __throw_bad_optional_access() { #ifndef _LIBCPP_NO_EXCEPTIONS throw bad_optional_access(); @@ -932,6 +934,7 @@ public: using __base::__get; _LIBCPP_INLINE_VISIBILITY + _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type const& value() const& { if (!this->has_value()) @@ -940,6 +943,7 @@ public: } _LIBCPP_INLINE_VISIBILITY + _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type& value() & { if (!this->has_value()) @@ -948,6 +952,7 @@ public: } _LIBCPP_INLINE_VISIBILITY + _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type&& value() && { if (!this->has_value()) @@ -956,6 +961,7 @@ public: } _LIBCPP_INLINE_VISIBILITY + _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type const&& value() const&& { if (!this->has_value()) diff --git a/contrib/libc++/include/ostream b/contrib/libc++/include/ostream index 5404e0dca6c..d700a369b34 100644 --- a/contrib/libc++/include/ostream +++ b/contrib/libc++/include/ostream @@ -140,6 +140,7 @@ template #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -160,7 +161,7 @@ public: typedef typename traits_type::off_type off_type; // 27.7.2.2 Constructor/destructor: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_ostream(basic_streambuf* __sb) { this->init(__sb); } virtual ~basic_ostream(); @@ -173,7 +174,7 @@ protected: inline _LIBCPP_INLINE_VISIBILITY basic_ostream& operator=(basic_ostream&& __rhs); #endif - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_ostream& __rhs) { basic_ios::swap(__rhs); } @@ -190,16 +191,16 @@ public: class _LIBCPP_TEMPLATE_VIS sentry; // 27.7.2.6 Formatted output: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(basic_ostream& (*__pf)(basic_ostream&)) { return __pf(*this); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(basic_ios& (*__pf)(basic_ios&)) { __pf(*this); return *this; } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(ios_base& (*__pf)(ios_base&)) { __pf(*this); return *this; } @@ -224,11 +225,11 @@ public: basic_ostream& flush(); // 27.7.2.5 seeks: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type tellp(); - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& seekp(pos_type __pos); - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& seekp(off_type __off, ios_base::seekdir __dir); protected: @@ -1092,7 +1093,7 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const bitset<_Size>& __x) use_facet >(__os.getloc()).widen('1')); } -#ifndef _LIBCPP_AVAILABILITY_NO_STREAMS_EXTERN_TEMPLATE +#ifndef _LIBCPP_DO_NOT_ASSUME_STREAMS_EXPLICIT_INSTANTIATION_IN_DYLIB _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ostream) _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_ostream) #endif diff --git a/contrib/libc++/include/random b/contrib/libc++/include/random index 89664a6ca33..724bd0fc215 100644 --- a/contrib/libc++/include/random +++ b/contrib/libc++/include/random @@ -2337,7 +2337,7 @@ mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, for (size_t __i = 1; __i < __n; ++__i) if (__x_[__i] != 0) return; - __x_[0] = _Max; + __x_[0] = result_type(1) << (__w - 1); } } @@ -2363,7 +2363,7 @@ mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, for (size_t __i = 1; __i < __n; ++__i) if (__x_[__i] != 0) return; - __x_[0] = _Max; + __x_[0] = result_type(1) << (__w - 1); } } diff --git a/contrib/libc++/include/regex b/contrib/libc++/include/regex index dcdb14af9af..bd83d7c10ca 100644 --- a/contrib/libc++/include/regex +++ b/contrib/libc++/include/regex @@ -769,6 +769,7 @@ typedef regex_token_iterator wsregex_token_iterator; #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -989,6 +990,10 @@ public: #if defined(__mips__) && defined(__GLIBC__) static const char_class_type __regex_word = static_cast(_ISbit(15)); +#elif defined(__NetBSD__) + // NetBSD defines classes up to 0x2000 + // see sys/ctype_bits.h, _CTYPE_Q + static const char_class_type __regex_word = 0x8000; #else static const char_class_type __regex_word = 0x80; #endif @@ -1351,9 +1356,9 @@ public: virtual ~__node() {} _LIBCPP_INLINE_VISIBILITY - virtual void __exec(__state&) const {}; + virtual void __exec(__state&) const {} _LIBCPP_INLINE_VISIBILITY - virtual void __exec_split(bool, __state&) const {}; + virtual void __exec_split(bool, __state&) const {} }; // __end_state diff --git a/contrib/libc++/include/scoped_allocator b/contrib/libc++/include/scoped_allocator index 4760d946def..bdbb0136b5c 100644 --- a/contrib/libc++/include/scoped_allocator +++ b/contrib/libc++/include/scoped_allocator @@ -108,6 +108,7 @@ template #include <__config> #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header diff --git a/contrib/libc++/include/set b/contrib/libc++/include/set index 108a9e97f88..a0155f0b275 100644 --- a/contrib/libc++/include/set +++ b/contrib/libc++/include/set @@ -128,6 +128,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(set& source); // C++17 + template + void merge(set&& source); // C++17 + template + void merge(multiset& source); // C++17 + template + void merge(multiset&& source); // C++17 + void swap(set& s) noexcept( __is_nothrow_swappable::value && @@ -207,6 +216,9 @@ void swap(set& x, set& y) noexcept(noexcept(x.swap(y))); +template + void erase_if(set& c, Predicate pred); // C++20 + template , class Allocator = allocator> class multiset @@ -316,6 +328,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(multiset& source); // C++17 + template + void merge(multiset&& source); // C++17 + template + void merge(set& source); // C++17 + template + void merge(set&& source); // C++17 + void swap(multiset& s) noexcept( __is_nothrow_swappable::value && @@ -394,6 +415,9 @@ void swap(multiset& x, multiset& y) noexcept(noexcept(x.swap(y))); +template + void erase_if(multiset& c, Predicate pred); // C++20 + } // std */ @@ -402,6 +426,7 @@ swap(multiset& x, multiset& y) #include <__tree> #include <__node_handle> #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -409,6 +434,9 @@ swap(multiset& x, multiset& y) _LIBCPP_BEGIN_NAMESPACE_STD +template +class multiset; + template , class _Allocator = allocator<_Key> > class _LIBCPP_TEMPLATE_VIS set @@ -423,6 +451,7 @@ public: typedef value_type& reference; typedef const value_type& const_reference; + static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); static_assert((is_same::value), "Allocator::value_type must be same type as value_type"); @@ -448,6 +477,11 @@ public: typedef __insert_return_type insert_return_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS set; + template + friend class _LIBCPP_TEMPLATE_VIS multiset; + _LIBCPP_INLINE_VISIBILITY set() _NOEXCEPT_( @@ -486,7 +520,7 @@ public: #if _LIBCPP_STD_VER > 11 template - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_INLINE_VISIBILITY set(_InputIterator __f, _InputIterator __l, const allocator_type& __a) : set(__f, __l, key_compare(), __a) {} #endif @@ -542,7 +576,7 @@ public: } #if _LIBCPP_STD_VER > 11 - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_INLINE_VISIBILITY set(initializer_list __il, const allocator_type& __a) : set(__il, key_compare(), __a) {} #endif @@ -680,6 +714,38 @@ public: { return __tree_.template __node_handle_extract(__it); } + template + _LIBCPP_INLINE_VISIBILITY + void merge(set& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(set&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multiset& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multiset&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_unique(__source.__tree_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -852,6 +918,13 @@ swap(set<_Key, _Compare, _Allocator>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(set<_Key, _Compare, _Allocator>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + template , class _Allocator = allocator<_Key> > class _LIBCPP_TEMPLATE_VIS multiset @@ -866,6 +939,7 @@ public: typedef value_type& reference; typedef const value_type& const_reference; + static_assert(sizeof(__diagnose_non_const_comparator<_Key, _Compare>()), ""); static_assert((is_same::value), "Allocator::value_type must be same type as value_type"); @@ -890,6 +964,11 @@ public: typedef __set_node_handle node_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS set; + template + friend class _LIBCPP_TEMPLATE_VIS multiset; + // construct/copy/destroy: _LIBCPP_INLINE_VISIBILITY multiset() @@ -920,7 +999,7 @@ public: #if _LIBCPP_STD_VER > 11 template - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_INLINE_VISIBILITY multiset(_InputIterator __f, _InputIterator __l, const allocator_type& __a) : multiset(__f, __l, key_compare(), __a) {} #endif @@ -984,7 +1063,7 @@ public: } #if _LIBCPP_STD_VER > 11 - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_INLINE_VISIBILITY multiset(initializer_list __il, const allocator_type& __a) : multiset(__il, key_compare(), __a) {} #endif @@ -1121,6 +1200,38 @@ public: { return __tree_.template __node_handle_extract(__it); } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multiset& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_multi(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(multiset&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_multi(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(set& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_multi(__source.__tree_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(set&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __tree_.__node_handle_merge_multi(__source.__tree_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -1294,6 +1405,13 @@ swap(multiset<_Key, _Compare, _Allocator>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(multiset<_Key, _Compare, _Allocator>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP_SET diff --git a/contrib/libc++/include/shared_mutex b/contrib/libc++/include/shared_mutex index a7735d6732c..3daf74d26c7 100644 --- a/contrib/libc++/include/shared_mutex +++ b/contrib/libc++/include/shared_mutex @@ -124,6 +124,7 @@ template */ #include <__config> +#include _LIBCPP_PUSH_MACROS #include <__undef_macros> @@ -143,7 +144,8 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -struct _LIBCPP_TYPE_VIS _LIBCPP_AVAILABILITY_SHARED_MUTEX __shared_mutex_base +struct _LIBCPP_TYPE_VIS _LIBCPP_AVAILABILITY_SHARED_MUTEX _LIBCPP_THREAD_SAFETY_ANNOTATION(capability("shared_mutex")) +__shared_mutex_base { mutex __mut_; condition_variable __gate1_; @@ -160,14 +162,14 @@ struct _LIBCPP_TYPE_VIS _LIBCPP_AVAILABILITY_SHARED_MUTEX __shared_mutex_base __shared_mutex_base& operator=(const __shared_mutex_base&) = delete; // Exclusive ownership - void lock(); // blocking - bool try_lock(); - void unlock(); + void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability()); // blocking + bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(try_acquire_capability(true)); + void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()); // Shared ownership - void lock_shared(); // blocking - bool try_lock_shared(); - void unlock_shared(); + void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_shared_capability()); // blocking + bool try_lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(try_acquire_shared_capability(true)); + void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_shared_capability()); // typedef implementation-defined native_handle_type; // See 30.2.3 // native_handle_type native_handle(); // See 30.2.3 diff --git a/contrib/libc++/include/span b/contrib/libc++/include/span index db8f836f918..cebe98760f2 100644 --- a/contrib/libc++/include/span +++ b/contrib/libc++/include/span @@ -23,27 +23,13 @@ inline constexpr ptrdiff_t dynamic_extent = -1; template class span; -// [span.comparison], span comparison operators -template - constexpr bool operator==(span l, span r); -template - constexpr bool operator!=(span l, span r); -template - constexpr bool operator<(span l, span r); -template - constexpr bool operator<=(span l, span r); -template - constexpr bool operator>(span l, span r); -template - constexpr bool operator>=(span l, span r); - // [span.objectrep], views of object representation template - span(sizeof(ElementType)) * Extent))> as_bytes(span s) noexcept; template - span< byte, ((Extent == dynamic_extent) ? dynamic_extent : + span< byte, ((Extent == dynamic_extent) ? dynamic_extent : (static_cast(sizeof(ElementType)) * Extent))> as_writable_bytes(span s) noexcept; @@ -123,7 +109,7 @@ private: template span(T (&)[N]) -> span; - + template span(array&) -> span; @@ -194,8 +180,8 @@ struct __is_span_compatible_container<_Tp, _ElementType, decltype(size(declval<_Tp>())), // remove_pointer_t(*)[] is convertible to ElementType(*)[] typename enable_if< - is_convertible_v()))>(*)[], - _ElementType(*)[]>, + is_convertible_v()))>(*)[], + _ElementType(*)[]>, nullptr_t>::type >> : public true_type {}; @@ -221,7 +207,7 @@ public: static constexpr index_type extent = _Extent; static_assert (_Extent >= 0, "Can't have a span with an extent < 0"); -// [span.cons], span constructors, copy, assignment, and destructor +// [span.cons], span constructors, copy, assignment, and destructor _LIBCPP_INLINE_VISIBILITY constexpr span() noexcept : __data{nullptr} { static_assert(_Extent == 0, "Can't default construct a statically sized span with size > 0"); } @@ -242,7 +228,7 @@ public: constexpr span( _Container& __c, enable_if_t<__is_span_compatible_container<_Container, _Tp>::value, nullptr_t> = nullptr) : __data{_VSTD::data(__c)} - { _LIBCPP_ASSERT(_Extent == _VSTD::size(__c), "size mismatch in span's constructor (container))"); } + { _LIBCPP_ASSERT(_Extent == _VSTD::size(__c), "size mismatch in span's constructor (container)"); } template inline _LIBCPP_INLINE_VISIBILITY @@ -287,7 +273,7 @@ public: static_assert(_Count <= _Extent, "Count out of range in span::last()"); return {data() + size() - _Count, _Count}; } - + _LIBCPP_INLINE_VISIBILITY constexpr span first(index_type __count) const noexcept { @@ -315,7 +301,7 @@ public: inline _LIBCPP_INLINE_VISIBILITY constexpr span subspan(index_type __offset, index_type __count = dynamic_extent) const noexcept - { + { _LIBCPP_ASSERT( __offset >= 0 && __offset <= size(), "Offset out of range in span::subspan(offset, count)"); _LIBCPP_ASSERT((__count >= 0 && __count <= size()) || __count == dynamic_extent, "Count out of range in span::subspan(offset, count)"); if (__count == dynamic_extent) @@ -331,14 +317,14 @@ public: _LIBCPP_INLINE_VISIBILITY constexpr reference operator[](index_type __idx) const noexcept { _LIBCPP_ASSERT(__idx >= 0 && __idx < size(), "span[] index out of bounds"); - return __data[__idx]; - } + return __data[__idx]; + } _LIBCPP_INLINE_VISIBILITY constexpr reference operator()(index_type __idx) const noexcept { _LIBCPP_ASSERT(__idx >= 0 && __idx < size(), "span() index out of bounds"); - return __data[__idx]; - } + return __data[__idx]; + } _LIBCPP_INLINE_VISIBILITY constexpr pointer data() const noexcept { return __data; } @@ -358,7 +344,7 @@ public: __data = __other.__data; __other.__data = __p; } - + _LIBCPP_INLINE_VISIBILITY span __as_bytes() const noexcept { return {reinterpret_cast(data()), size_bytes()}; } @@ -392,7 +378,7 @@ public: static constexpr index_type extent = dynamic_extent; -// [span.cons], span constructors, copy, assignment, and destructor +// [span.cons], span constructors, copy, assignment, and destructor _LIBCPP_INLINE_VISIBILITY constexpr span() noexcept : __data{nullptr}, __size{0} {} constexpr span (const span&) noexcept = default; @@ -408,7 +394,7 @@ public: template inline _LIBCPP_INLINE_VISIBILITY constexpr span(array& __arr) noexcept : __data{__arr.data()}, __size{_Sz} {} - + template inline _LIBCPP_INLINE_VISIBILITY constexpr span(const array& __arr) noexcept : __data{__arr.data()}, __size{_Sz} {} @@ -440,7 +426,7 @@ public: inline _LIBCPP_INLINE_VISIBILITY constexpr span first() const noexcept { - static_assert(_Count >= 0); + static_assert(_Count >= 0, "Count must be >= 0 in span::first()"); _LIBCPP_ASSERT(_Count <= size(), "Count out of range in span::first()"); return {data(), _Count}; } @@ -449,7 +435,7 @@ public: inline _LIBCPP_INLINE_VISIBILITY constexpr span last() const noexcept { - static_assert(_Count >= 0); + static_assert(_Count >= 0, "Count must be >= 0 in span::last()"); _LIBCPP_ASSERT(_Count <= size(), "Count out of range in span::last()"); return {data() + size() - _Count, _Count}; } @@ -460,7 +446,7 @@ public: _LIBCPP_ASSERT(__count >= 0 && __count <= size(), "Count out of range in span::first(count)"); return {data(), __count}; } - + _LIBCPP_INLINE_VISIBILITY constexpr span last (index_type __count) const noexcept { @@ -480,7 +466,7 @@ public: constexpr span inline _LIBCPP_INLINE_VISIBILITY subspan(index_type __offset, index_type __count = dynamic_extent) const noexcept - { + { _LIBCPP_ASSERT( __offset >= 0 && __offset <= size(), "Offset out of range in span::subspan(offset, count)"); _LIBCPP_ASSERT((__count >= 0 && __count <= size()) || __count == dynamic_extent, "count out of range in span::subspan(offset, count)"); if (__count == dynamic_extent) @@ -496,14 +482,14 @@ public: _LIBCPP_INLINE_VISIBILITY constexpr reference operator[](index_type __idx) const noexcept { _LIBCPP_ASSERT(__idx >= 0 && __idx < size(), "span[] index out of bounds"); - return __data[__idx]; - } + return __data[__idx]; + } _LIBCPP_INLINE_VISIBILITY constexpr reference operator()(index_type __idx) const noexcept { _LIBCPP_ASSERT(__idx >= 0 && __idx < size(), "span() index out of bounds"); - return __data[__idx]; - } + return __data[__idx]; + } _LIBCPP_INLINE_VISIBILITY constexpr pointer data() const noexcept { return __data; } @@ -539,39 +525,9 @@ private: index_type __size; }; -template - constexpr bool - operator==(const span<_Tp1, _Extent1>& __lhs, const span<_Tp2, _Extent2>& __rhs) - { return equal(__lhs.begin(), __lhs.end(), __rhs.begin(), __rhs.end()); } - -template - constexpr bool - operator!=(const span<_Tp1, _Extent1>& __lhs, const span<_Tp2, _Extent2>& __rhs) - { return !(__rhs == __lhs); } - -template - constexpr bool - operator< (const span<_Tp1, _Extent1>& __lhs, const span<_Tp2, _Extent2>& __rhs) - { return lexicographical_compare (__lhs.begin(), __lhs.end(), __rhs.begin(), __rhs.end()); } - -template - constexpr bool - operator<=(const span<_Tp1, _Extent1>& __lhs, const span<_Tp2, _Extent2>& __rhs) - { return !(__rhs < __lhs); } - -template - constexpr bool - operator> (const span<_Tp1, _Extent1>& __lhs, const span<_Tp2, _Extent2>& __rhs) - { return __rhs < __lhs; } - -template - constexpr bool - operator>=(const span<_Tp1, _Extent1>& __lhs, const span<_Tp2, _Extent2>& __rhs) - { return !(__lhs < __rhs); } - // as_bytes & as_writeable_bytes template - auto as_bytes(span<_Tp, _Extent> __s) noexcept + auto as_bytes(span<_Tp, _Extent> __s) noexcept -> decltype(__s.__as_bytes()) { return __s.__as_bytes(); } @@ -588,7 +544,7 @@ template // Deduction guides template span(_Tp (&)[_Sz]) -> span<_Tp, _Sz>; - + template span(array<_Tp, _Sz>&) -> span<_Tp, _Sz>; diff --git a/contrib/libc++/include/sstream b/contrib/libc++/include/sstream index b01f47b6872..9c3ee13bfba 100644 --- a/contrib/libc++/include/sstream +++ b/contrib/libc++/include/sstream @@ -473,12 +473,12 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::str(const string_type& __s) { while (__sz > INT_MAX) { - this->pbump(INT_MAX); - __sz -= INT_MAX; + this->pbump(INT_MAX); + __sz -= INT_MAX; } if (__sz > 0) - this->pbump(__sz); - } + this->pbump(__sz); + } } } diff --git a/contrib/libc++/include/stddef.h b/contrib/libc++/include/stddef.h index faf8552d8ce..f65065d869a 100644 --- a/contrib/libc++/include/stddef.h +++ b/contrib/libc++/include/stddef.h @@ -54,7 +54,7 @@ using std::nullptr_t; // Re-use the compiler's max_align_t where possible. #if !defined(__CLANG_MAX_ALIGN_T_DEFINED) && !defined(_GCC_MAX_ALIGN_T) && \ - !defined(__DEFINED_max_align_t) + !defined(__DEFINED_max_align_t) && !defined(__NetBSD__) typedef long double max_align_t; #endif diff --git a/contrib/libc++/include/stdexcept b/contrib/libc++/include/stdexcept index 6533497e4ea..3ec79349aa9 100644 --- a/contrib/libc++/include/stdexcept +++ b/contrib/libc++/include/stdexcept @@ -192,7 +192,7 @@ void __throw_logic_error(const char*__msg) throw logic_error(__msg); #else ((void)__msg); - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -203,7 +203,7 @@ void __throw_domain_error(const char*__msg) throw domain_error(__msg); #else ((void)__msg); - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -214,7 +214,7 @@ void __throw_invalid_argument(const char*__msg) throw invalid_argument(__msg); #else ((void)__msg); - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -225,7 +225,7 @@ void __throw_length_error(const char*__msg) throw length_error(__msg); #else ((void)__msg); - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -236,7 +236,7 @@ void __throw_out_of_range(const char*__msg) throw out_of_range(__msg); #else ((void)__msg); - _VSTD::abort(); + _VSTD::abort(); #endif } @@ -247,7 +247,7 @@ void __throw_range_error(const char*__msg) throw range_error(__msg); #else ((void)__msg); - _VSTD::abort(); + _VSTD::abort(); #endif } diff --git a/contrib/libc++/include/streambuf b/contrib/libc++/include/streambuf index 37a65323716..dd293dc639b 100644 --- a/contrib/libc++/include/streambuf +++ b/contrib/libc++/include/streambuf @@ -138,7 +138,7 @@ public: virtual ~basic_streambuf(); // 27.6.2.2.1 locales: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 locale pubimbue(const locale& __loc) { imbue(__loc); locale __r = __loc_; @@ -146,70 +146,70 @@ public: return __r; } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 locale getloc() const { return __loc_; } // 27.6.2.2.2 buffer and positioning: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_streambuf* pubsetbuf(char_type* __s, streamsize __n) { return setbuf(__s, __n); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type pubseekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __which = ios_base::in | ios_base::out) { return seekoff(__off, __way, __which); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type pubseekpos(pos_type __sp, ios_base::openmode __which = ios_base::in | ios_base::out) { return seekpos(__sp, __which); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int pubsync() { return sync(); } // Get and put areas: // 27.6.2.2.3 Get area: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize in_avail() { if (__ninp_ < __einp_) return static_cast(__einp_ - __ninp_); return showmanyc(); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type snextc() { if (sbumpc() == traits_type::eof()) return traits_type::eof(); return sgetc(); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sbumpc() { if (__ninp_ == __einp_) return uflow(); return traits_type::to_int_type(*__ninp_++); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sgetc() { if (__ninp_ == __einp_) return underflow(); return traits_type::to_int_type(*__ninp_); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize sgetn(char_type* __s, streamsize __n) { return xsgetn(__s, __n); } // 27.6.2.2.4 Putback: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputbackc(char_type __c) { if (__binp_ == __ninp_ || !traits_type::eq(__c, __ninp_[-1])) return pbackfail(traits_type::to_int_type(__c)); return traits_type::to_int_type(*--__ninp_); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sungetc() { if (__binp_ == __ninp_) return pbackfail(); @@ -217,7 +217,7 @@ public: } // 27.6.2.2.5 Put area: - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputc(char_type __c) { if (__nout_ == __eout_) return overflow(traits_type::to_int_type(__c)); @@ -225,7 +225,7 @@ public: return traits_type::to_int_type(__c); } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize sputn(const char_type* __s, streamsize __n) { return xsputn(__s, __n); } @@ -240,10 +240,10 @@ protected: _LIBCPP_INLINE_VISIBILITY char_type* gptr() const {return __ninp_;} _LIBCPP_INLINE_VISIBILITY char_type* egptr() const {return __einp_;} - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void gbump(int __n) { __ninp_ += __n; } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void setg(char_type* __gbeg, char_type* __gnext, char_type* __gend) { __binp_ = __gbeg; __ninp_ = __gnext; @@ -255,13 +255,13 @@ protected: _LIBCPP_INLINE_VISIBILITY char_type* pptr() const {return __nout_;} _LIBCPP_INLINE_VISIBILITY char_type* epptr() const {return __eout_;} - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void pbump(int __n) { __nout_ += __n; } _LIBCPP_INLINE_VISIBILITY void __pbump(streamsize __n) { __nout_ += __n; } - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void setp(char_type* __pbeg, char_type* __pend) { __bout_ = __nout_ = __pbeg; __eout_ = __pend; @@ -486,7 +486,7 @@ basic_streambuf<_CharT, _Traits>::overflow(int_type) return traits_type::eof(); } -#ifndef _LIBCPP_AVAILABILITY_NO_STREAMS_EXTERN_TEMPLATE +#ifndef _LIBCPP_DO_NOT_ASSUME_STREAMS_EXPLICIT_INSTANTIATION_IN_DYLIB _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_streambuf) _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS basic_streambuf) diff --git a/contrib/libc++/include/string b/contrib/libc++/include/string index 162e54058c4..fb838d1e5dd 100644 --- a/contrib/libc++/include/string +++ b/contrib/libc++/include/string @@ -437,6 +437,11 @@ template basic_istream& getline(basic_istream& is, basic_string& str); +template +void erase(basic_string& c, const U& value); // C++20 +template +void erase_if(basic_string& c, Predicate pred); // C++20 + typedef basic_string string; typedef basic_string wstring; typedef basic_string u16string; @@ -510,6 +515,7 @@ basic_string operator "" s( const char32_t *str, size_t len ); // C++1 #include #include #include <__functional_base> +#include #ifndef _LIBCPP_HAS_NO_UNICODE_CHARS #include #endif @@ -579,6 +585,7 @@ basic_string<_CharT, _Traits, _Allocator> operator+(_CharT __x, const basic_string<_CharT,_Traits,_Allocator>& __y); template +inline _LIBCPP_INLINE_VISIBILITY basic_string<_CharT, _Traits, _Allocator> operator+(const basic_string<_CharT, _Traits, _Allocator>& __x, const _CharT* __y); @@ -806,7 +813,9 @@ public: basic_string(basic_string&& __str, const allocator_type& __a); #endif // _LIBCPP_CXX03_LANG +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES template ::value, nullptr_t>::type> +#endif _LIBCPP_INLINE_VISIBILITY basic_string(const _CharT* __s) { _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*) detected nullptr"); @@ -816,7 +825,9 @@ public: # endif } +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES template ::value, nullptr_t>::type> +#endif _LIBCPP_INLINE_VISIBILITY basic_string(const _CharT* __s, const _Allocator& __a); @@ -827,7 +838,9 @@ public: _LIBCPP_INLINE_VISIBILITY basic_string(size_type __n, _CharT __c); +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES template ::value, nullptr_t>::type> +#endif _LIBCPP_INLINE_VISIBILITY basic_string(size_type __n, _CharT __c, const _Allocator& __a); @@ -948,7 +961,11 @@ public: void resize(size_type __n, value_type __c); _LIBCPP_INLINE_VISIBILITY void resize(size_type __n) {resize(__n, value_type());} - void reserve(size_type __res_arg = 0); + void reserve(size_type __res_arg); + _LIBCPP_INLINE_VISIBILITY void __resize_default_init(size_type __n); + + _LIBCPP_INLINE_VISIBILITY + void reserve() _NOEXCEPT {reserve(0);} _LIBCPP_INLINE_VISIBILITY void shrink_to_fit() _NOEXCEPT {reserve();} _LIBCPP_INLINE_VISIBILITY @@ -1002,6 +1019,10 @@ public: basic_string& append(const value_type* __s, size_type __n); basic_string& append(const value_type* __s); basic_string& append(size_type __n, value_type __c); + + _LIBCPP_INLINE_VISIBILITY + void __append_default_init(size_type __n); + template _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS basic_string& __append_forward_unsafe(_ForwardIterator, _ForwardIterator); @@ -1664,7 +1685,7 @@ basic_string(basic_string_view<_CharT, _Traits>, _Sz, _Sz, const _Allocator& = _ template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::__invalidate_all_iterators() { @@ -1674,7 +1695,7 @@ basic_string<_CharT, _Traits, _Allocator>::__invalidate_all_iterators() } template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::__invalidate_iterators_past(size_type #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1704,7 +1725,7 @@ basic_string<_CharT, _Traits, _Allocator>::__invalidate_iterators_past(size_type } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string() _NOEXCEPT_(is_nothrow_default_constructible::value) { @@ -1715,7 +1736,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string() } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(const allocator_type& __a) #if _LIBCPP_STD_VER <= 14 _NOEXCEPT_(is_nothrow_copy_constructible::value) @@ -1780,7 +1801,9 @@ basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_ty } template +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES template +#endif basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a) : __r_(__second_tag(), __a) { @@ -1792,7 +1815,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_type __n) { _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "basic_string(const char*, n) detected nullptr"); @@ -1803,7 +1826,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_ } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_type __n, const _Allocator& __a) : __r_(__second_tag(), __a) { @@ -1844,7 +1867,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string( #ifndef _LIBCPP_CXX03_LANG template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(basic_string&& __str) #if _LIBCPP_STD_VER <= 14 _NOEXCEPT_(is_nothrow_move_constructible::value) @@ -1862,7 +1885,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(basic_string&& __str) } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(basic_string&& __str, const allocator_type& __a) : __r_(__second_tag(), __a) { @@ -1907,7 +1930,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(size_type __n, value_type __c) } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(size_type __n, _CharT __c) { __init(__n, __c); @@ -1917,7 +1940,9 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(size_type __n, _CharT __ } template +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES template +#endif basic_string<_CharT, _Traits, _Allocator>::basic_string(size_type __n, _CharT __c, const _Allocator& __a) : __r_(__second_tag(), __a) { @@ -1943,7 +1968,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __st } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str, size_type __pos, const _Allocator& __a) : __r_(__second_tag(), __a) @@ -2054,7 +2079,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(_ForwardIterator __first, _For template template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first, _InputIterator __last) { __init(__first, __last); @@ -2065,7 +2090,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first, template template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first, _InputIterator __last, const allocator_type& __a) : __r_(__second_tag(), __a) @@ -2079,7 +2104,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first, #ifndef _LIBCPP_CXX03_LANG template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string( initializer_list<_CharT> __il) { @@ -2090,7 +2115,7 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string( } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>::basic_string( initializer_list<_CharT> __il, const _Allocator& __a) @@ -2254,7 +2279,7 @@ basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str) #ifndef _LIBCPP_CXX03_LANG template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, false_type) _NOEXCEPT_(__alloc_traits::is_always_equal::value) @@ -2266,7 +2291,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, fa } template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, true_type) #if _LIBCPP_STD_VER > 14 @@ -2282,7 +2307,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::operator=(basic_string&& __str) _NOEXCEPT_((__noexcept_move_assign_container<_Allocator, __alloc_traits>::value)) @@ -2415,6 +2440,23 @@ basic_string<_CharT, _Traits, _Allocator>::append(size_type __n, value_type __c) return *this; } +template +inline void +basic_string<_CharT, _Traits, _Allocator>::__append_default_init(size_type __n) +{ + if (__n) + { + size_type __cap = capacity(); + size_type __sz = size(); + if (__cap - __sz < __n) + __grow_by(__cap, __sz + __n - __cap, __sz, __sz, 0); + pointer __p = __get_pointer(); + __sz += __n; + __set_size(__sz); + traits_type::assign(__p[__sz], value_type()); + } +} + template void basic_string<_CharT, _Traits, _Allocator>::push_back(value_type __c) @@ -2499,7 +2541,7 @@ basic_string<_CharT, _Traits, _Allocator>::__append_forward_unsafe( } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::append(const basic_string& __str) { @@ -2676,7 +2718,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, _Forward } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos1, const basic_string& __str) { @@ -2746,7 +2788,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, value_ty } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::iterator basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, size_type __n, value_type __c) { @@ -2866,7 +2908,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_it } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos1, size_type __n1, const basic_string& __str) { @@ -2910,7 +2952,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __ } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_iterator __i2, const basic_string& __str) { @@ -2919,7 +2961,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_it } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_iterator __i2, const value_type* __s, size_type __n) { @@ -2927,7 +2969,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_it } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_iterator __i2, const value_type* __s) { @@ -2935,7 +2977,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_it } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::replace(const_iterator __i1, const_iterator __i2, size_type __n, value_type __c) { @@ -2967,7 +3009,7 @@ basic_string<_CharT, _Traits, _Allocator>::erase(size_type __pos, size_type __n) } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::iterator basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __pos) { @@ -2985,7 +3027,7 @@ basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __pos) } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::iterator basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __first, const_iterator __last) { @@ -3002,7 +3044,7 @@ basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __first, const_i } template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::pop_back() { @@ -3024,7 +3066,7 @@ basic_string<_CharT, _Traits, _Allocator>::pop_back() } template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT { @@ -3042,7 +3084,7 @@ basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT } template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::__erase_to_end(size_type __pos) { @@ -3071,7 +3113,18 @@ basic_string<_CharT, _Traits, _Allocator>::resize(size_type __n, value_type __c) } template -inline _LIBCPP_INLINE_VISIBILITY +inline void +basic_string<_CharT, _Traits, _Allocator>::__resize_default_init(size_type __n) +{ + size_type __sz = size(); + if (__n > __sz) { + __append_default_init(__n - __sz); + } else + __erase_to_end(__n); +} + +template +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::max_size() const _NOEXCEPT { @@ -3147,7 +3200,7 @@ basic_string<_CharT, _Traits, _Allocator>::reserve(size_type __res_arg) } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::const_reference basic_string<_CharT, _Traits, _Allocator>::operator[](size_type __pos) const _NOEXCEPT { @@ -3156,7 +3209,7 @@ basic_string<_CharT, _Traits, _Allocator>::operator[](size_type __pos) const _NO } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::reference basic_string<_CharT, _Traits, _Allocator>::operator[](size_type __pos) _NOEXCEPT { @@ -3183,7 +3236,7 @@ basic_string<_CharT, _Traits, _Allocator>::at(size_type __n) } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::reference basic_string<_CharT, _Traits, _Allocator>::front() { @@ -3192,7 +3245,7 @@ basic_string<_CharT, _Traits, _Allocator>::front() } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::const_reference basic_string<_CharT, _Traits, _Allocator>::front() const { @@ -3201,7 +3254,7 @@ basic_string<_CharT, _Traits, _Allocator>::front() const } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::reference basic_string<_CharT, _Traits, _Allocator>::back() { @@ -3210,7 +3263,7 @@ basic_string<_CharT, _Traits, _Allocator>::back() } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::const_reference basic_string<_CharT, _Traits, _Allocator>::back() const { @@ -3231,7 +3284,7 @@ basic_string<_CharT, _Traits, _Allocator>::copy(value_type* __s, size_type __n, } template -inline _LIBCPP_INLINE_VISIBILITY +inline basic_string<_CharT, _Traits, _Allocator> basic_string<_CharT, _Traits, _Allocator>::substr(size_type __pos, size_type __n) const { @@ -3239,7 +3292,7 @@ basic_string<_CharT, _Traits, _Allocator>::substr(size_type __pos, size_type __n } template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::swap(basic_string& __str) #if _LIBCPP_STD_VER >= 14 @@ -3287,7 +3340,7 @@ basic_string<_CharT, _Traits, _Allocator>::find(const value_type* __s, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find(const basic_string& __str, size_type __pos) const _NOEXCEPT @@ -3312,7 +3365,7 @@ basic_string<_CharT, _Traits, _Allocator>::find(const _Tp &__t, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find(const value_type* __s, size_type __pos) const _NOEXCEPT @@ -3345,7 +3398,7 @@ basic_string<_CharT, _Traits, _Allocator>::rfind(const value_type* __s, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::rfind(const basic_string& __str, size_type __pos) const _NOEXCEPT @@ -3370,7 +3423,7 @@ basic_string<_CharT, _Traits, _Allocator>::rfind(const _Tp& __t, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::rfind(const value_type* __s, size_type __pos) const _NOEXCEPT @@ -3403,7 +3456,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_first_of(const value_type* __s, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_first_of(const basic_string& __str, size_type __pos) const _NOEXCEPT @@ -3428,7 +3481,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_first_of(const _Tp& __t, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_first_of(const value_type* __s, size_type __pos) const _NOEXCEPT @@ -3439,7 +3492,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_first_of(const value_type* __s, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_first_of(value_type __c, size_type __pos) const _NOEXCEPT @@ -3461,7 +3514,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_last_of(const value_type* __s, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_last_of(const basic_string& __str, size_type __pos) const _NOEXCEPT @@ -3486,7 +3539,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_last_of(const _Tp& __t, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_last_of(const value_type* __s, size_type __pos) const _NOEXCEPT @@ -3497,7 +3550,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_last_of(const value_type* __s, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_last_of(value_type __c, size_type __pos) const _NOEXCEPT @@ -3519,7 +3572,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const value_type* _ } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const basic_string& __str, size_type __pos) const _NOEXCEPT @@ -3544,7 +3597,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const _Tp& __t, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const value_type* __s, size_type __pos) const _NOEXCEPT @@ -3555,7 +3608,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(const value_type* _ } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_first_not_of(value_type __c, size_type __pos) const _NOEXCEPT @@ -3578,7 +3631,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const value_type* __ } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const basic_string& __str, size_type __pos) const _NOEXCEPT @@ -3603,7 +3656,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const _Tp& __t, } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const value_type* __s, size_type __pos) const _NOEXCEPT @@ -3614,7 +3667,7 @@ basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(const value_type* __ } template -inline _LIBCPP_INLINE_VISIBILITY +inline typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::find_last_not_of(value_type __c, size_type __pos) const _NOEXCEPT @@ -3649,7 +3702,7 @@ basic_string<_CharT, _Traits, _Allocator>::compare(const _Tp& __t) const } template -inline _LIBCPP_INLINE_VISIBILITY +inline int basic_string<_CharT, _Traits, _Allocator>::compare(const basic_string& __str) const _NOEXCEPT { @@ -3695,7 +3748,7 @@ basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, } template -inline _LIBCPP_INLINE_VISIBILITY +inline int basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, @@ -3753,7 +3806,7 @@ basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, // __invariants template -inline _LIBCPP_INLINE_VISIBILITY +inline bool basic_string<_CharT, _Traits, _Allocator>::__invariants() const { @@ -3771,7 +3824,7 @@ basic_string<_CharT, _Traits, _Allocator>::__invariants() const // __clear_and_shrink template -inline _LIBCPP_INLINE_VISIBILITY +inline void basic_string<_CharT, _Traits, _Allocator>::__clear_and_shrink() _NOEXCEPT { @@ -4025,6 +4078,7 @@ operator+(_CharT __lhs, const basic_string<_CharT,_Traits,_Allocator>& __rhs) } template +inline basic_string<_CharT, _Traits, _Allocator> operator+(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) { @@ -4121,11 +4175,13 @@ swap(basic_string<_CharT, _Traits, _Allocator>& __lhs, __lhs.swap(__rhs); } -#ifndef _LIBCPP_HAS_NO_UNICODE_CHARS +#ifndef _LIBCPP_NO_HAS_CHAR8_T +typedef basic_string u8string; +#endif +#ifndef _LIBCPP_HAS_NO_UNICODE_CHARS typedef basic_string u16string; typedef basic_string u32string; - #endif // _LIBCPP_HAS_NO_UNICODE_CHARS _LIBCPP_FUNC_VIS int stoi (const string& __str, size_t* __idx = 0, int __base = 10); @@ -4225,6 +4281,18 @@ getline(basic_istream<_CharT, _Traits>&& __is, #endif // _LIBCPP_CXX03_LANG +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase(basic_string<_CharT, _Traits, _Allocator>& __str, const _Up& __v) +{ __str.erase(_VSTD::remove(__str.begin(), __str.end(), __v), __str.end()); } + +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(basic_string<_CharT, _Traits, _Allocator>& __str, _Predicate __pred) +{ __str.erase(_VSTD::remove_if(__str.begin(), __str.end(), __pred), __str.end()); } +#endif + #if _LIBCPP_DEBUG_LEVEL >= 2 template @@ -4282,6 +4350,14 @@ inline namespace literals return basic_string (__str, __len); } +#ifndef _LIBCPP_NO_HAS_CHAR8_T + inline _LIBCPP_INLINE_VISIBILITY + basic_string operator "" s(const char8_t *__str, size_t __len) _NOEXCEPT + { + return basic_string (__str, __len); + } +#endif + inline _LIBCPP_INLINE_VISIBILITY basic_string operator "" s( const char16_t *__str, size_t __len ) { diff --git a/contrib/libc++/include/string_view b/contrib/libc++/include/string_view index 6377aeb6d64..7d783122f12 100644 --- a/contrib/libc++/include/string_view +++ b/contrib/libc++/include/string_view @@ -178,6 +178,7 @@ namespace std { #include #include #include +#include #include <__debug> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -768,6 +769,9 @@ bool operator>=(typename common_type >::type } typedef basic_string_view string_view; +#ifndef _LIBCPP_NO_HAS_CHAR8_T +typedef basic_string_view u8string_view; +#endif typedef basic_string_view u16string_view; typedef basic_string_view u32string_view; typedef basic_string_view wstring_view; @@ -777,17 +781,12 @@ template struct _LIBCPP_TEMPLATE_VIS hash > : public unary_function, size_t> { - size_t operator()(const basic_string_view<_CharT, _Traits> __val) const _NOEXCEPT; + _LIBCPP_INLINE_VISIBILITY + size_t operator()(const basic_string_view<_CharT, _Traits> __val) const _NOEXCEPT { + return __do_string_hash(__val.data(), __val.data() + __val.size()); + } }; -template -size_t -hash >::operator()( - const basic_string_view<_CharT, _Traits> __val) const _NOEXCEPT -{ - return __do_string_hash(__val.data(), __val.data() + __val.size()); -} - #if _LIBCPP_STD_VER > 11 inline namespace literals @@ -806,6 +805,14 @@ inline namespace literals return basic_string_view (__str, __len); } +#ifndef _LIBCPP_NO_HAS_CHAR8_T + inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR + basic_string_view operator "" sv(const char8_t *__str, size_t __len) _NOEXCEPT + { + return basic_string_view (__str, __len); + } +#endif + inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR basic_string_view operator "" sv(const char16_t *__str, size_t __len) _NOEXCEPT { diff --git a/contrib/libc++/include/thread b/contrib/libc++/include/thread index 0629d70efda..8c0115f8708 100644 --- a/contrib/libc++/include/thread +++ b/contrib/libc++/include/thread @@ -151,7 +151,7 @@ class __thread_specific_ptr __thread_specific_ptr(const __thread_specific_ptr&); __thread_specific_ptr& operator=(const __thread_specific_ptr&); - static void _LIBCPP_TLS_DESTRUCTOR_CC __at_thread_exit(void*); + _LIBCPP_HIDDEN static void _LIBCPP_TLS_DESTRUCTOR_CC __at_thread_exit(void*); public: typedef _Tp* pointer; diff --git a/contrib/libc++/include/tuple b/contrib/libc++/include/tuple index b3a17e7b735..4cc69030b9a 100644 --- a/contrib/libc++/include/tuple +++ b/contrib/libc++/include/tuple @@ -65,7 +65,7 @@ public: template tuple& operator=(const pair&); // iff sizeof...(T) == 2 template - tuple& operator=(pair&&); //iffsizeof...(T) == 2 + tuple& operator=(pair&&); // iff sizeof...(T) == 2 void swap(tuple&) noexcept(AND(swap(declval(), declval())...)); }; @@ -84,8 +84,8 @@ template constexpr T make_from_tuple(Tuple&& t); // C++17 // 20.4.1.4, tuple helper classes: -template class tuple_size; // undefined -template class tuple_size>; +template struct tuple_size; // undefined +template struct tuple_size>; template inline constexpr size_t tuple_size_v = tuple_size::value; // C++17 template class tuple_element; // undefined @@ -141,6 +141,7 @@ template #include #include <__functional_base> #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -1077,30 +1078,12 @@ namespace { _LIBCPP_INLINE_VAR constexpr __ignore_t ignore = __ignore_t(); } -template -struct __make_tuple_return_impl -{ - typedef _Tp type; -}; - -template -struct __make_tuple_return_impl > -{ - typedef _Tp& type; -}; - -template -struct __make_tuple_return -{ - typedef typename __make_tuple_return_impl::type>::type type; -}; - template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 -tuple::type...> +tuple::type...> make_tuple(_Tp&&... __t) { - return tuple::type...>(_VSTD::forward<_Tp>(__t)...); + return tuple::type...>(_VSTD::forward<_Tp>(__t)...); } template diff --git a/contrib/libc++/include/type_traits b/contrib/libc++/include/type_traits index 479c2e039ff..8b30511a4ec 100644 --- a/contrib/libc++/include/type_traits +++ b/contrib/libc++/include/type_traits @@ -75,6 +75,10 @@ namespace std template struct remove_pointer; template struct add_pointer; + template struct type_identity; // C++20 + template + using type_identity_t = typename type_identity::type; // C++20 + // Integral properties: template struct is_signed; template struct is_unsigned; @@ -400,6 +404,7 @@ namespace std */ #include <__config> #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -704,6 +709,9 @@ template <> struct __libcpp_is_integral : public tr template <> struct __libcpp_is_integral : public true_type {}; template <> struct __libcpp_is_integral : public true_type {}; template <> struct __libcpp_is_integral : public true_type {}; +#ifndef _LIBCPP_NO_HAS_CHAR8_T +template <> struct __libcpp_is_integral : public true_type {}; +#endif #ifndef _LIBCPP_HAS_NO_UNICODE_CHARS template <> struct __libcpp_is_integral : public true_type {}; template <> struct __libcpp_is_integral : public true_type {}; @@ -1220,6 +1228,12 @@ template struct _LIBCPP_TEMPLATE_VIS add_pointer template using add_pointer_t = typename add_pointer<_Tp>::type; #endif +// type_identity +#if _LIBCPP_STD_VER > 17 +template struct type_identity { typedef _Tp type; }; +template using type_identity_t = typename type_identity<_Tp>::type; +#endif + // is_signed template ::value> @@ -1638,7 +1652,7 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool has_unique_object_representations_v // alignment_of template struct _LIBCPP_TEMPLATE_VIS alignment_of - : public integral_constant {}; + : public integral_constant {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) template @@ -1668,7 +1682,7 @@ struct __nat template struct __align_type { - static const size_t value = alignment_of<_Tp>::value; + static const size_t value = _LIBCPP_PREFERRED_ALIGNOF(_Tp); typedef _Tp type; }; @@ -1801,8 +1815,8 @@ struct __static_max<_I0, _I1, _In...> template struct aligned_union { - static const size_t alignment_value = __static_max<__alignof__(_Type0), - __alignof__(_Types)...>::value; + static const size_t alignment_value = __static_max<_LIBCPP_PREFERRED_ALIGNOF(_Type0), + _LIBCPP_PREFERRED_ALIGNOF(_Types)...>::value; static const size_t __len = __static_max<_Len, sizeof(_Type0), sizeof(_Types)...>::value; typedef typename aligned_storage<__len, alignment_value>::type type; @@ -4742,7 +4756,6 @@ struct __has_operator_addressof #if _LIBCPP_STD_VER > 14 -#define __cpp_lib_void_t 201411 template using void_t = void; # ifndef _LIBCPP_HAS_NO_VARIADICS diff --git a/contrib/libc++/include/typeinfo b/contrib/libc++/include/typeinfo index f32ea6e76f1..8411532860b 100644 --- a/contrib/libc++/include/typeinfo +++ b/contrib/libc++/include/typeinfo @@ -73,12 +73,8 @@ public: #include #else -#if !defined(_LIBCPP_ABI_MICROSOFT) -#if defined(_LIBCPP_NONUNIQUE_RTTI_BIT) -#define _LIBCPP_HAS_NONUNIQUE_TYPEINFO -#else -#define _LIBCPP_HAS_UNIQUE_TYPEINFO -#endif +#if defined(_LIBCPP_NONUNIQUE_RTTI_BIT) && !defined(_LIBCPP_ABI_MICROSOFT) +# define _LIBCPP_HAS_NONUNIQUE_TYPEINFO #endif namespace std // purposefully not using versioning namespace @@ -232,7 +228,7 @@ void __throw_bad_cast() #ifndef _LIBCPP_NO_EXCEPTIONS throw bad_cast(); #else - _VSTD::abort(); + _VSTD::abort(); #endif } _LIBCPP_END_NAMESPACE_STD diff --git a/contrib/libc++/include/unordered_map b/contrib/libc++/include/unordered_map index 348f57923b5..6035b05dc61 100644 --- a/contrib/libc++/include/unordered_map +++ b/contrib/libc++/include/unordered_map @@ -153,6 +153,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(unordered_map& source); // C++17 + template + void merge(unordered_map&& source); // C++17 + template + void merge(unordered_multimap& source); // C++17 + template + void merge(unordered_multimap&& source); // C++17 + void swap(unordered_map&) noexcept( (!allocator_type::propagate_on_container_swap::value || @@ -325,6 +334,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(unordered_multimap& source); // C++17 + template + void merge(unordered_multimap&& source); // C++17 + template + void merge(unordered_map& source); // C++17 + template + void merge(unordered_map&& source); // C++17 + void swap(unordered_multimap&) noexcept( (!allocator_type::propagate_on_container_swap::value || @@ -366,6 +384,12 @@ template unordered_multimap& y) noexcept(noexcept(x.swap(y))); +template + void erase_if(unordered_set& c, Predicate pred); // C++20 + +template + void erase_if(unordered_multiset& c, Predicate pred); // C++20 + template bool operator==(const unordered_multimap& x, @@ -386,6 +410,7 @@ template #include #include #include +#include #include <__debug> @@ -395,7 +420,8 @@ template _LIBCPP_BEGIN_NAMESPACE_STD -template +template ::value && !__libcpp_is_final<_Hash>::value> class __unordered_map_hasher : private _Hash { @@ -463,7 +489,8 @@ swap(__unordered_map_hasher<_Key, _Cp, _Hash, __b>& __x, __x.swap(__y); } -template +template ::value && !__libcpp_is_final<_Pred>::value> class __unordered_map_equal : private _Pred { @@ -807,6 +834,9 @@ public: template friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator; }; +template +class unordered_multimap; + template , class _Pred = equal_to<_Key>, class _Alloc = allocator > > class _LIBCPP_TEMPLATE_VIS unordered_map @@ -823,6 +853,7 @@ public: typedef const value_type& const_reference; static_assert((is_same::value), "Invalid allocator::value_type"); + static_assert(sizeof(__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), ""); private: typedef __hash_value_type __value_type; @@ -864,6 +895,11 @@ public: typedef __insert_return_type insert_return_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS unordered_map; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; + _LIBCPP_INLINE_VISIBILITY unordered_map() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) @@ -1187,6 +1223,39 @@ public: return __table_.template __node_handle_extract( __it.__i_); } + + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_map& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_unique(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_map&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_unique(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multimap& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_unique(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multimap&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_unique(__source.__table_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -1563,6 +1632,13 @@ swap(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + template bool operator==(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, @@ -1607,6 +1683,7 @@ public: typedef const value_type& const_reference; static_assert((is_same::value), "Invalid allocator::value_type"); + static_assert(sizeof(__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), ""); private: typedef __hash_value_type __value_type; @@ -1645,6 +1722,11 @@ public: typedef __map_node_handle<__node, allocator_type> node_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS unordered_map; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; + _LIBCPP_INLINE_VISIBILITY unordered_multimap() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) @@ -1846,6 +1928,39 @@ public: return __table_.template __node_handle_extract( __it.__i_); } + + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multimap& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multimap&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_map& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_map&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -2141,6 +2256,13 @@ swap(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + template bool operator==(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x, diff --git a/contrib/libc++/include/unordered_set b/contrib/libc++/include/unordered_set index 9b8560da494..b4e61da89ee 100644 --- a/contrib/libc++/include/unordered_set +++ b/contrib/libc++/include/unordered_set @@ -127,6 +127,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(unordered_set& source); // C++17 + template + void merge(unordered_set&& source); // C++17 + template + void merge(unordered_multiset& source); // C++17 + template + void merge(unordered_multiset&& source); // C++17 + void swap(unordered_set&) noexcept(allocator_traits::is_always_equal::value && noexcept(swap(declval(), declval())) && @@ -282,6 +291,15 @@ public: iterator erase(const_iterator first, const_iterator last); void clear() noexcept; + template + void merge(unordered_multiset& source); // C++17 + template + void merge(unordered_multiset&& source); // C++17 + template + void merge(unordered_set& source); // C++17 + template + void merge(unordered_set&& source); // C++17 + void swap(unordered_multiset&) noexcept(allocator_traits::is_always_equal::value && noexcept(swap(declval(), declval())) && @@ -321,6 +339,13 @@ template unordered_multiset& y) noexcept(noexcept(x.swap(y))); +template + void erase_if(unordered_set& c, Predicate pred); // C++20 + +template + void erase_if(unordered_multiset& c, Predicate pred); // C++20 + + template bool operator==(const unordered_multiset& x, @@ -338,6 +363,7 @@ template #include <__hash_table> #include <__node_handle> #include +#include #include <__debug> @@ -347,6 +373,9 @@ template _LIBCPP_BEGIN_NAMESPACE_STD +template +class unordered_multiset; + template , class _Pred = equal_to<_Value>, class _Alloc = allocator<_Value> > class _LIBCPP_TEMPLATE_VIS unordered_set @@ -362,6 +391,7 @@ public: typedef const value_type& const_reference; static_assert((is_same::value), "Invalid allocator::value_type"); + static_assert(sizeof(__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), ""); private: typedef __hash_table __table; @@ -384,6 +414,11 @@ public: typedef __insert_return_type insert_return_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS unordered_set; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multiset; + _LIBCPP_INLINE_VISIBILITY unordered_set() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) @@ -589,6 +624,39 @@ public: { return __table_.template __node_handle_extract(__it); } + + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_set& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __table_.__node_handle_merge_unique(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_set&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __table_.__node_handle_merge_unique(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multiset& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __table_.__node_handle_merge_unique(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multiset&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + __table_.__node_handle_merge_unique(__source.__table_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -873,6 +941,13 @@ swap(unordered_set<_Value, _Hash, _Pred, _Alloc>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(unordered_set<_Value, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + template bool operator==(const unordered_set<_Value, _Hash, _Pred, _Alloc>& __x, @@ -916,6 +991,7 @@ public: typedef const value_type& const_reference; static_assert((is_same::value), "Invalid allocator::value_type"); + static_assert(sizeof(__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), ""); private: typedef __hash_table __table; @@ -937,6 +1013,11 @@ public: typedef __set_node_handle node_type; #endif + template + friend class _LIBCPP_TEMPLATE_VIS unordered_set; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multiset; + _LIBCPP_INLINE_VISIBILITY unordered_multiset() _NOEXCEPT_(is_nothrow_default_constructible<__table>::value) @@ -1101,6 +1182,39 @@ public: { return __table_.template __node_handle_extract(__key); } + + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multiset& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_multiset&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_set& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } + template + _LIBCPP_INLINE_VISIBILITY + void merge(unordered_set&& __source) + { + _LIBCPP_ASSERT(__source.get_allocator() == get_allocator(), + "merging container with incompatible allocator"); + return __table_.__node_handle_merge_multi(__source.__table_); + } #endif _LIBCPP_INLINE_VISIBILITY @@ -1397,6 +1511,13 @@ swap(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x, __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __c, _Predicate __pred) +{ __libcpp_erase_if_container(__c, __pred); } +#endif + template bool operator==(const unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __x, diff --git a/contrib/libc++/include/utility b/contrib/libc++/include/utility index ed9bf030d49..74bbc5cf34f 100644 --- a/contrib/libc++/include/utility +++ b/contrib/libc++/include/utility @@ -103,7 +103,7 @@ swap(pair& x, pair& y) noexcept(noexcept(x.swap(y))); struct piecewise_construct_t { }; inline constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t(); -template class tuple_size; +template struct tuple_size; template class tuple_element; template struct tuple_size >; @@ -203,6 +203,7 @@ template #include #include #include +#include #include <__debug> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -296,12 +297,13 @@ template void as_const(const _Tp&&) = delete; struct _LIBCPP_TEMPLATE_VIS piecewise_construct_t { }; #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY) -extern const piecewise_construct_t piecewise_construct;// = piecewise_construct_t(); +extern _LIBCPP_EXPORTED_FROM_ABI const piecewise_construct_t piecewise_construct;// = piecewise_construct_t(); #else /* _LIBCPP_INLINE_VAR */ constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t(); #endif #if defined(_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR) +template struct __non_trivially_copyable_base { _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY __non_trivially_copyable_base() _NOEXCEPT {} @@ -313,7 +315,7 @@ struct __non_trivially_copyable_base { template struct _LIBCPP_TEMPLATE_VIS pair #if defined(_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR) -: private __non_trivially_copyable_base +: private __non_trivially_copyable_base<_T1, _T2> #endif { typedef _T1 first_type; @@ -408,13 +410,17 @@ struct _LIBCPP_TEMPLATE_VIS pair _CheckArgsDep<_Dummy>::template __enable_default<_T1, _T2>() > = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR - pair() : first(), second() {} + pair() _NOEXCEPT_(is_nothrow_default_constructible::value && + is_nothrow_default_constructible::value) + : first(), second() {} template ::template __enable_explicit<_T1 const&, _T2 const&>() > = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 explicit pair(_T1 const& __t1, _T2 const& __t2) + _NOEXCEPT_(is_nothrow_copy_constructible::value && + is_nothrow_copy_constructible::value) : first(__t1), second(__t2) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 pair(_T1 const& __t1, _T2 const& __t2) + _NOEXCEPT_(is_nothrow_copy_constructible::value && + is_nothrow_copy_constructible::value) : first(__t1), second(__t2) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 explicit pair(_U1&& __u1, _U2&& __u2) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : first(_VSTD::forward<_U1>(__u1)), second(_VSTD::forward<_U2>(__u2)) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 pair(_U1&& __u1, _U2&& __u2) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : first(_VSTD::forward<_U1>(__u1)), second(_VSTD::forward<_U2>(__u2)) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 explicit pair(pair<_U1, _U2> const& __p) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : first(__p.first), second(__p.second) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 pair(pair<_U1, _U2> const& __p) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : first(__p.first), second(__p.second) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 explicit pair(pair<_U1, _U2>&&__p) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : first(_VSTD::forward<_U1>(__p.first)), second(_VSTD::forward<_U2>(__p.second)) {} template = false> _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 pair(pair<_U1, _U2>&& __p) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : first(_VSTD::forward<_U1>(__p.first)), second(_VSTD::forward<_U2>(__p.second)) {} template __first_args, tuple<_Args2...> __second_args) + _NOEXCEPT_((is_nothrow_constructible::value && + is_nothrow_constructible::value)) : pair(__pc, __first_args, __second_args, typename __make_tuple_indices::type(), typename __make_tuple_indices::type()) {} @@ -615,32 +637,37 @@ swap(pair<_T1, _T2>& __x, pair<_T1, _T2>& __y) __x.swap(__y); } +template +struct __unwrap_reference { typedef _Tp type; }; + +template +struct __unwrap_reference > { typedef _Tp& type; }; + +#if _LIBCPP_STD_VER > 17 +template +struct unwrap_reference : __unwrap_reference<_Tp> { }; + +template +struct unwrap_ref_decay : unwrap_reference::type> { }; +#endif // > C++17 + +template +struct __unwrap_ref_decay +#if _LIBCPP_STD_VER > 17 + : unwrap_ref_decay<_Tp> +#else + : __unwrap_reference::type> +#endif +{ }; + #ifndef _LIBCPP_CXX03_LANG -template -struct __make_pair_return_impl -{ - typedef _Tp type; -}; - -template -struct __make_pair_return_impl> -{ - typedef _Tp& type; -}; - -template -struct __make_pair_return -{ - typedef typename __make_pair_return_impl::type>::type type; -}; - template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 -pair::type, typename __make_pair_return<_T2>::type> +pair::type, typename __unwrap_ref_decay<_T2>::type> make_pair(_T1&& __t1, _T2&& __t2) { - return pair::type, typename __make_pair_return<_T2>::type> + return pair::type, typename __unwrap_ref_decay<_T2>::type> (_VSTD::forward<_T1>(__t1), _VSTD::forward<_T2>(__t2)); } @@ -657,7 +684,7 @@ make_pair(_T1 __x, _T2 __y) #endif // _LIBCPP_CXX03_LANG template - class _LIBCPP_TEMPLATE_VIS tuple_size > + struct _LIBCPP_TEMPLATE_VIS tuple_size > : public integral_constant {}; template @@ -988,8 +1015,10 @@ __murmur2_or_cityhash<_Size, 32>::operator()(const void* __key, _Size __len) { case 3: __h ^= __data[2] << 16; + _LIBCPP_FALLTHROUGH(); case 2: __h ^= __data[1] << 8; + _LIBCPP_FALLTHROUGH(); case 1: __h ^= __data[0]; __h *= __m; @@ -1453,7 +1482,7 @@ struct _LIBCPP_TEMPLATE_VIS hash size_t operator()(float __v) const _NOEXCEPT { // -0.0 and 0.0 should return same hash - if (__v == 0) + if (__v == 0.0) return 0; return __scalar_hash::operator()(__v); } @@ -1467,7 +1496,7 @@ struct _LIBCPP_TEMPLATE_VIS hash size_t operator()(double __v) const _NOEXCEPT { // -0.0 and 0.0 should return same hash - if (__v == 0) + if (__v == 0.0) return 0; return __scalar_hash::operator()(__v); } @@ -1481,7 +1510,7 @@ struct _LIBCPP_TEMPLATE_VIS hash size_t operator()(long double __v) const _NOEXCEPT { // -0.0 and 0.0 should return same hash - if (__v == 0) + if (__v == 0.0) return 0; #if defined(__i386__) // Zero out padding bits diff --git a/contrib/libc++/include/valarray b/contrib/libc++/include/valarray index 8d3892ad35d..07f38c81150 100644 --- a/contrib/libc++/include/valarray +++ b/contrib/libc++/include/valarray @@ -803,7 +803,7 @@ public: // construct/destroy: _LIBCPP_INLINE_VISIBILITY valarray() : __begin_(0), __end_(0) {} - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit valarray(size_t __n); _LIBCPP_INLINE_VISIBILITY valarray(const value_type& __x, size_t __n); @@ -818,7 +818,7 @@ public: valarray(const gslice_array& __ga); valarray(const mask_array& __ma); valarray(const indirect_array& __ia); - inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY + inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 ~valarray(); // assignment: @@ -1054,7 +1054,8 @@ private: const _Up* end(const valarray<_Up>& __v); - void __clear(); + _LIBCPP_INLINE_VISIBILITY + void __clear(size_t __capacity); valarray& __assign_range(const value_type* __f, const value_type* __l); }; @@ -2739,7 +2740,7 @@ __val_expr<_ValExpr>::operator valarray<__val_expr::result_type>() const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(result_type), __alignof(result_type))); + _VSTD::__libcpp_allocate(__n * sizeof(result_type), _LIBCPP_ALIGNOF(result_type))); for (size_t __i = 0; __i != __n; ++__r.__end_, ++__i) ::new (__r.__end_) result_type(__expr_[__i]); } @@ -2757,18 +2758,18 @@ valarray<_Tp>::valarray(size_t __n) if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif // _LIBCPP_NO_EXCEPTIONS - for (; __n; --__n, ++__end_) + for (size_t __n_left = __n; __n_left; --__n_left, ++__end_) ::new (__end_) value_type(); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2792,18 +2793,18 @@ valarray<_Tp>::valarray(const value_type* __p, size_t __n) if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif // _LIBCPP_NO_EXCEPTIONS - for (; __n; ++__end_, ++__p, --__n) + for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left) ::new (__end_) value_type(*__p); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2818,7 +2819,7 @@ valarray<_Tp>::valarray(const valarray& __v) if (__v.size()) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__v.size() * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__v.size() * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { @@ -2829,7 +2830,7 @@ valarray<_Tp>::valarray(const valarray& __v) } catch (...) { - __clear(); + __clear(__v.size()); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2852,22 +2853,23 @@ valarray<_Tp>::valarray(initializer_list __il) : __begin_(0), __end_(0) { - size_t __n = __il.size(); + const size_t __n = __il.size(); if (__n) { __begin_ = __end_ = static_cast( -_VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); +_VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif // _LIBCPP_NO_EXCEPTIONS - for (const value_type* __p = __il.begin(); __n; ++__end_, ++__p, --__n) + size_t __n_left = __n; + for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left) ::new (__end_) value_type(*__p); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2881,22 +2883,23 @@ valarray<_Tp>::valarray(const slice_array& __sa) : __begin_(0), __end_(0) { - size_t __n = __sa.__size_; + const size_t __n = __sa.__size_; if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif // _LIBCPP_NO_EXCEPTIONS - for (const value_type* __p = __sa.__vp_; __n; ++__end_, __p += __sa.__stride_, --__n) + size_t __n_left = __n; + for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left) ::new (__end_) value_type(*__p); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2908,11 +2911,11 @@ valarray<_Tp>::valarray(const gslice_array& __ga) : __begin_(0), __end_(0) { - size_t __n = __ga.__1d_.size(); + const size_t __n = __ga.__1d_.size(); if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { @@ -2926,7 +2929,7 @@ valarray<_Tp>::valarray(const gslice_array& __ga) } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2938,11 +2941,11 @@ valarray<_Tp>::valarray(const mask_array& __ma) : __begin_(0), __end_(0) { - size_t __n = __ma.__1d_.size(); + const size_t __n = __ma.__1d_.size(); if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { @@ -2956,7 +2959,7 @@ valarray<_Tp>::valarray(const mask_array& __ma) } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2968,11 +2971,11 @@ valarray<_Tp>::valarray(const indirect_array& __ia) : __begin_(0), __end_(0) { - size_t __n = __ia.__1d_.size(); + const size_t __n = __ia.__1d_.size(); if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { @@ -2986,7 +2989,7 @@ valarray<_Tp>::valarray(const indirect_array& __ia) } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS @@ -2997,7 +3000,7 @@ template inline valarray<_Tp>::~valarray() { - __clear(); + __clear(size()); } template @@ -3007,9 +3010,9 @@ valarray<_Tp>::__assign_range(const value_type* __f, const value_type* __l) size_t __n = __l - __f; if (size() != __n) { - __clear(); + __clear(size()); __begin_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); __end_ = __begin_ + __n; _VSTD::uninitialized_copy(__f, __l, __begin_); } else { @@ -3034,7 +3037,7 @@ inline valarray<_Tp>& valarray<_Tp>::operator=(valarray&& __v) _NOEXCEPT { - __clear(); + __clear(size()); __begin_ = __v.__begin_; __end_ = __v.__end_; __v.__begin_ = nullptr; @@ -3265,7 +3268,7 @@ valarray<_Tp>::operator+() const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n) ::new (__r.__end_) value_type(+*__p); } @@ -3283,7 +3286,7 @@ valarray<_Tp>::operator-() const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n) ::new (__r.__end_) value_type(-*__p); } @@ -3301,7 +3304,7 @@ valarray<_Tp>::operator~() const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n) ::new (__r.__end_) value_type(~*__p); } @@ -3318,7 +3321,7 @@ valarray<_Tp>::operator!() const { __r.__begin_ = __r.__end_ = - static_cast(_VSTD::__libcpp_allocate(__n * sizeof(bool), __alignof(bool))); + static_cast(_VSTD::__libcpp_allocate(__n * sizeof(bool), _LIBCPP_ALIGNOF(bool))); for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n) ::new (__r.__end_) bool(!*__p); } @@ -3639,7 +3642,7 @@ valarray<_Tp>::shift(int __i) const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); const value_type* __sb; value_type* __tb; value_type* __te; @@ -3678,7 +3681,7 @@ valarray<_Tp>::cshift(int __i) const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); __i %= static_cast(__n); const value_type* __m = __i >= 0 ? __begin_ + __i : __end_ + __i; for (const value_type* __s = __m; __s != __end_; ++__r.__end_, ++__s) @@ -3700,7 +3703,7 @@ valarray<_Tp>::apply(value_type __f(value_type)) const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n) ::new (__r.__end_) value_type(__f(*__p)); } @@ -3718,7 +3721,7 @@ valarray<_Tp>::apply(value_type __f(const value_type&)) const __r.__begin_ = __r.__end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n) ::new (__r.__end_) value_type(__f(*__p)); } @@ -3726,38 +3729,38 @@ valarray<_Tp>::apply(value_type __f(const value_type&)) const } template -void -valarray<_Tp>::__clear() +inline +void valarray<_Tp>::__clear(size_t __capacity) { - if (__begin_ != nullptr) - { - while (__end_ != __begin_) - (--__end_)->~value_type(); - _VSTD::__libcpp_deallocate(__begin_, __alignof(value_type)); - __begin_ = __end_ = nullptr; - } + if (__begin_ != nullptr) + { + while (__end_ != __begin_) + (--__end_)->~value_type(); + _VSTD::__libcpp_deallocate(__begin_, __capacity * sizeof(value_type), _LIBCPP_ALIGNOF(value_type)); + __begin_ = __end_ = nullptr; + } } template void valarray<_Tp>::resize(size_t __n, value_type __x) { - __clear(); + __clear(size()); if (__n) { __begin_ = __end_ = static_cast( - _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type))); + _VSTD::__libcpp_allocate(__n * sizeof(value_type), _LIBCPP_ALIGNOF(value_type))); #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif // _LIBCPP_NO_EXCEPTIONS - for (; __n; --__n, ++__end_) + for (size_t __n_left = __n; __n_left; --__n_left, ++__end_) ::new (__end_) value_type(__x); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - __clear(); + __clear(__n); throw; } #endif // _LIBCPP_NO_EXCEPTIONS diff --git a/contrib/libc++/include/variant b/contrib/libc++/include/variant index f9098f42249..a4339de6cde 100644 --- a/contrib/libc++/include/variant +++ b/contrib/libc++/include/variant @@ -23,8 +23,8 @@ namespace std { // 20.7.2.1, constructors constexpr variant() noexcept(see below); - variant(const variant&); - variant(variant&&) noexcept(see below); + variant(const variant&); // constexpr in C++20 + variant(variant&&) noexcept(see below); // constexpr in C++20 template constexpr variant(T&&) noexcept(see below); @@ -46,8 +46,8 @@ namespace std { ~variant(); // 20.7.2.3, assignment - variant& operator=(const variant&); - variant& operator=(variant&&) noexcept(see below); + variant& operator=(const variant&); // constexpr in C++20 + variant& operator=(variant&&) noexcept(see below); // constexpr in C++20 template variant& operator=(T&&) noexcept(see below); @@ -208,6 +208,7 @@ namespace std { #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -218,7 +219,7 @@ _LIBCPP_PUSH_MACROS namespace std { // explicitly not using versioning namespace -class _LIBCPP_EXCEPTION_ABI bad_variant_access : public exception { +class _LIBCPP_EXCEPTION_ABI _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS bad_variant_access : public exception { public: virtual const char* what() const _NOEXCEPT; }; @@ -231,6 +232,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS void __throw_bad_variant_access() { #ifndef _LIBCPP_NO_EXCEPTIONS throw bad_variant_access(); @@ -1064,7 +1066,7 @@ public: #ifndef _LIBCPP_NO_EXCEPTIONS // EXTENSION: When the move construction of `__lhs` into `__rhs` throws // and `__tmp` is nothrow move constructible then we move `__tmp` back - // into `__rhs` and provide the strong exception safety guarentee. + // into `__rhs` and provide the strong exception safety guarantee. try { this->__generic_construct(*__rhs, _VSTD::move(*__lhs)); } catch (...) { @@ -1320,7 +1322,8 @@ constexpr bool holds_alternative(const variant<_Types...>& __v) noexcept { template inline _LIBCPP_INLINE_VISIBILITY -static constexpr auto&& __generic_get(_Vp&& __v) { +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS +constexpr auto&& __generic_get(_Vp&& __v) { using __variant_detail::__access::__variant; if (!__holds_alternative<_Ip>(__v)) { __throw_bad_variant_access(); @@ -1330,6 +1333,7 @@ static constexpr auto&& __generic_get(_Vp&& __v) { template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>& get( variant<_Types...>& __v) { static_assert(_Ip < sizeof...(_Types)); @@ -1339,6 +1343,7 @@ constexpr variant_alternative_t<_Ip, variant<_Types...>>& get( template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&& get( variant<_Types...>&& __v) { static_assert(_Ip < sizeof...(_Types)); @@ -1348,6 +1353,7 @@ constexpr variant_alternative_t<_Ip, variant<_Types...>>&& get( template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>& get( const variant<_Types...>& __v) { static_assert(_Ip < sizeof...(_Types)); @@ -1357,6 +1363,7 @@ constexpr const variant_alternative_t<_Ip, variant<_Types...>>& get( template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&& get( const variant<_Types...>&& __v) { static_assert(_Ip < sizeof...(_Types)); @@ -1366,6 +1373,7 @@ constexpr const variant_alternative_t<_Ip, variant<_Types...>>&& get( template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp& get(variant<_Types...>& __v) { static_assert(!is_void_v<_Tp>); return _VSTD::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v); @@ -1373,6 +1381,7 @@ constexpr _Tp& get(variant<_Types...>& __v) { template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp&& get(variant<_Types...>&& __v) { static_assert(!is_void_v<_Tp>); return _VSTD::get<__find_exactly_one_t<_Tp, _Types...>::value>( @@ -1381,6 +1390,7 @@ constexpr _Tp&& get(variant<_Types...>&& __v) { template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp& get(const variant<_Types...>& __v) { static_assert(!is_void_v<_Tp>); return _VSTD::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v); @@ -1388,6 +1398,7 @@ constexpr const _Tp& get(const variant<_Types...>& __v) { template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp&& get(const variant<_Types...>&& __v) { static_assert(!is_void_v<_Tp>); return _VSTD::get<__find_exactly_one_t<_Tp, _Types...>::value>( @@ -1437,6 +1448,16 @@ get_if(const variant<_Types...>* __v) noexcept { return _VSTD::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v); } +template +struct __convert_to_bool { + template + _LIBCPP_INLINE_VISIBILITY constexpr bool operator()(_T1 && __t1, _T2&& __t2) const { + static_assert(std::is_convertible(__t1), _VSTD::forward<_T2>(__t2))), bool>::value, + "the relational operator does not return a type which is implicitly convertible to bool"); + return _Operator{}(_VSTD::forward<_T1>(__t1), _VSTD::forward<_T2>(__t2)); + } +}; + template inline _LIBCPP_INLINE_VISIBILITY constexpr bool operator==(const variant<_Types...>& __lhs, @@ -1444,7 +1465,7 @@ constexpr bool operator==(const variant<_Types...>& __lhs, using __variant_detail::__visitation::__variant; if (__lhs.index() != __rhs.index()) return false; if (__lhs.valueless_by_exception()) return true; - return __variant::__visit_value_at(__lhs.index(), equal_to<>{}, __lhs, __rhs); + return __variant::__visit_value_at(__lhs.index(), __convert_to_bool>{}, __lhs, __rhs); } template @@ -1455,7 +1476,7 @@ constexpr bool operator!=(const variant<_Types...>& __lhs, if (__lhs.index() != __rhs.index()) return true; if (__lhs.valueless_by_exception()) return false; return __variant::__visit_value_at( - __lhs.index(), not_equal_to<>{}, __lhs, __rhs); + __lhs.index(), __convert_to_bool>{}, __lhs, __rhs); } template @@ -1467,7 +1488,7 @@ constexpr bool operator<(const variant<_Types...>& __lhs, if (__lhs.valueless_by_exception()) return true; if (__lhs.index() < __rhs.index()) return true; if (__lhs.index() > __rhs.index()) return false; - return __variant::__visit_value_at(__lhs.index(), less<>{}, __lhs, __rhs); + return __variant::__visit_value_at(__lhs.index(), __convert_to_bool>{}, __lhs, __rhs); } template @@ -1479,7 +1500,7 @@ constexpr bool operator>(const variant<_Types...>& __lhs, if (__rhs.valueless_by_exception()) return true; if (__lhs.index() > __rhs.index()) return true; if (__lhs.index() < __rhs.index()) return false; - return __variant::__visit_value_at(__lhs.index(), greater<>{}, __lhs, __rhs); + return __variant::__visit_value_at(__lhs.index(), __convert_to_bool>{}, __lhs, __rhs); } template @@ -1492,7 +1513,7 @@ constexpr bool operator<=(const variant<_Types...>& __lhs, if (__lhs.index() < __rhs.index()) return true; if (__lhs.index() > __rhs.index()) return false; return __variant::__visit_value_at( - __lhs.index(), less_equal<>{}, __lhs, __rhs); + __lhs.index(), __convert_to_bool>{}, __lhs, __rhs); } template @@ -1505,11 +1526,12 @@ constexpr bool operator>=(const variant<_Types...>& __lhs, if (__lhs.index() > __rhs.index()) return true; if (__lhs.index() < __rhs.index()) return false; return __variant::__visit_value_at( - __lhs.index(), greater_equal<>{}, __lhs, __rhs); + __lhs.index(), __convert_to_bool>{}, __lhs, __rhs); } template inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) { using __variant_detail::__visitation::__variant; bool __results[] = {__vs.valueless_by_exception()...}; diff --git a/contrib/libc++/include/vector b/contrib/libc++/include/vector index 0f5006f3756..edb6d3e09f5 100644 --- a/contrib/libc++/include/vector +++ b/contrib/libc++/include/vector @@ -261,6 +261,11 @@ template void swap(vector& x, vector& y) noexcept(noexcept(x.swap(y))); +template + void erase(vector& c, const U& value); // C++20 +template + void erase_if(vector& c, Predicate pred); // C++20 + } // std */ @@ -276,6 +281,7 @@ void swap(vector& x, vector& y) #include #include #include +#include #include <__split_buffer> #include <__functional_base> @@ -540,13 +546,14 @@ public: value_type, typename iterator_traits<_ForwardIterator>::reference>::value>::type* = 0); -#if _LIBCPP_DEBUG_LEVEL >= 2 _LIBCPP_INLINE_VISIBILITY ~vector() { + __annotate_delete(); +#if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->__erase_c(this); - } #endif + } vector(const vector& __x); vector(const vector& __x, const allocator_type& __a); @@ -2453,7 +2460,7 @@ private: void __vdeallocate() _NOEXCEPT; _LIBCPP_INLINE_VISIBILITY static size_type __align_it(size_type __new_size) _NOEXCEPT - {return __new_size + (__bits_per_word-1) & ~((size_type)__bits_per_word-1);}; + {return __new_size + (__bits_per_word-1) & ~((size_type)__bits_per_word-1);} _LIBCPP_INLINE_VISIBILITY size_type __recommend(size_type __new_size) const; _LIBCPP_INLINE_VISIBILITY void __construct_at_end(size_type __n, bool __x); template @@ -2604,6 +2611,13 @@ vector::__construct_at_end(size_type __n, bool __x) { size_type __old_size = this->__size_; this->__size_ += __n; + if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) + { + if (this->__size_ <= __bits_per_word) + this->__begin_[0] = __storage_type(0); + else + this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0); + } _VSTD::fill_n(__make_iter(__old_size), __n, __x); } @@ -2618,6 +2632,13 @@ vector::__construct_at_end(_ForwardIterator __first, _ForwardI { size_type __old_size = this->__size_; this->__size_ += _VSTD::distance(__first, __last); + if (__old_size == 0 || ((__old_size - 1) / __bits_per_word) != ((this->__size_ - 1) / __bits_per_word)) + { + if (this->__size_ <= __bits_per_word) + this->__begin_[0] = __storage_type(0); + else + this->__begin_[(this->__size_ - 1) / __bits_per_word] = __storage_type(0); + } _VSTD::copy(__first, __last, __make_iter(__old_size)); } @@ -3392,6 +3413,18 @@ swap(vector<_Tp, _Allocator>& __x, vector<_Tp, _Allocator>& __y) __x.swap(__y); } +#if _LIBCPP_STD_VER > 17 +template +inline _LIBCPP_INLINE_VISIBILITY +void erase(vector<_Tp, _Allocator>& __c, const _Up& __v) +{ __c.erase(_VSTD::remove(__c.begin(), __c.end(), __v), __c.end()); } + +template +inline _LIBCPP_INLINE_VISIBILITY +void erase_if(vector<_Tp, _Allocator>& __c, _Predicate __pred) +{ __c.erase(_VSTD::remove_if(__c.begin(), __c.end(), __pred), __c.end()); } +#endif + _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS diff --git a/contrib/libc++/include/version b/contrib/libc++/include/version index 23a872ea85a..e37afc44860 100644 --- a/contrib/libc++/include/version +++ b/contrib/libc++/include/version @@ -12,7 +12,105 @@ #define _LIBCPP_VERSIONH /* - version synopsis + version synopsis + +Macro name Value Headers +__cpp_lib_addressof_constexpr 201603L +__cpp_lib_allocator_traits_is_always_equal 201411L + + + +__cpp_lib_any 201606L +__cpp_lib_apply 201603L +__cpp_lib_array_constexpr 201603L +__cpp_lib_as_const 201510L +__cpp_lib_atomic_is_always_lock_free 201603L +__cpp_lib_atomic_ref 201806L +__cpp_lib_bind_front 201811L +__cpp_lib_bit_cast 201806L +__cpp_lib_bool_constant 201505L +__cpp_lib_boyer_moore_searcher 201603L +__cpp_lib_byte 201603L +__cpp_lib_char8_t 201811L + + +__cpp_lib_chrono 201611L +__cpp_lib_chrono_udls 201304L +__cpp_lib_clamp 201603L +__cpp_lib_complex_udls 201309L +__cpp_lib_concepts 201806L +__cpp_lib_constexpr_misc 201811L + +__cpp_lib_constexpr_swap_algorithms 201806L +__cpp_lib_destroying_delete 201806L +__cpp_lib_enable_shared_from_this 201603L +__cpp_lib_erase_if 201811L + + +__cpp_lib_exchange_function 201304L +__cpp_lib_execution 201603L +__cpp_lib_filesystem 201703L +__cpp_lib_gcd_lcm 201606L +__cpp_lib_generic_associative_lookup 201304L +__cpp_lib_generic_unordered_lookup 201811L +__cpp_lib_hardware_interference_size 201703L +__cpp_lib_has_unique_object_representations 201606L +__cpp_lib_hypot 201603L +__cpp_lib_incomplete_container_elements 201505L +__cpp_lib_integer_sequence 201304L +__cpp_lib_integral_constant_callable 201304L +__cpp_lib_invoke 201411L +__cpp_lib_is_aggregate 201703L +__cpp_lib_is_constant_evaluated 201811L +__cpp_lib_is_final 201402L +__cpp_lib_is_invocable 201703L +__cpp_lib_is_null_pointer 201309L +__cpp_lib_is_swappable 201603L +__cpp_lib_launder 201606L +__cpp_lib_list_remove_return_type 201806L +__cpp_lib_logical_traits 201510L +__cpp_lib_make_from_tuple 201606L +__cpp_lib_make_reverse_iterator 201402L +__cpp_lib_make_unique 201304L +__cpp_lib_map_try_emplace 201411L +__cpp_lib_math_special_functions 201603L +__cpp_lib_memory_resource 201603L +__cpp_lib_node_extract 201606L + +__cpp_lib_nonmember_container_access 201411L + + + +__cpp_lib_not_fn 201603L +__cpp_lib_null_iterators 201304L +__cpp_lib_optional 201606L +__cpp_lib_parallel_algorithm 201603L +__cpp_lib_quoted_string_io 201304L +__cpp_lib_ranges 201811L + +__cpp_lib_raw_memory_algorithms 201606L +__cpp_lib_result_of_sfinae 201210L +__cpp_lib_robust_nonmodifying_seq_ops 201304L +__cpp_lib_sample 201603L +__cpp_lib_scoped_lock 201703L +__cpp_lib_shared_mutex 201505L +__cpp_lib_shared_ptr_arrays 201611L +__cpp_lib_shared_ptr_weak_type 201606L +__cpp_lib_shared_timed_mutex 201402L +__cpp_lib_string_udls 201304L +__cpp_lib_string_view 201606L +__cpp_lib_three_way_comparison 201711L +__cpp_lib_to_chars 201611L +__cpp_lib_transformation_trait_aliases 201304L +__cpp_lib_transparent_operators 201510L + 201210L // C++14 +__cpp_lib_tuple_element_t 201402L +__cpp_lib_tuples_by_type 201304L +__cpp_lib_type_trait_variable_templates 201510L +__cpp_lib_uncaught_exceptions 201411L +__cpp_lib_unordered_map_try_emplace 201411L +__cpp_lib_variant 201606L +__cpp_lib_void_t 201411L */ @@ -22,4 +120,113 @@ #pragma GCC system_header #endif -#endif // _LIBCPP_VERSIONH +#if _LIBCPP_STD_VER > 11 +# define __cpp_lib_chrono_udls 201304L +# define __cpp_lib_complex_udls 201309L +# define __cpp_lib_exchange_function 201304L +# define __cpp_lib_generic_associative_lookup 201304L +# define __cpp_lib_integer_sequence 201304L +# define __cpp_lib_integral_constant_callable 201304L +# define __cpp_lib_is_final 201402L +# define __cpp_lib_is_null_pointer 201309L +# define __cpp_lib_make_reverse_iterator 201402L +# define __cpp_lib_make_unique 201304L +# define __cpp_lib_null_iterators 201304L +# define __cpp_lib_quoted_string_io 201304L +# define __cpp_lib_result_of_sfinae 201210L +# define __cpp_lib_robust_nonmodifying_seq_ops 201304L +# if !defined(_LIBCPP_HAS_NO_THREADS) +# define __cpp_lib_shared_timed_mutex 201402L +# endif +# define __cpp_lib_string_udls 201304L +# define __cpp_lib_transformation_trait_aliases 201304L +# define __cpp_lib_transparent_operators 201210L +# define __cpp_lib_tuple_element_t 201402L +# define __cpp_lib_tuples_by_type 201304L +#endif + +#if _LIBCPP_STD_VER > 14 +# if !defined(_LIBCPP_HAS_NO_BUILTIN_ADDRESSOF) +# define __cpp_lib_addressof_constexpr 201603L +# endif +# define __cpp_lib_allocator_traits_is_always_equal 201411L +# define __cpp_lib_any 201606L +# define __cpp_lib_apply 201603L +# define __cpp_lib_array_constexpr 201603L +# define __cpp_lib_as_const 201510L +# if !defined(_LIBCPP_HAS_NO_THREADS) +# define __cpp_lib_atomic_is_always_lock_free 201603L +# endif +# define __cpp_lib_bool_constant 201505L +// # define __cpp_lib_boyer_moore_searcher 201603L +# define __cpp_lib_byte 201603L +# define __cpp_lib_chrono 201611L +# define __cpp_lib_clamp 201603L +# define __cpp_lib_enable_shared_from_this 201603L +// # define __cpp_lib_execution 201603L +# define __cpp_lib_filesystem 201703L +# define __cpp_lib_gcd_lcm 201606L +# define __cpp_lib_hardware_interference_size 201703L +# if defined(_LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS) +# define __cpp_lib_has_unique_object_representations 201606L +# endif +# define __cpp_lib_hypot 201603L +# define __cpp_lib_incomplete_container_elements 201505L +# define __cpp_lib_invoke 201411L +# if !defined(_LIBCPP_HAS_NO_IS_AGGREGATE) +# define __cpp_lib_is_aggregate 201703L +# endif +# define __cpp_lib_is_invocable 201703L +# define __cpp_lib_is_swappable 201603L +# define __cpp_lib_launder 201606L +# define __cpp_lib_logical_traits 201510L +# define __cpp_lib_make_from_tuple 201606L +# define __cpp_lib_map_try_emplace 201411L +// # define __cpp_lib_math_special_functions 201603L +// # define __cpp_lib_memory_resource 201603L +# define __cpp_lib_node_extract 201606L +# define __cpp_lib_nonmember_container_access 201411L +# define __cpp_lib_not_fn 201603L +# define __cpp_lib_optional 201606L +// # define __cpp_lib_parallel_algorithm 201603L +# define __cpp_lib_raw_memory_algorithms 201606L +# define __cpp_lib_sample 201603L +# define __cpp_lib_scoped_lock 201703L +# if !defined(_LIBCPP_HAS_NO_THREADS) +# define __cpp_lib_shared_mutex 201505L +# endif +// # define __cpp_lib_shared_ptr_arrays 201611L +# define __cpp_lib_shared_ptr_weak_type 201606L +# define __cpp_lib_string_view 201606L +// # define __cpp_lib_to_chars 201611L +# undef __cpp_lib_transparent_operators +# define __cpp_lib_transparent_operators 201510L +# define __cpp_lib_type_trait_variable_templates 201510L +# define __cpp_lib_uncaught_exceptions 201411L +# define __cpp_lib_unordered_map_try_emplace 201411L +# define __cpp_lib_variant 201606L +# define __cpp_lib_void_t 201411L +#endif + +#if _LIBCPP_STD_VER > 17 +# if !defined(_LIBCPP_HAS_NO_THREADS) +// # define __cpp_lib_atomic_ref 201806L +# endif +// # define __cpp_lib_bind_front 201811L +// # define __cpp_lib_bit_cast 201806L +# if !defined(_LIBCPP_NO_HAS_CHAR8_T) +# define __cpp_lib_char8_t 201811L +# endif +// # define __cpp_lib_concepts 201806L +// # define __cpp_lib_constexpr_misc 201811L +// # define __cpp_lib_constexpr_swap_algorithms 201806L +// # define __cpp_lib_destroying_delete 201806L +# define __cpp_lib_erase_if 201811L +// # define __cpp_lib_generic_unordered_lookup 201811L +// # define __cpp_lib_is_constant_evaluated 201811L +// # define __cpp_lib_list_remove_return_type 201806L +// # define __cpp_lib_ranges 201811L +// # define __cpp_lib_three_way_comparison 201711L +#endif + +#endif // _LIBCPP_VERSIONH diff --git a/contrib/libc++/src/experimental/memory_resource.cpp b/contrib/libc++/src/experimental/memory_resource.cpp index a3b64cc8b49..7f0052f2b50 100644 --- a/contrib/libc++/src/experimental/memory_resource.cpp +++ b/contrib/libc++/src/experimental/memory_resource.cpp @@ -33,8 +33,9 @@ protected: virtual void* do_allocate(size_t __size, size_t __align) { return _VSTD::__libcpp_allocate(__size, __align); /* FIXME */} - virtual void do_deallocate(void * __p, size_t, size_t __align) - { _VSTD::__libcpp_deallocate(__p, __align); /* FIXME */ } + virtual void do_deallocate(void* __p, size_t __n, size_t __align) { + _VSTD::__libcpp_deallocate(__p, __n, __align); /* FIXME */ + } virtual bool do_is_equal(memory_resource const & __other) const _NOEXCEPT { return &__other == this; } diff --git a/contrib/libc++/src/filesystem/filesystem_common.h b/contrib/libc++/src/filesystem/filesystem_common.h index ed92877c425..40419ee35e6 100644 --- a/contrib/libc++/src/filesystem/filesystem_common.h +++ b/contrib/libc++/src/filesystem/filesystem_common.h @@ -67,24 +67,31 @@ static string format_string_imp(const char* msg, ...) { va_copy(args_cp, args); GuardVAList args_copy_guard(args_cp); + std::string result; + array local_buff; - size_t size = local_buff.size(); - auto ret = ::vsnprintf(local_buff.data(), size, msg, args_cp); + size_t size_with_null = local_buff.size(); + auto ret = ::vsnprintf(local_buff.data(), size_with_null, msg, args_cp); args_copy_guard.clear(); // handle empty expansion if (ret == 0) - return string{}; - if (static_cast(ret) < size) - return string(local_buff.data()); + return result; + if (static_cast(ret) < size_with_null) { + result.assign(local_buff.data(), static_cast(ret)); + return result; + } - // we did not provide a long enough buffer on our first attempt. - // add 1 to size to account for null-byte in size cast to prevent overflow - size = static_cast(ret) + 1; - auto buff_ptr = unique_ptr(new char[size]); - ret = ::vsnprintf(buff_ptr.get(), size, msg, args); - return string(buff_ptr.get()); + // we did not provide a long enough buffer on our first attempt. The + // return value is the number of bytes (excluding the null byte) that are + // needed for formatting. + size_with_null = static_cast(ret) + 1; + result.__resize_default_init(size_with_null - 1); + ret = ::vsnprintf(&result[0], size_with_null, msg, args); + _LIBCPP_ASSERT(static_cast(ret) == (size_with_null - 1), "TODO"); + + return result; } const char* unwrap(string const& s) { return s.c_str(); } diff --git a/contrib/libc++/src/filesystem/operations.cpp b/contrib/libc++/src/filesystem/operations.cpp index c9396b59cae..b4106188872 100644 --- a/contrib/libc++/src/filesystem/operations.cpp +++ b/contrib/libc++/src/filesystem/operations.cpp @@ -206,8 +206,20 @@ public: return *this; } + bool atEnd() const noexcept { + return State == PS_AtEnd; + } + + bool inRootDir() const noexcept { + return State == PS_InRootDir; + } + + bool inRootName() const noexcept { + return State == PS_InRootName; + } + bool inRootPath() const noexcept { - return State == PS_InRootDir || State == PS_InRootName; + return inRootName() || inRootDir(); } private: @@ -427,7 +439,8 @@ file_status posix_lstat(path const& p, error_code* ec) { return posix_lstat(p, path_stat, ec); } -bool posix_ftruncate(const FileDescriptor& fd, size_t to_size, error_code& ec) { +// http://pubs.opengroup.org/onlinepubs/9699919799/functions/ftruncate.html +bool posix_ftruncate(const FileDescriptor& fd, off_t to_size, error_code& ec) { if (::ftruncate(fd.fd, to_size) == -1) { ec = capture_errno(); return true; @@ -1294,7 +1307,19 @@ string_view_t path::__root_path_raw() const { return {}; } +static bool ConsumeRootName(PathParser *PP) { + static_assert(PathParser::PS_BeforeBegin == 1 && + PathParser::PS_InRootName == 2, + "Values for enums are incorrect"); + while (PP->State <= PathParser::PS_InRootName) + ++(*PP); + return PP->State == PathParser::PS_AtEnd; +} + static bool ConsumeRootDir(PathParser* PP) { + static_assert(PathParser::PS_BeforeBegin == 1 && + PathParser::PS_InRootName == 2 && + PathParser::PS_InRootDir == 3, "Values for enums are incorrect"); while (PP->State <= PathParser::PS_InRootDir) ++(*PP); return PP->State == PathParser::PS_AtEnd; @@ -1454,7 +1479,7 @@ static int DetermineLexicalElementCount(PathParser PP) { auto Elem = *PP; if (Elem == "..") --Count; - else if (Elem != ".") + else if (Elem != "." && Elem != "") ++Count; } return Count; @@ -1468,8 +1493,7 @@ path path::lexically_relative(const path& base) const { return PP.State != PPBase.State && (PP.inRootPath() || PPBase.inRootPath()); }; - if (PP.State == PathParser::PS_InRootName && - PPBase.State == PathParser::PS_InRootName) { + if (PP.inRootName() && PPBase.inRootName()) { if (*PP != *PPBase) return {}; } else if (CheckIterMismatchAtBase()) @@ -1501,6 +1525,10 @@ path path::lexically_relative(const path& base) const { if (ElemCount < 0) return {}; + // if n == 0 and (a == end() || a->empty()), returns path("."); otherwise + if (ElemCount == 0 && (PP.atEnd() || *PP == "")) + return "."; + // return a path constructed with 'n' dot-dot elements, followed by the the // elements of '*this' after the mismatch. path Result; @@ -1514,21 +1542,68 @@ path path::lexically_relative(const path& base) const { //////////////////////////////////////////////////////////////////////////// // path.comparisons -int path::__compare(string_view_t __s) const { - auto PP = PathParser::CreateBegin(__pn_); - auto PP2 = PathParser::CreateBegin(__s); - while (PP && PP2) { - int res = (*PP).compare(*PP2); - if (res != 0) - return res; - ++PP; - ++PP2; - } - if (PP.State == PP2.State && !PP) +static int CompareRootName(PathParser *LHS, PathParser *RHS) { + if (!LHS->inRootName() && !RHS->inRootName()) return 0; - if (!PP) + + auto GetRootName = [](PathParser *Parser) -> string_view_t { + return Parser->inRootName() ? **Parser : ""; + }; + int res = GetRootName(LHS).compare(GetRootName(RHS)); + ConsumeRootName(LHS); + ConsumeRootName(RHS); + return res; +} + +static int CompareRootDir(PathParser *LHS, PathParser *RHS) { + if (!LHS->inRootDir() && RHS->inRootDir()) return -1; - return 1; + else if (LHS->inRootDir() && !RHS->inRootDir()) + return 1; + else { + ConsumeRootDir(LHS); + ConsumeRootDir(RHS); + return 0; + } +} + +static int CompareRelative(PathParser *LHSPtr, PathParser *RHSPtr) { + auto &LHS = *LHSPtr; + auto &RHS = *RHSPtr; + + int res; + while (LHS && RHS) { + if ((res = (*LHS).compare(*RHS)) != 0) + return res; + ++LHS; + ++RHS; + } + return 0; +} + +static int CompareEndState(PathParser *LHS, PathParser *RHS) { + if (LHS->atEnd() && !RHS->atEnd()) + return -1; + else if (!LHS->atEnd() && RHS->atEnd()) + return 1; + return 0; +} + +int path::__compare(string_view_t __s) const { + auto LHS = PathParser::CreateBegin(__pn_); + auto RHS = PathParser::CreateBegin(__s); + int res; + + if ((res = CompareRootName(&LHS, &RHS)) != 0) + return res; + + if ((res = CompareRootDir(&LHS, &RHS)) != 0) + return res; + + if ((res = CompareRelative(&LHS, &RHS)) != 0) + return res; + + return CompareEndState(&LHS, &RHS); } //////////////////////////////////////////////////////////////////////////// diff --git a/contrib/libc++/src/future.cpp b/contrib/libc++/src/future.cpp index 07e4602f567..cbcd2e7b728 100644 --- a/contrib/libc++/src/future.cpp +++ b/contrib/libc++/src/future.cpp @@ -179,10 +179,7 @@ __assoc_sub_state::__execute() future::future(__assoc_sub_state* __state) : __state_(__state) { - if (__state_->__has_future_attached()) - __throw_future_error(future_errc::future_already_retrieved); - __state_->__add_shared(); - __state_->__set_future_attached(); + __state_->__attach_future(); } future::~future() diff --git a/contrib/libc++/src/iostream.cpp b/contrib/libc++/src/iostream.cpp index 2b47cf25b64..11bfb486208 100644 --- a/contrib/libc++/src/iostream.cpp +++ b/contrib/libc++/src/iostream.cpp @@ -13,21 +13,21 @@ #define _str(s) #s #define str(s) _str(s) -#define _LIBCPP_NAMESPACE_STR str(_LIBCPP_NAMESPACE) +#define _LIBCPP_ABI_NAMESPACE_STR str(_LIBCPP_ABI_NAMESPACE) _LIBCPP_BEGIN_NAMESPACE_STD #ifndef _LIBCPP_HAS_NO_STDIN _ALIGNAS_TYPE (istream) _LIBCPP_FUNC_VIS char cin[sizeof(istream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?cin@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_istream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?cin@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_istream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdinbuf ) static char __cin[sizeof(__stdinbuf )]; static mbstate_t mb_cin; _ALIGNAS_TYPE (wistream) _LIBCPP_FUNC_VIS char wcin[sizeof(wistream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wcin@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_istream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?wcin@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_istream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdinbuf ) static char __wcin[sizeof(__stdinbuf )]; @@ -37,14 +37,14 @@ static mbstate_t mb_wcin; #ifndef _LIBCPP_HAS_NO_STDOUT _ALIGNAS_TYPE (ostream) _LIBCPP_FUNC_VIS char cout[sizeof(ostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?cout@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?cout@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __cout[sizeof(__stdoutbuf)]; static mbstate_t mb_cout; _ALIGNAS_TYPE (wostream) _LIBCPP_FUNC_VIS char wcout[sizeof(wostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wcout@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?wcout@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __wcout[sizeof(__stdoutbuf)]; @@ -53,14 +53,14 @@ static mbstate_t mb_wcout; _ALIGNAS_TYPE (ostream) _LIBCPP_FUNC_VIS char cerr[sizeof(ostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?cerr@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?cerr@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __cerr[sizeof(__stdoutbuf)]; static mbstate_t mb_cerr; _ALIGNAS_TYPE (wostream) _LIBCPP_FUNC_VIS char wcerr[sizeof(wostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wcerr@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?wcerr@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __wcerr[sizeof(__stdoutbuf)]; @@ -68,16 +68,16 @@ static mbstate_t mb_wcerr; _ALIGNAS_TYPE (ostream) _LIBCPP_FUNC_VIS char clog[sizeof(ostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?clog@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?clog@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (wostream) _LIBCPP_FUNC_VIS char wclog[sizeof(wostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wclog@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") +__asm__("?wclog@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_ABI_NAMESPACE_STR "@std@@@12@A") #endif ; -ios_base::Init __start_std_streams; +_LIBCPP_HIDDEN ios_base::Init __start_std_streams; ios_base::Init::Init() { diff --git a/contrib/libc++/src/new.cpp b/contrib/libc++/src/new.cpp index 8013d89ae3c..cc8383d4f14 100644 --- a/contrib/libc++/src/new.cpp +++ b/contrib/libc++/src/new.cpp @@ -55,7 +55,7 @@ __throw_bad_alloc() } // std #if !defined(__GLIBCXX__) && \ - (!defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME)) && \ + !defined(_LIBCPP_DEFER_NEW_TO_VCRUNTIME) && \ !defined(_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS) // Implement all new and delete operators as weak definitions @@ -135,8 +135,7 @@ _LIBCPP_WEAK void operator delete(void* ptr) _NOEXCEPT { - if (ptr) - ::free(ptr); + ::free(ptr); } _LIBCPP_WEAK @@ -174,7 +173,7 @@ operator delete[] (void* ptr, size_t) _NOEXCEPT ::operator delete[](ptr); } -#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) +#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) _LIBCPP_WEAK void * @@ -257,11 +256,10 @@ _LIBCPP_WEAK void operator delete(void* ptr, std::align_val_t) _NOEXCEPT { - if (ptr) #if defined(_LIBCPP_MSVCRT_LIKE) - ::_aligned_free(ptr); + ::_aligned_free(ptr); #else - ::free(ptr); + ::free(ptr); #endif } @@ -300,5 +298,5 @@ operator delete[] (void* ptr, size_t, std::align_val_t alignment) _NOEXCEPT ::operator delete[](ptr, alignment); } -#endif // !_LIBCPP_HAS_NO_ALIGNED_ALLOCATION +#endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION #endif // !__GLIBCXX__ && (!_LIBCPP_ABI_MICROSOFT || _LIBCPP_NO_VCRUNTIME) && !_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS diff --git a/contrib/libc++/src/support/runtime/exception_fallback.ipp b/contrib/libc++/src/support/runtime/exception_fallback.ipp index 664e7f48c09..16d387b99e8 100644 --- a/contrib/libc++/src/support/runtime/exception_fallback.ipp +++ b/contrib/libc++/src/support/runtime/exception_fallback.ipp @@ -134,22 +134,6 @@ bad_array_new_length::what() const _NOEXCEPT return "bad_array_new_length"; } - -bad_array_length::bad_array_length() _NOEXCEPT -{ -} - -bad_array_length::~bad_array_length() _NOEXCEPT -{ -} - -const char* -bad_array_length::what() const _NOEXCEPT -{ - return "bad_array_length"; -} - - bad_cast::bad_cast() _NOEXCEPT { } diff --git a/contrib/libc++/src/support/runtime/exception_glibcxx.ipp b/contrib/libc++/src/support/runtime/exception_glibcxx.ipp index 0f78932f6f9..dda4432b508 100644 --- a/contrib/libc++/src/support/runtime/exception_glibcxx.ipp +++ b/contrib/libc++/src/support/runtime/exception_glibcxx.ipp @@ -22,11 +22,6 @@ bad_array_new_length::bad_array_new_length() _NOEXCEPT { } -bad_array_length::bad_array_length() _NOEXCEPT -{ -} - - bad_cast::bad_cast() _NOEXCEPT { } diff --git a/contrib/libc++/src/support/runtime/exception_libcxxrt.ipp b/contrib/libc++/src/support/runtime/exception_libcxxrt.ipp index 6d9e0cff58d..52fe8635db7 100644 --- a/contrib/libc++/src/support/runtime/exception_libcxxrt.ipp +++ b/contrib/libc++/src/support/runtime/exception_libcxxrt.ipp @@ -23,19 +23,4 @@ const char* bad_exception::what() const _NOEXCEPT return "std::bad_exception"; } - -bad_array_length::bad_array_length() _NOEXCEPT -{ -} - -bad_array_length::~bad_array_length() _NOEXCEPT -{ -} - -const char* -bad_array_length::what() const _NOEXCEPT -{ - return "bad_array_length"; -} - } // namespace std diff --git a/contrib/libc++/src/support/runtime/exception_msvc.ipp b/contrib/libc++/src/support/runtime/exception_msvc.ipp index 87d5a66fc8f..042d3add6c7 100644 --- a/contrib/libc++/src/support/runtime/exception_msvc.ipp +++ b/contrib/libc++/src/support/runtime/exception_msvc.ipp @@ -83,20 +83,6 @@ int uncaught_exceptions() _NOEXCEPT { return __uncaught_exceptions(); } -bad_array_length::bad_array_length() _NOEXCEPT -{ -} - -bad_array_length::~bad_array_length() _NOEXCEPT -{ -} - -const char* -bad_array_length::what() const _NOEXCEPT -{ - return "bad_array_length"; -} - #if defined(_LIBCPP_NO_VCRUNTIME) bad_cast::bad_cast() _NOEXCEPT { diff --git a/contrib/libc++/src/thread.cpp b/contrib/libc++/src/thread.cpp index 550da8ea71f..241cfee23c5 100644 --- a/contrib/libc++/src/thread.cpp +++ b/contrib/libc++/src/thread.cpp @@ -19,9 +19,9 @@ #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) # include -# if defined(BSD) +# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__APPLE__) # include -# endif // defined(BSD) +# endif #endif // defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) || defined(__CloudABI__) || defined(__Fuchsia__) diff --git a/contrib/llvm/FREEBSD-Xlist b/contrib/llvm/FREEBSD-Xlist index 9fbda5f3054..f694223a183 100644 --- a/contrib/llvm/FREEBSD-Xlist +++ b/contrib/llvm/FREEBSD-Xlist @@ -2,17 +2,15 @@ .arcconfig .clang-format .clang-tidy +.gitattributes .gitignore CMakeLists.txt CODE_OWNERS.TXT CREDITS.TXT LLVMBuild.txt -Makefile -Makefile.common -Makefile.config.in -Makefile.rules README.txt -autoconf/ +RELEASE_TESTERS.TXT +benchmarks/ bindings/ cmake/ configure @@ -21,473 +19,485 @@ examples/ include/llvm/CMakeLists.txt include/llvm/Config/ include/llvm/IR/CMakeLists.txt -include/llvm/Support/DataTypes.h.cmake +include/llvm/Support/CMakeLists.txt include/llvm/Support/LICENSE.TXT lib/Analysis/CMakeLists.txt -lib/Analysis/IPA/CMakeLists.txt -lib/Analysis/IPA/LLVMBuild.txt -lib/Analysis/IPA/Makefile lib/Analysis/LLVMBuild.txt -lib/Analysis/Makefile lib/Analysis/README.txt lib/AsmParser/CMakeLists.txt lib/AsmParser/LLVMBuild.txt -lib/AsmParser/Makefile +lib/BinaryFormat/CMakeLists.txt +lib/BinaryFormat/LLVMBuild.txt lib/Bitcode/CMakeLists.txt lib/Bitcode/LLVMBuild.txt -lib/Bitcode/Makefile lib/Bitcode/Reader/CMakeLists.txt lib/Bitcode/Reader/LLVMBuild.txt -lib/Bitcode/Reader/Makefile lib/Bitcode/Writer/CMakeLists.txt lib/Bitcode/Writer/LLVMBuild.txt -lib/Bitcode/Writer/Makefile lib/CMakeLists.txt lib/CodeGen/AsmPrinter/CMakeLists.txt lib/CodeGen/AsmPrinter/LLVMBuild.txt -lib/CodeGen/AsmPrinter/Makefile lib/CodeGen/CMakeLists.txt +lib/CodeGen/GlobalISel/CMakeLists.txt +lib/CodeGen/GlobalISel/LLVMBuild.txt lib/CodeGen/LLVMBuild.txt -lib/CodeGen/Makefile +lib/CodeGen/MIRParser/CMakeLists.txt +lib/CodeGen/MIRParser/LLVMBuild.txt lib/CodeGen/README.txt lib/CodeGen/SelectionDAG/CMakeLists.txt lib/CodeGen/SelectionDAG/LLVMBuild.txt -lib/CodeGen/SelectionDAG/Makefile lib/DebugInfo/CMakeLists.txt +lib/DebugInfo/CodeView/CMakeLists.txt +lib/DebugInfo/CodeView/LLVMBuild.txt lib/DebugInfo/DWARF/CMakeLists.txt lib/DebugInfo/DWARF/LLVMBuild.txt -lib/DebugInfo/DWARF/Makefile lib/DebugInfo/LLVMBuild.txt -lib/DebugInfo/Makefile +lib/DebugInfo/MSF/CMakeLists.txt +lib/DebugInfo/MSF/LLVMBuild.txt lib/DebugInfo/PDB/CMakeLists.txt lib/DebugInfo/PDB/LLVMBuild.txt -lib/DebugInfo/PDB/Makefile +lib/DebugInfo/Symbolize/CMakeLists.txt +lib/DebugInfo/Symbolize/LLVMBuild.txt +lib/Demangle/CMakeLists.txt +lib/Demangle/LLVMBuild.txt lib/ExecutionEngine/CMakeLists.txt lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt -lib/ExecutionEngine/IntelJITEvents/Makefile lib/ExecutionEngine/Interpreter/CMakeLists.txt lib/ExecutionEngine/Interpreter/LLVMBuild.txt -lib/ExecutionEngine/Interpreter/Makefile lib/ExecutionEngine/LLVMBuild.txt lib/ExecutionEngine/MCJIT/CMakeLists.txt lib/ExecutionEngine/MCJIT/LLVMBuild.txt -lib/ExecutionEngine/MCJIT/Makefile -lib/ExecutionEngine/Makefile lib/ExecutionEngine/OProfileJIT/CMakeLists.txt lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt -lib/ExecutionEngine/OProfileJIT/Makefile lib/ExecutionEngine/Orc/CMakeLists.txt lib/ExecutionEngine/Orc/LLVMBuild.txt -lib/ExecutionEngine/Orc/Makefile +lib/ExecutionEngine/PerfJITEvents/CMakeLists.txt +lib/ExecutionEngine/PerfJITEvents/LLVMBuild.txt lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt lib/ExecutionEngine/RuntimeDyld/LLVMBuild.txt -lib/ExecutionEngine/RuntimeDyld/Makefile +lib/FuzzMutate/CMakeLists.txt +lib/FuzzMutate/LLVMBuild.txt lib/Fuzzer/ lib/IR/CMakeLists.txt lib/IR/LLVMBuild.txt -lib/IR/Makefile lib/IRReader/CMakeLists.txt lib/IRReader/LLVMBuild.txt -lib/IRReader/Makefile lib/LLVMBuild.txt lib/LTO/CMakeLists.txt lib/LTO/LLVMBuild.txt -lib/LTO/Makefile lib/LineEditor/CMakeLists.txt lib/LineEditor/LLVMBuild.txt -lib/LineEditor/Makefile lib/Linker/CMakeLists.txt lib/Linker/LLVMBuild.txt -lib/Linker/Makefile lib/MC/CMakeLists.txt lib/MC/LLVMBuild.txt lib/MC/MCDisassembler/CMakeLists.txt lib/MC/MCDisassembler/LLVMBuild.txt -lib/MC/MCDisassembler/Makefile lib/MC/MCParser/CMakeLists.txt lib/MC/MCParser/LLVMBuild.txt -lib/MC/MCParser/Makefile -lib/MC/Makefile -lib/Makefile +lib/MCA/CMakeLists.txt +lib/MCA/LLVMBuild.txt lib/Object/CMakeLists.txt lib/Object/LLVMBuild.txt -lib/Object/Makefile +lib/ObjectYAML/CMakeLists.txt +lib/ObjectYAML/LLVMBuild.txt +lib/OptRemarks/CMakeLists.txt +lib/OptRemarks/LLVMBuild.txt lib/Option/CMakeLists.txt lib/Option/LLVMBuild.txt -lib/Option/Makefile lib/Passes/CMakeLists.txt lib/Passes/LLVMBuild.txt -lib/Passes/Makefile lib/ProfileData/CMakeLists.txt +lib/ProfileData/Coverage/CMakeLists.txt +lib/ProfileData/Coverage/LLVMBuild.txt lib/ProfileData/LLVMBuild.txt -lib/ProfileData/Makefile lib/Support/CMakeLists.txt lib/Support/LLVMBuild.txt -lib/Support/Makefile lib/Support/README.txt.system lib/TableGen/CMakeLists.txt lib/TableGen/LLVMBuild.txt -lib/TableGen/Makefile lib/Target/AArch64/AsmParser/CMakeLists.txt lib/Target/AArch64/AsmParser/LLVMBuild.txt -lib/Target/AArch64/AsmParser/Makefile lib/Target/AArch64/CMakeLists.txt lib/Target/AArch64/Disassembler/CMakeLists.txt lib/Target/AArch64/Disassembler/LLVMBuild.txt -lib/Target/AArch64/Disassembler/Makefile lib/Target/AArch64/InstPrinter/CMakeLists.txt lib/Target/AArch64/InstPrinter/LLVMBuild.txt -lib/Target/AArch64/InstPrinter/Makefile lib/Target/AArch64/LLVMBuild.txt lib/Target/AArch64/MCTargetDesc/CMakeLists.txt lib/Target/AArch64/MCTargetDesc/LLVMBuild.txt -lib/Target/AArch64/MCTargetDesc/Makefile -lib/Target/AArch64/Makefile lib/Target/AArch64/TargetInfo/CMakeLists.txt lib/Target/AArch64/TargetInfo/LLVMBuild.txt -lib/Target/AArch64/TargetInfo/Makefile lib/Target/AArch64/Utils/CMakeLists.txt lib/Target/AArch64/Utils/LLVMBuild.txt -lib/Target/AArch64/Utils/Makefile +lib/Target/AMDGPU/AsmParser/CMakeLists.txt +lib/Target/AMDGPU/AsmParser/LLVMBuild.txt +lib/Target/AMDGPU/CMakeLists.txt +lib/Target/AMDGPU/Disassembler/CMakeLists.txt +lib/Target/AMDGPU/Disassembler/LLVMBuild.txt +lib/Target/AMDGPU/InstPrinter/CMakeLists.txt +lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt +lib/Target/AMDGPU/LLVMBuild.txt +lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt +lib/Target/AMDGPU/TargetInfo/CMakeLists.txt +lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt +lib/Target/AMDGPU/Utils/CMakeLists.txt +lib/Target/AMDGPU/Utils/LLVMBuild.txt +lib/Target/ARC/CMakeLists.txt +lib/Target/ARC/Disassembler/CMakeLists.txt +lib/Target/ARC/Disassembler/LLVMBuild.txt +lib/Target/ARC/InstPrinter/CMakeLists.txt +lib/Target/ARC/InstPrinter/LLVMBuild.txt +lib/Target/ARC/LLVMBuild.txt +lib/Target/ARC/MCTargetDesc/CMakeLists.txt +lib/Target/ARC/MCTargetDesc/LLVMBuild.txt +lib/Target/ARC/TargetInfo/CMakeLists.txt +lib/Target/ARC/TargetInfo/LLVMBuild.txt lib/Target/ARM/AsmParser/CMakeLists.txt lib/Target/ARM/AsmParser/LLVMBuild.txt -lib/Target/ARM/AsmParser/Makefile lib/Target/ARM/CMakeLists.txt lib/Target/ARM/Disassembler/CMakeLists.txt lib/Target/ARM/Disassembler/LLVMBuild.txt -lib/Target/ARM/Disassembler/Makefile lib/Target/ARM/InstPrinter/CMakeLists.txt lib/Target/ARM/InstPrinter/LLVMBuild.txt -lib/Target/ARM/InstPrinter/Makefile lib/Target/ARM/LLVMBuild.txt lib/Target/ARM/MCTargetDesc/CMakeLists.txt lib/Target/ARM/MCTargetDesc/LLVMBuild.txt -lib/Target/ARM/MCTargetDesc/Makefile -lib/Target/ARM/Makefile lib/Target/ARM/README-Thumb.txt lib/Target/ARM/README-Thumb2.txt lib/Target/ARM/README.txt lib/Target/ARM/TargetInfo/CMakeLists.txt lib/Target/ARM/TargetInfo/LLVMBuild.txt -lib/Target/ARM/TargetInfo/Makefile +lib/Target/ARM/Utils/CMakeLists.txt +lib/Target/ARM/Utils/LLVMBuild.txt +lib/Target/AVR/AsmParser/CMakeLists.txt +lib/Target/AVR/AsmParser/LLVMBuild.txt +lib/Target/AVR/CMakeLists.txt +lib/Target/AVR/Disassembler/CMakeLists.txt +lib/Target/AVR/Disassembler/LLVMBuild.txt +lib/Target/AVR/InstPrinter/CMakeLists.txt +lib/Target/AVR/InstPrinter/LLVMBuild.txt +lib/Target/AVR/LLVMBuild.txt +lib/Target/AVR/MCTargetDesc/CMakeLists.txt +lib/Target/AVR/MCTargetDesc/LLVMBuild.txt +lib/Target/AVR/TargetInfo/CMakeLists.txt +lib/Target/AVR/TargetInfo/LLVMBuild.txt +lib/Target/BPF/AsmParser/CMakeLists.txt +lib/Target/BPF/AsmParser/LLVMBuild.txt lib/Target/BPF/CMakeLists.txt +lib/Target/BPF/Disassembler/CMakeLists.txt +lib/Target/BPF/Disassembler/LLVMBuild.txt lib/Target/BPF/InstPrinter/CMakeLists.txt lib/Target/BPF/InstPrinter/LLVMBuild.txt -lib/Target/BPF/InstPrinter/Makefile lib/Target/BPF/LLVMBuild.txt lib/Target/BPF/MCTargetDesc/CMakeLists.txt lib/Target/BPF/MCTargetDesc/LLVMBuild.txt -lib/Target/BPF/MCTargetDesc/Makefile -lib/Target/BPF/Makefile lib/Target/BPF/TargetInfo/CMakeLists.txt lib/Target/BPF/TargetInfo/LLVMBuild.txt -lib/Target/BPF/TargetInfo/Makefile lib/Target/CMakeLists.txt -lib/Target/CppBackend/CMakeLists.txt -lib/Target/CppBackend/LLVMBuild.txt -lib/Target/CppBackend/Makefile -lib/Target/CppBackend/TargetInfo/CMakeLists.txt -lib/Target/CppBackend/TargetInfo/LLVMBuild.txt -lib/Target/CppBackend/TargetInfo/Makefile +lib/Target/Hexagon/AsmParser/CMakeLists.txt +lib/Target/Hexagon/AsmParser/LLVMBuild.txt lib/Target/Hexagon/CMakeLists.txt lib/Target/Hexagon/Disassembler/CMakeLists.txt lib/Target/Hexagon/Disassembler/LLVMBuild.txt -lib/Target/Hexagon/Disassembler/Makefile lib/Target/Hexagon/LLVMBuild.txt lib/Target/Hexagon/MCTargetDesc/CMakeLists.txt lib/Target/Hexagon/MCTargetDesc/LLVMBuild.txt -lib/Target/Hexagon/MCTargetDesc/Makefile -lib/Target/Hexagon/Makefile lib/Target/Hexagon/TargetInfo/CMakeLists.txt lib/Target/Hexagon/TargetInfo/LLVMBuild.txt -lib/Target/Hexagon/TargetInfo/Makefile lib/Target/LLVMBuild.txt +lib/Target/Lanai/AsmParser/CMakeLists.txt +lib/Target/Lanai/AsmParser/LLVMBuild.txt +lib/Target/Lanai/CMakeLists.txt +lib/Target/Lanai/Disassembler/CMakeLists.txt +lib/Target/Lanai/Disassembler/LLVMBuild.txt +lib/Target/Lanai/InstPrinter/CMakeLists.txt +lib/Target/Lanai/InstPrinter/LLVMBuild.txt +lib/Target/Lanai/LLVMBuild.txt +lib/Target/Lanai/MCTargetDesc/CMakeLists.txt +lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt +lib/Target/Lanai/TargetInfo/CMakeLists.txt +lib/Target/Lanai/TargetInfo/LLVMBuild.txt +lib/Target/MSP430/AsmParser/CMakeLists.txt +lib/Target/MSP430/AsmParser/LLVMBuild.txt lib/Target/MSP430/CMakeLists.txt +lib/Target/MSP430/Disassembler/CMakeLists.txt +lib/Target/MSP430/Disassembler/LLVMBuild.txt lib/Target/MSP430/InstPrinter/CMakeLists.txt lib/Target/MSP430/InstPrinter/LLVMBuild.txt -lib/Target/MSP430/InstPrinter/Makefile lib/Target/MSP430/LLVMBuild.txt lib/Target/MSP430/MCTargetDesc/CMakeLists.txt lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt -lib/Target/MSP430/MCTargetDesc/Makefile -lib/Target/MSP430/Makefile lib/Target/MSP430/README.txt lib/Target/MSP430/TargetInfo/CMakeLists.txt lib/Target/MSP430/TargetInfo/LLVMBuild.txt -lib/Target/MSP430/TargetInfo/Makefile -lib/Target/Makefile lib/Target/Mips/AsmParser/CMakeLists.txt lib/Target/Mips/AsmParser/LLVMBuild.txt -lib/Target/Mips/AsmParser/Makefile lib/Target/Mips/CMakeLists.txt lib/Target/Mips/Disassembler/CMakeLists.txt lib/Target/Mips/Disassembler/LLVMBuild.txt -lib/Target/Mips/Disassembler/Makefile lib/Target/Mips/InstPrinter/CMakeLists.txt lib/Target/Mips/InstPrinter/LLVMBuild.txt -lib/Target/Mips/InstPrinter/Makefile lib/Target/Mips/LLVMBuild.txt lib/Target/Mips/MCTargetDesc/CMakeLists.txt lib/Target/Mips/MCTargetDesc/LLVMBuild.txt -lib/Target/Mips/MCTargetDesc/Makefile -lib/Target/Mips/Makefile lib/Target/Mips/TargetInfo/CMakeLists.txt lib/Target/Mips/TargetInfo/LLVMBuild.txt -lib/Target/Mips/TargetInfo/Makefile lib/Target/NVPTX/CMakeLists.txt lib/Target/NVPTX/InstPrinter/CMakeLists.txt lib/Target/NVPTX/InstPrinter/LLVMBuild.txt -lib/Target/NVPTX/InstPrinter/Makefile lib/Target/NVPTX/LLVMBuild.txt lib/Target/NVPTX/MCTargetDesc/CMakeLists.txt lib/Target/NVPTX/MCTargetDesc/LLVMBuild.txt -lib/Target/NVPTX/MCTargetDesc/Makefile -lib/Target/NVPTX/Makefile lib/Target/NVPTX/TargetInfo/CMakeLists.txt lib/Target/NVPTX/TargetInfo/LLVMBuild.txt -lib/Target/NVPTX/TargetInfo/Makefile lib/Target/PowerPC/AsmParser/CMakeLists.txt lib/Target/PowerPC/AsmParser/LLVMBuild.txt -lib/Target/PowerPC/AsmParser/Makefile lib/Target/PowerPC/CMakeLists.txt lib/Target/PowerPC/Disassembler/CMakeLists.txt lib/Target/PowerPC/Disassembler/LLVMBuild.txt -lib/Target/PowerPC/Disassembler/Makefile lib/Target/PowerPC/InstPrinter/CMakeLists.txt lib/Target/PowerPC/InstPrinter/LLVMBuild.txt -lib/Target/PowerPC/InstPrinter/Makefile lib/Target/PowerPC/LLVMBuild.txt lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt lib/Target/PowerPC/MCTargetDesc/LLVMBuild.txt -lib/Target/PowerPC/MCTargetDesc/Makefile -lib/Target/PowerPC/Makefile lib/Target/PowerPC/README.txt lib/Target/PowerPC/README_ALTIVEC.txt lib/Target/PowerPC/TargetInfo/CMakeLists.txt lib/Target/PowerPC/TargetInfo/LLVMBuild.txt -lib/Target/PowerPC/TargetInfo/Makefile -lib/Target/R600/AsmParser/CMakeLists.txt -lib/Target/R600/AsmParser/LLVMBuild.txt -lib/Target/R600/AsmParser/Makefile -lib/Target/R600/CMakeLists.txt -lib/Target/R600/InstPrinter/CMakeLists.txt -lib/Target/R600/InstPrinter/LLVMBuild.txt -lib/Target/R600/InstPrinter/Makefile -lib/Target/R600/LLVMBuild.txt -lib/Target/R600/MCTargetDesc/CMakeLists.txt -lib/Target/R600/MCTargetDesc/LLVMBuild.txt -lib/Target/R600/MCTargetDesc/Makefile -lib/Target/R600/Makefile -lib/Target/R600/TargetInfo/CMakeLists.txt -lib/Target/R600/TargetInfo/LLVMBuild.txt -lib/Target/R600/TargetInfo/Makefile lib/Target/README.txt +lib/Target/RISCV/AsmParser/CMakeLists.txt +lib/Target/RISCV/AsmParser/LLVMBuild.txt +lib/Target/RISCV/CMakeLists.txt +lib/Target/RISCV/Disassembler/CMakeLists.txt +lib/Target/RISCV/Disassembler/LLVMBuild.txt +lib/Target/RISCV/InstPrinter/CMakeLists.txt +lib/Target/RISCV/InstPrinter/LLVMBuild.txt +lib/Target/RISCV/LLVMBuild.txt +lib/Target/RISCV/MCTargetDesc/CMakeLists.txt +lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt +lib/Target/RISCV/TargetInfo/CMakeLists.txt +lib/Target/RISCV/TargetInfo/LLVMBuild.txt +lib/Target/RISCV/Utils/CMakeLists.txt +lib/Target/RISCV/Utils/LLVMBuild.txt lib/Target/Sparc/AsmParser/CMakeLists.txt lib/Target/Sparc/AsmParser/LLVMBuild.txt -lib/Target/Sparc/AsmParser/Makefile lib/Target/Sparc/CMakeLists.txt lib/Target/Sparc/Disassembler/CMakeLists.txt lib/Target/Sparc/Disassembler/LLVMBuild.txt -lib/Target/Sparc/Disassembler/Makefile lib/Target/Sparc/InstPrinter/CMakeLists.txt lib/Target/Sparc/InstPrinter/LLVMBuild.txt -lib/Target/Sparc/InstPrinter/Makefile lib/Target/Sparc/LLVMBuild.txt lib/Target/Sparc/MCTargetDesc/CMakeLists.txt lib/Target/Sparc/MCTargetDesc/LLVMBuild.txt -lib/Target/Sparc/MCTargetDesc/Makefile -lib/Target/Sparc/Makefile lib/Target/Sparc/README.txt lib/Target/Sparc/TargetInfo/CMakeLists.txt lib/Target/Sparc/TargetInfo/LLVMBuild.txt -lib/Target/Sparc/TargetInfo/Makefile lib/Target/SystemZ/AsmParser/CMakeLists.txt lib/Target/SystemZ/AsmParser/LLVMBuild.txt -lib/Target/SystemZ/AsmParser/Makefile lib/Target/SystemZ/CMakeLists.txt lib/Target/SystemZ/Disassembler/CMakeLists.txt lib/Target/SystemZ/Disassembler/LLVMBuild.txt -lib/Target/SystemZ/Disassembler/Makefile lib/Target/SystemZ/InstPrinter/CMakeLists.txt lib/Target/SystemZ/InstPrinter/LLVMBuild.txt -lib/Target/SystemZ/InstPrinter/Makefile lib/Target/SystemZ/LLVMBuild.txt lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt lib/Target/SystemZ/MCTargetDesc/LLVMBuild.txt -lib/Target/SystemZ/MCTargetDesc/Makefile -lib/Target/SystemZ/Makefile lib/Target/SystemZ/TargetInfo/CMakeLists.txt lib/Target/SystemZ/TargetInfo/LLVMBuild.txt -lib/Target/SystemZ/TargetInfo/Makefile +lib/Target/WebAssembly/AsmParser/CMakeLists.txt +lib/Target/WebAssembly/AsmParser/LLVMBuild.txt +lib/Target/WebAssembly/CMakeLists.txt +lib/Target/WebAssembly/Disassembler/CMakeLists.txt +lib/Target/WebAssembly/Disassembler/LLVMBuild.txt +lib/Target/WebAssembly/InstPrinter/CMakeLists.txt +lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt +lib/Target/WebAssembly/LLVMBuild.txt +lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt +lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt +lib/Target/WebAssembly/TargetInfo/CMakeLists.txt +lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt lib/Target/X86/AsmParser/CMakeLists.txt lib/Target/X86/AsmParser/LLVMBuild.txt -lib/Target/X86/AsmParser/Makefile lib/Target/X86/CMakeLists.txt lib/Target/X86/Disassembler/CMakeLists.txt lib/Target/X86/Disassembler/LLVMBuild.txt -lib/Target/X86/Disassembler/Makefile lib/Target/X86/InstPrinter/CMakeLists.txt lib/Target/X86/InstPrinter/LLVMBuild.txt -lib/Target/X86/InstPrinter/Makefile lib/Target/X86/LLVMBuild.txt lib/Target/X86/MCTargetDesc/CMakeLists.txt lib/Target/X86/MCTargetDesc/LLVMBuild.txt -lib/Target/X86/MCTargetDesc/Makefile -lib/Target/X86/Makefile lib/Target/X86/README-FPStack.txt -lib/Target/X86/README-MMX.txt lib/Target/X86/README-SSE.txt -lib/Target/X86/README-UNIMPLEMENTED.txt lib/Target/X86/README-X86-64.txt lib/Target/X86/README.txt lib/Target/X86/TargetInfo/CMakeLists.txt lib/Target/X86/TargetInfo/LLVMBuild.txt -lib/Target/X86/TargetInfo/Makefile lib/Target/X86/Utils/CMakeLists.txt lib/Target/X86/Utils/LLVMBuild.txt -lib/Target/X86/Utils/Makefile -lib/Target/X86/X86CompilationCallback_Win64.asm lib/Target/XCore/CMakeLists.txt lib/Target/XCore/Disassembler/CMakeLists.txt lib/Target/XCore/Disassembler/LLVMBuild.txt -lib/Target/XCore/Disassembler/Makefile lib/Target/XCore/InstPrinter/CMakeLists.txt lib/Target/XCore/InstPrinter/LLVMBuild.txt -lib/Target/XCore/InstPrinter/Makefile lib/Target/XCore/LLVMBuild.txt lib/Target/XCore/MCTargetDesc/CMakeLists.txt lib/Target/XCore/MCTargetDesc/LLVMBuild.txt -lib/Target/XCore/MCTargetDesc/Makefile -lib/Target/XCore/Makefile lib/Target/XCore/README.txt lib/Target/XCore/TargetInfo/CMakeLists.txt lib/Target/XCore/TargetInfo/LLVMBuild.txt -lib/Target/XCore/TargetInfo/Makefile +lib/Testing/CMakeLists.txt +lib/Testing/LLVMBuild.txt +lib/Testing/Support/CMakeLists.txt +lib/Testing/Support/LLVMBuild.txt +lib/TextAPI/CMakeLists.txt +lib/TextAPI/LLVMBuild.txt +lib/ToolDrivers/CMakeLists.txt +lib/ToolDrivers/LLVMBuild.txt +lib/ToolDrivers/llvm-dlltool/CMakeLists.txt +lib/ToolDrivers/llvm-dlltool/LLVMBuild.txt +lib/ToolDrivers/llvm-lib/CMakeLists.txt +lib/ToolDrivers/llvm-lib/LLVMBuild.txt +lib/Transforms/AggressiveInstCombine/CMakeLists.txt +lib/Transforms/AggressiveInstCombine/LLVMBuild.txt lib/Transforms/CMakeLists.txt +lib/Transforms/Coroutines/CMakeLists.txt +lib/Transforms/Coroutines/LLVMBuild.txt lib/Transforms/Hello/ lib/Transforms/IPO/CMakeLists.txt lib/Transforms/IPO/LLVMBuild.txt -lib/Transforms/IPO/Makefile lib/Transforms/InstCombine/CMakeLists.txt lib/Transforms/InstCombine/LLVMBuild.txt -lib/Transforms/InstCombine/Makefile lib/Transforms/Instrumentation/CMakeLists.txt lib/Transforms/Instrumentation/LLVMBuild.txt -lib/Transforms/Instrumentation/Makefile lib/Transforms/LLVMBuild.txt -lib/Transforms/Makefile lib/Transforms/ObjCARC/CMakeLists.txt lib/Transforms/ObjCARC/LLVMBuild.txt -lib/Transforms/ObjCARC/Makefile lib/Transforms/Scalar/CMakeLists.txt lib/Transforms/Scalar/LLVMBuild.txt -lib/Transforms/Scalar/Makefile lib/Transforms/Utils/CMakeLists.txt lib/Transforms/Utils/LLVMBuild.txt -lib/Transforms/Utils/Makefile lib/Transforms/Vectorize/CMakeLists.txt lib/Transforms/Vectorize/LLVMBuild.txt -lib/Transforms/Vectorize/Makefile +lib/WindowsManifest/CMakeLists.txt +lib/WindowsManifest/LLVMBuild.txt +lib/XRay/CMakeLists.txt +lib/XRay/LLVMBuild.txt llvm.spec.in -projects/ +projects/CMakeLists.txt +projects/LLVMBuild.txt +resources/ +runtimes/ test/ tools/CMakeLists.txt tools/LLVMBuild.txt -tools/Makefile +tools/bugpoint-passes/ tools/bugpoint/CMakeLists.txt tools/bugpoint/LLVMBuild.txt -tools/bugpoint/Makefile -tools/bugpoint-passes/ tools/dsymutil/ tools/gold/ tools/llc/CMakeLists.txt tools/llc/LLVMBuild.txt -tools/llc/Makefile tools/lli/CMakeLists.txt tools/lli/ChildTarget/CMakeLists.txt tools/lli/ChildTarget/LLVMBuild.txt -tools/lli/ChildTarget/Makefile tools/lli/LLVMBuild.txt -tools/lli/Makefile tools/llvm-ar/CMakeLists.txt tools/llvm-ar/LLVMBuild.txt -tools/llvm-ar/Makefile -tools/llvm-ar/install_symlink.cmake +tools/llvm-as-fuzzer/ +tools/llvm-as-parasitic-coverage-repro/ tools/llvm-as/CMakeLists.txt tools/llvm-as/LLVMBuild.txt -tools/llvm-as/Makefile tools/llvm-bcanalyzer/CMakeLists.txt tools/llvm-bcanalyzer/LLVMBuild.txt -tools/llvm-bcanalyzer/Makefile tools/llvm-c-test/ +tools/llvm-cat/ +tools/llvm-cfi-verify/ tools/llvm-config/ tools/llvm-cov/CMakeLists.txt tools/llvm-cov/LLVMBuild.txt -tools/llvm-cov/Makefile +tools/llvm-cvtres/ tools/llvm-cxxdump/CMakeLists.txt tools/llvm-cxxdump/LLVMBuild.txt -tools/llvm-cxxdump/Makefile +tools/llvm-cxxfilt/CMakeLists.txt +tools/llvm-cxxmap/CMakeLists.txt +tools/llvm-cxxmap/LLVMBuild.txt tools/llvm-diff/CMakeLists.txt tools/llvm-diff/LLVMBuild.txt -tools/llvm-diff/Makefile tools/llvm-dis/CMakeLists.txt tools/llvm-dis/LLVMBuild.txt -tools/llvm-dis/Makefile tools/llvm-dwarfdump/CMakeLists.txt tools/llvm-dwarfdump/LLVMBuild.txt -tools/llvm-dwarfdump/Makefile +tools/llvm-dwarfdump/fuzzer/ +tools/llvm-dwp/ +tools/llvm-elfabi/ +tools/llvm-exegesis/ tools/llvm-extract/CMakeLists.txt tools/llvm-extract/LLVMBuild.txt -tools/llvm-extract/Makefile tools/llvm-go/ +tools/llvm-isel-fuzzer/ +tools/llvm-itanium-demangle-fuzzer/ tools/llvm-jitlistener/ tools/llvm-link/CMakeLists.txt tools/llvm-link/LLVMBuild.txt -tools/llvm-link/Makefile tools/llvm-lto/CMakeLists.txt tools/llvm-lto/LLVMBuild.txt -tools/llvm-lto/Makefile +tools/llvm-lto2/CMakeLists.txt +tools/llvm-lto2/LLVMBuild.txt +tools/llvm-mc-assemble-fuzzer/ +tools/llvm-mc-disassemble-fuzzer/ tools/llvm-mc/CMakeLists.txt tools/llvm-mc/LLVMBuild.txt -tools/llvm-mc/Makefile +tools/llvm-mca/CMakeLists.txt +tools/llvm-mca/LLVMBuild.txt tools/llvm-mcmarkup/ +tools/llvm-microsoft-demangle-fuzzer/ +tools/llvm-modextract/CMakeLists.txt +tools/llvm-modextract/LLVMBuild.txt +tools/llvm-mt/ tools/llvm-nm/CMakeLists.txt tools/llvm-nm/LLVMBuild.txt -tools/llvm-nm/Makefile +tools/llvm-objcopy/CMakeLists.txt +tools/llvm-objcopy/LLVMBuild.txt tools/llvm-objdump/CMakeLists.txt tools/llvm-objdump/LLVMBuild.txt -tools/llvm-objdump/Makefile -tools/llvm-pdbdump/CMakeLists.txt -tools/llvm-pdbdump/LLVMBuild.txt -tools/llvm-pdbdump/Makefile +tools/llvm-opt-fuzzer/ +tools/llvm-opt-report/ +tools/llvm-pdbutil/CMakeLists.txt +tools/llvm-pdbutil/LLVMBuild.txt +tools/llvm-pdbutil/fuzzer/ tools/llvm-profdata/CMakeLists.txt tools/llvm-profdata/LLVMBuild.txt -tools/llvm-profdata/Makefile +tools/llvm-rc/ tools/llvm-readobj/CMakeLists.txt tools/llvm-readobj/LLVMBuild.txt -tools/llvm-readobj/Makefile tools/llvm-rtdyld/CMakeLists.txt tools/llvm-rtdyld/LLVMBuild.txt -tools/llvm-rtdyld/Makefile tools/llvm-shlib/ tools/llvm-size/ +tools/llvm-special-case-list-fuzzer/ +tools/llvm-split/ tools/llvm-stress/CMakeLists.txt tools/llvm-stress/LLVMBuild.txt -tools/llvm-stress/Makefile +tools/llvm-strings/ tools/llvm-symbolizer/CMakeLists.txt -tools/llvm-symbolizer/Makefile +tools/llvm-undname/ tools/llvm-vtabledump/ +tools/llvm-xray/CMakeLists.txt +tools/llvm-yaml-numeric-parser-fuzzer/ tools/lto/ -tools/macho-dump/CMakeLists.txt -tools/macho-dump/LLVMBuild.txt -tools/macho-dump/Makefile tools/msbuild/ tools/obj2yaml/ +tools/opt-remarks/ +tools/opt-viewer/ tools/opt/CMakeLists.txt tools/opt/LLVMBuild.txt -tools/opt/Makefile +tools/sancov/ +tools/sanstats/ tools/verify-uselistorder/ +tools/xcode-toolchain/ tools/yaml2obj/ unittests/ utils/DSAclean.py @@ -498,33 +508,47 @@ utils/GetRepositoryPath utils/GetSourceVersion utils/KillTheDoctor/ utils/LLVMBuild.txt -utils/Makefile +utils/LLVMVisualizers/ utils/Misc/ utils/PerfectShuffle/ +utils/Reviewing/ utils/TableGen/CMakeLists.txt utils/TableGen/LLVMBuild.txt -utils/TableGen/Makefile utils/TableGen/tdtags utils/Target/ utils/UpdateCMakeLists.pl +utils/UpdateTestChecks/ +utils/abtest/ +utils/abtest.py +utils/benchmark/ utils/bisect +utils/bisect-skip-count utils/bugpoint/ +utils/bugpoint_gisel_reducer.py utils/buildit/ utils/check-each-file utils/clang-parse-diagnostics-file utils/codegen-diff +utils/collect_and_build_with_pgo.py utils/count/ utils/countloc.sh utils/create_ladder_graph.py utils/crosstool/ +utils/demangle_tree.py +utils/docker/ utils/emacs/ +utils/extract_symbols.py +utils/extract_vplan.py utils/findmisopt utils/findoptdiff utils/findsym.pl utils/fpcmp/ +utils/gdb-scripts/ utils/getsrcs.sh utils/git/ utils/git-svn/ +utils/gn/ +utils/indirect_calls.py utils/jedit/ utils/kate/ utils/lint/ @@ -532,23 +556,33 @@ utils/lit/ utils/lldbDataFormatters.py utils/llvm-build/ utils/llvm-compilers-check +utils/llvm-gisel-cov.py utils/llvm-lit/ utils/llvm-native-gxx utils/llvm.grm -utils/llvm.natvis utils/llvmdo utils/llvmgrep -utils/makellvm utils/not/ +utils/opt-viewer/ +utils/prepare-code-coverage-artifact.py utils/release/ +utils/sanitizers/ +utils/schedcover.py utils/shuffle_fuzz.py +utils/shuffle_select_fuzz_tester.py utils/sort_includes.py -utils/test_debuginfo.pl utils/testgen/ utils/textmate/ +utils/unicode-case-fold.py utils/unittest/ +utils/update_analyze_test_checks.py +utils/update_cc_test_checks.py utils/update_llc_test_checks.py +utils/update_mca_test_checks.py +utils/update_mir_test_checks.py +utils/update_test_checks.py utils/valgrind/ utils/vim/ +utils/vscode/ utils/wciia.py utils/yaml-bench/ diff --git a/contrib/llvm/LICENSE.TXT b/contrib/llvm/LICENSE.TXT index 461398bab7a..e4d67d16fea 100644 --- a/contrib/llvm/LICENSE.TXT +++ b/contrib/llvm/LICENSE.TXT @@ -4,7 +4,7 @@ LLVM Release License University of Illinois/NCSA Open Source License -Copyright (c) 2003-2018 University of Illinois at Urbana-Champaign. +Copyright (c) 2003-2019 University of Illinois at Urbana-Champaign. All rights reserved. Developed by: diff --git a/contrib/llvm/include/llvm-c/Core.h b/contrib/llvm/include/llvm-c/Core.h index 6792219f873..06de058bdc5 100644 --- a/contrib/llvm/include/llvm-c/Core.h +++ b/contrib/llvm/include/llvm-c/Core.h @@ -54,6 +54,8 @@ extern "C" { * @{ */ +/// External users depend on the following values being stable. It is not safe +/// to reorder them. typedef enum { /* Terminator Instructions */ LLVMRet = 1, @@ -64,6 +66,9 @@ typedef enum { /* removed 6 due to API changes */ LLVMUnreachable = 7, + /* Standard Unary Operators */ + LLVMFNeg = 66, + /* Standard Binary Operators */ LLVMAdd = 8, LLVMFAdd = 9, @@ -515,6 +520,23 @@ void *LLVMContextGetDiagnosticContext(LLVMContextRef C); void LLVMContextSetYieldCallback(LLVMContextRef C, LLVMYieldCallback Callback, void *OpaqueHandle); +/** + * Retrieve whether the given context is set to discard all value names. + * + * @see LLVMContext::shouldDiscardValueNames() + */ +LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C); + +/** + * Set whether the given context discards all value names. + * + * If true, only the names of GlobalValue objects will be available in the IR. + * This can be used to save memory and runtime, especially in release mode. + * + * @see LLVMContext::setDiscardValueNames() + */ +void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard); + /** * Destroy a context instance. * @@ -842,6 +864,63 @@ LLVMContextRef LLVMGetModuleContext(LLVMModuleRef M); */ LLVMTypeRef LLVMGetTypeByName(LLVMModuleRef M, const char *Name); +/** + * Obtain an iterator to the first NamedMDNode in a Module. + * + * @see llvm::Module::named_metadata_begin() + */ +LLVMNamedMDNodeRef LLVMGetFirstNamedMetadata(LLVMModuleRef M); + +/** + * Obtain an iterator to the last NamedMDNode in a Module. + * + * @see llvm::Module::named_metadata_end() + */ +LLVMNamedMDNodeRef LLVMGetLastNamedMetadata(LLVMModuleRef M); + +/** + * Advance a NamedMDNode iterator to the next NamedMDNode. + * + * Returns NULL if the iterator was already at the end and there are no more + * named metadata nodes. + */ +LLVMNamedMDNodeRef LLVMGetNextNamedMetadata(LLVMNamedMDNodeRef NamedMDNode); + +/** + * Decrement a NamedMDNode iterator to the previous NamedMDNode. + * + * Returns NULL if the iterator was already at the beginning and there are + * no previous named metadata nodes. + */ +LLVMNamedMDNodeRef LLVMGetPreviousNamedMetadata(LLVMNamedMDNodeRef NamedMDNode); + +/** + * Retrieve a NamedMDNode with the given name, returning NULL if no such + * node exists. + * + * @see llvm::Module::getNamedMetadata() + */ +LLVMNamedMDNodeRef LLVMGetNamedMetadata(LLVMModuleRef M, + const char *Name, size_t NameLen); + +/** + * Retrieve a NamedMDNode with the given name, creating a new node if no such + * node exists. + * + * @see llvm::Module::getOrInsertNamedMetadata() + */ +LLVMNamedMDNodeRef LLVMGetOrInsertNamedMetadata(LLVMModuleRef M, + const char *Name, + size_t NameLen); + +/** + * Retrieve the name of a NamedMDNode. + * + * @see llvm::NamedMDNode::getName() + */ +const char *LLVMGetNamedMetadataName(LLVMNamedMDNodeRef NamedMD, + size_t *NameLen); + /** * Obtain the number of operands for named metadata in a module. * @@ -872,6 +951,44 @@ void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char *Name, void LLVMAddNamedMetadataOperand(LLVMModuleRef M, const char *Name, LLVMValueRef Val); +/** + * Return the directory of the debug location for this value, which must be + * an llvm::Instruction, llvm::GlobalVariable, or llvm::Function. + * + * @see llvm::Instruction::getDebugLoc() + * @see llvm::GlobalVariable::getDebugInfo() + * @see llvm::Function::getSubprogram() + */ +const char *LLVMGetDebugLocDirectory(LLVMValueRef Val, unsigned *Length); + +/** + * Return the filename of the debug location for this value, which must be + * an llvm::Instruction, llvm::GlobalVariable, or llvm::Function. + * + * @see llvm::Instruction::getDebugLoc() + * @see llvm::GlobalVariable::getDebugInfo() + * @see llvm::Function::getSubprogram() + */ +const char *LLVMGetDebugLocFilename(LLVMValueRef Val, unsigned *Length); + +/** + * Return the line number of the debug location for this value, which must be + * an llvm::Instruction, llvm::GlobalVariable, or llvm::Function. + * + * @see llvm::Instruction::getDebugLoc() + * @see llvm::GlobalVariable::getDebugInfo() + * @see llvm::Function::getSubprogram() + */ +unsigned LLVMGetDebugLocLine(LLVMValueRef Val); + +/** + * Return the column number of the debug location for this value, which must be + * an llvm::Instruction. + * + * @see llvm::Instruction::getDebugLoc() + */ +unsigned LLVMGetDebugLocColumn(LLVMValueRef Val); + /** * Add a function to a module under a specified name. * @@ -1221,6 +1338,13 @@ LLVMBool LLVMIsPackedStruct(LLVMTypeRef StructTy); */ LLVMBool LLVMIsOpaqueStruct(LLVMTypeRef StructTy); +/** + * Determine whether a structure is literal. + * + * @see llvm::StructType::isLiteral() + */ +LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy); + /** * @} */ @@ -1408,6 +1532,7 @@ LLVMTypeRef LLVMX86MMXType(void); macro(ConstantVector) \ macro(GlobalValue) \ macro(GlobalAlias) \ + macro(GlobalIFunc) \ macro(GlobalObject) \ macro(Function) \ macro(GlobalVariable) \ @@ -1417,7 +1542,9 @@ LLVMTypeRef LLVMX86MMXType(void); macro(CallInst) \ macro(IntrinsicInst) \ macro(DbgInfoIntrinsic) \ - macro(DbgDeclareInst) \ + macro(DbgVariableIntrinsic) \ + macro(DbgDeclareInst) \ + macro(DbgLabelInst) \ macro(MemIntrinsic) \ macro(MemCpyInst) \ macro(MemMoveInst) \ @@ -1434,16 +1561,15 @@ LLVMTypeRef LLVMX86MMXType(void); macro(SelectInst) \ macro(ShuffleVectorInst) \ macro(StoreInst) \ - macro(TerminatorInst) \ - macro(BranchInst) \ - macro(IndirectBrInst) \ - macro(InvokeInst) \ - macro(ReturnInst) \ - macro(SwitchInst) \ - macro(UnreachableInst) \ - macro(ResumeInst) \ - macro(CleanupReturnInst) \ - macro(CatchReturnInst) \ + macro(BranchInst) \ + macro(IndirectBrInst) \ + macro(InvokeInst) \ + macro(ReturnInst) \ + macro(SwitchInst) \ + macro(UnreachableInst) \ + macro(ResumeInst) \ + macro(CleanupReturnInst) \ + macro(CatchReturnInst) \ macro(FuncletPadInst) \ macro(CatchPadInst) \ macro(CleanupPadInst) \ @@ -1959,9 +2085,14 @@ LLVMValueRef LLVMConstLShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant); LLVMValueRef LLVMConstAShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant); LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal, LLVMValueRef *ConstantIndices, unsigned NumIndices); +LLVMValueRef LLVMConstGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal, + LLVMValueRef *ConstantIndices, unsigned NumIndices); LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal, LLVMValueRef *ConstantIndices, unsigned NumIndices); +LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal, + LLVMValueRef *ConstantIndices, + unsigned NumIndices); LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType); LLVMValueRef LLVMConstSExt(LLVMValueRef ConstantVal, LLVMTypeRef ToType); LLVMValueRef LLVMConstZExt(LLVMValueRef ConstantVal, LLVMTypeRef ToType); @@ -2037,6 +2168,14 @@ void LLVMSetDLLStorageClass(LLVMValueRef Global, LLVMDLLStorageClass Class); LLVMUnnamedAddr LLVMGetUnnamedAddress(LLVMValueRef Global); void LLVMSetUnnamedAddress(LLVMValueRef Global, LLVMUnnamedAddr UnnamedAddr); +/** + * Returns the "value type" of a global value. This differs from the formal + * type of a global value which is always a pointer type. + * + * @see llvm::GlobalValue::getValueType() + */ +LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global); + /** Deprecated: Use LLVMGetUnnamedAddress instead. */ LLVMBool LLVMHasUnnamedAddr(LLVMValueRef Global); /** Deprecated: Use LLVMSetUnnamedAddress instead. */ @@ -2067,6 +2206,58 @@ unsigned LLVMGetAlignment(LLVMValueRef V); */ void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes); +/** + * Sets a metadata attachment, erasing the existing metadata attachment if + * it already exists for the given kind. + * + * @see llvm::GlobalObject::setMetadata() + */ +void LLVMGlobalSetMetadata(LLVMValueRef Global, unsigned Kind, + LLVMMetadataRef MD); + +/** + * Erases a metadata attachment of the given kind if it exists. + * + * @see llvm::GlobalObject::eraseMetadata() + */ +void LLVMGlobalEraseMetadata(LLVMValueRef Global, unsigned Kind); + +/** + * Removes all metadata attachments from this value. + * + * @see llvm::GlobalObject::clearMetadata() + */ +void LLVMGlobalClearMetadata(LLVMValueRef Global); + +/** + * Retrieves an array of metadata entries representing the metadata attached to + * this value. The caller is responsible for freeing this array by calling + * \c LLVMDisposeValueMetadataEntries. + * + * @see llvm::GlobalObject::getAllMetadata() + */ +LLVMValueMetadataEntry *LLVMGlobalCopyAllMetadata(LLVMValueRef Value, + size_t *NumEntries); + +/** + * Destroys value metadata entries. + */ +void LLVMDisposeValueMetadataEntries(LLVMValueMetadataEntry *Entries); + +/** + * Returns the kind of a value metadata entry at a specific index. + */ +unsigned LLVMValueMetadataEntriesGetKind(LLVMValueMetadataEntry *Entries, + unsigned Index); + +/** + * Returns the underlying metadata node of a value metadata entry at a + * specific index. + */ +LLVMMetadataRef +LLVMValueMetadataEntriesGetMetadata(LLVMValueMetadataEntry *Entries, + unsigned Index); + /** * @} */ @@ -2217,6 +2408,54 @@ void LLVMSetPersonalityFn(LLVMValueRef Fn, LLVMValueRef PersonalityFn); */ unsigned LLVMGetIntrinsicID(LLVMValueRef Fn); +/** + * Create or insert the declaration of an intrinsic. For overloaded intrinsics, + * parameter types must be provided to uniquely identify an overload. + * + * @see llvm::Intrinsic::getDeclaration() + */ +LLVMValueRef LLVMGetIntrinsicDeclaration(LLVMModuleRef Mod, + unsigned ID, + LLVMTypeRef *ParamTypes, + size_t ParamCount); + +/** + * Retrieves the type of an intrinsic. For overloaded intrinsics, parameter + * types must be provided to uniquely identify an overload. + * + * @see llvm::Intrinsic::getType() + */ +LLVMTypeRef LLVMIntrinsicGetType(LLVMContextRef Ctx, unsigned ID, + LLVMTypeRef *ParamTypes, size_t ParamCount); + +/** + * Retrieves the name of an intrinsic. + * + * @see llvm::Intrinsic::getName() + */ +const char *LLVMIntrinsicGetName(unsigned ID, size_t *NameLength); + +/** + * Copies the name of an overloaded intrinsic identified by a given list of + * parameter types. + * + * Unlike LLVMIntrinsicGetName, the caller is responsible for freeing the + * returned string. + * + * @see llvm::Intrinsic::getName() + */ +const char *LLVMIntrinsicCopyOverloadedName(unsigned ID, + LLVMTypeRef *ParamTypes, + size_t ParamCount, + size_t *NameLength); + +/** + * Obtain if the intrinsic identified by the given ID is overloaded. + * + * @see llvm::Intrinsic::isOverloaded() + */ +LLVMBool LLVMIntrinsicIsOverloaded(unsigned ID); + /** * Obtain the calling function of a function. * @@ -2514,7 +2753,7 @@ LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB); * If the basic block does not have a terminator (it is not well-formed * if it doesn't), then NULL is returned. * - * The returned LLVMValueRef corresponds to a llvm::TerminatorInst. + * The returned LLVMValueRef corresponds to an llvm::Instruction. * * @see llvm::BasicBlock::getTerminator() */ @@ -2572,6 +2811,14 @@ LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB); */ LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn); +/** + * Create a new basic block without inserting it into a function. + * + * @see llvm::BasicBlock::Create() + */ +LLVMBasicBlockRef LLVMCreateBasicBlockInContext(LLVMContextRef C, + const char *Name); + /** * Append a basic block to the end of a function. * @@ -2694,6 +2941,16 @@ LLVMValueRef LLVMGetMetadata(LLVMValueRef Val, unsigned KindID); */ void LLVMSetMetadata(LLVMValueRef Val, unsigned KindID, LLVMValueRef Node); +/** + * Returns the metadata associated with an instruction value, but filters out + * all the debug locations. + * + * @see llvm::Instruction::getAllMetadataOtherThanDebugLoc() + */ +LLVMValueMetadataEntry * +LLVMInstructionGetAllMetadataOtherThanDebugLoc(LLVMValueRef Instr, + size_t *NumEntries); + /** * Obtain the basic block to which an instruction belongs. * @@ -2776,6 +3033,15 @@ LLVMRealPredicate LLVMGetFCmpPredicate(LLVMValueRef Inst); */ LLVMValueRef LLVMInstructionClone(LLVMValueRef Inst); +/** + * Determine whether an instruction is a terminator. This routine is named to + * be compatible with historical functions that did this by querying the + * underlying C++ type. + * + * @see llvm::Instruction::isTerminator() + */ +LLVMValueRef LLVMIsATerminatorInst(LLVMValueRef Inst); + /** * @defgroup LLVMCCoreValueInstructionCall Call Sites and Invocations * @@ -2838,6 +3104,13 @@ void LLVMRemoveCallSiteEnumAttribute(LLVMValueRef C, LLVMAttributeIndex Idx, void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C, LLVMAttributeIndex Idx, const char *K, unsigned KLen); +/** + * Obtain the function type called by this instruction. + * + * @see llvm::CallBase::getFunctionType() + */ +LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C); + /** * Obtain the pointer to the function invoked by this instruction. * @@ -2916,8 +3189,8 @@ void LLVMSetUnwindDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B); /** * @defgroup LLVMCCoreValueInstructionTerminator Terminators * - * Functions in this group only apply to instructions that map to - * llvm::TerminatorInst instances. + * Functions in this group only apply to instructions for which + * LLVMIsATerminatorInst returns true. * * @{ */ @@ -2925,21 +3198,21 @@ void LLVMSetUnwindDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B); /** * Return the number of successors that this terminator has. * - * @see llvm::TerminatorInst::getNumSuccessors + * @see llvm::Instruction::getNumSuccessors */ unsigned LLVMGetNumSuccessors(LLVMValueRef Term); /** * Return the specified successor. * - * @see llvm::TerminatorInst::getSuccessor + * @see llvm::Instruction::getSuccessor */ LLVMBasicBlockRef LLVMGetSuccessor(LLVMValueRef Term, unsigned i); /** * Update the specified successor to point at the provided block. * - * @see llvm::TerminatorInst::setSuccessor + * @see llvm::Instruction::setSuccessor */ void LLVMSetSuccessor(LLVMValueRef Term, unsigned i, LLVMBasicBlockRef block); @@ -3130,10 +3403,16 @@ LLVMValueRef LLVMBuildSwitch(LLVMBuilderRef, LLVMValueRef V, LLVMBasicBlockRef Else, unsigned NumCases); LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr, unsigned NumDests); +// LLVMBuildInvoke is deprecated in favor of LLVMBuildInvoke2, in preparation +// for opaque pointer types. LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, const char *Name); +LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + const char *Name); LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef); /* Exception Handling */ @@ -3290,13 +3569,48 @@ LLVMValueRef LLVMBuildNot(LLVMBuilderRef, LLVMValueRef V, const char *Name); LLVMValueRef LLVMBuildMalloc(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name); LLVMValueRef LLVMBuildArrayMalloc(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Val, const char *Name); + +/** + * Creates and inserts a memset to the specified pointer and the + * specified value. + * + * @see llvm::IRRBuilder::CreateMemSet() + */ +LLVMValueRef LLVMBuildMemSet(LLVMBuilderRef B, LLVMValueRef Ptr, + LLVMValueRef Val, LLVMValueRef Len, + unsigned Align); +/** + * Creates and inserts a memcpy between the specified pointers. + * + * @see llvm::IRRBuilder::CreateMemCpy() + */ +LLVMValueRef LLVMBuildMemCpy(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size); +/** + * Creates and inserts a memmove between the specified pointers. + * + * @see llvm::IRRBuilder::CreateMemMove() + */ +LLVMValueRef LLVMBuildMemMove(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size); + LLVMValueRef LLVMBuildAlloca(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name); LLVMValueRef LLVMBuildArrayAlloca(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Val, const char *Name); LLVMValueRef LLVMBuildFree(LLVMBuilderRef, LLVMValueRef PointerVal); +// LLVMBuildLoad is deprecated in favor of LLVMBuildLoad2, in preparation for +// opaque pointer types. LLVMValueRef LLVMBuildLoad(LLVMBuilderRef, LLVMValueRef PointerVal, const char *Name); +LLVMValueRef LLVMBuildLoad2(LLVMBuilderRef, LLVMTypeRef Ty, + LLVMValueRef PointerVal, const char *Name); LLVMValueRef LLVMBuildStore(LLVMBuilderRef, LLVMValueRef Val, LLVMValueRef Ptr); +// LLVMBuildGEP, LLVMBuildInBoundsGEP, and LLVMBuildStructGEP are deprecated in +// favor of LLVMBuild*GEP2, in preparation for opaque pointer types. LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer, LLVMValueRef *Indices, unsigned NumIndices, const char *Name); @@ -3305,6 +3619,15 @@ LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer, const char *Name); LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer, unsigned Idx, const char *Name); +LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Pointer, LLVMValueRef *Indices, + unsigned NumIndices, const char *Name); +LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Pointer, LLVMValueRef *Indices, + unsigned NumIndices, const char *Name); +LLVMValueRef LLVMBuildStructGEP2(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Pointer, unsigned Idx, + const char *Name); LLVMValueRef LLVMBuildGlobalString(LLVMBuilderRef B, const char *Str, const char *Name); LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B, const char *Str, @@ -3351,11 +3674,16 @@ LLVMValueRef LLVMBuildCast(LLVMBuilderRef B, LLVMOpcode Op, LLVMValueRef Val, LLVMTypeRef DestTy, const char *Name); LLVMValueRef LLVMBuildPointerCast(LLVMBuilderRef, LLVMValueRef Val, LLVMTypeRef DestTy, const char *Name); -LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef, LLVMValueRef Val, /*Signed cast!*/ - LLVMTypeRef DestTy, const char *Name); +LLVMValueRef LLVMBuildIntCast2(LLVMBuilderRef, LLVMValueRef Val, + LLVMTypeRef DestTy, LLVMBool IsSigned, + const char *Name); LLVMValueRef LLVMBuildFPCast(LLVMBuilderRef, LLVMValueRef Val, LLVMTypeRef DestTy, const char *Name); +/** Deprecated: This cast is always signed. Use LLVMBuildIntCast2 instead. */ +LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef, LLVMValueRef Val, /*Signed cast!*/ + LLVMTypeRef DestTy, const char *Name); + /* Comparisons */ LLVMValueRef LLVMBuildICmp(LLVMBuilderRef, LLVMIntPredicate Op, LLVMValueRef LHS, LLVMValueRef RHS, @@ -3366,9 +3694,14 @@ LLVMValueRef LLVMBuildFCmp(LLVMBuilderRef, LLVMRealPredicate Op, /* Miscellaneous instructions */ LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name); +// LLVMBuildCall is deprecated in favor of LLVMBuildCall2, in preparation for +// opaque pointer types. LLVMValueRef LLVMBuildCall(LLVMBuilderRef, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, const char *Name); +LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + const char *Name); LLVMValueRef LLVMBuildSelect(LLVMBuilderRef, LLVMValueRef If, LLVMValueRef Then, LLVMValueRef Else, const char *Name); diff --git a/contrib/llvm/include/llvm-c/DebugInfo.h b/contrib/llvm/include/llvm-c/DebugInfo.h index cee6755f187..87a72034b0e 100644 --- a/contrib/llvm/include/llvm-c/DebugInfo.h +++ b/contrib/llvm/include/llvm-c/DebugInfo.h @@ -54,9 +54,12 @@ typedef enum { LLVMDIFlagMainSubprogram = 1 << 21, LLVMDIFlagTypePassByValue = 1 << 22, LLVMDIFlagTypePassByReference = 1 << 23, - LLVMDIFlagFixedEnum = 1 << 24, + LLVMDIFlagEnumClass = 1 << 24, + LLVMDIFlagFixedEnum = LLVMDIFlagEnumClass, // Deprecated. LLVMDIFlagThunk = 1 << 25, LLVMDIFlagTrivial = 1 << 26, + LLVMDIFlagBigEndian = 1 << 27, + LLVMDIFlagLittleEndian = 1 << 28, LLVMDIFlagIndirectVirtualBase = (1 << 2) | (1 << 5), LLVMDIFlagAccessibility = LLVMDIFlagPrivate | LLVMDIFlagProtected | LLVMDIFlagPublic, @@ -124,6 +127,44 @@ typedef enum { LLVMDWARFEmissionLineTablesOnly } LLVMDWARFEmissionKind; +/** + * The kind of metadata nodes. + */ +enum { + LLVMMDStringMetadataKind, + LLVMConstantAsMetadataMetadataKind, + LLVMLocalAsMetadataMetadataKind, + LLVMDistinctMDOperandPlaceholderMetadataKind, + LLVMMDTupleMetadataKind, + LLVMDILocationMetadataKind, + LLVMDIExpressionMetadataKind, + LLVMDIGlobalVariableExpressionMetadataKind, + LLVMGenericDINodeMetadataKind, + LLVMDISubrangeMetadataKind, + LLVMDIEnumeratorMetadataKind, + LLVMDIBasicTypeMetadataKind, + LLVMDIDerivedTypeMetadataKind, + LLVMDICompositeTypeMetadataKind, + LLVMDISubroutineTypeMetadataKind, + LLVMDIFileMetadataKind, + LLVMDICompileUnitMetadataKind, + LLVMDISubprogramMetadataKind, + LLVMDILexicalBlockMetadataKind, + LLVMDILexicalBlockFileMetadataKind, + LLVMDINamespaceMetadataKind, + LLVMDIModuleMetadataKind, + LLVMDITemplateTypeParameterMetadataKind, + LLVMDITemplateValueParameterMetadataKind, + LLVMDIGlobalVariableMetadataKind, + LLVMDILocalVariableMetadataKind, + LLVMDILabelMetadataKind, + LLVMDIObjCPropertyMetadataKind, + LLVMDIImportedEntityMetadataKind, + LLVMDIMacroMetadataKind, + LLVMDIMacroFileMetadataKind +}; +typedef unsigned LLVMMetadataKind; + /** * An LLVM DWARF type encoding. */ @@ -531,11 +572,13 @@ LLVMDIBuilderCreateUnspecifiedType(LLVMDIBuilderRef Builder, const char *Name, * \param NameLen Length of type name. * \param SizeInBits Size of the type. * \param Encoding DWARF encoding code, e.g. \c LLVMDWARFTypeEncoding_float. + * \param Flags Flags to encode optional attribute like endianity */ LLVMMetadataRef LLVMDIBuilderCreateBasicType(LLVMDIBuilderRef Builder, const char *Name, size_t NameLen, uint64_t SizeInBits, - LLVMDWARFTypeEncoding Encoding); + LLVMDWARFTypeEncoding Encoding, + LLVMDIFlags Flags); /** * Create debugging information entry for a pointer. @@ -965,21 +1008,15 @@ LLVMDIBuilderCreateConstantValueExpression(LLVMDIBuilderRef Builder, * \param Expr The location of the global relative to the attached * GlobalVariable. * \param Decl Reference to the corresponding declaration. + * variables. * \param AlignInBits Variable alignment(or 0 if no alignment attr was * specified) */ -LLVMMetadataRef -LLVMDIBuilderCreateGlobalVariableExpression(LLVMDIBuilderRef Builder, - LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - const char *Linkage, size_t LinkLen, - LLVMMetadataRef File, - unsigned LineNo, - LLVMMetadataRef Ty, - LLVMBool LocalToUnit, - LLVMMetadataRef Expr, - LLVMMetadataRef Decl, - uint32_t AlignInBits); +LLVMMetadataRef LLVMDIBuilderCreateGlobalVariableExpression( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, const char *Linkage, size_t LinkLen, LLVMMetadataRef File, + unsigned LineNo, LLVMMetadataRef Ty, LLVMBool LocalToUnit, + LLVMMetadataRef Expr, LLVMMetadataRef Decl, uint32_t AlignInBits); /** * Create a new temporary \c MDNode. Suitable for use in constructing cyclic * \c MDNode structures. A temporary \c MDNode is not uniqued, may be RAUW'd, @@ -1025,17 +1062,11 @@ void LLVMMetadataReplaceAllUsesWith(LLVMMetadataRef TempTargetMetadata, * \param AlignInBits Variable alignment(or 0 if no alignment attr was * specified) */ -LLVMMetadataRef -LLVMDIBuilderCreateTempGlobalVariableFwdDecl(LLVMDIBuilderRef Builder, - LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - const char *Linkage, size_t LnkLen, - LLVMMetadataRef File, - unsigned LineNo, - LLVMMetadataRef Ty, - LLVMBool LocalToUnit, - LLVMMetadataRef Decl, - uint32_t AlignInBits); +LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, const char *Linkage, size_t LnkLen, LLVMMetadataRef File, + unsigned LineNo, LLVMMetadataRef Ty, LLVMBool LocalToUnit, + LLVMMetadataRef Decl, uint32_t AlignInBits); /** * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1149,6 +1180,13 @@ LLVMMetadataRef LLVMGetSubprogram(LLVMValueRef Func); */ void LLVMSetSubprogram(LLVMValueRef Func, LLVMMetadataRef SP); +/** + * Obtain the enumerated type of a Metadata instance. + * + * @see llvm::Metadata::getMetadataID() + */ +LLVMMetadataKind LLVMGetMetadataKind(LLVMMetadataRef Metadata); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/contrib/llvm/include/llvm-c/Error.h b/contrib/llvm/include/llvm-c/Error.h new file mode 100644 index 00000000000..71e84661222 --- /dev/null +++ b/contrib/llvm/include/llvm-c/Error.h @@ -0,0 +1,69 @@ +/*===------- llvm-c/Error.h - llvm::Error class C Interface -------*- C -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This file defines the C interface to LLVM's Error class. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_ERROR_H +#define LLVM_C_ERROR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define LLVMErrorSuccess 0 + +/** + * Opaque reference to an error instance. Null serves as the 'success' value. + */ +typedef struct LLVMOpaqueError *LLVMErrorRef; + +/** + * Error type identifier. + */ +typedef const void *LLVMErrorTypeId; + +/** + * Returns the type id for the given error instance, which must be a failure + * value (i.e. non-null). + */ +LLVMErrorTypeId LLVMGetErrorTypeId(LLVMErrorRef Err); + +/** + * Dispose of the given error without handling it. This operation consumes the + * error, and the given LLVMErrorRef value is not usable once this call returns. + * Note: This method *only* needs to be called if the error is not being passed + * to some other consuming operation, e.g. LLVMGetErrorMessage. + */ +void LLVMConsumeError(LLVMErrorRef Err); + +/** + * Returns the given string's error message. This operation consumes the error, + * and the given LLVMErrorRef value is not usable once this call returns. + * The caller is responsible for disposing of the string by calling + * LLVMDisposeErrorMessage. + */ +char *LLVMGetErrorMessage(LLVMErrorRef Err); + +/** + * Dispose of the given error message. + */ +void LLVMDisposeErrorMessage(char *ErrMsg); + +/** + * Returns the type id for llvm StringError. + */ +LLVMErrorTypeId LLVMGetStringErrorTypeId(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/llvm/include/llvm-c/ExecutionEngine.h b/contrib/llvm/include/llvm-c/ExecutionEngine.h index 49ae6fee45f..e8ebef9ab15 100644 --- a/contrib/llvm/include/llvm-c/ExecutionEngine.h +++ b/contrib/llvm/include/llvm-c/ExecutionEngine.h @@ -186,7 +186,7 @@ void LLVMDisposeMCJITMemoryManager(LLVMMCJITMemoryManagerRef MM); LLVMJITEventListenerRef LLVMCreateGDBRegistrationListener(void); LLVMJITEventListenerRef LLVMCreateIntelJITEventListener(void); -LLVMJITEventListenerRef LLVMCreateOprofileJITEventListener(void); +LLVMJITEventListenerRef LLVMCreateOProfileJITEventListener(void); LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void); /** diff --git a/contrib/llvm/include/llvm-c/OptRemarks.h b/contrib/llvm/include/llvm-c/OptRemarks.h new file mode 100644 index 00000000000..6a90394e711 --- /dev/null +++ b/contrib/llvm/include/llvm-c/OptRemarks.h @@ -0,0 +1,204 @@ +/*===-- llvm-c/OptRemarks.h - OptRemarks Public C Interface -------*- C -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header provides a public interface to an opt-remark library. *| +|* LLVM provides an implementation of this interface. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_OPT_REMARKS_H +#define LLVM_C_OPT_REMARKS_H + +#include "llvm-c/Core.h" +#include "llvm-c/Types.h" +#ifdef __cplusplus +#include +extern "C" { +#else +#include +#endif /* !defined(__cplusplus) */ + +/** + * @defgroup LLVMCOPTREMARKS OptRemarks + * @ingroup LLVMC + * + * @{ + */ + +#define OPT_REMARKS_API_VERSION 0 + +/** + * String containing a buffer and a length. The buffer is not guaranteed to be + * zero-terminated. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +typedef struct { + const char *Str; + uint32_t Len; +} LLVMOptRemarkStringRef; + +/** + * DebugLoc containing File, Line and Column. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +typedef struct { + // File: + LLVMOptRemarkStringRef SourceFile; + // Line: + uint32_t SourceLineNumber; + // Column: + uint32_t SourceColumnNumber; +} LLVMOptRemarkDebugLoc; + +/** + * Element of the "Args" list. The key might give more information about what + * are the semantics of the value, e.g. "Callee" will tell you that the value + * is a symbol that names a function. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +typedef struct { + // e.g. "Callee" + LLVMOptRemarkStringRef Key; + // e.g. "malloc" + LLVMOptRemarkStringRef Value; + + // "DebugLoc": Optional + LLVMOptRemarkDebugLoc DebugLoc; +} LLVMOptRemarkArg; + +/** + * One remark entry. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +typedef struct { + // e.g. !Missed, !Passed + LLVMOptRemarkStringRef RemarkType; + // "Pass": Required + LLVMOptRemarkStringRef PassName; + // "Name": Required + LLVMOptRemarkStringRef RemarkName; + // "Function": Required + LLVMOptRemarkStringRef FunctionName; + + // "DebugLoc": Optional + LLVMOptRemarkDebugLoc DebugLoc; + // "Hotness": Optional + uint32_t Hotness; + // "Args": Optional. It is an array of `num_args` elements. + uint32_t NumArgs; + LLVMOptRemarkArg *Args; +} LLVMOptRemarkEntry; + +typedef struct LLVMOptRemarkOpaqueParser *LLVMOptRemarkParserRef; + +/** + * Creates a remark parser that can be used to read and parse the buffer located + * in \p Buf of size \p Size. + * + * \p Buf cannot be NULL. + * + * This function should be paired with LLVMOptRemarkParserDispose() to avoid + * leaking resources. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +extern LLVMOptRemarkParserRef LLVMOptRemarkParserCreate(const void *Buf, + uint64_t Size); + +/** + * Returns the next remark in the file. + * + * The value pointed to by the return value is invalidated by the next call to + * LLVMOptRemarkParserGetNext(). + * + * If the parser reaches the end of the buffer, the return value will be NULL. + * + * In the case of an error, the return value will be NULL, and: + * + * 1) LLVMOptRemarkParserHasError() will return `1`. + * + * 2) LLVMOptRemarkParserGetErrorMessage() will return a descriptive error + * message. + * + * An error may occur if: + * + * 1) An argument is invalid. + * + * 2) There is a YAML parsing error. This type of error aborts parsing + * immediately and returns `1`. It can occur on malformed YAML. + * + * 3) Remark parsing error. If this type of error occurs, the parser won't call + * the handler and will continue to the next one. It can occur on malformed + * remarks, like missing or extra fields in the file. + * + * Here is a quick example of the usage: + * + * ``` + * LLVMOptRemarkParserRef Parser = LLVMOptRemarkParserCreate(Buf, Size); + * LLVMOptRemarkEntry *Remark = NULL; + * while ((Remark == LLVMOptRemarkParserGetNext(Parser))) { + * // use Remark + * } + * bool HasError = LLVMOptRemarkParserHasError(Parser); + * LLVMOptRemarkParserDispose(Parser); + * ``` + * + * \since OPT_REMARKS_API_VERSION=0 + */ +extern LLVMOptRemarkEntry * +LLVMOptRemarkParserGetNext(LLVMOptRemarkParserRef Parser); + +/** + * Returns `1` if the parser encountered an error while parsing the buffer. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +extern LLVMBool LLVMOptRemarkParserHasError(LLVMOptRemarkParserRef Parser); + +/** + * Returns a null-terminated string containing an error message. + * + * In case of no error, the result is `NULL`. + * + * The memory of the string is bound to the lifetime of \p Parser. If + * LLVMOptRemarkParserDispose() is called, the memory of the string will be + * released. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +extern const char * +LLVMOptRemarkParserGetErrorMessage(LLVMOptRemarkParserRef Parser); + +/** + * Releases all the resources used by \p Parser. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +extern void LLVMOptRemarkParserDispose(LLVMOptRemarkParserRef Parser); + +/** + * Returns the version of the opt-remarks dylib. + * + * \since OPT_REMARKS_API_VERSION=0 + */ +extern uint32_t LLVMOptRemarkVersion(void); + +/** + * @} // endgoup LLVMCOPTREMARKS + */ + +#ifdef __cplusplus +} +#endif /* !defined(__cplusplus) */ + +#endif /* LLVM_C_OPT_REMARKS_H */ diff --git a/contrib/llvm/include/llvm-c/OrcBindings.h b/contrib/llvm/include/llvm-c/OrcBindings.h index 9497f0d4077..570db87fee9 100644 --- a/contrib/llvm/include/llvm-c/OrcBindings.h +++ b/contrib/llvm/include/llvm-c/OrcBindings.h @@ -22,6 +22,7 @@ #ifndef LLVM_C_ORCBINDINGS_H #define LLVM_C_ORCBINDINGS_H +#include "llvm-c/Error.h" #include "llvm-c/Object.h" #include "llvm-c/TargetMachine.h" @@ -36,8 +37,6 @@ typedef uint64_t (*LLVMOrcSymbolResolverFn)(const char *Name, void *LookupCtx); typedef uint64_t (*LLVMOrcLazyCompileCallbackFn)(LLVMOrcJITStackRef JITStack, void *CallbackCtx); -typedef enum { LLVMOrcErrSuccess = 0, LLVMOrcErrGeneric } LLVMOrcErrorCode; - /** * Create an ORC JIT stack. * @@ -72,43 +71,41 @@ void LLVMOrcDisposeMangledSymbol(char *MangledSymbol); /** * Create a lazy compile callback. */ -LLVMOrcErrorCode -LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - LLVMOrcLazyCompileCallbackFn Callback, - void *CallbackCtx); +LLVMErrorRef LLVMOrcCreateLazyCompileCallback( + LLVMOrcJITStackRef JITStack, LLVMOrcTargetAddress *RetAddr, + LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx); /** * Create a named indirect call stub. */ -LLVMOrcErrorCode LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, - const char *StubName, - LLVMOrcTargetAddress InitAddr); +LLVMErrorRef LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress InitAddr); /** * Set the pointer for the given indirect stub. */ -LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, - const char *StubName, - LLVMOrcTargetAddress NewAddr); +LLVMErrorRef LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress NewAddr); /** * Add module to be eagerly compiled. */ -LLVMOrcErrorCode -LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx); +LLVMErrorRef LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); /** * Add module to be lazily compiled one function at a time. */ -LLVMOrcErrorCode -LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx); +LLVMErrorRef LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); /** * Add an object file. @@ -118,11 +115,11 @@ LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, * Clients should *not* dispose of the 'Obj' argument: the JIT will manage it * from this call onwards. */ -LLVMOrcErrorCode LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMMemoryBufferRef Obj, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx); +LLVMErrorRef LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMMemoryBufferRef Obj, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); /** * Remove a module set from the JIT. @@ -130,29 +127,29 @@ LLVMOrcErrorCode LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, * This works for all modules that can be added via OrcAdd*, including object * files. */ -LLVMOrcErrorCode LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle H); +LLVMErrorRef LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle H); /** * Get symbol address from JIT instance. */ -LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - const char *SymbolName); +LLVMErrorRef LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + const char *SymbolName); /** * Get symbol address from JIT instance, searching only the specified * handle. */ -LLVMOrcErrorCode LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - LLVMOrcModuleHandle H, - const char *SymbolName); +LLVMErrorRef LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + LLVMOrcModuleHandle H, + const char *SymbolName); /** * Dispose of an ORC JIT stack. */ -LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack); +LLVMErrorRef LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack); /** * Register a JIT Event Listener. diff --git a/contrib/llvm/include/llvm-c/TargetMachine.h b/contrib/llvm/include/llvm-c/TargetMachine.h index 7f672b5d10d..c06e9edc9aa 100644 --- a/contrib/llvm/include/llvm-c/TargetMachine.h +++ b/contrib/llvm/include/llvm-c/TargetMachine.h @@ -39,12 +39,16 @@ typedef enum { LLVMRelocDefault, LLVMRelocStatic, LLVMRelocPIC, - LLVMRelocDynamicNoPic + LLVMRelocDynamicNoPic, + LLVMRelocROPI, + LLVMRelocRWPI, + LLVMRelocROPI_RWPI } LLVMRelocMode; typedef enum { LLVMCodeModelDefault, LLVMCodeModelJITDefault, + LLVMCodeModelTiny, LLVMCodeModelSmall, LLVMCodeModelKernel, LLVMCodeModelMedium, diff --git a/contrib/llvm/include/llvm-c/Transforms/AggressiveInstCombine.h b/contrib/llvm/include/llvm-c/Transforms/AggressiveInstCombine.h new file mode 100644 index 00000000000..8756a22e917 --- /dev/null +++ b/contrib/llvm/include/llvm-c/Transforms/AggressiveInstCombine.h @@ -0,0 +1,43 @@ +/*===-- AggressiveInstCombine.h ---------------------------------*- C++ -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header declares the C interface to libLLVMAggressiveInstCombine.a, *| +|* which combines instructions to form fewer, simple IR instructions. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_TRANSFORMS_AGGRESSIVEINSTCOMBINE_H +#define LLVM_C_TRANSFORMS_AGGRESSIVEINSTCOMBINE_H + +#include "llvm-c/Types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @defgroup LLVMCTransformsAggressiveInstCombine Aggressive Instruction Combining transformations + * @ingroup LLVMCTransforms + * + * @{ + */ + +/** See llvm::createAggressiveInstCombinerPass function. */ +void LLVMAddAggressiveInstCombinerPass(LLVMPassManagerRef PM); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif /* defined(__cplusplus) */ + +#endif + diff --git a/contrib/llvm/include/llvm-c/Transforms/Coroutines.h b/contrib/llvm/include/llvm-c/Transforms/Coroutines.h new file mode 100644 index 00000000000..827e30fb2d7 --- /dev/null +++ b/contrib/llvm/include/llvm-c/Transforms/Coroutines.h @@ -0,0 +1,55 @@ +/*===-- Coroutines.h - Coroutines Library C Interface -----------*- C++ -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header declares the C interface to libLLVMCoroutines.a, which *| +|* implements various scalar transformations of the LLVM IR. *| +|* *| +|* Many exotic languages can interoperate with C code but have a harder time *| +|* with C++ due to name mangling. So in addition to C, this interface enables *| +|* tools written in such languages. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_TRANSFORMS_COROUTINES_H +#define LLVM_C_TRANSFORMS_COROUTINES_H + +#include "llvm-c/Types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @defgroup LLVMCTransformsCoroutines Coroutine transformations + * @ingroup LLVMCTransforms + * + * @{ + */ + +/** See llvm::createCoroEarlyPass function. */ +void LLVMAddCoroEarlyPass(LLVMPassManagerRef PM); + +/** See llvm::createCoroSplitPass function. */ +void LLVMAddCoroSplitPass(LLVMPassManagerRef PM); + +/** See llvm::createCoroElidePass function. */ +void LLVMAddCoroElidePass(LLVMPassManagerRef PM); + +/** See llvm::createCoroCleanupPass function. */ +void LLVMAddCoroCleanupPass(LLVMPassManagerRef PM); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif /* defined(__cplusplus) */ + +#endif diff --git a/contrib/llvm/include/llvm-c/Transforms/Scalar.h b/contrib/llvm/include/llvm-c/Transforms/Scalar.h index f55cdce86be..3c3bb4eb9b8 100644 --- a/contrib/llvm/include/llvm-c/Transforms/Scalar.h +++ b/contrib/llvm/include/llvm-c/Transforms/Scalar.h @@ -35,9 +35,6 @@ extern "C" { /** See llvm::createAggressiveDCEPass function. */ void LLVMAddAggressiveDCEPass(LLVMPassManagerRef PM); -/** See llvm::createAggressiveInstCombinerPass function. */ -void LLVMAddAggressiveInstCombinerPass(LLVMPassManagerRef PM); - /** See llvm::createBitTrackingDCEPass function. */ void LLVMAddBitTrackingDCEPass(LLVMPassManagerRef PM); @@ -95,6 +92,9 @@ void LLVMAddLoopUnrollAndJamPass(LLVMPassManagerRef PM); /** See llvm::createLoopUnswitchPass function. */ void LLVMAddLoopUnswitchPass(LLVMPassManagerRef PM); +/** See llvm::createLowerAtomicPass function. */ +void LLVMAddLowerAtomicPass(LLVMPassManagerRef PM); + /** See llvm::createMemCpyOptPass function. */ void LLVMAddMemCpyOptPass(LLVMPassManagerRef PM); @@ -153,6 +153,9 @@ void LLVMAddScopedNoAliasAAPass(LLVMPassManagerRef PM); /** See llvm::createBasicAliasAnalysisPass function */ void LLVMAddBasicAliasAnalysisPass(LLVMPassManagerRef PM); +/** See llvm::createUnifyFunctionExitNodesPass function */ +void LLVMAddUnifyFunctionExitNodesPass(LLVMPassManagerRef PM); + /** * @} */ diff --git a/contrib/llvm/include/llvm-c/Types.h b/contrib/llvm/include/llvm-c/Types.h index 4a33542e86c..ce1acf3e042 100644 --- a/contrib/llvm/include/llvm-c/Types.h +++ b/contrib/llvm/include/llvm-c/Types.h @@ -89,6 +89,20 @@ typedef struct LLVMOpaqueBasicBlock *LLVMBasicBlockRef; */ typedef struct LLVMOpaqueMetadata *LLVMMetadataRef; +/** + * Represents an LLVM Named Metadata Node. + * + * This models llvm::NamedMDNode. + */ +typedef struct LLVMOpaqueNamedMDNode *LLVMNamedMDNodeRef; + +/** + * Represents an entry in a Global Object's metadata attachments. + * + * This models std::pair + */ +typedef struct LLVMOpaqueValueMetadataEntry LLVMValueMetadataEntry; + /** * Represents an LLVM basic block builder. * diff --git a/contrib/llvm/include/llvm-c/lto.h b/contrib/llvm/include/llvm-c/lto.h index 1acd610f70a..090cd34af4e 100644 --- a/contrib/llvm/include/llvm-c/lto.h +++ b/contrib/llvm/include/llvm-c/lto.h @@ -44,7 +44,7 @@ typedef bool lto_bool_t; * @{ */ -#define LTO_API_VERSION 22 +#define LTO_API_VERSION 23 /** * \since prior to LTO_API_VERSION=3 @@ -827,6 +827,16 @@ extern void thinlto_codegen_set_cache_entry_expiration(thinlto_code_gen_t cg, extern void thinlto_codegen_set_cache_size_bytes(thinlto_code_gen_t cg, unsigned max_size_bytes); +/** + * Same as thinlto_codegen_set_cache_size_bytes, except the maximum size is in + * megabytes (2^20 bytes). + * + * \since LTO_API_VERSION=23 + */ +extern void +thinlto_codegen_set_cache_size_megabytes(thinlto_code_gen_t cg, + unsigned max_size_megabytes); + /** * Sets the maximum number of files in the cache directory. An unspecified * default value will be applied. A value of 0 will be ignored. diff --git a/contrib/llvm/include/llvm/ADT/APFloat.h b/contrib/llvm/include/llvm/ADT/APFloat.h index 5c59af4c04b..c6fa5ad674f 100644 --- a/contrib/llvm/include/llvm/ADT/APFloat.h +++ b/contrib/llvm/include/llvm/ADT/APFloat.h @@ -870,13 +870,13 @@ public: /// Factory for NaN values. /// /// \param Negative - True iff the NaN generated should be negative. - /// \param type - The unspecified fill bits for creating the NaN, 0 by + /// \param payload - The unspecified fill bits for creating the NaN, 0 by /// default. The value is truncated as necessary. static APFloat getNaN(const fltSemantics &Sem, bool Negative = false, - unsigned type = 0) { - if (type) { - APInt fill(64, type); - return getQNaN(Sem, Negative, &fill); + uint64_t payload = 0) { + if (payload) { + APInt intPayload(64, payload); + return getQNaN(Sem, Negative, &intPayload); } else { return getQNaN(Sem, Negative, nullptr); } @@ -1243,6 +1243,32 @@ inline APFloat maxnum(const APFloat &A, const APFloat &B) { return (A.compare(B) == APFloat::cmpLessThan) ? B : A; } +/// Implements IEEE 754-2018 minimum semantics. Returns the smaller of 2 +/// arguments, propagating NaNs and treating -0 as less than +0. +LLVM_READONLY +inline APFloat minimum(const APFloat &A, const APFloat &B) { + if (A.isNaN()) + return A; + if (B.isNaN()) + return B; + if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative())) + return A.isNegative() ? A : B; + return (B.compare(A) == APFloat::cmpLessThan) ? B : A; +} + +/// Implements IEEE 754-2018 maximum semantics. Returns the larger of 2 +/// arguments, propagating NaNs and treating -0 as less than +0. +LLVM_READONLY +inline APFloat maximum(const APFloat &A, const APFloat &B) { + if (A.isNaN()) + return A; + if (B.isNaN()) + return B; + if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative())) + return A.isNegative() ? B : A; + return (A.compare(B) == APFloat::cmpLessThan) ? B : A; +} + } // namespace llvm #undef APFLOAT_DISPATCH_ON_SEMANTICS diff --git a/contrib/llvm/include/llvm/ADT/APInt.h b/contrib/llvm/include/llvm/ADT/APInt.h index 6bf6b22fb01..6e106ff8bf5 100644 --- a/contrib/llvm/include/llvm/ADT/APInt.h +++ b/contrib/llvm/include/llvm/ADT/APInt.h @@ -31,6 +31,7 @@ class raw_ostream; template class SmallVectorImpl; template class ArrayRef; +template class Optional; class APInt; @@ -84,7 +85,7 @@ public: UP, }; - static const WordType WORD_MAX = ~WordType(0); + static const WordType WORDTYPE_MAX = ~WordType(0); private: /// This union is used to store the integer value. When the @@ -149,7 +150,7 @@ private: unsigned WordBits = ((BitWidth-1) % APINT_BITS_PER_WORD) + 1; // Mask out the high bits. - uint64_t mask = WORD_MAX >> (APINT_BITS_PER_WORD - WordBits); + uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - WordBits); if (isSingleWord()) U.VAL &= mask; else @@ -394,7 +395,7 @@ public: /// This checks to see if the value has all bits of the APInt are set or not. bool isAllOnesValue() const { if (isSingleWord()) - return U.VAL == WORD_MAX >> (APINT_BITS_PER_WORD - BitWidth); + return U.VAL == WORDTYPE_MAX >> (APINT_BITS_PER_WORD - BitWidth); return countTrailingOnesSlowCase() == BitWidth; } @@ -495,7 +496,7 @@ public: assert(numBits != 0 && "numBits must be non-zero"); assert(numBits <= BitWidth && "numBits out of range"); if (isSingleWord()) - return U.VAL == (WORD_MAX >> (APINT_BITS_PER_WORD - numBits)); + return U.VAL == (WORDTYPE_MAX >> (APINT_BITS_PER_WORD - numBits)); unsigned Ones = countTrailingOnesSlowCase(); return (numBits == Ones) && ((Ones + countLeadingZerosSlowCase()) == BitWidth); @@ -559,7 +560,7 @@ public: /// /// \returns the all-ones value for an APInt of the specified bit-width. static APInt getAllOnesValue(unsigned numBits) { - return APInt(numBits, WORD_MAX, true); + return APInt(numBits, WORDTYPE_MAX, true); } /// Get the '0' value. @@ -1104,6 +1105,12 @@ public: APInt sshl_ov(const APInt &Amt, bool &Overflow) const; APInt ushl_ov(const APInt &Amt, bool &Overflow) const; + // Operations that saturate + APInt sadd_sat(const APInt &RHS) const; + APInt uadd_sat(const APInt &RHS) const; + APInt ssub_sat(const APInt &RHS) const; + APInt usub_sat(const APInt &RHS) const; + /// Array-indexing support. /// /// \returns the bit value at bitPosition @@ -1382,7 +1389,7 @@ public: /// Set every bit to 1. void setAllBits() { if (isSingleWord()) - U.VAL = WORD_MAX; + U.VAL = WORDTYPE_MAX; else // Set all the bits in all the words. memset(U.pVal, -1, getNumWords() * APINT_WORD_SIZE); @@ -1394,7 +1401,7 @@ public: /// /// Set the given bit to 1 whose position is given as "bitPosition". void setBit(unsigned BitPosition) { - assert(BitPosition <= BitWidth && "BitPosition out of range"); + assert(BitPosition < BitWidth && "BitPosition out of range"); WordType Mask = maskBit(BitPosition); if (isSingleWord()) U.VAL |= Mask; @@ -1415,7 +1422,7 @@ public: if (loBit == hiBit) return; if (loBit < APINT_BITS_PER_WORD && hiBit <= APINT_BITS_PER_WORD) { - uint64_t mask = WORD_MAX >> (APINT_BITS_PER_WORD - (hiBit - loBit)); + uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - (hiBit - loBit)); mask <<= loBit; if (isSingleWord()) U.VAL |= mask; @@ -1453,7 +1460,7 @@ public: /// /// Set the given bit to 0 whose position is given as "bitPosition". void clearBit(unsigned BitPosition) { - assert(BitPosition <= BitWidth && "BitPosition out of range"); + assert(BitPosition < BitWidth && "BitPosition out of range"); WordType Mask = ~maskBit(BitPosition); if (isSingleWord()) U.VAL &= Mask; @@ -1469,7 +1476,7 @@ public: /// Toggle every bit to its opposite value. void flipAllBits() { if (isSingleWord()) { - U.VAL ^= WORD_MAX; + U.VAL ^= WORDTYPE_MAX; clearUnusedBits(); } else { flipAllBitsSlowCase(); @@ -1758,7 +1765,7 @@ public: /// referencing 2 in a space where 2 does no exist. unsigned nearestLogBase2() const { // Special case when we have a bitwidth of 1. If VAL is 1, then we - // get 0. If VAL is 0, we get WORD_MAX which gets truncated to + // get 0. If VAL is 0, we get WORDTYPE_MAX which gets truncated to // UINT32_MAX. if (BitWidth == 1) return U.VAL - 1; @@ -2166,6 +2173,41 @@ APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM); /// Return A sign-divided by B, rounded by the given rounding mode. APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM); +/// Let q(n) = An^2 + Bn + C, and BW = bit width of the value range +/// (e.g. 32 for i32). +/// This function finds the smallest number n, such that +/// (a) n >= 0 and q(n) = 0, or +/// (b) n >= 1 and q(n-1) and q(n), when evaluated in the set of all +/// integers, belong to two different intervals [Rk, Rk+R), +/// where R = 2^BW, and k is an integer. +/// The idea here is to find when q(n) "overflows" 2^BW, while at the +/// same time "allowing" subtraction. In unsigned modulo arithmetic a +/// subtraction (treated as addition of negated numbers) would always +/// count as an overflow, but here we want to allow values to decrease +/// and increase as long as they are within the same interval. +/// Specifically, adding of two negative numbers should not cause an +/// overflow (as long as the magnitude does not exceed the bith width). +/// On the other hand, given a positive number, adding a negative +/// number to it can give a negative result, which would cause the +/// value to go from [-2^BW, 0) to [0, 2^BW). In that sense, zero is +/// treated as a special case of an overflow. +/// +/// This function returns None if after finding k that minimizes the +/// positive solution to q(n) = kR, both solutions are contained between +/// two consecutive integers. +/// +/// There are cases where q(n) > T, and q(n+1) < T (assuming evaluation +/// in arithmetic modulo 2^BW, and treating the values as signed) by the +/// virtue of *signed* overflow. This function will *not* find such an n, +/// however it may find a value of n satisfying the inequalities due to +/// an *unsigned* overflow (if the values are treated as unsigned). +/// To find a solution for a signed overflow, treat it as a problem of +/// finding an unsigned overflow with a range with of BW-1. +/// +/// The returned value may have a different bit width from the input +/// coefficients. +Optional SolveQuadraticEquationWrap(APInt A, APInt B, APInt C, + unsigned RangeWidth); } // End of APIntOps namespace // See friend declaration above. This additional declaration is required in diff --git a/contrib/llvm/include/llvm/ADT/Any.h b/contrib/llvm/include/llvm/ADT/Any.h index c64c3998754..7faa4c963d3 100644 --- a/contrib/llvm/include/llvm/ADT/Any.h +++ b/contrib/llvm/include/llvm/ADT/Any.h @@ -65,6 +65,16 @@ public: typename std::enable_if< llvm::conjunction< llvm::negation::type, Any>>, + // We also disable this overload when an `Any` object can be + // converted to the parameter type because in that case, this + // constructor may combine with that conversion during overload + // resolution for determining copy constructibility, and then + // when we try to determine copy constructibility below we may + // infinitely recurse. This is being evaluated by the standards + // committee as a potential DR in `std::any` as well, but we're + // going ahead and adopting it to work-around usage of `Any` with + // types that need to be implicitly convertible from an `Any`. + llvm::negation::type>>, std::is_copy_constructible::type>>::value, int>::type = 0> Any(T &&Value) { diff --git a/contrib/llvm/include/llvm/ADT/BitVector.h b/contrib/llvm/include/llvm/ADT/BitVector.h index 438c7d84c58..9ab1da7c691 100644 --- a/contrib/llvm/include/llvm/ADT/BitVector.h +++ b/contrib/llvm/include/llvm/ADT/BitVector.h @@ -503,6 +503,23 @@ public: return (*this)[Idx]; } + // Push single bit to end of vector. + void push_back(bool Val) { + unsigned OldSize = Size; + unsigned NewSize = Size + 1; + + // Resize, which will insert zeros. + // If we already fit then the unused bits will be already zero. + if (NewSize > getBitCapacity()) + resize(NewSize, false); + else + Size = NewSize; + + // If true, set single bit. + if (Val) + set(OldSize); + } + /// Test if any common bits are set. bool anyCommon(const BitVector &RHS) const { unsigned ThisWords = NumBitWords(size()); diff --git a/contrib/llvm/include/llvm/ADT/DenseMap.h b/contrib/llvm/include/llvm/ADT/DenseMap.h index ba60b7972a8..1f50502fff9 100644 --- a/contrib/llvm/include/llvm/ADT/DenseMap.h +++ b/contrib/llvm/include/llvm/ADT/DenseMap.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,34 @@ namespace detail { // implementation without requiring two members. template struct DenseMapPair : public std::pair { + + // FIXME: Switch to inheriting constructors when we drop support for older + // clang versions. + // NOTE: This default constructor is declared with '{}' rather than + // '= default' to work around a separate bug in clang-3.8. This can + // also go when we switch to inheriting constructors. + DenseMapPair() {} + + DenseMapPair(const KeyT &Key, const ValueT &Value) + : std::pair(Key, Value) {} + + DenseMapPair(KeyT &&Key, ValueT &&Value) + : std::pair(std::move(Key), std::move(Value)) {} + + template + DenseMapPair(AltKeyT &&AltKey, AltValueT &&AltValue, + typename std::enable_if< + std::is_convertible::value && + std::is_convertible::value>::type * = 0) + : std::pair(std::forward(AltKey), + std::forward(AltValue)) {} + + template + DenseMapPair(AltPairT &&AltPair, + typename std::enable_if>::value>::type * = 0) + : std::pair(std::forward(AltPair)) {} + KeyT &getFirst() { return std::pair::first; } const KeyT &getFirst() const { return std::pair::first; } ValueT &getSecond() { return std::pair::second; } @@ -46,9 +75,10 @@ struct DenseMapPair : public std::pair { } // end namespace detail -template < - typename KeyT, typename ValueT, typename KeyInfoT = DenseMapInfo, - typename Bucket = detail::DenseMapPair, bool IsConst = false> +template , + typename Bucket = llvm::detail::DenseMapPair, + bool IsConst = false> class DenseMapIterator; template ::value && isPodLike::value) - memcpy(getBuckets(), other.getBuckets(), + memcpy(reinterpret_cast(getBuckets()), other.getBuckets(), getNumBuckets() * sizeof(BucketT)); else for (size_t i = 0; i < getNumBuckets(); ++i) { @@ -639,9 +669,43 @@ public: } }; +/// Equality comparison for DenseMap. +/// +/// Iterates over elements of LHS confirming that each (key, value) pair in LHS +/// is also in RHS, and that no additional pairs are in RHS. +/// Equivalent to N calls to RHS.find and N value comparisons. Amortized +/// complexity is linear, worst case is O(N^2) (if every hash collides). +template +bool operator==( + const DenseMapBase &LHS, + const DenseMapBase &RHS) { + if (LHS.size() != RHS.size()) + return false; + + for (auto &KV : LHS) { + auto I = RHS.find(KV.first); + if (I == RHS.end() || I->second != KV.second) + return false; + } + + return true; +} + +/// Inequality comparison for DenseMap. +/// +/// Equivalent to !(LHS == RHS). See operator== for performance notes. +template +bool operator!=( + const DenseMapBase &LHS, + const DenseMapBase &RHS) { + return !(LHS == RHS); +} + template , - typename BucketT = detail::DenseMapPair> + typename BucketT = llvm::detail::DenseMapPair> class DenseMap : public DenseMapBase, KeyT, ValueT, KeyInfoT, BucketT> { friend class DenseMapBase; @@ -676,6 +740,11 @@ public: this->insert(I, E); } + DenseMap(std::initializer_list Vals) { + init(Vals.size()); + this->insert(Vals.begin(), Vals.end()); + } + ~DenseMap() { this->destroyAll(); operator delete(Buckets); @@ -798,7 +867,7 @@ private: template , - typename BucketT = detail::DenseMapPair> + typename BucketT = llvm::detail::DenseMapPair> class SmallDenseMap : public DenseMapBase< SmallDenseMap, KeyT, diff --git a/contrib/llvm/include/llvm/ADT/DenseSet.h b/contrib/llvm/include/llvm/ADT/DenseSet.h index b495e25dd5e..e85a38587e4 100644 --- a/contrib/llvm/include/llvm/ADT/DenseSet.h +++ b/contrib/llvm/include/llvm/ADT/DenseSet.h @@ -16,6 +16,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/type_traits.h" #include #include @@ -67,7 +68,7 @@ public: explicit DenseSetImpl(unsigned InitialReserve = 0) : TheMap(InitialReserve) {} DenseSetImpl(std::initializer_list Elems) - : DenseSetImpl(Elems.size()) { + : DenseSetImpl(PowerOf2Ceil(Elems.size())) { insert(Elems.begin(), Elems.end()); } @@ -136,8 +137,8 @@ public: public: using difference_type = typename MapTy::const_iterator::difference_type; using value_type = ValueT; - using pointer = value_type *; - using reference = value_type &; + using pointer = const value_type *; + using reference = const value_type &; using iterator_category = std::forward_iterator_tag; ConstIterator() = default; @@ -214,6 +215,34 @@ public: } }; +/// Equality comparison for DenseSet. +/// +/// Iterates over elements of LHS confirming that each element is also a member +/// of RHS, and that RHS contains no additional values. +/// Equivalent to N calls to RHS.count. Amortized complexity is linear, worst +/// case is O(N^2) (if every hash collides). +template +bool operator==(const DenseSetImpl &LHS, + const DenseSetImpl &RHS) { + if (LHS.size() != RHS.size()) + return false; + + for (auto &E : LHS) + if (!RHS.count(E)) + return false; + + return true; +} + +/// Inequality comparison for DenseSet. +/// +/// Equivalent to !(LHS == RHS). See operator== for performance notes. +template +bool operator!=(const DenseSetImpl &LHS, + const DenseSetImpl &RHS) { + return !(LHS == RHS); +} + } // end namespace detail /// Implements a dense probed hash-table based set. diff --git a/contrib/llvm/include/llvm/ADT/GraphTraits.h b/contrib/llvm/include/llvm/ADT/GraphTraits.h index 27c647f4bbb..d39b50fdc48 100644 --- a/contrib/llvm/include/llvm/ADT/GraphTraits.h +++ b/contrib/llvm/include/llvm/ADT/GraphTraits.h @@ -25,6 +25,13 @@ namespace llvm { // GraphTraits - This class should be specialized by different graph types... // which is why the default version is empty. // +// This template evolved from supporting `BasicBlock` to also later supporting +// more complex types (e.g. CFG and DomTree). +// +// GraphTraits can be used to create a view over a graph interpreting it +// differently without requiring a copy of the original graph. This could +// be achieved by carrying more data in NodeRef. See LoopBodyTraits for one +// example. template struct GraphTraits { // Elements to provide: diff --git a/contrib/llvm/include/llvm/ADT/Hashing.h b/contrib/llvm/include/llvm/ADT/Hashing.h index 9f830baa424..9175c545b7c 100644 --- a/contrib/llvm/include/llvm/ADT/Hashing.h +++ b/contrib/llvm/include/llvm/ADT/Hashing.h @@ -133,7 +133,7 @@ hash_code hash_value(const std::basic_string &arg); /// undone. This makes it thread-hostile and very hard to use outside of /// immediately on start of a simple program designed for reproducible /// behavior. -void set_fixed_execution_hash_seed(size_t fixed_value); +void set_fixed_execution_hash_seed(uint64_t fixed_value); // All of the implementation details of actually computing the various hash @@ -316,9 +316,9 @@ struct hash_state { /// This variable can be set using the \see llvm::set_fixed_execution_seed /// function. See that function for details. Do not, under any circumstances, /// set or read this variable. -extern size_t fixed_seed_override; +extern uint64_t fixed_seed_override; -inline size_t get_execution_seed() { +inline uint64_t get_execution_seed() { // FIXME: This needs to be a per-execution seed. This is just a placeholder // implementation. Switching to a per-execution seed is likely to flush out // instability bugs and so will happen as its own commit. @@ -326,8 +326,7 @@ inline size_t get_execution_seed() { // However, if there is a fixed seed override set the first time this is // called, return that instead of the per-execution seed. const uint64_t seed_prime = 0xff51afd7ed558ccdULL; - static size_t seed = fixed_seed_override ? fixed_seed_override - : (size_t)seed_prime; + static uint64_t seed = fixed_seed_override ? fixed_seed_override : seed_prime; return seed; } @@ -402,7 +401,7 @@ bool store_and_advance(char *&buffer_ptr, char *buffer_end, const T& value, /// combining them, this (as an optimization) directly combines the integers. template hash_code hash_combine_range_impl(InputIteratorT first, InputIteratorT last) { - const size_t seed = get_execution_seed(); + const uint64_t seed = get_execution_seed(); char buffer[64], *buffer_ptr = buffer; char *const buffer_end = std::end(buffer); while (first != last && store_and_advance(buffer_ptr, buffer_end, @@ -446,7 +445,7 @@ hash_code hash_combine_range_impl(InputIteratorT first, InputIteratorT last) { template typename std::enable_if::value, hash_code>::type hash_combine_range_impl(ValueT *first, ValueT *last) { - const size_t seed = get_execution_seed(); + const uint64_t seed = get_execution_seed(); const char *s_begin = reinterpret_cast(first); const char *s_end = reinterpret_cast(last); const size_t length = std::distance(s_begin, s_end); @@ -496,7 +495,7 @@ namespace detail { struct hash_combine_recursive_helper { char buffer[64]; hash_state state; - const size_t seed; + const uint64_t seed; public: /// Construct a recursive hash combining helper. diff --git a/contrib/llvm/include/llvm/ADT/ImmutableList.h b/contrib/llvm/include/llvm/ADT/ImmutableList.h index 1f5e9813798..0541dc2566e 100644 --- a/contrib/llvm/include/llvm/ADT/ImmutableList.h +++ b/contrib/llvm/include/llvm/ADT/ImmutableList.h @@ -31,8 +31,9 @@ class ImmutableListImpl : public FoldingSetNode { T Head; const ImmutableListImpl* Tail; - ImmutableListImpl(const T& head, const ImmutableListImpl* tail = nullptr) - : Head(head), Tail(tail) {} + template + ImmutableListImpl(ElemT &&head, const ImmutableListImpl *tail = nullptr) + : Head(std::forward(head)), Tail(tail) {} public: ImmutableListImpl(const ImmutableListImpl &) = delete; @@ -66,6 +67,9 @@ public: using value_type = T; using Factory = ImmutableListFactory; + static_assert(std::is_trivially_destructible::value, + "T must be trivially destructible!"); + private: const ImmutableListImpl* X; @@ -90,6 +94,9 @@ public: bool operator==(const iterator& I) const { return L == I.L; } bool operator!=(const iterator& I) const { return L != I.L; } const value_type& operator*() const { return L->getHead(); } + const typename std::remove_reference::type* operator->() const { + return &L->getHead(); + } ImmutableList getList() const { return L; } }; @@ -123,14 +130,14 @@ public: bool operator==(const ImmutableList& L) const { return isEqual(L); } /// getHead - Returns the head of the list. - const T& getHead() { + const T& getHead() const { assert(!isEmpty() && "Cannot get the head of an empty list."); return X->getHead(); } /// getTail - Returns the tail of the list, which is another (possibly empty) /// ImmutableList. - ImmutableList getTail() { + ImmutableList getTail() const { return X ? X->getTail() : nullptr; } @@ -166,7 +173,8 @@ public: if (ownsAllocator()) delete &getAllocator(); } - LLVM_NODISCARD ImmutableList concat(const T &Head, ImmutableList Tail) { + template + LLVM_NODISCARD ImmutableList concat(ElemT &&Head, ImmutableList Tail) { // Profile the new list to see if it already exists in our cache. FoldingSetNodeID ID; void* InsertPos; @@ -179,7 +187,7 @@ public: // The list does not exist in our cache. Create it. BumpPtrAllocator& A = getAllocator(); L = (ListTy*) A.Allocate(); - new (L) ListTy(Head, TailImpl); + new (L) ListTy(std::forward(Head), TailImpl); // Insert the new list into the cache. Cache.InsertNode(L, InsertPos); @@ -188,16 +196,24 @@ public: return L; } - LLVM_NODISCARD ImmutableList add(const T& D, ImmutableList L) { - return concat(D, L); + template + LLVM_NODISCARD ImmutableList add(ElemT &&Data, ImmutableList L) { + return concat(std::forward(Data), L); + } + + template + LLVM_NODISCARD ImmutableList emplace(ImmutableList Tail, + CtorArgs &&...Args) { + return concat(T(std::forward(Args)...), Tail); } ImmutableList getEmptyList() const { return ImmutableList(nullptr); } - ImmutableList create(const T& X) { - return Concat(X, getEmptyList()); + template + ImmutableList create(ElemT &&Data) { + return concat(std::forward(Data), getEmptyList()); } }; diff --git a/contrib/llvm/include/llvm/ADT/IntervalMap.h b/contrib/llvm/include/llvm/ADT/IntervalMap.h index f7136681121..2af61049e5a 100644 --- a/contrib/llvm/include/llvm/ADT/IntervalMap.h +++ b/contrib/llvm/include/llvm/ADT/IntervalMap.h @@ -101,6 +101,7 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/bit.h" #include "llvm/Support/AlignOf.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/RecyclingAllocator.h" @@ -963,6 +964,7 @@ public: private: // The root data is either a RootLeaf or a RootBranchData instance. + LLVM_ALIGNAS(RootLeaf) LLVM_ALIGNAS(RootBranchData) AlignedCharArrayUnion data; // Tree height. @@ -977,15 +979,10 @@ private: // Allocator used for creating external nodes. Allocator &allocator; - /// dataAs - Represent data as a node type without breaking aliasing rules. + /// Represent data as a node type without breaking aliasing rules. template T &dataAs() const { - union { - const char *d; - T *t; - } u; - u.d = data.buffer; - return *u.t; + return *bit_cast(const_cast(data.buffer)); } const RootLeaf &rootLeaf() const { @@ -1137,6 +1134,19 @@ public: I.find(x); return I; } + + /// overlaps(a, b) - Return true if the intervals in this map overlap with the + /// interval [a;b]. + bool overlaps(KeyT a, KeyT b) { + assert(Traits::nonEmpty(a, b)); + const_iterator I = find(a); + if (!I.valid()) + return false; + // [a;b] and [x;y] overlap iff x<=b and a<=y. The find() call guarantees the + // second part (y = find(a).stop()), so it is sufficient to check the first + // one. + return !Traits::stopLess(b, I.start()); + } }; /// treeSafeLookup - Return the mapped value at x or NotFound, assuming a diff --git a/contrib/llvm/include/llvm/ADT/Optional.h b/contrib/llvm/include/llvm/ADT/Optional.h index 353e5d0ec9d..76937d632ae 100644 --- a/contrib/llvm/include/llvm/ADT/Optional.h +++ b/contrib/llvm/include/llvm/ADT/Optional.h @@ -29,7 +29,7 @@ namespace llvm { namespace optional_detail { /// Storage for any type. -template struct OptionalStorage { +template ::value> struct OptionalStorage { AlignedCharArrayUnion storage; bool hasVal = false; @@ -108,28 +108,10 @@ template struct OptionalStorage { } }; -#if !defined(__GNUC__) || defined(__clang__) // GCC up to GCC7 miscompiles this. -/// Storage for trivially copyable types only. -template struct OptionalStorage { - AlignedCharArrayUnion storage; - bool hasVal = false; - - OptionalStorage() = default; - - OptionalStorage(const T &y) : hasVal(true) { new (storage.buffer) T(y); } - OptionalStorage &operator=(const T &y) { - *reinterpret_cast(storage.buffer) = y; - hasVal = true; - return *this; - } - - void reset() { hasVal = false; } -}; -#endif } // namespace optional_detail template class Optional { - optional_detail::OptionalStorage::value> Storage; + optional_detail::OptionalStorage Storage; public: using value_type = T; diff --git a/contrib/llvm/include/llvm/ADT/PointerIntPair.h b/contrib/llvm/include/llvm/ADT/PointerIntPair.h index 884d05155bf..6d1b53a90ad 100644 --- a/contrib/llvm/include/llvm/ADT/PointerIntPair.h +++ b/contrib/llvm/include/llvm/ADT/PointerIntPair.h @@ -42,6 +42,8 @@ template , typename Info = PointerIntPairInfo> class PointerIntPair { + // Used by MSVC visualizer and generally helpful for debugging/visualizing. + using InfoTy = Info; intptr_t Value = 0; public: diff --git a/contrib/llvm/include/llvm/ADT/PointerSumType.h b/contrib/llvm/include/llvm/ADT/PointerSumType.h index e37957160d9..a19e45a4621 100644 --- a/contrib/llvm/include/llvm/ADT/PointerSumType.h +++ b/contrib/llvm/include/llvm/ADT/PointerSumType.h @@ -10,6 +10,7 @@ #ifndef LLVM_ADT_POINTERSUMTYPE_H #define LLVM_ADT_POINTERSUMTYPE_H +#include "llvm/ADT/bit.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include @@ -58,56 +59,142 @@ template struct PointerSumTypeHelper; /// and may be desirable to set to a state that is particularly desirable to /// default construct. /// +/// Having a supported zero-valued tag also enables getting the address of a +/// pointer stored with that tag provided it is stored in its natural bit +/// representation. This works because in the case of a zero-valued tag, the +/// pointer's value is directly stored into this object and we can expose the +/// address of that internal storage. This is especially useful when building an +/// `ArrayRef` of a single pointer stored in a sum type. +/// /// There is no support for constructing or accessing with a dynamic tag as /// that would fundamentally violate the type safety provided by the sum type. template class PointerSumType { - uintptr_t Value = 0; - using HelperT = detail::PointerSumTypeHelper; + // We keep both the raw value and the min tag value's pointer in a union. When + // the minimum tag value is zero, this allows code below to cleanly expose the + // address of the zero-tag pointer instead of just the zero-tag pointer + // itself. This is especially useful when building `ArrayRef`s out of a single + // pointer. However, we have to carefully access the union due to the active + // member potentially changing. When we *store* a new value, we directly + // access the union to allow us to store using the obvious types. However, + // when we *read* a value, we copy the underlying storage out to avoid relying + // on one member or the other being active. + union StorageT { + // Ensure we get a null default constructed value. We don't use a member + // initializer because some compilers seem to not implement those correctly + // for a union. + StorageT() : Value(0) {} + + uintptr_t Value; + + typename HelperT::template Lookup::PointerT MinTagPointer; + }; + + StorageT Storage; + public: constexpr PointerSumType() = default; + /// A typed setter to a given tagged member of the sum type. + template + void set(typename HelperT::template Lookup::PointerT Pointer) { + void *V = HelperT::template Lookup::TraitsT::getAsVoidPointer(Pointer); + assert((reinterpret_cast(V) & HelperT::TagMask) == 0 && + "Pointer is insufficiently aligned to store the discriminant!"); + Storage.Value = reinterpret_cast(V) | N; + } + /// A typed constructor for a specific tagged member of the sum type. template static PointerSumType create(typename HelperT::template Lookup::PointerT Pointer) { PointerSumType Result; - void *V = HelperT::template Lookup::TraitsT::getAsVoidPointer(Pointer); - assert((reinterpret_cast(V) & HelperT::TagMask) == 0 && - "Pointer is insufficiently aligned to store the discriminant!"); - Result.Value = reinterpret_cast(V) | N; + Result.set(Pointer); return Result; } - TagT getTag() const { return static_cast(Value & HelperT::TagMask); } + /// Clear the value to null with the min tag type. + void clear() { set(nullptr); } + + TagT getTag() const { + return static_cast(getOpaqueValue() & HelperT::TagMask); + } template bool is() const { return N == getTag(); } template typename HelperT::template Lookup::PointerT get() const { - void *P = is() ? getImpl() : nullptr; + void *P = is() ? getVoidPtr() : nullptr; return HelperT::template Lookup::TraitsT::getFromVoidPointer(P); } template typename HelperT::template Lookup::PointerT cast() const { assert(is() && "This instance has a different active member."); - return HelperT::template Lookup::TraitsT::getFromVoidPointer(getImpl()); + return HelperT::template Lookup::TraitsT::getFromVoidPointer( + getVoidPtr()); } - explicit operator bool() const { return Value & HelperT::PointerMask; } - bool operator==(const PointerSumType &R) const { return Value == R.Value; } - bool operator!=(const PointerSumType &R) const { return Value != R.Value; } - bool operator<(const PointerSumType &R) const { return Value < R.Value; } - bool operator>(const PointerSumType &R) const { return Value > R.Value; } - bool operator<=(const PointerSumType &R) const { return Value <= R.Value; } - bool operator>=(const PointerSumType &R) const { return Value >= R.Value; } + /// If the tag is zero and the pointer's value isn't changed when being + /// stored, get the address of the stored value type-punned to the zero-tag's + /// pointer type. + typename HelperT::template Lookup::PointerT const * + getAddrOfZeroTagPointer() const { + return const_cast(this)->getAddrOfZeroTagPointer(); + } - uintptr_t getOpaqueValue() const { return Value; } + /// If the tag is zero and the pointer's value isn't changed when being + /// stored, get the address of the stored value type-punned to the zero-tag's + /// pointer type. + typename HelperT::template Lookup::PointerT * + getAddrOfZeroTagPointer() { + static_assert(HelperT::MinTag == 0, "Non-zero minimum tag value!"); + assert(is() && "The active tag is not zero!"); + // Store the initial value of the pointer when read out of our storage. + auto InitialPtr = get(); + // Now update the active member of the union to be the actual pointer-typed + // member so that accessing it indirectly through the returned address is + // valid. + Storage.MinTagPointer = InitialPtr; + // Finally, validate that this was a no-op as expected by reading it back + // out using the same underlying-storage read as above. + assert(InitialPtr == get() && + "Switching to typed storage changed the pointer returned!"); + // Now we can correctly return an address to typed storage. + return &Storage.MinTagPointer; + } + + explicit operator bool() const { + return getOpaqueValue() & HelperT::PointerMask; + } + bool operator==(const PointerSumType &R) const { + return getOpaqueValue() == R.getOpaqueValue(); + } + bool operator!=(const PointerSumType &R) const { + return getOpaqueValue() != R.getOpaqueValue(); + } + bool operator<(const PointerSumType &R) const { + return getOpaqueValue() < R.getOpaqueValue(); + } + bool operator>(const PointerSumType &R) const { + return getOpaqueValue() > R.getOpaqueValue(); + } + bool operator<=(const PointerSumType &R) const { + return getOpaqueValue() <= R.getOpaqueValue(); + } + bool operator>=(const PointerSumType &R) const { + return getOpaqueValue() >= R.getOpaqueValue(); + } + + uintptr_t getOpaqueValue() const { + // Read the underlying storage of the union, regardless of the active + // member. + return bit_cast(Storage); + } protected: - void *getImpl() const { - return reinterpret_cast(Value & HelperT::PointerMask); + void *getVoidPtr() const { + return reinterpret_cast(getOpaqueValue() & HelperT::PointerMask); } }; @@ -151,8 +238,9 @@ struct PointerSumTypeHelper : MemberTs... { enum { NumTagBits = Min::value }; // Also compute the smallest discriminant and various masks for convenience. + constexpr static TagT MinTag = + static_cast(Min::value); enum : uint64_t { - MinTag = Min::value, PointerMask = static_cast(-1) << NumTagBits, TagMask = ~PointerMask }; diff --git a/contrib/llvm/include/llvm/ADT/PostOrderIterator.h b/contrib/llvm/include/llvm/ADT/PostOrderIterator.h index dc8a9b6e78b..d77b12228cb 100644 --- a/contrib/llvm/include/llvm/ADT/PostOrderIterator.h +++ b/contrib/llvm/include/llvm/ADT/PostOrderIterator.h @@ -296,12 +296,15 @@ class ReversePostOrderTraversal { public: using rpo_iterator = typename std::vector::reverse_iterator; + using const_rpo_iterator = typename std::vector::const_reverse_iterator; ReversePostOrderTraversal(GraphT G) { Initialize(GT::getEntryNode(G)); } // Because we want a reverse post order, use reverse iterators from the vector rpo_iterator begin() { return Blocks.rbegin(); } + const_rpo_iterator begin() const { return Blocks.crbegin(); } rpo_iterator end() { return Blocks.rend(); } + const_rpo_iterator end() const { return Blocks.crend(); } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/ADT/STLExtras.h b/contrib/llvm/include/llvm/ADT/STLExtras.h index 94365dd9ced..f66ca7c08a7 100644 --- a/contrib/llvm/include/llvm/ADT/STLExtras.h +++ b/contrib/llvm/include/llvm/ADT/STLExtras.h @@ -21,6 +21,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/Config/abi-breaking.h" #include "llvm/Support/ErrorHandling.h" #include #include @@ -70,6 +71,16 @@ template struct conjunction : std::conditional, B1>::type {}; +template struct make_const_ptr { + using type = + typename std::add_pointer::type>::type; +}; + +template struct make_const_ref { + using type = typename std::add_lvalue_reference< + typename std::add_const::type>::type; +}; + //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -194,6 +205,12 @@ void adl_swap(T &&lhs, T &&rhs) noexcept( adl_detail::adl_swap(std::forward(lhs), std::forward(rhs)); } +/// Test whether \p RangeOrContainer is empty. Similar to C++17 std::empty. +template +constexpr bool empty(const T &RangeOrContainer) { + return adl_begin(RangeOrContainer) == adl_end(RangeOrContainer); +} + // mapped_iterator - This is a simple iterator adapter that causes a function to // be applied whenever operator* is invoked on the iterator. @@ -418,9 +435,94 @@ make_filter_range(RangeT &&Range, PredicateT Pred) { std::end(std::forward(Range)), Pred)); } -// forward declarations required by zip_shortest/zip_first +/// A pseudo-iterator adaptor that is designed to implement "early increment" +/// style loops. +/// +/// This is *not a normal iterator* and should almost never be used directly. It +/// is intended primarily to be used with range based for loops and some range +/// algorithms. +/// +/// The iterator isn't quite an `OutputIterator` or an `InputIterator` but +/// somewhere between them. The constraints of these iterators are: +/// +/// - On construction or after being incremented, it is comparable and +/// dereferencable. It is *not* incrementable. +/// - After being dereferenced, it is neither comparable nor dereferencable, it +/// is only incrementable. +/// +/// This means you can only dereference the iterator once, and you can only +/// increment it once between dereferences. +template +class early_inc_iterator_impl + : public iterator_adaptor_base, + WrappedIteratorT, std::input_iterator_tag> { + using BaseT = + iterator_adaptor_base, + WrappedIteratorT, std::input_iterator_tag>; + + using PointerT = typename std::iterator_traits::pointer; + +protected: +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + bool IsEarlyIncremented = false; +#endif + +public: + early_inc_iterator_impl(WrappedIteratorT I) : BaseT(I) {} + + using BaseT::operator*; + typename BaseT::reference operator*() { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + assert(!IsEarlyIncremented && "Cannot dereference twice!"); + IsEarlyIncremented = true; +#endif + return *(this->I)++; + } + + using BaseT::operator++; + early_inc_iterator_impl &operator++() { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + assert(IsEarlyIncremented && "Cannot increment before dereferencing!"); + IsEarlyIncremented = false; +#endif + return *this; + } + + using BaseT::operator==; + bool operator==(const early_inc_iterator_impl &RHS) const { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + assert(!IsEarlyIncremented && "Cannot compare after dereferencing!"); +#endif + return BaseT::operator==(RHS); + } +}; + +/// Make a range that does early increment to allow mutation of the underlying +/// range without disrupting iteration. +/// +/// The underlying iterator will be incremented immediately after it is +/// dereferenced, allowing deletion of the current node or insertion of nodes to +/// not disrupt iteration provided they do not invalidate the *next* iterator -- +/// the current iterator can be invalidated. +/// +/// This requires a very exact pattern of use that is only really suitable to +/// range based for loops and other range algorithms that explicitly guarantee +/// to dereference exactly once each element, and to increment exactly once each +/// element. +template +iterator_range>> +make_early_inc_range(RangeT &&Range) { + using EarlyIncIteratorT = + early_inc_iterator_impl>; + return make_range(EarlyIncIteratorT(std::begin(std::forward(Range))), + EarlyIncIteratorT(std::end(std::forward(Range)))); +} + +// forward declarations required by zip_shortest/zip_first/zip_longest template bool all_of(R &&range, UnaryPredicate P); +template +bool any_of(R &&range, UnaryPredicate P); template struct index_sequence; @@ -571,6 +673,132 @@ detail::zippy zip_first(T &&t, U &&u, std::forward(t), std::forward(u), std::forward(args)...); } +namespace detail { +template +static Iter next_or_end(const Iter &I, const Iter &End) { + if (I == End) + return End; + return std::next(I); +} + +template +static auto deref_or_none(const Iter &I, const Iter &End) + -> llvm::Optional::type>::type> { + if (I == End) + return None; + return *I; +} + +template struct ZipLongestItemType { + using type = + llvm::Optional())>::type>::type>; +}; + +template struct ZipLongestTupleType { + using type = std::tuple::type...>; +}; + +template +class zip_longest_iterator + : public iterator_facade_base< + zip_longest_iterator, + typename std::common_type< + std::forward_iterator_tag, + typename std::iterator_traits::iterator_category...>::type, + typename ZipLongestTupleType::type, + typename std::iterator_traits>::type>::difference_type, + typename ZipLongestTupleType::type *, + typename ZipLongestTupleType::type> { +public: + using value_type = typename ZipLongestTupleType::type; + +private: + std::tuple iterators; + std::tuple end_iterators; + + template + bool test(const zip_longest_iterator &other, + index_sequence) const { + return llvm::any_of( + std::initializer_list{std::get(this->iterators) != + std::get(other.iterators)...}, + identity{}); + } + + template value_type deref(index_sequence) const { + return value_type( + deref_or_none(std::get(iterators), std::get(end_iterators))...); + } + + template + decltype(iterators) tup_inc(index_sequence) const { + return std::tuple( + next_or_end(std::get(iterators), std::get(end_iterators))...); + } + +public: + zip_longest_iterator(std::pair... ts) + : iterators(std::forward(ts.first)...), + end_iterators(std::forward(ts.second)...) {} + + value_type operator*() { return deref(index_sequence_for{}); } + + value_type operator*() const { return deref(index_sequence_for{}); } + + zip_longest_iterator &operator++() { + iterators = tup_inc(index_sequence_for{}); + return *this; + } + + bool operator==(const zip_longest_iterator &other) const { + return !test(other, index_sequence_for{}); + } +}; + +template class zip_longest_range { +public: + using iterator = + zip_longest_iterator()))...>; + using iterator_category = typename iterator::iterator_category; + using value_type = typename iterator::value_type; + using difference_type = typename iterator::difference_type; + using pointer = typename iterator::pointer; + using reference = typename iterator::reference; + +private: + std::tuple ts; + + template iterator begin_impl(index_sequence) const { + return iterator(std::make_pair(adl_begin(std::get(ts)), + adl_end(std::get(ts)))...); + } + + template iterator end_impl(index_sequence) const { + return iterator(std::make_pair(adl_end(std::get(ts)), + adl_end(std::get(ts)))...); + } + +public: + zip_longest_range(Args &&... ts_) : ts(std::forward(ts_)...) {} + + iterator begin() const { return begin_impl(index_sequence_for{}); } + iterator end() const { return end_impl(index_sequence_for{}); } +}; +} // namespace detail + +/// Iterate over two or more iterators at the same time. Iteration continues +/// until all iterators reach the end. The llvm::Optional only contains a value +/// if the iterator has not reached the end. +template +detail::zip_longest_range zip_longest(T &&t, U &&u, + Args &&... args) { + return detail::zip_longest_range( + std::forward(t), std::forward(u), std::forward(args)...); +} + /// Iterator wrapper that concatenates sequences together. /// /// This can concatenate different iterators, even with different types, into @@ -593,18 +821,20 @@ class concat_iterator /// Note that something like iterator_range seems nice at first here, but the /// range properties are of little benefit and end up getting in the way /// because we need to do mutation on the current iterators. - std::tuple...> IterPairs; + std::tuple Begins; + std::tuple Ends; /// Attempts to increment a specific iterator. /// /// Returns true if it was able to increment the iterator. Returns false if /// the iterator is already at the end iterator. template bool incrementHelper() { - auto &IterPair = std::get(IterPairs); - if (IterPair.first == IterPair.second) + auto &Begin = std::get(Begins); + auto &End = std::get(Ends); + if (Begin == End) return false; - ++IterPair.first; + ++Begin; return true; } @@ -628,11 +858,12 @@ class concat_iterator /// dereferences the iterator and returns the address of the resulting /// reference. template ValueT *getHelper() const { - auto &IterPair = std::get(IterPairs); - if (IterPair.first == IterPair.second) + auto &Begin = std::get(Begins); + auto &End = std::get(Ends); + if (Begin == End) return nullptr; - return &*IterPair.first; + return &*Begin; } /// Finds the first non-end iterator, dereferences, and returns the resulting @@ -659,7 +890,7 @@ public: /// iterators. template explicit concat_iterator(RangeTs &&... Ranges) - : IterPairs({std::begin(Ranges), std::end(Ranges)}...) {} + : Begins(std::begin(Ranges)...), Ends(std::end(Ranges)...) {} using BaseT::operator++; @@ -671,7 +902,7 @@ public: ValueT &operator*() const { return get(index_sequence_for()); } bool operator==(const concat_iterator &RHS) const { - return IterPairs == RHS.IterPairs; + return Begins == RHS.Begins && Ends == RHS.Ends; } }; @@ -740,6 +971,19 @@ struct less_second { } }; +/// \brief Function object to apply a binary function to the first component of +/// a std::pair. +template +struct on_first { + FuncTy func; + + template + auto operator()(const T &lhs, const T &rhs) const + -> decltype(func(lhs.first, rhs.first)) { + return func(lhs.first, rhs.first); + } +}; + // A subset of N3658. More stuff can be added as-needed. /// Represents a compile-time sequence of integers. @@ -877,6 +1121,10 @@ inline void sort(IteratorTy Start, IteratorTy End) { std::sort(Start, End); } +template inline void sort(Container &&C) { + llvm::sort(adl_begin(C), adl_end(C)); +} + template inline void sort(IteratorTy Start, IteratorTy End, Compare Comp) { #ifdef EXPENSIVE_CHECKS @@ -886,6 +1134,11 @@ inline void sort(IteratorTy Start, IteratorTy End, Compare Comp) { std::sort(Start, End, Comp); } +template +inline void sort(Container &&C, Compare Comp) { + llvm::sort(adl_begin(C), adl_end(C), Comp); +} + //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -908,6 +1161,18 @@ void DeleteContainerSeconds(Container &C) { C.clear(); } +/// Get the size of a range. This is a wrapper function around std::distance +/// which is only enabled when the operation is O(1). +template +auto size(R &&Range, typename std::enable_if< + std::is_same::iterator_category, + std::random_access_iterator_tag>::value, + void>::type * = nullptr) + -> decltype(std::distance(Range.begin(), Range.end())) { + return std::distance(Range.begin(), Range.end()); +} + /// Provide wrappers to std::for_each which take ranges instead of having to /// pass begin/end explicitly. template @@ -1018,6 +1283,33 @@ auto lower_bound(R &&Range, ForwardIt I) -> decltype(adl_begin(Range)) { return std::lower_bound(adl_begin(Range), adl_end(Range), I); } +template +auto lower_bound(R &&Range, ForwardIt I, Compare C) + -> decltype(adl_begin(Range)) { + return std::lower_bound(adl_begin(Range), adl_end(Range), I, C); +} + +/// Provide wrappers to std::upper_bound which take ranges instead of having to +/// pass begin/end explicitly. +template +auto upper_bound(R &&Range, ForwardIt I) -> decltype(adl_begin(Range)) { + return std::upper_bound(adl_begin(Range), adl_end(Range), I); +} + +template +auto upper_bound(R &&Range, ForwardIt I, Compare C) + -> decltype(adl_begin(Range)) { + return std::upper_bound(adl_begin(Range), adl_end(Range), I, C); +} +/// Wrapper function around std::equal to detect if all elements +/// in a container are same. +template +bool is_splat(R &&Range) { + size_t range_size = size(Range); + return range_size != 0 && (range_size == 1 || + std::equal(adl_begin(Range) + 1, adl_end(Range), adl_begin(Range))); +} + /// Given a range of type R, iterate the entire range and return a /// SmallVector with elements of the vector. This is useful, for example, /// when you want to iterate a range and then sort the results. @@ -1039,18 +1331,6 @@ void erase_if(Container &C, UnaryPredicate P) { C.erase(remove_if(C, P), C.end()); } -/// Get the size of a range. This is a wrapper function around std::distance -/// which is only enabled when the operation is O(1). -template -auto size(R &&Range, typename std::enable_if< - std::is_same::iterator_category, - std::random_access_iterator_tag>::value, - void>::type * = nullptr) - -> decltype(std::distance(Range.begin(), Range.end())) { - return std::distance(Range.begin(), Range.end()); -} - //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -1263,6 +1543,40 @@ auto apply_tuple(F &&f, Tuple &&t) -> decltype(detail::apply_tuple_impl( Indices{}); } +/// Return true if the sequence [Begin, End) has exactly N items. Runs in O(N) +/// time. Not meant for use with random-access iterators. +template +bool hasNItems( + IterTy &&Begin, IterTy &&End, unsigned N, + typename std::enable_if< + !std::is_same< + typename std::iterator_traits::type>::iterator_category, + std::random_access_iterator_tag>::value, + void>::type * = nullptr) { + for (; N; --N, ++Begin) + if (Begin == End) + return false; // Too few. + return Begin == End; +} + +/// Return true if the sequence [Begin, End) has N or more items. Runs in O(N) +/// time. Not meant for use with random-access iterators. +template +bool hasNItemsOrMore( + IterTy &&Begin, IterTy &&End, unsigned N, + typename std::enable_if< + !std::is_same< + typename std::iterator_traits::type>::iterator_category, + std::random_access_iterator_tag>::value, + void>::type * = nullptr) { + for (; N; --N, ++Begin) + if (Begin == End) + return false; // Too few. + return true; +} + } // end namespace llvm #endif // LLVM_ADT_STLEXTRAS_H diff --git a/contrib/llvm/include/llvm/ADT/SmallBitVector.h b/contrib/llvm/include/llvm/ADT/SmallBitVector.h index b6391746639..0a73dbd6067 100644 --- a/contrib/llvm/include/llvm/ADT/SmallBitVector.h +++ b/contrib/llvm/include/llvm/ADT/SmallBitVector.h @@ -92,10 +92,6 @@ public: }; private: - bool isSmall() const { - return X & uintptr_t(1); - } - BitVector *getPointer() const { assert(!isSmall()); return reinterpret_cast(X); @@ -186,6 +182,8 @@ public: return make_range(set_bits_begin(), set_bits_end()); } + bool isSmall() const { return X & uintptr_t(1); } + /// Tests whether there are no bits in this bitvector. bool empty() const { return isSmall() ? getSmallSize() == 0 : getPointer()->empty(); @@ -242,7 +240,7 @@ public: uintptr_t Bits = getSmallBits(); if (Bits == 0) return -1; - return NumBaseBits - countLeadingZeros(Bits); + return NumBaseBits - countLeadingZeros(Bits) - 1; } return getPointer()->find_last(); } @@ -265,7 +263,9 @@ public: return -1; uintptr_t Bits = getSmallBits(); - return NumBaseBits - countLeadingOnes(Bits); + // Set unused bits. + Bits |= ~uintptr_t(0) << getSmallSize(); + return NumBaseBits - countLeadingOnes(Bits) - 1; } return getPointer()->find_last_unset(); } @@ -465,6 +465,11 @@ public: return (*this)[Idx]; } + // Push single bit to end of vector. + void push_back(bool Val) { + resize(size() + 1, Val); + } + /// Test if any common bits are set. bool anyCommon(const SmallBitVector &RHS) const { if (isSmall() && RHS.isSmall()) @@ -482,10 +487,17 @@ public: bool operator==(const SmallBitVector &RHS) const { if (size() != RHS.size()) return false; - if (isSmall()) + if (isSmall() && RHS.isSmall()) return getSmallBits() == RHS.getSmallBits(); - else + else if (!isSmall() && !RHS.isSmall()) return *getPointer() == *RHS.getPointer(); + else { + for (size_t i = 0, e = size(); i != e; ++i) { + if ((*this)[i] != RHS[i]) + return false; + } + return true; + } } bool operator!=(const SmallBitVector &RHS) const { @@ -493,16 +505,19 @@ public: } // Intersection, union, disjoint union. + // FIXME BitVector::operator&= does not resize the LHS but this does SmallBitVector &operator&=(const SmallBitVector &RHS) { resize(std::max(size(), RHS.size())); - if (isSmall()) + if (isSmall() && RHS.isSmall()) setSmallBits(getSmallBits() & RHS.getSmallBits()); - else if (!RHS.isSmall()) + else if (!isSmall() && !RHS.isSmall()) getPointer()->operator&=(*RHS.getPointer()); else { - SmallBitVector Copy = RHS; - Copy.resize(size()); - getPointer()->operator&=(*Copy.getPointer()); + size_t i, e; + for (i = 0, e = std::min(size(), RHS.size()); i != e; ++i) + (*this)[i] = test(i) && RHS.test(i); + for (e = size(); i != e; ++i) + reset(i); } return *this; } @@ -542,28 +557,26 @@ public: SmallBitVector &operator|=(const SmallBitVector &RHS) { resize(std::max(size(), RHS.size())); - if (isSmall()) + if (isSmall() && RHS.isSmall()) setSmallBits(getSmallBits() | RHS.getSmallBits()); - else if (!RHS.isSmall()) + else if (!isSmall() && !RHS.isSmall()) getPointer()->operator|=(*RHS.getPointer()); else { - SmallBitVector Copy = RHS; - Copy.resize(size()); - getPointer()->operator|=(*Copy.getPointer()); + for (size_t i = 0, e = RHS.size(); i != e; ++i) + (*this)[i] = test(i) || RHS.test(i); } return *this; } SmallBitVector &operator^=(const SmallBitVector &RHS) { resize(std::max(size(), RHS.size())); - if (isSmall()) + if (isSmall() && RHS.isSmall()) setSmallBits(getSmallBits() ^ RHS.getSmallBits()); - else if (!RHS.isSmall()) + else if (!isSmall() && !RHS.isSmall()) getPointer()->operator^=(*RHS.getPointer()); else { - SmallBitVector Copy = RHS; - Copy.resize(size()); - getPointer()->operator^=(*Copy.getPointer()); + for (size_t i = 0, e = RHS.size(); i != e; ++i) + (*this)[i] = test(i) != RHS.test(i); } return *this; } diff --git a/contrib/llvm/include/llvm/ADT/SmallVector.h b/contrib/llvm/include/llvm/ADT/SmallVector.h index acb4426b4f4..0636abbb1fb 100644 --- a/contrib/llvm/include/llvm/ADT/SmallVector.h +++ b/contrib/llvm/include/llvm/ADT/SmallVector.h @@ -182,7 +182,7 @@ public: /// SmallVectorTemplateBase - This is where we put method /// implementations that are designed to work with non-POD-like T's. -template +template ::value> class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} @@ -299,7 +299,7 @@ protected: // use memcpy here. Note that I and E are iterators and thus might be // invalid for memcpy if they are equal. if (I != E) - memcpy(Dest, I, (E - I) * sizeof(T)); + memcpy(reinterpret_cast(Dest), I, (E - I) * sizeof(T)); } /// Double the size of the allocated memory, guaranteeing space for at @@ -310,7 +310,7 @@ public: void push_back(const T &Elt) { if (LLVM_UNLIKELY(this->size() >= this->capacity())) this->grow(); - memcpy(this->end(), &Elt, sizeof(T)); + memcpy(reinterpret_cast(this->end()), &Elt, sizeof(T)); this->set_size(this->size() + 1); } @@ -320,8 +320,8 @@ public: /// This class consists of common code factored out of the SmallVector class to /// reduce code duplication based on the SmallVector 'N' template parameter. template -class SmallVectorImpl : public SmallVectorTemplateBase::value> { - using SuperClass = SmallVectorTemplateBase::value>; +class SmallVectorImpl : public SmallVectorTemplateBase { + using SuperClass = SmallVectorTemplateBase; public: using iterator = typename SuperClass::iterator; diff --git a/contrib/llvm/include/llvm/ADT/SparseBitVector.h b/contrib/llvm/include/llvm/ADT/SparseBitVector.h index 4cbf40c7680..84e73bcbace 100644 --- a/contrib/llvm/include/llvm/ADT/SparseBitVector.h +++ b/contrib/llvm/include/llvm/ADT/SparseBitVector.h @@ -261,21 +261,33 @@ class SparseBitVector { BITWORD_SIZE = SparseBitVectorElement::BITWORD_SIZE }; - // Pointer to our current Element. - ElementListIter CurrElementIter; ElementList Elements; + // Pointer to our current Element. This has no visible effect on the external + // state of a SparseBitVector, it's just used to improve performance in the + // common case of testing/modifying bits with similar indices. + mutable ElementListIter CurrElementIter; // This is like std::lower_bound, except we do linear searching from the // current position. - ElementListIter FindLowerBound(unsigned ElementIndex) { + ElementListIter FindLowerBoundImpl(unsigned ElementIndex) const { + + // We cache a non-const iterator so we're forced to resort to const_cast to + // get the begin/end in the case where 'this' is const. To avoid duplication + // of code with the only difference being whether the const cast is present + // 'this' is always const in this particular function and we sort out the + // difference in FindLowerBound and FindLowerBoundConst. + ElementListIter Begin = + const_cast *>(this)->Elements.begin(); + ElementListIter End = + const_cast *>(this)->Elements.end(); if (Elements.empty()) { - CurrElementIter = Elements.begin(); - return Elements.begin(); + CurrElementIter = Begin; + return CurrElementIter; } // Make sure our current iterator is valid. - if (CurrElementIter == Elements.end()) + if (CurrElementIter == End) --CurrElementIter; // Search from our current iterator, either backwards or forwards, @@ -284,17 +296,23 @@ class SparseBitVector { if (CurrElementIter->index() == ElementIndex) { return ElementIter; } else if (CurrElementIter->index() > ElementIndex) { - while (ElementIter != Elements.begin() + while (ElementIter != Begin && ElementIter->index() > ElementIndex) --ElementIter; } else { - while (ElementIter != Elements.end() && + while (ElementIter != End && ElementIter->index() < ElementIndex) ++ElementIter; } CurrElementIter = ElementIter; return ElementIter; } + ElementListConstIter FindLowerBoundConst(unsigned ElementIndex) const { + return FindLowerBoundImpl(ElementIndex); + } + ElementListIter FindLowerBound(unsigned ElementIndex) { + return FindLowerBoundImpl(ElementIndex); + } // Iterator to walk set bits in the bitmap. This iterator is a lot uglier // than it would be, in order to be efficient. @@ -423,22 +441,12 @@ class SparseBitVector { public: using iterator = SparseBitVectorIterator; - SparseBitVector() { - CurrElementIter = Elements.begin(); - } + SparseBitVector() : Elements(), CurrElementIter(Elements.begin()) {} - // SparseBitVector copy ctor. - SparseBitVector(const SparseBitVector &RHS) { - ElementListConstIter ElementIter = RHS.Elements.begin(); - while (ElementIter != RHS.Elements.end()) { - Elements.push_back(SparseBitVectorElement(*ElementIter)); - ++ElementIter; - } - - CurrElementIter = Elements.begin (); - } - - ~SparseBitVector() = default; + SparseBitVector(const SparseBitVector &RHS) + : Elements(RHS.Elements), CurrElementIter(Elements.begin()) {} + SparseBitVector(SparseBitVector &&RHS) + : Elements(std::move(RHS.Elements)), CurrElementIter(Elements.begin()) {} // Clear. void clear() { @@ -450,26 +458,23 @@ public: if (this == &RHS) return *this; - Elements.clear(); - - ElementListConstIter ElementIter = RHS.Elements.begin(); - while (ElementIter != RHS.Elements.end()) { - Elements.push_back(SparseBitVectorElement(*ElementIter)); - ++ElementIter; - } - - CurrElementIter = Elements.begin (); - + Elements = RHS.Elements; + CurrElementIter = Elements.begin(); + return *this; + } + SparseBitVector &operator=(SparseBitVector &&RHS) { + Elements = std::move(RHS.Elements); + CurrElementIter = Elements.begin(); return *this; } // Test, Reset, and Set a bit in the bitmap. - bool test(unsigned Idx) { + bool test(unsigned Idx) const { if (Elements.empty()) return false; unsigned ElementIndex = Idx / ElementSize; - ElementListIter ElementIter = FindLowerBound(ElementIndex); + ElementListConstIter ElementIter = FindLowerBoundConst(ElementIndex); // If we can't find an element that is supposed to contain this bit, there // is nothing more to do. diff --git a/contrib/llvm/include/llvm/ADT/StringExtras.h b/contrib/llvm/include/llvm/ADT/StringExtras.h index 71b0e7527cb..60a03633a8a 100644 --- a/contrib/llvm/include/llvm/ADT/StringExtras.h +++ b/contrib/llvm/include/llvm/ADT/StringExtras.h @@ -139,22 +139,23 @@ inline std::string utohexstr(uint64_t X, bool LowerCase = false) { /// Convert buffer \p Input to its hexadecimal representation. /// The returned string is double the size of \p Input. -inline std::string toHex(StringRef Input) { +inline std::string toHex(StringRef Input, bool LowerCase = false) { static const char *const LUT = "0123456789ABCDEF"; + const uint8_t Offset = LowerCase ? 32 : 0; size_t Length = Input.size(); std::string Output; Output.reserve(2 * Length); for (size_t i = 0; i < Length; ++i) { const unsigned char c = Input[i]; - Output.push_back(LUT[c >> 4]); - Output.push_back(LUT[c & 15]); + Output.push_back(LUT[c >> 4] | Offset); + Output.push_back(LUT[c & 15] | Offset); } return Output; } -inline std::string toHex(ArrayRef Input) { - return toHex(toStringRef(Input)); +inline std::string toHex(ArrayRef Input, bool LowerCase = false) { + return toHex(toStringRef(Input), LowerCase); } inline uint8_t hexFromNibbles(char MSB, char LSB) { diff --git a/contrib/llvm/include/llvm/ADT/Triple.h b/contrib/llvm/include/llvm/ADT/Triple.h index c95b16dd4e8..e06a68e2731 100644 --- a/contrib/llvm/include/llvm/ADT/Triple.h +++ b/contrib/llvm/include/llvm/ADT/Triple.h @@ -55,12 +55,11 @@ public: bpfel, // eBPF or extended BPF or 64-bit BPF (little endian) bpfeb, // eBPF or extended BPF or 64-bit BPF (big endian) hexagon, // Hexagon: hexagon - mips, // MIPS: mips, mipsallegrex - mipsel, // MIPSEL: mipsel, mipsallegrexel - mips64, // MIPS64: mips64 - mips64el, // MIPS64EL: mips64el + mips, // MIPS: mips, mipsallegrex, mipsr6 + mipsel, // MIPSEL: mipsel, mipsallegrexe, mipsr6el + mips64, // MIPS64: mips64, mips64r6, mipsn32, mipsn32r6 + mips64el, // MIPS64EL: mips64el, mips64r6el, mipsn32el, mipsn32r6el msp430, // MSP430: msp430 - nios2, // NIOSII: nios2 ppc, // PPC: powerpc ppc64, // PPC64: powerpc64, ppu ppc64le, // PPC64LE: powerpc64le @@ -101,6 +100,7 @@ public: enum SubArchType { NoSubArch, + ARMSubArch_v8_5a, ARMSubArch_v8_4a, ARMSubArch_v8_3a, ARMSubArch_v8_2a, @@ -125,7 +125,9 @@ public: KalimbaSubArch_v3, KalimbaSubArch_v4, - KalimbaSubArch_v5 + KalimbaSubArch_v5, + + MipsSubArch_r6 }; enum VendorType { UnknownVendor, @@ -182,7 +184,10 @@ public: Mesa3D, Contiki, AMDPAL, // AMD PAL Runtime - LastOSType = AMDPAL + HermitCore, // HermitCore Unikernel/Multikernel + Hurd, // GNU/Hurd + WASI, // Experimental WebAssembly OS + LastOSType = WASI }; enum EnvironmentType { UnknownEnvironment, @@ -578,9 +583,20 @@ public: return getOS() == Triple::KFreeBSD; } + /// Tests whether the OS is Hurd. + bool isOSHurd() const { + return getOS() == Triple::Hurd; + } + + /// Tests whether the OS is WASI. + bool isOSWASI() const { + return getOS() == Triple::WASI; + } + /// Tests whether the OS uses glibc. bool isOSGlibc() const { - return (getOS() == Triple::Linux || getOS() == Triple::KFreeBSD) && + return (getOS() == Triple::Linux || getOS() == Triple::KFreeBSD || + getOS() == Triple::Hurd) && !isAndroid(); } diff --git a/contrib/llvm/include/llvm/ADT/bit.h b/contrib/llvm/include/llvm/ADT/bit.h new file mode 100644 index 00000000000..a4aba7b6a9e --- /dev/null +++ b/contrib/llvm/include/llvm/ADT/bit.h @@ -0,0 +1,59 @@ +//===-- llvm/ADT/bit.h - C++20 ----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the C++20 header. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_BIT_H +#define LLVM_ADT_BIT_H + +#include "llvm/Support/Compiler.h" +#include +#include + +namespace llvm { + +// This implementation of bit_cast is different from the C++17 one in two ways: +// - It isn't constexpr because that requires compiler support. +// - It requires trivially-constructible To, to avoid UB in the implementation. +template ::type +#if (__has_feature(is_trivially_constructible) && defined(_LIBCPP_VERSION)) || \ + (defined(__GNUC__) && __GNUC__ >= 5) + , typename = typename std::is_trivially_constructible::type +#elif __has_feature(is_trivially_constructible) + , typename = typename std::enable_if<__is_trivially_constructible(To)>::type +#else + // See comment below. +#endif +#if (__has_feature(is_trivially_copyable) && defined(_LIBCPP_VERSION)) || \ + (defined(__GNUC__) && __GNUC__ >= 5) + , typename = typename std::enable_if::value>::type + , typename = typename std::enable_if::value>::type +#elif __has_feature(is_trivially_copyable) + , typename = typename std::enable_if<__is_trivially_copyable(To)>::type + , typename = typename std::enable_if<__is_trivially_copyable(From)>::type +#else + // This case is GCC 4.x. clang with libc++ or libstdc++ never get here. Unlike + // llvm/Support/type_traits.h's isPodLike we don't want to provide a + // good-enough answer here: developers in that configuration will hit + // compilation failures on the bots instead of locally. That's acceptable + // because it's very few developers, and only until we move past C++11. +#endif +> +inline To bit_cast(const From &from) noexcept { + To to; + std::memcpy(&to, &from, sizeof(To)); + return to; +} + +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/ADT/iterator.h b/contrib/llvm/include/llvm/ADT/iterator.h index 549c5221173..40e490cf786 100644 --- a/contrib/llvm/include/llvm/ADT/iterator.h +++ b/contrib/llvm/include/llvm/ADT/iterator.h @@ -202,9 +202,7 @@ template < typename ReferenceT = typename std::conditional< std::is_same::value_type>::value, - typename std::iterator_traits::reference, T &>::type, - // Don't provide these, they are mostly to act as aliases below. - typename WrappedTraitsT = std::iterator_traits> + typename std::iterator_traits::reference, T &>::type> class iterator_adaptor_base : public iterator_facade_base { @@ -311,8 +309,10 @@ make_pointee_range(RangeT &&Range) { template ())> class pointer_iterator - : public iterator_adaptor_base, - WrappedIteratorT, T> { + : public iterator_adaptor_base< + pointer_iterator, WrappedIteratorT, + typename std::iterator_traits::iterator_category, + T> { mutable T Ptr; public: @@ -334,6 +334,34 @@ make_pointer_range(RangeT &&Range) { PointerIteratorT(std::end(std::forward(Range)))); } +// Wrapper iterator over iterator ItType, adding DataRef to the type of ItType, +// to create NodeRef = std::pair. +template +class WrappedPairNodeDataIterator + : public iterator_adaptor_base< + WrappedPairNodeDataIterator, ItType, + typename std::iterator_traits::iterator_category, NodeRef, + std::ptrdiff_t, NodeRef *, NodeRef &> { + using BaseT = iterator_adaptor_base< + WrappedPairNodeDataIterator, ItType, + typename std::iterator_traits::iterator_category, NodeRef, + std::ptrdiff_t, NodeRef *, NodeRef &>; + + const DataRef DR; + mutable NodeRef NR; + +public: + WrappedPairNodeDataIterator(ItType Begin, const DataRef DR) + : BaseT(Begin), DR(DR) { + NR.first = DR; + } + + NodeRef &operator*() const { + NR.second = *this->I; + return NR; + } +}; + } // end namespace llvm #endif // LLVM_ADT_ITERATOR_H diff --git a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h index be3496bbd95..e2a2ac0622e 100644 --- a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -43,7 +43,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" @@ -335,8 +334,7 @@ public: /// A convenience wrapper around the primary \c alias interface. AliasResult alias(const Value *V1, const Value *V2) { - return alias(V1, MemoryLocation::UnknownSize, V2, - MemoryLocation::UnknownSize); + return alias(V1, LocationSize::unknown(), V2, LocationSize::unknown()); } /// A trivial helper function to check to see if the specified pointers are @@ -364,7 +362,8 @@ public: /// A convenience wrapper around the \c isMustAlias helper interface. bool isMustAlias(const Value *V1, const Value *V2) { - return alias(V1, 1, V2, 1) == MustAlias; + return alias(V1, LocationSize::precise(1), V2, LocationSize::precise(1)) == + MustAlias; } /// Checks whether the given location points to constant memory, or if @@ -382,15 +381,15 @@ public: /// \name Simple mod/ref information /// @{ - /// Get the ModRef info associated with a pointer argument of a callsite. The + /// Get the ModRef info associated with a pointer argument of a call. The /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note /// that these bits do not necessarily account for the overall behavior of /// the function, but rather only provide additional per-argument /// information. This never sets ModRefInfo::Must. - ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx); + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx); /// Return the behavior of the given call site. - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + FunctionModRefBehavior getModRefBehavior(const CallBase *Call); /// Return the behavior when calling the given function. FunctionModRefBehavior getModRefBehavior(const Function *F); @@ -406,8 +405,8 @@ public: /// property (e.g. calls to 'sin' and 'cos'). /// /// This property corresponds to the GCC 'const' attribute. - bool doesNotAccessMemory(ImmutableCallSite CS) { - return getModRefBehavior(CS) == FMRB_DoesNotAccessMemory; + bool doesNotAccessMemory(const CallBase *Call) { + return getModRefBehavior(Call) == FMRB_DoesNotAccessMemory; } /// Checks if the specified function is known to never read or write memory. @@ -434,8 +433,8 @@ public: /// absence of interfering store instructions, such as CSE of strlen calls. /// /// This property corresponds to the GCC 'pure' attribute. - bool onlyReadsMemory(ImmutableCallSite CS) { - return onlyReadsMemory(getModRefBehavior(CS)); + bool onlyReadsMemory(const CallBase *Call) { + return onlyReadsMemory(getModRefBehavior(Call)); } /// Checks if the specified function is known to only read from non-volatile @@ -500,36 +499,12 @@ public: /// getModRefInfo (for call sites) - Return information about whether /// a particular call site modifies or reads the specified memory location. - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); /// getModRefInfo (for call sites) - A convenience wrapper. - ModRefInfo getModRefInfo(ImmutableCallSite CS, const Value *P, + ModRefInfo getModRefInfo(const CallBase *Call, const Value *P, LocationSize Size) { - return getModRefInfo(CS, MemoryLocation(P, Size)); - } - - /// getModRefInfo (for calls) - Return information about whether - /// a particular call modifies or reads the specified memory location. - ModRefInfo getModRefInfo(const CallInst *C, const MemoryLocation &Loc) { - return getModRefInfo(ImmutableCallSite(C), Loc); - } - - /// getModRefInfo (for calls) - A convenience wrapper. - ModRefInfo getModRefInfo(const CallInst *C, const Value *P, - LocationSize Size) { - return getModRefInfo(C, MemoryLocation(P, Size)); - } - - /// getModRefInfo (for invokes) - Return information about whether - /// a particular invoke modifies or reads the specified memory location. - ModRefInfo getModRefInfo(const InvokeInst *I, const MemoryLocation &Loc) { - return getModRefInfo(ImmutableCallSite(I), Loc); - } - - /// getModRefInfo (for invokes) - A convenience wrapper. - ModRefInfo getModRefInfo(const InvokeInst *I, const Value *P, - LocationSize Size) { - return getModRefInfo(I, MemoryLocation(P, Size)); + return getModRefInfo(Call, MemoryLocation(P, Size)); } /// getModRefInfo (for loads) - Return information about whether @@ -569,7 +544,7 @@ public: /// getModRefInfo (for cmpxchges) - A convenience wrapper. ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, const Value *P, - unsigned Size) { + LocationSize Size) { return getModRefInfo(CX, MemoryLocation(P, Size)); } @@ -579,7 +554,7 @@ public: /// getModRefInfo (for atomicrmws) - A convenience wrapper. ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const Value *P, - unsigned Size) { + LocationSize Size) { return getModRefInfo(RMW, MemoryLocation(P, Size)); } @@ -626,8 +601,8 @@ public: ModRefInfo getModRefInfo(const Instruction *I, const Optional &OptLoc) { if (OptLoc == None) { - if (auto CS = ImmutableCallSite(I)) { - return createModRefInfo(getModRefBehavior(CS)); + if (const auto *Call = dyn_cast(I)) { + return createModRefInfo(getModRefBehavior(Call)); } } @@ -661,12 +636,12 @@ public: /// Return information about whether a call and an instruction may refer to /// the same memory locations. - ModRefInfo getModRefInfo(Instruction *I, ImmutableCallSite Call); + ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call); /// Return information about whether two call sites may refer to the same set /// of memory locations. See the AA documentation for details: /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo - ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2); /// Return information about whether a particular call site modifies /// or reads the specified memory location \p MemLoc before instruction \p I @@ -777,25 +752,25 @@ public: /// that these bits do not necessarily account for the overall behavior of /// the function, but rather only provide additional per-argument /// information. - virtual ModRefInfo getArgModRefInfo(ImmutableCallSite CS, + virtual ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) = 0; /// Return the behavior of the given call site. - virtual FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) = 0; + virtual FunctionModRefBehavior getModRefBehavior(const CallBase *Call) = 0; /// Return the behavior when calling the given function. virtual FunctionModRefBehavior getModRefBehavior(const Function *F) = 0; /// getModRefInfo (for call sites) - Return information about whether /// a particular call site modifies or reads the specified memory location. - virtual ModRefInfo getModRefInfo(ImmutableCallSite CS, + virtual ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) = 0; /// Return information about whether two call sites may refer to the same set /// of memory locations. See the AA documentation for details: /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo - virtual ModRefInfo getModRefInfo(ImmutableCallSite CS1, - ImmutableCallSite CS2) = 0; + virtual ModRefInfo getModRefInfo(const CallBase *Call1, + const CallBase *Call2) = 0; /// @} }; @@ -827,26 +802,26 @@ public: return Result.pointsToConstantMemory(Loc, OrLocal); } - ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) override { - return Result.getArgModRefInfo(CS, ArgIdx); + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) override { + return Result.getArgModRefInfo(Call, ArgIdx); } - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override { - return Result.getModRefBehavior(CS); + FunctionModRefBehavior getModRefBehavior(const CallBase *Call) override { + return Result.getModRefBehavior(Call); } FunctionModRefBehavior getModRefBehavior(const Function *F) override { return Result.getModRefBehavior(F); } - ModRefInfo getModRefInfo(ImmutableCallSite CS, + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) override { - return Result.getModRefInfo(CS, Loc); + return Result.getModRefInfo(Call, Loc); } - ModRefInfo getModRefInfo(ImmutableCallSite CS1, - ImmutableCallSite CS2) override { - return Result.getModRefInfo(CS1, CS2); + ModRefInfo getModRefInfo(const CallBase *Call1, + const CallBase *Call2) override { + return Result.getModRefInfo(Call1, Call2); } }; @@ -901,25 +876,28 @@ protected: : CurrentResult.pointsToConstantMemory(Loc, OrLocal); } - ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) { - return AAR ? AAR->getArgModRefInfo(CS, ArgIdx) : CurrentResult.getArgModRefInfo(CS, ArgIdx); + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { + return AAR ? AAR->getArgModRefInfo(Call, ArgIdx) + : CurrentResult.getArgModRefInfo(Call, ArgIdx); } - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) { - return AAR ? AAR->getModRefBehavior(CS) : CurrentResult.getModRefBehavior(CS); + FunctionModRefBehavior getModRefBehavior(const CallBase *Call) { + return AAR ? AAR->getModRefBehavior(Call) + : CurrentResult.getModRefBehavior(Call); } FunctionModRefBehavior getModRefBehavior(const Function *F) { return AAR ? AAR->getModRefBehavior(F) : CurrentResult.getModRefBehavior(F); } - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc) { - return AAR ? AAR->getModRefInfo(CS, Loc) - : CurrentResult.getModRefInfo(CS, Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { + return AAR ? AAR->getModRefInfo(Call, Loc) + : CurrentResult.getModRefInfo(Call, Loc); } - ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) { - return AAR ? AAR->getModRefInfo(CS1, CS2) : CurrentResult.getModRefInfo(CS1, CS2); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) { + return AAR ? AAR->getModRefInfo(Call1, Call2) + : CurrentResult.getModRefInfo(Call1, Call2); } }; @@ -951,11 +929,11 @@ public: return false; } - ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) { + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { return ModRefInfo::ModRef; } - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) { + FunctionModRefBehavior getModRefBehavior(const CallBase *Call) { return FMRB_UnknownModRefBehavior; } @@ -963,11 +941,11 @@ public: return FMRB_UnknownModRefBehavior; } - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc) { + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { return ModRefInfo::ModRef; } - ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) { + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) { return ModRefInfo::ModRef; } }; @@ -1075,6 +1053,29 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override; }; +/// A wrapper pass for external alias analyses. This just squirrels away the +/// callback used to run any analyses and register their results. +struct ExternalAAWrapperPass : ImmutablePass { + using CallbackT = std::function; + + CallbackT CB; + + static char ID; + + ExternalAAWrapperPass() : ImmutablePass(ID) { + initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry()); + } + + explicit ExternalAAWrapperPass(CallbackT CB) + : ImmutablePass(ID), CB(std::move(CB)) { + initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } +}; + FunctionPass *createAAResultsWrapperPass(); /// A wrapper pass around a callback which can be used to populate the diff --git a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h index c9680ff40d1..7ed5cd5c473 100644 --- a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h +++ b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h @@ -52,9 +52,13 @@ class AliasSet : public ilist_node { PointerRec **PrevInList = nullptr; PointerRec *NextInList = nullptr; AliasSet *AS = nullptr; - LocationSize Size = 0; + LocationSize Size = LocationSize::mapEmpty(); AAMDNodes AAInfo; + // Whether the size for this record has been set at all. This makes no + // guarantees about the size being known. + bool isSizeSet() const { return Size != LocationSize::mapEmpty(); } + public: PointerRec(Value *V) : Val(V), AAInfo(DenseMapInfo::getEmptyKey()) {} @@ -71,9 +75,10 @@ class AliasSet : public ilist_node { bool updateSizeAndAAInfo(LocationSize NewSize, const AAMDNodes &NewAAInfo) { bool SizeChanged = false; - if (NewSize > Size) { - Size = NewSize; - SizeChanged = true; + if (NewSize != Size) { + LocationSize OldSize = Size; + Size = isSizeSet() ? Size.unionWith(NewSize) : NewSize; + SizeChanged = OldSize != Size; } if (AAInfo == DenseMapInfo::getEmptyKey()) @@ -91,7 +96,10 @@ class AliasSet : public ilist_node { return SizeChanged; } - LocationSize getSize() const { return Size; } + LocationSize getSize() const { + assert(isSizeSet() && "Getting an unset size!"); + return Size; + } /// Return the AAInfo, or null if there is no information or conflicting /// information. @@ -175,9 +183,6 @@ class AliasSet : public ilist_node { }; unsigned Alias : 1; - /// True if this alias set contains volatile loads or stores. - unsigned Volatile : 1; - unsigned SetSize = 0; void addRef() { ++RefCount; } @@ -203,9 +208,6 @@ public: bool isMustAlias() const { return Alias == SetMustAlias; } bool isMayAlias() const { return Alias == SetMayAlias; } - /// Return true if this alias set contains volatile loads or stores. - bool isVolatile() const { return Volatile; } - /// Return true if this alias set should be ignored as part of the /// AliasSetTracker object. bool isForwardingAliasSet() const { return Forward; } @@ -224,6 +226,10 @@ public: // track of the list's exact size. unsigned size() { return SetSize; } + /// If this alias set is known to contain a single instruction and *only* a + /// single unique instruction, return it. Otherwise, return nullptr. + Instruction* getUniqueInstruction(); + void print(raw_ostream &OS) const; void dump() const; @@ -264,7 +270,7 @@ private: // Can only be created by AliasSetTracker. AliasSet() : PtrListEnd(&PtrList), RefCount(0), AliasAny(false), Access(NoAccess), - Alias(SetMustAlias), Volatile(false) {} + Alias(SetMustAlias) {} PointerRec *getSomePointer() const { return PtrList; @@ -303,8 +309,6 @@ private: dropRef(AST); } - void setVolatile() { Volatile = true; } - public: /// Return true if the specified pointer "may" (or must) alias one of the /// members in the set. @@ -379,23 +383,11 @@ public: /// Return the alias sets that are active. const ilist &getAliasSets() const { return AliasSets; } - /// Return the alias set that the specified pointer lives in. If the New - /// argument is non-null, this method sets the value to true if a new alias - /// set is created to contain the pointer (because the pointer didn't alias - /// anything). - AliasSet &getAliasSetForPointer(Value *P, LocationSize Size, - const AAMDNodes &AAInfo); - - /// Return the alias set containing the location specified if one exists, - /// otherwise return null. - AliasSet *getAliasSetForPointerIfExists(const Value *P, LocationSize Size, - const AAMDNodes &AAInfo) { - return mergeAliasSetsForPointer(P, Size, AAInfo); - } - - /// Return true if the specified instruction "may" (or must) alias one of the - /// members in any of the sets. - bool containsUnknown(const Instruction *I) const; + /// Return the alias set which contains the specified memory location. If + /// the memory location aliases two or more existing alias sets, will have + /// the effect of merging those alias sets before the single resulting alias + /// set is returned. + AliasSet &getAliasSetFor(const MemoryLocation &MemLoc); /// Return the underlying alias analysis object used by this tracker. AliasAnalysis &getAliasAnalysis() const { return AA; } @@ -445,8 +437,7 @@ private: return *Entry; } - AliasSet &addPointer(Value *P, LocationSize Size, const AAMDNodes &AAInfo, - AliasSet::AccessLattice E); + AliasSet &addPointer(MemoryLocation Loc, AliasSet::AccessLattice E); AliasSet *mergeAliasSetsForPointer(const Value *Ptr, LocationSize Size, const AAMDNodes &AAInfo); diff --git a/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h index 6344e84b58e..820d7ac0935 100644 --- a/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h @@ -21,7 +21,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/MemoryLocation.h" -#include "llvm/IR/CallSite.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include @@ -84,18 +84,18 @@ public: AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); - ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2); /// Chases pointers until we find a (constant global) or not. bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); /// Get the location associated with a pointer argument of a callsite. - ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx); + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx); /// Returns the behavior when calling the given call site. - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + FunctionModRefBehavior getModRefBehavior(const CallBase *Call); /// Returns the behavior when calling the given function. For use when the /// call site is not known. @@ -115,7 +115,7 @@ private: unsigned ZExtBits; unsigned SExtBits; - int64_t Scale; + APInt Scale; bool operator==(const VariableGEPIndex &Other) const { return V == Other.V && ZExtBits == Other.ZExtBits && @@ -133,10 +133,10 @@ private: // Base pointer of the GEP const Value *Base; // Total constant offset w.r.t the base from indexing into structs - int64_t StructOffset; + APInt StructOffset; // Total constant offset w.r.t the base from indexing through // pointers/arrays/vectors - int64_t OtherOffset; + APInt OtherOffset; // Scaled variable (non-constant) indices. SmallVector VarIndices; }; @@ -189,7 +189,7 @@ private: bool constantOffsetHeuristic(const SmallVectorImpl &VarIndices, LocationSize V1Size, LocationSize V2Size, - int64_t BaseOffset, AssumptionCache *AC, + APInt BaseOffset, AssumptionCache *AC, DominatorTree *DT); bool isValueEqualInPotentialCycles(const Value *V1, const Value *V2); diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h index ca12db6208b..0b261873569 100644 --- a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h +++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h @@ -56,7 +56,7 @@ public: const Function *getFunction() const; const BranchProbabilityInfo *getBPI() const; - void view() const; + void view(StringRef = "BlockFrequencyDAGs") const; /// getblockFreq - Return block frequency. Return 0 if we don't have the /// information. Please note that initial frequency is equal to ENTRY_FREQ. It diff --git a/contrib/llvm/include/llvm/Analysis/CFG.h b/contrib/llvm/include/llvm/Analysis/CFG.h index cccdd163741..caae0b6e2a8 100644 --- a/contrib/llvm/include/llvm/Analysis/CFG.h +++ b/contrib/llvm/include/llvm/Analysis/CFG.h @@ -25,7 +25,6 @@ class DominatorTree; class Function; class Instruction; class LoopInfo; -class TerminatorInst; /// Analyze the specified function to find all of the loop backedges in the /// function and return them. This is a relatively cheap (compared to @@ -46,7 +45,7 @@ unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ); /// edges from a block with multiple successors to a block with multiple /// predecessors. /// -bool isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum, +bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges = false); /// Determine whether instruction 'To' is reachable from 'From', diff --git a/contrib/llvm/include/llvm/Analysis/CFGPrinter.h b/contrib/llvm/include/llvm/Analysis/CFGPrinter.h index 5786769cc50..5996dd90bcf 100644 --- a/contrib/llvm/include/llvm/Analysis/CFGPrinter.h +++ b/contrib/llvm/include/llvm/Analysis/CFGPrinter.h @@ -150,7 +150,7 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { /// Display the raw branch weights from PGO. std::string getEdgeAttributes(const BasicBlock *Node, succ_const_iterator I, const Function *F) { - const TerminatorInst *TI = Node->getTerminator(); + const Instruction *TI = Node->getTerminator(); if (TI->getNumSuccessors() == 1) return ""; @@ -172,8 +172,7 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { // Prepend a 'W' to indicate that this is a weight rather than the actual // profile count (due to scaling). - Twine Attrs = "label=\"W:" + Twine(Weight->getZExtValue()) + "\""; - return Attrs.str(); + return ("label=\"W:" + Twine(Weight->getZExtValue()) + "\"").str(); } }; } // End llvm namespace diff --git a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h index 5e83ea2a6e2..61b99f6c3e6 100644 --- a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -364,6 +364,10 @@ public: InvalidSCCSet, nullptr, nullptr, InlinedInternalEdges}; + // Request PassInstrumentation from analysis manager, will use it to run + // instrumenting callbacks for the passes later. + PassInstrumentation PI = AM.getResult(M); + PreservedAnalyses PA = PreservedAnalyses::all(); CG.buildRefSCCs(); for (auto RCI = CG.postorder_ref_scc_begin(), @@ -428,8 +432,20 @@ public: UR.UpdatedRC = nullptr; UR.UpdatedC = nullptr; + + // Check the PassInstrumentation's BeforePass callbacks before + // running the pass, skip its execution completely if asked to + // (callback returns false). + if (!PI.runBeforePass(Pass, *C)) + continue; + PreservedAnalyses PassPA = Pass.run(*C, CGAM, CG, UR); + if (UR.InvalidatedSCCs.count(C)) + PI.runAfterPassInvalidated(Pass); + else + PI.runAfterPass(Pass, *C); + // Update the SCC and RefSCC if necessary. C = UR.UpdatedC ? UR.UpdatedC : C; RC = UR.UpdatedRC ? UR.UpdatedRC : RC; @@ -615,12 +631,20 @@ public: if (CG.lookupSCC(*N) != CurrentC) continue; - PreservedAnalyses PassPA = Pass.run(N->getFunction(), FAM); + Function &F = N->getFunction(); + + PassInstrumentation PI = FAM.getResult(F); + if (!PI.runBeforePass(Pass, F)) + continue; + + PreservedAnalyses PassPA = Pass.run(F, FAM); + + PI.runAfterPass(Pass, F); // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. - FAM.invalidate(N->getFunction(), PassPA); + FAM.invalidate(F, PassPA); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -690,6 +714,8 @@ public: PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR) { PreservedAnalyses PA = PreservedAnalyses::all(); + PassInstrumentation PI = + AM.getResult(InitialC, CG); // The SCC may be refined while we are running passes over it, so set up // a pointer that we can update. @@ -733,8 +759,17 @@ public: auto CallCounts = ScanSCC(*C, CallHandles); for (int Iteration = 0;; ++Iteration) { + + if (!PI.runBeforePass(Pass, *C)) + continue; + PreservedAnalyses PassPA = Pass.run(*C, AM, CG, UR); + if (UR.InvalidatedSCCs.count(C)) + PI.runAfterPassInvalidated(Pass); + else + PI.runAfterPass(Pass, *C); + // If the SCC structure has changed, bail immediately and let the outer // CGSCC layer handle any iteration to reflect the refined structure. if (UR.UpdatedC && UR.UpdatedC != C) { diff --git a/contrib/llvm/include/llvm/Analysis/CaptureTracking.h b/contrib/llvm/include/llvm/Analysis/CaptureTracking.h index 7a869a51233..aaaaff9ae25 100644 --- a/contrib/llvm/include/llvm/Analysis/CaptureTracking.h +++ b/contrib/llvm/include/llvm/Analysis/CaptureTracking.h @@ -22,6 +22,14 @@ namespace llvm { class DominatorTree; class OrderedBasicBlock; + /// The default value for MaxUsesToExplore argument. It's relatively small to + /// keep the cost of analysis reasonable for clients like BasicAliasAnalysis, + /// where the results can't be cached. + /// TODO: we should probably introduce a caching CaptureTracking analysis and + /// use it where possible. The caching version can use much higher limit or + /// don't have this cap at all. + unsigned constexpr DefaultMaxUsesToExplore = 20; + /// PointerMayBeCaptured - Return true if this pointer value may be captured /// by the enclosing function (which is required to exist). This routine can /// be expensive, so consider caching the results. The boolean ReturnCaptures @@ -29,9 +37,12 @@ namespace llvm { /// counts as capturing it or not. The boolean StoreCaptures specified /// whether storing the value (or part of it) into memory anywhere /// automatically counts as capturing it or not. + /// MaxUsesToExplore specifies how many uses should the analysis explore for + /// one value before giving up due too "too many uses". bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, - bool StoreCaptures); + bool StoreCaptures, + unsigned MaxUsesToExplore = DefaultMaxUsesToExplore); /// PointerMayBeCapturedBefore - Return true if this pointer value may be /// captured by the enclosing function (which is required to exist). If a @@ -44,10 +55,13 @@ namespace llvm { /// or not. Captures by the provided instruction are considered if the /// final parameter is true. An ordered basic block in \p OBB could be used /// to speed up capture-tracker queries. + /// MaxUsesToExplore specifies how many uses should the analysis explore for + /// one value before giving up due too "too many uses". bool PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, bool StoreCaptures, const Instruction *I, const DominatorTree *DT, bool IncludeI = false, - OrderedBasicBlock *OBB = nullptr); + OrderedBasicBlock *OBB = nullptr, + unsigned MaxUsesToExplore = DefaultMaxUsesToExplore); /// This callback is used in conjunction with PointerMayBeCaptured. In /// addition to the interface here, you'll need to provide your own getters @@ -75,7 +89,10 @@ namespace llvm { /// PointerMayBeCaptured - Visit the value and the values derived from it and /// find values which appear to be capturing the pointer value. This feeds /// results into and is controlled by the CaptureTracker object. - void PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker); + /// MaxUsesToExplore specifies how many uses should the analysis explore for + /// one value before giving up due too "too many uses". + void PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker, + unsigned MaxUsesToExplore = DefaultMaxUsesToExplore); } // end namespace llvm #endif diff --git a/contrib/llvm/include/llvm/Analysis/CmpInstAnalysis.h b/contrib/llvm/include/llvm/Analysis/CmpInstAnalysis.h index 3cc69d9fea2..0e9c6a96b0f 100644 --- a/contrib/llvm/include/llvm/Analysis/CmpInstAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/CmpInstAnalysis.h @@ -46,19 +46,18 @@ namespace llvm { /// unsigned getICmpCode(const ICmpInst *ICI, bool InvertPred = false); - /// This is the complement of getICmpCode, which turns an opcode and two - /// operands into either a constant true or false, or the predicate for a new - /// ICmp instruction. The sign is passed in to determine which kind of - /// predicate to use in the new icmp instruction. + /// This is the complement of getICmpCode. It turns a predicate code into + /// either a constant true or false or the predicate for a new ICmp. + /// The sign is passed in to determine which kind of predicate to use in the + /// new ICmp instruction. /// Non-NULL return value will be a true or false constant. - /// NULL return means a new ICmp is needed. The predicate for which is output - /// in NewICmpPred. - Value *getICmpValue(bool Sign, unsigned Code, Value *LHS, Value *RHS, - CmpInst::Predicate &NewICmpPred); + /// NULL return means a new ICmp is needed. The predicate is output in Pred. + Constant *getPredForICmpCode(unsigned Code, bool Sign, Type *OpTy, + CmpInst::Predicate &Pred); /// Return true if both predicates match sign or if at least one of them is an /// equality comparison (which is signless). - bool PredicatesFoldable(CmpInst::Predicate p1, CmpInst::Predicate p2); + bool predicatesFoldable(CmpInst::Predicate P1, CmpInst::Predicate P2); /// Decompose an icmp into the form ((X & Mask) pred 0) if possible. The /// returned predicate is either == or !=. Returns false if decomposition diff --git a/contrib/llvm/include/llvm/Analysis/DemandedBits.h b/contrib/llvm/include/llvm/Analysis/DemandedBits.h index d4384609762..4c4e3f6c99e 100644 --- a/contrib/llvm/include/llvm/Analysis/DemandedBits.h +++ b/contrib/llvm/include/llvm/Analysis/DemandedBits.h @@ -44,19 +44,30 @@ public: F(F), AC(AC), DT(DT) {} /// Return the bits demanded from instruction I. + /// + /// For vector instructions individual vector elements are not distinguished: + /// A bit is demanded if it is demanded for any of the vector elements. The + /// size of the return value corresponds to the type size in bits of the + /// scalar type. + /// + /// Instructions that do not have integer or vector of integer type are + /// accepted, but will always produce a mask with all bits set. APInt getDemandedBits(Instruction *I); /// Return true if, during analysis, I could not be reached. bool isInstructionDead(Instruction *I); + /// Return whether this use is dead by means of not having any demanded bits. + bool isUseDead(Use *U); + void print(raw_ostream &OS); private: void performAnalysis(); void determineLiveOperandBits(const Instruction *UserI, - const Instruction *I, unsigned OperandNo, + const Value *Val, unsigned OperandNo, const APInt &AOut, APInt &AB, - KnownBits &Known, KnownBits &Known2); + KnownBits &Known, KnownBits &Known2, bool &KnownBitsComputed); Function &F; AssumptionCache &AC; @@ -67,6 +78,9 @@ private: // The set of visited instructions (non-integer-typed only). SmallPtrSet Visited; DenseMap AliveBits; + // Uses with no demanded bits. If the user also has no demanded bits, the use + // might not be stored explicitly in this map, to save memory during analysis. + SmallPtrSet DeadUses; }; class DemandedBitsWrapperPass : public FunctionPass { diff --git a/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h index c8ec737a2cb..69d0e2c1513 100644 --- a/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -936,6 +936,17 @@ template class ArrayRef; friend struct AnalysisInfoMixin; }; // class DependenceAnalysis + /// Printer pass to dump DA results. + struct DependenceAnalysisPrinterPass + : public PassInfoMixin { + DependenceAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + + private: + raw_ostream &OS; + }; // class DependenceAnalysisPrinterPass + /// Legacy pass manager pass to access dependence information class DependenceAnalysisWrapperPass : public FunctionPass { public: diff --git a/contrib/llvm/include/llvm/Analysis/DivergenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/DivergenceAnalysis.h index 328c8645d3c..d834862db09 100644 --- a/contrib/llvm/include/llvm/Analysis/DivergenceAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/DivergenceAnalysis.h @@ -7,55 +7,199 @@ // //===----------------------------------------------------------------------===// // -// The divergence analysis is an LLVM pass which can be used to find out -// if a branch instruction in a GPU program is divergent or not. It can help -// branch optimizations such as jump threading and loop unswitching to make -// better decisions. +// \file +// The divergence analysis determines which instructions and branches are +// divergent given a set of divergent source instructions. // //===----------------------------------------------------------------------===// + #ifndef LLVM_ANALYSIS_DIVERGENCE_ANALYSIS_H #define LLVM_ANALYSIS_DIVERGENCE_ANALYSIS_H #include "llvm/ADT/DenseSet.h" +#include "llvm/Analysis/SyncDependenceAnalysis.h" #include "llvm/IR/Function.h" #include "llvm/Pass.h" +#include namespace llvm { +class Module; class Value; -class DivergenceAnalysis : public FunctionPass { +class Instruction; +class Loop; +class raw_ostream; +class TargetTransformInfo; + +/// \brief Generic divergence analysis for reducible CFGs. +/// +/// This analysis propagates divergence in a data-parallel context from sources +/// of divergence to all users. It requires reducible CFGs. All assignments +/// should be in SSA form. +class DivergenceAnalysis { public: - static char ID; + /// \brief This instance will analyze the whole function \p F or the loop \p + /// RegionLoop. + /// + /// \param RegionLoop if non-null the analysis is restricted to \p RegionLoop. + /// Otherwise the whole function is analyzed. + /// \param IsLCSSAForm whether the analysis may assume that the IR in the + /// region in in LCSSA form. + DivergenceAnalysis(const Function &F, const Loop *RegionLoop, + const DominatorTree &DT, const LoopInfo &LI, + SyncDependenceAnalysis &SDA, bool IsLCSSAForm); - DivergenceAnalysis() : FunctionPass(ID) { - initializeDivergenceAnalysisPass(*PassRegistry::getPassRegistry()); - } + /// \brief The loop that defines the analyzed region (if any). + const Loop *getRegionLoop() const { return RegionLoop; } + const Function &getFunction() const { return F; } - void getAnalysisUsage(AnalysisUsage &AU) const override; + /// \brief Whether \p BB is part of the region. + bool inRegion(const BasicBlock &BB) const; + /// \brief Whether \p I is part of the region. + bool inRegion(const Instruction &I) const; - bool runOnFunction(Function &F) override; + /// \brief Mark \p UniVal as a value that is always uniform. + void addUniformOverride(const Value &UniVal); - // Print all divergent branches in the function. - void print(raw_ostream &OS, const Module *) const override; + /// \brief Mark \p DivVal as a value that is always divergent. + void markDivergent(const Value &DivVal); - // Returns true if V is divergent at its definition. - // - // Even if this function returns false, V may still be divergent when used - // in a different basic block. - bool isDivergent(const Value *V) const { return DivergentValues.count(V); } + /// \brief Propagate divergence to all instructions in the region. + /// Divergence is seeded by calls to \p markDivergent. + void compute(); - // Returns true if V is uniform/non-divergent. - // - // Even if this function returns true, V may still be divergent when used - // in a different basic block. - bool isUniform(const Value *V) const { return !isDivergent(V); } + /// \brief Whether any value was marked or analyzed to be divergent. + bool hasDetectedDivergence() const { return !DivergentValues.empty(); } - // Keep the analysis results uptodate by removing an erased value. - void removeValue(const Value *V) { DivergentValues.erase(V); } + /// \brief Whether \p Val will always return a uniform value regardless of its + /// operands + bool isAlwaysUniform(const Value &Val) const; + + /// \brief Whether \p Val is a divergent value + bool isDivergent(const Value &Val) const; + + void print(raw_ostream &OS, const Module *) const; private: - // Stores all divergent values. - DenseSet DivergentValues; -}; -} // End llvm namespace + bool updateTerminator(const Instruction &Term) const; + bool updatePHINode(const PHINode &Phi) const; -#endif //LLVM_ANALYSIS_DIVERGENCE_ANALYSIS_H \ No newline at end of file + /// \brief Computes whether \p Inst is divergent based on the + /// divergence of its operands. + /// + /// \returns Whether \p Inst is divergent. + /// + /// This should only be called for non-phi, non-terminator instructions. + bool updateNormalInstruction(const Instruction &Inst) const; + + /// \brief Mark users of live-out users as divergent. + /// + /// \param LoopHeader the header of the divergent loop. + /// + /// Marks all users of live-out values of the loop headed by \p LoopHeader + /// as divergent and puts them on the worklist. + void taintLoopLiveOuts(const BasicBlock &LoopHeader); + + /// \brief Push all users of \p Val (in the region) to the worklist + void pushUsers(const Value &I); + + /// \brief Push all phi nodes in @block to the worklist + void pushPHINodes(const BasicBlock &Block); + + /// \brief Mark \p Block as join divergent + /// + /// A block is join divergent if two threads may reach it from different + /// incoming blocks at the same time. + void markBlockJoinDivergent(const BasicBlock &Block) { + DivergentJoinBlocks.insert(&Block); + } + + /// \brief Whether \p Val is divergent when read in \p ObservingBlock. + bool isTemporalDivergent(const BasicBlock &ObservingBlock, + const Value &Val) const; + + /// \brief Whether \p Block is join divergent + /// + /// (see markBlockJoinDivergent). + bool isJoinDivergent(const BasicBlock &Block) const { + return DivergentJoinBlocks.find(&Block) != DivergentJoinBlocks.end(); + } + + /// \brief Propagate control-induced divergence to users (phi nodes and + /// instructions). + // + // \param JoinBlock is a divergent loop exit or join point of two disjoint + // paths. + // \returns Whether \p JoinBlock is a divergent loop exit of \p TermLoop. + bool propagateJoinDivergence(const BasicBlock &JoinBlock, + const Loop *TermLoop); + + /// \brief Propagate induced value divergence due to control divergence in \p + /// Term. + void propagateBranchDivergence(const Instruction &Term); + + /// \brief Propagate divergent caused by a divergent loop exit. + /// + /// \param ExitingLoop is a divergent loop. + void propagateLoopDivergence(const Loop &ExitingLoop); + +private: + const Function &F; + // If regionLoop != nullptr, analysis is only performed within \p RegionLoop. + // Otw, analyze the whole function + const Loop *RegionLoop; + + const DominatorTree &DT; + const LoopInfo &LI; + + // Recognized divergent loops + DenseSet DivergentLoops; + + // The SDA links divergent branches to divergent control-flow joins. + SyncDependenceAnalysis &SDA; + + // Use simplified code path for LCSSA form. + bool IsLCSSAForm; + + // Set of known-uniform values. + DenseSet UniformOverrides; + + // Blocks with joining divergent control from different predecessors. + DenseSet DivergentJoinBlocks; + + // Detected/marked divergent values. + DenseSet DivergentValues; + + // Internal worklist for divergence propagation. + std::vector Worklist; +}; + +/// \brief Divergence analysis frontend for GPU kernels. +class GPUDivergenceAnalysis { + SyncDependenceAnalysis SDA; + DivergenceAnalysis DA; + +public: + /// Runs the divergence analysis on @F, a GPU kernel + GPUDivergenceAnalysis(Function &F, const DominatorTree &DT, + const PostDominatorTree &PDT, const LoopInfo &LI, + const TargetTransformInfo &TTI); + + /// Whether any divergence was detected. + bool hasDivergence() const { return DA.hasDetectedDivergence(); } + + /// The GPU kernel this analysis result is for + const Function &getFunction() const { return DA.getFunction(); } + + /// Whether \p V is divergent. + bool isDivergent(const Value &V) const; + + /// Whether \p V is uniform/non-divergent + bool isUniform(const Value &V) const { return !isDivergent(V); } + + /// Print all divergent values in the kernel. + void print(raw_ostream &OS, const Module *) const; +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_DIVERGENCE_ANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/GlobalsModRef.h b/contrib/llvm/include/llvm/Analysis/GlobalsModRef.h index 09cef68ce70..3a664ca6ef5 100644 --- a/contrib/llvm/include/llvm/Analysis/GlobalsModRef.h +++ b/contrib/llvm/include/llvm/Analysis/GlobalsModRef.h @@ -88,7 +88,7 @@ public: AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); using AAResultBase::getModRefInfo; - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); /// getModRefBehavior - Return the behavior of the specified function if /// called from the specified call site. The call site may be null in which @@ -98,7 +98,7 @@ public: /// getModRefBehavior - Return the behavior of the specified function if /// called from the specified call site. The call site may be null in which /// case the most generic behavior of this function should be returned. - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + FunctionModRefBehavior getModRefBehavior(const CallBase *Call); private: FunctionInfo *getFunctionInfo(const Function *F); @@ -113,7 +113,7 @@ private: void CollectSCCMembership(CallGraph &CG); bool isNonEscapingGlobalNoAlias(const GlobalValue *GV, const Value *V); - ModRefInfo getModRefInfoForArgument(ImmutableCallSite CS, + ModRefInfo getModRefInfoForArgument(const CallBase *Call, const GlobalValue *GV); }; diff --git a/contrib/llvm/include/llvm/Analysis/GuardUtils.h b/contrib/llvm/include/llvm/Analysis/GuardUtils.h new file mode 100644 index 00000000000..3b151eeafc8 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/GuardUtils.h @@ -0,0 +1,26 @@ +//===-- GuardUtils.h - Utils for work with guards ---------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Utils that are used to perform analyzes related to guards and their +// conditions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_GUARDUTILS_H +#define LLVM_ANALYSIS_GUARDUTILS_H + +namespace llvm { + +class User; + +/// Returns true iff \p U has semantics of a guard. +bool isGuard(const User *U); + +} // llvm + +#endif // LLVM_ANALYSIS_GUARDUTILS_H + diff --git a/contrib/llvm/include/llvm/Analysis/IVDescriptors.h b/contrib/llvm/include/llvm/Analysis/IVDescriptors.h new file mode 100644 index 00000000000..64b4ae23cc5 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/IVDescriptors.h @@ -0,0 +1,357 @@ +//===- llvm/Analysis/IVDescriptors.h - IndVar Descriptors -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file "describes" induction and recurrence variables. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_IVDESCRIPTORS_H +#define LLVM_ANALYSIS_IVDESCRIPTORS_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/DemandedBits.h" +#include "llvm/Analysis/EHPersonalities.h" +#include "llvm/Analysis/MustExecute.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Casting.h" + +namespace llvm { + +class AliasSet; +class AliasSetTracker; +class BasicBlock; +class DataLayout; +class Loop; +class LoopInfo; +class OptimizationRemarkEmitter; +class PredicatedScalarEvolution; +class PredIteratorCache; +class ScalarEvolution; +class SCEV; +class TargetLibraryInfo; +class TargetTransformInfo; + +/// The RecurrenceDescriptor is used to identify recurrences variables in a +/// loop. Reduction is a special case of recurrence that has uses of the +/// recurrence variable outside the loop. The method isReductionPHI identifies +/// reductions that are basic recurrences. +/// +/// Basic recurrences are defined as the summation, product, OR, AND, XOR, min, +/// or max of a set of terms. For example: for(i=0; i &CI) + : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK), + UnsafeAlgebraInst(UAI), RecurrenceType(RT), IsSigned(Signed) { + CastInsts.insert(CI.begin(), CI.end()); + } + + /// This POD struct holds information about a potential recurrence operation. + class InstDesc { + public: + InstDesc(bool IsRecur, Instruction *I, Instruction *UAI = nullptr) + : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid), + UnsafeAlgebraInst(UAI) {} + + InstDesc(Instruction *I, MinMaxRecurrenceKind K, Instruction *UAI = nullptr) + : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K), + UnsafeAlgebraInst(UAI) {} + + bool isRecurrence() { return IsRecurrence; } + + bool hasUnsafeAlgebra() { return UnsafeAlgebraInst != nullptr; } + + Instruction *getUnsafeAlgebraInst() { return UnsafeAlgebraInst; } + + MinMaxRecurrenceKind getMinMaxKind() { return MinMaxKind; } + + Instruction *getPatternInst() { return PatternLastInst; } + + private: + // Is this instruction a recurrence candidate. + bool IsRecurrence; + // The last instruction in a min/max pattern (select of the select(icmp()) + // pattern), or the current recurrence instruction otherwise. + Instruction *PatternLastInst; + // If this is a min/max pattern the comparison predicate. + MinMaxRecurrenceKind MinMaxKind; + // Recurrence has unsafe algebra. + Instruction *UnsafeAlgebraInst; + }; + + /// Returns a struct describing if the instruction 'I' can be a recurrence + /// variable of type 'Kind'. If the recurrence is a min/max pattern of + /// select(icmp()) this function advances the instruction pointer 'I' from the + /// compare instruction to the select instruction and stores this pointer in + /// 'PatternLastInst' member of the returned struct. + static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, + InstDesc &Prev, bool HasFunNoNaNAttr); + + /// Returns true if instruction I has multiple uses in Insts + static bool hasMultipleUsesOf(Instruction *I, + SmallPtrSetImpl &Insts, + unsigned MaxNumUses); + + /// Returns true if all uses of the instruction I is within the Set. + static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl &Set); + + /// Returns a struct describing if the instruction if the instruction is a + /// Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y) + /// or max(X, Y). + static InstDesc isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev); + + /// Returns a struct describing if the instruction is a + /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern. + static InstDesc isConditionalRdxPattern(RecurrenceKind Kind, Instruction *I); + + /// Returns identity corresponding to the RecurrenceKind. + static Constant *getRecurrenceIdentity(RecurrenceKind K, Type *Tp); + + /// Returns the opcode of binary operation corresponding to the + /// RecurrenceKind. + static unsigned getRecurrenceBinOp(RecurrenceKind Kind); + + /// Returns true if Phi is a reduction of type Kind and adds it to the + /// RecurrenceDescriptor. If either \p DB is non-null or \p AC and \p DT are + /// non-null, the minimal bit width needed to compute the reduction will be + /// computed. + static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop, + bool HasFunNoNaNAttr, + RecurrenceDescriptor &RedDes, + DemandedBits *DB = nullptr, + AssumptionCache *AC = nullptr, + DominatorTree *DT = nullptr); + + /// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor + /// is returned in RedDes. If either \p DB is non-null or \p AC and \p DT are + /// non-null, the minimal bit width needed to compute the reduction will be + /// computed. + static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, + RecurrenceDescriptor &RedDes, + DemandedBits *DB = nullptr, + AssumptionCache *AC = nullptr, + DominatorTree *DT = nullptr); + + /// Returns true if Phi is a first-order recurrence. A first-order recurrence + /// is a non-reduction recurrence relation in which the value of the + /// recurrence in the current loop iteration equals a value defined in the + /// previous iteration. \p SinkAfter includes pairs of instructions where the + /// first will be rescheduled to appear after the second if/when the loop is + /// vectorized. It may be augmented with additional pairs if needed in order + /// to handle Phi as a first-order recurrence. + static bool + isFirstOrderRecurrence(PHINode *Phi, Loop *TheLoop, + DenseMap &SinkAfter, + DominatorTree *DT); + + RecurrenceKind getRecurrenceKind() { return Kind; } + + MinMaxRecurrenceKind getMinMaxRecurrenceKind() { return MinMaxKind; } + + TrackingVH getRecurrenceStartValue() { return StartValue; } + + Instruction *getLoopExitInstr() { return LoopExitInstr; } + + /// Returns true if the recurrence has unsafe algebra which requires a relaxed + /// floating-point model. + bool hasUnsafeAlgebra() { return UnsafeAlgebraInst != nullptr; } + + /// Returns first unsafe algebra instruction in the PHI node's use-chain. + Instruction *getUnsafeAlgebraInst() { return UnsafeAlgebraInst; } + + /// Returns true if the recurrence kind is an integer kind. + static bool isIntegerRecurrenceKind(RecurrenceKind Kind); + + /// Returns true if the recurrence kind is a floating point kind. + static bool isFloatingPointRecurrenceKind(RecurrenceKind Kind); + + /// Returns true if the recurrence kind is an arithmetic kind. + static bool isArithmeticRecurrenceKind(RecurrenceKind Kind); + + /// Returns the type of the recurrence. This type can be narrower than the + /// actual type of the Phi if the recurrence has been type-promoted. + Type *getRecurrenceType() { return RecurrenceType; } + + /// Returns a reference to the instructions used for type-promoting the + /// recurrence. + SmallPtrSet &getCastInsts() { return CastInsts; } + + /// Returns true if all source operands of the recurrence are SExtInsts. + bool isSigned() { return IsSigned; } + +private: + // The starting value of the recurrence. + // It does not have to be zero! + TrackingVH StartValue; + // The instruction who's value is used outside the loop. + Instruction *LoopExitInstr = nullptr; + // The kind of the recurrence. + RecurrenceKind Kind = RK_NoRecurrence; + // If this a min/max recurrence the kind of recurrence. + MinMaxRecurrenceKind MinMaxKind = MRK_Invalid; + // First occurrence of unasfe algebra in the PHI's use-chain. + Instruction *UnsafeAlgebraInst = nullptr; + // The type of the recurrence. + Type *RecurrenceType = nullptr; + // True if all source operands of the recurrence are SExtInsts. + bool IsSigned = false; + // Instructions used for type-promoting the recurrence. + SmallPtrSet CastInsts; +}; + +/// A struct for saving information about induction variables. +class InductionDescriptor { +public: + /// This enum represents the kinds of inductions that we support. + enum InductionKind { + IK_NoInduction, ///< Not an induction variable. + IK_IntInduction, ///< Integer induction variable. Step = C. + IK_PtrInduction, ///< Pointer induction var. Step = C / sizeof(elem). + IK_FpInduction ///< Floating point induction variable. + }; + +public: + /// Default constructor - creates an invalid induction. + InductionDescriptor() = default; + + /// Get the consecutive direction. Returns: + /// 0 - unknown or non-consecutive. + /// 1 - consecutive and increasing. + /// -1 - consecutive and decreasing. + int getConsecutiveDirection() const; + + Value *getStartValue() const { return StartValue; } + InductionKind getKind() const { return IK; } + const SCEV *getStep() const { return Step; } + BinaryOperator *getInductionBinOp() const { return InductionBinOp; } + ConstantInt *getConstIntStepValue() const; + + /// Returns true if \p Phi is an induction in the loop \p L. If \p Phi is an + /// induction, the induction descriptor \p D will contain the data describing + /// this induction. If by some other means the caller has a better SCEV + /// expression for \p Phi than the one returned by the ScalarEvolution + /// analysis, it can be passed through \p Expr. If the def-use chain + /// associated with the phi includes casts (that we know we can ignore + /// under proper runtime checks), they are passed through \p CastsToIgnore. + static bool + isInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, + InductionDescriptor &D, const SCEV *Expr = nullptr, + SmallVectorImpl *CastsToIgnore = nullptr); + + /// Returns true if \p Phi is a floating point induction in the loop \p L. + /// If \p Phi is an induction, the induction descriptor \p D will contain + /// the data describing this induction. + static bool isFPInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, + InductionDescriptor &D); + + /// Returns true if \p Phi is a loop \p L induction, in the context associated + /// with the run-time predicate of PSE. If \p Assume is true, this can add + /// further SCEV predicates to \p PSE in order to prove that \p Phi is an + /// induction. + /// If \p Phi is an induction, \p D will contain the data describing this + /// induction. + static bool isInductionPHI(PHINode *Phi, const Loop *L, + PredicatedScalarEvolution &PSE, + InductionDescriptor &D, bool Assume = false); + + /// Returns true if the induction type is FP and the binary operator does + /// not have the "fast-math" property. Such operation requires a relaxed FP + /// mode. + bool hasUnsafeAlgebra() { + return InductionBinOp && !cast(InductionBinOp)->isFast(); + } + + /// Returns induction operator that does not have "fast-math" property + /// and requires FP unsafe mode. + Instruction *getUnsafeAlgebraInst() { + if (!InductionBinOp || cast(InductionBinOp)->isFast()) + return nullptr; + return InductionBinOp; + } + + /// Returns binary opcode of the induction operator. + Instruction::BinaryOps getInductionOpcode() const { + return InductionBinOp ? InductionBinOp->getOpcode() + : Instruction::BinaryOpsEnd; + } + + /// Returns a reference to the type cast instructions in the induction + /// update chain, that are redundant when guarded with a runtime + /// SCEV overflow check. + const SmallVectorImpl &getCastInsts() const { + return RedundantCasts; + } + +private: + /// Private constructor - used by \c isInductionPHI. + InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, + BinaryOperator *InductionBinOp = nullptr, + SmallVectorImpl *Casts = nullptr); + + /// Start value. + TrackingVH StartValue; + /// Induction kind. + InductionKind IK = IK_NoInduction; + /// Step value. + const SCEV *Step = nullptr; + // Instruction that advances induction variable. + BinaryOperator *InductionBinOp = nullptr; + // Instructions used for type-casts of the induction variable, + // that are redundant when guarded with a runtime SCEV overflow check. + SmallVector RedundantCasts; +}; + +} // end namespace llvm + +#endif // LLVM_ANALYSIS_IVDESCRIPTORS_H diff --git a/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h b/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h deleted file mode 100644 index dde56a143c5..00000000000 --- a/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h +++ /dev/null @@ -1,35 +0,0 @@ -//===-- IndirectCallSiteVisitor.h - indirect call-sites visitor -----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements defines a visitor class and a helper function that find -// all indirect call-sites in a function. - -#include "llvm/IR/InstVisitor.h" -#include - -namespace llvm { -// Visitor class that finds all indirect call sites. -struct PGOIndirectCallSiteVisitor - : public InstVisitor { - std::vector IndirectCallInsts; - PGOIndirectCallSiteVisitor() {} - - void visitCallSite(CallSite CS) { - if (CS.isIndirectCall()) - IndirectCallInsts.push_back(CS.getInstruction()); - } -}; - -// Helper function that finds all indirect call sites. -inline std::vector findIndirectCallSites(Function &F) { - PGOIndirectCallSiteVisitor ICV; - ICV.visit(F); - return ICV.IndirectCallInsts; -} -} diff --git a/contrib/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/contrib/llvm/include/llvm/Analysis/IndirectCallVisitor.h new file mode 100644 index 00000000000..d00cf63368f --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/IndirectCallVisitor.h @@ -0,0 +1,39 @@ +//===-- IndirectCallVisitor.h - indirect call visitor ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements defines a visitor class and a helper function that find +// all indirect call-sites in a function. + +#ifndef LLVM_ANALYSIS_INDIRECTCALLVISITOR_H +#define LLVM_ANALYSIS_INDIRECTCALLVISITOR_H + +#include "llvm/IR/InstVisitor.h" +#include + +namespace llvm { +// Visitor class that finds all indirect call. +struct PGOIndirectCallVisitor : public InstVisitor { + std::vector IndirectCalls; + PGOIndirectCallVisitor() {} + + void visitCallBase(CallBase &Call) { + if (Call.isIndirectCall()) + IndirectCalls.push_back(&Call); + } +}; + +// Helper function that finds all indirect call sites. +inline std::vector findIndirectCalls(Function &F) { + PGOIndirectCallVisitor ICV; + ICV.visit(F); + return ICV.IndirectCalls; +} +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/Analysis/InlineCost.h b/contrib/llvm/include/llvm/Analysis/InlineCost.h index 8c412057fb8..4c270354b0c 100644 --- a/contrib/llvm/include/llvm/Analysis/InlineCost.h +++ b/contrib/llvm/include/llvm/Analysis/InlineCost.h @@ -46,7 +46,6 @@ const int IndirectCallThreshold = 100; const int CallPenalty = 25; const int LastCallToStaticBonus = 15000; const int ColdccPenalty = 2000; -const int NoreturnPenalty = 10000; /// Do not inline functions which allocate this many bytes on the stack /// when the caller is recursive. const unsigned TotalAllocaSizeRecursiveCaller = 1024; @@ -74,8 +73,15 @@ class InlineCost { /// The adjusted threshold against which this cost was computed. const int Threshold; + /// Must be set for Always and Never instances. + const char *Reason = nullptr; + // Trivial constructor, interesting logic in the factory functions below. - InlineCost(int Cost, int Threshold) : Cost(Cost), Threshold(Threshold) {} + InlineCost(int Cost, int Threshold, const char *Reason = nullptr) + : Cost(Cost), Threshold(Threshold), Reason(Reason) { + assert((isVariable() || Reason) && + "Reason must be provided for Never or Always"); + } public: static InlineCost get(int Cost, int Threshold) { @@ -83,11 +89,11 @@ public: assert(Cost < NeverInlineCost && "Cost crosses sentinel value"); return InlineCost(Cost, Threshold); } - static InlineCost getAlways() { - return InlineCost(AlwaysInlineCost, 0); + static InlineCost getAlways(const char *Reason) { + return InlineCost(AlwaysInlineCost, 0, Reason); } - static InlineCost getNever() { - return InlineCost(NeverInlineCost, 0); + static InlineCost getNever(const char *Reason) { + return InlineCost(NeverInlineCost, 0, Reason); } /// Test whether the inline cost is low enough for inlining. @@ -112,12 +118,30 @@ public: return Threshold; } + /// Get the reason of Always or Never. + const char *getReason() const { + assert((Reason || isVariable()) && + "InlineCost reason must be set for Always or Never"); + return Reason; + } + /// Get the cost delta from the threshold for inlining. /// Only valid if the cost is of the variable kind. Returns a negative /// value if the cost is too high to inline. int getCostDelta() const { return Threshold - getCost(); } }; +/// InlineResult is basically true or false. For false results the message +/// describes a reason why it is decided not to inline. +struct InlineResult { + const char *message = nullptr; + InlineResult(bool result, const char *message = nullptr) + : message(result ? nullptr : (message ? message : "cost > threshold")) {} + InlineResult(const char *message = nullptr) : message(message) {} + operator bool() const { return !message; } + operator const char *() const { return message; } +}; + /// Thresholds to tune inline cost analysis. The inline cost analysis decides /// the condition to apply a threshold and applies it. Otherwise, /// DefaultThreshold is used. If a threshold is Optional, it is applied only diff --git a/contrib/llvm/include/llvm/Analysis/InstructionPrecedenceTracking.h b/contrib/llvm/include/llvm/Analysis/InstructionPrecedenceTracking.h new file mode 100644 index 00000000000..073e6ec3b7f --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/InstructionPrecedenceTracking.h @@ -0,0 +1,150 @@ +//===-- InstructionPrecedenceTracking.h -------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Implements a class that is able to define some instructions as "special" +// (e.g. as having implicit control flow, or writing memory, or having another +// interesting property) and then efficiently answers queries of the types: +// 1. Are there any special instructions in the block of interest? +// 2. Return first of the special instructions in the given block; +// 3. Check if the given instruction is preceeded by the first special +// instruction in the same block. +// The class provides caching that allows to answer these queries quickly. The +// user must make sure that the cached data is invalidated properly whenever +// a content of some tracked block is changed. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_INSTRUCTIONPRECEDENCETRACKING_H +#define LLVM_ANALYSIS_INSTRUCTIONPRECEDENCETRACKING_H + +#include "llvm/IR/Dominators.h" +#include "llvm/Analysis/OrderedInstructions.h" + +namespace llvm { + +class InstructionPrecedenceTracking { + // Maps a block to the topmost special instruction in it. If the value is + // nullptr, it means that it is known that this block does not contain any + // special instructions. + DenseMap FirstSpecialInsts; + // Allows to answer queries about precedence of instructions within one block. + OrderedInstructions OI; + + // Fills information about the given block's special instructions. + void fill(const BasicBlock *BB); + +#ifndef NDEBUG + /// Asserts that the cached info for \p BB is up-to-date. This helps to catch + /// the usage error of accessing a block without properly invalidating after a + /// previous transform. + void validate(const BasicBlock *BB) const; + + /// Asserts whether or not the contents of this tracking is up-to-date. This + /// helps to catch the usage error of accessing a block without properly + /// invalidating after a previous transform. + void validateAll() const; +#endif + +protected: + InstructionPrecedenceTracking(DominatorTree *DT) + : OI(OrderedInstructions(DT)) {} + + /// Returns the topmost special instruction from the block \p BB. Returns + /// nullptr if there is no special instructions in the block. + const Instruction *getFirstSpecialInstruction(const BasicBlock *BB); + + /// Returns true iff at least one instruction from the basic block \p BB is + /// special. + bool hasSpecialInstructions(const BasicBlock *BB); + + /// Returns true iff the first special instruction of \p Insn's block exists + /// and dominates \p Insn. + bool isPreceededBySpecialInstruction(const Instruction *Insn); + + /// A predicate that defines whether or not the instruction \p Insn is + /// considered special and needs to be tracked. Implementing this method in + /// children classes allows to implement tracking of implicit control flow, + /// memory writing instructions or any other kinds of instructions we might + /// be interested in. + virtual bool isSpecialInstruction(const Instruction *Insn) const = 0; + + virtual ~InstructionPrecedenceTracking() = default; + +public: + /// Notifies this tracking that we are going to insert a new instruction \p + /// Inst to the basic block \p BB. It makes all necessary updates to internal + /// caches to keep them consistent. + void insertInstructionTo(const Instruction *Inst, const BasicBlock *BB); + + /// Notifies this tracking that we are going to remove the instruction \p Inst + /// It makes all necessary updates to internal caches to keep them consistent. + void removeInstruction(const Instruction *Inst); + + /// Invalidates all information from this tracking. + void clear(); +}; + +/// This class allows to keep track on instructions with implicit control flow. +/// These are instructions that may not pass execution to their successors. For +/// example, throwing calls and guards do not always do this. If we need to know +/// for sure that some instruction is guaranteed to execute if the given block +/// is reached, then we need to make sure that there is no implicit control flow +/// instruction (ICFI) preceeding it. For example, this check is required if we +/// perform PRE moving non-speculable instruction to other place. +class ImplicitControlFlowTracking : public InstructionPrecedenceTracking { +public: + ImplicitControlFlowTracking(DominatorTree *DT) + : InstructionPrecedenceTracking(DT) {} + + /// Returns the topmost instruction with implicit control flow from the given + /// basic block. Returns nullptr if there is no such instructions in the block. + const Instruction *getFirstICFI(const BasicBlock *BB) { + return getFirstSpecialInstruction(BB); + } + + /// Returns true if at least one instruction from the given basic block has + /// implicit control flow. + bool hasICF(const BasicBlock *BB) { + return hasSpecialInstructions(BB); + } + + /// Returns true if the first ICFI of Insn's block exists and dominates Insn. + bool isDominatedByICFIFromSameBlock(const Instruction *Insn) { + return isPreceededBySpecialInstruction(Insn); + } + + virtual bool isSpecialInstruction(const Instruction *Insn) const; +}; + +class MemoryWriteTracking : public InstructionPrecedenceTracking { +public: + MemoryWriteTracking(DominatorTree *DT) : InstructionPrecedenceTracking(DT) {} + + /// Returns the topmost instruction that may write memory from the given + /// basic block. Returns nullptr if there is no such instructions in the block. + const Instruction *getFirstMemoryWrite(const BasicBlock *BB) { + return getFirstSpecialInstruction(BB); + } + + /// Returns true if at least one instruction from the given basic block may + /// write memory. + bool mayWriteToMemory(const BasicBlock *BB) { + return hasSpecialInstructions(BB); + } + + /// Returns true if the first memory writing instruction of Insn's block + /// exists and dominates Insn. + bool isDominatedByMemoryWriteFromSameBlock(const Instruction *Insn) { + return isPreceededBySpecialInstruction(Insn); + } + + virtual bool isSpecialInstruction(const Instruction *Insn) const; +}; + +} // llvm + +#endif // LLVM_ANALYSIS_INSTRUCTIONPRECEDENCETRACKING_H diff --git a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h index 4f896bddff8..6662e91037e 100644 --- a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -32,6 +32,8 @@ #ifndef LLVM_ANALYSIS_INSTRUCTIONSIMPLIFY_H #define LLVM_ANALYSIS_INSTRUCTIONSIMPLIFY_H +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Operator.h" #include "llvm/IR/User.h" namespace llvm { @@ -40,7 +42,6 @@ template class AnalysisManager; template class ArrayRef; class AssumptionCache; class DominatorTree; -class Instruction; class ImmutableCallSite; class DataLayout; class FastMathFlags; @@ -50,6 +51,41 @@ class Pass; class TargetLibraryInfo; class Type; class Value; +class MDNode; +class BinaryOperator; + +/// InstrInfoQuery provides an interface to query additional information for +/// instructions like metadata or keywords like nsw, which provides conservative +/// results if the users specified it is safe to use. +struct InstrInfoQuery { + InstrInfoQuery(bool UMD) : UseInstrInfo(UMD) {} + InstrInfoQuery() : UseInstrInfo(true) {} + bool UseInstrInfo = true; + + MDNode *getMetadata(const Instruction *I, unsigned KindID) const { + if (UseInstrInfo) + return I->getMetadata(KindID); + return nullptr; + } + + template bool hasNoUnsignedWrap(const InstT *Op) const { + if (UseInstrInfo) + return Op->hasNoUnsignedWrap(); + return false; + } + + template bool hasNoSignedWrap(const InstT *Op) const { + if (UseInstrInfo) + return Op->hasNoSignedWrap(); + return false; + } + + bool isExact(const BinaryOperator *Op) const { + if (UseInstrInfo && isa(Op)) + return cast(Op)->isExact(); + return false; + } +}; struct SimplifyQuery { const DataLayout &DL; @@ -58,14 +94,19 @@ struct SimplifyQuery { AssumptionCache *AC = nullptr; const Instruction *CxtI = nullptr; + // Wrapper to query additional information for instructions like metadata or + // keywords like nsw, which provides conservative results if those cannot + // be safely used. + const InstrInfoQuery IIQ; + SimplifyQuery(const DataLayout &DL, const Instruction *CXTI = nullptr) : DL(DL), CxtI(CXTI) {} SimplifyQuery(const DataLayout &DL, const TargetLibraryInfo *TLI, const DominatorTree *DT = nullptr, AssumptionCache *AC = nullptr, - const Instruction *CXTI = nullptr) - : DL(DL), TLI(TLI), DT(DT), AC(AC), CxtI(CXTI) {} + const Instruction *CXTI = nullptr, bool UseInstrInfo = true) + : DL(DL), TLI(TLI), DT(DT), AC(AC), CxtI(CXTI), IIQ(UseInstrInfo) {} SimplifyQuery getWithInstruction(Instruction *I) const { SimplifyQuery Copy(*this); Copy.CxtI = I; diff --git a/contrib/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h b/contrib/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h index 6b195073324..3083db75b81 100644 --- a/contrib/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h +++ b/contrib/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h @@ -6,7 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// +/// \file /// Compute iterated dominance frontiers using a linear time algorithm. /// /// The algorithm used here is based on: @@ -28,6 +28,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFGDiff.h" #include "llvm/IR/Dominators.h" namespace llvm { @@ -45,17 +46,21 @@ namespace llvm { template class IDFCalculator { public: - IDFCalculator(DominatorTreeBase &DT) - : DT(DT), useLiveIn(false) {} + IDFCalculator(DominatorTreeBase &DT) + : DT(DT), GD(nullptr), useLiveIn(false) {} - /// Give the IDF calculator the set of blocks in which the value is - /// defined. This is equivalent to the set of starting blocks it should be - /// calculating the IDF for (though later gets pruned based on liveness). - /// - /// Note: This set *must* live for the entire lifetime of the IDF calculator. - void setDefiningBlocks(const SmallPtrSetImpl &Blocks) { - DefBlocks = &Blocks; - } + IDFCalculator(DominatorTreeBase &DT, + const GraphDiff *GD) + : DT(DT), GD(GD), useLiveIn(false) {} + + /// Give the IDF calculator the set of blocks in which the value is + /// defined. This is equivalent to the set of starting blocks it should be + /// calculating the IDF for (though later gets pruned based on liveness). + /// + /// Note: This set *must* live for the entire lifetime of the IDF calculator. + void setDefiningBlocks(const SmallPtrSetImpl &Blocks) { + DefBlocks = &Blocks; + } /// Give the IDF calculator the set of blocks in which the value is /// live on entry to the block. This is used to prune the IDF calculation to @@ -85,6 +90,7 @@ class IDFCalculator { private: DominatorTreeBase &DT; + const GraphDiff *GD; bool useLiveIn; const SmallPtrSetImpl *LiveInBlocks; const SmallPtrSetImpl *DefBlocks; diff --git a/contrib/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h new file mode 100644 index 00000000000..fc426ad7fb6 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h @@ -0,0 +1,69 @@ +//===- llvm/Analysis/LegacyDivergenceAnalysis.h - KernelDivergence Analysis -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// The kernel divergence analysis is an LLVM pass which can be used to find out +// if a branch instruction in a GPU program (kernel) is divergent or not. It can help +// branch optimizations such as jump threading and loop unswitching to make +// better decisions. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_ANALYSIS_LEGACY_DIVERGENCE_ANALYSIS_H +#define LLVM_ANALYSIS_LEGACY_DIVERGENCE_ANALYSIS_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/IR/Function.h" +#include "llvm/Pass.h" +#include "llvm/Analysis/DivergenceAnalysis.h" + +namespace llvm { +class Value; +class GPUDivergenceAnalysis; +class LegacyDivergenceAnalysis : public FunctionPass { +public: + static char ID; + + LegacyDivergenceAnalysis() : FunctionPass(ID) { + initializeLegacyDivergenceAnalysisPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnFunction(Function &F) override; + + // Print all divergent branches in the function. + void print(raw_ostream &OS, const Module *) const override; + + // Returns true if V is divergent at its definition. + // + // Even if this function returns false, V may still be divergent when used + // in a different basic block. + bool isDivergent(const Value *V) const; + + // Returns true if V is uniform/non-divergent. + // + // Even if this function returns true, V may still be divergent when used + // in a different basic block. + bool isUniform(const Value *V) const { return !isDivergent(V); } + + // Keep the analysis results uptodate by removing an erased value. + void removeValue(const Value *V) { DivergentValues.erase(V); } + +private: + // Whether analysis should be performed by GPUDivergenceAnalysis. + bool shouldUseGPUDivergenceAnalysis(const Function &F) const; + + // (optional) handle to new DivergenceAnalysis + std::unique_ptr gpuDA; + + // Stores all divergent values. + DenseSet DivergentValues; +}; +} // End llvm namespace + +#endif //LLVM_ANALYSIS_LEGACY_DIVERGENCE_ANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index d27b3e42bbe..4ed00e20775 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -97,6 +97,19 @@ public: /// Set of potential dependent memory accesses. typedef EquivalenceClasses DepCandidates; + /// Type to keep track of the status of the dependence check. The order of + /// the elements is important and has to be from most permissive to least + /// permissive. + enum class VectorizationSafetyStatus { + // Can vectorize safely without RT checks. All dependences are known to be + // safe. + Safe, + // Can possibly vectorize with RT checks to overcome unknown dependencies. + PossiblySafeWithRtChecks, + // Cannot vectorize due to known unsafe dependencies. + Unsafe, + }; + /// Dependece between memory access instructions. struct Dependence { /// The type of the dependence. @@ -146,7 +159,7 @@ public: Instruction *getDestination(const LoopAccessInfo &LAI) const; /// Dependence types that don't prevent vectorization. - static bool isSafeForVectorization(DepType Type); + static VectorizationSafetyStatus isSafeForVectorization(DepType Type); /// Lexically forward dependence. bool isForward() const; @@ -164,8 +177,8 @@ public: MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeRegisterWidth(-1U), - ShouldRetryWithRuntimeCheck(false), SafeForVectorization(true), - RecordDependences(true) {} + FoundNonConstantDistanceDependence(false), + Status(VectorizationSafetyStatus::Safe), RecordDependences(true) {} /// Register the location (instructions are given increasing numbers) /// of a write access. @@ -193,7 +206,9 @@ public: /// No memory dependence was encountered that would inhibit /// vectorization. - bool isSafeForVectorization() const { return SafeForVectorization; } + bool isSafeForVectorization() const { + return Status == VectorizationSafetyStatus::Safe; + } /// The maximum number of bytes of a vector register we can vectorize /// the accesses safely with. @@ -205,7 +220,10 @@ public: /// In same cases when the dependency check fails we can still /// vectorize the loop with a dynamic array access check. - bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; } + bool shouldRetryWithRuntimeCheck() const { + return FoundNonConstantDistanceDependence && + Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks; + } /// Returns the memory dependences. If null is returned we exceeded /// the MaxDependences threshold and this information is not @@ -267,11 +285,12 @@ private: /// If we see a non-constant dependence distance we can still try to /// vectorize this loop with runtime checks. - bool ShouldRetryWithRuntimeCheck; + bool FoundNonConstantDistanceDependence; - /// No memory dependence was encountered that would inhibit - /// vectorization. - bool SafeForVectorization; + /// Result of the dependence checks, indicating whether the checked + /// dependences are safe for vectorization, require RT checks or are known to + /// be unsafe. + VectorizationSafetyStatus Status; //// True if Dependences reflects the dependences in the //// loop. If false we exceeded MaxDependences and @@ -304,6 +323,11 @@ private: /// \return false if we shouldn't vectorize at all or avoid larger /// vectorization factors by limiting MaxSafeDepDistBytes. bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize); + + /// Updates the current safety status with \p S. We can go from Safe to + /// either PossiblySafeWithRtChecks or Unsafe and from + /// PossiblySafeWithRtChecks to Unsafe. + void mergeInStatus(VectorizationSafetyStatus S); }; /// Holds information about the memory runtime legality checks to verify @@ -564,11 +588,10 @@ public: /// Print the information about the memory accesses in the loop. void print(raw_ostream &OS, unsigned Depth = 0) const; - /// Checks existence of store to invariant address inside loop. - /// If the loop has any store to invariant address, then it returns true, - /// else returns false. - bool hasStoreToLoopInvariantAddress() const { - return StoreToLoopInvariantAddress; + /// If the loop has memory dependence involving an invariant address, i.e. two + /// stores or a store and a load, then return true, else return false. + bool hasDependenceInvolvingLoopInvariantAddress() const { + return HasDependenceInvolvingLoopInvariantAddress; } /// Used to add runtime SCEV checks. Simplifies SCEV expressions and converts @@ -621,9 +644,8 @@ private: /// Cache the result of analyzeLoop. bool CanVecMem; - /// Indicator for storing to uniform addresses. - /// If a loop has write to a loop invariant address then it should be true. - bool StoreToLoopInvariantAddress; + /// Indicator that there are non vectorizable stores to a uniform address. + bool HasDependenceInvolvingLoopInvariantAddress; /// The diagnostics report generated for the analysis. E.g. why we /// couldn't analyze the loop. diff --git a/contrib/llvm/include/llvm/Analysis/LoopInfo.h b/contrib/llvm/include/llvm/Analysis/LoopInfo.h index 30b29d66a1d..72873546a06 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopInfo.h +++ b/contrib/llvm/include/llvm/Analysis/LoopInfo.h @@ -408,6 +408,12 @@ public: /// Verify loop structure of this loop and all nested loops. void verifyLoopNest(DenseSet *Loops) const; + /// Returns true if the loop is annotated parallel. + /// + /// Derived classes can override this method using static template + /// polymorphism. + bool isAnnotatedParallel() const { return false; } + /// Print loop with all the BBs inside it. void print(raw_ostream &OS, unsigned Depth = 0, bool Verbose = false) const; @@ -989,6 +995,26 @@ public: /// Function to print a loop's contents as LLVM's text IR assembly. void printLoop(Loop &L, raw_ostream &OS, const std::string &Banner = ""); +/// Find and return the loop attribute node for the attribute @p Name in +/// @p LoopID. Return nullptr if there is no such attribute. +MDNode *findOptionMDForLoopID(MDNode *LoopID, StringRef Name); + +/// Find string metadata for a loop. +/// +/// Returns the MDNode where the first operand is the metadata's name. The +/// following operands are the metadata's values. If no metadata with @p Name is +/// found, return nullptr. +MDNode *findOptionMDForLoop(const Loop *TheLoop, StringRef Name); + +/// Return whether an MDNode might represent an access group. +/// +/// Access group metadata nodes have to be distinct and empty. Being +/// always-empty ensures that it never needs to be changed (which -- because +/// MDNodes are designed immutable -- would require creating a new MDNode). Note +/// that this is not a sufficient condition: not every distinct and empty NDNode +/// is representing an access group. +bool isValidAsAccessGroup(MDNode *AccGroup); + } // End llvm namespace #endif diff --git a/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h b/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h index 94138985886..2b807919fed 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h @@ -392,7 +392,10 @@ void LoopBase::verifyLoopNest( template void LoopBase::print(raw_ostream &OS, unsigned Depth, bool Verbose) const { - OS.indent(Depth * 2) << "Loop at depth " << getLoopDepth() << " containing: "; + OS.indent(Depth * 2); + if (static_cast(this)->isAnnotatedParallel()) + OS << "Parallel "; + OS << "Loop at depth " << getLoopDepth() << " containing: "; BlockT *H = getHeader(); for (unsigned i = 0; i < getBlocks().size(); ++i) { @@ -640,8 +643,8 @@ void LoopInfoBase::print(raw_ostream &OS) const { template bool compareVectors(std::vector &BB1, std::vector &BB2) { - llvm::sort(BB1.begin(), BB1.end()); - llvm::sort(BB2.begin(), BB2.end()); + llvm::sort(BB1); + llvm::sort(BB2); return BB1 == BB2; } diff --git a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h index 1c40cffc7f6..958d4fe4b83 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -37,7 +37,6 @@ namespace llvm { class AssumptionCache; -class CallSite; class DominatorTree; class Function; class Instruction; @@ -304,7 +303,7 @@ private: /// The maximum size of the dereferences of the pointer. /// /// May be UnknownSize if the sizes are unknown. - LocationSize Size = MemoryLocation::UnknownSize; + LocationSize Size = LocationSize::unknown(); /// The AA tags associated with dereferences of the pointer. /// /// The members may be null if there are no tags or conflicting tags. @@ -398,7 +397,7 @@ public: /// invalidated on the next non-local query or when an instruction is /// removed. Clients must copy this data if they want it around longer than /// that. - const NonLocalDepInfo &getNonLocalCallDependency(CallSite QueryCS); + const NonLocalDepInfo &getNonLocalCallDependency(CallBase *QueryCall); /// Perform a full dependency query for an access to the QueryInst's /// specified memory location, returning the set of instructions that either @@ -482,9 +481,9 @@ public: void releaseMemory(); private: - MemDepResult getCallSiteDependencyFrom(CallSite C, bool isReadOnlyCall, - BasicBlock::iterator ScanIt, - BasicBlock *BB); + MemDepResult getCallDependencyFrom(CallBase *Call, bool isReadOnlyCall, + BasicBlock::iterator ScanIt, + BasicBlock *BB); bool getNonLocalPointerDepFromBB(Instruction *QueryInst, const PHITransAddr &Pointer, const MemoryLocation &Loc, bool isLoad, diff --git a/contrib/llvm/include/llvm/Analysis/MemoryLocation.h b/contrib/llvm/include/llvm/Analysis/MemoryLocation.h index 6b680000312..fca18c1b599 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryLocation.h @@ -16,9 +16,9 @@ #ifndef LLVM_ANALYSIS_MEMORYLOCATION_H #define LLVM_ANALYSIS_MEMORYLOCATION_H -#include "llvm/ADT/Optional.h" #include "llvm/ADT/DenseMapInfo.h" -#include "llvm/IR/CallSite.h" +#include "llvm/ADT/Optional.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" namespace llvm { @@ -34,8 +34,134 @@ class AnyMemIntrinsic; class TargetLibraryInfo; // Represents the size of a MemoryLocation. Logically, it's an -// Optional, with a special UnknownSize value from `MemoryLocation`. -using LocationSize = uint64_t; +// Optional that also carries a bit to represent whether the integer +// it contains, N, is 'precise'. Precise, in this context, means that we know +// that the area of storage referenced by the given MemoryLocation must be +// precisely N bytes. An imprecise value is formed as the union of two or more +// precise values, and can conservatively represent all of the values unioned +// into it. Importantly, imprecise values are an *upper-bound* on the size of a +// MemoryLocation. +// +// Concretely, a precise MemoryLocation is (%p, 4) in +// store i32 0, i32* %p +// +// Since we know that %p must be at least 4 bytes large at this point. +// Otherwise, we have UB. An example of an imprecise MemoryLocation is (%p, 4) +// at the memcpy in +// +// %n = select i1 %foo, i64 1, i64 4 +// call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %baz, i64 %n, i32 1, +// i1 false) +// +// ...Since we'll copy *up to* 4 bytes into %p, but we can't guarantee that +// we'll ever actually do so. +// +// If asked to represent a pathologically large value, this will degrade to +// None. +class LocationSize { + enum : uint64_t { + Unknown = ~uint64_t(0), + ImpreciseBit = uint64_t(1) << 63, + MapEmpty = Unknown - 1, + MapTombstone = Unknown - 2, + + // The maximum value we can represent without falling back to 'unknown'. + MaxValue = (MapTombstone - 1) & ~ImpreciseBit, + }; + + uint64_t Value; + + // Hack to support implicit construction. This should disappear when the + // public LocationSize ctor goes away. + enum DirectConstruction { Direct }; + + constexpr LocationSize(uint64_t Raw, DirectConstruction): Value(Raw) {} + + static_assert(Unknown & ImpreciseBit, "Unknown is imprecise by definition."); +public: + // FIXME: Migrate all users to construct via either `precise` or `upperBound`, + // to make it more obvious at the callsite the kind of size that they're + // providing. + // + // Since the overwhelming majority of users of this provide precise values, + // this assumes the provided value is precise. + constexpr LocationSize(uint64_t Raw) + : Value(Raw > MaxValue ? Unknown : Raw) {} + + static LocationSize precise(uint64_t Value) { return LocationSize(Value); } + + static LocationSize upperBound(uint64_t Value) { + // You can't go lower than 0, so give a precise result. + if (LLVM_UNLIKELY(Value == 0)) + return precise(0); + if (LLVM_UNLIKELY(Value > MaxValue)) + return unknown(); + return LocationSize(Value | ImpreciseBit, Direct); + } + + constexpr static LocationSize unknown() { + return LocationSize(Unknown, Direct); + } + + // Sentinel values, generally used for maps. + constexpr static LocationSize mapTombstone() { + return LocationSize(MapTombstone, Direct); + } + constexpr static LocationSize mapEmpty() { + return LocationSize(MapEmpty, Direct); + } + + // Returns a LocationSize that can correctly represent either `*this` or + // `Other`. + LocationSize unionWith(LocationSize Other) const { + if (Other == *this) + return *this; + + if (!hasValue() || !Other.hasValue()) + return unknown(); + + return upperBound(std::max(getValue(), Other.getValue())); + } + + bool hasValue() const { return Value != Unknown; } + uint64_t getValue() const { + assert(hasValue() && "Getting value from an unknown LocationSize!"); + return Value & ~ImpreciseBit; + } + + // Returns whether or not this value is precise. Note that if a value is + // precise, it's guaranteed to not be `unknown()`. + bool isPrecise() const { + return (Value & ImpreciseBit) == 0; + } + + // Convenience method to check if this LocationSize's value is 0. + bool isZero() const { return hasValue() && getValue() == 0; } + + bool operator==(const LocationSize &Other) const { + return Value == Other.Value; + } + + bool operator!=(const LocationSize &Other) const { + return !(*this == Other); + } + + // Ordering operators are not provided, since it's unclear if there's only one + // reasonable way to compare: + // - values that don't exist against values that do, and + // - precise values to imprecise values + + void print(raw_ostream &OS) const; + + // Returns an opaque value that represents this LocationSize. Cannot be + // reliably converted back into a LocationSize. + uint64_t toRaw() const { return Value; } +}; + +inline raw_ostream &operator<<(raw_ostream &OS, LocationSize Size) { + Size.print(OS); + return OS; +} /// Representation for a specific memory location. /// @@ -108,11 +234,15 @@ public: static MemoryLocation getForDest(const AnyMemIntrinsic *MI); /// Return a location representing a particular argument of a call. - static MemoryLocation getForArgument(ImmutableCallSite CS, unsigned ArgIdx, - const TargetLibraryInfo &TLI); + static MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx, + const TargetLibraryInfo *TLI); + static MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx, + const TargetLibraryInfo &TLI) { + return getForArgument(Call, ArgIdx, &TLI); + } explicit MemoryLocation(const Value *Ptr = nullptr, - LocationSize Size = UnknownSize, + LocationSize Size = LocationSize::unknown(), const AAMDNodes &AATags = AAMDNodes()) : Ptr(Ptr), Size(Size), AATags(AATags) {} @@ -139,13 +269,30 @@ public: } }; -// Specialize DenseMapInfo for MemoryLocation. +// Specialize DenseMapInfo. +template <> struct DenseMapInfo { + static inline LocationSize getEmptyKey() { + return LocationSize::mapEmpty(); + } + static inline LocationSize getTombstoneKey() { + return LocationSize::mapTombstone(); + } + static unsigned getHashValue(const LocationSize &Val) { + return DenseMapInfo::getHashValue(Val.toRaw()); + } + static bool isEqual(const LocationSize &LHS, const LocationSize &RHS) { + return LHS == RHS; + } +}; + template <> struct DenseMapInfo { static inline MemoryLocation getEmptyKey() { - return MemoryLocation(DenseMapInfo::getEmptyKey(), 0); + return MemoryLocation(DenseMapInfo::getEmptyKey(), + DenseMapInfo::getEmptyKey()); } static inline MemoryLocation getTombstoneKey() { - return MemoryLocation(DenseMapInfo::getTombstoneKey(), 0); + return MemoryLocation(DenseMapInfo::getTombstoneKey(), + DenseMapInfo::getTombstoneKey()); } static unsigned getHashValue(const MemoryLocation &Val) { return DenseMapInfo::getHashValue(Val.Ptr) ^ diff --git a/contrib/llvm/include/llvm/Analysis/MemorySSA.h b/contrib/llvm/include/llvm/Analysis/MemorySSA.h index d445e4430e5..17e2d0c7397 100644 --- a/contrib/llvm/include/llvm/Analysis/MemorySSA.h +++ b/contrib/llvm/include/llvm/Analysis/MemorySSA.h @@ -280,9 +280,10 @@ protected: friend class MemorySSAUpdater; MemoryUseOrDef(LLVMContext &C, MemoryAccess *DMA, unsigned Vty, - DeleteValueTy DeleteValue, Instruction *MI, BasicBlock *BB) - : MemoryAccess(C, Vty, DeleteValue, BB, 1), MemoryInstruction(MI), - OptimizedAccessAlias(MayAlias) { + DeleteValueTy DeleteValue, Instruction *MI, BasicBlock *BB, + unsigned NumOperands) + : MemoryAccess(C, Vty, DeleteValue, BB, NumOperands), + MemoryInstruction(MI), OptimizedAccessAlias(MayAlias) { setDefiningAccess(DMA); } @@ -308,11 +309,6 @@ private: Optional OptimizedAccessAlias; }; -template <> -struct OperandTraits - : public FixedNumOperandTraits {}; -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryUseOrDef, MemoryAccess) - /// Represents read-only accesses to memory /// /// In particular, the set of Instructions that will be represented by @@ -323,7 +319,8 @@ public: DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); MemoryUse(LLVMContext &C, MemoryAccess *DMA, Instruction *MI, BasicBlock *BB) - : MemoryUseOrDef(C, DMA, MemoryUseVal, deleteMe, MI, BB) {} + : MemoryUseOrDef(C, DMA, MemoryUseVal, deleteMe, MI, BB, + /*NumOperands=*/1) {} // allocate space for exactly one operand void *operator new(size_t s) { return User::operator new(s, 1); } @@ -381,31 +378,33 @@ public: MemoryDef(LLVMContext &C, MemoryAccess *DMA, Instruction *MI, BasicBlock *BB, unsigned Ver) - : MemoryUseOrDef(C, DMA, MemoryDefVal, deleteMe, MI, BB), ID(Ver) {} + : MemoryUseOrDef(C, DMA, MemoryDefVal, deleteMe, MI, BB, + /*NumOperands=*/2), + ID(Ver) {} - // allocate space for exactly one operand - void *operator new(size_t s) { return User::operator new(s, 1); } + // allocate space for exactly two operands + void *operator new(size_t s) { return User::operator new(s, 2); } static bool classof(const Value *MA) { return MA->getValueID() == MemoryDefVal; } void setOptimized(MemoryAccess *MA) { - Optimized = MA; - OptimizedID = getDefiningAccess()->getID(); + setOperand(1, MA); + OptimizedID = MA->getID(); } MemoryAccess *getOptimized() const { - return cast_or_null(Optimized); + return cast_or_null(getOperand(1)); } bool isOptimized() const { - return getOptimized() && getDefiningAccess() && - OptimizedID == getDefiningAccess()->getID(); + return getOptimized() && OptimizedID == getOptimized()->getID(); } void resetOptimized() { OptimizedID = INVALID_MEMORYACCESS_ID; + setOperand(1, nullptr); } void print(raw_ostream &OS) const; @@ -417,13 +416,34 @@ private: const unsigned ID; unsigned OptimizedID = INVALID_MEMORYACCESS_ID; - WeakVH Optimized; }; template <> -struct OperandTraits : public FixedNumOperandTraits {}; +struct OperandTraits : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryDef, MemoryAccess) +template <> +struct OperandTraits { + static Use *op_begin(MemoryUseOrDef *MUD) { + if (auto *MU = dyn_cast(MUD)) + return OperandTraits::op_begin(MU); + return OperandTraits::op_begin(cast(MUD)); + } + + static Use *op_end(MemoryUseOrDef *MUD) { + if (auto *MU = dyn_cast(MUD)) + return OperandTraits::op_end(MU); + return OperandTraits::op_end(cast(MUD)); + } + + static unsigned operands(const MemoryUseOrDef *MUD) { + if (const auto *MU = dyn_cast(MUD)) + return OperandTraits::operands(MU); + return OperandTraits::operands(cast(MUD)); + } +}; +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryUseOrDef, MemoryAccess) + /// Represents phi nodes for memory accesses. /// /// These have the same semantic as regular phi nodes, with the exception that @@ -684,13 +704,19 @@ public: ~MemorySSA(); MemorySSAWalker *getWalker(); + MemorySSAWalker *getSkipSelfWalker(); /// Given a memory Mod/Ref'ing instruction, get the MemorySSA /// access associated with it. If passed a basic block gets the memory phi /// node that exists for that block, if there is one. Otherwise, this will get /// a MemoryUseOrDef. - MemoryUseOrDef *getMemoryAccess(const Instruction *) const; - MemoryPhi *getMemoryAccess(const BasicBlock *BB) const; + MemoryUseOrDef *getMemoryAccess(const Instruction *I) const { + return cast_or_null(ValueToMemoryAccess.lookup(I)); + } + + MemoryPhi *getMemoryAccess(const BasicBlock *BB) const { + return cast_or_null(ValueToMemoryAccess.lookup(cast(BB))); + } void dump() const; void print(raw_ostream &) const; @@ -750,6 +776,9 @@ public: /// all uses, uses appear in the right places). This is used by unit tests. void verifyMemorySSA() const; + /// Check clobber sanity for an access. + void checkClobberSanityAccess(const MemoryAccess *MA) const; + /// Used in various insertion functions to specify whether we are talking /// about the beginning or end of a block. enum InsertionPlace { Beginning, End }; @@ -764,6 +793,7 @@ protected: void verifyDomination(Function &F) const; void verifyOrdering(Function &F) const; void verifyDominationNumbers(const Function &F) const; + void verifyClobberSanity(const Function &F) const; // This is used by the use optimizer and updater. AccessList *getWritableBlockAccesses(const BasicBlock *BB) const { @@ -796,16 +826,20 @@ protected: InsertionPlace); void insertIntoListsBefore(MemoryAccess *, const BasicBlock *, AccessList::iterator); - MemoryUseOrDef *createDefinedAccess(Instruction *, MemoryAccess *); + MemoryUseOrDef *createDefinedAccess(Instruction *, MemoryAccess *, + const MemoryUseOrDef *Template = nullptr); private: + class ClobberWalkerBase; class CachingWalker; + class SkipSelfWalker; class OptimizeUses; CachingWalker *getWalkerImpl(); void buildMemorySSA(); void optimizeUses(); + void prepareForMoveTo(MemoryAccess *, BasicBlock *); void verifyUseInDefs(MemoryAccess *, MemoryAccess *) const; using AccessMap = DenseMap>; @@ -816,7 +850,8 @@ private: void markUnreachableAsLiveOnEntry(BasicBlock *BB); bool dominatesUse(const MemoryAccess *, const MemoryAccess *) const; MemoryPhi *createMemoryPhi(BasicBlock *BB); - MemoryUseOrDef *createNewAccess(Instruction *); + MemoryUseOrDef *createNewAccess(Instruction *, + const MemoryUseOrDef *Template = nullptr); MemoryAccess *findDominatingDef(BasicBlock *, enum InsertionPlace); void placePHINodes(const SmallPtrSetImpl &); MemoryAccess *renameBlock(BasicBlock *, MemoryAccess *, bool); @@ -851,7 +886,9 @@ private: mutable DenseMap BlockNumbering; // Memory SSA building info + std::unique_ptr WalkerBase; std::unique_ptr Walker; + std::unique_ptr SkipWalker; unsigned NextID; }; diff --git a/contrib/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/contrib/llvm/include/llvm/Analysis/MemorySSAUpdater.h index 38f08c1eebd..169d5bd9fa8 100644 --- a/contrib/llvm/include/llvm/Analysis/MemorySSAUpdater.h +++ b/contrib/llvm/include/llvm/Analysis/MemorySSAUpdater.h @@ -35,8 +35,11 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFGDiff.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Module.h" #include "llvm/IR/OperandTraits.h" @@ -45,6 +48,7 @@ #include "llvm/IR/User.h" #include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/IR/ValueMap.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" @@ -57,6 +61,12 @@ class MemoryAccess; class LLVMContext; class raw_ostream; +using ValueToValueMapTy = ValueMap; +using PhiToDefMap = SmallDenseMap; +using CFGUpdate = cfg::Update; +using GraphDiffInvBBPair = + std::pair *, Inverse>; + class MemorySSAUpdater { private: MemorySSA *MSSA; @@ -70,6 +80,7 @@ private: public: MemorySSAUpdater(MemorySSA *MSSA) : MSSA(MSSA) {} + /// Insert a definition into the MemorySSA IR. RenameUses will rename any use /// below the new def block (and any inserted phis). RenameUses should be set /// to true if the definition may cause new aliases for loads below it. This @@ -89,15 +100,48 @@ public: /// Where a mayalias b, *does* require RenameUses be set to true. void insertDef(MemoryDef *Def, bool RenameUses = false); void insertUse(MemoryUse *Use); + /// Update the MemoryPhi in `To` following an edge deletion between `From` and + /// `To`. If `To` becomes unreachable, a call to removeBlocks should be made. + void removeEdge(BasicBlock *From, BasicBlock *To); + /// Update the MemoryPhi in `To` to have a single incoming edge from `From`, + /// following a CFG change that replaced multiple edges (switch) with a direct + /// branch. + void removeDuplicatePhiEdgesBetween(BasicBlock *From, BasicBlock *To); + /// Update MemorySSA after a loop was cloned, given the blocks in RPO order, + /// the exit blocks and a 1:1 mapping of all blocks and instructions + /// cloned. This involves duplicating all defs and uses in the cloned blocks + /// Updating phi nodes in exit block successors is done separately. + void updateForClonedLoop(const LoopBlocksRPO &LoopBlocks, + ArrayRef ExitBlocks, + const ValueToValueMapTy &VM, + bool IgnoreIncomingWithNoClones = false); + // Block BB was fully or partially cloned into its predecessor P1. Map + // contains the 1:1 mapping of instructions cloned and VM[BB]=P1. + void updateForClonedBlockIntoPred(BasicBlock *BB, BasicBlock *P1, + const ValueToValueMapTy &VM); + /// Update phi nodes in exit block successors following cloning. Exit blocks + /// that were not cloned don't have additional predecessors added. + void updateExitBlocksForClonedLoop(ArrayRef ExitBlocks, + const ValueToValueMapTy &VMap, + DominatorTree &DT); + void updateExitBlocksForClonedLoop( + ArrayRef ExitBlocks, + ArrayRef> VMaps, DominatorTree &DT); + + /// Apply CFG updates, analogous with the DT edge updates. + void applyUpdates(ArrayRef Updates, DominatorTree &DT); + /// Apply CFG insert updates, analogous with the DT edge updates. + void applyInsertUpdates(ArrayRef Updates, DominatorTree &DT); + void moveBefore(MemoryUseOrDef *What, MemoryUseOrDef *Where); void moveAfter(MemoryUseOrDef *What, MemoryUseOrDef *Where); void moveToPlace(MemoryUseOrDef *What, BasicBlock *BB, MemorySSA::InsertionPlace Where); - /// `From` block was spliced into `From` and `To`. - /// Move all accesses from `From` to `To` starting at instruction `Start`. - /// `To` is newly created BB, so empty of MemorySSA::MemoryAccesses. - /// Edges are already updated, so successors of `To` with MPhi nodes need to - /// update incoming block. + /// `From` block was spliced into `From` and `To`. There is a CFG edge from + /// `From` to `To`. Move all accesses from `From` to `To` starting at + /// instruction `Start`. `To` is newly created BB, so empty of + /// MemorySSA::MemoryAccesses. Edges are already updated, so successors of + /// `To` with MPhi nodes need to update incoming block. /// |------| |------| /// | From | | From | /// | | |------| @@ -108,12 +152,12 @@ public: /// |------| |------| void moveAllAfterSpliceBlocks(BasicBlock *From, BasicBlock *To, Instruction *Start); - /// `From` block was merged into `To`. All instructions were moved and - /// `From` is an empty block with successor edges; `From` is about to be - /// deleted. Move all accesses from `From` to `To` starting at instruction - /// `Start`. `To` may have multiple successors, `From` has a single - /// predecessor. `From` may have successors with MPhi nodes, replace their - /// incoming block with `To`. + /// `From` block was merged into `To`. There is a CFG edge from `To` to + /// `From`.`To` still branches to `From`, but all instructions were moved and + /// `From` is now an empty block; `From` is about to be deleted. Move all + /// accesses from `From` to `To` starting at instruction `Start`. `To` may + /// have multiple successors, `From` has a single predecessor. `From` may have + /// successors with MPhi nodes, replace their incoming block with `To`. /// |------| |------| /// | To | | To | /// |------| | | @@ -124,15 +168,14 @@ public: /// |------| |------| void moveAllAfterMergeBlocks(BasicBlock *From, BasicBlock *To, Instruction *Start); - /// BasicBlock Old had New, an empty BasicBlock, added directly before it, - /// and the predecessors in Preds that used to point to Old, now point to - /// New. If New is the only predecessor, move Old's Phi, if present, to New. + /// A new empty BasicBlock (New) now branches directly to Old. Some of + /// Old's predecessors (Preds) are now branching to New instead of Old. + /// If New is the only predecessor, move Old's Phi, if present, to New. /// Otherwise, add a new Phi in New with appropriate incoming values, and /// update the incoming values in Old's Phi node too, if present. - void - wireOldPredecessorsToNewImmediatePredecessor(BasicBlock *Old, BasicBlock *New, - ArrayRef Preds); - + void wireOldPredecessorsToNewImmediatePredecessor( + BasicBlock *Old, BasicBlock *New, ArrayRef Preds, + bool IdenticalEdgesWereMerged = true); // The below are utility functions. Other than creation of accesses to pass // to insertDef, and removeAccess to remove accesses, you should generally // not attempt to update memoryssa yourself. It is very non-trivial to get @@ -220,6 +263,23 @@ private: template MemoryAccess *tryRemoveTrivialPhi(MemoryPhi *Phi, RangeType &Operands); void fixupDefs(const SmallVectorImpl &); + // Clone all uses and defs from BB to NewBB given a 1:1 map of all + // instructions and blocks cloned, and a map of MemoryPhi : Definition + // (MemoryAccess Phi or Def). VMap maps old instructions to cloned + // instructions and old blocks to cloned blocks. MPhiMap, is created in the + // caller of this private method, and maps existing MemoryPhis to new + // definitions that new MemoryAccesses must point to. These definitions may + // not necessarily be MemoryPhis themselves, they may be MemoryDefs. As such, + // the map is between MemoryPhis and MemoryAccesses, where the MemoryAccesses + // may be MemoryPhis or MemoryDefs and not MemoryUses. + void cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB, + const ValueToValueMapTy &VMap, PhiToDefMap &MPhiMap); + template + void privateUpdateExitBlocksForClonedLoop(ArrayRef ExitBlocks, + Iter ValuesBegin, Iter ValuesEnd, + DominatorTree &DT); + void applyInsertUpdates(ArrayRef, DominatorTree &DT, + const GraphDiff *GD); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Analysis/MustExecute.h b/contrib/llvm/include/llvm/Analysis/MustExecute.h index 97ad76d451c..ad3222c17e6 100644 --- a/contrib/llvm/include/llvm/Analysis/MustExecute.h +++ b/contrib/llvm/include/llvm/Analysis/MustExecute.h @@ -19,6 +19,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/Analysis/EHPersonalities.h" +#include "llvm/Analysis/InstructionPrecedenceTracking.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" @@ -31,33 +32,138 @@ class DominatorTree; class Loop; /// Captures loop safety information. -/// It keep information for loop & its header may throw exception or otherwise +/// It keep information for loop blocks may throw exception or otherwise /// exit abnormaly on any iteration of the loop which might actually execute /// at runtime. The primary way to consume this infromation is via /// isGuaranteedToExecute below, but some callers bailout or fallback to /// alternate reasoning if a loop contains any implicit control flow. -struct LoopSafetyInfo { - bool MayThrow = false; // The current loop contains an instruction which - // may throw. - bool HeaderMayThrow = false; // Same as previous, but specific to loop header +/// NOTE: LoopSafetyInfo contains cached information regarding loops and their +/// particular blocks. This information is only dropped on invocation of +/// computeLoopSafetyInfo. If the loop or any of its block is deleted, or if +/// any thrower instructions have been added or removed from them, or if the +/// control flow has changed, or in case of other meaningful modifications, the +/// LoopSafetyInfo needs to be recomputed. If a meaningful modifications to the +/// loop were made and the info wasn't recomputed properly, the behavior of all +/// methods except for computeLoopSafetyInfo is undefined. +class LoopSafetyInfo { // Used to update funclet bundle operands. DenseMap BlockColors; +protected: + /// Computes block colors. + void computeBlockColors(const Loop *CurLoop); + +public: + /// Returns block colors map that is used to update funclet operand bundles. + const DenseMap &getBlockColors() const; + + /// Copy colors of block \p Old into the block \p New. + void copyColors(BasicBlock *New, BasicBlock *Old); + + /// Returns true iff the block \p BB potentially may throw exception. It can + /// be false-positive in cases when we want to avoid complex analysis. + virtual bool blockMayThrow(const BasicBlock *BB) const = 0; + + /// Returns true iff any block of the loop for which this info is contains an + /// instruction that may throw or otherwise exit abnormally. + virtual bool anyBlockMayThrow() const = 0; + + /// Return true if we must reach the block \p BB under assumption that the + /// loop \p CurLoop is entered. + bool allLoopPathsLeadToBlock(const Loop *CurLoop, const BasicBlock *BB, + const DominatorTree *DT) const; + + /// Computes safety information for a loop checks loop body & header for + /// the possibility of may throw exception, it takes LoopSafetyInfo and loop + /// as argument. Updates safety information in LoopSafetyInfo argument. + /// Note: This is defined to clear and reinitialize an already initialized + /// LoopSafetyInfo. Some callers rely on this fact. + virtual void computeLoopSafetyInfo(const Loop *CurLoop) = 0; + + /// Returns true if the instruction in a loop is guaranteed to execute at + /// least once (under the assumption that the loop is entered). + virtual bool isGuaranteedToExecute(const Instruction &Inst, + const DominatorTree *DT, + const Loop *CurLoop) const = 0; + LoopSafetyInfo() = default; + + virtual ~LoopSafetyInfo() = default; }; -/// Computes safety information for a loop checks loop body & header for -/// the possibility of may throw exception, it takes LoopSafetyInfo and loop as -/// argument. Updates safety information in LoopSafetyInfo argument. -/// Note: This is defined to clear and reinitialize an already initialized -/// LoopSafetyInfo. Some callers rely on this fact. -void computeLoopSafetyInfo(LoopSafetyInfo *, Loop *); -/// Returns true if the instruction in a loop is guaranteed to execute at least -/// once (under the assumption that the loop is entered). -bool isGuaranteedToExecute(const Instruction &Inst, const DominatorTree *DT, - const Loop *CurLoop, - const LoopSafetyInfo *SafetyInfo); +/// Simple and conservative implementation of LoopSafetyInfo that can give +/// false-positive answers to its queries in order to avoid complicated +/// analysis. +class SimpleLoopSafetyInfo: public LoopSafetyInfo { + bool MayThrow = false; // The current loop contains an instruction which + // may throw. + bool HeaderMayThrow = false; // Same as previous, but specific to loop header + +public: + virtual bool blockMayThrow(const BasicBlock *BB) const; + + virtual bool anyBlockMayThrow() const; + + virtual void computeLoopSafetyInfo(const Loop *CurLoop); + + virtual bool isGuaranteedToExecute(const Instruction &Inst, + const DominatorTree *DT, + const Loop *CurLoop) const; + + SimpleLoopSafetyInfo() : LoopSafetyInfo() {}; + + virtual ~SimpleLoopSafetyInfo() {}; +}; + +/// This implementation of LoopSafetyInfo use ImplicitControlFlowTracking to +/// give precise answers on "may throw" queries. This implementation uses cache +/// that should be invalidated by calling the methods insertInstructionTo and +/// removeInstruction whenever we modify a basic block's contents by adding or +/// removing instructions. +class ICFLoopSafetyInfo: public LoopSafetyInfo { + bool MayThrow = false; // The current loop contains an instruction which + // may throw. + // Contains information about implicit control flow in this loop's blocks. + mutable ImplicitControlFlowTracking ICF; + // Contains information about instruction that may possibly write memory. + mutable MemoryWriteTracking MW; + +public: + virtual bool blockMayThrow(const BasicBlock *BB) const; + + virtual bool anyBlockMayThrow() const; + + virtual void computeLoopSafetyInfo(const Loop *CurLoop); + + virtual bool isGuaranteedToExecute(const Instruction &Inst, + const DominatorTree *DT, + const Loop *CurLoop) const; + + /// Returns true if we could not execute a memory-modifying instruction before + /// we enter \p BB under assumption that \p CurLoop is entered. + bool doesNotWriteMemoryBefore(const BasicBlock *BB, const Loop *CurLoop) + const; + + /// Returns true if we could not execute a memory-modifying instruction before + /// we execute \p I under assumption that \p CurLoop is entered. + bool doesNotWriteMemoryBefore(const Instruction &I, const Loop *CurLoop) + const; + + /// Inform the safety info that we are planning to insert a new instruction + /// \p Inst into the basic block \p BB. It will make all cache updates to keep + /// it correct after this insertion. + void insertInstructionTo(const Instruction *Inst, const BasicBlock *BB); + + /// Inform safety info that we are planning to remove the instruction \p Inst + /// from its block. It will make all cache updates to keep it correct after + /// this removal. + void removeInstruction(const Instruction *Inst); + + ICFLoopSafetyInfo(DominatorTree *DT) : LoopSafetyInfo(), ICF(DT), MW(DT) {}; + + virtual ~ICFLoopSafetyInfo() {}; +}; } diff --git a/contrib/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h index 559c77c3081..58a67042ea2 100644 --- a/contrib/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h @@ -60,7 +60,7 @@ public: FunctionModRefBehavior getModRefBehavior(const Function *F); using AAResultBase::getModRefInfo; - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); }; /// Analysis pass providing a never-invalidated alias analysis result. diff --git a/contrib/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/contrib/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h index 07beb0bb60a..1f497fab35d 100644 --- a/contrib/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h +++ b/contrib/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h @@ -51,25 +51,25 @@ extern bool EnableARCOpts; /// on. inline bool ModuleHasARC(const Module &M) { return - M.getNamedValue("objc_retain") || - M.getNamedValue("objc_release") || - M.getNamedValue("objc_autorelease") || - M.getNamedValue("objc_retainAutoreleasedReturnValue") || - M.getNamedValue("objc_unsafeClaimAutoreleasedReturnValue") || - M.getNamedValue("objc_retainBlock") || - M.getNamedValue("objc_autoreleaseReturnValue") || - M.getNamedValue("objc_autoreleasePoolPush") || - M.getNamedValue("objc_loadWeakRetained") || - M.getNamedValue("objc_loadWeak") || - M.getNamedValue("objc_destroyWeak") || - M.getNamedValue("objc_storeWeak") || - M.getNamedValue("objc_initWeak") || - M.getNamedValue("objc_moveWeak") || - M.getNamedValue("objc_copyWeak") || - M.getNamedValue("objc_retainedObject") || - M.getNamedValue("objc_unretainedObject") || - M.getNamedValue("objc_unretainedPointer") || - M.getNamedValue("clang.arc.use"); + M.getNamedValue("llvm.objc.retain") || + M.getNamedValue("llvm.objc.release") || + M.getNamedValue("llvm.objc.autorelease") || + M.getNamedValue("llvm.objc.retainAutoreleasedReturnValue") || + M.getNamedValue("llvm.objc.unsafeClaimAutoreleasedReturnValue") || + M.getNamedValue("llvm.objc.retainBlock") || + M.getNamedValue("llvm.objc.autoreleaseReturnValue") || + M.getNamedValue("llvm.objc.autoreleasePoolPush") || + M.getNamedValue("llvm.objc.loadWeakRetained") || + M.getNamedValue("llvm.objc.loadWeak") || + M.getNamedValue("llvm.objc.destroyWeak") || + M.getNamedValue("llvm.objc.storeWeak") || + M.getNamedValue("llvm.objc.initWeak") || + M.getNamedValue("llvm.objc.moveWeak") || + M.getNamedValue("llvm.objc.copyWeak") || + M.getNamedValue("llvm.objc.retainedObject") || + M.getNamedValue("llvm.objc.unretainedObject") || + M.getNamedValue("llvm.objc.unretainedPointer") || + M.getNamedValue("llvm.objc.clang.arc.use"); } /// This is a wrapper around getUnderlyingObject which also knows how to diff --git a/contrib/llvm/include/llvm/Analysis/ObjCARCInstKind.h b/contrib/llvm/include/llvm/Analysis/ObjCARCInstKind.h index 0b92d8b4835..018ea1f851b 100644 --- a/contrib/llvm/include/llvm/Analysis/ObjCARCInstKind.h +++ b/contrib/llvm/include/llvm/Analysis/ObjCARCInstKind.h @@ -11,6 +11,7 @@ #define LLVM_ANALYSIS_OBJCARCINSTKIND_H #include "llvm/IR/Function.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Instructions.h" namespace llvm { @@ -48,7 +49,7 @@ enum class ARCInstKind { CopyWeak, ///< objc_copyWeak (derived) DestroyWeak, ///< objc_destroyWeak (derived) StoreStrong, ///< objc_storeStrong (derived) - IntrinsicUser, ///< clang.arc.use + IntrinsicUser, ///< llvm.objc.clang.arc.use CallOrUser, ///< could call objc_release and/or "use" pointers Call, ///< could call objc_release User, ///< could "use" a pointer diff --git a/contrib/llvm/include/llvm/Transforms/Utils/OrderedInstructions.h b/contrib/llvm/include/llvm/Analysis/OrderedInstructions.h similarity index 93% rename from contrib/llvm/include/llvm/Transforms/Utils/OrderedInstructions.h rename to contrib/llvm/include/llvm/Analysis/OrderedInstructions.h index 7f57fde638b..7e3850b87c5 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/OrderedInstructions.h +++ b/contrib/llvm/include/llvm/Analysis/OrderedInstructions.h @@ -17,8 +17,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TRANSFORMS_UTILS_ORDEREDINSTRUCTIONS_H -#define LLVM_TRANSFORMS_UTILS_ORDEREDINSTRUCTIONS_H +#ifndef LLVM_ANALYSIS_ORDEREDINSTRUCTIONS_H +#define LLVM_ANALYSIS_ORDEREDINSTRUCTIONS_H #include "llvm/ADT/DenseMap.h" #include "llvm/Analysis/OrderedBasicBlock.h" @@ -62,4 +62,4 @@ public: } // end namespace llvm -#endif // LLVM_TRANSFORMS_UTILS_ORDEREDINSTRUCTIONS_H +#endif // LLVM_ANALYSIS_ORDEREDINSTRUCTIONS_H diff --git a/contrib/llvm/include/llvm/Analysis/Passes.h b/contrib/llvm/include/llvm/Analysis/Passes.h index 09b28a0b088..081dd500083 100644 --- a/contrib/llvm/include/llvm/Analysis/Passes.h +++ b/contrib/llvm/include/llvm/Analysis/Passes.h @@ -61,10 +61,10 @@ namespace llvm { //===--------------------------------------------------------------------===// // - // createDivergenceAnalysisPass - This pass determines which branches in a GPU + // createLegacyDivergenceAnalysisPass - This pass determines which branches in a GPU // program are divergent. // - FunctionPass *createDivergenceAnalysisPass(); + FunctionPass *createLegacyDivergenceAnalysisPass(); //===--------------------------------------------------------------------===// // diff --git a/contrib/llvm/include/llvm/Analysis/PhiValues.h b/contrib/llvm/include/llvm/Analysis/PhiValues.h index 6607b329c04..76204ac1bc6 100644 --- a/contrib/llvm/include/llvm/Analysis/PhiValues.h +++ b/contrib/llvm/include/llvm/Analysis/PhiValues.h @@ -88,6 +88,22 @@ private: /// All values reachable from each component. DenseMap ReachableMap; + /// A CallbackVH to notify PhiValues when a value is deleted or replaced, so + /// that the cached information for that value can be cleared to avoid + /// dangling pointers to invalid values. + class PhiValuesCallbackVH final : public CallbackVH { + PhiValues *PV; + void deleted() override; + void allUsesReplacedWith(Value *New) override; + + public: + PhiValuesCallbackVH(Value *V, PhiValues *PV = nullptr) + : CallbackVH(V), PV(PV) {} + }; + + /// A set of callbacks to the values that processPhi has seen. + DenseSet> TrackedValues; + /// The function that the PhiValues is for. const Function &F; diff --git a/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h index 58b67e74ba5..3aef4be72d7 100644 --- a/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -98,14 +98,14 @@ public: bool isFunctionEntryCold(const Function *F); /// Returns true if \p F contains only cold code. bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI); - /// Returns true if \p F is a hot function. + /// Returns true if count \p C is considered hot. bool isHotCount(uint64_t C); /// Returns true if count \p C is considered cold. bool isColdCount(uint64_t C); - /// Returns true if BasicBlock \p B is considered hot. - bool isHotBB(const BasicBlock *B, BlockFrequencyInfo *BFI); - /// Returns true if BasicBlock \p B is considered cold. - bool isColdBB(const BasicBlock *B, BlockFrequencyInfo *BFI); + /// Returns true if BasicBlock \p BB is considered hot. + bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); + /// Returns true if BasicBlock \p BB is considered cold. + bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if CallSite \p CS is considered hot. bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); /// Returns true if Callsite \p CS is considered cold. @@ -134,9 +134,8 @@ public: static char ID; ProfileSummaryInfoWrapperPass(); - ProfileSummaryInfo *getPSI() { - return &*PSI; - } + ProfileSummaryInfo &getPSI() { return *PSI; } + const ProfileSummaryInfo &getPSI() const { return *PSI; } bool doInitialization(Module &M) override; bool doFinalization(Module &M) override; diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h index 89918e3c205..8f4200b07e5 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1833,6 +1833,10 @@ private: const SCEV *getOrCreateMulExpr(SmallVectorImpl &Ops, SCEV::NoWrapFlags Flags); + // Get addrec expr already created or create a new one. + const SCEV *getOrCreateAddRecExpr(SmallVectorImpl &Ops, + const Loop *L, SCEV::NoWrapFlags Flags); + /// Return x if \p Val is f(x) where f is a 1-1 function. const SCEV *stripInjectiveFunctions(const SCEV *Val) const; diff --git a/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h b/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h index 508968e16e5..1356c6e9198 100644 --- a/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h +++ b/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h @@ -16,7 +16,7 @@ #define LLVM_ANALYSIS_SCOPEDNOALIASAA_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/CallSite.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include @@ -41,8 +41,8 @@ public: } AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); - ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2); private: bool mayAliasInScopes(const MDNode *Scopes, const MDNode *NoAlias) const; diff --git a/contrib/llvm/include/llvm/Analysis/SparsePropagation.h b/contrib/llvm/include/llvm/Analysis/SparsePropagation.h index defcf96afb2..02a2e64268b 100644 --- a/contrib/llvm/include/llvm/Analysis/SparsePropagation.h +++ b/contrib/llvm/include/llvm/Analysis/SparsePropagation.h @@ -189,12 +189,12 @@ private: /// getFeasibleSuccessors - Return a vector of booleans to indicate which /// successors are reachable from a given terminator instruction. - void getFeasibleSuccessors(TerminatorInst &TI, SmallVectorImpl &Succs, + void getFeasibleSuccessors(Instruction &TI, SmallVectorImpl &Succs, bool AggressiveUndef); void visitInst(Instruction &I); void visitPHINode(PHINode &I); - void visitTerminatorInst(TerminatorInst &TI); + void visitTerminator(Instruction &TI); }; //===----------------------------------------------------------------------===// @@ -286,7 +286,7 @@ void SparseSolver::markEdgeExecutable( template void SparseSolver::getFeasibleSuccessors( - TerminatorInst &TI, SmallVectorImpl &Succs, bool AggressiveUndef) { + Instruction &TI, SmallVectorImpl &Succs, bool AggressiveUndef) { Succs.resize(TI.getNumSuccessors()); if (TI.getNumSuccessors() == 0) return; @@ -330,7 +330,7 @@ void SparseSolver::getFeasibleSuccessors( return; } - if (TI.isExceptional()) { + if (TI.isExceptionalTerminator()) { Succs.assign(Succs.size(), true); return; } @@ -374,7 +374,7 @@ template bool SparseSolver::isEdgeFeasible( BasicBlock *From, BasicBlock *To, bool AggressiveUndef) { SmallVector SuccFeasible; - TerminatorInst *TI = From->getTerminator(); + Instruction *TI = From->getTerminator(); getFeasibleSuccessors(*TI, SuccFeasible, AggressiveUndef); for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) @@ -385,8 +385,8 @@ bool SparseSolver::isEdgeFeasible( } template -void SparseSolver::visitTerminatorInst( - TerminatorInst &TI) { +void SparseSolver::visitTerminator( + Instruction &TI) { SmallVector SuccFeasible; getFeasibleSuccessors(TI, SuccFeasible, true); @@ -465,8 +465,8 @@ void SparseSolver::visitInst(Instruction &I) { if (ChangedValue.second != LatticeFunc->getUntrackedVal()) UpdateState(ChangedValue.first, ChangedValue.second); - if (TerminatorInst *TI = dyn_cast(&I)) - visitTerminatorInst(*TI); + if (I.isTerminator()) + visitTerminator(I); } template diff --git a/contrib/llvm/include/llvm/Analysis/StackSafetyAnalysis.h b/contrib/llvm/include/llvm/Analysis/StackSafetyAnalysis.h new file mode 100644 index 00000000000..8a151650a34 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/StackSafetyAnalysis.h @@ -0,0 +1,120 @@ +//===- StackSafetyAnalysis.h - Stack memory safety analysis -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Stack Safety Analysis detects allocas and arguments with safe access. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_STACKSAFETYANALYSIS_H +#define LLVM_ANALYSIS_STACKSAFETYANALYSIS_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" + +namespace llvm { + +/// Interface to access stack safety analysis results for single function. +class StackSafetyInfo { +public: + struct FunctionInfo; + +private: + std::unique_ptr Info; + +public: + StackSafetyInfo(); + StackSafetyInfo(FunctionInfo &&Info); + StackSafetyInfo(StackSafetyInfo &&); + StackSafetyInfo &operator=(StackSafetyInfo &&); + ~StackSafetyInfo(); + + // TODO: Add useful for client methods. + void print(raw_ostream &O) const; +}; + +/// StackSafetyInfo wrapper for the new pass manager. +class StackSafetyAnalysis : public AnalysisInfoMixin { + friend AnalysisInfoMixin; + static AnalysisKey Key; + +public: + using Result = StackSafetyInfo; + StackSafetyInfo run(Function &F, FunctionAnalysisManager &AM); +}; + +/// Printer pass for the \c StackSafetyAnalysis results. +class StackSafetyPrinterPass : public PassInfoMixin { + raw_ostream &OS; + +public: + explicit StackSafetyPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +/// StackSafetyInfo wrapper for the legacy pass manager +class StackSafetyInfoWrapperPass : public FunctionPass { + StackSafetyInfo SSI; + +public: + static char ID; + StackSafetyInfoWrapperPass(); + + const StackSafetyInfo &getResult() const { return SSI; } + + void print(raw_ostream &O, const Module *M) const override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnFunction(Function &F) override; +}; + +using StackSafetyGlobalInfo = std::map; + +/// This pass performs the global (interprocedural) stack safety analysis (new +/// pass manager). +class StackSafetyGlobalAnalysis + : public AnalysisInfoMixin { + friend AnalysisInfoMixin; + static AnalysisKey Key; + +public: + using Result = StackSafetyGlobalInfo; + Result run(Module &M, ModuleAnalysisManager &AM); +}; + +/// Printer pass for the \c StackSafetyGlobalAnalysis results. +class StackSafetyGlobalPrinterPass + : public PassInfoMixin { + raw_ostream &OS; + +public: + explicit StackSafetyGlobalPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +/// This pass performs the global (interprocedural) stack safety analysis +/// (legacy pass manager). +class StackSafetyGlobalInfoWrapperPass : public ModulePass { + StackSafetyGlobalInfo SSI; + +public: + static char ID; + + StackSafetyGlobalInfoWrapperPass(); + + const StackSafetyGlobalInfo &getResult() const { return SSI; } + + void print(raw_ostream &O, const Module *M) const override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnModule(Module &M) override; +}; + +} // end namespace llvm + +#endif // LLVM_ANALYSIS_STACKSAFETYANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h new file mode 100644 index 00000000000..df693d9d8e8 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h @@ -0,0 +1,86 @@ +//===- SyncDependenceAnalysis.h - Divergent Branch Dependence -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// \file +// This file defines the SyncDependenceAnalysis class, which computes for +// every divergent branch the set of phi nodes that the branch will make +// divergent. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_SYNC_DEPENDENCE_ANALYSIS_H +#define LLVM_ANALYSIS_SYNC_DEPENDENCE_ANALYSIS_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Analysis/LoopInfo.h" +#include + +namespace llvm { + +class BasicBlock; +class DominatorTree; +class Loop; +class PostDominatorTree; + +using ConstBlockSet = SmallPtrSet; + +/// \brief Relates points of divergent control to join points in +/// reducible CFGs. +/// +/// This analysis relates points of divergent control to points of converging +/// divergent control. The analysis requires all loops to be reducible. +class SyncDependenceAnalysis { + void visitSuccessor(const BasicBlock &succBlock, const Loop *termLoop, + const BasicBlock *defBlock); + +public: + bool inRegion(const BasicBlock &BB) const; + + ~SyncDependenceAnalysis(); + SyncDependenceAnalysis(const DominatorTree &DT, const PostDominatorTree &PDT, + const LoopInfo &LI); + + /// \brief Computes divergent join points and loop exits caused by branch + /// divergence in \p Term. + /// + /// The set of blocks which are reachable by disjoint paths from \p Term. + /// The set also contains loop exits if there two disjoint paths: + /// one from \p Term to the loop exit and another from \p Term to the loop + /// header. Those exit blocks are added to the returned set. + /// If L is the parent loop of \p Term and an exit of L is in the returned + /// set then L is a divergent loop. + const ConstBlockSet &join_blocks(const Instruction &Term); + + /// \brief Computes divergent join points and loop exits (in the surrounding + /// loop) caused by the divergent loop exits of\p Loop. + /// + /// The set of blocks which are reachable by disjoint paths from the + /// loop exits of \p Loop. + /// This treats the loop as a single node in \p Loop's parent loop. + /// The returned set has the same properties as for join_blocks(TermInst&). + const ConstBlockSet &join_blocks(const Loop &Loop); + +private: + static ConstBlockSet EmptyBlockSet; + + ReversePostOrderTraversal FuncRPOT; + const DominatorTree &DT; + const PostDominatorTree &PDT; + const LoopInfo &LI; + + std::map> CachedLoopExitJoins; + std::map> + CachedBranchJoins; +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_SYNC_DEPENDENCE_ANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/SyntheticCountsUtils.h b/contrib/llvm/include/llvm/Analysis/SyntheticCountsUtils.h index 87f4a0100b3..db80bef001e 100644 --- a/contrib/llvm/include/llvm/Analysis/SyntheticCountsUtils.h +++ b/contrib/llvm/include/llvm/Analysis/SyntheticCountsUtils.h @@ -36,16 +36,17 @@ public: using EdgeRef = typename CGT::EdgeRef; using SccTy = std::vector; - using GetRelBBFreqTy = function_ref(EdgeRef)>; - using GetCountTy = function_ref; - using AddCountTy = function_ref; + // Not all EdgeRef have information about the source of the edge. Hence + // NodeRef corresponding to the source of the EdgeRef is explicitly passed. + using GetProfCountTy = function_ref(NodeRef, EdgeRef)>; + using AddCountTy = function_ref; - static void propagate(const CallGraphType &CG, GetRelBBFreqTy GetRelBBFreq, - GetCountTy GetCount, AddCountTy AddCount); + static void propagate(const CallGraphType &CG, GetProfCountTy GetProfCount, + AddCountTy AddCount); private: - static void propagateFromSCC(const SccTy &SCC, GetRelBBFreqTy GetRelBBFreq, - GetCountTy GetCount, AddCountTy AddCount); + static void propagateFromSCC(const SccTy &SCC, GetProfCountTy GetProfCount, + AddCountTy AddCount); }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def index f94debba9c5..518a85ee1a0 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def +++ b/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def @@ -565,6 +565,30 @@ TLI_DEFINE_STRING_INTERNAL("cosl") /// char *ctermid(char *s); TLI_DEFINE_ENUM_INTERNAL(ctermid) TLI_DEFINE_STRING_INTERNAL("ctermid") +/// int execl(const char *path, const char *arg, ...); +TLI_DEFINE_ENUM_INTERNAL(execl) +TLI_DEFINE_STRING_INTERNAL("execl") +/// int execle(const char *file, const char *arg, ..., char * const envp[]); +TLI_DEFINE_ENUM_INTERNAL(execle) +TLI_DEFINE_STRING_INTERNAL("execle") +/// int execlp(const char *file, const char *arg, ...); +TLI_DEFINE_ENUM_INTERNAL(execlp) +TLI_DEFINE_STRING_INTERNAL("execlp") +/// int execv(const char *path, char *const argv[]); +TLI_DEFINE_ENUM_INTERNAL(execv) +TLI_DEFINE_STRING_INTERNAL("execv") +/// int execvP(const char *file, const char *search_path, char *const argv[]); +TLI_DEFINE_ENUM_INTERNAL(execvP) +TLI_DEFINE_STRING_INTERNAL("execvP") +/// int execve(const char *filename, char *const argv[], char *const envp[]); +TLI_DEFINE_ENUM_INTERNAL(execve) +TLI_DEFINE_STRING_INTERNAL("execve") +/// int execvp(const char *file, char *const argv[]); +TLI_DEFINE_ENUM_INTERNAL(execvp) +TLI_DEFINE_STRING_INTERNAL("execvp") +/// int execvpe(const char *file, char *const argv[], char *const envp[]); +TLI_DEFINE_ENUM_INTERNAL(execvpe) +TLI_DEFINE_STRING_INTERNAL("execvpe") /// double exp(double x); TLI_DEFINE_ENUM_INTERNAL(exp) TLI_DEFINE_STRING_INTERNAL("exp") @@ -709,6 +733,9 @@ TLI_DEFINE_STRING_INTERNAL("fopen") /// FILE *fopen64(const char *filename, const char *opentype) TLI_DEFINE_ENUM_INTERNAL(fopen64) TLI_DEFINE_STRING_INTERNAL("fopen64") +/// int fork(); +TLI_DEFINE_ENUM_INTERNAL(fork) +TLI_DEFINE_STRING_INTERNAL("fork") /// int fprintf(FILE *stream, const char *format, ...); TLI_DEFINE_ENUM_INTERNAL(fprintf) TLI_DEFINE_STRING_INTERNAL("fprintf") diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h index 59657cca40f..223175d17c2 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -289,7 +289,7 @@ public: /// Returns whether V is a source of divergence. /// /// This function provides the target-dependent information for - /// the target-independent DivergenceAnalysis. DivergenceAnalysis first + /// the target-independent LegacyDivergenceAnalysis. LegacyDivergenceAnalysis first /// builds the dependency graph, and then runs the reachability algorithm /// starting with the sources of divergence. bool isSourceOfDivergence(const Value *V) const; @@ -581,12 +581,21 @@ public: struct MemCmpExpansionOptions { // The list of available load sizes (in bytes), sorted in decreasing order. SmallVector LoadSizes; + // Set to true to allow overlapping loads. For example, 7-byte compares can + // be done with two 4-byte compares instead of 4+2+1-byte compares. This + // requires all loads in LoadSizes to be doable in an unaligned way. + bool AllowOverlappingLoads = false; }; const MemCmpExpansionOptions *enableMemCmpExpansion(bool IsZeroCmp) const; /// Enable matching of interleaved access groups. bool enableInterleavedAccessVectorization() const; + /// Enable matching of interleaved access groups that contain predicated + /// accesses or gaps and therefore vectorized using masked + /// vector loads/stores. + bool enableMaskedInterleavedAccessVectorization() const; + /// Indicate that it is potentially unsafe to automatically vectorize /// floating-point operations because the semantics of vector and scalar /// floating-point semantics may differ. For example, ARM NEON v7 SIMD math @@ -739,6 +748,10 @@ public: /// and the number of execution units in the CPU. unsigned getMaxInterleaveFactor(unsigned VF) const; + /// Collect properties of V used in cost analysis, e.g. OP_PowerOf2. + static OperandValueKind getOperandInfo(Value *V, + OperandValueProperties &OpProps); + /// This is an approximation of reciprocal throughput of a math/logic op. /// A higher cost indicates less expected throughput. /// From Agner Fog's guides, reciprocal throughput is "the average number of @@ -762,7 +775,9 @@ public: /// \return The cost of a shuffle instruction of kind Kind and of type Tp. /// The index and subtype parameters are used by the subvector insertion and - /// extraction shuffle kinds. + /// extraction shuffle kinds to show the insert/extract point and the type of + /// the subvector being inserted/extracted. + /// NOTE: For subvector extractions Tp represents the source type. int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index = 0, Type *SubTp = nullptr) const; @@ -817,9 +832,13 @@ public: /// load allows gaps) /// \p Alignment is the alignment of the memory operation /// \p AddressSpace is address space of the pointer. + /// \p UseMaskForCond indicates if the memory access is predicated. + /// \p UseMaskForGaps indicates if gaps should be masked. int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, unsigned Alignment, - unsigned AddressSpace) const; + unsigned AddressSpace, + bool UseMaskForCond = false, + bool UseMaskForGaps = false) const; /// Calculate the cost of performing a vector reduction. /// @@ -915,6 +934,14 @@ public: bool areInlineCompatible(const Function *Caller, const Function *Callee) const; + /// \returns True if the caller and callee agree on how \p Args will be passed + /// to the callee. + /// \param[out] Args The list of compatible arguments. The implementation may + /// filter out any incompatible args from this list. + bool areFunctionArgsABICompatible(const Function *Caller, + const Function *Callee, + SmallPtrSetImpl &Args) const; + /// The type of load/store indexing. enum MemIndexedMode { MIM_Unindexed, ///< No indexing. @@ -1068,6 +1095,7 @@ public: virtual const MemCmpExpansionOptions *enableMemCmpExpansion( bool IsZeroCmp) const = 0; virtual bool enableInterleavedAccessVectorization() = 0; + virtual bool enableMaskedInterleavedAccessVectorization() = 0; virtual bool isFPVectorizationPotentiallyUnsafe() = 0; virtual bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth, @@ -1128,7 +1156,9 @@ public: unsigned Factor, ArrayRef Indices, unsigned Alignment, - unsigned AddressSpace) = 0; + unsigned AddressSpace, + bool UseMaskForCond = false, + bool UseMaskForGaps = false) = 0; virtual int getArithmeticReductionCost(unsigned Opcode, Type *Ty, bool IsPairwiseForm) = 0; virtual int getMinMaxReductionCost(Type *Ty, Type *CondTy, @@ -1157,6 +1187,9 @@ public: unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const = 0; virtual bool areInlineCompatible(const Function *Caller, const Function *Callee) const = 0; + virtual bool + areFunctionArgsABICompatible(const Function *Caller, const Function *Callee, + SmallPtrSetImpl &Args) const = 0; virtual bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const = 0; virtual bool isIndexedStoreLegal(MemIndexedMode Mode,Type *Ty) const = 0; virtual unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const = 0; @@ -1342,6 +1375,9 @@ public: bool enableInterleavedAccessVectorization() override { return Impl.enableInterleavedAccessVectorization(); } + bool enableMaskedInterleavedAccessVectorization() override { + return Impl.enableMaskedInterleavedAccessVectorization(); + } bool isFPVectorizationPotentiallyUnsafe() override { return Impl.isFPVectorizationPotentiallyUnsafe(); } @@ -1467,9 +1503,11 @@ public: } int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, unsigned Alignment, - unsigned AddressSpace) override { + unsigned AddressSpace, bool UseMaskForCond, + bool UseMaskForGaps) override { return Impl.getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, - Alignment, AddressSpace); + Alignment, AddressSpace, + UseMaskForCond, UseMaskForGaps); } int getArithmeticReductionCost(unsigned Opcode, Type *Ty, bool IsPairwiseForm) override { @@ -1530,6 +1568,11 @@ public: const Function *Callee) const override { return Impl.areInlineCompatible(Caller, Callee); } + bool areFunctionArgsABICompatible( + const Function *Caller, const Function *Callee, + SmallPtrSetImpl &Args) const override { + return Impl.areFunctionArgsABICompatible(Caller, Callee, Args); + } bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const override { return Impl.isIndexedLoadLegal(Mode, Ty, getDataLayout()); } diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index d80ae1d6845..c9a234deeb7 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -158,6 +158,9 @@ public: case Intrinsic::dbg_label: case Intrinsic::invariant_start: case Intrinsic::invariant_end: + case Intrinsic::launder_invariant_group: + case Intrinsic::strip_invariant_group: + case Intrinsic::is_constant: case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: case Intrinsic::objectsize: @@ -311,6 +314,8 @@ public: bool enableInterleavedAccessVectorization() { return false; } + bool enableMaskedInterleavedAccessVectorization() { return false; } + bool isFPVectorizationPotentiallyUnsafe() { return false; } bool allowsMisalignedMemoryAccesses(LLVMContext &Context, @@ -448,8 +453,9 @@ public: unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, - unsigned Alignment, - unsigned AddressSpace) { + unsigned Alignment, unsigned AddressSpace, + bool UseMaskForCond = false, + bool UseMaskForGaps = false) { return 1; } @@ -520,6 +526,14 @@ public: Callee->getFnAttribute("target-features")); } + bool areFunctionArgsABICompatible(const Function *Caller, const Function *Callee, + SmallPtrSetImpl &Args) const { + return (Caller->getFnAttribute("target-cpu") == + Callee->getFnAttribute("target-cpu")) && + (Caller->getFnAttribute("target-features") == + Callee->getFnAttribute("target-features")); + } + bool isIndexedLoadLegal(TTI::MemIndexedMode Mode, Type *Ty, const DataLayout &DL) const { return false; diff --git a/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h index 7fcfdb3a817..d2e6df22425 100644 --- a/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h @@ -17,7 +17,7 @@ #define LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/CallSite.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include @@ -43,10 +43,10 @@ public: AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); - FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + FunctionModRefBehavior getModRefBehavior(const CallBase *Call); FunctionModRefBehavior getModRefBehavior(const Function *F); - ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); - ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2); private: bool Aliases(const MDNode *A, const MDNode *B) const; diff --git a/contrib/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/contrib/llvm/include/llvm/Analysis/TypeMetadataUtils.h index 6764563f683..3bf9c5d2074 100644 --- a/contrib/llvm/include/llvm/Analysis/TypeMetadataUtils.h +++ b/contrib/llvm/include/llvm/Analysis/TypeMetadataUtils.h @@ -20,6 +20,8 @@ namespace llvm { +class DominatorTree; + /// The type of CFI jumptable needed for a function. enum CfiFunctionLinkage { CFL_Definition = 0, @@ -39,7 +41,8 @@ struct DevirtCallSite { /// call sites based on the call and return them in DevirtCalls. void findDevirtualizableCallsForTypeTest( SmallVectorImpl &DevirtCalls, - SmallVectorImpl &Assumes, const CallInst *CI); + SmallVectorImpl &Assumes, const CallInst *CI, + DominatorTree &DT); /// Given a call to the intrinsic \@llvm.type.checked.load, find all /// devirtualizable call sites based on the call and return them in DevirtCalls. @@ -47,7 +50,7 @@ void findDevirtualizableCallsForTypeCheckedLoad( SmallVectorImpl &DevirtCalls, SmallVectorImpl &LoadedPtrs, SmallVectorImpl &Preds, bool &HasNonCallUses, - const CallInst *CI); + const CallInst *CI, DominatorTree &DT); } #endif diff --git a/contrib/llvm/include/llvm/Analysis/ValueTracking.h b/contrib/llvm/include/llvm/Analysis/ValueTracking.h index c1a91a8e598..f46fdfcb608 100644 --- a/contrib/llvm/include/llvm/Analysis/ValueTracking.h +++ b/contrib/llvm/include/llvm/Analysis/ValueTracking.h @@ -55,14 +55,16 @@ class Value; AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr, - OptimizationRemarkEmitter *ORE = nullptr); + OptimizationRemarkEmitter *ORE = nullptr, + bool UseInstrInfo = true); /// Returns the known bits rather than passing by reference. KnownBits computeKnownBits(const Value *V, const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr, - OptimizationRemarkEmitter *ORE = nullptr); + OptimizationRemarkEmitter *ORE = nullptr, + bool UseInstrInfo = true); /// Compute known bits from the range metadata. /// \p KnownZero the set of bits that are known to be zero @@ -75,7 +77,8 @@ class Value; const DataLayout &DL, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Return true if the given value is known to have exactly one bit set when /// defined. For vectors return true if every element is known to be a power @@ -86,7 +89,8 @@ class Value; bool OrZero = false, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI); @@ -99,7 +103,8 @@ class Value; bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Return true if the two given values are negation. /// Currently can recoginze Value pair: @@ -112,28 +117,32 @@ class Value; unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Returns true if the given value is known be positive (i.e. non-negative /// and non-zero). bool isKnownPositive(const Value *V, const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Returns true if the given value is known be negative (i.e. non-positive /// and non-zero). bool isKnownNegative(const Value *V, const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Return true if the given values are known to be non-equal when defined. /// Supports scalar integer types only. bool isKnownNonEqual(const Value *V1, const Value *V2, const DataLayout &DL, - AssumptionCache *AC = nullptr, - const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Return true if 'V & Mask' is known to be zero. We use this predicate to /// simplify operations downstream. Mask is known to be zero for bits that V @@ -148,7 +157,8 @@ class Value; const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// Return the number of times the sign bit of the register is replicated into /// the other bits. We know that at least 1 bit is always equal to the sign @@ -160,7 +170,8 @@ class Value; unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool UseInstrInfo = true); /// This function computes the integer multiple of Base that equals V. If /// successful, it returns true and returns the multiple in Multiple. If @@ -194,7 +205,8 @@ class Value; /// Return true if the floating-point scalar value is not a NaN or if the /// floating-point vector value has no NaN elements. Return false if a value /// could ever be NaN. - bool isKnownNeverNaN(const Value *V); + bool isKnownNeverNaN(const Value *V, const TargetLibraryInfo *TLI, + unsigned Depth = 0); /// Return true if we can prove that the specified FP value's sign bit is 0. /// @@ -209,7 +221,8 @@ class Value; /// return the i8 value that it is represented with. This is true for all i8 /// values obviously, but is also true for i32 0, i32 -1, i16 0xF0F0, double /// 0.0 etc. If the value can't be handled with a repeated byte store (e.g. - /// i16 0x1234), return null. + /// i16 0x1234), return null. If the value is entirely undef and padding, + /// return undef. Value *isBytewiseValue(Value *V); /// Given an aggregrate and an sequence of indices, see if the scalar value @@ -284,10 +297,10 @@ class Value; /// This function returns call pointer argument that is considered the same by /// aliasing rules. You CAN'T use it to replace one value with another. - const Value *getArgumentAliasingToReturnedPointer(ImmutableCallSite CS); - inline Value *getArgumentAliasingToReturnedPointer(CallSite CS) { - return const_cast( - getArgumentAliasingToReturnedPointer(ImmutableCallSite(CS))); + const Value *getArgumentAliasingToReturnedPointer(const CallBase *Call); + inline Value *getArgumentAliasingToReturnedPointer(CallBase *Call) { + return const_cast(getArgumentAliasingToReturnedPointer( + const_cast(Call))); } // {launder,strip}.invariant.group returns pointer that aliases its argument, @@ -296,7 +309,7 @@ class Value; // considered as capture. The arguments are not marked as returned neither, // because it would make it useless. bool isIntrinsicReturningPointerAliasingArgumentWithoutCapturing( - ImmutableCallSite CS); + const CallBase *Call); /// This method strips off any GEP address adjustments and pointer casts from /// the specified value, returning the original object being addressed. Note @@ -405,18 +418,21 @@ class Value; const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, - const DominatorTree *DT); + const DominatorTree *DT, + bool UseInstrInfo = true); OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, - const DominatorTree *DT); + const DominatorTree *DT, + bool UseInstrInfo = true); OverflowResult computeOverflowForUnsignedAdd(const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, - const DominatorTree *DT); + const DominatorTree *DT, + bool UseInstrInfo = true); OverflowResult computeOverflowForSignedAdd(const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC = nullptr, @@ -594,6 +610,12 @@ class Value; Optional isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue = true, unsigned Depth = 0); + + /// Return the boolean condition value in the context of the given instruction + /// if it is known based on dominating conditions. + Optional isImpliedByDomCondition(const Value *Cond, + const Instruction *ContextI, + const DataLayout &DL); } // end namespace llvm #endif // LLVM_ANALYSIS_VALUETRACKING_H diff --git a/contrib/llvm/include/llvm/Analysis/VectorUtils.h b/contrib/llvm/include/llvm/Analysis/VectorUtils.h index 9fde36d6109..be4d4f17b9a 100644 --- a/contrib/llvm/include/llvm/Analysis/VectorUtils.h +++ b/contrib/llvm/include/llvm/Analysis/VectorUtils.h @@ -15,6 +15,7 @@ #define LLVM_ANALYSIS_VECTORUTILS_H #include "llvm/ADT/MapVector.h" +#include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" @@ -23,6 +24,7 @@ namespace llvm { template class ArrayRef; class DemandedBits; class GetElementPtrInst; +template class InterleaveGroup; class Loop; class ScalarEvolution; class TargetTransformInfo; @@ -115,8 +117,24 @@ computeMinimumValueSizes(ArrayRef Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr); +/// Compute the union of two access-group lists. +/// +/// If the list contains just one access group, it is returned directly. If the +/// list is empty, returns nullptr. +MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2); + +/// Compute the access-group list of access groups that @p Inst1 and @p Inst2 +/// are both in. If either instruction does not access memory at all, it is +/// considered to be in every list. +/// +/// If the list contains just one access group, it is returned directly. If the +/// list is empty, returns nullptr. +MDNode *intersectAccessGroups(const Instruction *Inst1, + const Instruction *Inst2); + /// Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, -/// MD_nontemporal]. For K in Kinds, we get the MDNode for K from each of the +/// MD_nontemporal, MD_access_group]. +/// For K in Kinds, we get the MDNode for K from each of the /// elements of VL, compute their "intersection" (i.e., the most generic /// metadata value that covers all of the individual values), and set I's /// metadata for M equal to the intersection value. @@ -124,6 +142,35 @@ computeMinimumValueSizes(ArrayRef Blocks, /// This function always sets a (possibly null) value for each K in Kinds. Instruction *propagateMetadata(Instruction *I, ArrayRef VL); +/// Create a mask that filters the members of an interleave group where there +/// are gaps. +/// +/// For example, the mask for \p Group with interleave-factor 3 +/// and \p VF 4, that has only its first member present is: +/// +/// <1,0,0,1,0,0,1,0,0,1,0,0> +/// +/// Note: The result is a mask of 0's and 1's, as opposed to the other +/// create[*]Mask() utilities which create a shuffle mask (mask that +/// consists of indices). +Constant *createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF, + const InterleaveGroup &Group); + +/// Create a mask with replicated elements. +/// +/// This function creates a shuffle mask for replicating each of the \p VF +/// elements in a vector \p ReplicationFactor times. It can be used to +/// transform a mask of \p VF elements into a mask of +/// \p VF * \p ReplicationFactor elements used by a predicated +/// interleaved-group of loads/stores whose Interleaved-factor == +/// \p ReplicationFactor. +/// +/// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is: +/// +/// <0,0,0,1,1,1,2,2,2,3,3,3> +Constant *createReplicatedMask(IRBuilder<> &Builder, unsigned ReplicationFactor, + unsigned VF); + /// Create an interleave shuffle mask. /// /// This function creates a shuffle mask for interleaving \p NumVecs vectors of @@ -176,6 +223,381 @@ Constant *createSequentialMask(IRBuilder<> &Builder, unsigned Start, /// elements, it will be padded with undefs. Value *concatenateVectors(IRBuilder<> &Builder, ArrayRef Vecs); +/// The group of interleaved loads/stores sharing the same stride and +/// close to each other. +/// +/// Each member in this group has an index starting from 0, and the largest +/// index should be less than interleaved factor, which is equal to the absolute +/// value of the access's stride. +/// +/// E.g. An interleaved load group of factor 4: +/// for (unsigned i = 0; i < 1024; i+=4) { +/// a = A[i]; // Member of index 0 +/// b = A[i+1]; // Member of index 1 +/// d = A[i+3]; // Member of index 3 +/// ... +/// } +/// +/// An interleaved store group of factor 4: +/// for (unsigned i = 0; i < 1024; i+=4) { +/// ... +/// A[i] = a; // Member of index 0 +/// A[i+1] = b; // Member of index 1 +/// A[i+2] = c; // Member of index 2 +/// A[i+3] = d; // Member of index 3 +/// } +/// +/// Note: the interleaved load group could have gaps (missing members), but +/// the interleaved store group doesn't allow gaps. +template class InterleaveGroup { +public: + InterleaveGroup(unsigned Factor, bool Reverse, unsigned Align) + : Factor(Factor), Reverse(Reverse), Align(Align), InsertPos(nullptr) {} + + InterleaveGroup(InstTy *Instr, int Stride, unsigned Align) + : Align(Align), InsertPos(Instr) { + assert(Align && "The alignment should be non-zero"); + + Factor = std::abs(Stride); + assert(Factor > 1 && "Invalid interleave factor"); + + Reverse = Stride < 0; + Members[0] = Instr; + } + + bool isReverse() const { return Reverse; } + unsigned getFactor() const { return Factor; } + unsigned getAlignment() const { return Align; } + unsigned getNumMembers() const { return Members.size(); } + + /// Try to insert a new member \p Instr with index \p Index and + /// alignment \p NewAlign. The index is related to the leader and it could be + /// negative if it is the new leader. + /// + /// \returns false if the instruction doesn't belong to the group. + bool insertMember(InstTy *Instr, int Index, unsigned NewAlign) { + assert(NewAlign && "The new member's alignment should be non-zero"); + + int Key = Index + SmallestKey; + + // Skip if there is already a member with the same index. + if (Members.find(Key) != Members.end()) + return false; + + if (Key > LargestKey) { + // The largest index is always less than the interleave factor. + if (Index >= static_cast(Factor)) + return false; + + LargestKey = Key; + } else if (Key < SmallestKey) { + // The largest index is always less than the interleave factor. + if (LargestKey - Key >= static_cast(Factor)) + return false; + + SmallestKey = Key; + } + + // It's always safe to select the minimum alignment. + Align = std::min(Align, NewAlign); + Members[Key] = Instr; + return true; + } + + /// Get the member with the given index \p Index + /// + /// \returns nullptr if contains no such member. + InstTy *getMember(unsigned Index) const { + int Key = SmallestKey + Index; + auto Member = Members.find(Key); + if (Member == Members.end()) + return nullptr; + + return Member->second; + } + + /// Get the index for the given member. Unlike the key in the member + /// map, the index starts from 0. + unsigned getIndex(const InstTy *Instr) const { + for (auto I : Members) { + if (I.second == Instr) + return I.first - SmallestKey; + } + + llvm_unreachable("InterleaveGroup contains no such member"); + } + + InstTy *getInsertPos() const { return InsertPos; } + void setInsertPos(InstTy *Inst) { InsertPos = Inst; } + + /// Add metadata (e.g. alias info) from the instructions in this group to \p + /// NewInst. + /// + /// FIXME: this function currently does not add noalias metadata a'la + /// addNewMedata. To do that we need to compute the intersection of the + /// noalias info from all members. + void addMetadata(InstTy *NewInst) const; + + /// Returns true if this Group requires a scalar iteration to handle gaps. + bool requiresScalarEpilogue() const { + // If the last member of the Group exists, then a scalar epilog is not + // needed for this group. + if (getMember(getFactor() - 1)) + return false; + + // We have a group with gaps. It therefore cannot be a group of stores, + // and it can't be a reversed access, because such groups get invalidated. + assert(!getMember(0)->mayWriteToMemory() && + "Group should have been invalidated"); + assert(!isReverse() && "Group should have been invalidated"); + + // This is a group of loads, with gaps, and without a last-member + return true; + } + +private: + unsigned Factor; // Interleave Factor. + bool Reverse; + unsigned Align; + DenseMap Members; + int SmallestKey = 0; + int LargestKey = 0; + + // To avoid breaking dependences, vectorized instructions of an interleave + // group should be inserted at either the first load or the last store in + // program order. + // + // E.g. %even = load i32 // Insert Position + // %add = add i32 %even // Use of %even + // %odd = load i32 + // + // store i32 %even + // %odd = add i32 // Def of %odd + // store i32 %odd // Insert Position + InstTy *InsertPos; +}; + +/// Drive the analysis of interleaved memory accesses in the loop. +/// +/// Use this class to analyze interleaved accesses only when we can vectorize +/// a loop. Otherwise it's meaningless to do analysis as the vectorization +/// on interleaved accesses is unsafe. +/// +/// The analysis collects interleave groups and records the relationships +/// between the member and the group in a map. +class InterleavedAccessInfo { +public: + InterleavedAccessInfo(PredicatedScalarEvolution &PSE, Loop *L, + DominatorTree *DT, LoopInfo *LI, + const LoopAccessInfo *LAI) + : PSE(PSE), TheLoop(L), DT(DT), LI(LI), LAI(LAI) {} + + ~InterleavedAccessInfo() { reset(); } + + /// Analyze the interleaved accesses and collect them in interleave + /// groups. Substitute symbolic strides using \p Strides. + /// Consider also predicated loads/stores in the analysis if + /// \p EnableMaskedInterleavedGroup is true. + void analyzeInterleaving(bool EnableMaskedInterleavedGroup); + + /// Invalidate groups, e.g., in case all blocks in loop will be predicated + /// contrary to original assumption. Although we currently prevent group + /// formation for predicated accesses, we may be able to relax this limitation + /// in the future once we handle more complicated blocks. + void reset() { + SmallPtrSet *, 4> DelSet; + // Avoid releasing a pointer twice. + for (auto &I : InterleaveGroupMap) + DelSet.insert(I.second); + for (auto *Ptr : DelSet) + delete Ptr; + InterleaveGroupMap.clear(); + RequiresScalarEpilogue = false; + } + + + /// Check if \p Instr belongs to any interleave group. + bool isInterleaved(Instruction *Instr) const { + return InterleaveGroupMap.find(Instr) != InterleaveGroupMap.end(); + } + + /// Get the interleave group that \p Instr belongs to. + /// + /// \returns nullptr if doesn't have such group. + InterleaveGroup * + getInterleaveGroup(const Instruction *Instr) const { + if (InterleaveGroupMap.count(Instr)) + return InterleaveGroupMap.find(Instr)->second; + return nullptr; + } + + iterator_range *>> + getInterleaveGroups() { + return make_range(InterleaveGroups.begin(), InterleaveGroups.end()); + } + + /// Returns true if an interleaved group that may access memory + /// out-of-bounds requires a scalar epilogue iteration for correctness. + bool requiresScalarEpilogue() const { return RequiresScalarEpilogue; } + + /// Invalidate groups that require a scalar epilogue (due to gaps). This can + /// happen when optimizing for size forbids a scalar epilogue, and the gap + /// cannot be filtered by masking the load/store. + void invalidateGroupsRequiringScalarEpilogue(); + +private: + /// A wrapper around ScalarEvolution, used to add runtime SCEV checks. + /// Simplifies SCEV expressions in the context of existing SCEV assumptions. + /// The interleaved access analysis can also add new predicates (for example + /// by versioning strides of pointers). + PredicatedScalarEvolution &PSE; + + Loop *TheLoop; + DominatorTree *DT; + LoopInfo *LI; + const LoopAccessInfo *LAI; + + /// True if the loop may contain non-reversed interleaved groups with + /// out-of-bounds accesses. We ensure we don't speculatively access memory + /// out-of-bounds by executing at least one scalar epilogue iteration. + bool RequiresScalarEpilogue = false; + + /// Holds the relationships between the members and the interleave group. + DenseMap *> InterleaveGroupMap; + + SmallPtrSet *, 4> InterleaveGroups; + + /// Holds dependences among the memory accesses in the loop. It maps a source + /// access to a set of dependent sink accesses. + DenseMap> Dependences; + + /// The descriptor for a strided memory access. + struct StrideDescriptor { + StrideDescriptor() = default; + StrideDescriptor(int64_t Stride, const SCEV *Scev, uint64_t Size, + unsigned Align) + : Stride(Stride), Scev(Scev), Size(Size), Align(Align) {} + + // The access's stride. It is negative for a reverse access. + int64_t Stride = 0; + + // The scalar expression of this access. + const SCEV *Scev = nullptr; + + // The size of the memory object. + uint64_t Size = 0; + + // The alignment of this access. + unsigned Align = 0; + }; + + /// A type for holding instructions and their stride descriptors. + using StrideEntry = std::pair; + + /// Create a new interleave group with the given instruction \p Instr, + /// stride \p Stride and alignment \p Align. + /// + /// \returns the newly created interleave group. + InterleaveGroup * + createInterleaveGroup(Instruction *Instr, int Stride, unsigned Align) { + assert(!InterleaveGroupMap.count(Instr) && + "Already in an interleaved access group"); + InterleaveGroupMap[Instr] = + new InterleaveGroup(Instr, Stride, Align); + InterleaveGroups.insert(InterleaveGroupMap[Instr]); + return InterleaveGroupMap[Instr]; + } + + /// Release the group and remove all the relationships. + void releaseGroup(InterleaveGroup *Group) { + for (unsigned i = 0; i < Group->getFactor(); i++) + if (Instruction *Member = Group->getMember(i)) + InterleaveGroupMap.erase(Member); + + InterleaveGroups.erase(Group); + delete Group; + } + + /// Collect all the accesses with a constant stride in program order. + void collectConstStrideAccesses( + MapVector &AccessStrideInfo, + const ValueToValueMap &Strides); + + /// Returns true if \p Stride is allowed in an interleaved group. + static bool isStrided(int Stride); + + /// Returns true if \p BB is a predicated block. + bool isPredicated(BasicBlock *BB) const { + return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT); + } + + /// Returns true if LoopAccessInfo can be used for dependence queries. + bool areDependencesValid() const { + return LAI && LAI->getDepChecker().getDependences(); + } + + /// Returns true if memory accesses \p A and \p B can be reordered, if + /// necessary, when constructing interleaved groups. + /// + /// \p A must precede \p B in program order. We return false if reordering is + /// not necessary or is prevented because \p A and \p B may be dependent. + bool canReorderMemAccessesForInterleavedGroups(StrideEntry *A, + StrideEntry *B) const { + // Code motion for interleaved accesses can potentially hoist strided loads + // and sink strided stores. The code below checks the legality of the + // following two conditions: + // + // 1. Potentially moving a strided load (B) before any store (A) that + // precedes B, or + // + // 2. Potentially moving a strided store (A) after any load or store (B) + // that A precedes. + // + // It's legal to reorder A and B if we know there isn't a dependence from A + // to B. Note that this determination is conservative since some + // dependences could potentially be reordered safely. + + // A is potentially the source of a dependence. + auto *Src = A->first; + auto SrcDes = A->second; + + // B is potentially the sink of a dependence. + auto *Sink = B->first; + auto SinkDes = B->second; + + // Code motion for interleaved accesses can't violate WAR dependences. + // Thus, reordering is legal if the source isn't a write. + if (!Src->mayWriteToMemory()) + return true; + + // At least one of the accesses must be strided. + if (!isStrided(SrcDes.Stride) && !isStrided(SinkDes.Stride)) + return true; + + // If dependence information is not available from LoopAccessInfo, + // conservatively assume the instructions can't be reordered. + if (!areDependencesValid()) + return false; + + // If we know there is a dependence from source to sink, assume the + // instructions can't be reordered. Otherwise, reordering is legal. + return Dependences.find(Src) == Dependences.end() || + !Dependences.lookup(Src).count(Sink); + } + + /// Collect the dependences from LoopAccessInfo. + /// + /// We process the dependences once during the interleaved access analysis to + /// enable constant-time dependence queries. + void collectDependences() { + if (!areDependencesValid()) + return; + auto *Deps = LAI->getDepChecker().getDependences(); + for (auto Dep : *Deps) + Dependences[Dep.getSource(*LAI)].insert(Dep.getDestination(*LAI)); + } +}; + } // llvm namespace #endif diff --git a/contrib/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h b/contrib/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h new file mode 100644 index 00000000000..de44f41720e --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h @@ -0,0 +1,70 @@ +//===- AMDGPUMetadataVerifier.h - MsgPack Types -----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This is a verifier for AMDGPU HSA metadata, which can verify both +/// well-typed metadata and untyped metadata. When verifying in the non-strict +/// mode, untyped metadata is coerced into the correct type if possible. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H +#define LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H + +#include "llvm/BinaryFormat/MsgPackTypes.h" + +namespace llvm { +namespace AMDGPU { +namespace HSAMD { +namespace V3 { + +/// Verifier for AMDGPU HSA metadata. +/// +/// Operates in two modes: +/// +/// In strict mode, metadata must already be well-typed. +/// +/// In non-strict mode, metadata is coerced into expected types when possible. +class MetadataVerifier { + bool Strict; + + bool verifyScalar(msgpack::Node &Node, msgpack::ScalarNode::ScalarKind SKind, + function_ref verifyValue = {}); + bool verifyInteger(msgpack::Node &Node); + bool verifyArray(msgpack::Node &Node, + function_ref verifyNode, + Optional Size = None); + bool verifyEntry(msgpack::MapNode &MapNode, StringRef Key, bool Required, + function_ref verifyNode); + bool + verifyScalarEntry(msgpack::MapNode &MapNode, StringRef Key, bool Required, + msgpack::ScalarNode::ScalarKind SKind, + function_ref verifyValue = {}); + bool verifyIntegerEntry(msgpack::MapNode &MapNode, StringRef Key, + bool Required); + bool verifyKernelArgs(msgpack::Node &Node); + bool verifyKernel(msgpack::Node &Node); + +public: + /// Construct a MetadataVerifier, specifying whether it will operate in \p + /// Strict mode. + MetadataVerifier(bool Strict) : Strict(Strict) {} + + /// Verify given HSA metadata. + /// + /// \returns True when successful, false when metadata is invalid. + bool verify(msgpack::Node &HSAMetadataRoot); +}; + +} // end namespace V3 +} // end namespace HSAMD +} // end namespace AMDGPU +} // end namespace llvm + +#endif // LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H diff --git a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def index 944c5dd1c15..6ad3cb57f62 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def +++ b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def @@ -18,9 +18,11 @@ defined HANDLE_DW_VIRTUALITY || defined HANDLE_DW_DEFAULTED || \ defined HANDLE_DW_CC || defined HANDLE_DW_LNS || defined HANDLE_DW_LNE || \ defined HANDLE_DW_LNCT || defined HANDLE_DW_MACRO || \ - defined HANDLE_DW_RLE || defined HANDLE_DW_CFA || \ + defined HANDLE_DW_RLE || \ + (defined HANDLE_DW_CFA && defined HANDLE_DW_CFA_PRED) || \ defined HANDLE_DW_APPLE_PROPERTY || defined HANDLE_DW_UT || \ - defined HANDLE_DWARF_SECTION || defined HANDLE_DW_IDX) + defined HANDLE_DWARF_SECTION || defined HANDLE_DW_IDX || \ + defined HANDLE_DW_END) #error "Missing macro definition of HANDLE_DW*" #endif @@ -41,7 +43,7 @@ #endif #ifndef HANDLE_DW_LANG -#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR) +#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR) #endif #ifndef HANDLE_DW_ATE @@ -84,6 +86,10 @@ #define HANDLE_DW_CFA(ID, NAME) #endif +#ifndef HANDLE_DW_CFA_PRED +#define HANDLE_DW_CFA_PRED(ID, NAME, PRED) +#endif + #ifndef HANDLE_DW_APPLE_PROPERTY #define HANDLE_DW_APPLE_PROPERTY(ID, NAME) #endif @@ -100,6 +106,10 @@ #define HANDLE_DW_IDX(ID, NAME) #endif +#ifndef HANDLE_DW_END +#define HANDLE_DW_END(ID, NAME) +#endif + HANDLE_DW_TAG(0x0000, null, 2, DWARF) HANDLE_DW_TAG(0x0001, array_type, 2, DWARF) HANDLE_DW_TAG(0x0002, class_type, 2, DWARF) @@ -622,50 +632,50 @@ HANDLE_DW_OP(0xfb, GNU_addr_index, 0, GNU) HANDLE_DW_OP(0xfc, GNU_const_index, 0, GNU) // DWARF languages. -HANDLE_DW_LANG(0x0001, C89, 2, DWARF) -HANDLE_DW_LANG(0x0002, C, 2, DWARF) -HANDLE_DW_LANG(0x0003, Ada83, 2, DWARF) -HANDLE_DW_LANG(0x0004, C_plus_plus, 2, DWARF) -HANDLE_DW_LANG(0x0005, Cobol74, 2, DWARF) -HANDLE_DW_LANG(0x0006, Cobol85, 2, DWARF) -HANDLE_DW_LANG(0x0007, Fortran77, 2, DWARF) -HANDLE_DW_LANG(0x0008, Fortran90, 2, DWARF) -HANDLE_DW_LANG(0x0009, Pascal83, 2, DWARF) -HANDLE_DW_LANG(0x000a, Modula2, 2, DWARF) +HANDLE_DW_LANG(0x0001, C89, 0, 2, DWARF) +HANDLE_DW_LANG(0x0002, C, 0, 2, DWARF) +HANDLE_DW_LANG(0x0003, Ada83, 1, 2, DWARF) +HANDLE_DW_LANG(0x0004, C_plus_plus, 0, 2, DWARF) +HANDLE_DW_LANG(0x0005, Cobol74, 1, 2, DWARF) +HANDLE_DW_LANG(0x0006, Cobol85, 1, 2, DWARF) +HANDLE_DW_LANG(0x0007, Fortran77, 1, 2, DWARF) +HANDLE_DW_LANG(0x0008, Fortran90, 1, 2, DWARF) +HANDLE_DW_LANG(0x0009, Pascal83, 1, 2, DWARF) +HANDLE_DW_LANG(0x000a, Modula2, 1, 2, DWARF) // New in DWARF v3: -HANDLE_DW_LANG(0x000b, Java, 3, DWARF) -HANDLE_DW_LANG(0x000c, C99, 3, DWARF) -HANDLE_DW_LANG(0x000d, Ada95, 3, DWARF) -HANDLE_DW_LANG(0x000e, Fortran95, 3, DWARF) -HANDLE_DW_LANG(0x000f, PLI, 3, DWARF) -HANDLE_DW_LANG(0x0010, ObjC, 3, DWARF) -HANDLE_DW_LANG(0x0011, ObjC_plus_plus, 3, DWARF) -HANDLE_DW_LANG(0x0012, UPC, 3, DWARF) -HANDLE_DW_LANG(0x0013, D, 3, DWARF) +HANDLE_DW_LANG(0x000b, Java, 0, 3, DWARF) +HANDLE_DW_LANG(0x000c, C99, 0, 3, DWARF) +HANDLE_DW_LANG(0x000d, Ada95, 1, 3, DWARF) +HANDLE_DW_LANG(0x000e, Fortran95, 1, 3, DWARF) +HANDLE_DW_LANG(0x000f, PLI, 1, 3, DWARF) +HANDLE_DW_LANG(0x0010, ObjC, 0, 3, DWARF) +HANDLE_DW_LANG(0x0011, ObjC_plus_plus, 0, 3, DWARF) +HANDLE_DW_LANG(0x0012, UPC, 0, 3, DWARF) +HANDLE_DW_LANG(0x0013, D, 0, 3, DWARF) // New in DWARF v4: -HANDLE_DW_LANG(0x0014, Python, 4, DWARF) +HANDLE_DW_LANG(0x0014, Python, 0, 4, DWARF) // New in DWARF v5: -HANDLE_DW_LANG(0x0015, OpenCL, 5, DWARF) -HANDLE_DW_LANG(0x0016, Go, 5, DWARF) -HANDLE_DW_LANG(0x0017, Modula3, 5, DWARF) -HANDLE_DW_LANG(0x0018, Haskell, 5, DWARF) -HANDLE_DW_LANG(0x0019, C_plus_plus_03, 5, DWARF) -HANDLE_DW_LANG(0x001a, C_plus_plus_11, 5, DWARF) -HANDLE_DW_LANG(0x001b, OCaml, 5, DWARF) -HANDLE_DW_LANG(0x001c, Rust, 5, DWARF) -HANDLE_DW_LANG(0x001d, C11, 5, DWARF) -HANDLE_DW_LANG(0x001e, Swift, 5, DWARF) -HANDLE_DW_LANG(0x001f, Julia, 5, DWARF) -HANDLE_DW_LANG(0x0020, Dylan, 5, DWARF) -HANDLE_DW_LANG(0x0021, C_plus_plus_14, 5, DWARF) -HANDLE_DW_LANG(0x0022, Fortran03, 5, DWARF) -HANDLE_DW_LANG(0x0023, Fortran08, 5, DWARF) -HANDLE_DW_LANG(0x0024, RenderScript, 5, DWARF) -HANDLE_DW_LANG(0x0025, BLISS, 5, DWARF) +HANDLE_DW_LANG(0x0015, OpenCL, 0, 5, DWARF) +HANDLE_DW_LANG(0x0016, Go, 0, 5, DWARF) +HANDLE_DW_LANG(0x0017, Modula3, 1, 5, DWARF) +HANDLE_DW_LANG(0x0018, Haskell, 0, 5, DWARF) +HANDLE_DW_LANG(0x0019, C_plus_plus_03, 0, 5, DWARF) +HANDLE_DW_LANG(0x001a, C_plus_plus_11, 0, 5, DWARF) +HANDLE_DW_LANG(0x001b, OCaml, 0, 5, DWARF) +HANDLE_DW_LANG(0x001c, Rust, 0, 5, DWARF) +HANDLE_DW_LANG(0x001d, C11, 0, 5, DWARF) +HANDLE_DW_LANG(0x001e, Swift, 0, 5, DWARF) +HANDLE_DW_LANG(0x001f, Julia, 1, 5, DWARF) +HANDLE_DW_LANG(0x0020, Dylan, 0, 5, DWARF) +HANDLE_DW_LANG(0x0021, C_plus_plus_14, 0, 5, DWARF) +HANDLE_DW_LANG(0x0022, Fortran03, 1, 5, DWARF) +HANDLE_DW_LANG(0x0023, Fortran08, 1, 5, DWARF) +HANDLE_DW_LANG(0x0024, RenderScript, 0, 5, DWARF) +HANDLE_DW_LANG(0x0025, BLISS, 0, 5, DWARF) // Vendor extensions: -HANDLE_DW_LANG(0x8001, Mips_Assembler, 0, MIPS) -HANDLE_DW_LANG(0x8e57, GOOGLE_RenderScript, 0, GOOGLE) -HANDLE_DW_LANG(0xb000, BORLAND_Delphi, 0, BORLAND) +HANDLE_DW_LANG(0x8001, Mips_Assembler, None, 0, MIPS) +HANDLE_DW_LANG(0x8e57, GOOGLE_RenderScript, 0, 0, GOOGLE) +HANDLE_DW_LANG(0xb000, BORLAND_Delphi, 0, 0, BORLAND) // DWARF attribute type encodings. HANDLE_DW_ATE(0x01, address, 2, DWARF) @@ -690,6 +700,11 @@ HANDLE_DW_ATE(0x10, UTF, 4, DWARF) HANDLE_DW_ATE(0x11, UCS, 5, DWARF) HANDLE_DW_ATE(0x12, ASCII, 5, DWARF) +// DWARF attribute endianity +HANDLE_DW_END(0x00, default) +HANDLE_DW_END(0x01, big) +HANDLE_DW_END(0x02, little) + // DWARF virtuality codes. HANDLE_DW_VIRTUALITY(0x00, none) HANDLE_DW_VIRTUALITY(0x01, virtual) @@ -821,9 +836,10 @@ HANDLE_DW_CFA(0x14, val_offset) HANDLE_DW_CFA(0x15, val_offset_sf) HANDLE_DW_CFA(0x16, val_expression) // Vendor extensions: -HANDLE_DW_CFA(0x1d, MIPS_advance_loc8) -HANDLE_DW_CFA(0x2d, GNU_window_save) -HANDLE_DW_CFA(0x2e, GNU_args_size) +HANDLE_DW_CFA_PRED(0x1d, MIPS_advance_loc8, SELECT_MIPS64) +HANDLE_DW_CFA_PRED(0x2d, GNU_window_save, SELECT_SPARC) +HANDLE_DW_CFA_PRED(0x2d, AARCH64_negate_ra_state, SELECT_AARCH64) +HANDLE_DW_CFA_PRED(0x2e, GNU_args_size, SELECT_X86) // Apple Objective-C Property Attributes. // Keep this list in sync with clang's DeclSpec.h ObjCPropertyAttributeKind! @@ -863,6 +879,7 @@ HANDLE_DWARF_SECTION(DebugTypes, ".debug_types", "debug-types") HANDLE_DWARF_SECTION(DebugLine, ".debug_line", "debug-line") HANDLE_DWARF_SECTION(DebugLineStr, ".debug_line_str", "debug-line-str") HANDLE_DWARF_SECTION(DebugLoc, ".debug_loc", "debug-loc") +HANDLE_DWARF_SECTION(DebugLoclists, ".debug_loclists", "debug-loclists") HANDLE_DWARF_SECTION(DebugFrame, ".debug_frame", "debug-frame") HANDLE_DWARF_SECTION(DebugMacro, ".debug_macro", "debug-macro") HANDLE_DWARF_SECTION(DebugNames, ".debug_names", "debug-names") @@ -905,7 +922,9 @@ HANDLE_DW_IDX(0x05, type_hash) #undef HANDLE_DW_MACRO #undef HANDLE_DW_RLE #undef HANDLE_DW_CFA +#undef HANDLE_DW_CFA_PRED #undef HANDLE_DW_APPLE_PROPERTY #undef HANDLE_DW_UT #undef HANDLE_DWARF_SECTION #undef HANDLE_DW_IDX +#undef HANDLE_DW_END diff --git a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h index 9036f405eae..525a04d5e6c 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h +++ b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h @@ -26,6 +26,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/FormatVariadicDetails.h" +#include "llvm/ADT/Triple.h" namespace llvm { class StringRef; @@ -150,9 +151,8 @@ enum DecimalSignEncoding { enum EndianityEncoding { // Endianity attribute values - DW_END_default = 0x00, - DW_END_big = 0x01, - DW_END_little = 0x02, +#define HANDLE_DW_END(ID, NAME) DW_END_##NAME = ID, +#include "llvm/BinaryFormat/Dwarf.def" DW_END_lo_user = 0x40, DW_END_hi_user = 0xff }; @@ -184,7 +184,8 @@ enum DefaultedMemberAttribute { }; enum SourceLanguage { -#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR) DW_LANG_##NAME = ID, +#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR) \ + DW_LANG_##NAME = ID, #include "llvm/BinaryFormat/Dwarf.def" DW_LANG_lo_user = 0x8000, DW_LANG_hi_user = 0xffff @@ -273,6 +274,7 @@ enum RangeListEntries { /// Call frame instruction encodings. enum CallFrameInfo { #define HANDLE_DW_CFA(ID, NAME) DW_CFA_##NAME = ID, +#define HANDLE_DW_CFA_PRED(ID, NAME, ARCH) DW_CFA_##NAME = ID, #include "llvm/BinaryFormat/Dwarf.def" DW_CFA_extended = 0x00, @@ -431,7 +433,7 @@ StringRef LNStandardString(unsigned Standard); StringRef LNExtendedString(unsigned Encoding); StringRef MacinfoString(unsigned Encoding); StringRef RangeListEncodingString(unsigned Encoding); -StringRef CallFrameString(unsigned Encoding); +StringRef CallFrameString(unsigned Encoding, Triple::ArchType Arch); StringRef ApplePropertyString(unsigned); StringRef UnitTypeString(unsigned); StringRef AtomTypeString(unsigned Atom); @@ -489,6 +491,8 @@ unsigned AttributeEncodingVendor(TypeKind E); unsigned LanguageVendor(SourceLanguage L); /// @} +Optional LanguageLowerBound(SourceLanguage L); + /// A helper struct providing information about the byte size of DW_FORM /// values that vary in size depending on the DWARF version, address byte /// size, or DWARF32/DWARF64. diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELF.h b/contrib/llvm/include/llvm/BinaryFormat/ELF.h index 2e778779117..ce35d127d43 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/ELF.h +++ b/contrib/llvm/include/llvm/BinaryFormat/ELF.h @@ -582,6 +582,7 @@ enum { EF_HEXAGON_MACH_V60 = 0x00000060, // Hexagon V60 EF_HEXAGON_MACH_V62 = 0x00000062, // Hexagon V62 EF_HEXAGON_MACH_V65 = 0x00000065, // Hexagon V65 + EF_HEXAGON_MACH_V66 = 0x00000066, // Hexagon V66 // Highest ISA version flags EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[11:0] @@ -594,6 +595,7 @@ enum { EF_HEXAGON_ISA_V60 = 0x00000060, // Hexagon V60 ISA EF_HEXAGON_ISA_V62 = 0x00000062, // Hexagon V62 ISA EF_HEXAGON_ISA_V65 = 0x00000065, // Hexagon V65 ISA + EF_HEXAGON_ISA_V66 = 0x00000066, // Hexagon V66 ISA }; // Hexagon-specific section indexes for common small data @@ -701,6 +703,7 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX902 = 0x02d, EF_AMDGPU_MACH_AMDGCN_GFX904 = 0x02e, EF_AMDGPU_MACH_AMDGCN_GFX906 = 0x02f, + EF_AMDGPU_MACH_AMDGCN_GFX909 = 0x031, // Reserved for AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_RESERVED0 = 0x027, @@ -708,11 +711,14 @@ enum : unsigned { // First/last AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600, - EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX906, + EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX909, - // Indicates if the xnack target feature is enabled for all code contained in - // the object. + // Indicates if the "xnack" target feature is enabled for all code contained + // in the object. EF_AMDGPU_XNACK = 0x100, + // Indicates if the "sram-ecc" target feature is enabled for all code + // contained in the object. + EF_AMDGPU_SRAM_ECC = 0x200, }; // ELF Relocation types for AMDGPU @@ -725,6 +731,38 @@ enum { #include "ELFRelocs/BPF.def" }; +// MSP430 specific e_flags +enum : unsigned { + EF_MSP430_MACH_MSP430x11 = 11, + EF_MSP430_MACH_MSP430x11x1 = 110, + EF_MSP430_MACH_MSP430x12 = 12, + EF_MSP430_MACH_MSP430x13 = 13, + EF_MSP430_MACH_MSP430x14 = 14, + EF_MSP430_MACH_MSP430x15 = 15, + EF_MSP430_MACH_MSP430x16 = 16, + EF_MSP430_MACH_MSP430x20 = 20, + EF_MSP430_MACH_MSP430x22 = 22, + EF_MSP430_MACH_MSP430x23 = 23, + EF_MSP430_MACH_MSP430x24 = 24, + EF_MSP430_MACH_MSP430x26 = 26, + EF_MSP430_MACH_MSP430x31 = 31, + EF_MSP430_MACH_MSP430x32 = 32, + EF_MSP430_MACH_MSP430x33 = 33, + EF_MSP430_MACH_MSP430x41 = 41, + EF_MSP430_MACH_MSP430x42 = 42, + EF_MSP430_MACH_MSP430x43 = 43, + EF_MSP430_MACH_MSP430x44 = 44, + EF_MSP430_MACH_MSP430X = 45, + EF_MSP430_MACH_MSP430x46 = 46, + EF_MSP430_MACH_MSP430x47 = 47, + EF_MSP430_MACH_MSP430x54 = 54, +}; + +// ELF Relocation types for MSP430 +enum { +#include "ELFRelocs/MSP430.def" +}; + #undef ELF_RELOC // Section header. @@ -829,6 +867,8 @@ enum : unsigned { SHT_MIPS_DWARF = 0x7000001e, // DWARF debugging section. SHT_MIPS_ABIFLAGS = 0x7000002a, // ABI information. + SHT_MSP430_ATTRIBUTES = 0x70000003U, + SHT_HIPROC = 0x7fffffff, // Highest processor arch-specific type. SHT_LOUSER = 0x80000000, // Lowest type reserved for applications. SHT_HIUSER = 0xffffffff // Highest type reserved for applications. @@ -1321,7 +1361,7 @@ enum { GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1 }; -// AMDGPU specific notes. +// AMD specific notes. (Code Object V2) enum { // Note types with values between 0 and 9 (inclusive) are reserved. NT_AMD_AMDGPU_HSA_METADATA = 10, @@ -1329,6 +1369,12 @@ enum { NT_AMD_AMDGPU_PAL_METADATA = 12 }; +// AMDGPU specific notes. (Code Object V3) +enum { + // Note types with values between 0 and 31 (inclusive) are reserved. + NT_AMDGPU_METADATA = 32 +}; + enum { GNU_ABI_TAG_LINUX = 0, GNU_ABI_TAG_HURD = 1, @@ -1339,6 +1385,8 @@ enum { GNU_ABI_TAG_NACL = 6, }; +constexpr const char *ELF_NOTE_GNU = "GNU"; + // Android packed relocation group flags. enum { RELOCATION_GROUPED_BY_INFO_FLAG = 1, diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/MSP430.def b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/MSP430.def new file mode 100644 index 00000000000..96990abf2db --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/MSP430.def @@ -0,0 +1,16 @@ + +#ifndef ELF_RELOC +#error "ELF_RELOC must be defined" +#endif + +ELF_RELOC(R_MSP430_NONE, 0) +ELF_RELOC(R_MSP430_32, 1) +ELF_RELOC(R_MSP430_10_PCREL, 2) +ELF_RELOC(R_MSP430_16, 3) +ELF_RELOC(R_MSP430_16_PCREL, 4) +ELF_RELOC(R_MSP430_16_BYTE, 5) +ELF_RELOC(R_MSP430_16_PCREL_BYTE, 6) +ELF_RELOC(R_MSP430_2X_PCREL, 7) +ELF_RELOC(R_MSP430_RL_PCREL, 8) +ELF_RELOC(R_MSP430_8, 9) +ELF_RELOC(R_MSP430_SYM_DIFF, 10) diff --git a/contrib/llvm/include/llvm/BinaryFormat/MachO.h b/contrib/llvm/include/llvm/BinaryFormat/MachO.h index c5294c76ebf..b3d60984249 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/MachO.h +++ b/contrib/llvm/include/llvm/BinaryFormat/MachO.h @@ -486,7 +486,10 @@ enum PlatformType { PLATFORM_IOS = 2, PLATFORM_TVOS = 3, PLATFORM_WATCHOS = 4, - PLATFORM_BRIDGEOS = 5 + PLATFORM_BRIDGEOS = 5, + PLATFORM_IOSSIMULATOR = 7, + PLATFORM_TVOSSIMULATOR = 8, + PLATFORM_WATCHOSSIMULATOR = 9 }; // Values for tools enum in build_tool_version. diff --git a/contrib/llvm/include/llvm/BinaryFormat/MsgPack.def b/contrib/llvm/include/llvm/BinaryFormat/MsgPack.def new file mode 100644 index 00000000000..781b49f46ae --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/MsgPack.def @@ -0,0 +1,108 @@ +//===- MsgPack.def - MessagePack definitions --------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Macros for running through MessagePack enumerators. +/// +//===----------------------------------------------------------------------===// + +#if !( \ + defined HANDLE_MP_FIRST_BYTE || defined HANDLE_MP_FIX_BITS || \ + defined HANDLE_MP_FIX_BITS_MASK || defined HANDLE_MP_FIX_MAX || \ + defined HANDLE_MP_FIX_LEN || defined HANDLE_MP_FIX_MIN) +#error "Missing macro definition of HANDLE_MP*" +#endif + +#ifndef HANDLE_MP_FIRST_BYTE +#define HANDLE_MP_FIRST_BYTE(ID, NAME) +#endif + +#ifndef HANDLE_MP_FIX_BITS +#define HANDLE_MP_FIX_BITS(ID, NAME) +#endif + +#ifndef HANDLE_MP_FIX_BITS_MASK +#define HANDLE_MP_FIX_BITS_MASK(ID, NAME) +#endif + +#ifndef HANDLE_MP_FIX_MAX +#define HANDLE_MP_FIX_MAX(ID, NAME) +#endif + +#ifndef HANDLE_MP_FIX_LEN +#define HANDLE_MP_FIX_LEN(ID, NAME) +#endif + +#ifndef HANDLE_MP_FIX_MIN +#define HANDLE_MP_FIX_MIN(ID, NAME) +#endif + +HANDLE_MP_FIRST_BYTE(0xc0, Nil) +HANDLE_MP_FIRST_BYTE(0xc2, False) +HANDLE_MP_FIRST_BYTE(0xc3, True) +HANDLE_MP_FIRST_BYTE(0xc4, Bin8) +HANDLE_MP_FIRST_BYTE(0xc5, Bin16) +HANDLE_MP_FIRST_BYTE(0xc6, Bin32) +HANDLE_MP_FIRST_BYTE(0xc7, Ext8) +HANDLE_MP_FIRST_BYTE(0xc8, Ext16) +HANDLE_MP_FIRST_BYTE(0xc9, Ext32) +HANDLE_MP_FIRST_BYTE(0xca, Float32) +HANDLE_MP_FIRST_BYTE(0xcb, Float64) +HANDLE_MP_FIRST_BYTE(0xcc, UInt8) +HANDLE_MP_FIRST_BYTE(0xcd, UInt16) +HANDLE_MP_FIRST_BYTE(0xce, UInt32) +HANDLE_MP_FIRST_BYTE(0xcf, UInt64) +HANDLE_MP_FIRST_BYTE(0xd0, Int8) +HANDLE_MP_FIRST_BYTE(0xd1, Int16) +HANDLE_MP_FIRST_BYTE(0xd2, Int32) +HANDLE_MP_FIRST_BYTE(0xd3, Int64) +HANDLE_MP_FIRST_BYTE(0xd4, FixExt1) +HANDLE_MP_FIRST_BYTE(0xd5, FixExt2) +HANDLE_MP_FIRST_BYTE(0xd6, FixExt4) +HANDLE_MP_FIRST_BYTE(0xd7, FixExt8) +HANDLE_MP_FIRST_BYTE(0xd8, FixExt16) +HANDLE_MP_FIRST_BYTE(0xd9, Str8) +HANDLE_MP_FIRST_BYTE(0xda, Str16) +HANDLE_MP_FIRST_BYTE(0xdb, Str32) +HANDLE_MP_FIRST_BYTE(0xdc, Array16) +HANDLE_MP_FIRST_BYTE(0xdd, Array32) +HANDLE_MP_FIRST_BYTE(0xde, Map16) +HANDLE_MP_FIRST_BYTE(0xdf, Map32) + +HANDLE_MP_FIX_BITS(0x00, PositiveInt) +HANDLE_MP_FIX_BITS(0x80, Map) +HANDLE_MP_FIX_BITS(0x90, Array) +HANDLE_MP_FIX_BITS(0xa0, String) +HANDLE_MP_FIX_BITS(0xe0, NegativeInt) + +HANDLE_MP_FIX_BITS_MASK(0x80, PositiveInt) +HANDLE_MP_FIX_BITS_MASK(0xf0, Map) +HANDLE_MP_FIX_BITS_MASK(0xf0, Array) +HANDLE_MP_FIX_BITS_MASK(0xe0, String) +HANDLE_MP_FIX_BITS_MASK(0xe0, NegativeInt) + +HANDLE_MP_FIX_MAX(0x7f, PositiveInt) +HANDLE_MP_FIX_MAX(0x0f, Map) +HANDLE_MP_FIX_MAX(0x0f, Array) +HANDLE_MP_FIX_MAX(0x1f, String) + +HANDLE_MP_FIX_LEN(0x01, Ext1) +HANDLE_MP_FIX_LEN(0x02, Ext2) +HANDLE_MP_FIX_LEN(0x04, Ext4) +HANDLE_MP_FIX_LEN(0x08, Ext8) +HANDLE_MP_FIX_LEN(0x10, Ext16) + +HANDLE_MP_FIX_MIN(-0x20, NegativeInt) + +#undef HANDLE_MP_FIRST_BYTE +#undef HANDLE_MP_FIX_BITS +#undef HANDLE_MP_FIX_BITS_MASK +#undef HANDLE_MP_FIX_MAX +#undef HANDLE_MP_FIX_LEN +#undef HANDLE_MP_FIX_MIN diff --git a/contrib/llvm/include/llvm/BinaryFormat/MsgPack.h b/contrib/llvm/include/llvm/BinaryFormat/MsgPack.h new file mode 100644 index 00000000000..d431912a53e --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/MsgPack.h @@ -0,0 +1,93 @@ +//===-- MsgPack.h - MessagePack Constants -----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains constants used for implementing MessagePack support. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_MSGPACK_H +#define LLVM_BINARYFORMAT_MSGPACK_H + +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/Endian.h" + +namespace llvm { +namespace msgpack { + +/// The endianness of all multi-byte encoded values in MessagePack. +constexpr support::endianness Endianness = support::big; + +/// The first byte identifiers of MessagePack object formats. +namespace FirstByte { +#define HANDLE_MP_FIRST_BYTE(ID, NAME) constexpr uint8_t NAME = ID; +#include "llvm/BinaryFormat/MsgPack.def" +} + +/// Most significant bits used to identify "Fix" variants in MessagePack. +/// +/// For example, FixStr objects encode their size in the five least significant +/// bits of their first byte, which is identified by the bit pattern "101" in +/// the three most significant bits. So FixBits::String contains 0b10100000. +/// +/// A corresponding mask of the bit pattern is found in \c FixBitsMask. +namespace FixBits { +#define HANDLE_MP_FIX_BITS(ID, NAME) constexpr uint8_t NAME = ID; +#include "llvm/BinaryFormat/MsgPack.def" +} + +/// Mask of bits used to identify "Fix" variants in MessagePack. +/// +/// For example, FixStr objects encode their size in the five least significant +/// bits of their first byte, which is identified by the bit pattern "101" in +/// the three most significant bits. So FixBitsMask::String contains +/// 0b11100000. +/// +/// The corresponding bit pattern to mask for is found in FixBits. +namespace FixBitsMask { +#define HANDLE_MP_FIX_BITS_MASK(ID, NAME) constexpr uint8_t NAME = ID; +#include "llvm/BinaryFormat/MsgPack.def" +} + +/// The maximum value or size encodable in "Fix" variants of formats. +/// +/// For example, FixStr objects encode their size in the five least significant +/// bits of their first byte, so the largest encodable size is 0b00011111. +namespace FixMax { +#define HANDLE_MP_FIX_MAX(ID, NAME) constexpr uint8_t NAME = ID; +#include "llvm/BinaryFormat/MsgPack.def" +} + +/// The exact size encodable in "Fix" variants of formats. +/// +/// The only objects for which an exact size makes sense are of Extension type. +/// +/// For example, FixExt4 stores an extension type containing exactly four bytes. +namespace FixLen { +#define HANDLE_MP_FIX_LEN(ID, NAME) constexpr uint8_t NAME = ID; +#include "llvm/BinaryFormat/MsgPack.def" +} + +/// The minimum value or size encodable in "Fix" variants of formats. +/// +/// The only object for which a minimum makes sense is a negative FixNum. +/// +/// Negative FixNum objects encode their signed integer value in one byte, but +/// they must have the pattern "111" as their three most significant bits. This +/// means all values are negative, and the smallest representable value is +/// 0b11100000. +namespace FixMin { +#define HANDLE_MP_FIX_MIN(ID, NAME) constexpr int8_t NAME = ID; +#include "llvm/BinaryFormat/MsgPack.def" +} + +} // end namespace msgpack +} // end namespace llvm + +#endif // LLVM_BINARYFORMAT_MSGPACK_H diff --git a/contrib/llvm/include/llvm/BinaryFormat/MsgPackReader.h b/contrib/llvm/include/llvm/BinaryFormat/MsgPackReader.h new file mode 100644 index 00000000000..511c3140745 --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/MsgPackReader.h @@ -0,0 +1,148 @@ +//===- MsgPackReader.h - Simple MsgPack reader ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This is a MessagePack reader. +/// +/// See https://github.com/msgpack/msgpack/blob/master/spec.md for the full +/// standard. +/// +/// Typical usage: +/// \code +/// StringRef input = GetInput(); +/// msgpack::Reader MPReader(input); +/// msgpack::Object Obj; +/// +/// while (MPReader.read(Obj)) { +/// switch (Obj.Kind) { +/// case msgpack::Type::Int: +// // Use Obj.Int +/// break; +/// // ... +/// } +/// } +/// \endcode +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_MSGPACKREADER_H +#define LLVM_SUPPORT_MSGPACKREADER_H + +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" +#include + +namespace llvm { +namespace msgpack { + +/// MessagePack types as defined in the standard, with the exception of Integer +/// being divided into a signed Int and unsigned UInt variant in order to map +/// directly to C++ types. +/// +/// The types map onto corresponding union members of the \c Object struct. +enum class Type : uint8_t { + Int, + UInt, + Nil, + Boolean, + Float, + String, + Binary, + Array, + Map, + Extension, +}; + +/// Extension types are composed of a user-defined type ID and an uninterpreted +/// sequence of bytes. +struct ExtensionType { + /// User-defined extension type. + int8_t Type; + /// Raw bytes of the extension object. + StringRef Bytes; +}; + +/// MessagePack object, represented as a tagged union of C++ types. +/// +/// All types except \c Type::Nil (which has only one value, and so is +/// completely represented by the \c Kind itself) map to a exactly one union +/// member. +struct Object { + Type Kind; + union { + /// Value for \c Type::Int. + int64_t Int; + /// Value for \c Type::Uint. + uint64_t UInt; + /// Value for \c Type::Boolean. + bool Bool; + /// Value for \c Type::Float. + double Float; + /// Value for \c Type::String and \c Type::Binary. + StringRef Raw; + /// Value for \c Type::Array and \c Type::Map. + size_t Length; + /// Value for \c Type::Extension. + ExtensionType Extension; + }; + + Object() : Kind(Type::Int), Int(0) {} +}; + +/// Reads MessagePack objects from memory, one at a time. +class Reader { +public: + /// Construct a reader, keeping a reference to the \p InputBuffer. + Reader(MemoryBufferRef InputBuffer); + /// Construct a reader, keeping a reference to the \p Input. + Reader(StringRef Input); + + Reader(const Reader &) = delete; + Reader &operator=(const Reader &) = delete; + + /// Read one object from the input buffer, advancing past it. + /// + /// The \p Obj is updated with the kind of the object read, and the + /// corresponding union member is updated. + /// + /// For the collection objects (Array and Map), only the length is read, and + /// the caller must make and additional \c N calls (in the case of Array) or + /// \c N*2 calls (in the case of Map) to \c Read to retrieve the collection + /// elements. + /// + /// \param [out] Obj filled with next object on success. + /// + /// \returns true when object successfully read, false when at end of + /// input (and so \p Obj was not updated), otherwise an error. + Expected read(Object &Obj); + +private: + MemoryBufferRef InputBuffer; + StringRef::iterator Current; + StringRef::iterator End; + + size_t remainingSpace() { + // The rest of the code maintains the invariant that End >= Current, so + // that this cast is always defined behavior. + return static_cast(End - Current); + } + + template Expected readRaw(Object &Obj); + template Expected readInt(Object &Obj); + template Expected readUInt(Object &Obj); + template Expected readLength(Object &Obj); + template Expected readExt(Object &Obj); + Expected createRaw(Object &Obj, uint32_t Size); + Expected createExt(Object &Obj, uint32_t Size); +}; + +} // end namespace msgpack +} // end namespace llvm + +#endif // LLVM_SUPPORT_MSGPACKREADER_H diff --git a/contrib/llvm/include/llvm/BinaryFormat/MsgPackTypes.h b/contrib/llvm/include/llvm/BinaryFormat/MsgPackTypes.h new file mode 100644 index 00000000000..f96cd4c338f --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/MsgPackTypes.h @@ -0,0 +1,372 @@ +//===- MsgPackTypes.h - MsgPack Types ---------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This is a data structure for representing MessagePack "documents", with +/// methods to go to and from MessagePack. The types also specialize YAMLIO +/// traits in order to go to and from YAML. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Optional.h" +#include "llvm/BinaryFormat/MsgPackReader.h" +#include "llvm/BinaryFormat/MsgPackWriter.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/YAMLTraits.h" +#include + +#ifndef LLVM_BINARYFORMAT_MSGPACKTYPES_H +#define LLVM_BINARYFORMAT_MSGPACKTYPES_H + +namespace llvm { +namespace msgpack { + +class Node; + +/// Short-hand for a Node pointer. +using NodePtr = std::shared_ptr; + +/// Short-hand for an Optional Node pointer. +using OptNodePtr = Optional; + +/// Abstract base-class which can be any MessagePack type. +class Node { +public: + enum NodeKind { + NK_Scalar, + NK_Array, + NK_Map, + }; + +private: + virtual void anchor() = 0; + const NodeKind Kind; + + static Expected readArray(Reader &MPReader, size_t Length); + static Expected readMap(Reader &MPReader, size_t Length); + +public: + NodeKind getKind() const { return Kind; } + + /// Construct a Node. Used by derived classes to track kind information. + Node(NodeKind Kind) : Kind(Kind) {} + + virtual ~Node() = default; + + /// Read from a MessagePack reader \p MPReader, returning an error if one is + /// encountered, or None if \p MPReader is at the end of stream, or some Node + /// pointer if some type is read. + static Expected read(Reader &MPReader); + + /// Write to a MessagePack writer \p MPWriter. + virtual void write(Writer &MPWriter) = 0; +}; + +/// A MessagePack scalar. +class ScalarNode : public Node { +public: + enum ScalarKind { + SK_Int, + SK_UInt, + SK_Nil, + SK_Boolean, + SK_Float, + SK_String, + SK_Binary, + }; + +private: + void anchor() override; + + void destroy(); + + ScalarKind SKind; + + union { + int64_t IntValue; + uint64_t UIntValue; + bool BoolValue; + double FloatValue; + std::string StringValue; + }; + +public: + /// Construct an Int ScalarNode. + ScalarNode(int64_t IntValue); + /// Construct an Int ScalarNode. + ScalarNode(int32_t IntValue); + /// Construct an UInt ScalarNode. + ScalarNode(uint64_t UIntValue); + /// Construct an UInt ScalarNode. + ScalarNode(uint32_t UIntValue); + /// Construct a Nil ScalarNode. + ScalarNode(); + /// Construct a Boolean ScalarNode. + ScalarNode(bool BoolValue); + /// Construct a Float ScalarNode. + ScalarNode(double FloatValue); + /// Construct a String ScalarNode. + ScalarNode(StringRef StringValue); + /// Construct a String ScalarNode. + ScalarNode(const char *StringValue); + /// Construct a String ScalarNode. + ScalarNode(std::string &&StringValue); + /// Construct a Binary ScalarNode. + ScalarNode(MemoryBufferRef BinaryValue); + + ~ScalarNode(); + + ScalarNode &operator=(const ScalarNode &RHS) = delete; + /// A ScalarNode can only be move assigned. + ScalarNode &operator=(ScalarNode &&RHS); + + /// Change the kind of this ScalarNode, zero initializing it to the new type. + void setScalarKind(ScalarKind SKind) { + switch (SKind) { + case SK_Int: + *this = int64_t(0); + break; + case SK_UInt: + *this = uint64_t(0); + break; + case SK_Boolean: + *this = false; + break; + case SK_Float: + *this = 0.0; + break; + case SK_String: + *this = StringRef(); + break; + case SK_Binary: + *this = MemoryBufferRef("", ""); + break; + case SK_Nil: + *this = ScalarNode(); + break; + } + } + + /// Get the current kind of ScalarNode. + ScalarKind getScalarKind() { return SKind; } + + /// Get the value of an Int scalar. + /// + /// \warning Assumes getScalarKind() == SK_Int + int64_t getInt() { + assert(SKind == SK_Int); + return IntValue; + } + + /// Get the value of a UInt scalar. + /// + /// \warning Assumes getScalarKind() == SK_UInt + uint64_t getUInt() { + assert(SKind == SK_UInt); + return UIntValue; + } + + /// Get the value of an Boolean scalar. + /// + /// \warning Assumes getScalarKind() == SK_Boolean + bool getBool() { + assert(SKind == SK_Boolean); + return BoolValue; + } + + /// Get the value of an Float scalar. + /// + /// \warning Assumes getScalarKind() == SK_Float + double getFloat() { + assert(SKind == SK_Float); + return FloatValue; + } + + /// Get the value of a String scalar. + /// + /// \warning Assumes getScalarKind() == SK_String + StringRef getString() { + assert(SKind == SK_String); + return StringValue; + } + + /// Get the value of a Binary scalar. + /// + /// \warning Assumes getScalarKind() == SK_Binary + StringRef getBinary() { + assert(SKind == SK_Binary); + return StringValue; + } + + static bool classof(const Node *N) { return N->getKind() == NK_Scalar; } + + void write(Writer &MPWriter) override; + + /// Parse a YAML scalar of the current ScalarKind from \p ScalarStr. + /// + /// \returns An empty string on success, otherwise an error message. + StringRef inputYAML(StringRef ScalarStr); + + /// Output a YAML scalar of the current ScalarKind into \p OS. + void outputYAML(raw_ostream &OS) const; + + /// Determine which YAML quoting type the current value would need when + /// output. + yaml::QuotingType mustQuoteYAML(StringRef ScalarStr) const; + + /// Get the YAML tag for the current ScalarKind. + StringRef getYAMLTag() const; + + /// Flag which affects how the type handles YAML tags when reading and + /// writing. + /// + /// When false, tags are used when reading and writing. When reading, the tag + /// is used to decide the ScalarKind before parsing. When writing, the tag is + /// output along with the value. + /// + /// When true, tags are ignored when reading and writing. When reading, the + /// ScalarKind is always assumed to be String. When writing, the tag is not + /// output. + bool IgnoreTag = false; + + static const char *IntTag; + static const char *NilTag; + static const char *BooleanTag; + static const char *FloatTag; + static const char *StringTag; + static const char *BinaryTag; +}; + +class ArrayNode : public Node, public std::vector { + void anchor() override; + +public: + ArrayNode() : Node(NK_Array) {} + static bool classof(const Node *N) { return N->getKind() == NK_Array; } + + void write(Writer &MPWriter) override { + MPWriter.writeArraySize(this->size()); + for (auto &N : *this) + N->write(MPWriter); + } +}; + +class MapNode : public Node, public StringMap { + void anchor() override; + +public: + MapNode() : Node(NK_Map) {} + static bool classof(const Node *N) { return N->getKind() == NK_Map; } + + void write(Writer &MPWriter) override { + MPWriter.writeMapSize(this->size()); + for (auto &N : *this) { + MPWriter.write(N.first()); + N.second->write(MPWriter); + } + } +}; + +} // end namespace msgpack + +namespace yaml { + +template <> struct PolymorphicTraits { + static NodeKind getKind(const msgpack::NodePtr &N) { + if (isa(*N)) + return NodeKind::Scalar; + if (isa(*N)) + return NodeKind::Map; + if (isa(*N)) + return NodeKind::Sequence; + llvm_unreachable("NodeKind not supported"); + } + static msgpack::ScalarNode &getAsScalar(msgpack::NodePtr &N) { + if (!N || !isa(*N)) + N.reset(new msgpack::ScalarNode()); + return *cast(N.get()); + } + static msgpack::MapNode &getAsMap(msgpack::NodePtr &N) { + if (!N || !isa(*N)) + N.reset(new msgpack::MapNode()); + return *cast(N.get()); + } + static msgpack::ArrayNode &getAsSequence(msgpack::NodePtr &N) { + if (!N || !isa(*N)) + N.reset(new msgpack::ArrayNode()); + return *cast(N.get()); + } +}; + +template <> struct TaggedScalarTraits { + static void output(const msgpack::ScalarNode &S, void *Ctxt, + raw_ostream &ScalarOS, raw_ostream &TagOS) { + if (!S.IgnoreTag) + TagOS << S.getYAMLTag(); + S.outputYAML(ScalarOS); + } + + static StringRef input(StringRef ScalarStr, StringRef Tag, void *Ctxt, + msgpack::ScalarNode &S) { + if (Tag == msgpack::ScalarNode::IntTag) { + S.setScalarKind(msgpack::ScalarNode::SK_UInt); + if (S.inputYAML(ScalarStr) == StringRef()) + return StringRef(); + S.setScalarKind(msgpack::ScalarNode::SK_Int); + return S.inputYAML(ScalarStr); + } + + if (S.IgnoreTag || Tag == msgpack::ScalarNode::StringTag || + Tag == "tag:yaml.org,2002:str") + S.setScalarKind(msgpack::ScalarNode::SK_String); + else if (Tag == msgpack::ScalarNode::NilTag) + S.setScalarKind(msgpack::ScalarNode::SK_Nil); + else if (Tag == msgpack::ScalarNode::BooleanTag) + S.setScalarKind(msgpack::ScalarNode::SK_Boolean); + else if (Tag == msgpack::ScalarNode::FloatTag) + S.setScalarKind(msgpack::ScalarNode::SK_Float); + else if (Tag == msgpack::ScalarNode::StringTag) + S.setScalarKind(msgpack::ScalarNode::SK_String); + else if (Tag == msgpack::ScalarNode::BinaryTag) + S.setScalarKind(msgpack::ScalarNode::SK_Binary); + else + return "Unsupported messagepack tag"; + + return S.inputYAML(ScalarStr); + } + + static QuotingType mustQuote(const msgpack::ScalarNode &S, StringRef Str) { + return S.mustQuoteYAML(Str); + } +}; + +template <> struct CustomMappingTraits { + static void inputOne(IO &IO, StringRef Key, msgpack::MapNode &M) { + IO.mapRequired(Key.str().c_str(), M[Key]); + } + static void output(IO &IO, msgpack::MapNode &M) { + for (auto &N : M) + IO.mapRequired(N.getKey().str().c_str(), N.getValue()); + } +}; + +template <> struct SequenceTraits { + static size_t size(IO &IO, msgpack::ArrayNode &A) { return A.size(); } + static msgpack::NodePtr &element(IO &IO, msgpack::ArrayNode &A, + size_t Index) { + if (Index >= A.size()) + A.resize(Index + 1); + return A[Index]; + } +}; + +} // end namespace yaml +} // end namespace llvm + +#endif // LLVM_BINARYFORMAT_MSGPACKTYPES_H diff --git a/contrib/llvm/include/llvm/BinaryFormat/MsgPackWriter.h b/contrib/llvm/include/llvm/BinaryFormat/MsgPackWriter.h new file mode 100644 index 00000000000..98af422c9f1 --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/MsgPackWriter.h @@ -0,0 +1,131 @@ +//===- MsgPackWriter.h - Simple MsgPack writer ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains a MessagePack writer. +/// +/// See https://github.com/msgpack/msgpack/blob/master/spec.md for the full +/// specification. +/// +/// Typical usage: +/// \code +/// raw_ostream output = GetOutputStream(); +/// msgpack::Writer MPWriter(output); +/// MPWriter.writeNil(); +/// MPWriter.write(false); +/// MPWriter.write("string"); +/// // ... +/// \endcode +/// +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_MSGPACKPARSER_H +#define LLVM_SUPPORT_MSGPACKPARSER_H + +#include "llvm/BinaryFormat/MsgPack.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace msgpack { + +/// Writes MessagePack objects to an output stream, one at a time. +class Writer { +public: + /// Construct a writer, optionally enabling "Compatibility Mode" as defined + /// in the MessagePack specification. + /// + /// When in \p Compatible mode, the writer will write \c Str16 formats + /// instead of \c Str8 formats, and will refuse to write any \c Bin formats. + /// + /// \param OS stream to output MessagePack objects to. + /// \param Compatible when set, write in "Compatibility Mode". + Writer(raw_ostream &OS, bool Compatible = false); + + Writer(const Writer &) = delete; + Writer &operator=(const Writer &) = delete; + + /// Write a \em Nil to the output stream. + /// + /// The output will be the \em nil format. + void writeNil(); + + /// Write a \em Boolean to the output stream. + /// + /// The output will be a \em bool format. + void write(bool b); + + /// Write a signed integer to the output stream. + /// + /// The output will be in the smallest possible \em int format. + /// + /// The format chosen may be for an unsigned integer. + void write(int64_t i); + + /// Write an unsigned integer to the output stream. + /// + /// The output will be in the smallest possible \em int format. + void write(uint64_t u); + + /// Write a floating point number to the output stream. + /// + /// The output will be in the smallest possible \em float format. + void write(double d); + + /// Write a string to the output stream. + /// + /// The output will be in the smallest possible \em str format. + void write(StringRef s); + + /// Write a memory buffer to the output stream. + /// + /// The output will be in the smallest possible \em bin format. + /// + /// \warning Do not use this overload if in \c Compatible mode. + void write(MemoryBufferRef Buffer); + + /// Write the header for an \em Array of the given size. + /// + /// The output will be in the smallest possible \em array format. + // + /// The header contains an identifier for the \em array format used, as well + /// as an encoding of the size of the array. + /// + /// N.B. The caller must subsequently call \c Write an additional \p Size + /// times to complete the array. + void writeArraySize(uint32_t Size); + + /// Write the header for a \em Map of the given size. + /// + /// The output will be in the smallest possible \em map format. + // + /// The header contains an identifier for the \em map format used, as well + /// as an encoding of the size of the map. + /// + /// N.B. The caller must subsequently call \c Write and additional \c Size*2 + /// times to complete the map. Each even numbered call to \c Write defines a + /// new key, and each odd numbered call defines the previous key's value. + void writeMapSize(uint32_t Size); + + /// Write a typed memory buffer (an extension type) to the output stream. + /// + /// The output will be in the smallest possible \em ext format. + void writeExt(int8_t Type, MemoryBufferRef Buffer); + +private: + support::endian::Writer EW; + bool Compatible; +}; + +} // end namespace msgpack +} // end namespace llvm + +#endif // LLVM_SUPPORT_MSGPACKPARSER_H diff --git a/contrib/llvm/include/llvm/BinaryFormat/Wasm.h b/contrib/llvm/include/llvm/BinaryFormat/Wasm.h index fa5448dacec..b02ddb6b7e2 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/contrib/llvm/include/llvm/BinaryFormat/Wasm.h @@ -16,6 +16,7 @@ #define LLVM_BINARYFORMAT_WASM_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" namespace llvm { namespace wasm { @@ -25,7 +26,7 @@ const char WasmMagic[] = {'\0', 'a', 's', 'm'}; // Wasm binary format version const uint32_t WasmVersion = 0x1; // Wasm linking metadata version -const uint32_t WasmMetadataVersion = 0x1; +const uint32_t WasmMetadataVersion = 0x2; // Wasm uses a 64k page size const uint32_t WasmPageSize = 65536; @@ -34,9 +35,12 @@ struct WasmObjectHeader { uint32_t Version; }; -struct WasmSignature { - std::vector ParamTypes; - uint8_t ReturnType; +struct WasmDylinkInfo { + uint32_t MemorySize; // Memory size in bytes + uint32_t MemoryAlignment; // P2 alignment of memory + uint32_t TableSize; // Table size in elements + uint32_t TableAlignment; // P2 alignment of table + std::vector Needed; // Shared library depenedencies }; struct WasmExport { @@ -79,6 +83,18 @@ struct WasmGlobal { StringRef SymbolName; // from the "linking" section }; +struct WasmEventType { + // Kind of event. Currently only WASM_EVENT_ATTRIBUTE_EXCEPTION is possible. + uint32_t Attribute; + uint32_t SigIndex; +}; + +struct WasmEvent { + uint32_t Index; + WasmEventType Type; + StringRef SymbolName; // from the "linking" section +}; + struct WasmImport { StringRef Module; StringRef Field; @@ -88,6 +104,7 @@ struct WasmImport { WasmGlobalType Global; WasmTable Table; WasmLimits Memory; + WasmEventType Event; }; }; @@ -104,8 +121,8 @@ struct WasmFunction { uint32_t Size; uint32_t CodeOffset; // start of Locals and Body StringRef SymbolName; // from the "linking" section - StringRef DebugName; // from the "name" section - uint32_t Comdat; // from the "comdat info" section + StringRef DebugName; // from the "name" section + uint32_t Comdat; // from the "comdat info" section }; struct WasmDataSegment { @@ -148,7 +165,8 @@ struct WasmSymbolInfo { StringRef Name; uint8_t Kind; uint32_t Flags; - StringRef Module; // For undefined symbols the module name of the import + StringRef ImportModule; // For undefined symbols the module of the import + StringRef ImportName; // For undefined symbols the name of the import union { // For function or global symbols, the index in function or global index // space. @@ -171,18 +189,20 @@ struct WasmLinkingData { }; enum : unsigned { - WASM_SEC_CUSTOM = 0, // Custom / User-defined section - WASM_SEC_TYPE = 1, // Function signature declarations - WASM_SEC_IMPORT = 2, // Import declarations - WASM_SEC_FUNCTION = 3, // Function declarations - WASM_SEC_TABLE = 4, // Indirect function table and other tables - WASM_SEC_MEMORY = 5, // Memory attributes - WASM_SEC_GLOBAL = 6, // Global declarations - WASM_SEC_EXPORT = 7, // Exports - WASM_SEC_START = 8, // Start function declaration - WASM_SEC_ELEM = 9, // Elements section - WASM_SEC_CODE = 10, // Function bodies (code) - WASM_SEC_DATA = 11 // Data segments + WASM_SEC_CUSTOM = 0, // Custom / User-defined section + WASM_SEC_TYPE = 1, // Function signature declarations + WASM_SEC_IMPORT = 2, // Import declarations + WASM_SEC_FUNCTION = 3, // Function declarations + WASM_SEC_TABLE = 4, // Indirect function table and other tables + WASM_SEC_MEMORY = 5, // Memory attributes + WASM_SEC_GLOBAL = 6, // Global declarations + WASM_SEC_EXPORT = 7, // Exports + WASM_SEC_START = 8, // Start function declaration + WASM_SEC_ELEM = 9, // Elements section + WASM_SEC_CODE = 10, // Function bodies (code) + WASM_SEC_DATA = 11, // Data segments + WASM_SEC_DATACOUNT = 12, // Data segment count + WASM_SEC_EVENT = 13 // Event declarations }; // Type immediate encodings used in various contexts. @@ -191,7 +211,8 @@ enum : unsigned { WASM_TYPE_I64 = 0x7E, WASM_TYPE_F32 = 0x7D, WASM_TYPE_F64 = 0x7C, - WASM_TYPE_ANYFUNC = 0x70, + WASM_TYPE_V128 = 0x7B, + WASM_TYPE_FUNCREF = 0x70, WASM_TYPE_EXCEPT_REF = 0x68, WASM_TYPE_FUNC = 0x60, WASM_TYPE_NORESULT = 0x40, // for blocks with no result values @@ -203,12 +224,13 @@ enum : unsigned { WASM_EXTERNAL_TABLE = 0x1, WASM_EXTERNAL_MEMORY = 0x2, WASM_EXTERNAL_GLOBAL = 0x3, + WASM_EXTERNAL_EVENT = 0x4, }; // Opcodes used in initializer expressions. enum : unsigned { WASM_OPCODE_END = 0x0b, - WASM_OPCODE_GET_GLOBAL = 0x23, + WASM_OPCODE_GLOBAL_GET = 0x23, WASM_OPCODE_I32_CONST = 0x41, WASM_OPCODE_I64_CONST = 0x42, WASM_OPCODE_F32_CONST = 0x43, @@ -217,35 +239,27 @@ enum : unsigned { enum : unsigned { WASM_LIMITS_FLAG_HAS_MAX = 0x1, -}; - -// Subset of types that a value can have -enum class ValType { - I32 = WASM_TYPE_I32, - I64 = WASM_TYPE_I64, - F32 = WASM_TYPE_F32, - F64 = WASM_TYPE_F64, - EXCEPT_REF = WASM_TYPE_EXCEPT_REF, + WASM_LIMITS_FLAG_IS_SHARED = 0x2, }; // Kind codes used in the custom "name" section enum : unsigned { WASM_NAMES_FUNCTION = 0x1, - WASM_NAMES_LOCAL = 0x2, + WASM_NAMES_LOCAL = 0x2, }; // Kind codes used in the custom "linking" section enum : unsigned { - WASM_SEGMENT_INFO = 0x5, - WASM_INIT_FUNCS = 0x6, - WASM_COMDAT_INFO = 0x7, - WASM_SYMBOL_TABLE = 0x8, + WASM_SEGMENT_INFO = 0x5, + WASM_INIT_FUNCS = 0x6, + WASM_COMDAT_INFO = 0x7, + WASM_SYMBOL_TABLE = 0x8, }; // Kind codes used in the custom "linking" section in the WASM_COMDAT_INFO enum : unsigned { - WASM_COMDAT_DATA = 0x0, - WASM_COMDAT_FUNCTION = 0x1, + WASM_COMDAT_DATA = 0x0, + WASM_COMDAT_FUNCTION = 0x1, }; // Kind codes used in the custom "linking" section in the WASM_SYMBOL_TABLE @@ -254,17 +268,24 @@ enum WasmSymbolType : unsigned { WASM_SYMBOL_TYPE_DATA = 0x1, WASM_SYMBOL_TYPE_GLOBAL = 0x2, WASM_SYMBOL_TYPE_SECTION = 0x3, + WASM_SYMBOL_TYPE_EVENT = 0x4, }; -const unsigned WASM_SYMBOL_BINDING_MASK = 0x3; -const unsigned WASM_SYMBOL_VISIBILITY_MASK = 0xc; +// Kinds of event attributes. +enum WasmEventAttribute : unsigned { + WASM_EVENT_ATTRIBUTE_EXCEPTION = 0x0, +}; -const unsigned WASM_SYMBOL_BINDING_GLOBAL = 0x0; -const unsigned WASM_SYMBOL_BINDING_WEAK = 0x1; -const unsigned WASM_SYMBOL_BINDING_LOCAL = 0x2; +const unsigned WASM_SYMBOL_BINDING_MASK = 0x3; +const unsigned WASM_SYMBOL_VISIBILITY_MASK = 0xc; + +const unsigned WASM_SYMBOL_BINDING_GLOBAL = 0x0; +const unsigned WASM_SYMBOL_BINDING_WEAK = 0x1; +const unsigned WASM_SYMBOL_BINDING_LOCAL = 0x2; const unsigned WASM_SYMBOL_VISIBILITY_DEFAULT = 0x0; -const unsigned WASM_SYMBOL_VISIBILITY_HIDDEN = 0x4; -const unsigned WASM_SYMBOL_UNDEFINED = 0x10; +const unsigned WASM_SYMBOL_VISIBILITY_HIDDEN = 0x4; +const unsigned WASM_SYMBOL_UNDEFINED = 0x10; +const unsigned WASM_SYMBOL_EXPLICIT_NAME = 0x40; #define WASM_RELOC(name, value) name = value, @@ -274,9 +295,32 @@ enum : unsigned { #undef WASM_RELOC +// Subset of types that a value can have +enum class ValType { + I32 = WASM_TYPE_I32, + I64 = WASM_TYPE_I64, + F32 = WASM_TYPE_F32, + F64 = WASM_TYPE_F64, + V128 = WASM_TYPE_V128, + EXCEPT_REF = WASM_TYPE_EXCEPT_REF, +}; + +struct WasmSignature { + SmallVector Returns; + SmallVector Params; + // Support empty and tombstone instances, needed by DenseMap. + enum { Plain, Empty, Tombstone } State = Plain; + + WasmSignature(SmallVector &&InReturns, + SmallVector &&InParams) + : Returns(InReturns), Params(InParams) {} + WasmSignature() = default; +}; + // Useful comparison operators inline bool operator==(const WasmSignature &LHS, const WasmSignature &RHS) { - return LHS.ReturnType == RHS.ReturnType && LHS.ParamTypes == RHS.ParamTypes; + return LHS.State == RHS.State && LHS.Returns == RHS.Returns && + LHS.Params == RHS.Params; } inline bool operator!=(const WasmSignature &LHS, const WasmSignature &RHS) { diff --git a/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs.def b/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs.def index 8ffd51e483f..b3a08e70c1d 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs.def +++ b/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs.def @@ -1,4 +1,3 @@ - #ifndef WASM_RELOC #error "WASM_RELOC must be defined" #endif @@ -13,3 +12,4 @@ WASM_RELOC(R_WEBASSEMBLY_TYPE_INDEX_LEB, 6) WASM_RELOC(R_WEBASSEMBLY_GLOBAL_INDEX_LEB, 7) WASM_RELOC(R_WEBASSEMBLY_FUNCTION_OFFSET_I32, 8) WASM_RELOC(R_WEBASSEMBLY_SECTION_OFFSET_I32, 9) +WASM_RELOC(R_WEBASSEMBLY_EVENT_INDEX_LEB, 10) diff --git a/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h b/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h index ce8bdd9cf0b..0d7cc141f2c 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h +++ b/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h @@ -51,6 +51,7 @@ class Module; struct BitcodeLTOInfo { bool IsThinLTO; bool HasSummary; + bool EnableSplitLTOUnit; }; /// Represents a module in a bitcode file. diff --git a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 6723cf42dd2..f0d11e9c168 100644 --- a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -342,6 +342,7 @@ enum ConstantsCodes { CST_CODE_INLINEASM = 23, // INLINEASM: [sideeffect|alignstack| // asmdialect,asmstr,conststr] CST_CODE_CE_GEP_WITH_INRANGE_INDEX = 24, // [opty, flags, n x operands] + CST_CODE_CE_UNOP = 25, // CE_UNOP: [opcode, opval] }; /// CastOpcodes - These are values used in the bitcode files to encode which @@ -364,6 +365,14 @@ enum CastOpcodes { CAST_ADDRSPACECAST = 12 }; +/// UnaryOpcodes - These are values used in the bitcode files to encode which +/// unop a CST_CODE_CE_UNOP or a XXX refers to. The values of these enums +/// have no fixed relation to the LLVM IR enum values. Changing these will +/// break compatibility with old files. +enum UnaryOpcodes { + UNOP_NEG = 0 +}; + /// BinaryOpcodes - These are values used in the bitcode files to encode which /// binop a CST_CODE_CE_BINOP or a XXX refers to. The values of these enums /// have no fixed relation to the LLVM IR enum values. Changing these will @@ -524,6 +533,7 @@ enum FunctionCodes { // 53 is unused. // 54 is unused. FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...] + FUNC_CODE_INST_UNOP = 56, // UNOP: [opcode, ty, opval] }; enum UseListCodes { @@ -591,6 +601,7 @@ enum AttributeKindCodes { ATTR_KIND_NOCF_CHECK = 56, ATTR_KIND_OPT_FOR_FUZZING = 57, ATTR_KIND_SHADOWCALLSTACK = 58, + ATTR_KIND_SPECULATIVE_LOAD_HARDENING = 59, }; enum ComdatSelectionKindCodes { diff --git a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h index b6056380916..413901d218f 100644 --- a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -71,6 +71,7 @@ class MCTargetOptions; class MDNode; class Module; class raw_ostream; +class StackMaps; class TargetLoweringObjectFile; class TargetMachine; @@ -137,6 +138,9 @@ private: static char ID; +protected: + /// Protected struct HandlerInfo and Handlers permit target extended + /// AsmPrinter adds their own handlers. struct HandlerInfo { AsmPrinterHandler *Handler; const char *TimerName; @@ -365,6 +369,9 @@ public: /// emit the proxies we previously omitted in EmitGlobalVariable. void emitGlobalGOTEquivs(); + /// Emit the stack maps. + void emitStackMaps(StackMaps &SM); + //===------------------------------------------------------------------===// // Overridable Hooks //===------------------------------------------------------------------===// @@ -542,7 +549,7 @@ public: /// /// \p Value - The value to emit. /// \p Size - The size of the integer (in bytes) to emit. - virtual void EmitDebugThreadLocal(const MCExpr *Value, unsigned Size) const; + virtual void EmitDebugValue(const MCExpr *Value, unsigned Size) const; //===------------------------------------------------------------------===// // Dwarf Lowering Routines @@ -631,6 +638,11 @@ private: /// inline asm. void EmitInlineAsm(const MachineInstr *MI) const; + /// Add inline assembly info to the diagnostics machinery, so we can + /// emit file and position info. Returns SrcMgr memory buffer position. + unsigned addInlineAsmDiagBuffer(StringRef AsmStr, + const MDNode *LocMDNode) const; + //===------------------------------------------------------------------===// // Internal Implementation Details //===------------------------------------------------------------------===// @@ -647,6 +659,8 @@ private: void EmitLLVMUsedList(const ConstantArray *InitList); /// Emit llvm.ident metadata in an '.ident' directive. void EmitModuleIdents(Module &M); + /// Emit bytes for llvm.commandline metadata. + void EmitModuleCommandLines(Module &M); void EmitXXStructorList(const DataLayout &DL, const Constant *List, bool isCtor); diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h b/contrib/llvm/include/llvm/CodeGen/AsmPrinterHandler.h similarity index 92% rename from contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h rename to contrib/llvm/include/llvm/CodeGen/AsmPrinterHandler.h index f5ac95a20b1..a8b13200dd4 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterHandler.h +++ b/contrib/llvm/include/llvm/CodeGen/AsmPrinterHandler.h @@ -1,4 +1,4 @@ -//===-- lib/CodeGen/AsmPrinter/AsmPrinterHandler.h -------------*- C++ -*--===// +//===-- llvm/CodeGen/AsmPrinterHandler.h -----------------------*- C++ -*--===// // // The LLVM Compiler Infrastructure // @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_ASMPRINTERHANDLER_H -#define LLVM_LIB_CODEGEN_ASMPRINTER_ASMPRINTERHANDLER_H +#ifndef LLVM_CODEGEN_ASMPRINTERHANDLER_H +#define LLVM_CODEGEN_ASMPRINTERHANDLER_H #include "llvm/Support/DataTypes.h" diff --git a/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h index f76a2426377..f105d887c39 100644 --- a/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -80,6 +80,23 @@ private: using BaseT = TargetTransformInfoImplCRTPBase; using TTI = TargetTransformInfo; + /// Estimate a cost of Broadcast as an extract and sequence of insert + /// operations. + unsigned getBroadcastShuffleOverhead(Type *Ty) { + assert(Ty->isVectorTy() && "Can only shuffle vectors"); + unsigned Cost = 0; + // Broadcast cost is equal to the cost of extracting the zero'th element + // plus the cost of inserting it into every element of the result vector. + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, Ty, 0); + + for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) { + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, Ty, i); + } + return Cost; + } + /// Estimate a cost of shuffle as a sequence of extract and insert /// operations. unsigned getPermuteShuffleOverhead(Type *Ty) { @@ -101,6 +118,50 @@ private: return Cost; } + /// Estimate a cost of subvector extraction as a sequence of extract and + /// insert operations. + unsigned getExtractSubvectorOverhead(Type *Ty, int Index, Type *SubTy) { + assert(Ty && Ty->isVectorTy() && SubTy && SubTy->isVectorTy() && + "Can only extract subvectors from vectors"); + int NumSubElts = SubTy->getVectorNumElements(); + assert((Index + NumSubElts) <= (int)Ty->getVectorNumElements() && + "SK_ExtractSubvector index out of range"); + + unsigned Cost = 0; + // Subvector extraction cost is equal to the cost of extracting element from + // the source type plus the cost of inserting them into the result vector + // type. + for (int i = 0; i != NumSubElts; ++i) { + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, Ty, i + Index); + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, SubTy, i); + } + return Cost; + } + + /// Estimate a cost of subvector insertion as a sequence of extract and + /// insert operations. + unsigned getInsertSubvectorOverhead(Type *Ty, int Index, Type *SubTy) { + assert(Ty && Ty->isVectorTy() && SubTy && SubTy->isVectorTy() && + "Can only insert subvectors into vectors"); + int NumSubElts = SubTy->getVectorNumElements(); + assert((Index + NumSubElts) <= (int)Ty->getVectorNumElements() && + "SK_InsertSubvector index out of range"); + + unsigned Cost = 0; + // Subvector insertion cost is equal to the cost of extracting element from + // the source type plus the cost of inserting them into the result vector + // type. + for (int i = 0; i != NumSubElts; ++i) { + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, SubTy, i); + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, Ty, i + Index); + } + return Cost; + } + /// Local query method delegates up to T which *must* implement this! const TargetSubtargetInfo *getST() const { return static_cast(this)->getST(); @@ -554,14 +615,20 @@ public: unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp) { switch (Kind) { + case TTI::SK_Broadcast: + return getBroadcastShuffleOverhead(Tp); case TTI::SK_Select: + case TTI::SK_Reverse: case TTI::SK_Transpose: case TTI::SK_PermuteSingleSrc: case TTI::SK_PermuteTwoSrc: return getPermuteShuffleOverhead(Tp); - default: - return 1; + case TTI::SK_ExtractSubvector: + return getExtractSubvectorOverhead(Tp, Index, SubTp); + case TTI::SK_InsertSubvector: + return getInsertSubvectorOverhead(Tp, Index, SubTp); } + llvm_unreachable("Unknown TTI::ShuffleKind"); } unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, @@ -783,8 +850,9 @@ public: unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, - unsigned Alignment, - unsigned AddressSpace) { + unsigned Alignment, unsigned AddressSpace, + bool UseMaskForCond = false, + bool UseMaskForGaps = false) { VectorType *VT = dyn_cast(VecTy); assert(VT && "Expect a vector type for interleaved memory op"); @@ -795,8 +863,13 @@ public: VectorType *SubVT = VectorType::get(VT->getElementType(), NumSubElts); // Firstly, the cost of load/store operation. - unsigned Cost = static_cast(this)->getMemoryOpCost( - Opcode, VecTy, Alignment, AddressSpace); + unsigned Cost; + if (UseMaskForCond || UseMaskForGaps) + Cost = static_cast(this)->getMaskedMemoryOpCost( + Opcode, VecTy, Alignment, AddressSpace); + else + Cost = static_cast(this)->getMemoryOpCost(Opcode, VecTy, Alignment, + AddressSpace); // Legalize the vector type, and get the legalized and unlegalized type // sizes. @@ -892,6 +965,40 @@ public: ->getVectorInstrCost(Instruction::InsertElement, VT, i); } + if (!UseMaskForCond) + return Cost; + + Type *I8Type = Type::getInt8Ty(VT->getContext()); + VectorType *MaskVT = VectorType::get(I8Type, NumElts); + SubVT = VectorType::get(I8Type, NumSubElts); + + // The Mask shuffling cost is extract all the elements of the Mask + // and insert each of them Factor times into the wide vector: + // + // E.g. an interleaved group with factor 3: + // %mask = icmp ult <8 x i32> %vec1, %vec2 + // %interleaved.mask = shufflevector <8 x i1> %mask, <8 x i1> undef, + // <24 x i32> <0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7> + // The cost is estimated as extract all mask elements from the <8xi1> mask + // vector and insert them factor times into the <24xi1> shuffled mask + // vector. + for (unsigned i = 0; i < NumSubElts; i++) + Cost += static_cast(this)->getVectorInstrCost( + Instruction::ExtractElement, SubVT, i); + + for (unsigned i = 0; i < NumElts; i++) + Cost += static_cast(this)->getVectorInstrCost( + Instruction::InsertElement, MaskVT, i); + + // The Gaps mask is invariant and created outside the loop, therefore the + // cost of creating it is not accounted for here. However if we have both + // a MaskForGaps and some other mask that guards the execution of the + // memory access, we need to account for the cost of And-ing the two masks + // inside the loop. + if (UseMaskForGaps) + Cost += static_cast(this)->getArithmeticInstrCost( + BinaryOperator::And, MaskVT); + return Cost; } @@ -901,6 +1008,7 @@ public: unsigned VF = 1) { unsigned RetVF = (RetTy->isVectorTy() ? RetTy->getVectorNumElements() : 1); assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type"); + auto *ConcreteTTI = static_cast(this); switch (IID) { default: { @@ -926,29 +1034,24 @@ public: ScalarizationCost += getOperandsScalarizationOverhead(Args, VF); } - return static_cast(this)-> - getIntrinsicInstrCost(IID, RetTy, Types, FMF, ScalarizationCost); + return ConcreteTTI->getIntrinsicInstrCost(IID, RetTy, Types, FMF, + ScalarizationCost); } case Intrinsic::masked_scatter: { assert(VF == 1 && "Can't vectorize types here."); Value *Mask = Args[3]; bool VarMask = !isa(Mask); unsigned Alignment = cast(Args[2])->getZExtValue(); - return - static_cast(this)->getGatherScatterOpCost(Instruction::Store, - Args[0]->getType(), - Args[1], VarMask, - Alignment); + return ConcreteTTI->getGatherScatterOpCost( + Instruction::Store, Args[0]->getType(), Args[1], VarMask, Alignment); } case Intrinsic::masked_gather: { assert(VF == 1 && "Can't vectorize types here."); Value *Mask = Args[2]; bool VarMask = !isa(Mask); unsigned Alignment = cast(Args[1])->getZExtValue(); - return - static_cast(this)->getGatherScatterOpCost(Instruction::Load, - RetTy, Args[0], VarMask, - Alignment); + return ConcreteTTI->getGatherScatterOpCost(Instruction::Load, RetTy, + Args[0], VarMask, Alignment); } case Intrinsic::experimental_vector_reduce_add: case Intrinsic::experimental_vector_reduce_mul: @@ -964,6 +1067,45 @@ public: case Intrinsic::experimental_vector_reduce_umax: case Intrinsic::experimental_vector_reduce_umin: return getIntrinsicInstrCost(IID, RetTy, Args[0]->getType(), FMF); + case Intrinsic::fshl: + case Intrinsic::fshr: { + Value *X = Args[0]; + Value *Y = Args[1]; + Value *Z = Args[2]; + TTI::OperandValueProperties OpPropsX, OpPropsY, OpPropsZ, OpPropsBW; + TTI::OperandValueKind OpKindX = TTI::getOperandInfo(X, OpPropsX); + TTI::OperandValueKind OpKindY = TTI::getOperandInfo(Y, OpPropsY); + TTI::OperandValueKind OpKindZ = TTI::getOperandInfo(Z, OpPropsZ); + TTI::OperandValueKind OpKindBW = TTI::OK_UniformConstantValue; + OpPropsBW = isPowerOf2_32(RetTy->getScalarSizeInBits()) ? TTI::OP_PowerOf2 + : TTI::OP_None; + // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) + // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) + unsigned Cost = 0; + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Or, RetTy); + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Sub, RetTy); + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Shl, RetTy, + OpKindX, OpKindZ, OpPropsX); + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::LShr, RetTy, + OpKindY, OpKindZ, OpPropsY); + // Non-constant shift amounts requires a modulo. + if (OpKindZ != TTI::OK_UniformConstantValue && + OpKindZ != TTI::OK_NonUniformConstantValue) + Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::URem, RetTy, + OpKindZ, OpKindBW, OpPropsZ, + OpPropsBW); + // For non-rotates (X != Y) we must add shift-by-zero handling costs. + if (X != Y) { + Type *CondTy = Type::getInt1Ty(RetTy->getContext()); + if (RetVF > 1) + CondTy = VectorType::get(CondTy, RetVF); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, + CondTy, nullptr); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy, + CondTy, nullptr); + } + return Cost; + } } } @@ -1036,15 +1178,18 @@ public: case Intrinsic::fabs: ISDs.push_back(ISD::FABS); break; + case Intrinsic::canonicalize: + ISDs.push_back(ISD::FCANONICALIZE); + break; case Intrinsic::minnum: ISDs.push_back(ISD::FMINNUM); if (FMF.noNaNs()) - ISDs.push_back(ISD::FMINNAN); + ISDs.push_back(ISD::FMINIMUM); break; case Intrinsic::maxnum: ISDs.push_back(ISD::FMAXNUM); if (FMF.noNaNs()) - ISDs.push_back(ISD::FMAXNAN); + ISDs.push_back(ISD::FMAXIMUM); break; case Intrinsic::copysign: ISDs.push_back(ISD::FCOPYSIGN); @@ -1136,7 +1281,8 @@ public: SmallVector CustomCost; for (unsigned ISD : ISDs) { if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { - if (IID == Intrinsic::fabs && TLI->isFAbsFree(LT.second)) { + if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() && + TLI->isFAbsFree(LT.second)) { return 0; } @@ -1280,24 +1426,36 @@ public: LT.second.isVector() ? LT.second.getVectorNumElements() : 1; while (NumVecElts > MVTLen) { NumVecElts /= 2; + Type *SubTy = VectorType::get(ScalarTy, NumVecElts); // Assume the pairwise shuffles add a cost. ShuffleCost += (IsPairwise + 1) * ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); - ArithCost += ConcreteTTI->getArithmeticInstrCost(Opcode, Ty); - Ty = VectorType::get(ScalarTy, NumVecElts); + NumVecElts, SubTy); + ArithCost += ConcreteTTI->getArithmeticInstrCost(Opcode, SubTy); + Ty = SubTy; ++LongVectorCount; } + + NumReduxLevels -= LongVectorCount; + // The minimal length of the vector is limited by the real length of vector // operations performed on the current platform. That's why several final // reduction operations are performed on the vectors with the same // architecture-dependent length. - ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) * - ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); - ArithCost += (NumReduxLevels - LongVectorCount) * + + // Non pairwise reductions need one shuffle per reduction level. Pairwise + // reductions need two shuffles on every level, but the last one. On that + // level one of the shuffles is <0, u, u, ...> which is identity. + unsigned NumShuffles = NumReduxLevels; + if (IsPairwise && NumReduxLevels >= 1) + NumShuffles += NumReduxLevels - 1; + ShuffleCost += NumShuffles * + ConcreteTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, + 0, Ty); + ArithCost += NumReduxLevels * ConcreteTTI->getArithmeticInstrCost(Opcode, Ty); - return ShuffleCost + ArithCost + getScalarizationOverhead(Ty, false, true); + return ShuffleCost + ArithCost + + ConcreteTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, 0); } /// Try to calculate op costs for min/max reduction operations. @@ -1327,37 +1485,46 @@ public: LT.second.isVector() ? LT.second.getVectorNumElements() : 1; while (NumVecElts > MVTLen) { NumVecElts /= 2; + Type *SubTy = VectorType::get(ScalarTy, NumVecElts); + CondTy = VectorType::get(ScalarCondTy, NumVecElts); + // Assume the pairwise shuffles add a cost. ShuffleCost += (IsPairwise + 1) * ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); + NumVecElts, SubTy); MinMaxCost += - ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, nullptr) + - ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy, + ConcreteTTI->getCmpSelInstrCost(CmpOpcode, SubTy, CondTy, nullptr) + + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, SubTy, CondTy, nullptr); - Ty = VectorType::get(ScalarTy, NumVecElts); - CondTy = VectorType::get(ScalarCondTy, NumVecElts); + Ty = SubTy; ++LongVectorCount; } + + NumReduxLevels -= LongVectorCount; + // The minimal length of the vector is limited by the real length of vector // operations performed on the current platform. That's why several final // reduction opertions are perfomed on the vectors with the same // architecture-dependent length. - ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) * - ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, - NumVecElts, Ty); + + // Non pairwise reductions need one shuffle per reduction level. Pairwise + // reductions need two shuffles on every level, but the last one. On that + // level one of the shuffles is <0, u, u, ...> which is identity. + unsigned NumShuffles = NumReduxLevels; + if (IsPairwise && NumReduxLevels >= 1) + NumShuffles += NumReduxLevels - 1; + ShuffleCost += NumShuffles * + ConcreteTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, + 0, Ty); MinMaxCost += - (NumReduxLevels - LongVectorCount) * + NumReduxLevels * (ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, nullptr) + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy, nullptr)); - // Need 3 extractelement instructions for scalarization + an additional - // scalar select instruction. + // The last min/max should be in vector registers and we counted it above. + // So just need a single extractelement. return ShuffleCost + MinMaxCost + - 3 * getScalarizationOverhead(Ty, /*Insert=*/false, - /*Extract=*/true) + - ConcreteTTI->getCmpSelInstrCost(Instruction::Select, ScalarTy, - ScalarCondTy, nullptr); + ConcreteTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, 0); } unsigned getVectorSplitCost() { return 1; } diff --git a/contrib/llvm/include/llvm/CodeGen/GCs.h b/contrib/llvm/include/llvm/CodeGen/BuiltinGCs.h similarity index 56% rename from contrib/llvm/include/llvm/CodeGen/GCs.h rename to contrib/llvm/include/llvm/CodeGen/BuiltinGCs.h index 5207f801c84..1767922fb5a 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCs.h +++ b/contrib/llvm/include/llvm/CodeGen/BuiltinGCs.h @@ -1,4 +1,4 @@ -//===-- GCs.h - Garbage collector linkage hacks ---------------------------===// +//===-- BuiltinGCs.h - Garbage collector linkage hacks --------------------===// // // The LLVM Compiler Infrastructure // @@ -7,7 +7,8 @@ // //===----------------------------------------------------------------------===// // -// This file contains hack functions to force linking in the GC components. +// This file contains hack functions to force linking in the builtin GC +// components. // //===----------------------------------------------------------------------===// @@ -15,32 +16,18 @@ #define LLVM_CODEGEN_GCS_H namespace llvm { -class GCStrategy; -class GCMetadataPrinter; /// FIXME: Collector instances are not useful on their own. These no longer /// serve any purpose except to link in the plugins. -/// Creates a CoreCLR-compatible garbage collector. -void linkCoreCLRGC(); - -/// Creates an ocaml-compatible garbage collector. -void linkOcamlGC(); +/// Ensure the definition of the builtin GCs gets linked in +void linkAllBuiltinGCs(); /// Creates an ocaml-compatible metadata printer. void linkOcamlGCPrinter(); -/// Creates an erlang-compatible garbage collector. -void linkErlangGC(); - /// Creates an erlang-compatible metadata printer. void linkErlangGCPrinter(); - -/// Creates a shadow stack garbage collector. This collector requires no code -/// generator support. -void linkShadowStackGC(); - -void linkStatepointExampleGC(); } #endif diff --git a/contrib/llvm/include/llvm/CodeGen/CommandFlags.inc b/contrib/llvm/include/llvm/CodeGen/CommandFlags.inc index 7d2d167289e..568d329a5e8 100644 --- a/contrib/llvm/include/llvm/CodeGen/CommandFlags.inc +++ b/contrib/llvm/include/llvm/CodeGen/CommandFlags.inc @@ -74,7 +74,8 @@ static cl::opt TMModel( static cl::opt CMModel( "code-model", cl::desc("Choose code model"), - cl::values(clEnumValN(CodeModel::Small, "small", "Small code model"), + cl::values(clEnumValN(CodeModel::Tiny, "tiny", "Tiny code model"), + clEnumValN(CodeModel::Small, "small", "Small code model"), clEnumValN(CodeModel::Kernel, "kernel", "Kernel code model"), clEnumValN(CodeModel::Medium, "medium", "Medium code model"), clEnumValN(CodeModel::Large, "large", "Large code model"))); @@ -113,10 +114,16 @@ static cl::opt FileType( clEnumValN(TargetMachine::CGFT_Null, "null", "Emit nothing, for performance testing"))); -static cl::opt - DisableFPElim("disable-fp-elim", - cl::desc("Disable frame pointer elimination optimization"), - cl::init(false)); +static cl::opt FramePointerUsage( + "frame-pointer", cl::desc("Specify frame pointer elimination optimization"), + cl::init(llvm::FramePointer::None), + cl::values( + clEnumValN(llvm::FramePointer::All, "all", + "Disable frame pointer elimination"), + clEnumValN(llvm::FramePointer::NonLeaf, "non-leaf", + "Disable frame pointer elimination for non-leaf frame"), + clEnumValN(llvm::FramePointer::None, "none", + "Enable frame pointer elimination"))); static cl::opt EnableUnsafeFPMath( "enable-unsafe-fp-math", @@ -367,9 +374,14 @@ setFunctionAttributes(StringRef CPU, StringRef Features, Module &M) { NewAttrs.addAttribute("target-cpu", CPU); if (!Features.empty()) NewAttrs.addAttribute("target-features", Features); - if (DisableFPElim.getNumOccurrences() > 0) - NewAttrs.addAttribute("no-frame-pointer-elim", - DisableFPElim ? "true" : "false"); + if (FramePointerUsage.getNumOccurrences() > 0) { + if (FramePointerUsage == llvm::FramePointer::All) + NewAttrs.addAttribute("frame-pointer", "all"); + else if (FramePointerUsage == llvm::FramePointer::NonLeaf) + NewAttrs.addAttribute("frame-pointer", "non-leaf"); + else if (FramePointerUsage == llvm::FramePointer::None) + NewAttrs.addAttribute("frame-pointer", "none"); + } if (DisableTailCalls.getNumOccurrences() > 0) NewAttrs.addAttribute("disable-tail-calls", toStringRef(DisableTailCalls)); diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h b/contrib/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h similarity index 53% rename from contrib/llvm/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h rename to contrib/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h index a262cb38b17..befc28f084e 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h +++ b/contrib/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h @@ -1,4 +1,4 @@ -//===- llvm/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h ------*- C++ -*-===// +//===- llvm/CodeGen/DbgEntityHistoryCalculator.h ----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_DBGVALUEHISTORYCALCULATOR_H -#define LLVM_LIB_CODEGEN_ASMPRINTER_DBGVALUEHISTORYCALCULATOR_H +#ifndef LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H +#define LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" @@ -33,20 +33,19 @@ class DbgValueHistoryMap { public: using InstrRange = std::pair; using InstrRanges = SmallVector; - using InlinedVariable = - std::pair; - using InstrRangesMap = MapVector; + using InlinedEntity = std::pair; + using InstrRangesMap = MapVector; private: InstrRangesMap VarInstrRanges; public: - void startInstrRange(InlinedVariable Var, const MachineInstr &MI); - void endInstrRange(InlinedVariable Var, const MachineInstr &MI); + void startInstrRange(InlinedEntity Var, const MachineInstr &MI); + void endInstrRange(InlinedEntity Var, const MachineInstr &MI); // Returns register currently describing @Var. If @Var is currently // unaccessible or is not described by a register, returns 0. - unsigned getRegisterForVar(InlinedVariable Var) const; + unsigned getRegisterForVar(InlinedEntity Var) const; bool empty() const { return VarInstrRanges.empty(); } void clear() { VarInstrRanges.clear(); } @@ -58,10 +57,31 @@ public: #endif }; -void calculateDbgValueHistory(const MachineFunction *MF, - const TargetRegisterInfo *TRI, - DbgValueHistoryMap &Result); +/// For each inlined instance of a source-level label, keep the corresponding +/// DBG_LABEL instruction. The DBG_LABEL instruction could be used to generate +/// a temporary (assembler) label before it. +class DbgLabelInstrMap { +public: + using InlinedEntity = std::pair; + using InstrMap = MapVector; + +private: + InstrMap LabelInstr; + +public: + void addInstr(InlinedEntity Label, const MachineInstr &MI); + + bool empty() const { return LabelInstr.empty(); } + void clear() { LabelInstr.clear(); } + InstrMap::const_iterator begin() const { return LabelInstr.begin(); } + InstrMap::const_iterator end() const { return LabelInstr.end(); } +}; + +void calculateDbgEntityHistory(const MachineFunction *MF, + const TargetRegisterInfo *TRI, + DbgValueHistoryMap &DbgValues, + DbgLabelInstrMap &DbgLabels); } // end namespace llvm -#endif // LLVM_LIB_CODEGEN_ASMPRINTER_DBGVALUEHISTORYCALCULATOR_H +#endif // LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.h b/contrib/llvm/include/llvm/CodeGen/DebugHandlerBase.h similarity index 88% rename from contrib/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.h rename to contrib/llvm/include/llvm/CodeGen/DebugHandlerBase.h index 1ccefe32be7..4f0d14d317f 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.h +++ b/contrib/llvm/include/llvm/CodeGen/DebugHandlerBase.h @@ -1,4 +1,4 @@ -//===-- llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.h --------*- C++ -*--===// +//===-- llvm/CodeGen/DebugHandlerBase.h -----------------------*- C++ -*--===// // // The LLVM Compiler Infrastructure // @@ -12,12 +12,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGHANDLERBASE_H -#define LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGHANDLERBASE_H +#ifndef LLVM_CODEGEN_DEBUGHANDLERBASE_H +#define LLVM_CODEGEN_DEBUGHANDLERBASE_H -#include "AsmPrinterHandler.h" -#include "DbgValueHistoryCalculator.h" #include "llvm/ADT/Optional.h" +#include "llvm/CodeGen/AsmPrinterHandler.h" +#include "llvm/CodeGen/DbgEntityHistoryCalculator.h" #include "llvm/CodeGen/LexicalScopes.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -82,6 +82,9 @@ protected: /// variable. Variables are listed in order of appearance. DbgValueHistoryMap DbgValues; + /// Mapping of inlined labels and DBG_LABEL machine instruction. + DbgLabelInstrMap DbgLabels; + /// Maps instruction with label emitted before instruction. /// FIXME: Make this private from DwarfDebug, we have the necessary accessors /// for it. @@ -122,6 +125,10 @@ public: /// Return Label immediately following the instruction. MCSymbol *getLabelAfterInsn(const MachineInstr *MI); + /// Return the function-local offset of an instruction. A label for the + /// instruction \p MI should exist (\ref getLabelAfterInsn). + const MCExpr *getFunctionLocalOffsetAfterInsn(const MachineInstr *MI); + /// If this type is derived from a base type then return base type size. static uint64_t getBaseTypeSize(const DITypeRef TyRef); }; diff --git a/contrib/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h b/contrib/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h index e6c0483cfc3..8b1a7af17bb 100644 --- a/contrib/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h +++ b/contrib/llvm/include/llvm/CodeGen/DwarfStringPoolEntry.h @@ -10,6 +10,7 @@ #ifndef LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H #define LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/StringMap.h" namespace llvm { @@ -18,34 +19,52 @@ class MCSymbol; /// Data for a string pool entry. struct DwarfStringPoolEntry { + static constexpr unsigned NotIndexed = -1; + MCSymbol *Symbol; unsigned Offset; unsigned Index; + + bool isIndexed() const { return Index != NotIndexed; } }; /// String pool entry reference. -struct DwarfStringPoolEntryRef { - const StringMapEntry *I = nullptr; +class DwarfStringPoolEntryRef { + PointerIntPair *, 1, bool> + MapEntryAndIndexed; + + const StringMapEntry *getMapEntry() const { + return MapEntryAndIndexed.getPointer(); + } public: DwarfStringPoolEntryRef() = default; - explicit DwarfStringPoolEntryRef( - const StringMapEntry &I) - : I(&I) {} + DwarfStringPoolEntryRef(const StringMapEntry &Entry, + bool Indexed) + : MapEntryAndIndexed(&Entry, Indexed) {} - explicit operator bool() const { return I; } + explicit operator bool() const { return getMapEntry(); } MCSymbol *getSymbol() const { - assert(I->second.Symbol && "No symbol available!"); - return I->second.Symbol; + assert(getMapEntry()->second.Symbol && "No symbol available!"); + return getMapEntry()->second.Symbol; } - unsigned getOffset() const { return I->second.Offset; } - unsigned getIndex() const { return I->second.Index; } - StringRef getString() const { return I->first(); } + unsigned getOffset() const { return getMapEntry()->second.Offset; } + bool isIndexed() const { return MapEntryAndIndexed.getInt(); } + unsigned getIndex() const { + assert(isIndexed()); + assert(getMapEntry()->getValue().isIndexed()); + return getMapEntry()->second.Index; + } + StringRef getString() const { return getMapEntry()->first(); } /// Return the entire string pool entry for convenience. - DwarfStringPoolEntry getEntry() const { return I->getValue(); } + DwarfStringPoolEntry getEntry() const { return getMapEntry()->getValue(); } - bool operator==(const DwarfStringPoolEntryRef &X) const { return I == X.I; } - bool operator!=(const DwarfStringPoolEntryRef &X) const { return I != X.I; } + bool operator==(const DwarfStringPoolEntryRef &X) const { + return getMapEntry() == X.getMapEntry(); + } + bool operator!=(const DwarfStringPoolEntryRef &X) const { + return getMapEntry() != X.getMapEntry(); + } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index 2da00b7d61a..7c658515de0 100644 --- a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -1,4 +1,4 @@ -//===- FunctionLoweringInfo.h - Lower functions from LLVM IR to CodeGen ---===// +//===- FunctionLoweringInfo.h - Lower functions from LLVM IR ---*- C++ -*--===// // // The LLVM Compiler Infrastructure // @@ -246,6 +246,7 @@ public: return 0; unsigned &R = ValueMap[V]; assert(R == 0 && "Already initialized this value register!"); + assert(VirtReg2Value.empty()); return R = CreateRegs(V->getType()); } diff --git a/contrib/llvm/include/llvm/CodeGen/GCMetadata.h b/contrib/llvm/include/llvm/CodeGen/GCMetadata.h index ad2599fc120..7fb27202c12 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCMetadata.h +++ b/contrib/llvm/include/llvm/CodeGen/GCMetadata.h @@ -55,12 +55,11 @@ class MCSymbol; /// GCPoint - Metadata for a collector-safe point in machine code. /// struct GCPoint { - GC::PointKind Kind; ///< The kind of the safe point. MCSymbol *Label; ///< A label. DebugLoc Loc; - GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL) - : Kind(K), Label(L), Loc(std::move(DL)) {} + GCPoint(MCSymbol *L, DebugLoc DL) + : Label(L), Loc(std::move(DL)) {} }; /// GCRoot - Metadata for a pointer to an object managed by the garbage @@ -124,8 +123,8 @@ public: /// addSafePoint - Notes the existence of a safe point. Num is the ID of the /// label just prior to the safe point (if the code generator is using /// MachineModuleInfo). - void addSafePoint(GC::PointKind Kind, MCSymbol *Label, const DebugLoc &DL) { - SafePoints.emplace_back(Kind, Label, DL); + void addSafePoint(MCSymbol *Label, const DebugLoc &DL) { + SafePoints.emplace_back(Label, DL); } /// getFrameSize/setFrameSize - Records the function's frame size. diff --git a/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h b/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h index 1cc69a7b71a..5f1efb2ce02 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h +++ b/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h @@ -29,6 +29,7 @@ class GCMetadataPrinter; class GCModuleInfo; class GCStrategy; class Module; +class StackMaps; /// GCMetadataPrinterRegistry - The GC assembly printer registry uses all the /// defaults from Registry. @@ -60,6 +61,11 @@ public: /// Called after the assembly for the module is generated by /// the AsmPrinter (but before target specific hooks) virtual void finishAssembly(Module &M, GCModuleInfo &Info, AsmPrinter &AP) {} + + /// Called when the stack maps are generated. Return true if + /// stack maps with a custom format are generated. Otherwise + /// returns false and the default format will be used. + virtual bool emitStackMaps(StackMaps &SM, AsmPrinter &AP) { return false; } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/GCStrategy.h b/contrib/llvm/include/llvm/CodeGen/GCStrategy.h index f835bacfb54..5a60cd7cb82 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCStrategy.h +++ b/contrib/llvm/include/llvm/CodeGen/GCStrategy.h @@ -59,19 +59,6 @@ namespace llvm { class Type; -namespace GC { - -/// PointKind - Used to indicate whether the address of the call instruction -/// or the address after the call instruction is listed in the stackmap. For -/// most runtimes, PostCall safepoints are appropriate. -/// -enum PointKind { - PreCall, ///< Instr is a call instruction. - PostCall ///< Instr is the return address of a call. -}; - -} // end namespace GC - /// GCStrategy describes a garbage collector algorithm's code generation /// requirements, and provides overridable hooks for those needs which cannot /// be abstractly described. GCStrategy objects must be looked up through @@ -88,11 +75,7 @@ protected: /// if set, none of the other options can be /// anything but their default values. - unsigned NeededSafePoints = 0; ///< Bitmask of required safe points. - bool CustomReadBarriers = false; ///< Default is to insert loads. - bool CustomWriteBarriers = false; ///< Default is to insert stores. - bool CustomRoots = false; ///< Default is to pass through to backend. - bool InitRoots= true; ///< If set, roots are nulled during lowering. + bool NeededSafePoints = false; ///< if set, calls are inferred to be safepoints bool UsesMetadata = false; ///< If set, backend must emit metadata tables. public: @@ -103,16 +86,6 @@ public: /// name string specified on functions which use this strategy. const std::string &getName() const { return Name; } - /// By default, write barriers are replaced with simple store - /// instructions. If true, you must provide a custom pass to lower - /// calls to \@llvm.gcwrite. - bool customWriteBarrier() const { return CustomWriteBarriers; } - - /// By default, read barriers are replaced with simple load - /// instructions. If true, you must provide a custom pass to lower - /// calls to \@llvm.gcread. - bool customReadBarrier() const { return CustomReadBarriers; } - /// Returns true if this strategy is expecting the use of gc.statepoints, /// and false otherwise. bool useStatepoints() const { return UseStatepoints; } @@ -135,25 +108,8 @@ public: */ ///@{ - /// True if safe points of any kind are required. By default, none are - /// recorded. - bool needsSafePoints() const { return NeededSafePoints != 0; } - - /// True if the given kind of safe point is required. By default, none are - /// recorded. - bool needsSafePoint(GC::PointKind Kind) const { - return (NeededSafePoints & 1 << Kind) != 0; - } - - /// By default, roots are left for the code generator so it can generate a - /// stack map. If true, you must provide a custom pass to lower - /// calls to \@llvm.gcroot. - bool customRoots() const { return CustomRoots; } - - /// If set, gcroot intrinsics should initialize their allocas to null - /// before the first use. This is necessary for most GCs and is enabled by - /// default. - bool initializeRoots() const { return InitRoots; } + /// True if safe points need to be inferred on call sites + bool needsSafePoints() const { return NeededSafePoints; } /// If set, appropriate metadata tables must be emitted by the back-end /// (assembler, JIT, or otherwise). For statepoint, this method is diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h new file mode 100644 index 00000000000..ce2d285a99e --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -0,0 +1,237 @@ +//===- llvm/CodeGen/GlobalISel/CSEInfo.h ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Provides analysis for continuously CSEing during GISel passes. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_CSEINFO_H +#define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H + +#include "llvm/ADT/FoldingSet.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/GlobalISel/GISelWorkList.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" +#include "llvm/Support/Allocator.h" + +namespace llvm { + +/// A class that wraps MachineInstrs and derives from FoldingSetNode in order to +/// be uniqued in a CSEMap. The tradeoff here is extra memory allocations for +/// UniqueMachineInstr vs making MachineInstr bigger. +class UniqueMachineInstr : public FoldingSetNode { + friend class GISelCSEInfo; + const MachineInstr *MI; + explicit UniqueMachineInstr(const MachineInstr *MI) : MI(MI) {} + +public: + void Profile(FoldingSetNodeID &ID); +}; + +// Class representing some configuration that can be done during CSE analysis. +// Currently it only supports shouldCSE method that each pass can set. +class CSEConfig { +public: + virtual ~CSEConfig() = default; + // Hook for defining which Generic instructions should be CSEd. + // GISelCSEInfo currently only calls this hook when dealing with generic + // opcodes. + virtual bool shouldCSEOpc(unsigned Opc); +}; + +// TODO: Find a better place for this. +// Commonly used for O0 config. +class CSEConfigConstantOnly : public CSEConfig { +public: + virtual ~CSEConfigConstantOnly() = default; + virtual bool shouldCSEOpc(unsigned Opc) override; +}; + +/// The CSE Analysis object. +/// This installs itself as a delegate to the MachineFunction to track +/// new instructions as well as deletions. It however will not be able to +/// track instruction mutations. In such cases, recordNewInstruction should be +/// called (for eg inside MachineIRBuilder::recordInsertion). +/// Also because of how just the instruction can be inserted without adding any +/// operands to the instruction, instructions are uniqued and inserted lazily. +/// CSEInfo should assert when trying to enter an incomplete instruction into +/// the CSEMap. There is Opcode level granularity on which instructions can be +/// CSE'd and for now, only Generic instructions are CSEable. +class GISelCSEInfo : public GISelChangeObserver { + // Make it accessible only to CSEMIRBuilder. + friend class CSEMIRBuilder; + + BumpPtrAllocator UniqueInstrAllocator; + FoldingSet CSEMap; + MachineRegisterInfo *MRI = nullptr; + MachineFunction *MF = nullptr; + std::unique_ptr CSEOpt; + /// Keep a cache of UniqueInstrs for each MachineInstr. In GISel, + /// often instructions are mutated (while their ID has completely changed). + /// Whenever mutation happens, invalidate the UniqueMachineInstr for the + /// MachineInstr + DenseMap InstrMapping; + + /// Store instructions that are not fully formed in TemporaryInsts. + /// Also because CSE insertion happens lazily, we can remove insts from this + /// list and avoid inserting and then removing from the CSEMap. + GISelWorkList<8> TemporaryInsts; + + // Only used in asserts. + DenseMap OpcodeHitTable; + + bool isUniqueMachineInstValid(const UniqueMachineInstr &UMI) const; + + void invalidateUniqueMachineInstr(UniqueMachineInstr *UMI); + + UniqueMachineInstr *getNodeIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, void *&InsertPos); + + /// Allocate and construct a new UniqueMachineInstr for MI and return. + UniqueMachineInstr *getUniqueInstrForMI(const MachineInstr *MI); + + void insertNode(UniqueMachineInstr *UMI, void *InsertPos = nullptr); + + /// Get the MachineInstr(Unique) if it exists already in the CSEMap and the + /// same MachineBasicBlock. + MachineInstr *getMachineInstrIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, + void *&InsertPos); + + /// Use this method to allocate a new UniqueMachineInstr for MI and insert it + /// into the CSEMap. MI should return true for shouldCSE(MI->getOpcode()) + void insertInstr(MachineInstr *MI, void *InsertPos = nullptr); + +public: + GISelCSEInfo() = default; + + virtual ~GISelCSEInfo(); + + void setMF(MachineFunction &MF); + + /// Records a newly created inst in a list and lazily insert it to the CSEMap. + /// Sometimes, this method might be called with a partially constructed + /// MachineInstr, + // (right after BuildMI without adding any operands) - and in such cases, + // defer the hashing of the instruction to a later stage. + void recordNewInstruction(MachineInstr *MI); + + /// Use this callback to inform CSE about a newly fully created instruction. + void handleRecordedInst(MachineInstr *MI); + + /// Use this callback to insert all the recorded instructions. At this point, + /// all of these insts need to be fully constructed and should not be missing + /// any operands. + void handleRecordedInsts(); + + /// Remove this inst from the CSE map. If this inst has not been inserted yet, + /// it will be removed from the Tempinsts list if it exists. + void handleRemoveInst(MachineInstr *MI); + + void releaseMemory(); + + void setCSEConfig(std::unique_ptr Opt) { CSEOpt = std::move(Opt); } + + bool shouldCSE(unsigned Opc) const; + + void analyze(MachineFunction &MF); + + void countOpcodeHit(unsigned Opc); + + void print(); + + // Observer API + void erasingInstr(MachineInstr &MI) override; + void createdInstr(MachineInstr &MI) override; + void changingInstr(MachineInstr &MI) override; + void changedInstr(MachineInstr &MI) override; +}; + +class TargetRegisterClass; +class RegisterBank; + +// Simple builder class to easily profile properties about MIs. +class GISelInstProfileBuilder { + FoldingSetNodeID &ID; + const MachineRegisterInfo &MRI; + +public: + GISelInstProfileBuilder(FoldingSetNodeID &ID, const MachineRegisterInfo &MRI) + : ID(ID), MRI(MRI) {} + // Profiling methods. + const GISelInstProfileBuilder &addNodeIDOpcode(unsigned Opc) const; + const GISelInstProfileBuilder &addNodeIDRegType(const LLT &Ty) const; + const GISelInstProfileBuilder &addNodeIDRegType(const unsigned) const; + + const GISelInstProfileBuilder & + addNodeIDRegType(const TargetRegisterClass *RC) const; + const GISelInstProfileBuilder &addNodeIDRegType(const RegisterBank *RB) const; + + const GISelInstProfileBuilder &addNodeIDRegNum(unsigned Reg) const; + + const GISelInstProfileBuilder &addNodeIDImmediate(int64_t Imm) const; + const GISelInstProfileBuilder & + addNodeIDMBB(const MachineBasicBlock *MBB) const; + + const GISelInstProfileBuilder & + addNodeIDMachineOperand(const MachineOperand &MO) const; + + const GISelInstProfileBuilder &addNodeIDFlag(unsigned Flag) const; + const GISelInstProfileBuilder &addNodeID(const MachineInstr *MI) const; +}; + +/// Simple wrapper that does the following. +/// 1) Lazily evaluate the MachineFunction to compute CSEable instructions. +/// 2) Allows configuration of which instructions are CSEd through CSEConfig +/// object. Provides a method called get which takes a CSEConfig object. +class GISelCSEAnalysisWrapper { + GISelCSEInfo Info; + MachineFunction *MF = nullptr; + bool AlreadyComputed = false; + +public: + /// Takes a CSEConfig object that defines what opcodes get CSEd. + /// If CSEConfig is already set, and the CSE Analysis has been preserved, + /// it will not use the new CSEOpt(use Recompute to force using the new + /// CSEOpt). + GISelCSEInfo &get(std::unique_ptr CSEOpt, bool ReCompute = false); + void setMF(MachineFunction &MFunc) { MF = &MFunc; } + void setComputed(bool Computed) { AlreadyComputed = Computed; } + void releaseMemory() { Info.releaseMemory(); } +}; + +/// The actual analysis pass wrapper. +class GISelCSEAnalysisWrapperPass : public MachineFunctionPass { + GISelCSEAnalysisWrapper Wrapper; + +public: + static char ID; + GISelCSEAnalysisWrapperPass() : MachineFunctionPass(ID) { + initializeGISelCSEAnalysisWrapperPassPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + const GISelCSEAnalysisWrapper &getCSEWrapper() const { return Wrapper; } + GISelCSEAnalysisWrapper &getCSEWrapper() { return Wrapper; } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void releaseMemory() override { + Wrapper.releaseMemory(); + Wrapper.setComputed(false); + } +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h new file mode 100644 index 00000000000..a8fb736ebbb --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h @@ -0,0 +1,110 @@ +//===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.h --*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements a version of MachineIRBuilder which CSEs insts within +/// a MachineBasicBlock. +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H +#define LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H + +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" + +namespace llvm { + +/// Defines a builder that does CSE of MachineInstructions using GISelCSEInfo. +/// Eg usage. +/// +/// +/// GISelCSEInfo *Info = +/// &getAnalysis().getCSEInfo(); CSEMIRBuilder +/// CB(Builder.getState()); CB.setCSEInfo(Info); auto A = CB.buildConstant(s32, +/// 42); auto B = CB.buildConstant(s32, 42); assert(A == B); unsigned CReg = +/// MRI.createGenericVirtualRegister(s32); auto C = CB.buildConstant(CReg, 42); +/// assert(C->getOpcode() == TargetOpcode::COPY); +/// Explicitly passing in a register would materialize a copy if possible. +/// CSEMIRBuilder also does trivial constant folding for binary ops. +class CSEMIRBuilder : public MachineIRBuilder { + + /// Returns true if A dominates B (within the same basic block). + /// Both iterators must be in the same basic block. + // + // TODO: Another approach for checking dominance is having two iterators and + // making them go towards each other until they meet or reach begin/end. Which + // approach is better? Should this even change dynamically? For G_CONSTANTS + // most of which will be at the top of the BB, the top down approach would be + // a better choice. Does IRTranslator placing constants at the beginning still + // make sense? Should this change based on Opcode? + bool dominates(MachineBasicBlock::const_iterator A, + MachineBasicBlock::const_iterator B) const; + + /// For given ID, find a machineinstr in the CSE Map. If found, check if it + /// dominates the current insertion point and if not, move it just before the + /// current insertion point and return it. If not found, return Null + /// MachineInstrBuilder. + MachineInstrBuilder getDominatingInstrForID(FoldingSetNodeID &ID, + void *&NodeInsertPos); + /// Simple check if we can CSE (we have the CSEInfo) or if this Opcode is + /// safe to CSE. + bool canPerformCSEForOpc(unsigned Opc) const; + + void profileDstOp(const DstOp &Op, GISelInstProfileBuilder &B) const; + + void profileDstOps(ArrayRef Ops, GISelInstProfileBuilder &B) const { + for (const DstOp &Op : Ops) + profileDstOp(Op, B); + } + + void profileSrcOp(const SrcOp &Op, GISelInstProfileBuilder &B) const; + + void profileSrcOps(ArrayRef Ops, GISelInstProfileBuilder &B) const { + for (const SrcOp &Op : Ops) + profileSrcOp(Op, B); + } + + void profileMBBOpcode(GISelInstProfileBuilder &B, unsigned Opc) const; + + void profileEverything(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, Optional Flags, + GISelInstProfileBuilder &B) const; + + // Takes a MachineInstrBuilder and inserts it into the CSEMap using the + // NodeInsertPos. + MachineInstrBuilder memoizeMI(MachineInstrBuilder MIB, void *NodeInsertPos); + + // If we have can CSE an instruction, but still need to materialize to a VReg, + // we emit a copy from the CSE'd inst to the VReg. + MachineInstrBuilder generateCopiesIfRequired(ArrayRef DstOps, + MachineInstrBuilder &MIB); + + // If we have can CSE an instruction, but still need to materialize to a VReg, + // check if we can generate copies. It's not possible to return a single MIB, + // while emitting copies to multiple vregs. + bool checkCopyToDefsPossible(ArrayRef DstOps); + +public: + // Pull in base class constructors. + using MachineIRBuilder::MachineIRBuilder; + // Unhide buildInstr + MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flag = None) override; + // Bring in the other overload from the base class. + using MachineIRBuilder::buildConstant; + + MachineInstrBuilder buildConstant(const DstOp &Res, + const ConstantInt &Val) override; + + // Bring in the other overload from the base class. + using MachineIRBuilder::buildFConstant; + MachineInstrBuilder buildFConstant(const DstOp &Res, + const ConstantFP &Val) override; +}; +} // namespace llvm +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 58eb412d8c2..ab498e8f070 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -40,6 +40,7 @@ class Value; class CallLowering { const TargetLowering *TLI; + virtual void anchor(); public: struct ArgInfo { unsigned Reg; @@ -108,6 +109,9 @@ public: MachineIRBuilder &MIRBuilder; MachineRegisterInfo &MRI; CCAssignFn *AssignFn; + + private: + virtual void anchor(); }; protected: @@ -138,12 +142,12 @@ public: virtual ~CallLowering() = default; /// This hook must be implemented to lower outgoing return values, described - /// by \p Val, into the specified virtual register \p VReg. + /// by \p Val, into the specified virtual registers \p VRegs. /// This hook is used by GlobalISel. /// /// \return True if the lowering succeeds, false otherwise. - virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, - const Value *Val, unsigned VReg) const { + virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, + ArrayRef VRegs) const { return false; } diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h index 36a33deb4a6..b097c781776 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -21,6 +21,7 @@ namespace llvm { class MachineRegisterInfo; class CombinerInfo; +class GISelCSEInfo; class TargetPassConfig; class MachineFunction; @@ -28,14 +29,17 @@ class Combiner { public: Combiner(CombinerInfo &CombinerInfo, const TargetPassConfig *TPC); - bool combineMachineInstrs(MachineFunction &MF); + /// If CSEInfo is not null, then the Combiner will setup observer for + /// CSEInfo and instantiate a CSEMIRBuilder. Pass nullptr if CSE is not + /// needed. + bool combineMachineInstrs(MachineFunction &MF, GISelCSEInfo *CSEInfo); protected: CombinerInfo &CInfo; MachineRegisterInfo *MRI = nullptr; const TargetPassConfig *TPC; - MachineIRBuilder Builder; + std::unique_ptr Builder; }; } // End namespace llvm. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 5d5b8398452..6e9ac01c1ee 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -1,4 +1,4 @@ -//== llvm/CodeGen/GlobalISel/CombinerHelper.h -------------- -*- C++ -*-==// +//===-- llvm/CodeGen/GlobalISel/CombinerHelper.h --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -20,21 +20,36 @@ namespace llvm { +class GISelChangeObserver; class MachineIRBuilder; class MachineRegisterInfo; class MachineInstr; +class MachineOperand; class CombinerHelper { MachineIRBuilder &Builder; MachineRegisterInfo &MRI; + GISelChangeObserver &Observer; public: - CombinerHelper(MachineIRBuilder &B); + CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B); + + /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes + void replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, unsigned ToReg) const; + + /// Replace a single register operand with a new register and inform the + /// observer of the changes. + void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, + unsigned ToReg) const; /// If \p MI is COPY, try to combine it. /// Returns true if MI changed. bool tryCombineCopy(MachineInstr &MI); + /// If \p MI is extend that consumes the result of a load, try to combine it. + /// Returns true if MI changed. + bool tryCombineExtendingLoads(MachineInstr &MI); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index 1d248547adb..d21aa3f725d 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -17,10 +17,12 @@ #include namespace llvm { +class GISelChangeObserver; class LegalizerInfo; class MachineInstr; class MachineIRBuilder; class MachineRegisterInfo; + // Contains information relevant to enabling/disabling various combines for a // pass. class CombinerInfo { @@ -41,7 +43,19 @@ public: /// illegal ops that are created. bool LegalizeIllegalOps; // TODO: Make use of this. const LegalizerInfo *LInfo; - virtual bool combine(MachineInstr &MI, MachineIRBuilder &B) const = 0; + + /// Attempt to combine instructions using MI as the root. + /// + /// Use Observer to report the creation, modification, and erasure of + /// instructions. GISelChangeObserver will automatically report certain + /// kinds of operations. These operations are: + /// * Instructions that are newly inserted into the MachineFunction + /// * Instructions that are erased from the MachineFunction. + /// + /// However, it is important to report instruction modification and this is + /// not automatic. + virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI, + MachineIRBuilder &B) const = 0; }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h index 8d61f9a6827..220a571b21d 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -15,91 +15,20 @@ namespace llvm { -static Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, - const unsigned Op2, - const MachineRegisterInfo &MRI) { - auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI); - auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI); - if (MaybeOp1Cst && MaybeOp2Cst) { - LLT Ty = MRI.getType(Op1); - APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true); - APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true); - switch (Opcode) { - default: - break; - case TargetOpcode::G_ADD: - return C1 + C2; - case TargetOpcode::G_AND: - return C1 & C2; - case TargetOpcode::G_ASHR: - return C1.ashr(C2); - case TargetOpcode::G_LSHR: - return C1.lshr(C2); - case TargetOpcode::G_MUL: - return C1 * C2; - case TargetOpcode::G_OR: - return C1 | C2; - case TargetOpcode::G_SHL: - return C1 << C2; - case TargetOpcode::G_SUB: - return C1 - C2; - case TargetOpcode::G_XOR: - return C1 ^ C2; - case TargetOpcode::G_UDIV: - if (!C2.getBoolValue()) - break; - return C1.udiv(C2); - case TargetOpcode::G_SDIV: - if (!C2.getBoolValue()) - break; - return C1.sdiv(C2); - case TargetOpcode::G_UREM: - if (!C2.getBoolValue()) - break; - return C1.urem(C2); - case TargetOpcode::G_SREM: - if (!C2.getBoolValue()) - break; - return C1.srem(C2); - } - } - return None; -} - /// An MIRBuilder which does trivial constant folding of binary ops. /// Calls to buildInstr will also try to constant fold binary ops. -class ConstantFoldingMIRBuilder - : public FoldableInstructionsBuilder { +class ConstantFoldingMIRBuilder : public MachineIRBuilder { public: // Pull in base class constructors. - using FoldableInstructionsBuilder< - ConstantFoldingMIRBuilder>::FoldableInstructionsBuilder; - // Unhide buildInstr - using FoldableInstructionsBuilder::buildInstr; + using MachineIRBuilder::MachineIRBuilder; - // Implement buildBinaryOp required by FoldableInstructionsBuilder which - // tries to constant fold. - MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Dst, - unsigned Src0, unsigned Src1) { - validateBinaryOp(Dst, Src0, Src1); - auto MaybeCst = ConstantFoldBinOp(Opcode, Src0, Src1, getMF().getRegInfo()); - if (MaybeCst) - return buildConstant(Dst, MaybeCst->getSExtValue()); - return buildInstr(Opcode).addDef(Dst).addUse(Src0).addUse(Src1); - } - - template - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, UseArg1Ty &&Arg1, - UseArg2Ty &&Arg2) { - unsigned Dst = getDestFromArg(Ty); - return buildInstr(Opc, Dst, getRegFromArg(std::forward(Arg1)), - getRegFromArg(std::forward(Arg2))); - } + virtual ~ConstantFoldingMIRBuilder() = default; // Try to provide an overload for buildInstr for binary ops in order to // constant fold. - MachineInstrBuilder buildInstr(unsigned Opc, unsigned Dst, unsigned Src0, - unsigned Src1) { + MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flags = None) override { switch (Opc) { default: break; @@ -116,19 +45,18 @@ public: case TargetOpcode::G_SDIV: case TargetOpcode::G_UREM: case TargetOpcode::G_SREM: { - return buildBinaryOp(Opc, Dst, Src0, Src1); + assert(DstOps.size() == 1 && "Invalid dst ops"); + assert(SrcOps.size() == 2 && "Invalid src ops"); + const DstOp &Dst = DstOps[0]; + const SrcOp &Src0 = SrcOps[0]; + const SrcOp &Src1 = SrcOps[1]; + if (auto MaybeCst = + ConstantFoldBinOp(Opc, Src0.getReg(), Src1.getReg(), *getMRI())) + return buildConstant(Dst, MaybeCst->getSExtValue()); + break; } } - return buildInstr(Opc).addDef(Dst).addUse(Src0).addUse(Src1); - } - - // Fallback implementation of buildInstr. - template - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, - UseArgsTy &&... Args) { - auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); - addUsesFromArgs(MIB, std::forward(Args)...); - return MIB; + return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps); } }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h new file mode 100644 index 00000000000..c8e8a7a5a7c --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h @@ -0,0 +1,111 @@ +//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// This contains common code to allow clients to notify changes to machine +/// instr. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H +#define LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H + +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/CodeGen/MachineFunction.h" + +namespace llvm { +class MachineInstr; +class MachineRegisterInfo; + +/// Abstract class that contains various methods for clients to notify about +/// changes. This should be the preferred way for APIs to notify changes. +/// Typically calling erasingInstr/createdInstr multiple times should not affect +/// the result. The observer would likely need to check if it was already +/// notified earlier (consider using GISelWorkList). +class GISelChangeObserver { + SmallPtrSet ChangingAllUsesOfReg; + +public: + virtual ~GISelChangeObserver() {} + + /// An instruction is about to be erased. + virtual void erasingInstr(MachineInstr &MI) = 0; + /// An instruction was created and inserted into the function. + virtual void createdInstr(MachineInstr &MI) = 0; + /// This instruction is about to be mutated in some way. + virtual void changingInstr(MachineInstr &MI) = 0; + /// This instruction was mutated in some way. + virtual void changedInstr(MachineInstr &MI) = 0; + + /// All the instructions using the given register are being changed. + /// For convenience, finishedChangingAllUsesOfReg() will report the completion + /// of the changes. The use list may change between this call and + /// finishedChangingAllUsesOfReg(). + void changingAllUsesOfReg(const MachineRegisterInfo &MRI, unsigned Reg); + /// All instructions reported as changing by changingAllUsesOfReg() have + /// finished being changed. + void finishedChangingAllUsesOfReg(); + +}; + +/// Simple wrapper observer that takes several observers, and calls +/// each one for each event. If there are multiple observers (say CSE, +/// Legalizer, Combiner), it's sufficient to register this to the machine +/// function as the delegate. +class GISelObserverWrapper : public MachineFunction::Delegate, + public GISelChangeObserver { + SmallVector Observers; + +public: + GISelObserverWrapper() = default; + GISelObserverWrapper(ArrayRef Obs) + : Observers(Obs.begin(), Obs.end()) {} + // Adds an observer. + void addObserver(GISelChangeObserver *O) { Observers.push_back(O); } + // Removes an observer from the list and does nothing if observer is not + // present. + void removeObserver(GISelChangeObserver *O) { + auto It = std::find(Observers.begin(), Observers.end(), O); + if (It != Observers.end()) + Observers.erase(It); + } + // API for Observer. + void erasingInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->erasingInstr(MI); + } + void createdInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->createdInstr(MI); + } + void changingInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->changingInstr(MI); + } + void changedInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->changedInstr(MI); + } + // API for MachineFunction::Delegate + void MF_HandleInsertion(MachineInstr &MI) override { createdInstr(MI); } + void MF_HandleRemoval(MachineInstr &MI) override { erasingInstr(MI); } +}; + +/// A simple RAII based CSEInfo installer. +/// Use this in a scope to install a delegate to the MachineFunction and reset +/// it at the end of the scope. +class RAIIDelegateInstaller { + MachineFunction &MF; + MachineFunction::Delegate *Delegate; + +public: + RAIIDelegateInstaller(MachineFunction &MF, MachineFunction::Delegate *Del); + ~RAIIDelegateInstaller(); +}; + +} // namespace llvm +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index 167905dc9aa..1571841a208 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -12,38 +12,42 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineInstr.h" #include "llvm/Support/Debug.h" namespace llvm { class MachineInstr; +class MachineFunction; -// Worklist which mostly works similar to InstCombineWorkList, but on MachineInstrs. -// The main difference with something like a SetVector is that erasing an element doesn't -// move all elements over one place - instead just nulls out the element of the vector. -// FIXME: Does it make sense to factor out common code with the instcombinerWorkList? +// Worklist which mostly works similar to InstCombineWorkList, but on +// MachineInstrs. The main difference with something like a SetVector is that +// erasing an element doesn't move all elements over one place - instead just +// nulls out the element of the vector. +// +// FIXME: Does it make sense to factor out common code with the +// instcombinerWorkList? template class GISelWorkList { - SmallVector Worklist; - DenseMap WorklistMap; + SmallVector Worklist; + DenseMap WorklistMap; public: - GISelWorkList() = default; + GISelWorkList() {} bool empty() const { return WorklistMap.empty(); } unsigned size() const { return WorklistMap.size(); } - /// Add - Add the specified instruction to the worklist if it isn't already - /// in it. + /// Add the specified instruction to the worklist if it isn't already in it. void insert(MachineInstr *I) { - if (WorklistMap.try_emplace(I, Worklist.size()).second) { + if (WorklistMap.try_emplace(I, Worklist.size()).second) Worklist.push_back(I); - } } - /// Remove - remove I from the worklist if it exists. - void remove(MachineInstr *I) { + /// Remove I from the worklist if it exists. + void remove(const MachineInstr *I) { auto It = WorklistMap.find(I); if (It == WorklistMap.end()) return; // Not in worklist. @@ -53,6 +57,11 @@ public: WorklistMap.erase(It); } + void clear() { + Worklist.clear(); + WorklistMap.clear(); + } + MachineInstr *pop_back_val() { MachineInstr *I; do { diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 2498ee93321..d1770bf6e4c 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -21,11 +21,11 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/Support/Allocator.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Allocator.h" #include #include @@ -300,6 +300,8 @@ private: bool translateFSub(const User &U, MachineIRBuilder &MIRBuilder); + bool translateFNeg(const User &U, MachineIRBuilder &MIRBuilder); + bool translateAdd(const User &U, MachineIRBuilder &MIRBuilder) { return translateBinaryOp(TargetOpcode::G_ADD, U, MIRBuilder); } @@ -442,11 +444,13 @@ private: // I.e., compared to regular MIBuilder, this one also inserts the instruction // in the current block, it can creates block, etc., basically a kind of // IRBuilder, but for Machine IR. - MachineIRBuilder CurBuilder; + // CSEMIRBuilder CurBuilder; + std::unique_ptr CurBuilder; // Builder set to the entry block (just after ABI lowering instructions). Used // as a convenient location for Constants. - MachineIRBuilder EntryBuilder; + // CSEMIRBuilder EntryBuilder; + std::unique_ptr EntryBuilder; // The MachineFunction currently being translated. MachineFunction *MF; diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 873587651ef..20bec765017 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -1,4 +1,4 @@ -//===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h --===========// +//===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h -----*- C++ -*-// // // The LLVM Compiler Infrastructure // @@ -14,12 +14,14 @@ #include "llvm/CodeGen/GlobalISel/Legalizer.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "legalizer" +using namespace llvm::MIPatternMatch; namespace llvm { class LegalizationArtifactCombiner { @@ -36,15 +38,29 @@ public: SmallVectorImpl &DeadInsts) { if (MI.getOpcode() != TargetOpcode::G_ANYEXT) return false; - if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, - MI.getOperand(1).getReg(), MRI)) { + + Builder.setInstr(MI); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // aext(trunc x) - > aext/copy/trunc x + unsigned TruncSrc; + if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = DefMI->getOperand(1).getReg(); - Builder.setInstr(MI); - // We get a copy/trunc/extend depending on the sizes - Builder.buildAnyExtOrTrunc(DstReg, SrcReg); - markInstAndDefDead(MI, *DefMI, DeadInsts); + Builder.buildAnyExtOrTrunc(DstReg, TruncSrc); + markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); + return true; + } + + // aext([asz]ext x) -> [asz]ext x + unsigned ExtSrc; + MachineInstr *ExtMI; + if (mi_match(SrcReg, MRI, + m_all_of(m_MInstr(ExtMI), m_any_of(m_GAnyExt(m_Reg(ExtSrc)), + m_GSExt(m_Reg(ExtSrc)), + m_GZExt(m_Reg(ExtSrc)))))) { + Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc}); + markInstAndDefDead(MI, *ExtMI, DeadInsts); return true; } return tryFoldImplicitDef(MI, DeadInsts); @@ -55,24 +71,25 @@ public: if (MI.getOpcode() != TargetOpcode::G_ZEXT) return false; - if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, - MI.getOperand(1).getReg(), MRI)) { - unsigned DstReg = MI.getOperand(0).getReg(); + + Builder.setInstr(MI); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // zext(trunc x) - > and (aext/copy/trunc x), mask + unsigned TruncSrc; + if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) || isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); - Builder.setInstr(MI); - unsigned ZExtSrc = MI.getOperand(1).getReg(); - LLT ZExtSrcTy = MRI.getType(ZExtSrc); - APInt Mask = APInt::getAllOnesValue(ZExtSrcTy.getSizeInBits()); - auto MaskCstMIB = Builder.buildConstant(DstTy, Mask.getZExtValue()); - unsigned TruncSrc = DefMI->getOperand(1).getReg(); - // We get a copy/trunc/extend depending on the sizes - auto SrcCopyOrTrunc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc); - Builder.buildAnd(DstReg, SrcCopyOrTrunc, MaskCstMIB); - markInstAndDefDead(MI, *DefMI, DeadInsts); + LLT SrcTy = MRI.getType(SrcReg); + APInt Mask = APInt::getAllOnesValue(SrcTy.getSizeInBits()); + auto MIBMask = Builder.buildConstant(DstTy, Mask.getZExtValue()); + Builder.buildAnd(DstReg, Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), + MIBMask); + markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } return tryFoldImplicitDef(MI, DeadInsts); @@ -83,33 +100,34 @@ public: if (MI.getOpcode() != TargetOpcode::G_SEXT) return false; - if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, - MI.getOperand(1).getReg(), MRI)) { - unsigned DstReg = MI.getOperand(0).getReg(); + + Builder.setInstr(MI); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c + unsigned TruncSrc; + if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy}}) || isInstUnsupported({TargetOpcode::G_ASHR, {DstTy}}) || isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); - Builder.setInstr(MI); - unsigned SExtSrc = MI.getOperand(1).getReg(); - LLT SExtSrcTy = MRI.getType(SExtSrc); - unsigned SizeDiff = DstTy.getSizeInBits() - SExtSrcTy.getSizeInBits(); - auto SizeDiffMIB = Builder.buildConstant(DstTy, SizeDiff); - unsigned TruncSrcReg = DefMI->getOperand(1).getReg(); - // We get a copy/trunc/extend depending on the sizes - auto SrcCopyExtOrTrunc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrcReg); - auto ShlMIB = Builder.buildInstr(TargetOpcode::G_SHL, DstTy, - SrcCopyExtOrTrunc, SizeDiffMIB); - Builder.buildInstr(TargetOpcode::G_ASHR, DstReg, ShlMIB, SizeDiffMIB); - markInstAndDefDead(MI, *DefMI, DeadInsts); + LLT SrcTy = MRI.getType(SrcReg); + unsigned ShAmt = DstTy.getSizeInBits() - SrcTy.getSizeInBits(); + auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt); + auto MIBShl = Builder.buildInstr( + TargetOpcode::G_SHL, {DstTy}, + {Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBShAmt}); + Builder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {MIBShl, MIBShAmt}); + markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } return tryFoldImplicitDef(MI, DeadInsts); } - /// Try to fold sb = EXTEND (G_IMPLICIT_DEF sa) -> sb = G_IMPLICIT_DEF + /// Try to fold G_[ASZ]EXT (G_IMPLICIT_DEF). bool tryFoldImplicitDef(MachineInstr &MI, SmallVectorImpl &DeadInsts) { unsigned Opcode = MI.getOpcode(); @@ -119,13 +137,25 @@ public: if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(), MRI)) { + Builder.setInstr(MI); unsigned DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) - return false; - LLVM_DEBUG(dbgs() << ".. Combine EXT(IMPLICIT_DEF) " << MI;); - Builder.setInstr(MI); - Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, DstReg); + + if (Opcode == TargetOpcode::G_ANYEXT) { + // G_ANYEXT (G_IMPLICIT_DEF) -> G_IMPLICIT_DEF + if (isInstUnsupported({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) + return false; + LLVM_DEBUG(dbgs() << ".. Combine G_ANYEXT(G_IMPLICIT_DEF): " << MI;); + Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, {DstReg}, {}); + } else { + // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top + // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT. + if (isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + return false; + LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;); + Builder.buildConstant(DstReg, 0); + } + markInstAndDefDead(MI, *DefMI, DeadInsts); return true; } @@ -139,8 +169,20 @@ public: return false; unsigned NumDefs = MI.getNumOperands() - 1; - MachineInstr *MergeI = getOpcodeDef(TargetOpcode::G_MERGE_VALUES, - MI.getOperand(NumDefs).getReg(), MRI); + + unsigned MergingOpcode; + LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg()); + LLT DestTy = MRI.getType(MI.getOperand(0).getReg()); + if (OpTy.isVector() && DestTy.isVector()) + MergingOpcode = TargetOpcode::G_CONCAT_VECTORS; + else if (OpTy.isVector() && !DestTy.isVector()) + MergingOpcode = TargetOpcode::G_BUILD_VECTOR; + else + MergingOpcode = TargetOpcode::G_MERGE_VALUES; + + MachineInstr *MergeI = + getOpcodeDef(MergingOpcode, MI.getOperand(NumDefs).getReg(), MRI); + if (!MergeI) return false; @@ -277,6 +319,19 @@ private: auto Step = LI.getAction(Query); return Step.Action == Unsupported || Step.Action == NotFound; } + + /// Looks through copy instructions and returns the actual + /// source register. + unsigned lookThroughCopyInstrs(unsigned Reg) { + unsigned TmpReg; + while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) { + if (MRI.getType(TmpReg).isValid()) + Reg = TmpReg; + else + break; + } + return Reg; + } }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index d122e67b87b..9b4ecf9284e 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -32,6 +32,7 @@ namespace llvm { class LegalizerInfo; class Legalizer; class MachineRegisterInfo; +class GISelChangeObserver; class LegalizerHelper { public: @@ -48,7 +49,10 @@ public: UnableToLegalize, }; - LegalizerHelper(MachineFunction &MF); + LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, + MachineIRBuilder &B); + LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, + GISelChangeObserver &Observer, MachineIRBuilder &B); /// Replace \p MI by a sequence of legal instructions that can implement the /// same operation. Note that this means \p MI may be deleted, so any iterator @@ -87,7 +91,7 @@ public: /// Expose MIRBuilder so clients can set their own RecordInsertInstruction /// functions - MachineIRBuilder MIRBuilder; + MachineIRBuilder &MIRBuilder; /// Expose LegalizerInfo so the clients can re-use. const LegalizerInfo &getLegalizerInfo() const { return LI; } @@ -112,8 +116,12 @@ private: void extractParts(unsigned Reg, LLT Ty, int NumParts, SmallVectorImpl &VRegs); + LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + MachineRegisterInfo &MRI; const LegalizerInfo &LI; + /// To keep track of changes made by the LegalizerHelper. + GISelChangeObserver &Observer; }; /// Helper function that creates the given libcall. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index a8c26082f22..13776dd3e87 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -39,6 +39,7 @@ class MachineInstr; class MachineIRBuilder; class MachineRegisterInfo; class MCInstrInfo; +class GISelChangeObserver; namespace LegalizeActions { enum LegalizeAction : std::uint8_t { @@ -121,7 +122,7 @@ struct LegalityQuery { ArrayRef Types; struct MemDesc { - uint64_t Size; + uint64_t SizeInBits; AtomicOrdering Ordering; }; @@ -651,6 +652,20 @@ public: return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); } + /// Widen the scalar to match the size of another. + LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { + typeIdx(TypeIdx); + return widenScalarIf( + [=](const LegalityQuery &Query) { + return Query.Types[LargeTypeIdx].getScalarSizeInBits() > + Query.Types[TypeIdx].getSizeInBits(); + }, + [=](const LegalityQuery &Query) { + return std::make_pair(TypeIdx, + Query.Types[LargeTypeIdx].getElementType()); + }); + } + /// Add more elements to the vector to reach the next power of two. /// No effect if the type is not a vector or the element count is a power of /// two. @@ -693,6 +708,8 @@ public: }, [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; + if (MaxElements == 1) + return std::make_pair(TypeIdx, VecTy.getElementType()); return std::make_pair( TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits())); }); @@ -947,9 +964,9 @@ public: bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; - virtual bool legalizeCustom(MachineInstr &MI, - MachineRegisterInfo &MRI, - MachineIRBuilder &MIRBuilder) const; + virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const; private: /// Determine what action should be taken to legalize the given generic diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index ac1673de5f3..37de8f03041 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H #define LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/LowLevelType.h" @@ -30,6 +31,7 @@ namespace llvm { class MachineFunction; class MachineInstr; class TargetInstrInfo; +class GISelChangeObserver; /// Class which stores all the state required in a MachineIRBuilder. /// Since MachineIRBuilders will only store state in this object, it allows @@ -50,63 +52,178 @@ struct MachineIRBuilderState { MachineBasicBlock::iterator II; /// @} - std::function InsertedInstr; + GISelChangeObserver *Observer; + + GISelCSEInfo *CSEInfo; }; +class DstOp { + union { + LLT LLTTy; + unsigned Reg; + const TargetRegisterClass *RC; + }; + +public: + enum class DstType { Ty_LLT, Ty_Reg, Ty_RC }; + DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {} + DstOp(const LLT &T) : LLTTy(T), Ty(DstType::Ty_LLT) {} + DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {} + + void addDefToMIB(MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) const { + switch (Ty) { + case DstType::Ty_Reg: + MIB.addDef(Reg); + break; + case DstType::Ty_LLT: + MIB.addDef(MRI.createGenericVirtualRegister(LLTTy)); + break; + case DstType::Ty_RC: + MIB.addDef(MRI.createVirtualRegister(RC)); + break; + } + } + + LLT getLLTTy(const MachineRegisterInfo &MRI) const { + switch (Ty) { + case DstType::Ty_RC: + return LLT{}; + case DstType::Ty_LLT: + return LLTTy; + case DstType::Ty_Reg: + return MRI.getType(Reg); + } + llvm_unreachable("Unrecognised DstOp::DstType enum"); + } + + unsigned getReg() const { + assert(Ty == DstType::Ty_Reg && "Not a register"); + return Reg; + } + + const TargetRegisterClass *getRegClass() const { + switch (Ty) { + case DstType::Ty_RC: + return RC; + default: + llvm_unreachable("Not a RC Operand"); + } + } + + DstType getDstOpKind() const { return Ty; } + +private: + DstType Ty; +}; + +class SrcOp { + union { + MachineInstrBuilder SrcMIB; + unsigned Reg; + CmpInst::Predicate Pred; + }; + +public: + enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate }; + SrcOp(unsigned R) : Reg(R), Ty(SrcType::Ty_Reg) {} + SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {} + SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {} + + void addSrcToMIB(MachineInstrBuilder &MIB) const { + switch (Ty) { + case SrcType::Ty_Predicate: + MIB.addPredicate(Pred); + break; + case SrcType::Ty_Reg: + MIB.addUse(Reg); + break; + case SrcType::Ty_MIB: + MIB.addUse(SrcMIB->getOperand(0).getReg()); + break; + } + } + + LLT getLLTTy(const MachineRegisterInfo &MRI) const { + switch (Ty) { + case SrcType::Ty_Predicate: + llvm_unreachable("Not a register operand"); + case SrcType::Ty_Reg: + return MRI.getType(Reg); + case SrcType::Ty_MIB: + return MRI.getType(SrcMIB->getOperand(0).getReg()); + } + llvm_unreachable("Unrecognised SrcOp::SrcType enum"); + } + + unsigned getReg() const { + switch (Ty) { + case SrcType::Ty_Predicate: + llvm_unreachable("Not a register operand"); + case SrcType::Ty_Reg: + return Reg; + case SrcType::Ty_MIB: + return SrcMIB->getOperand(0).getReg(); + } + llvm_unreachable("Unrecognised SrcOp::SrcType enum"); + } + + CmpInst::Predicate getPredicate() const { + switch (Ty) { + case SrcType::Ty_Predicate: + return Pred; + default: + llvm_unreachable("Not a register operand"); + } + } + + SrcType getSrcOpKind() const { return Ty; } + +private: + SrcType Ty; +}; + +class FlagsOp { + Optional Flags; + +public: + explicit FlagsOp(unsigned F) : Flags(F) {} + FlagsOp() : Flags(None) {} + Optional getFlags() const { return Flags; } +}; /// Helper class to build MachineInstr. /// It keeps internally the insertion point and debug location for all /// the new instructions we want to create. /// This information can be modify via the related setters. -class MachineIRBuilderBase { +class MachineIRBuilder { MachineIRBuilderState State; + +protected: + void validateTruncExt(const LLT &Dst, const LLT &Src, bool IsExtend); + + void validateBinaryOp(const LLT &Res, const LLT &Op0, const LLT &Op1); + + void validateSelectOp(const LLT &ResTy, const LLT &TstTy, const LLT &Op0Ty, + const LLT &Op1Ty); + void recordInsertion(MachineInstr *MI) const; + +public: + /// Some constructors for easy use. + MachineIRBuilder() = default; + MachineIRBuilder(MachineFunction &MF) { setMF(MF); } + MachineIRBuilder(MachineInstr &MI) : MachineIRBuilder(*MI.getMF()) { + setInstr(MI); + } + + virtual ~MachineIRBuilder() = default; + + MachineIRBuilder(const MachineIRBuilderState &BState) : State(BState) {} + const TargetInstrInfo &getTII() { assert(State.TII && "TargetInstrInfo is not set"); return *State.TII; } - void validateTruncExt(unsigned Dst, unsigned Src, bool IsExtend); - -protected: - unsigned getDestFromArg(unsigned Reg) { return Reg; } - unsigned getDestFromArg(LLT Ty) { - return getMF().getRegInfo().createGenericVirtualRegister(Ty); - } - unsigned getDestFromArg(const TargetRegisterClass *RC) { - return getMF().getRegInfo().createVirtualRegister(RC); - } - - void addUseFromArg(MachineInstrBuilder &MIB, unsigned Reg) { - MIB.addUse(Reg); - } - - void addUseFromArg(MachineInstrBuilder &MIB, const MachineInstrBuilder &UseMIB) { - MIB.addUse(UseMIB->getOperand(0).getReg()); - } - - void addUsesFromArgs(MachineInstrBuilder &MIB) { } - template - void addUsesFromArgs(MachineInstrBuilder &MIB, UseArgTy &&Arg1, UseArgsTy &&... Args) { - addUseFromArg(MIB, Arg1); - addUsesFromArgs(MIB, std::forward(Args)...); - } - unsigned getRegFromArg(unsigned Reg) { return Reg; } - unsigned getRegFromArg(const MachineInstrBuilder &MIB) { - return MIB->getOperand(0).getReg(); - } - - void validateBinaryOp(unsigned Res, unsigned Op0, unsigned Op1); - -public: - /// Some constructors for easy use. - MachineIRBuilderBase() = default; - MachineIRBuilderBase(MachineFunction &MF) { setMF(MF); } - MachineIRBuilderBase(MachineInstr &MI) : MachineIRBuilderBase(*MI.getMF()) { - setInstr(MI); - } - - MachineIRBuilderBase(const MachineIRBuilderState &BState) : State(BState) {} - /// Getter for the function we currently build. MachineFunction &getMF() { assert(State.MF && "MachineFunction is not set"); @@ -118,16 +235,25 @@ public: /// Getter for MRI MachineRegisterInfo *getMRI() { return State.MRI; } + const MachineRegisterInfo *getMRI() const { return State.MRI; } /// Getter for the State MachineIRBuilderState &getState() { return State; } /// Getter for the basic block we currently build. - MachineBasicBlock &getMBB() { + const MachineBasicBlock &getMBB() const { assert(State.MBB && "MachineBasicBlock is not set"); return *State.MBB; } + MachineBasicBlock &getMBB() { + return const_cast( + const_cast(this)->getMBB()); + } + + GISelCSEInfo *getCSEInfo() { return State.CSEInfo; } + const GISelCSEInfo *getCSEInfo() const { return State.CSEInfo; } + /// Current insertion point for new instructions. MachineBasicBlock::iterator getInsertPt() { return State.II; } @@ -137,10 +263,12 @@ public: void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II); /// @} + void setCSEInfo(GISelCSEInfo *Info); + /// \name Setters for the insertion point. /// @{ /// Set the MachineFunction where to build instructions. - void setMF(MachineFunction &); + void setMF(MachineFunction &MF); /// Set the insertion point to the end of \p MBB. /// \pre \p MBB must be contained by getMF(). @@ -151,12 +279,8 @@ public: void setInstr(MachineInstr &MI); /// @} - /// \name Control where instructions we create are recorded (typically for - /// visiting again later during legalization). - /// @{ - void recordInsertion(MachineInstr *InsertedInstr) const; - void recordInsertions(std::function InsertedInstr); - void stopRecordingInsertions(); + void setChangeObserver(GISelChangeObserver &Observer); + void stopObservingChanges(); /// @} /// Set the debug location to \p DL for all the next build instructions. @@ -208,6 +332,10 @@ public: const MDNode *Variable, const MDNode *Expr); + /// Build and insert a DBG_LABEL instructions specifying that \p Label is + /// given. Convert "llvm.dbg.label Label" to "DBG_LABEL Label". + MachineInstrBuilder buildDbgLabel(const MDNode *Label); + /// Build and insert \p Res = G_FRAME_INDEX \p Idx /// /// G_FRAME_INDEX materializes the address of an alloca value or other @@ -296,9 +424,9 @@ public: /// registers with the same scalar type (typically s1) /// /// \return The newly created instruction. - MachineInstrBuilder buildUAdde(unsigned Res, unsigned CarryOut, unsigned Op0, - unsigned Op1, unsigned CarryIn); - + MachineInstrBuilder buildUAdde(const DstOp &Res, const DstOp &CarryOut, + const SrcOp &Op0, const SrcOp &Op1, + const SrcOp &CarryIn); /// Build and insert \p Res = G_ANYEXT \p Op0 /// @@ -314,11 +442,7 @@ public: /// /// \return The newly created instruction. - MachineInstrBuilder buildAnyExt(unsigned Res, unsigned Op); - template - MachineInstrBuilder buildAnyExt(DstType &&Res, ArgType &&Arg) { - return buildAnyExt(getDestFromArg(Res), getRegFromArg(Arg)); - } + MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_SEXT \p Op /// @@ -332,11 +456,7 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. - template - MachineInstrBuilder buildSExt(DstType &&Res, ArgType &&Arg) { - return buildSExt(getDestFromArg(Res), getRegFromArg(Arg)); - } - MachineInstrBuilder buildSExt(unsigned Res, unsigned Op); + MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_ZEXT \p Op /// @@ -350,11 +470,7 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. - template - MachineInstrBuilder buildZExt(DstType &&Res, ArgType &&Arg) { - return buildZExt(getDestFromArg(Res), getRegFromArg(Arg)); - } - MachineInstrBuilder buildZExt(unsigned Res, unsigned Op); + MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_SEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. @@ -364,11 +480,7 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildSExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { - return buildSExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); - } - MachineInstrBuilder buildSExtOrTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_ZEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. @@ -378,11 +490,7 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildZExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { - return buildZExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); - } - MachineInstrBuilder buildZExtOrTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op); // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. @@ -392,11 +500,7 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildAnyExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { - return buildAnyExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); - } - MachineInstrBuilder buildAnyExtOrTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = \p ExtOpc, \p Res = G_TRUNC \p /// Op, or \p Res = COPY \p Op depending on the differing sizes of \p Res and @@ -407,15 +511,11 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. - MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, unsigned Res, - unsigned Op); + MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, + const SrcOp &Op); /// Build and insert an appropriate cast between two registers of equal size. - template - MachineInstrBuilder buildCast(DstType &&Res, ArgType &&Arg) { - return buildCast(getDestFromArg(Res), getRegFromArg(Arg)); - } - MachineInstrBuilder buildCast(unsigned Dst, unsigned Src); + MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src); /// Build and insert G_BR \p Dest /// @@ -460,7 +560,8 @@ public: /// type. /// /// \return The newly created instruction. - MachineInstrBuilder buildConstant(unsigned Res, const ConstantInt &Val); + virtual MachineInstrBuilder buildConstant(const DstOp &Res, + const ConstantInt &Val); /// Build and insert \p Res = G_CONSTANT \p Val /// @@ -470,12 +571,8 @@ public: /// \pre \p Res must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildConstant(unsigned Res, int64_t Val); + MachineInstrBuilder buildConstant(const DstOp &Res, int64_t Val); - template - MachineInstrBuilder buildConstant(DstType &&Res, int64_t Val) { - return buildConstant(getDestFromArg(Res), Val); - } /// Build and insert \p Res = G_FCONSTANT \p Val /// /// G_FCONSTANT is a floating-point constant with the specified size and @@ -485,17 +582,10 @@ public: /// \pre \p Res must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - template - MachineInstrBuilder buildFConstant(DstType &&Res, const ConstantFP &Val) { - return buildFConstant(getDestFromArg(Res), Val); - } - MachineInstrBuilder buildFConstant(unsigned Res, const ConstantFP &Val); + virtual MachineInstrBuilder buildFConstant(const DstOp &Res, + const ConstantFP &Val); - template - MachineInstrBuilder buildFConstant(DstType &&Res, double Val) { - return buildFConstant(getDestFromArg(Res), Val); - } - MachineInstrBuilder buildFConstant(unsigned Res, double Val); + MachineInstrBuilder buildFConstant(const DstOp &Res, double Val); /// Build and insert \p Res = COPY Op /// @@ -504,11 +594,7 @@ public: /// \pre setBasicBlock or setMI must have been called. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildCopy(unsigned Res, unsigned Op); - template - MachineInstrBuilder buildCopy(DstType &&Res, SrcType &&Src) { - return buildCopy(getDestFromArg(Res), getRegFromArg(Src)); - } + MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op); /// Build and insert `Res = G_LOAD Addr, MMO`. /// @@ -555,10 +641,7 @@ public: MachineInstrBuilder buildExtract(unsigned Res, unsigned Src, uint64_t Index); /// Build and insert \p Res = IMPLICIT_DEF. - template MachineInstrBuilder buildUndef(DstType &&Res) { - return buildUndef(getDestFromArg(Res)); - } - MachineInstrBuilder buildUndef(unsigned Res); + MachineInstrBuilder buildUndef(const DstOp &Res); /// Build and insert instructions to put \p Ops together at the specified p /// Indices to form a larger register. @@ -587,7 +670,7 @@ public: /// \pre The type of all \p Ops registers must be identical. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMerge(unsigned Res, ArrayRef Ops); + MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef Ops); /// Build and insert \p Res0, ... = G_UNMERGE_VALUES \p Op /// @@ -599,7 +682,50 @@ public: /// \pre The type of all \p Res registers must be identical. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildUnmerge(ArrayRef Res, unsigned Op); + MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); + MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); + + /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... + /// + /// G_BUILD_VECTOR creates a vector value from multiple scalar registers. + /// \pre setBasicBlock or setMI must have been called. + /// \pre The entire register \p Res (and no more) must be covered by the + /// input scalar registers. + /// \pre The type of all \p Ops registers must be identical. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildBuildVector(const DstOp &Res, + ArrayRef Ops); + + /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... + /// + /// G_BUILD_VECTOR_TRUNC creates a vector value from multiple scalar registers + /// which have types larger than the destination vector element type, and + /// truncates the values to fit. + /// + /// If the operands given are already the same size as the vector elt type, + /// then this method will instead create a G_BUILD_VECTOR instruction. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre The type of all \p Ops registers must be identical. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildBuildVectorTrunc(const DstOp &Res, + ArrayRef Ops); + + /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... + /// + /// G_CONCAT_VECTORS creates a vector from the concatenation of 2 or more + /// vectors. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre The entire register \p Res (and no more) must be covered by the input + /// registers. + /// \pre The type of all source operands must be identical. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildConcatVectors(const DstOp &Res, + ArrayRef Ops); MachineInstrBuilder buildInsert(unsigned Res, unsigned Src, unsigned Op, unsigned Index); @@ -627,11 +753,7 @@ public: /// \pre \p Res must be smaller than \p Op /// /// \return The newly created instruction. - template - MachineInstrBuilder buildFPTrunc(DstType &&Res, SrcType &&Src) { - return buildFPTrunc(getDestFromArg(Res), getRegFromArg(Src)); - } - MachineInstrBuilder buildFPTrunc(unsigned Res, unsigned Op); + MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert \p Res = G_TRUNC \p Op /// @@ -644,11 +766,7 @@ public: /// \pre \p Res must be smaller than \p Op /// /// \return The newly created instruction. - MachineInstrBuilder buildTrunc(unsigned Res, unsigned Op); - template - MachineInstrBuilder buildTrunc(DstType &&Res, SrcType &&Src) { - return buildTrunc(getDestFromArg(Res), getRegFromArg(Src)); - } + MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op); /// Build and insert a \p Res = G_ICMP \p Pred, \p Op0, \p Op1 /// @@ -662,8 +780,8 @@ public: /// \pre \p Pred must be an integer predicate. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, - unsigned Res, unsigned Op0, unsigned Op1); + MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, + const SrcOp &Op0, const SrcOp &Op1); /// Build and insert a \p Res = G_FCMP \p Pred\p Op0, \p Op1 /// @@ -677,8 +795,8 @@ public: /// \pre \p Pred must be a floating-point predicate. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, - unsigned Res, unsigned Op0, unsigned Op1); + MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, + const SrcOp &Op0, const SrcOp &Op1); /// Build and insert a \p Res = G_SELECT \p Tst, \p Op0, \p Op1 /// @@ -690,8 +808,8 @@ public: /// elements as the other parameters. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildSelect(unsigned Res, unsigned Tst, - unsigned Op0, unsigned Op1); + MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, + const SrcOp &Op0, const SrcOp &Op1); /// Build and insert \p Res = G_INSERT_VECTOR_ELT \p Val, /// \p Elt, \p Idx @@ -703,8 +821,10 @@ public: /// with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildInsertVectorElement(unsigned Res, unsigned Val, - unsigned Elt, unsigned Idx); + MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, + const SrcOp &Val, + const SrcOp &Elt, + const SrcOp &Idx); /// Build and insert \p Res = G_EXTRACT_VECTOR_ELT \p Val, \p Idx /// @@ -714,8 +834,9 @@ public: /// \pre \p Idx must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val, - unsigned Idx); + MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, + const SrcOp &Val, + const SrcOp &Idx); /// Build and insert `OldValRes, SuccessRes = /// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`. @@ -952,19 +1073,7 @@ public: /// /// \return The newly created instruction. MachineInstrBuilder buildBlockAddress(unsigned Res, const BlockAddress *BA); -}; -/// A CRTP class that contains methods for building instructions that can -/// be constant folded. MachineIRBuilders that want to inherit from this will -/// need to implement buildBinaryOp (for constant folding binary ops). -/// Alternatively, they can implement buildInstr(Opc, Dst, Uses...) to perform -/// additional folding for Opc. -template -class FoldableInstructionsBuilder : public MachineIRBuilderBase { - Base &base() { return static_cast(*this); } - -public: - using MachineIRBuilderBase::MachineIRBuilderBase; /// Build and insert \p Res = G_ADD \p Op0, \p Op1 /// /// G_ADD sets \p Res to the sum of integer parameters \p Op0 and \p Op1, @@ -976,13 +1085,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAdd(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_ADD, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildAdd(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildAdd(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_ADD, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_SUB \p Op0, \p Op1 @@ -996,13 +1102,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildSub(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_SUB, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildSub(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildSub(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_SUB, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_MUL \p Op0, \p Op1 @@ -1015,13 +1118,10 @@ public: /// with the same (scalar or vector) type). /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMul(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_MUL, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildMul(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildMul(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_MUL, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_AND \p Op0, \p Op1 @@ -1035,13 +1135,9 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAnd(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_AND, Dst, Src0, Src1); - } - template - MachineInstrBuilder buildAnd(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildAnd(Res, (base().getRegFromArg(UseArgs))...); + MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_AND, {Dst}, {Src0, Src1}); } /// Build and insert \p Res = G_OR \p Op0, \p Op1 @@ -1054,39 +1150,14 @@ public: /// with the same (scalar or vector) type). /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildOr(unsigned Dst, unsigned Src0, unsigned Src1) { - return base().buildBinaryOp(TargetOpcode::G_OR, Dst, Src0, Src1); + MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1}); } - template - MachineInstrBuilder buildOr(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = base().getDestFromArg(Ty); - return base().buildOr(Res, (base().getRegFromArg(UseArgs))...); - } -}; -class MachineIRBuilder : public FoldableInstructionsBuilder { -public: - using FoldableInstructionsBuilder< - MachineIRBuilder>::FoldableInstructionsBuilder; - MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Dst, - unsigned Src0, unsigned Src1) { - validateBinaryOp(Dst, Src0, Src1); - return buildInstr(Opcode).addDef(Dst).addUse(Src0).addUse(Src1); - } - using FoldableInstructionsBuilder::buildInstr; - /// DAG like Generic method for building arbitrary instructions as above. - /// \Opc opcode for the instruction. - /// \Ty Either LLT/TargetRegisterClass/unsigned types for Dst - /// \Args Variadic list of uses of types(unsigned/MachineInstrBuilder) - /// Uses of type MachineInstrBuilder will perform - /// getOperand(0).getReg() to convert to register. - template - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, - UseArgsTy &&... Args) { - auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); - addUsesFromArgs(MIB, std::forward(Args)...); - return MIB; - } + virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flags = None); }; } // End namespace llvm. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index 82fd7eddb68..c33b32b2db4 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -103,8 +103,8 @@ public: /// Currently the TableGen-like file would look like: /// \code /// PartialMapping[] = { - /// /*32-bit add*/ {0, 32, GPR}, - /// /*2x32-bit add*/ {0, 32, GPR}, {0, 32, GPR}, // <-- Same entry 3x + /// /*32-bit add*/ {0, 32, GPR}, // Scalar entry repeated for first vec elt. + /// /*2x32-bit add*/ {0, 32, GPR}, {32, 32, GPR}, /// /*<2x32-bit> vadd {0, 64, VPR} /// }; // PartialMapping duplicated. /// @@ -118,14 +118,15 @@ public: /// With the array of pointer, we would have: /// \code /// PartialMapping[] = { - /// /*32-bit add*/ {0, 32, GPR}, + /// /*32-bit add lower */ {0, 32, GPR}, + /// /*32-bit add upper */ {32, 32, GPR}, /// /*<2x32-bit> vadd {0, 64, VPR} /// }; // No more duplication. /// /// BreakDowns[] = { /// /*AddBreakDown*/ &PartialMapping[0], - /// /*2xAddBreakDown*/ &PartialMapping[0], &PartialMapping[0], - /// /*VAddBreakDown*/ &PartialMapping[1] + /// /*2xAddBreakDown*/ &PartialMapping[0], &PartialMapping[1], + /// /*VAddBreakDown*/ &PartialMapping[2] /// }; // Addresses of PartialMapping duplicated (smaller). /// /// ValueMapping[] { diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index 51e3a273297..82b791d35b2 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -108,5 +108,8 @@ APFloat getAPFloatFromSize(double Val, unsigned Size); /// fallback. void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); +Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, + const unsigned Op2, + const MachineRegisterInfo &MRI); } // End namespace llvm. #endif diff --git a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h index 80bd796d537..9c918ae1104 100644 --- a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -70,7 +70,7 @@ namespace ISD { /// of the frame or return address to return. An index of zero corresponds /// to the current function's frame or return address, an index of one to /// the parent's frame or return address, and so on. - FRAMEADDR, RETURNADDR, ADDROFRETURNADDR, + FRAMEADDR, RETURNADDR, ADDROFRETURNADDR, SPONENTRY, /// LOCAL_RECOVER - Represents the llvm.localrecover intrinsic. /// Materializes the offset from the local object pointer of another @@ -256,6 +256,29 @@ namespace ISD { /// Same for multiplication. SMULO, UMULO, + /// RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 + /// integers with the same bit width (W). If the true value of LHS + RHS + /// exceeds the largest value that can be represented by W bits, the + /// resulting value is this maximum value. Otherwise, if this value is less + /// than the smallest value that can be represented by W bits, the + /// resulting value is this minimum value. + SADDSAT, UADDSAT, + + /// RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 + /// integers with the same bit width (W). If the true value of LHS - RHS + /// exceeds the largest value that can be represented by W bits, the + /// resulting value is this maximum value. Otherwise, if this value is less + /// than the smallest value that can be represented by W bits, the + /// resulting value is this minimum value. + SSUBSAT, USUBSAT, + + /// RESULT = SMULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on + /// 2 integers with the same width and scale. SCALE represents the scale of + /// both operands as fixed point numbers. This SCALE parameter must be a + /// constant integer. A scale of zero is effectively performing + /// multiplication on 2 integers. + SMULFIX, + /// Simple binary floating point operators. FADD, FSUB, FMUL, FDIV, FREM, @@ -272,7 +295,8 @@ namespace ISD { /// They are used to limit optimizations while the DAG is being optimized. STRICT_FSQRT, STRICT_FPOW, STRICT_FPOWI, STRICT_FSIN, STRICT_FCOS, STRICT_FEXP, STRICT_FEXP2, STRICT_FLOG, STRICT_FLOG10, STRICT_FLOG2, - STRICT_FRINT, STRICT_FNEARBYINT, + STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM, + STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC, /// FMA - Perform a * b + c with no intermediate rounding step. FMA, @@ -377,9 +401,13 @@ namespace ISD { /// When the 1st operand is a vector, the shift amount must be in the same /// type. (TLI.getShiftAmountTy() will return the same type when the input /// type is a vector.) - /// For rotates, the shift amount is treated as an unsigned amount modulo - /// the element size of the first operand. - SHL, SRA, SRL, ROTL, ROTR, + /// For rotates and funnel shifts, the shift amount is treated as an unsigned + /// amount modulo the element size of the first operand. + /// + /// Funnel 'double' shifts take 3 operands, 2 inputs and the shift amount. + /// fshl(X,Y,Z): (X << (Z % BW)) | (Y >> (BW - (Z % BW))) + /// fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW)) + SHL, SRA, SRL, ROTL, ROTR, FSHL, FSHR, /// Byte Swap and Counting operators. BSWAP, CTTZ, CTLZ, CTPOP, BITREVERSE, @@ -461,31 +489,33 @@ namespace ISD { /// in-register any-extension of the low lanes of an integer vector. The /// result type must have fewer elements than the operand type, and those /// elements must be larger integer types such that the total size of the - /// operand type and the result type match. Each of the low operand - /// elements is any-extended into the corresponding, wider result - /// elements with the high bits becoming undef. + /// operand type is less than or equal to the size of the result type. Each + /// of the low operand elements is any-extended into the corresponding, + /// wider result elements with the high bits becoming undef. + /// NOTE: The type legalizer prefers to make the operand and result size + /// the same to allow expansion to shuffle vector during op legalization. ANY_EXTEND_VECTOR_INREG, /// SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an /// in-register sign-extension of the low lanes of an integer vector. The /// result type must have fewer elements than the operand type, and those /// elements must be larger integer types such that the total size of the - /// operand type and the result type match. Each of the low operand - /// elements is sign-extended into the corresponding, wider result - /// elements. - // FIXME: The SIGN_EXTEND_INREG node isn't specifically limited to - // scalars, but it also doesn't handle vectors well. Either it should be - // restricted to scalars or this node (and its handling) should be merged - // into it. + /// operand type is less than or equal to the size of the result type. Each + /// of the low operand elements is sign-extended into the corresponding, + /// wider result elements. + /// NOTE: The type legalizer prefers to make the operand and result size + /// the same to allow expansion to shuffle vector during op legalization. SIGN_EXTEND_VECTOR_INREG, /// ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an /// in-register zero-extension of the low lanes of an integer vector. The /// result type must have fewer elements than the operand type, and those /// elements must be larger integer types such that the total size of the - /// operand type and the result type match. Each of the low operand - /// elements is zero-extended into the corresponding, wider result - /// elements. + /// operand type is less than or equal to the size of the result type. Each + /// of the low operand elements is zero-extended into the corresponding, + /// wider result elements. + /// NOTE: The type legalizer prefers to make the operand and result size + /// the same to allow expansion to shuffle vector during op legalization. ZERO_EXTEND_VECTOR_INREG, /// FP_TO_[US]INT - Convert a floating point value to a signed or unsigned @@ -550,22 +580,29 @@ namespace ISD { /// is often a storage-only type but has native conversions. FP16_TO_FP, FP_TO_FP16, - /// FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, - /// FLOG, FLOG2, FLOG10, FEXP, FEXP2, - /// FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary - /// floating point operations. These are inspired by libm. - FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, + /// Perform various unary floating-point operations inspired by libm. + FNEG, FABS, FSQRT, FCBRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR, /// FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two /// values. - /// In the case where a single input is NaN, the non-NaN input is returned. + // + /// In the case where a single input is a NaN (either signaling or quiet), + /// the non-NaN input is returned. /// /// The return value of (FMINNUM 0.0, -0.0) could be either 0.0 or -0.0. FMINNUM, FMAXNUM, - /// FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that - /// when a single input is NaN, NaN is returned. - FMINNAN, FMAXNAN, + + /// FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on + /// two values, following the IEEE-754 2008 definition. This differs from + /// FMINNUM/FMAXNUM in the handling of signaling NaNs. If one input is a + /// signaling NaN, returns a quiet NaN. + FMINNUM_IEEE, FMAXNUM_IEEE, + + /// FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 + /// as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008 + /// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2018 draft semantics. + FMINIMUM, FMAXIMUM, /// FSINCOS - Compute both fsin and fcos as a single operation. FSINCOS, @@ -786,11 +823,20 @@ namespace ISD { // Masked load and store - consecutive vector load and store operations // with additional mask operand that prevents memory accesses to the // masked-off lanes. + // + // Val, OutChain = MLOAD(BasePtr, Mask, PassThru) + // OutChain = MSTORE(Value, BasePtr, Mask) MLOAD, MSTORE, // Masked gather and scatter - load and store operations for a vector of // random addresses with additional mask operand that prevents memory // accesses to the masked-off lanes. + // + // Val, OutChain = GATHER(InChain, PassThru, Mask, BasePtr, Index, Scale) + // OutChain = SCATTER(InChain, Value, Mask, BasePtr, Index, Scale) + // + // The Index operand can have more vector elements than the other operands + // due to type legalization. The extra elements are ignored. MGATHER, MSCATTER, /// This corresponds to the llvm.lifetime.* intrinsics. The first operand diff --git a/contrib/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h b/contrib/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h index c3046da90b8..38fcb37b1e6 100644 --- a/contrib/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h +++ b/contrib/llvm/include/llvm/CodeGen/LinkAllAsmWriterComponents.h @@ -15,7 +15,7 @@ #ifndef LLVM_CODEGEN_LINKALLASMWRITERCOMPONENTS_H #define LLVM_CODEGEN_LINKALLASMWRITERCOMPONENTS_H -#include "llvm/CodeGen/GCs.h" +#include "llvm/CodeGen/BuiltinGCs.h" #include namespace { diff --git a/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h b/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h index fee131e4a3c..18c13ca8f59 100644 --- a/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -15,7 +15,7 @@ #ifndef LLVM_CODEGEN_LINKALLCODEGENCOMPONENTS_H #define LLVM_CODEGEN_LINKALLCODEGENCOMPONENTS_H -#include "llvm/CodeGen/GCs.h" +#include "llvm/CodeGen/BuiltinGCs.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/Target/TargetMachine.h" @@ -36,11 +36,7 @@ namespace { (void) llvm::createGreedyRegisterAllocator(); (void) llvm::createDefaultPBQPRegisterAllocator(); - llvm::linkCoreCLRGC(); - llvm::linkOcamlGC(); - llvm::linkErlangGC(); - llvm::linkShadowStackGC(); - llvm::linkStatepointExampleGC(); + llvm::linkAllBuiltinGCs(); (void) llvm::createBURRListDAGScheduler(nullptr, llvm::CodeGenOpt::Default); diff --git a/contrib/llvm/include/llvm/CodeGen/LiveIntervals.h b/contrib/llvm/include/llvm/CodeGen/LiveIntervals.h index 291a07a712c..16ab1dc475c 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveIntervals.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveIntervals.h @@ -198,10 +198,10 @@ class VirtRegMap; void pruneValue(LiveRange &LR, SlotIndex Kill, SmallVectorImpl *EndPoints); - /// This function should not be used. Its intend is to tell you that - /// you are doing something wrong if you call pruveValue directly on a + /// This function should not be used. Its intent is to tell you that you are + /// doing something wrong if you call pruneValue directly on a /// LiveInterval. Indeed, you are supposed to call pruneValue on the main - /// LiveRange and all the LiveRange of the subranges if any. + /// LiveRange and all the LiveRanges of the subranges if any. LLVM_ATTRIBUTE_UNUSED void pruneValue(LiveInterval &, SlotIndex, SmallVectorImpl *) { llvm_unreachable( diff --git a/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h b/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h index 301a45066b4..7312902e21b 100644 --- a/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -48,7 +48,8 @@ class raw_ostream; /// when walking backward/forward through a basic block. class LivePhysRegs { const TargetRegisterInfo *TRI = nullptr; - SparseSet LiveRegs; + using RegisterSet = SparseSet>; + RegisterSet LiveRegs; public: /// Constructs an unitialized set. init() needs to be called to initialize it. @@ -76,7 +77,7 @@ public: bool empty() const { return LiveRegs.empty(); } /// Adds a physical register and all its sub-registers to the set. - void addReg(unsigned Reg) { + void addReg(MCPhysReg Reg) { assert(TRI && "LivePhysRegs is not initialized."); assert(Reg <= TRI->getNumRegs() && "Expected a physical register."); for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); @@ -86,7 +87,7 @@ public: /// Removes a physical register, all its sub-registers, and all its /// super-registers from the set. - void removeReg(unsigned Reg) { + void removeReg(MCPhysReg Reg) { assert(TRI && "LivePhysRegs is not initialized."); assert(Reg <= TRI->getNumRegs() && "Expected a physical register."); for (MCRegAliasIterator R(Reg, TRI, true); R.isValid(); ++R) @@ -95,7 +96,7 @@ public: /// Removes physical registers clobbered by the regmask operand \p MO. void removeRegsInMask(const MachineOperand &MO, - SmallVectorImpl> *Clobbers = + SmallVectorImpl> *Clobbers = nullptr); /// Returns true if register \p Reg is contained in the set. This also @@ -103,10 +104,10 @@ public: /// addReg() always adds all sub-registers to the set as well. /// Note: Returns false if just some sub registers are live, use available() /// when searching a free register. - bool contains(unsigned Reg) const { return LiveRegs.count(Reg); } + bool contains(MCPhysReg Reg) const { return LiveRegs.count(Reg); } /// Returns true if register \p Reg and no aliasing register is in the set. - bool available(const MachineRegisterInfo &MRI, unsigned Reg) const; + bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const; /// Remove defined registers and regmask kills from the set. void removeDefs(const MachineInstr &MI); @@ -126,7 +127,7 @@ public: /// defined or clobbered by a regmask. The operand will identify whether this /// is a regmask or register operand. void stepForward(const MachineInstr &MI, - SmallVectorImpl> &Clobbers); + SmallVectorImpl> &Clobbers); /// Adds all live-in registers of basic block \p MBB. /// Live in registers are the registers in the blocks live-in list and the @@ -143,7 +144,7 @@ public: /// registers. void addLiveOutsNoPristines(const MachineBasicBlock &MBB); - using const_iterator = SparseSet::const_iterator; + using const_iterator = RegisterSet::const_iterator; const_iterator begin() const { return LiveRegs.begin(); } const_iterator end() const { return LiveRegs.end(); } diff --git a/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h b/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h index 249545906e0..5e9dd8b3cdf 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h @@ -85,14 +85,14 @@ public: bool empty() const { return Units.none(); } /// Adds register units covered by physical register \p Reg. - void addReg(unsigned Reg) { + void addReg(MCPhysReg Reg) { for (MCRegUnitIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) Units.set(*Unit); } /// Adds register units covered by physical register \p Reg that are /// part of the lanemask \p Mask. - void addRegMasked(unsigned Reg, LaneBitmask Mask) { + void addRegMasked(MCPhysReg Reg, LaneBitmask Mask) { for (MCRegUnitMaskIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) { LaneBitmask UnitMask = (*Unit).second; if (UnitMask.none() || (UnitMask & Mask).any()) @@ -101,7 +101,7 @@ public: } /// Removes all register units covered by physical register \p Reg. - void removeReg(unsigned Reg) { + void removeReg(MCPhysReg Reg) { for (MCRegUnitIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) Units.reset(*Unit); } @@ -115,7 +115,7 @@ public: void addRegsInMask(const uint32_t *RegMask); /// Returns true if no part of physical register \p Reg is live. - bool available(unsigned Reg) const { + bool available(MCPhysReg Reg) const { for (MCRegUnitIterator Unit(Reg, TRI); Unit.isValid(); ++Unit) { if (Units.test(*Unit)) return false; diff --git a/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h index 7f46406c478..98ac81915dc 100644 --- a/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -425,6 +425,7 @@ struct MachineFrameInfo { StringValue StackProtector; // TODO: Serialize FunctionContextIdx unsigned MaxCallFrameSize = ~0u; ///< ~0u means: not computed yet. + unsigned CVBytesOfCalleeSavedRegisters = 0; bool HasOpaqueSPAdjustment = false; bool HasVAStart = false; bool HasMustTailInVarArgFunc = false; @@ -443,6 +444,8 @@ struct MachineFrameInfo { AdjustsStack == Other.AdjustsStack && HasCalls == Other.HasCalls && StackProtector == Other.StackProtector && MaxCallFrameSize == Other.MaxCallFrameSize && + CVBytesOfCalleeSavedRegisters == + Other.CVBytesOfCalleeSavedRegisters && HasOpaqueSPAdjustment == Other.HasOpaqueSPAdjustment && HasVAStart == Other.HasVAStart && HasMustTailInVarArgFunc == Other.HasMustTailInVarArgFunc && @@ -465,6 +468,8 @@ template <> struct MappingTraits { YamlIO.mapOptional("stackProtector", MFI.StackProtector, StringValue()); // Don't print it out when it's empty. YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize, (unsigned)~0); + YamlIO.mapOptional("cvBytesOfCalleeSavedRegisters", + MFI.CVBytesOfCalleeSavedRegisters, 0U); YamlIO.mapOptional("hasOpaqueSPAdjustment", MFI.HasOpaqueSPAdjustment, false); YamlIO.mapOptional("hasVAStart", MFI.HasVAStart, false); @@ -489,6 +494,7 @@ struct MachineFunction { bool FailedISel = false; // Register information bool TracksRegLiveness = false; + bool HasWinCFI = false; std::vector VirtualRegisters; std::vector LiveIns; Optional> CalleeSavedRegisters; @@ -512,6 +518,7 @@ template <> struct MappingTraits { YamlIO.mapOptional("selected", MF.Selected, false); YamlIO.mapOptional("failedISel", MF.FailedISel, false); YamlIO.mapOptional("tracksRegLiveness", MF.TracksRegLiveness, false); + YamlIO.mapOptional("hasWinCFI", MF.HasWinCFI, false); YamlIO.mapOptional("registers", MF.VirtualRegisters, std::vector()); YamlIO.mapOptional("liveins", MF.LiveIns, diff --git a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h index ace33efd871..ec2f270fcb3 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -569,6 +569,12 @@ public: return !empty() && back().isReturn(); } + /// Convenience function that returns true if the bock ends in a EH scope + /// return instruction. + bool isEHScopeReturnBlock() const { + return !empty() && back().isEHScopeReturn(); + } + /// Split the critical edge from this block to the given successor block, and /// return the newly created block, or null if splitting is not possible. /// diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h index 2d6081f3577..c2706a21a17 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -28,9 +28,14 @@ class AllocaInst; /// The CalleeSavedInfo class tracks the information need to locate where a /// callee saved register is in the current frame. +/// Callee saved reg can also be saved to a different register rather than +/// on the stack by setting DstReg instead of FrameIdx. class CalleeSavedInfo { unsigned Reg; - int FrameIdx; + union { + int FrameIdx; + unsigned DstReg; + }; /// Flag indicating whether the register is actually restored in the epilog. /// In most cases, if a register is saved, it is also restored. There are /// some situations, though, when this is not the case. For example, the @@ -44,17 +49,29 @@ class CalleeSavedInfo { /// by implicit uses on the return instructions, however, the required /// changes in the ARM backend would be quite extensive. bool Restored; + /// Flag indicating whether the register is spilled to stack or another + /// register. + bool SpilledToReg; public: explicit CalleeSavedInfo(unsigned R, int FI = 0) - : Reg(R), FrameIdx(FI), Restored(true) {} + : Reg(R), FrameIdx(FI), Restored(true), SpilledToReg(false) {} // Accessors. unsigned getReg() const { return Reg; } int getFrameIdx() const { return FrameIdx; } - void setFrameIdx(int FI) { FrameIdx = FI; } + unsigned getDstReg() const { return DstReg; } + void setFrameIdx(int FI) { + FrameIdx = FI; + SpilledToReg = false; + } + void setDstReg(unsigned SpillReg) { + DstReg = SpillReg; + SpilledToReg = true; + } bool isRestored() const { return Restored; } void setRestored(bool R) { Restored = R; } + bool isSpilledToReg() const { return SpilledToReg; } }; /// The MachineFrameInfo class represents an abstract stack frame until @@ -266,10 +283,14 @@ private: /// It is only valid during and after prolog/epilog code insertion. unsigned MaxCallFrameSize = ~0u; + /// The number of bytes of callee saved registers that the target wants to + /// report for the current function in the CodeView S_FRAMEPROC record. + unsigned CVBytesOfCalleeSavedRegisters = 0; + /// The prolog/epilog code inserter fills in this vector with each - /// callee saved register saved in the frame. Beyond its use by the prolog/ - /// epilog code inserter, this data used for debug info and exception - /// handling. + /// callee saved register saved in either the frame or a different + /// register. Beyond its use by the prolog/ epilog code inserter, + /// this data is used for debug info and exception handling. std::vector CSInfo; /// Has CSInfo been set yet? @@ -603,6 +624,15 @@ public: } void setMaxCallFrameSize(unsigned S) { MaxCallFrameSize = S; } + /// Returns how many bytes of callee-saved registers the target pushed in the + /// prologue. Only used for debug info. + unsigned getCVBytesOfCalleeSavedRegisters() const { + return CVBytesOfCalleeSavedRegisters; + } + void setCVBytesOfCalleeSavedRegisters(unsigned S) { + CVBytesOfCalleeSavedRegisters = S; + } + /// Create a new object at a fixed location on the stack. /// All fixed objects should be created before other objects are created for /// efficiency. By default, fixed objects are not pointed to by LLVM IR diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h index e8a4d529faa..6dbe1650ada 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h @@ -58,6 +58,7 @@ class DILocalVariable; class DILocation; class Function; class GlobalValue; +class LLVMTargetMachine; class MachineConstantPool; class MachineFrameInfo; class MachineFunction; @@ -70,7 +71,6 @@ class Pass; class PseudoSourceValueManager; class raw_ostream; class SlotIndexes; -class TargetMachine; class TargetRegisterClass; class TargetSubtargetInfo; struct WasmEHFuncInfo; @@ -225,7 +225,7 @@ struct LandingPadInfo { class MachineFunction { const Function &F; - const TargetMachine &Target; + const LLVMTargetMachine &Target; const TargetSubtargetInfo *STI; MCContext &Ctx; MachineModuleInfo &MMI; @@ -294,7 +294,7 @@ class MachineFunction { bool HasInlineAsm = false; /// True if any WinCFI instruction have been emitted in this function. - Optional HasWinCFI; + bool HasWinCFI = false; /// Current high-level properties of the IR of the function (e.g. is in SSA /// form or whether registers have been allocated) @@ -316,6 +316,9 @@ class MachineFunction { /// Map a landing pad's EH symbol to the call site indexes. DenseMap> LPadToCallSiteMap; + /// Map a landing pad to its index. + DenseMap WasmLPadToIndexMap; + /// Map of invoke call site index values to associated begin EH_LABEL. DenseMap CallSiteMap; @@ -326,6 +329,7 @@ class MachineFunction { bool CallsUnwindInit = false; bool HasEHScopes = false; bool HasEHFunclets = false; + bool HasLocalEscape = false; /// List of C++ TypeInfo used. std::vector TypeInfos; @@ -363,10 +367,31 @@ public: int Slot, const DILocation *Loc) : Var(Var), Expr(Expr), Slot(Slot), Loc(Loc) {} }; + + class Delegate { + virtual void anchor(); + + public: + virtual ~Delegate() = default; + /// Callback after an insertion. This should not modify the MI directly. + virtual void MF_HandleInsertion(MachineInstr &MI) = 0; + /// Callback before a removal. This should not modify the MI directly. + virtual void MF_HandleRemoval(MachineInstr &MI) = 0; + }; + +private: + Delegate *TheDelegate = nullptr; + + // Callbacks for insertion and removal. + void handleInsertion(MachineInstr &MI); + void handleRemoval(MachineInstr &MI); + friend struct ilist_traits; + +public: using VariableDbgInfoMapTy = SmallVector; VariableDbgInfoMapTy VariableDbgInfos; - MachineFunction(const Function &F, const TargetMachine &Target, + MachineFunction(const Function &F, const LLVMTargetMachine &Target, const TargetSubtargetInfo &STI, unsigned FunctionNum, MachineModuleInfo &MMI); MachineFunction(const MachineFunction &) = delete; @@ -379,6 +404,23 @@ public: init(); } + /// Reset the currently registered delegate - otherwise assert. + void resetDelegate(Delegate *delegate) { + assert(TheDelegate == delegate && + "Only the current delegate can perform reset!"); + TheDelegate = nullptr; + } + + /// Set the delegate. resetDelegate must be called before attempting + /// to set. + void setDelegate(Delegate *delegate) { + assert(delegate && !TheDelegate && + "Attempted to set delegate to null, or to change it without " + "first resetting it!"); + + TheDelegate = delegate; + } + MachineModuleInfo &getMMI() const { return MMI; } MCContext &getContext() const { return Ctx; } @@ -397,7 +439,7 @@ public: unsigned getFunctionNumber() const { return FunctionNumber; } /// getTarget - Return the target machine this machine code is compiled with - const TargetMachine &getTarget() const { return Target; } + const LLVMTargetMachine &getTarget() const { return Target; } /// getSubtarget - Return the subtarget for which this machine code is being /// compiled. @@ -484,8 +526,7 @@ public: } bool hasWinCFI() const { - assert(HasWinCFI.hasValue() && "HasWinCFI not set yet!"); - return *HasWinCFI; + return HasWinCFI; } void setHasWinCFI(bool v) { HasWinCFI = v; } @@ -619,6 +660,14 @@ public: BasicBlocks.sort(comp); } + /// Return the number of \p MachineInstrs in this \p MachineFunction. + unsigned getInstructionCount() const { + unsigned InstrCount = 0; + for (const MachineBasicBlock &MBB : BasicBlocks) + InstrCount += MBB.size(); + return InstrCount; + } + //===--------------------------------------------------------------------===// // Internal functions used to automatically number MachineBasicBlocks @@ -711,23 +760,14 @@ public: /// Allocate and initialize a register mask with @p NumRegister bits. uint32_t *allocateRegMask(); - /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand - /// pointers. This array is owned by the MachineFunction. - MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num); - - /// extractLoadMemRefs - Allocate an array and populate it with just the - /// load information from the given MachineMemOperand sequence. - std::pair - extractLoadMemRefs(MachineInstr::mmo_iterator Begin, - MachineInstr::mmo_iterator End); - - /// extractStoreMemRefs - Allocate an array and populate it with just the - /// store information from the given MachineMemOperand sequence. - std::pair - extractStoreMemRefs(MachineInstr::mmo_iterator Begin, - MachineInstr::mmo_iterator End); + /// Allocate and construct an extra info structure for a `MachineInstr`. + /// + /// This is allocated on the function's allocator and so lives the life of + /// the function. + MachineInstr::ExtraInfo * + createMIExtraInfo(ArrayRef MMOs, + MCSymbol *PreInstrSymbol = nullptr, + MCSymbol *PostInstrSymbol = nullptr); /// Allocate a string and populate it with the given external symbol name. const char *createExternalSymbolName(StringRef Name); @@ -772,11 +812,15 @@ public: bool hasEHFunclets() const { return HasEHFunclets; } void setHasEHFunclets(bool V) { HasEHFunclets = V; } + bool hasLocalEscape() const { return HasLocalEscape; } + void setHasLocalEscape(bool V) { HasLocalEscape = V; } + /// Find or create an LandingPadInfo for the specified MachineBasicBlock. LandingPadInfo &getOrCreateLandingPadInfo(MachineBasicBlock *LandingPad); /// Remap landing pad labels and remove any deleted landing pads. - void tidyLandingPads(DenseMap *LPMap = nullptr); + void tidyLandingPads(DenseMap *LPMap = nullptr, + bool TidyIfNoBeginLabels = true); /// Return a reference to the landing pad info for the current function. const std::vector &getLandingPads() const { @@ -788,7 +832,9 @@ public: void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel); - /// Add a new panding pad. Returns the label ID for the landing pad entry. + /// Add a new panding pad, and extract the exception handling information from + /// the landingpad instruction. Returns the label ID for the landing pad + /// entry. MCSymbol *addLandingPad(MachineBasicBlock *LandingPad); /// Provide the catch typeinfo for a landing pad. @@ -817,6 +863,22 @@ public: /// Map the landing pad's EH symbol to the call site indexes. void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef Sites); + /// Map the landing pad to its index. Used for Wasm exception handling. + void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index) { + WasmLPadToIndexMap[LPad] = Index; + } + + /// Returns true if the landing pad has an associate index in wasm EH. + bool hasWasmLandingPadIndex(const MachineBasicBlock *LPad) const { + return WasmLPadToIndexMap.count(LPad); + } + + /// Get the index in wasm EH for a given landing pad. + unsigned getWasmLandingPadIndex(const MachineBasicBlock *LPad) const { + assert(hasWasmLandingPadIndex(LPad)); + return WasmLPadToIndexMap.lookup(LPad); + } + /// Get the call site indexes for a landing pad EH symbol. SmallVectorImpl &getCallSiteLandingPad(MCSymbol *Sym) { assert(hasCallSiteLandingPad(Sym) && @@ -880,15 +942,6 @@ public: } }; -/// \name Exception Handling -/// \{ - -/// Extract the exception handling information from the landingpad instruction -/// and add them to the specified machine module info. -void addLandingPadInfo(const LandingPadInst &I, MachineBasicBlock &MBB); - -/// \} - //===--------------------------------------------------------------------===// // GraphTraits specializations for function basic block graphs (CFGs) //===--------------------------------------------------------------------===// diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h index 88e13cdf413..ea1a2a536fc 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h @@ -17,16 +17,20 @@ #define LLVM_CODEGEN_MACHINEINSTR_H #include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/PointerSumType.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Support/ArrayRecycler.h" +#include "llvm/Support/TrailingObjects.h" #include #include #include @@ -61,7 +65,7 @@ class MachineInstr : public ilist_node_with_parent> { public: - using mmo_iterator = MachineMemOperand **; + using mmo_iterator = ArrayRef::iterator; /// Flags to specify different kinds of comments to output in /// assembly code. These flags carry semantic information not @@ -93,8 +97,14 @@ public: // contraction operations like fma. FmAfn = 1 << 9, // Instruction may map to Fast math // instrinsic approximation. - FmReassoc = 1 << 10 // Instruction supports Fast math + FmReassoc = 1 << 10, // Instruction supports Fast math // reassociation of operand order. + NoUWrap = 1 << 11, // Instruction supports binary operator + // no unsigned wrap. + NoSWrap = 1 << 12, // Instruction supports binary operator + // no signed wrap. + IsExact = 1 << 13 // Instruction supports division is + // known to be exact. }; private: @@ -118,14 +128,102 @@ private: // anything other than to convey comment // information to AsmPrinter. - uint8_t NumMemRefs = 0; // Information on memory references. - // Note that MemRefs == nullptr, means 'don't know', not 'no memory access'. - // Calling code must treat missing information conservatively. If the number - // of memory operands required to be precise exceeds the maximum value of - // NumMemRefs - currently 256 - we remove the operands entirely. Note also - // that this is a non-owning reference to a shared copy on write buffer owned - // by the MachineFunction and created via MF.allocateMemRefsArray. - mmo_iterator MemRefs = nullptr; + /// Internal implementation detail class that provides out-of-line storage for + /// extra info used by the machine instruction when this info cannot be stored + /// in-line within the instruction itself. + /// + /// This has to be defined eagerly due to the implementation constraints of + /// `PointerSumType` where it is used. + class ExtraInfo final + : TrailingObjects { + public: + static ExtraInfo *create(BumpPtrAllocator &Allocator, + ArrayRef MMOs, + MCSymbol *PreInstrSymbol = nullptr, + MCSymbol *PostInstrSymbol = nullptr) { + bool HasPreInstrSymbol = PreInstrSymbol != nullptr; + bool HasPostInstrSymbol = PostInstrSymbol != nullptr; + auto *Result = new (Allocator.Allocate( + totalSizeToAlloc( + MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol), + alignof(ExtraInfo))) + ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol); + + // Copy the actual data into the trailing objects. + std::copy(MMOs.begin(), MMOs.end(), + Result->getTrailingObjects()); + + if (HasPreInstrSymbol) + Result->getTrailingObjects()[0] = PreInstrSymbol; + if (HasPostInstrSymbol) + Result->getTrailingObjects()[HasPreInstrSymbol] = + PostInstrSymbol; + + return Result; + } + + ArrayRef getMMOs() const { + return makeArrayRef(getTrailingObjects(), NumMMOs); + } + + MCSymbol *getPreInstrSymbol() const { + return HasPreInstrSymbol ? getTrailingObjects()[0] : nullptr; + } + + MCSymbol *getPostInstrSymbol() const { + return HasPostInstrSymbol + ? getTrailingObjects()[HasPreInstrSymbol] + : nullptr; + } + + private: + friend TrailingObjects; + + // Description of the extra info, used to interpret the actual optional + // data appended. + // + // Note that this is not terribly space optimized. This leaves a great deal + // of flexibility to fit more in here later. + const int NumMMOs; + const bool HasPreInstrSymbol; + const bool HasPostInstrSymbol; + + // Implement the `TrailingObjects` internal API. + size_t numTrailingObjects(OverloadToken) const { + return NumMMOs; + } + size_t numTrailingObjects(OverloadToken) const { + return HasPreInstrSymbol + HasPostInstrSymbol; + } + + // Just a boring constructor to allow us to initialize the sizes. Always use + // the `create` routine above. + ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol) + : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), + HasPostInstrSymbol(HasPostInstrSymbol) {} + }; + + /// Enumeration of the kinds of inline extra info available. It is important + /// that the `MachineMemOperand` inline kind has a tag value of zero to make + /// it accessible as an `ArrayRef`. + enum ExtraInfoInlineKinds { + EIIK_MMO = 0, + EIIK_PreInstrSymbol, + EIIK_PostInstrSymbol, + EIIK_OutOfLine + }; + + // We store extra information about the instruction here. The common case is + // expected to be nothing or a single pointer (typically a MMO or a symbol). + // We work to optimize this common case by storing it inline here rather than + // requiring a separate allocation, but we fall back to an allocation when + // multiple pointers are needed. + PointerSumType, + PointerSumTypeMember, + PointerSumTypeMember, + PointerSumTypeMember> + Info; DebugLoc debugLoc; // Source line information. @@ -310,7 +408,7 @@ public: /// Returns the opcode of this MachineInstr. unsigned getOpcode() const { return MCID->Opcode; } - /// Access to explicit operands of the instruction. + /// Retuns the total number of operands. unsigned getNumOperands() const { return NumOperands; } const MachineOperand& getOperand(unsigned i) const { @@ -412,28 +510,70 @@ public: return I - operands_begin(); } - /// Access to memory operands of the instruction - mmo_iterator memoperands_begin() const { return MemRefs; } - mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; } + /// Access to memory operands of the instruction. If there are none, that does + /// not imply anything about whether the function accesses memory. Instead, + /// the caller must behave conservatively. + ArrayRef memoperands() const { + if (!Info) + return {}; + + if (Info.is()) + return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); + + if (ExtraInfo *EI = Info.get()) + return EI->getMMOs(); + + return {}; + } + + /// Access to memory operands of the instruction. + /// + /// If `memoperands_begin() == memoperands_end()`, that does not imply + /// anything about whether the function accesses memory. Instead, the caller + /// must behave conservatively. + mmo_iterator memoperands_begin() const { return memoperands().begin(); } + + /// Access to memory operands of the instruction. + /// + /// If `memoperands_begin() == memoperands_end()`, that does not imply + /// anything about whether the function accesses memory. Instead, the caller + /// must behave conservatively. + mmo_iterator memoperands_end() const { return memoperands().end(); } + /// Return true if we don't have any memory operands which described the /// memory access done by this instruction. If this is true, calling code /// must be conservative. - bool memoperands_empty() const { return NumMemRefs == 0; } - - iterator_range memoperands() { - return make_range(memoperands_begin(), memoperands_end()); - } - iterator_range memoperands() const { - return make_range(memoperands_begin(), memoperands_end()); - } + bool memoperands_empty() const { return memoperands().empty(); } /// Return true if this instruction has exactly one MachineMemOperand. - bool hasOneMemOperand() const { - return NumMemRefs == 1; - } + bool hasOneMemOperand() const { return memoperands().size() == 1; } /// Return the number of memory operands. - unsigned getNumMemOperands() const { return NumMemRefs; } + unsigned getNumMemOperands() const { return memoperands().size(); } + + /// Helper to extract a pre-instruction symbol if one has been added. + MCSymbol *getPreInstrSymbol() const { + if (!Info) + return nullptr; + if (MCSymbol *S = Info.get()) + return S; + if (ExtraInfo *EI = Info.get()) + return EI->getPreInstrSymbol(); + + return nullptr; + } + + /// Helper to extract a post-instruction symbol if one has been added. + MCSymbol *getPostInstrSymbol() const { + if (!Info) + return nullptr; + if (MCSymbol *S = Info.get()) + return S; + if (ExtraInfo *EI = Info.get()) + return EI->getPostInstrSymbol(); + + return nullptr; + } /// API for querying MachineInstr properties. They are the same as MCInstrDesc /// queries but they are bundle aware. @@ -450,6 +590,8 @@ public: /// The second argument indicates whether the query should look inside /// instruction bundles. bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { + assert(MCFlag < 64 && + "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle."); // Inline the fast path for unbundled or bundle-internal instructions. if (Type == IgnoreBundle || !isBundled() || isBundledWithPred()) return getDesc().getFlags() & (1ULL << MCFlag); @@ -482,6 +624,12 @@ public: return hasProperty(MCID::Return, Type); } + /// Return true if this is an instruction that marks the end of an EH scope, + /// i.e., a catchpad or a cleanuppad instruction. + bool isEHScopeReturn(QueryType Type = AnyInBundle) const { + return hasProperty(MCID::EHScopeReturn, Type); + } + bool isCall(QueryType Type = AnyInBundle) const { return hasProperty(MCID::Call, Type); } @@ -1323,47 +1471,63 @@ public: /// fewer operand than it started with. void RemoveOperand(unsigned OpNo); + /// Clear this MachineInstr's memory reference descriptor list. This resets + /// the memrefs to their most conservative state. This should be used only + /// as a last resort since it greatly pessimizes our knowledge of the memory + /// access performed by the instruction. + void dropMemRefs(MachineFunction &MF); + + /// Assign this MachineInstr's memory reference descriptor list. + /// + /// Unlike other methods, this *will* allocate them into a new array + /// associated with the provided `MachineFunction`. + void setMemRefs(MachineFunction &MF, ArrayRef MemRefs); + /// Add a MachineMemOperand to the machine instruction. /// This function should be used only occasionally. The setMemRefs function /// is the primary method for setting up a MachineInstr's MemRefs list. void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); - /// Assign this MachineInstr's memory reference descriptor list. - /// This does not transfer ownership. - void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { - setMemRefs(std::make_pair(NewMemRefs, NewMemRefsEnd-NewMemRefs)); - } + /// Clone another MachineInstr's memory reference descriptor list and replace + /// ours with it. + /// + /// Note that `*this` may be the incoming MI! + /// + /// Prefer this API whenever possible as it can avoid allocations in common + /// cases. + void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); - /// Assign this MachineInstr's memory reference descriptor list. First - /// element in the pair is the begin iterator/pointer to the array; the - /// second is the number of MemoryOperands. This does not transfer ownership - /// of the underlying memory. - void setMemRefs(std::pair NewMemRefs) { - MemRefs = NewMemRefs.first; - NumMemRefs = uint8_t(NewMemRefs.second); - assert(NumMemRefs == NewMemRefs.second && - "Too many memrefs - must drop memory operands"); - } + /// Clone the merge of multiple MachineInstrs' memory reference descriptors + /// list and replace ours with it. + /// + /// Note that `*this` may be one of the incoming MIs! + /// + /// Prefer this API whenever possible as it can avoid allocations in common + /// cases. + void cloneMergedMemRefs(MachineFunction &MF, + ArrayRef MIs); - /// Return a set of memrefs (begin iterator, size) which conservatively - /// describe the memory behavior of both MachineInstrs. This is appropriate - /// for use when merging two MachineInstrs into one. This routine does not - /// modify the memrefs of the this MachineInstr. - std::pair mergeMemRefsWith(const MachineInstr& Other); + /// Set a symbol that will be emitted just prior to the instruction itself. + /// + /// Setting this to a null pointer will remove any such symbol. + /// + /// FIXME: This is not fully implemented yet. + void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); + + /// Set a symbol that will be emitted just after the instruction itself. + /// + /// Setting this to a null pointer will remove any such symbol. + /// + /// FIXME: This is not fully implemented yet. + void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); /// Return the MIFlags which represent both MachineInstrs. This /// should be used when merging two MachineInstrs into one. This routine does /// not modify the MIFlags of this MachineInstr. uint16_t mergeFlagsWith(const MachineInstr& Other) const; - /// Clear this MachineInstr's memory reference descriptor list. This resets - /// the memrefs to their most conservative state. This should be used only - /// as a last resort since it greatly pessimizes our knowledge of the memory - /// access performed by the instruction. - void dropMemRefs() { - MemRefs = nullptr; - NumMemRefs = 0; - } + /// Copy all flags to MachineInst MIFlags + void copyIRFlags(const Instruction &I); /// Break any tie involving OpIdx. void untieRegOperand(unsigned OpIdx) { @@ -1377,6 +1541,13 @@ public: /// Add all implicit def and use operands to this instruction. void addImplicitDefUseOperands(MachineFunction &MF); + /// Scan instructions following MI and collect any matching DBG_VALUEs. + void collectDebugValues(SmallVectorImpl &DbgValues); + + /// Find all DBG_VALUEs immediately following this instruction that point + /// to a register def in this instruction and point them to \p Reg instead. + void changeDebugValuesDefReg(unsigned Reg); + private: /// If this instruction is embedded into a MachineFunction, return the /// MachineRegisterInfo object for the current function, otherwise @@ -1394,7 +1565,7 @@ private: void AddRegOperandsToUseLists(MachineRegisterInfo&); /// Slow path for hasProperty when we're dealing with a bundle. - bool hasPropertyInBundle(unsigned Mask, QueryType Type) const; + bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; /// Implements the logic of getRegClassConstraintEffectForVReg for the /// this MI and the given operand index \p OpIdx. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 66560875574..b5e523f655e 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -191,15 +191,20 @@ public: return *this; } - const MachineInstrBuilder &setMemRefs(MachineInstr::mmo_iterator b, - MachineInstr::mmo_iterator e) const { - MI->setMemRefs(b, e); + const MachineInstrBuilder & + setMemRefs(ArrayRef MMOs) const { + MI->setMemRefs(*MF, MMOs); return *this; } - const MachineInstrBuilder &setMemRefs(std::pair MemOperandsRef) const { - MI->setMemRefs(MemOperandsRef); + const MachineInstrBuilder &cloneMemRefs(const MachineInstr &OtherMI) const { + MI->cloneMemRefs(*MF, OtherMI); + return *this; + } + + const MachineInstrBuilder & + cloneMergedMemRefs(ArrayRef OtherMIs) const { + MI->cloneMergedMemRefs(*MF, OtherMIs); return *this; } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h index 554e89019b7..4371420bc7a 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h @@ -46,10 +46,10 @@ namespace llvm { class BasicBlock; class CallInst; class Function; -class MachineFunction; +class LLVMTargetMachine; class MMIAddrLabelMap; +class MachineFunction; class Module; -class TargetMachine; //===----------------------------------------------------------------------===// /// This class can be derived from and used by targets to hold private @@ -76,7 +76,7 @@ protected: /// for specific use. /// class MachineModuleInfo : public ImmutablePass { - const TargetMachine &TM; + const LLVMTargetMachine &TM; /// This is the MCContext used for the entire code generator. MCContext Context; @@ -145,7 +145,7 @@ class MachineModuleInfo : public ImmutablePass { public: static char ID; // Pass identification, replacement for typeid - explicit MachineModuleInfo(const TargetMachine *TM = nullptr); + explicit MachineModuleInfo(const LLVMTargetMachine *TM = nullptr); ~MachineModuleInfo() override; // Initialization and Finalization diff --git a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h index 6a87fa2fbf0..17df1fa792b 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -80,6 +80,28 @@ public: SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } }; +/// MachineModuleInfoCOFF - This is a MachineModuleInfoImpl implementation +/// for COFF targets. +class MachineModuleInfoCOFF : public MachineModuleInfoImpl { + /// GVStubs - These stubs are used to materialize global addresses in PIC + /// mode. + DenseMap GVStubs; + + virtual void anchor(); // Out of line virtual method. + +public: + MachineModuleInfoCOFF(const MachineModuleInfo &) {} + + StubValueTy &getGVStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return GVStubs[Sym]; + } + + /// Accessor methods to return the set of stubs in sorted order. + + SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } +}; + } // end namespace llvm #endif // LLVM_CODEGEN_MACHINEMODULEINFOIMPLS_H diff --git a/contrib/llvm/include/llvm/CodeGen/MachineOutliner.h b/contrib/llvm/include/llvm/CodeGen/MachineOutliner.h index 95bfc24b57f..bfd1e994053 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineOutliner.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineOutliner.h @@ -61,9 +61,6 @@ public: /// \p OutlinedFunctions. unsigned FunctionIdx; - /// Set to false if the candidate overlapped with another candidate. - bool InCandidateList = true; - /// Identifier denoting the instructions to emit to call an outlined function /// from this point. Defined by the target. unsigned CallConstructionID; @@ -82,6 +79,12 @@ public: /// been used across the sequence. LiveRegUnits UsedInSequence; + /// Target-specific flags for this Candidate's MBB. + unsigned Flags = 0x0; + + /// True if initLRU has been called on this Candidate. + bool LRUWasSet = false; + /// Return the number of instructions in this Candidate. unsigned getLength() const { return Len; } @@ -99,9 +102,7 @@ public: } /// Returns the call overhead of this candidate if it is in the list. - unsigned getCallOverhead() const { - return InCandidateList ? CallOverhead : 0; - } + unsigned getCallOverhead() const { return CallOverhead; } MachineBasicBlock::iterator &front() { return FirstInst; } MachineBasicBlock::iterator &back() { return LastInst; } @@ -120,9 +121,9 @@ public: Candidate(unsigned StartIdx, unsigned Len, MachineBasicBlock::iterator &FirstInst, MachineBasicBlock::iterator &LastInst, MachineBasicBlock *MBB, - unsigned FunctionIdx) + unsigned FunctionIdx, unsigned Flags) : StartIdx(StartIdx), Len(Len), FirstInst(FirstInst), LastInst(LastInst), - MBB(MBB), FunctionIdx(FunctionIdx) {} + MBB(MBB), FunctionIdx(FunctionIdx), Flags(Flags) {} Candidate() {} /// Used to ensure that \p Candidates are outlined in an order that @@ -138,6 +139,10 @@ public: void initLRU(const TargetRegisterInfo &TRI) { assert(MBB->getParent()->getRegInfo().tracksLiveness() && "Candidate's Machine Function must track liveness"); + // Only initialize once. + if (LRUWasSet) + return; + LRUWasSet = true; LRU.init(TRI); LRU.addLiveOuts(*MBB); @@ -158,24 +163,13 @@ public: /// class of candidate. struct OutlinedFunction { -private: - /// The number of candidates for this \p OutlinedFunction. - unsigned OccurrenceCount = 0; - public: - std::vector> Candidates; + std::vector Candidates; /// The actual outlined function created. /// This is initialized after we go through and create the actual function. MachineFunction *MF = nullptr; - /// A number assigned to this function which appears at the end of its name. - unsigned Name; - - /// The sequence of integers corresponding to the instructions in this - /// function. - std::vector Sequence; - /// Represents the size of a sequence in bytes. (Some instructions vary /// widely in size, so just counting the instructions isn't very useful.) unsigned SequenceSize; @@ -187,49 +181,41 @@ public: unsigned FrameConstructionID; /// Return the number of candidates for this \p OutlinedFunction. - unsigned getOccurrenceCount() { return OccurrenceCount; } - - /// Decrement the occurrence count of this OutlinedFunction and return the - /// new count. - unsigned decrement() { - assert(OccurrenceCount > 0 && "Can't decrement an empty function!"); - OccurrenceCount--; - return getOccurrenceCount(); - } + unsigned getOccurrenceCount() const { return Candidates.size(); } /// Return the number of bytes it would take to outline this /// function. - unsigned getOutliningCost() { + unsigned getOutliningCost() const { unsigned CallOverhead = 0; - for (std::shared_ptr &C : Candidates) - CallOverhead += C->getCallOverhead(); + for (const Candidate &C : Candidates) + CallOverhead += C.getCallOverhead(); return CallOverhead + SequenceSize + FrameOverhead; } /// Return the size in bytes of the unoutlined sequences. - unsigned getNotOutlinedCost() { return OccurrenceCount * SequenceSize; } + unsigned getNotOutlinedCost() const { + return getOccurrenceCount() * SequenceSize; + } /// Return the number of instructions that would be saved by outlining /// this function. - unsigned getBenefit() { + unsigned getBenefit() const { unsigned NotOutlinedCost = getNotOutlinedCost(); unsigned OutlinedCost = getOutliningCost(); return (NotOutlinedCost < OutlinedCost) ? 0 : NotOutlinedCost - OutlinedCost; } - OutlinedFunction(std::vector &Cands, - unsigned SequenceSize, unsigned FrameOverhead, - unsigned FrameConstructionID) - : SequenceSize(SequenceSize), FrameOverhead(FrameOverhead), - FrameConstructionID(FrameConstructionID) { - OccurrenceCount = Cands.size(); - for (Candidate &C : Cands) - Candidates.push_back(std::make_shared(C)); + /// Return the number of instructions in this sequence. + unsigned getNumInstrs() const { return Candidates[0].getLength(); } - unsigned B = getBenefit(); - for (std::shared_ptr &C : Candidates) - C->Benefit = B; + OutlinedFunction(std::vector &Candidates, unsigned SequenceSize, + unsigned FrameOverhead, unsigned FrameConstructionID) + : Candidates(Candidates), SequenceSize(SequenceSize), + FrameOverhead(FrameOverhead), FrameConstructionID(FrameConstructionID) { + const unsigned B = getBenefit(); + for (Candidate &C : Candidates) + C.Benefit = B; } OutlinedFunction() {} diff --git a/contrib/llvm/include/llvm/CodeGen/MachinePassRegistry.h b/contrib/llvm/include/llvm/CodeGen/MachinePassRegistry.h index 3aba0bba7d1..a031c92d914 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachinePassRegistry.h +++ b/contrib/llvm/include/llvm/CodeGen/MachinePassRegistry.h @@ -24,22 +24,20 @@ namespace llvm { -using MachinePassCtor = void *(*)(); - //===----------------------------------------------------------------------===// /// /// MachinePassRegistryListener - Listener to adds and removals of nodes in /// registration list. /// //===----------------------------------------------------------------------===// -class MachinePassRegistryListener { - virtual void anchor(); +template class MachinePassRegistryListener { + virtual void anchor() {} public: MachinePassRegistryListener() = default; virtual ~MachinePassRegistryListener() = default; - virtual void NotifyAdd(StringRef N, MachinePassCtor C, StringRef D) = 0; + virtual void NotifyAdd(StringRef N, PassCtorTy C, StringRef D) = 0; virtual void NotifyRemove(StringRef N) = 0; }; @@ -48,15 +46,15 @@ public: /// MachinePassRegistryNode - Machine pass node stored in registration list. /// //===----------------------------------------------------------------------===// -class MachinePassRegistryNode { +template class MachinePassRegistryNode { private: MachinePassRegistryNode *Next = nullptr; // Next function pass in list. StringRef Name; // Name of function pass. StringRef Description; // Description string. - MachinePassCtor Ctor; // Function pass creator. + PassCtorTy Ctor; // Pass creator. public: - MachinePassRegistryNode(const char *N, const char *D, MachinePassCtor C) + MachinePassRegistryNode(const char *N, const char *D, PassCtorTy C) : Name(N), Description(D), Ctor(C) {} // Accessors @@ -64,7 +62,7 @@ public: MachinePassRegistryNode **getNextAddress() { return &Next; } StringRef getName() const { return Name; } StringRef getDescription() const { return Description; } - MachinePassCtor getCtor() const { return Ctor; } + PassCtorTy getCtor() const { return Ctor; } void setNext(MachinePassRegistryNode *N) { Next = N; } }; @@ -73,11 +71,12 @@ public: /// MachinePassRegistry - Track the registration of machine passes. /// //===----------------------------------------------------------------------===// -class MachinePassRegistry { +template class MachinePassRegistry { private: - MachinePassRegistryNode *List; // List of registry nodes. - MachinePassCtor Default; // Default function pass creator. - MachinePassRegistryListener *Listener; // Listener for list adds are removes. + MachinePassRegistryNode *List; // List of registry nodes. + PassCtorTy Default; // Default function pass creator. + MachinePassRegistryListener + *Listener; // Listener for list adds are removes. public: // NO CONSTRUCTOR - we don't want static constructor ordering to mess @@ -85,19 +84,47 @@ public: // Accessors. // - MachinePassRegistryNode *getList() { return List; } - MachinePassCtor getDefault() { return Default; } - void setDefault(MachinePassCtor C) { Default = C; } - void setDefault(StringRef Name); - void setListener(MachinePassRegistryListener *L) { Listener = L; } + MachinePassRegistryNode *getList() { return List; } + PassCtorTy getDefault() { return Default; } + void setDefault(PassCtorTy C) { Default = C; } + /// setDefault - Set the default constructor by name. + void setDefault(StringRef Name) { + PassCtorTy Ctor = nullptr; + for (MachinePassRegistryNode *R = getList(); R; + R = R->getNext()) { + if (R->getName() == Name) { + Ctor = R->getCtor(); + break; + } + } + assert(Ctor && "Unregistered pass name"); + setDefault(Ctor); + } + void setListener(MachinePassRegistryListener *L) { Listener = L; } /// Add - Adds a function pass to the registration list. /// - void Add(MachinePassRegistryNode *Node); + void Add(MachinePassRegistryNode *Node) { + Node->setNext(List); + List = Node; + if (Listener) + Listener->NotifyAdd(Node->getName(), Node->getCtor(), + Node->getDescription()); + } /// Remove - Removes a function pass from the registration list. /// - void Remove(MachinePassRegistryNode *Node); + void Remove(MachinePassRegistryNode *Node) { + for (MachinePassRegistryNode **I = &List; *I; + I = (*I)->getNextAddress()) { + if (*I == Node) { + if (Listener) + Listener->NotifyRemove(Node->getName()); + *I = (*I)->getNext(); + break; + } + } + } }; //===----------------------------------------------------------------------===// @@ -105,9 +132,11 @@ public: /// RegisterPassParser class - Handle the addition of new machine passes. /// //===----------------------------------------------------------------------===// -template -class RegisterPassParser : public MachinePassRegistryListener, - public cl::parser { +template +class RegisterPassParser + : public MachinePassRegistryListener< + typename RegistryClass::FunctionPassCtor>, + public cl::parser { public: RegisterPassParser(cl::Option &O) : cl::parser(O) {} @@ -129,8 +158,9 @@ public: } // Implement the MachinePassRegistryListener callbacks. - void NotifyAdd(StringRef N, MachinePassCtor C, StringRef D) override { - this->addLiteralOption(N, (typename RegistryClass::FunctionPassCtor)C, D); + void NotifyAdd(StringRef N, typename RegistryClass::FunctionPassCtor C, + StringRef D) override { + this->addLiteralOption(N, C, D); } void NotifyRemove(StringRef N) override { this->removeLiteralOption(N); diff --git a/contrib/llvm/include/llvm/CodeGen/MachinePipeliner.h b/contrib/llvm/include/llvm/CodeGen/MachinePipeliner.h new file mode 100644 index 00000000000..38cb33e90e6 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/MachinePipeliner.h @@ -0,0 +1,608 @@ +//===- MachinePipeliner.h - Machine Software Pipeliner Pass -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// An implementation of the Swing Modulo Scheduling (SMS) software pipeliner. +// +// Software pipelining (SWP) is an instruction scheduling technique for loops +// that overlap loop iterations and exploits ILP via a compiler transformation. +// +// Swing Modulo Scheduling is an implementation of software pipelining +// that generates schedules that are near optimal in terms of initiation +// interval, register requirements, and stage count. See the papers: +// +// "Swing Modulo Scheduling: A Lifetime-Sensitive Approach", by J. Llosa, +// A. Gonzalez, E. Ayguade, and M. Valero. In PACT '96 Proceedings of the 1996 +// Conference on Parallel Architectures and Compilation Techiniques. +// +// "Lifetime-Sensitive Modulo Scheduling in a Production Environment", by J. +// Llosa, E. Ayguade, A. Gonzalez, M. Valero, and J. Eckhardt. In IEEE +// Transactions on Computers, Vol. 50, No. 3, 2001. +// +// "An Implementation of Swing Modulo Scheduling With Extensions for +// Superblocks", by T. Lattner, Master's Thesis, University of Illinois at +// Urbana-Champaign, 2005. +// +// +// The SMS algorithm consists of three main steps after computing the minimal +// initiation interval (MII). +// 1) Analyze the dependence graph and compute information about each +// instruction in the graph. +// 2) Order the nodes (instructions) by priority based upon the heuristics +// described in the algorithm. +// 3) Attempt to schedule the nodes in the specified order using the MII. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIB_CODEGEN_MACHINEPIPELINER_H +#define LLVM_LIB_CODEGEN_MACHINEPIPELINER_H + +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/CodeGen/TargetInstrInfo.h" + +namespace llvm { + +class NodeSet; +class SMSchedule; + +extern cl::opt SwpEnableCopyToPhi; + +/// The main class in the implementation of the target independent +/// software pipeliner pass. +class MachinePipeliner : public MachineFunctionPass { +public: + MachineFunction *MF = nullptr; + const MachineLoopInfo *MLI = nullptr; + const MachineDominatorTree *MDT = nullptr; + const InstrItineraryData *InstrItins; + const TargetInstrInfo *TII = nullptr; + RegisterClassInfo RegClassInfo; + +#ifndef NDEBUG + static int NumTries; +#endif + + /// Cache the target analysis information about the loop. + struct LoopInfo { + MachineBasicBlock *TBB = nullptr; + MachineBasicBlock *FBB = nullptr; + SmallVector BrCond; + MachineInstr *LoopInductionVar = nullptr; + MachineInstr *LoopCompare = nullptr; + }; + LoopInfo LI; + + static char ID; + + MachinePipeliner() : MachineFunctionPass(ID) { + initializeMachinePipelinerPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + void preprocessPhiNodes(MachineBasicBlock &B); + bool canPipelineLoop(MachineLoop &L); + bool scheduleLoop(MachineLoop &L); + bool swingModuloScheduler(MachineLoop &L); +}; + +/// This class builds the dependence graph for the instructions in a loop, +/// and attempts to schedule the instructions using the SMS algorithm. +class SwingSchedulerDAG : public ScheduleDAGInstrs { + MachinePipeliner &Pass; + /// The minimum initiation interval between iterations for this schedule. + unsigned MII = 0; + /// Set to true if a valid pipelined schedule is found for the loop. + bool Scheduled = false; + MachineLoop &Loop; + LiveIntervals &LIS; + const RegisterClassInfo &RegClassInfo; + + /// A toplogical ordering of the SUnits, which is needed for changing + /// dependences and iterating over the SUnits. + ScheduleDAGTopologicalSort Topo; + + struct NodeInfo { + int ASAP = 0; + int ALAP = 0; + int ZeroLatencyDepth = 0; + int ZeroLatencyHeight = 0; + + NodeInfo() = default; + }; + /// Computed properties for each node in the graph. + std::vector ScheduleInfo; + + enum OrderKind { BottomUp = 0, TopDown = 1 }; + /// Computed node ordering for scheduling. + SetVector NodeOrder; + + using NodeSetType = SmallVector; + using ValueMapTy = DenseMap; + using MBBVectorTy = SmallVectorImpl; + using InstrMapTy = DenseMap; + + /// Instructions to change when emitting the final schedule. + DenseMap> InstrChanges; + + /// We may create a new instruction, so remember it because it + /// must be deleted when the pass is finished. + SmallPtrSet NewMIs; + + /// Ordered list of DAG postprocessing steps. + std::vector> Mutations; + + /// Helper class to implement Johnson's circuit finding algorithm. + class Circuits { + std::vector &SUnits; + SetVector Stack; + BitVector Blocked; + SmallVector, 10> B; + SmallVector, 16> AdjK; + // Node to Index from ScheduleDAGTopologicalSort + std::vector *Node2Idx; + unsigned NumPaths; + static unsigned MaxPaths; + + public: + Circuits(std::vector &SUs, ScheduleDAGTopologicalSort &Topo) + : SUnits(SUs), Blocked(SUs.size()), B(SUs.size()), AdjK(SUs.size()) { + Node2Idx = new std::vector(SUs.size()); + unsigned Idx = 0; + for (const auto &NodeNum : Topo) + Node2Idx->at(NodeNum) = Idx++; + } + + ~Circuits() { delete Node2Idx; } + + /// Reset the data structures used in the circuit algorithm. + void reset() { + Stack.clear(); + Blocked.reset(); + B.assign(SUnits.size(), SmallPtrSet()); + NumPaths = 0; + } + + void createAdjacencyStructure(SwingSchedulerDAG *DAG); + bool circuit(int V, int S, NodeSetType &NodeSets, bool HasBackedge = false); + void unblock(int U); + }; + + struct CopyToPhiMutation : public ScheduleDAGMutation { + void apply(ScheduleDAGInstrs *DAG) override; + }; + +public: + SwingSchedulerDAG(MachinePipeliner &P, MachineLoop &L, LiveIntervals &lis, + const RegisterClassInfo &rci) + : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis), + RegClassInfo(rci), Topo(SUnits, &ExitSU) { + P.MF->getSubtarget().getSMSMutations(Mutations); + if (SwpEnableCopyToPhi) + Mutations.push_back(llvm::make_unique()); + } + + void schedule() override; + void finishBlock() override; + + /// Return true if the loop kernel has been scheduled. + bool hasNewSchedule() { return Scheduled; } + + /// Return the earliest time an instruction may be scheduled. + int getASAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ASAP; } + + /// Return the latest time an instruction my be scheduled. + int getALAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ALAP; } + + /// The mobility function, which the number of slots in which + /// an instruction may be scheduled. + int getMOV(SUnit *Node) { return getALAP(Node) - getASAP(Node); } + + /// The depth, in the dependence graph, for a node. + unsigned getDepth(SUnit *Node) { return Node->getDepth(); } + + /// The maximum unweighted length of a path from an arbitrary node to the + /// given node in which each edge has latency 0 + int getZeroLatencyDepth(SUnit *Node) { + return ScheduleInfo[Node->NodeNum].ZeroLatencyDepth; + } + + /// The height, in the dependence graph, for a node. + unsigned getHeight(SUnit *Node) { return Node->getHeight(); } + + /// The maximum unweighted length of a path from the given node to an + /// arbitrary node in which each edge has latency 0 + int getZeroLatencyHeight(SUnit *Node) { + return ScheduleInfo[Node->NodeNum].ZeroLatencyHeight; + } + + /// Return true if the dependence is a back-edge in the data dependence graph. + /// Since the DAG doesn't contain cycles, we represent a cycle in the graph + /// using an anti dependence from a Phi to an instruction. + bool isBackedge(SUnit *Source, const SDep &Dep) { + if (Dep.getKind() != SDep::Anti) + return false; + return Source->getInstr()->isPHI() || Dep.getSUnit()->getInstr()->isPHI(); + } + + bool isLoopCarriedDep(SUnit *Source, const SDep &Dep, bool isSucc = true); + + /// The distance function, which indicates that operation V of iteration I + /// depends on operations U of iteration I-distance. + unsigned getDistance(SUnit *U, SUnit *V, const SDep &Dep) { + // Instructions that feed a Phi have a distance of 1. Computing larger + // values for arrays requires data dependence information. + if (V->getInstr()->isPHI() && Dep.getKind() == SDep::Anti) + return 1; + return 0; + } + + /// Set the Minimum Initiation Interval for this schedule attempt. + void setMII(unsigned mii) { MII = mii; } + + void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule); + + void fixupRegisterOverlaps(std::deque &Instrs); + + /// Return the new base register that was stored away for the changed + /// instruction. + unsigned getInstrBaseReg(SUnit *SU) { + DenseMap>::iterator It = + InstrChanges.find(SU); + if (It != InstrChanges.end()) + return It->second.first; + return 0; + } + + void addMutation(std::unique_ptr Mutation) { + Mutations.push_back(std::move(Mutation)); + } + + static bool classof(const ScheduleDAGInstrs *DAG) { return true; } + +private: + void addLoopCarriedDependences(AliasAnalysis *AA); + void updatePhiDependences(); + void changeDependences(); + unsigned calculateResMII(); + unsigned calculateRecMII(NodeSetType &RecNodeSets); + void findCircuits(NodeSetType &NodeSets); + void fuseRecs(NodeSetType &NodeSets); + void removeDuplicateNodes(NodeSetType &NodeSets); + void computeNodeFunctions(NodeSetType &NodeSets); + void registerPressureFilter(NodeSetType &NodeSets); + void colocateNodeSets(NodeSetType &NodeSets); + void checkNodeSets(NodeSetType &NodeSets); + void groupRemainingNodes(NodeSetType &NodeSets); + void addConnectedNodes(SUnit *SU, NodeSet &NewSet, + SetVector &NodesAdded); + void computeNodeOrder(NodeSetType &NodeSets); + void checkValidNodeOrder(const NodeSetType &Circuits) const; + bool schedulePipeline(SMSchedule &Schedule); + void generatePipelinedLoop(SMSchedule &Schedule); + void generateProlog(SMSchedule &Schedule, unsigned LastStage, + MachineBasicBlock *KernelBB, ValueMapTy *VRMap, + MBBVectorTy &PrologBBs); + void generateEpilog(SMSchedule &Schedule, unsigned LastStage, + MachineBasicBlock *KernelBB, ValueMapTy *VRMap, + MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs); + void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, + MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, + SMSchedule &Schedule, ValueMapTy *VRMap, + InstrMapTy &InstrMap, unsigned LastStageNum, + unsigned CurStageNum, bool IsLast); + void generatePhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1, + MachineBasicBlock *BB2, MachineBasicBlock *KernelBB, + SMSchedule &Schedule, ValueMapTy *VRMap, + InstrMapTy &InstrMap, unsigned LastStageNum, + unsigned CurStageNum, bool IsLast); + void removeDeadInstructions(MachineBasicBlock *KernelBB, + MBBVectorTy &EpilogBBs); + void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, + SMSchedule &Schedule); + void addBranches(MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB, + MBBVectorTy &EpilogBBs, SMSchedule &Schedule, + ValueMapTy *VRMap); + bool computeDelta(MachineInstr &MI, unsigned &Delta); + void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI, + unsigned Num); + MachineInstr *cloneInstr(MachineInstr *OldMI, unsigned CurStageNum, + unsigned InstStageNum); + MachineInstr *cloneAndChangeInstr(MachineInstr *OldMI, unsigned CurStageNum, + unsigned InstStageNum, + SMSchedule &Schedule); + void updateInstruction(MachineInstr *NewMI, bool LastDef, + unsigned CurStageNum, unsigned InstrStageNum, + SMSchedule &Schedule, ValueMapTy *VRMap); + MachineInstr *findDefInLoop(unsigned Reg); + unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal, + unsigned LoopStage, ValueMapTy *VRMap, + MachineBasicBlock *BB); + void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum, + SMSchedule &Schedule, ValueMapTy *VRMap, + InstrMapTy &InstrMap); + void rewriteScheduledInstr(MachineBasicBlock *BB, SMSchedule &Schedule, + InstrMapTy &InstrMap, unsigned CurStageNum, + unsigned PhiNum, MachineInstr *Phi, + unsigned OldReg, unsigned NewReg, + unsigned PrevReg = 0); + bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos, + unsigned &OffsetPos, unsigned &NewBase, + int64_t &NewOffset); + void postprocessDAG(); +}; + +/// A NodeSet contains a set of SUnit DAG nodes with additional information +/// that assigns a priority to the set. +class NodeSet { + SetVector Nodes; + bool HasRecurrence = false; + unsigned RecMII = 0; + int MaxMOV = 0; + unsigned MaxDepth = 0; + unsigned Colocate = 0; + SUnit *ExceedPressure = nullptr; + unsigned Latency = 0; + +public: + using iterator = SetVector::const_iterator; + + NodeSet() = default; + NodeSet(iterator S, iterator E) : Nodes(S, E), HasRecurrence(true) { + Latency = 0; + for (unsigned i = 0, e = Nodes.size(); i < e; ++i) + for (const SDep &Succ : Nodes[i]->Succs) + if (Nodes.count(Succ.getSUnit())) + Latency += Succ.getLatency(); + } + + bool insert(SUnit *SU) { return Nodes.insert(SU); } + + void insert(iterator S, iterator E) { Nodes.insert(S, E); } + + template bool remove_if(UnaryPredicate P) { + return Nodes.remove_if(P); + } + + unsigned count(SUnit *SU) const { return Nodes.count(SU); } + + bool hasRecurrence() { return HasRecurrence; }; + + unsigned size() const { return Nodes.size(); } + + bool empty() const { return Nodes.empty(); } + + SUnit *getNode(unsigned i) const { return Nodes[i]; }; + + void setRecMII(unsigned mii) { RecMII = mii; }; + + void setColocate(unsigned c) { Colocate = c; }; + + void setExceedPressure(SUnit *SU) { ExceedPressure = SU; } + + bool isExceedSU(SUnit *SU) { return ExceedPressure == SU; } + + int compareRecMII(NodeSet &RHS) { return RecMII - RHS.RecMII; } + + int getRecMII() { return RecMII; } + + /// Summarize node functions for the entire node set. + void computeNodeSetInfo(SwingSchedulerDAG *SSD) { + for (SUnit *SU : *this) { + MaxMOV = std::max(MaxMOV, SSD->getMOV(SU)); + MaxDepth = std::max(MaxDepth, SSD->getDepth(SU)); + } + } + + unsigned getLatency() { return Latency; } + + unsigned getMaxDepth() { return MaxDepth; } + + void clear() { + Nodes.clear(); + RecMII = 0; + HasRecurrence = false; + MaxMOV = 0; + MaxDepth = 0; + Colocate = 0; + ExceedPressure = nullptr; + } + + operator SetVector &() { return Nodes; } + + /// Sort the node sets by importance. First, rank them by recurrence MII, + /// then by mobility (least mobile done first), and finally by depth. + /// Each node set may contain a colocate value which is used as the first + /// tie breaker, if it's set. + bool operator>(const NodeSet &RHS) const { + if (RecMII == RHS.RecMII) { + if (Colocate != 0 && RHS.Colocate != 0 && Colocate != RHS.Colocate) + return Colocate < RHS.Colocate; + if (MaxMOV == RHS.MaxMOV) + return MaxDepth > RHS.MaxDepth; + return MaxMOV < RHS.MaxMOV; + } + return RecMII > RHS.RecMII; + } + + bool operator==(const NodeSet &RHS) const { + return RecMII == RHS.RecMII && MaxMOV == RHS.MaxMOV && + MaxDepth == RHS.MaxDepth; + } + + bool operator!=(const NodeSet &RHS) const { return !operator==(RHS); } + + iterator begin() { return Nodes.begin(); } + iterator end() { return Nodes.end(); } + void print(raw_ostream &os) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif +}; + +/// This class represents the scheduled code. The main data structure is a +/// map from scheduled cycle to instructions. During scheduling, the +/// data structure explicitly represents all stages/iterations. When +/// the algorithm finshes, the schedule is collapsed into a single stage, +/// which represents instructions from different loop iterations. +/// +/// The SMS algorithm allows negative values for cycles, so the first cycle +/// in the schedule is the smallest cycle value. +class SMSchedule { +private: + /// Map from execution cycle to instructions. + DenseMap> ScheduledInstrs; + + /// Map from instruction to execution cycle. + std::map InstrToCycle; + + /// Map for each register and the max difference between its uses and def. + /// The first element in the pair is the max difference in stages. The + /// second is true if the register defines a Phi value and loop value is + /// scheduled before the Phi. + std::map> RegToStageDiff; + + /// Keep track of the first cycle value in the schedule. It starts + /// as zero, but the algorithm allows negative values. + int FirstCycle = 0; + + /// Keep track of the last cycle value in the schedule. + int LastCycle = 0; + + /// The initiation interval (II) for the schedule. + int InitiationInterval = 0; + + /// Target machine information. + const TargetSubtargetInfo &ST; + + /// Virtual register information. + MachineRegisterInfo &MRI; + + std::unique_ptr Resources; + +public: + SMSchedule(MachineFunction *mf) + : ST(mf->getSubtarget()), MRI(mf->getRegInfo()), + Resources(ST.getInstrInfo()->CreateTargetScheduleState(ST)) {} + + void reset() { + ScheduledInstrs.clear(); + InstrToCycle.clear(); + RegToStageDiff.clear(); + FirstCycle = 0; + LastCycle = 0; + InitiationInterval = 0; + } + + /// Set the initiation interval for this schedule. + void setInitiationInterval(int ii) { InitiationInterval = ii; } + + /// Return the first cycle in the completed schedule. This + /// can be a negative value. + int getFirstCycle() const { return FirstCycle; } + + /// Return the last cycle in the finalized schedule. + int getFinalCycle() const { return FirstCycle + InitiationInterval - 1; } + + /// Return the cycle of the earliest scheduled instruction in the dependence + /// chain. + int earliestCycleInChain(const SDep &Dep); + + /// Return the cycle of the latest scheduled instruction in the dependence + /// chain. + int latestCycleInChain(const SDep &Dep); + + void computeStart(SUnit *SU, int *MaxEarlyStart, int *MinLateStart, + int *MinEnd, int *MaxStart, int II, SwingSchedulerDAG *DAG); + bool insert(SUnit *SU, int StartCycle, int EndCycle, int II); + + /// Iterators for the cycle to instruction map. + using sched_iterator = DenseMap>::iterator; + using const_sched_iterator = + DenseMap>::const_iterator; + + /// Return true if the instruction is scheduled at the specified stage. + bool isScheduledAtStage(SUnit *SU, unsigned StageNum) { + return (stageScheduled(SU) == (int)StageNum); + } + + /// Return the stage for a scheduled instruction. Return -1 if + /// the instruction has not been scheduled. + int stageScheduled(SUnit *SU) const { + std::map::const_iterator it = InstrToCycle.find(SU); + if (it == InstrToCycle.end()) + return -1; + return (it->second - FirstCycle) / InitiationInterval; + } + + /// Return the cycle for a scheduled instruction. This function normalizes + /// the first cycle to be 0. + unsigned cycleScheduled(SUnit *SU) const { + std::map::const_iterator it = InstrToCycle.find(SU); + assert(it != InstrToCycle.end() && "Instruction hasn't been scheduled."); + return (it->second - FirstCycle) % InitiationInterval; + } + + /// Return the maximum stage count needed for this schedule. + unsigned getMaxStageCount() { + return (LastCycle - FirstCycle) / InitiationInterval; + } + + /// Return the max. number of stages/iterations that can occur between a + /// register definition and its uses. + unsigned getStagesForReg(int Reg, unsigned CurStage) { + std::pair Stages = RegToStageDiff[Reg]; + if (CurStage > getMaxStageCount() && Stages.first == 0 && Stages.second) + return 1; + return Stages.first; + } + + /// The number of stages for a Phi is a little different than other + /// instructions. The minimum value computed in RegToStageDiff is 1 + /// because we assume the Phi is needed for at least 1 iteration. + /// This is not the case if the loop value is scheduled prior to the + /// Phi in the same stage. This function returns the number of stages + /// or iterations needed between the Phi definition and any uses. + unsigned getStagesForPhi(int Reg) { + std::pair Stages = RegToStageDiff[Reg]; + if (Stages.second) + return Stages.first; + return Stages.first - 1; + } + + /// Return the instructions that are scheduled at the specified cycle. + std::deque &getInstructions(int cycle) { + return ScheduledInstrs[cycle]; + } + + bool isValidSchedule(SwingSchedulerDAG *SSD); + void finalizeSchedule(SwingSchedulerDAG *SSD); + void orderDependence(SwingSchedulerDAG *SSD, SUnit *SU, + std::deque &Insts); + bool isLoopCarried(SwingSchedulerDAG *SSD, MachineInstr &Phi); + bool isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD, MachineInstr *Def, + MachineOperand &MO); + void print(raw_ostream &os) const; + void dump() const; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_CODEGEN_MACHINEPIPELINER_H diff --git a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h index 5bf4a49c8b3..fef010a23ef 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h @@ -689,15 +689,14 @@ public: unsigned MinNumRegs = 0); /// Constrain the register class or the register bank of the virtual register - /// \p Reg to be a common subclass and a common bank of both registers - /// provided respectively. Do nothing if any of the attributes (classes, - /// banks, or low-level types) of the registers are deemed incompatible, or if - /// the resulting register will have a class smaller than before and of size - /// less than \p MinNumRegs. Return true if such register attributes exist, - /// false otherwise. + /// \p Reg (and low-level type) to be a common subclass or a common bank of + /// both registers provided respectively (and a common low-level type). Do + /// nothing if any of the attributes (classes, banks, or low-level types) of + /// the registers are deemed incompatible, or if the resulting register will + /// have a class smaller than before and of size less than \p MinNumRegs. + /// Return true if such register attributes exist, false otherwise. /// - /// \note Assumes that each register has either a low-level type or a class - /// assigned, but not both. Use this method instead of constrainRegClass and + /// \note Use this method instead of constrainRegClass and /// RegisterBankInfo::constrainGenericRegister everywhere but SelectionDAG /// ISel / FastISel and GlobalISel's InstructionSelect pass respectively. bool constrainRegAttrs(unsigned Reg, unsigned ConstrainingReg, @@ -717,6 +716,10 @@ public: unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name = ""); + /// Create and return a new virtual register in the function with the same + /// attributes as the given register. + unsigned cloneVirtualRegister(unsigned VReg, StringRef Name = ""); + /// Get the low-level type of \p Reg or LLT{} if Reg is not a generic /// (target independent) virtual register. LLT getType(unsigned Reg) const { diff --git a/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h b/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h index 85ffa4eda2b..4bc31ae7c61 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -132,17 +132,19 @@ struct MachineSchedContext { /// MachineSchedRegistry provides a selection of available machine instruction /// schedulers. -class MachineSchedRegistry : public MachinePassRegistryNode { +class MachineSchedRegistry + : public MachinePassRegistryNode< + ScheduleDAGInstrs *(*)(MachineSchedContext *)> { public: using ScheduleDAGCtor = ScheduleDAGInstrs *(*)(MachineSchedContext *); // RegisterPassParser requires a (misnamed) FunctionPassCtor type. using FunctionPassCtor = ScheduleDAGCtor; - static MachinePassRegistry Registry; + static MachinePassRegistry Registry; MachineSchedRegistry(const char *N, const char *D, ScheduleDAGCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) { + : MachinePassRegistryNode(N, D, C) { Registry.Add(this); } @@ -158,7 +160,7 @@ public: return (MachineSchedRegistry *)Registry.getList(); } - static void setListener(MachinePassRegistryListener *L) { + static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } }; @@ -466,6 +468,9 @@ public: PressureDiff &getPressureDiff(const SUnit *SU) { return SUPressureDiffs[SU->NodeNum]; } + const PressureDiff &getPressureDiff(const SUnit *SU) const { + return SUPressureDiffs[SU->NodeNum]; + } /// Compute a DFSResult after DAG building is complete, and before any /// queue comparisons. @@ -491,6 +496,8 @@ public: /// Compute the cyclic critical path through the DAG. unsigned computeCyclicCriticalPath(); + void dump() const override; + protected: // Top-Level entry points for the schedule() driver... @@ -787,7 +794,7 @@ public: /// Represent the type of SchedCandidate found within a single queue. /// pickNodeBidirectional depends on these listed by decreasing priority. enum CandReason : uint8_t { - NoCand, Only1, PhysRegCopy, RegExcess, RegCritical, Stall, Cluster, Weak, + NoCand, Only1, PhysReg, RegExcess, RegCritical, Stall, Cluster, Weak, RegMax, ResourceReduce, ResourceDemand, BotHeightReduce, BotPathReduce, TopDepthReduce, TopPathReduce, NextDefUse, NodeOrder}; @@ -895,6 +902,10 @@ protected: #ifndef NDEBUG void traceCandidate(const SchedCandidate &Cand); #endif + +private: + bool shouldReduceLatency(const CandPolicy &Policy, SchedBoundary &CurrZone, + bool ComputeRemLatency, unsigned &RemLatency) const; }; // Utility functions used by heuristics in tryCandidate(). @@ -917,7 +928,7 @@ bool tryPressure(const PressureChange &TryP, const TargetRegisterInfo *TRI, const MachineFunction &MF); unsigned getWeakLeft(const SUnit *SU, bool isTop); -int biasPhysRegCopy(const SUnit *SU, bool isTop); +int biasPhysReg(const SUnit *SU, bool isTop); /// GenericScheduler shrinks the unscheduled zone using heuristics to balance /// the schedule. @@ -995,7 +1006,7 @@ protected: const RegPressureTracker &RPTracker, SchedCandidate &Candidate); - void reschedulePhysRegCopies(SUnit *SU, bool isTop); + void reschedulePhysReg(SUnit *SU, bool isTop); }; /// PostGenericScheduler - Interface to the scheduling algorithm used by diff --git a/contrib/llvm/include/llvm/CodeGen/Passes.h b/contrib/llvm/include/llvm/CodeGen/Passes.h index cb12b14f443..acf1ebb5bc8 100644 --- a/contrib/llvm/include/llvm/CodeGen/Passes.h +++ b/contrib/llvm/include/llvm/CodeGen/Passes.h @@ -379,14 +379,20 @@ namespace llvm { /// FunctionPass *createInterleavedAccessPass(); + /// InterleavedLoadCombines Pass - This pass identifies interleaved loads and + /// combines them into wide loads detectable by InterleavedAccessPass + /// + FunctionPass *createInterleavedLoadCombinePass(); + /// LowerEmuTLS - This pass generates __emutls_[vt].xyz variables for all /// TLS variables for the emulated TLS model. /// ModulePass *createLowerEmuTLSPass(); - /// This pass lowers the \@llvm.load.relative intrinsic to instructions. - /// This is unsafe to do earlier because a pass may combine the constant - /// initializer into the load, which may result in an overflowing evaluation. + /// This pass lowers the \@llvm.load.relative and \@llvm.objc.* intrinsics to + /// instructions. This is unsafe to do earlier because a pass may combine the + /// constant initializer into the load, which may result in an overflowing + /// evaluation. ModulePass *createPreISelIntrinsicLoweringPass(); /// GlobalMerge - This pass merges internal (by default) globals into structs diff --git a/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h b/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h index 7a007eb8bce..b7f83e515b7 100644 --- a/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h @@ -7,7 +7,8 @@ // //===----------------------------------------------------------------------===// // -// This pass implements IR lowering for the llvm.load.relative intrinsic. +// This pass implements IR lowering for the llvm.load.relative and llvm.objc.* +// intrinsics. // //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_PREISELINTRINSICLOWERING_H diff --git a/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h b/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h index bdf0bb73154..f66191bc9fb 100644 --- a/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h +++ b/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h @@ -36,7 +36,7 @@ raw_ostream &operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); /// below the stack frame (e.g., argument space), or constant pool. class PseudoSourceValue { public: - enum PSVKind { + enum PSVKind : unsigned { Stack, GOT, JumpTable, @@ -48,7 +48,7 @@ public: }; private: - PSVKind Kind; + unsigned Kind; unsigned AddressSpace; friend raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); @@ -60,11 +60,11 @@ private: virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(PSVKind Kind, const TargetInstrInfo &TII); + explicit PseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII); virtual ~PseudoSourceValue(); - PSVKind kind() const { return Kind; } + unsigned kind() const { return Kind; } bool isStack() const { return Kind == Stack; } bool isGOT() const { return Kind == GOT; } @@ -116,7 +116,7 @@ public: class CallEntryPseudoSourceValue : public PseudoSourceValue { protected: - CallEntryPseudoSourceValue(PSVKind Kind, const TargetInstrInfo &TII); + CallEntryPseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII); public: bool isConstant(const MachineFrameInfo *) const override; diff --git a/contrib/llvm/include/llvm/CodeGen/RegAllocRegistry.h b/contrib/llvm/include/llvm/CodeGen/RegAllocRegistry.h index 481747dc163..b518fbb9c9d 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegAllocRegistry.h +++ b/contrib/llvm/include/llvm/CodeGen/RegAllocRegistry.h @@ -26,14 +26,14 @@ class FunctionPass; /// RegisterRegAlloc class - Track the registration of register allocators. /// //===----------------------------------------------------------------------===// -class RegisterRegAlloc : public MachinePassRegistryNode { +class RegisterRegAlloc : public MachinePassRegistryNode { public: using FunctionPassCtor = FunctionPass *(*)(); - static MachinePassRegistry Registry; + static MachinePassRegistry Registry; RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) { + : MachinePassRegistryNode(N, D, C) { Registry.Add(this); } @@ -48,15 +48,11 @@ public: return (RegisterRegAlloc *)Registry.getList(); } - static FunctionPassCtor getDefault() { - return (FunctionPassCtor)Registry.getDefault(); - } + static FunctionPassCtor getDefault() { return Registry.getDefault(); } - static void setDefault(FunctionPassCtor C) { - Registry.setDefault((MachinePassCtor)C); - } + static void setDefault(FunctionPassCtor C) { Registry.setDefault(C); } - static void setListener(MachinePassRegistryListener *L) { + static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } }; diff --git a/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h b/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h index efd175eeed3..efecc61d9c3 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h @@ -29,7 +29,7 @@ namespace llvm { class Function; -class TargetMachine; +class LLVMTargetMachine; class PhysicalRegisterUsageInfo : public ImmutablePass { public: @@ -41,7 +41,7 @@ public: } /// Set TargetMachine which is used to print analysis. - void setTargetMachine(const TargetMachine &TM); + void setTargetMachine(const LLVMTargetMachine &TM); bool doInitialization(Module &M) override; @@ -63,7 +63,7 @@ private: /// and 1 means content of register will be preserved around function call. DenseMap> RegMasks; - const TargetMachine *TM; + const LLVMTargetMachine *TM; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h index 56adc2e2fbf..0870d67db39 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h @@ -33,15 +33,15 @@ namespace llvm { template class GraphWriter; +class LLVMTargetMachine; class MachineFunction; class MachineRegisterInfo; class MCInstrDesc; struct MCSchedClassDesc; -class ScheduleDAG; class SDNode; class SUnit; +class ScheduleDAG; class TargetInstrInfo; -class TargetMachine; class TargetRegisterClass; class TargetRegisterInfo; @@ -236,8 +236,7 @@ class TargetRegisterInfo; Contents.Reg = Reg; } - raw_ostream &print(raw_ostream &O, - const TargetRegisterInfo *TRI = nullptr) const; + void dump(const TargetRegisterInfo *TRI = nullptr) const; }; template <> @@ -459,12 +458,7 @@ class TargetRegisterInfo; /// edge occurs first. void biasCriticalPath(); - void dump(const ScheduleDAG *G) const; - void dumpAll(const ScheduleDAG *G) const; - raw_ostream &print(raw_ostream &O, - const SUnit *Entry = nullptr, - const SUnit *Exit = nullptr) const; - raw_ostream &print(raw_ostream &O, const ScheduleDAG *G) const; + void dumpAttributes() const; private: void ComputeDepth(); @@ -564,7 +558,7 @@ class TargetRegisterInfo; class ScheduleDAG { public: - const TargetMachine &TM; ///< Target processor + const LLVMTargetMachine &TM; ///< Target processor const TargetInstrInfo *TII; ///< Target instruction information const TargetRegisterInfo *TRI; ///< Target processor register info MachineFunction &MF; ///< Machine function @@ -597,7 +591,9 @@ class TargetRegisterInfo; virtual void viewGraph(const Twine &Name, const Twine &Title); virtual void viewGraph(); - virtual void dumpNode(const SUnit *SU) const = 0; + virtual void dumpNode(const SUnit &SU) const = 0; + virtual void dump() const = 0; + void dumpNodeName(const SUnit &SU) const; /// Returns a label for an SUnit node in a visualization of the ScheduleDAG. virtual std::string getGraphNodeLabel(const SUnit *SU) const = 0; @@ -614,6 +610,9 @@ class TargetRegisterInfo; unsigned VerifyScheduledDAG(bool isBottomUp); #endif + protected: + void dumpNodeAll(const SUnit &SU) const; + private: /// Returns the MCInstrDesc of this SDNode or NULL. const MCInstrDesc *getNodeDesc(const SDNode *Node) const; diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index 520a23846f6..daad18125db 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -327,7 +327,8 @@ namespace llvm { /// whole MachineFunction. By default does nothing. virtual void finalizeSchedule() {} - void dumpNode(const SUnit *SU) const override; + void dumpNode(const SUnit &SU) const override; + void dump() const override; /// Returns a label for a DAG node that points to an instruction. std::string getGraphNodeLabel(const SUnit *SU) const override; diff --git a/contrib/llvm/include/llvm/CodeGen/SchedulerRegistry.h b/contrib/llvm/include/llvm/CodeGen/SchedulerRegistry.h index badf927d0e9..fbe559f2555 100644 --- a/contrib/llvm/include/llvm/CodeGen/SchedulerRegistry.h +++ b/contrib/llvm/include/llvm/CodeGen/SchedulerRegistry.h @@ -29,16 +29,19 @@ namespace llvm { class ScheduleDAGSDNodes; class SelectionDAGISel; -class RegisterScheduler : public MachinePassRegistryNode { +class RegisterScheduler + : public MachinePassRegistryNode< + ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level)> { public: using FunctionPassCtor = ScheduleDAGSDNodes *(*)(SelectionDAGISel*, CodeGenOpt::Level); - static MachinePassRegistry Registry; + static MachinePassRegistry Registry; RegisterScheduler(const char *N, const char *D, FunctionPassCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) - { Registry.Add(this); } + : MachinePassRegistryNode(N, D, C) { + Registry.Add(this); + } ~RegisterScheduler() { Registry.Remove(this); } @@ -51,7 +54,7 @@ public: return (RegisterScheduler *)Registry.getList(); } - static void setListener(MachinePassRegistryListener *L) { + static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } }; diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h index 888f9425ff9..67fe87fc96a 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -28,7 +28,7 @@ #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/ISDOpcodes.h" @@ -188,8 +188,8 @@ public: return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty(); } - ArrayRef getSDDbgValues(const SDNode *Node) { - DbgValMapType::iterator I = DbgValMap.find(Node); + ArrayRef getSDDbgValues(const SDNode *Node) const { + auto I = DbgValMap.find(Node); if (I != DbgValMap.end()) return I->second; return ArrayRef(); @@ -229,7 +229,7 @@ class SelectionDAG { LLVMContext *Context; CodeGenOpt::Level OptLevel; - DivergenceAnalysis * DA = nullptr; + LegacyDivergenceAnalysis * DA = nullptr; FunctionLoweringInfo * FLI = nullptr; /// The function-level optimization remark emitter. Used to emit remarks @@ -308,6 +308,9 @@ public: : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } + + private: + virtual void anchor(); }; /// When true, additional steps are taken to @@ -382,7 +385,7 @@ public: /// Prepare this SelectionDAG to process code in the given MachineFunction. void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, - DivergenceAnalysis * Divergence); + LegacyDivergenceAnalysis * Divergence); void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { FLI = FuncInfo; @@ -471,7 +474,9 @@ public: return Root; } +#ifndef NDEBUG void VerifyDAGDiverence(); +#endif /// This iterates over the nodes in the SelectionDAG, folding /// certain types of nodes together, or eliminating superfluous nodes. The @@ -784,24 +789,6 @@ public: /// value assuming it was the smaller SrcTy value. SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); - /// Return an operation which will any-extend the low lanes of the operand - /// into the specified vector type. For example, - /// this can convert a v16i8 into a v4i32 by any-extending the low four - /// lanes of the operand from i8 to i32. - SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT); - - /// Return an operation which will sign extend the low lanes of the operand - /// into the specified vector type. For example, - /// this can convert a v16i8 into a v4i32 by sign extending the low four - /// lanes of the operand from i8 to i32. - SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT); - - /// Return an operation which will zero extend the low lanes of the operand - /// into the specified vector type. For example, - /// this can convert a v16i8 into a v4i32 by zero extending the low four - /// lanes of the operand from i8 to i32. - SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT); - /// Convert Op, which must be of integer type, to the integer type VT, /// by using an extension appropriate for the target's /// BooleanContent for type OpVT or truncating it. @@ -945,41 +932,45 @@ public: Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo); - /// Helper function to make it easier to build SetCC's if you just - /// have an ISD::CondCode instead of an SDValue. - /// + /// Helper function to make it easier to build SetCC's if you just have an + /// ISD::CondCode instead of an SDValue. SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond) { assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() && - "Cannot compare scalars to vectors"); + "Cannot compare scalars to vectors"); assert(LHS.getValueType().isVector() == VT.isVector() && - "Cannot compare scalars to vectors"); + "Cannot compare scalars to vectors"); assert(Cond != ISD::SETCC_INVALID && - "Cannot create a setCC of an invalid node."); + "Cannot create a setCC of an invalid node."); return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); } - /// Helper function to make it easier to build Select's if you just - /// have operands and don't want to check for vector. + /// Helper function to make it easier to build Select's if you just have + /// operands and don't want to check for vector. SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS) { assert(LHS.getValueType() == RHS.getValueType() && "Cannot use select on differing types"); assert(VT.isVector() == LHS.getValueType().isVector() && "Cannot mix vectors and scalars"); - return getNode(Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, - Cond, LHS, RHS); + auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; + return getNode(Opcode, DL, VT, Cond, LHS, RHS); } - /// Helper function to make it easier to build SelectCC's if you - /// just have an ISD::CondCode instead of an SDValue. - /// + /// Helper function to make it easier to build SelectCC's if you just have an + /// ISD::CondCode instead of an SDValue. SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond) { - return getNode(ISD::SELECT_CC, DL, True.getValueType(), - LHS, RHS, True, False, getCondCode(Cond)); + return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True, + False, getCondCode(Cond)); } + /// Try to simplify a select/vselect into 1 of its operands or a constant. + SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal); + + /// Try to simplify a shift into 1 of its operands or a constant. + SDValue simplifyShift(SDValue X, SDValue Y); + /// VAArg produces a result and token chain, and takes a pointer /// and a source value as input. SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, @@ -1140,6 +1131,13 @@ public: /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. SDValue expandVACopy(SDNode *Node); + /// Returs an GlobalAddress of the function from the current module with + /// name matching the given ExternalSymbol. Additionally can provide the + /// matched function. + /// Panics the function doesn't exists. + SDValue getSymbolFunctionGlobalAddress(SDValue Op, + Function **TargetFunction = nullptr); + /// *Mutate* the specified node in-place to have the /// specified operands. If the resultant node already exists in the DAG, /// this does not modify the specified node, instead it returns the node that @@ -1156,6 +1154,11 @@ public: SDValue Op3, SDValue Op4, SDValue Op5); SDNode *UpdateNodeOperands(SDNode *N, ArrayRef Ops); + /// *Mutate* the specified machine node's memory references to the provided + /// list. + void setNodeMemRefs(MachineSDNode *N, + ArrayRef NewMemRefs); + // Propagates the change in divergence to users void updateDivergence(SDNode * N); @@ -1346,7 +1349,7 @@ public: void AddDbgLabel(SDDbgLabel *DB); /// Get the debug values which reference the given SDNode. - ArrayRef GetDbgValues(const SDNode* SD) { + ArrayRef GetDbgValues(const SDNode* SD) const { return DbgInfo->getSDDbgValues(SD); } @@ -1429,15 +1432,15 @@ public: /// every vector element. /// Targets can implement the computeKnownBitsForTargetNode method in the /// TargetLowering class to allow target nodes to be understood. - void computeKnownBits(SDValue Op, KnownBits &Known, unsigned Depth = 0) const; + KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const; /// Determine which bits of Op are known to be either zero or one and return /// them in Known. The DemandedElts argument allows us to only collect the /// known bits that are shared by the requested vector elements. /// Targets can implement the computeKnownBitsForTargetNode method in the /// TargetLowering class to allow target nodes to be understood. - void computeKnownBits(SDValue Op, KnownBits &Known, const APInt &DemandedElts, - unsigned Depth = 0) const; + KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts, + unsigned Depth = 0) const; /// Used to represent the possible overflow behavior of an operation. /// Never: the operation cannot overflow. @@ -1484,8 +1487,15 @@ public: /// X|Cst == X+Cst iff X&Cst = 0. bool isBaseWithConstantOffset(SDValue Op) const; - /// Test whether the given SDValue is known to never be NaN. - bool isKnownNeverNaN(SDValue Op) const; + /// Test whether the given SDValue is known to never be NaN. If \p SNaN is + /// true, returns if \p Op is known to never be a signaling NaN (it may still + /// be a qNaN). + bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const; + + /// \returns true if \p Op is known to never be a signaling NaN. + bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const { + return isKnownNeverNaN(Op, true, Depth); + } /// Test whether the given floating point SDValue is known to never be /// positive or negative zero. @@ -1503,6 +1513,27 @@ public: /// allow an 'add' to be transformed into an 'or'. bool haveNoCommonBitsSet(SDValue A, SDValue B) const; + /// Test whether \p V has a splatted value for all the demanded elements. + /// + /// On success \p UndefElts will indicate the elements that have UNDEF + /// values instead of the splat value, this is only guaranteed to be correct + /// for \p DemandedElts. + /// + /// NOTE: The function will return true for a demanded splat of UNDEF values. + bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts); + + /// Test whether \p V has a splatted value. + bool isSplatValue(SDValue V, bool AllowUndefs = false); + + /// Match a binop + shuffle pyramid that represents a horizontal reduction + /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p + /// Extract. The reduction must use one of the opcodes listed in /p + /// CandidateBinOps and on success /p BinOp will contain the matching opcode. + /// Returns the vector that is being reduced on, or SDValue() if a reduction + /// was not matched. + SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, + ArrayRef CandidateBinOps); + /// Utility function used by legalize and lowering to /// "unroll" a vector operation by splitting out the scalars and operating /// on each element individually. If the ResNE is 0, fully unroll the vector diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h index 580606441a9..2b2c48d57bc 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h @@ -45,18 +45,21 @@ public: IsIndexSignExt(IsIndexSignExt) {} SDValue getBase() { return Base; } + SDValue getBase() const { return Base; } SDValue getIndex() { return Index; } + SDValue getIndex() const { return Index; } - bool equalBaseIndex(BaseIndexOffset &Other, const SelectionDAG &DAG) { + bool equalBaseIndex(const BaseIndexOffset &Other, + const SelectionDAG &DAG) const { int64_t Off; return equalBaseIndex(Other, DAG, Off); } - bool equalBaseIndex(BaseIndexOffset &Other, const SelectionDAG &DAG, - int64_t &Off); + bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG, + int64_t &Off) const; /// Parses tree in Ptr for base, index, offset addresses. - static BaseIndexOffset match(LSBaseSDNode *N, const SelectionDAG &DAG); + static BaseIndexOffset match(const LSBaseSDNode *N, const SelectionDAG &DAG); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 86df0af7303..6758c55c696 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -132,6 +132,7 @@ public: OPC_CheckChild2Same, OPC_CheckChild3Same, OPC_CheckPatternPredicate, OPC_CheckPredicate, + OPC_CheckPredicateWithOperands, OPC_CheckOpcode, OPC_SwitchOpcode, OPC_CheckType, @@ -267,6 +268,17 @@ public: llvm_unreachable("Tblgen should generate the implementation of this!"); } + /// CheckNodePredicateWithOperands - This function is generated by tblgen in + /// the target. + /// It runs node predicate number PredNo and returns true if it succeeds or + /// false if it fails. The number is a private implementation detail to the + /// code tblgen produces. + virtual bool CheckNodePredicateWithOperands( + SDNode *N, unsigned PredNo, + const SmallVectorImpl &Operands) const { + llvm_unreachable("Tblgen should generate the implementation of this!"); + } + virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl > &Result) { diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 1af22185d36..10f28417908 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -672,6 +672,12 @@ public: case ISD::STRICT_FLOG2: case ISD::STRICT_FRINT: case ISD::STRICT_FNEARBYINT: + case ISD::STRICT_FMAXNUM: + case ISD::STRICT_FMINNUM: + case ISD::STRICT_FCEIL: + case ISD::STRICT_FFLOOR: + case ISD::STRICT_FROUND: + case ISD::STRICT_FTRUNC: return true; } } @@ -1589,15 +1595,38 @@ bool isAllOnesConstant(SDValue V); /// Returns true if \p V is a constant integer one. bool isOneConstant(SDValue V); +/// Return the non-bitcasted source operand of \p V if it exists. +/// If \p V is not a bitcasted value, it is returned as-is. +SDValue peekThroughBitcasts(SDValue V); + +/// Return the non-bitcasted and one-use source operand of \p V if it exists. +/// If \p V is not a bitcasted one-use value, it is returned as-is. +SDValue peekThroughOneUseBitcasts(SDValue V); + /// Returns true if \p V is a bitwise not operation. Assumes that an all ones /// constant is canonicalized to be operand 1. bool isBitwiseNot(SDValue V); /// Returns the SDNode if it is a constant splat BuildVector or constant int. -ConstantSDNode *isConstOrConstSplat(SDValue N); +ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false); /// Returns the SDNode if it is a constant splat BuildVector or constant float. -ConstantFPSDNode *isConstOrConstSplatFP(SDValue N); +ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); + +/// Return true if the value is a constant 0 integer or a splatted vector of +/// a constant 0 integer (with no undefs). +/// Build vector implicit truncation is not an issue for null values. +bool isNullOrNullSplat(SDValue V); + +/// Return true if the value is a constant 1 integer or a splatted vector of a +/// constant 1 integer (with no undefs). +/// Does not permit build vector implicit truncation. +bool isOneOrOneSplat(SDValue V); + +/// Return true if the value is a constant -1 integer or a splatted vector of a +/// constant -1 integer (with no undefs). +/// Does not permit build vector implicit truncation. +bool isAllOnesOrAllOnesSplat(SDValue V); class GlobalAddressSDNode : public SDNode { friend class SelectionDAG; @@ -2113,12 +2142,15 @@ public: MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} - // In the both nodes address is Op1, mask is Op2: - // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value - // MaskedStoreSDNode (Chain, ptr, mask, data) + // MaskedLoadSDNode (Chain, ptr, mask, passthru) + // MaskedStoreSDNode (Chain, data, ptr, mask) // Mask is a vector of i1 elements - const SDValue &getBasePtr() const { return getOperand(1); } - const SDValue &getMask() const { return getOperand(2); } + const SDValue &getBasePtr() const { + return getOperand(getOpcode() == ISD::MLOAD ? 1 : 2); + } + const SDValue &getMask() const { + return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); + } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD || @@ -2143,7 +2175,10 @@ public: return static_cast(LoadSDNodeBits.ExtTy); } - const SDValue &getSrc0() const { return getOperand(3); } + const SDValue &getBasePtr() const { return getOperand(1); } + const SDValue &getMask() const { return getOperand(2); } + const SDValue &getPassThru() const { return getOperand(3); } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; } @@ -2175,7 +2210,9 @@ public: /// memory at base_addr. bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } - const SDValue &getValue() const { return getOperand(3); } + const SDValue &getValue() const { return getOperand(1); } + const SDValue &getBasePtr() const { return getOperand(2); } + const SDValue &getMask() const { return getOperand(3); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSTORE; @@ -2201,7 +2238,6 @@ public: const SDValue &getBasePtr() const { return getOperand(3); } const SDValue &getIndex() const { return getOperand(4); } const SDValue &getMask() const { return getOperand(2); } - const SDValue &getValue() const { return getOperand(1); } const SDValue &getScale() const { return getOperand(5); } static bool classof(const SDNode *N) { @@ -2220,6 +2256,8 @@ public: EVT MemVT, MachineMemOperand *MMO) : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {} + const SDValue &getPassThru() const { return getOperand(1); } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MGATHER; } @@ -2235,6 +2273,8 @@ public: EVT MemVT, MachineMemOperand *MMO) : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {} + const SDValue &getValue() const { return getOperand(1); } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSCATTER; } @@ -2243,32 +2283,60 @@ public: /// An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. +/// +/// Note that the only supported way to set the `memoperands` is by calling the +/// `SelectionDAG::setNodeMemRefs` function as the memory management happens +/// inside the DAG rather than in the node. class MachineSDNode : public SDNode { -public: - using mmo_iterator = MachineMemOperand **; - private: friend class SelectionDAG; MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) : SDNode(Opc, Order, DL, VTs) {} - /// Memory reference descriptions for this instruction. - mmo_iterator MemRefs = nullptr; - mmo_iterator MemRefsEnd = nullptr; + // We use a pointer union between a single `MachineMemOperand` pointer and + // a pointer to an array of `MachineMemOperand` pointers. This is null when + // the number of these is zero, the single pointer variant used when the + // number is one, and the array is used for larger numbers. + // + // The array is allocated via the `SelectionDAG`'s allocator and so will + // always live until the DAG is cleaned up and doesn't require ownership here. + // + // We can't use something simpler like `TinyPtrVector` here because `SDNode` + // subclasses aren't managed in a conforming C++ manner. See the comments on + // `SelectionDAG::MorphNodeTo` which details what all goes on, but the + // constraint here is that these don't manage memory with their constructor or + // destructor and can be initialized to a good state even if they start off + // uninitialized. + PointerUnion MemRefs = {}; + + // Note that this could be folded into the above `MemRefs` member if doing so + // is advantageous at some point. We don't need to store this in most cases. + // However, at the moment this doesn't appear to make the allocation any + // smaller and makes the code somewhat simpler to read. + int NumMemRefs = 0; public: - mmo_iterator memoperands_begin() const { return MemRefs; } - mmo_iterator memoperands_end() const { return MemRefsEnd; } - bool memoperands_empty() const { return MemRefsEnd == MemRefs; } + using mmo_iterator = ArrayRef::const_iterator; - /// Assign this MachineSDNodes's memory reference descriptor - /// list. This does not transfer ownership. - void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { - for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI) - assert(*MMI && "Null mem ref detected!"); - MemRefs = NewMemRefs; - MemRefsEnd = NewMemRefsEnd; + ArrayRef memoperands() const { + // Special case the common cases. + if (NumMemRefs == 0) + return {}; + if (NumMemRefs == 1) + return makeArrayRef(MemRefs.getAddrOfPtr1(), 1); + + // Otherwise we have an actual array. + return makeArrayRef(MemRefs.get(), NumMemRefs); + } + mmo_iterator memoperands_begin() const { return memoperands().begin(); } + mmo_iterator memoperands_end() const { return memoperands().end(); } + bool memoperands_empty() const { return memoperands().empty(); } + + /// Clear out the memory reference descriptor list. + void clearMemRefs() { + MemRefs = nullptr; + NumMemRefs = 0; } static bool classof(const SDNode *N) { @@ -2405,17 +2473,32 @@ namespace ISD { cast(N)->getAddressingMode() == ISD::UNINDEXED; } + /// Return true if the node is a math/logic binary operator. This corresponds + /// to the IR function of the same name. + inline bool isBinaryOp(const SDNode *N) { + auto Op = N->getOpcode(); + return (Op == ISD::ADD || Op == ISD::SUB || Op == ISD::MUL || + Op == ISD::AND || Op == ISD::OR || Op == ISD::XOR || + Op == ISD::SHL || Op == ISD::SRL || Op == ISD::SRA || + Op == ISD::SDIV || Op == ISD::UDIV || Op == ISD::SREM || + Op == ISD::UREM || Op == ISD::FADD || Op == ISD::FSUB || + Op == ISD::FMUL || Op == ISD::FDIV || Op == ISD::FREM); + } + /// Attempt to match a unary predicate against a scalar/splat constant or /// every element of a constant BUILD_VECTOR. + /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. bool matchUnaryPredicate(SDValue Op, - std::function Match); + std::function Match, + bool AllowUndefs = false); /// Attempt to match a binary predicate against a pair of scalar/splat /// constants or every element of a pair of constant BUILD_VECTORs. + /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. bool matchBinaryPredicate( SDValue LHS, SDValue RHS, - std::function Match); - + std::function Match, + bool AllowUndefs = false); } // end namespace ISD } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h b/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h index 334267d9828..8c8a7be459f 100644 --- a/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h +++ b/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h @@ -413,8 +413,14 @@ class raw_ostream; /// Returns the base index for the given instruction. SlotIndex getInstructionIndex(const MachineInstr &MI) const { // Instructions inside a bundle have the same number as the bundle itself. - const MachineInstr &BundleStart = *getBundleStart(MI.getIterator()); - Mi2IndexMap::const_iterator itr = mi2iMap.find(&BundleStart); + auto BundleStart = getBundleStart(MI.getIterator()); + auto BundleEnd = getBundleEnd(MI.getIterator()); + // Use the first non-debug instruction in the bundle to get SlotIndex. + const MachineInstr &BundleNonDebug = + *skipDebugInstructionsForward(BundleStart, BundleEnd); + assert(!BundleNonDebug.isDebugInstr() && + "Could not use a debug instruction to query mi2iMap."); + Mi2IndexMap::const_iterator itr = mi2iMap.find(&BundleNonDebug); assert(itr != mi2iMap.end() && "Instruction not found in maps."); return itr->second; } @@ -442,7 +448,7 @@ class raw_ostream; /// MI is not required to have an index. SlotIndex getIndexBefore(const MachineInstr &MI) const { const MachineBasicBlock *MBB = MI.getParent(); - assert(MBB && "MI must be inserted inna basic block"); + assert(MBB && "MI must be inserted in a basic block"); MachineBasicBlock::const_iterator I = MI, B = MBB->begin(); while (true) { if (I == B) @@ -459,7 +465,7 @@ class raw_ostream; /// MI is not required to have an index. SlotIndex getIndexAfter(const MachineInstr &MI) const { const MachineBasicBlock *MBB = MI.getParent(); - assert(MBB && "MI must be inserted inna basic block"); + assert(MBB && "MI must be inserted in a basic block"); MachineBasicBlock::const_iterator I = MI, E = MBB->end(); while (true) { ++I; @@ -674,7 +680,7 @@ class raw_ostream; idx2MBBMap.push_back(IdxMBBPair(startIdx, mbb)); renumberIndexes(newItr); - llvm::sort(idx2MBBMap.begin(), idx2MBBMap.end(), Idx2MBBCompare()); + llvm::sort(idx2MBBMap, Idx2MBBCompare()); } /// Free the resources that were required to maintain a SlotIndex. diff --git a/contrib/llvm/include/llvm/CodeGen/StackMaps.h b/contrib/llvm/include/llvm/CodeGen/StackMaps.h index e584a4136e4..8be9ae37855 100644 --- a/contrib/llvm/include/llvm/CodeGen/StackMaps.h +++ b/contrib/llvm/include/llvm/CodeGen/StackMaps.h @@ -236,25 +236,6 @@ public: FnInfos.clear(); } - /// Generate a stackmap record for a stackmap instruction. - /// - /// MI must be a raw STACKMAP, not a PATCHPOINT. - void recordStackMap(const MachineInstr &MI); - - /// Generate a stackmap record for a patchpoint instruction. - void recordPatchPoint(const MachineInstr &MI); - - /// Generate a stackmap record for a statepoint instruction. - void recordStatepoint(const MachineInstr &MI); - - /// If there is any stack map data, create a stack map section and serialize - /// the map info into it. This clears the stack map data structures - /// afterwards. - void serializeToStackMapSection(); - -private: - static const char *WSMP; - using LocationVec = SmallVector; using LiveOutVec = SmallVector; using ConstantPool = MapVector; @@ -283,6 +264,31 @@ private: using FnInfoMap = MapVector; using CallsiteInfoList = std::vector; + /// Generate a stackmap record for a stackmap instruction. + /// + /// MI must be a raw STACKMAP, not a PATCHPOINT. + void recordStackMap(const MachineInstr &MI); + + /// Generate a stackmap record for a patchpoint instruction. + void recordPatchPoint(const MachineInstr &MI); + + /// Generate a stackmap record for a statepoint instruction. + void recordStatepoint(const MachineInstr &MI); + + /// If there is any stack map data, create a stack map section and serialize + /// the map info into it. This clears the stack map data structures + /// afterwards. + void serializeToStackMapSection(); + + /// Get call site info. + CallsiteInfoList &getCSInfos() { return CSInfos; } + + /// Get function info. + FnInfoMap &getFnInfos() { return FnInfos; } + +private: + static const char *WSMP; + AsmPrinter &AP; CallsiteInfoList CSInfos; ConstantPool ConstPool; diff --git a/contrib/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/contrib/llvm/include/llvm/CodeGen/TargetFrameLowering.h index f8effee998e..b4d1da94143 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetFrameLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetFrameLowering.h @@ -207,8 +207,11 @@ public: return false; } - /// Return true if the target needs to disable frame pointer elimination. - virtual bool noFramePointerElim(const MachineFunction &MF) const; + /// Return true if the target wants to keep the frame pointer regardless of + /// the function attribute "frame-pointer". + virtual bool keepFramePointer(const MachineFunction &MF) const { + return false; + } /// hasFP - Return true if the specified function should have a dedicated /// frame pointer register. For most targets this is true only if the function diff --git a/contrib/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/contrib/llvm/include/llvm/CodeGen/TargetInstrInfo.h index b5bc561d834..961b90e9bc1 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -246,14 +246,14 @@ public: } /// If the specified machine instruction has a load from a stack slot, - /// return true along with the FrameIndex of the loaded stack slot and the - /// machine mem operand containing the reference. + /// return true along with the FrameIndices of the loaded stack slot and the + /// machine mem operands containing the reference. /// If not, return false. Unlike isLoadFromStackSlot, this returns true for /// any instructions that loads from the stack. This is just a hint, as some /// cases may be missed. - virtual bool hasLoadFromStackSlot(const MachineInstr &MI, - const MachineMemOperand *&MMO, - int &FrameIndex) const; + virtual bool hasLoadFromStackSlot( + const MachineInstr &MI, + SmallVectorImpl &Accesses) const; /// If the specified machine instruction is a direct /// store to a stack slot, return the virtual or physical register number of @@ -284,14 +284,14 @@ public: } /// If the specified machine instruction has a store to a stack slot, - /// return true along with the FrameIndex of the loaded stack slot and the - /// machine mem operand containing the reference. + /// return true along with the FrameIndices of the loaded stack slot and the + /// machine mem operands containing the reference. /// If not, return false. Unlike isStoreToStackSlot, /// this returns true for any instructions that stores to the /// stack. This is just a hint, as some cases may be missed. - virtual bool hasStoreToStackSlot(const MachineInstr &MI, - const MachineMemOperand *&MMO, - int &FrameIndex) const; + virtual bool hasStoreToStackSlot( + const MachineInstr &MI, + SmallVectorImpl &Accesses) const; /// Return true if the specified machine instruction /// is a copy of one stack slot to another and has no other effect. @@ -846,15 +846,33 @@ public: llvm_unreachable("Target didn't implement TargetInstrInfo::copyPhysReg!"); } +protected: + /// Target-dependent implemenation for IsCopyInstr. /// If the specific machine instruction is a instruction that moves/copies /// value from one register to another register return true along with /// @Source machine operand and @Destination machine operand. - virtual bool isCopyInstr(const MachineInstr &MI, - const MachineOperand *&SourceOpNum, - const MachineOperand *&Destination) const { + virtual bool isCopyInstrImpl(const MachineInstr &MI, + const MachineOperand *&Source, + const MachineOperand *&Destination) const { return false; } +public: + /// If the specific machine instruction is a instruction that moves/copies + /// value from one register to another register return true along with + /// @Source machine operand and @Destination machine operand. + /// For COPY-instruction the method naturally returns true, for all other + /// instructions the method calls target-dependent implementation. + bool isCopyInstr(const MachineInstr &MI, const MachineOperand *&Source, + const MachineOperand *&Destination) const { + if (MI.isCopy()) { + Destination = &MI.getOperand(0); + Source = &MI.getOperand(1); + return true; + } + return isCopyInstrImpl(MI, Source, Destination); + } + /// Store the specified register of the given register class to the specified /// stack frame index. The store instruction is to be added to the given /// machine basic block before the specified machine instruction. If isKill @@ -1063,7 +1081,7 @@ public: /// getAddressSpaceForPseudoSourceKind - Given the kind of memory /// (e.g. stack) the target returns the corresponding address space. virtual unsigned - getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const { + getAddressSpaceForPseudoSourceKind(unsigned Kind) const { return 0; } @@ -1118,11 +1136,11 @@ public: return false; } - /// Get the base register and byte offset of an instruction that reads/writes + /// Get the base operand and byte offset of an instruction that reads/writes /// memory. - virtual bool getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, - int64_t &Offset, - const TargetRegisterInfo *TRI) const { + virtual bool getMemOperandWithOffset(MachineInstr &MI, + MachineOperand *&BaseOp, int64_t &Offset, + const TargetRegisterInfo *TRI) const { return false; } @@ -1146,8 +1164,8 @@ public: /// or /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); /// to TargetPassConfig::createMachineScheduler() to have an effect. - virtual bool shouldClusterMemOps(MachineInstr &FirstLdSt, unsigned BaseReg1, - MachineInstr &SecondLdSt, unsigned BaseReg2, + virtual bool shouldClusterMemOps(MachineOperand &BaseOp1, + MachineOperand &BaseOp2, unsigned NumLoads) const { llvm_unreachable("target did not implement shouldClusterMemOps()"); } @@ -1617,10 +1635,11 @@ public: "Target didn't implement TargetInstrInfo::getOutliningType!"); } - /// Returns target-defined flags defining properties of the MBB for - /// the outliner. - virtual unsigned getMachineOutlinerMBBFlags(MachineBasicBlock &MBB) const { - return 0x0; + /// Optional target hook that returns true if \p MBB is safe to outline from, + /// and returns any target-specific information in \p Flags. + virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, + unsigned &Flags) const { + return true; } /// Insert a custom frame for outlined functions. diff --git a/contrib/llvm/include/llvm/CodeGen/TargetLowering.h b/contrib/llvm/include/llvm/CodeGen/TargetLowering.h index 847da671c42..23dbaac03eb 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetLowering.h @@ -29,7 +29,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/RuntimeLibcalls.h" @@ -163,6 +163,7 @@ public: LLOnly, // Expand the (load) instruction into just a load-linked, which has // greater atomic guarantees than a normal load. CmpXChg, // Expand the instruction into cmpxchg; used by at least X86. + MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop. }; /// Enum that specifies when a multiplication should be expanded. @@ -268,6 +269,14 @@ public: return true; } + /// Return true if it is profitable to convert a select of FP constants into + /// a constant pool load whose address depends on the select condition. The + /// parameter may be used to differentiate a select with FP compare from + /// integer compare. + virtual bool reduceSelectOfFPConstantLoads(bool IsFPSetCC) const { + return true; + } + /// Return true if multiple condition registers are available. bool hasMultipleConditionRegisters() const { return HasMultipleConditionRegisters; @@ -278,7 +287,7 @@ public: /// Return the preferred vector type legalization action. virtual TargetLoweringBase::LegalizeTypeAction - getPreferredVectorAction(EVT VT) const { + getPreferredVectorAction(MVT VT) const { // The default action for one element vectors is to scalarize if (VT.getVectorNumElements() == 1) return TypeScalarizeVector; @@ -545,6 +554,12 @@ public: return false; } + /// Return true if inserting a scalar into a variable element of an undef + /// vector is more efficiently handled by splatting the scalar instead. + virtual bool shouldSplatInsEltVarIndex(EVT) const { + return false; + } + /// Return true if target supports floating point exceptions. bool hasFloatingPointExceptions() const { return HasFloatingPointExceptions; @@ -790,6 +805,38 @@ public: return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op]; } + /// Custom method defined by each target to indicate if an operation which + /// may require a scale is supported natively by the target. + /// If not, the operation is illegal. + virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT, + unsigned Scale) const { + return false; + } + + /// Some fixed point operations may be natively supported by the target but + /// only for specific scales. This method allows for checking + /// if the width is supported by the target for a given operation that may + /// depend on scale. + LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, + unsigned Scale) const { + auto Action = getOperationAction(Op, VT); + if (Action != Legal) + return Action; + + // This operation is supported in this type but may only work on specific + // scales. + bool Supported; + switch (Op) { + default: + llvm_unreachable("Unexpected fixed point operation."); + case ISD::SMULFIX: + Supported = isSupportedFixedPointOperation(Op, VT, Scale); + break; + } + + return Supported ? Action : Expand; + } + LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const { unsigned EqOpc; switch (Op) { @@ -798,6 +845,7 @@ public: case ISD::STRICT_FSUB: EqOpc = ISD::FSUB; break; case ISD::STRICT_FMUL: EqOpc = ISD::FMUL; break; case ISD::STRICT_FDIV: EqOpc = ISD::FDIV; break; + case ISD::STRICT_FREM: EqOpc = ISD::FREM; break; case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break; case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break; case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break; @@ -811,6 +859,12 @@ public: case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break; case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break; case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break; + case ISD::STRICT_FMAXNUM: EqOpc = ISD::FMAXNUM; break; + case ISD::STRICT_FMINNUM: EqOpc = ISD::FMINNUM; break; + case ISD::STRICT_FCEIL: EqOpc = ISD::FCEIL; break; + case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break; + case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break; + case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break; } auto Action = getOperationAction(EqOpc, VT); @@ -1199,13 +1253,15 @@ public: /// reduce runtime. virtual bool ShouldShrinkFPConstant(EVT) const { return true; } - // Return true if it is profitable to reduce the given load node to a smaller - // type. - // - // e.g. (i16 (trunc (i32 (load x))) -> i16 load x should be performed - virtual bool shouldReduceLoadWidth(SDNode *Load, - ISD::LoadExtType ExtTy, + /// Return true if it is profitable to reduce a load to a smaller type. + /// Example: (i16 (trunc (i32 (load x))) -> i16 load x + virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const { + // By default, assume that it is cheaper to extract a subvector from a wide + // vector load rather than creating multiple narrow vector loads. + if (NewVT.isVector() && !Load->hasOneUse()) + return false; + return true; } @@ -1428,6 +1484,12 @@ public: return PrefLoopAlignment; } + /// Should loops be aligned even when the function is marked OptSize (but not + /// MinSize). + virtual bool alignLoopsWithOptSize() const { + return false; + } + /// If the target has a standard location for the stack protector guard, /// returns the address of that location. Otherwise, returns nullptr. /// DEPRECATED: please override useLoadStackGuardNode and customize @@ -1549,6 +1611,26 @@ public: llvm_unreachable("Store conditional unimplemented on this target"); } + /// Perform a masked atomicrmw using a target-specific intrinsic. This + /// represents the core LL/SC loop which will be lowered at a late stage by + /// the backend. + virtual Value *emitMaskedAtomicRMWIntrinsic(IRBuilder<> &Builder, + AtomicRMWInst *AI, + Value *AlignedAddr, Value *Incr, + Value *Mask, Value *ShiftAmt, + AtomicOrdering Ord) const { + llvm_unreachable("Masked atomicrmw expansion unimplemented on this target"); + } + + /// Perform a masked cmpxchg using a target-specific intrinsic. This + /// represents the core LL/SC loop which will be lowered at a late stage by + /// the backend. + virtual Value *emitMaskedAtomicCmpXchgIntrinsic( + IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, + Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { + llvm_unreachable("Masked cmpxchg expansion unimplemented on this target"); + } + /// Inserts in the IR a target-specific intrinsic specifying a fence. /// It is called by AtomicExpandPass before expanding an /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad @@ -1625,11 +1707,11 @@ public: return AtomicExpansionKind::None; } - /// Returns true if the given atomic cmpxchg should be expanded by the - /// IR-level AtomicExpand pass into a load-linked/store-conditional sequence - /// (through emitLoadLinked() and emitStoreConditional()). - virtual bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { - return false; + /// Returns how the given atomic cmpxchg should be expanded by the IR-level + /// AtomicExpand pass. + virtual AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { + return AtomicExpansionKind::None; } /// Returns how the IR-level AtomicExpand pass should expand the given @@ -1687,6 +1769,25 @@ public: return false; } + /// Return true if it is profitable to transform an integer + /// multiplication-by-constant into simpler operations like shifts and adds. + /// This may be true if the target does not directly support the + /// multiplication operation for the specified type or the sequence of simpler + /// ops is faster than the multiply. + virtual bool decomposeMulByConstant(EVT VT, SDValue C) const { + return false; + } + + /// Return true if it is more correct/profitable to use strict FP_TO_INT + /// conversion operations - canonicalizing the FP source value instead of + /// converting all cases and then selecting based on value. + /// This may be true if the target throws exceptions for out of bounds + /// conversions or has fast FP CMOV. + virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, + bool IsSigned) const { + return false; + } + //===--------------------------------------------------------------------===// // TargetLowering Configuration Methods - These methods should be invoked by // the derived class constructor to configure this object for the target. @@ -2015,6 +2116,14 @@ public: return true; } + /// Return true if the specified immediate is legal for the value input of a + /// store instruction. + virtual bool isLegalStoreImmediate(int64_t Value) const { + // Default implementation assumes that at least 0 works since it is likely + // that a zero register exists or a zero immediate is allowed. + return Value == 0; + } + /// Return true if it's significantly cheaper to shift a vector by a uniform /// scalar than by an amount which will vary across each lane. On x86, for /// example, there is a "psllw" instruction for the former case, but no simple @@ -2046,10 +2155,12 @@ public: case ISD::UADDO: case ISD::ADDC: case ISD::ADDE: + case ISD::SADDSAT: + case ISD::UADDSAT: case ISD::FMINNUM: case ISD::FMAXNUM: - case ISD::FMINNAN: - case ISD::FMAXNAN: + case ISD::FMINIMUM: + case ISD::FMAXIMUM: return true; default: return false; } @@ -2153,6 +2264,12 @@ public: return false; } + /// Return true if sign-extension from FromTy to ToTy is cheaper than + /// zero-extension. + virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const { + return false; + } + /// Return true if the target supplies and combines to a paired load /// two loaded values of type LoadedType next to each other in memory. /// RequiredAlignment gives the minimal alignment constraints that must be met @@ -2292,6 +2409,12 @@ public: return false; } + /// Try to convert an extract element of a vector binary operation into an + /// extract element followed by a scalar operation. + virtual bool shouldScalarizeBinop(SDValue VecOp) const { + return false; + } + // Return true if it is profitable to use a scalar input to a BUILD_VECTOR // even if the vector itself has multiple uses. virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const { @@ -2648,7 +2771,7 @@ public: virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, - DivergenceAnalysis *DA) const { + LegacyDivergenceAnalysis *DA) const { return false; } @@ -2774,36 +2897,33 @@ public: bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &Demanded, TargetLoweringOpt &TLO) const; - /// Helper for SimplifyDemandedBits that can simplify an operation with - /// multiple uses. This function simplifies operand \p OpIdx of \p User and - /// then updates \p User with the simplified version. No other uses of - /// \p OpIdx are updated. If \p User is the only user of \p OpIdx, this - /// function behaves exactly like function SimplifyDemandedBits declared - /// below except that it also updates the DAG by calling - /// DCI.CommitTargetLoweringOpt. - bool SimplifyDemandedBits(SDNode *User, unsigned OpIdx, const APInt &Demanded, - DAGCombinerInfo &DCI, TargetLoweringOpt &TLO) const; - - /// Look at Op. At this point, we know that only the DemandedMask bits of the + /// Look at Op. At this point, we know that only the DemandedBits bits of the /// result of Op are ever used downstream. If we can use this information to /// simplify Op, create a new simplified DAG node and return true, returning /// the original and new nodes in Old and New. Otherwise, analyze the /// expression and return a mask of KnownOne and KnownZero bits for the /// expression (used to simplify the caller). The KnownZero/One bits may only - /// be accurate for those bits in the DemandedMask. + /// be accurate for those bits in the Demanded masks. /// \p AssumeSingleUse When this parameter is true, this function will /// attempt to simplify \p Op even if there are multiple uses. /// Callers are responsible for correctly updating the DAG based on the /// results of this function, because simply replacing replacing TLO.Old /// with TLO.New will be incorrect when this parameter is true and TLO.Old /// has multiple uses. - bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, - KnownBits &Known, - TargetLoweringOpt &TLO, + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts, KnownBits &Known, + TargetLoweringOpt &TLO, unsigned Depth = 0, + bool AssumeSingleUse = false) const; + + /// Helper wrapper around SimplifyDemandedBits, demanding all elements. + /// Adds Op back to the worklist upon success. + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth = 0, bool AssumeSingleUse = false) const; - /// Helper wrapper around SimplifyDemandedBits + /// Helper wrapper around SimplifyDemandedBits. + /// Adds Op back to the worklist upon success. bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, DAGCombinerInfo &DCI) const; @@ -2826,7 +2946,8 @@ public: TargetLoweringOpt &TLO, unsigned Depth = 0, bool AssumeSingleUse = false) const; - /// Helper wrapper around SimplifyDemandedVectorElts + /// Helper wrapper around SimplifyDemandedVectorElts. + /// Adds Op back to the worklist upon success. bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, DAGCombinerInfo &DCI) const; @@ -2863,11 +2984,30 @@ public: /// elements, returning true on success. Otherwise, analyze the expression and /// return a mask of KnownUndef and KnownZero elements for the expression /// (used to simplify the caller). The KnownUndef/Zero elements may only be - /// accurate for those bits in the DemandedMask + /// accurate for those bits in the DemandedMask. virtual bool SimplifyDemandedVectorEltsForTargetNode( SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth = 0) const; + /// Attempt to simplify any target nodes based on the demanded bits/elts, + /// returning true on success. Otherwise, analyze the + /// expression and return a mask of KnownOne and KnownZero bits for the + /// expression (used to simplify the caller). The KnownZero/One bits may only + /// be accurate for those bits in the Demanded masks. + virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, + const APInt &DemandedBits, + const APInt &DemandedElts, + KnownBits &Known, + TargetLoweringOpt &TLO, + unsigned Depth = 0) const; + + /// If \p SNaN is false, \returns true if \p Op is known to never be any + /// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling + /// NaN. + virtual bool isKnownNeverNaNForTargetNode(SDValue Op, + const SelectionDAG &DAG, + bool SNaN = false, + unsigned Depth = 0) const; struct DAGCombinerInfo { void *DC; // The DAG Combiner object. CombineLevel Level; @@ -2948,6 +3088,15 @@ public: return true; } + /// Return true if it is profitable to fold a pair of shifts into a mask. + /// This is usually true on most targets. But some targets, like Thumb1, + /// have immediate shift instructions, but no immediate "and" instruction; + /// this makes the fold unprofitable. + virtual bool shouldFoldShiftPairToMask(const SDNode *N, + CombineLevel Level) const { + return true; + } + // Return true if it is profitable to combine a BUILD_VECTOR with a stride-pattern // to a shuffle and a truncate. // Example of such a combine: @@ -3492,11 +3641,9 @@ public: //===--------------------------------------------------------------------===// // Div utility functions // - SDValue BuildSDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, - bool IsAfterLegalization, + SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl &Created) const; - SDValue BuildUDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, - bool IsAfterLegalization, + SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl &Created) const; /// Targets may override this function to provide custom SDIV lowering for @@ -3588,12 +3735,68 @@ public: SDValue LL = SDValue(), SDValue LH = SDValue(), SDValue RL = SDValue(), SDValue RH = SDValue()) const; + /// Expand funnel shift. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandFunnelShift(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand rotations. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandROT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + /// Expand float(f32) to SINT(i64) conversion /// \param N Node to expand /// \param Result output after conversion /// \returns True, if the expansion was successful, false otherwise bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + /// Expand float to UINT conversion + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand UINT(i64) to double(f64) conversion + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs. + SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const; + + /// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandCTPOP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand CTLZ/CTLZ_ZERO_UNDEF nodes. Expands vector/scalar CTLZ nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandCTLZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand CTTZ/CTTZ_ZERO_UNDEF nodes. Expands vector/scalar CTTZ nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandCTTZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + + /// Expand ABS nodes. Expands vector/scalar ABS nodes, + /// vector nodes can only succeed if all operations are legal/custom. + /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size)) + /// \param N Node to expand + /// \param Result output after conversion + /// \returns True, if the expansion was successful, false otherwise + bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + /// Turn load of vector type into a load of the individual elements. /// \param LD load to expand /// \returns MERGE_VALUEs of the scalar loads with their chains. @@ -3631,6 +3834,15 @@ public: SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const; + /// Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT. This + /// method accepts integers as its arguments. + SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const; + + /// Method for building the DAG expansion of ISD::SMULFIX. This method accepts + /// integers as its arguments. + SDValue getExpandedFixedPointMultiplication(SDNode *Node, + SelectionDAG &DAG) const; + //===--------------------------------------------------------------------===// // Instruction Emitting Hooks // diff --git a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index f5c7fc824ab..052d1f8bc68 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -90,6 +90,8 @@ public: const MCExpr *lowerRelativeReference(const GlobalValue *LHS, const GlobalValue *RHS, const TargetMachine &TM) const override; + + MCSection *getSectionForCommandLines() const override; }; class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile { diff --git a/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h b/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h index 8f5c9cb8c3f..3288711a335 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -90,6 +90,19 @@ private: AnalysisID StartAfter = nullptr; AnalysisID StopBefore = nullptr; AnalysisID StopAfter = nullptr; + + unsigned StartBeforeInstanceNum = 0; + unsigned StartBeforeCount = 0; + + unsigned StartAfterInstanceNum = 0; + unsigned StartAfterCount = 0; + + unsigned StopBeforeInstanceNum = 0; + unsigned StopBeforeCount = 0; + + unsigned StopAfterInstanceNum = 0; + unsigned StopAfterCount = 0; + bool Started = true; bool Stopped = false; bool AddingMachinePasses = false; @@ -145,13 +158,13 @@ public: CodeGenOpt::Level getOptLevel() const; - /// Describe the status of the codegen - /// pipeline set by this target pass config. - /// Having a limited codegen pipeline means that options - /// have been used to restrict what codegen is doing. - /// In particular, that means that codegen won't emit - /// assembly code. - bool hasLimitedCodeGenPipeline() const; + /// Returns true if one of the `-start-after`, `-start-before`, `-stop-after` + /// or `-stop-before` options is set. + static bool hasLimitedCodeGenPipeline(); + + /// Returns true if none of the `-stop-before` and `-stop-after` options is + /// set. + static bool willCompleteCodeGenPipeline(); /// If hasLimitedCodeGenPipeline is true, this method /// returns a string with the name of the options, separated @@ -159,13 +172,6 @@ public: std::string getLimitedCodeGenPipelineReason(const char *Separator = "/") const; - /// Check if the codegen pipeline is limited in such a way that it - /// won't be complete. When the codegen pipeline is not complete, - /// this means it may not be possible to generate assembly from it. - bool willCompleteCodeGenPipeline() const { - return !hasLimitedCodeGenPipeline() || (!StopAfter && !StopBefore); - } - void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); } bool getEnableTailMerge() const { return EnableTailMerge; } diff --git a/contrib/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/contrib/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index 55a8ba630a5..0fbff313765 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -510,6 +510,13 @@ public: /// markSuperRegs() and checkAllSuperRegsMarked() in this case. virtual BitVector getReservedRegs(const MachineFunction &MF) const = 0; + /// Returns false if we can't guarantee that Physreg, specified as an IR asm + /// clobber constraint, will be preserved across the statement. + virtual bool isAsmClobberable(const MachineFunction &MF, + unsigned PhysReg) const { + return true; + } + /// Returns true if PhysReg is unallocatable and constant throughout the /// function. Used by MachineRegisterInfo::isConstantPhysReg(). virtual bool isConstantPhysReg(unsigned PhysReg) const { return false; } @@ -817,13 +824,6 @@ public: // Do nothing. } - /// The creation of multiple copy hints have been implemented in - /// weightCalcHelper(), but since this affects so many tests for many - /// targets, this is temporarily disabled per default. THIS SHOULD BE - /// "GENERAL GOODNESS" and hopefully all targets will update their tests - /// and enable this soon. This hook should then be removed. - virtual bool enableMultipleCopyHints() const { return false; } - /// Allow the target to reverse allocation order of local live ranges. This /// will generally allocate shorter local live ranges first. For targets with /// many registers, this could reduce regalloc compile time by a large diff --git a/contrib/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/contrib/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h index 227e591f5a7..968e4c4b810 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_TARGETSUBTARGETINFO_H #define LLVM_CODEGEN_TARGETSUBTARGETINFO_H +#include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -144,6 +145,43 @@ public: return 0; } + /// Returns true if MI is a dependency breaking zero-idiom instruction for the + /// subtarget. + /// + /// This function also sets bits in Mask related to input operands that + /// are not in a data dependency relationship. There is one bit for each + /// machine operand; implicit operands follow explicit operands in the bit + /// representation used for Mask. An empty (i.e. a mask with all bits + /// cleared) means: data dependencies are "broken" for all the explicit input + /// machine operands of MI. + virtual bool isZeroIdiom(const MachineInstr *MI, APInt &Mask) const { + return false; + } + + /// Returns true if MI is a dependency breaking instruction for the subtarget. + /// + /// Similar in behavior to `isZeroIdiom`. However, it knows how to identify + /// all dependency breaking instructions (i.e. not just zero-idioms). + /// + /// As for `isZeroIdiom`, this method returns a mask of "broken" dependencies. + /// (See method `isZeroIdiom` for a detailed description of Mask). + virtual bool isDependencyBreaking(const MachineInstr *MI, APInt &Mask) const { + return isZeroIdiom(MI, Mask); + } + + /// Returns true if MI is a candidate for move elimination. + /// + /// A candidate for move elimination may be optimized out at register renaming + /// stage. Subtargets can specify the set of optimizable moves by + /// instantiating tablegen class `IsOptimizableRegisterMove` (see + /// llvm/Target/TargetInstrPredicate.td). + /// + /// SubtargetEmitter is responsible for processing all the definitions of class + /// IsOptimizableRegisterMove, and auto-generate an override for this method. + virtual bool isOptimizableRegisterMove(const MachineInstr *MI) const { + return false; + } + /// True if the subtarget should run MachineScheduler after aggressive /// coalescing. /// diff --git a/contrib/llvm/include/llvm/CodeGen/WasmEHFuncInfo.h b/contrib/llvm/include/llvm/CodeGen/WasmEHFuncInfo.h index 3ad6760d881..219fff988f6 100644 --- a/contrib/llvm/include/llvm/CodeGen/WasmEHFuncInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/WasmEHFuncInfo.h @@ -14,13 +14,15 @@ #ifndef LLVM_CODEGEN_WASMEHFUNCINFO_H #define LLVM_CODEGEN_WASMEHFUNCINFO_H -#include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/BasicBlock.h" namespace llvm { +enum EventTag { CPP_EXCEPTION = 0, C_LONGJMP = 1 }; + using BBOrMBB = PointerUnion; struct WasmEHFuncInfo { diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h index 9dbeb438f4a..11ca9ff108d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h @@ -45,13 +45,8 @@ public: return RecordData.drop_front(sizeof(RecordPrefix)); } - Optional hash() const { return Hash; } - - void setHash(uint32_t Value) { Hash = Value; } - Kind Type; ArrayRef RecordData; - Optional Hash; }; template struct RemappedRecord { diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h index 4ce9f68cffd..8e0d9f608e9 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h @@ -231,6 +231,8 @@ enum class FrameProcedureOptions : uint32_t { Inlined = 0x00000800, StrictSecurityChecks = 0x00001000, SafeBuffers = 0x00002000, + EncodedLocalBasePointerMask = 0x0000C000, + EncodedParamBasePointerMask = 0x00030000, ProfileGuidedOptimization = 0x00040000, ValidProfileCounts = 0x00080000, OptimizedForSpeed = 0x00100000, @@ -356,7 +358,9 @@ enum class PointerOptions : uint32_t { Const = 0x00000400, Unaligned = 0x00000800, Restrict = 0x00001000, - WinRTSmartPointer = 0x00080000 + WinRTSmartPointer = 0x00080000, + LValueRefThisPointer = 0x00100000, + RValueRefThisPointer = 0x00200000 }; CV_DEFINE_ENUM_CLASS_FLAGS_OPERATORS(PointerOptions) @@ -510,6 +514,19 @@ enum class RegisterId : uint16_t { #undef CV_REGISTER }; +/// Two-bit value indicating which register is the designated frame pointer +/// register. Appears in the S_FRAMEPROC record flags. +enum class EncodedFramePtrReg : uint8_t { + None = 0, + StackPtr = 1, + FramePtr = 2, + BasePtr = 3, +}; + +RegisterId decodeFramePtrReg(EncodedFramePtrReg EncodedReg, CPUType CPU); + +EncodedFramePtrReg encodeFramePtrReg(RegisterId Reg, CPUType CPU); + /// These values correspond to the THUNK_ORDINAL enumeration. enum class ThunkOrdinal : uint8_t { Standard, diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h index 586a720ce6e..d4615d02220 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h @@ -24,23 +24,32 @@ enum class cv_error_code { no_records, unknown_member_record, }; +} // namespace codeview +} // namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} // namespace std + +namespace llvm { +namespace codeview { +const std::error_category &CVErrorCategory(); + +inline std::error_code make_error_code(cv_error_code E) { + return std::error_code(static_cast(E), CVErrorCategory()); +} /// Base class for errors originating when parsing raw PDB files -class CodeViewError : public ErrorInfo { +class CodeViewError : public ErrorInfo { public: + using ErrorInfo::ErrorInfo; // inherit constructors + CodeViewError(const Twine &S) : ErrorInfo(S, cv_error_code::unspecified) {} static char ID; - CodeViewError(cv_error_code C); - CodeViewError(const std::string &Context); - CodeViewError(cv_error_code C, const std::string &Context); - - void log(raw_ostream &OS) const override; - const std::string &getErrorMessage() const; - std::error_code convertToErrorCode() const override; - -private: - std::string ErrMsg; - cv_error_code Code; }; -} -} + +} // namespace codeview +} // namespace llvm + #endif diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def index 6da8893bd61..fdfcf4d53a2 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def @@ -18,251 +18,342 @@ // This currently only contains the "register subset shared by all processor // types" (ERR etc.) and the x86 registers. -CV_REGISTER(CVRegERR, 30000) -CV_REGISTER(CVRegTEB, 30001) -CV_REGISTER(CVRegTIMER, 30002) -CV_REGISTER(CVRegEFAD1, 30003) -CV_REGISTER(CVRegEFAD2, 30004) -CV_REGISTER(CVRegEFAD3, 30005) -CV_REGISTER(CVRegVFRAME, 30006) -CV_REGISTER(CVRegHANDLE, 30007) -CV_REGISTER(CVRegPARAMS, 30008) -CV_REGISTER(CVRegLOCALS, 30009) -CV_REGISTER(CVRegTID, 30010) -CV_REGISTER(CVRegENV, 30011) -CV_REGISTER(CVRegCMDLN, 30012) +// Some system headers define macros that conflict with our enums. Every +// compiler supported by LLVM has the push_macro and pop_macro pragmas, so use +// them to avoid the conflict. +#pragma push_macro("CR0") +#pragma push_macro("CR1") +#pragma push_macro("CR2") +#pragma push_macro("CR3") +#pragma push_macro("CR4") -CV_REGISTER(CVRegNONE, 0) -CV_REGISTER(CVRegAL, 1) -CV_REGISTER(CVRegCL, 2) -CV_REGISTER(CVRegDL, 3) -CV_REGISTER(CVRegBL, 4) -CV_REGISTER(CVRegAH, 5) -CV_REGISTER(CVRegCH, 6) -CV_REGISTER(CVRegDH, 7) -CV_REGISTER(CVRegBH, 8) -CV_REGISTER(CVRegAX, 9) -CV_REGISTER(CVRegCX, 10) -CV_REGISTER(CVRegDX, 11) -CV_REGISTER(CVRegBX, 12) -CV_REGISTER(CVRegSP, 13) -CV_REGISTER(CVRegBP, 14) -CV_REGISTER(CVRegSI, 15) -CV_REGISTER(CVRegDI, 16) -CV_REGISTER(CVRegEAX, 17) -CV_REGISTER(CVRegECX, 18) -CV_REGISTER(CVRegEDX, 19) -CV_REGISTER(CVRegEBX, 20) -CV_REGISTER(CVRegESP, 21) -CV_REGISTER(CVRegEBP, 22) -CV_REGISTER(CVRegESI, 23) -CV_REGISTER(CVRegEDI, 24) -CV_REGISTER(CVRegES, 25) -CV_REGISTER(CVRegCS, 26) -CV_REGISTER(CVRegSS, 27) -CV_REGISTER(CVRegDS, 28) -CV_REGISTER(CVRegFS, 29) -CV_REGISTER(CVRegGS, 30) -CV_REGISTER(CVRegIP, 31) -CV_REGISTER(CVRegFLAGS, 32) -CV_REGISTER(CVRegEIP, 33) -CV_REGISTER(CVRegEFLAGS, 34) -CV_REGISTER(CVRegTEMP, 40) -CV_REGISTER(CVRegTEMPH, 41) -CV_REGISTER(CVRegQUOTE, 42) -CV_REGISTER(CVRegPCDR3, 43) -CV_REGISTER(CVRegPCDR4, 44) -CV_REGISTER(CVRegPCDR5, 45) -CV_REGISTER(CVRegPCDR6, 46) -CV_REGISTER(CVRegPCDR7, 47) -CV_REGISTER(CVRegCR0, 80) -CV_REGISTER(CVRegCR1, 81) -CV_REGISTER(CVRegCR2, 82) -CV_REGISTER(CVRegCR3, 83) -CV_REGISTER(CVRegCR4, 84) -CV_REGISTER(CVRegDR0, 90) -CV_REGISTER(CVRegDR1, 91) -CV_REGISTER(CVRegDR2, 92) -CV_REGISTER(CVRegDR3, 93) -CV_REGISTER(CVRegDR4, 94) -CV_REGISTER(CVRegDR5, 95) -CV_REGISTER(CVRegDR6, 96) -CV_REGISTER(CVRegDR7, 97) -CV_REGISTER(CVRegGDTR, 110) -CV_REGISTER(CVRegGDTL, 111) -CV_REGISTER(CVRegIDTR, 112) -CV_REGISTER(CVRegIDTL, 113) -CV_REGISTER(CVRegLDTR, 114) -CV_REGISTER(CVRegTR, 115) +CV_REGISTER(ERR, 30000) +CV_REGISTER(TEB, 30001) +CV_REGISTER(TIMER, 30002) +CV_REGISTER(EFAD1, 30003) +CV_REGISTER(EFAD2, 30004) +CV_REGISTER(EFAD3, 30005) +CV_REGISTER(VFRAME, 30006) +CV_REGISTER(HANDLE, 30007) +CV_REGISTER(PARAMS, 30008) +CV_REGISTER(LOCALS, 30009) +CV_REGISTER(TID, 30010) +CV_REGISTER(ENV, 30011) +CV_REGISTER(CMDLN, 30012) -CV_REGISTER(CVRegPSEUDO1, 116) -CV_REGISTER(CVRegPSEUDO2, 117) -CV_REGISTER(CVRegPSEUDO3, 118) -CV_REGISTER(CVRegPSEUDO4, 119) -CV_REGISTER(CVRegPSEUDO5, 120) -CV_REGISTER(CVRegPSEUDO6, 121) -CV_REGISTER(CVRegPSEUDO7, 122) -CV_REGISTER(CVRegPSEUDO8, 123) -CV_REGISTER(CVRegPSEUDO9, 124) +CV_REGISTER(NONE, 0) +CV_REGISTER(AL, 1) +CV_REGISTER(CL, 2) +CV_REGISTER(DL, 3) +CV_REGISTER(BL, 4) +CV_REGISTER(AH, 5) +CV_REGISTER(CH, 6) +CV_REGISTER(DH, 7) +CV_REGISTER(BH, 8) +CV_REGISTER(AX, 9) +CV_REGISTER(CX, 10) +CV_REGISTER(DX, 11) +CV_REGISTER(BX, 12) +CV_REGISTER(SP, 13) +CV_REGISTER(BP, 14) +CV_REGISTER(SI, 15) +CV_REGISTER(DI, 16) +CV_REGISTER(EAX, 17) +CV_REGISTER(ECX, 18) +CV_REGISTER(EDX, 19) +CV_REGISTER(EBX, 20) +CV_REGISTER(ESP, 21) +CV_REGISTER(EBP, 22) +CV_REGISTER(ESI, 23) +CV_REGISTER(EDI, 24) +CV_REGISTER(ES, 25) +CV_REGISTER(CS, 26) +CV_REGISTER(SS, 27) +CV_REGISTER(DS, 28) +CV_REGISTER(FS, 29) +CV_REGISTER(GS, 30) +CV_REGISTER(IP, 31) +CV_REGISTER(FLAGS, 32) +CV_REGISTER(EIP, 33) +CV_REGISTER(EFLAGS, 34) +CV_REGISTER(TEMP, 40) +CV_REGISTER(TEMPH, 41) +CV_REGISTER(QUOTE, 42) +CV_REGISTER(PCDR3, 43) +CV_REGISTER(PCDR4, 44) +CV_REGISTER(PCDR5, 45) +CV_REGISTER(PCDR6, 46) +CV_REGISTER(PCDR7, 47) +CV_REGISTER(CR0, 80) +CV_REGISTER(CR1, 81) +CV_REGISTER(CR2, 82) +CV_REGISTER(CR3, 83) +CV_REGISTER(CR4, 84) +CV_REGISTER(DR0, 90) +CV_REGISTER(DR1, 91) +CV_REGISTER(DR2, 92) +CV_REGISTER(DR3, 93) +CV_REGISTER(DR4, 94) +CV_REGISTER(DR5, 95) +CV_REGISTER(DR6, 96) +CV_REGISTER(DR7, 97) +CV_REGISTER(GDTR, 110) +CV_REGISTER(GDTL, 111) +CV_REGISTER(IDTR, 112) +CV_REGISTER(IDTL, 113) +CV_REGISTER(LDTR, 114) +CV_REGISTER(TR, 115) -CV_REGISTER(CVRegST0, 128) -CV_REGISTER(CVRegST1, 129) -CV_REGISTER(CVRegST2, 130) -CV_REGISTER(CVRegST3, 131) -CV_REGISTER(CVRegST4, 132) -CV_REGISTER(CVRegST5, 133) -CV_REGISTER(CVRegST6, 134) -CV_REGISTER(CVRegST7, 135) -CV_REGISTER(CVRegCTRL, 136) -CV_REGISTER(CVRegSTAT, 137) -CV_REGISTER(CVRegTAG, 138) -CV_REGISTER(CVRegFPIP, 139) -CV_REGISTER(CVRegFPCS, 140) -CV_REGISTER(CVRegFPDO, 141) -CV_REGISTER(CVRegFPDS, 142) -CV_REGISTER(CVRegISEM, 143) -CV_REGISTER(CVRegFPEIP, 144) -CV_REGISTER(CVRegFPEDO, 145) +CV_REGISTER(PSEUDO1, 116) +CV_REGISTER(PSEUDO2, 117) +CV_REGISTER(PSEUDO3, 118) +CV_REGISTER(PSEUDO4, 119) +CV_REGISTER(PSEUDO5, 120) +CV_REGISTER(PSEUDO6, 121) +CV_REGISTER(PSEUDO7, 122) +CV_REGISTER(PSEUDO8, 123) +CV_REGISTER(PSEUDO9, 124) -CV_REGISTER(CVRegMM0, 146) -CV_REGISTER(CVRegMM1, 147) -CV_REGISTER(CVRegMM2, 148) -CV_REGISTER(CVRegMM3, 149) -CV_REGISTER(CVRegMM4, 150) -CV_REGISTER(CVRegMM5, 151) -CV_REGISTER(CVRegMM6, 152) -CV_REGISTER(CVRegMM7, 153) +CV_REGISTER(ST0, 128) +CV_REGISTER(ST1, 129) +CV_REGISTER(ST2, 130) +CV_REGISTER(ST3, 131) +CV_REGISTER(ST4, 132) +CV_REGISTER(ST5, 133) +CV_REGISTER(ST6, 134) +CV_REGISTER(ST7, 135) +CV_REGISTER(CTRL, 136) +CV_REGISTER(STAT, 137) +CV_REGISTER(TAG, 138) +CV_REGISTER(FPIP, 139) +CV_REGISTER(FPCS, 140) +CV_REGISTER(FPDO, 141) +CV_REGISTER(FPDS, 142) +CV_REGISTER(ISEM, 143) +CV_REGISTER(FPEIP, 144) +CV_REGISTER(FPEDO, 145) -CV_REGISTER(CVRegXMM0, 154) -CV_REGISTER(CVRegXMM1, 155) -CV_REGISTER(CVRegXMM2, 156) -CV_REGISTER(CVRegXMM3, 157) -CV_REGISTER(CVRegXMM4, 158) -CV_REGISTER(CVRegXMM5, 159) -CV_REGISTER(CVRegXMM6, 160) -CV_REGISTER(CVRegXMM7, 161) +CV_REGISTER(MM0, 146) +CV_REGISTER(MM1, 147) +CV_REGISTER(MM2, 148) +CV_REGISTER(MM3, 149) +CV_REGISTER(MM4, 150) +CV_REGISTER(MM5, 151) +CV_REGISTER(MM6, 152) +CV_REGISTER(MM7, 153) -CV_REGISTER(CVRegMXCSR, 211) +CV_REGISTER(XMM0, 154) +CV_REGISTER(XMM1, 155) +CV_REGISTER(XMM2, 156) +CV_REGISTER(XMM3, 157) +CV_REGISTER(XMM4, 158) +CV_REGISTER(XMM5, 159) +CV_REGISTER(XMM6, 160) +CV_REGISTER(XMM7, 161) -CV_REGISTER(CVRegEDXEAX, 212) +CV_REGISTER(MXCSR, 211) -CV_REGISTER(CVRegEMM0L, 220) -CV_REGISTER(CVRegEMM1L, 221) -CV_REGISTER(CVRegEMM2L, 222) -CV_REGISTER(CVRegEMM3L, 223) -CV_REGISTER(CVRegEMM4L, 224) -CV_REGISTER(CVRegEMM5L, 225) -CV_REGISTER(CVRegEMM6L, 226) -CV_REGISTER(CVRegEMM7L, 227) +CV_REGISTER(EDXEAX, 212) -CV_REGISTER(CVRegEMM0H, 228) -CV_REGISTER(CVRegEMM1H, 229) -CV_REGISTER(CVRegEMM2H, 230) -CV_REGISTER(CVRegEMM3H, 231) -CV_REGISTER(CVRegEMM4H, 232) -CV_REGISTER(CVRegEMM5H, 233) -CV_REGISTER(CVRegEMM6H, 234) -CV_REGISTER(CVRegEMM7H, 235) +CV_REGISTER(EMM0L, 220) +CV_REGISTER(EMM1L, 221) +CV_REGISTER(EMM2L, 222) +CV_REGISTER(EMM3L, 223) +CV_REGISTER(EMM4L, 224) +CV_REGISTER(EMM5L, 225) +CV_REGISTER(EMM6L, 226) +CV_REGISTER(EMM7L, 227) -CV_REGISTER(CVRegMM00, 236) -CV_REGISTER(CVRegMM01, 237) -CV_REGISTER(CVRegMM10, 238) -CV_REGISTER(CVRegMM11, 239) -CV_REGISTER(CVRegMM20, 240) -CV_REGISTER(CVRegMM21, 241) -CV_REGISTER(CVRegMM30, 242) -CV_REGISTER(CVRegMM31, 243) -CV_REGISTER(CVRegMM40, 244) -CV_REGISTER(CVRegMM41, 245) -CV_REGISTER(CVRegMM50, 246) -CV_REGISTER(CVRegMM51, 247) -CV_REGISTER(CVRegMM60, 248) -CV_REGISTER(CVRegMM61, 249) -CV_REGISTER(CVRegMM70, 250) -CV_REGISTER(CVRegMM71, 251) +CV_REGISTER(EMM0H, 228) +CV_REGISTER(EMM1H, 229) +CV_REGISTER(EMM2H, 230) +CV_REGISTER(EMM3H, 231) +CV_REGISTER(EMM4H, 232) +CV_REGISTER(EMM5H, 233) +CV_REGISTER(EMM6H, 234) +CV_REGISTER(EMM7H, 235) -CV_REGISTER(CVRegBND0, 396) -CV_REGISTER(CVRegBND1, 397) -CV_REGISTER(CVRegBND2, 398) +CV_REGISTER(MM00, 236) +CV_REGISTER(MM01, 237) +CV_REGISTER(MM10, 238) +CV_REGISTER(MM11, 239) +CV_REGISTER(MM20, 240) +CV_REGISTER(MM21, 241) +CV_REGISTER(MM30, 242) +CV_REGISTER(MM31, 243) +CV_REGISTER(MM40, 244) +CV_REGISTER(MM41, 245) +CV_REGISTER(MM50, 246) +CV_REGISTER(MM51, 247) +CV_REGISTER(MM60, 248) +CV_REGISTER(MM61, 249) +CV_REGISTER(MM70, 250) +CV_REGISTER(MM71, 251) + +CV_REGISTER(BND0, 396) +CV_REGISTER(BND1, 397) +CV_REGISTER(BND2, 398) -CV_REGISTER(CVRegXMM8, 252) -CV_REGISTER(CVRegXMM9, 253) -CV_REGISTER(CVRegXMM10, 254) -CV_REGISTER(CVRegXMM11, 255) -CV_REGISTER(CVRegXMM12, 256) -CV_REGISTER(CVRegXMM13, 257) -CV_REGISTER(CVRegXMM14, 258) -CV_REGISTER(CVRegXMM15, 259) +CV_REGISTER(XMM8, 252) +CV_REGISTER(XMM9, 253) +CV_REGISTER(XMM10, 254) +CV_REGISTER(XMM11, 255) +CV_REGISTER(XMM12, 256) +CV_REGISTER(XMM13, 257) +CV_REGISTER(XMM14, 258) +CV_REGISTER(XMM15, 259) -CV_REGISTER(CVRegSIL, 324) -CV_REGISTER(CVRegDIL, 325) -CV_REGISTER(CVRegBPL, 326) -CV_REGISTER(CVRegSPL, 327) +CV_REGISTER(SIL, 324) +CV_REGISTER(DIL, 325) +CV_REGISTER(BPL, 326) +CV_REGISTER(SPL, 327) -CV_REGISTER(CVRegRAX, 328) -CV_REGISTER(CVRegRBX, 329) -CV_REGISTER(CVRegRCX, 330) -CV_REGISTER(CVRegRDX, 331) -CV_REGISTER(CVRegRSI, 332) -CV_REGISTER(CVRegRDI, 333) -CV_REGISTER(CVRegRBP, 334) -CV_REGISTER(CVRegRSP, 335) +CV_REGISTER(RAX, 328) +CV_REGISTER(RBX, 329) +CV_REGISTER(RCX, 330) +CV_REGISTER(RDX, 331) +CV_REGISTER(RSI, 332) +CV_REGISTER(RDI, 333) +CV_REGISTER(RBP, 334) +CV_REGISTER(RSP, 335) -CV_REGISTER(CVRegR8, 336) -CV_REGISTER(CVRegR9, 337) -CV_REGISTER(CVRegR10, 338) -CV_REGISTER(CVRegR11, 339) -CV_REGISTER(CVRegR12, 340) -CV_REGISTER(CVRegR13, 341) -CV_REGISTER(CVRegR14, 342) -CV_REGISTER(CVRegR15, 343) +CV_REGISTER(R8, 336) +CV_REGISTER(R9, 337) +CV_REGISTER(R10, 338) +CV_REGISTER(R11, 339) +CV_REGISTER(R12, 340) +CV_REGISTER(R13, 341) +CV_REGISTER(R14, 342) +CV_REGISTER(R15, 343) -CV_REGISTER(CVRegR8B, 344) -CV_REGISTER(CVRegR9B, 345) -CV_REGISTER(CVRegR10B, 346) -CV_REGISTER(CVRegR11B, 347) -CV_REGISTER(CVRegR12B, 348) -CV_REGISTER(CVRegR13B, 349) -CV_REGISTER(CVRegR14B, 350) -CV_REGISTER(CVRegR15B, 351) +CV_REGISTER(R8B, 344) +CV_REGISTER(R9B, 345) +CV_REGISTER(R10B, 346) +CV_REGISTER(R11B, 347) +CV_REGISTER(R12B, 348) +CV_REGISTER(R13B, 349) +CV_REGISTER(R14B, 350) +CV_REGISTER(R15B, 351) -CV_REGISTER(CVRegR8W, 352) -CV_REGISTER(CVRegR9W, 353) -CV_REGISTER(CVRegR10W, 354) -CV_REGISTER(CVRegR11W, 355) -CV_REGISTER(CVRegR12W, 356) -CV_REGISTER(CVRegR13W, 357) -CV_REGISTER(CVRegR14W, 358) -CV_REGISTER(CVRegR15W, 359) +CV_REGISTER(R8W, 352) +CV_REGISTER(R9W, 353) +CV_REGISTER(R10W, 354) +CV_REGISTER(R11W, 355) +CV_REGISTER(R12W, 356) +CV_REGISTER(R13W, 357) +CV_REGISTER(R14W, 358) +CV_REGISTER(R15W, 359) -CV_REGISTER(CVRegR8D, 360) -CV_REGISTER(CVRegR9D, 361) -CV_REGISTER(CVRegR10D, 362) -CV_REGISTER(CVRegR11D, 363) -CV_REGISTER(CVRegR12D, 364) -CV_REGISTER(CVRegR13D, 365) -CV_REGISTER(CVRegR14D, 366) -CV_REGISTER(CVRegR15D, 367) +CV_REGISTER(R8D, 360) +CV_REGISTER(R9D, 361) +CV_REGISTER(R10D, 362) +CV_REGISTER(R11D, 363) +CV_REGISTER(R12D, 364) +CV_REGISTER(R13D, 365) +CV_REGISTER(R14D, 366) +CV_REGISTER(R15D, 367) // cvconst.h defines both CV_REG_YMM0 (252) and CV_AMD64_YMM0 (368). Keep the // original prefix to distinguish them. -CV_REGISTER(CVRegAMD64_YMM0, 368) -CV_REGISTER(CVRegAMD64_YMM1, 369) -CV_REGISTER(CVRegAMD64_YMM2, 370) -CV_REGISTER(CVRegAMD64_YMM3, 371) -CV_REGISTER(CVRegAMD64_YMM4, 372) -CV_REGISTER(CVRegAMD64_YMM5, 373) -CV_REGISTER(CVRegAMD64_YMM6, 374) -CV_REGISTER(CVRegAMD64_YMM7, 375) -CV_REGISTER(CVRegAMD64_YMM8, 376) -CV_REGISTER(CVRegAMD64_YMM9, 377) -CV_REGISTER(CVRegAMD64_YMM10, 378) -CV_REGISTER(CVRegAMD64_YMM11, 379) -CV_REGISTER(CVRegAMD64_YMM12, 380) -CV_REGISTER(CVRegAMD64_YMM13, 381) -CV_REGISTER(CVRegAMD64_YMM14, 382) -CV_REGISTER(CVRegAMD64_YMM15, 383) +CV_REGISTER(AMD64_YMM0, 368) +CV_REGISTER(AMD64_YMM1, 369) +CV_REGISTER(AMD64_YMM2, 370) +CV_REGISTER(AMD64_YMM3, 371) +CV_REGISTER(AMD64_YMM4, 372) +CV_REGISTER(AMD64_YMM5, 373) +CV_REGISTER(AMD64_YMM6, 374) +CV_REGISTER(AMD64_YMM7, 375) +CV_REGISTER(AMD64_YMM8, 376) +CV_REGISTER(AMD64_YMM9, 377) +CV_REGISTER(AMD64_YMM10, 378) +CV_REGISTER(AMD64_YMM11, 379) +CV_REGISTER(AMD64_YMM12, 380) +CV_REGISTER(AMD64_YMM13, 381) +CV_REGISTER(AMD64_YMM14, 382) +CV_REGISTER(AMD64_YMM15, 383) + +CV_REGISTER(AMD64_XMM16, 694) +CV_REGISTER(AMD64_XMM17, 695) +CV_REGISTER(AMD64_XMM18, 696) +CV_REGISTER(AMD64_XMM19, 697) +CV_REGISTER(AMD64_XMM20, 698) +CV_REGISTER(AMD64_XMM21, 699) +CV_REGISTER(AMD64_XMM22, 700) +CV_REGISTER(AMD64_XMM23, 701) +CV_REGISTER(AMD64_XMM24, 702) +CV_REGISTER(AMD64_XMM25, 703) +CV_REGISTER(AMD64_XMM26, 704) +CV_REGISTER(AMD64_XMM27, 705) +CV_REGISTER(AMD64_XMM28, 706) +CV_REGISTER(AMD64_XMM29, 707) +CV_REGISTER(AMD64_XMM30, 708) +CV_REGISTER(AMD64_XMM31, 709) + +CV_REGISTER(AMD64_YMM16, 710) +CV_REGISTER(AMD64_YMM17, 711) +CV_REGISTER(AMD64_YMM18, 712) +CV_REGISTER(AMD64_YMM19, 713) +CV_REGISTER(AMD64_YMM20, 714) +CV_REGISTER(AMD64_YMM21, 715) +CV_REGISTER(AMD64_YMM22, 716) +CV_REGISTER(AMD64_YMM23, 717) +CV_REGISTER(AMD64_YMM24, 718) +CV_REGISTER(AMD64_YMM25, 719) +CV_REGISTER(AMD64_YMM26, 720) +CV_REGISTER(AMD64_YMM27, 721) +CV_REGISTER(AMD64_YMM28, 722) +CV_REGISTER(AMD64_YMM29, 723) +CV_REGISTER(AMD64_YMM30, 724) +CV_REGISTER(AMD64_YMM31, 725) + +CV_REGISTER(AMD64_ZMM0, 726) +CV_REGISTER(AMD64_ZMM1, 727) +CV_REGISTER(AMD64_ZMM2, 728) +CV_REGISTER(AMD64_ZMM3, 729) +CV_REGISTER(AMD64_ZMM4, 730) +CV_REGISTER(AMD64_ZMM5, 731) +CV_REGISTER(AMD64_ZMM6, 732) +CV_REGISTER(AMD64_ZMM7, 733) +CV_REGISTER(AMD64_ZMM8, 734) +CV_REGISTER(AMD64_ZMM9, 735) +CV_REGISTER(AMD64_ZMM10, 736) +CV_REGISTER(AMD64_ZMM11, 737) +CV_REGISTER(AMD64_ZMM12, 738) +CV_REGISTER(AMD64_ZMM13, 739) +CV_REGISTER(AMD64_ZMM14, 740) +CV_REGISTER(AMD64_ZMM15, 741) +CV_REGISTER(AMD64_ZMM16, 742) +CV_REGISTER(AMD64_ZMM17, 743) +CV_REGISTER(AMD64_ZMM18, 744) +CV_REGISTER(AMD64_ZMM19, 745) +CV_REGISTER(AMD64_ZMM20, 746) +CV_REGISTER(AMD64_ZMM21, 747) +CV_REGISTER(AMD64_ZMM22, 748) +CV_REGISTER(AMD64_ZMM23, 749) +CV_REGISTER(AMD64_ZMM24, 750) +CV_REGISTER(AMD64_ZMM25, 751) +CV_REGISTER(AMD64_ZMM26, 752) +CV_REGISTER(AMD64_ZMM27, 753) +CV_REGISTER(AMD64_ZMM28, 754) +CV_REGISTER(AMD64_ZMM29, 755) +CV_REGISTER(AMD64_ZMM30, 756) +CV_REGISTER(AMD64_ZMM31, 757) + +CV_REGISTER(AMD64_K0, 758) +CV_REGISTER(AMD64_K1, 759) +CV_REGISTER(AMD64_K2, 760) +CV_REGISTER(AMD64_K3, 761) +CV_REGISTER(AMD64_K4, 762) +CV_REGISTER(AMD64_K5, 763) +CV_REGISTER(AMD64_K6, 764) +CV_REGISTER(AMD64_K7, 765) + +#pragma pop_macro("CR0") +#pragma pop_macro("CR1") +#pragma pop_macro("CR2") +#pragma pop_macro("CR3") +#pragma pop_macro("CR4") diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h index 1e329c7c3f1..847d93f0e98 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h @@ -13,6 +13,7 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" #include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" namespace llvm { @@ -26,21 +27,23 @@ public: } Error initialize(BinaryStreamReader Reader); + Error initialize(BinaryStreamRef Stream); FixedStreamArray::Iterator begin() const { return Frames.begin(); } FixedStreamArray::Iterator end() const { return Frames.end(); } - const void *getRelocPtr() const { return RelocPtr; } + const support::ulittle32_t *getRelocPtr() const { return RelocPtr; } private: - const uint32_t *RelocPtr = nullptr; + const support::ulittle32_t *RelocPtr = nullptr; FixedStreamArray Frames; }; class DebugFrameDataSubsection final : public DebugSubsection { public: - DebugFrameDataSubsection() - : DebugSubsection(DebugSubsectionKind::FrameData) {} + DebugFrameDataSubsection(bool IncludeRelocPtr) + : DebugSubsection(DebugSubsectionKind::FrameData), + IncludeRelocPtr(IncludeRelocPtr) {} static bool classof(const DebugSubsection *S) { return S->kind() == DebugSubsectionKind::FrameData; } @@ -52,6 +55,7 @@ public: void setFrames(ArrayRef Frames); private: + bool IncludeRelocPtr = false; std::vector Frames; }; } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h index 58449c2c756..36237e1a4d9 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h @@ -180,26 +180,6 @@ template serialize_numeric_impl serialize_numeric(T &Item) { return serialize_numeric_impl(Item); } -// This field is only present in the byte record if the condition is true. The -// condition is evaluated lazily, so it can depend on items that were -// deserialized -// earlier. -#define CV_CONDITIONAL_FIELD(I, C) \ - serialize_conditional(I, [&]() { return !!(C); }) - -// This is an array of N items, where N is evaluated lazily, so it can refer -// to a field deserialized earlier. -#define CV_ARRAY_FIELD_N(I, N) serialize_array(I, [&]() { return N; }) - -// This is an array that exhausts the remainder of the input buffer. -#define CV_ARRAY_FIELD_TAIL(I) serialize_array_tail(I) - -// This is an array that consumes null terminated strings until a double null -// is encountered. -#define CV_STRING_ARRAY_NULL_TERM(I) serialize_null_term_string_array(I) - -#define CV_NUMERIC_FIELD(I) serialize_numeric(I) - template Error consume(BinaryStreamReader &Reader, const serialize_conditional_impl &Item) { @@ -242,9 +222,6 @@ Error consume(BinaryStreamReader &Reader, T &&X, U &&Y, Args &&... Rest) { return consume(Reader, Y, std::forward(Rest)...); } -#define CV_DESERIALIZE(...) \ - if (auto EC = consume(__VA_ARGS__)) \ - return std::move(EC); } } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h index b5479db97a1..6b5dd2d20d1 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h @@ -47,7 +47,7 @@ public: return Error::success(); } template static Expected deserializeAs(CVSymbol Symbol) { - T Record(Symbol.kind()); + T Record(static_cast(Symbol.kind())); if (auto EC = deserializeAs(Symbol, Record)) return std::move(EC); return Record; diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h index 293daa851bd..215da2e2b52 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h @@ -27,10 +27,10 @@ class CVSymbolDumper { public: CVSymbolDumper(ScopedPrinter &W, TypeCollection &Types, CodeViewContainer Container, - std::unique_ptr ObjDelegate, + std::unique_ptr ObjDelegate, CPUType CPU, bool PrintRecordBytes) : W(W), Types(Types), Container(Container), - ObjDelegate(std::move(ObjDelegate)), + ObjDelegate(std::move(ObjDelegate)), CompilationCPUType(CPU), PrintRecordBytes(PrintRecordBytes) {} /// Dumps one type record. Returns false if there was a type parsing error, @@ -43,12 +43,14 @@ public: /// parse error, and true otherwise. Error dump(const CVSymbolArray &Symbols); + CPUType getCompilationCPUType() const { return CompilationCPUType; } + private: ScopedPrinter &W; TypeCollection &Types; CodeViewContainer Container; std::unique_ptr ObjDelegate; - + CPUType CompilationCPUType; bool PrintRecordBytes; }; } // end namespace codeview diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h index 93306824012..b58825c4a78 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -358,6 +358,7 @@ public: // S_PUB32 class PublicSym32 : public SymbolRecord { public: + PublicSym32() : SymbolRecord(SymbolRecordKind::PublicSym32) {} explicit PublicSym32(SymbolRecordKind Kind) : SymbolRecord(Kind) {} explicit PublicSym32(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::PublicSym32), @@ -399,6 +400,7 @@ public: uint16_t Module; StringRef Name; + uint16_t modi() const { return Module - 1; } uint32_t RecordOffset; }; @@ -636,6 +638,7 @@ public: // S_OBJNAME class ObjNameSym : public SymbolRecord { public: + explicit ObjNameSym() : SymbolRecord(SymbolRecordKind::ObjNameSym) {} explicit ObjNameSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} ObjNameSym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::ObjNameSym), RecordOffset(RecordOffset) { @@ -718,6 +721,7 @@ public: // S_COMPILE3 class Compile3Sym : public SymbolRecord { public: + Compile3Sym() : SymbolRecord(SymbolRecordKind::Compile3Sym) {} explicit Compile3Sym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} Compile3Sym(uint32_t RecordOffset) : SymbolRecord(SymbolRecordKind::Compile3Sym), @@ -739,8 +743,17 @@ public: Flags = CompileSym3Flags((uint32_t(Flags) & 0xFFFFFF00) | uint32_t(Lang)); } - uint8_t getLanguage() const { return static_cast(Flags) & 0xFF; } - uint32_t getFlags() const { return static_cast(Flags) & ~0xFF; } + SourceLanguage getLanguage() const { + return static_cast(static_cast(Flags) & 0xFF); + } + CompileSym3Flags getFlags() const { + return static_cast(static_cast(Flags) & ~0xFF); + } + + bool hasOptimizations() const { + return CompileSym3Flags::None != + (getFlags() & (CompileSym3Flags::PGO | CompileSym3Flags::LTCG)); + } uint32_t RecordOffset; }; @@ -761,7 +774,21 @@ public: uint16_t SectionIdOfExceptionHandler; FrameProcedureOptions Flags; + /// Extract the register this frame uses to refer to local variables. + RegisterId getLocalFramePtrReg(CPUType CPU) const { + return decodeFramePtrReg( + EncodedFramePtrReg((uint32_t(Flags) >> 14U) & 0x3U), CPU); + } + + /// Extract the register this frame uses to refer to parameters. + RegisterId getParamFramePtrReg(CPUType CPU) const { + return decodeFramePtrReg( + EncodedFramePtrReg((uint32_t(Flags) >> 16U) & 0x3U), CPU); + } + uint32_t RecordOffset; + +private: }; // S_CALLSITEINFO diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h new file mode 100644 index 00000000000..3713fe118ea --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h @@ -0,0 +1,62 @@ +//===- SymbolRecordHelpers.h ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_SYMBOLRECORDHELPERS_H +#define LLVM_DEBUGINFO_CODEVIEW_SYMBOLRECORDHELPERS_H + +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" + +namespace llvm { +namespace codeview { +/// Return true if this symbol opens a scope. This implies that the symbol has +/// "parent" and "end" fields, which contain the offset of the S_END or +/// S_INLINESITE_END record. +inline bool symbolOpensScope(SymbolKind Kind) { + switch (Kind) { + case SymbolKind::S_GPROC32: + case SymbolKind::S_LPROC32: + case SymbolKind::S_LPROC32_ID: + case SymbolKind::S_GPROC32_ID: + case SymbolKind::S_BLOCK32: + case SymbolKind::S_SEPCODE: + case SymbolKind::S_THUNK32: + case SymbolKind::S_INLINESITE: + case SymbolKind::S_INLINESITE2: + return true; + default: + break; + } + return false; +} + +/// Return true if this ssymbol ends a scope. +inline bool symbolEndsScope(SymbolKind Kind) { + switch (Kind) { + case SymbolKind::S_END: + case SymbolKind::S_PROC_ID_END: + case SymbolKind::S_INLINESITE_END: + return true; + default: + break; + } + return false; +} + +/// Given a symbol P for which symbolOpensScope(P) == true, return the +/// corresponding end offset. +uint32_t getScopeEndOffset(const CVSymbol &Symbol); +uint32_t getScopeParentOffset(const CVSymbol &Symbol); + +CVSymbolArray limitSymbolArrayToScope(const CVSymbolArray &Symbols, + uint32_t ScopeBegin); + +} // namespace codeview +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h index c71281de714..58463a6b13d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -134,6 +134,8 @@ public: return static_cast(Index & SimpleModeMask); } + TypeIndex makeDirect() const { return TypeIndex{getSimpleKind()}; } + static TypeIndex None() { return TypeIndex(SimpleTypeKind::None); } static TypeIndex Void() { return TypeIndex(SimpleTypeKind::Void); } static TypeIndex VoidPointer32() { @@ -143,6 +145,13 @@ public: return TypeIndex(SimpleTypeKind::Void, SimpleTypeMode::NearPointer64); } + static TypeIndex NullptrT() { + // std::nullptr_t uses the pointer mode that doesn't indicate bit-width, + // presumably because std::nullptr_t is intended to be compatible with any + // pointer type. + return TypeIndex(SimpleTypeKind::Void, SimpleTypeMode::NearPointer); + } + static TypeIndex SignedCharacter() { return TypeIndex(SimpleTypeKind::SignedCharacter); } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h index 61ebdf878ce..7b4a30ee622 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -95,6 +95,11 @@ struct MemberAttributes { return MP == MethodKind::IntroducingVirtual || MP == MethodKind::PureIntroducingVirtual; } + + /// Is this method static. + bool isStatic() const { + return getMethodKind() == MethodKind::Static; + } }; // Does not correspond to any tag, this is the tail of an LF_POINTER record @@ -264,14 +269,18 @@ public: // LF_POINTER class PointerRecord : public TypeRecord { public: + // ---------------------------XXXXX static const uint32_t PointerKindShift = 0; static const uint32_t PointerKindMask = 0x1F; + // ------------------------XXX----- static const uint32_t PointerModeShift = 5; static const uint32_t PointerModeMask = 0x07; - static const uint32_t PointerOptionMask = 0xFF; + // ----------XXX------XXXXX-------- + static const uint32_t PointerOptionMask = 0x381f00; + // -------------XXXXXX------------ static const uint32_t PointerSizeShift = 13; static const uint32_t PointerSizeMask = 0xFF; @@ -305,7 +314,7 @@ public: } PointerOptions getOptions() const { - return static_cast(Attrs); + return static_cast(Attrs & PointerOptionMask); } uint8_t getSize() const { @@ -334,6 +343,14 @@ public: return !!(Attrs & uint32_t(PointerOptions::Restrict)); } + bool isLValueReferenceThisPtr() const { + return !!(Attrs & uint32_t(PointerOptions::LValueRefThisPointer)); + } + + bool isRValueReferenceThisPtr() const { + return !!(Attrs & uint32_t(PointerOptions::RValueRefThisPointer)); + } + TypeIndex ReferentType; uint32_t Attrs; Optional MemberInfo; @@ -429,6 +446,14 @@ public: return (Options & ClassOptions::ForwardReference) != ClassOptions::None; } + bool containsNestedClass() const { + return (Options & ClassOptions::ContainsNestedClass) != ClassOptions::None; + } + + bool isScoped() const { + return (Options & ClassOptions::Scoped) != ClassOptions::None; + } + uint16_t getMemberCount() const { return MemberCount; } ClassOptions getOptions() const { return Options; } TypeIndex getFieldList() const { return FieldList; } @@ -655,7 +680,17 @@ public: ArrayRef getArgs() const { return ArgIndices; } - SmallVector ArgIndices; + /// Indices of known build info arguments. + enum BuildInfoArg { + CurrentDirectory, ///< Absolute CWD path + BuildTool, ///< Absolute compiler path + SourceFile, ///< Path to main source file, relative or absolute + TypeServerPDB, ///< Absolute path of type server PDB (/Fd) + CommandLine, ///< Full canonical command line (maybe -cc1) + MaxArgs + }; + + SmallVector ArgIndices; }; // LF_VFTABLE @@ -923,6 +958,7 @@ public: uint32_t Signature; }; + } // end namespace codeview } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h new file mode 100644 index 00000000000..389472ed1ae --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h @@ -0,0 +1,28 @@ +//===- TypeRecordHelpers.h --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPERECORDHELPERS_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPERECORDHELPERS_H + +#include "llvm/DebugInfo/CodeView/TypeRecord.h" + +namespace llvm { + namespace codeview { + /// Given an arbitrary codeview type, determine if it is an LF_STRUCTURE, + /// LF_CLASS, LF_INTERFACE, LF_UNION, or LF_ENUM with the forward ref class + /// option. + bool isUdtForwardRef(CVType CVT); + + /// Given a CVType which is assumed to be an LF_MODIFIER, return the + /// TypeIndex of the type that the LF_MODIFIER modifies. + TypeIndex getModifiedType(const CVType &CVT); + } +} + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h index 583740d2eb4..0b9f54ec60b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h @@ -83,18 +83,21 @@ Error mergeIdRecords(MergingTypeTableBuilder &Dest, ArrayRef Types, Error mergeTypeAndIdRecords(MergingTypeTableBuilder &DestIds, MergingTypeTableBuilder &DestTypes, SmallVectorImpl &SourceToDest, - const CVTypeArray &IdsAndTypes); + const CVTypeArray &IdsAndTypes, + Optional &PCHSignature); Error mergeTypeAndIdRecords(GlobalTypeTableBuilder &DestIds, GlobalTypeTableBuilder &DestTypes, SmallVectorImpl &SourceToDest, const CVTypeArray &IdsAndTypes, - ArrayRef Hashes); + ArrayRef Hashes, + Optional &PCHSignature); Error mergeTypeRecords(GlobalTypeTableBuilder &Dest, SmallVectorImpl &SourceToDest, const CVTypeArray &Types, - ArrayRef Hashes); + ArrayRef Hashes, + Optional &PCHSignature); Error mergeIdRecords(GlobalTypeTableBuilder &Dest, ArrayRef Types, SmallVectorImpl &SourceToDest, diff --git a/contrib/llvm/include/llvm/DebugInfo/DIContext.h b/contrib/llvm/include/llvm/DebugInfo/DIContext.h index bbdd5e0d9c3..85e96402a24 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DIContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DIContext.h @@ -81,7 +81,7 @@ class DIInliningInfo { public: DIInliningInfo() = default; - DILineInfo getFrame(unsigned Index) const { + const DILineInfo & getFrame(unsigned Index) const { assert(Index < Frames.size()); return Frames[Index]; } @@ -98,6 +98,11 @@ public: void addFrame(const DILineInfo &Frame) { Frames.push_back(Frame); } + + void resize(unsigned i) { + Frames.resize(i); + } + }; /// Container for description of a global variable. diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h index c219ca75e64..33797419a7b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h @@ -18,20 +18,20 @@ namespace llvm { class DWARFCompileUnit : public DWARFUnit { public: DWARFCompileUnit(DWARFContext &Context, const DWARFSection &Section, - const DWARFUnitHeader &Header, - const DWARFDebugAbbrev *DA, const DWARFSection *RS, + const DWARFUnitHeader &Header, const DWARFDebugAbbrev *DA, + const DWARFSection *RS, const DWARFSection *LocSection, StringRef SS, const DWARFSection &SOS, const DWARFSection *AOS, const DWARFSection &LS, bool LE, - bool IsDWO, const DWARFUnitSectionBase &UnitSection) - : DWARFUnit(Context, Section, Header, DA, RS, SS, SOS, AOS, LS, LE, IsDWO, - UnitSection) {} + bool IsDWO, const DWARFUnitVector &UnitVector) + : DWARFUnit(Context, Section, Header, DA, RS, LocSection, SS, SOS, AOS, + LS, LE, IsDWO, UnitVector) {} - // VTable anchor. + /// VTable anchor. ~DWARFCompileUnit() override; - - void dump(raw_ostream &OS, DIDumpOptions DumpOpts); - - static const DWARFSectionKind Section = DW_SECT_INFO; + /// Dump this compile unit to \p OS. + void dump(raw_ostream &OS, DIDumpOptions DumpOpts) override; + /// Enable LLVM-style RTTI. + static bool classof(const DWARFUnit *U) { return !U->isTypeUnit(); } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h index f5419fe0242..dbb6be04544 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -57,8 +57,7 @@ enum class ErrorPolicy { Halt, Continue }; /// This data structure is the top level entity that deals with dwarf debug /// information parsing. The actual data is supplied through DWARFObj. class DWARFContext : public DIContext { - DWARFUnitSection CUs; - std::deque> TUs; + DWARFUnitVector NormalUnits; std::unique_ptr CUIndex; std::unique_ptr GdbIndex; std::unique_ptr TUIndex; @@ -75,10 +74,9 @@ class DWARFContext : public DIContext { std::unique_ptr AppleNamespaces; std::unique_ptr AppleObjC; - DWARFUnitSection DWOCUs; - std::deque> DWOTUs; + DWARFUnitVector DWOUnits; std::unique_ptr AbbrevDWO; - std::unique_ptr LocDWO; + std::unique_ptr LocDWO; /// The maximum DWARF version of all units. unsigned MaxVersion = 0; @@ -95,22 +93,17 @@ class DWARFContext : public DIContext { std::unique_ptr RegInfo; /// Read compile units from the debug_info section (if necessary) - /// and store them in CUs. - void parseCompileUnits(); - - /// Read type units from the debug_types sections (if necessary) - /// and store them in TUs. - void parseTypeUnits(); + /// and type units from the debug_types sections (if necessary) + /// and store them in NormalUnits. + void parseNormalUnits(); /// Read compile units from the debug_info.dwo section (if necessary) - /// and store them in DWOCUs. - void parseDWOCompileUnits(); + /// and type units from the debug_types.dwo section (if necessary) + /// and store them in DWOUnits. + /// If \p Lazy is true, set up to parse but don't actually parse them. + enum { EagerParse = false, LazyParse = true }; + void parseDWOUnits(bool Lazy = false); - /// Read type units from the debug_types.dwo section (if necessary) - /// and store them in DWOTUs. - void parseDWOTypeUnits(); - -protected: std::unique_ptr DObj; public: @@ -139,68 +132,95 @@ public: bool verify(raw_ostream &OS, DIDumpOptions DumpOpts = {}) override; - using cu_iterator_range = DWARFUnitSection::iterator_range; - using tu_iterator_range = DWARFUnitSection::iterator_range; - using tu_section_iterator_range = iterator_range; + using unit_iterator_range = DWARFUnitVector::iterator_range; - /// Get compile units in this context. - cu_iterator_range compile_units() { - parseCompileUnits(); - return cu_iterator_range(CUs.begin(), CUs.end()); + /// Get units from .debug_info in this context. + unit_iterator_range info_section_units() { + parseNormalUnits(); + return unit_iterator_range(NormalUnits.begin(), + NormalUnits.begin() + + NormalUnits.getNumInfoUnits()); } + /// Get units from .debug_types in this context. + unit_iterator_range types_section_units() { + parseNormalUnits(); + return unit_iterator_range( + NormalUnits.begin() + NormalUnits.getNumInfoUnits(), NormalUnits.end()); + } + + /// Get compile units in this context. + unit_iterator_range compile_units() { return info_section_units(); } + /// Get type units in this context. - tu_section_iterator_range type_unit_sections() { - parseTypeUnits(); - return tu_section_iterator_range(TUs.begin(), TUs.end()); + unit_iterator_range type_units() { return types_section_units(); } + + /// Get all normal compile/type units in this context. + unit_iterator_range normal_units() { + parseNormalUnits(); + return unit_iterator_range(NormalUnits.begin(), NormalUnits.end()); + } + + /// Get units from .debug_info..dwo in the DWO context. + unit_iterator_range dwo_info_section_units() { + parseDWOUnits(); + return unit_iterator_range(DWOUnits.begin(), + DWOUnits.begin() + DWOUnits.getNumInfoUnits()); + } + + /// Get units from .debug_types.dwo in the DWO context. + unit_iterator_range dwo_types_section_units() { + parseDWOUnits(); + return unit_iterator_range(DWOUnits.begin() + DWOUnits.getNumInfoUnits(), + DWOUnits.end()); } /// Get compile units in the DWO context. - cu_iterator_range dwo_compile_units() { - parseDWOCompileUnits(); - return cu_iterator_range(DWOCUs.begin(), DWOCUs.end()); - } + unit_iterator_range dwo_compile_units() { return dwo_info_section_units(); } /// Get type units in the DWO context. - tu_section_iterator_range dwo_type_unit_sections() { - parseDWOTypeUnits(); - return tu_section_iterator_range(DWOTUs.begin(), DWOTUs.end()); + unit_iterator_range dwo_type_units() { return dwo_types_section_units(); } + + /// Get all units in the DWO context. + unit_iterator_range dwo_units() { + parseDWOUnits(); + return unit_iterator_range(DWOUnits.begin(), DWOUnits.end()); } /// Get the number of compile units in this context. unsigned getNumCompileUnits() { - parseCompileUnits(); - return CUs.size(); + parseNormalUnits(); + return NormalUnits.getNumInfoUnits(); } - /// Get the number of compile units in this context. + /// Get the number of type units in this context. unsigned getNumTypeUnits() { - parseTypeUnits(); - return TUs.size(); + parseNormalUnits(); + return NormalUnits.getNumTypesUnits(); } /// Get the number of compile units in the DWO context. unsigned getNumDWOCompileUnits() { - parseDWOCompileUnits(); - return DWOCUs.size(); + parseDWOUnits(); + return DWOUnits.getNumInfoUnits(); } - /// Get the number of compile units in the DWO context. + /// Get the number of type units in the DWO context. unsigned getNumDWOTypeUnits() { - parseDWOTypeUnits(); - return DWOTUs.size(); + parseDWOUnits(); + return DWOUnits.getNumTypesUnits(); } - /// Get the compile unit at the specified index for this compile unit. - DWARFCompileUnit *getCompileUnitAtIndex(unsigned index) { - parseCompileUnits(); - return CUs[index].get(); + /// Get the unit at the specified index. + DWARFUnit *getUnitAtIndex(unsigned index) { + parseNormalUnits(); + return NormalUnits[index].get(); } - /// Get the compile unit at the specified index for the DWO compile units. - DWARFCompileUnit *getDWOCompileUnitAtIndex(unsigned index) { - parseDWOCompileUnits(); - return DWOCUs[index].get(); + /// Get the unit at the specified index for the DWO units. + DWARFUnit *getDWOUnitAtIndex(unsigned index) { + parseDWOUnits(); + return DWOUnits[index].get(); } DWARFCompileUnit *getDWOCompileUnitForHash(uint64_t Hash); @@ -211,7 +231,17 @@ public: /// Get a DIE given an exact offset. DWARFDie getDIEForOffset(uint32_t Offset); - unsigned getMaxVersion() const { return MaxVersion; } + unsigned getMaxVersion() { + // Ensure info units have been parsed to discover MaxVersion + info_section_units(); + return MaxVersion; + } + + unsigned getMaxDWOVersion() { + // Ensure DWO info units have been parsed to discover MaxVersion + dwo_info_section_units(); + return MaxVersion; + } void setMaxVersionIfGreater(unsigned Version) { if (Version > MaxVersion) @@ -232,7 +262,7 @@ public: const DWARFDebugAbbrev *getDebugAbbrevDWO(); /// Get a pointer to the parsed DebugLoc object. - const DWARFDebugLocDWO *getDebugLocDWO(); + const DWARFDebugLoclists *getDebugLocDWO(); /// Get a pointer to the parsed DebugAranges object. const DWARFDebugAranges *getDebugAranges(); @@ -327,6 +357,13 @@ public: /// TODO: refactor compile_units() to make this const. uint8_t getCUAddrSize(); + /// Dump Error as warning message to stderr. + static void dumpWarning(Error Warning); + + Triple::ArchType getArch() const { + return getDWARFObj().getFile()->getArch(); + } + private: /// Return the compile unit which contains instruction with provided /// address. diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h index ff1c7fb3838..7dc07d774ab 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h @@ -13,6 +13,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Triple.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFExpression.h" #include "llvm/Support/Error.h" @@ -59,9 +60,11 @@ public: unsigned size() const { return (unsigned)Instructions.size(); } bool empty() const { return Instructions.empty(); } - CFIProgram(uint64_t CodeAlignmentFactor, int64_t DataAlignmentFactor) + CFIProgram(uint64_t CodeAlignmentFactor, int64_t DataAlignmentFactor, + Triple::ArchType Arch) : CodeAlignmentFactor(CodeAlignmentFactor), - DataAlignmentFactor(DataAlignmentFactor) {} + DataAlignmentFactor(DataAlignmentFactor), + Arch(Arch) {} /// Parse and store a sequence of CFI instructions from Data, /// starting at *Offset and ending at EndOffset. *Offset is updated @@ -76,6 +79,7 @@ private: std::vector Instructions; const uint64_t CodeAlignmentFactor; const int64_t DataAlignmentFactor; + Triple::ArchType Arch; /// Convenience method to add a new instruction with the given opcode. void addInstruction(uint8_t Opcode) { @@ -130,8 +134,9 @@ public: enum FrameKind { FK_CIE, FK_FDE }; FrameEntry(FrameKind K, uint64_t Offset, uint64_t Length, uint64_t CodeAlign, - int64_t DataAlign) - : Kind(K), Offset(Offset), Length(Length), CFIs(CodeAlign, DataAlign) {} + int64_t DataAlign, Triple::ArchType Arch) + : Kind(K), Offset(Offset), Length(Length), + CFIs(CodeAlign, DataAlign, Arch) {} virtual ~FrameEntry() {} @@ -168,9 +173,9 @@ public: int64_t DataAlignmentFactor, uint64_t ReturnAddressRegister, SmallString<8> AugmentationData, uint32_t FDEPointerEncoding, uint32_t LSDAPointerEncoding, Optional Personality, - Optional PersonalityEnc) + Optional PersonalityEnc, Triple::ArchType Arch) : FrameEntry(FK_CIE, Offset, Length, CodeAlignmentFactor, - DataAlignmentFactor), + DataAlignmentFactor, Arch), Version(Version), Augmentation(std::move(Augmentation)), AddressSize(AddressSize), SegmentDescriptorSize(SegmentDescriptorSize), CodeAlignmentFactor(CodeAlignmentFactor), @@ -224,10 +229,11 @@ public: // is obtained lazily once it's actually required. FDE(uint64_t Offset, uint64_t Length, int64_t LinkedCIEOffset, uint64_t InitialLocation, uint64_t AddressRange, CIE *Cie, - Optional LSDAAddress) + Optional LSDAAddress, Triple::ArchType Arch) : FrameEntry(FK_FDE, Offset, Length, Cie ? Cie->getCodeAlignmentFactor() : 0, - Cie ? Cie->getDataAlignmentFactor() : 0), + Cie ? Cie->getDataAlignmentFactor() : 0, + Arch), LinkedCIEOffset(LinkedCIEOffset), InitialLocation(InitialLocation), AddressRange(AddressRange), LinkedCIE(Cie), LSDAAddress(LSDAAddress) {} @@ -256,6 +262,7 @@ private: /// A parsed .debug_frame or .eh_frame section class DWARFDebugFrame { + const Triple::ArchType Arch; // True if this is parsing an eh_frame section. const bool IsEH; // Not zero for sane pointer values coming out of eh_frame @@ -272,7 +279,8 @@ public: // it is a .debug_frame section. EHFrameAddress should be different // than zero for correct parsing of .eh_frame addresses when they // use a PC-relative encoding. - DWARFDebugFrame(bool IsEH = false, uint64_t EHFrameAddress = 0); + DWARFDebugFrame(Triple::ArchType Arch, + bool IsEH = false, uint64_t EHFrameAddress = 0); ~DWARFDebugFrame(); /// Dump the section data into the given stream. diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index 5b2af34bbcf..d50af5a057f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -247,10 +247,11 @@ public: void clear(); /// Parse prologue and all rows. - Error parse(DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, - const DWARFContext &Ctx, const DWARFUnit *U, - std::function RecoverableErrorCallback = warn, - raw_ostream *OS = nullptr); + Error parse( + DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, + const DWARFContext &Ctx, const DWARFUnit *U, + std::function RecoverableErrorCallback, + raw_ostream *OS = nullptr); using RowVector = std::vector; using RowIter = RowVector::const_iterator; @@ -273,14 +274,13 @@ public: Expected getOrParseLineTable( DWARFDataExtractor &DebugLineData, uint32_t Offset, const DWARFContext &Ctx, const DWARFUnit *U, - std::function RecoverableErrorCallback = warn); + std::function RecoverableErrorCallback); /// Helper to allow for parsing of an entire .debug_line section in sequence. class SectionParser { public: - using cu_range = DWARFUnitSection::iterator_range; - using tu_range = - iterator_range>::iterator>; + using cu_range = DWARFUnitVector::iterator_range; + using tu_range = DWARFUnitVector::iterator_range; using LineToUnitMap = std::map; SectionParser(DWARFDataExtractor &Data, const DWARFContext &C, cu_range CUs, @@ -296,16 +296,17 @@ public: /// \param OS - if not null, the parser will print information about the /// table as it parses it. LineTable - parseNext(function_ref RecoverableErrorCallback = warn, - function_ref UnrecoverableErrorCallback = warn, - raw_ostream *OS = nullptr); + parseNext( + function_ref RecoverableErrorCallback, + function_ref UnrecoverableErrorCallback, + raw_ostream *OS = nullptr); /// Skip the current line table and go to the following line table (if /// present) immediately. /// /// \param ErrorCallback - report any prologue parsing issues via this /// callback. - void skip(function_ref ErrorCallback = warn); + void skip(function_ref ErrorCallback); /// Indicates if the parser has parsed as much as possible. /// @@ -328,12 +329,6 @@ public: bool Done = false; }; - /// Helper function for DWARFDebugLine parse functions, to report issues - /// identified during parsing. - /// - /// \param Err The Error to report. - static void warn(Error Err); - private: struct ParsingState { ParsingState(struct LineTable *LT); diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h index 9a73745fb6b..da2098e1540 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h @@ -73,19 +73,21 @@ public: uint32_t *Offset); }; -class DWARFDebugLocDWO { +class DWARFDebugLoclists { public: struct Entry { - uint64_t Start; - uint32_t Length; + uint8_t Kind; + uint64_t Value0; + uint64_t Value1; SmallVector Loc; }; struct LocationList { unsigned Offset; SmallVector Entries; - void dump(raw_ostream &OS, bool IsLittleEndian, unsigned AddressSize, - const MCRegisterInfo *RegInfo, unsigned Indent) const; + void dump(raw_ostream &OS, uint64_t BaseAddr, bool IsLittleEndian, + unsigned AddressSize, const MCRegisterInfo *RegInfo, + unsigned Indent) const; }; private: @@ -98,15 +100,15 @@ private: bool IsLittleEndian; public: - void parse(DataExtractor data); - void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, + void parse(DataExtractor data, unsigned Version); + void dump(raw_ostream &OS, uint64_t BaseAddr, const MCRegisterInfo *RegInfo, Optional Offset) const; /// Return the location list at the given offset or nullptr. LocationList const *getLocationListAtOffset(uint64_t Offset) const; - static Optional parseOneLocationList(DataExtractor Data, - uint32_t *Offset); + static Optional + parseOneLocationList(DataExtractor Data, unsigned *Offset, unsigned Version); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h index cae4804e61d..9e1656eb161 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h @@ -13,6 +13,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/DebugInfo/DWARF/DWARFObject.h" #include #include @@ -67,7 +68,8 @@ private: bool GnuStyle; public: - DWARFDebugPubTable(StringRef Data, bool LittleEndian, bool GnuStyle); + DWARFDebugPubTable(const DWARFObject &Obj, const DWARFSection &Sec, + bool LittleEndian, bool GnuStyle); void dump(raw_ostream &OS) const; diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h index ce7436d9faa..bc26edf0064 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h @@ -18,7 +18,6 @@ namespace llvm { -struct BaseAddress; class raw_ostream; class DWARFDebugRangeList { @@ -78,7 +77,7 @@ public: /// list. Has to be passed base address of the compile unit referencing this /// range list. DWARFAddressRangesVector - getAbsoluteRanges(llvm::Optional BaseAddr) const; + getAbsoluteRanges(llvm::Optional BaseAddr) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h index e2e8ab5ed21..5cc8d789e59 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h @@ -10,6 +10,7 @@ #ifndef LLVM_DEBUGINFO_DWARFDEBUGRNGLISTS_H #define LLVM_DEBUGINFO_DWARFDEBUGRNGLISTS_H +#include "llvm/ADT/Optional.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" @@ -23,6 +24,7 @@ namespace llvm { class Error; class raw_ostream; +class DWARFUnit; /// A class representing a single range list entry. struct RangeListEntry : public DWARFListEntryBase { @@ -35,7 +37,9 @@ struct RangeListEntry : public DWARFListEntryBase { Error extract(DWARFDataExtractor Data, uint32_t End, uint32_t *OffsetPtr); void dump(raw_ostream &OS, uint8_t AddrSize, uint8_t MaxEncodingStringLength, - uint64_t &CurrentBase, DIDumpOptions DumpOpts) const; + uint64_t &CurrentBase, DIDumpOptions DumpOpts, + llvm::function_ref(uint32_t)> + LookupPooledAddress) const; bool isSentinel() const { return EntryKind == dwarf::DW_RLE_end_of_list; } }; @@ -44,7 +48,8 @@ class DWARFDebugRnglist : public DWARFListType { public: /// Build a DWARFAddressRangesVector from a rangelist. DWARFAddressRangesVector - getAbsoluteRanges(llvm::Optional BaseAddr) const; + getAbsoluteRanges(llvm::Optional BaseAddr, + DWARFUnit &U) const; }; class DWARFDebugRnglistTable : public DWARFListTableBase { diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h index c77034f6348..56d46cd739a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -180,6 +180,7 @@ public: /// \returns a valid DWARFDie instance if the attribute exists, or an invalid /// DWARFDie object if it doesn't. DWARFDie getAttributeValueAsReferencedDie(dwarf::Attribute Attr) const; + DWARFDie getAttributeValueAsReferencedDie(const DWARFFormValue &V) const; /// Extract the range base attribute from this DIE as absolute section offset. /// @@ -404,6 +405,10 @@ public: Die = Die.getPreviousSibling(); } + llvm::DWARFDie::iterator base() const { + return llvm::DWARFDie::iterator(AtEnd ? Die : Die.getSibling()); + } + reverse_iterator &operator++() { assert(!AtEnd && "Incrementing rend"); llvm::DWARFDie D = Die.getPreviousSibling(); diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h index 1b5f71c946f..727e853c09f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h @@ -61,7 +61,6 @@ public: dwarf::Form getForm() const { return Form; } uint64_t getRawUValue() const { return Value.uval; } - uint64_t getSectionIndex() const { return Value.SectionIndex; } void setForm(dwarf::Form F) { Form = F; } void setUValue(uint64_t V) { Value.uval = V; } void setSValue(int64_t V) { Value.sval = V; } @@ -75,6 +74,10 @@ public: bool isFormClass(FormClass FC) const; const DWARFUnit *getUnit() const { return U; } void dump(raw_ostream &OS, DIDumpOptions DumpOpts = DIDumpOptions()) const; + void dumpSectionedAddress(raw_ostream &OS, DIDumpOptions DumpOpts, + SectionedAddress SA) const; + static void dumpAddressSection(const DWARFObject &Obj, raw_ostream &OS, + DIDumpOptions DumpOpts, uint64_t SectionIndex); /// Extracts a value in \p Data at offset \p *OffsetPtr. The information /// in \p FormParams is needed to interpret some forms. The optional @@ -101,6 +104,7 @@ public: Optional getAsSignedConstant() const; Optional getAsCString() const; Optional getAsAddress() const; + Optional getAsSectionedAddress() const; Optional getAsSectionOffset() const; Optional> getAsBlock() const; Optional getAsCStringOffset() const; @@ -238,6 +242,13 @@ inline Optional toAddress(const Optional &V) { return None; } +inline Optional +toSectionedAddress(const Optional &V) { + if (V) + return V->getAsSectionedAddress(); + return None; +} + /// Take an optional DWARFFormValue and extract a address. /// /// \param V and optional DWARFFormValue to attempt to extract the value from. diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h index 8d1ac5c83c2..073e02903c3 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h @@ -24,6 +24,7 @@ class DWARFGdbIndex { uint32_t Version; uint32_t CuListOffset; + uint32_t TuListOffset; uint32_t AddressAreaOffset; uint32_t SymbolTableOffset; uint32_t ConstantPoolOffset; @@ -34,6 +35,13 @@ class DWARFGdbIndex { }; SmallVector CuList; + struct TypeUnitEntry { + uint64_t Offset; + uint64_t TypeOffset; + uint64_t TypeSignature; + }; + SmallVector TuList; + struct AddressEntry { uint64_t LowAddress; /// The low address. uint64_t HighAddress; /// The high address. @@ -55,6 +63,7 @@ class DWARFGdbIndex { uint32_t StringPoolOffset; void dumpCUList(raw_ostream &OS) const; + void dumpTUList(raw_ostream &OS) const; void dumpAddressArea(raw_ostream &OS) const; void dumpSymbolTable(raw_ostream &OS) const; void dumpConstantPool(raw_ostream &OS) const; diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h index ab12f3bc08b..9b987314f20 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h @@ -13,6 +13,7 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" +#include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" @@ -43,10 +44,6 @@ protected: ListEntries Entries; public: - // FIXME: We need to consolidate the various verions of "createError" - // that are used in the DWARF consumer. Until then, this is a workaround. - Error createError(const char *, const char *, uint32_t); - const ListEntries &getEntries() const { return Entries; } bool empty() const { return Entries.empty(); } void clear() { Entries.clear(); } @@ -102,6 +99,7 @@ public: uint32_t getHeaderOffset() const { return HeaderOffset; } uint8_t getAddrSize() const { return HeaderData.AddrSize; } uint32_t getLength() const { return HeaderData.Length; } + uint16_t getVersion() const { return HeaderData.Version; } StringRef getSectionName() const { return SectionName; } StringRef getListTypeString() const { return ListTypeString; } dwarf::DwarfFormat getFormat() const { return Format; } @@ -159,7 +157,10 @@ public: uint32_t getHeaderOffset() const { return Header.getHeaderOffset(); } uint8_t getAddrSize() const { return Header.getAddrSize(); } - void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}) const; + void dump(raw_ostream &OS, + llvm::function_ref(uint32_t)> + LookupPooledAddress, + DIDumpOptions DumpOpts = {}) const; /// Return the contents of the offset entry designated by a given index. Optional getOffsetEntry(uint32_t Index) const { @@ -213,7 +214,8 @@ Error DWARFListType::extract(DWARFDataExtractor Data, StringRef SectionName, StringRef ListTypeString) { if (*OffsetPtr < HeaderOffset || *OffsetPtr >= End) - return createError("invalid %s list offset 0x%" PRIx32, + return createStringError(errc::invalid_argument, + "invalid %s list offset 0x%" PRIx32, ListTypeString.data(), *OffsetPtr); Entries.clear(); while (*OffsetPtr < End) { @@ -224,14 +226,18 @@ Error DWARFListType::extract(DWARFDataExtractor Data, if (Entry.isSentinel()) return Error::success(); } - return createError("no end of list marker detected at end of %s table " + return createStringError(errc::illegal_byte_sequence, + "no end of list marker detected at end of %s table " "starting at offset 0x%" PRIx32, SectionName.data(), HeaderOffset); } template -void DWARFListTableBase::dump(raw_ostream &OS, - DIDumpOptions DumpOpts) const { +void DWARFListTableBase::dump( + raw_ostream &OS, + llvm::function_ref(uint32_t)> + LookupPooledAddress, + DIDumpOptions DumpOpts) const { Header.dump(OS, DumpOpts); OS << HeaderString << "\n"; @@ -250,7 +256,7 @@ void DWARFListTableBase::dump(raw_ostream &OS, for (const auto &List : ListMap) for (const auto &Entry : List.second.getEntries()) Entry.dump(OS, getAddrSize(), MaxEncodingStringLength, CurrentBase, - DumpOpts); + DumpOpts, LookupPooledAddress); } template diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h index 6e8f370f4ae..d611b5d075c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h @@ -33,11 +33,13 @@ public: virtual ArrayRef getSectionNames() const { return {}; } virtual bool isLittleEndian() const = 0; virtual uint8_t getAddressSize() const { llvm_unreachable("unimplemented"); } - virtual const DWARFSection &getInfoSection() const { return Dummy; } + virtual void + forEachInfoSections(function_ref F) const {} virtual void forEachTypesSections(function_ref F) const {} virtual StringRef getAbbrevSection() const { return ""; } virtual const DWARFSection &getLocSection() const { return Dummy; } + virtual const DWARFSection &getLoclistsSection() const { return Dummy; } virtual StringRef getARangeSection() const { return ""; } virtual StringRef getDebugFrameSection() const { return ""; } virtual StringRef getEHFrameSection() const { return ""; } @@ -47,12 +49,13 @@ public: virtual const DWARFSection &getRangeSection() const { return Dummy; } virtual const DWARFSection &getRnglistsSection() const { return Dummy; } virtual StringRef getMacinfoSection() const { return ""; } - virtual StringRef getPubNamesSection() const { return ""; } - virtual StringRef getPubTypesSection() const { return ""; } - virtual StringRef getGnuPubNamesSection() const { return ""; } - virtual StringRef getGnuPubTypesSection() const { return ""; } + virtual const DWARFSection &getPubNamesSection() const { return Dummy; } + virtual const DWARFSection &getPubTypesSection() const { return Dummy; } + virtual const DWARFSection &getGnuPubNamesSection() const { return Dummy; } + virtual const DWARFSection &getGnuPubTypesSection() const { return Dummy; } virtual const DWARFSection &getStringOffsetSection() const { return Dummy; } - virtual const DWARFSection &getInfoDWOSection() const { return Dummy; } + virtual void + forEachInfoDWOSections(function_ref F) const {} virtual void forEachTypesDWOSections(function_ref F) const {} virtual StringRef getAbbrevDWOSection() const { return ""; } diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h index 77045f0794a..7f823596529 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h @@ -23,6 +23,11 @@ struct SectionName { bool IsNameUnique; }; +struct SectionedAddress { + uint64_t Address; + uint64_t SectionIndex; +}; + } // end namespace llvm #endif // LLVM_DEBUGINFO_DWARF_DWARFSECTION_H diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h index cb5a78ee3db..8ca5ba13fc2 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h @@ -26,19 +26,20 @@ class raw_ostream; class DWARFTypeUnit : public DWARFUnit { public: DWARFTypeUnit(DWARFContext &Context, const DWARFSection &Section, - const DWARFUnitHeader &Header, - const DWARFDebugAbbrev *DA, const DWARFSection *RS, + const DWARFUnitHeader &Header, const DWARFDebugAbbrev *DA, + const DWARFSection *RS, const DWARFSection *LocSection, StringRef SS, const DWARFSection &SOS, const DWARFSection *AOS, const DWARFSection &LS, bool LE, bool IsDWO, - const DWARFUnitSectionBase &UnitSection) - : DWARFUnit(Context, Section, Header, DA, RS, SS, SOS, AOS, LS, LE, IsDWO, - UnitSection) {} + const DWARFUnitVector &UnitVector) + : DWARFUnit(Context, Section, Header, DA, RS, LocSection, SS, SOS, AOS, + LS, LE, IsDWO, UnitVector) {} uint64_t getTypeHash() const { return getHeader().getTypeHash(); } uint32_t getTypeOffset() const { return getHeader().getTypeOffset(); } - void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}); - static const DWARFSectionKind Section = DW_SECT_TYPES; + void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}) override; + // Enable LLVM-style RTTI. + static bool classof(const DWARFUnit *U) { return U->isTypeUnit(); } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h index 988a7958184..79c3ce1106d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -72,7 +72,8 @@ public: /// Parse a unit header from \p debug_info starting at \p offset_ptr. bool extract(DWARFContext &Context, const DWARFDataExtractor &debug_info, uint32_t *offset_ptr, DWARFSectionKind Kind = DW_SECT_INFO, - const DWARFUnitIndex *Index = nullptr); + const DWARFUnitIndex *Index = nullptr, + const DWARFUnitIndex::Entry *Entry = nullptr); uint32_t getOffset() const { return Offset; } const dwarf::FormParams &getFormParams() const { return FormParams; } uint16_t getVersion() const { return FormParams.Version; } @@ -101,133 +102,66 @@ public: uint32_t getNextUnitOffset() const { return Offset + Length + 4; } }; -/// Base class for all DWARFUnitSection classes. This provides the -/// functionality common to all unit types. -class DWARFUnitSectionBase { -public: - /// Returns the Unit that contains the given section offset in the - /// same section this Unit originated from. - virtual DWARFUnit *getUnitForOffset(uint32_t Offset) const = 0; - virtual DWARFUnit *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E) = 0; - - void parse(DWARFContext &C, const DWARFSection &Section); - void parseDWO(DWARFContext &C, const DWARFSection &DWOSection, - bool Lazy = false); - -protected: - ~DWARFUnitSectionBase() = default; - - virtual void parseImpl(DWARFContext &Context, const DWARFObject &Obj, - const DWARFSection &Section, - const DWARFDebugAbbrev *DA, const DWARFSection *RS, - StringRef SS, const DWARFSection &SOS, - const DWARFSection *AOS, const DWARFSection &LS, - bool isLittleEndian, bool isDWO, bool Lazy) = 0; -}; - const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context, DWARFSectionKind Kind); -/// Concrete instance of DWARFUnitSection, specialized for one Unit type. -template -class DWARFUnitSection final : public SmallVector, 1>, - public DWARFUnitSectionBase { - bool Parsed = false; - std::function(uint32_t)> Parser; +/// Describe a collection of units. Intended to hold all units either from +/// .debug_info and .debug_types, or from .debug_info.dwo and .debug_types.dwo. +class DWARFUnitVector final : public SmallVector, 1> { + std::function(uint32_t, DWARFSectionKind, + const DWARFSection *, + const DWARFUnitIndex::Entry *)> + Parser; + int NumInfoUnits = -1; public: - using UnitVector = SmallVectorImpl>; + using UnitVector = SmallVectorImpl>; using iterator = typename UnitVector::iterator; using iterator_range = llvm::iterator_range; - UnitType *getUnitForOffset(uint32_t Offset) const override { - auto *CU = std::upper_bound( - this->begin(), this->end(), Offset, - [](uint32_t LHS, const std::unique_ptr &RHS) { - return LHS < RHS->getNextUnitOffset(); - }); - if (CU != this->end() && (*CU)->getOffset() <= Offset) - return CU->get(); - return nullptr; - } - UnitType *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E) override { - const auto *CUOff = E.getOffset(DW_SECT_INFO); - if (!CUOff) - return nullptr; - - auto Offset = CUOff->Offset; - - auto *CU = std::upper_bound( - this->begin(), this->end(), CUOff->Offset, - [](uint32_t LHS, const std::unique_ptr &RHS) { - return LHS < RHS->getNextUnitOffset(); - }); - if (CU != this->end() && (*CU)->getOffset() <= Offset) - return CU->get(); - - if (!Parser) - return nullptr; - - auto U = Parser(Offset); - if (!U) - U = nullptr; - - auto *NewCU = U.get(); - this->insert(CU, std::move(U)); - return NewCU; + DWARFUnit *getUnitForOffset(uint32_t Offset) const; + DWARFUnit *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E); + + /// Read units from a .debug_info or .debug_types section. Calls made + /// before finishedInfoUnits() are assumed to be for .debug_info sections, + /// calls after finishedInfoUnits() are for .debug_types sections. Caller + /// must not mix calls to addUnitsForSection and addUnitsForDWOSection. + void addUnitsForSection(DWARFContext &C, const DWARFSection &Section, + DWARFSectionKind SectionKind); + /// Read units from a .debug_info.dwo or .debug_types.dwo section. Calls + /// made before finishedInfoUnits() are assumed to be for .debug_info.dwo + /// sections, calls after finishedInfoUnits() are for .debug_types.dwo + /// sections. Caller must not mix calls to addUnitsForSection and + /// addUnitsForDWOSection. + void addUnitsForDWOSection(DWARFContext &C, const DWARFSection &DWOSection, + DWARFSectionKind SectionKind, bool Lazy = false); + + /// Add an existing DWARFUnit to this UnitVector. This is used by the DWARF + /// verifier to process unit separately. + DWARFUnit *addUnit(std::unique_ptr Unit); + + /// Returns number of all units held by this instance. + unsigned getNumUnits() const { return size(); } + /// Returns number of units from all .debug_info[.dwo] sections. + unsigned getNumInfoUnits() const { + return NumInfoUnits == -1 ? size() : NumInfoUnits; } + /// Returns number of units from all .debug_types[.dwo] sections. + unsigned getNumTypesUnits() const { return size() - NumInfoUnits; } + /// Indicate that parsing .debug_info[.dwo] is done, and remaining units + /// will be from .debug_types[.dwo]. + void finishedInfoUnits() { NumInfoUnits = size(); } private: - void parseImpl(DWARFContext &Context, const DWARFObject &Obj, - const DWARFSection &Section, const DWARFDebugAbbrev *DA, - const DWARFSection *RS, StringRef SS, const DWARFSection &SOS, - const DWARFSection *AOS, const DWARFSection &LS, bool LE, - bool IsDWO, bool Lazy) override { - if (Parsed) - return; - DWARFDataExtractor Data(Obj, Section, LE, 0); - if (!Parser) { - const DWARFUnitIndex *Index = nullptr; - if (IsDWO) - Index = &getDWARFUnitIndex(Context, UnitType::Section); - Parser = [=, &Context, &Section, &SOS, - &LS](uint32_t Offset) -> std::unique_ptr { - if (!Data.isValidOffset(Offset)) - return nullptr; - DWARFUnitHeader Header; - if (!Header.extract(Context, Data, &Offset, UnitType::Section, Index)) - return nullptr; - auto U = llvm::make_unique( - Context, Section, Header, DA, RS, SS, SOS, AOS, LS, LE, IsDWO, - *this); - return U; - }; - } - if (Lazy) - return; - auto I = this->begin(); - uint32_t Offset = 0; - while (Data.isValidOffset(Offset)) { - if (I != this->end() && (*I)->getOffset() == Offset) { - ++I; - continue; - } - auto U = Parser(Offset); - if (!U) - break; - Offset = U->getNextUnitOffset(); - I = std::next(this->insert(I, std::move(U))); - } - Parsed = true; - } + void addUnitsImpl(DWARFContext &Context, const DWARFObject &Obj, + const DWARFSection &Section, const DWARFDebugAbbrev *DA, + const DWARFSection *RS, const DWARFSection *LocSection, + StringRef SS, const DWARFSection &SOS, + const DWARFSection *AOS, const DWARFSection &LS, bool LE, + bool IsDWO, bool Lazy, DWARFSectionKind SectionKind); }; /// Represents base address of the CU. -struct BaseAddress { - uint64_t Address; - uint64_t SectionIndex; -}; - /// Represents a unit's contribution to the string offsets table. struct StrOffsetsContributionDescriptor { uint64_t Base = 0; @@ -261,14 +195,20 @@ class DWARFUnit { const DWARFDebugAbbrev *Abbrev; const DWARFSection *RangeSection; uint32_t RangeSectionBase; + /// We either keep track of the location list section or its data, depending + /// on whether we are handling a split DWARF section or not. + union { + const DWARFSection *LocSection; + StringRef LocSectionData; + }; const DWARFSection &LineSection; StringRef StringSection; const DWARFSection &StringOffsetSection; const DWARFSection *AddrOffsetSection; uint32_t AddrOffsetSectionBase = 0; bool isLittleEndian; - bool isDWO; - const DWARFUnitSectionBase &UnitSection; + bool IsDWO; + const DWARFUnitVector &UnitVector; /// Start, length, and DWARF format of the unit's contribution to the string /// offsets table (DWARF v5). @@ -278,7 +218,7 @@ class DWARFUnit { Optional RngListTable; mutable const DWARFAbbreviationDeclarationSet *Abbrevs; - llvm::Optional BaseAddr; + llvm::Optional BaseAddr; /// The compile unit debug information entry items. std::vector DieArray; @@ -308,28 +248,30 @@ protected: /// length and form. The given offset is expected to be derived from the unit /// DIE's DW_AT_str_offsets_base attribute. Optional - determineStringOffsetsTableContribution(DWARFDataExtractor &DA, - uint64_t Offset); + determineStringOffsetsTableContribution(DWARFDataExtractor &DA); /// Find the unit's contribution to the string offsets table and determine its /// length and form. The given offset is expected to be 0 in a dwo file or, /// in a dwp file, the start of the unit's contribution to the string offsets /// table section (as determined by the index table). Optional - determineStringOffsetsTableContributionDWO(DWARFDataExtractor &DA, - uint64_t Offset); + determineStringOffsetsTableContributionDWO(DWARFDataExtractor &DA); public: DWARFUnit(DWARFContext &Context, const DWARFSection &Section, - const DWARFUnitHeader &Header, - const DWARFDebugAbbrev *DA, const DWARFSection *RS, StringRef SS, - const DWARFSection &SOS, const DWARFSection *AOS, + const DWARFUnitHeader &Header, const DWARFDebugAbbrev *DA, + const DWARFSection *RS, const DWARFSection *LocSection, + StringRef SS, const DWARFSection &SOS, const DWARFSection *AOS, const DWARFSection &LS, bool LE, bool IsDWO, - const DWARFUnitSectionBase &UnitSection); + const DWARFUnitVector &UnitVector); virtual ~DWARFUnit(); + bool isDWOUnit() const { return IsDWO; } DWARFContext& getContext() const { return Context; } + const DWARFSection &getInfoSection() const { return InfoSection; } + const DWARFSection *getLocSection() const { return LocSection; } + StringRef getLocSectionData() const { return LocSectionData; } uint32_t getOffset() const { return Header.getOffset(); } const dwarf::FormParams &getFormParams() const { return Header.getFormParams(); @@ -342,6 +284,7 @@ public: } uint32_t getLength() const { return Header.getLength(); } uint8_t getUnitType() const { return Header.getUnitType(); } + bool isTypeUnit() const { return Header.isTypeUnit(); } uint32_t getNextUnitOffset() const { return Header.getNextUnitOffset(); } const DWARFSection &getLineSection() const { return LineSection; } StringRef getStringSection() const { return StringSection; } @@ -362,8 +305,8 @@ public: RangeSectionBase = Base; } - bool getAddrOffsetSectionItem(uint32_t Index, uint64_t &Result) const; - bool getStringOffsetSectionItem(uint32_t Index, uint64_t &Result) const; + Optional getAddrOffsetSectionItem(uint32_t Index) const; + Optional getStringOffsetSectionItem(uint32_t Index) const; DWARFDataExtractor getDebugInfoExtractor() const; @@ -433,7 +376,7 @@ public: llvm_unreachable("Invalid UnitType."); } - llvm::Optional getBaseAddress(); + llvm::Optional getBaseAddress(); DWARFDie getUnitDIE(bool ExtractUnitDIEOnly = true) { extractDIEsIfNeeded(ExtractUnitDIEOnly); @@ -467,7 +410,7 @@ public: return None; } - void collectAddressRanges(DWARFAddressRangesVector &CURanges); + Expected collectAddressRanges(); /// Returns subprogram DIE with address range encompassing the provided /// address. The pointer is alive as long as parsed compile unit DIEs are not @@ -480,8 +423,8 @@ public: void getInlinedChainForAddress(uint64_t Address, SmallVectorImpl &InlinedChain); - /// getUnitSection - Return the DWARFUnitSection containing this unit. - const DWARFUnitSectionBase &getUnitSection() const { return UnitSection; } + /// Return the DWARFUnitVector containing this unit. + const DWARFUnitVector &getUnitVector() const { return UnitVector; } /// Returns the number of DIEs in the unit. Parses the unit /// if necessary. @@ -541,6 +484,7 @@ public: return die_iterator_range(DieArray.begin(), DieArray.end()); } + virtual void dump(raw_ostream &OS, DIDumpOptions DumpOpts) = 0; private: /// Size in bytes of the .debug_info data associated with this compile unit. size_t getDebugInfoSize() const { diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h index 49ed4bb222f..16be5f9401c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h @@ -74,6 +74,7 @@ private: int InfoColumn = -1; std::unique_ptr ColumnKinds; std::unique_ptr Rows; + mutable std::vector OffsetLookup; static StringRef getColumnHeader(DWARFSectionKind DS); diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h index a829510a219..e47fbea5646 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h @@ -14,6 +14,7 @@ #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h" #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" +#include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" #include #include @@ -96,10 +97,14 @@ private: /// lies between to valid DIEs. std::map> ReferenceToDIEOffsets; uint32_t NumDebugLineErrors = 0; + // Used to relax some checks that do not currently work portably + bool IsObjectFile; + bool IsMachOObject; raw_ostream &error() const; raw_ostream &warn() const; raw_ostream ¬e() const; + raw_ostream &dump(const DWARFDie &Die, unsigned indent = 0) const; /// Verifies the abbreviations section. /// @@ -113,20 +118,20 @@ private: /// \returns The number of errors that occurred during verification. unsigned verifyAbbrevSection(const DWARFDebugAbbrev *Abbrev); - /// Verifies the header of a unit in the .debug_info section. + /// Verifies the header of a unit in a .debug_info or .debug_types section. /// /// This function currently checks for: /// - Unit is in 32-bit DWARF format. The function can be modified to /// support 64-bit format. /// - The DWARF version is valid /// - The unit type is valid (if unit is in version >=5) - /// - The unit doesn't extend beyond .debug_info section + /// - The unit doesn't extend beyond the containing section /// - The address size is valid /// - The offset in the .debug_abbrev section is valid /// - /// \param DebugInfoData The .debug_info section data + /// \param DebugInfoData The section data /// \param Offset A reference to the offset start of the unit. The offset will - /// be updated to point to the next unit in .debug_info + /// be updated to point to the next unit in the section /// \param UnitIndex The index of the unit to be verified /// \param UnitType A reference to the type of the unit /// \param isUnitDWARF64 A reference to a flag that shows whether the unit is @@ -137,7 +142,7 @@ private: uint32_t *Offset, unsigned UnitIndex, uint8_t &UnitType, bool &isUnitDWARF64); - /// Verifies the header of a unit in the .debug_info section. + /// Verifies the header of a unit in a .debug_info or .debug_types section. /// /// This function currently verifies: /// - The debug info attributes. @@ -146,13 +151,29 @@ private: /// - That the root DIE is a unit DIE. /// - If a unit type is provided, that the unit DIE matches the unit type. /// - The DIE ranges. + /// - That call site entries are only nested within subprograms with a + /// DW_AT_call attribute. /// - /// \param Unit The DWARF Unit to verifiy. - /// \param UnitType An optional unit type which will be used to verify the - /// type of the unit DIE. + /// \param Unit The DWARF Unit to verify. /// - /// \returns true if the content is verified successfully, false otherwise. - bool verifyUnitContents(DWARFUnit &Unit, uint8_t UnitType = 0); + /// \returns The number of errors that occurred during verification. + unsigned verifyUnitContents(DWARFUnit &Unit); + + /// Verifies the unit headers and contents in a .debug_info or .debug_types + /// section. + /// + /// \param S The DWARF Section to verify. + /// \param SectionKind The object-file section kind that S comes from. + /// + /// \returns The number of errors that occurred during verification. + unsigned verifyUnitSection(const DWARFSection &S, + DWARFSectionKind SectionKind); + + /// Verifies that a call site entry is nested within a subprogram with a + /// DW_AT_call attribute. + /// + /// \returns Number of errors that occurred during verification. + unsigned verifyDebugInfoCallSite(const DWARFDie &Die); /// Verify that all Die ranges are valid. /// @@ -172,7 +193,7 @@ private: /// \param AttrValue The DWARF attribute value to check /// /// \returns NumErrors The number of errors occurred during verification of - /// attributes' values in a .debug_info section unit + /// attributes' values in a unit unsigned verifyDebugInfoAttribute(const DWARFDie &Die, DWARFAttribute &AttrValue); @@ -180,14 +201,14 @@ private: /// /// This function currently checks for: /// - All DW_FORM_ref values that are CU relative have valid CU offsets - /// - All DW_FORM_ref_addr values have valid .debug_info offsets + /// - All DW_FORM_ref_addr values have valid section offsets /// - All DW_FORM_strp values have valid .debug_str offsets /// /// \param Die The DWARF DIE that owns the attribute value /// \param AttrValue The DWARF attribute value to check /// /// \returns NumErrors The number of errors occurred during verification of - /// attributes' forms in a .debug_info section unit + /// attributes' forms in a unit unsigned verifyDebugInfoForm(const DWARFDie &Die, DWARFAttribute &AttrValue); /// Verifies the all valid references that were found when iterating through @@ -199,7 +220,7 @@ private: /// CU relative and absolute references. /// /// \returns NumErrors The number of errors occurred during verification of - /// references for the .debug_info section + /// references for the .debug_info and .debug_types sections unsigned verifyDebugInfoReferences(); /// Verify the DW_AT_stmt_list encoding and value and ensure that no @@ -268,8 +289,8 @@ private: public: DWARFVerifier(raw_ostream &S, DWARFContext &D, - DIDumpOptions DumpOpts = DIDumpOptions::getForSingleDIE()) - : OS(S), DCtx(D), DumpOpts(std::move(DumpOpts)) {} + DIDumpOptions DumpOpts = DIDumpOptions::getForSingleDIE()); + /// Verify the information in any of the following sections, if available: /// .debug_abbrev, debug_abbrev.dwo /// @@ -280,12 +301,12 @@ public: /// false otherwise. bool handleDebugAbbrev(); - /// Verify the information in the .debug_info section. + /// Verify the information in the .debug_info and .debug_types sections. /// - /// Any errors are reported to the stream that was this object was + /// Any errors are reported to the stream that this object was /// constructed with. /// - /// \returns true if the .debug_info verifies successfully, false otherwise. + /// \returns true if all sections verify successfully, false otherwise. bool handleDebugInfo(); /// Verify the information in the .debug_line section. diff --git a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFError.h b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFError.h index e66aeca3cd4..5c043a7837b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFError.h +++ b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFError.h @@ -24,22 +24,28 @@ enum class msf_error_code { invalid_format, block_in_use }; +} // namespace msf +} // namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} // namespace std + +namespace llvm { +namespace msf { +const std::error_category &MSFErrCategory(); + +inline std::error_code make_error_code(msf_error_code E) { + return std::error_code(static_cast(E), MSFErrCategory()); +} /// Base class for errors originating when parsing raw PDB files -class MSFError : public ErrorInfo { +class MSFError : public ErrorInfo { public: + using ErrorInfo::ErrorInfo; // inherit constructors + MSFError(const Twine &S) : ErrorInfo(S, msf_error_code::unspecified) {} static char ID; - MSFError(msf_error_code C); - MSFError(const std::string &Context); - MSFError(msf_error_code C, const std::string &Context); - - void log(raw_ostream &OS) const override; - const std::string &getErrorMessage() const; - std::error_code convertToErrorCode() const override; - -private: - std::string ErrMsg; - msf_error_code Code; }; } // namespace msf } // namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h b/contrib/llvm/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h index 9713dce362d..ac7f19637ab 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h @@ -43,11 +43,6 @@ public: void reset() override { Enumerator->reset(); } - ConcreteSymbolEnumerator *clone() const override { - std::unique_ptr WrappedClone(Enumerator->clone()); - return new ConcreteSymbolEnumerator(std::move(WrappedClone)); - } - private: std::unique_ptr Enumerator; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h index 930bea6060b..881d7329ab6 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h @@ -24,7 +24,6 @@ public: llvm::Optional getItemAtIndex(uint32_t Index) const override; bool getNext(RecordType &Record) override; void reset() override; - DIADataStream *clone() const override; private: CComPtr StreamData; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h index ffae6645e94..1f129052d03 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h @@ -27,7 +27,6 @@ public: ChildTypePtr getChildAtIndex(uint32_t Index) const override; ChildTypePtr getNext() override; void reset() override; - DIAEnumDebugStreams *clone() const override; private: CComPtr Enumerator; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h new file mode 100644 index 00000000000..f3b02f07e64 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h @@ -0,0 +1,36 @@ +//==- DIAEnumFrameData.h --------------------------------------- -*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_DIA_DIAENUMFRAMEDATA_H +#define LLVM_DEBUGINFO_PDB_DIA_DIAENUMFRAMEDATA_H + +#include "DIASupport.h" +#include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" +#include "llvm/DebugInfo/PDB/IPDBFrameData.h" + +namespace llvm { +namespace pdb { + +class DIAEnumFrameData : public IPDBEnumChildren { +public: + explicit DIAEnumFrameData(CComPtr DiaEnumerator); + + uint32_t getChildCount() const override; + ChildTypePtr getChildAtIndex(uint32_t Index) const override; + ChildTypePtr getNext() override; + void reset() override; + +private: + CComPtr Enumerator; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h index 39490a4b220..4669a8d3119 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h @@ -16,22 +16,18 @@ namespace llvm { namespace pdb { -class DIASession; class DIAEnumInjectedSources : public IPDBEnumChildren { public: explicit DIAEnumInjectedSources( - const DIASession &PDBSession, CComPtr DiaEnumerator); uint32_t getChildCount() const override; ChildTypePtr getChildAtIndex(uint32_t Index) const override; ChildTypePtr getNext() override; void reset() override; - DIAEnumInjectedSources *clone() const override; private: - const DIASession &Session; CComPtr Enumerator; }; } // namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h index 08f0de124ed..f1cb6268a26 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h @@ -26,7 +26,6 @@ public: ChildTypePtr getChildAtIndex(uint32_t Index) const override; ChildTypePtr getNext() override; void reset() override; - DIAEnumLineNumbers *clone() const override; private: CComPtr Enumerator; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h index 52c9563b5d5..ac2ae317d26 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h @@ -28,7 +28,6 @@ public: ChildTypePtr getChildAtIndex(uint32_t Index) const override; ChildTypePtr getNext() override; void reset() override; - DIAEnumSectionContribs *clone() const override; private: const DIASession &Session; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h index e69d18f5ba3..dac3df06a17 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h @@ -27,7 +27,6 @@ public: ChildTypePtr getChildAtIndex(uint32_t Index) const override; ChildTypePtr getNext() override; void reset() override; - DIAEnumSourceFiles *clone() const override; private: const DIASession &Session; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h index f779cd1f4be..9689859ae0f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h @@ -27,7 +27,6 @@ public: std::unique_ptr getChildAtIndex(uint32_t Index) const override; std::unique_ptr getNext() override; void reset() override; - DIAEnumSymbols *clone() const override; private: const DIASession &Session; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h index 926fcfe6964..f4f856ebb6f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h @@ -26,7 +26,6 @@ public: std::unique_ptr getChildAtIndex(uint32_t Index) const override; std::unique_ptr getNext() override; void reset() override; - DIAEnumTables *clone() const override; private: CComPtr Enumerator; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAError.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAError.h index 35a39a0df5c..2b33a65a0a1 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAError.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAError.h @@ -23,23 +23,29 @@ enum class dia_error_code { already_loaded, debug_info_mismatch, }; +} // namespace pdb +} // namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} // namespace std + +namespace llvm { +namespace pdb { +const std::error_category &DIAErrCategory(); + +inline std::error_code make_error_code(dia_error_code E) { + return std::error_code(static_cast(E), DIAErrCategory()); +} /// Base class for errors originating in DIA SDK, e.g. COM calls -class DIAError : public ErrorInfo { +class DIAError : public ErrorInfo { public: + using ErrorInfo::ErrorInfo; + DIAError(const Twine &S) : ErrorInfo(S, dia_error_code::unspecified) {} static char ID; - DIAError(dia_error_code C); - DIAError(StringRef Context); - DIAError(dia_error_code C, StringRef Context); - - void log(raw_ostream &OS) const override; - StringRef getErrorMessage() const; - std::error_code convertToErrorCode() const override; - -private: - std::string ErrMsg; - dia_error_code Code; }; -} -} +} // namespace pdb +} // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h new file mode 100644 index 00000000000..0ce6cfc9303 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h @@ -0,0 +1,39 @@ +//===- DIAFrameData.h - DIA Impl. of IPDBFrameData ---------------- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_DIA_DIAFRAMEDATA_H +#define LLVM_DEBUGINFO_PDB_DIA_DIAFRAMEDATA_H + +#include "DIASupport.h" +#include "llvm/DebugInfo/PDB/IPDBFrameData.h" + +namespace llvm { +namespace pdb { + +class DIASession; + +class DIAFrameData : public IPDBFrameData { +public: + explicit DIAFrameData(CComPtr DiaFrameData); + + uint32_t getAddressOffset() const override; + uint32_t getAddressSection() const override; + uint32_t getLengthBlock() const override; + std::string getProgram() const override; + uint32_t getRelativeVirtualAddress() const override; + uint64_t getVirtualAddress() const override; + +private: + CComPtr FrameData; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h index dfb35647055..5d4f855c63c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h @@ -20,7 +20,8 @@ class DIARawSymbol : public IPDBRawSymbol { public: DIARawSymbol(const DIASession &PDBSession, CComPtr DiaSymbol); - void dump(raw_ostream &OS, int Indent) const override; + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; CComPtr getDiaSymbol() const { return Symbol; } @@ -63,25 +64,25 @@ public: uint32_t getAddressOffset() const override; uint32_t getAddressSection() const override; uint32_t getAge() const override; - uint32_t getArrayIndexTypeId() const override; + SymIndexId getArrayIndexTypeId() const override; uint32_t getBaseDataOffset() const override; uint32_t getBaseDataSlot() const override; - uint32_t getBaseSymbolId() const override; + SymIndexId getBaseSymbolId() const override; PDB_BuiltinType getBuiltinType() const override; uint32_t getBitPosition() const override; PDB_CallingConv getCallingConvention() const override; - uint32_t getClassParentId() const override; + SymIndexId getClassParentId() const override; std::string getCompilerName() const override; uint32_t getCount() const override; uint32_t getCountLiveRanges() const override; PDB_Lang getLanguage() const override; - uint32_t getLexicalParentId() const override; + SymIndexId getLexicalParentId() const override; std::string getLibraryName() const override; uint32_t getLiveRangeStartAddressOffset() const override; uint32_t getLiveRangeStartAddressSection() const override; uint32_t getLiveRangeStartRelativeVirtualAddress() const override; codeview::RegisterId getLocalBasePointerRegisterId() const override; - uint32_t getLowerBoundId() const override; + SymIndexId getLowerBoundId() const override; uint32_t getMemorySpaceKind() const override; std::string getName() const override; uint32_t getNumberOfAcceleratorPointerTags() const override; @@ -91,7 +92,7 @@ public: uint32_t getNumberOfRows() const override; std::string getObjectFileName() const override; uint32_t getOemId() const override; - uint32_t getOemSymbolId() const override; + SymIndexId getOemSymbolId() const override; uint32_t getOffsetInUdt() const override; PDB_Cpu getPlatform() const override; uint32_t getRank() const override; @@ -105,9 +106,9 @@ public: std::string getSourceFileName() const override; std::unique_ptr getSrcLineOnTypeDefn() const override; uint32_t getStride() const override; - uint32_t getSubTypeId() const override; + SymIndexId getSubTypeId() const override; std::string getSymbolsFileName() const override; - uint32_t getSymIndexId() const override; + SymIndexId getSymIndexId() const override; uint32_t getTargetOffset() const override; uint32_t getTargetRelativeVirtualAddress() const override; uint64_t getTargetVirtualAddress() const override; @@ -115,16 +116,16 @@ public: uint32_t getTextureSlot() const override; uint32_t getTimeStamp() const override; uint32_t getToken() const override; - uint32_t getTypeId() const override; + SymIndexId getTypeId() const override; uint32_t getUavSlot() const override; std::string getUndecoratedName() const override; std::string getUndecoratedNameEx(PDB_UndnameFlags Flags) const override; - uint32_t getUnmodifiedTypeId() const override; - uint32_t getUpperBoundId() const override; + SymIndexId getUnmodifiedTypeId() const override; + SymIndexId getUpperBoundId() const override; Variant getValue() const override; uint32_t getVirtualBaseDispIndex() const override; uint32_t getVirtualBaseOffset() const override; - uint32_t getVirtualTableShapeId() const override; + SymIndexId getVirtualTableShapeId() const override; std::unique_ptr getVirtualBaseTableType() const override; PDB_DataKind getDataKind() const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h index a6365943938..592e061a8d8 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h @@ -32,7 +32,7 @@ public: uint64_t getLoadAddress() const override; bool setLoadAddress(uint64_t Address) override; std::unique_ptr getGlobalScope() override; - std::unique_ptr getSymbolById(uint32_t SymbolId) const override; + std::unique_ptr getSymbolById(SymIndexId SymbolId) const override; bool addressForVA(uint64_t VA, uint32_t &Section, uint32_t &Offset) const override; @@ -85,6 +85,7 @@ public: std::unique_ptr getSectionContribs() const override; + std::unique_ptr getFrameData() const override; private: CComPtr Session; }; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/GenericError.h b/contrib/llvm/include/llvm/DebugInfo/PDB/GenericError.h index 03205a986f1..997f13f5f30 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/GenericError.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/GenericError.h @@ -16,29 +16,37 @@ namespace llvm { namespace pdb { -enum class generic_error_code { - invalid_path = 1, +enum class pdb_error_code { + invalid_utf8_path = 1, dia_sdk_not_present, - type_server_not_found, + dia_failed_loading, + signature_out_of_date, + external_cmdline_ref, unspecified, }; +} // namespace pdb +} // namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} // namespace std + +namespace llvm { +namespace pdb { +const std::error_category &PDBErrCategory(); + +inline std::error_code make_error_code(pdb_error_code E) { + return std::error_code(static_cast(E), PDBErrCategory()); +} /// Base class for errors originating when parsing raw PDB files -class GenericError : public ErrorInfo { +class PDBError : public ErrorInfo { public: + using ErrorInfo::ErrorInfo; // inherit constructors + PDBError(const Twine &S) : ErrorInfo(S, pdb_error_code::unspecified) {} static char ID; - GenericError(generic_error_code C); - GenericError(StringRef Context); - GenericError(generic_error_code C, StringRef Context); - - void log(raw_ostream &OS) const override; - StringRef getErrorMessage() const; - std::error_code convertToErrorCode() const override; - -private: - std::string ErrMsg; - generic_error_code Code; }; -} -} +} // namespace pdb +} // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBDataStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBDataStream.h index 67b5a06d7c5..0d7a286a11a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBDataStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBDataStream.h @@ -32,7 +32,6 @@ public: virtual Optional getItemAtIndex(uint32_t Index) const = 0; virtual bool getNext(RecordType &Record) = 0; virtual void reset() = 0; - virtual IPDBDataStream *clone() const = 0; }; } // end namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h index b6b7d95f628..7017f2600e9 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h @@ -10,6 +10,7 @@ #ifndef LLVM_DEBUGINFO_PDB_IPDBENUMCHILDREN_H #define LLVM_DEBUGINFO_PDB_IPDBENUMCHILDREN_H +#include #include #include @@ -27,7 +28,19 @@ public: virtual ChildTypePtr getChildAtIndex(uint32_t Index) const = 0; virtual ChildTypePtr getNext() = 0; virtual void reset() = 0; - virtual MyType *clone() const = 0; +}; + +template +class NullEnumerator : public IPDBEnumChildren { + virtual uint32_t getChildCount() const override { return 0; } + virtual std::unique_ptr + getChildAtIndex(uint32_t Index) const override { + return nullptr; + } + virtual std::unique_ptr getNext() override { + return nullptr; + } + virtual void reset() override {} }; } // end namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h new file mode 100644 index 00000000000..74679215b88 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h @@ -0,0 +1,36 @@ +//===- IPDBFrameData.h - base interface for frame data ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_IPDBFRAMEDATA_H +#define LLVM_DEBUGINFO_PDB_IPDBFRAMEDATA_H + +#include +#include + +namespace llvm { +namespace pdb { + +/// IPDBFrameData defines an interface used to represent a frame data of some +/// code block. +class IPDBFrameData { +public: + virtual ~IPDBFrameData(); + + virtual uint32_t getAddressOffset() const = 0; + virtual uint32_t getAddressSection() const = 0; + virtual uint32_t getLengthBlock() const = 0; + virtual std::string getProgram() const = 0; + virtual uint32_t getRelativeVirtualAddress() const = 0; + virtual uint64_t getVirtualAddress() const = 0; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h index bcb2eaa3563..7c818d7cade 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h @@ -11,6 +11,7 @@ #define LLVM_DEBUGINFO_PDB_IPDBRAWSYMBOL_H #include "PDBTypes.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -21,9 +22,26 @@ class raw_ostream; namespace pdb { +class IPDBSession; class PDBSymbolTypeVTable; class PDBSymbolTypeVTableShape; +enum class PdbSymbolIdField : uint32_t { + None = 0, + SymIndexId = 1 << 0, + LexicalParent = 1 << 1, + ClassParent = 1 << 2, + Type = 1 << 3, + UnmodifiedType = 1 << 4, + All = 0xFFFFFFFF, + LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ All) +}; + +void dumpSymbolIdField(raw_ostream &OS, StringRef Name, SymIndexId Value, + int Indent, const IPDBSession &Session, + PdbSymbolIdField FieldId, PdbSymbolIdField ShowFlags, + PdbSymbolIdField RecurseFlags); + /// IPDBRawSymbol defines an interface used to represent an arbitrary symbol. /// It exposes a monolithic interface consisting of accessors for the union of /// all properties that are valid for any symbol type. This interface is then @@ -33,7 +51,8 @@ class IPDBRawSymbol { public: virtual ~IPDBRawSymbol(); - virtual void dump(raw_ostream &OS, int Indent) const = 0; + virtual void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const = 0; virtual std::unique_ptr findChildren(PDB_SymType Type) const = 0; @@ -74,26 +93,26 @@ public: virtual uint32_t getAddressOffset() const = 0; virtual uint32_t getAddressSection() const = 0; virtual uint32_t getAge() const = 0; - virtual uint32_t getArrayIndexTypeId() const = 0; + virtual SymIndexId getArrayIndexTypeId() const = 0; virtual uint32_t getBaseDataOffset() const = 0; virtual uint32_t getBaseDataSlot() const = 0; - virtual uint32_t getBaseSymbolId() const = 0; + virtual SymIndexId getBaseSymbolId() const = 0; virtual PDB_BuiltinType getBuiltinType() const = 0; virtual uint32_t getBitPosition() const = 0; virtual PDB_CallingConv getCallingConvention() const = 0; - virtual uint32_t getClassParentId() const = 0; + virtual SymIndexId getClassParentId() const = 0; virtual std::string getCompilerName() const = 0; virtual uint32_t getCount() const = 0; virtual uint32_t getCountLiveRanges() const = 0; virtual void getFrontEndVersion(VersionInfo &Version) const = 0; virtual PDB_Lang getLanguage() const = 0; - virtual uint32_t getLexicalParentId() const = 0; + virtual SymIndexId getLexicalParentId() const = 0; virtual std::string getLibraryName() const = 0; virtual uint32_t getLiveRangeStartAddressOffset() const = 0; virtual uint32_t getLiveRangeStartAddressSection() const = 0; virtual uint32_t getLiveRangeStartRelativeVirtualAddress() const = 0; virtual codeview::RegisterId getLocalBasePointerRegisterId() const = 0; - virtual uint32_t getLowerBoundId() const = 0; + virtual SymIndexId getLowerBoundId() const = 0; virtual uint32_t getMemorySpaceKind() const = 0; virtual std::string getName() const = 0; virtual uint32_t getNumberOfAcceleratorPointerTags() const = 0; @@ -103,7 +122,7 @@ public: virtual uint32_t getNumberOfRows() const = 0; virtual std::string getObjectFileName() const = 0; virtual uint32_t getOemId() const = 0; - virtual uint32_t getOemSymbolId() const = 0; + virtual SymIndexId getOemSymbolId() const = 0; virtual uint32_t getOffsetInUdt() const = 0; virtual PDB_Cpu getPlatform() const = 0; virtual uint32_t getRank() const = 0; @@ -118,9 +137,9 @@ public: virtual std::unique_ptr getSrcLineOnTypeDefn() const = 0; virtual uint32_t getStride() const = 0; - virtual uint32_t getSubTypeId() const = 0; + virtual SymIndexId getSubTypeId() const = 0; virtual std::string getSymbolsFileName() const = 0; - virtual uint32_t getSymIndexId() const = 0; + virtual SymIndexId getSymIndexId() const = 0; virtual uint32_t getTargetOffset() const = 0; virtual uint32_t getTargetRelativeVirtualAddress() const = 0; virtual uint64_t getTargetVirtualAddress() const = 0; @@ -128,18 +147,18 @@ public: virtual uint32_t getTextureSlot() const = 0; virtual uint32_t getTimeStamp() const = 0; virtual uint32_t getToken() const = 0; - virtual uint32_t getTypeId() const = 0; + virtual SymIndexId getTypeId() const = 0; virtual uint32_t getUavSlot() const = 0; virtual std::string getUndecoratedName() const = 0; virtual std::string getUndecoratedNameEx(PDB_UndnameFlags Flags) const = 0; - virtual uint32_t getUnmodifiedTypeId() const = 0; - virtual uint32_t getUpperBoundId() const = 0; + virtual SymIndexId getUnmodifiedTypeId() const = 0; + virtual SymIndexId getUpperBoundId() const = 0; virtual Variant getValue() const = 0; virtual uint32_t getVirtualBaseDispIndex() const = 0; virtual uint32_t getVirtualBaseOffset() const = 0; virtual std::unique_ptr getVirtualBaseTableType() const = 0; - virtual uint32_t getVirtualTableShapeId() const = 0; + virtual SymIndexId getVirtualTableShapeId() const = 0; virtual PDB_DataKind getDataKind() const = 0; virtual PDB_SymType getSymTag() const = 0; virtual codeview::GUID getGuid() const = 0; @@ -237,6 +256,8 @@ public: virtual std::string getUnused() const = 0; }; +LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); + } // namespace pdb } // namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h index 88ec517bc4a..88fd02c0a34 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h @@ -30,7 +30,8 @@ public: virtual uint64_t getLoadAddress() const = 0; virtual bool setLoadAddress(uint64_t Address) = 0; virtual std::unique_ptr getGlobalScope() = 0; - virtual std::unique_ptr getSymbolById(uint32_t SymbolId) const = 0; + virtual std::unique_ptr + getSymbolById(SymIndexId SymbolId) const = 0; virtual bool addressForVA(uint64_t VA, uint32_t &Section, uint32_t &Offset) const = 0; @@ -38,7 +39,7 @@ public: uint32_t &Offset) const = 0; template - std::unique_ptr getConcreteSymbolById(uint32_t SymbolId) const { + std::unique_ptr getConcreteSymbolById(SymIndexId SymbolId) const { return unique_dyn_cast_or_null(getSymbolById(SymbolId)); } @@ -90,6 +91,9 @@ public: virtual std::unique_ptr getSectionContribs() const = 0; + + virtual std::unique_ptr + getFrameData() const = 0; }; } // namespace pdb } // namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h index ce4d0791775..ac7f741afef 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h @@ -51,6 +51,7 @@ public: void setObjFileName(StringRef Name); void setFirstSectionContrib(const SectionContrib &SC); void addSymbol(codeview::CVSymbol Symbol); + void addSymbolsInBulk(ArrayRef BulkSymbols); void addDebugSubsection(std::shared_ptr Subsection); @@ -91,7 +92,7 @@ private: std::string ModuleName; std::string ObjFileName; std::vector SourceFiles; - std::vector Symbols; + std::vector> Symbols; std::vector> C13Builders; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h index 280615bdb50..a3ca607efbe 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h @@ -78,7 +78,7 @@ public: const DbiModuleList &modules() const; - FixedStreamArray getSectionHeaders(); + FixedStreamArray getSectionHeaders() const; FixedStreamArray getFpoRecords(); diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h index 51befcdac77..b538de57667 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h @@ -15,6 +15,7 @@ #include "llvm/BinaryFormat/COFF.h" #include "llvm/Support/Error.h" +#include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" @@ -24,11 +25,15 @@ #include "llvm/Support/Endian.h" namespace llvm { +namespace codeview { +struct FrameData; +} namespace msf { class MSFBuilder; } namespace object { struct coff_section; +struct FpoData; } namespace pdb { class DbiStream; @@ -65,6 +70,8 @@ public: void setGlobalsStreamIndex(uint32_t Index); void setPublicsStreamIndex(uint32_t Index); void setSymbolRecordStreamIndex(uint32_t Index); + void addNewFpoData(const codeview::FrameData &FD); + void addOldFpoData(const object::FpoData &Fpo); Expected addModuleInfo(StringRef ModuleName); Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module, StringRef File); @@ -84,7 +91,8 @@ public: private: struct DebugStream { - ArrayRef Data; + std::function WriteFn; + uint32_t Size = 0; uint16_t StreamNumber = kInvalidStreamIndex; }; @@ -117,6 +125,9 @@ private: std::vector> ModiList; + Optional NewFpoData; + std::vector OldFpoData; + StringMap SourceFileNames; PDBStringTableBuilder ECNamesBuilder; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h index 1a4f89d607d..4c39ca762b5 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h @@ -61,7 +61,6 @@ public: void addGlobalSymbol(const codeview::ProcRefSym &Sym); void addGlobalSymbol(const codeview::DataSym &Sym); void addGlobalSymbol(const codeview::ConstantSym &Sym); - void addGlobalSymbol(const codeview::UDTSym &Sym); void addGlobalSymbol(const codeview::CVSymbol &Sym); private: diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h index dd04b5c5681..7f84564ee98 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h @@ -10,18 +10,20 @@ #ifndef LLVM_DEBUGINFO_PDB_RAW_GLOBALS_STREAM_H #define LLVM_DEBUGINFO_PDB_RAW_GLOBALS_STREAM_H +#include "llvm/ADT/iterator.h" +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/Error.h" -#include "llvm/ADT/iterator.h" namespace llvm { namespace pdb { class DbiStream; class PDBFile; +class SymbolStream; /// Iterator over hash records producing symbol record offsets. Abstracts away /// the fact that symbol record offsets on disk are off-by-one. @@ -50,8 +52,9 @@ class GSIHashTable { public: const GSIHashHeader *HashHdr; FixedStreamArray HashRecords; - ArrayRef HashBitmap; + FixedStreamArray HashBitmap; FixedStreamArray HashBuckets; + std::array BucketMap; Error read(BinaryStreamReader &Reader); @@ -72,6 +75,9 @@ public: const GSIHashTable &getGlobalsTable() const { return GlobalsTable; } Error reload(); + std::vector> + findRecordsByName(StringRef Name, const SymbolStream &Symbols) const; + private: GSIHashTable GlobalsTable; std::unique_ptr Stream; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h index 419e8ada06f..101127a355f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h @@ -35,11 +35,18 @@ public: InfoStreamBuilder &operator=(const InfoStreamBuilder &) = delete; void setVersion(PdbRaw_ImplVer V); + void addFeature(PdbRaw_FeatureSig Sig); + + // If this is true, the PDB contents are hashed and this hash is used as + // PDB GUID and as Signature. The age is always 1. + void setHashPDBContentsToGUID(bool B); + + // These only have an effect if hashPDBContentsToGUID() is false. void setSignature(uint32_t S); void setAge(uint32_t A); void setGuid(codeview::GUID G); - void addFeature(PdbRaw_FeatureSig Sig); + bool hashPDBContentsToGUID() const { return HashPDBContentsToGUID; } uint32_t getAge() const { return Age; } codeview::GUID getGuid() const { return Guid; } Optional getSignature() const { return Signature; } @@ -60,6 +67,8 @@ private: Optional Signature; codeview::GUID Guid; + bool HashPDBContentsToGUID = false; + NamedStreamMap &NamedStreams; }; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h index efc25e0559b..8d590df288f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h @@ -15,6 +15,7 @@ #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" +#include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Error.h" #include @@ -43,6 +44,8 @@ public: symbols(bool *HadError) const; const codeview::CVSymbolArray &getSymbolArray() const { return SymbolArray; } + const codeview::CVSymbolArray + getSymbolArrayForScope(uint32_t ScopeBegin) const; BinarySubstreamRef getSymbolsSubstream() const; BinarySubstreamRef getC11LinesSubstream() const; @@ -51,6 +54,8 @@ public: ModuleDebugStreamRef &operator=(ModuleDebugStreamRef &&Other) = delete; + codeview::CVSymbol readSymbolAtOffset(uint32_t Offset) const; + iterator_range subsections() const; codeview::DebugSubsectionArray getSubsectionsArray() const { return Subsections; @@ -64,7 +69,7 @@ public: findChecksumsSubsection() const; private: - const DbiModuleDescriptor &Mod; + DbiModuleDescriptor Mod; uint32_t Signature; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h index bd5c09e5ff7..3cd46550304 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h @@ -21,11 +21,12 @@ public: NativeCompilandSymbol(NativeSession &Session, SymIndexId SymbolId, DbiModuleDescriptor MI); - std::unique_ptr clone() const override; + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; PDB_SymType getSymTag() const override; bool isEditAndContinueEnabled() const override; - uint32_t getLexicalParentId() const override; + SymIndexId getLexicalParentId() const override; std::string getLibraryName() const override; std::string getName() const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h new file mode 100644 index 00000000000..4442a1ec41f --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h @@ -0,0 +1,43 @@ +//==- NativeEnumGlobals.h - Native Global Enumerator impl --------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMGLOBALS_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMGLOBALS_H + +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" +#include "llvm/DebugInfo/PDB/PDBSymbol.h" + +#include + +namespace llvm { +namespace pdb { + +class NativeSession; + +class NativeEnumGlobals : public IPDBEnumChildren { +public: + NativeEnumGlobals(NativeSession &Session, + std::vector Kinds); + + uint32_t getChildCount() const override; + std::unique_ptr getChildAtIndex(uint32_t Index) const override; + std::unique_ptr getNext() override; + void reset() override; + +private: + std::vector MatchOffsets; + uint32_t Index; + NativeSession &Session; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h index 6aa1460dbb4..c268641a100 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h @@ -11,28 +11,23 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMMODULES_H #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" -#include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" #include "llvm/DebugInfo/PDB/PDBSymbol.h" namespace llvm { namespace pdb { -class DbiModuleList; class NativeSession; class NativeEnumModules : public IPDBEnumChildren { public: - NativeEnumModules(NativeSession &Session, const DbiModuleList &Modules, - uint32_t Index = 0); + NativeEnumModules(NativeSession &Session, uint32_t Index = 0); uint32_t getChildCount() const override; std::unique_ptr getChildAtIndex(uint32_t Index) const override; std::unique_ptr getNext() override; void reset() override; - NativeEnumModules *clone() const override; private: NativeSession &Session; - const DbiModuleList &Modules; uint32_t Index; }; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbol.h deleted file mode 100644 index 41b7b78b8d8..00000000000 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbol.h +++ /dev/null @@ -1,60 +0,0 @@ -//===- NativeEnumSymbol.h - info about enum type ----------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOL_H -#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOL_H - -#include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" -#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" - -namespace llvm { -namespace pdb { - -class NativeEnumSymbol : public NativeRawSymbol, - public codeview::TypeVisitorCallbacks { -public: - NativeEnumSymbol(NativeSession &Session, SymIndexId Id, - const codeview::CVType &CV); - ~NativeEnumSymbol() override; - - std::unique_ptr clone() const override; - - std::unique_ptr - findChildren(PDB_SymType Type) const override; - - Error visitKnownRecord(codeview::CVType &CVR, - codeview::EnumRecord &Record) override; - Error visitKnownMember(codeview::CVMemberRecord &CVM, - codeview::EnumeratorRecord &Record) override; - - PDB_SymType getSymTag() const override; - uint32_t getClassParentId() const override; - uint32_t getUnmodifiedTypeId() const override; - bool hasConstructor() const override; - bool hasAssignmentOperator() const override; - bool hasCastOperator() const override; - uint64_t getLength() const override; - std::string getName() const override; - bool isNested() const override; - bool hasOverloadedOperator() const override; - bool isPacked() const override; - bool isScoped() const override; - uint32_t getTypeId() const override; - -protected: - codeview::CVType CV; - codeview::EnumRecord Record; -}; - -} // namespace pdb -} // namespace llvm - -#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOL_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h index e0a5c8d9ad8..f8ac1655dc6 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h @@ -26,23 +26,20 @@ class NativeEnumTypes : public IPDBEnumChildren { public: NativeEnumTypes(NativeSession &Session, codeview::LazyRandomTypeCollection &TypeCollection, - codeview::TypeLeafKind Kind); + std::vector Kinds); + + NativeEnumTypes(NativeSession &Session, + std::vector Indices); uint32_t getChildCount() const override; std::unique_ptr getChildAtIndex(uint32_t Index) const override; std::unique_ptr getNext() override; void reset() override; - NativeEnumTypes *clone() const override; private: - NativeEnumTypes(NativeSession &Session, - const std::vector &Matches, - codeview::TypeLeafKind Kind); - std::vector Matches; uint32_t Index; NativeSession &Session; - codeview::TypeLeafKind Kind; }; } // namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h index 587c7ff2b09..f4030da1d02 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h @@ -16,11 +16,14 @@ namespace llvm { namespace pdb { -class NativeExeSymbol : public NativeRawSymbol { -public: - NativeExeSymbol(NativeSession &Session, SymIndexId SymbolId); +class DbiStream; - std::unique_ptr clone() const override; +class NativeExeSymbol : public NativeRawSymbol { + // EXE symbol is the authority on the various symbol types. + DbiStream *Dbi = nullptr; + +public: + NativeExeSymbol(NativeSession &Session, SymIndexId Id); std::unique_ptr findChildren(PDB_SymType Type) const override; @@ -30,9 +33,6 @@ public: codeview::GUID getGuid() const override; bool hasCTypes() const override; bool hasPrivateSymbols() const override; - -private: - PDBFile &File; }; } // namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h index 5b70ecfa205..6505a7d3957 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h @@ -19,15 +19,16 @@ namespace pdb { class NativeSession; -typedef uint32_t SymIndexId; - class NativeRawSymbol : public IPDBRawSymbol { + friend class SymbolCache; + virtual void initialize() {} + public: - NativeRawSymbol(NativeSession &PDBSession, SymIndexId SymbolId); + NativeRawSymbol(NativeSession &PDBSession, PDB_SymType Tag, + SymIndexId SymbolId); - virtual std::unique_ptr clone() const = 0; - - void dump(raw_ostream &OS, int Indent) const override; + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; std::unique_ptr findChildren(PDB_SymType Type) const override; @@ -68,25 +69,25 @@ public: uint32_t getAddressOffset() const override; uint32_t getAddressSection() const override; uint32_t getAge() const override; - uint32_t getArrayIndexTypeId() const override; + SymIndexId getArrayIndexTypeId() const override; uint32_t getBaseDataOffset() const override; uint32_t getBaseDataSlot() const override; - uint32_t getBaseSymbolId() const override; + SymIndexId getBaseSymbolId() const override; PDB_BuiltinType getBuiltinType() const override; uint32_t getBitPosition() const override; PDB_CallingConv getCallingConvention() const override; - uint32_t getClassParentId() const override; + SymIndexId getClassParentId() const override; std::string getCompilerName() const override; uint32_t getCount() const override; uint32_t getCountLiveRanges() const override; PDB_Lang getLanguage() const override; - uint32_t getLexicalParentId() const override; + SymIndexId getLexicalParentId() const override; std::string getLibraryName() const override; uint32_t getLiveRangeStartAddressOffset() const override; uint32_t getLiveRangeStartAddressSection() const override; uint32_t getLiveRangeStartRelativeVirtualAddress() const override; codeview::RegisterId getLocalBasePointerRegisterId() const override; - uint32_t getLowerBoundId() const override; + SymIndexId getLowerBoundId() const override; uint32_t getMemorySpaceKind() const override; std::string getName() const override; uint32_t getNumberOfAcceleratorPointerTags() const override; @@ -96,7 +97,7 @@ public: uint32_t getNumberOfRows() const override; std::string getObjectFileName() const override; uint32_t getOemId() const override; - uint32_t getOemSymbolId() const override; + SymIndexId getOemSymbolId() const override; uint32_t getOffsetInUdt() const override; PDB_Cpu getPlatform() const override; uint32_t getRank() const override; @@ -110,9 +111,9 @@ public: std::string getSourceFileName() const override; std::unique_ptr getSrcLineOnTypeDefn() const override; uint32_t getStride() const override; - uint32_t getSubTypeId() const override; + SymIndexId getSubTypeId() const override; std::string getSymbolsFileName() const override; - uint32_t getSymIndexId() const override; + SymIndexId getSymIndexId() const override; uint32_t getTargetOffset() const override; uint32_t getTargetRelativeVirtualAddress() const override; uint64_t getTargetVirtualAddress() const override; @@ -120,16 +121,16 @@ public: uint32_t getTextureSlot() const override; uint32_t getTimeStamp() const override; uint32_t getToken() const override; - uint32_t getTypeId() const override; + SymIndexId getTypeId() const override; uint32_t getUavSlot() const override; std::string getUndecoratedName() const override; std::string getUndecoratedNameEx(PDB_UndnameFlags Flags) const override; - uint32_t getUnmodifiedTypeId() const override; - uint32_t getUpperBoundId() const override; + SymIndexId getUnmodifiedTypeId() const override; + SymIndexId getUpperBoundId() const override; Variant getValue() const override; uint32_t getVirtualBaseDispIndex() const override; uint32_t getVirtualBaseOffset() const override; - uint32_t getVirtualTableShapeId() const override; + SymIndexId getVirtualTableShapeId() const override; std::unique_ptr getVirtualBaseTableType() const override; PDB_DataKind getDataKind() const override; @@ -230,6 +231,7 @@ public: protected: NativeSession &Session; + PDB_SymType Tag; SymIndexId SymbolId; }; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h index aff7ef2f8f2..4878e47d312 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h @@ -15,9 +15,8 @@ #include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/IPDBSession.h" -#include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" -#include "llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/SymbolCache.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" @@ -25,6 +24,7 @@ namespace llvm { class MemoryBuffer; namespace pdb { class PDBFile; +class NativeExeSymbol; class NativeSession : public IPDBSession { public: @@ -37,21 +37,10 @@ public: static Error createFromExe(StringRef Path, std::unique_ptr &Session); - std::unique_ptr - createCompilandSymbol(DbiModuleDescriptor MI); - - std::unique_ptr - createEnumSymbol(codeview::TypeIndex Index); - - std::unique_ptr - createTypeEnumerator(codeview::TypeLeafKind Kind); - - SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI); - uint64_t getLoadAddress() const override; bool setLoadAddress(uint64_t Address) override; std::unique_ptr getGlobalScope() override; - std::unique_ptr getSymbolById(uint32_t SymbolId) const override; + std::unique_ptr getSymbolById(SymIndexId SymbolId) const override; bool addressForVA(uint64_t VA, uint32_t &Section, uint32_t &Offset) const override; @@ -104,14 +93,23 @@ public: std::unique_ptr getSectionContribs() const override; + std::unique_ptr getFrameData() const override; + PDBFile &getPDBFile() { return *Pdb; } const PDBFile &getPDBFile() const { return *Pdb; } + NativeExeSymbol &getNativeGlobalScope() const; + SymbolCache &getSymbolCache() { return Cache; } + const SymbolCache &getSymbolCache() const { return Cache; } + private: + void initializeExeSymbol(); + std::unique_ptr Pdb; std::unique_ptr Allocator; - std::vector> SymbolCache; - DenseMap TypeIndexToSymbolId; + + SymbolCache Cache; + SymIndexId ExeSymbol = 0; }; } // namespace pdb } // namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h new file mode 100644 index 00000000000..acc5eb8ff2c --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h @@ -0,0 +1,51 @@ +//===- NativeSymbolEnumerator.h - info about enumerator values --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVESYMBOLENUMERATOR_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVESYMBOLENUMERATOR_H + +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { +class NativeTypeEnum; + +class NativeSymbolEnumerator : public NativeRawSymbol { +public: + NativeSymbolEnumerator(NativeSession &Session, SymIndexId Id, + const NativeTypeEnum &Parent, + codeview::EnumeratorRecord Record); + + ~NativeSymbolEnumerator() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + SymIndexId getClassParentId() const override; + SymIndexId getLexicalParentId() const override; + std::string getName() const override; + SymIndexId getTypeId() const override; + PDB_DataKind getDataKind() const override; + PDB_LocType getLocationType() const override; + bool isConstType() const override; + bool isVolatileType() const override; + bool isUnalignedType() const override; + Variant getValue() const override; + +protected: + const NativeTypeEnum &Parent; + codeview::EnumeratorRecord Record; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEENUM_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h new file mode 100644 index 00000000000..10e68e6df45 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h @@ -0,0 +1,50 @@ +//===- NativeTypeArray.h ------------------------------------------ C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEARRAY_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEARRAY_H + +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" + +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/PDBTypes.h" + +namespace llvm { +namespace pdb { + +class NativeSession; + +class NativeTypeArray : public NativeRawSymbol { +public: + NativeTypeArray(NativeSession &Session, SymIndexId Id, codeview::TypeIndex TI, + codeview::ArrayRecord Record); + ~NativeTypeArray() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + SymIndexId getArrayIndexTypeId() const override; + + bool isConstType() const override; + bool isUnalignedType() const override; + bool isVolatileType() const override; + + uint32_t getCount() const override; + SymIndexId getTypeId() const override; + uint64_t getLength() const override; + +protected: + codeview::ArrayRecord Record; + codeview::TypeIndex Index; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h similarity index 58% rename from contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h rename to contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h index 4f532c6e382..725dfb89222 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h @@ -1,4 +1,4 @@ -//===- NativeBuiltinSymbol.h -------------------------------------- C++ -*-===// +//===- NativeTypeBuiltin.h ---------------------------------------- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEBUILTINSYMBOL_H -#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEBUILTINSYMBOL_H +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEBUILTIN_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEBUILTIN_H #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" @@ -19,15 +19,15 @@ namespace pdb { class NativeSession; -class NativeBuiltinSymbol : public NativeRawSymbol { +class NativeTypeBuiltin : public NativeRawSymbol { public: - NativeBuiltinSymbol(NativeSession &PDBSession, SymIndexId Id, - PDB_BuiltinType T, uint64_t L); - ~NativeBuiltinSymbol() override; + NativeTypeBuiltin(NativeSession &PDBSession, SymIndexId Id, + codeview::ModifierOptions Mods, PDB_BuiltinType T, + uint64_t L); + ~NativeTypeBuiltin() override; - virtual std::unique_ptr clone() const override; - - void dump(raw_ostream &OS, int Indent) const override; + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; PDB_SymType getSymTag() const override; @@ -39,6 +39,7 @@ public: protected: NativeSession &Session; + codeview::ModifierOptions Mods; PDB_BuiltinType Type; uint64_t Length; }; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h new file mode 100644 index 00000000000..a5cbefc1811 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h @@ -0,0 +1,75 @@ +//===- NativeTypeEnum.h - info about enum type ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEENUM_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEENUM_H + +#include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeTypeBuiltin; + +class NativeTypeEnum : public NativeRawSymbol { +public: + NativeTypeEnum(NativeSession &Session, SymIndexId Id, codeview::TypeIndex TI, + codeview::EnumRecord Record); + + NativeTypeEnum(NativeSession &Session, SymIndexId Id, + NativeTypeEnum &UnmodifiedType, + codeview::ModifierRecord Modifier); + ~NativeTypeEnum() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + std::unique_ptr + findChildren(PDB_SymType Type) const override; + + PDB_BuiltinType getBuiltinType() const override; + PDB_SymType getSymTag() const override; + SymIndexId getUnmodifiedTypeId() const override; + bool hasConstructor() const override; + bool hasAssignmentOperator() const override; + bool hasCastOperator() const override; + uint64_t getLength() const override; + std::string getName() const override; + bool isConstType() const override; + bool isVolatileType() const override; + bool isUnalignedType() const override; + bool isNested() const override; + bool hasOverloadedOperator() const override; + bool hasNestedTypes() const override; + bool isIntrinsic() const override; + bool isPacked() const override; + bool isScoped() const override; + SymIndexId getTypeId() const override; + bool isRefUdt() const override; + bool isValueUdt() const override; + bool isInterfaceUdt() const override; + + const NativeTypeBuiltin &getUnderlyingBuiltinType() const; + const codeview::EnumRecord &getEnumRecord() const { return *Record; } + +protected: + codeview::TypeIndex Index; + Optional Record; + NativeTypeEnum *UnmodifiedType = nullptr; + Optional Modifiers; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEENUM_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h new file mode 100644 index 00000000000..1b1b87f6581 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h @@ -0,0 +1,74 @@ +//===- NativeTypeFunctionSig.h - info about function signature ---*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEFUNCTIONSIG_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEFUNCTIONSIG_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeTypeUDT; + +class NativeTypeFunctionSig : public NativeRawSymbol { +protected: + void initialize() override; + +public: + NativeTypeFunctionSig(NativeSession &Session, SymIndexId Id, + codeview::TypeIndex TI, codeview::ProcedureRecord Proc); + + NativeTypeFunctionSig(NativeSession &Session, SymIndexId Id, + codeview::TypeIndex TI, + codeview::MemberFunctionRecord MemberFunc); + + ~NativeTypeFunctionSig() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + std::unique_ptr + findChildren(PDB_SymType Type) const override; + + SymIndexId getClassParentId() const override; + PDB_CallingConv getCallingConvention() const override; + uint32_t getCount() const override; + SymIndexId getTypeId() const override; + int32_t getThisAdjust() const override; + bool hasConstructor() const override; + bool isConstType() const override; + bool isConstructorVirtualBase() const override; + bool isCxxReturnUdt() const override; + bool isUnalignedType() const override; + bool isVolatileType() const override; + +private: + void initializeArgList(codeview::TypeIndex ArgListTI); + + union { + codeview::MemberFunctionRecord MemberFunc; + codeview::ProcedureRecord Proc; + }; + + SymIndexId ClassParentId = 0; + codeview::TypeIndex Index; + codeview::ArgListRecord ArgList; + bool IsMemberFunction = false; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEPOINTER_H \ No newline at end of file diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h new file mode 100644 index 00000000000..bcb7431fecf --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h @@ -0,0 +1,61 @@ +//===- NativeTypePointer.h - info about pointer type -------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEPOINTER_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEPOINTER_H + +#include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeTypePointer : public NativeRawSymbol { +public: + // Create a pointer record for a simple type. + NativeTypePointer(NativeSession &Session, SymIndexId Id, + codeview::TypeIndex TI); + + // Create a pointer record for a non-simple type. + NativeTypePointer(NativeSession &Session, SymIndexId Id, + codeview::TypeIndex TI, codeview::PointerRecord PR); + ~NativeTypePointer() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + SymIndexId getClassParentId() const override; + bool isConstType() const override; + uint64_t getLength() const override; + bool isReference() const override; + bool isRValueReference() const override; + bool isPointerToDataMember() const override; + bool isPointerToMemberFunction() const override; + SymIndexId getTypeId() const override; + bool isRestrictedType() const override; + bool isVolatileType() const override; + bool isUnalignedType() const override; + + bool isSingleInheritance() const override; + bool isMultipleInheritance() const override; + bool isVirtualInheritance() const override; + +protected: + bool isMemberPointer() const; + codeview::TypeIndex TI; + Optional Record; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEPOINTER_H \ No newline at end of file diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h new file mode 100644 index 00000000000..06eb6fcf376 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h @@ -0,0 +1,42 @@ +//===- NativeTypeTypedef.h - info about typedef ------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPETYPEDEF_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPETYPEDEF_H + +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeTypeTypedef : public NativeRawSymbol { +public: + // Create a pointer record for a non-simple type. + NativeTypeTypedef(NativeSession &Session, SymIndexId Id, + codeview::UDTSym Typedef); + + ~NativeTypeTypedef() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + std::string getName() const override; + SymIndexId getTypeId() const override; + +protected: + codeview::UDTSym Record; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEPOINTER_H \ No newline at end of file diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h new file mode 100644 index 00000000000..84821d8731b --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h @@ -0,0 +1,74 @@ +//===- NativeTypeUDT.h - info about class/struct type ------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEUDT_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEUDT_H + +#include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeTypeUDT : public NativeRawSymbol { +public: + NativeTypeUDT(NativeSession &Session, SymIndexId Id, codeview::TypeIndex TI, + codeview::ClassRecord Class); + + NativeTypeUDT(NativeSession &Session, SymIndexId Id, codeview::TypeIndex TI, + codeview::UnionRecord Union); + + NativeTypeUDT(NativeSession &Session, SymIndexId Id, + NativeTypeUDT &UnmodifiedType, + codeview::ModifierRecord Modifier); + + ~NativeTypeUDT() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + std::string getName() const override; + SymIndexId getLexicalParentId() const override; + SymIndexId getUnmodifiedTypeId() const override; + SymIndexId getVirtualTableShapeId() const override; + uint64_t getLength() const override; + PDB_UdtType getUdtKind() const override; + bool hasConstructor() const override; + bool isConstType() const override; + bool hasAssignmentOperator() const override; + bool hasCastOperator() const override; + bool hasNestedTypes() const override; + bool hasOverloadedOperator() const override; + bool isInterfaceUdt() const override; + bool isIntrinsic() const override; + bool isNested() const override; + bool isPacked() const override; + bool isRefUdt() const override; + bool isScoped() const override; + bool isValueUdt() const override; + bool isUnalignedType() const override; + bool isVolatileType() const override; + +protected: + codeview::TypeIndex Index; + + Optional Class; + Optional Union; + NativeTypeUDT *UnmodifiedType = nullptr; + codeview::TagRecord *Tag = nullptr; + Optional Modifiers; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEUDT_H \ No newline at end of file diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h new file mode 100644 index 00000000000..a996f34ef85 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h @@ -0,0 +1,46 @@ +//===- NativeTypeVTShape.h - info about virtual table shape ------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEVTSHAPE_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEVTSHAPE_H + +#include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeTypeVTShape : public NativeRawSymbol { +public: + // Create a pointer record for a non-simple type. + NativeTypeVTShape(NativeSession &Session, SymIndexId Id, + codeview::TypeIndex TI, codeview::VFTableShapeRecord SR); + + ~NativeTypeVTShape() override; + + void dump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowIdFields, + PdbSymbolIdField RecurseIdFields) const override; + + bool isConstType() const override; + bool isVolatileType() const override; + bool isUnalignedType() const override; + uint32_t getCount() const override; + +protected: + codeview::TypeIndex TI; + codeview::VFTableShapeRecord Record; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVETYPEVTSHAPE_H \ No newline at end of file diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h index 7f9c4cf9fa8..37458749a8d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -53,7 +53,9 @@ public: PDBStringTableBuilder &getStringTableBuilder(); GSIStreamBuilder &getGsiBuilder(); - Error commit(StringRef Filename); + // If HashPDBContentsToGUID is true on the InfoStreamBuilder, Guid is filled + // with the computed PDB GUID on return. + Error commit(StringRef Filename, codeview::GUID *Guid); Expected getNamedStreamIndex(StringRef Name) const; Error addNamedStream(StringRef Name, StringRef Data); diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h index 3624a7682e3..97d11b4f20d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h @@ -31,23 +31,29 @@ enum class raw_error_code { stream_too_long, invalid_tpi_hash, }; +} // namespace pdb +} // namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} // namespace std + +namespace llvm { +namespace pdb { +const std::error_category &RawErrCategory(); + +inline std::error_code make_error_code(raw_error_code E) { + return std::error_code(static_cast(E), RawErrCategory()); +} /// Base class for errors originating when parsing raw PDB files -class RawError : public ErrorInfo { +class RawError : public ErrorInfo { public: + using ErrorInfo::ErrorInfo; // inherit constructors + RawError(const Twine &S) : ErrorInfo(S, raw_error_code::unspecified) {} static char ID; - RawError(raw_error_code C); - RawError(const std::string &Context); - RawError(raw_error_code C, const std::string &Context); - - void log(raw_ostream &OS) const override; - const std::string &getErrorMessage() const; - std::error_code convertToErrorCode() const override; - -private: - std::string ErrMsg; - raw_error_code Code; }; -} -} +} // namespace pdb +} // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h index 19f592d562e..8f6d6611c03 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h @@ -343,7 +343,6 @@ struct SrcHeaderBlockEntry { char Reserved[8]; }; -constexpr int I = sizeof(SrcHeaderBlockEntry); static_assert(sizeof(SrcHeaderBlockEntry) == 40, "Incorrect struct size!"); } // namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h new file mode 100644 index 00000000000..08e1d41e6ee --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h @@ -0,0 +1,148 @@ +//==- SymbolCache.h - Cache of native symbols and ids ------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_SYMBOLCACHE_H +#define LLVM_DEBUGINFO_PDB_NATIVE_SYMBOLCACHE_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/DebugInfo/CodeView/TypeDeserializer.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/Support/Allocator.h" + +#include +#include + +namespace llvm { +namespace pdb { +class DbiStream; +class PDBFile; + +class SymbolCache { + NativeSession &Session; + DbiStream *Dbi = nullptr; + + /// Cache of all stable symbols, indexed by SymIndexId. Just because a + /// symbol has been parsed does not imply that it will be stable and have + /// an Id. Id allocation is an implementation, with the only guarantee + /// being that once an Id is allocated, the symbol can be assumed to be + /// cached. + std::vector> Cache; + + /// For type records from the TPI stream which have been paresd and cached, + /// stores a mapping to SymIndexId of the cached symbol. + DenseMap TypeIndexToSymbolId; + + /// For field list members which have been parsed and cached, stores a mapping + /// from (IndexOfClass, MemberIndex) to the corresponding SymIndexId of the + /// cached symbol. + DenseMap, SymIndexId> + FieldListMembersToSymbolId; + + /// List of SymIndexIds for each compiland, indexed by compiland index as they + /// appear in the PDB file. + std::vector Compilands; + + /// Map from global symbol offset to SymIndexId. + DenseMap GlobalOffsetToSymbolId; + + SymIndexId createSymbolPlaceholder() { + SymIndexId Id = Cache.size(); + Cache.push_back(nullptr); + return Id; + } + + template + SymIndexId createSymbolForType(codeview::TypeIndex TI, codeview::CVType CVT, + Args &&... ConstructorArgs) { + CVRecordT Record; + if (auto EC = + codeview::TypeDeserializer::deserializeAs(CVT, Record)) { + consumeError(std::move(EC)); + return 0; + } + + return createSymbol( + TI, std::move(Record), std::forward(ConstructorArgs)...); + } + + SymIndexId createSymbolForModifiedType(codeview::TypeIndex ModifierTI, + codeview::CVType CVT); + + SymIndexId createSimpleType(codeview::TypeIndex TI, + codeview::ModifierOptions Mods); + +public: + SymbolCache(NativeSession &Session, DbiStream *Dbi); + + template + SymIndexId createSymbol(Args &&... ConstructorArgs) { + SymIndexId Id = Cache.size(); + + // Initial construction must not access the cache, since it must be done + // atomically. + auto Result = llvm::make_unique( + Session, Id, std::forward(ConstructorArgs)...); + Result->SymbolId = Id; + + NativeRawSymbol *NRS = static_cast(Result.get()); + Cache.push_back(std::move(Result)); + + // After the item is in the cache, we can do further initialization which + // is then allowed to access the cache. + NRS->initialize(); + return Id; + } + + std::unique_ptr + createTypeEnumerator(codeview::TypeLeafKind Kind); + + std::unique_ptr + createTypeEnumerator(std::vector Kinds); + + std::unique_ptr + createGlobalsEnumerator(codeview::SymbolKind Kind); + + SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI); + + template + SymIndexId getOrCreateFieldListMember(codeview::TypeIndex FieldListTI, + uint32_t Index, + Args &&... ConstructorArgs) { + SymIndexId SymId = Cache.size(); + std::pair Key{FieldListTI, Index}; + auto Result = FieldListMembersToSymbolId.try_emplace(Key, SymId); + if (Result.second) + SymId = + createSymbol(std::forward(ConstructorArgs)...); + else + SymId = Result.first->second; + return SymId; + } + + SymIndexId getOrCreateGlobalSymbolByOffset(uint32_t Offset); + + std::unique_ptr getOrCreateCompiland(uint32_t Index); + uint32_t getNumCompilands() const; + + std::unique_ptr getSymbolById(SymIndexId SymbolId) const; + + NativeRawSymbol &getNativeSymbolById(SymIndexId SymbolId) const; + + template + ConcreteT &getNativeSymbolById(SymIndexId SymbolId) const { + return static_cast(getNativeSymbolById(SymbolId)); + } +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h index c1edec7a26f..c2996ccf182 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h @@ -18,6 +18,54 @@ namespace pdb { Expected hashTypeRecord(const llvm::codeview::CVType &Type); +struct TagRecordHash { + explicit TagRecordHash(codeview::ClassRecord CR, uint32_t Full, + uint32_t Forward) + : FullRecordHash(Full), ForwardDeclHash(Forward), Class(std::move(CR)) { + State = 0; + } + + explicit TagRecordHash(codeview::EnumRecord ER, uint32_t Full, + uint32_t Forward) + : FullRecordHash(Full), ForwardDeclHash(Forward), Enum(std::move(ER)) { + State = 1; + } + + explicit TagRecordHash(codeview::UnionRecord UR, uint32_t Full, + uint32_t Forward) + : FullRecordHash(Full), ForwardDeclHash(Forward), Union(std::move(UR)) { + State = 2; + } + + uint32_t FullRecordHash; + uint32_t ForwardDeclHash; + + codeview::TagRecord &getRecord() { + switch (State) { + case 0: + return Class; + case 1: + return Enum; + case 2: + return Union; + } + llvm_unreachable("unreachable!"); + } + +private: + union { + codeview::ClassRecord Class; + codeview::EnumRecord Enum; + codeview::UnionRecord Union; + }; + + uint8_t State = 0; +}; + +/// Given a CVType referring to a class, structure, union, or enum, compute +/// the hash of its forward decl and full decl. +Expected hashTagRecord(const codeview::CVType &Type); + } // end namespace pdb } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h index b77939929ec..b76576a7a26 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h @@ -58,10 +58,21 @@ public: codeview::LazyRandomTypeCollection &typeCollection() { return *Types; } + Expected + findFullDeclForForwardRef(codeview::TypeIndex ForwardRefTI) const; + + std::vector findRecordsByName(StringRef Name) const; + + codeview::CVType getType(codeview::TypeIndex Index); + BinarySubstreamRef getTypeRecordsSubstream() const; Error commit(); + void buildHashMap(); + + bool supportsTypeLookup() const; + private: PDBFile &Pdb; std::unique_ptr Stream; @@ -77,6 +88,8 @@ private: FixedStreamArray TypeIndexOffsets; HashTable HashAdjusters; + std::vector> HashMap; + const TpiStreamHeader *Header; }; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h index 3c9a19801f8..aaec71aa8c9 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h @@ -12,6 +12,8 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" +#include "llvm/Support/raw_ostream.h" + #include namespace llvm { @@ -24,6 +26,7 @@ using TagStats = std::unordered_map; raw_ostream &operator<<(raw_ostream &OS, const PDB_VariantType &Value); raw_ostream &operator<<(raw_ostream &OS, const PDB_CallingConv &Conv); +raw_ostream &operator<<(raw_ostream &OS, const PDB_BuiltinType &Type); raw_ostream &operator<<(raw_ostream &OS, const PDB_DataKind &Data); raw_ostream &operator<<(raw_ostream &OS, const codeview::RegisterId &Reg); raw_ostream &operator<<(raw_ostream &OS, const PDB_LocType &Loc); @@ -41,6 +44,15 @@ raw_ostream &operator<<(raw_ostream &OS, const Variant &Value); raw_ostream &operator<<(raw_ostream &OS, const VersionInfo &Version); raw_ostream &operator<<(raw_ostream &OS, const TagStats &Stats); + +template +void dumpSymbolField(raw_ostream &OS, StringRef Name, T Value, int Indent) { + OS << "\n"; + OS.indent(Indent); + OS << Name << ": " << Value; +} + + } // end namespace pdb } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h index 04373463212..3a74f7c3aac 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h @@ -49,9 +49,22 @@ class IPDBRawSymbol; class IPDBSession; #define DECLARE_PDB_SYMBOL_CONCRETE_TYPE(TagValue) \ +private: \ + using PDBSymbol::PDBSymbol; \ + friend class PDBSymbol; \ + \ +public: \ static const PDB_SymType Tag = TagValue; \ static bool classof(const PDBSymbol *S) { return S->getSymTag() == Tag; } +#define DECLARE_PDB_SYMBOL_CUSTOM_TYPE(Condition) \ +private: \ + using PDBSymbol::PDBSymbol; \ + friend class PDBSymbol; \ + \ +public: \ + static bool classof(const PDBSymbol *S) { return Condition; } + /// PDBSymbol defines the base of the inheritance hierarchy for concrete symbol /// types (e.g. functions, executables, vtables, etc). All concrete symbol /// types inherit from PDBSymbol and expose the exact set of methods that are @@ -59,14 +72,33 @@ class IPDBSession; /// reference "Lexical and Class Hierarchy of Symbol Types": /// https://msdn.microsoft.com/en-us/library/370hs6k4.aspx class PDBSymbol { + static std::unique_ptr createSymbol(const IPDBSession &PDBSession, + PDB_SymType Tag); + protected: - PDBSymbol(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - PDBSymbol(PDBSymbol &Symbol); + explicit PDBSymbol(const IPDBSession &PDBSession); + PDBSymbol(PDBSymbol &&Other); public: static std::unique_ptr - create(const IPDBSession &PDBSession, std::unique_ptr Symbol); + create(const IPDBSession &PDBSession, + std::unique_ptr RawSymbol); + static std::unique_ptr create(const IPDBSession &PDBSession, + IPDBRawSymbol &RawSymbol); + + template + static std::unique_ptr + createAs(const IPDBSession &PDBSession, + std::unique_ptr RawSymbol) { + std::unique_ptr S = create(PDBSession, std::move(RawSymbol)); + return unique_dyn_cast_or_null(std::move(S)); + } + template + static std::unique_ptr createAs(const IPDBSession &PDBSession, + IPDBRawSymbol &RawSymbol) { + std::unique_ptr S = create(PDBSession, RawSymbol); + return unique_dyn_cast_or_null(std::move(S)); + } virtual ~PDBSymbol(); @@ -80,7 +112,8 @@ public: /// normally goes on the right side of the symbol. virtual void dumpRight(PDBSymDumper &Dumper) const {} - void defaultDump(raw_ostream &OS, int Indent) const; + void defaultDump(raw_ostream &OS, int Indent, PdbSymbolIdField ShowFlags, + PdbSymbolIdField RecurseFlags) const; void dumpProperties() const; void dumpChildStats() const; @@ -94,8 +127,6 @@ public: return Enumerator->getNext(); } - std::unique_ptr clone() const; - template std::unique_ptr> findAllChildren() const { auto BaseIter = RawSymbol->findChildren(T::Tag); @@ -131,7 +162,8 @@ protected: } const IPDBSession &Session; - std::unique_ptr RawSymbol; + std::unique_ptr OwnedRawSymbol; + IPDBRawSymbol *RawSymbol = nullptr; }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h index 3169146e5b1..ef00df15cb0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h @@ -18,12 +18,9 @@ class raw_ostream; namespace pdb { class PDBSymbolAnnotation : public PDBSymbol { -public: - PDBSymbolAnnotation(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Annotation) +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAddressOffset) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h index d81da1eaa02..2cf9c72a888 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolBlock : public PDBSymbol { -public: - PDBSymbolBlock(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Block) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAddressOffset) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h index 9549089c7eb..04dbd962ebd 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h @@ -20,12 +20,8 @@ class raw_ostream; namespace pdb { class PDBSymbolCompiland : public PDBSymbol { -public: - PDBSymbolCompiland(const IPDBSession &PDBSession, - std::unique_ptr CompilandSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Compiland) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(isEditAndContinueEnabled) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h index dba50c42cf8..3d651a464d9 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolCompilandDetails : public PDBSymbol { -public: - PDBSymbolCompilandDetails(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::CompilandDetails) - +public: void dump(PDBSymDumper &Dumper) const override; void getFrontEndVersion(VersionInfo &Version) const { diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h index 7868f045908..ffc408314d9 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h @@ -18,12 +18,8 @@ namespace llvm { class raw_ostream; namespace pdb { class PDBSymbolCompilandEnv : public PDBSymbol { -public: - PDBSymbolCompilandEnv(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::CompilandEnv) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_ID_METHOD(getLexicalParent) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h index 54f08940426..c29e4c31d3f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h @@ -23,12 +23,8 @@ namespace pdb { /// fit anywhere else in the lexical hierarchy. /// https://msdn.microsoft.com/en-us/library/d88sf09h.aspx class PDBSymbolCustom : public PDBSymbol { -public: - PDBSymbolCustom(const IPDBSession &PDBSession, - std::unique_ptr CustomSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Custom) - +public: void dump(PDBSymDumper &Dumper) const override; void getDataBytes(llvm::SmallVector &bytes); diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h index 76b14bf1778..217e1e976e6 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h @@ -21,12 +21,8 @@ class raw_ostream; namespace pdb { class PDBSymbolData : public PDBSymbol { -public: - PDBSymbolData(const IPDBSession &PDBSession, - std::unique_ptr DataSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Data) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAccess) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h index 2c2d7466504..366d0cf4777 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h @@ -20,12 +20,8 @@ class raw_ostream; namespace pdb { class PDBSymbolExe : public PDBSymbol { -public: - PDBSymbolExe(const IPDBSession &PDBSession, - std::unique_ptr ExeSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Exe) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAge) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h index 05d585d2576..129e557c7f2 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h @@ -22,18 +22,14 @@ class raw_ostream; namespace pdb { class PDBSymbolFunc : public PDBSymbol { + DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Function) public: - PDBSymbolFunc(const IPDBSession &PDBSession, - std::unique_ptr FuncSymbol); - void dump(PDBSymDumper &Dumper) const override; bool isDestructor() const; std::unique_ptr> getArguments() const; - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Function) - FORWARD_SYMBOL_METHOD(getAccess) FORWARD_SYMBOL_METHOD(getAddressOffset) FORWARD_SYMBOL_METHOD(getAddressSection) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h index 3341bd9b30f..18db8a50fd1 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h @@ -20,12 +20,8 @@ class raw_ostream; namespace pdb { class PDBSymbolFuncDebugEnd : public PDBSymbol { -public: - PDBSymbolFuncDebugEnd(const IPDBSession &PDBSession, - std::unique_ptr FuncDebugEndSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FuncDebugEnd) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAddressOffset) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h index 6729838597c..83d82f0cbcc 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolFuncDebugStart : public PDBSymbol { -public: - PDBSymbolFuncDebugStart(const IPDBSession &PDBSession, - std::unique_ptr FuncDebugStartSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FuncDebugStart) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAddressOffset) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h index c2b1c28c929..8b2617fcd75 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolLabel : public PDBSymbol { -public: - PDBSymbolLabel(const IPDBSession &PDBSession, - std::unique_ptr LabelSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Label) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAddressOffset) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h index c9e6ee67c57..9def3edb469 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolPublicSymbol : public PDBSymbol { -public: - PDBSymbolPublicSymbol(const IPDBSession &PDBSession, - std::unique_ptr PublicSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::PublicSymbol) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAddressOffset) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h index 614fad86caa..7bb0555362d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolThunk : public PDBSymbol { -public: - PDBSymbolThunk(const IPDBSession &PDBSession, - std::unique_ptr ThunkSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Thunk) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAccess) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h index 39b7d3b300e..488f668bdc1 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeArray : public PDBSymbol { -public: - PDBSymbolTypeArray(const IPDBSession &PDBSession, - std::unique_ptr ArrayTypeSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::ArrayType) - +public: void dump(PDBSymDumper &Dumper) const override; void dumpRight(PDBSymDumper &Dumper) const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h index d607a3d8117..550deedd750 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h @@ -22,12 +22,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeBaseClass : public PDBSymbol { -public: - PDBSymbolTypeBaseClass(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BaseClass) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getAccess) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h index 5b1863c42a0..e07e88802b8 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeBuiltin : public PDBSymbol { -public: - PDBSymbolTypeBuiltin(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BuiltinType) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getBuiltinType) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h index 199b3f8b304..0d8979c9c5c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeCustom : public PDBSymbol { -public: - PDBSymbolTypeCustom(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::CustomType) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getOemId) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h index e635eb5bbf6..58292a63501 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeDimension : public PDBSymbol { -public: - PDBSymbolTypeDimension(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Dimension) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getLowerBoundId) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h index ddbe7e58f18..f463047bb5b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h @@ -21,12 +21,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeEnum : public PDBSymbol { -public: - PDBSymbolTypeEnum(const IPDBSession &PDBSession, - std::unique_ptr EnumTypeSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Enum) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getBuiltinType) @@ -38,6 +34,7 @@ public: FORWARD_SYMBOL_METHOD(hasNestedTypes) FORWARD_SYMBOL_METHOD(getLength) FORWARD_SYMBOL_ID_METHOD(getLexicalParent) + FORWARD_SYMBOL_ID_METHOD(getUnmodifiedType) FORWARD_SYMBOL_METHOD(getName) FORWARD_SYMBOL_METHOD(getSrcLineOnTypeDefn) FORWARD_SYMBOL_METHOD(isNested) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h index 24c13128111..5b940b0737a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeFriend : public PDBSymbol { -public: - PDBSymbolTypeFriend(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Friend) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_ID_METHOD(getClassParent) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h index 3855999c473..074cb418fc8 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeFunctionArg : public PDBSymbol { -public: - PDBSymbolTypeFunctionArg(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FunctionArg) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_ID_METHOD(getClassParent) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h index abd4cf5effa..dfdf436197c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeFunctionSig : public PDBSymbol { -public: - PDBSymbolTypeFunctionSig(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FunctionSig) - +public: std::unique_ptr getArguments() const; void dump(PDBSymDumper &Dumper) const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h index 31cf5363dde..d716abd640c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeManaged : public PDBSymbol { -public: - PDBSymbolTypeManaged(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::ManagedType) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getName) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h index 7612ebac31d..300d6722fc4 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h @@ -19,16 +19,13 @@ class raw_ostream; namespace pdb { class PDBSymbolTypePointer : public PDBSymbol { -public: - PDBSymbolTypePointer(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::PointerType) - +public: void dump(PDBSymDumper &Dumper) const override; void dumpRight(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(isConstType) + FORWARD_SYMBOL_ID_METHOD(getClassParent) FORWARD_SYMBOL_METHOD(getLength) FORWARD_SYMBOL_ID_METHOD(getLexicalParent) FORWARD_SYMBOL_METHOD(isReference) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h index 16c1d1b88c6..d6e2a36486d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeTypedef : public PDBSymbol { -public: - PDBSymbolTypeTypedef(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Typedef) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(getBuiltinType) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h index e259b6dca3d..937dd6c8722 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h @@ -23,17 +23,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeUDT : public PDBSymbol { -public: - PDBSymbolTypeUDT(const IPDBSession &PDBSession, - std::unique_ptr UDTSymbol); - - std::unique_ptr clone() const { - return getSession().getConcreteSymbolById( - getSymIndexId()); - } - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::UDT) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_ID_METHOD(getClassParent) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h index e270c2b7eb9..6efce4bbd68 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeVTable : public PDBSymbol { -public: - PDBSymbolTypeVTable(const IPDBSession &PDBSession, - std::unique_ptr VtblSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::VTable) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_ID_METHOD(getClassParent) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h index 8acaabea5bb..8949052b0c0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h @@ -19,12 +19,8 @@ class raw_ostream; namespace pdb { class PDBSymbolTypeVTableShape : public PDBSymbol { -public: - PDBSymbolTypeVTableShape(const IPDBSession &PDBSession, - std::unique_ptr VtblShapeSymbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::VTableShape) - +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_METHOD(isConstType) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h index de43e47badb..e935ac6ce0d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h @@ -18,16 +18,11 @@ class raw_ostream; namespace pdb { class PDBSymbolUnknown : public PDBSymbol { + DECLARE_PDB_SYMBOL_CUSTOM_TYPE(S->getSymTag() == PDB_SymType::None || + S->getSymTag() >= PDB_SymType::Max) + public: - PDBSymbolUnknown(const IPDBSession &PDBSession, - std::unique_ptr UnknownSymbol); - void dump(PDBSymDumper &Dumper) const override; - - static bool classof(const PDBSymbol *S) { - return (S->getSymTag() == PDB_SymType::None || - S->getSymTag() >= PDB_SymType::Max); - } }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h index 70fbd5b84c3..4e8c99fc8d8 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h @@ -19,12 +19,9 @@ class raw_ostream; namespace pdb { class PDBSymbolUsingNamespace : public PDBSymbol { -public: - PDBSymbolUsingNamespace(const IPDBSession &PDBSession, - std::unique_ptr Symbol); - DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::UsingNamespace) +public: void dump(PDBSymDumper &Dumper) const override; FORWARD_SYMBOL_ID_METHOD(getLexicalParent) diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h index da6cb1d2677..917f3ed7391 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h @@ -12,6 +12,7 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" +#include "llvm/DebugInfo/PDB/IPDBFrameData.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include #include @@ -22,6 +23,8 @@ namespace llvm { namespace pdb { +typedef uint32_t SymIndexId; + class IPDBDataStream; class IPDBInjectedSource; class IPDBLineNumber; @@ -69,6 +72,7 @@ using IPDBEnumLineNumbers = IPDBEnumChildren; using IPDBEnumTables = IPDBEnumChildren; using IPDBEnumInjectedSources = IPDBEnumChildren; using IPDBEnumSectionContribs = IPDBEnumChildren; +using IPDBEnumFrameData = IPDBEnumChildren; /// Specifies which PDB reader implementation is to be used. Only a value /// of PDB_ReaderType::DIA is currently supported, but Native is in the works. @@ -208,6 +212,18 @@ enum class PDB_SymType { CustomType, ManagedType, Dimension, + CallSite, + InlineSite, + BaseInterface, + VectorType, + MatrixType, + HLSLType, + Caller, + Callee, + Export, + HeapAllocationSite, + CoffGroup, + Inlinee, Max }; @@ -334,6 +350,36 @@ enum PDB_VariantType { struct Variant { Variant() = default; + explicit Variant(bool V) : Type(PDB_VariantType::Bool) { Value.Bool = V; } + explicit Variant(int8_t V) : Type(PDB_VariantType::Int8) { Value.Int8 = V; } + explicit Variant(int16_t V) : Type(PDB_VariantType::Int16) { + Value.Int16 = V; + } + explicit Variant(int32_t V) : Type(PDB_VariantType::Int32) { + Value.Int32 = V; + } + explicit Variant(int64_t V) : Type(PDB_VariantType::Int64) { + Value.Int64 = V; + } + explicit Variant(float V) : Type(PDB_VariantType::Single) { + Value.Single = V; + } + explicit Variant(double V) : Type(PDB_VariantType::Double) { + Value.Double = V; + } + explicit Variant(uint8_t V) : Type(PDB_VariantType::UInt8) { + Value.UInt8 = V; + } + explicit Variant(uint16_t V) : Type(PDB_VariantType::UInt16) { + Value.UInt16 = V; + } + explicit Variant(uint32_t V) : Type(PDB_VariantType::UInt32) { + Value.UInt32 = V; + } + explicit Variant(uint64_t V) : Type(PDB_VariantType::UInt64) { + Value.UInt64 = V; + } + Variant(const Variant &Other) { *this = Other; } diff --git a/contrib/llvm/lib/Demangle/Compiler.h b/contrib/llvm/include/llvm/Demangle/Compiler.h similarity index 100% rename from contrib/llvm/lib/Demangle/Compiler.h rename to contrib/llvm/include/llvm/Demangle/Compiler.h diff --git a/contrib/llvm/include/llvm/Demangle/Demangle.h b/contrib/llvm/include/llvm/Demangle/Demangle.h index df7753f23b8..4c9dc9569e1 100644 --- a/contrib/llvm/include/llvm/Demangle/Demangle.h +++ b/contrib/llvm/include/llvm/Demangle/Demangle.h @@ -7,6 +7,9 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_DEMANGLE_DEMANGLE_H +#define LLVM_DEMANGLE_DEMANGLE_H + #include namespace llvm { @@ -27,8 +30,11 @@ enum : int { char *itaniumDemangle(const char *mangled_name, char *buf, size_t *n, int *status); + + +enum MSDemangleFlags { MSDF_None = 0, MSDF_DumpBackrefs = 1 << 0 }; char *microsoftDemangle(const char *mangled_name, char *buf, size_t *n, - int *status); + int *status, MSDemangleFlags Flags = MSDF_None); /// "Partial" demangler. This supports demangling a string into an AST /// (typically an intermediate stage in itaniumDemangle) and querying certain @@ -86,3 +92,5 @@ private: void *Context; }; } // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/Demangle/ItaniumDemangle.h b/contrib/llvm/include/llvm/Demangle/ItaniumDemangle.h new file mode 100644 index 00000000000..0b9187f30a5 --- /dev/null +++ b/contrib/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -0,0 +1,5184 @@ +//===------------------------- ItaniumDemangle.h ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEMANGLE_ITANIUMDEMANGLE_H +#define LLVM_DEMANGLE_ITANIUMDEMANGLE_H + +// FIXME: (possibly) incomplete list of features that clang mangles that this +// file does not yet support: +// - C++ modules TS + +#include "llvm/Demangle/Compiler.h" +#include "llvm/Demangle/StringView.h" +#include "llvm/Demangle/Utility.h" + +#include +#include +#include +#include +#include +#include +#include + +#define FOR_EACH_NODE_KIND(X) \ + X(NodeArrayNode) \ + X(DotSuffix) \ + X(VendorExtQualType) \ + X(QualType) \ + X(ConversionOperatorType) \ + X(PostfixQualifiedType) \ + X(ElaboratedTypeSpefType) \ + X(NameType) \ + X(AbiTagAttr) \ + X(EnableIfAttr) \ + X(ObjCProtoName) \ + X(PointerType) \ + X(ReferenceType) \ + X(PointerToMemberType) \ + X(ArrayType) \ + X(FunctionType) \ + X(NoexceptSpec) \ + X(DynamicExceptionSpec) \ + X(FunctionEncoding) \ + X(LiteralOperator) \ + X(SpecialName) \ + X(CtorVtableSpecialName) \ + X(QualifiedName) \ + X(NestedName) \ + X(LocalName) \ + X(VectorType) \ + X(PixelVectorType) \ + X(ParameterPack) \ + X(TemplateArgumentPack) \ + X(ParameterPackExpansion) \ + X(TemplateArgs) \ + X(ForwardTemplateReference) \ + X(NameWithTemplateArgs) \ + X(GlobalQualifiedName) \ + X(StdQualifiedName) \ + X(ExpandedSpecialSubstitution) \ + X(SpecialSubstitution) \ + X(CtorDtorName) \ + X(DtorName) \ + X(UnnamedTypeName) \ + X(ClosureTypeName) \ + X(StructuredBindingName) \ + X(BinaryExpr) \ + X(ArraySubscriptExpr) \ + X(PostfixExpr) \ + X(ConditionalExpr) \ + X(MemberExpr) \ + X(EnclosingExpr) \ + X(CastExpr) \ + X(SizeofParamPackExpr) \ + X(CallExpr) \ + X(NewExpr) \ + X(DeleteExpr) \ + X(PrefixExpr) \ + X(FunctionParam) \ + X(ConversionExpr) \ + X(InitListExpr) \ + X(FoldExpr) \ + X(ThrowExpr) \ + X(BoolExpr) \ + X(IntegerCastExpr) \ + X(IntegerLiteral) \ + X(FloatLiteral) \ + X(DoubleLiteral) \ + X(LongDoubleLiteral) \ + X(BracedExpr) \ + X(BracedRangeExpr) + +namespace llvm { +namespace itanium_demangle { +// Base class of all AST nodes. The AST is built by the parser, then is +// traversed by the printLeft/Right functions to produce a demangled string. +class Node { +public: + enum Kind : unsigned char { +#define ENUMERATOR(NodeKind) K ## NodeKind, + FOR_EACH_NODE_KIND(ENUMERATOR) +#undef ENUMERATOR + }; + + /// Three-way bool to track a cached value. Unknown is possible if this node + /// has an unexpanded parameter pack below it that may affect this cache. + enum class Cache : unsigned char { Yes, No, Unknown, }; + +private: + Kind K; + + // FIXME: Make these protected. +public: + /// Tracks if this node has a component on its right side, in which case we + /// need to call printRight. + Cache RHSComponentCache; + + /// Track if this node is a (possibly qualified) array type. This can affect + /// how we format the output string. + Cache ArrayCache; + + /// Track if this node is a (possibly qualified) function type. This can + /// affect how we format the output string. + Cache FunctionCache; + +public: + Node(Kind K_, Cache RHSComponentCache_ = Cache::No, + Cache ArrayCache_ = Cache::No, Cache FunctionCache_ = Cache::No) + : K(K_), RHSComponentCache(RHSComponentCache_), ArrayCache(ArrayCache_), + FunctionCache(FunctionCache_) {} + + /// Visit the most-derived object corresponding to this object. + template void visit(Fn F) const; + + // The following function is provided by all derived classes: + // + // Call F with arguments that, when passed to the constructor of this node, + // would construct an equivalent node. + //template void match(Fn F) const; + + bool hasRHSComponent(OutputStream &S) const { + if (RHSComponentCache != Cache::Unknown) + return RHSComponentCache == Cache::Yes; + return hasRHSComponentSlow(S); + } + + bool hasArray(OutputStream &S) const { + if (ArrayCache != Cache::Unknown) + return ArrayCache == Cache::Yes; + return hasArraySlow(S); + } + + bool hasFunction(OutputStream &S) const { + if (FunctionCache != Cache::Unknown) + return FunctionCache == Cache::Yes; + return hasFunctionSlow(S); + } + + Kind getKind() const { return K; } + + virtual bool hasRHSComponentSlow(OutputStream &) const { return false; } + virtual bool hasArraySlow(OutputStream &) const { return false; } + virtual bool hasFunctionSlow(OutputStream &) const { return false; } + + // Dig through "glue" nodes like ParameterPack and ForwardTemplateReference to + // get at a node that actually represents some concrete syntax. + virtual const Node *getSyntaxNode(OutputStream &) const { + return this; + } + + void print(OutputStream &S) const { + printLeft(S); + if (RHSComponentCache != Cache::No) + printRight(S); + } + + // Print the "left" side of this Node into OutputStream. + virtual void printLeft(OutputStream &) const = 0; + + // Print the "right". This distinction is necessary to represent C++ types + // that appear on the RHS of their subtype, such as arrays or functions. + // Since most types don't have such a component, provide a default + // implementation. + virtual void printRight(OutputStream &) const {} + + virtual StringView getBaseName() const { return StringView(); } + + // Silence compiler warnings, this dtor will never be called. + virtual ~Node() = default; + +#ifndef NDEBUG + LLVM_DUMP_METHOD void dump() const; +#endif +}; + +class NodeArray { + Node **Elements; + size_t NumElements; + +public: + NodeArray() : Elements(nullptr), NumElements(0) {} + NodeArray(Node **Elements_, size_t NumElements_) + : Elements(Elements_), NumElements(NumElements_) {} + + bool empty() const { return NumElements == 0; } + size_t size() const { return NumElements; } + + Node **begin() const { return Elements; } + Node **end() const { return Elements + NumElements; } + + Node *operator[](size_t Idx) const { return Elements[Idx]; } + + void printWithComma(OutputStream &S) const { + bool FirstElement = true; + for (size_t Idx = 0; Idx != NumElements; ++Idx) { + size_t BeforeComma = S.getCurrentPosition(); + if (!FirstElement) + S += ", "; + size_t AfterComma = S.getCurrentPosition(); + Elements[Idx]->print(S); + + // Elements[Idx] is an empty parameter pack expansion, we should erase the + // comma we just printed. + if (AfterComma == S.getCurrentPosition()) { + S.setCurrentPosition(BeforeComma); + continue; + } + + FirstElement = false; + } + } +}; + +struct NodeArrayNode : Node { + NodeArray Array; + NodeArrayNode(NodeArray Array_) : Node(KNodeArrayNode), Array(Array_) {} + + template void match(Fn F) const { F(Array); } + + void printLeft(OutputStream &S) const override { + Array.printWithComma(S); + } +}; + +class DotSuffix final : public Node { + const Node *Prefix; + const StringView Suffix; + +public: + DotSuffix(const Node *Prefix_, StringView Suffix_) + : Node(KDotSuffix), Prefix(Prefix_), Suffix(Suffix_) {} + + template void match(Fn F) const { F(Prefix, Suffix); } + + void printLeft(OutputStream &s) const override { + Prefix->print(s); + s += " ("; + s += Suffix; + s += ")"; + } +}; + +class VendorExtQualType final : public Node { + const Node *Ty; + StringView Ext; + +public: + VendorExtQualType(const Node *Ty_, StringView Ext_) + : Node(KVendorExtQualType), Ty(Ty_), Ext(Ext_) {} + + template void match(Fn F) const { F(Ty, Ext); } + + void printLeft(OutputStream &S) const override { + Ty->print(S); + S += " "; + S += Ext; + } +}; + +enum FunctionRefQual : unsigned char { + FrefQualNone, + FrefQualLValue, + FrefQualRValue, +}; + +enum Qualifiers { + QualNone = 0, + QualConst = 0x1, + QualVolatile = 0x2, + QualRestrict = 0x4, +}; + +inline Qualifiers operator|=(Qualifiers &Q1, Qualifiers Q2) { + return Q1 = static_cast(Q1 | Q2); +} + +class QualType : public Node { +protected: + const Qualifiers Quals; + const Node *Child; + + void printQuals(OutputStream &S) const { + if (Quals & QualConst) + S += " const"; + if (Quals & QualVolatile) + S += " volatile"; + if (Quals & QualRestrict) + S += " restrict"; + } + +public: + QualType(const Node *Child_, Qualifiers Quals_) + : Node(KQualType, Child_->RHSComponentCache, + Child_->ArrayCache, Child_->FunctionCache), + Quals(Quals_), Child(Child_) {} + + template void match(Fn F) const { F(Child, Quals); } + + bool hasRHSComponentSlow(OutputStream &S) const override { + return Child->hasRHSComponent(S); + } + bool hasArraySlow(OutputStream &S) const override { + return Child->hasArray(S); + } + bool hasFunctionSlow(OutputStream &S) const override { + return Child->hasFunction(S); + } + + void printLeft(OutputStream &S) const override { + Child->printLeft(S); + printQuals(S); + } + + void printRight(OutputStream &S) const override { Child->printRight(S); } +}; + +class ConversionOperatorType final : public Node { + const Node *Ty; + +public: + ConversionOperatorType(const Node *Ty_) + : Node(KConversionOperatorType), Ty(Ty_) {} + + template void match(Fn F) const { F(Ty); } + + void printLeft(OutputStream &S) const override { + S += "operator "; + Ty->print(S); + } +}; + +class PostfixQualifiedType final : public Node { + const Node *Ty; + const StringView Postfix; + +public: + PostfixQualifiedType(Node *Ty_, StringView Postfix_) + : Node(KPostfixQualifiedType), Ty(Ty_), Postfix(Postfix_) {} + + template void match(Fn F) const { F(Ty, Postfix); } + + void printLeft(OutputStream &s) const override { + Ty->printLeft(s); + s += Postfix; + } +}; + +class NameType final : public Node { + const StringView Name; + +public: + NameType(StringView Name_) : Node(KNameType), Name(Name_) {} + + template void match(Fn F) const { F(Name); } + + StringView getName() const { return Name; } + StringView getBaseName() const override { return Name; } + + void printLeft(OutputStream &s) const override { s += Name; } +}; + +class ElaboratedTypeSpefType : public Node { + StringView Kind; + Node *Child; +public: + ElaboratedTypeSpefType(StringView Kind_, Node *Child_) + : Node(KElaboratedTypeSpefType), Kind(Kind_), Child(Child_) {} + + template void match(Fn F) const { F(Kind, Child); } + + void printLeft(OutputStream &S) const override { + S += Kind; + S += ' '; + Child->print(S); + } +}; + +struct AbiTagAttr : Node { + Node *Base; + StringView Tag; + + AbiTagAttr(Node* Base_, StringView Tag_) + : Node(KAbiTagAttr, Base_->RHSComponentCache, + Base_->ArrayCache, Base_->FunctionCache), + Base(Base_), Tag(Tag_) {} + + template void match(Fn F) const { F(Base, Tag); } + + void printLeft(OutputStream &S) const override { + Base->printLeft(S); + S += "[abi:"; + S += Tag; + S += "]"; + } +}; + +class EnableIfAttr : public Node { + NodeArray Conditions; +public: + EnableIfAttr(NodeArray Conditions_) + : Node(KEnableIfAttr), Conditions(Conditions_) {} + + template void match(Fn F) const { F(Conditions); } + + void printLeft(OutputStream &S) const override { + S += " [enable_if:"; + Conditions.printWithComma(S); + S += ']'; + } +}; + +class ObjCProtoName : public Node { + const Node *Ty; + StringView Protocol; + + friend class PointerType; + +public: + ObjCProtoName(const Node *Ty_, StringView Protocol_) + : Node(KObjCProtoName), Ty(Ty_), Protocol(Protocol_) {} + + template void match(Fn F) const { F(Ty, Protocol); } + + bool isObjCObject() const { + return Ty->getKind() == KNameType && + static_cast(Ty)->getName() == "objc_object"; + } + + void printLeft(OutputStream &S) const override { + Ty->print(S); + S += "<"; + S += Protocol; + S += ">"; + } +}; + +class PointerType final : public Node { + const Node *Pointee; + +public: + PointerType(const Node *Pointee_) + : Node(KPointerType, Pointee_->RHSComponentCache), + Pointee(Pointee_) {} + + template void match(Fn F) const { F(Pointee); } + + bool hasRHSComponentSlow(OutputStream &S) const override { + return Pointee->hasRHSComponent(S); + } + + void printLeft(OutputStream &s) const override { + // We rewrite objc_object* into id. + if (Pointee->getKind() != KObjCProtoName || + !static_cast(Pointee)->isObjCObject()) { + Pointee->printLeft(s); + if (Pointee->hasArray(s)) + s += " "; + if (Pointee->hasArray(s) || Pointee->hasFunction(s)) + s += "("; + s += "*"; + } else { + const auto *objcProto = static_cast(Pointee); + s += "id<"; + s += objcProto->Protocol; + s += ">"; + } + } + + void printRight(OutputStream &s) const override { + if (Pointee->getKind() != KObjCProtoName || + !static_cast(Pointee)->isObjCObject()) { + if (Pointee->hasArray(s) || Pointee->hasFunction(s)) + s += ")"; + Pointee->printRight(s); + } + } +}; + +enum class ReferenceKind { + LValue, + RValue, +}; + +// Represents either a LValue or an RValue reference type. +class ReferenceType : public Node { + const Node *Pointee; + ReferenceKind RK; + + mutable bool Printing = false; + + // Dig through any refs to refs, collapsing the ReferenceTypes as we go. The + // rule here is rvalue ref to rvalue ref collapses to a rvalue ref, and any + // other combination collapses to a lvalue ref. + std::pair collapse(OutputStream &S) const { + auto SoFar = std::make_pair(RK, Pointee); + for (;;) { + const Node *SN = SoFar.second->getSyntaxNode(S); + if (SN->getKind() != KReferenceType) + break; + auto *RT = static_cast(SN); + SoFar.second = RT->Pointee; + SoFar.first = std::min(SoFar.first, RT->RK); + } + return SoFar; + } + +public: + ReferenceType(const Node *Pointee_, ReferenceKind RK_) + : Node(KReferenceType, Pointee_->RHSComponentCache), + Pointee(Pointee_), RK(RK_) {} + + template void match(Fn F) const { F(Pointee, RK); } + + bool hasRHSComponentSlow(OutputStream &S) const override { + return Pointee->hasRHSComponent(S); + } + + void printLeft(OutputStream &s) const override { + if (Printing) + return; + SwapAndRestore SavePrinting(Printing, true); + std::pair Collapsed = collapse(s); + Collapsed.second->printLeft(s); + if (Collapsed.second->hasArray(s)) + s += " "; + if (Collapsed.second->hasArray(s) || Collapsed.second->hasFunction(s)) + s += "("; + + s += (Collapsed.first == ReferenceKind::LValue ? "&" : "&&"); + } + void printRight(OutputStream &s) const override { + if (Printing) + return; + SwapAndRestore SavePrinting(Printing, true); + std::pair Collapsed = collapse(s); + if (Collapsed.second->hasArray(s) || Collapsed.second->hasFunction(s)) + s += ")"; + Collapsed.second->printRight(s); + } +}; + +class PointerToMemberType final : public Node { + const Node *ClassType; + const Node *MemberType; + +public: + PointerToMemberType(const Node *ClassType_, const Node *MemberType_) + : Node(KPointerToMemberType, MemberType_->RHSComponentCache), + ClassType(ClassType_), MemberType(MemberType_) {} + + template void match(Fn F) const { F(ClassType, MemberType); } + + bool hasRHSComponentSlow(OutputStream &S) const override { + return MemberType->hasRHSComponent(S); + } + + void printLeft(OutputStream &s) const override { + MemberType->printLeft(s); + if (MemberType->hasArray(s) || MemberType->hasFunction(s)) + s += "("; + else + s += " "; + ClassType->print(s); + s += "::*"; + } + + void printRight(OutputStream &s) const override { + if (MemberType->hasArray(s) || MemberType->hasFunction(s)) + s += ")"; + MemberType->printRight(s); + } +}; + +class NodeOrString { + const void *First; + const void *Second; + +public: + /* implicit */ NodeOrString(StringView Str) { + const char *FirstChar = Str.begin(); + const char *SecondChar = Str.end(); + if (SecondChar == nullptr) { + assert(FirstChar == SecondChar); + ++FirstChar, ++SecondChar; + } + First = static_cast(FirstChar); + Second = static_cast(SecondChar); + } + + /* implicit */ NodeOrString(Node *N) + : First(static_cast(N)), Second(nullptr) {} + NodeOrString() : First(nullptr), Second(nullptr) {} + + bool isString() const { return Second && First; } + bool isNode() const { return First && !Second; } + bool isEmpty() const { return !First && !Second; } + + StringView asString() const { + assert(isString()); + return StringView(static_cast(First), + static_cast(Second)); + } + + const Node *asNode() const { + assert(isNode()); + return static_cast(First); + } +}; + +class ArrayType final : public Node { + const Node *Base; + NodeOrString Dimension; + +public: + ArrayType(const Node *Base_, NodeOrString Dimension_) + : Node(KArrayType, + /*RHSComponentCache=*/Cache::Yes, + /*ArrayCache=*/Cache::Yes), + Base(Base_), Dimension(Dimension_) {} + + template void match(Fn F) const { F(Base, Dimension); } + + bool hasRHSComponentSlow(OutputStream &) const override { return true; } + bool hasArraySlow(OutputStream &) const override { return true; } + + void printLeft(OutputStream &S) const override { Base->printLeft(S); } + + void printRight(OutputStream &S) const override { + if (S.back() != ']') + S += " "; + S += "["; + if (Dimension.isString()) + S += Dimension.asString(); + else if (Dimension.isNode()) + Dimension.asNode()->print(S); + S += "]"; + Base->printRight(S); + } +}; + +class FunctionType final : public Node { + const Node *Ret; + NodeArray Params; + Qualifiers CVQuals; + FunctionRefQual RefQual; + const Node *ExceptionSpec; + +public: + FunctionType(const Node *Ret_, NodeArray Params_, Qualifiers CVQuals_, + FunctionRefQual RefQual_, const Node *ExceptionSpec_) + : Node(KFunctionType, + /*RHSComponentCache=*/Cache::Yes, /*ArrayCache=*/Cache::No, + /*FunctionCache=*/Cache::Yes), + Ret(Ret_), Params(Params_), CVQuals(CVQuals_), RefQual(RefQual_), + ExceptionSpec(ExceptionSpec_) {} + + template void match(Fn F) const { + F(Ret, Params, CVQuals, RefQual, ExceptionSpec); + } + + bool hasRHSComponentSlow(OutputStream &) const override { return true; } + bool hasFunctionSlow(OutputStream &) const override { return true; } + + // Handle C++'s ... quirky decl grammar by using the left & right + // distinction. Consider: + // int (*f(float))(char) {} + // f is a function that takes a float and returns a pointer to a function + // that takes a char and returns an int. If we're trying to print f, start + // by printing out the return types's left, then print our parameters, then + // finally print right of the return type. + void printLeft(OutputStream &S) const override { + Ret->printLeft(S); + S += " "; + } + + void printRight(OutputStream &S) const override { + S += "("; + Params.printWithComma(S); + S += ")"; + Ret->printRight(S); + + if (CVQuals & QualConst) + S += " const"; + if (CVQuals & QualVolatile) + S += " volatile"; + if (CVQuals & QualRestrict) + S += " restrict"; + + if (RefQual == FrefQualLValue) + S += " &"; + else if (RefQual == FrefQualRValue) + S += " &&"; + + if (ExceptionSpec != nullptr) { + S += ' '; + ExceptionSpec->print(S); + } + } +}; + +class NoexceptSpec : public Node { + const Node *E; +public: + NoexceptSpec(const Node *E_) : Node(KNoexceptSpec), E(E_) {} + + template void match(Fn F) const { F(E); } + + void printLeft(OutputStream &S) const override { + S += "noexcept("; + E->print(S); + S += ")"; + } +}; + +class DynamicExceptionSpec : public Node { + NodeArray Types; +public: + DynamicExceptionSpec(NodeArray Types_) + : Node(KDynamicExceptionSpec), Types(Types_) {} + + template void match(Fn F) const { F(Types); } + + void printLeft(OutputStream &S) const override { + S += "throw("; + Types.printWithComma(S); + S += ')'; + } +}; + +class FunctionEncoding final : public Node { + const Node *Ret; + const Node *Name; + NodeArray Params; + const Node *Attrs; + Qualifiers CVQuals; + FunctionRefQual RefQual; + +public: + FunctionEncoding(const Node *Ret_, const Node *Name_, NodeArray Params_, + const Node *Attrs_, Qualifiers CVQuals_, + FunctionRefQual RefQual_) + : Node(KFunctionEncoding, + /*RHSComponentCache=*/Cache::Yes, /*ArrayCache=*/Cache::No, + /*FunctionCache=*/Cache::Yes), + Ret(Ret_), Name(Name_), Params(Params_), Attrs(Attrs_), + CVQuals(CVQuals_), RefQual(RefQual_) {} + + template void match(Fn F) const { + F(Ret, Name, Params, Attrs, CVQuals, RefQual); + } + + Qualifiers getCVQuals() const { return CVQuals; } + FunctionRefQual getRefQual() const { return RefQual; } + NodeArray getParams() const { return Params; } + const Node *getReturnType() const { return Ret; } + + bool hasRHSComponentSlow(OutputStream &) const override { return true; } + bool hasFunctionSlow(OutputStream &) const override { return true; } + + const Node *getName() const { return Name; } + + void printLeft(OutputStream &S) const override { + if (Ret) { + Ret->printLeft(S); + if (!Ret->hasRHSComponent(S)) + S += " "; + } + Name->print(S); + } + + void printRight(OutputStream &S) const override { + S += "("; + Params.printWithComma(S); + S += ")"; + if (Ret) + Ret->printRight(S); + + if (CVQuals & QualConst) + S += " const"; + if (CVQuals & QualVolatile) + S += " volatile"; + if (CVQuals & QualRestrict) + S += " restrict"; + + if (RefQual == FrefQualLValue) + S += " &"; + else if (RefQual == FrefQualRValue) + S += " &&"; + + if (Attrs != nullptr) + Attrs->print(S); + } +}; + +class LiteralOperator : public Node { + const Node *OpName; + +public: + LiteralOperator(const Node *OpName_) + : Node(KLiteralOperator), OpName(OpName_) {} + + template void match(Fn F) const { F(OpName); } + + void printLeft(OutputStream &S) const override { + S += "operator\"\" "; + OpName->print(S); + } +}; + +class SpecialName final : public Node { + const StringView Special; + const Node *Child; + +public: + SpecialName(StringView Special_, const Node *Child_) + : Node(KSpecialName), Special(Special_), Child(Child_) {} + + template void match(Fn F) const { F(Special, Child); } + + void printLeft(OutputStream &S) const override { + S += Special; + Child->print(S); + } +}; + +class CtorVtableSpecialName final : public Node { + const Node *FirstType; + const Node *SecondType; + +public: + CtorVtableSpecialName(const Node *FirstType_, const Node *SecondType_) + : Node(KCtorVtableSpecialName), + FirstType(FirstType_), SecondType(SecondType_) {} + + template void match(Fn F) const { F(FirstType, SecondType); } + + void printLeft(OutputStream &S) const override { + S += "construction vtable for "; + FirstType->print(S); + S += "-in-"; + SecondType->print(S); + } +}; + +struct NestedName : Node { + Node *Qual; + Node *Name; + + NestedName(Node *Qual_, Node *Name_) + : Node(KNestedName), Qual(Qual_), Name(Name_) {} + + template void match(Fn F) const { F(Qual, Name); } + + StringView getBaseName() const override { return Name->getBaseName(); } + + void printLeft(OutputStream &S) const override { + Qual->print(S); + S += "::"; + Name->print(S); + } +}; + +struct LocalName : Node { + Node *Encoding; + Node *Entity; + + LocalName(Node *Encoding_, Node *Entity_) + : Node(KLocalName), Encoding(Encoding_), Entity(Entity_) {} + + template void match(Fn F) const { F(Encoding, Entity); } + + void printLeft(OutputStream &S) const override { + Encoding->print(S); + S += "::"; + Entity->print(S); + } +}; + +class QualifiedName final : public Node { + // qualifier::name + const Node *Qualifier; + const Node *Name; + +public: + QualifiedName(const Node *Qualifier_, const Node *Name_) + : Node(KQualifiedName), Qualifier(Qualifier_), Name(Name_) {} + + template void match(Fn F) const { F(Qualifier, Name); } + + StringView getBaseName() const override { return Name->getBaseName(); } + + void printLeft(OutputStream &S) const override { + Qualifier->print(S); + S += "::"; + Name->print(S); + } +}; + +class VectorType final : public Node { + const Node *BaseType; + const NodeOrString Dimension; + +public: + VectorType(const Node *BaseType_, NodeOrString Dimension_) + : Node(KVectorType), BaseType(BaseType_), + Dimension(Dimension_) {} + + template void match(Fn F) const { F(BaseType, Dimension); } + + void printLeft(OutputStream &S) const override { + BaseType->print(S); + S += " vector["; + if (Dimension.isNode()) + Dimension.asNode()->print(S); + else if (Dimension.isString()) + S += Dimension.asString(); + S += "]"; + } +}; + +class PixelVectorType final : public Node { + const NodeOrString Dimension; + +public: + PixelVectorType(NodeOrString Dimension_) + : Node(KPixelVectorType), Dimension(Dimension_) {} + + template void match(Fn F) const { F(Dimension); } + + void printLeft(OutputStream &S) const override { + // FIXME: This should demangle as "vector pixel". + S += "pixel vector["; + S += Dimension.asString(); + S += "]"; + } +}; + +/// An unexpanded parameter pack (either in the expression or type context). If +/// this AST is correct, this node will have a ParameterPackExpansion node above +/// it. +/// +/// This node is created when some are found that apply to an +/// , and is stored in the TemplateParams table. In order for this to +/// appear in the final AST, it has to referenced via a (ie, +/// T_). +class ParameterPack final : public Node { + NodeArray Data; + + // Setup OutputStream for a pack expansion unless we're already expanding one. + void initializePackExpansion(OutputStream &S) const { + if (S.CurrentPackMax == std::numeric_limits::max()) { + S.CurrentPackMax = static_cast(Data.size()); + S.CurrentPackIndex = 0; + } + } + +public: + ParameterPack(NodeArray Data_) : Node(KParameterPack), Data(Data_) { + ArrayCache = FunctionCache = RHSComponentCache = Cache::Unknown; + if (std::all_of(Data.begin(), Data.end(), [](Node* P) { + return P->ArrayCache == Cache::No; + })) + ArrayCache = Cache::No; + if (std::all_of(Data.begin(), Data.end(), [](Node* P) { + return P->FunctionCache == Cache::No; + })) + FunctionCache = Cache::No; + if (std::all_of(Data.begin(), Data.end(), [](Node* P) { + return P->RHSComponentCache == Cache::No; + })) + RHSComponentCache = Cache::No; + } + + template void match(Fn F) const { F(Data); } + + bool hasRHSComponentSlow(OutputStream &S) const override { + initializePackExpansion(S); + size_t Idx = S.CurrentPackIndex; + return Idx < Data.size() && Data[Idx]->hasRHSComponent(S); + } + bool hasArraySlow(OutputStream &S) const override { + initializePackExpansion(S); + size_t Idx = S.CurrentPackIndex; + return Idx < Data.size() && Data[Idx]->hasArray(S); + } + bool hasFunctionSlow(OutputStream &S) const override { + initializePackExpansion(S); + size_t Idx = S.CurrentPackIndex; + return Idx < Data.size() && Data[Idx]->hasFunction(S); + } + const Node *getSyntaxNode(OutputStream &S) const override { + initializePackExpansion(S); + size_t Idx = S.CurrentPackIndex; + return Idx < Data.size() ? Data[Idx]->getSyntaxNode(S) : this; + } + + void printLeft(OutputStream &S) const override { + initializePackExpansion(S); + size_t Idx = S.CurrentPackIndex; + if (Idx < Data.size()) + Data[Idx]->printLeft(S); + } + void printRight(OutputStream &S) const override { + initializePackExpansion(S); + size_t Idx = S.CurrentPackIndex; + if (Idx < Data.size()) + Data[Idx]->printRight(S); + } +}; + +/// A variadic template argument. This node represents an occurrence of +/// JE in some . It isn't itself unexpanded, unless +/// one of it's Elements is. The parser inserts a ParameterPack into the +/// TemplateParams table if the this pack belongs to apply to an +/// . +class TemplateArgumentPack final : public Node { + NodeArray Elements; +public: + TemplateArgumentPack(NodeArray Elements_) + : Node(KTemplateArgumentPack), Elements(Elements_) {} + + template void match(Fn F) const { F(Elements); } + + NodeArray getElements() const { return Elements; } + + void printLeft(OutputStream &S) const override { + Elements.printWithComma(S); + } +}; + +/// A pack expansion. Below this node, there are some unexpanded ParameterPacks +/// which each have Child->ParameterPackSize elements. +class ParameterPackExpansion final : public Node { + const Node *Child; + +public: + ParameterPackExpansion(const Node *Child_) + : Node(KParameterPackExpansion), Child(Child_) {} + + template void match(Fn F) const { F(Child); } + + const Node *getChild() const { return Child; } + + void printLeft(OutputStream &S) const override { + constexpr unsigned Max = std::numeric_limits::max(); + SwapAndRestore SavePackIdx(S.CurrentPackIndex, Max); + SwapAndRestore SavePackMax(S.CurrentPackMax, Max); + size_t StreamPos = S.getCurrentPosition(); + + // Print the first element in the pack. If Child contains a ParameterPack, + // it will set up S.CurrentPackMax and print the first element. + Child->print(S); + + // No ParameterPack was found in Child. This can occur if we've found a pack + // expansion on a . + if (S.CurrentPackMax == Max) { + S += "..."; + return; + } + + // We found a ParameterPack, but it has no elements. Erase whatever we may + // of printed. + if (S.CurrentPackMax == 0) { + S.setCurrentPosition(StreamPos); + return; + } + + // Else, iterate through the rest of the elements in the pack. + for (unsigned I = 1, E = S.CurrentPackMax; I < E; ++I) { + S += ", "; + S.CurrentPackIndex = I; + Child->print(S); + } + } +}; + +class TemplateArgs final : public Node { + NodeArray Params; + +public: + TemplateArgs(NodeArray Params_) : Node(KTemplateArgs), Params(Params_) {} + + template void match(Fn F) const { F(Params); } + + NodeArray getParams() { return Params; } + + void printLeft(OutputStream &S) const override { + S += "<"; + Params.printWithComma(S); + if (S.back() == '>') + S += " "; + S += ">"; + } +}; + +/// A forward-reference to a template argument that was not known at the point +/// where the template parameter name was parsed in a mangling. +/// +/// This is created when demangling the name of a specialization of a +/// conversion function template: +/// +/// \code +/// struct A { +/// template operator T*(); +/// }; +/// \endcode +/// +/// When demangling a specialization of the conversion function template, we +/// encounter the name of the template (including the \c T) before we reach +/// the template argument list, so we cannot substitute the parameter name +/// for the corresponding argument while parsing. Instead, we create a +/// \c ForwardTemplateReference node that is resolved after we parse the +/// template arguments. +struct ForwardTemplateReference : Node { + size_t Index; + Node *Ref = nullptr; + + // If we're currently printing this node. It is possible (though invalid) for + // a forward template reference to refer to itself via a substitution. This + // creates a cyclic AST, which will stack overflow printing. To fix this, bail + // out if more than one print* function is active. + mutable bool Printing = false; + + ForwardTemplateReference(size_t Index_) + : Node(KForwardTemplateReference, Cache::Unknown, Cache::Unknown, + Cache::Unknown), + Index(Index_) {} + + // We don't provide a matcher for these, because the value of the node is + // not determined by its construction parameters, and it generally needs + // special handling. + template void match(Fn F) const = delete; + + bool hasRHSComponentSlow(OutputStream &S) const override { + if (Printing) + return false; + SwapAndRestore SavePrinting(Printing, true); + return Ref->hasRHSComponent(S); + } + bool hasArraySlow(OutputStream &S) const override { + if (Printing) + return false; + SwapAndRestore SavePrinting(Printing, true); + return Ref->hasArray(S); + } + bool hasFunctionSlow(OutputStream &S) const override { + if (Printing) + return false; + SwapAndRestore SavePrinting(Printing, true); + return Ref->hasFunction(S); + } + const Node *getSyntaxNode(OutputStream &S) const override { + if (Printing) + return this; + SwapAndRestore SavePrinting(Printing, true); + return Ref->getSyntaxNode(S); + } + + void printLeft(OutputStream &S) const override { + if (Printing) + return; + SwapAndRestore SavePrinting(Printing, true); + Ref->printLeft(S); + } + void printRight(OutputStream &S) const override { + if (Printing) + return; + SwapAndRestore SavePrinting(Printing, true); + Ref->printRight(S); + } +}; + +struct NameWithTemplateArgs : Node { + // name + Node *Name; + Node *TemplateArgs; + + NameWithTemplateArgs(Node *Name_, Node *TemplateArgs_) + : Node(KNameWithTemplateArgs), Name(Name_), TemplateArgs(TemplateArgs_) {} + + template void match(Fn F) const { F(Name, TemplateArgs); } + + StringView getBaseName() const override { return Name->getBaseName(); } + + void printLeft(OutputStream &S) const override { + Name->print(S); + TemplateArgs->print(S); + } +}; + +class GlobalQualifiedName final : public Node { + Node *Child; + +public: + GlobalQualifiedName(Node* Child_) + : Node(KGlobalQualifiedName), Child(Child_) {} + + template void match(Fn F) const { F(Child); } + + StringView getBaseName() const override { return Child->getBaseName(); } + + void printLeft(OutputStream &S) const override { + S += "::"; + Child->print(S); + } +}; + +struct StdQualifiedName : Node { + Node *Child; + + StdQualifiedName(Node *Child_) : Node(KStdQualifiedName), Child(Child_) {} + + template void match(Fn F) const { F(Child); } + + StringView getBaseName() const override { return Child->getBaseName(); } + + void printLeft(OutputStream &S) const override { + S += "std::"; + Child->print(S); + } +}; + +enum class SpecialSubKind { + allocator, + basic_string, + string, + istream, + ostream, + iostream, +}; + +class ExpandedSpecialSubstitution final : public Node { + SpecialSubKind SSK; + +public: + ExpandedSpecialSubstitution(SpecialSubKind SSK_) + : Node(KExpandedSpecialSubstitution), SSK(SSK_) {} + + template void match(Fn F) const { F(SSK); } + + StringView getBaseName() const override { + switch (SSK) { + case SpecialSubKind::allocator: + return StringView("allocator"); + case SpecialSubKind::basic_string: + return StringView("basic_string"); + case SpecialSubKind::string: + return StringView("basic_string"); + case SpecialSubKind::istream: + return StringView("basic_istream"); + case SpecialSubKind::ostream: + return StringView("basic_ostream"); + case SpecialSubKind::iostream: + return StringView("basic_iostream"); + } + LLVM_BUILTIN_UNREACHABLE; + } + + void printLeft(OutputStream &S) const override { + switch (SSK) { + case SpecialSubKind::allocator: + S += "std::allocator"; + break; + case SpecialSubKind::basic_string: + S += "std::basic_string"; + break; + case SpecialSubKind::string: + S += "std::basic_string, " + "std::allocator >"; + break; + case SpecialSubKind::istream: + S += "std::basic_istream >"; + break; + case SpecialSubKind::ostream: + S += "std::basic_ostream >"; + break; + case SpecialSubKind::iostream: + S += "std::basic_iostream >"; + break; + } + } +}; + +class SpecialSubstitution final : public Node { +public: + SpecialSubKind SSK; + + SpecialSubstitution(SpecialSubKind SSK_) + : Node(KSpecialSubstitution), SSK(SSK_) {} + + template void match(Fn F) const { F(SSK); } + + StringView getBaseName() const override { + switch (SSK) { + case SpecialSubKind::allocator: + return StringView("allocator"); + case SpecialSubKind::basic_string: + return StringView("basic_string"); + case SpecialSubKind::string: + return StringView("string"); + case SpecialSubKind::istream: + return StringView("istream"); + case SpecialSubKind::ostream: + return StringView("ostream"); + case SpecialSubKind::iostream: + return StringView("iostream"); + } + LLVM_BUILTIN_UNREACHABLE; + } + + void printLeft(OutputStream &S) const override { + switch (SSK) { + case SpecialSubKind::allocator: + S += "std::allocator"; + break; + case SpecialSubKind::basic_string: + S += "std::basic_string"; + break; + case SpecialSubKind::string: + S += "std::string"; + break; + case SpecialSubKind::istream: + S += "std::istream"; + break; + case SpecialSubKind::ostream: + S += "std::ostream"; + break; + case SpecialSubKind::iostream: + S += "std::iostream"; + break; + } + } +}; + +class CtorDtorName final : public Node { + const Node *Basename; + const bool IsDtor; + const int Variant; + +public: + CtorDtorName(const Node *Basename_, bool IsDtor_, int Variant_) + : Node(KCtorDtorName), Basename(Basename_), IsDtor(IsDtor_), + Variant(Variant_) {} + + template void match(Fn F) const { F(Basename, IsDtor, Variant); } + + void printLeft(OutputStream &S) const override { + if (IsDtor) + S += "~"; + S += Basename->getBaseName(); + } +}; + +class DtorName : public Node { + const Node *Base; + +public: + DtorName(const Node *Base_) : Node(KDtorName), Base(Base_) {} + + template void match(Fn F) const { F(Base); } + + void printLeft(OutputStream &S) const override { + S += "~"; + Base->printLeft(S); + } +}; + +class UnnamedTypeName : public Node { + const StringView Count; + +public: + UnnamedTypeName(StringView Count_) : Node(KUnnamedTypeName), Count(Count_) {} + + template void match(Fn F) const { F(Count); } + + void printLeft(OutputStream &S) const override { + S += "'unnamed"; + S += Count; + S += "\'"; + } +}; + +class ClosureTypeName : public Node { + NodeArray Params; + StringView Count; + +public: + ClosureTypeName(NodeArray Params_, StringView Count_) + : Node(KClosureTypeName), Params(Params_), Count(Count_) {} + + template void match(Fn F) const { F(Params, Count); } + + void printLeft(OutputStream &S) const override { + S += "\'lambda"; + S += Count; + S += "\'("; + Params.printWithComma(S); + S += ")"; + } +}; + +class StructuredBindingName : public Node { + NodeArray Bindings; +public: + StructuredBindingName(NodeArray Bindings_) + : Node(KStructuredBindingName), Bindings(Bindings_) {} + + template void match(Fn F) const { F(Bindings); } + + void printLeft(OutputStream &S) const override { + S += '['; + Bindings.printWithComma(S); + S += ']'; + } +}; + +// -- Expression Nodes -- + +class BinaryExpr : public Node { + const Node *LHS; + const StringView InfixOperator; + const Node *RHS; + +public: + BinaryExpr(const Node *LHS_, StringView InfixOperator_, const Node *RHS_) + : Node(KBinaryExpr), LHS(LHS_), InfixOperator(InfixOperator_), RHS(RHS_) { + } + + template void match(Fn F) const { F(LHS, InfixOperator, RHS); } + + void printLeft(OutputStream &S) const override { + // might be a template argument expression, then we need to disambiguate + // with parens. + if (InfixOperator == ">") + S += "("; + + S += "("; + LHS->print(S); + S += ") "; + S += InfixOperator; + S += " ("; + RHS->print(S); + S += ")"; + + if (InfixOperator == ">") + S += ")"; + } +}; + +class ArraySubscriptExpr : public Node { + const Node *Op1; + const Node *Op2; + +public: + ArraySubscriptExpr(const Node *Op1_, const Node *Op2_) + : Node(KArraySubscriptExpr), Op1(Op1_), Op2(Op2_) {} + + template void match(Fn F) const { F(Op1, Op2); } + + void printLeft(OutputStream &S) const override { + S += "("; + Op1->print(S); + S += ")["; + Op2->print(S); + S += "]"; + } +}; + +class PostfixExpr : public Node { + const Node *Child; + const StringView Operator; + +public: + PostfixExpr(const Node *Child_, StringView Operator_) + : Node(KPostfixExpr), Child(Child_), Operator(Operator_) {} + + template void match(Fn F) const { F(Child, Operator); } + + void printLeft(OutputStream &S) const override { + S += "("; + Child->print(S); + S += ")"; + S += Operator; + } +}; + +class ConditionalExpr : public Node { + const Node *Cond; + const Node *Then; + const Node *Else; + +public: + ConditionalExpr(const Node *Cond_, const Node *Then_, const Node *Else_) + : Node(KConditionalExpr), Cond(Cond_), Then(Then_), Else(Else_) {} + + template void match(Fn F) const { F(Cond, Then, Else); } + + void printLeft(OutputStream &S) const override { + S += "("; + Cond->print(S); + S += ") ? ("; + Then->print(S); + S += ") : ("; + Else->print(S); + S += ")"; + } +}; + +class MemberExpr : public Node { + const Node *LHS; + const StringView Kind; + const Node *RHS; + +public: + MemberExpr(const Node *LHS_, StringView Kind_, const Node *RHS_) + : Node(KMemberExpr), LHS(LHS_), Kind(Kind_), RHS(RHS_) {} + + template void match(Fn F) const { F(LHS, Kind, RHS); } + + void printLeft(OutputStream &S) const override { + LHS->print(S); + S += Kind; + RHS->print(S); + } +}; + +class EnclosingExpr : public Node { + const StringView Prefix; + const Node *Infix; + const StringView Postfix; + +public: + EnclosingExpr(StringView Prefix_, Node *Infix_, StringView Postfix_) + : Node(KEnclosingExpr), Prefix(Prefix_), Infix(Infix_), + Postfix(Postfix_) {} + + template void match(Fn F) const { F(Prefix, Infix, Postfix); } + + void printLeft(OutputStream &S) const override { + S += Prefix; + Infix->print(S); + S += Postfix; + } +}; + +class CastExpr : public Node { + // cast_kind(from) + const StringView CastKind; + const Node *To; + const Node *From; + +public: + CastExpr(StringView CastKind_, const Node *To_, const Node *From_) + : Node(KCastExpr), CastKind(CastKind_), To(To_), From(From_) {} + + template void match(Fn F) const { F(CastKind, To, From); } + + void printLeft(OutputStream &S) const override { + S += CastKind; + S += "<"; + To->printLeft(S); + S += ">("; + From->printLeft(S); + S += ")"; + } +}; + +class SizeofParamPackExpr : public Node { + const Node *Pack; + +public: + SizeofParamPackExpr(const Node *Pack_) + : Node(KSizeofParamPackExpr), Pack(Pack_) {} + + template void match(Fn F) const { F(Pack); } + + void printLeft(OutputStream &S) const override { + S += "sizeof...("; + ParameterPackExpansion PPE(Pack); + PPE.printLeft(S); + S += ")"; + } +}; + +class CallExpr : public Node { + const Node *Callee; + NodeArray Args; + +public: + CallExpr(const Node *Callee_, NodeArray Args_) + : Node(KCallExpr), Callee(Callee_), Args(Args_) {} + + template void match(Fn F) const { F(Callee, Args); } + + void printLeft(OutputStream &S) const override { + Callee->print(S); + S += "("; + Args.printWithComma(S); + S += ")"; + } +}; + +class NewExpr : public Node { + // new (expr_list) type(init_list) + NodeArray ExprList; + Node *Type; + NodeArray InitList; + bool IsGlobal; // ::operator new ? + bool IsArray; // new[] ? +public: + NewExpr(NodeArray ExprList_, Node *Type_, NodeArray InitList_, bool IsGlobal_, + bool IsArray_) + : Node(KNewExpr), ExprList(ExprList_), Type(Type_), InitList(InitList_), + IsGlobal(IsGlobal_), IsArray(IsArray_) {} + + template void match(Fn F) const { + F(ExprList, Type, InitList, IsGlobal, IsArray); + } + + void printLeft(OutputStream &S) const override { + if (IsGlobal) + S += "::operator "; + S += "new"; + if (IsArray) + S += "[]"; + S += ' '; + if (!ExprList.empty()) { + S += "("; + ExprList.printWithComma(S); + S += ")"; + } + Type->print(S); + if (!InitList.empty()) { + S += "("; + InitList.printWithComma(S); + S += ")"; + } + + } +}; + +class DeleteExpr : public Node { + Node *Op; + bool IsGlobal; + bool IsArray; + +public: + DeleteExpr(Node *Op_, bool IsGlobal_, bool IsArray_) + : Node(KDeleteExpr), Op(Op_), IsGlobal(IsGlobal_), IsArray(IsArray_) {} + + template void match(Fn F) const { F(Op, IsGlobal, IsArray); } + + void printLeft(OutputStream &S) const override { + if (IsGlobal) + S += "::"; + S += "delete"; + if (IsArray) + S += "[] "; + Op->print(S); + } +}; + +class PrefixExpr : public Node { + StringView Prefix; + Node *Child; + +public: + PrefixExpr(StringView Prefix_, Node *Child_) + : Node(KPrefixExpr), Prefix(Prefix_), Child(Child_) {} + + template void match(Fn F) const { F(Prefix, Child); } + + void printLeft(OutputStream &S) const override { + S += Prefix; + S += "("; + Child->print(S); + S += ")"; + } +}; + +class FunctionParam : public Node { + StringView Number; + +public: + FunctionParam(StringView Number_) : Node(KFunctionParam), Number(Number_) {} + + template void match(Fn F) const { F(Number); } + + void printLeft(OutputStream &S) const override { + S += "fp"; + S += Number; + } +}; + +class ConversionExpr : public Node { + const Node *Type; + NodeArray Expressions; + +public: + ConversionExpr(const Node *Type_, NodeArray Expressions_) + : Node(KConversionExpr), Type(Type_), Expressions(Expressions_) {} + + template void match(Fn F) const { F(Type, Expressions); } + + void printLeft(OutputStream &S) const override { + S += "("; + Type->print(S); + S += ")("; + Expressions.printWithComma(S); + S += ")"; + } +}; + +class InitListExpr : public Node { + const Node *Ty; + NodeArray Inits; +public: + InitListExpr(const Node *Ty_, NodeArray Inits_) + : Node(KInitListExpr), Ty(Ty_), Inits(Inits_) {} + + template void match(Fn F) const { F(Ty, Inits); } + + void printLeft(OutputStream &S) const override { + if (Ty) + Ty->print(S); + S += '{'; + Inits.printWithComma(S); + S += '}'; + } +}; + +class BracedExpr : public Node { + const Node *Elem; + const Node *Init; + bool IsArray; +public: + BracedExpr(const Node *Elem_, const Node *Init_, bool IsArray_) + : Node(KBracedExpr), Elem(Elem_), Init(Init_), IsArray(IsArray_) {} + + template void match(Fn F) const { F(Elem, Init, IsArray); } + + void printLeft(OutputStream &S) const override { + if (IsArray) { + S += '['; + Elem->print(S); + S += ']'; + } else { + S += '.'; + Elem->print(S); + } + if (Init->getKind() != KBracedExpr && Init->getKind() != KBracedRangeExpr) + S += " = "; + Init->print(S); + } +}; + +class BracedRangeExpr : public Node { + const Node *First; + const Node *Last; + const Node *Init; +public: + BracedRangeExpr(const Node *First_, const Node *Last_, const Node *Init_) + : Node(KBracedRangeExpr), First(First_), Last(Last_), Init(Init_) {} + + template void match(Fn F) const { F(First, Last, Init); } + + void printLeft(OutputStream &S) const override { + S += '['; + First->print(S); + S += " ... "; + Last->print(S); + S += ']'; + if (Init->getKind() != KBracedExpr && Init->getKind() != KBracedRangeExpr) + S += " = "; + Init->print(S); + } +}; + +class FoldExpr : public Node { + const Node *Pack, *Init; + StringView OperatorName; + bool IsLeftFold; + +public: + FoldExpr(bool IsLeftFold_, StringView OperatorName_, const Node *Pack_, + const Node *Init_) + : Node(KFoldExpr), Pack(Pack_), Init(Init_), OperatorName(OperatorName_), + IsLeftFold(IsLeftFold_) {} + + template void match(Fn F) const { + F(IsLeftFold, OperatorName, Pack, Init); + } + + void printLeft(OutputStream &S) const override { + auto PrintPack = [&] { + S += '('; + ParameterPackExpansion(Pack).print(S); + S += ')'; + }; + + S += '('; + + if (IsLeftFold) { + // init op ... op pack + if (Init != nullptr) { + Init->print(S); + S += ' '; + S += OperatorName; + S += ' '; + } + // ... op pack + S += "... "; + S += OperatorName; + S += ' '; + PrintPack(); + } else { // !IsLeftFold + // pack op ... + PrintPack(); + S += ' '; + S += OperatorName; + S += " ..."; + // pack op ... op init + if (Init != nullptr) { + S += ' '; + S += OperatorName; + S += ' '; + Init->print(S); + } + } + S += ')'; + } +}; + +class ThrowExpr : public Node { + const Node *Op; + +public: + ThrowExpr(const Node *Op_) : Node(KThrowExpr), Op(Op_) {} + + template void match(Fn F) const { F(Op); } + + void printLeft(OutputStream &S) const override { + S += "throw "; + Op->print(S); + } +}; + +class BoolExpr : public Node { + bool Value; + +public: + BoolExpr(bool Value_) : Node(KBoolExpr), Value(Value_) {} + + template void match(Fn F) const { F(Value); } + + void printLeft(OutputStream &S) const override { + S += Value ? StringView("true") : StringView("false"); + } +}; + +class IntegerCastExpr : public Node { + // ty(integer) + const Node *Ty; + StringView Integer; + +public: + IntegerCastExpr(const Node *Ty_, StringView Integer_) + : Node(KIntegerCastExpr), Ty(Ty_), Integer(Integer_) {} + + template void match(Fn F) const { F(Ty, Integer); } + + void printLeft(OutputStream &S) const override { + S += "("; + Ty->print(S); + S += ")"; + S += Integer; + } +}; + +class IntegerLiteral : public Node { + StringView Type; + StringView Value; + +public: + IntegerLiteral(StringView Type_, StringView Value_) + : Node(KIntegerLiteral), Type(Type_), Value(Value_) {} + + template void match(Fn F) const { F(Type, Value); } + + void printLeft(OutputStream &S) const override { + if (Type.size() > 3) { + S += "("; + S += Type; + S += ")"; + } + + if (Value[0] == 'n') { + S += "-"; + S += Value.dropFront(1); + } else + S += Value; + + if (Type.size() <= 3) + S += Type; + } +}; + +template struct FloatData; + +namespace float_literal_impl { +constexpr Node::Kind getFloatLiteralKind(float *) { + return Node::KFloatLiteral; +} +constexpr Node::Kind getFloatLiteralKind(double *) { + return Node::KDoubleLiteral; +} +constexpr Node::Kind getFloatLiteralKind(long double *) { + return Node::KLongDoubleLiteral; +} +} + +template class FloatLiteralImpl : public Node { + const StringView Contents; + + static constexpr Kind KindForClass = + float_literal_impl::getFloatLiteralKind((Float *)nullptr); + +public: + FloatLiteralImpl(StringView Contents_) + : Node(KindForClass), Contents(Contents_) {} + + template void match(Fn F) const { F(Contents); } + + void printLeft(OutputStream &s) const override { + const char *first = Contents.begin(); + const char *last = Contents.end() + 1; + + const size_t N = FloatData::mangled_size; + if (static_cast(last - first) > N) { + last = first + N; + union { + Float value; + char buf[sizeof(Float)]; + }; + const char *t = first; + char *e = buf; + for (; t != last; ++t, ++e) { + unsigned d1 = isdigit(*t) ? static_cast(*t - '0') + : static_cast(*t - 'a' + 10); + ++t; + unsigned d0 = isdigit(*t) ? static_cast(*t - '0') + : static_cast(*t - 'a' + 10); + *e = static_cast((d1 << 4) + d0); + } +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + std::reverse(buf, e); +#endif + char num[FloatData::max_demangled_size] = {0}; + int n = snprintf(num, sizeof(num), FloatData::spec, value); + s += StringView(num, num + n); + } + } +}; + +using FloatLiteral = FloatLiteralImpl; +using DoubleLiteral = FloatLiteralImpl; +using LongDoubleLiteral = FloatLiteralImpl; + +/// Visit the node. Calls \c F(P), where \c P is the node cast to the +/// appropriate derived class. +template +void Node::visit(Fn F) const { + switch (K) { +#define CASE(X) case K ## X: return F(static_cast(this)); + FOR_EACH_NODE_KIND(CASE) +#undef CASE + } + assert(0 && "unknown mangling node kind"); +} + +/// Determine the kind of a node from its type. +template struct NodeKind; +#define SPECIALIZATION(X) \ + template<> struct NodeKind { \ + static constexpr Node::Kind Kind = Node::K##X; \ + static constexpr const char *name() { return #X; } \ + }; +FOR_EACH_NODE_KIND(SPECIALIZATION) +#undef SPECIALIZATION + +#undef FOR_EACH_NODE_KIND + +template +class PODSmallVector { + static_assert(std::is_pod::value, + "T is required to be a plain old data type"); + + T* First; + T* Last; + T* Cap; + T Inline[N]; + + bool isInline() const { return First == Inline; } + + void clearInline() { + First = Inline; + Last = Inline; + Cap = Inline + N; + } + + void reserve(size_t NewCap) { + size_t S = size(); + if (isInline()) { + auto* Tmp = static_cast(std::malloc(NewCap * sizeof(T))); + if (Tmp == nullptr) + std::terminate(); + std::copy(First, Last, Tmp); + First = Tmp; + } else { + First = static_cast(std::realloc(First, NewCap * sizeof(T))); + if (First == nullptr) + std::terminate(); + } + Last = First + S; + Cap = First + NewCap; + } + +public: + PODSmallVector() : First(Inline), Last(First), Cap(Inline + N) {} + + PODSmallVector(const PODSmallVector&) = delete; + PODSmallVector& operator=(const PODSmallVector&) = delete; + + PODSmallVector(PODSmallVector&& Other) : PODSmallVector() { + if (Other.isInline()) { + std::copy(Other.begin(), Other.end(), First); + Last = First + Other.size(); + Other.clear(); + return; + } + + First = Other.First; + Last = Other.Last; + Cap = Other.Cap; + Other.clearInline(); + } + + PODSmallVector& operator=(PODSmallVector&& Other) { + if (Other.isInline()) { + if (!isInline()) { + std::free(First); + clearInline(); + } + std::copy(Other.begin(), Other.end(), First); + Last = First + Other.size(); + Other.clear(); + return *this; + } + + if (isInline()) { + First = Other.First; + Last = Other.Last; + Cap = Other.Cap; + Other.clearInline(); + return *this; + } + + std::swap(First, Other.First); + std::swap(Last, Other.Last); + std::swap(Cap, Other.Cap); + Other.clear(); + return *this; + } + + void push_back(const T& Elem) { + if (Last == Cap) + reserve(size() * 2); + *Last++ = Elem; + } + + void pop_back() { + assert(Last != First && "Popping empty vector!"); + --Last; + } + + void dropBack(size_t Index) { + assert(Index <= size() && "dropBack() can't expand!"); + Last = First + Index; + } + + T* begin() { return First; } + T* end() { return Last; } + + bool empty() const { return First == Last; } + size_t size() const { return static_cast(Last - First); } + T& back() { + assert(Last != First && "Calling back() on empty vector!"); + return *(Last - 1); + } + T& operator[](size_t Index) { + assert(Index < size() && "Invalid access!"); + return *(begin() + Index); + } + void clear() { Last = First; } + + ~PODSmallVector() { + if (!isInline()) + std::free(First); + } +}; + +template struct AbstractManglingParser { + const char *First; + const char *Last; + + // Name stack, this is used by the parser to hold temporary names that were + // parsed. The parser collapses multiple names into new nodes to construct + // the AST. Once the parser is finished, names.size() == 1. + PODSmallVector Names; + + // Substitution table. Itanium supports name substitutions as a means of + // compression. The string "S42_" refers to the 44nd entry (base-36) in this + // table. + PODSmallVector Subs; + + // Template parameter table. Like the above, but referenced like "T42_". + // This has a smaller size compared to Subs and Names because it can be + // stored on the stack. + PODSmallVector TemplateParams; + + // Set of unresolved forward references. These can occur in a + // conversion operator's type, and are resolved in the enclosing . + PODSmallVector ForwardTemplateRefs; + + bool TryToParseTemplateArgs = true; + bool PermitForwardTemplateReferences = false; + bool ParsingLambdaParams = false; + + Alloc ASTAllocator; + + AbstractManglingParser(const char *First_, const char *Last_) + : First(First_), Last(Last_) {} + + Derived &getDerived() { return static_cast(*this); } + + void reset(const char *First_, const char *Last_) { + First = First_; + Last = Last_; + Names.clear(); + Subs.clear(); + TemplateParams.clear(); + ParsingLambdaParams = false; + TryToParseTemplateArgs = true; + PermitForwardTemplateReferences = false; + ASTAllocator.reset(); + } + + template Node *make(Args &&... args) { + return ASTAllocator.template makeNode(std::forward(args)...); + } + + template NodeArray makeNodeArray(It begin, It end) { + size_t sz = static_cast(end - begin); + void *mem = ASTAllocator.allocateNodeArray(sz); + Node **data = new (mem) Node *[sz]; + std::copy(begin, end, data); + return NodeArray(data, sz); + } + + NodeArray popTrailingNodeArray(size_t FromPosition) { + assert(FromPosition <= Names.size()); + NodeArray res = + makeNodeArray(Names.begin() + (long)FromPosition, Names.end()); + Names.dropBack(FromPosition); + return res; + } + + bool consumeIf(StringView S) { + if (StringView(First, Last).startsWith(S)) { + First += S.size(); + return true; + } + return false; + } + + bool consumeIf(char C) { + if (First != Last && *First == C) { + ++First; + return true; + } + return false; + } + + char consume() { return First != Last ? *First++ : '\0'; } + + char look(unsigned Lookahead = 0) { + if (static_cast(Last - First) <= Lookahead) + return '\0'; + return First[Lookahead]; + } + + size_t numLeft() const { return static_cast(Last - First); } + + StringView parseNumber(bool AllowNegative = false); + Qualifiers parseCVQualifiers(); + bool parsePositiveInteger(size_t *Out); + StringView parseBareSourceName(); + + bool parseSeqId(size_t *Out); + Node *parseSubstitution(); + Node *parseTemplateParam(); + Node *parseTemplateArgs(bool TagTemplates = false); + Node *parseTemplateArg(); + + /// Parse the production. + Node *parseExpr(); + Node *parsePrefixExpr(StringView Kind); + Node *parseBinaryExpr(StringView Kind); + Node *parseIntegerLiteral(StringView Lit); + Node *parseExprPrimary(); + template Node *parseFloatingLiteral(); + Node *parseFunctionParam(); + Node *parseNewExpr(); + Node *parseConversionExpr(); + Node *parseBracedExpr(); + Node *parseFoldExpr(); + + /// Parse the production. + Node *parseType(); + Node *parseFunctionType(); + Node *parseVectorType(); + Node *parseDecltype(); + Node *parseArrayType(); + Node *parsePointerToMemberType(); + Node *parseClassEnumType(); + Node *parseQualifiedType(); + + Node *parseEncoding(); + bool parseCallOffset(); + Node *parseSpecialName(); + + /// Holds some extra information about a that is being parsed. This + /// information is only pertinent if the refers to an . + struct NameState { + bool CtorDtorConversion = false; + bool EndsWithTemplateArgs = false; + Qualifiers CVQualifiers = QualNone; + FunctionRefQual ReferenceQualifier = FrefQualNone; + size_t ForwardTemplateRefsBegin; + + NameState(AbstractManglingParser *Enclosing) + : ForwardTemplateRefsBegin(Enclosing->ForwardTemplateRefs.size()) {} + }; + + bool resolveForwardTemplateRefs(NameState &State) { + size_t I = State.ForwardTemplateRefsBegin; + size_t E = ForwardTemplateRefs.size(); + for (; I < E; ++I) { + size_t Idx = ForwardTemplateRefs[I]->Index; + if (Idx >= TemplateParams.size()) + return true; + ForwardTemplateRefs[I]->Ref = TemplateParams[Idx]; + } + ForwardTemplateRefs.dropBack(State.ForwardTemplateRefsBegin); + return false; + } + + /// Parse the production> + Node *parseName(NameState *State = nullptr); + Node *parseLocalName(NameState *State); + Node *parseOperatorName(NameState *State); + Node *parseUnqualifiedName(NameState *State); + Node *parseUnnamedTypeName(NameState *State); + Node *parseSourceName(NameState *State); + Node *parseUnscopedName(NameState *State); + Node *parseNestedName(NameState *State); + Node *parseCtorDtorName(Node *&SoFar, NameState *State); + + Node *parseAbiTags(Node *N); + + /// Parse the production. + Node *parseUnresolvedName(); + Node *parseSimpleId(); + Node *parseBaseUnresolvedName(); + Node *parseUnresolvedType(); + Node *parseDestructorName(); + + /// Top-level entry point into the parser. + Node *parse(); +}; + +const char* parse_discriminator(const char* first, const char* last); + +// ::= // N +// ::= # See Scope Encoding below // Z +// ::= +// ::= +// +// ::= +// ::= +template +Node *AbstractManglingParser::parseName(NameState *State) { + consumeIf('L'); // extension + + if (look() == 'N') + return getDerived().parseNestedName(State); + if (look() == 'Z') + return getDerived().parseLocalName(State); + + // ::= + if (look() == 'S' && look(1) != 't') { + Node *S = getDerived().parseSubstitution(); + if (S == nullptr) + return nullptr; + if (look() != 'I') + return nullptr; + Node *TA = getDerived().parseTemplateArgs(State != nullptr); + if (TA == nullptr) + return nullptr; + if (State) State->EndsWithTemplateArgs = true; + return make(S, TA); + } + + Node *N = getDerived().parseUnscopedName(State); + if (N == nullptr) + return nullptr; + // ::= + if (look() == 'I') { + Subs.push_back(N); + Node *TA = getDerived().parseTemplateArgs(State != nullptr); + if (TA == nullptr) + return nullptr; + if (State) State->EndsWithTemplateArgs = true; + return make(N, TA); + } + // ::= + return N; +} + +// := Z E [] +// := Z E s [] +// := Z Ed [ ] _ +template +Node *AbstractManglingParser::parseLocalName(NameState *State) { + if (!consumeIf('Z')) + return nullptr; + Node *Encoding = getDerived().parseEncoding(); + if (Encoding == nullptr || !consumeIf('E')) + return nullptr; + + if (consumeIf('s')) { + First = parse_discriminator(First, Last); + auto *StringLitName = make("string literal"); + if (!StringLitName) + return nullptr; + return make(Encoding, StringLitName); + } + + if (consumeIf('d')) { + parseNumber(true); + if (!consumeIf('_')) + return nullptr; + Node *N = getDerived().parseName(State); + if (N == nullptr) + return nullptr; + return make(Encoding, N); + } + + Node *Entity = getDerived().parseName(State); + if (Entity == nullptr) + return nullptr; + First = parse_discriminator(First, Last); + return make(Encoding, Entity); +} + +// ::= +// ::= St # ::std:: +// extension ::= StL +template +Node * +AbstractManglingParser::parseUnscopedName(NameState *State) { + if (consumeIf("StL") || consumeIf("St")) { + Node *R = getDerived().parseUnqualifiedName(State); + if (R == nullptr) + return nullptr; + return make(R); + } + return getDerived().parseUnqualifiedName(State); +} + +// ::= [abi-tags] +// ::= +// ::= +// ::= +// ::= DC + E # structured binding declaration +template +Node * +AbstractManglingParser::parseUnqualifiedName(NameState *State) { + // s are special-cased in parseNestedName(). + Node *Result; + if (look() == 'U') + Result = getDerived().parseUnnamedTypeName(State); + else if (look() >= '1' && look() <= '9') + Result = getDerived().parseSourceName(State); + else if (consumeIf("DC")) { + size_t BindingsBegin = Names.size(); + do { + Node *Binding = getDerived().parseSourceName(State); + if (Binding == nullptr) + return nullptr; + Names.push_back(Binding); + } while (!consumeIf('E')); + Result = make(popTrailingNodeArray(BindingsBegin)); + } else + Result = getDerived().parseOperatorName(State); + if (Result != nullptr) + Result = getDerived().parseAbiTags(Result); + return Result; +} + +// ::= Ut [] _ +// ::= +// +// ::= Ul E [ ] _ +// +// ::= + # Parameter types or "v" if the lambda has no parameters +template +Node * +AbstractManglingParser::parseUnnamedTypeName(NameState *) { + if (consumeIf("Ut")) { + StringView Count = parseNumber(); + if (!consumeIf('_')) + return nullptr; + return make(Count); + } + if (consumeIf("Ul")) { + NodeArray Params; + SwapAndRestore SwapParams(ParsingLambdaParams, true); + if (!consumeIf("vE")) { + size_t ParamsBegin = Names.size(); + do { + Node *P = getDerived().parseType(); + if (P == nullptr) + return nullptr; + Names.push_back(P); + } while (!consumeIf('E')); + Params = popTrailingNodeArray(ParamsBegin); + } + StringView Count = parseNumber(); + if (!consumeIf('_')) + return nullptr; + return make(Params, Count); + } + return nullptr; +} + +// ::= +template +Node *AbstractManglingParser::parseSourceName(NameState *) { + size_t Length = 0; + if (parsePositiveInteger(&Length)) + return nullptr; + if (numLeft() < Length || Length == 0) + return nullptr; + StringView Name(First, First + Length); + First += Length; + if (Name.startsWith("_GLOBAL__N")) + return make("(anonymous namespace)"); + return make(Name); +} + +// ::= aa # && +// ::= ad # & (unary) +// ::= an # & +// ::= aN # &= +// ::= aS # = +// ::= cl # () +// ::= cm # , +// ::= co # ~ +// ::= cv # (cast) +// ::= da # delete[] +// ::= de # * (unary) +// ::= dl # delete +// ::= dv # / +// ::= dV # /= +// ::= eo # ^ +// ::= eO # ^= +// ::= eq # == +// ::= ge # >= +// ::= gt # > +// ::= ix # [] +// ::= le # <= +// ::= li # operator "" +// ::= ls # << +// ::= lS # <<= +// ::= lt # < +// ::= mi # - +// ::= mI # -= +// ::= ml # * +// ::= mL # *= +// ::= mm # -- (postfix in context) +// ::= na # new[] +// ::= ne # != +// ::= ng # - (unary) +// ::= nt # ! +// ::= nw # new +// ::= oo # || +// ::= or # | +// ::= oR # |= +// ::= pm # ->* +// ::= pl # + +// ::= pL # += +// ::= pp # ++ (postfix in context) +// ::= ps # + (unary) +// ::= pt # -> +// ::= qu # ? +// ::= rm # % +// ::= rM # %= +// ::= rs # >> +// ::= rS # >>= +// ::= ss # <=> C++2a +// ::= v # vendor extended operator +template +Node * +AbstractManglingParser::parseOperatorName(NameState *State) { + switch (look()) { + case 'a': + switch (look(1)) { + case 'a': + First += 2; + return make("operator&&"); + case 'd': + case 'n': + First += 2; + return make("operator&"); + case 'N': + First += 2; + return make("operator&="); + case 'S': + First += 2; + return make("operator="); + } + return nullptr; + case 'c': + switch (look(1)) { + case 'l': + First += 2; + return make("operator()"); + case 'm': + First += 2; + return make("operator,"); + case 'o': + First += 2; + return make("operator~"); + // ::= cv # (cast) + case 'v': { + First += 2; + SwapAndRestore SaveTemplate(TryToParseTemplateArgs, false); + // If we're parsing an encoding, State != nullptr and the conversion + // operators' could have a that refers to some + // s further ahead in the mangled name. + SwapAndRestore SavePermit(PermitForwardTemplateReferences, + PermitForwardTemplateReferences || + State != nullptr); + Node *Ty = getDerived().parseType(); + if (Ty == nullptr) + return nullptr; + if (State) State->CtorDtorConversion = true; + return make(Ty); + } + } + return nullptr; + case 'd': + switch (look(1)) { + case 'a': + First += 2; + return make("operator delete[]"); + case 'e': + First += 2; + return make("operator*"); + case 'l': + First += 2; + return make("operator delete"); + case 'v': + First += 2; + return make("operator/"); + case 'V': + First += 2; + return make("operator/="); + } + return nullptr; + case 'e': + switch (look(1)) { + case 'o': + First += 2; + return make("operator^"); + case 'O': + First += 2; + return make("operator^="); + case 'q': + First += 2; + return make("operator=="); + } + return nullptr; + case 'g': + switch (look(1)) { + case 'e': + First += 2; + return make("operator>="); + case 't': + First += 2; + return make("operator>"); + } + return nullptr; + case 'i': + if (look(1) == 'x') { + First += 2; + return make("operator[]"); + } + return nullptr; + case 'l': + switch (look(1)) { + case 'e': + First += 2; + return make("operator<="); + // ::= li # operator "" + case 'i': { + First += 2; + Node *SN = getDerived().parseSourceName(State); + if (SN == nullptr) + return nullptr; + return make(SN); + } + case 's': + First += 2; + return make("operator<<"); + case 'S': + First += 2; + return make("operator<<="); + case 't': + First += 2; + return make("operator<"); + } + return nullptr; + case 'm': + switch (look(1)) { + case 'i': + First += 2; + return make("operator-"); + case 'I': + First += 2; + return make("operator-="); + case 'l': + First += 2; + return make("operator*"); + case 'L': + First += 2; + return make("operator*="); + case 'm': + First += 2; + return make("operator--"); + } + return nullptr; + case 'n': + switch (look(1)) { + case 'a': + First += 2; + return make("operator new[]"); + case 'e': + First += 2; + return make("operator!="); + case 'g': + First += 2; + return make("operator-"); + case 't': + First += 2; + return make("operator!"); + case 'w': + First += 2; + return make("operator new"); + } + return nullptr; + case 'o': + switch (look(1)) { + case 'o': + First += 2; + return make("operator||"); + case 'r': + First += 2; + return make("operator|"); + case 'R': + First += 2; + return make("operator|="); + } + return nullptr; + case 'p': + switch (look(1)) { + case 'm': + First += 2; + return make("operator->*"); + case 'l': + First += 2; + return make("operator+"); + case 'L': + First += 2; + return make("operator+="); + case 'p': + First += 2; + return make("operator++"); + case 's': + First += 2; + return make("operator+"); + case 't': + First += 2; + return make("operator->"); + } + return nullptr; + case 'q': + if (look(1) == 'u') { + First += 2; + return make("operator?"); + } + return nullptr; + case 'r': + switch (look(1)) { + case 'm': + First += 2; + return make("operator%"); + case 'M': + First += 2; + return make("operator%="); + case 's': + First += 2; + return make("operator>>"); + case 'S': + First += 2; + return make("operator>>="); + } + return nullptr; + case 's': + if (look(1) == 's') { + First += 2; + return make("operator<=>"); + } + return nullptr; + // ::= v # vendor extended operator + case 'v': + if (std::isdigit(look(1))) { + First += 2; + Node *SN = getDerived().parseSourceName(State); + if (SN == nullptr) + return nullptr; + return make(SN); + } + return nullptr; + } + return nullptr; +} + +// ::= C1 # complete object constructor +// ::= C2 # base object constructor +// ::= C3 # complete object allocating constructor +// extension ::= C5 # ? +// ::= D0 # deleting destructor +// ::= D1 # complete object destructor +// ::= D2 # base object destructor +// extension ::= D5 # ? +template +Node * +AbstractManglingParser::parseCtorDtorName(Node *&SoFar, + NameState *State) { + if (SoFar->getKind() == Node::KSpecialSubstitution) { + auto SSK = static_cast(SoFar)->SSK; + switch (SSK) { + case SpecialSubKind::string: + case SpecialSubKind::istream: + case SpecialSubKind::ostream: + case SpecialSubKind::iostream: + SoFar = make(SSK); + if (!SoFar) + return nullptr; + break; + default: + break; + } + } + + if (consumeIf('C')) { + bool IsInherited = consumeIf('I'); + if (look() != '1' && look() != '2' && look() != '3' && look() != '5') + return nullptr; + int Variant = look() - '0'; + ++First; + if (State) State->CtorDtorConversion = true; + if (IsInherited) { + if (getDerived().parseName(State) == nullptr) + return nullptr; + } + return make(SoFar, false, Variant); + } + + if (look() == 'D' && + (look(1) == '0' || look(1) == '1' || look(1) == '2' || look(1) == '5')) { + int Variant = look(1) - '0'; + First += 2; + if (State) State->CtorDtorConversion = true; + return make(SoFar, true, Variant); + } + + return nullptr; +} + +// ::= N [] [] E +// ::= N [] [] E +// +// ::= +// ::= +// ::= +// ::= +// ::= # empty +// ::= +// ::= +// extension ::= L +// +// := [] M +// +// ::=