From 5d00c5a6571cdf533228338fa7ca532c91af1d66 Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Fri, 29 Mar 2019 17:52:57 +0000 Subject: [PATCH] Fix initial exec TLS mode for dynamically loaded shared objects. If dso uses initial exec TLS mode, rtld tries to allocate TLS in static space. If there is no space left, the dlopen(3) fails. If space if allocated, initial content from PT_TLS segment is distributed to all threads' pcbs, which was missed and caused un-initialized TLS segment for such dso after dlopen(3). The mode is auto-detected either due to the relocation used, or if the DF_STATIC_TLS dynamic flag is set. In the later case, the TLS segment is tried to allocate earlier, which increases chance of the dlopen(3) to succeed. LLD was recently fixed to properly emit the flag, ld.bdf did it always. Initial test by: dumbbell Tested by: emaste (amd64), ian (arm) Tested by: Gerald Aryeetey (arm64) Sponsored by: The FreeBSD Foundation MFC after: 2 weeks Differential revision: https://reviews.freebsd.org/D19072 --- lib/libc/aarch64/static_tls.h | 46 +++++++++++++ lib/libc/amd64/static_tls.h | 46 +++++++++++++ lib/libc/arm/static_tls.h | 51 +++++++++++++++ lib/libc/gen/Symbol.map | 1 + lib/libc/gen/elf_utils.c | 24 +++++++ lib/libc/i386/static_tls.h | 46 +++++++++++++ lib/libc/include/libc_private.h | 3 + lib/libc/mips/static_tls.h | 64 +++++++++++++++++++ lib/libc/powerpc/static_tls.h | 46 +++++++++++++ lib/libc/powerpc64/static_tls.h | 46 +++++++++++++ lib/libc/riscv/static_tls.h | 46 +++++++++++++ lib/libc/sparc64/static_tls.h | 44 +++++++++++++ lib/libc/sys/interposing_table.c | 1 + lib/libthr/Makefile | 6 +- lib/libthr/arch/aarch64/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/amd64/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/arm/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/i386/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/mips/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/powerpc/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/riscv/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/arch/sparc64/include/pthread_tls.h | 46 +++++++++++++ lib/libthr/pthread.map | 1 + lib/libthr/thread/thr_list.c | 33 ++++++++++ lib/libthr/thread/thr_private.h | 2 + libexec/rtld-elf/rtld.c | 46 +++++++++++-- libexec/rtld-elf/rtld.h | 2 + 27 files changed, 914 insertions(+), 8 deletions(-) create mode 100644 lib/libc/aarch64/static_tls.h create mode 100644 lib/libc/amd64/static_tls.h create mode 100644 lib/libc/arm/static_tls.h create mode 100644 lib/libc/i386/static_tls.h create mode 100644 lib/libc/mips/static_tls.h create mode 100644 lib/libc/powerpc/static_tls.h create mode 100644 lib/libc/powerpc64/static_tls.h create mode 100644 lib/libc/riscv/static_tls.h create mode 100644 lib/libc/sparc64/static_tls.h create mode 100644 lib/libthr/arch/aarch64/include/pthread_tls.h create mode 100644 lib/libthr/arch/amd64/include/pthread_tls.h create mode 100644 lib/libthr/arch/arm/include/pthread_tls.h create mode 100644 lib/libthr/arch/i386/include/pthread_tls.h create mode 100644 lib/libthr/arch/mips/include/pthread_tls.h create mode 100644 lib/libthr/arch/powerpc/include/pthread_tls.h create mode 100644 lib/libthr/arch/riscv/include/pthread_tls.h create mode 100644 lib/libthr/arch/sparc64/include/pthread_tls.h diff --git a/lib/libc/aarch64/static_tls.h b/lib/libc/aarch64/static_tls.h new file mode 100644 index 00000000000..78f6b50f012 --- /dev/null +++ b/lib/libc/aarch64/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_AARCH64_STATIC_TLS_H +#define _LIBC_AARCH64_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("mrs %x0, tpidr_el0" : "=r" (tlsbase)); + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libc/amd64/static_tls.h b/lib/libc/amd64/static_tls.h new file mode 100644 index 00000000000..1e9b76f5822 --- /dev/null +++ b/lib/libc/amd64/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_AMD64_STATIC_TLS_H +#define _LIBC_AMD64_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movq %%fs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} + +#endif diff --git a/lib/libc/arm/static_tls.h b/lib/libc/arm/static_tls.h new file mode 100644 index 00000000000..e6be9adfcff --- /dev/null +++ b/lib/libc/arm/static_tls.h @@ -0,0 +1,51 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_ARM_STATIC_TLS_H +#define _LIBC_ARM_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + +#ifdef ARM_TP_ADDRESS + tlsbase = *(uintptr_t *)ARM_TP_ADDRESS; +#else + __asm __volatile("mrc p15, 0, %0, c13, c0, 3" : "=r" (tlsbase)); +#endif + + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map index 010358ba9bc..9c6b969ea8d 100644 --- a/lib/libc/gen/Symbol.map +++ b/lib/libc/gen/Symbol.map @@ -542,6 +542,7 @@ FBSDprivate_1.0 { __libc_tcdrain; __elf_aux_vector; + __pthread_distribute_static_tls; __pthread_map_stacks_exec; __fillcontextx; __fillcontextx2; diff --git a/lib/libc/gen/elf_utils.c b/lib/libc/gen/elf_utils.c index 417f1842aad..819620c1e19 100644 --- a/lib/libc/gen/elf_utils.c +++ b/lib/libc/gen/elf_utils.c @@ -34,10 +34,13 @@ #include #include #include +#include #include "libc_private.h" +#include "static_tls.h" int __elf_phdr_match_addr(struct dl_phdr_info *, void *); void __pthread_map_stacks_exec(void); +void __pthread_distribute_static_tls(size_t, void *, size_t, size_t); int __elf_phdr_match_addr(struct dl_phdr_info *phdr_info, void *addr) @@ -97,3 +100,24 @@ __pthread_map_stacks_exec(void) ((void (*)(void))__libc_interposing[INTERPOS_map_stacks_exec])(); } + +void +__libc_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + uintptr_t tlsbase; + + tlsbase = _libc_get_static_tls_base(offset); + memcpy((void *)tlsbase, src, len); + memset((char *)tlsbase + len, 0, total_len - len); +} + +#pragma weak __pthread_distribute_static_tls +void +__pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + + ((void (*)(size_t, void *, size_t, size_t))__libc_interposing[ + INTERPOS_distribute_static_tls])(offset, src, len, total_len); +} diff --git a/lib/libc/i386/static_tls.h b/lib/libc/i386/static_tls.h new file mode 100644 index 00000000000..b0685d2e0f8 --- /dev/null +++ b/lib/libc/i386/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_I386_STATIC_TLS_H +#define _LIBC_I386_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movl %%gs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} + +#endif diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h index aaa40deeb08..116764b399b 100644 --- a/lib/libc/include/libc_private.h +++ b/lib/libc/include/libc_private.h @@ -235,6 +235,7 @@ enum { INTERPOS_map_stacks_exec, INTERPOS_fdatasync, INTERPOS_clock_nanosleep, + INTERPOS_distribute_static_tls, INTERPOS_MAX }; @@ -413,6 +414,8 @@ struct dl_phdr_info; int __elf_phdr_match_addr(struct dl_phdr_info *, void *); void __init_elf_aux_vector(void); void __libc_map_stacks_exec(void); +void __libc_distribute_static_tls(__size_t, void *, __size_t, __size_t); +__uintptr_t __libc_static_tls_base(__size_t); void _pthread_cancel_enter(int); void _pthread_cancel_leave(int); diff --git a/lib/libc/mips/static_tls.h b/lib/libc/mips/static_tls.h new file mode 100644 index 00000000000..67ee8afe14c --- /dev/null +++ b/lib/libc/mips/static_tls.h @@ -0,0 +1,64 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_MIPS_STATIC_TLS_H +#define _LIBC_MIPS_STATIC_TLS_H + +#include + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + +#if defined(__mips_n64) + __asm__ __volatile__ ( + ".set\tpush\n\t" + ".set\tmips64r2\n\t" + "rdhwr\t%0, $29\n\t" + ".set\tpop" + : "=r" (tlsbase)); + tlsbase -= TLS_TP_OFFSET + TLS_TCB_SIZE; +#else /* mips 32 */ + __asm__ __volatile__ ( + ".set\tpush\n\t" + ".set\tmips32r2\n\t" + "rdhwr\t%0, $29\n\t" + ".set\tpop" + : "=r" (tlsbase)); + tlsbase -= TLS_TP_OFFSET + TLS_TCB_SIZE; +#endif /* ! __mips_n64 */ + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libc/powerpc/static_tls.h b/lib/libc/powerpc/static_tls.h new file mode 100644 index 00000000000..fd7a80d4923 --- /dev/null +++ b/lib/libc/powerpc/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_POWERPC_STATIC_TLS_H +#define _LIBC_POWERPC_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("mr %0,2" : "=r"(tlsbase)); + tlsbase += offset - 0x7008; + return (tlsbase); +} + +#endif diff --git a/lib/libc/powerpc64/static_tls.h b/lib/libc/powerpc64/static_tls.h new file mode 100644 index 00000000000..63adba39bab --- /dev/null +++ b/lib/libc/powerpc64/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_POWERPC64_STATIC_TLS_H +#define _LIBC_POWERPC64_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("mr %0,13" : "=r"(tlsbase)); + tlsbase += offset - 0x7010; + return (tlsbase); +} + +#endif diff --git a/lib/libc/riscv/static_tls.h b/lib/libc/riscv/static_tls.h new file mode 100644 index 00000000000..dcdc459b8c1 --- /dev/null +++ b/lib/libc/riscv/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_RISCV_STATIC_TLS_H +#define _LIBC_RISCV_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("mv %0, tp" : "=r"(tlsbase)); + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libc/sparc64/static_tls.h b/lib/libc/sparc64/static_tls.h new file mode 100644 index 00000000000..dd59de97e7b --- /dev/null +++ b/lib/libc/sparc64/static_tls.h @@ -0,0 +1,44 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LIBC_SPARC64_STATIC_TLS_H +#define _LIBC_SPARC64_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + register uintptr_t tlsbase __asm("%g7"); + + return (tlsbase + offset); +} + +#endif diff --git a/lib/libc/sys/interposing_table.c b/lib/libc/sys/interposing_table.c index faf1e6ceeb6..670e9fd7dd0 100644 --- a/lib/libc/sys/interposing_table.c +++ b/lib/libc/sys/interposing_table.c @@ -81,6 +81,7 @@ interpos_func_t __libc_interposing[INTERPOS_MAX] = { SLOT(map_stacks_exec, __libc_map_stacks_exec), SLOT(fdatasync, __sys_fdatasync), SLOT(clock_nanosleep, __sys_clock_nanosleep), + SLOT(distribute_static_tls, __libc_distribute_static_tls), }; #undef SLOT diff --git a/lib/libthr/Makefile b/lib/libthr/Makefile index e2e571e1341..5f3d52661c9 100644 --- a/lib/libthr/Makefile +++ b/lib/libthr/Makefile @@ -16,8 +16,10 @@ SHLIB_MAJOR= 3 NO_WTHREAD_SAFETY=1 NO_WCAST_ALIGN.gcc=1 # for gcc 4.2 CFLAGS+=-DPTHREAD_KERNEL -CFLAGS+=-I${SRCTOP}/lib/libc/include -I${.CURDIR}/thread \ - -I${SRCTOP}/include +CFLAGS+=-I${SRCTOP}/lib/libc/include +CFLAGS+=-I${SRCTOP}/lib/libc/${MACHINE_CPUARCH} +CFLAGS+=-I${.CURDIR}/thread +CFLAGS+=-I${SRCTOP}/include CFLAGS+=-I${.CURDIR}/arch/${MACHINE_CPUARCH}/include CFLAGS+=-I${.CURDIR}/sys CFLAGS+=-I${SRCTOP}/libexec/rtld-elf diff --git a/lib/libthr/arch/aarch64/include/pthread_tls.h b/lib/libthr/arch/aarch64/include/pthread_tls.h new file mode 100644 index 00000000000..bd3a4fbb202 --- /dev/null +++ b/lib/libthr/arch/aarch64/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_AARCH64_PTHREAD_TLS_H +#define _ARCH_AARCH64_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/amd64/include/pthread_tls.h b/lib/libthr/arch/amd64/include/pthread_tls.h new file mode 100644 index 00000000000..c0ca6201d41 --- /dev/null +++ b/lib/libthr/arch/amd64/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_AMD64_PTHREAD_TLS_H +#define _ARCH_AMD64_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/arm/include/pthread_tls.h b/lib/libthr/arch/arm/include/pthread_tls.h new file mode 100644 index 00000000000..6aef9c22f36 --- /dev/null +++ b/lib/libthr/arch/arm/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_ARM_PTHREAD_TLS_H +#define _ARCH_ARM_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/i386/include/pthread_tls.h b/lib/libthr/arch/i386/include/pthread_tls.h new file mode 100644 index 00000000000..f48147fd645 --- /dev/null +++ b/lib/libthr/arch/i386/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_I386_PTHREAD_TLS_H +#define _ARCH_I386_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/mips/include/pthread_tls.h b/lib/libthr/arch/mips/include/pthread_tls.h new file mode 100644 index 00000000000..ed69d6d2a17 --- /dev/null +++ b/lib/libthr/arch/mips/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_MIPS_PTHREAD_TLS_H +#define _ARCH_MIPS_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/powerpc/include/pthread_tls.h b/lib/libthr/arch/powerpc/include/pthread_tls.h new file mode 100644 index 00000000000..5c8db311f8d --- /dev/null +++ b/lib/libthr/arch/powerpc/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_POWERPC_PTHREAD_TLS_H +#define _ARCH_POWERPC_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/riscv/include/pthread_tls.h b/lib/libthr/arch/riscv/include/pthread_tls.h new file mode 100644 index 00000000000..eb3e47b7026 --- /dev/null +++ b/lib/libthr/arch/riscv/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_RISCV_PTHREAD_TLS_H +#define _ARCH_RISCV_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase += offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/arch/sparc64/include/pthread_tls.h b/lib/libthr/arch/sparc64/include/pthread_tls.h new file mode 100644 index 00000000000..7dd194f0860 --- /dev/null +++ b/lib/libthr/arch/sparc64/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARCH_SPARC64_PTHREAD_TLS_H +#define _ARCH_SPARC64_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif diff --git a/lib/libthr/pthread.map b/lib/libthr/pthread.map index 19541b9e800..cd94078e602 100644 --- a/lib/libthr/pthread.map +++ b/lib/libthr/pthread.map @@ -137,6 +137,7 @@ FBSDprivate_1.0 { __pthread_mutex_lock; __pthread_mutex_timedlock; __pthread_mutex_trylock; + __pthread_distribute_static_tls; _pthread_atfork; _pthread_barrier_destroy; _pthread_barrier_init; diff --git a/lib/libthr/thread/thr_list.c b/lib/libthr/thread/thr_list.c index 530e49721a4..f28f059c290 100644 --- a/lib/libthr/thread/thr_list.c +++ b/lib/libthr/thread/thr_list.c @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include "libc_private.h" #include "thr_private.h" +#include "static_tls.h" /*#define DEBUG_THREAD_LIST */ #ifdef DEBUG_THREAD_LIST @@ -362,3 +363,35 @@ _thr_find_thread(struct pthread *curthread, struct pthread *thread, THREAD_LIST_UNLOCK(curthread); return (ret); } + +#include "pthread_tls.h" + +static void +thr_distribute_static_tls(uintptr_t tlsbase, void *src, size_t len, + size_t total_len) +{ + + memcpy((void *)tlsbase, src, len); + memset((char *)tlsbase + len, 0, total_len - len); +} + +void +__pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + struct pthread *curthread, *thrd; + uintptr_t tlsbase; + + if (!_thr_is_inited()) { + tlsbase = _libc_get_static_tls_base(offset); + thr_distribute_static_tls(tlsbase, src, len, total_len); + return; + } + curthread = _get_curthread(); + THREAD_LIST_RDLOCK(curthread); + TAILQ_FOREACH(thrd, &_thread_list, tle) { + tlsbase = _get_static_tls_base(thrd, offset); + thr_distribute_static_tls(tlsbase, src, len, total_len); + } + THREAD_LIST_UNLOCK(curthread); +} diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h index 897784ad721..a9720812d68 100644 --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -973,6 +973,8 @@ void __pthread_cxa_finalize(struct dl_phdr_info *phdr_info); void _thr_tsd_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_sigact_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_stack_fix_protection(struct pthread *thrd); +void __pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len); int *__error_threaded(void) __hidden; void __thr_interpose_libc(void) __hidden; diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c index c027fc89f41..b518ed045b1 100644 --- a/libexec/rtld-elf/rtld.c +++ b/libexec/rtld-elf/rtld.c @@ -92,6 +92,7 @@ static void digest_dynamic2(Obj_Entry *, const Elf_Dyn *, const Elf_Dyn *, const Elf_Dyn *); static void digest_dynamic(Obj_Entry *, int); static Obj_Entry *digest_phdr(const Elf_Phdr *, int, caddr_t, const char *); +static void distribute_static_tls(Objlist *, RtldLockState *); static Obj_Entry *dlcheck(void *); static int dlclose_locked(void *, RtldLockState *); static Obj_Entry *dlopen_object(const char *name, int fd, Obj_Entry *refobj, @@ -1247,8 +1248,8 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath, obj->textrel = true; if (dynp->d_un.d_val & DF_BIND_NOW) obj->bind_now = true; - /*if (dynp->d_un.d_val & DF_STATIC_TLS) - ;*/ + if (dynp->d_un.d_val & DF_STATIC_TLS) + obj->static_tls = true; break; #ifdef __mips__ case DT_MIPS_LOCAL_GOTNO: @@ -3330,8 +3331,16 @@ dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags, if (globallist_next(old_obj_tail) != NULL) { /* We loaded something new. */ assert(globallist_next(old_obj_tail) == obj); - result = load_needed_objects(obj, - lo_flags & (RTLD_LO_DLOPEN | RTLD_LO_EARLY)); + result = 0; + if ((lo_flags & RTLD_LO_EARLY) == 0 && obj->static_tls && + !allocate_tls_offset(obj)) { + _rtld_error("%s: No space available " + "for static Thread Local Storage", obj->path); + result = -1; + } + if (result != -1) + result = load_needed_objects(obj, lo_flags & (RTLD_LO_DLOPEN | + RTLD_LO_EARLY)); init_dag(obj); ref_dag(obj); if (result != -1) @@ -3391,8 +3400,10 @@ dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags, name); GDB_STATE(RT_CONSISTENT,obj ? &obj->linkmap : NULL); - if (!(lo_flags & RTLD_LO_EARLY)) { + if ((lo_flags & RTLD_LO_EARLY) == 0) { map_stacks_exec(lockstate); + if (obj != NULL) + distribute_static_tls(&initlist, lockstate); } if (initlist_objects_ifunc(&initlist, (mode & RTLD_MODEMASK) == RTLD_NOW, @@ -4917,8 +4928,10 @@ allocate_tls(Obj_Entry *objs, void *oldtls, size_t tcbsize, size_t tcbalign) addr = segbase - obj->tlsoffset; memset((void*)(addr + obj->tlsinitsize), 0, obj->tlssize - obj->tlsinitsize); - if (obj->tlsinit) + if (obj->tlsinit) { memcpy((void*) addr, obj->tlsinit, obj->tlsinitsize); + obj->static_tls_copied = true; + } dtv[obj->tlsindex + 1] = addr; } } @@ -5380,6 +5393,27 @@ map_stacks_exec(RtldLockState *lockstate) } } +static void +distribute_static_tls(Objlist *list, RtldLockState *lockstate) +{ + Objlist_Entry *elm; + Obj_Entry *obj; + void (*distrib)(size_t, void *, size_t, size_t); + + distrib = (void (*)(size_t, void *, size_t, size_t))(uintptr_t) + get_program_var_addr("__pthread_distribute_static_tls", lockstate); + if (distrib == NULL) + return; + STAILQ_FOREACH(elm, list, link) { + obj = elm->obj; + if (obj->marker || !obj->tls_done || obj->static_tls_copied) + continue; + distrib(obj->tlsoffset, obj->tlsinit, obj->tlsinitsize, + obj->tlssize); + obj->static_tls_copied = true; + } +} + void symlook_init(SymLook *dst, const char *name) { diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h index 735df1f5809..d7ef9d56b8d 100644 --- a/libexec/rtld-elf/rtld.h +++ b/libexec/rtld-elf/rtld.h @@ -256,6 +256,8 @@ typedef struct Struct_Obj_Entry { bool z_interpose : 1; /* Interpose all objects but main */ bool z_nodeflib : 1; /* Don't search default library path */ bool z_global : 1; /* Make the object global */ + bool static_tls : 1; /* Needs static TLS allocation */ + bool static_tls_copied : 1; /* Needs static TLS copying */ bool ref_nodel : 1; /* Refcount increased to prevent dlclose */ bool init_scanned: 1; /* Object is already on init list. */ bool on_fini_list: 1; /* Object is already on fini list. */