From 2bf9501287cd79dfb3c0b558692ef2a9057a06a3 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Thu, 5 Jul 2018 17:13:37 +0000 Subject: [PATCH] Create a new macro for static DPCPU data. On arm64 (and possible other architectures) we are unable to use static DPCPU data in kernel modules. This is because the compiler will generate PC-relative accesses, however the runtime-linker expects to be able to relocate these. In preparation to fix this create two macros depending on if the data is global or static. Reviewed by: bz, emaste, markj Sponsored by: ABT Systems Ltd Differential Revision: https://reviews.freebsd.org/D16140 --- share/man/man9/dpcpu.9 | 16 +++++++++------- sys/compat/linuxkpi/common/src/linux_idr.c | 2 +- sys/compat/linuxkpi/common/src/linux_rcu.c | 2 +- sys/compat/linuxkpi/common/src/linux_tasklet.c | 2 +- sys/kern/kern_clock.c | 2 +- sys/kern/kern_clocksource.c | 2 +- sys/kern/kern_exec.c | 2 +- sys/kern/kern_tc.c | 4 ++-- sys/kern/sched_4bsd.c | 2 +- sys/kern/sched_ule.c | 2 +- sys/kern/subr_pcpu.c | 2 +- sys/mips/mips/tick.c | 10 +++++----- sys/mips/nlm/tick.c | 10 +++++----- sys/netinet/siftr.c | 2 +- sys/netpfil/ipfw/ip_fw_dynamic.c | 2 +- sys/powerpc/powerpc/clock.c | 2 +- sys/sys/pcpu.h | 6 +++++- sys/vm/vm_page.c | 2 +- sys/x86/xen/xen_intr.c | 2 +- 19 files changed, 40 insertions(+), 34 deletions(-) diff --git a/share/man/man9/dpcpu.9 b/share/man/man9/dpcpu.9 index 39e04e1a9e6..a7867b46f31 100644 --- a/share/man/man9/dpcpu.9 +++ b/share/man/man9/dpcpu.9 @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd March 26, 2017 +.Dd July 5, 2018 .Dt DPCPU 9 .Os .Sh NAME @@ -35,6 +35,7 @@ .In sys/pcpu.h .Ss Per-CPU Variable Definition and Declaration .Fn DPCPU_DEFINE "type" "name" +.Fn DPCPU_DEFINE_STATIC "type" "name" .Fn DPCPU_DECLARE "type" "name" .Ss Current CPU Accessor Functions .Fn DPCPU_PTR "name" @@ -66,11 +67,12 @@ per-CPU instance to be initialized with the value: DPCPU_DEFINE(int, foo_int) = 1; .Ed .Pp -Syntactically, the definition may be treated as a variable. -For example, a dynamic per-CPU variable may be declared as -.Dv static : +Values that can be defined as +.Dv static +must use +.Fn DPCPU_DEFINE_STATIC : .Bd -literal -offset 1234 -static DPCPU_DEFINE(int, foo_int); +DPCPU_DEFINE_STATIC(int, foo_int); .Ed .Pp .Fn DPCPU_DECLARE @@ -111,8 +113,8 @@ Alternatively, it may be desirable to cache the CPU ID at the start of a sequence of accesses, using suitable synchronization to make non-atomic sequences safe in the presence of migration. .Bd -literal -offset 1234 -static DPCPU_DEFINE(int, foo_int); -static DPCPU_DEFINE(struct mutex, foo_lock); +DPCPU_DEFINE_STATIC(int, foo_int); +DPCPU_DEFINE_STATIC(struct mutex, foo_lock); void foo_int_increment(void) diff --git a/sys/compat/linuxkpi/common/src/linux_idr.c b/sys/compat/linuxkpi/common/src/linux_idr.c index 513be67ac2b..773ebd3b841 100644 --- a/sys/compat/linuxkpi/common/src/linux_idr.c +++ b/sys/compat/linuxkpi/common/src/linux_idr.c @@ -55,7 +55,7 @@ struct linux_idr_cache { unsigned count; }; -static DPCPU_DEFINE(struct linux_idr_cache, linux_idr_cache); +DPCPU_DEFINE_STATIC(struct linux_idr_cache, linux_idr_cache); /* * IDR Implementation. diff --git a/sys/compat/linuxkpi/common/src/linux_rcu.c b/sys/compat/linuxkpi/common/src/linux_rcu.c index 32283ee1849..b89dc2b63c0 100644 --- a/sys/compat/linuxkpi/common/src/linux_rcu.c +++ b/sys/compat/linuxkpi/common/src/linux_rcu.c @@ -92,7 +92,7 @@ CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); static ck_epoch_t linux_epoch; static struct linux_epoch_head linux_epoch_head; -static DPCPU_DEFINE(struct linux_epoch_record, linux_epoch_record); +DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record); static void linux_rcu_cleaner_func(void *, int); diff --git a/sys/compat/linuxkpi/common/src/linux_tasklet.c b/sys/compat/linuxkpi/common/src/linux_tasklet.c index 549af864747..049d9caac69 100644 --- a/sys/compat/linuxkpi/common/src/linux_tasklet.c +++ b/sys/compat/linuxkpi/common/src/linux_tasklet.c @@ -61,7 +61,7 @@ struct tasklet_worker { #define TASKLET_WORKER_LOCK(tw) mtx_lock(&(tw)->mtx) #define TASKLET_WORKER_UNLOCK(tw) mtx_unlock(&(tw)->mtx) -static DPCPU_DEFINE(struct tasklet_worker, tasklet_worker); +DPCPU_DEFINE_STATIC(struct tasklet_worker, tasklet_worker); static void tasklet_handler(void *arg) diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 6b3a19e0a62..f3c68c408bb 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -382,7 +382,7 @@ int profprocs; volatile int ticks; int psratio; -static DPCPU_DEFINE(int, pcputicks); /* Per-CPU version of ticks. */ +DPCPU_DEFINE_STATIC(int, pcputicks); /* Per-CPU version of ticks. */ #ifdef DEVICE_POLLING static int devpoll_run = 0; #endif diff --git a/sys/kern/kern_clocksource.c b/sys/kern/kern_clocksource.c index 9676923104e..ed7b67f62cf 100644 --- a/sys/kern/kern_clocksource.c +++ b/sys/kern/kern_clocksource.c @@ -126,7 +126,7 @@ struct pcpu_state { int idle; /* This CPU is in idle mode. */ }; -static DPCPU_DEFINE(struct pcpu_state, timerstate); +DPCPU_DEFINE_STATIC(struct pcpu_state, timerstate); DPCPU_DEFINE(sbintime_t, hardclocktime); /* diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 4b99c90a2e8..e6b3782d93d 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -1331,7 +1331,7 @@ struct exec_args_kva { SLIST_ENTRY(exec_args_kva) next; }; -static DPCPU_DEFINE(struct exec_args_kva *, exec_args_kva); +DPCPU_DEFINE_STATIC(struct exec_args_kva *, exec_args_kva); static SLIST_HEAD(, exec_args_kva) exec_args_kva_freelist; static struct mtx exec_args_kva_mtx; diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c index c4e94762fe3..e3eb37bce32 100644 --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -2001,8 +2001,8 @@ SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); static int cpu_tick_variable; static uint64_t cpu_tick_frequency; -static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base); -static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last); +DPCPU_DEFINE_STATIC(uint64_t, tc_cpu_ticks_base); +DPCPU_DEFINE_STATIC(unsigned, tc_cpu_ticks_last); static uint64_t tc_cpu_ticks(void) diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 8dc4443259b..a8535610793 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -176,7 +176,7 @@ struct pcpuidlestat { u_int idlecalls; u_int oldidlecalls; }; -static DPCPU_DEFINE(struct pcpuidlestat, idlestat); +DPCPU_DEFINE_STATIC(struct pcpuidlestat, idlestat); static void setup_runqs(void) diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index a12d2d5d5f9..ef5bef38586 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -283,7 +283,7 @@ static int trysteal_limit = 2; static struct tdq tdq_cpu[MAXCPU]; static struct tdq *balance_tdq; static int balance_ticks; -static DPCPU_DEFINE(uint32_t, randomval); +DPCPU_DEFINE_STATIC(uint32_t, randomval); #define TDQ_SELF() (&tdq_cpu[PCPU_GET(cpuid)]) #define TDQ_CPU(x) (&tdq_cpu[(x)]) diff --git a/sys/kern/subr_pcpu.c b/sys/kern/subr_pcpu.c index 069b5ec02fb..86f140a0f07 100644 --- a/sys/kern/subr_pcpu.c +++ b/sys/kern/subr_pcpu.c @@ -72,7 +72,7 @@ struct dpcpu_free { TAILQ_ENTRY(dpcpu_free) df_link; }; -static DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]); +DPCPU_DEFINE_STATIC(char, modspace[DPCPU_MODMIN]); static TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); static struct sx dpcpu_lock; uintptr_t dpcpu_off[MAXCPU]; diff --git a/sys/mips/mips/tick.c b/sys/mips/mips/tick.c index 74296a0bd6d..c2417edd0ad 100644 --- a/sys/mips/mips/tick.c +++ b/sys/mips/mips/tick.c @@ -61,13 +61,13 @@ uint64_t counter_freq; struct timecounter *platform_timecounter; -static DPCPU_DEFINE(uint32_t, cycles_per_tick); +DPCPU_DEFINE_STATIC(uint32_t, cycles_per_tick); static uint32_t cycles_per_usec; -static DPCPU_DEFINE(volatile uint32_t, counter_upper); -static DPCPU_DEFINE(volatile uint32_t, counter_lower_last); -static DPCPU_DEFINE(uint32_t, compare_ticks); -static DPCPU_DEFINE(uint32_t, lost_ticks); +DPCPU_DEFINE_STATIC(volatile uint32_t, counter_upper); +DPCPU_DEFINE_STATIC(volatile uint32_t, counter_lower_last); +DPCPU_DEFINE_STATIC(uint32_t, compare_ticks); +DPCPU_DEFINE_STATIC(uint32_t, lost_ticks); struct clock_softc { int intr_rid; diff --git a/sys/mips/nlm/tick.c b/sys/mips/nlm/tick.c index 66d192445fb..fafd397e8bf 100644 --- a/sys/mips/nlm/tick.c +++ b/sys/mips/nlm/tick.c @@ -62,13 +62,13 @@ uint64_t counter_freq; struct timecounter *platform_timecounter; -static DPCPU_DEFINE(uint32_t, cycles_per_tick); +DPCPU_DEFINE_STATIC(uint32_t, cycles_per_tick); static uint32_t cycles_per_usec; -static DPCPU_DEFINE(volatile uint32_t, counter_upper); -static DPCPU_DEFINE(volatile uint32_t, counter_lower_last); -static DPCPU_DEFINE(uint32_t, compare_ticks); -static DPCPU_DEFINE(uint32_t, lost_ticks); +DPCPU_DEFINE_STATIC(volatile uint32_t, counter_upper); +DPCPU_DEFINE_STATIC(volatile uint32_t, counter_lower_last); +DPCPU_DEFINE_STATIC(uint32_t, compare_ticks); +DPCPU_DEFINE_STATIC(uint32_t, lost_ticks); struct clock_softc { int intr_rid; diff --git a/sys/netinet/siftr.c b/sys/netinet/siftr.c index 0c44e774f27..83af89570d5 100644 --- a/sys/netinet/siftr.c +++ b/sys/netinet/siftr.c @@ -268,7 +268,7 @@ struct siftr_stats uint32_t nskip_out_dejavu; }; -static DPCPU_DEFINE(struct siftr_stats, ss); +DPCPU_DEFINE_STATIC(struct siftr_stats, ss); static volatile unsigned int siftr_exit_pkt_manager_thread = 0; static unsigned int siftr_enabled = 0; diff --git a/sys/netpfil/ipfw/ip_fw_dynamic.c b/sys/netpfil/ipfw/ip_fw_dynamic.c index d16f16d932b..6f877edf794 100644 --- a/sys/netpfil/ipfw/ip_fw_dynamic.c +++ b/sys/netpfil/ipfw/ip_fw_dynamic.c @@ -219,7 +219,7 @@ static VNET_DEFINE(struct dyn_ipv6_slist, dyn_expired_ipv6); * and must not be reclaimed by expiration callout. */ static void **dyn_hp_cache; -static DPCPU_DEFINE(void *, dyn_hp); +DPCPU_DEFINE_STATIC(void *, dyn_hp); #define DYNSTATE_GET(cpu) ck_pr_load_ptr(DPCPU_ID_PTR((cpu), dyn_hp)) #define DYNSTATE_PROTECT(v) ck_pr_store_ptr(DPCPU_PTR(dyn_hp), (v)) #define DYNSTATE_RELEASE() DYNSTATE_PROTECT(NULL) diff --git a/sys/powerpc/powerpc/clock.c b/sys/powerpc/powerpc/clock.c index 1d0583d5b68..890ce6616f5 100644 --- a/sys/powerpc/powerpc/clock.c +++ b/sys/powerpc/powerpc/clock.c @@ -95,7 +95,7 @@ struct decr_state { int mode; /* 0 - off, 1 - periodic, 2 - one-shot. */ int32_t div; /* Periodic divisor. */ }; -static DPCPU_DEFINE(struct decr_state, decr_state); +DPCPU_DEFINE_STATIC(struct decr_state, decr_state); static struct eventtimer decr_et; static struct timecounter decr_tc = { diff --git a/sys/sys/pcpu.h b/sys/sys/pcpu.h index a116b62d253..127c10d6e11 100644 --- a/sys/sys/pcpu.h +++ b/sys/sys/pcpu.h @@ -81,7 +81,11 @@ extern uintptr_t dpcpu_off[]; */ #define DPCPU_NAME(n) pcpu_entry_##n #define DPCPU_DECLARE(t, n) extern t DPCPU_NAME(n) -#define DPCPU_DEFINE(t, n) t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used +/* struct _hack is to stop this from being used with the static keyword. */ +#define DPCPU_DEFINE(t, n) \ + struct _hack; t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used +#define DPCPU_DEFINE_STATIC(t, n) \ + static t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used /* * Accessors with a given base. diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 2137d54652b..e7b6891d47f 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -134,7 +134,7 @@ extern int vmem_startup_count(void); struct vm_domain vm_dom[MAXMEMDOM]; -static DPCPU_DEFINE(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]); +DPCPU_DEFINE_STATIC(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]); struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT]; diff --git a/sys/x86/xen/xen_intr.c b/sys/x86/xen/xen_intr.c index cbc90a99d30..f823b9303f6 100644 --- a/sys/x86/xen/xen_intr.c +++ b/sys/x86/xen/xen_intr.c @@ -103,7 +103,7 @@ struct xen_intr_pcpu_data { * Start the scan at port 0 by initializing the last scanned * location as the highest numbered event channel port. */ -static DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = { +DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = { .last_processed_l1i = LONG_BIT - 1, .last_processed_l2i = LONG_BIT - 1 };