From 2c6946dca29a0481f1af751eda1ed9d97cdb92d5 Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Tue, 9 Jun 2015 11:49:56 +0000 Subject: [PATCH] When updating/accessing the timehands, barriers are needed to ensure that: - th_generation update is visible after the parameters update is visible; - the read of parameters is not reordered before initial read of th_generation. On UP kernels, compiler barriers are enough. For SMP machines, CPU barriers must be used too, as was confirmed by submitter by testing on the Freescale T4240 platform with 24 PowerPC processors. Submitted by: Sebastian Huber MFC after: 1 week --- sys/kern/kern_tc.c | 110 ++++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c index 9dca0e80e5e..01c61bd0944 100644 --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include /* * A large step happens on boot. This constant detects such steps. @@ -71,7 +72,7 @@ struct timehands { struct timeval th_microtime; struct timespec th_nanotime; /* Fields not to be copied in tc_windup start with th_generation. */ - volatile u_int th_generation; + u_int th_generation; struct timehands *th_next; }; @@ -189,6 +190,33 @@ tc_delta(struct timehands *th) tc->tc_counter_mask); } +static u_int +tc_getgen(struct timehands *th) +{ + +#ifdef SMP + return (atomic_load_acq_int(&th->th_generation)); +#else + u_int gen; + + gen = th->th_generation; + __compiler_membar(); + return (gen); +#endif +} + +static void +tc_setgen(struct timehands *th, u_int newgen) +{ + +#ifdef SMP + atomic_store_rel_int(&th->th_generation, newgen); +#else + __compiler_membar(); + th->th_generation = newgen; +#endif +} + /* * Functions for reading the time. We have to loop until we are sure that * the timehands that we operated on was not updated under our feet. See @@ -204,10 +232,10 @@ fbclock_binuptime(struct bintime *bt) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *bt = th->th_offset; bintime_addx(bt, th->th_scale * tc_delta(th)); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -262,9 +290,9 @@ fbclock_getbinuptime(struct bintime *bt) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *bt = th->th_offset; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -275,9 +303,9 @@ fbclock_getnanouptime(struct timespec *tsp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); bintime2timespec(&th->th_offset, tsp); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -288,9 +316,9 @@ fbclock_getmicrouptime(struct timeval *tvp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); bintime2timeval(&th->th_offset, tvp); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -301,9 +329,9 @@ fbclock_getbintime(struct bintime *bt) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *bt = th->th_offset; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); bintime_add(bt, &boottimebin); } @@ -315,9 +343,9 @@ fbclock_getnanotime(struct timespec *tsp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *tsp = th->th_nanotime; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -328,9 +356,9 @@ fbclock_getmicrotime(struct timeval *tvp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *tvp = th->th_microtime; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } #else /* !FFCLOCK */ void @@ -341,10 +369,10 @@ binuptime(struct bintime *bt) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *bt = th->th_offset; bintime_addx(bt, th->th_scale * tc_delta(th)); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -399,9 +427,9 @@ getbinuptime(struct bintime *bt) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *bt = th->th_offset; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -412,9 +440,9 @@ getnanouptime(struct timespec *tsp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); bintime2timespec(&th->th_offset, tsp); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -425,9 +453,9 @@ getmicrouptime(struct timeval *tvp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); bintime2timeval(&th->th_offset, tvp); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -438,9 +466,9 @@ getbintime(struct bintime *bt) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *bt = th->th_offset; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); bintime_add(bt, &boottimebin); } @@ -452,9 +480,9 @@ getnanotime(struct timespec *tsp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *tsp = th->th_nanotime; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } void @@ -465,9 +493,9 @@ getmicrotime(struct timeval *tvp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *tvp = th->th_microtime; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } #endif /* FFCLOCK */ @@ -880,11 +908,11 @@ ffclock_read_counter(ffcounter *ffcount) */ do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); ffth = fftimehands; delta = tc_delta(th); *ffcount = ffth->tick_ffcount; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); *ffcount += delta; } @@ -988,9 +1016,9 @@ dtrace_getnanotime(struct timespec *tsp) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); *tsp = th->th_nanotime; - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); } /* @@ -1028,7 +1056,7 @@ sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) do { th = timehands; - gen = th->th_generation; + gen = tc_getgen(th); fbi->th_scale = th->th_scale; fbi->tick_time = th->th_offset; #ifdef FFCLOCK @@ -1042,7 +1070,7 @@ sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) #endif if (!fast) delta = tc_delta(th); - } while (gen == 0 || gen != th->th_generation); + } while (gen == 0 || gen != tc_getgen(th)); clock_snap->delta = delta; clock_snap->sysclock_active = sysclock_active; @@ -1260,7 +1288,7 @@ tc_windup(void) tho = timehands; th = tho->th_next; ogen = th->th_generation; - th->th_generation = 0; + tc_setgen(th, 0); bcopy(tho, th, offsetof(struct timehands, th_generation)); /* @@ -1377,7 +1405,7 @@ tc_windup(void) */ if (++ogen == 0) ogen = 1; - th->th_generation = ogen; + tc_setgen(th, ogen); /* Go live with the new struct timehands. */ #ifdef FFCLOCK @@ -1651,13 +1679,13 @@ pps_capture(struct pps_state *pps) KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); th = timehands; - pps->capgen = th->th_generation; + pps->capgen = tc_getgen(th); pps->capth = th; #ifdef FFCLOCK pps->capffth = fftimehands; #endif pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); - if (pps->capgen != th->th_generation) + if (pps->capgen != tc_getgen(th)) pps->capgen = 0; } @@ -1677,7 +1705,7 @@ pps_event(struct pps_state *pps, int event) KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); /* If the timecounter was wound up underneath us, bail out. */ - if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation) + if (pps->capgen == 0 || pps->capgen != tc_getgen(pps->capth)) return; /* Things would be easier with arrays. */ @@ -1727,7 +1755,7 @@ pps_event(struct pps_state *pps, int event) bintime2timespec(&bt, &ts); /* If the timecounter was wound up underneath us, bail out. */ - if (pps->capgen != pps->capth->th_generation) + if (pps->capgen != tc_getgen(pps->capth)) return; *pcount = pps->capcount;