Add the atomic_thread_fence() family of functions with intent to

provide a semantic defined by the C11 fences with corresponding
memory_order.

atomic_thread_fence_acq() gives r | r, w, where r and w are read and
write accesses, and | denotes the fence itself.

atomic_thread_fence_rel() is r, w | w.

atomic_thread_fence_acq_rel() is the combination of the acquire and
release in single operation.  Note that reads after the acq+rel fence
could be made visible before writes preceeding the fence.

atomic_thread_fence_seq_cst() orders all accesses before/after the
fence, and the fence itself is globally ordered against other
sequentially consistent atomic operations.

Reviewed by:	alc
Discussed with:	bde
Sponsored by:	The FreeBSD Foundation
MFC after:	3 weeks

sys/amd64/include/atomic.h

@@ -84,6 +84,10 @@ u_int	atomic_fetchadd_int(volatile u_int *p, u_int v);
 u_long	atomic_fetchadd_long(volatile u_long *p, u_long v);
 int	atomic_testandset_int(volatile u_int *p, u_int v);
 int	atomic_testandset_long(volatile u_long *p, u_int v);
+void	atomic_thread_fence_acq(void);
+void	atomic_thread_fence_acq_rel(void);
+void	atomic_thread_fence_rel(void);
+void	atomic_thread_fence_seq_cst(void);
 
 #define	ATOMIC_LOAD(TYPE)					\
 u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p)
@@ -328,6 +332,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)	\
 }								\
 struct __hack
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	__storeload_barrier();
+}
+
 #endif /* KLD_MODULE || !__GNUCLIKE_ASM */
 
 ATOMIC_ASM(set,	     char,  "orb %b1,%0",  "iq",  v);

sys/arm/include/atomic.h

@@ -1103,6 +1103,34 @@ atomic_store_long(volatile u_long *dst, u_long src)
 	*dst = src;
 }
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	dmb();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+	dmb();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+	dmb();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	dmb();
+}
+
 #define atomic_clear_ptr		atomic_clear_32
 #define atomic_set_ptr			atomic_set_32
 #define atomic_cmpset_ptr		atomic_cmpset_32

sys/arm64/include/atomic.h

@@ -728,6 +728,34 @@ atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val)
 	);
 }
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	dmb(ld);
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+	dmb(sy);
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+	dmb(sy);
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	dmb(sy);
+}
+
 #define	atomic_add_rel_long		atomic_add_rel_64
 #define	atomic_clear_rel_long		atomic_clear_rel_64
 #define	atomic_cmpset_rel_long		atomic_cmpset_rel_64

sys/i386/include/atomic.h

@@ -86,6 +86,10 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 int	atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
 u_int	atomic_fetchadd_int(volatile u_int *p, u_int v);
 int	atomic_testandset_int(volatile u_int *p, u_int v);
+void	atomic_thread_fence_acq(void);
+void	atomic_thread_fence_acq_rel(void);
+void	atomic_thread_fence_rel(void);
+void	atomic_thread_fence_seq_cst(void);
 
 #define	ATOMIC_LOAD(TYPE)					\
 u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p)
@@ -310,6 +314,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)	\
 }								\
 struct __hack
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	__storeload_barrier();
+}
+
 #ifdef _KERNEL
 
 #ifdef WANT_FUNCTIONS

sys/mips/include/atomic.h

@@ -496,6 +496,34 @@ atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v)
 }
 #endif
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	mips_sync();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+	mips_sync();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+	mips_sync();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	mips_sync();
+}
+
 /* Operations on chars. */
 #define	atomic_set_char		atomic_set_8
 #define	atomic_set_acq_char	atomic_set_acq_8

sys/powerpc/include/atomic.h

@@ -730,4 +730,45 @@ atomic_swap_64(volatile u_long *p, u_long v)
 #undef __ATOMIC_REL
 #undef __ATOMIC_ACQ
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	/* See above comment about lwsync being broken on Book-E. */
+#ifdef __powerpc64__
+	__asm __volatile("lwsync" : : : "memory");
+#else
+	__asm __volatile("sync" : : : "memory");
+#endif
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+#ifdef __powerpc64__
+	__asm __volatile("lwsync" : : : "memory");
+#else
+	__asm __volatile("sync" : : : "memory");
+#endif
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+#ifdef __powerpc64__
+	__asm __volatile("lwsync" : : : "memory");
+#else
+	__asm __volatile("sync" : : : "memory");
+#endif
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	__asm __volatile("sync" : : : "memory");
+}
+
 #endif /* ! _MACHINE_ATOMIC_H_ */

sys/sparc64/include/atomic.h

@@ -279,6 +279,35 @@ atomic_store_rel_ ## name(volatile ptype p, vtype v)			\
 	atomic_st_rel((p), (v), sz);					\
 }
 
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+	__compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+	membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
+}
+
+
 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);