implement fetch-and-add array queue data structure

this is a lockless queue based on hazard pointers.

config.h.in

@@ -366,6 +366,9 @@
 /* define if struct stat has st_mtim.tv_nsec field */
 #undef HAVE_STAT_NSEC
 
+/* Define to 1 if you have the <stdalign.h> header file. */
+#undef HAVE_STDALIGN_H
+
 /* Define to 1 if you have the <stdatomic.h> header file. */
 #undef HAVE_STDATOMIC_H
 

configure

@@ -848,6 +848,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -1018,6 +1019,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1270,6 +1272,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1407,7 +1418,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1560,6 +1571,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -4000,7 +4012,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4046,7 +4058,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4070,7 +4082,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4115,7 +4127,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4139,7 +4151,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -19672,6 +19684,19 @@ done
 
 LIBS="$LIBS $ISC_ATOMIC_LIBS"
 
+for ac_header in stdalign.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "stdalign.h" "ac_cv_header_stdalign_h" "$ac_includes_default"
+if test "x$ac_cv_header_stdalign_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_STDALIGN_H 1
+_ACEOF
+
+fi
+
+done
+
+
 for ac_header in uchar.h
 do :
   ac_fn_c_check_header_mongrel "$LINENO" "uchar.h" "ac_cv_header_uchar_h" "$ac_includes_default"

configure.ac

@@ -1796,6 +1796,8 @@ AC_CHECK_HEADERS(
   ])
 LIBS="$LIBS $ISC_ATOMIC_LIBS"
 
+AC_CHECK_HEADERS([stdalign.h])
+
 AC_CHECK_HEADERS([uchar.h])
 
 #

lib/isc/Makefile.in

@@ -55,9 +55,9 @@ OBJS =		pk11.@O@ pk11_result.@O@ \
 		lex.@O@ lfsr.@O@ lib.@O@ log.@O@ \
 		md.@O@ mem.@O@ mutexblock.@O@ \
 		netaddr.@O@ netscope.@O@ nonce.@O@ openssl_shim.@O@ pool.@O@ \
-		parseint.@O@ portset.@O@ quota.@O@ radix.@O@ random.@O@ \
-		ratelimiter.@O@ region.@O@ regex.@O@ result.@O@ \
-		rwlock.@O@ \
+		parseint.@O@ portset.@O@ queue.@O@ quota.@O@ \
+		radix.@O@ random.@O@ ratelimiter.@O@ \
+		region.@O@ regex.@O@ result.@O@ rwlock.@O@ \
 		serial.@O@ siphash.@O@ sockaddr.@O@ stats.@O@ \
 		string.@O@ symtab.@O@ task.@O@ taskpool.@O@ \
 		tm.@O@ timer.@O@ version.@O@ \
@@ -74,7 +74,7 @@ SRCS =		pk11.c pk11_result.c \
 		lex.c lfsr.c lib.c log.c \
 		md.c mem.c mutexblock.c \
 		netaddr.c netscope.c nonce.c openssl_shim.c pool.c \
-		parseint.c portset.c quota.c radix.c random.c \
+		parseint.c portset.c queue.c quota.c radix.c random.c \
 		ratelimiter.c region.c regex.c result.c rwlock.c \
 		serial.c siphash.c sockaddr.c stats.c string.c \
 		symtab.c task.c taskpool.c timer.c \

lib/isc/include/isc/queue.h

@@ -9,7 +9,46 @@
  * information regarding copyright ownership.
  */
 
-#ifndef ISC_QUEUE_H
-#define ISC_QUEUE_H 1
+#pragma once
+#include <isc/mem.h>
 
-#endif /* ISC_QUEUE_H */
+typedef struct isc_queue isc_queue_t;
+
+isc_queue_t *
+isc_queue_new(isc_mem_t *mctx, int max_threads);
+/*%<
+ * Create a new fetch-and-add array queue.
+ *
+ * 'max_threads' is currently unused. In the future it can be used
+ * to pass a maximum threads parameter when creating hazard pointers,
+ * but currently `isc_hp_t` uses a hard-coded value.
+ */
+
+void
+isc_queue_enqueue(isc_queue_t *queue, uintptr_t item);
+/*%<
+ * Enqueue an object pointer 'item' at the tail of the queue.
+ *
+ * Requires:
+ * \li	'item' is not null.
+ */
+
+uintptr_t
+isc_queue_dequeue(isc_queue_t *queue);
+/*%<
+ * Remove an object pointer from the head of the queue and return the
+ * pointer. If the queue is empty, return `nulluintptr` (the uintptr_t
+ * representation of NULL).
+ *
+ * Requires:
+ * \li	'queue' is not null.
+ */
+
+void
+isc_queue_destroy(isc_queue_t *queue);
+/*%<
+ * Destroy a queue.
+ *
+ * Requires:
+ * \li	'queue' is not null.
+ */

lib/isc/queue.c

@@ -0,0 +1,219 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <inttypes.h>
+
+#include <isc/align.h>
+#include <isc/atomic.h>
+#include <isc/queue.h>
+#include <isc/string.h>
+#include <isc/mem.h>
+#include <isc/hp.h>
+
+#define BUFFER_SIZE 1024
+
+#define MAX_THREADS 128
+
+static uintptr_t nulluintptr = (uintptr_t)NULL;
+
+typedef struct node {
+	atomic_uint_fast32_t	deqidx;
+	atomic_uintptr_t	items[BUFFER_SIZE];
+	atomic_uint_fast32_t	enqidx;
+	atomic_uintptr_t	next;
+	isc_mem_t		*mctx;
+} node_t;
+
+/* we just need one Hazard Pointer */
+#define HP_TAIL 0
+#define HP_HEAD 0
+
+struct isc_queue {
+	alignas(128) atomic_uintptr_t	head;
+	alignas(128) atomic_uintptr_t	tail;
+	isc_mem_t			*mctx;
+	int				max_threads;
+	int				taken;
+	isc_hp_t			*hp;
+};
+
+static node_t *
+node_new(isc_mem_t *mctx, uintptr_t item) {
+	node_t *node = isc_mem_get(mctx, sizeof(*node));
+	*node = (node_t){
+		.mctx = NULL
+	};
+
+	atomic_init(&node->deqidx, 0);
+	atomic_init(&node->enqidx, 1);
+	atomic_init(&node->next, 0);
+	atomic_init(&node->items[0], item);
+
+	for (int i = 1; i < BUFFER_SIZE; i++) {
+		atomic_init(&node->items[i], 0);
+	}
+
+	isc_mem_attach(mctx, &node->mctx);
+
+	return (node);
+}
+
+static void
+node_destroy(void *node0) {
+	node_t *node = (node_t *)node0;
+
+	isc_mem_putanddetach(&node->mctx, node, sizeof(*node));
+}
+
+static bool
+node_cas_next(node_t *node, node_t *cmp, const node_t *val) {
+	return (atomic_compare_exchange_strong(&node->next,
+					       (uintptr_t *)&cmp,
+					       (uintptr_t)val));
+}
+
+static bool
+queue_cas_tail(isc_queue_t *queue, node_t *cmp, const node_t *val) {
+	return (atomic_compare_exchange_strong(&queue->tail,
+					       (uintptr_t *)&cmp,
+					       (uintptr_t)val));
+}
+
+static bool
+queue_cas_head(isc_queue_t *queue, node_t *cmp, const node_t *val) {
+	return (atomic_compare_exchange_strong(&queue->head,
+					       (uintptr_t *)&cmp,
+					       (uintptr_t)val));
+}
+
+isc_queue_t *
+isc_queue_new(isc_mem_t *mctx, int max_threads) {
+	isc_queue_t *queue = isc_mem_get(mctx, sizeof(*queue));
+	node_t *sentinel = node_new(mctx, nulluintptr);
+
+	if (max_threads == 0) {
+		max_threads = MAX_THREADS;
+	}
+
+	*queue = (isc_queue_t){
+		.max_threads = max_threads,
+	};
+
+	isc_mem_attach(mctx, &queue->mctx);
+
+	queue->hp = isc_hp_new(mctx, 1, node_destroy);
+
+	atomic_init(&sentinel->enqidx, 0);
+	atomic_init(&queue->head, (uintptr_t)sentinel);
+	atomic_init(&queue->tail, (uintptr_t)sentinel);
+
+	return (queue);
+}
+
+void
+isc_queue_enqueue(isc_queue_t *queue, uintptr_t item) {
+	REQUIRE(item != nulluintptr);
+
+	while (true) {
+		node_t *lt = NULL;
+		uint_fast32_t idx;
+		uintptr_t n = nulluintptr;
+
+		lt = (node_t *)isc_hp_protect(queue->hp, 0, &queue->tail);
+		idx = atomic_fetch_add(&lt->enqidx, 1);
+		if (idx > BUFFER_SIZE-1) {
+			node_t *lnext = NULL;
+
+			if (lt != (node_t *)atomic_load(&queue->tail)) {
+				continue;
+			}
+
+			lnext = (node_t *)atomic_load(&lt->next);
+			if (lnext == NULL) {
+				node_t *newnode = node_new(queue->mctx, item);
+				if (node_cas_next(lt, NULL, newnode)) {
+					queue_cas_tail(queue, lt, newnode);
+					isc_hp_clear(queue->hp);
+					return;
+				}
+				node_destroy(newnode);
+			} else {
+				queue_cas_tail(queue, lt, lnext);
+			}
+
+			continue;
+		}
+
+		if (atomic_compare_exchange_strong(&lt->items[idx], &n, item)) {
+			isc_hp_clear(queue->hp);
+			return;
+		}
+	}
+}
+
+uintptr_t
+isc_queue_dequeue(isc_queue_t *queue) {
+	REQUIRE(queue != NULL);
+
+	while (true) {
+		node_t *lh = NULL;
+		uint_fast32_t idx;
+		uintptr_t item;
+
+		lh = (node_t *)isc_hp_protect(queue->hp, 0, &queue->head);
+		if (atomic_load(&lh->deqidx) >= atomic_load(&lh->enqidx) &&
+		    atomic_load(&lh->next) == nulluintptr)
+		{
+			break;
+		}
+
+		idx = atomic_fetch_add(&lh->deqidx, 1);
+		if (idx > BUFFER_SIZE-1) {
+			node_t *lnext = (node_t *)atomic_load(&lh->next);
+			if (lnext == NULL) {
+				break;
+			}
+			if (queue_cas_head(queue, lh, lnext)) {
+				isc_hp_retire(queue->hp, (uintptr_t)lh);
+			}
+
+			continue;
+		}
+
+		item = atomic_exchange(&(lh->items[idx]),
+				       (uintptr_t)&queue->taken);
+		if (item == nulluintptr) {
+			continue;
+		}
+
+		isc_hp_clear(queue->hp);
+		return (item);
+	}
+
+	isc_hp_clear(queue->hp);
+	return (nulluintptr);
+}
+
+void
+isc_queue_destroy(isc_queue_t *queue) {
+	node_t *last = NULL;
+
+	REQUIRE(queue != NULL);
+
+	while (isc_queue_dequeue(queue) != nulluintptr) {
+		/* do nothing */
+	}
+
+	last = (node_t *)atomic_load_relaxed(&queue->head);
+	node_destroy(last);
+	isc_hp_destroy(queue->hp);
+	isc_mem_putanddetach(&queue->mctx, queue, sizeof(*queue));
+}

lib/isc/unix/include/isc/align.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#ifdef HAVE_STDALIGN_H
+#include <stdalign.h>
+#else
+#define alignas(x) __attribute__ ((__aligned__ (x)))
+#endif

lib/isc/win32/include/isc/align.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+#define alignas(x) __declspec(align(x))

lib/isc/win32/libisc.def.in

@@ -452,6 +452,10 @@ isc_portset_isset
 isc_portset_nports
 isc_portset_remove
 isc_portset_removerange
+isc_queue_enqueue
+isc_queue_dequeue
+isc_queue_destroy
+isc_queue_new
 isc_quota_attach
 isc_quota_destroy
 isc_quota_detach

lib/isc/win32/libisc.vcxproj.filters.in

@@ -554,6 +554,9 @@
     <ClCompile Include="..\portset.c">
       <Filter>Library Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\queue.c">
+      <Filter>Library Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\quota.c">
       <Filter>Library Source Files</Filter>
     </ClCompile>

lib/isc/win32/libisc.vcxproj.in

@@ -440,6 +440,7 @@ copy InstallFiles ..\Build\Release\
     <ClCompile Include="..\parseint.c" />
     <ClCompile Include="..\pool.c" />
     <ClCompile Include="..\portset.c" />
+    <ClCompile Include="..\queue.c" />
     <ClCompile Include="..\quota.c" />
     <ClCompile Include="..\radix.c" />
     <ClCompile Include="..\random.c" />

util/copyrights

@@ -2270,6 +2270,7 @@
 ./lib/isc/pthreads/include/isc/thread.h		C	1998,1999,2000,2001,2004,2005,2007,2013,2016,2017,2018,2019
 ./lib/isc/pthreads/mutex.c			C	2000,2001,2002,2004,2005,2007,2008,2011,2012,2014,2015,2016,2018,2019
 ./lib/isc/pthreads/thread.c			C	2000,2001,2003,2004,2005,2007,2013,2016,2017,2018,2019
+./lib/isc/queue.c				C	2019
 ./lib/isc/quota.c				C	2000,2001,2004,2005,2007,2016,2018,2019
 ./lib/isc/radix.c				C	2007,2008,2009,2011,2012,2013,2014,2015,2016,2018,2019
 ./lib/isc/random.c				C	1999,2000,2001,2002,2003,2004,2005,2007,2009,2013,2014,2016,2017,2018,2019
@@ -2331,6 +2332,7 @@
 ./lib/isc/unix/file.c				C	2000,2001,2002,2004,2005,2007,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019
 ./lib/isc/unix/fsaccess.c			C	2000,2001,2004,2005,2006,2007,2016,2018,2019
 ./lib/isc/unix/ifiter_getifaddrs.c		C	2003,2004,2005,2007,2008,2009,2014,2016,2018,2019
+./lib/isc/unix/include/isc/align.h		C	2019
 ./lib/isc/unix/include/isc/dir.h		C	1999,2000,2001,2004,2005,2007,2016,2018,2019
 ./lib/isc/unix/include/isc/net.h		C	1999,2000,2001,2002,2003,2004,2005,2007,2008,2012,2013,2014,2016,2017,2018,2019
 ./lib/isc/unix/include/isc/netdb.h		C	1999,2000,2001,2004,2005,2007,2016,2018,2019
@@ -2363,6 +2365,7 @@
 ./lib/isc/win32/fsaccess.c			C	2000,2001,2002,2004,2007,2013,2016,2017,2018,2019
 ./lib/isc/win32/include/Makefile.in		MAKE	1999,2000,2001,2004,2007,2012,2014,2016,2018,2019
 ./lib/isc/win32/include/isc/Makefile.in		MAKE	1999,2000,2001,2004,2007,2012,2013,2014,2015,2016,2018,2019
+./lib/isc/win32/include/isc/align.h		C	2019
 ./lib/isc/win32/include/isc/bind_registry.h	C	2001,2004,2007,2016,2018,2019
 ./lib/isc/win32/include/isc/bindevt.h		C	2001,2004,2007,2016,2018,2019
 ./lib/isc/win32/include/isc/condition.h		C	1998,1999,2000,2001,2004,2007,2016,2018,2019