libc: allow __cxa_atexit handlers to be added during __cxa_finalize

science/dlib-cpp reveals an interesting scenario that works fine on
other platforms but not on FreeBSD; notably, it ends up creating a new
global object from some destructor which is called during
__cxa_finalize.  This breaks when libdlib is dlopen()ed and then
subsequently dlclose()ed, as we never end up invoking the created
object's dtor until program exit when the shlib is already unmapped.

Fix it by noting when we're in the middle of __cxa_finalize for a dso,
and then restarting the search if __cxa_atexit() was called in the
middle somewhere.

We wait until we've processed the initial set before starting over and
processing the newly added handlers as if it were a complete set of
handlers added during runtime.  The alternative is calling them as
they're added to maintain a LIFO in terms of total ordering, but in
theory a constructor could add another global object that also needs to
be destroyed, and that object needs to be destroyed after the one that
constructed it to avoid creating unexpected lifetime issues.

This manifests in the pdlib PHP extension for dlib crashing, see [0].

[0] https://github.com/goodspb/pdlib/issues/39

PR:		285870
Reviewed by:	kevans (also supplied commit message)
MFC after:	1 week

lib/libc/stdlib/atexit.c

@@ -35,6 +35,7 @@
 #include "namespace.h"
 #include <errno.h>
 #include <link.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -56,6 +57,8 @@ _Block_copy(void*);
 #define	ATEXIT_FN_CXA	2
 
 static pthread_mutex_t atexit_mutex = PTHREAD_MUTEX_INITIALIZER;
+static void *current_finalize_dso = NULL;
+static bool call_finalize_again = false;
 
 #define _MUTEX_LOCK(x)		if (__isthreaded) _pthread_mutex_lock(x)
 #define _MUTEX_UNLOCK(x)	if (__isthreaded) _pthread_mutex_unlock(x)
@@ -115,6 +118,9 @@ atexit_register(struct atexit_fn *fptr)
 		__atexit = p;
 	}
 	p->fns[p->ind++] = *fptr;
+	if (current_finalize_dso != NULL &&
+	    current_finalize_dso == fptr->fn_dso)
+		call_finalize_again = true;
 	_MUTEX_UNLOCK(&atexit_mutex);
 	return 0;
 }
@@ -208,33 +214,38 @@ __cxa_finalize(void *dso)
 	}
 
 	_MUTEX_LOCK(&atexit_mutex);
-	for (p = __atexit; p; p = p->next) {
-		for (n = p->ind; --n >= 0;) {
-			if (p->fns[n].fn_type == ATEXIT_FN_EMPTY)
-				continue; /* already been called */
-			fn = p->fns[n];
-			if (dso != NULL && dso != fn.fn_dso) {
-				/* wrong DSO ? */
-				if (!has_phdr || global_exit ||
-				    !__elf_phdr_match_addr(&phdr_info,
-				    fn.fn_ptr.cxa_func))
-					continue;
+	current_finalize_dso = dso;
+	do {
+		call_finalize_again = false;
+		for (p = __atexit; p; p = p->next) {
+			for (n = p->ind; --n >= 0;) {
+				if (p->fns[n].fn_type == ATEXIT_FN_EMPTY)
+					continue; /* already been called */
+				fn = p->fns[n];
+				if (dso != NULL && dso != fn.fn_dso) {
+					/* wrong DSO ? */
+					if (!has_phdr || global_exit ||
+					    !__elf_phdr_match_addr(&phdr_info,
+					    fn.fn_ptr.cxa_func))
+						continue;
+				}
+				/*
+				  Mark entry to indicate that this particular
+				  handler has already been called.
+				*/
+				p->fns[n].fn_type = ATEXIT_FN_EMPTY;
+				_MUTEX_UNLOCK(&atexit_mutex);
+
+				/* Call the function of correct type. */
+				if (fn.fn_type == ATEXIT_FN_CXA)
+					fn.fn_ptr.cxa_func(fn.fn_arg);
+				else if (fn.fn_type == ATEXIT_FN_STD)
+					fn.fn_ptr.std_func();
+				_MUTEX_LOCK(&atexit_mutex);
 			}
-			/*
-			  Mark entry to indicate that this particular handler
-			  has already been called.
-			*/
-			p->fns[n].fn_type = ATEXIT_FN_EMPTY;
-		        _MUTEX_UNLOCK(&atexit_mutex);
-		
-			/* Call the function of correct type. */
-			if (fn.fn_type == ATEXIT_FN_CXA)
-				fn.fn_ptr.cxa_func(fn.fn_arg);
-			else if (fn.fn_type == ATEXIT_FN_STD)
-				fn.fn_ptr.std_func();
-			_MUTEX_LOCK(&atexit_mutex);
 		}
-	}
+	} while (call_finalize_again);
+	current_finalize_dso = NULL;
 	_MUTEX_UNLOCK(&atexit_mutex);
 	if (dso == NULL)
 		_MUTEX_DESTROY(&atexit_mutex);