Handle fatal and FIPS provider interactions

When fatal is called we may be holding memory allocated by OpenSSL.
This may result in the reference count for the FIPS provider not
going to zero and the shared library not being unloaded during
OPENSSL_cleanup.  When the shared library is ultimately unloaded,
when all remaining dynamically loaded libraries are freed, we have
already destroyed the memory context we where using to track memory
leaks / late frees resulting in INSIST being called.

Disable triggering the INSIST when fatal has being called.

bin/confgen/util.c

@@ -19,6 +19,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include <isc/tls.h>
+
 extern bool verbose;
 extern const char *progname;
 
@@ -43,5 +45,6 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	exit(1);
 }

bin/delv/delv.c

@@ -263,6 +263,7 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	exit(1);
 }
 

bin/dig/dighost.c

@@ -411,6 +411,7 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	digexit();
 }
 

bin/dnssec/dnssectool.c

@@ -32,6 +32,7 @@
 #include <isc/result.h>
 #include <isc/string.h>
 #include <isc/time.h>
+#include <isc/tls.h>
 #include <isc/tm.h>
 #include <isc/util.h>
 
@@ -82,6 +83,7 @@ fatal(const char *format, ...) {
 	if (fatalcallback != NULL) {
 		(*fatalcallback)();
 	}
+	isc__tls_setfatalmode();
 	exit(1);
 }
 

bin/named/server.c

@@ -10301,6 +10301,7 @@ fatal(const char *msg, isc_result_t result) {
 		      NAMED_LOGMODULE_SERVER, ISC_LOG_CRITICAL,
 		      "exiting (due to fatal error)");
 	named_os_shutdown();
+	isc__tls_setfatalmode();
 	exit(1);
 }
 

bin/nsupdate/nsupdate.c

@@ -278,6 +278,7 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	exit(1);
 }
 

bin/rndc/util.c

@@ -19,6 +19,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include <isc/tls.h>
+
 extern bool verbose;
 extern const char *progname;
 
@@ -43,5 +45,6 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	exit(1);
 }

bin/tools/mdig.c

@@ -889,6 +889,7 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	exit(-2);
 }
 

bin/tools/nsec3hash.c

@@ -24,6 +24,7 @@
 #include <isc/iterated_hash.h>
 #include <isc/result.h>
 #include <isc/string.h>
+#include <isc/tls.h>
 #include <isc/types.h>
 #include <isc/util.h>
 
@@ -46,6 +47,7 @@ fatal(const char *format, ...) {
 	vfprintf(stderr, format, args);
 	va_end(args);
 	fprintf(stderr, "\n");
+	isc__tls_setfatalmode();
 	exit(1);
 }
 

lib/isc/include/isc/tls.h

@@ -594,3 +594,6 @@ isc__tls_shutdown(void);
 
 void
 isc__tls_setdestroycheck(bool check);
+
+void
+isc__tls_setfatalmode(void);

lib/isc/tls.c

@@ -77,6 +77,8 @@ isc__tls_set_thread_id(CRYPTO_THREADID *id) {
 }
 #endif
 
+static atomic_bool handle_fatal = false;
+
 #if !defined(LIBRESSL_VERSION_NUMBER)
 /*
  * This was crippled with LibreSSL, so just skip it:
@@ -109,7 +111,9 @@ isc__tls_free_ex(void *ptr, const char *file, int line) {
 	if (ptr == NULL) {
 		return;
 	}
-	isc__mem_free(isc__tls_mctx, ptr, 0, file, (unsigned int)line);
+	if (!atomic_load(&handle_fatal) || isc__tls_mctx != NULL) {
+		isc__mem_free(isc__tls_mctx, ptr, 0, file, (unsigned int)line);
+	}
 }
 
 #else /* ISC_MEM_TRACKLINES */
@@ -135,7 +139,9 @@ isc__tls_free_ex(void *ptr, const char *file, int line) {
 	if (ptr == NULL) {
 		return;
 	}
-	isc__mem_free(isc__tls_mctx, ptr, 0);
+	if (!atomic_load(&handle_fatal) || isc__tls_mctx != NULL) {
+		isc__mem_free(isc__tls_mctx, ptr, 0);
+	}
 }
 
 #endif /* ISC_MEM_TRACKLINES */
@@ -1744,3 +1750,8 @@ isc_tlsctx_set_random_session_id_context(isc_tlsctx_t *ctx) {
 	RUNTIME_CHECK(
 		SSL_CTX_set_session_id_context(ctx, session_id_ctx, len) == 1);
 }
+
+void
+isc__tls_setfatalmode(void) {
+	atomic_store(&handle_fatal, true);
+}