Provide more information when the memory allocation fails

Instead of just crashing when memory allocation fails, also print a
message saying "Out of memory!", the size of the allocation that failed,
total allocated memory from all memory contexts and value of errno.

lib/isc/jemalloc_shim.h

@@ -103,7 +103,9 @@ mallocx(size_t size, int flags) {
 
 	size_t bytes = ISC_CHECKED_ADD(size, sizeof(size_info));
 	size_info *si = malloc(bytes);
-	INSIST(si != NULL);
+	if (si == NULL) {
+		return NULL;
+	}
 
 	si->size = size;
 	ptr = &si[1];
@@ -131,8 +133,11 @@ sallocx(void *ptr, int flags ISC_ATTR_UNUSED) {
 
 static inline void *
 rallocx(void *ptr, size_t size, int flags) {
-	size_info *si = realloc(&(((size_info *)ptr)[-1]), size + sizeof(*si));
-	INSIST(si != NULL);
+	size_t bytes = ISC_CHECKED_ADD(size, sizeof(size_info));
+	size_info *si = realloc(&(((size_info *)ptr)[-1]), bytes);
+	if (si == NULL) {
+		return NULL;
+	}
 
 	if (MALLOCX_ZERO_GET(flags) && size > si->size) {
 		memset((uint8_t *)si + sizeof(*si) + si->size, 0,

lib/isc/mem.c

@@ -20,8 +20,10 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <unistd.h>
 
 #include <isc/atomic.h>
+#include <isc/backtrace.h>
 #include <isc/hash.h>
 #include <isc/magic.h>
 #include <isc/mem.h>
@@ -172,6 +174,84 @@ struct isc_mempool {
  * Private Inline-able.
  */
 
+static size_t
+total_inuse(void) {
+	size_t inuse = 0;
+	LOCK(&contextslock);
+	ISC_LIST_FOREACH(contexts, ctx, link) {
+		inuse += isc_mem_inuse(ctx);
+	}
+	UNLOCK(&contextslock);
+
+	return inuse;
+}
+
+static void
+write_string(int fd, const char *str) {
+	int r = write(fd, str, strlen(str));
+	if (r == -1) {
+		abort();
+	}
+}
+
+#define STRINGIFY(x) #x
+#define TOSTRING(x)  STRINGIFY(x)
+static void
+write_size(int fd, size_t size) {
+	char buf[sizeof(TOSTRING(SIZE_MAX)) + 1] = { 0 };
+
+	char *str = buf + (sizeof(buf) - 1);
+
+	if (size == 0) {
+		*--str = '0';
+	} else {
+		while (size) {
+			*--str = '0' + (size % 10);
+			size /= 10;
+		}
+	}
+
+	write_string(fd, str);
+}
+#undef TOSTRING
+#undef STRINGIFY
+
+static void
+write_errno(int fd, int errnum) {
+	char buf[BUFSIZ] = { 0 };
+	int ret = isc_string_strerror_r(errnum, buf, sizeof(buf));
+	if (ret == 0) {
+		write_string(fd, buf);
+	}
+}
+
+static void
+write_backtrace(int fd) {
+	void *tracebuf[ISC_BACKTRACE_MAXFRAME];
+	int nframes = isc_backtrace(tracebuf, ISC_BACKTRACE_MAXFRAME);
+
+	if (nframes > 0) {
+		isc_backtrace_symbols_fd(tracebuf, nframes, fd);
+	}
+}
+
+#define CHECK_OOM(ptr, size) (void)((ptr != NULL) || (oom(size), false))
+
+ISC_NORETURN static void
+oom(size_t size) {
+	int fd = fileno(stderr);
+	write_string(fd, "Out of memory (trying to allocate ");
+	write_size(fd, size);
+	write_string(fd, ", total ");
+	write_size(fd, total_inuse());
+	write_string(fd, "): ");
+	write_errno(fd, errno);
+	write_string(fd, "\n");
+	write_backtrace(fd);
+
+	abort();
+}
+
 #if !ISC_MEM_TRACKLINES
 #define ADD_TRACE(mctx, ptr, size, func, file, line)
 #define DELETE_TRACE(mctx, ptr, size, func, file, line)
@@ -239,7 +319,7 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) {
 	idx = hash % DEBUG_TABLE_COUNT;
 
 	dl = mallocx(dlsize, mctx->jemalloc_flags);
-	INSIST(dl != NULL);
+	CHECK_OOM(dl, dlsize);
 
 	ISC_LINK_INIT(dl, link);
 	dl->ptr = ptr;
@@ -310,14 +390,12 @@ unlock:
  */
 static void *
 mem_get(isc_mem_t *ctx, size_t size, int flags) {
-	char *ret = NULL;
-
 	ADJUST_ZERO_ALLOCATION_SIZE(size);
 
-	ret = mallocx(size, flags | ctx->jemalloc_flags);
-	INSIST(ret != NULL);
+	void *ptr = mallocx(size, flags | ctx->jemalloc_flags);
+	CHECK_OOM(ptr, size);
 
-	return ret;
+	return ptr;
 }
 
 /*!
@@ -338,7 +416,7 @@ mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t new_size, int flags) {
 	ADJUST_ZERO_ALLOCATION_SIZE(new_size);
 
 	new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags);
-	INSIST(new_ptr != NULL);
+	CHECK_OOM(new_ptr, new_size);
 
 	return new_ptr;
 }
@@ -469,7 +547,7 @@ mem_create(const char *name, isc_mem_t **ctxp, unsigned int debugging,
 	REQUIRE(name != NULL);
 
 	ctx = mallocx(sizeof(*ctx), jemalloc_flags);
-	INSIST(ctx != NULL);
+	CHECK_OOM(ctx, sizeof(*ctx));
 
 	*ctx = (isc_mem_t){
 		.magic = MEM_MAGIC,
@@ -498,11 +576,11 @@ mem_create(const char *name, isc_mem_t **ctxp, unsigned int debugging,
 #if ISC_MEM_TRACKLINES
 	if ((ctx->debugging & ISC_MEM_DEBUGRECORD) != 0) {
 		unsigned int i;
+		size_t debuglist_size = ISC_CHECKED_MUL(DEBUG_TABLE_COUNT,
+							sizeof(debuglist_t));
 
-		ctx->debuglist = mallocx(
-			ISC_CHECKED_MUL(DEBUG_TABLE_COUNT, sizeof(debuglist_t)),
-			jemalloc_flags);
-		INSIST(ctx->debuglist != NULL);
+		ctx->debuglist = mallocx(debuglist_size, jemalloc_flags);
+		CHECK_OOM(ctx->debuglist, debuglist_size);
 
 		for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
 			ISC_LIST_INIT(ctx->debuglist[i]);