MEDIUM: chunk: Add support for large chunks

Because there is now a memory pool for large buffers, we must also add the
support for large chunks. So, if large buffers are configured, a dedicated
memory pool is created to allocate large chunks. alloc_large_trash_chunk()
must be used to allocate a large chunk. alloc_trash_chunk_sz() can be used to
allocate a chunk with the best size. However free_trash_chunk() remains the
only way to release a chunk, regular or large.

In addition, large trash buffers are also created, using the same mechanism
than for regular trash buffers. So three thread-local trash buffers are
created. get_large_trash_chunk() must be used to get a large trash buffer.
And get_trash_chunk_sz() may be used to get a trash buffer with the best
size.

doc/configuration.txt

@@ -4127,6 +4127,9 @@ tune.bufsize.large <size>
   data must be bufferized without changing the size of regular buffers. The
   large buffers are not implicitly used.
 
+  Note that when large buffers are configured, three special large buffers will
+  be allocated for each threads during startup for internal usage.
+
 tune.bufsize.small <size>
   Sets the size in bytes for small buffers. The defaults value is 1024.
 

include/haproxy/chunk.h

@@ -32,6 +32,7 @@
 
 
 extern struct pool_head *pool_head_trash;
+extern struct pool_head *pool_head_large_trash;
 
 /* function prototypes */
 
@@ -46,6 +47,9 @@ int chunk_asciiencode(struct buffer *dst, struct buffer *src, char qc);
 int chunk_strcmp(const struct buffer *chk, const char *str);
 int chunk_strcasecmp(const struct buffer *chk, const char *str);
 struct buffer *get_trash_chunk(void);
+struct buffer *get_large_trash_chunk(void);
+struct buffer *get_trash_chunk_sz(size_t size);
+struct buffer *get_larger_trash_chunk(struct buffer *chunk);
 int init_trash_buffers(int first);
 
 static inline void chunk_reset(struct buffer *chk)
@@ -106,12 +110,53 @@ static forceinline struct buffer *alloc_trash_chunk(void)
 	return chunk;
 }
 
+/*
+ * Allocate a large trash chunk from the reentrant pool. The buffer starts at
+ * the end of the chunk. This chunk must be freed using free_trash_chunk(). This
+ * call may fail and the caller is responsible for checking that the returned
+ * pointer is not NULL.
+ */
+static forceinline struct buffer *alloc_large_trash_chunk(void)
+{
+	struct buffer *chunk;
+
+	if (!pool_head_large_trash)
+		return NULL;
+
+	chunk = pool_alloc(pool_head_large_trash);
+	if (chunk) {
+		char *buf = (char *)chunk + sizeof(struct buffer);
+		*buf = 0;
+		chunk_init(chunk, buf,
+			   pool_head_large_trash->size - sizeof(struct buffer));
+	}
+	return chunk;
+}
+
+/*
+ * Allocate a trash chunk accordingly to the requested size. This chunk must be
+ * freed using free_trash_chunk(). This call may fail and the caller is
+ * responsible for checking that the returned pointer is not NULL.
+ */
+static forceinline struct buffer *alloc_trash_chunk_sz(size_t size)
+{
+	if (likely(size <= pool_head_trash->size))
+		return alloc_trash_chunk();
+	else if (pool_head_large_trash && size <= pool_head_large_trash->size)
+		return alloc_large_trash_chunk();
+	else
+		return NULL;
+}
+
 /*
  * free a trash chunk allocated by alloc_trash_chunk(). NOP on NULL.
  */
 static forceinline void free_trash_chunk(struct buffer *chunk)
 {
-	pool_free(pool_head_trash, chunk);
+	if (likely(chunk && chunk->size == pool_head_trash->size - sizeof(struct buffer)))
+		pool_free(pool_head_trash, chunk);
+	else
+		pool_free(pool_head_large_trash, chunk);
 }
 
 /* copies chunk <src> into <chk>. Returns 0 in case of failure. */

src/chunk.c

@@ -16,6 +16,7 @@
 #include <string.h>
 
 #include <haproxy/api.h>
+#include <haproxy/buf.h>
 #include <haproxy/chunk.h>
 #include <haproxy/global.h>
 #include <haproxy/tools.h>
@@ -36,6 +37,22 @@ struct pool_head *pool_head_trash __read_mostly = NULL;
 /* this is used to drain data, and as a temporary buffer for sprintf()... */
 THREAD_LOCAL struct buffer trash = { };
 
+/* large trash chunks used for various conversions */
+static THREAD_LOCAL struct buffer *large_trash_chunk;
+static THREAD_LOCAL struct buffer large_trash_chunk1;
+static THREAD_LOCAL struct buffer large_trash_chunk2;
+
+/* large trash buffers used for various conversions */
+static int large_trash_size __read_mostly = 0;
+static THREAD_LOCAL char *large_trash_buf1 = NULL;
+static THREAD_LOCAL char *large_trash_buf2 = NULL;
+
+/* the trash pool for reentrant allocations */
+struct pool_head *pool_head_large_trash __read_mostly = NULL;
+
+/* this is used to drain data, and as a temporary large buffer */
+THREAD_LOCAL struct buffer trash_large = { };
+
 /*
 * Returns a pre-allocated and initialized trash chunk that can be used for any
 * type of conversion. Two chunks and their respective buffers are alternatively
@@ -62,6 +79,63 @@ struct buffer *get_trash_chunk(void)
 	return trash_chunk;
 }
 
+/* Similar to get_trash_chunk() but return a pre-allocated large chunk
+ * instead. Becasuse large buffers are not enabled by default, this function may
+ * return NULL.
+ */
+struct buffer *get_large_trash_chunk(void)
+{
+	char *large_trash_buf;
+
+	if (!large_trash_size)
+		return NULL;
+
+	if (large_trash_chunk == &large_trash_chunk1) {
+		large_trash_chunk = &large_trash_chunk2;
+		large_trash_buf = large_trash_buf2;
+	}
+	else {
+		large_trash_chunk = &large_trash_chunk1;
+		large_trash_buf = large_trash_buf1;
+	}
+	*large_trash_buf = 0;
+	chunk_init(large_trash_chunk, large_trash_buf, large_trash_size);
+	return large_trash_chunk;
+}
+
+/* Returns a trash chunk accordingly to the requested size. This function may
+ * fail if the requested size is too big or if the large chubks are not
+ * configured.
+ */
+struct buffer *get_trash_chunk_sz(size_t size)
+{
+	if (likely(size <= trash_size))
+	    return get_trash_chunk();
+	else if (large_trash_size && size <= large_trash_size)
+		return get_large_trash_chunk();
+	else
+		return NULL;
+}
+
+/* Returns a larger buffer than <chk> if possible or NULL otherwise. If a larger
+ * buffer is returned, content of <chk> are copied.
+ */
+struct buffer *get_larger_trash_chunk(struct buffer *chk)
+{
+	struct buffer *chunk;
+
+	if (!chk)
+		return get_trash_chunk();
+
+	/* No large buffers or current chunk is alread a large trash chunk */
+	if (!large_trash_size || chk->size == large_trash_size)
+		return NULL;
+
+	chunk = get_large_trash_chunk();
+	b_xfer(chunk, chk, b_data(chk));
+	return chunk;
+}
+
 /* (re)allocates the trash buffers. Returns 0 in case of failure. It is
  * possible to call this function multiple times if the trash size changes.
  */
@@ -74,9 +148,27 @@ static int alloc_trash_buffers(int bufsize)
 	return trash.area && trash_buf1 && trash_buf2;
 }
 
+/* allocates the trash large buffers if necessary. Returns 0 in case of
+ * failure. Unlike alloc_trash_buffers(), It is unexpected to call this function
+ * multiple times. Large buffers are not used during configuration parsing.
+ */
+static int alloc_large_trash_buffers(int bufsize)
+{
+	large_trash_size = bufsize;
+	if (!large_trash_size)
+		return 1;
+
+	BUG_ON(trash_large.area && large_trash_buf1 && large_trash_buf2);
+
+	chunk_init(&trash_large, my_realloc2(trash_large.area, bufsize), bufsize);
+	large_trash_buf1 = (char *)my_realloc2(large_trash_buf1, bufsize);
+	large_trash_buf2 = (char *)my_realloc2(large_trash_buf2, bufsize);
+	return trash_large.area && large_trash_buf1 && large_trash_buf2;
+}
+
 static int alloc_trash_buffers_per_thread()
 {
-	return alloc_trash_buffers(global.tune.bufsize);
+	return alloc_trash_buffers(global.tune.bufsize) && alloc_large_trash_buffers(global.tune.bufsize_large);
 }
 
 static void free_trash_buffers_per_thread()
@@ -84,6 +176,10 @@ static void free_trash_buffers_per_thread()
 	chunk_destroy(&trash);
 	ha_free(&trash_buf2);
 	ha_free(&trash_buf1);
+
+	chunk_destroy(&trash_large);
+	ha_free(&large_trash_buf2);
+	ha_free(&large_trash_buf1);
 }
 
 /* Initialize the trash buffers. It returns 0 if an error occurred. */
@@ -103,6 +199,14 @@ int init_trash_buffers(int first)
 				      MEM_F_EXACT);
 	if (!pool_head_trash || !alloc_trash_buffers(global.tune.bufsize))
 		return 0;
+
+	if (!first && global.tune.bufsize_large) {
+		pool_head_large_trash = create_pool("large_trash",
+						    sizeof(struct buffer) + global.tune.bufsize_large,
+						    MEM_F_EXACT);
+		if (!pool_head_large_trash)
+			return 0;
+	}
 	return 1;
 }