Prevent buffer overrun in unicode_normalize().

Some UTF8 characters decompose to more than a dozen codepoints.
It is possible for an input string that fits into well under
1GB to produce more than 4G decomposed codepoints, causing
unicode_normalize()'s decomp_size variable to wrap around to a
small positive value.  This results in a small output buffer
allocation and subsequent buffer overrun.

To fix, test after each addition to see if we've overrun MaxAllocSize,
and break out of the loop early if so.  In frontend code we want to
just return NULL for this failure (treating it like OOM).  In the
backend, we can rely on the following palloc() call to throw error.

I also tightened things up in the calling functions in varlena.c,
using size_t rather than int and allocating the input workspace
with palloc_array().  These changes are probably unnecessary
given the knowledge that the original input and the normalized
output_chars array must fit into 1GB, but it's a lot easier to
believe the code is safe with these changes.

Reported-by: Xint Code
Reported-by: Bruce Dang <bruce@calif.io>
Author: Tom Lane <tgl@sss.pgh.pa.us>
Co-authored-by: Heikki Linnakangas <hlinnaka@iki.fi>
Backpatch-through: 14
Security: CVE-2026-6473

src/backend/utils/adt/varlena.c

@@ -5490,18 +5490,18 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
 	text	   *input = PG_GETARG_TEXT_PP(0);
 	char	   *formstr = text_to_cstring(PG_GETARG_TEXT_PP(1));
 	UnicodeNormalizationForm form;
-	int			size;
+	size_t		size;
 	char32_t   *input_chars;
 	char32_t   *output_chars;
 	unsigned char *p;
 	text	   *result;
-	int			i;
+	size_t		i;
 
 	form = unicode_norm_form_from_string(formstr);
 
 	/* convert to char32_t */
 	size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
-	input_chars = palloc((size + 1) * sizeof(char32_t));
+	input_chars = palloc_array(char32_t, size + 1);
 	p = (unsigned char *) VARDATA_ANY(input);
 	for (i = 0; i < size; i++)
 	{
@@ -5556,20 +5556,20 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
 	text	   *input = PG_GETARG_TEXT_PP(0);
 	char	   *formstr = text_to_cstring(PG_GETARG_TEXT_PP(1));
 	UnicodeNormalizationForm form;
-	int			size;
+	size_t		size;
 	char32_t   *input_chars;
 	char32_t   *output_chars;
 	unsigned char *p;
-	int			i;
+	size_t		i;
 	UnicodeNormalizationQC quickcheck;
-	int			output_size;
+	size_t		output_size;
 	bool		result;
 
 	form = unicode_norm_form_from_string(formstr);
 
 	/* convert to char32_t */
 	size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
-	input_chars = palloc((size + 1) * sizeof(char32_t));
+	input_chars = palloc_array(char32_t, size + 1);
 	p = (unsigned char *) VARDATA_ANY(input);
 	for (i = 0; i < size; i++)
 	{

src/common/unicode_norm.c

@@ -23,6 +23,7 @@
 #include "common/unicode_norm_hashfunc.h"
 #include "common/unicode_normprops_table.h"
 #include "port/pg_bswap.h"
+#include "utils/memutils.h"
 #else
 #include "common/unicode_norm_table.h"
 #endif
@@ -420,10 +421,28 @@ unicode_normalize(UnicodeNormalizationForm form, const char32_t *input)
 
 	/*
 	 * Calculate how many characters long the decomposed version will be.
+	 *
+	 * Some characters decompose to quite a few code points, so that the
+	 * decomposed version's size could overrun MaxAllocSize, and even 32-bit
+	 * size_t, even though the input string presumably fits in that.  In
+	 * frontend we want to just return NULL in that case, so monitor the sum
+	 * and exit early once we'd need more than MaxAllocSize bytes.
 	 */
 	decomp_size = 0;
 	for (p = input; *p; p++)
+	{
 		decomp_size += get_decomposed_size(*p, compat);
+		if (unlikely(decomp_size > MaxAllocSize / sizeof(char32_t)))
+		{
+#ifndef FRONTEND
+			/* Exit loop and let palloc() throw error below */
+			break;
+#else
+			/* Just return NULL with no explicit error */
+			return NULL;
+#endif
+		}
+	}
 
 	decomp_chars = (char32_t *) ALLOC((decomp_size + 1) * sizeof(char32_t));
 	if (decomp_chars == NULL)