diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c index 01d4f55b07e..7c36fd5091b 100644 --- a/src/backend/utils/adt/pg_locale_builtin.c +++ b/src/backend/utils/adt/pg_locale_builtin.c @@ -62,21 +62,33 @@ initcap_wbnext(void *state) while (wbstate->offset < wbstate->len) { - char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str + + int ulen = pg_utf_mblen((const unsigned char *) wbstate->str + wbstate->offset); - bool curr_alnum = pg_u_isalnum(u, wbstate->posix); + char32_t u; + bool curr_alnum; + size_t prev_offset = wbstate->offset; + + /* invalid UTF8 */ + if (wbstate->offset + ulen > wbstate->len) + { + wbstate->init = true; + wbstate->offset = wbstate->len; + return prev_offset; + } + + u = utf8_to_unicode((const unsigned char *) wbstate->str + + wbstate->offset); + curr_alnum = pg_u_isalnum(u, wbstate->posix); if (!wbstate->init || curr_alnum != wbstate->prev_alnum) { - size_t prev_offset = wbstate->offset; - wbstate->init = true; - wbstate->offset += unicode_utf8len(u); + wbstate->offset += ulen; wbstate->prev_alnum = curr_alnum; return prev_offset; } - wbstate->offset += unicode_utf8len(u); + wbstate->offset += ulen; } return wbstate->len; diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c index a0dbf00b671..31ea94513bf 100644 --- a/src/common/unicode/case_test.c +++ b/src/common/unicode/case_test.c @@ -296,6 +296,8 @@ tfunc_fold(char *dst, size_t dstsize, const char *src, static void test_convert_case(void) { + size_t needed; + /* test string with no case changes */ test_convert(tfunc_lower, "√∞", "√∞"); /* test adjust-to-cased behavior */ @@ -320,6 +322,12 @@ test_convert_case(void) /* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */ test_convert(tfunc_title, "\uFF11a", "\uFF11a"); + /* invalid UTF8: truncated multibyte sequence */ + needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false); + Assert(needed == 3); + /* invalid UTF8: invalid byte */ + needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false); + Assert(needed == 3); #ifdef USE_ICU icu_test_full(""); diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c index d6ee00b7d9c..42eb7d22211 100644 --- a/src/common/unicode_case.c +++ b/src/common/unicode_case.c @@ -189,6 +189,22 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen, NULL); } +/* local version of pg_utf_mblen() to be inlinable */ +static int +utf8_mblen(const unsigned char *s) +{ + if ((*s & 0x80) == 0) + return 1; + else if ((*s & 0xe0) == 0xc0) + return 2; + else if ((*s & 0xf0) == 0xe0) + return 3; + else if ((*s & 0xf8) == 0xf0) + return 4; + else + return -1; +} + /* * Implement Unicode Default Case Conversion algorithm. * @@ -227,12 +243,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, while (srcoff < srclen) { - char32_t u1 = utf8_to_unicode((const unsigned char *) src + srcoff); - int u1len = unicode_utf8len(u1); + int u1len = utf8_mblen((const unsigned char *) src + srcoff); + char32_t u1; char32_t simple = 0; const char32_t *special = NULL; enum CaseMapResult casemap_result; + /* invalid UTF8 */ + if (u1len < 0 || srcoff + u1len > srclen) + break; + + u1 = utf8_to_unicode((const unsigned char *) src + srcoff); + if (str_casekind == CaseTitle) { if (srcoff == boundary) @@ -316,7 +338,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) { if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0) { - char32_t curr = utf8_to_unicode(str + i); + int u1len = utf8_mblen((const unsigned char *) str + i); + char32_t curr; + + /* invalid UTF8 */ + if (u1len < 0 || i + u1len > len) + return false; + + curr = utf8_to_unicode(str + i); if (pg_u_prop_case_ignorable(curr)) continue; @@ -327,8 +356,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) } else if ((str[i] & 0xC0) == 0x80) continue; - - Assert(false); /* invalid UTF-8 */ + else + return false; /* invalid UTF8 */ } /* end of string is not followed by a Cased character */ @@ -340,7 +369,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) { if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0) { - char32_t curr = utf8_to_unicode(str + i); + int u1len = utf8_mblen((const unsigned char *) str + i); + char32_t curr; + + /* invalid UTF8 */ + if (u1len < 0 || i + u1len > len) + return false; + + curr = utf8_to_unicode(str + i); if (pg_u_prop_case_ignorable(curr)) continue; @@ -351,8 +387,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) } else if ((str[i] & 0xC0) == 0x80) continue; - - Assert(false); /* invalid UTF-8 */ + else + return false; /* invalid UTF8 */ } return true;