mirror of
https://github.com/isc-projects/bind9.git
synced 2026-07-10 22:01:01 -04:00
fix: dev: Detect UTF-16 surrogates in isc_utf8_valid()
UTF-8 standard forbid usage of unicode character between the range of 0xD800..0xDFFF (reserved, and used as UTF-16 surrogates, see RFC 3629). However, `usc_utf8_valid()` was not checking if the encoded unicode character was in this range, which then would accept invalid UTF-8 strings. This is now fixed. Closes #6151 Merge branch '6151-utf16-surrogates-detection' into 'main' See merge request isc-projects/bind9!12345
This commit is contained in:
Colin Vidal
commit
dfefde8481
3 changed files with 202 additions and 0 deletions
|
|
@ -33,9 +33,20 @@ isc_utf8_valid(const unsigned char *buf, size_t len) {
|
|||
REQUIRE(buf != NULL);
|
||||
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
/*
|
||||
* ASCII character range (first row).
|
||||
*/
|
||||
if (buf[i] <= 0x7f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* 0x80 -> 1000 0000
|
||||
* 0xC0 -> 1100 0000
|
||||
* 0xE0 -> 1110 0000
|
||||
*
|
||||
* Is unicode character is encoded using 2 bytes (second row).
|
||||
*/
|
||||
if ((i + 1) < len && (buf[i] & 0xe0) == 0xc0 &&
|
||||
(buf[i + 1] & 0xc0) == 0x80)
|
||||
{
|
||||
|
|
@ -47,6 +58,15 @@ isc_utf8_valid(const unsigned char *buf, size_t len) {
|
|||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* 0x80 -> 1000 0000
|
||||
* 0xC0 -> 1100 0000
|
||||
* 0xE0 -> 1110 0000
|
||||
* 0xF0 -> 1111 0000
|
||||
*
|
||||
* Is unicode character is encoded within 3 bytes (third row).
|
||||
*/
|
||||
if ((i + 2) < len && (buf[i] & 0xf0) == 0xe0 &&
|
||||
(buf[i + 1] & 0xc0) == 0x80 && (buf[i + 2] & 0xc0) == 0x80)
|
||||
{
|
||||
|
|
@ -57,8 +77,26 @@ isc_utf8_valid(const unsigned char *buf, size_t len) {
|
|||
if (w < 0x0800) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Unicode range 0xD800..0xDFFF is reserved (UTF16
|
||||
* surrogates)
|
||||
*/
|
||||
if (w >= 0xD800 && w <= 0xDFFF) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* 0x80 -> 1000 0000
|
||||
* 0xC0 -> 1100 0000
|
||||
* 0xE0 -> 1110 0000
|
||||
* 0xF0 -> 1111 0000
|
||||
* 0xF8 -> 1111 1000
|
||||
*
|
||||
* Is unicode character is encoded within 4 bytes (fourth row).
|
||||
*/
|
||||
if ((i + 3) < len && (buf[i] & 0xf8) == 0xf0 &&
|
||||
(buf[i + 1] & 0xc0) == 0x80 &&
|
||||
(buf[i + 2] & 0xc0) == 0x80 && (buf[i + 3] & 0xc0) == 0x80)
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ isc_test = [
|
|||
'tlsdns',
|
||||
'udp',
|
||||
'url',
|
||||
'utf8',
|
||||
'work',
|
||||
]
|
||||
|
||||
|
|
|
|||
163
tests/isc/utf8_test.c
Normal file
163
tests/isc/utf8_test.c
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
/*
|
||||
* Copyright (C) Internet Systems Consortium, Inc. ("ISC")
|
||||
*
|
||||
* SPDX-License-Identifier: MPL-2.0
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, you can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
*
|
||||
* See the COPYRIGHT file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
*/
|
||||
|
||||
#include <ctype.h>
|
||||
#include <inttypes.h>
|
||||
#include <sched.h> /* IWYU pragma: keep */
|
||||
#include <setjmp.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define UNIT_TESTING
|
||||
#include <cmocka.h>
|
||||
|
||||
#include <isc/lib.h>
|
||||
#include <isc/utf8.h>
|
||||
|
||||
#include <tests/isc.h>
|
||||
|
||||
static void
|
||||
valid(const char *str) {
|
||||
assert_true(isc_utf8_valid((const unsigned char *)str, strlen(str)));
|
||||
}
|
||||
|
||||
static void
|
||||
invalid(const char *str) {
|
||||
assert_false(isc_utf8_valid((const unsigned char *)str, strlen(str)));
|
||||
}
|
||||
|
||||
static void
|
||||
validbom(const char *str) {
|
||||
assert_true(isc_utf8_bom((const unsigned char *)str, strlen(str)));
|
||||
}
|
||||
|
||||
static void
|
||||
invalidbom(const char *str) {
|
||||
assert_false(isc_utf8_bom((const unsigned char *)str, strlen(str)));
|
||||
}
|
||||
|
||||
ISC_RUN_TEST_IMPL(test_utf8_ascii) {
|
||||
valid("#1@#'Been a long lonely, lonely, lonely, lonely, lonely time");
|
||||
}
|
||||
|
||||
ISC_RUN_TEST_IMPL(test_utf8_twobytes) {
|
||||
/*
|
||||
* 0xC280 -> 1100 0010 1000 0000 (unicode character 0x80, below it's
|
||||
* ASCII, so it would be invalid)
|
||||
*
|
||||
* 0xDFBF -> 1101 1111 1011 1111 (unicode character 0x07FF)
|
||||
*/
|
||||
invalid("invalid 2-bytes sequence: \xC0\x80 (invalid)");
|
||||
valid("2-bytes sequence: \xC2\x80 (begining of the range)");
|
||||
valid("2-bytes sequence: \xDF\xBF (end of range)");
|
||||
invalid("invalid (second bytes doesn't starts with 10__) \xDF\xFF");
|
||||
}
|
||||
|
||||
ISC_RUN_TEST_IMPL(test_utf8_threebytes) {
|
||||
/*
|
||||
* 0x0800 (first valid unicode character holding in 3 bytes in UTF-8) is
|
||||
* 0000 |1000 00|00 0000 which encodes in
|
||||
*
|
||||
* 1110 ____ 10__ ____ 10__ ____
|
||||
* 1110 0000 1010 0000 1000 0000
|
||||
*
|
||||
* So, 0xE0A080
|
||||
*
|
||||
* 0xEFBFBF is 1110 1111 1011 1111 1011 1111 which is the maximum
|
||||
* unicode character encoded in 3-bytes in UTF-8.
|
||||
*/
|
||||
|
||||
invalid("invalid 3-bytes sequence: \xE0\x80\x88");
|
||||
invalid("invalid 3-bytes sequence: \xE0\x84\x80");
|
||||
valid("valid 3-bytes sequence: \xE0\xA0\x80 (min value)");
|
||||
valid("valid 3-bytes sequence: \xEF\xBF\xBF (max value)");
|
||||
invalid("invalid 3-bytes sequence \xE0\xC1\xC1 (extra bytes does not "
|
||||
"start with 10__)");
|
||||
}
|
||||
|
||||
ISC_RUN_TEST_IMPL(test_utf8_utf16_surrogate) {
|
||||
/*
|
||||
* Right before the surrogate range.
|
||||
*/
|
||||
valid("a 3-bytes encoded UTF-8 character \xED\x90\x80 right before the "
|
||||
"surrogate range");
|
||||
|
||||
/*
|
||||
* 0xED 0xA0 0x80 sequence is UTF-8 encoding of 0xD800.
|
||||
*/
|
||||
invalid("a utf16 \xED\xA0\x80 test");
|
||||
|
||||
/*
|
||||
* Within the surrogate range.
|
||||
*/
|
||||
invalid("a utf16 \xED\xA0\x88 test");
|
||||
|
||||
/*
|
||||
* 0xED 0xBF 0xBF sequence is UTF-8 encoding of 0xDFFF.
|
||||
*/
|
||||
invalid("a utf16 \xED\xBF\xBF test");
|
||||
|
||||
/*
|
||||
* Right after the surrogate range.
|
||||
*/
|
||||
valid("a 3-bytes encoded UTF-8 character \xEE\x80\x80 right after the "
|
||||
"surrogate range");
|
||||
}
|
||||
|
||||
ISC_RUN_TEST_IMPL(test_utf8_fourbytes) {
|
||||
/*
|
||||
* 0x010000 (first valid unicode character holding in 4 bytes with
|
||||
* UTF-8) is 0000 00|01 0000 |0000 00|00 0000 which encodes in
|
||||
*
|
||||
* 1111 0___ 10__ ____ 10__ ____ 10__ ____
|
||||
* 1111 0000 1001 0000 1000 0000 1000 0000
|
||||
*
|
||||
* So, 0xF0908080.
|
||||
*
|
||||
* 0x10FFFF (last valid unicode character holding in 4 bytes with UTF-8
|
||||
* is 0001 0000 1111 |1111 11|11 1111 which encodes in
|
||||
|
||||
* 1111 0___ 10__ ____ 10__ ____ 10__ ____
|
||||
* 1111 0100 1000 1111 1011 1111 1011 1111
|
||||
*
|
||||
* So, 0xF48FBFBF.
|
||||
*/
|
||||
|
||||
invalid("invalid 4-bytes sequence: \xF0\x80\x80\x80 (below min)");
|
||||
valid("valid 4-bytes sequence: \xF0\x90\x80\x80 (min value)");
|
||||
valid("valid 4-bytes sequence: \xF4\x8F\xBF\xBF (max value)");
|
||||
invalid("invalid 4-bytes sequence: \xF4\x9F\xBF\xBF(above max)");
|
||||
invalid("invalid 4-bytes sequence: \xF0\xC1\xC1\xC1 (extra bytes does "
|
||||
"not start with 10__)");
|
||||
}
|
||||
|
||||
ISC_RUN_TEST_IMPL(test_utf8_bom) {
|
||||
validbom("\xEF\xBB\xBF");
|
||||
validbom("\xEF\xBB\xBF ab");
|
||||
validbom("\xEF\xBB\xBF\xF4\x8F\xBF\xBF");
|
||||
invalidbom("\xEF\xBB");
|
||||
invalidbom("\xEF\xBB\xBE");
|
||||
invalidbom("\xEF\xBB\xBE ab");
|
||||
invalidbom("\xEF\xBB\xBE\xF4\x8F\xBF\xBF");
|
||||
}
|
||||
|
||||
ISC_TEST_LIST_START
|
||||
ISC_TEST_ENTRY(test_utf8_ascii)
|
||||
ISC_TEST_ENTRY(test_utf8_twobytes)
|
||||
ISC_TEST_ENTRY(test_utf8_threebytes)
|
||||
ISC_TEST_ENTRY(test_utf8_utf16_surrogate)
|
||||
ISC_TEST_ENTRY(test_utf8_fourbytes)
|
||||
ISC_TEST_ENTRY(test_utf8_bom)
|
||||
ISC_TEST_LIST_END
|
||||
|
||||
ISC_TEST_MAIN
|
||||
Loading…
Reference in a new issue