From 6ea5f9b0608ca98863fe1164d65eca5243c7dd0e Mon Sep 17 00:00:00 2001 From: Baptiste Daroussin Date: Sat, 1 Oct 2016 00:11:09 +0000 Subject: [PATCH] Update libucl to snapshot 20160812 --- CMakeLists.txt | 3 +- Makefile.unix | 9 +- Makefile.w32 | 5 +- README.md | 30 ++-- haskell/hucl.hs | 123 ++++++++++++++ include/ucl++.h | 238 ++++++++++++++++++++++++---- include/ucl.h | 16 +- m4/gcov.m4 | 89 +++++++++++ python/setup.py | 54 ++++--- python/src/uclmodule.c | 107 ++++++++++--- python/test.sh | 6 - python/test_uclmodule.py | 148 ----------------- python/tests/__init__.py | 0 python/tests/compat.py | 8 + python/tests/test_dump.py | 66 ++++++++ python/tests/test_load.py | 107 +++++++++++++ python/tests/test_validation.py | 50 ++++++ src/mum.h | 2 +- src/ucl_parser.c | 14 +- src/ucl_schema.c | 30 ++-- src/ucl_util.c | 25 ++- tests/schema/patternProperties.json | 5 + tests/schema/refRemote.json | 2 +- 23 files changed, 852 insertions(+), 285 deletions(-) create mode 100644 haskell/hucl.hs create mode 100644 m4/gcov.m4 delete mode 100755 python/test.sh delete mode 100755 python/test_uclmodule.py create mode 100644 python/tests/__init__.py create mode 100644 python/tests/compat.py create mode 100644 python/tests/test_dump.py create mode 100644 python/tests/test_load.py create mode 100644 python/tests/test_validation.py diff --git a/CMakeLists.txt b/CMakeLists.txt index ae28a8908b3..7b55faf8243 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,8 +205,7 @@ SET(UCLSRC src/ucl_util.c src/ucl_hash.c src/ucl_schema.c src/ucl_msgpack.c - src/ucl_sexp.c - src/xxhash.c) + src/ucl_sexp.c) SET (LIB_TYPE STATIC) diff --git a/Makefile.unix b/Makefile.unix index 121403d0d46..0653d4843f7 100644 --- a/Makefile.unix +++ b/Makefile.unix @@ -1,7 +1,7 @@ CC ?= gcc DESTDIR ?= /usr/local LD ?= gcc -C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src +C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src -I./klib MAJOR_VERSION = 0 MINOR_VERSION = 2 PATCH_VERSION = 9 @@ -25,13 +25,12 @@ HDEPS = $(SRCDIR)/ucl_hash.h \ $(SRCDIR)/ucl_chartable.h \ $(SRCDIR)/ucl_internal.h \ $(INCLUDEDIR)/ucl.h \ - $(SRCDIR)/xxhash.h + $(SRCDIR)/mum.h OBJECTS = $(OBJDIR)/ucl_hash.o \ $(OBJDIR)/ucl_util.o \ $(OBJDIR)/ucl_parser.o \ $(OBJDIR)/ucl_emitter.o \ - $(OBJDIR)/ucl_schema.o \ - $(OBJDIR)/xxhash.o + $(OBJDIR)/ucl_schema.o all: $(OBJDIR) $(OBJDIR)/$(SONAME) @@ -55,8 +54,6 @@ $(OBJDIR)/ucl_hash.o: $(SRCDIR)/ucl_hash.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_hash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_hash.c $(OBJDIR)/ucl_schema.o: $(SRCDIR)/ucl_schema.c $(HDEPS) $(CC) -o $(OBJDIR)/ucl_schema.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_schema.c -$(OBJDIR)/xxhash.o: $(SRCDIR)/xxhash.c $(HDEPS) - $(CC) -o $(OBJDIR)/xxhash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/xxhash.c clean: $(RM) $(OBJDIR)/*.o $(OBJDIR)/$(SONAME_FULL) $(OBJDIR)/$(SONAME) $(OBJDIR)/chargen $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/objdump $(OBJDIR)/test_generate $(OBJDIR)/test_schema || true diff --git a/Makefile.w32 b/Makefile.w32 index 0e61274ff59..5d9398bf198 100644 --- a/Makefile.w32 +++ b/Makefile.w32 @@ -28,14 +28,13 @@ HDEPS = $(SRCDIR)/ucl_hash.h \ $(SRCDIR)/ucl_chartable.h \ $(SRCDIR)/ucl_internal.h \ $(INCLUDEDIR)/ucl.h \ - $(SRCDIR)/xxhash.h + $(SRCDIR)/mum.h OBJECTS = $(OBJDIR)/ucl_hash.o \ $(OBJDIR)/ucl_util.o \ $(OBJDIR)/ucl_parser.o \ $(OBJDIR)/ucl_emitter.o \ $(OBJDIR)/ucl_emitter_utils.o \ - $(OBJDIR)/ucl_schema.o \ - $(OBJDIR)/xxhash.o + $(OBJDIR)/ucl_schema.o all: $(OBJDIR) $(OBJDIR)/$(SONAME) diff --git a/README.md b/README.md index 0d76ca4a5e2..453a7ad66cd 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - [Named keys hierarchy](#named-keys-hierarchy) - [Convenient numbers and booleans](#convenient-numbers-and-booleans) - [General improvements](#general-improvements) - - [Commments](#commments) + - [Comments](#comments) - [Macros support](#macros-support) - [Variables support](#variables-support) - [Multiline strings](#multiline-strings) @@ -21,7 +21,7 @@ - [Performance](#performance) - [Conclusion](#conclusion) -## Introduction +## Introduction This document describes the main features and principles of the configuration language called `UCL` - universal configuration language. @@ -47,7 +47,7 @@ section { string = "something"; subsection { host = { - host = "hostname"; + host = "hostname"; port = 900; } host = { @@ -163,9 +163,9 @@ section { } } ``` - + Plain definitions may be more complex and contain more than a single level of nested objects: - + ```nginx section "blah" "foo" { key = value; @@ -174,7 +174,7 @@ section "blah" "foo" { is presented as: -```nginx +```nginx section { blah { foo { @@ -196,17 +196,17 @@ section { ## General improvements -### Commments +### Comments UCL supports different style of comments: -* single line: `#` +* single line: `#` * multiline: `/* ... */` Multiline comments may be nested: ```c # Sample single line comment -/* +/* some comment /* nested comment */ end of comment @@ -263,7 +263,7 @@ all files that matches the specified pattern (normally the format of patterns is for your operating system). This option is meaningless for URL includes. * `url` (default: **true**) - allow URL includes. * `path` (default: empty) - A UCL_ARRAY of directories to search for the include file. -Search ends after the first patch, unless `glob` is true, then all matches are included. +Search ends after the first match, unless `glob` is true, then all matches are included. * `prefix` (default false) - Put included contents inside an object, instead of loading them into the root. If no `key` is provided, one is automatically generated based on each files basename() * `key` (default: ) - Key to load contents of include into. If @@ -273,7 +273,7 @@ object or an array. * `priority` (default: 0) - specify priority for the include (see below). * `duplicate` (default: 'append') - specify policy of duplicates resolving: - `append` - default strategy, if we have new object of higher priority then it replaces old one, if we have new object with less priority it is ignored completely, and if we have two duplicate objects with the same priority then we have a multi-value key (implicit array) - - `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardeless of priorities) + - `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardless of priorities) - `error` - create error on duplicate keys and stop parsing - `rewrite` - always rewrite an old value with new one (ignoring priorities) @@ -320,7 +320,7 @@ Here are some rules for this syntax: * Multiline terminator must start just after `<<` symbols and it must consist of capital letters only (e.g. `< ParserHandle +foreign import ccall "ucl_parser_add_string" ucl_parser_add_string :: ParserHandle -> CString -> CUInt -> IO Bool +foreign import ccall "ucl_parser_add_file" ucl_parser_add_file :: ParserHandle -> CString -> IO Bool +foreign import ccall "ucl_parser_get_object" ucl_parser_get_object :: ParserHandle -> UCLObjectHandle +foreign import ccall "ucl_parser_get_error" ucl_parser_get_error :: ParserHandle -> CString + +foreign import ccall "ucl_object_iterate_new" ucl_object_iterate_new :: UCLObjectHandle -> UCLIterHandle +foreign import ccall "ucl_object_iterate_safe" ucl_object_iterate_safe :: UCLIterHandle -> Bool -> UCLObjectHandle +foreign import ccall "ucl_object_type" ucl_object_type :: UCLObjectHandle -> CUInt +foreign import ccall "ucl_object_key" ucl_object_key :: UCLObjectHandle -> CString +foreign import ccall "ucl_object_toint" ucl_object_toint :: UCLObjectHandle -> CInt +foreign import ccall "ucl_object_todouble" ucl_object_todouble :: UCLObjectHandle -> CDouble +foreign import ccall "ucl_object_tostring" ucl_object_tostring :: UCLObjectHandle -> CString +foreign import ccall "ucl_object_toboolean" ucl_object_toboolean :: UCLObjectHandle -> Bool + +foreign import ccall "ucl_object_emit" ucl_object_emit :: UCLObjectHandle -> UCLEmitterType -> CString +foreign import ccall "ucl_object_emit_len" ucl_object_emit_len :: UCLObjectHandle -> UCLEmitterType -> Ptr CSize -> IO CString + +type UCL_TYPE = CUInt +ucl_OBJECT :: UCL_TYPE +ucl_OBJECT = 0 +ucl_ARRAY :: UCL_TYPE +ucl_ARRAY = 1 +ucl_INT :: UCL_TYPE +ucl_INT = 2 +ucl_FLOAT :: UCL_TYPE +ucl_FLOAT = 3 +ucl_STRING :: UCL_TYPE +ucl_STRING = 4 +ucl_BOOLEAN :: UCL_TYPE +ucl_BOOLEAN = 5 +ucl_TIME :: UCL_TYPE +ucl_TIME = 6 +ucl_USERDATA :: UCL_TYPE +ucl_USERDATA = 7 +ucl_NULL :: UCL_TYPE +ucl_NULL = 8 + +ucl_emit_json :: UCLEmitterType +ucl_emit_json = 0 +ucl_emit_json_compact :: UCLEmitterType +ucl_emit_json_compact = 1 :: UCLEmitterType +ucl_emit_msgpack :: UCLEmitterType +ucl_emit_msgpack = 4 :: UCLEmitterType + +ucl_parser_parse_string_pure :: String -> Either UCLObjectHandle ErrorString +ucl_parser_parse_string_pure s = unsafePerformIO $ do + cs <- newCString s + let p = ucl_parser_new 0x4 + didParse <- ucl_parser_add_string p cs (toEnum $ length s) + if didParse + then return $ Left $ ucl_parser_get_object p + else Right <$> peekCString ( ucl_parser_get_error p) + +ucl_parser_add_file_pure :: String -> Either UCLObjectHandle ErrorString +ucl_parser_add_file_pure s = unsafePerformIO $ do + cs <- newCString s + let p = ucl_parser_new 0x4 + didParse <- ucl_parser_add_file p cs + if didParse + then return $ Left $ ucl_parser_get_object p + else Right <$> peekCString ( ucl_parser_get_error p) + +unpack :: MSG.MessagePack a => String -> Either a ErrorString +unpack s = case ucl_parser_parse_string_pure s of + (Right err) -> Right err + (Left obj) -> case MSG.fromObject (ucl_to_msgpack_object obj) of + Nothing -> Right "MessagePack fromObject Error" + (Just a) -> Left a + +ucl_to_msgpack_object :: UCLObjectHandle -> MSG.Object +ucl_to_msgpack_object o = toMsgPackObj (ucl_object_type o) o + where + toMsgPackObj n obj + |n==ucl_OBJECT = MSG.ObjectMap $ uclObjectToVector obj + |n==ucl_ARRAY = MSG.ObjectArray undefined + |n==ucl_INT = MSG.ObjectInt $ fromEnum $ ucl_object_toint obj + |n==ucl_FLOAT = MSG.ObjectDouble $ realToFrac $ ucl_object_todouble obj + |n==ucl_STRING = MSG.ObjectStr $ T.pack $ unsafePerformIO $ peekCString $ ucl_object_tostring obj + |n==ucl_BOOLEAN = MSG.ObjectBool $ ucl_object_toboolean obj + |n==ucl_TIME = error "time undefined" + |n==ucl_USERDATA = error "userdata undefined" + |n==ucl_NULL = error "null undefined" + |otherwise = error "\"Unknown Type\" Error" + +uclObjectToVector :: UCLObjectHandle -> V.Vector (MSG.Object,MSG.Object) +uclObjectToVector o = iterateObject (ucl_object_iterate_safe iter True ) iter V.empty + where + iter = ucl_object_iterate_new o + iterateObject obj it vec = if ucl_object_type obj == ucl_NULL + then vec + else iterateObject (ucl_object_iterate_safe it True) it (V.snoc vec ( getUclKey obj , ucl_to_msgpack_object obj)) + getUclKey obj = MSG.ObjectStr $ T.pack $ unsafePerformIO $ peekCString $ ucl_object_key obj + +uclArrayToVector :: UCLObjectHandle -> V.Vector MSG.Object +uclArrayToVector o = iterateArray (ucl_object_iterate_safe iter True ) iter V.empty + where + iter = ucl_object_iterate_new o + iterateArray obj it vec = if ucl_object_type obj == ucl_NULL + then vec + else iterateArray (ucl_object_iterate_safe it True) it (V.snoc vec (ucl_to_msgpack_object obj)) + diff --git a/include/ucl++.h b/include/ucl++.h index 00297e7069a..2c2bdde5155 100644 --- a/include/ucl++.h +++ b/include/ucl++.h @@ -24,6 +24,9 @@ #pragma once #include +#include +#include +#include #include #include @@ -100,6 +103,68 @@ private: return func; }; + static bool ucl_variable_getter(const unsigned char *data, size_t len, + unsigned char ** /*replace*/, size_t * /*replace_len*/, bool *need_free, void* ud) + { + *need_free = false; + + auto vars = reinterpret_cast *>(ud); + if (vars && data && len != 0) { + vars->emplace (data, data + len); + } + return false; + } + + static bool ucl_variable_replacer (const unsigned char *data, size_t len, + unsigned char **replace, size_t *replace_len, bool *need_free, void* ud) + { + *need_free = false; + + auto replacer = reinterpret_cast(ud); + if (!replacer) { + return false; + } + + std::string var_name (data, data + len); + if (!replacer->is_variable (var_name)) { + return false; + } + + std::string var_value = replacer->replace (var_name); + if (var_value.empty ()) { + return false; + } + + *replace = (unsigned char *)UCL_ALLOC (var_value.size ()); + memcpy (*replace, var_value.data (), var_value.size ()); + + *replace_len = var_value.size (); + *need_free = true; + + return true; + } + + template + static Ucl parse_with_strategy_function (C config_func, P parse_func, std::string &err) + { + auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + + config_func (parser); + + if (!parse_func (parser)) { + err.assign (ucl_parser_get_error (parser)); + ucl_parser_free (parser); + + return nullptr; + } + + auto obj = ucl_parser_get_object (parser); + ucl_parser_free (parser); + + // Obj will handle ownership + return Ucl (obj); + } + std::unique_ptr obj; public: @@ -117,9 +182,9 @@ public: const_iterator(const Ucl &obj) { it = std::shared_ptr(ucl_object_iterate_new (obj.obj.get()), - ucl_iter_deleter()); + ucl_iter_deleter()); cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); - if (!*cur) { + if (cur->type() == UCL_NULL) { it.reset (); cur.reset (); } @@ -153,7 +218,7 @@ public: cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); } - if (!*cur) { + if (cur && cur->type() == UCL_NULL) { it.reset (); cur.reset (); } @@ -171,6 +236,17 @@ public: } }; + struct variable_replacer { + virtual ~variable_replacer() {} + + virtual bool is_variable (const std::string &str) const + { + return !str.empty (); + } + + virtual std::string replace (const std::string &var) const = 0; + }; + // We grab ownership if get non-const ucl_object_t Ucl(ucl_object_t *other) { obj.reset (other); @@ -211,20 +287,20 @@ public: obj.reset (ucl_object_fromstring_common (value.data (), value.size (), UCL_STRING_RAW)); } - Ucl(const char * value) { + Ucl(const char *value) { obj.reset (ucl_object_fromstring_common (value, 0, UCL_STRING_RAW)); } // Implicit constructor: anything with a to_json() function. template - Ucl(const T & t) : Ucl(t.to_ucl()) {} + Ucl(const T &t) : Ucl(t.to_ucl()) {} // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) template ::value && std::is_constructible::value, int>::type = 0> - Ucl(const M & m) { + Ucl(const M &m) { obj.reset (ucl_object_typed_new (UCL_OBJECT)); auto cobj = obj.get (); @@ -238,7 +314,7 @@ public: template ::value, int>::type = 0> - Ucl(const V & v) { + Ucl(const V &v) { obj.reset (ucl_object_typed_new (UCL_ARRAY)); auto cobj = obj.get (); @@ -356,46 +432,138 @@ public: return out; } - static Ucl parse (const std::string & in, std::string & err) + static Ucl parse (const std::string &in, std::string &err) { - auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); - - if (!ucl_parser_add_chunk (parser, (const unsigned char *)in.data (), - in.size ())) { - err.assign (ucl_parser_get_error (parser)); - ucl_parser_free (parser); - - return nullptr; - } - - auto obj = ucl_parser_get_object (parser); - ucl_parser_free (parser); - - // Obj will handle ownership - return Ucl (obj); + return parse (in, std::map(), err); } - static Ucl parse (const char * in, std::string & err) + static Ucl parse (const std::string &in, const std::map &vars, std::string &err) { - if (in) { - return parse (std::string(in), err); - } else { + auto config_func = [&vars] (ucl_parser *parser) { + for (const auto & item : vars) { + ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ()); + } + }; + + auto parse_func = [&in] (ucl_parser *parser) { + return ucl_parser_add_chunk (parser, (unsigned char *)in.data (), in.size ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static Ucl parse (const std::string &in, const variable_replacer &replacer, std::string &err) + { + auto config_func = [&replacer] (ucl_parser *parser) { + ucl_parser_set_variables_handler (parser, ucl_variable_replacer, + &const_cast(replacer)); + }; + + auto parse_func = [&in] (ucl_parser *parser) { + return ucl_parser_add_chunk (parser, (unsigned char *) in.data (), in.size ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static Ucl parse (const char *in, std::string &err) + { + return parse (in, std::map(), err); + } + + static Ucl parse (const char *in, const std::map &vars, std::string &err) + { + if (!in) { err = "null input"; return nullptr; } + return parse (std::string (in), vars, err); } - static Ucl parse (std::istream &ifs, std::string &err) + static Ucl parse (const char *in, const variable_replacer &replacer, std::string &err) { - return Ucl::parse (std::string(std::istreambuf_iterator(ifs), - std::istreambuf_iterator()), err); + if (!in) { + err = "null input"; + return nullptr; + } + return parse (std::string(in), replacer, err); } - Ucl& operator= (Ucl rhs) - { - obj.swap (rhs.obj); - return *this; - } + static Ucl parse_from_file (const std::string &filename, std::string &err) + { + return parse_from_file (filename, std::map(), err); + } + + static Ucl parse_from_file (const std::string &filename, const std::map &vars, std::string &err) + { + auto config_func = [&vars] (ucl_parser *parser) { + for (const auto & item : vars) { + ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ()); + } + }; + + auto parse_func = [&filename] (ucl_parser *parser) { + return ucl_parser_add_file (parser, filename.c_str ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static Ucl parse_from_file (const std::string &filename, const variable_replacer &replacer, std::string &err) + { + auto config_func = [&replacer] (ucl_parser *parser) { + ucl_parser_set_variables_handler (parser, ucl_variable_replacer, + &const_cast(replacer)); + }; + + auto parse_func = [&filename] (ucl_parser *parser) { + return ucl_parser_add_file (parser, filename.c_str ()); + }; + + return parse_with_strategy_function (config_func, parse_func, err); + } + + static std::vector find_variable (const std::string &in) + { + auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + + std::set vars; + ucl_parser_set_variables_handler (parser, ucl_variable_getter, &vars); + ucl_parser_add_chunk (parser, (const unsigned char *)in.data (), in.size ()); + ucl_parser_free (parser); + + std::vector result; + std::move (vars.begin (), vars.end (), std::back_inserter (result)); + return result; + } + + static std::vector find_variable (const char *in) + { + if (!in) { + return std::vector(); + } + return find_variable (std::string (in)); + } + + static std::vector find_variable_from_file (const std::string &filename) + { + auto parser = ucl_parser_new (UCL_PARSER_DEFAULT); + + std::set vars; + ucl_parser_set_variables_handler (parser, ucl_variable_getter, &vars); + ucl_parser_add_file (parser, filename.c_str ()); + ucl_parser_free (parser); + + std::vector result; + std::move (vars.begin (), vars.end (), std::back_inserter (result)); + return std::move (result); + } + + Ucl& operator= (Ucl rhs) + { + obj.swap (rhs.obj); + return *this; + } bool operator== (const Ucl &rhs) const { diff --git a/include/ucl.h b/include/ucl.h index 024f5dd8fd3..304d3291c2e 100644 --- a/include/ucl.h +++ b/include/ucl.h @@ -1016,7 +1016,6 @@ UCL_EXTERN bool ucl_parser_add_string_priority (struct ucl_parser *parser, * Load and add data from a file * @param parser parser structure * @param filename the name of file - * @param err if *err is NULL it is set to parser error * @return true if chunk has been added and false in case of error */ UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser, @@ -1026,7 +1025,6 @@ UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser, * Load and add data from a file * @param parser parser structure * @param filename the name of file - * @param err if *err is NULL it is set to parser error * @param priority the desired priority of a chunk (only 4 least significant bits * are considered for this parameter) * @return true if chunk has been added and false in case of error @@ -1034,6 +1032,20 @@ UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser, UCL_EXTERN bool ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, unsigned priority); +/** + * Load and add data from a file + * @param parser parser structure + * @param filename the name of file + * @param priority the desired priority of a chunk (only 4 least significant bits + * are considered for this parameter) + * @param strat Merge strategy to use while parsing this file + * @param parse_type Parser type to use while parsing this file + * @return true if chunk has been added and false in case of error + */ +UCL_EXTERN bool ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename, + unsigned priority, enum ucl_duplicate_strategy strat, + enum ucl_parse_type parse_type); + /** * Load and add data from a file descriptor * @param parser parser structure diff --git a/m4/gcov.m4 b/m4/gcov.m4 new file mode 100644 index 00000000000..a1359a0de64 --- /dev/null +++ b/m4/gcov.m4 @@ -0,0 +1,89 @@ +# SYNOPSIS +# +# Add code coverage support with gcov/lcov. +# +# AX_CODE_COVERAGE() +# +# DESCRIPTION +# +# Provides a --enable-coverage option which checks for available +# gcov/lcov binaries and provides ENABLE_CODE_COVERAGE conditional. +# +# LAST MODIFICATION +# +# $Id: coverage.m4 40881 2013-08-20 17:54:39Z damon $ +# +# COPYLEFT +# +# Copyright (c) 2012 Roy H. Stogner +# Copyright (c) 2010 Karl W. Schulz +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +AC_DEFUN([AX_CODE_COVERAGE], +[ + +AC_ARG_ENABLE(coverage, AC_HELP_STRING([--enable-coverage],[configure code coverage analysis tools])) + +HAVE_GCOV_TOOLS=0 + +GCOV_FLAGS="" + +if test "x$enable_coverage" = "xyes"; then + + # ---------------------------- + # Check for gcov/lcov binaries + # ---------------------------- + + AC_ARG_VAR([GCOV], [Coverage testing command]) + if test "x$GCOV" = "x"; then + AC_PATH_PROG(GCOV, gcov, no) + else + AC_PATH_PROG(GCOV, $GCOV, no) + fi + + AC_PATH_PROG(LCOV, lcov, no) + AC_PATH_PROG(GENHTML, genhtml) + + # ---------------------------------- + # include coverage compiler options + # ---------------------------------- + AC_MSG_CHECKING([for clang]) + + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([], [[ + #ifndef __clang__ + not clang + #endif + ]])], + [CLANG=yes], [CLANG=no]) + + AC_MSG_RESULT([$CLANG]) + HAVE_GCOV_TOOLS=1 + COVERAGE_CFLAGS="-fprofile-arcs -ftest-coverage" + COVERAGE_LDFLAGS="--coverage -fprofile-arcs -ftest-coverage" + COVERAGE_OPTFLAGS="-O0" + + # Test for C... + CFLAGS="${GCOV_FLAGS} ${CFLAGS}" + CXXFLAGS="${GCOV_FLAGS} ${CXXFLAGS}" + if test "x$GCC" = "xyes" -a "x$CLANG" = "xno"; then + COVERAGE_LIBS="-lgcov" + else + COVERAGE_LIBS="" + fi +fi + +AC_SUBST([GCOV]) +AC_SUBST([LCOV]) +AC_SUBST([GENHTML]) +AC_SUBST([GENHTML_OPTIONS]) +AC_SUBST([COVERAGE_CFLAGS]) +AC_SUBST([COVERAGE_OPTFLAGS]) +AC_SUBST([COVERAGE_LDFLAGS]) +AC_SUBST([COVERAGE_LIBS]) +AM_CONDITIONAL(CODE_COVERAGE_ENABLED,test x$HAVE_GCOV_TOOLS = x1) + +]) diff --git a/python/setup.py b/python/setup.py index b2b8981705f..9e1151a1323 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,37 +1,43 @@ -import distutils.ccompiler -import distutils.sysconfig -from distutils.core import setup, Extension +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension + import os +import sys +tests_require = [] -compiler = distutils.ccompiler.new_compiler() -search_paths=[os.path.expanduser('~/{}'), '/opt/local/{}', '/usr/local/{}', '/usr/{}'] -lib_paths = [ a.format("lib") for a in search_paths] -inc_paths = [ a.format("include") for a in search_paths] +if sys.version < '2.7': + tests_require.append('unittest2') -uclmodule = Extension('ucl', - include_dirs = inc_paths, - library_dirs = lib_paths, - libraries = ['ucl'], - sources = ['src/uclmodule.c'], - runtime_library_dirs = lib_paths, - language='c') +uclmodule = Extension( + 'ucl', + libraries = ['ucl'], + sources = ['src/uclmodule.c'], + language = 'c' +) -setup(name='ucl', - version='1.0', - description='ucl parser and emmitter', +setup( + name = 'ucl', + version = '0.8', + description = 'ucl parser and emmitter', ext_modules = [uclmodule], - author="Eitan Adler", - author_email="lists@eitanadler.com", - url="https://github.com/vstakhov/libucl/", - license="MIT", - classifiers=["Development Status :: 3 - Alpha", + test_suite = 'tests', + tests_require = tests_require, + author = "Eitan Adler, Denis Volpato Martins", + author_email = "lists@eitanadler.com", + url = "https://github.com/vstakhov/libucl/", + license = "MIT", + classifiers = [ + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: DFSG approved", "License :: OSI Approved :: MIT License", "Programming Language :: C", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Libraries", - ] - ) + ] +) diff --git a/python/src/uclmodule.c b/python/src/uclmodule.c index bc13c704b9c..fce0dab14a4 100644 --- a/python/src/uclmodule.c +++ b/python/src/uclmodule.c @@ -2,6 +2,8 @@ #include #include +static PyObject *SchemaError; + static PyObject * _basic_ucl_type (ucl_object_t const *obj) { @@ -13,9 +15,11 @@ _basic_ucl_type (ucl_object_t const *obj) case UCL_STRING: return Py_BuildValue ("s", ucl_object_tostring (obj)); case UCL_BOOLEAN: - return ucl_object_toboolean (obj) ? Py_True : Py_False; + return PyBool_FromLong (ucl_object_toboolean (obj)); case UCL_TIME: return Py_BuildValue ("d", ucl_object_todouble (obj)); + case UCL_NULL: + Py_RETURN_NONE; } return NULL; } @@ -124,26 +128,60 @@ _iterate_python (PyObject *obj) { if (obj == Py_None) { return ucl_object_new(); - } else if (PyBool_Check (obj)) { - return ucl_object_frombool (obj == Py_True); - } else if (PyInt_Check (obj)) { - return ucl_object_fromint (PyInt_AsLong (obj)); - } else if (PyFloat_Check (obj)) { - return ucl_object_fromdouble (PyFloat_AsDouble (obj)); - } else if (PyString_Check (obj)) { - return ucl_object_fromstring (PyString_AsString (obj)); - // } else if (PyDateTime_Check (obj)) { } + else if (PyBool_Check (obj)) { + return ucl_object_frombool (obj == Py_True); + } +#if PY_MAJOR_VERSION < 3 + else if (PyInt_Check (obj)) { + return ucl_object_fromint (PyInt_AsLong (obj)); + } +#endif + else if (PyLong_Check (obj)) { + return ucl_object_fromint (PyLong_AsLong (obj)); + } + else if (PyFloat_Check (obj)) { + return ucl_object_fromdouble (PyFloat_AsDouble (obj)); + } + else if (PyUnicode_Check (obj)) { + ucl_object_t *ucl_str; + PyObject *str = PyUnicode_AsASCIIString(obj); + ucl_str = ucl_object_fromstring (PyBytes_AsString (str)); + Py_DECREF(str); + return ucl_str; + } +#if PY_MAJOR_VERSION < 3 + else if (PyString_Check (obj)) { + return ucl_object_fromstring (PyString_AsString (obj)); + } +#endif else if (PyDict_Check(obj)) { PyObject *key, *value; Py_ssize_t pos = 0; ucl_object_t *top, *elm; + char *keystr = NULL; top = ucl_object_typed_new (UCL_OBJECT); while (PyDict_Next(obj, &pos, &key, &value)) { elm = _iterate_python(value); - ucl_object_insert_key (top, elm, PyString_AsString(key), 0, true); + + if (PyUnicode_Check(key)) { + PyObject *keyascii = PyUnicode_AsASCIIString(key); + keystr = PyBytes_AsString(keyascii); + Py_DECREF(keyascii); + } +#if PY_MAJOR_VERSION < 3 + else if (PyString_Check(key)) { + keystr = PyString_AsString(key); + } +#endif + else { + PyErr_SetString(PyExc_TypeError, "Unknown key type"); + return NULL; + } + + ucl_object_insert_key (top, elm, keystr, 0, true); } return top; @@ -195,11 +233,6 @@ ucl_dump (PyObject *self, PyObject *args) Py_RETURN_NONE; } - if (!PyDict_Check(obj)) { - PyErr_SetString(PyExc_TypeError, "Argument must be dict"); - return NULL; - } - root = _iterate_python(obj); if (root) { PyObject *ret; @@ -207,7 +240,11 @@ ucl_dump (PyObject *self, PyObject *args) buf = (char *) ucl_object_emit (root, emitter); ucl_object_unref (root); +#if PY_MAJOR_VERSION < 3 ret = PyString_FromString (buf); +#else + ret = PyUnicode_FromString (buf); +#endif free(buf); return ret; @@ -219,17 +256,35 @@ ucl_dump (PyObject *self, PyObject *args) static PyObject * ucl_validate (PyObject *self, PyObject *args) { - char *uclstr, *schema; + PyObject *dataobj, *schemaobj; + ucl_object_t *data, *schema; + bool r; + struct ucl_schema_error err; - if (PyArg_ParseTuple(args, "zz", &uclstr, &schema)) { - if (!uclstr || !schema) { - Py_RETURN_NONE; - } - - PyErr_SetString(PyExc_NotImplementedError, "schema validation is not yet supported"); + if (!PyArg_ParseTuple (args, "OO", &schemaobj, &dataobj)) { + PyErr_SetString (PyExc_TypeError, "Unhandled object type"); + return NULL; } - return NULL; + schema = _iterate_python(schemaobj); + if (!schema) + return NULL; + + data = _iterate_python(dataobj); + if (!data) + return NULL; + + // validation + r = ucl_object_validate (schema, data, &err); + ucl_object_unref (schema); + ucl_object_unref (data); + + if (!r) { + PyErr_SetString (SchemaError, err.msg); + return NULL; + } + + Py_RETURN_TRUE; } static PyMethodDef uclMethods[] = { @@ -247,6 +302,10 @@ init_macros(PyObject *mod) PyModule_AddIntMacro(mod, UCL_EMIT_CONFIG); PyModule_AddIntMacro(mod, UCL_EMIT_YAML); PyModule_AddIntMacro(mod, UCL_EMIT_MSGPACK); + + SchemaError = PyErr_NewException("ucl.SchemaError", NULL, NULL); + Py_INCREF(SchemaError); + PyModule_AddObject(mod, "SchemaError", SchemaError); } #if PY_MAJOR_VERSION >= 3 diff --git a/python/test.sh b/python/test.sh deleted file mode 100755 index 53af6a3fd37..00000000000 --- a/python/test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -set -xe -python3.4 setup.py build_ext --inplace -./test_uclmodule.py -v -rm -rfv build -rm ucl.so diff --git a/python/test_uclmodule.py b/python/test_uclmodule.py deleted file mode 100755 index 0d774696aa4..00000000000 --- a/python/test_uclmodule.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python -import json -import unittest -import ucl -import sys - -if sys.version_info[:2] == (2, 7): - unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - - -class TestUcl(unittest.TestCase): - def test_no_args(self): - with self.assertRaises(TypeError): - ucl.load() - - def test_multi_args(self): - with self.assertRaises(TypeError): - ucl.load(0,0) - - def test_none(self): - r = ucl.load(None) - self.assertEqual(r, None) - - def test_int(self): - r = ucl.load("a : 1") - self.assertEqual(ucl.load("a : 1"), { "a" : 1 } ) - - def test_braced_int(self): - self.assertEqual(ucl.load("{a : 1}"), { "a" : 1 } ) - - def test_nested_int(self): - self.assertEqual(ucl.load("a : { b : 1 }"), { "a" : { "b" : 1 } }) - - def test_str(self): - self.assertEqual(ucl.load("a : b"), {"a" : "b"}) - - def test_float(self): - self.assertEqual(ucl.load("a : 1.1"), {"a" : 1.1}) - - def test_boolean(self): - totest = ( - "a : True;" \ - "b : False" - ) - correct = {"a" : True, "b" : False} - self.assertEqual(ucl.load(totest), correct) - - def test_empty_ucl(self): - r = ucl.load("{}") - self.assertEqual(r, {}) - - def test_single_brace(self): - self.assertEqual(ucl.load("{"), {}) - - def test_single_back_brace(self): - ucl.load("}") - - def test_single_square_forward(self): - self.assertEqual(ucl.load("["), []) - - def test_invalid_ucl(self): - with self.assertRaisesRegex(ValueError, "unfinished key$"): - ucl.load('{ "var"') - - def test_comment_ignored(self): - self.assertEqual(ucl.load("{/*1*/}"), {}) - - def test_1_in(self): - with open("../tests/basic/1.in", "r") as in1: - self.assertEqual(ucl.load(in1.read()), {'key1': 'value'}) - - def test_every_type(self): - totest="""{ - "key1": value; - "key2": value2; - "key3": "value;" - "key4": 1.0, - "key5": -0xdeadbeef - "key6": 0xdeadbeef.1 - "key7": 0xreadbeef - "key8": -1e-10, - "key9": 1 - "key10": true - "key11": no - "key12": yes - }""" - correct = { - 'key1': 'value', - 'key2': 'value2', - 'key3': 'value;', - 'key4': 1.0, - 'key5': -3735928559, - 'key6': '0xdeadbeef.1', - 'key7': '0xreadbeef', - 'key8': -1e-10, - 'key9': 1, - 'key10': True, - 'key11': False, - 'key12': True, - } - self.assertEqual(ucl.load(totest), correct) - - def test_validation_useless(self): - with self.assertRaises(NotImplementedError): - ucl.validate("","") - - -class TestUclDump(unittest.TestCase): - def test_no_args(self): - with self.assertRaises(TypeError): - ucl.dump() - - def test_multi_args(self): - with self.assertRaises(TypeError): - ucl.dump(0, 0) - - def test_none(self): - self.assertEqual(ucl.dump(None), None) - - def test_int(self): - self.assertEqual(ucl.dump({ "a" : 1 }), "a = 1;\n") - - def test_nested_int(self): - self.assertEqual(ucl.dump({ "a" : { "b" : 1 } }), "a {\n b = 1;\n}\n") - - def test_int_array(self): - self.assertEqual(ucl.dump({ "a" : [1,2,3,4]}), "a [\n 1,\n 2,\n 3,\n 4,\n]\n") - - def test_str(self): - self.assertEqual(ucl.dump({"a" : "b"}), "a = \"b\";\n") - - def test_float(self): - self.assertEqual(ucl.dump({"a" : 1.1}), "a = 1.100000;\n") - - def test_boolean(self): - totest = {"a" : True, "b" : False} - correct = "a = true;\nb = false;\n" - self.assertEqual(ucl.dump(totest), correct) - - def test_empty_ucl(self): - self.assertEqual(ucl.dump({}), "") - - def test_json(self): - self.assertEqual(ucl.dump({ "a" : 1, "b": "bleh;" }, ucl.UCL_EMIT_JSON), '{\n "a": 1,\n "b": "bleh;"\n}') - - -if __name__ == '__main__': - unittest.main() diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/tests/compat.py b/python/tests/compat.py new file mode 100644 index 00000000000..50138262e9b --- /dev/null +++ b/python/tests/compat.py @@ -0,0 +1,8 @@ +try: + import unittest2 as unittest +except ImportError: + import unittest + +# Python 2.7 - 3.1 +if not hasattr(unittest.TestCase, 'assertRaisesRegex'): + unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp diff --git a/python/tests/test_dump.py b/python/tests/test_dump.py new file mode 100644 index 00000000000..36924146850 --- /dev/null +++ b/python/tests/test_dump.py @@ -0,0 +1,66 @@ +from .compat import unittest +import ucl +import sys + +class DumpTest(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + ucl.dump() + + def test_none(self): + self.assertEqual(ucl.dump(None), None) + + def test_null(self): + data = { "a" : None } + valid = "a = null;\n" + self.assertEqual(ucl.dump(data), valid) + + def test_int(self): + data = { "a" : 1 } + valid = "a = 1;\n" + self.assertEqual(ucl.dump(data), valid) + + def test_nested_int(self): + data = { "a" : { "b" : 1 } } + valid = "a {\n b = 1;\n}\n" + self.assertEqual(ucl.dump(data), valid) + + def test_int_array(self): + data = { "a" : [1,2,3,4] } + valid = "a [\n 1,\n 2,\n 3,\n 4,\n]\n" + self.assertEqual(ucl.dump(data), valid) + + def test_str(self): + data = { "a" : "b" } + valid = "a = \"b\";\n" + self.assertEqual(ucl.dump(data), valid) + + @unittest.skipIf(sys.version_info[0] > 2, "Python3 uses unicode only") + def test_unicode(self): + data = { unicode("a") : unicode("b") } + valid = unicode("a = \"b\";\n") + self.assertEqual(ucl.dump(data), valid) + + def test_float(self): + data = { "a" : 1.1 } + valid = "a = 1.100000;\n" + self.assertEqual(ucl.dump(data), valid) + + def test_boolean(self): + data = { "a" : True, "b" : False } + valid = [ + "a = true;\nb = false;\n", + "b = false;\na = true;\n" + ] + self.assertIn(ucl.dump(data), valid) + + def test_empty_ucl(self): + self.assertEqual(ucl.dump({}), "") + + def test_json(self): + data = { "a" : 1, "b": "bleh;" } + valid = [ + '{\n "a": 1,\n "b": "bleh;"\n}', + '{\n "b": "bleh;",\n "a": 1\n}' + ] + self.assertIn(ucl.dump(data, ucl.UCL_EMIT_JSON), valid) diff --git a/python/tests/test_load.py b/python/tests/test_load.py new file mode 100644 index 00000000000..786587a67f3 --- /dev/null +++ b/python/tests/test_load.py @@ -0,0 +1,107 @@ +from .compat import unittest +import ucl + +class LoadTest(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + ucl.load() + + def test_multi_args(self): + with self.assertRaises(TypeError): + ucl.load(0,0) + + def test_none(self): + self.assertEqual(ucl.load(None), None) + + def test_null(self): + data = "a: null" + valid = { "a" : None } + self.assertEqual(ucl.load(data), valid) + + def test_int(self): + data = "a : 1" + valid = { "a" : 1 } + self.assertEqual(ucl.load(data), valid) + + def test_braced_int(self): + data = "{a : 1}" + valid = { "a" : 1 } + self.assertEqual(ucl.load(data), valid) + + def test_nested_int(self): + data = "a : { b : 1 }" + valid = { "a" : { "b" : 1 } } + self.assertEqual(ucl.load(data), valid) + + def test_str(self): + data = "a : b" + valid = { "a" : "b" } + self.assertEqual(ucl.load(data), valid) + + def test_float(self): + data = "a : 1.1" + valid = {"a" : 1.1} + self.assertEqual(ucl.load(data), valid) + + def test_boolean(self): + data = ( + "a : True;" \ + "b : False" + ) + valid = { "a" : True, "b" : False } + self.assertEqual(ucl.load(data), valid) + + def test_empty_ucl(self): + self.assertEqual(ucl.load("{}"), {}) + + def test_single_brace(self): + self.assertEqual(ucl.load("{"), {}) + + def test_single_back_brace(self): + self.assertEqual(ucl.load("}"), {}) + + def test_single_square_forward(self): + self.assertEqual(ucl.load("["), []) + + def test_invalid_ucl(self): + with self.assertRaisesRegex(ValueError, "unfinished key$"): + ucl.load('{ "var"') + + def test_comment_ignored(self): + self.assertEqual(ucl.load("{/*1*/}"), {}) + + def test_1_in(self): + valid = { 'key1': 'value' } + with open("../tests/basic/1.in", "r") as in1: + self.assertEqual(ucl.load(in1.read()), valid) + + def test_every_type(self): + data = ("""{ + "key1": value; + "key2": value2; + "key3": "value;" + "key4": 1.0, + "key5": -0xdeadbeef + "key6": 0xdeadbeef.1 + "key7": 0xreadbeef + "key8": -1e-10, + "key9": 1 + "key10": true + "key11": no + "key12": yes + }""") + valid = { + 'key1': 'value', + 'key2': 'value2', + 'key3': 'value;', + 'key4': 1.0, + 'key5': -3735928559, + 'key6': '0xdeadbeef.1', + 'key7': '0xreadbeef', + 'key8': -1e-10, + 'key9': 1, + 'key10': True, + 'key11': False, + 'key12': True, + } + self.assertEqual(ucl.load(data), valid) diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py new file mode 100644 index 00000000000..f7c853ad69a --- /dev/null +++ b/python/tests/test_validation.py @@ -0,0 +1,50 @@ +from .compat import unittest +import ucl +import json +import os.path +import glob +import re + +TESTS_SCHEMA_FOLDER = '../tests/schema/*.json' + +comment_re = re.compile('\/\*((?!\*\/).)*?\*\/', re.DOTALL | re.MULTILINE) +def json_remove_comments(content): + return comment_re.sub('', content) + +class ValidationTest(unittest.TestCase): + def validate(self, jsonfile): + def perform_test(schema, data, valid, description): + msg = '%s (valid=%r)' % (description, valid) + if valid: + self.assertTrue(ucl.validate(schema, data), msg) + else: + with self.assertRaises(ucl.SchemaError): + ucl.validate(schema, data) + self.fail(msg) # fail() will be called only if SchemaError is not raised + + with open(jsonfile) as f: + try: + # data = json.load(f) + data = json.loads(json_remove_comments(f.read())) + except ValueError as e: + raise self.skipTest('Failed to load JSON: %s' % str(e)) + + for testgroup in data: + for test in testgroup['tests']: + perform_test(testgroup['schema'], test['data'], + test['valid'], test['description']) + + @classmethod + def setupValidationTests(cls): + """Creates each test dynamically from a folder""" + def test_gen(filename): + def run_test(self): + self.validate(filename) + return run_test + + for jsonfile in glob.glob(TESTS_SCHEMA_FOLDER): + testname = os.path.splitext(os.path.basename(jsonfile))[0] + setattr(cls, 'test_%s' % testname, test_gen(jsonfile)) + + +ValidationTest.setupValidationTests() diff --git a/src/mum.h b/src/mum.h index ae6eec16c18..8bc8af8621c 100644 --- a/src/mum.h +++ b/src/mum.h @@ -73,7 +73,7 @@ typedef unsigned __int64 uint64_t; #define _MUM_FRESH_GCC #endif -#if defined(__GNUC__) && !defined(__llvm__) +#if defined(__GNUC__) && !defined(__llvm__) && defined(_MUM_FRESH_GCC) #define _MUM_ATTRIBUTE_UNUSED __attribute__((unused)) #define _MUM_OPTIMIZE(opts) __attribute__((__optimize__ (opts))) #define _MUM_TARGET(opts) __attribute__((__target__ (opts))) diff --git a/src/ucl_parser.c b/src/ucl_parser.c index fc7cea07feb..0f029ea6210 100644 --- a/src/ucl_parser.c +++ b/src/ucl_parser.c @@ -342,6 +342,7 @@ ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t rema /* Call generic handler */ if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free, parser->var_data)) { + *out_len += dstlen; *found = true; if (need_free) { free (dst); @@ -458,11 +459,18 @@ ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr, } if (!found) { if (strict && parser->var_handler != NULL) { - if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free, + size_t var_len = 0; + while (var_len < remain && p[var_len] != '}') + var_len ++; + + if (parser->var_handler (p, var_len, &dst, &dstlen, &need_free, parser->var_data)) { memcpy (d, dst, dstlen); - ret += dstlen; - d += remain; + ret += var_len; + d += dstlen; + if (need_free) { + free (dst); + } found = true; } } diff --git a/src/ucl_schema.c b/src/ucl_schema.c index 4d7c871ff9f..c3aa8a78040 100644 --- a/src/ucl_schema.c +++ b/src/ucl_schema.c @@ -69,7 +69,7 @@ ucl_schema_create_error (struct ucl_schema_error *err, * Check whether we have a pattern specified */ static const ucl_object_t * -ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) +ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern, bool recursive) { const ucl_object_t *res = NULL; #ifdef HAVE_REGEX_H @@ -78,11 +78,16 @@ ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) ucl_object_iter_t iter = NULL; if (regcomp (®, pattern, REG_EXTENDED | REG_NOSUB) == 0) { - while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { - if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { - res = elt; - break; + if (recursive) { + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { + if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { + res = elt; + break; + } } + } else { + if (regexec (®, ucl_object_key (obj), 0, NULL, 0) == 0) + res = obj; } regfree (®); } @@ -205,12 +210,17 @@ ucl_schema_validate_object (const ucl_object_t *schema, } } else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) { + const ucl_object_t *vobj; + ucl_object_iter_t viter; piter = NULL; while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { - found = ucl_schema_test_pattern (obj, ucl_object_key (prop)); - if (found) { - ret = ucl_schema_validate (prop, found, true, err, root, - ext_ref); + viter = NULL; + while (ret && (vobj = ucl_object_iterate (obj, &viter, true)) != NULL) { + found = ucl_schema_test_pattern (vobj, ucl_object_key (prop), false); + if (found) { + ret = ucl_schema_validate (prop, found, true, err, root, + ext_ref); + } } } } @@ -234,7 +244,7 @@ ucl_schema_validate_object (const ucl_object_t *schema, piter = NULL; pat = ucl_object_lookup (schema, "patternProperties"); while ((pelt = ucl_object_iterate (pat, &piter, true)) != NULL) { - found = ucl_schema_test_pattern (obj, ucl_object_key (pelt)); + found = ucl_schema_test_pattern (obj, ucl_object_key (pelt), true); if (found != NULL) { break; } diff --git a/src/ucl_util.c b/src/ucl_util.c index 900658bb6bb..ccc437384ae 100644 --- a/src/ucl_util.c +++ b/src/ucl_util.c @@ -1795,8 +1795,9 @@ ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool n } bool -ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, - unsigned priority) +ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename, + unsigned priority, enum ucl_duplicate_strategy strat, + enum ucl_parse_type parse_type) { unsigned char *buf; size_t len; @@ -1819,7 +1820,8 @@ ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, } parser->cur_file = strdup (realbuf); ucl_parser_set_filevars (parser, realbuf, false); - ret = ucl_parser_add_chunk_priority (parser, buf, len, priority); + ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat, + parse_type); if (len > 0) { ucl_munmap (buf, len); @@ -1828,6 +1830,18 @@ ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, return ret; } +bool +ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename, + unsigned priority) +{ + if (parser == NULL) { + return false; + } + + return ucl_parser_add_file_full(parser, filename, priority, + UCL_DUPLICATE_APPEND, UCL_PARSE_UCL); +} + bool ucl_parser_add_file (struct ucl_parser *parser, const char *filename) { @@ -1835,8 +1849,9 @@ ucl_parser_add_file (struct ucl_parser *parser, const char *filename) return false; } - return ucl_parser_add_file_priority(parser, filename, - parser->default_priority); + return ucl_parser_add_file_full(parser, filename, + parser->default_priority, UCL_DUPLICATE_APPEND, + UCL_PARSE_UCL); } bool diff --git a/tests/schema/patternProperties.json b/tests/schema/patternProperties.json index 18586e5daba..e386b6b4ccc 100644 --- a/tests/schema/patternProperties.json +++ b/tests/schema/patternProperties.json @@ -23,6 +23,11 @@ "data": {"foo": "bar", "fooooo": 2}, "valid": false }, + { + "description": "a single invalid match is invalid", + "data": {"fooooo": 2, "foo": "bar"}, + "valid": false + }, { "description": "multiple invalid matches is invalid", "data": {"foo": "bar", "foooooo" : "baz"}, diff --git a/tests/schema/refRemote.json b/tests/schema/refRemote.json index 0ba9adb94ca..067c666b0ec 100644 --- a/tests/schema/refRemote.json +++ b/tests/schema/refRemote.json @@ -48,7 +48,7 @@ "valid": false } ] - }, + } /* { "description": "change resolution scope",