From c2d9dc5bc662ea3fce1cdef1e7bb8a3acd6a11b6 Mon Sep 17 00:00:00 2001 From: GOB Date: Wed, 11 Jan 2023 03:27:25 +0900 Subject: [PATCH 1/2] Add build option to set each buffer size --- README.md | 18 +++++++++++++++++- platformio.ini | 2 ++ src/gob_json.cpp | 12 ++++++++---- src/gob_json.hpp | 17 +++++++++++------ src/gob_json_element_path.hpp | 26 ++++++++++++++++++++++++-- test/test_element.cpp | 4 ++-- 6 files changed, 64 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index af65bf8..91fbabf 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Log output when internal errors occur and added detection mechanism. ### Added a mechanism to help determine keys and retrieve values if elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif ... OMG! Added a helper for retrive values and a delegation handler for processing per JSON object. -see also ValuesHandler in [test_basic.cpp](test/test_basic.cpp) +see also [test_element.cpp](test/test_element.cpp) ### Unit test support with GoogleTest Even small test cases are useful. @@ -77,6 +77,22 @@ If you are using PlatformIO, you can write the following to download automatical ```ini lib_deps = https://github.com/GOB52/gob_json.git ``` + +## Setting +The build option allows you to set several items. + +|symbol| description|default value| +|---|---|---| +|GOB_JSON_PARSER_BUFFER_MAX_LENGTH| Token buffer size| 256| +|GOB_JSON_PARSER_KEY_MAX_LENGTH| JSON key token buffer size|32| +|GOB_JSON_PARSER_STACK_MAX_DEPTH|Maximum nesting level of JSON object/array|20| + +```ini +build_flags = -D GOB_JSON_PARSER_BUFFER_MAX_LENGTH=384 + -D GOB_JSON_PARSER_KEY_MAX_LENGTH=64 + -D GOB_JSON_PARSER_STACK_MAX_DEPTH=16 +``` + ## Usage If you use it like json-streaming-parser2, rename the headers and types to include. diff --git a/platformio.ini b/platformio.ini index e31fdff..bc11ea7 100644 --- a/platformio.ini +++ b/platformio.ini @@ -9,6 +9,8 @@ default_envs = native_11, native_14, native_17, native_20 [env] build_flags = -DUNIT_TEST -Wall -Wextra -Werror=format -D GOB_JSON_PARSER_BUFFER_MAX_LENGTH=512 + -D GOB_JSON_PARSER_KEY_MAX_LENGTH=64 + -D GOB_JSON_PARSER_STACK_MAX_DEPTH=16 -D GOB_JSON_LOG_LEVEL=5 test_framework = googletest diff --git a/src/gob_json.cpp b/src/gob_json.cpp index 3ad7d34..b38819f 100644 --- a/src/gob_json.cpp +++ b/src/gob_json.cpp @@ -12,6 +12,10 @@ namespace goblib { namespace json { +// Size of array. +template constexpr auto size(const C& c) -> decltype(c.size()) { return c.size(); } +template constexpr auto size(const T(&)[N]) noexcept -> size_t { return N; } + void StreamingParser::reset() { state = State::START_DOCUMENT; @@ -307,7 +311,7 @@ void StreamingParser::endArray() { } void StreamingParser::startKey() { - if(stackPos >= (int)sizeof(stack)) + if(stackPos >= (int)size(stack)) { PARSE_ERROR("stack overflow", curCh, characterCounter, path); return; @@ -555,7 +559,7 @@ void StreamingParser::endNull() { } void StreamingParser::startArray() { - if(stackPos >= (int)sizeof(stack)) + if(stackPos >= (int)size(stack)) { PARSE_ERROR("stack overflow", curCh, characterCounter, path); return; @@ -568,7 +572,7 @@ void StreamingParser::startArray() { } void StreamingParser::startObject() { - if(stackPos >= (int)sizeof(stack)) + if(stackPos >= (int)size(stack)) { PARSE_ERROR("stack overflow", curCh, characterCounter, path); return; @@ -581,7 +585,7 @@ void StreamingParser::startObject() { } void StreamingParser::startString() { - if(stackPos >= (int)sizeof(stack)) + if(stackPos >= (int)size(stack)) { PARSE_ERROR("stack overflow", curCh, characterCounter, path); return; diff --git a/src/gob_json.hpp b/src/gob_json.hpp index 84ff135..df856b3 100644 --- a/src/gob_json.hpp +++ b/src/gob_json.hpp @@ -29,13 +29,18 @@ namespace goblib { */ namespace json { +#ifndef GOB_JSON_STRINGIFY +# define GOB_JSON_STRINGIFY(x) GOB_JSON_STRINGIFY_AGAIN(x) +#endif +#ifndef GOB_JSON_STRINGIFY_AGAIN +# define GOB_JSON_STRINGIFY_AGAIN(x) #x +#endif + #ifndef GOB_JSON_PARSER_BUFFER_MAX_LENGTH -# pragma message "Buffer size as default" +# pragma message "Buffer length as default" # define GOB_JSON_PARSER_BUFFER_MAX_LENGTH (256) #else -# define GOB_JSON_STRINGIFY(x) GOB_JSON_STRINGIFY_AGAIN(x) -# define GOB_JSON_STRINGIFY_AGAIN(x) #x -# pragma message "Buffer size set by build option=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_BUFFER_MAX_LENGTH) +# pragma message "Defined buffer length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_BUFFER_MAX_LENGTH) #endif /*! @@ -136,13 +141,13 @@ class StreamingParser ElementPath path{}; State state{State::START_DOCUMENT}; - Stack stack[20]{}; + Stack stack[GOB_JSON_PARSER_STACK_MAX_DEPTH]{}; int stackPos{0}; bool recursive{false}; bool doEmitWhitespace{false}; - char buffer[GOB_JSON_PARSER_BUFFER_MAX_LENGTH]; + char buffer[GOB_JSON_PARSER_BUFFER_MAX_LENGTH]{}; int bufferPos{0}; char unicodeEscapeBuffer[10]; diff --git a/src/gob_json_element_path.hpp b/src/gob_json_element_path.hpp index 93a80c2..1271ec2 100644 --- a/src/gob_json_element_path.hpp +++ b/src/gob_json_element_path.hpp @@ -11,6 +11,28 @@ namespace goblib { namespace json { +#ifndef GOB_JSON_STRINGIFY +# define GOB_JSON_STRINGIFY(x) GOB_JSON_STRINGIFY_AGAIN(x) +#endif +#ifndef GOB_JSON_STRINGIFY_AGAIN +# define GOB_JSON_STRINGIFY_AGAIN(x) #x +#endif + +#ifndef GOB_JSON_PARSER_KEY_MAX_LENGTH +# pragma message "Key length as default" +# define GOB_JSON_PARSER_KEY_MAX_LENGTH (32) +#else +# pragma message "Defined key length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_KEY_MAX_LENGTH) +#endif + +// For ElementPath::selectors and StreamingParser::stack +#ifndef GOB_JSON_PARSER_STACK_MAX_DEPTH +# pragma message "Stack max depth as default" +# define GOB_JSON_PARSER_STACK_MAX_DEPTH (20) +#else +# pragma message "Defined stack max depth=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_STACK_MAX_DEPTH) +#endif + /*! @class ElementSelector @brief Unified element selector. @@ -36,7 +58,7 @@ class ElementSelector private: int index{-1}; - char key[32]{0,}; + char key[GOB_JSON_PARSER_KEY_MAX_LENGTH]{0,}; friend class ElementPath; friend class StreamingParser; }; @@ -97,7 +119,7 @@ class ElementPath private: int count{0}; ElementSelector* current{nullptr}; - ElementSelector selectors[20]{}; + ElementSelector selectors[GOB_JSON_PARSER_STACK_MAX_DEPTH]{}; friend class StreamingParser; }; diff --git a/test/test_element.cpp b/test/test_element.cpp index c8b98cc..5040e4c 100644 --- a/test/test_element.cpp +++ b/test/test_element.cpp @@ -94,8 +94,8 @@ class TestHandler : public goblib::json::Handler std::vector ba2; int32_t i{}; - int32_t ia0[9]{}; - std::array ia1; + int32_t ia0[3]{}; + std::array ia1; std::vector ia2; float f{}; From dac6b22f4adbf726b3b4be988dc977f07c7193e1 Mon Sep 17 00:00:00 2001 From: GOB Date: Sat, 18 Feb 2023 18:13:33 +0900 Subject: [PATCH 2/2] Fixes processing unicode escape --- README.md | 2 +- library.json | 2 +- library.properties | 2 +- platformio.ini | 2 +- src/gob_json.cpp | 79 ++++++------ src/gob_json.hpp | 8 +- src/gob_json_element_path.hpp | 8 +- src/gob_json_version.hpp | 2 +- src/internal/gob_json_log.hpp | 12 +- test/test_basic.cpp | 218 +++++++++------------------------- test/test_element.cpp | 162 +++++++++++++++++++++++++ 11 files changed, 282 insertions(+), 215 deletions(-) diff --git a/README.md b/README.md index 91fbabf..e0aefb1 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Log output when internal errors occur and added detection mechanism. ### Added a mechanism to help determine keys and retrieve values if elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif ... OMG! Added a helper for retrive values and a delegation handler for processing per JSON object. -see also [test_element.cpp](test/test_element.cpp) +see also [test_element.cpp](test/test_element.cpp), [test_basic.cpp](test/test_basic.cpp) ### Unit test support with GoogleTest Even small test cases are useful. diff --git a/library.json b/library.json index f721e28..94d1904 100644 --- a/library.json +++ b/library.json @@ -11,7 +11,7 @@ "type": "git", "url": "https://github.com/GOB52/gob_json.git" }, - "version": "0.0.3", + "version": "0.0.4", "build": { "libArchive": false }, diff --git a/library.properties b/library.properties index a5ac093..847a640 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=gob_json -version=0.0.3 +version=0.0.4 author=GOB maintainer=GOB sentence=Library for parsing potentially huge json streams on devices with scarce memory . diff --git a/platformio.ini b/platformio.ini index bc11ea7..481d747 100644 --- a/platformio.ini +++ b/platformio.ini @@ -7,7 +7,7 @@ default_envs = native_11, native_14, native_17, native_20 ;default_env = m5s_11, m5s_14, m5s_17, m5s_20 [env] -build_flags = -DUNIT_TEST -Wall -Wextra -Werror=format +build_flags = -DUNIT_TEST -Wall -Wextra -Wreturn-local-addr -Werror=format -Werror=return-local-addr -D GOB_JSON_PARSER_BUFFER_MAX_LENGTH=512 -D GOB_JSON_PARSER_KEY_MAX_LENGTH=64 -D GOB_JSON_PARSER_STACK_MAX_DEPTH=16 diff --git a/src/gob_json.cpp b/src/gob_json.cpp index b38819f..7c54a4b 100644 --- a/src/gob_json.cpp +++ b/src/gob_json.cpp @@ -97,12 +97,15 @@ void StreamingParser::parse(const char ch) startValue(c); break; case State::START_ESCAPE: + // GOB_JSON_LOGV("startEscape"); processEscapeCharacters(c); break; case State::UNICODE: + // GOB_JSON_LOGV("unicode:[%c]:0x%02x", c,c); processUnicodeCharacter(c); break; case State::UNICODE_SURROGATE: + // GOB_JSON_LOGV("surrogate:[%c]:0x%02x", c,c); unicodeEscapeBuffer[unicodeEscapeBufferPos] = c; unicodeEscapeBufferPos++; if (unicodeEscapeBufferPos == 2) { @@ -389,28 +392,18 @@ void StreamingParser::processUnicodeCharacter(char c) { if (unicodeBufferPos == 4) { int codepoint = getHexArrayAsDecimal(unicodeBuffer, unicodeBufferPos); - endUnicodeCharacter(codepoint); - return; - /*if (codepoint >= 0xD800 && codepoint < 0xDC00) { - unicodeHighSurrogate = codepoint; - unicodeBufferPos = 0; - state = State::UNICODE_SURROGATE; - } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { - if (unicodeHighSurrogate == -1) { - // throw new ParsingError($this->_line_number, - // $this->_char_number, - // "Missing high surrogate for Unicode low surrogate."); - } - int combinedCodePoint = ((unicodeHighSurrogate - 0xD800) * 0x400) + (codepoint - 0xDC00) + 0x10000; - endUnicodeCharacter(combinedCodePoint); - } else if (unicodeHighSurrogate != -1) { - // throw new ParsingError($this->_line_number, - // $this->_char_number, - // "Invalid low surrogate following Unicode high surrogate."); - endUnicodeCharacter(codepoint); - } else { - endUnicodeCharacter(codepoint); - }*/ + if(state != State::UNICODE_SURROGATE) + { + if (codepoint >= 0xD800 && codepoint < 0xDC00) { + unicodeHighSurrogate = codepoint; + unicodeBufferPos = 0; + state = State::UNICODE_SURROGATE; + } + else + { + endUnicodeCharacter(codepoint); + } + } } } bool StreamingParser::isHexCharacter(char c) { @@ -601,19 +594,37 @@ void StreamingParser::startNumber(char c) { increaseBufferPointer(); } -void StreamingParser::endUnicodeCharacter(int codepoint) { - if (codepoint < 0x80){ - buffer[bufferPos] = (char) (codepoint); - } else if (codepoint <= 0x800){ - buffer[bufferPos] = (char) ((codepoint >> 6) | 0b11000000); - increaseBufferPointer(); - buffer[bufferPos] = (char) ((codepoint & 0b00111111) | 0b10000000); - } else if (codepoint == 0x2019){ - buffer[bufferPos] = '\''; // \u2019 ’ - } else { - buffer[bufferPos] = ' '; +void StreamingParser::endUnicodeCharacter(uint32_t cp) { + // UTF-32 to UTF-8 + constexpr uint8_t mask = 0xBF; + constexpr uint8_t mark_bit = 0x80; + constexpr uint8_t markTable[] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + + uint8_t length{3}; + uint8_t buf[8]{}; + + // GOB_JSON_LOGV("cp:%x", cp); + + if(unicodeHighSurrogate != -1) + { + uint32_t high = unicodeHighSurrogate; + uint32_t low = cp; + cp = ((high - 0xD800) << 10) + (low - 0xDC00) + 0x10000; + // GOB_JSON_LOGV("SP:high:%x low:%x => %x", high, low, cp); + } + + if(cp < 0x80) { length = 1; } + else if(cp < 0x800) { length = 2; } + else if(cp < 0x10000) { length = 3; } + else if(cp < 0x110000) { length = 4; } + for(uint8_t i = length; i > 0 ; --i) + { + buf[i-1] = static_cast( (i==1) ? (cp | markTable[length]) : ((cp | mark_bit) & mask) ); + cp >>= 6; } - increaseBufferPointer(); + auto p = buf; + while(length--) { buffer[bufferPos] = *p++; increaseBufferPointer(); } + unicodeBufferPos = 0; unicodeHighSurrogate = -1; state = State::IN_STRING; diff --git a/src/gob_json.hpp b/src/gob_json.hpp index df856b3..5c89591 100644 --- a/src/gob_json.hpp +++ b/src/gob_json.hpp @@ -37,10 +37,10 @@ namespace json { #endif #ifndef GOB_JSON_PARSER_BUFFER_MAX_LENGTH -# pragma message "Buffer length as default" +# pragma message "[gob_json] Buffer length as default" # define GOB_JSON_PARSER_BUFFER_MAX_LENGTH (256) #else -# pragma message "Defined buffer length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_BUFFER_MAX_LENGTH) +# pragma message "[gob_json] Defined buffer length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_BUFFER_MAX_LENGTH) #endif /*! @@ -95,7 +95,7 @@ class StreamingParser void endTrue(); void endDocument(); void endUnicodeSurrogateInterstitial(); - void endUnicodeCharacter(int codepoint); + void endUnicodeCharacter(uint32_t codepoint); void increaseBufferPointer(); void processEscapeCharacters(char c); @@ -154,7 +154,7 @@ class StreamingParser int unicodeEscapeBufferPos{0};; char unicodeBuffer[10]; int unicodeBufferPos{0}; - int unicodeHighSurrogate{0}; + int unicodeHighSurrogate{-1}; size_t characterCounter{0}; int curCh{}; // for error information. diff --git a/src/gob_json_element_path.hpp b/src/gob_json_element_path.hpp index 1271ec2..572fe02 100644 --- a/src/gob_json_element_path.hpp +++ b/src/gob_json_element_path.hpp @@ -19,18 +19,18 @@ namespace goblib { namespace json { #endif #ifndef GOB_JSON_PARSER_KEY_MAX_LENGTH -# pragma message "Key length as default" +# pragma message "[gob_json] Key length as default" # define GOB_JSON_PARSER_KEY_MAX_LENGTH (32) #else -# pragma message "Defined key length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_KEY_MAX_LENGTH) +# pragma message "[gob_json] Defined key length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_KEY_MAX_LENGTH) #endif // For ElementPath::selectors and StreamingParser::stack #ifndef GOB_JSON_PARSER_STACK_MAX_DEPTH -# pragma message "Stack max depth as default" +# pragma message "[gob_json] Stack max depth as default" # define GOB_JSON_PARSER_STACK_MAX_DEPTH (20) #else -# pragma message "Defined stack max depth=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_STACK_MAX_DEPTH) +# pragma message "[gob_json] Defined stack max depth=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_STACK_MAX_DEPTH) #endif /*! diff --git a/src/gob_json_version.hpp b/src/gob_json_version.hpp index 1bd6704..95d3924 100644 --- a/src/gob_json_version.hpp +++ b/src/gob_json_version.hpp @@ -3,7 +3,7 @@ #define GOB_JSON_VERSION_MAJOR 0 #define GOB_JSON_VERSION_MINOR 0 -#define GOB_JSON_VERSION_PATCH 3 +#define GOB_JSON_VERSION_PATCH 4 #define GOB_JSON_VERSION_STRINGIFY_AGAIN(x) #x #define GOB_JSON_VERSION_STRINGIFY(x) GOB_JSON_VERSION_STRINGIFY_AGAIN(x) diff --git a/src/internal/gob_json_log.hpp b/src/internal/gob_json_log.hpp index 18bf5fb..b3e339e 100644 --- a/src/internal/gob_json_log.hpp +++ b/src/internal/gob_json_log.hpp @@ -16,14 +16,17 @@ # include # ifndef GOB_JSON_LOG_LEVEL # if defined(LOG_LOCAL_LEVEL) -# pragma message "[JSON]] Using LOG_LOCAL_LEVEL" +# pragma message "[gob_json Using LOG_LOCAL_LEVEL" # define GOB_JSON_LOG_LEVEL (LOG_LOCAL_LEVEL) # elif defined(CORE_DEBUG_LEVEL) -# pragma message "[JSON] Using CORE_DEBUG_LEVEL" +# pragma message "[gob_json] Using CORE_DEBUG_LEVEL" # define GOB_JSON_LOG_LEVEL (CORE_DEBUG_LEVEL) +# else +# pragma message "[gob_json] Using loglevel 3" +# define GOB_JSON_LOG_LEVEL (3)) # endif # else -# pragma message "[JSON] Using defined log level" +# pragma message "[gob_json] Using defined log level" # endif /*! @brief Error */ # define GOB_JSON_LOGE(format, ...) do { if(GOB_JSON_LOG_LEVEL >= ESP_LOG_ERROR) { log_printf(ARDUHAL_LOG_FORMAT(E, format), ##__VA_ARGS__); } } while(0) @@ -40,7 +43,10 @@ # include # ifndef GOB_JSON_LOG_LEVEL +# pragma message "[gob_json] Using loglevel 3" # define GOB_JSON_LOG_LEVEL (3) +# else +# pragma message "[gob_json] Using defined log level" # endif # define GOB_JSON_LOG(fmt, ...) do { printf("%s:%d ", __FILE__, __LINE__); printf(fmt, ##__VA_ARGS__); putchar('\n'); }while(0) diff --git a/test/test_basic.cpp b/test/test_basic.cpp index 6b2b77f..fd24275 100644 --- a/test/test_basic.cpp +++ b/test/test_basic.cpp @@ -6,6 +6,7 @@ #include #include +// TEST(Basic, Recursive) namespace { // Include 2 JSON documents. @@ -91,7 +92,7 @@ TEST(Basic, Recursive) } } -// +// TEST(Basic, Arithmetic) namespace { const char values_json[] = @@ -233,168 +234,7 @@ TEST(Basic, Arithmetic) } } -// - -namespace -{ -// Nested object and custom type values. -const char custom_types_json[] = -R"***( -{ - "rgba": "##12abcdef", - "datetime": - { - "start_at": "2009-08-07T12:34:56", - "end_at": "2009-08-07T13:57:00" - } -} -)***"; -// -} - -// RGBA color -struct RGBA -{ - static RGBA parse(const char* str) - { - int rr,gg,bb,aa; - sscanf(str, "##%02x%02x%02x%02x", &aa, &rr, &gg, &bb); - return RGBA(rr,gg,bb,aa); - } - RGBA() : RGBA(0,0,0,0) {} - RGBA(uint8_t rr, uint8_t gg, uint8_t bb, uint8_t aa) : r(rr), g(gg), b(bb), a(aa) {} - uint8_t r{}, g{}, b{}, a{}; -}; - -// Wrapping time_t (time_t is typedefed some integer, So to be a separate type. -struct Time_t -{ - static Time_t parse(const char* str) - { - struct tm tmp{}; - strptime(str, "%FT%T", &tmp); - auto tt = mktime(&tmp); - return Time_t(tt); - } - explicit Time_t(const time_t tt) : t(tt) {} - Time_t() : Time_t(0) {} - time_t t{}; -}; - -struct CustomTypes -{ - RGBA rgba{}; - struct Datetime - { - Time_t start_at; - Time_t end_at; - } datetime; -}; - -// You can make the specific element helper -template struct CustomElement : public Element -{ - CustomElement(const char*k, T* vp) : Element(k, vp) {} // Need constructor - virtual void store(const ElementValue& ev, const int index = -1) override { _store(ev, index); } - - // Declrare _store functions for each custom type.(SFINAE) - // For RGBA - template::value, std::nullptr_t>::type = nullptr> - void _store(const ElementValue& ev, const int) - { - *this->value = ev.isString() ? RGBA::parse(ev.getString()) : RGBA{}; - } - // For Time_t - template::value, std::nullptr_t>::type = nullptr> - void _store(const ElementValue& ev, const int) - { - *this->value = ev.isString() ? Time_t::parse(ev.getString()) : Time_t{}; - } -}; - -class CustomTypesHandler : public goblib::json::DelegateHandler -{ - public: - // The hierarchical structure shall be the same as in CustomTypes - struct CustomTypesDelegater : Delegater - { - struct DatetimeDelegater : Delegater - { - DatetimeDelegater(CustomTypes::Datetime& target) : _v(target) {} - virtual void value(const ElementPath& path, const ElementValue& value) override - { - CustomElement e_s { "start_at", &_v.start_at }; - CustomElement e_e { "end_at", &_v.end_at }; - ElementBase* tbl[] = { &e_s, &e_e }; - for(auto& e : tbl) { if(*e == path.getKey()) { e->store(value, path.getIndex()); return; } } - } - private: - CustomTypes::Datetime& _v; - }; - - CustomTypesDelegater(CustomTypes& target) : _v(target) {} - virtual Delegater* startObject(const ElementPath& path) override - { - if(strcmp(path.getKey(),"datetime")==0) { return new DatetimeDelegater(_v.datetime); } - return Delegater::startObject(path); - } - virtual void value(const ElementPath& path, const ElementValue& value) override - { - CustomElement e_rgba { "rgba", &_v.rgba }; - ElementBase* tbl[] = { &e_rgba }; - for(auto& e : tbl) { if(*e == path.getKey()) { e->store(value, path.getIndex()); return; } } - } - private: - CustomTypes& _v; - }; - - CustomTypesHandler(CustomTypes& v) : _value(v) {} - void startObject(const ElementPath& path) override - { - if(path.getCount() == 0) { pushDelegater(new CustomTypesDelegater(_value)); return; } - DelegateHandler::startObject(path); - } - private: - CustomTypes& _value; -}; - -TEST(Basic, CustomElements) -{ - { - CustomTypes cs{}; - CustomTypesHandler handler(cs); - goblib::json::StreamingParser parser(&handler); - for(auto& e: custom_types_json) { parser.parse(e); } - - EXPECT_FALSE(parser.hasError()); - - EXPECT_EQ(cs.rgba.r, 0xab); - EXPECT_EQ(cs.rgba.g, 0xcd); - EXPECT_EQ(cs.rgba.b, 0xef); - EXPECT_EQ(cs.rgba.a, 0x12); - - struct tm tmp; - tmp = *localtime(&cs.datetime.start_at.t); - EXPECT_EQ(tmp.tm_year + 1900, 2009); - EXPECT_EQ(tmp.tm_mon + 1, 8); - EXPECT_EQ(tmp.tm_mday, 7); - EXPECT_EQ(tmp.tm_hour,12); - EXPECT_EQ(tmp.tm_min, 34); - EXPECT_EQ(tmp.tm_sec, 56); - - tmp = *localtime(&cs.datetime.end_at.t); - EXPECT_EQ(tmp.tm_year + 1900, 2009); - EXPECT_EQ(tmp.tm_mon + 1, 8); - EXPECT_EQ(tmp.tm_mday, 7); - EXPECT_EQ(tmp.tm_hour,13); - EXPECT_EQ(tmp.tm_min, 57); - EXPECT_EQ(tmp.tm_sec, 0);; - } -} - - +// TEST(Basic, AnyType) namespace { const char any_type_json[] = @@ -407,7 +247,7 @@ R"***( } )***"; -struct TestHandler: public goblib::json::Handler +struct AnyTestHandler: public goblib::json::Handler { virtual void startDocument() override{} virtual void endDocument() override {} @@ -433,7 +273,7 @@ struct TestHandler: public goblib::json::Handler TEST(Basic, AnyType) { - TestHandler handler; + AnyTestHandler handler; goblib::json::StreamingParser parser(&handler); for(auto& e: any_type_json) { parser.parse(e); } @@ -443,3 +283,51 @@ TEST(Basic, AnyType) EXPECT_EQ(handler.i16, 42); EXPECT_STREQ(handler.str.c_str(), "漢字カナまじりANK☎️"); } + +// TEST(Basic, Unicode) +namespace +{ +const char unicode_escape_json[] = +R"***( +{ + "text":"\u30b31\u30de\\\u30f3&\u30c9@", + "surrogate":"\ud867\ude3d\ud840\udc0b\ud842\udfb7\ud853\udd14" +} +)***"; + +struct UnicodeEscapeTestHandler: public goblib::json::Handler +{ + virtual void startDocument() override{} + virtual void endDocument() override {} + virtual void startObject(const ElementPath& ) override {} + virtual void endObject(const ElementPath& ) override {} + virtual void startArray(const ElementPath& ) override {} + virtual void endArray(const ElementPath& ) override {} + virtual void whitespace(const char/*ch*/) override {} + virtual void value(const ElementPath& path, const ElementValue& value) override + { + if(strcmp(path.getKey(),"text") == 0) { str = value.getString(); } + else if(strcmp(path.getKey(),"surrogate") == 0) { str2 = value.getString(); } + else if(strcmp(path.getKey(),"JISX0201") == 0) { jisx0201 = value.getString(); } + else if(strcmp(path.getKey(),"JISX0213") == 0) { jisx0213 = value.getString(); } + } + goblib::json::string_t str{"deadbeaf"}; + goblib::json::string_t str2{"deadbeaf"}; + goblib::json::string_t jisx0201{"deadbeaf"}; + goblib::json::string_t jisx0213{"deadbeaf"}; +}; +// +} + +TEST(Basic, UnicodeEscape) +{ + UnicodeEscapeTestHandler handler; + goblib::json::StreamingParser parser(&handler); + for(auto& e: unicode_escape_json) { parser.parse(e); } + + EXPECT_FALSE(parser.hasError()); + EXPECT_STREQ(handler.str.c_str(), "コ1マ\\ン&ド@"); + EXPECT_STREQ(handler.str2.c_str(), "𩸽𠀋𠮷𤴔"); + +} + diff --git a/test/test_element.cpp b/test/test_element.cpp index 5040e4c..3fd9ccc 100644 --- a/test/test_element.cpp +++ b/test/test_element.cpp @@ -3,7 +3,9 @@ #include #include +#include +// TEST(Element, Basic) namespace { const char test_json[] = @@ -177,3 +179,163 @@ TEST(Element, Basic) EXPECT_STREQ(handler.sa2[2].c_str(), "😄"); } } + +// TEST(Element, Custom) +namespace +{ +// Nested object and custom type values. +const char custom_types_json[] = +R"***( +{ + "rgba": "##12abcdef", + "datetime": + { + "start_at": "2009-08-07T12:34:56", + "end_at": "2009-08-07T13:57:00" + } +} +)***"; +// +} + +// RGBA color +struct RGBA +{ + static RGBA parse(const char* str) + { + int rr,gg,bb,aa; + sscanf(str, "##%02x%02x%02x%02x", &aa, &rr, &gg, &bb); + return RGBA(rr,gg,bb,aa); + } + RGBA() : RGBA(0,0,0,0) {} + RGBA(uint8_t rr, uint8_t gg, uint8_t bb, uint8_t aa) : r(rr), g(gg), b(bb), a(aa) {} + uint8_t r{}, g{}, b{}, a{}; +}; + +// Wrapping time_t (time_t is typedefed some integer, So to be a separate type. +struct Time_t +{ + static Time_t parse(const char* str) + { + struct tm tmp{}; + strptime(str, "%FT%T", &tmp); + auto tt = mktime(&tmp); + return Time_t(tt); + } + explicit Time_t(const time_t tt) : t(tt) {} + Time_t() : Time_t(0) {} + time_t t{}; +}; + +struct CustomTypes +{ + RGBA rgba{}; + struct Datetime + { + Time_t start_at; + Time_t end_at; + } datetime; +}; + +// You can make the specific element helper +template struct CustomElement : public Element +{ + CustomElement(const char*k, T* vp) : Element(k, vp) {} // Need constructor + virtual void store(const ElementValue& ev, const int index = -1) override { _store(ev, index); } + + // Declrare _store functions for each custom type.(SFINAE) + // For RGBA + template::value, std::nullptr_t>::type = nullptr> + void _store(const ElementValue& ev, const int) + { + *this->value = ev.isString() ? RGBA::parse(ev.getString()) : RGBA{}; + } + // For Time_t + template::value, std::nullptr_t>::type = nullptr> + void _store(const ElementValue& ev, const int) + { + *this->value = ev.isString() ? Time_t::parse(ev.getString()) : Time_t{}; + } +}; + +class CustomTypesHandler : public goblib::json::DelegateHandler +{ + public: + // The hierarchical structure shall be the same as in CustomTypes + struct CustomTypesDelegater : Delegater + { + struct DatetimeDelegater : Delegater + { + DatetimeDelegater(CustomTypes::Datetime& target) : _v(target) {} + virtual void value(const ElementPath& path, const ElementValue& value) override + { + CustomElement e_s { "start_at", &_v.start_at }; + CustomElement e_e { "end_at", &_v.end_at }; + ElementBase* tbl[] = { &e_s, &e_e }; + for(auto& e : tbl) { if(*e == path.getKey()) { e->store(value, path.getIndex()); return; } } + } + private: + CustomTypes::Datetime& _v; + }; + + CustomTypesDelegater(CustomTypes& target) : _v(target) {} + virtual Delegater* startObject(const ElementPath& path) override + { + if(strcmp(path.getKey(),"datetime")==0) { return new DatetimeDelegater(_v.datetime); } + return Delegater::startObject(path); + } + virtual void value(const ElementPath& path, const ElementValue& value) override + { + CustomElement e_rgba { "rgba", &_v.rgba }; + ElementBase* tbl[] = { &e_rgba }; + for(auto& e : tbl) { if(*e == path.getKey()) { e->store(value, path.getIndex()); return; } } + } + private: + CustomTypes& _v; + }; + + CustomTypesHandler(CustomTypes& v) : _value(v) {} + void startObject(const ElementPath& path) override + { + if(path.getCount() == 0) { pushDelegater(new CustomTypesDelegater(_value)); return; } + DelegateHandler::startObject(path); + } + private: + CustomTypes& _value; +}; + +TEST(Element, Custom) +{ + { + CustomTypes cs{}; + CustomTypesHandler handler(cs); + goblib::json::StreamingParser parser(&handler); + for(auto& e: custom_types_json) { parser.parse(e); } + + EXPECT_FALSE(parser.hasError()); + + EXPECT_EQ(cs.rgba.r, 0xab); + EXPECT_EQ(cs.rgba.g, 0xcd); + EXPECT_EQ(cs.rgba.b, 0xef); + EXPECT_EQ(cs.rgba.a, 0x12); + + struct tm tmp; + tmp = *localtime(&cs.datetime.start_at.t); + EXPECT_EQ(tmp.tm_year + 1900, 2009); + EXPECT_EQ(tmp.tm_mon + 1, 8); + EXPECT_EQ(tmp.tm_mday, 7); + EXPECT_EQ(tmp.tm_hour,12); + EXPECT_EQ(tmp.tm_min, 34); + EXPECT_EQ(tmp.tm_sec, 56); + + tmp = *localtime(&cs.datetime.end_at.t); + EXPECT_EQ(tmp.tm_year + 1900, 2009); + EXPECT_EQ(tmp.tm_mon + 1, 8); + EXPECT_EQ(tmp.tm_mday, 7); + EXPECT_EQ(tmp.tm_hour,13); + EXPECT_EQ(tmp.tm_min, 57); + EXPECT_EQ(tmp.tm_sec, 0);; + } +}