Merge branch 'develop'

README.md

@@ -61,7 +61,7 @@ Log output when internal errors occur and added detection mechanism.
 ### Added a mechanism to help determine keys and retrieve values
 if elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif elseif ... OMG!  
 Added a helper for retrive values and a delegation handler for processing per JSON object.  
-see also ValuesHandler in [test_basic.cpp](test/test_basic.cpp)
+see also [test_element.cpp](test/test_element.cpp), [test_basic.cpp](test/test_basic.cpp)
 
 ### Unit test support with GoogleTest
 Even small test cases are useful.
@@ -77,6 +77,22 @@ If you are using PlatformIO, you can write the following to download automatical
 ```ini
 lib_deps = https://github.com/GOB52/gob_json.git
 ```
+
+## Setting
+The build option allows you to set several items.
+
+|symbol| description|default value|
+|---|---|---|
+|GOB_JSON_PARSER_BUFFER_MAX_LENGTH| Token buffer size| 256|
+|GOB_JSON_PARSER_KEY_MAX_LENGTH| JSON key token buffer size|32|
+|GOB_JSON_PARSER_STACK_MAX_DEPTH|Maximum nesting level of JSON object/array|20|
+
+```ini
+build_flags = -D GOB_JSON_PARSER_BUFFER_MAX_LENGTH=384 
+  -D GOB_JSON_PARSER_KEY_MAX_LENGTH=64
+  -D GOB_JSON_PARSER_STACK_MAX_DEPTH=16
+```
+
 ## Usage
 If you use it like json-streaming-parser2, rename the headers and types to include.
 

library.json

@@ -11,7 +11,7 @@
     "type": "git",
     "url": "https://github.com/GOB52/gob_json.git"
   },
-  "version": "0.0.3",
+  "version": "0.0.4",
   "build": {
     "libArchive": false
   },

library.properties

@@ -1,5 +1,5 @@
 name=gob_json
-version=0.0.3
+version=0.0.4
 author=GOB
 maintainer=GOB
 sentence=Library for parsing potentially huge json streams on devices with scarce memory .

platformio.ini

@@ -7,8 +7,10 @@ default_envs = native_11, native_14, native_17, native_20
 ;default_env = m5s_11, m5s_14, m5s_17, m5s_20
 
 [env]
-build_flags = -DUNIT_TEST -Wall -Wextra -Werror=format 
+build_flags = -DUNIT_TEST -Wall -Wextra  -Wreturn-local-addr -Werror=format -Werror=return-local-addr
   -D GOB_JSON_PARSER_BUFFER_MAX_LENGTH=512
+  -D GOB_JSON_PARSER_KEY_MAX_LENGTH=64
+  -D GOB_JSON_PARSER_STACK_MAX_DEPTH=16
   -D GOB_JSON_LOG_LEVEL=5
 
 test_framework = googletest

src/gob_json.cpp

@@ -12,6 +12,10 @@
 
 namespace goblib { namespace json {
 
+// Size of array.
+template <class C> constexpr auto size(const C& c) -> decltype(c.size()) { return c.size(); }
+template<typename T, size_t N> constexpr auto size(const T(&)[N]) noexcept -> size_t { return N; }
+
 void StreamingParser::reset()
 {
     state = State::START_DOCUMENT;
@@ -93,12 +97,15 @@ void StreamingParser::parse(const char ch)
         startValue(c);
         break;
     case State::START_ESCAPE:
+        //        GOB_JSON_LOGV("startEscape");
         processEscapeCharacters(c);
         break;
     case State::UNICODE:
+        //        GOB_JSON_LOGV("unicode:[%c]:0x%02x", c,c);
         processUnicodeCharacter(c);
         break;
     case State::UNICODE_SURROGATE:
+        //        GOB_JSON_LOGV("surrogate:[%c]:0x%02x", c,c);
         unicodeEscapeBuffer[unicodeEscapeBufferPos] = c;
         unicodeEscapeBufferPos++;
         if (unicodeEscapeBufferPos == 2) {
@@ -307,7 +314,7 @@ void StreamingParser::endArray() {
 }
 
 void StreamingParser::startKey() {
-    if(stackPos >= (int)sizeof(stack))
+    if(stackPos >= (int)size(stack))
     {
         PARSE_ERROR("stack overflow", curCh, characterCounter, path);
         return;
@@ -385,28 +392,18 @@ void StreamingParser::processUnicodeCharacter(char c) {
 
     if (unicodeBufferPos == 4) {
         int codepoint = getHexArrayAsDecimal(unicodeBuffer, unicodeBufferPos);
-        endUnicodeCharacter(codepoint);
-        return;
-        /*if (codepoint >= 0xD800 && codepoint < 0xDC00) {
-          unicodeHighSurrogate = codepoint;
-          unicodeBufferPos = 0;
-          state = State::UNICODE_SURROGATE;
-          } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
-          if (unicodeHighSurrogate == -1) {
-          // throw new ParsingError($this->_line_number,
-          // $this->_char_number,
-          // "Missing high surrogate for Unicode low surrogate.");
-          }
-          int combinedCodePoint = ((unicodeHighSurrogate - 0xD800) * 0x400) + (codepoint - 0xDC00) + 0x10000;
-          endUnicodeCharacter(combinedCodePoint);
-          } else if (unicodeHighSurrogate != -1) {
-          // throw new ParsingError($this->_line_number,
-          // $this->_char_number,
-          // "Invalid low surrogate following Unicode high surrogate.");
-          endUnicodeCharacter(codepoint);
-          } else {
-          endUnicodeCharacter(codepoint);
-          }*/
+        if(state != State::UNICODE_SURROGATE)
+        {
+            if (codepoint >= 0xD800 && codepoint < 0xDC00) {
+                unicodeHighSurrogate = codepoint;
+                unicodeBufferPos = 0;
+                state = State::UNICODE_SURROGATE;
+            }
+            else
+            {
+                endUnicodeCharacter(codepoint);
+            }
+        }
     }
 }
 bool StreamingParser::isHexCharacter(char c) {
@@ -555,7 +552,7 @@ void StreamingParser::endNull() {
 }
 
 void StreamingParser::startArray() {
-    if(stackPos >= (int)sizeof(stack))
+    if(stackPos >= (int)size(stack))
     {
         PARSE_ERROR("stack overflow", curCh, characterCounter, path);
         return;
@@ -568,7 +565,7 @@ void StreamingParser::startArray() {
 }
 
 void StreamingParser::startObject() {
-    if(stackPos >= (int)sizeof(stack))
+    if(stackPos >= (int)size(stack))
     {
         PARSE_ERROR("stack overflow", curCh, characterCounter, path);
         return;
@@ -581,7 +578,7 @@ void StreamingParser::startObject() {
 }
 
 void StreamingParser::startString() {
-    if(stackPos >= (int)sizeof(stack))
+    if(stackPos >= (int)size(stack))
     {
         PARSE_ERROR("stack overflow", curCh, characterCounter, path);
         return;
@@ -597,19 +594,37 @@ void StreamingParser::startNumber(char c) {
     increaseBufferPointer();
 }
 
-void StreamingParser::endUnicodeCharacter(int codepoint) {
-    if (codepoint < 0x80){
-        buffer[bufferPos] = (char) (codepoint);
-    } else if (codepoint <= 0x800){
-        buffer[bufferPos] = (char) ((codepoint >> 6) | 0b11000000);
-        increaseBufferPointer();
-        buffer[bufferPos] = (char) ((codepoint & 0b00111111) | 0b10000000);
-    } else if (codepoint == 0x2019){
-        buffer[bufferPos] = '\''; // \u2019 ’
-    } else {
-        buffer[bufferPos] = ' ';
+void StreamingParser::endUnicodeCharacter(uint32_t cp) {
+    // UTF-32 to UTF-8
+    constexpr uint8_t mask = 0xBF;
+    constexpr uint8_t mark_bit = 0x80;
+    constexpr uint8_t markTable[] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+    uint8_t length{3};
+    uint8_t buf[8]{};
+
+    //    GOB_JSON_LOGV("cp:%x", cp);
+
+    if(unicodeHighSurrogate != -1)
+    {
+        uint32_t high = unicodeHighSurrogate;
+        uint32_t low = cp;
+        cp = ((high - 0xD800) << 10) + (low - 0xDC00) + 0x10000;
+        //        GOB_JSON_LOGV("SP:high:%x low:%x => %x", high, low, cp);
+    }
+
+    if(cp < 0x80) { length = 1; }
+    else if(cp < 0x800) { length = 2; }
+    else if(cp < 0x10000) { length = 3; }
+    else if(cp < 0x110000) { length = 4; }
+    for(uint8_t i = length; i > 0 ; --i)
+    {
+        buf[i-1] = static_cast<uint8_t>( (i==1) ? (cp | markTable[length]) : ((cp | mark_bit) & mask) );
+        cp >>= 6;
     }
-    increaseBufferPointer();
+    auto p = buf;
+    while(length--) { buffer[bufferPos] = *p++; increaseBufferPointer(); }
+
     unicodeBufferPos = 0;
     unicodeHighSurrogate = -1;
     state = State::IN_STRING;

src/gob_json.hpp

@@ -29,13 +29,18 @@ namespace goblib {
 */
 namespace json {
 
+#ifndef GOB_JSON_STRINGIFY
+# define GOB_JSON_STRINGIFY(x) GOB_JSON_STRINGIFY_AGAIN(x)
+#endif
+#ifndef GOB_JSON_STRINGIFY_AGAIN
+# define GOB_JSON_STRINGIFY_AGAIN(x) #x
+#endif
+
 #ifndef GOB_JSON_PARSER_BUFFER_MAX_LENGTH
-# pragma message "Buffer size as default"
+# pragma message "[gob_json] Buffer length as default"
 # define GOB_JSON_PARSER_BUFFER_MAX_LENGTH  (256)
 #else
-# define GOB_JSON_STRINGIFY(x) GOB_JSON_STRINGIFY_AGAIN(x)
-# define GOB_JSON_STRINGIFY_AGAIN(x) #x
-# pragma message "Buffer size set by build option=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_BUFFER_MAX_LENGTH)
+# pragma message "[gob_json] Defined buffer length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_BUFFER_MAX_LENGTH)
 #endif
 
 /*!
@@ -90,7 +95,7 @@ class StreamingParser
     void endTrue();
     void endDocument();
     void endUnicodeSurrogateInterstitial();
-    void endUnicodeCharacter(int codepoint);
+    void endUnicodeCharacter(uint32_t codepoint);
 
     void increaseBufferPointer();
     void processEscapeCharacters(char c);
@@ -136,20 +141,20 @@ class StreamingParser
     ElementPath path{};
 
     State state{State::START_DOCUMENT};
-    Stack stack[20]{};
+    Stack stack[GOB_JSON_PARSER_STACK_MAX_DEPTH]{};
     int stackPos{0};
 
     bool recursive{false};
     bool doEmitWhitespace{false};
 
-    char buffer[GOB_JSON_PARSER_BUFFER_MAX_LENGTH];
+    char buffer[GOB_JSON_PARSER_BUFFER_MAX_LENGTH]{};
     int bufferPos{0};
 
     char unicodeEscapeBuffer[10];
     int unicodeEscapeBufferPos{0};;
     char unicodeBuffer[10];
     int unicodeBufferPos{0};
-    int unicodeHighSurrogate{0};
+    int unicodeHighSurrogate{-1};
 
     size_t characterCounter{0};
     int curCh{}; // for error information.

src/gob_json_element_path.hpp

@@ -11,6 +11,28 @@
 
 namespace goblib { namespace json {
 
+#ifndef GOB_JSON_STRINGIFY
+# define GOB_JSON_STRINGIFY(x) GOB_JSON_STRINGIFY_AGAIN(x)
+#endif
+#ifndef GOB_JSON_STRINGIFY_AGAIN
+# define GOB_JSON_STRINGIFY_AGAIN(x) #x
+#endif
+
+#ifndef GOB_JSON_PARSER_KEY_MAX_LENGTH
+# pragma message "[gob_json] Key length as default"
+# define GOB_JSON_PARSER_KEY_MAX_LENGTH  (32)
+#else
+# pragma message "[gob_json] Defined key length=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_KEY_MAX_LENGTH)
+#endif
+
+// For ElementPath::selectors and StreamingParser::stack
+#ifndef GOB_JSON_PARSER_STACK_MAX_DEPTH
+# pragma message "[gob_json] Stack max depth as default"
+# define GOB_JSON_PARSER_STACK_MAX_DEPTH (20)
+#else
+# pragma message "[gob_json] Defined stack max depth=" GOB_JSON_STRINGIFY(GOB_JSON_PARSER_STACK_MAX_DEPTH)
+#endif
+
 /*!
   @class ElementSelector
   @brief Unified element selector.
@@ -36,7 +58,7 @@ class ElementSelector
 
   private: 
     int index{-1};
-    char key[32]{0,};
+    char key[GOB_JSON_PARSER_KEY_MAX_LENGTH]{0,};
     friend class ElementPath;
     friend class StreamingParser;
 };
@@ -97,7 +119,7 @@ class ElementPath
   private:
     int count{0};
     ElementSelector* current{nullptr};
-    ElementSelector selectors[20]{};
+    ElementSelector selectors[GOB_JSON_PARSER_STACK_MAX_DEPTH]{};
     friend class StreamingParser;
 };
 

src/gob_json_version.hpp

@@ -3,7 +3,7 @@
 
 #define GOB_JSON_VERSION_MAJOR 0
 #define GOB_JSON_VERSION_MINOR 0
-#define GOB_JSON_VERSION_PATCH 3
+#define GOB_JSON_VERSION_PATCH 4
 
 #define GOB_JSON_VERSION_STRINGIFY_AGAIN(x) #x
 #define GOB_JSON_VERSION_STRINGIFY(x) GOB_JSON_VERSION_STRINGIFY_AGAIN(x)

src/internal/gob_json_log.hpp

@@ -16,14 +16,17 @@
 #   include <esp32-hal-log.h>
 #   ifndef GOB_JSON_LOG_LEVEL
 #     if defined(LOG_LOCAL_LEVEL)
-#       pragma message "[JSON]] Using LOG_LOCAL_LEVEL"
+#       pragma message "[gob_json Using LOG_LOCAL_LEVEL"
 #       define GOB_JSON_LOG_LEVEL (LOG_LOCAL_LEVEL)
 #     elif defined(CORE_DEBUG_LEVEL)
-#       pragma message "[JSON] Using CORE_DEBUG_LEVEL"
+#       pragma message "[gob_json] Using CORE_DEBUG_LEVEL"
 #       define GOB_JSON_LOG_LEVEL (CORE_DEBUG_LEVEL)
+#     else
+#       pragma message "[gob_json] Using loglevel 3"
+#       define GOB_JSON_LOG_LEVEL (3))
 #     endif
 #   else
-#     pragma message "[JSON] Using defined log level"
+#     pragma message "[gob_json] Using defined log level"
 #   endif
 /*! @brief Error */
 #   define GOB_JSON_LOGE(format, ...) do { if(GOB_JSON_LOG_LEVEL >= ESP_LOG_ERROR)   { log_printf(ARDUHAL_LOG_FORMAT(E, format), ##__VA_ARGS__); } } while(0)
@@ -40,7 +43,10 @@
 
 #   include <cstdio>
 #   ifndef GOB_JSON_LOG_LEVEL
+#     pragma message "[gob_json] Using loglevel 3"
 #     define GOB_JSON_LOG_LEVEL (3)
+#   else
+#     pragma message "[gob_json] Using defined log level"
 #   endif
 
 #   define GOB_JSON_LOG(fmt, ...) do { printf("%s:%d ", __FILE__, __LINE__); printf(fmt, ##__VA_ARGS__); putchar('\n'); }while(0)