Core/Utils: Added additional argument to Tokenizer class to make it behave like strtok - not returning empty tokens in case of multiple consecutive separators in input string

(cherry picked from commit dd1533b315bda37e1d43ebe0fb8bde87381c6e66)

src/common/Utilities/Util.cpp

@@ -30,7 +30,7 @@
   #include <arpa/inet.h>
 #endif
 
-Tokenizer::Tokenizer(const std::string &src, const char sep, uint32 vectorReserve)
+Tokenizer::Tokenizer(const std::string &src, const char sep, uint32 vectorReserve /*= 0*/, bool keepEmptyStrings /*= true*/)
 {
     m_str = new char[src.length() + 1];
     memcpy(m_str, src.c_str(), src.length() + 1);
@@ -45,9 +45,10 @@ Tokenizer::Tokenizer(const std::string &src, const char sep, uint32 vectorReserv
     {
         if (*posnew == sep)
         {
-            m_storage.push_back(posold);
-            posold = posnew + 1;
+            if (keepEmptyStrings || posold != posnew)
+                m_storage.push_back(posold);
 
+            posold = posnew + 1;
             *posnew = '\0';
         }
         else if (*posnew == '\0')

src/common/Utilities/Util.h

@@ -52,7 +52,7 @@ public:
     typedef StorageType::const_reference const_reference;
 
 public:
-    Tokenizer(const std::string &src, char const sep, uint32 vectorReserve = 0);
+    Tokenizer(const std::string &src, char const sep, uint32 vectorReserve = 0, bool keepEmptyStrings = true);
     ~Tokenizer() { delete[] m_str; }
 
     const_iterator begin() const { return m_storage.begin(); }