From 06d64b76f79ffabcf606afa9216009c3bd78fca5 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Sun, 2 Sep 2018 20:47:43 +0200 Subject: [PATCH] data: Reduced memory usage during indexing with LowMemoryStringMap * reduces node and local symbol index sizes by about 30-50% --- .../LowMemoryStringMapTestSuite/names.txt | 7 + src/lib/CMakeLists.txt | 1 + .../storage/sqlite/SqliteIndexStorage.cpp | 88 ++- .../data/storage/sqlite/SqliteIndexStorage.h | 9 +- src/lib/utility/LowMemoryStringMap.h | 601 ++++++++++++++++++ src/test/CMakeLists.txt | 5 +- src/test/LowMemoryStringMapTestSuite.h | 165 +++++ 7 files changed, 855 insertions(+), 21 deletions(-) create mode 100644 bin/test/data/LowMemoryStringMapTestSuite/names.txt create mode 100644 src/lib/utility/LowMemoryStringMap.h create mode 100644 src/test/LowMemoryStringMapTestSuite.h diff --git a/bin/test/data/LowMemoryStringMapTestSuite/names.txt b/bin/test/data/LowMemoryStringMapTestSuite/names.txt new file mode 100644 index 00000000..e10fe9ac --- /dev/null +++ b/bin/test/data/LowMemoryStringMapTestSuite/names.txt @@ -0,0 +1,7 @@ +Eberhard +Gräther +Gräber +Gröber +Österreich +Ostern +Öl \ No newline at end of file diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index fab08aca..55098c50 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -586,6 +586,7 @@ add_files( utility/AppPath.h utility/ConfigManager.cpp utility/ConfigManager.h + utility/LowMemoryStringMap.h utility/Optional.h utility/OrderedCache.h utility/OsType.h diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp index ca4fc279..44708223 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp @@ -24,9 +24,12 @@ size_t SqliteIndexStorage::getStaticVersion() const void SqliteIndexStorage::setMode(const StorageModeType mode) { - m_tempNodeIndex.clear(); + m_tempNodeNameIndex.clear(); + m_tempWNodeNameIndex.clear(); + m_tempNodeTypes.clear(); m_tempEdgeIndex.clear(); - m_tempLocalSymbolIndex.clear(); + m_tempLocalSymbolNameIndex.clear(); + m_tempWLocalSymbolNameIndex.clear(); m_tempSourceLocationIndices.clear(); std::vector> indices = getIndices(); @@ -55,24 +58,45 @@ void SqliteIndexStorage::setProjectSettingsText(std::string text) StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data) { - if (m_tempNodeIndex.empty()) + if (m_tempNodeNameIndex.empty() && m_tempWNodeNameIndex.empty()) { for (const StorageNode& node : getAll()) { - m_tempNodeIndex.emplace(utility::encodeToUtf8(node.serializedName), std::make_pair(node.id, node.type)); + std::string name = utility::encodeToUtf8(node.serializedName); + if (name.size() != node.serializedName.size()) + { + m_tempWNodeNameIndex.add(node.serializedName, node.id); + } + else + { + m_tempNodeNameIndex.add(name, node.id); + } + + m_tempNodeTypes.emplace(node.id, node.type); } } std::string name = utility::encodeToUtf8(data.serializedName); { - std::map>::const_iterator it = m_tempNodeIndex.find(name); - if (it != m_tempNodeIndex.end()) + Id nodeId; + if (name.size() != data.serializedName.size()) { - if (it->second.second < data.type) + nodeId = m_tempWNodeNameIndex.find(data.serializedName); + } + else + { + nodeId = m_tempNodeNameIndex.find(name); + } + + if (nodeId) + { + auto it = m_tempNodeTypes.find(nodeId); + if (it != m_tempNodeTypes.end() && it->second < data.type) { - setNodeType(data.type, it->second.first); + setNodeType(data.type, nodeId); + m_tempNodeTypes[nodeId] = data.type; } - return StorageNode(it->second.first, data); + return StorageNode(nodeId, data); } } @@ -90,7 +114,15 @@ StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data) m_inserNodeStmt.reset(); } - m_tempNodeIndex.emplace(name, std::make_pair(id, data.type)); + if (name.size() != data.serializedName.size()) + { + m_tempWNodeNameIndex.add(data.serializedName, id); + } + else + { + m_tempNodeNameIndex.add(name, id); + } + m_tempNodeTypes.emplace(id, data.type); return StorageNode(id, data); } @@ -187,20 +219,37 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data) StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data) { - if (m_tempLocalSymbolIndex.empty()) + if (m_tempLocalSymbolNameIndex.empty() && m_tempWLocalSymbolNameIndex.empty()) { for (const StorageLocalSymbol& localSymbol : getAll()) { - m_tempLocalSymbolIndex.emplace(utility::encodeToUtf8(localSymbol.name), localSymbol.id); + std::string name = utility::encodeToUtf8(localSymbol.name); + if (name.size() != localSymbol.name.size()) + { + m_tempWLocalSymbolNameIndex.add(localSymbol.name, localSymbol.id); + } + else + { + m_tempLocalSymbolNameIndex.add(name, localSymbol.id); + } } } std::string name = utility::encodeToUtf8(data.name); { - std::map::const_iterator it = m_tempLocalSymbolIndex.find(name); - if (it != m_tempLocalSymbolIndex.end()) + Id localSymbolId; + if (name.size() != data.name.size()) { - return StorageLocalSymbol(it->second, data); + localSymbolId = m_tempWLocalSymbolNameIndex.find(data.name); + } + else + { + localSymbolId = m_tempLocalSymbolNameIndex.find(name); + } + + if (localSymbolId) + { + return StorageLocalSymbol(localSymbolId, data); } } @@ -217,7 +266,14 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa m_inserLocalSymbolStmt.reset(); } - m_tempLocalSymbolIndex.emplace(name, id); + if (name.size() != data.name.size()) + { + m_tempWLocalSymbolNameIndex.add(data.name, id); + } + else + { + m_tempLocalSymbolNameIndex.add(name, id); + } return StorageLocalSymbol(id, data); } diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.h b/src/lib/data/storage/sqlite/SqliteIndexStorage.h index 0419c993..45e15ee4 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.h +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.h @@ -18,6 +18,7 @@ #include "data/storage/type/StorageOccurrence.h" #include "data/storage/type/StorageSourceLocation.h" #include "data/storage/type/StorageSymbol.h" +#include "utility/LowMemoryStringMap.h" #include "utility/types.h" #include "utility/utility.h" #include "utility/utilityString.h" @@ -215,10 +216,12 @@ private: return ResultType(); } - - std::map> m_tempNodeIndex; + LowMemoryStringMap m_tempNodeNameIndex; + LowMemoryStringMap m_tempWNodeNameIndex; + std::map m_tempNodeTypes; std::map m_tempEdgeIndex; - std::map m_tempLocalSymbolIndex; + LowMemoryStringMap m_tempLocalSymbolNameIndex; + LowMemoryStringMap m_tempWLocalSymbolNameIndex; std::map> m_tempSourceLocationIndices; CppSQLite3Statement m_insertElementStmt; diff --git a/src/lib/utility/LowMemoryStringMap.h b/src/lib/utility/LowMemoryStringMap.h new file mode 100644 index 00000000..cddfd180 --- /dev/null +++ b/src/lib/utility/LowMemoryStringMap.h @@ -0,0 +1,601 @@ +#ifndef LOW_MEMORY_STRING_MAP_H +#define LOW_MEMORY_STRING_MAP_H + +#include +#include +#include + +/* + * StringTraits + * + * Defines types related to either std::string or std::wstring + */ + +template +class StringTraits +{ +}; + +template <> +class StringTraits +{ +public: + typedef char CharT; + typedef std::ostream StreamT; + + static size_t SizeFn(const CharT* str) + { + return strlen(str); + } + + static CharT* CopyFn(CharT* destination, const CharT* source, size_t num) + { + return strncpy(destination, source, num); + } +}; + +template <> +class StringTraits +{ +public: + typedef wchar_t CharT; + typedef std::wostream StreamT; + + static size_t SizeFn(const CharT* str) + { + return wcslen(str); + } + + static CharT* CopyFn(CharT* destination, const CharT* source, size_t num) + { + return wcsncpy(destination, source, num); + } +}; + +/* + * LowMemoryStringMap + * + * Map of string - value pairs, where equal suffixes of strings are used to build tree structure reducing memory + * consumption. + * + * NOTE: It is not guaranteed that equal strings cannot be added twice. If uniqueness of strings is important, use + * find() first to check if the string was already added. + * + * - StringT: string type (supported: std::string, std::wstring) + * - ValueT: value type + * - defaultVal: default value of type ValueT (cannot be stored as value) + * - branchSplitThreshold: minimum number of characters necessary to split remaining string part into another branch. + * (e.g. storing 'code' and 'copy' as two strings takes up less space than storing them as tree 'co' -> 'de' | 'py', + * because that also needs pointers and a map. For that reason strings are only split into branches if the newly + * added one is above this threshold. This is also the reason std::multimap is used to reference children. + */ + +template +class LowMemoryStringMap +{ +public: + typedef typename StringTraits::CharT CharT; + typedef typename StringTraits::StreamT StreamT; + + LowMemoryStringMap() + : m_root(StringT()) + {} + + void clear() + { + m_root.clear(); + } + + bool empty() const + { + return m_root.empty(); + } + + /* + * Adds a new string - value pair to the map. + * NOTE: Adding a previously added string does not replace the value. It can also lead to a second entry, in which + * case it is not defined which value will be retrieved on calling find(). If uniqueness of keys is important + * always call find() first. + */ + void add(const StringT& str, const ValueT& val) + { + if (str.size()) + { + m_uncompressedByteSize += str.size() * sizeof(CharT) + sizeof(StringT) + sizeof(ValueT); + + Branch::addTo(&m_root, str, 0, val); + } + } + + /* + * Finds the value for a string. defaultValue is returned if the string is not found. + */ + ValueT find(const StringT& str) const + { + return m_root.find(str, 0); + } + + void print(StreamT& os) const + { + m_root.print(os, 0); + } + + /* + * Returns the number of bytes used to store this map. + */ + size_t getByteSize() const + { + return m_root.getByteSize(); + } + + /* + * Returns the number of bytes necessary to store all raw string - value pairs that were added. + */ + size_t getUncompressedByteSize() const + { + return m_uncompressedByteSize; + } + +private: + /* + * StringTypes + * + * These types are used to store strings of different lengths with as little memory as possible. + * (std::string allocates a certain default capacity and has multiple members for storing size etc., which take up + * more space than necessary if string length is already known.) + */ + + /* + * LongString + * + * Stores all characters in an array with terminating \0 character to avoid storing its size. + */ + class LongString + { + public: + LongString(const StringT& str) + { + m_str = std::unique_ptr(new CharT[str.size() + 1]); + StringTraits::CopyFn(m_str.get(), str.c_str(), str.size() + 1); + } + + StringT getString() const + { + return StringT(m_str.get()); + } + + std::pair compareString(const StringT& str, size_t idx) const + { + size_t size = StringTraits::SizeFn(m_str.get()); + return std::make_pair(str.compare(idx, size, m_str.get(), size) == 0, size); + } + + size_t getByteSize() const + { + size_t c = 0; + while (m_str.get()[c] != CharT(0)) + { + c++; + } + + return sizeof(CharT) * (c + 1); + } + + private: + std::unique_ptr m_str; + }; + + + /* + * ShortString + * + * Stores all characters in an array without terminating \0. Size is available as template argument. + * + * - Size: the number of characters + */ + template + class ShortString + { + public: + ShortString(const StringT& str) + { + StringTraits::CopyFn(m_str, str.c_str(), Size); + } + + StringT getString() const + { + return StringT(m_str, Size); + } + + std::pair compareString(const StringT& str, size_t idx) const + { + return std::make_pair(str.compare(idx, Size, m_str, Size) == 0, Size); + } + + size_t getByteSize() const + { + return 0; + } + + private: + CharT m_str[Size]; + }; + + /* + * EmptyString + * + * an empty string, taking up no memory. + */ + class EmptyString + { + public: + EmptyString(const StringT& str) {} + + StringT getString() const + { + return StringT(); + } + + std::pair compareString(const StringT& str, size_t idx) const + { + return std::make_pair(true, 0); + } + + size_t getByteSize() const + { + return 0; + } + }; + + + /* + * Nodes + * + * These types are used to build the tree structure. They are either branches or leafes and contain a StringType + */ + + /* + * Node + * + * Base type used to provide common interface of all elements in the tree. + */ + class Node + { + public: + virtual ~Node() = default; + + virtual ValueT getValue() const + { + return defaultVal; + } + + virtual StringT getString() const = 0; + + virtual std::pair compareString(const StringT& str, size_t idx) const = 0; + + virtual size_t getByteSize() const = 0; + + virtual ValueT find(const StringT& str, size_t idx) const = 0; + + virtual void print(StreamT& os, size_t depth) const = 0; + }; + + + /* + * Leaf + * + * Contains value. + */ + class Leaf + : public Node + { + public: + Leaf(const ValueT& val) + : m_value(val) + {} + + ValueT getValue() const override + { + return m_value; + } + + ValueT find(const StringT& str, size_t idx) const override + { + std::pair p = this->compareString(str, idx); + + if (p.first && str.size() - idx == p.second) + { + return getValue(); + } + + return defaultVal; + } + + void print(StreamT& os, size_t depth) const override + { + os << StringT(depth, ' ') << this->getString() << '=' << m_value << std::endl; + } + + private: + ValueT m_value; + }; + + + /* + * StringLeaf + * + * Combines Leaf and StringType to allow for creating leafes of different string lengths, that can still be stored + * in a collection of Node types. + */ + template + class StringLeaf + : public Leaf + , public StringType + { + public: + StringLeaf(const StringT& str, const ValueT& val) + : Leaf(val) + , StringType(str) + {} + + StringT getString() const override + { + return StringType::getString(); + } + + std::pair compareString(const StringT& str, size_t idx) const override + { + return StringType::compareString(str, idx); + } + + size_t getByteSize() const override + { + return sizeof(*this) + StringType::getByteSize(); + } + }; + + + /* + * Branch + * + * Has branches and leaves as children, each referenced by their first character (std::multimap is used, because + * it takes less memory to store multiple short string with an equal prefix in full lenght instead of splitting them + * into a tree). + */ + class Branch + : public Node + { + public: + bool empty() const + { + return m_children.empty(); + } + + void clear() + { + m_children.clear(); + } + + static void addTo(Branch* branch, const StringT& str, size_t idx, const ValueT& val) + { + CharT c(0); + if (idx < str.size()) + { + c = str[idx]; + } + + auto it = branch->m_children.find(c); + if (it == branch->m_children.end() || + (str.size() - idx <= branchSplitThreshold && branch->m_children.count(c) < MAX_EQUAL_RANGE_COUNT)) + { + size_t newIdx = idx + 1 >= str.size() ? str.size() : idx + 1; + branch->m_children.emplace(c, branch->createLeaf(str.substr(newIdx), val)); + return; + } + + idx++; + + std::unique_ptr child = std::move(it->second); + const StringT& childStr = child->getString(); + + auto p = std::mismatch(childStr.begin(), childStr.end(), str.begin() + idx); + + if (p.first == childStr.end() && p.second == str.end()) + { + // adding same string, abort + it->second = std::move(child); + return; + } + + size_t length = std::distance(childStr.begin(), p.first); + + std::unique_ptr newBranch; + if (!dynamic_cast(child.get()) || p.first != childStr.end()) + { + newBranch = branch->split(std::move(child), length); + } + else + { + newBranch = std::unique_ptr(dynamic_cast(child.release())); + } + + addTo(newBranch.get(), str, idx + length, val); + + it->second = std::move(newBranch); + } + + size_t getByteSize() const override + { + size_t s = m_children.size() * sizeof(std::pair>); + + for (const auto& p : m_children) + { + s += p.second->getByteSize(); + } + + return s; + } + + virtual ValueT find(const StringT& str, size_t idx) const override + { + if (idx > str.size()) + { + return defaultVal; + } + + std::pair p = this->compareString(str, idx); + if (!p.first) + { + return defaultVal; + } + idx += p.second; + + CharT c(0); + if (idx < str.size()) + { + c = str[idx]; + idx++; + } + + for (auto it = m_children.find(c); it != m_children.end() && it->first == c; it++) + { + ValueT val = it->second->find(str, idx); + if (val != defaultVal) + { + return val; + } + } + + return defaultVal; + } + + void print(StreamT& os, size_t depth) const override + { + const StringT& myStr = this->getString(); + if (myStr.size()) + { + os << StringT(depth, ' ') << myStr << std::endl; + depth += myStr.size(); + } + + for (const auto& p : m_children) + { + os << StringT(depth, ' ') << '|' << p.first << '|' << std::endl; + p.second->print(os, depth + 3); + } + } + + private: + virtual std::unique_ptr createBranch(const StringT& str) const = 0; + virtual std::unique_ptr createLeaf(const StringT& str, const ValueT& val) const = 0; + + std::unique_ptr split(std::unique_ptr node, size_t idx) const + { + const StringT& str = node->getString(); + std::unique_ptr frontBranch = createBranch(str.substr(0, idx)); + + CharT c(0); + if (idx < str.size()) + { + c = str[idx]; + idx++; + } + + Branch* oldBranch = dynamic_cast(node.get()); + if (oldBranch) + { + std::unique_ptr backBranch = createBranch(str.substr(idx)); + backBranch->m_children = std::move(oldBranch->m_children); + frontBranch->m_children.emplace(c, std::move(backBranch)); + } + else + { + frontBranch->m_children.emplace(c, createLeaf(str.substr(idx), node->getValue())); + } + + return frontBranch; + } + + /* + * Having lots of leaves in the multimap not split into branches makes finding slow. For that reason only a + * certain amount of leaves with the same start character are allowed. + */ + static const size_t MAX_EQUAL_RANGE_COUNT = 10; + + std::multimap> m_children; + }; + + + /* + * StringBranch + * + * Combines Branch and StringType to allow for creating branches of different string lengths, that can still be + * stored in a collection of Node types. + */ + template + class StringBranch + : public Branch + , public StringType + { + public: + StringBranch(const StringT& str) + : StringType(str) + {} + + StringT getString() const override + { + return StringType::getString(); + } + + std::pair compareString(const StringT& str, size_t idx) const override + { + return StringType::compareString(str, idx); + } + + size_t getByteSize() const override + { + return sizeof(*this) + Branch::getByteSize() + StringType::getByteSize(); + } + + private: + virtual std::unique_ptr createBranch(const StringT& str) const override + { + switch (str.size()) + { + case 0: return std::make_unique>(str); + case 1: return std::make_unique>>(str); + case 2: return std::make_unique>>(str); + case 3: return std::make_unique>>(str); + case 4: return std::make_unique>>(str); + case 5: return std::make_unique>>(str); + case 6: return std::make_unique>>(str); + case 7: return std::make_unique>>(str); + case 8: return std::make_unique>>(str); + default: return std::make_unique>(str); + } + } + + virtual std::unique_ptr createLeaf(const StringT& str, const ValueT& val) const override + { + switch (str.size()) + { + case 0: return std::make_unique>(str, val); + case 1: return std::make_unique>>(str, val); + case 2: return std::make_unique>>(str, val); + case 3: return std::make_unique>>(str, val); + case 4: return std::make_unique>>(str, val); + case 5: return std::make_unique>>(str, val); + case 6: return std::make_unique>>(str, val); + case 7: return std::make_unique>>(str, val); + case 8: return std::make_unique>>(str, val); + default: return std::make_unique>(str, val); + } + } + }; + + StringBranch m_root; + + size_t m_uncompressedByteSize = 0; +}; + +#endif // LOW_MEMORY_STRING_MAP_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 2ace570a..e0e96436 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -22,13 +22,14 @@ add_files( GeneratorTestSuite.h GraphTestSuite.h LogManagerTestSuite.h + LowMemoryStringMapTestSuite.h MatrixBaseTestSuite.h MessageQueueTestSuite.h NetworkProtocolHelperTestSuite.h RefreshInfoGeneratorTestSuite.h SearchIndexTestSuite.h SettingsMigratorTestSuite.h - SettingsTestSuite.h + SettingsTestSuite.h SharedMemoryTestSuite.h SonargraphProjectTestSuite.h SourceLocationCollectionTestSuite.h @@ -43,6 +44,6 @@ add_files( Vector2TestSuite.h # Java tests need to be executed last because of some linux related issues. - JavaParserTestSuite.h + JavaParserTestSuite.h JavaIndexSampleProjectsTestSuite.h ) diff --git a/src/test/LowMemoryStringMapTestSuite.h b/src/test/LowMemoryStringMapTestSuite.h new file mode 100644 index 00000000..d9e6e2af --- /dev/null +++ b/src/test/LowMemoryStringMapTestSuite.h @@ -0,0 +1,165 @@ +#include "cxxtest/TestSuite.h" + +#include "utility/LowMemoryStringMap.h" +#include "utility/text/TextAccess.h" +#include "utility/types.h" +#include "utility/utilityString.h" + +class LowMemoryStringMapTestSuite : public CxxTest::TestSuite +{ +public: + void test_roughly_everything() + { + LowMemoryStringMap map; + + map.add("abcdefg", 2); + map.add("abcdefgerlitz", 1); + map.add("abcdefghij", 3); + map.add("abc", 4); + + // map.print(std::cout); + // std::cout << std::endl << map.getByteSize() << " : " << map.getUncompressedByteSize() << std::endl; + + TS_ASSERT(map.find("abcdefgerlitz") == 1); + TS_ASSERT(map.find("abcdefg") == 2); + TS_ASSERT(map.find("abcdefghij") == 3); + TS_ASSERT(map.find("abc") == 4); + TS_ASSERT(map.find("bc") == 0); + TS_ASSERT(map.find("") == 0); + TS_ASSERT(map.find(";asdfl;kjasd;flkasdf") == 0); + + // TS_ASSERT(map.getByteSize() < map.getUncompressedByteSize()); + } + + void test_cannot_find_element_after_creation() + { + LowMemoryStringMap map; + + TS_ASSERT(map.find("a") == 0); + } + + void test_find_element() + { + LowMemoryStringMap map; + map.add("a", 1); + + TS_ASSERT(map.find("a") == 1); + TS_ASSERT(map.find("b") == 0); + } + + void test_find_fully_different_elements() + { + LowMemoryStringMap map; + map.add("a", 1); + map.add("b", 2); + + TS_ASSERT(map.find("a") == 1); + TS_ASSERT(map.find("b") == 2); + } + + void test_find_similar_short_elements() + { + LowMemoryStringMap map; + map.add("ab", 1); + map.add("ac", 2); + + TS_ASSERT(map.find("ab") == 1); + TS_ASSERT(map.find("ac") == 2); + TS_ASSERT(map.find("bc") == 0); + } + + void test_find_similar_long_elements() + { + LowMemoryStringMap map; + map.add("aaaaabbbbb", 1); + map.add("aaaaaccccc", 2); + map.add("aaaaccccc", 3); + map.add("aaaccccc", 4); + + TS_ASSERT(map.find("aaaaabbbbb") == 1); + TS_ASSERT(map.find("aaaaaccccc") == 2); + TS_ASSERT(map.find("aaaaacccccc") == 0); + TS_ASSERT(map.find("aacc") == 0); + } + + void test_add_twice() + { + LowMemoryStringMap map; + map.add("abba", 1); + map.add("abba", 2); + + TS_ASSERT(map.find("abba") == 1); + } + + void test_find_parent_child_elements() + { + LowMemoryStringMap map; + map.add("a", 1); + map.add("ab", 2); + + TS_ASSERT(map.find("a") == 1); + TS_ASSERT(map.find("ab") == 2); + TS_ASSERT(map.find("b") == 0); + } + + void test_find_long_parent_child_elements() + { + LowMemoryStringMap map; + map.add("ababaaa", 1); + map.add("aba", 2); + + TS_ASSERT(map.find("ababaaa") == 1); + TS_ASSERT(map.find("aba") == 2); + TS_ASSERT(map.find("ab") == 0); + TS_ASSERT(map.find("") == 0); + } + + void test_find_long_similar_prefix_elements() + { + LowMemoryStringMap map; + map.add("ababababaab", 1); + map.add("abababababababaccc", 2); + map.add("abababababababaer", 3); + map.add("abababababababber", 4); + map.add("abababababababaaaa", 5); + + // map.print(std::cout); + // std::cout << std::endl << map.getByteSize() << " : " << map.getUncompressedByteSize() << std::endl; + + TS_ASSERT(map.find("ababababaab") == 1); + TS_ASSERT(map.find("abababababababaccc") == 2); + TS_ASSERT(map.find("abababababababaer") == 3); + TS_ASSERT(map.find("abababababababber") == 4); + TS_ASSERT(map.find("abababababababaaaa") == 5); + TS_ASSERT(map.find("abababab") == 0); + } + + void test_wstring() + { + LowMemoryStringMap map; + + FilePath filePath(L"data/LowMemoryStringMapTestSuite/names.txt"); + std::shared_ptr textAccess = TextAccess::createFromFile(filePath); + + std::vector names; + for (std::string line : textAccess->getAllLines()) + { + names.emplace_back(utility::decodeFromUtf8(line.substr(0, line.find("\n")))); + } + + for (size_t i = 0; i < names.size(); i++) + { + map.add(names[i], i); + } + + // map.print(std::wcout); + // std::cout << std::endl << map.getByteSize() << " : " << map.getUncompressedByteSize() << std::endl; + + for (size_t i = 0; i < names.size(); i++) + { + TS_ASSERT(map.find(names[i]) == i); + } + + // TS_ASSERT(map.getByteSize() < map.getUncompressedByteSize()); + } +};