diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp index fd0bf685..d60a9b77 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp @@ -12,6 +12,38 @@ const size_t SqliteIndexStorage::s_storageVersion = 19; + +namespace +{ + std::tuple splitLocalSymbolName(const std::wstring& name) + { + const std::tuple res = std::make_tuple(L"", 0, 0); + + size_t pos = name.find_last_of(L'<'); + if (pos == std::wstring::npos) + { + return res; + } + + size_t pos2 = name.find(L':', pos + 1); + if (pos2 == std::wstring::npos) + { + return res; + } + + if (name.back() != L'>') + { + return res; + } + + return std::tuple( + name.substr(0, pos), + std::stoi(name.substr(pos + 1, pos2 - pos - 1)), + std::stoi(name.substr(pos2 + 1, name.size() - pos2 - 2)) + ); + } +} + SqliteIndexStorage::SqliteIndexStorage(const FilePath& dbFilePath) : SqliteStorage(dbFilePath.getCanonical()) { @@ -28,8 +60,7 @@ void SqliteIndexStorage::setMode(const StorageModeType mode) m_tempWNodeNameIndex.clear(); m_tempNodeTypes.clear(); m_tempEdgeIndex.clear(); - m_tempLocalSymbolNameIndex.clear(); - m_tempWLocalSymbolNameIndex.clear(); + m_tempLocalSymbolIndex.clear(); m_tempSourceLocationIndices.clear(); std::vector> indices = getIndices(); @@ -219,37 +250,33 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data) StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data) { - if (m_tempLocalSymbolNameIndex.empty() && m_tempWLocalSymbolNameIndex.empty()) + std::wstring name; + uint32_t line; + uint32_t col; + + if (m_tempLocalSymbolIndex.empty()) { for (const StorageLocalSymbol& localSymbol : getAll()) { - std::string name = utility::encodeToUtf8(localSymbol.name); - if (name.size() != localSymbol.name.size()) + std::tie(name, line, col) = splitLocalSymbolName(localSymbol.name); + if (name.size()) { - m_tempWLocalSymbolNameIndex.add(localSymbol.name, localSymbol.id); - } - else - { - m_tempLocalSymbolNameIndex.add(name, localSymbol.id); + m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), localSymbol.id); } } } - std::string name = utility::encodeToUtf8(data.name); + std::tie(name, line, col) = splitLocalSymbolName(data.name); + if (name.size()) { - Id localSymbolId; - if (name.size() != data.name.size()) + auto it = m_tempLocalSymbolIndex.find(name); + if (it != m_tempLocalSymbolIndex.end()) { - localSymbolId = m_tempWLocalSymbolNameIndex.find(data.name); - } - else - { - localSymbolId = m_tempLocalSymbolNameIndex.find(name); - } - - if (localSymbolId) - { - return StorageLocalSymbol(localSymbolId, data); + auto it2 = it->second.find(std::make_pair(line, col)); + if (it2 != it->second.end()) + { + return StorageLocalSymbol(it2->second, data); + } } } @@ -261,18 +288,14 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa } { m_inserLocalSymbolStmt.bind(1, int(id)); - m_inserLocalSymbolStmt.bind(2, name.c_str()); + m_inserLocalSymbolStmt.bind(2, utility::encodeToUtf8(data.name).c_str()); executeStatement(m_inserLocalSymbolStmt); m_inserLocalSymbolStmt.reset(); } - if (name.size() != data.name.size()) + if (name.size()) { - m_tempWLocalSymbolNameIndex.add(data.name, id); - } - else - { - m_tempLocalSymbolNameIndex.add(name, id); + m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id); } return StorageLocalSymbol(id, data); diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.h b/src/lib/data/storage/sqlite/SqliteIndexStorage.h index 45e15ee4..200b4da6 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.h +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.h @@ -216,12 +216,11 @@ private: return ResultType(); } - LowMemoryStringMap m_tempNodeNameIndex; - LowMemoryStringMap m_tempWNodeNameIndex; + LowMemoryStringMap m_tempNodeNameIndex; + LowMemoryStringMap m_tempWNodeNameIndex; std::map m_tempNodeTypes; std::map m_tempEdgeIndex; - LowMemoryStringMap m_tempLocalSymbolNameIndex; - LowMemoryStringMap m_tempWLocalSymbolNameIndex; + std::map, Id>> m_tempLocalSymbolIndex; std::map> m_tempSourceLocationIndices; CppSQLite3Statement m_insertElementStmt; diff --git a/src/lib/utility/LowMemoryStringMap.h b/src/lib/utility/LowMemoryStringMap.h index b68b94f7..05efdc0a 100644 --- a/src/lib/utility/LowMemoryStringMap.h +++ b/src/lib/utility/LowMemoryStringMap.h @@ -59,19 +59,12 @@ public: * Map of string - value pairs, where equal suffixes of strings are used to build tree structure reducing memory * consumption. * - * NOTE: It is not guaranteed that equal strings cannot be added twice. If uniqueness of strings is important, use - * find() first to check if the string was already added. - * * - StringT: string type (supported: std::string, std::wstring) * - ValueT: value type * - defaultVal: default value of type ValueT (cannot be stored as value) - * - branchSplitThreshold: minimum number of characters necessary to split remaining string part into another branch. - * (e.g. storing 'code' and 'copy' as two strings takes up less space than storing them as tree 'co' -> 'de' | 'py', - * because that also needs pointers and a map. For that reason strings are only split into branches if the newly - * added one is above this threshold. This is also the reason std::multimap is used to reference children. */ -template +template class LowMemoryStringMap { public: @@ -94,9 +87,6 @@ public: /* * Adds a new string - value pair to the map. - * NOTE: Adding a previously added string does not replace the value. It can also lead to a second entry, in which - * case it is not defined which value will be retrieved on calling find(). If uniqueness of keys is important - * always call find() first. */ void add(const StringT& str, const ValueT& val) { @@ -359,9 +349,7 @@ private: /* * Branch * - * Has branches and leaves as children, each referenced by their first character (std::multimap is used, because - * it takes less memory to store multiple short string with an equal prefix in full lenght instead of splitting them - * into a tree). + * Has branches and leaves as children, each referenced by their first character. */ class Branch : public Node @@ -386,8 +374,7 @@ private: } auto it = branch->m_children.find(c); - if (it == branch->m_children.end() || - (str.size() - idx <= branchSplitThreshold && branch->m_children.count(c) < MAX_EQUAL_RANGE_COUNT)) + if (it == branch->m_children.end()) { size_t newIdx = idx + 1 >= str.size() ? str.size() : idx + 1; branch->m_children.emplace(c, branch->createLeaf(str.substr(newIdx), val)); @@ -458,13 +445,10 @@ private: idx++; } - for (auto it = m_children.find(c); it != m_children.end() && it->first == c; it++) + auto it = m_children.find(c); + if (it != m_children.end()) { - ValueT val = it->second->find(str, idx); - if (val != defaultVal) - { - return val; - } + return it->second->find(str, idx); } return defaultVal; @@ -517,13 +501,7 @@ private: return frontBranch; } - /* - * Having lots of leaves in the multimap not split into branches makes finding slow. For that reason only a - * certain amount of leaves with the same start character are allowed. - */ - static const size_t MAX_EQUAL_RANGE_COUNT = 10; - - std::multimap> m_children; + std::map> m_children; };