ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: Removed std::multimap optimization from LowMemoryStringMap and store local symbol names in std::map

Browse Source
This commit is contained in:
Eberhard Graether
2018-09-18 12:32:55 +02:00
parent 9f453ef376
commit 07454f2045
3 changed files with 63 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/storage/sqlite/SqliteIndexStorage.cpp
+53 -30
View File
@@ -12,6 +12,38 @@
const size_t SqliteIndexStorage::s_storageVersion = 19;
namespace
{
std::tuple<std::wstring, uint32_t, uint32_t> splitLocalSymbolName(const std::wstring& name)
{
const std::tuple<std::wstring, uint32_t, uint32_t> res = std::make_tuple(L"", 0, 0);
size_t pos = name.find_last_of(L'<');
if (pos == std::wstring::npos)
{
return res;
}
size_t pos2 = name.find(L':', pos + 1);
if (pos2 == std::wstring::npos)
{
return res;
}
if (name.back() != L'>')
{
return res;
}
return std::tuple<std::wstring, uint32_t, uint32_t>(
name.substr(0, pos),
std::stoi(name.substr(pos + 1, pos2 - pos - 1)),
std::stoi(name.substr(pos2 + 1, name.size() - pos2 - 2))
);
}
}
SqliteIndexStorage::SqliteIndexStorage(const FilePath& dbFilePath)
: SqliteStorage(dbFilePath.getCanonical())
{
@@ -28,8 +60,7 @@ void SqliteIndexStorage::setMode(const StorageModeType mode)
m_tempWNodeNameIndex.clear();
m_tempNodeTypes.clear();
m_tempEdgeIndex.clear();
m_tempLocalSymbolNameIndex.clear();
m_tempWLocalSymbolNameIndex.clear();
m_tempLocalSymbolIndex.clear();
m_tempSourceLocationIndices.clear();
std::vector<std::pair<int, SqliteDatabaseIndex>> indices = getIndices();
@@ -219,37 +250,33 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data)
StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data)
{
if (m_tempLocalSymbolNameIndex.empty() && m_tempWLocalSymbolNameIndex.empty())
std::wstring name;
uint32_t line;
uint32_t col;
if (m_tempLocalSymbolIndex.empty())
{
for (const StorageLocalSymbol& localSymbol : getAll<StorageLocalSymbol>())
{
std::string name = utility::encodeToUtf8(localSymbol.name);
if (name.size() != localSymbol.name.size())
std::tie(name, line, col) = splitLocalSymbolName(localSymbol.name);
if (name.size())
{
m_tempWLocalSymbolNameIndex.add(localSymbol.name, localSymbol.id);
}
else
{
m_tempLocalSymbolNameIndex.add(name, localSymbol.id);
m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), localSymbol.id);
}
}
}
std::string name = utility::encodeToUtf8(data.name);
std::tie(name, line, col) = splitLocalSymbolName(data.name);
if (name.size())
{
Id localSymbolId;
if (name.size() != data.name.size())
auto it = m_tempLocalSymbolIndex.find(name);
if (it != m_tempLocalSymbolIndex.end())
{
localSymbolId = m_tempWLocalSymbolNameIndex.find(data.name);
}
else
{
localSymbolId = m_tempLocalSymbolNameIndex.find(name);
}
if (localSymbolId)
{
return StorageLocalSymbol(localSymbolId, data);
auto it2 = it->second.find(std::make_pair(line, col));
if (it2 != it->second.end())
{
return StorageLocalSymbol(it2->second, data);
}
}
}
@@ -261,18 +288,14 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa
}
{
m_inserLocalSymbolStmt.bind(1, int(id));
m_inserLocalSymbolStmt.bind(2, name.c_str());
m_inserLocalSymbolStmt.bind(2, utility::encodeToUtf8(data.name).c_str());
executeStatement(m_inserLocalSymbolStmt);
m_inserLocalSymbolStmt.reset();
}
if (name.size() != data.name.size())
if (name.size())
{
m_tempWLocalSymbolNameIndex.add(data.name, id);
}
else
{
m_tempLocalSymbolNameIndex.add(name, id);
m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id);
}
return StorageLocalSymbol(id, data);
src/lib/data/storage/sqlite/SqliteIndexStorage.h
+3 -4
View File
@@ -216,12 +216,11 @@ private:
return ResultType();
}
LowMemoryStringMap<std::string, Id, 0, 32> m_tempNodeNameIndex;
LowMemoryStringMap<std::wstring, Id, 0, 32> m_tempWNodeNameIndex;
LowMemoryStringMap<std::string, Id, 0> m_tempNodeNameIndex;
LowMemoryStringMap<std::wstring, Id, 0> m_tempWNodeNameIndex;
std::map<Id, int> m_tempNodeTypes;
std::map<StorageEdgeData, Id> m_tempEdgeIndex;
LowMemoryStringMap<std::string, Id, 0, 8> m_tempLocalSymbolNameIndex;
LowMemoryStringMap<std::wstring, Id, 0, 8> m_tempWLocalSymbolNameIndex;
std::map<std::wstring, std::map<std::pair<uint32_t, uint32_t>, Id>> m_tempLocalSymbolIndex;
std::map<Id, std::map<TempSourceLocation, Id>> m_tempSourceLocationIndices;
CppSQLite3Statement m_insertElementStmt;
src/lib/utility/LowMemoryStringMap.h
+7 -29
View File
@@ -59,19 +59,12 @@ public:
* Map of string - value pairs, where equal suffixes of strings are used to build tree structure reducing memory
* consumption.
*
* NOTE: It is not guaranteed that equal strings cannot be added twice. If uniqueness of strings is important, use
* find() first to check if the string was already added.
*
* - StringT: string type (supported: std::string, std::wstring)
* - ValueT: value type
* - defaultVal: default value of type ValueT (cannot be stored as value)
* - branchSplitThreshold: minimum number of characters necessary to split remaining string part into another branch.
* (e.g. storing 'code' and 'copy' as two strings takes up less space than storing them as tree 'co' -> 'de' | 'py',
* because that also needs pointers and a map. For that reason strings are only split into branches if the newly
* added one is above this threshold. This is also the reason std::multimap is used to reference children.
*/
template <typename StringT, typename ValueT, ValueT defaultVal, size_t branchSplitThreshold = 8>
template <typename StringT, typename ValueT, ValueT defaultVal>
class LowMemoryStringMap
{
public:
@@ -94,9 +87,6 @@ public:
/*
* Adds a new string - value pair to the map.
* NOTE: Adding a previously added string does not replace the value. It can also lead to a second entry, in which
* case it is not defined which value will be retrieved on calling find(). If uniqueness of keys is important
* always call find() first.
*/
void add(const StringT& str, const ValueT& val)
{
@@ -359,9 +349,7 @@ private:
/*
* Branch
*
* Has branches and leaves as children, each referenced by their first character (std::multimap is used, because
* it takes less memory to store multiple short string with an equal prefix in full lenght instead of splitting them
* into a tree).
* Has branches and leaves as children, each referenced by their first character.
*/
class Branch
: public Node
@@ -386,8 +374,7 @@ private:
}
auto it = branch->m_children.find(c);
if (it == branch->m_children.end() ||
(str.size() - idx <= branchSplitThreshold && branch->m_children.count(c) < MAX_EQUAL_RANGE_COUNT))
if (it == branch->m_children.end())
{
size_t newIdx = idx + 1 >= str.size() ? str.size() : idx + 1;
branch->m_children.emplace(c, branch->createLeaf(str.substr(newIdx), val));
@@ -458,13 +445,10 @@ private:
idx++;
}
for (auto it = m_children.find(c); it != m_children.end() && it->first == c; it++)
auto it = m_children.find(c);
if (it != m_children.end())
{
ValueT val = it->second->find(str, idx);
if (val != defaultVal)
{
return val;
}
return it->second->find(str, idx);
}
return defaultVal;
@@ -517,13 +501,7 @@ private:
return frontBranch;
}
/*
* Having lots of leaves in the multimap not split into branches makes finding slow. For that reason only a
* certain amount of leaves with the same start character are allowed.
*/
static const size_t MAX_EQUAL_RANGE_COUNT = 10;
std::multimap<CharT, std::unique_ptr<Node>> m_children;
std::map<CharT, std::unique_ptr<Node>> m_children;
};