logic: improved performance of storing index to database

* add compiled statement for batch inserting 100 occurrences at once
* removed unnecessary primary key from component_access to get rid of additional index table
* changed explicit check for existing symbols when injecting to implicit check in insert statement (insert or ignore)
* improved performance of adding edges by replacing edge multi part index with in-memory-index
* added in-memory-index for node table
* added in-memory-index for local_symbol table
* added in-memory-index for lource_location table
* moved setting storage mode to SqliteIndexStorage, because storage modes are not used anywhere else.
* don't store current mode as member because re-creating an existing index takes no time.
* add getter for name of SqliteDatabaseIndex
* removed unused method from persistent storage
This commit is contained in:
mlangkabel
2018-08-10 15:37:08 +02:00
parent 52d0236f9a
commit 337cf2d371
22 changed files with 262 additions and 265 deletions
-2
View File
@@ -45,8 +45,6 @@ public:
virtual NodeType getNodeTypeForNodeWithId(Id id) const = 0;
virtual Id getIdForEdge(
Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const = 0;
virtual StorageEdge getEdgeById(Id edgeId) const = 0;
virtual std::shared_ptr<SourceLocationCollection> getFullTextSearchLocations(
@@ -78,7 +78,6 @@ typedef std::map<Id, std::pair<Id, NameHierarchy>> NodeIdToParentFileMap;
DEF_GETTER_1(getNodeIdToParentFileMap, const std::vector<Id>&, NodeIdToParentFileMap, {})
DEF_GETTER_1(getNodeTypeForNodeWithId, Id, NodeType, NodeType(NodeType::NODE_SYMBOL))
DEF_GETTER_3(getIdForEdge, Edge::EdgeType, const NameHierarchy&, const NameHierarchy&, Id, 0)
DEF_GETTER_1(getEdgeById, Id, StorageEdge, StorageEdge())
DEF_GETTER_2(getFullTextSearchLocations, const std::wstring &, bool, std::shared_ptr<SourceLocationCollection>, std::make_shared<SourceLocationCollection>())
DEF_GETTER_2(getAutocompletionMatches, const std::wstring &, NodeTypeSet, std::vector<SearchMatch>, std::vector<SearchMatch>())
-2
View File
@@ -24,8 +24,6 @@ public:
NodeType getNodeTypeForNodeWithId(Id id) const override;
Id getIdForEdge(
Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const override;
StorageEdge getEdgeById(Id edgeId) const override;
std::shared_ptr<SourceLocationCollection> getFullTextSearchLocations(
@@ -174,9 +174,9 @@ void SharedIntermediateStorage::setStorageOccurrences(const std::vector<StorageO
}
}
std::vector<StorageComponentAccessData> SharedIntermediateStorage::getStorageComponentAccesses() const
std::vector<StorageComponentAccess> SharedIntermediateStorage::getStorageComponentAccesses() const
{
std::vector<StorageComponentAccessData> result;
std::vector<StorageComponentAccess> result;
for (unsigned int i = 0; i < m_storageComponentAccesses.size(); i++)
{
@@ -186,7 +186,7 @@ std::vector<StorageComponentAccessData> SharedIntermediateStorage::getStorageCom
return result;
}
void SharedIntermediateStorage::setStorageComponentAccesses(const std::vector<StorageComponentAccessData>& storageComponentAccesses)
void SharedIntermediateStorage::setStorageComponentAccesses(const std::vector<StorageComponentAccess>& storageComponentAccesses)
{
m_storageComponentAccesses.clear();
@@ -31,8 +31,8 @@ public:
std::vector<StorageOccurrence> getStorageOccurrences() const;
void setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurences);
std::vector<StorageComponentAccessData> getStorageComponentAccesses() const;
void setStorageComponentAccesses(const std::vector<StorageComponentAccessData>& storageComponentAccesses);
std::vector<StorageComponentAccess> getStorageComponentAccesses() const;
void setStorageComponentAccesses(const std::vector<StorageComponentAccess>& storageComponentAccesses);
std::vector<StorageCommentLocationData> getStorageCommentLocations() const;
void setStorageCommentLocations(const std::vector<StorageCommentLocationData>& commentLocations);
@@ -47,7 +47,7 @@ private:
SharedMemory::Vector<SharedStorageFile> m_storageFiles;
SharedMemory::Vector<SharedStorageSymbol> m_storageSymbols;
SharedMemory::Vector<SharedStorageOccurrence> m_storageOccurrences;
SharedMemory::Vector<SharedStorageComponentAccessData> m_storageComponentAccesses;
SharedMemory::Vector<SharedStorageComponentAccess> m_storageComponentAccesses;
SharedMemory::Vector<SharedStorageCommentLocationData> m_storageCommentLocations;
SharedMemory::Vector<SharedStorageNode> m_storageNodes;
SharedMemory::Vector<SharedStorageEdge> m_storageEdges;
@@ -36,8 +36,8 @@ CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageEdge, SharedStorageEdge )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageSymbol, SharedStorageSymbol )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageSourceLocation, SharedStorageSourceLocation )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageOccurrence, SharedStorageOccurrence )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageComponentAccessData, SharedStorageComponentAccessData)
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageCommentLocationData, SharedStorageCommentLocationData)
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageComponentAccess, SharedStorageComponentAccess)
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageCommentLocationData, SharedStorageCommentLocationData)
struct SharedStorageNode
@@ -151,9 +151,9 @@ struct SharedStorageErrorData
inline SharedStorageErrorData toShared(const StorageErrorData& error, SharedMemory::Allocator* allocator)
{
return SharedStorageErrorData(
utility::encodeToUtf8(error.message),
utility::encodeToUtf8(error.message),
utility::encodeToUtf8(error.filePath),
error.lineNumber,
error.lineNumber,
error.columnNumber,
utility::encodeToUtf8(error.translationUnit),
error.fatal,
@@ -164,12 +164,12 @@ inline SharedStorageErrorData toShared(const StorageErrorData& error, SharedMemo
inline StorageErrorData fromShared(const SharedStorageErrorData& error)
{
return StorageErrorData(
utility::decodeFromUtf8(error.message.c_str()),
utility::decodeFromUtf8(error.message.c_str()),
utility::decodeFromUtf8(error.filePath.c_str()),
error.lineNumber,
error.columnNumber,
error.columnNumber,
utility::decodeFromUtf8(error.translationUnit.c_str()),
error.fatal,
error.fatal,
error.indexed
);
}
+1 -1
View File
@@ -317,7 +317,7 @@ void ParserClientImpl::addComponentAccess(Id nodeId , int type)
return;
}
m_storage->addComponentAccess(StorageComponentAccessData(nodeId, type));
m_storage->addComponentAccess(StorageComponentAccess(nodeId, type));
}
void ParserClientImpl::addCommentLocation(const ParseLocation& location)
+9 -10
View File
@@ -56,7 +56,7 @@ size_t IntermediateStorage::getByteSize(size_t stringSize) const
byteSize += sizeof(StorageEdge) * getStorageEdges().size();
byteSize += sizeof(StorageCommentLocationData) * getCommentLocations().size();
byteSize += sizeof(StorageComponentAccessData) * getComponentAccesses().size();
byteSize += sizeof(StorageComponentAccess) * getComponentAccesses().size();
byteSize += sizeof(StorageOccurrence) * getStorageOccurrences().size();
byteSize += sizeof(StorageSymbol) * getStorageSymbols().size();
byteSize += sizeof(StorageSourceLocation) * getStorageSourceLocations().size();
@@ -147,7 +147,6 @@ void IntermediateStorage::addFile(const StorageFile& file)
Id IntermediateStorage::addEdge(const StorageEdgeData& edgeData)
{
const std::wstring serialized = serialize(edgeData);
std::unordered_map<std::wstring, size_t>::const_iterator it = m_edgesIndex.find(serialized);
if (it != m_edgesIndex.end())
@@ -209,13 +208,13 @@ void IntermediateStorage::addOccurrences(const std::vector<StorageOccurrence>& o
}
}
void IntermediateStorage::addComponentAccess(const StorageComponentAccessData& componentAccessData)
void IntermediateStorage::addComponentAccess(const StorageComponentAccess& componentAccess)
{
const std::wstring serialized = serialize(componentAccessData);
const std::wstring serialized = serialize(componentAccess);
if (m_serializedComponentAccesses.find(serialized) == m_serializedComponentAccesses.end())
{
m_componentAccesses.push_back(componentAccessData);
m_componentAccesses.push_back(componentAccess);
m_serializedComponentAccesses.insert(serialized);
}
}
@@ -300,9 +299,9 @@ void IntermediateStorage::forEachOccurrence(std::function<void(const StorageOccu
}
}
void IntermediateStorage::forEachComponentAccess(std::function<void(const StorageComponentAccessData& /*data*/)> callback) const
void IntermediateStorage::forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const
{
for (std::vector<StorageComponentAccessData>::const_iterator it = m_componentAccesses.begin(); it != m_componentAccesses.end(); it++)
for (std::vector<StorageComponentAccess>::const_iterator it = m_componentAccesses.begin(); it != m_componentAccesses.end(); it++)
{
callback(*it);
}
@@ -371,7 +370,7 @@ std::vector<StorageOccurrence> IntermediateStorage::getStorageOccurrences() cons
return m_occurrences;
}
std::vector<StorageComponentAccessData> IntermediateStorage::getComponentAccesses() const
std::vector<StorageComponentAccess> IntermediateStorage::getComponentAccesses() const
{
return m_componentAccesses;
}
@@ -441,7 +440,7 @@ void IntermediateStorage::setStorageOccurrences(const std::vector<StorageOccurre
m_occurrences = storageOccurrences;
}
void IntermediateStorage::setComponentAccesses(const std::vector<StorageComponentAccessData>& componentAccesses)
void IntermediateStorage::setComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses)
{
m_componentAccesses = componentAccesses;
}
@@ -508,7 +507,7 @@ std::wstring IntermediateStorage::serialize(const StorageOccurrence& occurrence)
}
std::wstring IntermediateStorage::serialize(const StorageComponentAccessData& componentAccessData) const
std::wstring IntermediateStorage::serialize(const StorageComponentAccess& componentAccessData) const
{
return std::to_wstring(componentAccessData.nodeId);
}
+6 -6
View File
@@ -39,7 +39,7 @@ public:
Id addSourceLocation(const StorageSourceLocationData& sourceLocationData) override;
void addOccurrence(const StorageOccurrence& occurrence) override;
void addOccurrences(const std::vector<StorageOccurrence>& occurrences) override;
void addComponentAccess(const StorageComponentAccessData& componentAccessData) override;
void addComponentAccess(const StorageComponentAccess& componentAccess) override;
void addCommentLocation(const StorageCommentLocationData& commentLocationData) override;
void addError(const StorageErrorData& errorData) override;
@@ -50,7 +50,7 @@ public:
void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const override;
void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const override;
void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const override;
void forEachComponentAccess(std::function<void(const StorageComponentAccessData& /*data*/)> callback) const override;
void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const override;
void forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const override;
void forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const override;
@@ -63,7 +63,7 @@ public:
std::vector<StorageLocalSymbol> getStorageLocalSymbols() const;
std::vector<StorageSourceLocation> getStorageSourceLocations() const;
std::vector<StorageOccurrence> getStorageOccurrences() const;
std::vector<StorageComponentAccessData> getComponentAccesses() const;
std::vector<StorageComponentAccess> getComponentAccesses() const;
std::vector<StorageCommentLocationData> getCommentLocations() const;
std::vector<StorageErrorData> getErrors() const;
@@ -74,7 +74,7 @@ public:
void setStorageLocalSymbols(const std::vector<StorageLocalSymbol>& storageLocalSymbols);
void setStorageSourceLocations(const std::vector<StorageSourceLocation>& storageSourceLocations);
void setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurrences);
void setComponentAccesses(const std::vector<StorageComponentAccessData>& componentAccesses);
void setComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses);
void setCommentLocations(const std::vector<StorageCommentLocationData>& commentLocations);
void setErrors(const std::vector<StorageErrorData>& errors);
@@ -88,7 +88,7 @@ private:
std::wstring serialize(const StorageLocalSymbolData& localSymbolData) const;
std::wstring serialize(const StorageSourceLocationData& sourceLocationData) const;
std::wstring serialize(const StorageOccurrence& occurrence) const;
std::wstring serialize(const StorageComponentAccessData& componentAccessData) const;
std::wstring serialize(const StorageComponentAccess& componentAccessData) const;
std::wstring serialize(const StorageCommentLocationData& commentLocationData) const;
std::wstring serialize(const StorageErrorData& errorData) const;
@@ -111,7 +111,7 @@ private:
std::vector<StorageOccurrence> m_occurrences;
std::unordered_set<std::wstring> m_serializedComponentAccesses; // this is used to prevent duplicates (unique)
std::vector<StorageComponentAccessData> m_componentAccesses;
std::vector<StorageComponentAccess> m_componentAccesses;
std::unordered_set<std::wstring> m_serializedCommentLocations; // this is used to prevent duplicates (unique)
std::vector<StorageCommentLocationData> m_commentLocations;
+7 -42
View File
@@ -48,28 +48,12 @@ PersistentStorage::PersistentStorage(const FilePath& dbPath, const FilePath& boo
Id PersistentStorage::addNode(const StorageNodeData& data)
{
const StorageNode storedNode = m_sqliteIndexStorage.getNodeBySerializedName(data.serializedName);
if (storedNode.id == 0)
{
return m_sqliteIndexStorage.addNode(data).id;
}
if (storedNode.type < data.type)
{
m_sqliteIndexStorage.setNodeType(data.type, storedNode.id);
return storedNode.id;
}
return storedNode.id;
return m_sqliteIndexStorage.addNode(data).id;
}
void PersistentStorage::addSymbol(const StorageSymbol& data)
{
if (m_sqliteIndexStorage.getFirstById<StorageSymbol>(data.id).id == 0)
{
m_sqliteIndexStorage.addSymbol(data);
}
m_sqliteIndexStorage.addSymbol(data);
}
void PersistentStorage::addFile(const StorageFile& data)
@@ -96,22 +80,12 @@ void PersistentStorage::addFile(const StorageFile& data)
Id PersistentStorage::addEdge(const StorageEdgeData& data)
{
const StorageEdge storedEdge = m_sqliteIndexStorage.getEdgeBySourceTargetType(data.sourceNodeId, data.targetNodeId, data.type);
if (storedEdge.id == 0)
{
return m_sqliteIndexStorage.addEdge(data).id;
}
return storedEdge.id;
return m_sqliteIndexStorage.addEdge(data).id;
}
Id PersistentStorage::addLocalSymbol(const StorageLocalSymbolData& data)
{
const StorageLocalSymbol storedLocalSymbol = m_sqliteIndexStorage.getLocalSymbolByName(data.name);
if (storedLocalSymbol.id == 0)
{
return m_sqliteIndexStorage.addLocalSymbol(data).id;
}
return storedLocalSymbol.id;
return m_sqliteIndexStorage.addLocalSymbol(data).id;
}
Id PersistentStorage::addSourceLocation(const StorageSourceLocationData& data)
@@ -129,9 +103,9 @@ void PersistentStorage::addOccurrences(const std::vector<StorageOccurrence>& occ
m_sqliteIndexStorage.addOccurrences(occurrences);
}
void PersistentStorage::addComponentAccess(const StorageComponentAccessData& data)
void PersistentStorage::addComponentAccess(const StorageComponentAccess& componentAccess)
{
m_sqliteIndexStorage.addComponentAccess(data);
m_sqliteIndexStorage.addComponentAccess(componentAccess);
}
void PersistentStorage::addCommentLocation(const StorageCommentLocationData& data)
@@ -200,7 +174,7 @@ void PersistentStorage::forEachOccurrence(std::function<void(const StorageOccurr
}
}
void PersistentStorage::forEachComponentAccess(std::function<void(const StorageComponentAccessData& /*data*/)> callback) const
void PersistentStorage::forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const
{
for (StorageComponentAccess& componentAccess: m_sqliteIndexStorage.getAll<StorageComponentAccess>())
{
@@ -540,15 +514,6 @@ NodeType PersistentStorage::getNodeTypeForNodeWithId(Id nodeId) const
return NodeType::intToType(m_sqliteIndexStorage.getFirstById<StorageNode>(nodeId).type);
}
Id PersistentStorage::getIdForEdge(
Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy
) const
{
const Id sourceId = getNodeIdForNameHierarchy(fromNameHierarchy);
const Id targetId = getNodeIdForNameHierarchy(toNameHierarchy);
return m_sqliteIndexStorage.getEdgeBySourceTargetType(sourceId, targetId, type).id;
}
StorageEdge PersistentStorage::getEdgeById(Id edgeId) const
{
return m_sqliteIndexStorage.getEdgeById(edgeId);
+2 -4
View File
@@ -27,7 +27,7 @@ public:
Id addSourceLocation(const StorageSourceLocationData& data) override;
void addOccurrence(const StorageOccurrence& data) override;
void addOccurrences(const std::vector<StorageOccurrence>& occurrences) override;
void addComponentAccess(const StorageComponentAccessData& data) override;
void addComponentAccess(const StorageComponentAccess& componentAccess) override;
void addCommentLocation(const StorageCommentLocationData& data) override;
void addError(const StorageErrorData& data) override;
@@ -38,7 +38,7 @@ public:
void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const override;
void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const override;
void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const override;
void forEachComponentAccess(std::function<void(const StorageComponentAccessData& /*data*/)> callback) const override;
void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const override;
void forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const override;
void forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const override;
@@ -85,8 +85,6 @@ public:
NodeType getNodeTypeForNodeWithId(Id nodeId) const override;
Id getIdForEdge(
Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const override;
StorageEdge getEdgeById(Id edgeId) const override;
std::shared_ptr<SourceLocationCollection> getFullTextSearchLocations(
+2 -2
View File
@@ -168,7 +168,7 @@ void Storage::inject(Storage* injected)
}
injected->forEachComponentAccess(
[&](const StorageComponentAccessData& injectedData)
[&](const StorageComponentAccess& injectedData)
{
std::unordered_map<Id, Id>::const_iterator it;
it = injectedIdToOwnElementId.find(injectedData.nodeId);
@@ -178,7 +178,7 @@ void Storage::inject(Storage* injected)
}
const Id ownNodeId = it->second;
addComponentAccess(StorageComponentAccessData(ownNodeId, injectedData.type));
addComponentAccess(StorageComponentAccess(ownNodeId, injectedData.type));
}
);
+2 -2
View File
@@ -31,7 +31,7 @@ public:
virtual Id addSourceLocation(const StorageSourceLocationData& data) = 0;
virtual void addOccurrence(const StorageOccurrence& data) = 0;
virtual void addOccurrences(const std::vector<StorageOccurrence>& occurrences) = 0;
virtual void addComponentAccess(const StorageComponentAccessData& data) = 0;
virtual void addComponentAccess(const StorageComponentAccess& componentAccess) = 0;
virtual void addCommentLocation(const StorageCommentLocationData& data) = 0;
virtual void addError(const StorageErrorData& data) = 0;
@@ -42,7 +42,7 @@ public:
virtual void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const = 0;
virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const = 0;
virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const = 0;
virtual void forEachComponentAccess(std::function<void(const StorageComponentAccessData& /*data*/)> callback) const = 0;
virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const = 0;
virtual void forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const = 0;
virtual void forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const = 0;
@@ -8,8 +8,9 @@ SqliteDatabaseIndex::SqliteDatabaseIndex(const std::string& indexName, const std
{
}
SqliteDatabaseIndex::~SqliteDatabaseIndex()
std::string SqliteDatabaseIndex::getName() const
{
return m_indexName;
}
void SqliteDatabaseIndex::createOnDatabase(CppSQLite3DB& database)
@@ -8,7 +8,8 @@ class SqliteDatabaseIndex
{
public:
SqliteDatabaseIndex(const std::string& indexName, const std::string& indexTarget);
~SqliteDatabaseIndex();
std::string getName() const;
void createOnDatabase(CppSQLite3DB& database);
void removeFromDatabase(CppSQLite3DB& database);
+182 -101
View File
@@ -9,7 +9,28 @@
#include "data/location/SourceLocationCollection.h"
#include "data/location/SourceLocationFile.h"
const size_t SqliteIndexStorage::s_storageVersion = 17;
namespace
{
std::string serialize(const StorageEdgeData& data)
{
return (
std::to_string(data.sourceNodeId) + "," +
std::to_string(data.targetNodeId) + "," +
std::to_string(data.type)
);
}
std::string serialize(const StorageSourceLocationData& loc)
{
return (
std::to_string(loc.startLine) + "," + std::to_string(loc.startCol) + "," +
std::to_string(loc.endLine) + "," + std::to_string(loc.endCol) + "," +
std::to_string(loc.type)
);
}
}
const size_t SqliteIndexStorage::s_storageVersion = 18;
SqliteIndexStorage::SqliteIndexStorage(const FilePath& dbFilePath)
: SqliteStorage(dbFilePath.getCanonical())
@@ -21,6 +42,27 @@ size_t SqliteIndexStorage::getStaticVersion() const
return s_storageVersion;
}
void SqliteIndexStorage::setMode(const StorageModeType mode)
{
m_tempNodeIndex.clear();
m_tempEdgeIndex.clear();
m_tempLocalSymbolIndex.clear();
m_tempSourceLocationIndices.clear();
std::vector<std::pair<int, SqliteDatabaseIndex>> indices = getIndices();
for (size_t i = 0; i < indices.size(); i++)
{
if (indices[i].first & mode)
{
indices[i].second.createOnDatabase(m_database);
}
else
{
indices[i].second.removeFromDatabase(m_database);
}
}
}
std::string SqliteIndexStorage::getProjectSettingsText() const
{
return getMetaValue("project_settings");
@@ -33,6 +75,27 @@ void SqliteIndexStorage::setProjectSettingsText(std::string text)
StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data)
{
if (m_tempNodeIndex.empty())
{
for (const StorageNode& node : getAll<StorageNode>())
{
m_tempNodeIndex[node.serializedName] = std::make_pair(node.id, node.type);
}
}
{
std::map<std::wstring, std::pair<Id, int>>::iterator it = m_tempNodeIndex.find(data.serializedName);
if (it != m_tempNodeIndex.end())
{
if (it->second.second < data.type)
{
it->second.second = data.type;
setNodeType(data.type, it->second.first);
}
return StorageNode(it->second.first, data);
}
}
Id id = 0;
{
executeStatement(m_insertElementStmt);
@@ -46,6 +109,9 @@ StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data)
executeStatement(m_inserNodeStmt);
m_inserNodeStmt.reset();
}
m_tempNodeIndex[data.serializedName] = std::make_pair(id, data.type);
return StorageNode(id, data);
}
@@ -95,6 +161,23 @@ void SqliteIndexStorage::addFile(const StorageFile& data)
StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data)
{
if (m_tempEdgeIndex.empty())
{
for (const StorageEdge& edge : getAll<StorageEdge>())
{
m_tempEdgeIndex[serialize(edge)] = edge.id;
}
}
const std::string serialized = serialize(data);
{
std::map<std::string, Id>::const_iterator it = m_tempEdgeIndex.find(serialized);
if (it != m_tempEdgeIndex.end())
{
return StorageEdge(it->second, data);
}
}
Id id = 0;
{
executeStatement(m_insertElementStmt);
@@ -109,11 +192,30 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data)
executeStatement(m_insertEdgeStmt);
m_insertEdgeStmt.reset();
}
m_tempEdgeIndex[serialized] = id;
return StorageEdge(id, data);
}
StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data)
{
if (m_tempLocalSymbolIndex.empty())
{
for (const StorageLocalSymbol& localSymbol : getAll<StorageLocalSymbol>())
{
m_tempLocalSymbolIndex[localSymbol.name] = localSymbol.id;
}
}
{
std::map<std::wstring, Id>::const_iterator it = m_tempLocalSymbolIndex.find(data.name);
if (it != m_tempLocalSymbolIndex.end())
{
return StorageLocalSymbol(it->second, data);
}
}
Id id = 0;
{
executeStatement(m_insertElementStmt);
@@ -126,64 +228,75 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa
executeStatement(m_inserLocalSymbolStmt);
m_inserLocalSymbolStmt.reset();
}
m_tempLocalSymbolIndex[data.name] = id;
return StorageLocalSymbol(id, data);
}
StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceLocationData& data)
{
if (m_tempSourceLocationIndices.empty())
{
for (const StorageSourceLocation& sourceLocation : getAll<StorageSourceLocation>())
{
m_tempSourceLocationIndices[sourceLocation.fileNodeId][serialize(sourceLocation)] = sourceLocation.id;
}
}
const std::string serialized = serialize(data);
std::map<std::string, Id>& index = m_tempSourceLocationIndices[data.fileNodeId];
{
std::map<std::string, Id>::const_iterator it = index.find(serialized);
if (it != index.end())
{
return StorageSourceLocation(it->second, data);
}
}
Id id = 0;
m_insertSourceLocationStmt.bind(1, int(data.fileNodeId));
m_insertSourceLocationStmt.bind(2, int(data.startLine));
m_insertSourceLocationStmt.bind(3, int(data.startCol));
m_insertSourceLocationStmt.bind(4, int(data.endLine));
m_insertSourceLocationStmt.bind(5, int(data.endCol));
m_insertSourceLocationStmt.bind(6, data.type);
const bool success = executeStatement(m_insertSourceLocationStmt);
if (success)
{
m_checkSourceLocationExistsStmt.bind(1, int(data.fileNodeId));
m_checkSourceLocationExistsStmt.bind(2, int(data.startLine));
m_checkSourceLocationExistsStmt.bind(3, int(data.startCol));
m_checkSourceLocationExistsStmt.bind(4, int(data.endLine));
m_checkSourceLocationExistsStmt.bind(5, int(data.endCol));
m_checkSourceLocationExistsStmt.bind(6, data.type);
CppSQLite3Query checkQuery = executeQuery(m_checkSourceLocationExistsStmt);
if (!checkQuery.eof() && checkQuery.numFields() > 0)
{
id = checkQuery.getIntField(0, 0);
}
m_checkSourceLocationExistsStmt.reset();
id = m_database.lastRowId();
index[serialized] = id;
}
if (id == 0)
{
m_insertSourceLocationStmt.bind(1, int(data.fileNodeId));
m_insertSourceLocationStmt.bind(2, int(data.startLine));
m_insertSourceLocationStmt.bind(3, int(data.startCol));
m_insertSourceLocationStmt.bind(4, int(data.endLine));
m_insertSourceLocationStmt.bind(5, int(data.endCol));
m_insertSourceLocationStmt.bind(6, data.type);
const bool success = executeStatement(m_insertSourceLocationStmt);
if (success)
{
id = m_database.lastRowId();
}
m_insertSourceLocationStmt.reset();
}
m_insertSourceLocationStmt.reset();
return StorageSourceLocation(id, data);
}
bool SqliteIndexStorage::addOccurrence(const StorageOccurrence& data)
{
{
m_insertOccurrenceStmt.bind(1, int(data.elementId));
m_insertOccurrenceStmt.bind(2, int(data.sourceLocationId));
executeStatement(m_insertOccurrenceStmt);
m_insertOccurrenceStmt.reset();
}
return true;
m_insertOccurrenceStmt.bind(1, int(data.elementId));
m_insertOccurrenceStmt.bind(2, int(data.sourceLocationId));
const bool success = executeStatement(m_insertOccurrenceStmt);
m_insertOccurrenceStmt.reset();
return success;
}
bool SqliteIndexStorage::addOccurrences(const std::vector<StorageOccurrence>& occurrences)
{
if (!occurrences.empty())
if (occurrences.size() == 100)
{
for (int i = 0; i < occurrences.size(); i++)
{
m_insert100OccurrencesStmt.bind((i * 2) + 1, int(occurrences[i].elementId));
m_insert100OccurrencesStmt.bind((i * 2) + 2, int(occurrences[i].sourceLocationId));
}
const bool success = executeStatement(m_insert100OccurrencesStmt);
m_insert100OccurrencesStmt.reset();
return success;
}
else if (!occurrences.empty())
{
std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES";
{
@@ -204,25 +317,13 @@ bool SqliteIndexStorage::addOccurrences(const std::vector<StorageOccurrence>& oc
return true;
}
StorageComponentAccess SqliteIndexStorage::addComponentAccess(const StorageComponentAccessData& data)
bool SqliteIndexStorage::addComponentAccess(const StorageComponentAccess& componentAccess)
{
Id id = getComponentAccessByNodeId(data.nodeId).id;
if (id == 0)
{
m_insertComponentAccessStmt.bind(1, int(data.nodeId));
m_insertComponentAccessStmt.bind(2, data.type);
const bool success = executeStatement(m_insertComponentAccessStmt);
if (success)
{
id = m_database.lastRowId();
}
m_insertComponentAccessStmt.reset();
}
return StorageComponentAccess(id, data);
m_insertComponentAccessStmt.bind(1, int(componentAccess.nodeId));
m_insertComponentAccessStmt.bind(2, componentAccess.type);
const bool success = executeStatement(m_insertComponentAccessStmt);
m_insertComponentAccessStmt.reset();
return success;
}
StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageCommentLocationData& data)
@@ -602,11 +703,6 @@ StorageNode SqliteIndexStorage::getNodeBySerializedName(const std::wstring& seri
return StorageNode();
}
StorageLocalSymbol SqliteIndexStorage::getLocalSymbolByName(const std::wstring& name) const
{
return doGetFirst<StorageLocalSymbol>("WHERE name == '" + utility::encodeToUtf8(name) + "'");
}
StorageFile SqliteIndexStorage::getFileByPath(const std::wstring& filePath) const
{
return doGetFirst<StorageFile>("WHERE file.path == '" + utility::encodeToUtf8(filePath) + "'");
@@ -862,10 +958,6 @@ int SqliteIndexStorage::getErrorCount() const
std::vector<std::pair<int, SqliteDatabaseIndex>> SqliteIndexStorage::getIndices() const
{
std::vector<std::pair<int, SqliteDatabaseIndex>> indices;
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("edge_multipart_index", "edge(type, source_node_id, target_node_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("edge_source_node_id_index", "edge(source_node_id)")
@@ -875,13 +967,9 @@ std::vector<std::pair<int, SqliteDatabaseIndex>> SqliteIndexStorage::getIndices(
SqliteDatabaseIndex("edge_target_node_id_index", "edge(target_node_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR,
STORAGE_MODE_READ | STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("node_serialized_name_index", "node(serialized_name)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("local_symbol_name_index", "local_symbol(name)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_READ | STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("source_location_file_node_id_index", "source_location(file_node_id)")
@@ -890,10 +978,6 @@ std::vector<std::pair<int, SqliteDatabaseIndex>> SqliteIndexStorage::getIndices(
STORAGE_MODE_READ,
SqliteDatabaseIndex("source_location_file_node_id_type_index", "source_location(file_node_id, type)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("comment_location_all_data_index", "comment_location(file_node_id, start_line, start_column, end_line, end_column)")
@@ -918,10 +1002,6 @@ std::vector<std::pair<int, SqliteDatabaseIndex>> SqliteIndexStorage::getIndices(
STORAGE_MODE_READ | STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("occurrence_source_location_id_index", "occurrence(source_location_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("component_access_node_id_index", "component_access(node_id)")
));
return indices;
}
@@ -1041,10 +1121,9 @@ void SqliteIndexStorage::setupTables()
m_database.execDML(
"CREATE TABLE IF NOT EXISTS component_access("
"id INTEGER NOT NULL, "
"node_id INTEGER, "
"node_id INTEGER NOT NULL, "
"type INTEGER NOT NULL, "
"PRIMARY KEY(id), "
"PRIMARY KEY(node_id), "
"FOREIGN KEY(node_id) REFERENCES node(id) ON DELETE CASCADE);"
);
@@ -1095,7 +1174,7 @@ void SqliteIndexStorage::setupPrecompiledStatements()
"INSERT INTO node(id, type, serialized_name) VALUES(?, ?, ?);"
);
m_insertSymbolStmt = m_database.compileStatement(
"INSERT INTO symbol(id, definition_kind) VALUES(?, ?);"
"INSERT OR IGNORE INTO symbol(id, definition_kind) VALUES(?, ?);"
);
m_insertFileStmt = m_database.compileStatement(
"INSERT INTO file(id, path, modification_time, indexed, complete, line_count) VALUES(?, ?, ?, ?, ?, ?);"
@@ -1106,24 +1185,27 @@ void SqliteIndexStorage::setupPrecompiledStatements()
m_inserLocalSymbolStmt = m_database.compileStatement(
"INSERT INTO local_symbol(id, name) VALUES(?, ?);"
);
m_checkSourceLocationExistsStmt = m_database.compileStatement(
"SELECT id FROM source_location WHERE "
"file_node_id = ? AND "
"start_line = ? AND "
"start_column = ? AND "
"end_line = ? AND "
"end_column = ? AND "
"type = ? "
"LIMIT 1;"
);
m_insertSourceLocationStmt = m_database.compileStatement(
"INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) VALUES(NULL, ?, ?, ?, ?, ?, ?);"
);
m_insertOccurrenceStmt = m_database.compileStatement(
"INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES(?, ?);"
);
{
std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES";
for (int i = 0; i < 100; i++)
{
if (i != 0)
{
stmt += ",";
}
stmt += "(?, ?)";
}
stmt += ";";
m_insert100OccurrencesStmt = m_database.compileStatement(stmt.c_str());
}
m_insertComponentAccessStmt = m_database.compileStatement(
"INSERT INTO component_access(id, node_id, type) VALUES(NULL, ?, ?);"
"INSERT OR IGNORE INTO component_access(node_id, type) VALUES(?, ?);"
);
m_checkCommentLocationExistsStmt = m_database.compileStatement(
"SELECT id FROM comment_location WHERE "
@@ -1339,20 +1421,19 @@ template <>
std::vector<StorageComponentAccess> SqliteIndexStorage::doGetAll<StorageComponentAccess>(const std::string& query) const
{
CppSQLite3Query q = executeQuery(
"SELECT id, node_id, type FROM component_access " + query + ";"
"SELECT node_id, type FROM component_access " + query + ";"
);
std::vector<StorageComponentAccess> componentAccesses;
while (!q.eof())
{
const Id id = q.getIntField(0, 0);
const Id nodeId = q.getIntField(1, 0);
const int type = q.getIntField(2, -1);
const Id nodeId = q.getIntField(0, 0);
const int type = q.getIntField(1, -1);
if (id != 0 && nodeId != 0 && type != -1)
if (nodeId != 0 && type != -1)
{
componentAccesses.emplace_back(id, nodeId, type);
componentAccesses.emplace_back(nodeId, type);
}
q.nextRow();
@@ -32,10 +32,19 @@ class SqliteIndexStorage
: public SqliteStorage
{
public:
enum StorageModeType
{
STORAGE_MODE_READ = 1,
STORAGE_MODE_WRITE = 2,
STORAGE_MODE_CLEAR = 4,
};
SqliteIndexStorage(const FilePath& dbFilePath);
virtual size_t getStaticVersion() const;
void setMode(const StorageModeType mode);
std::string getProjectSettingsText() const;
void setProjectSettingsText(std::string text);
@@ -47,7 +56,7 @@ public:
StorageSourceLocation addSourceLocation(const StorageSourceLocationData& data);
bool addOccurrence(const StorageOccurrence& data);
bool addOccurrences(const std::vector<StorageOccurrence>& occurrences);
StorageComponentAccess addComponentAccess(const StorageComponentAccessData& data);
bool addComponentAccess(const StorageComponentAccess& componentAccess);
StorageCommentLocation addCommentLocation(const StorageCommentLocationData& data);
StorageError addError(const StorageErrorData& data);
@@ -80,8 +89,6 @@ public:
StorageNode getNodeById(Id id) const;
StorageNode getNodeBySerializedName(const std::wstring& serializedName) const;
StorageLocalSymbol getLocalSymbolByName(const std::wstring& name) const;
StorageFile getFileByPath(const std::wstring& filePath) const;
std::vector<StorageFile> getFilesByPaths(const std::vector<FilePath>& filePaths) const;
@@ -147,7 +154,8 @@ public:
private:
static const size_t s_storageVersion;
virtual std::vector<std::pair<int, SqliteDatabaseIndex>> getIndices() const;
std::vector<std::pair<int, SqliteDatabaseIndex>> getIndices() const;
virtual void clearTables();
virtual void setupTables();
virtual void setupPrecompiledStatements();
@@ -166,6 +174,11 @@ private:
return ResultType();
}
std::map<std::wstring, std::pair<Id, int>> m_tempNodeIndex;
std::map<std::string, Id> m_tempEdgeIndex;
std::map<std::wstring, Id> m_tempLocalSymbolIndex;
std::map<Id, std::map<std::string, Id>> m_tempSourceLocationIndices;
CppSQLite3Statement m_insertElementStmt;
CppSQLite3Statement m_insertEdgeStmt;
CppSQLite3Statement m_inserNodeStmt;
@@ -173,10 +186,9 @@ private:
CppSQLite3Statement m_insertFileStmt;
CppSQLite3Statement m_insertFileContentStmt;
CppSQLite3Statement m_inserLocalSymbolStmt;
CppSQLite3Statement m_checkSourceLocationExistsStmt;
CppSQLite3Statement m_insertSourceLocationStmt;
CppSQLite3Statement m_checkOccurrenceExistsStmt;
CppSQLite3Statement m_insertOccurrenceStmt;
CppSQLite3Statement m_insert100OccurrencesStmt;
CppSQLite3Statement m_insertComponentAccessStmt;
CppSQLite3Statement m_checkCommentLocationExistsStmt;
CppSQLite3Statement m_insertCommentLocationStmt;
@@ -16,8 +16,6 @@ SqliteStorage::SqliteStorage(const FilePath& dbFilePath)
m_database.open(utility::encodeToUtf8(m_dbFilePath.wstr()).c_str());
executeStatement("PRAGMA foreign_keys=ON;");
m_mode = STORAGE_MODE_UNKNOWN;
}
SqliteStorage::~SqliteStorage()
@@ -34,8 +32,6 @@ SqliteStorage::~SqliteStorage()
void SqliteStorage::setup()
{
m_indices = getIndices();
executeStatement("PRAGMA foreign_keys=ON;");
setupMetaTable();
@@ -49,8 +45,6 @@ void SqliteStorage::setup()
m_precompiledStatementsInitialized = true;
}
}
m_mode = STORAGE_MODE_UNKNOWN;
}
void SqliteStorage::clear()
@@ -79,28 +73,6 @@ void SqliteStorage::setVersion(size_t version)
insertOrUpdateMetaValue("storage_version", std::to_string(version));
}
void SqliteStorage::setMode(const StorageModeType mode)
{
if (mode == m_mode)
{
return;
}
for (size_t i = 0; i < m_indices.size(); i++)
{
if (m_indices[i].first & mode)
{
m_indices[i].second.createOnDatabase(m_database);
}
else
{
m_indices[i].second.removeFromDatabase(m_database);
}
}
m_mode = mode;
}
void SqliteStorage::beginTransaction()
{
executeStatement("BEGIN TRANSACTION;");
@@ -12,14 +12,6 @@ class TimeStamp;
class SqliteStorage
{
public:
enum StorageModeType
{
STORAGE_MODE_UNKNOWN = 0,
STORAGE_MODE_READ = 1,
STORAGE_MODE_WRITE = 2,
STORAGE_MODE_CLEAR = 4,
};
SqliteStorage(const FilePath& dbFilePath);
virtual ~SqliteStorage();
@@ -29,8 +21,6 @@ public:
size_t getVersion() const;
void setVersion(size_t version);
void setMode(const StorageModeType mode);
void beginTransaction();
void commitTransaction();
void rollbackTransaction();
@@ -64,11 +54,8 @@ protected:
mutable CppSQLite3DB m_database;
FilePath m_dbFilePath;
StorageModeType m_mode;
private:
virtual size_t getStaticVersion() const = 0;
virtual std::vector<std::pair<int, SqliteDatabaseIndex>> getIndices() const = 0;
virtual void clearTables() = 0;
virtual void setupTables() = 0;
virtual void setupPrecompiledStatements() = 0;
@@ -3,14 +3,14 @@
#include "utility/types.h"
struct StorageComponentAccessData
struct StorageComponentAccess
{
StorageComponentAccessData()
StorageComponentAccess()
: nodeId(0)
, type(0)
{}
StorageComponentAccessData(Id nodeId, int type)
StorageComponentAccess(Id nodeId, int type)
: nodeId(nodeId)
, type(type)
{}
@@ -19,24 +19,4 @@ struct StorageComponentAccessData
int type;
};
struct StorageComponentAccess: public StorageComponentAccessData
{
StorageComponentAccess()
: StorageComponentAccessData()
, id(0)
{}
StorageComponentAccess(Id id, const StorageComponentAccessData data)
: StorageComponentAccessData(data)
, id(nodeId)
{}
StorageComponentAccess(Id id, Id nodeId, int type)
: StorageComponentAccessData(nodeId, type)
, id(nodeId)
{}
Id id;
};
#endif // STORAGE_COMPONENT_ACCESS_H
+1 -1
View File
@@ -194,7 +194,7 @@ void Project::load(std::shared_ptr<DialogView> dialogView)
if (canLoad)
{
m_storage->setMode(SqliteStorage::STORAGE_MODE_READ);
m_storage->setMode(SqliteIndexStorage::STORAGE_MODE_READ);
m_storage->buildCaches();
m_storageCache->setSubject(m_storage);
+12 -6
View File
@@ -65,8 +65,19 @@ public:
intermetiateStorage->addEdge(StorageEdgeData(Edge::typeToInt(Edge::EDGE_MEMBER), aId, bId));
storage.inject(intermetiateStorage.get());
bool foundEdge = false;
TS_ASSERT(storage.getIdForEdge(Edge::EDGE_MEMBER, a, b) != 0);
const Id sourceId = storage.getNodeIdForNameHierarchy(a);
const Id targetId = storage.getNodeIdForNameHierarchy(b);
storage.forEachEdge([&](const StorageEdge& edge)
{
if (edge.sourceNodeId == sourceId && edge.targetNodeId == targetId && edge.type == Edge::typeToInt(Edge::EDGE_MEMBER))
{
foundEdge = true;
}
}
);
TS_ASSERT(foundEdge);
}
@@ -244,11 +255,6 @@ private:
//{
// return getGraph().getEdgeCount();
//}
Id getEdgeId(Edge::EdgeType type, const NameHierarchy& fromName, const NameHierarchy& toName) const
{
return getIdForEdge(type, fromName, toName);
}
};
ParseLocation validLocation(Id locationId = 0) const