src: Improved project loading performance

* faster hierarchy cache building
* cache symbol definition kinds
* don't request files when caches can be used
* save SearchEdge by starting character in SearchNode to avoid iteration when adding to SearchIndex
* fixed disabled state of some search buttons
This commit is contained in:
Eberhard Graether
2017-07-15 15:21:08 +02:00
parent da756c9a4c
commit 9a36d48d1e
10 changed files with 169 additions and 178 deletions
+5 -1
View File
@@ -47,7 +47,7 @@ QAbstractItemView#search_box_popup {
background: <color:search/button/normal>;
border: none;
border-radius: 12px;
background-position: 15px 5px;
height: 30px;
width: 30px;
}
@@ -56,6 +56,10 @@ QAbstractItemView#search_box_popup {
border-top-left-radius: 0px;
}
#search_button:disabled, #home_button:disabled {
background: <color:search/button/disabled>;
}
#search_button:hover, #home_button:hover {
background: <color:search/button/hover>;
}
@@ -34,7 +34,7 @@ void ActivationController::handleMessage(MessageActivateEdge* message)
m.tokenIds = message->aggregationIds;
m.setKeepContent(false);
m.isAggregation = true;
m.tokenNames.push_back(NameHierarchy(message->getFullName(), message->sourceNameHierarchy.getDelimiterrr()));
m.tokenNames.push_back(NameHierarchy(message->getFullName(), message->sourceNameHierarchy.getDelimiter()));
m.dispatchImmediately();
}
else
@@ -42,7 +42,7 @@ void ActivationController::handleMessage(MessageActivateEdge* message)
MessageActivateTokens m(message);
m.tokenIds.push_back(message->tokenId);
m.isEdge = true;
m.tokenNames.push_back(NameHierarchy(message->getFullName(), message->sourceNameHierarchy.getDelimiterrr()));
m.tokenNames.push_back(NameHierarchy(message->getFullName(), message->sourceNameHierarchy.getDelimiter()));
m.dispatchImmediately();
}
}
+114 -128
View File
@@ -473,9 +473,12 @@ void PersistentStorage::clearCaches()
{
m_symbolIndex.clear();
m_fileIndex.clear();
m_fileNodeIds.clear();
m_fileNodePaths.clear();
m_fileNodeComplete.clear();
m_symbolDefinitionKinds.clear();
m_hierarchyCache.clear();
m_fullTextSearchIndex.clear();
}
@@ -600,6 +603,8 @@ NameHierarchy PersistentStorage::getNameHierarchyForNodeId(Id nodeId) const
std::vector<NameHierarchy> PersistentStorage::getNameHierarchiesForNodeIds(const std::vector<Id> nodeIds) const
{
TRACE();
std::vector<NameHierarchy> nameHierarchies;
for (const StorageNode& storageNode : m_sqliteIndexStorage.getAllByIds<StorageNode>(nodeIds))
{
@@ -820,7 +825,7 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionSymbolMatches(
match.text = name.getRange(idx, name.size()).getQualifiedName();
match.subtext = name.getRange(0, idx).getQualifiedName();
match.delimiter = name.getDelimiterrr();
match.delimiter = name.getDelimiter();
match.indices = result.indices;
match.score = result.score;
@@ -903,6 +908,8 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionCommandMatches(cons
std::vector<SearchMatch> PersistentStorage::getSearchMatchesForTokenIds(const std::vector<Id>& elementIds) const
{
TRACE();
// todo: what if all these elements share the same node in the searchindex?
// In that case there should be only one search match.
std::vector<SearchMatch> matches;
@@ -932,7 +939,7 @@ std::vector<SearchMatch> PersistentStorage::getSearchMatchesForTokenIds(const st
match.nodeType = Node::intToType(node.type);
match.searchType = SearchMatch::SEARCH_TOKEN;
match.delimiter = nameHierarchy.getDelimiterrr();
match.delimiter = nameHierarchy.getDelimiter();
if (match.nodeType == Node::NODE_FILE)
{
@@ -951,35 +958,23 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForAll() const
std::shared_ptr<Graph> graph = std::make_shared<Graph>();
std::unordered_set<Id> explicitlyDefinedSymbolIds;
for (StorageSymbol symbol: m_sqliteIndexStorage.getAll<StorageSymbol>())
{
if (intToDefinitionKind(symbol.definitionKind) == DEFINITION_EXPLICIT)
{
explicitlyDefinedSymbolIds.insert(symbol.id);
}
}
std::vector<Id> tokenIds;
for (StorageNode node: m_sqliteIndexStorage.getAll<StorageNode>())
{
if (explicitlyDefinedSymbolIds.find(node.id) != explicitlyDefinedSymbolIds.end() &&
auto it = m_symbolDefinitionKinds.find(node.id);
if (it != m_symbolDefinitionKinds.end() && it->second == DEFINITION_EXPLICIT &&
(
!m_hierarchyCache.isChildOfVisibleNodeOrInvisible(node.id) ||
(
Node::intToType(node.type) == Node::NODE_NAMESPACE || // TODO: use & operator here
Node::intToType(node.type) == Node::NODE_PACKAGE
)
)
Node::intToType(node.type) & (Node::NODE_NAMESPACE | Node::NODE_PACKAGE) ||
!m_hierarchyCache.isChildOfVisibleNodeOrInvisible(node.id)
)
{
){
tokenIds.push_back(node.id);
}
}
for (StorageFile file: m_sqliteIndexStorage.getAll<StorageFile>())
for (auto p : m_fileNodePaths)
{
tokenIds.push_back(file.id);
tokenIds.push_back(p.first);
}
addNodesToGraph(tokenIds, graph.get());
@@ -1220,6 +1215,8 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForTrail(
// TODO: rename: getActiveElementIdsForId; TODO: make separate function for declarationId
std::vector<Id> PersistentStorage::getActiveTokenIdsForId(Id tokenId, Id* declarationId) const
{
TRACE();
std::vector<Id> activeTokenIds;
if (!(m_sqliteIndexStorage.isEdge(tokenId) || m_sqliteIndexStorage.isNode(tokenId)))
@@ -1429,6 +1426,8 @@ std::shared_ptr<SourceLocationFile> PersistentStorage::getCommentLocationsInFile
std::shared_ptr<TextAccess> PersistentStorage::getFileContent(const FilePath& filePath) const
{
TRACE();
return m_sqliteIndexStorage.getFileContentByPath(filePath.str());
}
@@ -1620,11 +1619,14 @@ FilePath PersistentStorage::getFileNodePath(Id fileId) const
bool PersistentStorage::getFileNodeComplete(const FilePath& filePath) const
{
std::map<FilePath, bool>::const_iterator it = m_fileNodeComplete.find(filePath);
if (it != m_fileNodeComplete.end())
auto it = m_fileNodeIds.find(filePath);
if (it != m_fileNodeIds.end())
{
return it->second;
auto it2 = m_fileNodeComplete.find(it->second);
if (it2 != m_fileNodeComplete.end())
{
return it2->second;
}
}
return false;
@@ -1784,51 +1786,31 @@ std::set<FilePath> PersistentStorage::getReferencingByImports(const std::set<Fil
return paths;
}
Id PersistentStorage::getLastVisibleParentNodeId(const Id nodeId) const
{
return m_hierarchyCache.getLastVisibleParentNodeId(nodeId);
}
std::vector<Id> PersistentStorage::getAllChildNodeIds(const Id nodeId) const
{
std::set<Id> childNodeIds;
std::set<Id> edgeIds;
m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &childNodeIds, &edgeIds);
return utility::toVector(childNodeIds);
}
void PersistentStorage::addNodesToGraph(const std::vector<Id>& newNodeIds, Graph* graph) const
{
TRACE();
std::vector<Id> nodeIds;
for (Id id : newNodeIds)
if (graph->getNodeCount())
{
if (!graph->getNodeById(id))
for (Id id : newNodeIds)
{
nodeIds.push_back(id);
if (!graph->getNodeById(id))
{
nodeIds.push_back(id);
}
}
}
else
{
nodeIds = newNodeIds;
}
if (nodeIds.size() == 0)
{
return;
}
std::unordered_map<Id, StorageSymbol> symbolMap;
for (const StorageSymbol& symbol : m_sqliteIndexStorage.getAllByIds<StorageSymbol>(nodeIds))
{
symbolMap[symbol.id] = symbol;
}
std::unordered_map<Id, StorageFile> fileMap;
for (const StorageFile& file : m_sqliteIndexStorage.getAllByIds<StorageFile>(nodeIds))
{
fileMap[file.id] = file;
}
for (const StorageNode& storageNode : m_sqliteIndexStorage.getAllByIds<StorageNode>(nodeIds))
{
const Node::NodeType type = Node::intToType(storageNode.type);
@@ -1837,10 +1819,10 @@ void PersistentStorage::addNodesToGraph(const std::vector<Id>& newNodeIds, Graph
const FilePath filePath(NameHierarchy::deserialize(storageNode.serializedName).getRawName());
bool defined = true;
auto it = fileMap.find(storageNode.id);
if (it != fileMap.end())
auto it = m_fileNodeComplete.find(storageNode.id);
if (it != m_fileNodeComplete.end())
{
defined = it->second.complete;
defined = it->second;
}
Node* node = graph->createNode(
@@ -1857,10 +1839,10 @@ void PersistentStorage::addNodesToGraph(const std::vector<Id>& newNodeIds, Graph
const NameHierarchy nameHierarchy = NameHierarchy::deserialize(storageNode.serializedName);
DefinitionKind defKind = DEFINITION_NONE;
auto it = symbolMap.find(storageNode.id);
if (it != symbolMap.end())
auto it = m_symbolDefinitionKinds.find(storageNode.id);
if (it != m_symbolDefinitionKinds.end())
{
defKind = intToDefinitionKind(it->second.definitionKind);
defKind = it->second;
}
Node* node = graph->createNode(
@@ -1972,7 +1954,9 @@ void PersistentStorage::addAggregationEdgesToGraph(
// build aggregation edges:
// get all children of the active node
std::vector<Id> childNodeIds = getAllChildNodeIds(nodeId);
std::set<Id> childNodeIdsSet, edgeIdsSet;
m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &childNodeIdsSet, &edgeIdsSet);
std::vector<Id> childNodeIds = utility::toVector(childNodeIdsSet);
if (childNodeIds.size() == 0 && edgesToAggregate.size() == 0)
{
return;
@@ -2008,12 +1992,12 @@ void PersistentStorage::addAggregationEdgesToGraph(
}
// get all parent nodes of all connected nodes (up to last level except namespace/undefined)
Id nodeParentNodeId = getLastVisibleParentNodeId(nodeId);
Id nodeParentNodeId = m_hierarchyCache.getLastVisibleParentNodeId(nodeId);
std::map<Id, std::vector<EdgeInfo>> connectedParentNodeIds;
for (const std::pair<Id, std::vector<EdgeInfo>>& p : connectedNodeIds)
{
Id parentNodeId = getLastVisibleParentNodeId(p.first);
Id parentNodeId = m_hierarchyCache.getLastVisibleParentNodeId(p.first);
if (parentNodeId != nodeParentNodeId)
{
@@ -2084,6 +2068,8 @@ void PersistentStorage::addComponentAccessToGraph(Graph* graph) const
void PersistentStorage::addCompleteFlagsToSourceLocationCollection(SourceLocationCollection* collection) const
{
TRACE();
collection->forEachSourceLocationFile(
[this](std::shared_ptr<SourceLocationFile> file)
{
@@ -2092,56 +2078,6 @@ void PersistentStorage::addCompleteFlagsToSourceLocationCollection(SourceLocatio
);
}
void PersistentStorage::buildSearchIndex()
{
TRACE();
FilePath dbPath = getDbFilePath();
std::unordered_map<Id, StorageSymbol> symbolMap;
for (StorageSymbol symbol : m_sqliteIndexStorage.getAll<StorageSymbol>())
{
symbolMap[symbol.id] = symbol;
}
std::unordered_map<Id, StorageFile> fileMap;
for (StorageFile file : m_sqliteIndexStorage.getAll<StorageFile>())
{
fileMap[file.id] = file;
}
for (StorageNode node : m_sqliteIndexStorage.getAll<StorageNode>())
{
if (Node::intToType(node.type) == Node::NODE_FILE)
{
auto it = fileMap.find(node.id);
if (it != fileMap.end())
{
FilePath filePath(it->second.filePath);
if (filePath.exists())
{
filePath = filePath.relativeTo(dbPath);
}
m_fileIndex.addNode(it->second.id, filePath.str());
}
}
else
{
auto it = symbolMap.find(node.id);
if (it == symbolMap.end() || intToDefinitionKind(it->second.definitionKind) != DEFINITION_IMPLICIT)
{
// we don't use the signature here, so elements with the same signature share the same node.
m_symbolIndex.addNode(node.id, NameHierarchy::deserialize(node.serializedName).getQualifiedName());
}
}
}
m_symbolIndex.finishSetup();
m_fileIndex.finishSetup();
}
void PersistentStorage::buildFilePathMaps()
{
TRACE();
@@ -2150,8 +2086,51 @@ void PersistentStorage::buildFilePathMaps()
{
m_fileNodeIds.emplace(FilePath(file.filePath), file.id);
m_fileNodePaths.emplace(file.id, FilePath(file.filePath));
m_fileNodeComplete.emplace(FilePath(file.filePath), file.complete);
m_fileNodeComplete.emplace(file.id, file.complete);
}
for (StorageSymbol symbol : m_sqliteIndexStorage.getAll<StorageSymbol>())
{
m_symbolDefinitionKinds.emplace(symbol.id, intToDefinitionKind(symbol.definitionKind));
}
}
void PersistentStorage::buildSearchIndex()
{
TRACE();
FilePath dbPath = getDbFilePath();
for (StorageNode node : m_sqliteIndexStorage.getAll<StorageNode>())
{
if (Node::intToType(node.type) == Node::NODE_FILE)
{
auto it = m_fileNodePaths.find(node.id);
if (it != m_fileNodePaths.end())
{
FilePath filePath(it->second);
if (filePath.exists())
{
filePath = filePath.relativeTo(dbPath);
}
m_fileIndex.addNode(node.id, filePath.str());
}
}
else
{
auto it = m_symbolDefinitionKinds.find(node.id);
if (it == m_symbolDefinitionKinds.end() || it->second != DEFINITION_IMPLICIT)
{
// we don't use the signature here, so elements with the same signature share the same node.
m_symbolIndex.addNode(node.id, NameHierarchy::deserialize(node.serializedName).getQualifiedName());
}
}
}
m_symbolIndex.finishSetup();
m_fileIndex.finishSetup();
}
void PersistentStorage::buildFullTextSearchIndex() const
@@ -2170,23 +2149,30 @@ void PersistentStorage::buildHierarchyCache()
std::vector<StorageEdge> memberEdges = m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER));
Cache<Id, Node::NodeType> nodeTypeCache([this](Id id){
return Node::intToType(m_sqliteIndexStorage.getFirstById<StorageNode>(id).type);
});
std::vector<Id> sourceNodeIds;
for (const StorageEdge& edge : memberEdges)
{
sourceNodeIds.push_back(edge.sourceNodeId);
}
Cache<Id, DefinitionKind> definitionKindCache([this](Id id){
StorageSymbol symbol = m_sqliteIndexStorage.getFirstById<StorageSymbol>(id);
if (symbol.id > 0)
{
return intToDefinitionKind(symbol.definitionKind);
}
return DEFINITION_NONE;
});
std::vector<StorageNode> sourceNodes = m_sqliteIndexStorage.getAllByIds<StorageNode>(sourceNodeIds);
std::map<Id, Node::NodeType> sourceNodeTypeMap;
for (const StorageNode& node : sourceNodes)
{
sourceNodeTypeMap.emplace(node.id, Node::intToType(node.type));
}
for (const StorageEdge& edge : memberEdges)
{
bool sourceIsVisible = !(nodeTypeCache.getValue(edge.sourceNodeId) & Node::NODE_NOT_VISIBLE);
bool targetIsImplicit = definitionKindCache.getValue(edge.targetNodeId) == DEFINITION_IMPLICIT;
bool sourceIsVisible = !(sourceNodeTypeMap[edge.sourceNodeId] & Node::NODE_NOT_VISIBLE);
bool targetIsImplicit = false;
auto it = m_symbolDefinitionKinds.find(edge.targetNodeId);
if (it != m_symbolDefinitionKinds.end())
{
targetIsImplicit = (it->second == DEFINITION_IMPLICIT);
}
m_hierarchyCache.createConnection(
edge.id, edge.sourceNodeId, edge.targetNodeId, sourceIsVisible, targetIsImplicit);
+6 -7
View File
@@ -156,9 +156,6 @@ private:
std::set<FilePath> getReferencingByIncludes(const std::set<FilePath>& filePaths);
std::set<FilePath> getReferencingByImports(const std::set<FilePath>& filePaths);
Id getLastVisibleParentNodeId(const Id nodeId) const;
std::vector<Id> getAllChildNodeIds(const Id nodeId) const;
void addNodesToGraph(const std::vector<Id>& nodeIds, Graph* graph) const;
void addEdgesToGraph(const std::vector<Id>& edgeIds, Graph* graph) const;
void addNodesWithParentsAndEdgesToGraph(
@@ -169,8 +166,8 @@ private:
void addCompleteFlagsToSourceLocationCollection(SourceLocationCollection* collection) const;
void buildSearchIndex();
void buildFilePathMaps();
void buildSearchIndex();
void buildFullTextSearchIndex() const;
void buildHierarchyCache();
@@ -185,9 +182,11 @@ private:
SqliteIndexStorage m_sqliteIndexStorage;
SqliteBookmarkStorage m_sqliteBookmarkStorage;
std::map <FilePath, Id> m_fileNodeIds;
std::map <Id, FilePath> m_fileNodePaths;
std::map <FilePath, bool> m_fileNodeComplete;
std::map<FilePath, Id> m_fileNodeIds;
std::map<Id, FilePath> m_fileNodePaths;
std::map<Id, bool> m_fileNodeComplete;
std::map<Id, DefinitionKind> m_symbolDefinitionKinds;
HierarchyCache m_hierarchyCache;
};
+5
View File
@@ -10,6 +10,11 @@ std::string NameElement::Signature::serialize(Signature signature)
NameElement::Signature NameElement::Signature::deserialize(const std::string& serialized)
{
if (serialized == "\tp")
{
return Signature();
}
std::vector<std::string> serializedElements = utility::splitToVector(serialized, "\tp");
if (serializedElements.size() != 2)
{
+3 -3
View File
@@ -5,7 +5,7 @@
std::string NameHierarchy::serialize(NameHierarchy nameHierarchy)
{
std::string serializedName = nameDelimiterTypeToString(nameHierarchy.getDelimiterrr()) + "\tm";
std::string serializedName = nameDelimiterTypeToString(nameHierarchy.getDelimiter()) + "\tm";
for (size_t i = 0; i < nameHierarchy.size(); i++)
{
if (i > 0)
@@ -45,12 +45,12 @@ NameHierarchy NameHierarchy::deserialize(const std::string& serializedName)
return nameHierarchy;
}
NameDelimiterType NameHierarchy::getDelimiterrr() const
NameDelimiterType NameHierarchy::getDelimiter() const
{
return m_delimiter;
}
void NameHierarchy::setDelimiterrr(const NameDelimiterType delimiter)
void NameHierarchy::setDelimiter(const NameDelimiterType delimiter)
{
m_delimiter = delimiter;
}
+2 -2
View File
@@ -19,8 +19,8 @@ public:
NameHierarchy(const std::vector<std::string>& names, const NameDelimiterType delimiter);
~NameHierarchy();
NameDelimiterType getDelimiterrr() const;
void setDelimiterrr(const NameDelimiterType delimiter);
NameDelimiterType getDelimiter() const;
void setDelimiter(const NameDelimiterType delimiter);
void push(std::shared_ptr<NameElement> element);
void pop();
+1 -1
View File
@@ -188,7 +188,7 @@ Id ParserClientImpl::addNodeHierarchy(NameHierarchy nameHierarchy, Node::NodeTyp
}
Id parentNodeId = 0;
NameHierarchy currentNameHierarchy(nameHierarchy.getDelimiterrr());
NameHierarchy currentNameHierarchy(nameHierarchy.getDelimiter());
for (size_t i = 0; i < nameHierarchy.size(); i++)
{
+30 -33
View File
@@ -23,15 +23,14 @@ void SearchIndex::addNode(Id id, const std::string& name)
std::string remaining = name;
while (remaining.size() > 0)
{
bool matchingEdgeFound = false;
// has edge starting with c?
for (size_t i = 0; i < currentNode->edges.size(); i++)
auto it = currentNode->edges.find(remaining[0]);
if (it != currentNode->edges.end())
{
Edge* currentEdge = currentNode->edges[i];
Edge* currentEdge = it->second;
const std::string& edgeString = currentEdge->s;
size_t matchCount = 0;
for (size_t j = 0; j < edgeString.size() && j < remaining.size(); j++)
size_t matchCount = 1;
for (size_t j = 1; j < edgeString.size() && j < remaining.size(); j++)
{
if (edgeString[j] != remaining[j])
{
@@ -40,32 +39,27 @@ void SearchIndex::addNode(Id id, const std::string& name)
matchCount++;
}
if (matchCount != 0)
if (matchCount < edgeString.size())
{
remaining = remaining.substr(matchCount);
if (matchCount < edgeString.size())
{
// split current edge
std::shared_ptr<Node> n = std::make_shared<Node>();
m_nodes.push_back(n);
std::shared_ptr<Edge> e = std::make_shared<Edge>();
m_edges.push_back(e);
// split current edge
std::shared_ptr<Node> n = std::make_shared<Node>();
m_nodes.push_back(n);
std::shared_ptr<Edge> e = std::make_shared<Edge>();
m_edges.push_back(e);
n->edges.push_back(e.get());
e->s = edgeString.substr(matchCount);
e->target = currentEdge->target;
e->s = edgeString.substr(matchCount);
e->target = currentEdge->target;
n->edges.emplace(e->s[0], e.get());
currentEdge->s = edgeString.substr(0, matchCount);
currentEdge->target = n.get();
}
currentNode = currentEdge->target;
matchingEdgeFound = true;
break;
currentEdge->s = edgeString.substr(0, matchCount);
currentEdge->target = n.get();
}
}
if (!matchingEdgeFound)
remaining = remaining.substr(matchCount);
currentNode = currentEdge->target;
}
else
{
std::shared_ptr<Node> n = std::make_shared<Node>();
m_nodes.push_back(n);
@@ -75,7 +69,7 @@ void SearchIndex::addNode(Id id, const std::string& name)
e->s = remaining;
e->target = n.get();
currentNode->edges.push_back(e.get());
currentNode->edges.emplace(e->s[0], e.get());
currentNode = n.get();
remaining = "";
@@ -87,9 +81,9 @@ void SearchIndex::addNode(Id id, const std::string& name)
void SearchIndex::finishSetup()
{
for (size_t i = 0; i < m_root->edges.size(); i++)
for (auto p : m_root->edges)
{
populateEdgeGate(m_root->edges[i]);
populateEdgeGate(p.second);
}
}
@@ -139,9 +133,9 @@ std::vector<SearchResult> SearchIndex::search(
void SearchIndex::populateEdgeGate(Edge* e)
{
Node* target = e->target;
for (size_t i = 0; i < target->edges.size(); i++)
for (auto p : target->edges)
{
Edge* targetEdge = target->edges[i];
Edge* targetEdge = p.second;
populateEdgeGate(targetEdge);
utility::append(e->gate, targetEdge->gate);
}
@@ -160,8 +154,10 @@ void SearchIndex::searchRecursive(
return;
}
for (const Edge* currentEdge : path.node->edges)
for (auto p : path.node->edges)
{
const Edge* currentEdge = p.second;
// test if s passes the edge's gate.
bool passesGate = true;
for (const char& c : remainingQuery)
@@ -242,8 +238,9 @@ std::multiset<SearchResult> SearchIndex::createScoredResults(const std::vector<P
}
}
for (const Edge* edge : path.node->edges)
for (auto p : path.node->edges)
{
const Edge* edge = p.second;
Path nextPath;
nextPath.indices = path.indices;
nextPath.node = edge->target;
+1 -1
View File
@@ -43,7 +43,7 @@ private:
struct Node
{
std::set<Id> elementIds;
std::vector<Edge*> edges;
std::map<char, Edge*> edges;
};
struct Edge