data: added HierarchyCache for storing member relationships

This change introduces the HierarchyCache that caches the member relationships of nodes in memory for faster graph
retrieval. Storing these relationships allows for retrieving nodes and edges as lists from the SqliteStorage at once
which results in a 10 fold speed improvement.
This commit is contained in:
Eberhard Graether
2015-09-24 12:43:49 +02:00
parent 5c06dedf7c
commit 97e49d9b3f
7 changed files with 339 additions and 158 deletions
+2
View File
@@ -179,6 +179,8 @@ add_files(
data/type/ReferenceModifiedDataType.cpp
data/type/ReferenceModifiedDataType.h
data/HierarchyCache.cpp
data/HierarchyCache.h
data/SqliteStorage.cpp
data/SqliteStorage.h
data/Storage.cpp
+133
View File
@@ -0,0 +1,133 @@
#include "data/HierarchyCache.h"
HierarchyCache::HierarchyNode::HierarchyNode(Id nodeId)
: m_nodeId(nodeId)
, m_edgeId(0)
, m_parent(nullptr)
, m_isVisible(true)
{
}
Id HierarchyCache::HierarchyNode::getNodeId() const
{
return m_nodeId;
}
Id HierarchyCache::HierarchyNode::getEdgeId() const
{
return m_edgeId;
}
void HierarchyCache::HierarchyNode::setEdgeId(Id edgeId)
{
m_edgeId = edgeId;
}
HierarchyCache::HierarchyNode* HierarchyCache::HierarchyNode::getParent() const
{
return m_parent;
}
void HierarchyCache::HierarchyNode::setParent(HierarchyNode* parent)
{
m_parent = parent;
}
void HierarchyCache::HierarchyNode::addChild(HierarchyNode* child)
{
m_children.push_back(child);
}
const std::vector<HierarchyCache::HierarchyNode*>& HierarchyCache::HierarchyNode::getChildren() const
{
return m_children;
}
void HierarchyCache::HierarchyNode::addChildIdsRecursive(std::vector<Id>* nodeIds, std::vector<Id>* edgeIds) const
{
for (const HierarchyNode* child : m_children)
{
nodeIds->push_back(child->getNodeId());
edgeIds->push_back(child->getEdgeId());
child->addChildIdsRecursive(nodeIds, edgeIds);
}
}
bool HierarchyCache::HierarchyNode::isVisible() const
{
return m_isVisible;
}
void HierarchyCache::HierarchyNode::setIsVisible(bool isVisible)
{
m_isVisible = isVisible;
}
void HierarchyCache::clear()
{
m_nodes.clear();
}
void HierarchyCache::createConnection(Id edgeId, Id fromId, Id toId, bool fromVisible)
{
HierarchyNode* from = createNode(fromId);
HierarchyNode* to = createNode(toId);
from->addChild(to);
to->setParent(from);
to->setEdgeId(edgeId);
from->setIsVisible(fromVisible);
}
Id HierarchyCache::getLastVisibleParentNodeId(Id nodeId) const
{
HierarchyNode* node = nullptr;
HierarchyNode* parent = getNode(nodeId);
while (parent && parent->isVisible())
{
node = parent;
parent = node->getParent();
nodeId = node->getNodeId();
}
return nodeId;
}
void HierarchyCache::addAllChildIdsForNodeId(Id nodeId, std::vector<Id>* nodeIds, std::vector<Id>* edgeIds) const
{
HierarchyNode* node = getNode(nodeId);
if (node)
{
node->addChildIdsRecursive(nodeIds, edgeIds);
}
}
HierarchyCache::HierarchyNode* HierarchyCache::getNode(Id nodeId) const
{
std::map<Id, std::shared_ptr<HierarchyNode>>::const_iterator it = m_nodes.find(nodeId);
if (it != m_nodes.end())
{
return it->second.get();
}
return nullptr;
}
HierarchyCache::HierarchyNode* HierarchyCache::createNode(Id nodeId)
{
std::map<Id, std::shared_ptr<HierarchyNode>>::iterator it = m_nodes.find(nodeId);
if (it == m_nodes.end())
{
it = m_nodes.emplace(nodeId, std::make_shared<HierarchyNode>(nodeId)).first;
}
return it->second.get();
}
+57
View File
@@ -0,0 +1,57 @@
#ifndef HIERARCHY_CACHE_H
#define HIERARCHY_CACHE_H
#include <map>
#include <memory>
#include <vector>
#include "utility/types.h"
class HierarchyCache
{
public:
void clear();
void createConnection(Id edgeId, Id fromId, Id toId, bool fromVisible);
Id getLastVisibleParentNodeId(Id nodeId) const;
void addAllChildIdsForNodeId(Id nodeId, std::vector<Id>* nodeIds, std::vector<Id>* edgeIds) const;
private:
class HierarchyNode
{
public:
HierarchyNode(Id nodeId);
Id getNodeId() const;
Id getEdgeId() const;
void setEdgeId(Id edgeId);
HierarchyNode* getParent() const;
void setParent(HierarchyNode* parent);
void addChild(HierarchyNode* child);
const std::vector<HierarchyNode*>& getChildren() const;
void addChildIdsRecursive(std::vector<Id>* nodeIds, std::vector<Id>* edgeIds) const;
bool isVisible() const;
void setIsVisible(bool isVisible);
private:
const Id m_nodeId;
Id m_edgeId;
HierarchyNode* m_parent;
std::vector<HierarchyNode*> m_children;
bool m_isVisible;
};
HierarchyNode* getNode(Id nodeId) const;
HierarchyNode* createNode(Id nodeId);
std::map<Id, std::shared_ptr<HierarchyNode>> m_nodes;
};
#endif // HIERARCHY_CACHE_H
+59 -21
View File
@@ -226,6 +226,27 @@ bool SqliteStorage::isFile(Id elementId) const
return (count > 0);
}
StorageEdge SqliteStorage::getEdgeById(Id edgeId) const
{
CppSQLite3Query q = m_database.execQuery((
"SELECT type, source_node_id, target_node_id FROM edge WHERE "
"id == " + std::to_string(edgeId) + ";"
).c_str());
if (!q.eof())
{
const int type = q.getIntField(0, -1);
const Id sourceId = q.getIntField(1, 0);
const Id targetId = q.getIntField(2, 0);
if (type != -1 && sourceId != 0 && targetId != 0)
{
return StorageEdge(edgeId, type, sourceId, targetId);
}
}
return StorageEdge(0, -1, 0, 0);
}
StorageEdge SqliteStorage::getEdgeBySourceTargetType(Id sourceId, Id targetId, int type) const
{
StorageEdge edge(
@@ -240,6 +261,11 @@ StorageEdge SqliteStorage::getEdgeBySourceTargetType(Id sourceId, Id targetId, i
return edge;
}
std::vector<StorageEdge> SqliteStorage::getEdgesByIds(const std::vector<Id>& edgeIds) const
{
return getAllEdges("WHERE id IN (" + utility::join(utility::toStrings(edgeIds), ',') + ")");
}
std::vector<StorageEdge> SqliteStorage::getEdgesBySourceId(Id sourceId) const
{
return getAllEdges("WHERE source_node_id == " + std::to_string(sourceId));
@@ -265,6 +291,11 @@ std::vector<StorageEdge> SqliteStorage::getEdgesBySourceOrTargetId(Id id) const
return getAllEdges("WHERE source_node_id == " + std::to_string(id) + " OR target_node_id == " + std::to_string(id));
}
std::vector<StorageEdge> SqliteStorage::getEdgesByType(int type) const
{
return getAllEdges("WHERE type == " + std::to_string(type));
}
std::vector<StorageEdge> SqliteStorage::getEdgesBySourceType(Id sourceId, int type) const
{
std::vector<StorageEdge> edges;
@@ -315,27 +346,6 @@ std::vector<StorageEdge> SqliteStorage::getEdgesByTargetType(Id targetId, int ty
return edges;
}
StorageEdge SqliteStorage::getEdgeById(Id edgeId) const
{
CppSQLite3Query q = m_database.execQuery((
"SELECT type, source_node_id, target_node_id FROM edge WHERE "
"id == " + std::to_string(edgeId) + ";"
).c_str());
if (!q.eof())
{
const int type = q.getIntField(0, -1);
const Id sourceId = q.getIntField(1, 0);
const Id targetId = q.getIntField(2, 0);
if (type != -1 && sourceId != 0 && targetId != 0)
{
return StorageEdge(edgeId, type, sourceId, targetId);
}
}
return StorageEdge(0, -1, 0, 0);
}
StorageNode SqliteStorage::getNodeById(Id id) const
{
CppSQLite3Query q = m_database.execQuery((
@@ -394,6 +404,11 @@ StorageNode SqliteStorage::getNodeByName(const std::string& nodeName) const
return StorageNode(0, -1, 0);
}
std::vector<StorageNode> SqliteStorage::getNodesByIds(const std::vector<Id>& nodeIds) const
{
return getAllNodes("WHERE id IN (" + utility::join(utility::toStrings(nodeIds), ',') + ")");
}
StorageFile SqliteStorage::getFileById(const Id id) const
{
return getFirstFile(
@@ -873,3 +888,26 @@ std::vector<StorageEdge> SqliteStorage::getAllEdges(const std::string& query) co
}
return edges;
}
std::vector<StorageNode> SqliteStorage::getAllNodes(const std::string& query) const
{
CppSQLite3Query q = m_database.execQuery((
"SELECT id, type, name_id FROM node " + query + ";"
).c_str());
std::vector<StorageNode> nodes;
while (!q.eof())
{
const Id id = q.getIntField(0, 0);
const int type = q.getIntField(1, -1);
const Id nameId = q.getIntField(2, 0);
if (id != 0 && type != -1)
{
nodes.push_back(StorageNode(id, type, nameId));
}
q.nextRow();
}
return nodes;
}
+7 -1
View File
@@ -48,19 +48,24 @@ public:
bool isNode(Id elementId) const;
bool isFile(Id elementId) const;
StorageEdge getEdgeById(Id edgeId) const;
StorageEdge getEdgeBySourceTargetType(Id sourceId, Id targetId, int type) const;
std::vector<StorageEdge> getEdgesByIds(const std::vector<Id>& edgeIds) const;
std::vector<StorageEdge> getEdgesBySourceId(Id sourceId) const;
std::vector<StorageEdge> getEdgesBySourceIds(const std::vector<Id>& sourceIds) const;
std::vector<StorageEdge> getEdgesByTargetId(Id targetId) const;
std::vector<StorageEdge> getEdgesByTargetIds(const std::vector<Id>& targetIds) const;
std::vector<StorageEdge> getEdgesBySourceOrTargetId(Id id) const;
std::vector<StorageEdge> getEdgesByType(int type) const;
std::vector<StorageEdge> getEdgesBySourceType(Id sourceId, int type) const;
std::vector<StorageEdge> getEdgesByTargetType(Id targetId, int type) const;
StorageEdge getEdgeById(Id edgeId) const;
StorageNode getNodeById(Id id) const;
StorageNode getNodeByNameId(Id nameId) const;
StorageNode getNodeByName(const std::string& nodeName) const; // hmm... we need to use name hierarchy here...??
std::vector<StorageNode> getNodesByIds(const std::vector<Id>& nodeIds) const;
StorageFile getFileById(const Id id) const;
StorageFile getFileByName(const std::string& fileName) const;
@@ -101,6 +106,7 @@ private:
std::vector<StorageSourceLocation> getAllSourceLocations(const std::string& query) const;
std::vector<StorageEdge> getAllEdges(const std::string& query) const;
std::vector<StorageNode> getAllNodes(const std::string& query) const;
template <typename ResultType>
ResultType getFirstResult(const std::string& query) const;
+71 -132
View File
@@ -32,14 +32,20 @@ Storage::~Storage()
void Storage::clear()
{
m_sqliteStorage.clear();
m_tokenIndex.clear();
m_fileNodeIds.clear();
clearCaches();
m_errorMessages.clear();
m_errorLocationCollection.clear();
}
void Storage::clearCaches()
{
m_tokenIndex.clear();
m_fileNodeIds.clear();
m_hierarchyCache.clear();
}
void Storage::clearFileElements(const std::set<FilePath>& filePaths)
{
for (const FilePath& filePath: filePaths)
@@ -94,17 +100,8 @@ std::set<FilePath> Storage::getDependingFilePaths(const FilePath& filePath)
void Storage::removeUnusedNames()
{
m_sqliteStorage.removeUnusedNameHierarchyElements();
m_fileNodeIds.clear();
}
void Storage::buildSearchIndex()
{
m_tokenIndex.clear();
for (StorageNode node: m_sqliteStorage.getAllNodes())
{
m_tokenIndex.addTokenId(m_tokenIndex.addNode(m_sqliteStorage.getNameHierarchyById(node.nameId)), node.id);
}
clearCaches();
}
void Storage::logGraph() const
@@ -130,6 +127,7 @@ void Storage::startParsing()
void Storage::finishParsing()
{
buildSearchIndex();
buildHierarchyCache();
}
void Storage::prepareParsingFile()
@@ -772,9 +770,6 @@ std::vector<SearchMatch> Storage::getAutocompletionMatches(const std::string& qu
return matches;
}
#include "utility/utility.h"
#include <iostream>
std::shared_ptr<Graph> Storage::getGraphForActiveTokenIds(const std::vector<Id>& tokenIds) const
{
std::shared_ptr<Graph> g = std::make_shared<Graph>();
@@ -788,39 +783,19 @@ std::shared_ptr<Graph> Storage::getGraphForActiveTokenIds(const std::vector<Id>&
{
const StorageNode node = m_sqliteStorage.getNodeById(elementId);
float a = utility::duration(
[&]()
{
addNodeAndAllChildrenToGraph(getLastVisibleParentNodeId(node.id), graph);
}
);
std::cout << "add node and children " << a << std::endl;
addNodeAndAllChildrenToGraph(getLastVisibleParentNodeId(node.id), graph);
std::vector<StorageEdge> edges = m_sqliteStorage.getEdgesBySourceOrTargetId(node.id);
float b = utility::duration(
[&]()
for (size_t i = 0; i < edges.size(); i++)
{
if (Edge::intToType(edges[i].type) != Edge::EDGE_MEMBER)
{
for (size_t i = 0; i < edges.size(); i++)
{
if (Edge::intToType(edges[i].type) != Edge::EDGE_MEMBER)
{
addEdgeAndAllChildrenToGraph(edges[i].id, graph);
}
}
addEdgeAndAllChildrenToGraph(edges[i].id, graph);
}
);
std::cout << "add edge and children " << b << std::endl;
}
float c = utility::duration(
[&]()
{
addAggregationEdgesToGraph(elementId, graph);
}
);
std::cout << "add aggregation " << c << std::endl << std::endl;
addAggregationEdgesToGraph(elementId, graph);
}
else
{
@@ -1290,82 +1265,15 @@ Id Storage::getFileNodeId(const FilePath& filePath)
Id Storage::getLastVisibleParentNodeId(const Id nodeId) const
{
Id currentNodeId = 0;
Id parentNodeId = nodeId;
while (parentNodeId != 0)
{
currentNodeId = parentNodeId;
std::vector<StorageEdge> memberEdges = m_sqliteStorage.getEdgesByTargetType(currentNodeId, Edge::EDGE_MEMBER);
if (!memberEdges.size())
{
break;
}
parentNodeId = memberEdges[0].sourceNodeId;
StorageNode parentNode = m_sqliteStorage.getNodeById(parentNodeId);
if (Node::intToType(parentNode.type) & Node::NODE_NOT_VISIBLE)
{
break;
}
}
return currentNodeId;
}
std::vector<Id> Storage::getDirectChildNodeIds(const Id nodeId) const
{
std::vector<Id> childNodeIds;
std::vector<StorageEdge> edges = m_sqliteStorage.getEdgesBySourceType(nodeId, Edge::EDGE_MEMBER);
for (size_t i = 0; i < edges.size(); i++)
{
childNodeIds.push_back(edges[i].targetNodeId);
}
return childNodeIds;
return m_hierarchyCache.getLastVisibleParentNodeId(nodeId);
}
std::vector<Id> Storage::getAllChildNodeIds(const Id nodeId) const
{
std::vector<Id> childNodeIds;
std::queue<Id> parents;
std::vector<Id> edgeIds;
parents.push(nodeId);
while (parents.size())
{
Id parentId = parents.front();
parents.pop();
std::vector<Id> childs = getDirectChildNodeIds(parentId);
for (Id childId : childs)
{
childNodeIds.push_back(childId);
parents.push(childId);
}
}
return childNodeIds;
}
std::vector<Id> Storage::getAllChildNodeIds(const Id nodeId, const Graph* graph) const
{
std::vector<Id> childNodeIds;
std::queue<Id> parents;
parents.push(nodeId);
while (parents.size())
{
Id parentId = parents.front();
parents.pop();
Node* parent = graph->getNodeById(parentId);
parent->forEachChildNode(
[&](Node* node)
{
childNodeIds.push_back(node->getId());
parents.push(node->getId());
}
);
}
m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &childNodeIds, &edgeIds);
return childNodeIds;
}
@@ -1400,20 +1308,17 @@ Node* Storage::addNodeAndAllChildrenToGraph(const Id nodeId, Graph* graph) const
return node;
}
node = addNodeToGraph(nodeId, graph);
std::vector<Id> nodeIdsToAdd;
std::vector<Id> edgeIdsToAdd;
std::vector<StorageEdge> memberEdges = m_sqliteStorage.getEdgesBySourceType(nodeId, Edge::EDGE_MEMBER);
for (const StorageEdge& edge : memberEdges)
{
Node* targetNode = addNodeAndAllChildrenToGraph(edge.targetNodeId, graph);
nodeIdsToAdd.push_back(nodeId);
if (node && targetNode)
{
graph->createEdge(edge.id, Edge::intToType(edge.type), node, targetNode);
}
}
m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &nodeIdsToAdd, &edgeIdsToAdd);
return node;
addNodesToGraph(nodeIdsToAdd, graph);
addEdgesToGraph(edgeIdsToAdd, graph);
return graph->getNodeById(nodeId);
}
void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const
@@ -1426,7 +1331,7 @@ void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const
// build aggregation edges:
// get all children of the active node
std::vector<Id> childNodeIds = getAllChildNodeIds(nodeId, graph);
std::vector<Id> childNodeIds = getAllChildNodeIds(nodeId);
// get all edges of the children
std::map<Id, std::vector<EdgeInfo>> connectedNodeIds;
@@ -1493,22 +1398,37 @@ void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const
}
}
Node* Storage::addNodeToGraph(const Id nodeId, Graph* graph) const
void Storage::addNodesToGraph(const std::vector<Id> nodeIds, Graph* graph) const
{
Node* node = graph->getNodeById(nodeId);
std::vector<StorageNode> storageNodes = m_sqliteStorage.getNodesByIds(nodeIds);
if (!node)
for (const StorageNode& storageNode : storageNodes)
{
StorageNode storageNode = m_sqliteStorage.getNodeById(nodeId);
node = graph->createNode(
graph->createNode(
storageNode.id,
Node::intToType(storageNode.type),
m_tokenIndex.getNameHierarchyForTokenId(nodeId)
m_tokenIndex.getNameHierarchyForTokenId(storageNode.id)
);
}
}
return node;
void Storage::addEdgesToGraph(const std::vector<Id> edgeIds, Graph* graph) const
{
std::vector<StorageEdge> storageEdges = m_sqliteStorage.getEdgesByIds(edgeIds);
for (const StorageEdge& storageEdge : storageEdges)
{
Node* sourceNode = graph->getNodeById(storageEdge.sourceNodeId);
Node* targetNode = graph->getNodeById(storageEdge.targetNodeId);
if (sourceNode && targetNode)
{
graph->createEdge(storageEdge.id, Edge::intToType(storageEdge.type), sourceNode, targetNode);
}
else
{
LOG_ERROR("Can't add edge because nodes are not present");
}
}
}
TokenComponentAccess::AccessType Storage::convertAccessType(ParserClient::AccessType access) const
@@ -1570,6 +1490,25 @@ void Storage::addComponentAccessToGraph(Graph* graph) const
}
}
void Storage::buildSearchIndex()
{
for (StorageNode node: m_sqliteStorage.getAllNodes())
{
m_tokenIndex.addTokenId(m_tokenIndex.addNode(m_sqliteStorage.getNameHierarchyById(node.nameId)), node.id);
}
}
void Storage::buildHierarchyCache()
{
std::vector<StorageEdge> memberEdges = m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER));
for (const StorageEdge& edge : memberEdges)
{
bool isVisible = !(Node::intToType(m_sqliteStorage.getNodeById(edge.sourceNodeId).type) & Node::NODE_NOT_VISIBLE);
m_hierarchyCache.createConnection(edge.id, edge.sourceNodeId, edge.targetNodeId, isVisible);
}
}
void Storage::log(std::string type, std::string str, const ParseLocation& location) const
{
LOG_INFO_STREAM(
+10 -4
View File
@@ -8,6 +8,7 @@
#include "data/access/StorageAccess.h"
//#include "data/graph/token_component/TokenComponentAbstraction.h"
#include "data/HierarchyCache.h"
#include "data/graph/token_component/TokenComponentAccess.h"
#include "data/location/TokenLocationCollection.h"
#include "data/parser/ParserClient.h"
@@ -23,13 +24,14 @@ public:
virtual ~Storage();
void clear();
void clearCaches();
void clearFileElements(const std::set<FilePath>& filePaths);
void clearFileElements(const FilePath& filePath);
std::set<FilePath> getDependingFilePaths(const std::set<FilePath>& filePaths);
std::set<FilePath> getDependingFilePaths(const FilePath& filePath);
void removeUnusedNames();
void buildSearchIndex();
void logGraph() const;
void logLocations() const;
@@ -162,26 +164,30 @@ private:
Id getFileNodeId(const FilePath& filePath);
Id getLastVisibleParentNodeId(const Id nodeId) const;
std::vector<Id> getDirectChildNodeIds(const Id nodeId) const;
std::vector<Id> getAllChildNodeIds(const Id nodeId) const;
std::vector<Id> getAllChildNodeIds(const Id nodeId, const Graph* graph) const;
void addEdgeAndAllChildrenToGraph(const Id edgeId, Graph* graph) const;
Node* addNodeAndAllChildrenToGraph(const Id nodeId, Graph* graph) const;
void addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const;
Node* addNodeToGraph(const Id nodeId, Graph* graph) const;
void addNodesToGraph(const std::vector<Id> nodeIds, Graph* graph) const;
void addEdgesToGraph(const std::vector<Id> edgeIds, Graph* graph) const;
TokenComponentAccess::AccessType convertAccessType(ParserClient::AccessType access) const;
void addAccess(const Id nodeId, ParserClient::AccessType access);
void addComponentAccessToGraph(Graph* graph) const;
void buildSearchIndex();
void buildHierarchyCache();
void log(std::string type, std::string str, const ParseLocation& location) const;
SearchIndex m_tokenIndex;
SqliteStorage m_sqliteStorage;
std::map <FilePath, Id> m_fileNodeIds;
HierarchyCache m_hierarchyCache;
TokenLocationCollection m_errorLocationCollection;
std::vector<std::string> m_errorMessages;