From 97e49d9b3fc9e8b108911c9c01a8b3041dcf3211 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Thu, 24 Sep 2015 12:43:49 +0200 Subject: [PATCH] data: added HierarchyCache for storing member relationships This change introduces the HierarchyCache that caches the member relationships of nodes in memory for faster graph retrieval. Storing these relationships allows for retrieving nodes and edges as lists from the SqliteStorage at once which results in a 10 fold speed improvement. --- src/lib/CMakeLists.txt | 2 + src/lib/data/HierarchyCache.cpp | 133 +++++++++++++++++++++ src/lib/data/HierarchyCache.h | 57 +++++++++ src/lib/data/SqliteStorage.cpp | 80 +++++++++---- src/lib/data/SqliteStorage.h | 8 +- src/lib/data/Storage.cpp | 203 +++++++++++--------------------- src/lib/data/Storage.h | 14 ++- 7 files changed, 339 insertions(+), 158 deletions(-) create mode 100644 src/lib/data/HierarchyCache.cpp create mode 100644 src/lib/data/HierarchyCache.h diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 11a45b7b..de372d90 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -179,6 +179,8 @@ add_files( data/type/ReferenceModifiedDataType.cpp data/type/ReferenceModifiedDataType.h + data/HierarchyCache.cpp + data/HierarchyCache.h data/SqliteStorage.cpp data/SqliteStorage.h data/Storage.cpp diff --git a/src/lib/data/HierarchyCache.cpp b/src/lib/data/HierarchyCache.cpp new file mode 100644 index 00000000..d465a2db --- /dev/null +++ b/src/lib/data/HierarchyCache.cpp @@ -0,0 +1,133 @@ +#include "data/HierarchyCache.h" + +HierarchyCache::HierarchyNode::HierarchyNode(Id nodeId) + : m_nodeId(nodeId) + , m_edgeId(0) + , m_parent(nullptr) + , m_isVisible(true) +{ +} + +Id HierarchyCache::HierarchyNode::getNodeId() const +{ + return m_nodeId; +} + +Id HierarchyCache::HierarchyNode::getEdgeId() const +{ + return m_edgeId; +} + +void HierarchyCache::HierarchyNode::setEdgeId(Id edgeId) +{ + m_edgeId = edgeId; +} + +HierarchyCache::HierarchyNode* HierarchyCache::HierarchyNode::getParent() const +{ + return m_parent; +} + +void HierarchyCache::HierarchyNode::setParent(HierarchyNode* parent) +{ + m_parent = parent; +} + +void HierarchyCache::HierarchyNode::addChild(HierarchyNode* child) +{ + m_children.push_back(child); +} + +const std::vector& HierarchyCache::HierarchyNode::getChildren() const +{ + return m_children; +} + +void HierarchyCache::HierarchyNode::addChildIdsRecursive(std::vector* nodeIds, std::vector* edgeIds) const +{ + for (const HierarchyNode* child : m_children) + { + nodeIds->push_back(child->getNodeId()); + edgeIds->push_back(child->getEdgeId()); + + child->addChildIdsRecursive(nodeIds, edgeIds); + } +} + +bool HierarchyCache::HierarchyNode::isVisible() const +{ + return m_isVisible; +} + +void HierarchyCache::HierarchyNode::setIsVisible(bool isVisible) +{ + m_isVisible = isVisible; +} + + + +void HierarchyCache::clear() +{ + m_nodes.clear(); +} + +void HierarchyCache::createConnection(Id edgeId, Id fromId, Id toId, bool fromVisible) +{ + HierarchyNode* from = createNode(fromId); + HierarchyNode* to = createNode(toId); + + from->addChild(to); + to->setParent(from); + + to->setEdgeId(edgeId); + from->setIsVisible(fromVisible); +} + +Id HierarchyCache::getLastVisibleParentNodeId(Id nodeId) const +{ + HierarchyNode* node = nullptr; + HierarchyNode* parent = getNode(nodeId); + + while (parent && parent->isVisible()) + { + node = parent; + parent = node->getParent(); + + nodeId = node->getNodeId(); + } + + return nodeId; +} + +void HierarchyCache::addAllChildIdsForNodeId(Id nodeId, std::vector* nodeIds, std::vector* edgeIds) const +{ + HierarchyNode* node = getNode(nodeId); + if (node) + { + node->addChildIdsRecursive(nodeIds, edgeIds); + } +} + +HierarchyCache::HierarchyNode* HierarchyCache::getNode(Id nodeId) const +{ + std::map>::const_iterator it = m_nodes.find(nodeId); + + if (it != m_nodes.end()) + { + return it->second.get(); + } + + return nullptr; +} + +HierarchyCache::HierarchyNode* HierarchyCache::createNode(Id nodeId) +{ + std::map>::iterator it = m_nodes.find(nodeId); + + if (it == m_nodes.end()) + { + it = m_nodes.emplace(nodeId, std::make_shared(nodeId)).first; + } + + return it->second.get(); +} diff --git a/src/lib/data/HierarchyCache.h b/src/lib/data/HierarchyCache.h new file mode 100644 index 00000000..d14f22c7 --- /dev/null +++ b/src/lib/data/HierarchyCache.h @@ -0,0 +1,57 @@ +#ifndef HIERARCHY_CACHE_H +#define HIERARCHY_CACHE_H + +#include +#include +#include + +#include "utility/types.h" + +class HierarchyCache +{ +public: + void clear(); + + void createConnection(Id edgeId, Id fromId, Id toId, bool fromVisible); + + Id getLastVisibleParentNodeId(Id nodeId) const; + void addAllChildIdsForNodeId(Id nodeId, std::vector* nodeIds, std::vector* edgeIds) const; + +private: + class HierarchyNode + { + public: + HierarchyNode(Id nodeId); + + Id getNodeId() const; + + Id getEdgeId() const; + void setEdgeId(Id edgeId); + + HierarchyNode* getParent() const; + void setParent(HierarchyNode* parent); + + void addChild(HierarchyNode* child); + const std::vector& getChildren() const; + void addChildIdsRecursive(std::vector* nodeIds, std::vector* edgeIds) const; + + bool isVisible() const; + void setIsVisible(bool isVisible); + + private: + const Id m_nodeId; + Id m_edgeId; + + HierarchyNode* m_parent; + std::vector m_children; + + bool m_isVisible; + }; + + HierarchyNode* getNode(Id nodeId) const; + HierarchyNode* createNode(Id nodeId); + + std::map> m_nodes; +}; + +#endif // HIERARCHY_CACHE_H diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index 78a05f49..1514afe8 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -226,6 +226,27 @@ bool SqliteStorage::isFile(Id elementId) const return (count > 0); } +StorageEdge SqliteStorage::getEdgeById(Id edgeId) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT type, source_node_id, target_node_id FROM edge WHERE " + "id == " + std::to_string(edgeId) + ";" + ).c_str()); + + if (!q.eof()) + { + const int type = q.getIntField(0, -1); + const Id sourceId = q.getIntField(1, 0); + const Id targetId = q.getIntField(2, 0); + + if (type != -1 && sourceId != 0 && targetId != 0) + { + return StorageEdge(edgeId, type, sourceId, targetId); + } + } + return StorageEdge(0, -1, 0, 0); +} + StorageEdge SqliteStorage::getEdgeBySourceTargetType(Id sourceId, Id targetId, int type) const { StorageEdge edge( @@ -240,6 +261,11 @@ StorageEdge SqliteStorage::getEdgeBySourceTargetType(Id sourceId, Id targetId, i return edge; } +std::vector SqliteStorage::getEdgesByIds(const std::vector& edgeIds) const +{ + return getAllEdges("WHERE id IN (" + utility::join(utility::toStrings(edgeIds), ',') + ")"); +} + std::vector SqliteStorage::getEdgesBySourceId(Id sourceId) const { return getAllEdges("WHERE source_node_id == " + std::to_string(sourceId)); @@ -265,6 +291,11 @@ std::vector SqliteStorage::getEdgesBySourceOrTargetId(Id id) const return getAllEdges("WHERE source_node_id == " + std::to_string(id) + " OR target_node_id == " + std::to_string(id)); } +std::vector SqliteStorage::getEdgesByType(int type) const +{ + return getAllEdges("WHERE type == " + std::to_string(type)); +} + std::vector SqliteStorage::getEdgesBySourceType(Id sourceId, int type) const { std::vector edges; @@ -315,27 +346,6 @@ std::vector SqliteStorage::getEdgesByTargetType(Id targetId, int ty return edges; } -StorageEdge SqliteStorage::getEdgeById(Id edgeId) const -{ - CppSQLite3Query q = m_database.execQuery(( - "SELECT type, source_node_id, target_node_id FROM edge WHERE " - "id == " + std::to_string(edgeId) + ";" - ).c_str()); - - if (!q.eof()) - { - const int type = q.getIntField(0, -1); - const Id sourceId = q.getIntField(1, 0); - const Id targetId = q.getIntField(2, 0); - - if (type != -1 && sourceId != 0 && targetId != 0) - { - return StorageEdge(edgeId, type, sourceId, targetId); - } - } - return StorageEdge(0, -1, 0, 0); -} - StorageNode SqliteStorage::getNodeById(Id id) const { CppSQLite3Query q = m_database.execQuery(( @@ -394,6 +404,11 @@ StorageNode SqliteStorage::getNodeByName(const std::string& nodeName) const return StorageNode(0, -1, 0); } +std::vector SqliteStorage::getNodesByIds(const std::vector& nodeIds) const +{ + return getAllNodes("WHERE id IN (" + utility::join(utility::toStrings(nodeIds), ',') + ")"); +} + StorageFile SqliteStorage::getFileById(const Id id) const { return getFirstFile( @@ -873,3 +888,26 @@ std::vector SqliteStorage::getAllEdges(const std::string& query) co } return edges; } + +std::vector SqliteStorage::getAllNodes(const std::string& query) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT id, type, name_id FROM node " + query + ";" + ).c_str()); + + std::vector nodes; + while (!q.eof()) + { + const Id id = q.getIntField(0, 0); + const int type = q.getIntField(1, -1); + const Id nameId = q.getIntField(2, 0); + + if (id != 0 && type != -1) + { + nodes.push_back(StorageNode(id, type, nameId)); + } + + q.nextRow(); + } + return nodes; +} diff --git a/src/lib/data/SqliteStorage.h b/src/lib/data/SqliteStorage.h index 8ed8df99..d091bd1f 100644 --- a/src/lib/data/SqliteStorage.h +++ b/src/lib/data/SqliteStorage.h @@ -48,19 +48,24 @@ public: bool isNode(Id elementId) const; bool isFile(Id elementId) const; + StorageEdge getEdgeById(Id edgeId) const; StorageEdge getEdgeBySourceTargetType(Id sourceId, Id targetId, int type) const; + std::vector getEdgesByIds(const std::vector& edgeIds) const; + std::vector getEdgesBySourceId(Id sourceId) const; std::vector getEdgesBySourceIds(const std::vector& sourceIds) const; std::vector getEdgesByTargetId(Id targetId) const; std::vector getEdgesByTargetIds(const std::vector& targetIds) const; std::vector getEdgesBySourceOrTargetId(Id id) const; + + std::vector getEdgesByType(int type) const; std::vector getEdgesBySourceType(Id sourceId, int type) const; std::vector getEdgesByTargetType(Id targetId, int type) const; - StorageEdge getEdgeById(Id edgeId) const; StorageNode getNodeById(Id id) const; StorageNode getNodeByNameId(Id nameId) const; StorageNode getNodeByName(const std::string& nodeName) const; // hmm... we need to use name hierarchy here...?? + std::vector getNodesByIds(const std::vector& nodeIds) const; StorageFile getFileById(const Id id) const; StorageFile getFileByName(const std::string& fileName) const; @@ -101,6 +106,7 @@ private: std::vector getAllSourceLocations(const std::string& query) const; std::vector getAllEdges(const std::string& query) const; + std::vector getAllNodes(const std::string& query) const; template ResultType getFirstResult(const std::string& query) const; diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index b68b3010..7232dd97 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -32,14 +32,20 @@ Storage::~Storage() void Storage::clear() { m_sqliteStorage.clear(); - m_tokenIndex.clear(); - m_fileNodeIds.clear(); + clearCaches(); m_errorMessages.clear(); m_errorLocationCollection.clear(); } +void Storage::clearCaches() +{ + m_tokenIndex.clear(); + m_fileNodeIds.clear(); + m_hierarchyCache.clear(); +} + void Storage::clearFileElements(const std::set& filePaths) { for (const FilePath& filePath: filePaths) @@ -94,17 +100,8 @@ std::set Storage::getDependingFilePaths(const FilePath& filePath) void Storage::removeUnusedNames() { m_sqliteStorage.removeUnusedNameHierarchyElements(); - m_fileNodeIds.clear(); -} -void Storage::buildSearchIndex() -{ - m_tokenIndex.clear(); - - for (StorageNode node: m_sqliteStorage.getAllNodes()) - { - m_tokenIndex.addTokenId(m_tokenIndex.addNode(m_sqliteStorage.getNameHierarchyById(node.nameId)), node.id); - } + clearCaches(); } void Storage::logGraph() const @@ -130,6 +127,7 @@ void Storage::startParsing() void Storage::finishParsing() { buildSearchIndex(); + buildHierarchyCache(); } void Storage::prepareParsingFile() @@ -772,9 +770,6 @@ std::vector Storage::getAutocompletionMatches(const std::string& qu return matches; } -#include "utility/utility.h" -#include - std::shared_ptr Storage::getGraphForActiveTokenIds(const std::vector& tokenIds) const { std::shared_ptr g = std::make_shared(); @@ -788,39 +783,19 @@ std::shared_ptr Storage::getGraphForActiveTokenIds(const std::vector& { const StorageNode node = m_sqliteStorage.getNodeById(elementId); - float a = utility::duration( - [&]() - { - addNodeAndAllChildrenToGraph(getLastVisibleParentNodeId(node.id), graph); - } - ); - std::cout << "add node and children " << a << std::endl; + addNodeAndAllChildrenToGraph(getLastVisibleParentNodeId(node.id), graph); std::vector edges = m_sqliteStorage.getEdgesBySourceOrTargetId(node.id); - float b = utility::duration( - [&]() + for (size_t i = 0; i < edges.size(); i++) + { + if (Edge::intToType(edges[i].type) != Edge::EDGE_MEMBER) { - for (size_t i = 0; i < edges.size(); i++) - { - if (Edge::intToType(edges[i].type) != Edge::EDGE_MEMBER) - { - addEdgeAndAllChildrenToGraph(edges[i].id, graph); - } - } + addEdgeAndAllChildrenToGraph(edges[i].id, graph); } - ); - std::cout << "add edge and children " << b << std::endl; + } - - - float c = utility::duration( - [&]() - { - addAggregationEdgesToGraph(elementId, graph); - } - ); - std::cout << "add aggregation " << c << std::endl << std::endl; + addAggregationEdgesToGraph(elementId, graph); } else { @@ -1290,82 +1265,15 @@ Id Storage::getFileNodeId(const FilePath& filePath) Id Storage::getLastVisibleParentNodeId(const Id nodeId) const { - Id currentNodeId = 0; - Id parentNodeId = nodeId; - while (parentNodeId != 0) - { - currentNodeId = parentNodeId; - - std::vector memberEdges = m_sqliteStorage.getEdgesByTargetType(currentNodeId, Edge::EDGE_MEMBER); - if (!memberEdges.size()) - { - break; - } - - parentNodeId = memberEdges[0].sourceNodeId; - - StorageNode parentNode = m_sqliteStorage.getNodeById(parentNodeId); - if (Node::intToType(parentNode.type) & Node::NODE_NOT_VISIBLE) - { - break; - } - } - return currentNodeId; -} - -std::vector Storage::getDirectChildNodeIds(const Id nodeId) const -{ - std::vector childNodeIds; - std::vector edges = m_sqliteStorage.getEdgesBySourceType(nodeId, Edge::EDGE_MEMBER); - for (size_t i = 0; i < edges.size(); i++) - { - childNodeIds.push_back(edges[i].targetNodeId); - } - return childNodeIds; + return m_hierarchyCache.getLastVisibleParentNodeId(nodeId); } std::vector Storage::getAllChildNodeIds(const Id nodeId) const { std::vector childNodeIds; - std::queue parents; + std::vector edgeIds; - parents.push(nodeId); - while (parents.size()) - { - Id parentId = parents.front(); - parents.pop(); - - std::vector childs = getDirectChildNodeIds(parentId); - for (Id childId : childs) - { - childNodeIds.push_back(childId); - parents.push(childId); - } - } - - return childNodeIds; -} - -std::vector Storage::getAllChildNodeIds(const Id nodeId, const Graph* graph) const -{ - std::vector childNodeIds; - std::queue parents; - - parents.push(nodeId); - while (parents.size()) - { - Id parentId = parents.front(); - parents.pop(); - - Node* parent = graph->getNodeById(parentId); - parent->forEachChildNode( - [&](Node* node) - { - childNodeIds.push_back(node->getId()); - parents.push(node->getId()); - } - ); - } + m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &childNodeIds, &edgeIds); return childNodeIds; } @@ -1400,20 +1308,17 @@ Node* Storage::addNodeAndAllChildrenToGraph(const Id nodeId, Graph* graph) const return node; } - node = addNodeToGraph(nodeId, graph); + std::vector nodeIdsToAdd; + std::vector edgeIdsToAdd; - std::vector memberEdges = m_sqliteStorage.getEdgesBySourceType(nodeId, Edge::EDGE_MEMBER); - for (const StorageEdge& edge : memberEdges) - { - Node* targetNode = addNodeAndAllChildrenToGraph(edge.targetNodeId, graph); + nodeIdsToAdd.push_back(nodeId); - if (node && targetNode) - { - graph->createEdge(edge.id, Edge::intToType(edge.type), node, targetNode); - } - } + m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &nodeIdsToAdd, &edgeIdsToAdd); - return node; + addNodesToGraph(nodeIdsToAdd, graph); + addEdgesToGraph(edgeIdsToAdd, graph); + + return graph->getNodeById(nodeId); } void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const @@ -1426,7 +1331,7 @@ void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const // build aggregation edges: // get all children of the active node - std::vector childNodeIds = getAllChildNodeIds(nodeId, graph); + std::vector childNodeIds = getAllChildNodeIds(nodeId); // get all edges of the children std::map> connectedNodeIds; @@ -1493,22 +1398,37 @@ void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const } } -Node* Storage::addNodeToGraph(const Id nodeId, Graph* graph) const +void Storage::addNodesToGraph(const std::vector nodeIds, Graph* graph) const { - Node* node = graph->getNodeById(nodeId); + std::vector storageNodes = m_sqliteStorage.getNodesByIds(nodeIds); - if (!node) + for (const StorageNode& storageNode : storageNodes) { - StorageNode storageNode = m_sqliteStorage.getNodeById(nodeId); - - node = graph->createNode( + graph->createNode( storageNode.id, Node::intToType(storageNode.type), - m_tokenIndex.getNameHierarchyForTokenId(nodeId) + m_tokenIndex.getNameHierarchyForTokenId(storageNode.id) ); } +} - return node; +void Storage::addEdgesToGraph(const std::vector edgeIds, Graph* graph) const +{ + std::vector storageEdges = m_sqliteStorage.getEdgesByIds(edgeIds); + for (const StorageEdge& storageEdge : storageEdges) + { + Node* sourceNode = graph->getNodeById(storageEdge.sourceNodeId); + Node* targetNode = graph->getNodeById(storageEdge.targetNodeId); + + if (sourceNode && targetNode) + { + graph->createEdge(storageEdge.id, Edge::intToType(storageEdge.type), sourceNode, targetNode); + } + else + { + LOG_ERROR("Can't add edge because nodes are not present"); + } + } } TokenComponentAccess::AccessType Storage::convertAccessType(ParserClient::AccessType access) const @@ -1570,6 +1490,25 @@ void Storage::addComponentAccessToGraph(Graph* graph) const } } +void Storage::buildSearchIndex() +{ + for (StorageNode node: m_sqliteStorage.getAllNodes()) + { + m_tokenIndex.addTokenId(m_tokenIndex.addNode(m_sqliteStorage.getNameHierarchyById(node.nameId)), node.id); + } +} + +void Storage::buildHierarchyCache() +{ + std::vector memberEdges = m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER)); + + for (const StorageEdge& edge : memberEdges) + { + bool isVisible = !(Node::intToType(m_sqliteStorage.getNodeById(edge.sourceNodeId).type) & Node::NODE_NOT_VISIBLE); + m_hierarchyCache.createConnection(edge.id, edge.sourceNodeId, edge.targetNodeId, isVisible); + } +} + void Storage::log(std::string type, std::string str, const ParseLocation& location) const { LOG_INFO_STREAM( diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index 8fd3da36..b8cfee37 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -8,6 +8,7 @@ #include "data/access/StorageAccess.h" //#include "data/graph/token_component/TokenComponentAbstraction.h" +#include "data/HierarchyCache.h" #include "data/graph/token_component/TokenComponentAccess.h" #include "data/location/TokenLocationCollection.h" #include "data/parser/ParserClient.h" @@ -23,13 +24,14 @@ public: virtual ~Storage(); void clear(); + void clearCaches(); + void clearFileElements(const std::set& filePaths); void clearFileElements(const FilePath& filePath); std::set getDependingFilePaths(const std::set& filePaths); std::set getDependingFilePaths(const FilePath& filePath); void removeUnusedNames(); - void buildSearchIndex(); void logGraph() const; void logLocations() const; @@ -162,26 +164,30 @@ private: Id getFileNodeId(const FilePath& filePath); Id getLastVisibleParentNodeId(const Id nodeId) const; - std::vector getDirectChildNodeIds(const Id nodeId) const; std::vector getAllChildNodeIds(const Id nodeId) const; - std::vector getAllChildNodeIds(const Id nodeId, const Graph* graph) const; void addEdgeAndAllChildrenToGraph(const Id edgeId, Graph* graph) const; Node* addNodeAndAllChildrenToGraph(const Id nodeId, Graph* graph) const; void addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const; - Node* addNodeToGraph(const Id nodeId, Graph* graph) const; + + void addNodesToGraph(const std::vector nodeIds, Graph* graph) const; + void addEdgesToGraph(const std::vector edgeIds, Graph* graph) const; TokenComponentAccess::AccessType convertAccessType(ParserClient::AccessType access) const; void addAccess(const Id nodeId, ParserClient::AccessType access); void addComponentAccessToGraph(Graph* graph) const; + void buildSearchIndex(); + void buildHierarchyCache(); + void log(std::string type, std::string str, const ParseLocation& location) const; SearchIndex m_tokenIndex; SqliteStorage m_sqliteStorage; std::map m_fileNodeIds; + HierarchyCache m_hierarchyCache; TokenLocationCollection m_errorLocationCollection; std::vector m_errorMessages;