diff --git a/src/lib/data/storage/IntermediateStorage.cpp b/src/lib/data/storage/IntermediateStorage.cpp index 061ab7dd..38166596 100644 --- a/src/lib/data/storage/IntermediateStorage.cpp +++ b/src/lib/data/storage/IntermediateStorage.cpp @@ -121,11 +121,27 @@ std::pair IntermediateStorage::addNode(const StorageNodeData& nodeData return std::make_pair(nodeId, true); } +std::vector IntermediateStorage::addNodes(const std::vector& nodes) +{ + std::vector nodeIds; + nodeIds.reserve(nodes.size()); + for (const StorageNode& node : nodes) + { + nodeIds.emplace_back(addNode(node).first); + } + return nodeIds; +} + void IntermediateStorage::addSymbol(const StorageSymbol& symbol) { m_symbols.push_back(symbol); } +void IntermediateStorage::addSymbols(const std::vector& symbols) +{ + m_symbols.insert(m_symbols.end(), symbols.begin(), symbols.end()); +} + void IntermediateStorage::addFile(const StorageFile& file) { auto it = m_filesIndex.find(file); @@ -164,6 +180,17 @@ Id IntermediateStorage::addEdge(const StorageEdgeData& edgeData) return edgeId; } +std::vector IntermediateStorage::addEdges(const std::vector& edges) +{ + std::vector edgeIds; + edgeIds.reserve(edges.size()); + for (const StorageEdge& edge : edges) + { + edgeIds.emplace_back(addEdge(edge)); + } + return edgeIds; +} + Id IntermediateStorage::addLocalSymbol(const StorageLocalSymbolData& localSymbolData) { auto it = m_localSymbols.find(StorageLocalSymbol(0, localSymbolData)); @@ -177,6 +204,17 @@ Id IntermediateStorage::addLocalSymbol(const StorageLocalSymbolData& localSymbol return localSymbolId; } +std::vector IntermediateStorage::addLocalSymbols(const std::set& symbols) +{ + std::vector symbolIds; + symbolIds.reserve(symbols.size()); + for (const StorageLocalSymbol& symbol : symbols) + { + symbolIds.emplace_back(addLocalSymbol(symbol)); + } + return symbolIds; +} + Id IntermediateStorage::addSourceLocation(const StorageSourceLocationData& sourceLocationData) { auto it = m_sourceLocations.find(StorageSourceLocation(0, sourceLocationData)); @@ -190,6 +228,17 @@ Id IntermediateStorage::addSourceLocation(const StorageSourceLocationData& sourc return sourceLocationId; } +std::vector IntermediateStorage::addSourceLocations(const std::vector& locations) +{ + std::vector locationIds; + locationIds.reserve(locations.size()); + for (const StorageSourceLocation& location : locations) + { + locationIds.emplace_back(addSourceLocation(location)); + } + return locationIds; +} + void IntermediateStorage::addOccurrence(const StorageOccurrence& occurrence) { m_occurrences.emplace(occurrence); @@ -205,6 +254,11 @@ void IntermediateStorage::addComponentAccess(const StorageComponentAccess& compo m_componentAccesses.emplace(componentAccess); } +void IntermediateStorage::addComponentAccesses(const std::vector& componentAccesses) +{ + m_componentAccesses.insert(componentAccesses.begin(), componentAccesses.end()); +} + void IntermediateStorage::addCommentLocation(const StorageCommentLocationData& commentLocationData) { m_commentLocations.emplace(commentLocationData); @@ -219,86 +273,6 @@ void IntermediateStorage::addError(const StorageErrorData& errorData) } } -void IntermediateStorage::forEachNode(std::function callback) const -{ - for (const StorageNode& node : m_nodes) - { - callback(node); - } -} - -void IntermediateStorage::forEachFile(std::function callback) const -{ - for (const StorageFile& file : m_files) - { - callback(file); - } -} - -void IntermediateStorage::forEachSymbol(std::function callback) const -{ - for (const StorageSymbol& symbol : m_symbols) - { - callback(symbol); - } -} - -void IntermediateStorage::forEachEdge(std::function callback) const -{ - for (const StorageEdge& edge : m_edges) - { - callback(edge); - } -} - -void IntermediateStorage::forEachLocalSymbol(std::function callback) const -{ - for (const StorageLocalSymbol& localSymbol : m_localSymbols) - { - callback(localSymbol); - } -} - -void IntermediateStorage::forEachSourceLocation(std::function callback) const -{ - for (const StorageSourceLocation& sourceLocation : m_sourceLocations) - { - callback(sourceLocation); - } -} - -void IntermediateStorage::forEachOccurrence(std::function callback) const -{ - for (const StorageOccurrence& occurrence : m_occurrences) - { - callback(occurrence); - } -} - -void IntermediateStorage::forEachComponentAccess(std::function callback) const -{ - for (const StorageComponentAccess& componentAccess : m_componentAccesses) - { - callback(componentAccess); - } -} - -void IntermediateStorage::forEachCommentLocation(std::function callback) const -{ - for (const StorageCommentLocationData& commentLocation : m_commentLocations) - { - callback(commentLocation); - } -} - -void IntermediateStorage::forEachError(std::function callback) const -{ - for (const StorageErrorData& error : m_errors) - { - callback(error); - } -} - const std::vector& IntermediateStorage::getStorageNodes() const { return m_nodes; diff --git a/src/lib/data/storage/IntermediateStorage.h b/src/lib/data/storage/IntermediateStorage.h index c56df745..bb033708 100644 --- a/src/lib/data/storage/IntermediateStorage.h +++ b/src/lib/data/storage/IntermediateStorage.h @@ -31,40 +31,33 @@ public: void setFilesWithErrorsIncomplete(); std::pair addNode(const StorageNodeData& nodeData) override; + std::vector addNodes(const std::vector& nodes) override; void addSymbol(const StorageSymbol& symbol) override; + void addSymbols(const std::vector& symbols) override; void addFile(const StorageFile& file) override; Id addEdge(const StorageEdgeData& edgeData) override; + std::vector addEdges(const std::vector& edges) override; Id addLocalSymbol(const StorageLocalSymbolData& localSymbolData) override; + std::vector addLocalSymbols(const std::set& symbols) override; Id addSourceLocation(const StorageSourceLocationData& sourceLocationData) override; + std::vector addSourceLocations(const std::vector& locations) override; void addOccurrence(const StorageOccurrence& occurrence) override; void addOccurrences(const std::vector& occurrences) override; void addComponentAccess(const StorageComponentAccess& componentAccess) override; + void addComponentAccesses(const std::vector& componentAccesses) override; void addCommentLocation(const StorageCommentLocationData& commentLocationData) override; void addError(const StorageErrorData& errorData) override; - void forEachNode(std::function callback) const override; - void forEachFile(std::function callback) const override; - void forEachSymbol(std::function callback) const override; - void forEachEdge(std::function callback) const override; - void forEachLocalSymbol(std::function callback) const override; - void forEachSourceLocation(std::function callback) const override; - void forEachOccurrence(std::function callback) const override; - void forEachComponentAccess(std::function callback) const override; - void forEachCommentLocation(std::function callback) const override; - void forEachError(std::function callback) const override; - - // for conversion to and from 'SharedIntermediateStorage' - - const std::vector& getStorageNodes() const; - const std::vector& getStorageFiles() const; - const std::vector& getStorageSymbols() const; - const std::vector& getStorageEdges() const; - const std::set& getStorageLocalSymbols() const; - const std::set& getStorageSourceLocations() const; - const std::set& getStorageOccurrences() const; - const std::set& getComponentAccesses() const; - const std::set& getCommentLocations() const; - const std::vector& getErrors() const; + const std::vector& getStorageNodes() const override; + const std::vector& getStorageFiles() const override; + const std::vector& getStorageSymbols() const override; + const std::vector& getStorageEdges() const override; + const std::set& getStorageLocalSymbols() const override; + const std::set& getStorageSourceLocations() const override; + const std::set& getStorageOccurrences() const override; + const std::set& getComponentAccesses() const override; + const std::set& getCommentLocations() const override; + const std::vector& getErrors() const override; void setStorageNodes(std::vector storageNodes); void setStorageFiles(std::vector storageFiles); diff --git a/src/lib/data/storage/PersistentStorage.cpp b/src/lib/data/storage/PersistentStorage.cpp index 0b9813e9..33cdc0fd 100644 --- a/src/lib/data/storage/PersistentStorage.cpp +++ b/src/lib/data/storage/PersistentStorage.cpp @@ -48,7 +48,12 @@ PersistentStorage::PersistentStorage(const FilePath& dbPath, const FilePath& boo std::pair PersistentStorage::addNode(const StorageNodeData& data) { - return std::make_pair(m_sqliteIndexStorage.addNode(data).id, true); + return std::make_pair(m_sqliteIndexStorage.addNode(data), true); +} + +std::vector PersistentStorage::addNodes(const std::vector& nodes) +{ + return m_sqliteIndexStorage.addNodes(nodes); } void PersistentStorage::addSymbol(const StorageSymbol& data) @@ -56,6 +61,11 @@ void PersistentStorage::addSymbol(const StorageSymbol& data) m_sqliteIndexStorage.addSymbol(data); } +void PersistentStorage::addSymbols(const std::vector& symbols) +{ + m_sqliteIndexStorage.addSymbols(symbols); +} + void PersistentStorage::addFile(const StorageFile& data) { const StorageFile storedFile = m_sqliteIndexStorage.getFirstById(data.id); @@ -80,17 +90,32 @@ void PersistentStorage::addFile(const StorageFile& data) Id PersistentStorage::addEdge(const StorageEdgeData& data) { - return m_sqliteIndexStorage.addEdge(data).id; + return m_sqliteIndexStorage.addEdge(data); +} + +std::vector PersistentStorage::addEdges(const std::vector& edges) +{ + return m_sqliteIndexStorage.addEdges(edges); } Id PersistentStorage::addLocalSymbol(const StorageLocalSymbolData& data) { - return m_sqliteIndexStorage.addLocalSymbol(data).id; + return m_sqliteIndexStorage.addLocalSymbol(data); +} + +std::vector PersistentStorage::addLocalSymbols(const std::set& symbols) +{ + return m_sqliteIndexStorage.addLocalSymbols(symbols); } Id PersistentStorage::addSourceLocation(const StorageSourceLocationData& data) { - return m_sqliteIndexStorage.addSourceLocation(data).id; + return m_sqliteIndexStorage.addSourceLocation(data); +} + +std::vector PersistentStorage::addSourceLocations(const std::vector& locations) +{ + return m_sqliteIndexStorage.addSourceLocations(locations); } void PersistentStorage::addOccurrence(const StorageOccurrence& data) @@ -108,6 +133,11 @@ void PersistentStorage::addComponentAccess(const StorageComponentAccess& compone m_sqliteIndexStorage.addComponentAccess(componentAccess); } +void PersistentStorage::addComponentAccesses(const std::vector& componentAccesses) +{ + m_sqliteIndexStorage.addComponentAccesses(componentAccesses); +} + void PersistentStorage::addCommentLocation(const StorageCommentLocationData& data) { m_sqliteIndexStorage.addCommentLocation(data); @@ -118,84 +148,64 @@ void PersistentStorage::addError(const StorageErrorData& data) m_sqliteIndexStorage.addError(data); } -void PersistentStorage::forEachNode(std::function callback) const +const std::vector& PersistentStorage::getStorageNodes() const { - for (StorageNode& node: m_sqliteIndexStorage.getAll()) - { - callback(node); - } + return m_storageData.nodes = m_sqliteIndexStorage.getAll(); } -void PersistentStorage::forEachFile(std::function callback) const +const std::vector& PersistentStorage::getStorageFiles() const { - for (StorageFile& file: m_sqliteIndexStorage.getAll()) - { - callback(file); - } + return m_storageData.files = m_sqliteIndexStorage.getAll(); } -void PersistentStorage::forEachSymbol(std::function callback) const +const std::vector& PersistentStorage::getStorageSymbols() const { - for (StorageSymbol& symbol: m_sqliteIndexStorage.getAll()) - { - callback(symbol); - } + return m_storageData.symbols = m_sqliteIndexStorage.getAll(); } -void PersistentStorage::forEachEdge(std::function callback) const +const std::vector& PersistentStorage::getStorageEdges() const { - for (StorageEdge& edge: m_sqliteIndexStorage.getAll()) - { - callback(edge); - } + return m_storageData.edges = m_sqliteIndexStorage.getAll(); } -void PersistentStorage::forEachLocalSymbol(std::function callback) const +const std::set& PersistentStorage::getStorageLocalSymbols() const { - for (StorageLocalSymbol& localSymbol: m_sqliteIndexStorage.getAll()) - { - callback(localSymbol); - } + return m_storageData.locals = utility::toSet(m_sqliteIndexStorage.getAll()); } -void PersistentStorage::forEachSourceLocation(std::function callback) const +const std::set& PersistentStorage::getStorageSourceLocations() const { - for (StorageSourceLocation& sourceLocation: m_sqliteIndexStorage.getAll()) - { - callback(sourceLocation); - } + return m_storageData.locations = utility::toSet(m_sqliteIndexStorage.getAll()); } -void PersistentStorage::forEachOccurrence(std::function callback) const +const std::set& PersistentStorage::getStorageOccurrences() const { - for (StorageOccurrence& occurrence: m_sqliteIndexStorage.getAll()) - { - callback(occurrence); - } + return m_storageData.occurrences = utility::toSet(m_sqliteIndexStorage.getAll()); } -void PersistentStorage::forEachComponentAccess(std::function callback) const +const std::set& PersistentStorage::getComponentAccesses() const { - for (StorageComponentAccess& componentAccess: m_sqliteIndexStorage.getAll()) - { - callback(componentAccess); - } + return m_storageData.accesses = utility::toSet(m_sqliteIndexStorage.getAll()); } -void PersistentStorage::forEachCommentLocation(std::function callback) const +const std::set& PersistentStorage::getCommentLocations() const { - for (StorageCommentLocation& commentLocation: m_sqliteIndexStorage.getAll()) + std::set comments; + for (const StorageCommentLocation& comment : m_sqliteIndexStorage.getAll()) { - callback(commentLocation); + comments.emplace(comment); } + return m_storageData.comments = comments; } -void PersistentStorage::forEachError(std::function callback) const +const std::vector& PersistentStorage::getErrors() const { - for (StorageError& error: m_sqliteIndexStorage.getAll()) + std::vector errors; + for (const StorageError& error : m_sqliteIndexStorage.getAll()) { - callback(error); + errors.emplace_back(error); } + return m_storageData.errors = errors; } void PersistentStorage::startInjection() diff --git a/src/lib/data/storage/PersistentStorage.h b/src/lib/data/storage/PersistentStorage.h index c33d237c..055e7734 100644 --- a/src/lib/data/storage/PersistentStorage.h +++ b/src/lib/data/storage/PersistentStorage.h @@ -20,27 +20,33 @@ public: PersistentStorage(const FilePath& dbPath, const FilePath& bookmarkPath); std::pair addNode(const StorageNodeData& data) override; + std::vector addNodes(const std::vector& nodes) override; void addSymbol(const StorageSymbol& data) override; + void addSymbols(const std::vector& symbols) override; void addFile(const StorageFile& data) override; Id addEdge(const StorageEdgeData& data) override; + std::vector addEdges(const std::vector& edges) override; Id addLocalSymbol(const StorageLocalSymbolData& data) override; + std::vector addLocalSymbols(const std::set& symbols) override; Id addSourceLocation(const StorageSourceLocationData& data) override; + std::vector addSourceLocations(const std::vector& locations) override; void addOccurrence(const StorageOccurrence& data) override; void addOccurrences(const std::vector& occurrences) override; void addComponentAccess(const StorageComponentAccess& componentAccess) override; + void addComponentAccesses(const std::vector& componentAccesses) override; void addCommentLocation(const StorageCommentLocationData& data) override; void addError(const StorageErrorData& data) override; - void forEachNode(std::function callback) const override; - void forEachFile(std::function callback) const override; - void forEachSymbol(std::function callback) const override; - void forEachEdge(std::function callback) const override; - void forEachLocalSymbol(std::function callback) const override; - void forEachSourceLocation(std::function callback) const override; - void forEachOccurrence(std::function callback) const override; - void forEachComponentAccess(std::function callback) const override; - void forEachCommentLocation(std::function callback) const override; - void forEachError(std::function callback) const override; + const std::vector& getStorageNodes() const override; + const std::vector& getStorageFiles() const override; + const std::vector& getStorageSymbols() const override; + const std::vector& getStorageEdges() const override; + const std::set& getStorageLocalSymbols() const override; + const std::set& getStorageSourceLocations() const override; + const std::set& getStorageOccurrences() const override; + const std::set& getComponentAccesses() const override; + const std::set& getCommentLocations() const override; + const std::vector& getErrors() const override; void startInjection() override; void finishInjection() override; @@ -153,6 +159,19 @@ public: const std::vector& locationIds, const std::vector& localSymbolIds) const override; private: + mutable struct { + std::vector nodes; + std::vector files; + std::vector symbols; + std::vector edges; + std::set locals; + std::set locations; + std::set occurrences; + std::set accesses; + std::set comments; + std::vector errors; + } m_storageData; + Id getFileNodeId(const FilePath& filePath) const; std::vector getFileNodeIds(const std::vector& filePaths) const; std::set getFileNodeIds(const std::set& filePaths) const; diff --git a/src/lib/data/storage/Storage.cpp b/src/lib/data/storage/Storage.cpp index 7a36d8eb..32c15dcf 100644 --- a/src/lib/data/storage/Storage.cpp +++ b/src/lib/data/storage/Storage.cpp @@ -2,6 +2,7 @@ #include +#include "logging.h" #include "StorageCommentLocation.h" #include "StorageComponentAccess.h" #include "StorageEdge.h" @@ -22,181 +23,263 @@ void Storage::inject(Storage* injected) { std::lock_guard lock(m_dataMutex); + std::map injectedIdToOwnElementId; + std::map injectedIdToOwnSourceLocationId; + TRACE(); startInjection(); - injected->forEachError( - [&](const StorageErrorData& injectedData) + { + // TRACE("inject errors"); + + for (const StorageErrorData& error : injected->getErrors()) { - addError(injectedData); + addError(error); } - ); + } - std::unordered_map injectedIdToOwnElementId; + { + // TRACE("inject nodes"); - injected->forEachNode( - [&](const StorageNode& injectedData) + const std::vector& nodes = injected->getStorageNodes(); + + std::vector nodeIds = addNodes(nodes); + + for (size_t i = 0; i < nodes.size(); i++) { - const Id ownId = addNode(injectedData).first; - if (ownId != 0) + if (nodeIds[i]) { - injectedIdToOwnElementId[injectedData.id] = ownId; + injectedIdToOwnElementId.emplace(nodes[i].id, nodeIds[i]); } } - ); + } - injected->forEachFile( - [&](const StorageFile& injectedData) + { + // TRACE("inject files"); + + for (const StorageFile& file : injected->getStorageFiles()) { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.id); + auto it = injectedIdToOwnElementId.find(file.id); if (it != injectedIdToOwnElementId.end()) { - const Id ownId = it->second; - addFile(StorageFile(ownId, injectedData.filePath, injectedData.modificationTime, injectedData.indexed, injectedData.complete)); + addFile(StorageFile( + it->second, + file.filePath, + file.modificationTime, + file.indexed, + file.complete + )); } } - ); + } - injected->forEachSymbol( - [&](const StorageSymbol& injectedData) + { + // TRACE("inject symbols"); + + std::vector symbols = injected->getStorageSymbols(); + for (size_t i = 0; i < symbols.size(); i++) { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.id); + auto it = injectedIdToOwnElementId.find(symbols[i].id); if (it != injectedIdToOwnElementId.end()) { - const Id ownId = it->second; - addSymbol(StorageSymbol(ownId, injectedData.definitionKind)); + symbols[i].id = it->second; + } + else + { + LOG_WARNING("New symbol id could not be found."); + symbols.erase(symbols.begin() + i); + i--; } } - ); - injected->forEachEdge( - [&](const StorageEdge& injectedData) + addSymbols(symbols); + } + + { + // TRACE("inject edges"); + + std::vector edges = injected->getStorageEdges(); + for (size_t i = 0; i < edges.size(); i++) { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.sourceNodeId); - if (it == injectedIdToOwnElementId.end()) + StorageEdge& edge = edges[i]; + size_t updateCount = 0; + + auto it = injectedIdToOwnElementId.find(edge.sourceNodeId); + if (it != injectedIdToOwnElementId.end()) { - return; + edge.sourceNodeId = it->second; + updateCount++; } - const Id ownSourceId = it->second; - it = injectedIdToOwnElementId.find(injectedData.targetNodeId); - if (it == injectedIdToOwnElementId.end()) + it = injectedIdToOwnElementId.find(edge.targetNodeId); + if (it != injectedIdToOwnElementId.end()) { - return; + edge.targetNodeId = it->second; + updateCount++; } - const Id ownTargetId = it->second; - const Id ownId = addEdge(StorageEdgeData(injectedData.type, ownSourceId, ownTargetId)); - - if (ownId != 0) + if (updateCount != 2) { - injectedIdToOwnElementId[injectedData.id] = ownId; + LOG_WARNING("New edge source or target id could not be found."); + edges.erase(edges.begin() + i); + i--; } } - ); - injected->forEachLocalSymbol( - [&](const StorageLocalSymbol& injectedData) + std::vector edgeIds = addEdges(edges); + + if (edges.size() == edgeIds.size()) { - const Id ownId = addLocalSymbol(injectedData); - if (ownId != 0) + for (size_t i = 0; i < edgeIds.size(); i++) { - injectedIdToOwnElementId[injectedData.id] = ownId; + if (edgeIds[i]) + { + injectedIdToOwnElementId.emplace(edges[i].id, edgeIds[i]); + } } } - ); - - std::unordered_map injectedIdToOwnSourceLocationId; - - injected->forEachSourceLocation( - [&](const StorageSourceLocation& injectedData) + else { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.fileNodeId); + LOG_ERROR("Returned edge ids don't match injected count."); + } + } + + { + // TRACE("inject local symbols"); + + const std::set& symbols = injected->getStorageLocalSymbols(); + std::vector symbolIds = addLocalSymbols(symbols); + + auto it = symbols.begin(); + for (size_t i = 0; i < symbols.size(); i++) + { + if (symbolIds[i]) + { + injectedIdToOwnElementId.emplace(it->id, symbolIds[i]); + } + it++; + } + } + + { + // TRACE("inject locations"); + + const std::set& oldLocations = injected->getStorageSourceLocations(); + std::vector locations; + locations.reserve(oldLocations.size()); + + for (const StorageSourceLocation& location : oldLocations) + { + auto it = injectedIdToOwnElementId.find(location.fileNodeId); if (it != injectedIdToOwnElementId.end()) { const Id ownFileNodeId = it->second; - - const Id ownId = addSourceLocation(StorageSourceLocationData( + locations.emplace_back( + location.id, ownFileNodeId, - injectedData.startLine, - injectedData.startCol, - injectedData.endLine, - injectedData.endCol, - injectedData.type - )); - if (ownId != 0) + location.startLine, + location.startCol, + location.endLine, + location.endCol, + location.type + ); + } + } + + std::vector locationIds = addSourceLocations(locations); + + if (locations.size() == locationIds.size()) + { + for (size_t i = 0; i < locationIds.size(); i++) + { + if (locationIds[i]) { - injectedIdToOwnSourceLocationId[injectedData.id] = ownId; + injectedIdToOwnSourceLocationId.emplace(locations[i].id, locationIds[i]); } } } - ); + else + { + LOG_ERROR("Returned source locations ids don't match injected count."); + } + } { + // TRACE("inject occurrences"); + + const std::set& oldOccurences = injected->getStorageOccurrences(); + std::vector occurrences; - injected->forEachOccurrence( - [&](const StorageOccurrence& injectedData) + occurrences.reserve(oldOccurences.size()); + + for (const StorageOccurrence& occurrence : oldOccurences) + { + Id elementId = 0; + Id sourceLocationId = 0; + + auto it = injectedIdToOwnElementId.find(occurrence.elementId); + if (it != injectedIdToOwnElementId.end()) { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.elementId); - if (it == injectedIdToOwnElementId.end()) - { - return; - } - const Id ownElementId = it->second; - - it = injectedIdToOwnSourceLocationId.find(injectedData.sourceLocationId); - if (it == injectedIdToOwnSourceLocationId.end()) - { - return; - } - const Id ownSourceLocationId = it->second; - - occurrences.emplace_back(ownElementId, ownSourceLocationId); + elementId = it->second; } - ); + + it = injectedIdToOwnSourceLocationId.find(occurrence.sourceLocationId); + if (it != injectedIdToOwnSourceLocationId.end()) + { + sourceLocationId = it->second; + } + + if (elementId && sourceLocationId) + { + occurrences.emplace_back(elementId, sourceLocationId); + } + else + { + LOG_WARNING("New occurrence element or location id could not be found."); + } + } + addOccurrences(occurrences); } - injected->forEachComponentAccess( - [&](const StorageComponentAccess& injectedData) + { + // TRACE("inject accesses"); + + const std::set& oldAccesses = injected->getComponentAccesses(); + std::vector accesses; + accesses.reserve(oldAccesses.size()); + + for (const StorageComponentAccess& access : oldAccesses) { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.nodeId); - if (it == injectedIdToOwnElementId.end()) + auto it = injectedIdToOwnElementId.find(access.nodeId); + if (it != injectedIdToOwnElementId.end()) { - return; + accesses.emplace_back(it->second, access.type); } - const Id ownNodeId = it->second; - - addComponentAccess(StorageComponentAccess(ownNodeId, injectedData.type)); } - ); - injected->forEachCommentLocation( - [&](const StorageCommentLocationData& injectedData) + addComponentAccesses(accesses); + } + + { + // TRACE("inject comments"); + + for (const StorageCommentLocationData& location : injected->getCommentLocations()) { - std::unordered_map::const_iterator it; - it = injectedIdToOwnElementId.find(injectedData.fileNodeId); - if (it == injectedIdToOwnElementId.end()) + auto it = injectedIdToOwnElementId.find(location.fileNodeId); + if (it != injectedIdToOwnElementId.end()) { - return; + const Id ownFileNodeId = it->second; + addCommentLocation(StorageCommentLocationData( + ownFileNodeId, + location.startLine, + location.startCol, + location.endLine, + location.endCol + )); } - const Id ownFileNodeId = it->second; - - addCommentLocation(StorageCommentLocationData( - ownFileNodeId, - injectedData.startLine, - injectedData.startCol, - injectedData.endLine, - injectedData.endCol - )); } - ); + } finishInjection(); } diff --git a/src/lib/data/storage/Storage.h b/src/lib/data/storage/Storage.h index 8632080e..ba6afe41 100644 --- a/src/lib/data/storage/Storage.h +++ b/src/lib/data/storage/Storage.h @@ -3,6 +3,7 @@ #include #include +#include #include #include "StorageCommentLocation.h" @@ -24,27 +25,33 @@ public: virtual ~Storage() = default; virtual std::pair addNode(const StorageNodeData& data) = 0; + virtual std::vector addNodes(const std::vector& nodes) = 0; virtual void addSymbol(const StorageSymbol& data) = 0; + virtual void addSymbols(const std::vector& symbols) = 0; virtual void addFile(const StorageFile& data) = 0; virtual Id addEdge(const StorageEdgeData& data) = 0; + virtual std::vector addEdges(const std::vector& edges) = 0; virtual Id addLocalSymbol(const StorageLocalSymbolData& data) = 0; + virtual std::vector addLocalSymbols(const std::set& symbols) = 0; virtual Id addSourceLocation(const StorageSourceLocationData& data) = 0; + virtual std::vector addSourceLocations(const std::vector& locations) = 0; virtual void addOccurrence(const StorageOccurrence& data) = 0; virtual void addOccurrences(const std::vector& occurrences) = 0; virtual void addComponentAccess(const StorageComponentAccess& componentAccess) = 0; + virtual void addComponentAccesses(const std::vector& componentAccesses) = 0; virtual void addCommentLocation(const StorageCommentLocationData& data) = 0; virtual void addError(const StorageErrorData& data) = 0; - virtual void forEachNode(std::function callback) const = 0; - virtual void forEachFile(std::function callback) const = 0; - virtual void forEachSymbol(std::function callback) const = 0; - virtual void forEachEdge(std::function callback) const = 0; - virtual void forEachLocalSymbol(std::function callback) const = 0; - virtual void forEachSourceLocation(std::function callback) const = 0; - virtual void forEachOccurrence(std::function callback) const = 0; - virtual void forEachComponentAccess(std::function callback) const = 0; - virtual void forEachCommentLocation(std::function callback) const = 0; - virtual void forEachError(std::function callback) const = 0; + virtual const std::vector& getStorageNodes() const = 0; + virtual const std::vector& getStorageFiles() const = 0; + virtual const std::vector& getStorageSymbols() const = 0; + virtual const std::vector& getStorageEdges() const = 0; + virtual const std::set& getStorageLocalSymbols() const = 0; + virtual const std::set& getStorageSourceLocations() const = 0; + virtual const std::set& getStorageOccurrences() const = 0; + virtual const std::set& getComponentAccesses() const = 0; + virtual const std::set& getCommentLocations() const = 0; + virtual const std::vector& getErrors() const = 0; void inject(Storage* injected); diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp index 0b8c44f1..8cc6451e 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp @@ -1,5 +1,6 @@ #include "SqliteIndexStorage.h" +#include #include #include "FileSystem.h" @@ -12,7 +13,6 @@ const size_t SqliteIndexStorage::s_storageVersion = 19; - namespace { std::tuple splitLocalSymbolName(const std::wstring& name) @@ -87,7 +87,13 @@ void SqliteIndexStorage::setProjectSettingsText(std::string text) insertOrUpdateMetaValue("project_settings", text); } -StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data) +Id SqliteIndexStorage::addNode(const StorageNodeData& data) +{ + std::vector ids = addNodes({ StorageNode(0, data) }); + return ids.size() ? ids[0] : 0; +} + +std::vector SqliteIndexStorage::addNodes(const std::vector& nodes) { if (m_tempNodeNameIndex.empty() && m_tempWNodeNameIndex.empty()) { @@ -107,70 +113,78 @@ StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data) } } - std::string name = utility::encodeToUtf8(data.serializedName); + std::vector nodeIds(nodes.size(), 0); + std::vector nodesToInsert; + for (size_t i = 0; i < nodes.size(); i++) { - Id nodeId; - if (name.size() != data.serializedName.size()) + const StorageNodeData& data = nodes[i]; + std::string name = utility::encodeToUtf8(data.serializedName); { - nodeId = m_tempWNodeNameIndex.find(data.serializedName); - } - else - { - nodeId = m_tempNodeNameIndex.find(name); - } - - if (nodeId) - { - auto it = m_tempNodeTypes.find(nodeId); - if (it != m_tempNodeTypes.end() && it->second < data.type) + Id nodeId; + if (name.size() != data.serializedName.size()) { - setNodeType(data.type, nodeId); - m_tempNodeTypes[nodeId] = data.type; + nodeId = m_tempWNodeNameIndex.find(data.serializedName); + } + else + { + nodeId = m_tempNodeNameIndex.find(name); + } + + if (nodeId) + { + auto it = m_tempNodeTypes.find(nodeId); + if (it != m_tempNodeTypes.end() && it->second < data.type) + { + setNodeType(data.type, nodeId); + m_tempNodeTypes[nodeId] = data.type; + } + + nodeIds[i] = nodeId; + } + else + { + executeStatement(m_insertElementStmt); + Id id = m_database.lastRowId(); + + nodesToInsert.emplace_back(id, data); + nodeIds[i] = id; + + if (name.size() != data.serializedName.size()) + { + m_tempWNodeNameIndex.add(data.serializedName, id); + } + else + { + m_tempNodeNameIndex.add(name, id); + } + m_tempNodeTypes.emplace(id, data.type); } - return StorageNode(nodeId, data); } } - Id id = 0; + if (nodesToInsert.size()) { - executeStatement(m_insertElementStmt); - id = m_database.lastRowId(); - m_insertElementStmt.reset(); - } - { - m_inserNodeStmt.bind(1, int(id)); - m_inserNodeStmt.bind(2, data.type); - m_inserNodeStmt.bind(3, name.c_str()); - executeStatement(m_inserNodeStmt); - m_inserNodeStmt.reset(); + m_insertNodeBatchStatement.execute(nodesToInsert, this); } - if (name.size() != data.serializedName.size()) - { - m_tempWNodeNameIndex.add(data.serializedName, id); - } - else - { - m_tempNodeNameIndex.add(name, id); - } - m_tempNodeTypes.emplace(id, data.type); - - return StorageNode(id, data); + return nodeIds; } -void SqliteIndexStorage::addSymbol(const StorageSymbol& data) +bool SqliteIndexStorage::addSymbol(const StorageSymbol& data) { - m_insertSymbolStmt.bind(1, int(data.id)); - m_insertSymbolStmt.bind(2, data.definitionKind); - executeStatement(m_insertSymbolStmt); - m_insertSymbolStmt.reset(); + return addSymbols({ data }); } -void SqliteIndexStorage::addFile(const StorageFile& data) +bool SqliteIndexStorage::addSymbols(const std::vector& symbols) +{ + return m_insertSymbolBatchStatement.execute(symbols, this); +} + +bool SqliteIndexStorage::addFile(const StorageFile& data) { if (getFileByPath(data.filePath).id != 0) { - return; + return false; } FilePath filePath(data.filePath); @@ -198,19 +212,25 @@ void SqliteIndexStorage::addFile(const StorageFile& data) m_insertFileStmt.bind(5, data.complete); m_insertFileStmt.bind(6, lineCount); success = executeStatement(m_insertFileStmt); - m_insertFileStmt.reset(); } if (success && content) { m_insertFileContentStmt.bind(1, int(data.id)); m_insertFileContentStmt.bind(2, content->getText().c_str()); - executeStatement(m_insertFileContentStmt); - m_insertFileContentStmt.reset(); + success = executeStatement(m_insertFileContentStmt); } + + return success; } -StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data) +Id SqliteIndexStorage::addEdge(const StorageEdgeData& data) +{ + std::vector ids = addEdges({ StorageEdge(0, data) }); + return ids.size() ? ids[0] : 0; +} + +std::vector SqliteIndexStorage::addEdges(const std::vector& edges) { if (m_tempEdgeIndex.empty()) { @@ -220,35 +240,43 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data) } } + std::vector edgeIds(edges.size(), 0); + std::vector edgesToInsert; + for (size_t i = 0; i < edges.size(); i++) { + const StorageEdge& data = edges[i]; std::map::const_iterator it = m_tempEdgeIndex.find(data); if (it != m_tempEdgeIndex.end()) { - return StorageEdge(it->second, data); + edgeIds[i] = it->second; + } + else + { + executeStatement(m_insertElementStmt); + Id id = m_database.lastRowId(); + + edgeIds[i] = id; + edgesToInsert.emplace_back(id, data); + + m_tempEdgeIndex.emplace(data, id); } } - Id id = 0; + if (edgesToInsert.size()) { - executeStatement(m_insertElementStmt); - id = m_database.lastRowId(); - m_insertElementStmt.reset(); - } - { - m_insertEdgeStmt.bind(1, int(id)); - m_insertEdgeStmt.bind(2, data.type); - m_insertEdgeStmt.bind(3, int(data.sourceNodeId)); - m_insertEdgeStmt.bind(4, int(data.targetNodeId)); - executeStatement(m_insertEdgeStmt); - m_insertEdgeStmt.reset(); + m_insertEdgeBatchStatement.execute(edgesToInsert, this); } - m_tempEdgeIndex.emplace(data, id); - - return StorageEdge(id, data); + return edgeIds; } -StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data) +Id SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data) +{ + std::vector ids = addLocalSymbols({ StorageLocalSymbol(0, data) }); + return ids.size() ? ids[0] : 0; +} + +std::vector SqliteIndexStorage::addLocalSymbols(const std::set& symbols) { std::wstring name; uint32_t line; @@ -266,42 +294,55 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa } } - std::tie(name, line, col) = splitLocalSymbolName(data.name); - if (name.size()) + std::vector symbolIds(symbols.size(), 0); + std::vector symbolsToInsert; + auto it = symbols.begin(); + for (size_t i = 0; i < symbols.size(); i++) { - auto it = m_tempLocalSymbolIndex.find(name); - if (it != m_tempLocalSymbolIndex.end()) + const StorageLocalSymbol& data = *it; + std::tie(name, line, col) = splitLocalSymbolName(data.name); + if (name.size()) { - auto it2 = it->second.find(std::make_pair(line, col)); - if (it2 != it->second.end()) + auto it = m_tempLocalSymbolIndex.find(name); + if (it != m_tempLocalSymbolIndex.end()) { - return StorageLocalSymbol(it2->second, data); + auto it2 = it->second.find(std::make_pair(line, col)); + if (it2 != it->second.end()) + { + symbolIds[i] = it2->second; + } } } + + if (!symbolIds[i]) + { + executeStatement(m_insertElementStmt); + Id id = m_database.lastRowId(); + + symbolIds[i] = id; + symbolsToInsert.emplace_back(id, data); + + m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id); + } + + it++; } - Id id = 0; + if (symbolsToInsert.size()) { - executeStatement(m_insertElementStmt); - id = m_database.lastRowId(); - m_insertElementStmt.reset(); - } - { - m_inserLocalSymbolStmt.bind(1, int(id)); - m_inserLocalSymbolStmt.bind(2, utility::encodeToUtf8(data.name).c_str()); - executeStatement(m_inserLocalSymbolStmt); - m_inserLocalSymbolStmt.reset(); + m_insertLocalSymbolBatchStatement.execute(symbolsToInsert, this); } - if (name.size()) - { - m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id); - } - - return StorageLocalSymbol(id, data); + return symbolIds; } -StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceLocationData& data) +Id SqliteIndexStorage::addSourceLocation(const StorageSourceLocationData& data) +{ + std::vector ids = addSourceLocations({ StorageSourceLocation(0, data) }); + return ids.size() ? ids[0] : 0; +} + +std::vector SqliteIndexStorage::addSourceLocations(const std::vector& locations) { if (m_tempSourceLocationIndices.empty()) { @@ -314,98 +355,59 @@ StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceL } } - const TempSourceLocation tempLoc(data.startLine, data.endLine - data.startLine, data.startCol, data.endCol, data.type); + std::vector locationIds(locations.size(), 0); + std::vector locationsToInsert; + size_t lastRowId = executeStatementScalar("SELECT MAX(rowid) from source_location", 0); - std::map& index = m_tempSourceLocationIndices[data.fileNodeId]; + for (size_t i = 0; i < locations.size(); i++) { + const StorageSourceLocation& data = locations[i]; + const TempSourceLocation tempLoc(data.startLine, data.endLine - data.startLine, data.startCol, data.endCol, data.type); + + std::map& index = m_tempSourceLocationIndices[data.fileNodeId]; std::map::const_iterator it = index.find(tempLoc); if (it != index.end()) { - return StorageSourceLocation(it->second, data); + locationIds[i] = it->second; + } + else + { + executeStatement(m_insertElementStmt); + Id id = lastRowId + 1 + locationsToInsert.size(); + + locationIds[i] = id; + index.emplace(tempLoc, id); + + locationsToInsert.emplace_back(data); } } - Id id = 0; - m_insertSourceLocationStmt.bind(1, int(data.fileNodeId)); - m_insertSourceLocationStmt.bind(2, int(data.startLine)); - m_insertSourceLocationStmt.bind(3, int(data.startCol)); - m_insertSourceLocationStmt.bind(4, int(data.endLine)); - m_insertSourceLocationStmt.bind(5, int(data.endCol)); - m_insertSourceLocationStmt.bind(6, data.type); - - const bool success = executeStatement(m_insertSourceLocationStmt); - if (success) + if (locationsToInsert.size()) { - id = m_database.lastRowId(); - index.emplace(tempLoc, id); + m_insertSourceLocationBatchStatement.execute(locationsToInsert, this); } - m_insertSourceLocationStmt.reset(); - - return StorageSourceLocation(id, data); + return locationIds; } bool SqliteIndexStorage::addOccurrence(const StorageOccurrence& data) { - m_insertOccurrenceStmt.bind(1, int(data.elementId)); - m_insertOccurrenceStmt.bind(2, int(data.sourceLocationId)); - const bool success = executeStatement(m_insertOccurrenceStmt); - m_insertOccurrenceStmt.reset(); - return success; + return addOccurrences({ data }); } bool SqliteIndexStorage::addOccurrences(const std::vector& occurrences) { - size_t i = 0; - while (occurrences.size() - i >= 100) - { - for (size_t j = 0; j < 100; j++) - { - m_insert100OccurrencesStmt.bind((j * 2) + 1, int(occurrences[i + j].elementId)); - m_insert100OccurrencesStmt.bind((j * 2) + 2, int(occurrences[i + j].sourceLocationId)); - } - - const bool success = executeStatement(m_insert100OccurrencesStmt); - m_insert100OccurrencesStmt.reset(); - - if (!success) - { - return false; - } - - i += 100; - } - - if (i < occurrences.size()) - { - std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES"; - { - bool isFirst = true; - while (i < occurrences.size()) - { - if (!isFirst) - { - stmt += ","; - } - isFirst = false; - stmt += "(" + std::to_string(occurrences[i].elementId) + "," + std::to_string(occurrences[i].sourceLocationId) + ")"; - i++; - } - stmt += ";"; - } - return executeStatement(stmt); - } - - return true; + return m_insertOccurenceBatchStatement.execute(occurrences, this); } bool SqliteIndexStorage::addComponentAccess(const StorageComponentAccess& componentAccess) { - m_insertComponentAccessStmt.bind(1, int(componentAccess.nodeId)); - m_insertComponentAccessStmt.bind(2, componentAccess.type); - const bool success = executeStatement(m_insertComponentAccessStmt); - m_insertComponentAccessStmt.reset(); - return success; + return addComponentAccesses({ componentAccess }); +} + +bool SqliteIndexStorage::addComponentAccesses(const std::vector& componentAccesses) +{ + return m_insertComponentAccessBatchStatement.execute(componentAccesses, this); } StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageCommentLocationData& data) @@ -423,7 +425,6 @@ StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageComme { id = checkQuery.getIntField(0, 0); } - m_checkCommentLocationExistsStmt.reset(); } @@ -440,8 +441,6 @@ StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageComme { id = m_database.lastRowId(); } - - m_insertCommentLocationStmt.reset(); } return StorageCommentLocation(id, data); @@ -464,7 +463,6 @@ StorageError SqliteIndexStorage::addError(const StorageErrorData& data) { id = checkQuery.getIntField(0, -1); } - m_checkErrorExistsStmt.reset(); } @@ -483,8 +481,6 @@ StorageError SqliteIndexStorage::addError(const StorageErrorData& data) { id = m_database.lastRowId(); } - - m_insertErrorStmt.reset(); } return StorageError(id, data); @@ -787,6 +783,8 @@ StorageNode SqliteIndexStorage::getNodeBySerializedName(const std::wstring& seri } } + stmt.reset(); + return StorageNode(); } @@ -1252,50 +1250,93 @@ void SqliteIndexStorage::setupPrecompiledStatements() { try { + m_insertNodeBatchStatement.compile( + "INSERT INTO node(id, type, serialized_name) VALUES", + 3, + [](CppSQLite3Statement& stmt, const StorageNode& node, size_t index) + { + stmt.bind(index * 3 + 1, int(node.id)); + stmt.bind(index * 3 + 2, int(node.type)); + stmt.bind(index * 3 + 3, utility::encodeToUtf8(node.serializedName).c_str()); + }, + m_database + ); + m_insertEdgeBatchStatement.compile( + "INSERT INTO edge(id, type, source_node_id, target_node_id) VALUES", + 4, + [](CppSQLite3Statement& stmt, const StorageEdge& edge, size_t index) + { + stmt.bind(index * 4 + 1, int(edge.id)); + stmt.bind(index * 4 + 2, int(edge.type)); + stmt.bind(index * 4 + 3, int(edge.sourceNodeId)); + stmt.bind(index * 4 + 4, int(edge.targetNodeId)); + }, + m_database + ); + m_insertSymbolBatchStatement.compile( + "INSERT OR IGNORE INTO symbol(id, definition_kind) VALUES", + 2, + [](CppSQLite3Statement& stmt, const StorageSymbol& symbol, size_t index) + { + stmt.bind(index * 2 + 1, int(symbol.id)); + stmt.bind(index * 2 + 2, int(symbol.definitionKind)); + }, + m_database + ); + m_insertLocalSymbolBatchStatement.compile( + "INSERT INTO local_symbol(id, name) VALUES", + 2, + [](CppSQLite3Statement& stmt, const StorageLocalSymbol& symbol, size_t index) + { + stmt.bind(index * 2 + 1, int(symbol.id)); + stmt.bind(index * 2 + 2, utility::encodeToUtf8(symbol.name).c_str()); + }, + m_database + ); + m_insertSourceLocationBatchStatement.compile( + "INSERT INTO source_location(file_node_id, start_line, start_column, end_line, end_column, type) VALUES", + 6, + [](CppSQLite3Statement& stmt, const StorageSourceLocationData& location, size_t index) + { + stmt.bind(index * 6 + 1, int(location.fileNodeId)); + stmt.bind(index * 6 + 2, int(location.startLine)); + stmt.bind(index * 6 + 3, int(location.startCol)); + stmt.bind(index * 6 + 4, int(location.endLine)); + stmt.bind(index * 6 + 5, int(location.endCol)); + stmt.bind(index * 6 + 6, int(location.type)); + }, + m_database + ); + m_insertOccurenceBatchStatement.compile( + "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES", + 2, + [](CppSQLite3Statement& stmt, const StorageOccurrence& occurrence, size_t index) + { + stmt.bind(index * 2 + 1, int(occurrence.elementId)); + stmt.bind(index * 2 + 2, int(occurrence.sourceLocationId)); + }, + m_database + ); + m_insertComponentAccessBatchStatement.compile( + "INSERT OR IGNORE INTO component_access(node_id, type) VALUES", + 2, + [](CppSQLite3Statement& stmt, const StorageComponentAccess& componentAccess, size_t index) + { + stmt.bind(index * 2 + 1, int(componentAccess.nodeId)); + stmt.bind(index * 2 + 2, int(componentAccess.type)); + }, + m_database + ); + m_insertElementStmt = m_database.compileStatement( "INSERT INTO element(id) VALUES(NULL);" ); - m_insertEdgeStmt = m_database.compileStatement( - "INSERT INTO edge(id, type, source_node_id, target_node_id) VALUES(?, ?, ?, ?);" - ); - m_inserNodeStmt = m_database.compileStatement( - "INSERT INTO node(id, type, serialized_name) VALUES(?, ?, ?);" - ); - m_insertSymbolStmt = m_database.compileStatement( - "INSERT OR IGNORE INTO symbol(id, definition_kind) VALUES(?, ?);" - ); m_insertFileStmt = m_database.compileStatement( "INSERT INTO file(id, path, modification_time, indexed, complete, line_count) VALUES(?, ?, ?, ?, ?, ?);" ); m_insertFileContentStmt = m_database.compileStatement( "INSERT INTO filecontent(id, content) VALUES(?, ?);" ); - m_inserLocalSymbolStmt = m_database.compileStatement( - "INSERT INTO local_symbol(id, name) VALUES(?, ?);" - ); - m_insertSourceLocationStmt = m_database.compileStatement( - "INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) " - "VALUES(NULL, ?, ?, ?, ?, ?, ?);" - ); - m_insertOccurrenceStmt = m_database.compileStatement( - "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES(?, ?);" - ); - { - std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES"; - for (int i = 0; i < 100; i++) - { - if (i != 0) - { - stmt += ","; - } - stmt += "(?, ?)"; - } - stmt += ";"; - m_insert100OccurrencesStmt = m_database.compileStatement(stmt.c_str()); - } - m_insertComponentAccessStmt = m_database.compileStatement( - "INSERT OR IGNORE INTO component_access(node_id, type) VALUES(?, ?);" - ); m_checkCommentLocationExistsStmt = m_database.compileStatement( "SELECT id FROM comment_location WHERE " "file_node_id = ? AND " diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.h b/src/lib/data/storage/sqlite/SqliteIndexStorage.h index ddc966c3..c9007081 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.h +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.h @@ -49,15 +49,21 @@ public: std::string getProjectSettingsText() const; void setProjectSettingsText(std::string text); - StorageNode addNode(const StorageNodeData& data); - void addSymbol(const StorageSymbol& data); - void addFile(const StorageFile& data); - StorageEdge addEdge(const StorageEdgeData& data); - StorageLocalSymbol addLocalSymbol(const StorageLocalSymbolData& data); - StorageSourceLocation addSourceLocation(const StorageSourceLocationData& data); + Id addNode(const StorageNodeData& data); + std::vector addNodes(const std::vector& nodes); + bool addSymbol(const StorageSymbol& data); + bool addSymbols(const std::vector& symbols); + bool addFile(const StorageFile& data); + Id addEdge(const StorageEdgeData& data); + std::vector addEdges(const std::vector& edges); + Id addLocalSymbol(const StorageLocalSymbolData& data); + std::vector addLocalSymbols(const std::set& symbols); + Id addSourceLocation(const StorageSourceLocationData& data); + std::vector addSourceLocations(const std::vector& locations); bool addOccurrence(const StorageOccurrence& data); bool addOccurrences(const std::vector& occurrences); bool addComponentAccess(const StorageComponentAccess& componentAccess); + bool addComponentAccesses(const std::vector& componentAccesses); StorageCommentLocation addCommentLocation(const StorageCommentLocationData& data); StorageError addError(const StorageErrorData& data); @@ -223,23 +229,92 @@ private: std::map, Id>> m_tempLocalSymbolIndex; std::map> m_tempSourceLocationIndices; + template + class InsertBatchStatement + { + public: + void compile( + const std::string header, + size_t valueCount, + std::function bindValuesFunc, + CppSQLite3DB& database) + { + m_bindValuesFunc = bindValuesFunc; + + std::string valueStr = '(' + utility::join(std::vector(valueCount, "?"), ',') + ')'; + + for (size_t j = 0; j < 4; j++) + { + std::stringstream stmt; + stmt << header; + + for (size_t i = 0; i < BATCH_SIZES[j]; i++) + { + if (i != 0) + { + stmt << ','; + } + stmt << valueStr; + } + stmt << ';'; + + m_stmt[j] = database.compileStatement(stmt.str().c_str()); + } + } + + bool execute(const std::vector& types, SqliteIndexStorage* storage) + { + size_t i = 0; + for (size_t x = 0; x < 4; x++) + { + while (types.size() - i >= BATCH_SIZES[x]) + { + for (size_t j = 0; j < BATCH_SIZES[x]; j++) + { + m_bindValuesFunc(m_stmt[x], types[i + j], j); + } + + const bool success = storage->executeStatement(m_stmt[x]); + if (!success) + { + return false; + } + + i += BATCH_SIZES[x]; + } + } + + return true; + } + + private: + static const size_t BATCH_SIZES[4]; + + CppSQLite3Statement m_stmt[4]; + + std::function m_bindValuesFunc; + }; + + InsertBatchStatement m_insertNodeBatchStatement; + InsertBatchStatement m_insertEdgeBatchStatement; + InsertBatchStatement m_insertSymbolBatchStatement; + InsertBatchStatement m_insertLocalSymbolBatchStatement; + InsertBatchStatement m_insertSourceLocationBatchStatement; + InsertBatchStatement m_insertOccurenceBatchStatement; + InsertBatchStatement m_insertComponentAccessBatchStatement; + CppSQLite3Statement m_insertElementStmt; - CppSQLite3Statement m_insertEdgeStmt; - CppSQLite3Statement m_inserNodeStmt; - CppSQLite3Statement m_insertSymbolStmt; CppSQLite3Statement m_insertFileStmt; CppSQLite3Statement m_insertFileContentStmt; - CppSQLite3Statement m_inserLocalSymbolStmt; - CppSQLite3Statement m_insertSourceLocationStmt; - CppSQLite3Statement m_insertOccurrenceStmt; - CppSQLite3Statement m_insert100OccurrencesStmt; - CppSQLite3Statement m_insertComponentAccessStmt; CppSQLite3Statement m_checkCommentLocationExistsStmt; CppSQLite3Statement m_insertCommentLocationStmt; CppSQLite3Statement m_checkErrorExistsStmt; CppSQLite3Statement m_insertErrorStmt; }; +template +const size_t SqliteIndexStorage::InsertBatchStatement::BATCH_SIZES[4] = { 100, 27, 8, 1 }; + template <> std::vector SqliteIndexStorage::doGetAll(const std::string& query) const; template <> diff --git a/src/lib/data/storage/sqlite/SqliteStorage.cpp b/src/lib/data/storage/sqlite/SqliteStorage.cpp index 229ae8e6..3e55f157 100644 --- a/src/lib/data/storage/sqlite/SqliteStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteStorage.cpp @@ -182,6 +182,8 @@ bool SqliteStorage::executeStatement(CppSQLite3Statement& statement) const LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage()); return false; } + + statement.reset(); return true; } @@ -222,6 +224,7 @@ int SqliteStorage::executeStatementScalar(CppSQLite3Statement& statement, const { LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage()); } + return ret; } diff --git a/src/test/SqliteIndexStorageTestSuite.h b/src/test/SqliteIndexStorageTestSuite.h index 7de11b92..91e889c8 100644 --- a/src/test/SqliteIndexStorageTestSuite.h +++ b/src/test/SqliteIndexStorageTestSuite.h @@ -32,7 +32,7 @@ public: SqliteIndexStorage storage(databasePath); storage.setup(); storage.beginTransaction(); - int nodeId = storage.addNode(StorageNodeData(0, L"a")).id; + int nodeId = storage.addNode(StorageNodeData(0, L"a")); storage.removeElement(nodeId); storage.commitTransaction(); nodeCount = storage.getNodeCount(); @@ -50,8 +50,8 @@ public: SqliteIndexStorage storage(databasePath); storage.setup(); storage.beginTransaction(); - int sourceNodeId = storage.addNode(StorageNodeData(0, L"a")).id; - int targetNodeId = storage.addNode(StorageNodeData(0, L"b")).id; + int sourceNodeId = storage.addNode(StorageNodeData(0, L"a")); + int targetNodeId = storage.addNode(StorageNodeData(0, L"b")); storage.addEdge(StorageEdgeData(0, sourceNodeId, targetNodeId)); storage.commitTransaction(); edgeCount = storage.getEdgeCount(); @@ -69,9 +69,9 @@ public: SqliteIndexStorage storage(databasePath); storage.setup(); storage.beginTransaction(); - int sourceNodeId = storage.addNode(StorageNodeData(0, L"a")).id; - int targetNodeId = storage.addNode(StorageNodeData(0, L"b")).id; - int edgeId = storage.addEdge(StorageEdgeData(0, sourceNodeId, targetNodeId)).id; + int sourceNodeId = storage.addNode(StorageNodeData(0, L"a")); + int targetNodeId = storage.addNode(StorageNodeData(0, L"b")); + int edgeId = storage.addEdge(StorageEdgeData(0, sourceNodeId, targetNodeId)); storage.removeElement(edgeId); storage.commitTransaction(); edgeCount = storage.getEdgeCount(); diff --git a/src/test/StorageTestSuite.h b/src/test/StorageTestSuite.h index bbc9522b..c930e652 100644 --- a/src/test/StorageTestSuite.h +++ b/src/test/StorageTestSuite.h @@ -69,14 +69,13 @@ public: const Id sourceId = storage.getNodeIdForNameHierarchy(a); const Id targetId = storage.getNodeIdForNameHierarchy(b); - storage.forEachEdge([&](const StorageEdge& edge) + for (auto edge : storage.getStorageEdges()) + { + if (edge.sourceNodeId == sourceId && edge.targetNodeId == targetId && edge.type == Edge::typeToInt(Edge::EDGE_MEMBER)) { - if (edge.sourceNodeId == sourceId && edge.targetNodeId == targetId && edge.type == Edge::typeToInt(Edge::EDGE_MEMBER)) - { - foundEdge = true; - } + foundEdge = true; } - ); + } TS_ASSERT(foundEdge); }