From 6a576bccc98d29fc56c399aa2c4c36665186c448 Mon Sep 17 00:00:00 2001 From: malte_langkabel Date: Wed, 11 May 2016 13:23:31 +0200 Subject: [PATCH] src: storage refactoring * renamed Storage to PersistentStorage * added Storage as baseclass that implements data injection * cleaned ParserClient interface * added default constructors for storage types * cleaned SqlteStorage by implementing the private getAll methods as template specializations * added CXX flags to CMakeLists (this is needed for sqlite with visual studio) * removed TestStorage.h/.cpp since these files were not used anymore. --- CMakeLists.txt | 1 + src/app/data/parser/cxx/TaskParseCxx.cpp | 6 +- src/lib/CMakeLists.txt | 2 + src/lib/Project.cpp | 4 +- src/lib/Project.h | 4 +- src/lib/data/IntermediateStorage.cpp | 292 +--- src/lib/data/IntermediateStorage.h | 45 +- src/lib/data/PersistentStorage.cpp | 1308 +++++++++++++++++ src/lib/data/PersistentStorage.h | 153 ++ src/lib/data/SqliteStorage.cpp | 437 +++--- src/lib/data/SqliteStorage.h | 56 +- src/lib/data/Storage.cpp | 1257 ++-------------- src/lib/data/Storage.h | 140 +- src/lib/data/StorageTypes.h | 52 + src/lib/data/TaskCleanStorage.cpp | 4 +- src/lib/data/TaskCleanStorage.h | 6 +- src/lib/data/access/StorageAccess.h | 2 - src/lib/data/access/StorageAccessProxy.cpp | 10 - src/lib/data/access/StorageAccessProxy.h | 2 - .../token_component/TokenComponentAccess.h | 2 +- src/lib/data/parser/ParserClient.h | 3 - src/lib/data/parser/ParserClientImpl.cpp | 52 +- src/lib/data/parser/ParserClientImpl.h | 4 +- src/lib/data/parser/cxx/TaskParseCxx.h | 6 +- src/test/CMakeLists.txt | 2 - src/test/StorageTestSuite.h | 18 +- src/test/helper/TestStorage.cpp | 6 - src/test/helper/TestStorage.h | 14 - src/trial/data/parser/cxx/TaskParseCxx.cpp | 4 +- 29 files changed, 2072 insertions(+), 1820 deletions(-) create mode 100644 src/lib/data/PersistentStorage.cpp create mode 100644 src/lib/data/PersistentStorage.h delete mode 100644 src/test/helper/TestStorage.cpp delete mode 100644 src/test/helper/TestStorage.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 79e8b953..fcab942d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ endif () # enable fts4 module for sqlite set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS") # Clang ------------------------------------------------------------------------ diff --git a/src/app/data/parser/cxx/TaskParseCxx.cpp b/src/app/data/parser/cxx/TaskParseCxx.cpp index b3fb5bd0..b19fc4fa 100644 --- a/src/app/data/parser/cxx/TaskParseCxx.cpp +++ b/src/app/data/parser/cxx/TaskParseCxx.cpp @@ -5,14 +5,14 @@ #include "clang/Tooling/JSONCompilationDatabase.h" #include "data/parser/cxx/CxxParser.h" -#include "data/Storage.h" +#include "data/PersistentStorage.h" #include "utility/file/FileRegister.h" #include "utility/messaging/type/MessageFinishedParsing.h" #include "utility/messaging/type/MessageStatus.h" #include "utility/utility.h" TaskParseCxx::TaskParseCxx( - Storage* storage, + PersistentStorage* storage, const FileManager* fileManager, const Parser::Arguments& arguments, const std::vector& files @@ -126,7 +126,7 @@ Task::TaskState TaskParseCxx::update() m_parserClient->finishParsingFile(sourcePath); m_parserClient->resetStorage(); - m_storage->injectData(intermediateStorage); + m_storage->inject(intermediateStorage.get()); if (isSource) { diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 53325b86..bb76c99f 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -161,6 +161,8 @@ add_files( data/HierarchyCache.h data/IntermediateStorage.cpp data/IntermediateStorage.h + data/PersistentStorage.cpp + data/PersistentStorage.h data/SqliteIndex.cpp data/SqliteIndex.h data/SqliteStorage.cpp diff --git a/src/lib/Project.cpp b/src/lib/Project.cpp index 212400dc..73ba609e 100644 --- a/src/lib/Project.cpp +++ b/src/lib/Project.cpp @@ -10,7 +10,7 @@ #include "data/access/StorageAccessProxy.h" #include "data/graph/Token.h" #include "data/parser/cxx/TaskParseCxx.h" -#include "data/Storage.h" +#include "data/PersistentStorage.h" #include "data/TaskCleanStorage.h" #include "settings/ApplicationSettings.h" #include "settings/ProjectSettings.h" @@ -219,7 +219,7 @@ void Project::loadStorage(const FilePath& path) FilePath dbPath = FilePath(path).replaceExtension("coatidb"); if (!m_storage || path != m_projectSettingsFilepath || !dbPath.exists()) { - m_storage = std::make_shared(dbPath); + m_storage = std::make_shared(dbPath); } } diff --git a/src/lib/Project.h b/src/lib/Project.h index 0f57feae..e6fa400b 100644 --- a/src/lib/Project.h +++ b/src/lib/Project.h @@ -7,7 +7,7 @@ #include "data/parser/Parser.h" -class Storage; +class PersistentStorage; class StorageAccessProxy; class Project @@ -57,7 +57,7 @@ private: FilePath m_projectSettingsFilepath; FileManager m_fileManager; - std::shared_ptr m_storage; + std::shared_ptr m_storage; }; #endif // PROJECT_H diff --git a/src/lib/data/IntermediateStorage.cpp b/src/lib/data/IntermediateStorage.cpp index f1e0fd01..50859b13 100644 --- a/src/lib/data/IntermediateStorage.cpp +++ b/src/lib/data/IntermediateStorage.cpp @@ -12,32 +12,36 @@ IntermediateStorage::~IntermediateStorage() { } -Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId) +Id IntermediateStorage::addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime) { - std::shared_ptr edge = std::make_shared(0, type, sourceId, targetId); + std::shared_ptr file = std::make_shared(0, name, filePath, modificationTime); - std::string serialized = serialize(*(edge.get())); - std::unordered_map::const_iterator it = m_edgeNamesToIds.find(serialized); - if (it != m_edgeNamesToIds.end()) + std::string serialized = serialize(*(file.get())); + std::unordered_map::const_iterator it = m_fileNamesToIds.find(serialized); + if (it != m_fileNamesToIds.end()) { - return it->second; + Id id = it->second; + if (m_fileIdsToData[id]->filePath.size() == 0) // stored information is incomplete. + { + m_fileIdsToData[id]->filePath = filePath; // so we replace it. + } + if (m_fileIdsToData[id]->modificationTime.size() == 0) // stored information is incomplete. + { + m_fileIdsToData[id]->modificationTime = modificationTime; // so we replace it. + } + return id; } Id id = m_nextId++; - m_edgeNamesToIds[serialized] = id; - m_edgeIdsToData[id] = edge; - - if (type == Edge::EDGE_MEMBER) - { - m_nodeIdsToMemberEdgeIds[targetId] = id; - } + m_fileNamesToIds[serialized] = id; + m_fileIdsToData[id] = file; return id; } -Id IntermediateStorage::addNode(int type, const NameHierarchy& nameHierarchy, int definitionType) +Id IntermediateStorage::addNode(int type, const std::string& serializedName, int definitionType) { - std::shared_ptr node = std::make_shared(0, type, NameHierarchy::serialize(nameHierarchy), definitionType); + std::shared_ptr node = std::make_shared(0, type, serializedName, definitionType); std::string serialized = serialize(*(node.get())); std::unordered_map::const_iterator it = m_nodeNamesToIds.find(serialized); @@ -67,43 +71,20 @@ Id IntermediateStorage::addNode(int type, const NameHierarchy& nameHierarchy, in return id; } -Id IntermediateStorage::addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime) +Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId) { - std::shared_ptr file = std::make_shared(0, name, filePath, modificationTime); + std::shared_ptr edge = std::make_shared(0, type, sourceId, targetId); - std::string serialized = serialize(*(file.get())); - std::unordered_map::const_iterator it = m_fileNamesToIds.find(serialized); - if (it != m_fileNamesToIds.end()) - { - Id id = it->second; - if (m_fileIdsToData[id]->filePath.size() == 0) // stored information is incomplete. - { - m_fileIdsToData[id] = file; // so we replace it. - } - return id; - } - - Id id = m_nextId++; - m_fileNamesToIds[serialized] = id; - m_fileIdsToData[id] = file; - - return id; -} - -Id IntermediateStorage::addFile(const std::string& filePath) -{ - std::shared_ptr file = std::make_shared(0, "", filePath, ""); - - std::string serialized = serialize(*(file.get())); - std::unordered_map::const_iterator it = m_fileNamesToIds.find(serialized); - if (it != m_fileNamesToIds.end()) + std::string serialized = serialize(*(edge.get())); + std::unordered_map::const_iterator it = m_edgeNamesToIds.find(serialized); + if (it != m_edgeNamesToIds.end()) { return it->second; } Id id = m_nextId++; - m_fileNamesToIds[serialized] = id; - m_fileIdsToData[id] = file; + m_edgeNamesToIds[serialized] = id; + m_edgeIdsToData[id] = edge; return id; } @@ -126,229 +107,48 @@ Id IntermediateStorage::addLocalSymbol(const std::string& name) return id; } -void IntermediateStorage::addSourceLocation(Id elementId, const ParseLocation& location, int type) +void IntermediateStorage::addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) { - Id fileNodeId = addFile(location.filePath.str()); m_sourceLocations.push_back(StorageSourceLocation( 0, elementId, fileNodeId, - location.startLineNumber, - location.startColumnNumber, - location.endLineNumber, - location.endColumnNumber, + startLine, + startCol, + endLine, + endCol, type )); } -void IntermediateStorage::addComponentAccess(Id nodeId, int type) +void IntermediateStorage::addComponentAccess(Id edgeId, int type) { - std::unordered_map::const_iterator it = m_nodeIdsToMemberEdgeIds.find(nodeId); - if (it != m_nodeIdsToMemberEdgeIds.end()) - { - m_componentAccesses.push_back(StorageComponentAccess(it->second, type)); - } - else - { - LOG_ERROR_STREAM(<< "Cannot assign access" << type << " to node id " << nodeId << " because it's not a child node."); - } + m_componentAccesses.push_back(StorageComponentAccess(edgeId, type)); } -void IntermediateStorage::addCommentLocation(const ParseLocation& location) +void IntermediateStorage::addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) { - Id fileNodeId = addFile(location.filePath.str()); m_commentLocations.push_back(StorageCommentLocation( 0, fileNodeId, - location.startLineNumber, - location.startColumnNumber, - location.endLineNumber, - location.endColumnNumber + startLine, + startCol, + endLine, + endCol )); } -void IntermediateStorage::addError(const std::string& message, bool fatal, const ParseLocation& location) +void IntermediateStorage::addError(const std::string& message, bool fatal, const std::string& filePath, uint startLine, uint startCol) { m_errors.push_back(StorageError( message, fatal, - location.filePath.str(), - location.startLineNumber, - location.startColumnNumber + filePath, + startLine, + startCol )); } -void IntermediateStorage::transferToStorage(SqliteStorage& storage) -{ - storage.beginTransaction(); - - std::unordered_map clientIdToStorageId; - - for (std::unordered_map>::const_iterator it = m_fileIdsToData.begin(); it != m_fileIdsToData.end(); it++) - { - if (it->second->name.size() > 0) - { - Id fileNodeId = storage.getFileByPath(it->second->filePath).id; - if (fileNodeId == 0) - { - NameHierarchy nameHierarchy; - nameHierarchy.push(std::make_shared(it->second->name)); - - fileNodeId = storage.addFile( - NameHierarchy::serialize(nameHierarchy), - it->second->filePath, - it->second->modificationTime - ); - } - clientIdToStorageId[it->first] = fileNodeId; - } - } - - for (std::map>::const_iterator it = m_nodeIdsToData.begin(); it != m_nodeIdsToData.end(); it++) - { - StorageNode clientNode = *(it->second.get()); - StorageNode storageNode = storage.getNodeBySerializedName(clientNode.serializedName); - Id storageNodeId = storageNode.id; - if (storageNodeId) - { - if (clientNode.definitionType > 0) - { - if (storageNode.definitionType == 0) - { - storage.setNodeDefinitionType(clientNode.definitionType, storageNode.id); - if(storageNode.type < clientNode.type) - { - storage.setNodeType(clientNode.type, storageNode.id); - } - } - } - } - else - { - storageNodeId = storage.addNode(clientNode.type, clientNode.serializedName, clientNode.definitionType); - } - clientIdToStorageId[it->first] = storageNodeId; - } - - for (std::map>::const_iterator it = m_edgeIdsToData.begin(); it != m_edgeIdsToData.end(); it++) - { - std::unordered_map::const_iterator it2; - it2 = clientIdToStorageId.find(it->second->sourceNodeId); - if (it2 == clientIdToStorageId.end()) - { - continue; - } - Id storageSourceId = it2->second; - - it2 = clientIdToStorageId.find(it->second->targetNodeId); - if (it2 == clientIdToStorageId.end()) - { - continue; - } - Id storageTargetId = it2->second; - - Id edgeId = storage.getEdgeBySourceTargetType(storageSourceId, storageTargetId, it->second->type).id; - - if (!edgeId) - { - edgeId = storage.addEdge(it->second->type, storageSourceId, storageTargetId); - } - clientIdToStorageId[it->first] = edgeId; - } - - for (std::map>::const_iterator it = m_localSymbolIdsToData.begin(); it != m_localSymbolIdsToData.end(); it++) - { - StorageLocalSymbol clientLocalSymbol = *(it->second.get()); - StorageLocalSymbol storageLocalSymbol = storage.getLocalSymbolByName(clientLocalSymbol.name); - Id storageLocalSymbolId = storageLocalSymbol.id; - if (storageLocalSymbolId == 0) - { - storageLocalSymbolId = storage.addLocalSymbol(clientLocalSymbol.name); - } - clientIdToStorageId[it->first] = storageLocalSymbolId; - } - - for (size_t i = 0; i < m_sourceLocations.size(); i++) - { - StorageSourceLocation sourceLocation = m_sourceLocations[i]; - std::unordered_map::const_iterator it; - it = clientIdToStorageId.find(sourceLocation.elementId); - if (it == clientIdToStorageId.end()) - { - continue; - } - Id storageElementId = it->second; - - it = clientIdToStorageId.find(sourceLocation.fileNodeId); - if (it == clientIdToStorageId.end()) - { - continue; - } - Id storageFileNodeId = it->second; - - storage.addSourceLocation( - storageElementId, - storageFileNodeId, - sourceLocation.startLine, - sourceLocation.startCol, - sourceLocation.endLine, - sourceLocation.endCol, - sourceLocation.type - ); - } - - for (size_t i = 0; i < m_componentAccesses.size(); i++) - { - StorageComponentAccess componentAccess = m_componentAccesses[i]; - - std::unordered_map::const_iterator it; - it = clientIdToStorageId.find(componentAccess.memberEdgeId); - if (it == clientIdToStorageId.end()) - { - continue; - } - Id storageMemberEdgeId = it->second; - - storage.addComponentAccess(storageMemberEdgeId, componentAccess.type); - } - - for (size_t i = 0; i < m_commentLocations.size(); i++) - { - StorageCommentLocation commentLocation = m_commentLocations[i]; - - std::unordered_map::const_iterator it; - it = clientIdToStorageId.find(commentLocation.fileNodeId); - if (it == clientIdToStorageId.end()) - { - continue; - } - Id storageFileNodeId = it->second; - - storage.addCommentLocation( - storageFileNodeId, - commentLocation.startLine, - commentLocation.startCol, - commentLocation.endLine, - commentLocation.endCol - ); - } - - for (size_t i = 0; i < m_errors.size(); i++) - { - StorageError error = m_errors[i]; - - storage.addError( - error.message, - error.fatal, - error.filePath, - error.lineNumber, - error.columnNumber - ); - } - - storage.commitTransaction(); -} - void IntermediateStorage::forEachFile(std::function callback) const { for (std::unordered_map>::const_iterator it = m_fileIdsToData.begin(); it != m_fileIdsToData.end(); it++) @@ -373,6 +173,14 @@ void IntermediateStorage::forEachEdge(std::function callback) const +{ + for (std::map>::const_iterator it = m_localSymbolIdsToData.begin(); it != m_localSymbolIdsToData.end(); it++) + { + callback(it->first, *(it->second.get())); + } +} + void IntermediateStorage::forEachSourceLocation(std::function callback) const { for (std::vector::const_iterator it = m_sourceLocations.begin(); it != m_sourceLocations.end(); it++) diff --git a/src/lib/data/IntermediateStorage.h b/src/lib/data/IntermediateStorage.h index d2fb9567..4202d1ed 100644 --- a/src/lib/data/IntermediateStorage.h +++ b/src/lib/data/IntermediateStorage.h @@ -2,40 +2,35 @@ #define INTERMEDIATE_STORAGE_H #include -#include +#include #include -#include "utility/types.h" -#include "data/name/NameHierarchy.h" -#include "data/parser/ParseLocation.h" - -#include "data/SqliteStorage.h" #include "data/StorageTypes.h" +#include "data/Storage.h" -class IntermediateStorage +class IntermediateStorage: public Storage { public: IntermediateStorage(); - ~IntermediateStorage(); - Id addEdge(int type, Id sourceId, Id targetId); - Id addNode(int type, const NameHierarchy& nameHierarchy, int definitionType); - Id addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime); - Id addFile(const std::string& filePath); - Id addLocalSymbol(const std::string& name); - void addSourceLocation(Id elementId, const ParseLocation& location, int type); - void addComponentAccess(Id nodeId , int type); - void addCommentLocation(const ParseLocation& location); - void addError(const std::string& message, bool fatal, const ParseLocation& location); + virtual ~IntermediateStorage(); - void transferToStorage(SqliteStorage& storage); // TODO: remove this and use foreach-callbacks instead + virtual Id addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime); + virtual Id addNode(int type, const std::string& serializedName, int definitionType); + virtual Id addEdge(int type, Id sourceId, Id targetId); + virtual Id addLocalSymbol(const std::string& name); + virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); + virtual void addComponentAccess(Id edgeId , int type); + virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol); + virtual void addError(const std::string& message, bool fatal, const std::string& filePath, uint startLine, uint startCol); - void forEachFile(std::function callback) const; - void forEachNode(std::function callback) const; - void forEachEdge(std::function callback) const; - void forEachSourceLocation(std::function callback) const; - void forEachComponentAccess(std::function callback) const; - void forEachCommentLocation(std::function callback) const; - void forEachError(std::function callback) const; + virtual void forEachFile(std::function callback) const; + virtual void forEachNode(std::function callback) const; + virtual void forEachEdge(std::function callback) const; + virtual void forEachLocalSymbol(std::function callback) const; + virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachComponentAccess(std::function callback) const; + virtual void forEachCommentLocation(std::function callback) const; + virtual void forEachError(std::function callback) const; private: std::string serialize(const StorageEdge& edge); diff --git a/src/lib/data/PersistentStorage.cpp b/src/lib/data/PersistentStorage.cpp new file mode 100644 index 00000000..d75b0811 --- /dev/null +++ b/src/lib/data/PersistentStorage.cpp @@ -0,0 +1,1308 @@ +#include "data/PersistentStorage.h" + +#include +#include + +#include "utility/file/FileSystem.h" +#include "utility/logging/logging.h" +#include "utility/messaging/type/MessageClearErrorCount.h" +#include "utility/messaging/type/MessageShowErrors.h" +#include "utility/TimePoint.h" +#include "utility/utility.h" +#include "utility/utilityString.h" +#include "utility/Version.h" +#include "utility/Cache.h" +#include "utility/utilityString.h" + +#include "data/graph/token_component/TokenComponentAggregation.h" +#include "data/graph/token_component/TokenComponentSignature.h" +#include "data/graph/Graph.h" +#include "data/location/TokenLocation.h" +#include "data/location/TokenLocationFile.h" +#include "data/location/TokenLocationLine.h" +#include "data/parser/ParseLocation.h" +#include "data/type/DataType.h" +#include "settings/ApplicationSettings.h" + +PersistentStorage::PersistentStorage(const FilePath& dbPath) + : m_sqliteStorage(dbPath) +{ + m_commandIndex.addNode(0, NameHierarchy(SearchMatch::getCommandName(SearchMatch::COMMAND_ALL))); + m_commandIndex.addNode(0, NameHierarchy(SearchMatch::getCommandName(SearchMatch::COMMAND_ERROR))); + m_commandIndex.finishSetup(); +} + +PersistentStorage::~PersistentStorage() +{ +} + +Id PersistentStorage::addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime) +{ + Id fileId = m_sqliteStorage.getFileByPath(filePath).id; + if (fileId == 0) + { + NameHierarchy nameHierarchy; + nameHierarchy.push(std::make_shared(name)); + + fileId = m_sqliteStorage.addFile( + NameHierarchy::serialize(nameHierarchy), + filePath, + modificationTime + ); + } + return fileId; +} + +Id PersistentStorage::addNode(int type, const std::string& serializedName, int definitionType) +{ + const StorageNode storedNode = m_sqliteStorage.getNodeBySerializedName(serializedName); + + Id nodeId = storedNode.id; + + if (nodeId == 0) + { + nodeId = m_sqliteStorage.addNode(type, serializedName, definitionType); + } + else + { + if (storedNode.definitionType == 0 && definitionType > 0) + { + m_sqliteStorage.setNodeDefinitionType(definitionType, nodeId); + if(storedNode.type < type) + { + m_sqliteStorage.setNodeType(type, nodeId); + } + } + } + return nodeId; +} + +Id PersistentStorage::addEdge(int type, Id sourceId, Id targetId) +{ + Id edgeId = m_sqliteStorage.getEdgeBySourceTargetType(sourceId, targetId, type).id; + if (edgeId == 0) + { + edgeId = m_sqliteStorage.addEdge(type, sourceId, targetId); + } + return edgeId; +} + +Id PersistentStorage::addLocalSymbol(const std::string& name) +{ + Id localSymbolId = m_sqliteStorage.getLocalSymbolByName(name).id; + if (localSymbolId == 0) + { + localSymbolId = m_sqliteStorage.addLocalSymbol(name); + } + return localSymbolId; +} + +void PersistentStorage::addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) +{ + m_sqliteStorage.addSourceLocation( + elementId, + fileNodeId, + startLine, + startCol, + endLine, + endCol, + type + ); +} + +void PersistentStorage::addComponentAccess(Id edgeId , int type) +{ + m_sqliteStorage.addComponentAccess(edgeId, type); +} + +void PersistentStorage::addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) +{ + m_sqliteStorage.addCommentLocation( + fileNodeId, + startLine, + startCol, + endLine, + endCol + ); +} + +void PersistentStorage::addError(const std::string& message, bool fatal, const std::string& filePath, uint startLine, uint startCol) +{ + m_sqliteStorage.addError( + message, + fatal, + filePath, + startLine, + startCol + ); +} + +void PersistentStorage::forEachFile(std::function callback) const +{ + for (StorageFile& file: m_sqliteStorage.getAllFiles()) + { + callback(file.id, file); + } +} + +void PersistentStorage::forEachNode(std::function callback) const +{ + for (StorageNode& node: m_sqliteStorage.getAllNodes()) + { + callback(node.id, node); + } +} + +void PersistentStorage::forEachEdge(std::function callback) const +{ + for (StorageEdge& edge: m_sqliteStorage.getAllEdges()) + { + callback(edge.id, edge); + } +} + +void PersistentStorage::forEachLocalSymbol(std::function callback) const +{ + for (StorageLocalSymbol& localSymbol: m_sqliteStorage.getAllLocalSymbols()) + { + callback(localSymbol.id, localSymbol); + } +} + +void PersistentStorage::forEachSourceLocation(std::function callback) const +{ + for (StorageSourceLocation& sourceLocation: m_sqliteStorage.getAllSourceLocations()) + { + callback(sourceLocation); + } +} + +void PersistentStorage::forEachComponentAccess(std::function callback) const +{ + for (StorageComponentAccess& componentAccess: m_sqliteStorage.getAllComponentAccesses()) + { + callback(componentAccess); + } +} + +void PersistentStorage::forEachCommentLocation(std::function callback) const +{ + for (StorageCommentLocation& commentLocation: m_sqliteStorage.getAllCommentLocations()) + { + callback(commentLocation); + } +} + +void PersistentStorage::forEachError(std::function callback) const +{ + for (StorageError& error: m_sqliteStorage.getAllErrors()) + { + callback(error); + } +} + +void PersistentStorage::startInjection() +{ + m_preInjectionErrorCount = getErrorCount().total; + + m_sqliteStorage.beginTransaction(); +} + +void PersistentStorage::finishInjection() +{ + m_sqliteStorage.commitTransaction(); + + if (m_preInjectionErrorCount != -1 && + m_preInjectionErrorCount != getErrorCount().total) + { + MessageShowErrors msg(getErrorCount()); + msg.setSendAsTask(false); + msg.dispatch(); + } + m_preInjectionErrorCount = -1; +} + +FilePath PersistentStorage::getDbFilePath() const +{ + return m_sqliteStorage.getDbFilePath(); +} + +Version PersistentStorage::getVersion() const +{ + return m_sqliteStorage.getVersion(); +} + +void PersistentStorage::init() +{ + m_sqliteStorage.init(); +} + +void PersistentStorage::clear() +{ + m_sqliteStorage.clear(); + + clearCaches(); +} + +void PersistentStorage::clearCaches() +{ + m_elementIndex.clear(); + m_fileNodeIds.clear(); + m_hierarchyCache.clear(); +} + +std::set PersistentStorage::getDependingFilePaths(const std::set& filePaths) +{ + std::set dependingFilePaths; + for (const FilePath& filePath: filePaths) + { + std::set dependingFilePathsSubset = getDependingFilePaths(filePath); + dependingFilePaths.insert(dependingFilePathsSubset.begin(), dependingFilePathsSubset.end()); + } + return dependingFilePaths; +} + +std::set PersistentStorage::getDependingFilePaths(const FilePath& filePath) +{ + std::set dependingFilePaths; + + std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetType( + getFileNodeId(filePath), Edge::typeToInt(Edge::EDGE_INCLUDE) + ); + for (const StorageEdge& incomingEdge: incomingEdges) + { + FilePath dependingFilePath = getFileNodePath(incomingEdge.sourceNodeId); + dependingFilePaths.insert(dependingFilePath); + + std::set dependingFilePathsSubset = getDependingFilePaths(dependingFilePath); + dependingFilePaths.insert(dependingFilePathsSubset.begin(), dependingFilePathsSubset.end()); + } + + return dependingFilePaths; +} + +void PersistentStorage::clearFileElements(const std::vector& filePaths) +{ + std::vector fileNodeIds; + + for (const FilePath& path : filePaths) + { + fileNodeIds.push_back(getFileNodeId(path)); + } + + if (fileNodeIds.size()) + { + m_sqliteStorage.removeElementsWithLocationInFiles(fileNodeIds); + m_sqliteStorage.removeElements(fileNodeIds); + + m_sqliteStorage.removeErrorsInFiles(filePaths); + } +} + +void PersistentStorage::removeUnusedNames() // maybe rename this function. look for callers first. +{ +// m_sqliteStorage.removeUnusedNameHierarchyElements(); + + clearCaches(); +} + +std::vector PersistentStorage::getInfoOnAllFiles() const +{ + std::vector fileInfos; + + std::vector storageFiles = m_sqliteStorage.getAllFiles(); + for (size_t i = 0; i < storageFiles.size(); i++) + { + boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time; + if (storageFiles[i].modificationTime != "not-a-date-time") + { + modificationTime = boost::posix_time::time_from_string(storageFiles[i].modificationTime); + } + fileInfos.push_back(FileInfo( + FilePath(storageFiles[i].filePath), + modificationTime + )); + } + + return fileInfos; +} + +void PersistentStorage::logStats() const +{ + std::stringstream ss; + StorageStats stats = getStorageStats(); + + ss << "\nGraph:\n"; + ss << "\t" << stats.nodeCount << " Nodes\n"; + ss << "\t" << stats.edgeCount << " Edges\n"; + + ss << "\nCode:\n"; + ss << "\t" << stats.fileCount << " Files\n"; + ss << "\t" << stats.fileLOCCount << " Lines of Code\n"; + + ss << "\nErrors:\n"; + ss << "\t" << stats.errorCount.total << " Errors\n"; + ss << "\t" << stats.errorCount.fatal << " Fatal Errors\n"; + + LOG_WARNING(ss.str()); +} + +void PersistentStorage::startParsing() +{ + MessageClearErrorCount().dispatch(); + + m_sqliteStorage.setVersion(Version::getApplicationVersion()); +} + +void PersistentStorage::finishParsing() +{ + buildSearchIndex(); + buildHierarchyCache(); + optimizeFTSTable(); +} + +Id PersistentStorage::getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const +{ + return m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(nameHierarchy)).id; +} + +Id PersistentStorage::getIdForEdge( + Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy +) const +{ + Id sourceId = getIdForNodeWithNameHierarchy(fromNameHierarchy); + Id targetId = getIdForNodeWithNameHierarchy(toNameHierarchy); + return m_sqliteStorage.getEdgeBySourceTargetType(sourceId, targetId, type).id; +} + +NameHierarchy PersistentStorage::getNameHierarchyForNodeWithId(Id nodeId) const +{ + return NameHierarchy::deserialize(m_sqliteStorage.getNodeById(nodeId).serializedName); +} + +Node::NodeType PersistentStorage::getNodeTypeForNodeWithId(Id nodeId) const +{ + return Node::intToType(m_sqliteStorage.getNodeById(nodeId).type); +} + +std::shared_ptr PersistentStorage::getFullTextSearchLocations(const std::string& searchTerm) const +{ + std::shared_ptr collection = std::make_shared(); + + std::vector parseLocations = m_sqliteStorage.getFullTextSearch(searchTerm); + size_t i = 0; + for(ParseLocation location : parseLocations) + { + collection->addTokenLocation( + i, + 0, + location.filePath, + location.startLineNumber, + location.startColumnNumber, + location.endLineNumber, + location.endColumnNumber + )->setType(LOCATION_FULLTEXTSEARCH_MATCH); + i++; + } + + return collection; +} + +std::vector PersistentStorage::getAutocompletionMatches(const std::string& query) const +{ + std::vector commandResults = m_commandIndex.search(query, 0); + + const size_t maxResultCount = 100; + std::vector elementResults = m_elementIndex.search(query, maxResultCount); + + std::vector results; + utility::append(results, commandResults); + utility::append(results, elementResults); + + std::sort(results.begin(), results.end(), + [](const SearchResult& a, const SearchResult& b) + { + // should a be ranked higher than b? + if (a.score > b.score) + { + return true; + } + else if (a.score == b.score) + { + if (a.text.size() < b.text.size()) + { + return true; + } + else if (a.text.size() == b.text.size()) + { + for (size_t i = 0; i < a.text.size(); i++) + { + if (tolower(a.text[i]) != tolower(b.text[i])) + { + return tolower(a.text[i]) < tolower(b.text[i]); + } + else + { + if (a.text[i] < b.text[i]) + { + return true; + } + else if (a.text[i] > b.text[i]) + { + return false; + } + } + } + } + } + return false; + } + ); + + std::map storageNodesMap; + { + std::vector elementIds; + + for (const SearchResult& result : results) + { + elementIds.insert(elementIds.end(), result.elementIds.begin(), result.elementIds.end()); + } + + std::vector storageNodes = m_sqliteStorage.getNodesByIds(elementIds); + + for (StorageNode& node : storageNodes) + { + if (node.id > 0) + { + storageNodesMap.emplace(node.id, node); + } + } + } + + std::vector matches; + for (const SearchResult& result : results) + { + SearchMatch match; + + const StorageNode* firstNode = nullptr; + for (const Id& elementId : result.elementIds) + { + if (elementId != 0) + { + const StorageNode& node = storageNodesMap[elementId]; + match.nameHierarchies.push_back(NameHierarchy::deserialize(node.serializedName)); + + if (!firstNode) + { + firstNode = &node; + } + } + } + + match.text = result.text; + match.indices = result.indices; + + if (firstNode) + { + match.nodeType = Node::intToType(firstNode->type); + match.typeName = Node::getTypeString(match.nodeType); + + if (intToDefinitionType(firstNode->definitionType) == DEFINITION_NONE + && match.nodeType != Node::NODE_UNDEFINED) + { + match.typeName = "undefined " + match.typeName; + } + match.searchType = SearchMatch::SEARCH_TOKEN; + } + else + { + match.searchType = SearchMatch::SEARCH_COMMAND; + match.typeName = "command"; + } + + matches.push_back(match); + } + + return matches; +} + +std::vector PersistentStorage::getSearchMatchesForTokenIds(const std::vector& elementIds) const +{ + // todo: what if all these elements share the same node in the searchindex? + // In that case there should be only one search match. + std::vector matches; + + for (Id elementId : elementIds) + { + SearchMatch match; + + if (m_sqliteStorage.isFile(elementId)) + { + match.nodeType = Node::NODE_FILE; + } + else if (m_sqliteStorage.isNode(elementId)) + { + StorageNode node = m_sqliteStorage.getNodeById(elementId); + match.nodeType = Node::intToType(node.type); + } + else + { + continue; + } + + NameHierarchy nameHierarchy = NameHierarchy::deserialize(m_sqliteStorage.getNodeById(elementId).serializedName); + match.text = nameHierarchy.getQualifiedName(); + match.nameHierarchies.push_back(nameHierarchy.getQualifiedName()); + match.searchType = SearchMatch::SEARCH_TOKEN; + + matches.push_back(match); + } + + return matches; +} + +std::shared_ptr PersistentStorage::getGraphForAll() const +{ + std::shared_ptr graph = std::make_shared(); + + std::vector tokenIds; + for (StorageNode node: m_sqliteStorage.getAllNodes()) + { + if (intToDefinitionType(node.definitionType) == DEFINITION_EXPLICIT && + (!m_hierarchyCache.isChildOfVisibleNodeOrInvisible(node.id) || + Node::intToType(node.type) == Node::NODE_NAMESPACE)) + { + tokenIds.push_back(node.id); + } + } + + addNodesToGraph(tokenIds, graph.get()); + + return graph; +} + +std::shared_ptr PersistentStorage::getGraphForActiveTokenIds(const std::vector& tokenIds) const +{ + std::shared_ptr g = std::make_shared(); + Graph* graph = g.get(); + + std::vector ids(tokenIds); + bool isNamespace = false; + + std::vector nodeIds; + std::vector edgeIds; + bool addAggregations = false; + + //m_sqliteStorage.getFullTextSearch("const int"); + if (tokenIds.size() == 1) + { + const Id elementId = tokenIds[0]; + StorageNode node = m_sqliteStorage.getNodeById(elementId); + + if (node.id > 0) + { + if (Node::intToType(node.type) == Node::NODE_NAMESPACE) + { + ids.clear(); + m_hierarchyCache.addFirstChildIdsForNodeId(elementId, &ids); + + isNamespace = true; + } + else + { + nodeIds.push_back(elementId); + + std::vector edges = m_sqliteStorage.getEdgesBySourceOrTargetId(elementId); + for (const StorageEdge& edge : edges) + { + if (Edge::intToType(edge.type) != Edge::EDGE_MEMBER) + { + edgeIds.push_back(edge.id); + } + } + + addAggregations = true; + } + } + else if (m_sqliteStorage.isEdge(elementId)) + { + edgeIds.push_back(elementId); + } + } + + if (ids.size() >= 1 || isNamespace) + { + std::vector nodes = m_sqliteStorage.getNodesByIds(ids); + for (const StorageNode& node : nodes) + { + if (node.id > 0 && (!isNamespace || intToDefinitionType(node.definitionType) != DEFINITION_IMPLICIT)) + { + nodeIds.push_back(node.id); + } + } + + if (nodeIds.size() != ids.size()) + { + std::vector edges = m_sqliteStorage.getEdgesByIds(ids); + for (const StorageEdge& edge : edges) + { + if (edge.id > 0) + { + edgeIds.push_back(edge.id); + } + } + } + } + + if (isNamespace) + { + addNodesToGraph(nodeIds, graph); + } + else + { + addNodesWithChildrenAndEdgesToGraph(nodeIds, edgeIds, graph); + } + + if (addAggregations) + { + addAggregationEdgesToGraph(tokenIds[0], graph); + } + + addComponentAccessToGraph(graph); + + return g; +} + +// TODO: rename: getActiveElementIdsForId; TODO: make separate function for declarationId +std::vector PersistentStorage::getActiveTokenIdsForId(Id tokenId, Id* declarationId) const +{ + std::vector activeTokenIds; + + if (!(m_sqliteStorage.isEdge(tokenId) || m_sqliteStorage.isNode(tokenId))) + { + return activeTokenIds; + } + + activeTokenIds.push_back(tokenId); + + if (m_sqliteStorage.isNode(tokenId)) + { + *declarationId = tokenId; + + std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetId(tokenId); + for (size_t i = 0; i < incomingEdges.size(); i++) + { + activeTokenIds.push_back(incomingEdges[i].id); + } + } + + return activeTokenIds; +} + +std::vector PersistentStorage::getNodeIdsForLocationIds(const std::vector& locationIds) const +{ + std::set edgeIds; + std::set nodeIds; + std::set implicitNodeIds; + + for (Id locationId : locationIds) + { + Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId); + + StorageEdge edge = m_sqliteStorage.getEdgeById(elementId); + if (edge.id != 0) // here we test if location is an edge. + { + edgeIds.insert(edge.targetNodeId); + } + else if(m_sqliteStorage.isNode(elementId)) + { + StorageNode node = m_sqliteStorage.getNodeById(elementId); + if (node.id != 0) + { + if (intToDefinitionType(node.definitionType) == DEFINITION_IMPLICIT) + { + implicitNodeIds.insert(elementId); + } + else + { + nodeIds.insert(elementId); + } + } + } + } + + if (nodeIds.size() == 0) + { + nodeIds = implicitNodeIds; + } + + if (nodeIds.size()) + { + return utility::toVector(nodeIds); + } + + return utility::toVector(edgeIds); +} + +std::vector PersistentStorage::getLocalSymbolIdsForLocationIds(const std::vector& locationIds) const +{ + std::set localSymbolIds; + + for (Id locationId : locationIds) + { + Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId); + + if (m_sqliteStorage.getNodeById(elementId).id == 0 && m_sqliteStorage.getEdgeById(elementId).id == 0) + { + localSymbolIds.insert(elementId); + } + } + + return utility::toVector(localSymbolIds); +} + +std::vector PersistentStorage::getTokenIdsForMatches(const std::vector& matches) const +{ + std::set idSet; + for (const SearchMatch& match : matches) + { + for (size_t i = 0; i < match.nameHierarchies.size(); i++) + { + idSet.insert( + m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(match.nameHierarchies[i])).id + ); + } + } + + std::vector ids; + for (std::set::const_iterator it = idSet.begin(); it != idSet.end(); it++) + { + if (*it != 0) + { + ids.push_back(*it); + } + } + + return ids; +} + +Id PersistentStorage::getTokenIdForFileNode(const FilePath& filePath) const +{ + return m_sqliteStorage.getFileByPath(filePath.str()).id; +} + +std::vector PersistentStorage::getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const +{ + std::vector edgeIds; + + std::vector aggregationEndpointsA = getAllChildNodeIds(sourceId); + std::set aggregationEndpointsB; + aggregationEndpointsB.insert(targetId); + for (const Id targetChildId: getAllChildNodeIds(targetId)) + { + aggregationEndpointsB.insert(targetChildId); + } + + for (size_t i = 0; i < aggregationEndpointsA.size(); i++) + { + std::vector outgoingEdges = m_sqliteStorage.getEdgesBySourceId(aggregationEndpointsA[i]); + for (size_t j = 0; j < outgoingEdges.size(); j++) + { + if (aggregationEndpointsB.find(outgoingEdges[j].targetNodeId) != aggregationEndpointsB.end()) + { + edgeIds.push_back(outgoingEdges[j].id); + } + } + + std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetId(aggregationEndpointsA[i]); + for (size_t j = 0; j < incomingEdges.size(); j++) + { + if (aggregationEndpointsB.find(incomingEdges[j].sourceNodeId) != aggregationEndpointsB.end()) + { + edgeIds.push_back(incomingEdges[j].id); + } + } + } + + return edgeIds; +} + +std::shared_ptr PersistentStorage::getTokenLocationsForTokenIds(const std::vector& tokenIds) const +{ + std::shared_ptr collection = std::make_shared(); + + std::vector fileIds; + std::vector nonFileIds; + std::vector allFileIds = m_sqliteStorage.getAllFileIds(); + for (size_t i = 0; i < tokenIds.size(); i++) + { + if (std::find(allFileIds.begin(), allFileIds.end(),tokenIds[i]) != allFileIds.end()) + { + fileIds.push_back(tokenIds[i]); + } + else + { + nonFileIds.push_back(tokenIds[i]); + } + } + + for (Id fileId: fileIds) + { + StorageFile storageFile = m_sqliteStorage.getFileById(fileId); + collection->addTokenLocationFileAsPlainCopy( + m_sqliteStorage.getTokenLocationsForFile(storageFile.filePath).get() + ); + } + + Cache filePathCache( + [this](Id id) -> std::string + { + return m_sqliteStorage.getFileById(id).filePath; + } + ); + + std::vector locations = m_sqliteStorage.getTokenLocationsForElementIds(nonFileIds); + for (size_t i = 0; i < locations.size(); i++) + { + const StorageSourceLocation& location = locations[i]; + std::string filePath = filePathCache.getValue(location.fileNodeId); + + TokenLocation* loc = collection->addTokenLocation( + location.id, + location.elementId, + filePath, + location.startLine, + location.startCol, + location.endLine, + location.endCol + ); + + if (loc) + { + loc->setType(intToLocationType(location.type)); + } + } + + return collection; +} + +std::shared_ptr PersistentStorage::getTokenLocationsForLocationIds( + const std::vector& locationIds +) const +{ + std::shared_ptr collection = std::make_shared(); + + for (size_t i = 0; i < locationIds.size(); i++) + { + StorageSourceLocation location = m_sqliteStorage.getSourceLocationById(locationIds[i]); + collection->addTokenLocation( + location.id, + location.elementId, + m_sqliteStorage.getFileById(location.fileNodeId).filePath, // TODO: optimize: only once per file! + location.startLine, + location.startCol, + location.endLine, + location.endCol + )->setType(intToLocationType(location.type)); + } + + return collection; +} + +std::shared_ptr PersistentStorage::getTokenLocationsForFile(const std::string& filePath) const +{ + std::shared_ptr locationFile = m_sqliteStorage.getTokenLocationsForFile(filePath); + locationFile->isWholeCopy = true; + return locationFile; +} + +std::shared_ptr PersistentStorage::getTokenLocationsForLinesInFile( + const std::string& filePath, uint firstLineNumber, uint lastLineNumber +) const +{ + return m_sqliteStorage.getTokenLocationsForFile(filePath)->getFilteredByLines(firstLineNumber, lastLineNumber); +} + +TokenLocationCollection PersistentStorage::getErrorTokenLocations(std::vector* errors) const +{ + TokenLocationCollection errorCollection; + + std::vector storageErrors = m_sqliteStorage.getAllErrors(); + for (size_t i = 0; i < storageErrors.size(); i++) + { + const StorageError& error = storageErrors[i]; + errorCollection.addTokenLocation( + i, i, error.filePath, error.lineNumber, error.columnNumber, error.lineNumber, error.columnNumber); + errors->push_back(ErrorInfo(error.message, error.filePath, i, error.fatal)); + } + + return errorCollection; +} + +std::shared_ptr PersistentStorage::getCommentLocationsInFile(const FilePath& filePath) const +{ + std::shared_ptr file = std::make_shared(filePath); + + std::vector storageLocations = m_sqliteStorage.getCommentLocationsInFile(filePath); + for (size_t i = 0; i < storageLocations.size(); i++) + { + file->addTokenLocation( + storageLocations[i].id, + 0, // comment token location has no element. + storageLocations[i].startLine, + storageLocations[i].startCol, + storageLocations[i].endLine, + storageLocations[i].endCol + ); + } + + return file; +} + +std::shared_ptr PersistentStorage::getFileContent(const FilePath& filePath) const +{ + return m_sqliteStorage.getFileContentByPath(filePath.str()); +} + +FileInfo PersistentStorage::getFileInfoForFilePath(const FilePath& filePath) const +{ + return FileInfo(filePath, m_sqliteStorage.getFileByPath(filePath).modificationTime); +} + +std::vector PersistentStorage::getFileInfosForFilePaths(const std::vector& filePaths) const +{ + std::vector fileInfos; + + std::vector storageFiles = m_sqliteStorage.getFilesByPaths(filePaths); + for (const StorageFile& file : storageFiles) + { + fileInfos.push_back(FileInfo(FilePath(file.filePath), file.modificationTime)); + } + + return fileInfos; +} + +ErrorCountInfo PersistentStorage::getErrorCount() const +{ + return ErrorCountInfo(m_sqliteStorage.getAllErrors().size(), m_sqliteStorage.getFatalErrors().size()); +} + +StorageStats PersistentStorage::getStorageStats() const +{ + StorageStats stats; + + stats.nodeCount = m_sqliteStorage.getNodeCount(); + stats.edgeCount = m_sqliteStorage.getEdgeCount(); + + stats.fileCount = m_sqliteStorage.getFileCount(); + stats.fileLOCCount = m_sqliteStorage.getFileLOCCount(); + + stats.errorCount = getErrorCount(); + + return stats; +} + +Id PersistentStorage::getFileNodeId(const FilePath& filePath) const +{ + std::map::const_iterator it = m_fileNodeIds.find(filePath); + + if (it != m_fileNodeIds.end()) + { + return it->second; + } + + if (filePath.empty()) + { + LOG_ERROR("No file path set"); + return 0; + } + + StorageFile storageFile = m_sqliteStorage.getFileByPath(filePath.str()); + + if (storageFile.id == 0) + { + return 0; + } + + m_fileNodeIds.emplace(filePath, storageFile.id); + + return storageFile.id; +} + +FilePath PersistentStorage::getFileNodePath(Id fileId) const +{ + for (const std::pair& p : m_fileNodeIds) + { + if (p.second == fileId) + { + return p.first; + } + } + + return m_sqliteStorage.getFileById(fileId).filePath; +} + +Id PersistentStorage::getLastVisibleParentNodeId(const Id nodeId) const +{ + return m_hierarchyCache.getLastVisibleParentNodeId(nodeId); +} + +std::vector PersistentStorage::getAllChildNodeIds(const Id nodeId) const +{ + std::vector childNodeIds; + std::vector edgeIds; + + m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &childNodeIds, &edgeIds); + + return childNodeIds; +} + +void PersistentStorage::addNodesToGraph(const std::vector& nodeIds, Graph* graph) const +{ + if (nodeIds.size() == 0) + { + return; + } + + std::vector storageNodes = m_sqliteStorage.getNodesByIds(nodeIds); + + for (const StorageNode& storageNode : storageNodes) + { + NameHierarchy nameHierarchy = NameHierarchy::deserialize(storageNode.serializedName); + + Node::NodeType type = Node::intToType(storageNode.type); + DefinitionType defType = intToDefinitionType(storageNode.definitionType); + Node* node = graph->createNode( + storageNode.id, + type, + nameHierarchy, + defType != DEFINITION_NONE + ); + + if (defType == DEFINITION_IMPLICIT) + { + node->setImplicit(true); + } + else if (defType == DEFINITION_EXPLICIT) + { + node->setExplicit(true); + } + + if (type == Node::NODE_FUNCTION || type == Node::NODE_METHOD) + { + std::string signatureString = nameHierarchy.getRawNameWithSignature(); + if (signatureString.size() > 0) // this should always be the case since functions and methods must have sigs. + { + node->addComponentSignature( + std::make_shared(signatureString) + ); + } + } + } +} + +void PersistentStorage::addEdgesToGraph(const std::vector& edgeIds, Graph* graph) const +{ + if (edgeIds.size() == 0) + { + return; + } + + std::vector storageEdges = m_sqliteStorage.getEdgesByIds(edgeIds); + for (const StorageEdge& storageEdge : storageEdges) + { + Node* sourceNode = graph->getNodeById(storageEdge.sourceNodeId); + Node* targetNode = graph->getNodeById(storageEdge.targetNodeId); + + if (sourceNode && targetNode) + { + graph->createEdge(storageEdge.id, Edge::intToType(storageEdge.type), sourceNode, targetNode); + } + else + { + LOG_ERROR("Can't add edge because nodes are not present"); + } + } +} + +void PersistentStorage::addNodesWithChildrenAndEdgesToGraph( + const std::vector& nodeIds, const std::vector& edgeIds, Graph* graph +) const +{ + std::set parentNodeIds; + + for (Id nodeId : nodeIds) + { + parentNodeIds.insert(getLastVisibleParentNodeId(nodeId)); + } + + if (edgeIds.size() > 0) + { + std::vector storageEdges = m_sqliteStorage.getEdgesByIds(edgeIds); + for (const StorageEdge& storageEdge : storageEdges) + { + parentNodeIds.insert(getLastVisibleParentNodeId(storageEdge.sourceNodeId)); + parentNodeIds.insert(getLastVisibleParentNodeId(storageEdge.targetNodeId)); + } + } + + std::vector allNodeIds; + std::vector allEdgeIds = edgeIds; + + for (Id parentNodeId : parentNodeIds) + { + allNodeIds.push_back(parentNodeId); + m_hierarchyCache.addAllChildIdsForNodeId(parentNodeId, &allNodeIds, &allEdgeIds); + } + + addNodesToGraph(allNodeIds, graph); + addEdgesToGraph(allEdgeIds, graph); +} + +void PersistentStorage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const +{ + struct EdgeInfo + { + Id edgeId; + bool forward; + }; + + // build aggregation edges: + // get all children of the active node + std::vector childNodeIds = getAllChildNodeIds(nodeId); + if (childNodeIds.size() == 0) + { + return; + } + + // get all edges of the children + std::map> connectedNodeIds; + + std::vector outgoingEdges = m_sqliteStorage.getEdgesBySourceIds(childNodeIds); + for (size_t j = 0; j < outgoingEdges.size(); j++) + { + EdgeInfo edgeInfo; + edgeInfo.edgeId = outgoingEdges[j].id; + edgeInfo.forward = true; + connectedNodeIds[outgoingEdges[j].targetNodeId].push_back(edgeInfo); + } + + std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetIds(childNodeIds); + for (size_t j = 0; j < incomingEdges.size(); j++) + { + EdgeInfo edgeInfo; + edgeInfo.edgeId = incomingEdges[j].id; + edgeInfo.forward = false; + connectedNodeIds[incomingEdges[j].sourceNodeId].push_back(edgeInfo); + } + + // get all parent nodes of all connected nodes (up to last level except namespace/undefined) + Id nodeParentNodeId = getLastVisibleParentNodeId(nodeId); + + std::map> connectedParentNodeIds; + for (const std::pair>& p : connectedNodeIds) + { + Id parentNodeId = getLastVisibleParentNodeId(p.first); + + if (parentNodeId != nodeParentNodeId) + { + utility::append(connectedParentNodeIds[parentNodeId], p.second); + } + } + + // add hierarchies of these parents + std::vector nodeIdsToAdd; + for (const std::pair> p : connectedParentNodeIds) + { + const Id aggregationTargetNodeId = p.first; + if (!graph->getNodeById(aggregationTargetNodeId)) + { + nodeIdsToAdd.push_back(aggregationTargetNodeId); + } + } + addNodesWithChildrenAndEdgesToGraph(nodeIdsToAdd, std::vector(), graph); + + // create aggregation edges between parents and active node + Node* sourceNode = graph->getNodeById(nodeId); + for (const std::pair> p : connectedParentNodeIds) + { + const Id aggregationTargetNodeId = p.first; + + Node* targetNode = graph->getNodeById(aggregationTargetNodeId); + if (!targetNode) + { + LOG_ERROR("Aggregation target node not present."); + } + + std::shared_ptr componentAggregation = std::make_shared(); + for (const EdgeInfo& edgeInfo: p.second) + { + componentAggregation->addAggregationId(edgeInfo.edgeId, edgeInfo.forward); + } + + Edge* edge = graph->createEdge( + *componentAggregation->getAggregationIds().begin(), + Edge::EDGE_AGGREGATION, + sourceNode, + targetNode + ); + + edge->addComponentAggregation(componentAggregation); + } +} + +void PersistentStorage::addComponentAccessToGraph(Graph* graph) const +{ + std::vector memberEdgeIds; + + graph->forEachEdge( + [&memberEdgeIds](Edge* edge) + { + if (!edge->isType(Edge::EDGE_MEMBER)) + { + return; + } + + memberEdgeIds.push_back(edge->getId()); + } + ); + + std::vector accesses = m_sqliteStorage.getComponentAccessByMemberEdgeIds(memberEdgeIds); + for (const StorageComponentAccess& access : accesses) + { + if (access.memberEdgeId && access.type) + { + graph->getEdgeById(access.memberEdgeId)->addComponentAccess( + std::make_shared(TokenComponentAccess::intToType(access.type))); + } + } +} + +void PersistentStorage::buildSearchIndex() +{ + for (StorageNode node: m_sqliteStorage.getAllNodes()) + { + m_elementIndex.addNode(node.id, NameHierarchy::deserialize(node.serializedName)); + } + m_elementIndex.finishSetup(); +} + +void PersistentStorage::buildHierarchyCache() +{ + std::vector memberEdges = m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER)); + + Cache nodeTypeCache([this](Id id){ + return Node::intToType(m_sqliteStorage.getNodeById(id).type); + }); + + for (const StorageEdge& edge : memberEdges) + { + bool isVisible = !(nodeTypeCache.getValue(edge.sourceNodeId) & Node::NODE_NOT_VISIBLE); + m_hierarchyCache.createConnection(edge.id, edge.sourceNodeId, edge.targetNodeId, isVisible); + } +} + +void PersistentStorage::optimizeFTSTable() +{ + m_sqliteStorage.optimizeFTSTable(); +} diff --git a/src/lib/data/PersistentStorage.h b/src/lib/data/PersistentStorage.h new file mode 100644 index 00000000..6cd58fae --- /dev/null +++ b/src/lib/data/PersistentStorage.h @@ -0,0 +1,153 @@ +#ifndef PERSISTENT_STORAGE_H +#define PERSISTENT_STORAGE_H + +#include +#include + +#include "utility/file/FilePath.h" + +#include "data/access/StorageAccess.h" +#include "data/graph/token_component/TokenComponentAccess.h" +#include "data/location/TokenLocationCollection.h" +#include "data/parser/ParserClient.h" +#include "data/parser/ParseLocation.h" +#include "data/search/SearchIndex.h" +#include "data/HierarchyCache.h" +#include "data/SqliteStorage.h" +#include "data/Storage.h" + +#include "data/parser/ParserClientImpl.h" + +class PersistentStorage + : public Storage + , public StorageAccess +{ +public: + PersistentStorage(const FilePath& dbPath); + virtual ~PersistentStorage(); + + virtual Id addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime); + virtual Id addNode(int type, const std::string& serializedName, int definitionType); + virtual Id addEdge(int type, Id sourceId, Id targetId); + virtual Id addLocalSymbol(const std::string& name); + virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); + virtual void addComponentAccess(Id edgeId , int type); + virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol); + virtual void addError(const std::string& message, bool fatal, const std::string& filePath, uint startLine, uint startCol); + + virtual void forEachFile(std::function callback) const; + virtual void forEachNode(std::function callback) const; + virtual void forEachEdge(std::function callback) const; + virtual void forEachLocalSymbol(std::function callback) const; + virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachComponentAccess(std::function callback) const; + virtual void forEachCommentLocation(std::function callback) const; + virtual void forEachError(std::function callback) const; + + virtual void startInjection(); + virtual void finishInjection(); + + + + + FilePath getDbFilePath() const; + Version getVersion() const; + + void init(); + void clear(); + void clearCaches(); + + std::set getDependingFilePaths(const std::set& filePaths); + std::set getDependingFilePaths(const FilePath& filePath); + + void clearFileElements(const std::vector& filePaths); + void removeUnusedNames(); + + std::vector getInfoOnAllFiles() const; + + void logStats() const; + + void startParsing(); + void finishParsing(); + + // StorageAccess implementation + virtual Id getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const; + virtual Id getIdForEdge( + Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const; + + virtual NameHierarchy getNameHierarchyForNodeWithId(Id nodeId) const; + virtual Node::NodeType getNodeTypeForNodeWithId(Id nodeId) const; + + virtual std::shared_ptr getFullTextSearchLocations(const std::string& searchTerm) const; + virtual std::vector getAutocompletionMatches(const std::string& query) const; + virtual std::vector getSearchMatchesForTokenIds(const std::vector& elementIds) const; + + virtual std::shared_ptr getGraphForAll() const; + virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const; + + virtual std::vector getActiveTokenIdsForId(Id tokenId, Id* declarationId) const; + + virtual std::vector getNodeIdsForLocationIds(const std::vector& locationIds) const; + virtual std::vector getLocalSymbolIdsForLocationIds(const std::vector& locationIds) const; + + virtual std::vector getTokenIdsForMatches(const std::vector& matches) const; + virtual Id getTokenIdForFileNode(const FilePath& filePath) const; + virtual std::vector getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const; + + virtual std::shared_ptr getTokenLocationsForTokenIds( + const std::vector& tokenIds + ) const; + virtual std::shared_ptr getTokenLocationsForLocationIds( + const std::vector& locationIds + ) const; + virtual std::shared_ptr getTokenLocationsForFile(const std::string& filePath) const; + virtual std::shared_ptr getTokenLocationsForLinesInFile( + const std::string& filePath, uint firstLineNumber, uint lastLineNumber + ) const; + + virtual TokenLocationCollection getErrorTokenLocations(std::vector* errors) const; + virtual std::shared_ptr getCommentLocationsInFile(const FilePath& filePath) const; + + virtual std::shared_ptr getFileContent(const FilePath& filePath) const; + + virtual FileInfo getFileInfoForFilePath(const FilePath& filePath) const; + virtual std::vector getFileInfosForFilePaths(const std::vector& filePaths) const; + + virtual ErrorCountInfo getErrorCount() const; + virtual StorageStats getStorageStats() const; + +private: + Id getFileNodeId(const FilePath& filePath) const; + FilePath getFileNodePath(Id fileId) const; + + Id getLastVisibleParentNodeId(const Id nodeId) const; + std::vector getAllChildNodeIds(const Id nodeId) const; + + void addNodesToGraph(const std::vector& nodeIds, Graph* graph) const; + void addEdgesToGraph(const std::vector& edgeIds, Graph* graph) const; + void addNodesWithChildrenAndEdgesToGraph( + const std::vector& nodeIds, + const std::vector& edgeIds, Graph* graph + ) const; + + void addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const; + void addComponentAccessToGraph(Graph* graph) const; + + void buildSearchIndex(); + void buildHierarchyCache(); + void optimizeFTSTable(); + + void log(std::string type, std::string str, const ParseLocation& location) const; + + int m_preInjectionErrorCount; + + SearchIndex m_commandIndex; + SearchIndex m_elementIndex; + + SqliteStorage m_sqliteStorage; + + mutable std::map m_fileNodeIds; + HierarchyCache m_hierarchyCache; +}; + +#endif // PERSISTENT_STORAGE_H diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index ea8190b4..f681cdd6 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -217,9 +217,9 @@ Id SqliteStorage::addError(const std::string& message, bool fatal, const std::st void SqliteStorage::removeElement(Id id) { - m_database.execDML(( - "DELETE FROM element WHERE id == " + std::to_string(id) + ";" - ).c_str()); + std::vector ids; + ids.push_back(id); + removeElements(ids); } void SqliteStorage::removeElements(const std::vector& ids) @@ -264,22 +264,6 @@ void SqliteStorage::removeErrorsInFiles(const std::vector& filePaths) ).c_str()); } -StorageNode SqliteStorage::getFirstNode() const -{ - std::vector nodes = getAllNodes("LIMIT 1"); - if (nodes.size()) - { - return nodes[0]; - } - - return StorageNode(); -} - -std::vector SqliteStorage::getAllNodes() const -{ - return getAllNodes(""); -} - bool SqliteStorage::isEdge(Id elementId) const { int count = m_database.execScalar(("SELECT count(*) FROM edge WHERE id = " + std::to_string(elementId) + ";").c_str()); @@ -321,101 +305,56 @@ StorageEdge SqliteStorage::getEdgeById(Id edgeId) const StorageEdge SqliteStorage::getEdgeBySourceTargetType(Id sourceId, Id targetId, int type) const { - StorageEdge edge( - getFirstResult( - "SELECT id FROM edge WHERE " - "source_node_id == " + std::to_string(sourceId) + " AND " - "target_node_id == " + std::to_string(targetId) + " AND " - "type == " + std::to_string(type) + ";" - ), - type, sourceId, targetId + return getFirst("WHERE " + "source_node_id == " + std::to_string(sourceId) + " AND " + "target_node_id == " + std::to_string(targetId) + " AND " + "type == " + std::to_string(type) ); - return edge; } std::vector SqliteStorage::getEdgesByIds(const std::vector& edgeIds) const { - return getAllEdges("WHERE id IN (" + utility::join(utility::toStrings(edgeIds), ',') + ")"); + return getAll("WHERE id IN (" + utility::join(utility::toStrings(edgeIds), ',') + ")"); } std::vector SqliteStorage::getEdgesBySourceId(Id sourceId) const { - return getAllEdges("WHERE source_node_id == " + std::to_string(sourceId)); + return getAll("WHERE source_node_id == " + std::to_string(sourceId)); } std::vector SqliteStorage::getEdgesBySourceIds(const std::vector& sourceIds) const { - return getAllEdges("WHERE source_node_id IN (" + utility::join(utility::toStrings(sourceIds), ',') + ")"); + return getAll("WHERE source_node_id IN (" + utility::join(utility::toStrings(sourceIds), ',') + ")"); } std::vector SqliteStorage::getEdgesByTargetId(Id targetId) const { - return getAllEdges("WHERE target_node_id == " + std::to_string(targetId)); + return getAll("WHERE target_node_id == " + std::to_string(targetId)); } std::vector SqliteStorage::getEdgesByTargetIds(const std::vector& targetIds) const { - return getAllEdges("WHERE target_node_id IN (" + utility::join(utility::toStrings(targetIds), ',') + ")"); + return getAll("WHERE target_node_id IN (" + utility::join(utility::toStrings(targetIds), ',') + ")"); } std::vector SqliteStorage::getEdgesBySourceOrTargetId(Id id) const { - return getAllEdges("WHERE source_node_id == " + std::to_string(id) + " OR target_node_id == " + std::to_string(id)); + return getAll("WHERE source_node_id == " + std::to_string(id) + " OR target_node_id == " + std::to_string(id)); } std::vector SqliteStorage::getEdgesByType(int type) const { - return getAllEdges("WHERE type == " + std::to_string(type)); + return getAll("WHERE type == " + std::to_string(type)); } std::vector SqliteStorage::getEdgesBySourceType(Id sourceId, int type) const { - std::vector edges; - - CppSQLite3Query q = m_database.execQuery(( - "SELECT id, target_node_id FROM edge WHERE " - "source_node_id == " + std::to_string(sourceId) + " AND " - "type == " + std::to_string(type) + ";" - ).c_str()); - - while (!q.eof()) - { - const Id id = q.getIntField(0, 0); - const Id targetId = q.getIntField(1, 0); - - if (id != 0 && targetId != 0) - { - edges.push_back(StorageEdge(id, type, sourceId, targetId)); - } - - q.nextRow(); - } - return edges; + return getAll("WHERE source_node_id == " + std::to_string(sourceId) + " AND type == " + std::to_string(type)); } std::vector SqliteStorage::getEdgesByTargetType(Id targetId, int type) const { - std::vector edges; - - CppSQLite3Query q = m_database.execQuery(( - "SELECT id, source_node_id FROM edge WHERE " - "target_node_id == " + std::to_string(targetId) + " AND " - "type == " + std::to_string(type) + ";" - ).c_str()); - - while (!q.eof()) - { - const Id id = q.getIntField(0, 0); - const Id sourceId = q.getIntField(1, 0); - - if (id != 0 && sourceId != 0) - { - edges.push_back(StorageEdge(id, type, sourceId, targetId)); - } - - q.nextRow(); - } - return edges; + return getAll("WHERE target_node_id == " + std::to_string(targetId) + " AND type == " + std::to_string(type)); } void SqliteStorage::optimizeFTSTable() const @@ -508,59 +447,39 @@ StorageNode SqliteStorage::getNodeById(Id id) const { if (id != 0) { - return getFirstNode("WHERE id == " + std::to_string(id)); + return getFirst("WHERE id == " + std::to_string(id)); } return StorageNode(); } StorageNode SqliteStorage::getNodeBySerializedName(const std::string& serializedName) const { - return getFirstNode("WHERE serialized_name == '" + serializedName + "'"); + return getFirst("WHERE serialized_name == '" + serializedName + "'"); } std::vector SqliteStorage::getNodesByIds(const std::vector& nodeIds) const { - return getAllNodes("WHERE id IN (" + utility::join(utility::toStrings(nodeIds), ',') + ")"); + return getAll("WHERE id IN (" + utility::join(utility::toStrings(nodeIds), ',') + ")"); } StorageLocalSymbol SqliteStorage::getLocalSymbolByName(const std::string& name) const { - StorageLocalSymbol localSymbol( - getFirstResult( - "SELECT id FROM local_symbol WHERE " - "name == '" + name + "';" - ), - name - ); - return localSymbol; + return getFirst("WHERE name == '" + name + "'"); } StorageFile SqliteStorage::getFileById(const Id id) const { - return getFirstFile( - "SELECT node.id, node.serialized_name, file.path, file.modification_time FROM node INNER JOIN file ON node.id = file.id " - "WHERE node.id == " + std::to_string(id) + ";" - ); + return getFirst("WHERE node.id == " + std::to_string(id)); } StorageFile SqliteStorage::getFileByPath(const FilePath& filePath) const { - StorageFile storageFile = getFirstFile( - "SELECT node.id, node.serialized_name, file.path, file.modification_time FROM node INNER JOIN file ON node.id = file.id " - "WHERE file.path == '" + filePath.str() + "';" - ); - - return storageFile; + return getFirst("WHERE file.path == '" + filePath.str() + "'"); } std::vector SqliteStorage::getFilesByPaths(const std::vector& filePaths) const { - return getAllFiles("WHERE file.path IN ('" + utility::join(utility::toStrings(filePaths), "', '") + "')"); -} - -std::vector SqliteStorage::getAllFiles() const -{ - return getAllFiles(""); + return getAll("WHERE file.path IN ('" + utility::join(utility::toStrings(filePaths), "', '") + "')"); } std::vector SqliteStorage::getAllFileIds() const @@ -610,8 +529,8 @@ void SqliteStorage::setNodeDefinitionType(int definitionType, Id nodeId) StorageSourceLocation SqliteStorage::getSourceLocationById(const Id id) const { - return getFirstSourceLocation( - "SELECT id, element_id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location WHERE id == " + std::to_string(id) + ";" + return getFirst( + "WHERE id == " + std::to_string(id) + ";" ); } @@ -689,9 +608,14 @@ std::vector SqliteStorage::getTokenLocationsForElementIds Id SqliteStorage::getElementIdByLocationId(Id locationId) const { - return getFirstResult( - "SELECT element_id FROM source_location WHERE id == " + std::to_string(locationId) + ";" - ); + CppSQLite3Query q = m_database.execQuery(( + "SELECT element_id FROM source_location WHERE id == " + std::to_string(locationId) + " LIMIT 1;" + ).c_str()); + if (!q.eof()) + { + return q.getIntField(0, 0); + } + return 0; } StorageComponentAccess SqliteStorage::getComponentAccessByMemberEdgeId(Id memberEdgeId) const @@ -733,77 +657,52 @@ std::vector SqliteStorage::getComponentAccessByMemberEdg std::vector SqliteStorage::getCommentLocationsInFile(const FilePath& filePath) const { Id fileNodeId = getFileByPath(filePath.str()).id; - CppSQLite3Query q = m_database.execQuery(( - "SELECT id, file_node_id, start_line, start_column, end_line, end_column FROM comment_location " - "WHERE file_node_id == " + std::to_string(fileNodeId) + ";" - ).c_str()); - - std::vector commentLocations; - while (!q.eof()) - { - const Id id = q.getIntField(0, 0); - const Id fileNodeId = q.getIntField(1, 0); - const int startLineNumber = q.getIntField(2, -1); - const int startColNumber = q.getIntField(3, -1); - const int endLineNumber = q.getIntField(4, -1); - const int endColNumber = q.getIntField(5, -1); - - if (id != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1) - { - commentLocations.push_back(StorageCommentLocation( - id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber - )); - } - q.nextRow(); - } - - return commentLocations; -} - -std::vector SqliteStorage::getAllErrors() const -{ - CppSQLite3Query q = m_database.execQuery( - "SELECT message, fatal, file_path, line_number, column_number FROM error;" - ); - - std::vector errors; - while (!q.eof()) - { - const std::string message = q.getStringField(0, ""); - const bool fatal = q.getIntField(1, 0); - const std::string filePath = q.getStringField(2, ""); - const uint lineNumber = q.getIntField(3, 0); - const uint columnNumber = q.getIntField(4, 0); - - errors.push_back(StorageError(message, fatal, filePath, lineNumber, columnNumber)); - - q.nextRow(); - } - - return errors; + return getAll("WHERE file_node_id == " + std::to_string(fileNodeId)); } std::vector SqliteStorage::getFatalErrors() const { - CppSQLite3Query q = m_database.execQuery( - "SELECT message, fatal, file_path, line_number, column_number FROM error WHERE fatal == 1;" - ); + return getAll("WHERE fatal == 1"); +} - std::vector errors; - while (!q.eof()) - { - const std::string message = q.getStringField(0, ""); - const bool fatal = q.getIntField(1, 0); - const std::string filePath = q.getStringField(2, ""); - const uint lineNumber = q.getIntField(3, 0); - const uint columnNumber = q.getIntField(4, 0); +std::vector SqliteStorage::getAllFiles() const +{ + return getAll(""); +} - errors.push_back(StorageError(message, fatal, filePath, lineNumber, columnNumber)); +std::vector SqliteStorage::getAllNodes() const +{ + return getAll(""); +} - q.nextRow(); - } +std::vector SqliteStorage::getAllEdges() const +{ + return getAll(""); +} - return errors; +std::vector SqliteStorage::getAllLocalSymbols() const +{ + return getAll(""); +} + +std::vector SqliteStorage::getAllSourceLocations() const +{ + return getAll(""); +} + +std::vector SqliteStorage::getAllComponentAccesses() const +{ + return getAll(""); +} + +std::vector SqliteStorage::getAllCommentLocations() const +{ + return getAll(""); +} + +std::vector SqliteStorage::getAllErrors() const +{ + return getAll(""); } int SqliteStorage::getNodeCount() const @@ -895,7 +794,6 @@ void SqliteStorage::setupTables() { m_database.execDML( "CREATE VIRTUAL TABLE IF NOT EXISTS file USING fts4(" - //"CREATE TABLE IF NOT EXISTS file (" "id INTEGER NOT NULL, " "path TEXT, " "modification_time TEXT, " @@ -907,7 +805,7 @@ void SqliteStorage::setupTables() } catch (CppSQLite3Exception& e) { - std::cerr << e.errorCode() << ":" << e.errorMessage() << std::endl; + LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage()); } m_database.execDML( @@ -1010,26 +908,15 @@ void SqliteStorage::insertOrUpdateMetaValue(const std::string& key, const std::s ).c_str()); } -StorageFile SqliteStorage::getFirstFile(const std::string& query) const -{ - CppSQLite3Query q = m_database.execQuery(query.c_str()); - if (!q.eof()) - { - const Id id = q.getIntField(0, 0); - const std::string serializedName = q.getStringField(1, ""); - const std::string filePath = q.getStringField(2, ""); - const std::string modificationTime = q.getStringField(3, ""); - if (id != 0) - { - return StorageFile(id, serializedName, filePath, modificationTime); - } - } - return StorageFile(0, "", "", ""); -} -std::vector SqliteStorage::getAllFiles(const std::string& query) const + + + + +template <> +std::vector SqliteStorage::getAll(const std::string& query) const { CppSQLite3Query q = m_database.execQuery(( "SELECT file.id, node.serialized_name, file.path, file.modification_time FROM file " @@ -1054,33 +941,8 @@ std::vector SqliteStorage::getAllFiles(const std::string& query) co return files; } -StorageSourceLocation SqliteStorage::getFirstSourceLocation(const std::string& query) const -{ - CppSQLite3Query q = m_database.execQuery(query.c_str()); - - if (!q.eof()) - { - const Id id = q.getIntField(0, 0); - const Id elementId = q.getIntField(1, 0); - const Id fileNodeId = q.getIntField(2, 0); - const int startLineNumber = q.getIntField(3, -1); - const int startColNumber = q.getIntField(4, -1); - const int endLineNumber = q.getIntField(5, -1); - const int endColNumber = q.getIntField(6, -1); - const int type = q.getIntField(7, -1); - - if (id != 0 && elementId != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1 && type != -1) - { - return StorageSourceLocation( - id, elementId, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type - ); - } - } - - return StorageSourceLocation(0, 0, 0, -1, -1, -1, -1, -1); -} - -std::vector SqliteStorage::getAllEdges(const std::string& query) const +template <> +std::vector SqliteStorage::getAll(const std::string& query) const { CppSQLite3Query q = m_database.execQuery(( "SELECT id, type, source_node_id, target_node_id FROM edge " + query + ";" @@ -1104,7 +966,8 @@ std::vector SqliteStorage::getAllEdges(const std::string& query) co return edges; } -std::vector SqliteStorage::getAllNodes(const std::string& query) const +template <> +std::vector SqliteStorage::getAll(const std::string& query) const { CppSQLite3Query q = m_database.execQuery(( "SELECT id, type, serialized_name, definition_type FROM node " + query + ";" @@ -1128,12 +991,138 @@ std::vector SqliteStorage::getAllNodes(const std::string& query) co return nodes; } -StorageNode SqliteStorage::getFirstNode(const std::string& query) const +template <> +std::vector SqliteStorage::getAll(const std::string& query) const { - std::vector nodes = getAllNodes(query + " LIMIT 1"); - if (nodes.size() > 0) + CppSQLite3Query q = m_database.execQuery(( + "SELECT id, name FROM local_symbol " + query + ";" + ).c_str()); + + std::vector localSymbols; + + while (!q.eof()) { - return nodes[0]; + const Id id = q.getIntField(0, 0); + const std::string name = q.getStringField(1, ""); + + if (id != 0) + { + localSymbols.push_back(StorageLocalSymbol(id, name)); + } + + q.nextRow(); } - return StorageNode(); + return localSymbols; +} + +template <> +std::vector SqliteStorage::getAll(const std::string& query) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT id, element_id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location " + query + ";" + ).c_str()); + + std::vector sourceLocations; + + while (!q.eof()) + { + const Id id = q.getIntField(0, 0); + const Id elementId = q.getIntField(1, 0); + const Id fileNodeId = q.getIntField(2, 0); + const int startLineNumber = q.getIntField(3, -1); + const int startColNumber = q.getIntField(4, -1); + const int endLineNumber = q.getIntField(5, -1); + const int endColNumber = q.getIntField(6, -1); + const int type = q.getIntField(7, -1); + + if (id != 0 && elementId != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1 && type != -1) + { + sourceLocations.push_back(StorageSourceLocation(id, elementId, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type)); + } + + q.nextRow(); + } + return sourceLocations; +} + +template <> +std::vector SqliteStorage::getAll(const std::string& query) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT id, edge_id, type FROM component_access " + query + ";" + ).c_str()); + + std::vector componentAccesses; + + while (!q.eof()) + { + const Id id = q.getIntField(0, 0); + const Id edgeId = q.getIntField(1, 0); + const int type = q.getIntField(2, -1); + + if (id != 0 && edgeId != 0 && type != -1) + { + componentAccesses.push_back(StorageComponentAccess(edgeId, type)); + } + + q.nextRow(); + } + return componentAccesses; +} + +template <> +std::vector SqliteStorage::getAll(const std::string& query) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT id, file_node_id, start_line, start_column, end_line, end_column FROM comment_location " + query + ";" + ).c_str()); + + std::vector commentLocations; + + while (!q.eof()) + { + const Id id = q.getIntField(0, 0); + const Id fileNodeId = q.getIntField(1, 0); + const int startLineNumber = q.getIntField(2, -1); + const int startColNumber = q.getIntField(3, -1); + const int endLineNumber = q.getIntField(4, -1); + const int endColNumber = q.getIntField(5, -1); + + if (id != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1) + { + commentLocations.push_back(StorageCommentLocation( + id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber + )); + } + + q.nextRow(); + } + return commentLocations; +} + +template <> +std::vector SqliteStorage::getAll(const std::string& query) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT message, fatal, file_path, line_number, column_number FROM error " + query + ";" + ).c_str()); + + std::vector errors; + while (!q.eof()) + { + const std::string message = q.getStringField(0, ""); + const bool fatal = q.getIntField(1, 0); + const std::string filePath = q.getStringField(2, ""); + const uint lineNumber = q.getIntField(3, -1); + const uint columnNumber = q.getIntField(4, -1); + + if (lineNumber != -1 && columnNumber != -1) + { + errors.push_back(StorageError(message, fatal, filePath, lineNumber, columnNumber)); + } + + q.nextRow(); + } + + return errors; } diff --git a/src/lib/data/SqliteStorage.h b/src/lib/data/SqliteStorage.h index 2bab8b5a..bce3aa5f 100644 --- a/src/lib/data/SqliteStorage.h +++ b/src/lib/data/SqliteStorage.h @@ -42,9 +42,7 @@ public: Id addFile(const std::string& serializedName, const std::string& filePath, const std::string& modificationTime); Id addLocalSymbol(const std::string& name); Id addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); - Id addComponentAccess(Id memberEdgeId, int type); - Id addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol); Id addError(const std::string& message, bool fatal, const std::string& filePath, uint lineNumber, uint columnNumber); @@ -54,9 +52,6 @@ public: void removeErrorsInFiles(const std::vector& filePaths); - StorageNode getFirstNode() const; - std::vector getAllNodes() const; - bool isEdge(Id elementId) const; bool isNode(Id elementId) const; bool isFile(Id elementId) const; @@ -86,7 +81,6 @@ public: std::vector getAllFileIds() const; std::vector getFilesByPaths(const std::vector& filePaths) const; - std::vector getAllFiles() const; std::shared_ptr getFileContentByPath(const std::string& filePath) const; void setNodeType(int type, Id nodeId); @@ -106,9 +100,17 @@ public: void optimizeFTSTable() const; std::vector getCommentLocationsInFile(const FilePath& filePath) const; - std::vector getAllErrors() const; std::vector getFatalErrors() const; + std::vector getAllFiles() const; + std::vector getAllNodes() const; + std::vector getAllEdges() const; + std::vector getAllLocalSymbols() const; + std::vector getAllSourceLocations() const; + std::vector getAllComponentAccesses() const; + std::vector getAllCommentLocations() const; + std::vector getAllErrors() const; + int getNodeCount() const; int getEdgeCount() const; int getFileCount() const; @@ -124,26 +126,40 @@ private: std::string getMetaValue(const std::string& key) const; void insertOrUpdateMetaValue(const std::string& key, const std::string& value); - StorageFile getFirstFile(const std::string& query) const; - std::vector getAllFiles(const std::string& query) const; - StorageSourceLocation getFirstSourceLocation(const std::string& query) const; + template + std::vector getAll(const std::string& query) const; - std::vector getAllEdges(const std::string& query) const; - std::vector getAllNodes(const std::string& query) const; - StorageNode getFirstNode(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; + template <> + std::vector getAll(const std::string& query) const; template - ResultType getFirstResult(const std::string& query) const; + ResultType getFirst(const std::string& query) const + { + std::vector results = getAll(query + " LIMIT 1"); + if (results.size() > 0) + { + return results[0]; + } + return ResultType(); + } mutable CppSQLite3DB m_database; FilePath m_dbFilePath; }; -template -ResultType SqliteStorage::getFirstResult(const std::string& query) const -{ - CppSQLite3Query q = m_database.execQuery(query.c_str()); - return q.getIntField(0, 0); -} #endif // SQLITE_STORAGE_H diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index 25cbe942..a09b8a0d 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -1,1143 +1,178 @@ #include "data/Storage.h" -#include -#include +#include -#include "utility/file/FileSystem.h" +#include "data/graph/Edge.h" +#include "data/StorageTypes.h" #include "utility/logging/logging.h" -#include "utility/messaging/type/MessageClearErrorCount.h" -#include "utility/messaging/type/MessageShowErrors.h" -#include "utility/TimePoint.h" -#include "utility/utility.h" -#include "utility/utilityString.h" -#include "utility/Version.h" -#include "utility/Cache.h" -#include "utility/utilityString.h" -#include "data/graph/token_component/TokenComponentAggregation.h" -#include "data/graph/token_component/TokenComponentSignature.h" -#include "data/graph/Graph.h" -#include "data/location/TokenLocation.h" -#include "data/location/TokenLocationFile.h" -#include "data/location/TokenLocationLine.h" -#include "data/parser/ParseLocation.h" -#include "data/type/DataType.h" -#include "settings/ApplicationSettings.h" - -Storage::Storage(const FilePath& dbPath) - : m_sqliteStorage(dbPath) +Storage::Storage() { - m_commandIndex.addNode(0, NameHierarchy(SearchMatch::getCommandName(SearchMatch::COMMAND_ALL))); - m_commandIndex.addNode(0, NameHierarchy(SearchMatch::getCommandName(SearchMatch::COMMAND_ERROR))); - m_commandIndex.finishSetup(); } Storage::~Storage() { } -FilePath Storage::getDbFilePath() const +void Storage::startInjection() { - return m_sqliteStorage.getDbFilePath(); + // may be implemented in derived } -Version Storage::getVersion() const +void Storage::finishInjection() { - return m_sqliteStorage.getVersion(); + // may be implemented in derived } -void Storage::init() +void Storage::inject(Storage* injected) { - m_sqliteStorage.init(); -} + startInjection(); -void Storage::clear() -{ - m_sqliteStorage.clear(); + std::unordered_map injectedIdToOwnId; - clearCaches(); -} - -void Storage::clearCaches() -{ - m_elementIndex.clear(); - m_fileNodeIds.clear(); - m_hierarchyCache.clear(); -} - -std::set Storage::getDependingFilePaths(const std::set& filePaths) -{ - std::set dependingFilePaths; - for (const FilePath& filePath: filePaths) - { - std::set dependingFilePathsSubset = getDependingFilePaths(filePath); - dependingFilePaths.insert(dependingFilePathsSubset.begin(), dependingFilePathsSubset.end()); - } - return dependingFilePaths; -} - -std::set Storage::getDependingFilePaths(const FilePath& filePath) -{ - std::set dependingFilePaths; - - std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetType( - getFileNodeId(filePath), Edge::typeToInt(Edge::EDGE_INCLUDE) - ); - for (const StorageEdge& incomingEdge: incomingEdges) - { - FilePath dependingFilePath = getFileNodePath(incomingEdge.sourceNodeId); - dependingFilePaths.insert(dependingFilePath); - - std::set dependingFilePathsSubset = getDependingFilePaths(dependingFilePath); - dependingFilePaths.insert(dependingFilePathsSubset.begin(), dependingFilePathsSubset.end()); - } - - return dependingFilePaths; -} - -void Storage::clearFileElements(const std::vector& filePaths) -{ - std::vector fileNodeIds; - - for (const FilePath& path : filePaths) - { - fileNodeIds.push_back(getFileNodeId(path)); - } - - if (fileNodeIds.size()) - { - m_sqliteStorage.removeElementsWithLocationInFiles(fileNodeIds); - m_sqliteStorage.removeElements(fileNodeIds); - - m_sqliteStorage.removeErrorsInFiles(filePaths); - } -} - -void Storage::removeUnusedNames() // maybe rename this function. look for callers first. -{ -// m_sqliteStorage.removeUnusedNameHierarchyElements(); - - clearCaches(); -} - -std::vector Storage::getInfoOnAllFiles() const -{ - std::vector fileInfos; - - std::vector storageFiles = m_sqliteStorage.getAllFiles(); - for (size_t i = 0; i < storageFiles.size(); i++) - { - boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time; - if (storageFiles[i].modificationTime != "not-a-date-time") + injected->forEachFile( + [&](Id injectedId, const StorageFile& injectedData) { - modificationTime = boost::posix_time::time_from_string(storageFiles[i].modificationTime); - } - fileInfos.push_back(FileInfo( - FilePath(storageFiles[i].filePath), - modificationTime - )); - } - - return fileInfos; -} - -void Storage::logStats() const -{ - std::stringstream ss; - StorageStats stats = getStorageStats(); - - ss << "\nGraph:\n"; - ss << "\t" << stats.nodeCount << " Nodes\n"; - ss << "\t" << stats.edgeCount << " Edges\n"; - - ss << "\nCode:\n"; - ss << "\t" << stats.fileCount << " Files\n"; - ss << "\t" << stats.fileLOCCount << " Lines of Code\n"; - - ss << "\nErrors:\n"; - ss << "\t" << stats.errorCount.total << " Errors\n"; - ss << "\t" << stats.errorCount.fatal << " Fatal Errors\n"; - - LOG_WARNING(ss.str()); -} - -void Storage::startParsing() -{ - MessageClearErrorCount().dispatch(); - - m_sqliteStorage.setVersion(Version::getApplicationVersion()); -} - -void Storage::finishParsing() -{ - buildSearchIndex(); - buildHierarchyCache(); - optimizeFTSTable(); -} - -void Storage::injectData(std::shared_ptr injectedStorage) -{ - int totalErrorCount = getErrorCount().total; - - injectedStorage->transferToStorage(m_sqliteStorage); - - if (totalErrorCount != getErrorCount().total) - { - MessageShowErrors msg(getErrorCount()); - msg.setSendAsTask(false); - msg.dispatch(); - } -} - -Id Storage::getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const -{ - return m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(nameHierarchy)).id; -} - -Id Storage::getIdForEdge( - Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy -) const -{ - Id sourceId = getIdForNodeWithNameHierarchy(fromNameHierarchy); - Id targetId = getIdForNodeWithNameHierarchy(toNameHierarchy); - return m_sqliteStorage.getEdgeBySourceTargetType(sourceId, targetId, type).id; -} - -Id Storage::getIdForFirstNode() const -{ - return m_sqliteStorage.getFirstNode().id; -} - -NameHierarchy Storage::getNameHierarchyForNodeWithId(Id nodeId) const -{ - return NameHierarchy::deserialize(m_sqliteStorage.getNodeById(nodeId).serializedName); -} - -Node::NodeType Storage::getNodeTypeForNodeWithId(Id nodeId) const -{ - return Node::intToType(m_sqliteStorage.getNodeById(nodeId).type); -} - -std::shared_ptr Storage::getFullTextSearchLocations(const std::string& searchTerm) const -{ - std::shared_ptr collection = std::make_shared(); - - std::vector parseLocations = m_sqliteStorage.getFullTextSearch(searchTerm); - size_t i = 0; - for(ParseLocation location : parseLocations) - { - collection->addTokenLocation( - i, - 0, - location.filePath, - location.startLineNumber, - location.startColumnNumber, - location.endLineNumber, - location.endColumnNumber - )->setType(LOCATION_FULLTEXTSEARCH_MATCH); - i++; - } - - return collection; -} - -std::vector Storage::getAutocompletionMatches(const std::string& query) const -{ - std::vector commandResults = m_commandIndex.search(query, 0); - - const size_t maxResultCount = 100; - std::vector elementResults = m_elementIndex.search(query, maxResultCount); - - std::vector results; - utility::append(results, commandResults); - utility::append(results, elementResults); - - std::sort(results.begin(), results.end(), - [](const SearchResult& a, const SearchResult& b) - { - // should a be ranked higher than b? - if (a.score > b.score) - { - return true; - } - else if (a.score == b.score) - { - if (a.text.size() < b.text.size()) - { - return true; - } - else if (a.text.size() == b.text.size()) - { - for (size_t i = 0; i < a.text.size(); i++) - { - if (tolower(a.text[i]) != tolower(b.text[i])) - { - return tolower(a.text[i]) < tolower(b.text[i]); - } - else - { - if (a.text[i] < b.text[i]) - { - return true; - } - else if (a.text[i] > b.text[i]) - { - return false; - } - } - } - } - } - return false; - } - ); - - std::map storageNodesMap; - { - std::vector elementIds; - - for (const SearchResult& result : results) - { - elementIds.insert(elementIds.end(), result.elementIds.begin(), result.elementIds.end()); - } - - std::vector storageNodes = m_sqliteStorage.getNodesByIds(elementIds); - - for (StorageNode& node : storageNodes) - { - if (node.id > 0) - { - storageNodesMap.emplace(node.id, node); - } - } - } - - std::vector matches; - for (const SearchResult& result : results) - { - SearchMatch match; - - const StorageNode* firstNode = nullptr; - for (const Id& elementId : result.elementIds) - { - if (elementId != 0) - { - const StorageNode& node = storageNodesMap[elementId]; - match.nameHierarchies.push_back(NameHierarchy::deserialize(node.serializedName)); - - if (!firstNode) - { - firstNode = &node; - } - } - } - - match.text = result.text; - match.indices = result.indices; - - if (firstNode) - { - match.nodeType = Node::intToType(firstNode->type); - match.typeName = Node::getTypeString(match.nodeType); - - if (intToDefinitionType(firstNode->definitionType) == DEFINITION_NONE - && match.nodeType != Node::NODE_UNDEFINED) - { - match.typeName = "undefined " + match.typeName; - } - match.searchType = SearchMatch::SEARCH_TOKEN; - } - else - { - match.searchType = SearchMatch::SEARCH_COMMAND; - match.typeName = "command"; - } - - matches.push_back(match); - } - - return matches; -} - -std::vector Storage::getSearchMatchesForTokenIds(const std::vector& elementIds) const -{ - // todo: what if all these elements share the same node in the searchindex? - // In that case there should be only one search match. - std::vector matches; - - for (Id elementId : elementIds) - { - SearchMatch match; - - if (m_sqliteStorage.isFile(elementId)) - { - match.nodeType = Node::NODE_FILE; - } - else if (m_sqliteStorage.isNode(elementId)) - { - StorageNode node = m_sqliteStorage.getNodeById(elementId); - match.nodeType = Node::intToType(node.type); - } - else - { - continue; - } - - NameHierarchy nameHierarchy = NameHierarchy::deserialize(m_sqliteStorage.getNodeById(elementId).serializedName); - match.text = nameHierarchy.getQualifiedName(); - match.nameHierarchies.push_back(nameHierarchy.getQualifiedName()); - match.searchType = SearchMatch::SEARCH_TOKEN; - - matches.push_back(match); - } - - return matches; -} - -std::shared_ptr Storage::getGraphForAll() const -{ - std::shared_ptr graph = std::make_shared(); - - std::vector tokenIds; - for (StorageNode node: m_sqliteStorage.getAllNodes()) - { - if (intToDefinitionType(node.definitionType) == DEFINITION_EXPLICIT && - (!m_hierarchyCache.isChildOfVisibleNodeOrInvisible(node.id) || - Node::intToType(node.type) == Node::NODE_NAMESPACE)) - { - tokenIds.push_back(node.id); - } - } - - addNodesToGraph(tokenIds, graph.get()); - - return graph; -} - -std::shared_ptr Storage::getGraphForActiveTokenIds(const std::vector& tokenIds) const -{ - std::shared_ptr g = std::make_shared(); - Graph* graph = g.get(); - - std::vector ids(tokenIds); - bool isNamespace = false; - - std::vector nodeIds; - std::vector edgeIds; - bool addAggregations = false; - - //m_sqliteStorage.getFullTextSearch("const int"); - if (tokenIds.size() == 1) - { - const Id elementId = tokenIds[0]; - StorageNode node = m_sqliteStorage.getNodeById(elementId); - - if (node.id > 0) - { - if (Node::intToType(node.type) == Node::NODE_NAMESPACE) - { - ids.clear(); - m_hierarchyCache.addFirstChildIdsForNodeId(elementId, &ids); - - isNamespace = true; - } - else - { - nodeIds.push_back(elementId); - - std::vector edges = m_sqliteStorage.getEdgesBySourceOrTargetId(elementId); - for (const StorageEdge& edge : edges) - { - if (Edge::intToType(edge.type) != Edge::EDGE_MEMBER) - { - edgeIds.push_back(edge.id); - } - } - - addAggregations = true; - } - } - else if (m_sqliteStorage.isEdge(elementId)) - { - edgeIds.push_back(elementId); - } - } - - if (ids.size() >= 1 || isNamespace) - { - std::vector nodes = m_sqliteStorage.getNodesByIds(ids); - for (const StorageNode& node : nodes) - { - if (node.id > 0 && (!isNamespace || intToDefinitionType(node.definitionType) != DEFINITION_IMPLICIT)) - { - nodeIds.push_back(node.id); - } - } - - if (nodeIds.size() != ids.size()) - { - std::vector edges = m_sqliteStorage.getEdgesByIds(ids); - for (const StorageEdge& edge : edges) - { - if (edge.id > 0) - { - edgeIds.push_back(edge.id); - } - } - } - } - - if (isNamespace) - { - addNodesToGraph(nodeIds, graph); - } - else - { - addNodesWithChildrenAndEdgesToGraph(nodeIds, edgeIds, graph); - } - - if (addAggregations) - { - addAggregationEdgesToGraph(tokenIds[0], graph); - } - - addComponentAccessToGraph(graph); - - return g; -} - -// TODO: rename: getActiveElementIdsForId; TODO: make separate function for declarationId -std::vector Storage::getActiveTokenIdsForId(Id tokenId, Id* declarationId) const -{ - std::vector activeTokenIds; - - if (!(m_sqliteStorage.isEdge(tokenId) || m_sqliteStorage.isNode(tokenId))) - { - return activeTokenIds; - } - - activeTokenIds.push_back(tokenId); - - if (m_sqliteStorage.isNode(tokenId)) - { - *declarationId = tokenId; - - std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetId(tokenId); - for (size_t i = 0; i < incomingEdges.size(); i++) - { - activeTokenIds.push_back(incomingEdges[i].id); - } - } - - return activeTokenIds; -} - -std::vector Storage::getNodeIdsForLocationIds(const std::vector& locationIds) const -{ - std::set edgeIds; - std::set nodeIds; - std::set implicitNodeIds; - - for (Id locationId : locationIds) - { - Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId); - - StorageEdge edge = m_sqliteStorage.getEdgeById(elementId); - if (edge.id != 0) // here we test if location is an edge. - { - edgeIds.insert(edge.targetNodeId); - } - else if(m_sqliteStorage.isNode(elementId)) - { - StorageNode node = m_sqliteStorage.getNodeById(elementId); - if (node.id != 0) - { - if (intToDefinitionType(node.definitionType) == DEFINITION_IMPLICIT) - { - implicitNodeIds.insert(elementId); - } - else - { - nodeIds.insert(elementId); - } - } - } - } - - if (nodeIds.size() == 0) - { - nodeIds = implicitNodeIds; - } - - if (nodeIds.size()) - { - return utility::toVector(nodeIds); - } - - return utility::toVector(edgeIds); -} - -std::vector Storage::getLocalSymbolIdsForLocationIds(const std::vector& locationIds) const -{ - std::set localSymbolIds; - - for (Id locationId : locationIds) - { - Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId); - - if (m_sqliteStorage.getNodeById(elementId).id == 0 && m_sqliteStorage.getEdgeById(elementId).id == 0) - { - localSymbolIds.insert(elementId); - } - } - - return utility::toVector(localSymbolIds); -} - -std::vector Storage::getTokenIdsForMatches(const std::vector& matches) const -{ - std::set idSet; - for (const SearchMatch& match : matches) - { - for (size_t i = 0; i < match.nameHierarchies.size(); i++) - { - idSet.insert( - m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(match.nameHierarchies[i])).id - ); - } - } - - std::vector ids; - for (std::set::const_iterator it = idSet.begin(); it != idSet.end(); it++) - { - if (*it != 0) - { - ids.push_back(*it); - } - } - - return ids; -} - -Id Storage::getTokenIdForFileNode(const FilePath& filePath) const -{ - return m_sqliteStorage.getFileByPath(filePath.str()).id; -} - -std::vector Storage::getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const -{ - std::vector edgeIds; - - std::vector aggregationEndpointsA = getAllChildNodeIds(sourceId); - std::set aggregationEndpointsB; - aggregationEndpointsB.insert(targetId); - for (const Id targetChildId: getAllChildNodeIds(targetId)) - { - aggregationEndpointsB.insert(targetChildId); - } - - for (size_t i = 0; i < aggregationEndpointsA.size(); i++) - { - std::vector outgoingEdges = m_sqliteStorage.getEdgesBySourceId(aggregationEndpointsA[i]); - for (size_t j = 0; j < outgoingEdges.size(); j++) - { - if (aggregationEndpointsB.find(outgoingEdges[j].targetNodeId) != aggregationEndpointsB.end()) - { - edgeIds.push_back(outgoingEdges[j].id); - } - } - - std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetId(aggregationEndpointsA[i]); - for (size_t j = 0; j < incomingEdges.size(); j++) - { - if (aggregationEndpointsB.find(incomingEdges[j].sourceNodeId) != aggregationEndpointsB.end()) - { - edgeIds.push_back(incomingEdges[j].id); - } - } - } - - return edgeIds; -} - -std::shared_ptr Storage::getTokenLocationsForTokenIds(const std::vector& tokenIds) const -{ - std::shared_ptr collection = std::make_shared(); - - std::vector fileIds; - std::vector nonFileIds; - std::vector allFileIds = m_sqliteStorage.getAllFileIds(); - for (size_t i = 0; i < tokenIds.size(); i++) - { - if (std::find(allFileIds.begin(), allFileIds.end(),tokenIds[i]) != allFileIds.end()) - { - fileIds.push_back(tokenIds[i]); - } - else - { - nonFileIds.push_back(tokenIds[i]); - } - } - - for (Id fileId: fileIds) - { - StorageFile storageFile = m_sqliteStorage.getFileById(fileId); - collection->addTokenLocationFileAsPlainCopy( - m_sqliteStorage.getTokenLocationsForFile(storageFile.filePath).get() - ); - } - - Cache filePathCache( - [this](Id id) -> std::string - { - return m_sqliteStorage.getFileById(id).filePath; - } - ); - - std::vector locations = m_sqliteStorage.getTokenLocationsForElementIds(nonFileIds); - for (size_t i = 0; i < locations.size(); i++) - { - const StorageSourceLocation& location = locations[i]; - std::string filePath = filePathCache.getValue(location.fileNodeId); - - TokenLocation* loc = collection->addTokenLocation( - location.id, - location.elementId, - filePath, - location.startLine, - location.startCol, - location.endLine, - location.endCol - ); - - if (loc) - { - loc->setType(intToLocationType(location.type)); - } - } - - return collection; -} - -std::shared_ptr Storage::getTokenLocationsForLocationIds( - const std::vector& locationIds -) const -{ - std::shared_ptr collection = std::make_shared(); - - for (size_t i = 0; i < locationIds.size(); i++) - { - StorageSourceLocation location = m_sqliteStorage.getSourceLocationById(locationIds[i]); - collection->addTokenLocation( - location.id, - location.elementId, - m_sqliteStorage.getFileById(location.fileNodeId).filePath, // TODO: optimize: only once per file! - location.startLine, - location.startCol, - location.endLine, - location.endCol - )->setType(intToLocationType(location.type)); - } - - return collection; -} - -std::shared_ptr Storage::getTokenLocationsForFile(const std::string& filePath) const -{ - std::shared_ptr locationFile = m_sqliteStorage.getTokenLocationsForFile(filePath); - locationFile->isWholeCopy = true; - return locationFile; -} - -std::shared_ptr Storage::getTokenLocationsForLinesInFile( - const std::string& filePath, uint firstLineNumber, uint lastLineNumber -) const -{ - return m_sqliteStorage.getTokenLocationsForFile(filePath)->getFilteredByLines(firstLineNumber, lastLineNumber); -} - -TokenLocationCollection Storage::getErrorTokenLocations(std::vector* errors) const -{ - TokenLocationCollection errorCollection; - - std::vector storageErrors = m_sqliteStorage.getAllErrors(); - for (size_t i = 0; i < storageErrors.size(); i++) - { - const StorageError& error = storageErrors[i]; - errorCollection.addTokenLocation( - i, i, error.filePath, error.lineNumber, error.columnNumber, error.lineNumber, error.columnNumber); - errors->push_back(ErrorInfo(error.message, error.filePath, i, error.fatal)); - } - - return errorCollection; -} - -std::shared_ptr Storage::getCommentLocationsInFile(const FilePath& filePath) const -{ - std::shared_ptr file = std::make_shared(filePath); - - std::vector storageLocations = m_sqliteStorage.getCommentLocationsInFile(filePath); - for (size_t i = 0; i < storageLocations.size(); i++) - { - file->addTokenLocation( - storageLocations[i].id, - 0, // comment token location has no element. - storageLocations[i].startLine, - storageLocations[i].startCol, - storageLocations[i].endLine, - storageLocations[i].endCol - ); - } - - return file; -} - -std::shared_ptr Storage::getFileContent(const FilePath& filePath) const -{ - return m_sqliteStorage.getFileContentByPath(filePath.str()); -} - -FileInfo Storage::getFileInfoForFilePath(const FilePath& filePath) const -{ - return FileInfo(filePath, m_sqliteStorage.getFileByPath(filePath).modificationTime); -} - -std::vector Storage::getFileInfosForFilePaths(const std::vector& filePaths) const -{ - std::vector fileInfos; - - std::vector storageFiles = m_sqliteStorage.getFilesByPaths(filePaths); - for (const StorageFile& file : storageFiles) - { - fileInfos.push_back(FileInfo(FilePath(file.filePath), file.modificationTime)); - } - - return fileInfos; -} - -ErrorCountInfo Storage::getErrorCount() const -{ - return ErrorCountInfo(m_sqliteStorage.getAllErrors().size(), m_sqliteStorage.getFatalErrors().size()); -} - -StorageStats Storage::getStorageStats() const -{ - StorageStats stats; - - stats.nodeCount = m_sqliteStorage.getNodeCount(); - stats.edgeCount = m_sqliteStorage.getEdgeCount(); - - stats.fileCount = m_sqliteStorage.getFileCount(); - stats.fileLOCCount = m_sqliteStorage.getFileLOCCount(); - - stats.errorCount = getErrorCount(); - - return stats; -} - -Id Storage::getFileNodeId(const FilePath& filePath) const -{ - std::map::const_iterator it = m_fileNodeIds.find(filePath); - - if (it != m_fileNodeIds.end()) - { - return it->second; - } - - if (filePath.empty()) - { - LOG_ERROR("No file path set"); - return 0; - } - - StorageFile storageFile = m_sqliteStorage.getFileByPath(filePath.str()); - - if (storageFile.id == 0) - { - return 0; - } - - m_fileNodeIds.emplace(filePath, storageFile.id); - - return storageFile.id; -} - -FilePath Storage::getFileNodePath(Id fileId) const -{ - for (const std::pair& p : m_fileNodeIds) - { - if (p.second == fileId) - { - return p.first; - } - } - - return m_sqliteStorage.getFileById(fileId).filePath; -} - -Id Storage::getLastVisibleParentNodeId(const Id nodeId) const -{ - return m_hierarchyCache.getLastVisibleParentNodeId(nodeId); -} - -std::vector Storage::getAllChildNodeIds(const Id nodeId) const -{ - std::vector childNodeIds; - std::vector edgeIds; - - m_hierarchyCache.addAllChildIdsForNodeId(nodeId, &childNodeIds, &edgeIds); - - return childNodeIds; -} - -void Storage::addNodesToGraph(const std::vector& nodeIds, Graph* graph) const -{ - if (nodeIds.size() == 0) - { - return; - } - - std::vector storageNodes = m_sqliteStorage.getNodesByIds(nodeIds); - - for (const StorageNode& storageNode : storageNodes) - { - NameHierarchy nameHierarchy = NameHierarchy::deserialize(storageNode.serializedName); - - Node::NodeType type = Node::intToType(storageNode.type); - DefinitionType defType = intToDefinitionType(storageNode.definitionType); - Node* node = graph->createNode( - storageNode.id, - type, - nameHierarchy, - defType != DEFINITION_NONE - ); - - if (defType == DEFINITION_IMPLICIT) - { - node->setImplicit(true); - } - else if (defType == DEFINITION_EXPLICIT) - { - node->setExplicit(true); - } - - if (type == Node::NODE_FUNCTION || type == Node::NODE_METHOD) - { - std::string signatureString = nameHierarchy.getRawNameWithSignature(); - if (signatureString.size() > 0) // this should always be the case since functions and methods must have sigs. - { - node->addComponentSignature( - std::make_shared(signatureString) - ); - } - } - } -} - -void Storage::addEdgesToGraph(const std::vector& edgeIds, Graph* graph) const -{ - if (edgeIds.size() == 0) - { - return; - } - - std::vector storageEdges = m_sqliteStorage.getEdgesByIds(edgeIds); - for (const StorageEdge& storageEdge : storageEdges) - { - Node* sourceNode = graph->getNodeById(storageEdge.sourceNodeId); - Node* targetNode = graph->getNodeById(storageEdge.targetNodeId); - - if (sourceNode && targetNode) - { - graph->createEdge(storageEdge.id, Edge::intToType(storageEdge.type), sourceNode, targetNode); - } - else - { - LOG_ERROR("Can't add edge because nodes are not present"); - } - } -} - -void Storage::addNodesWithChildrenAndEdgesToGraph( - const std::vector& nodeIds, - const std::vector& edgeIds, - Graph* graph -) const -{ - std::set parentNodeIds; - - for (Id nodeId : nodeIds) - { - parentNodeIds.insert(getLastVisibleParentNodeId(nodeId)); - } - - if (edgeIds.size() > 0) - { - std::vector storageEdges = m_sqliteStorage.getEdgesByIds(edgeIds); - for (const StorageEdge& storageEdge : storageEdges) - { - parentNodeIds.insert(getLastVisibleParentNodeId(storageEdge.sourceNodeId)); - parentNodeIds.insert(getLastVisibleParentNodeId(storageEdge.targetNodeId)); - } - } - - std::vector allNodeIds; - std::vector allEdgeIds = edgeIds; - - for (Id parentNodeId : parentNodeIds) - { - allNodeIds.push_back(parentNodeId); - m_hierarchyCache.addAllChildIdsForNodeId(parentNodeId, &allNodeIds, &allEdgeIds); - } - - addNodesToGraph(allNodeIds, graph); - addEdgesToGraph(allEdgeIds, graph); -} - -void Storage::addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const -{ - struct EdgeInfo - { - Id edgeId; - bool forward; - }; - - // build aggregation edges: - // get all children of the active node - std::vector childNodeIds = getAllChildNodeIds(nodeId); - if (childNodeIds.size() == 0) - { - return; - } - - // get all edges of the children - std::map> connectedNodeIds; - - std::vector outgoingEdges = m_sqliteStorage.getEdgesBySourceIds(childNodeIds); - for (size_t j = 0; j < outgoingEdges.size(); j++) - { - EdgeInfo edgeInfo; - edgeInfo.edgeId = outgoingEdges[j].id; - edgeInfo.forward = true; - connectedNodeIds[outgoingEdges[j].targetNodeId].push_back(edgeInfo); - } - - std::vector incomingEdges = m_sqliteStorage.getEdgesByTargetIds(childNodeIds); - for (size_t j = 0; j < incomingEdges.size(); j++) - { - EdgeInfo edgeInfo; - edgeInfo.edgeId = incomingEdges[j].id; - edgeInfo.forward = false; - connectedNodeIds[incomingEdges[j].sourceNodeId].push_back(edgeInfo); - } - - // get all parent nodes of all connected nodes (up to last level except namespace/undefined) - Id nodeParentNodeId = getLastVisibleParentNodeId(nodeId); - - std::map> connectedParentNodeIds; - for (const std::pair>& p : connectedNodeIds) - { - Id parentNodeId = getLastVisibleParentNodeId(p.first); - - if (parentNodeId != nodeParentNodeId) - { - utility::append(connectedParentNodeIds[parentNodeId], p.second); - } - } - - // add hierarchies of these parents - std::vector nodeIdsToAdd; - for (const std::pair> p : connectedParentNodeIds) - { - const Id aggregationTargetNodeId = p.first; - if (!graph->getNodeById(aggregationTargetNodeId)) - { - nodeIdsToAdd.push_back(aggregationTargetNodeId); - } - } - addNodesWithChildrenAndEdgesToGraph(nodeIdsToAdd, std::vector(), graph); - - // create aggregation edges between parents and active node - Node* sourceNode = graph->getNodeById(nodeId); - for (const std::pair> p : connectedParentNodeIds) - { - const Id aggregationTargetNodeId = p.first; - - Node* targetNode = graph->getNodeById(aggregationTargetNodeId); - if (!targetNode) - { - LOG_ERROR("Aggregation target node not present."); - } - - std::shared_ptr componentAggregation = std::make_shared(); - for (const EdgeInfo& edgeInfo: p.second) - { - componentAggregation->addAggregationId(edgeInfo.edgeId, edgeInfo.forward); - } - - Edge* edge = graph->createEdge( - *componentAggregation->getAggregationIds().begin(), - Edge::EDGE_AGGREGATION, - sourceNode, - targetNode - ); - - edge->addComponentAggregation(componentAggregation); - } -} - -void Storage::addComponentAccessToGraph(Graph* graph) const -{ - std::vector memberEdgeIds; - - graph->forEachEdge( - [&memberEdgeIds](Edge* edge) - { - if (!edge->isType(Edge::EDGE_MEMBER)) + if (injectedData.name.size() == 0) { return; } - memberEdgeIds.push_back(edge->getId()); + Id ownId = addFile(injectedData.name, injectedData.filePath, injectedData.modificationTime); + if (ownId != 0) + { + injectedIdToOwnId[injectedId] = ownId; + } } ); - std::vector accesses = m_sqliteStorage.getComponentAccessByMemberEdgeIds(memberEdgeIds); - for (const StorageComponentAccess& access : accesses) - { - if (access.memberEdgeId && access.type) + injected->forEachNode( + [&](Id injectedId, const StorageNode& injectedData) { - graph->getEdgeById(access.memberEdgeId)->addComponentAccess( - std::make_shared(TokenComponentAccess::intToType(access.type))); + Id ownId = addNode(injectedData.type, injectedData.serializedName, injectedData.definitionType); + if (ownId != 0) + { + injectedIdToOwnId[injectedId] = ownId; + } } - } -} - -void Storage::buildSearchIndex() -{ - for (StorageNode node: m_sqliteStorage.getAllNodes()) - { - m_elementIndex.addNode(node.id, NameHierarchy::deserialize(node.serializedName)); - } - m_elementIndex.finishSetup(); -} - -void Storage::buildHierarchyCache() -{ - std::vector memberEdges = m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER)); - - Cache nodeTypeCache([this](Id id){ - return Node::intToType(m_sqliteStorage.getNodeById(id).type); - }); - - for (const StorageEdge& edge : memberEdges) - { - bool isVisible = !(nodeTypeCache.getValue(edge.sourceNodeId) & Node::NODE_NOT_VISIBLE); - m_hierarchyCache.createConnection(edge.id, edge.sourceNodeId, edge.targetNodeId, isVisible); - } -} - -void Storage::optimizeFTSTable() -{ - m_sqliteStorage.optimizeFTSTable(); + ); + + injected->forEachEdge( + [&](Id injectedId, const StorageEdge& injectedData) + { + std::unordered_map::const_iterator it; + it = injectedIdToOwnId.find(injectedData.sourceNodeId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownSourceId = it->second; + + it = injectedIdToOwnId.find(injectedData.targetNodeId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownTargetId = it->second; + + Id ownId = addEdge(injectedData.type, ownSourceId, ownTargetId); + + if (ownId != 0) + { + injectedIdToOwnId[injectedId] = ownId; + } + } + ); + + injected->forEachLocalSymbol( + [&](const Id injectedId, const StorageLocalSymbol& injectedData) + { + Id ownId = addLocalSymbol(injectedData.name); + if (ownId != 0) + { + injectedIdToOwnId[injectedId] = ownId; + } + } + ); + + injected->forEachSourceLocation( + [&](const StorageSourceLocation& injectedData) + { + std::unordered_map::const_iterator it; + it = injectedIdToOwnId.find(injectedData.elementId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownElementId = it->second; + + it = injectedIdToOwnId.find(injectedData.fileNodeId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownFileNodeId = it->second; + + addSourceLocation( + ownElementId, + ownFileNodeId, + injectedData.startLine, + injectedData.startCol, + injectedData.endLine, + injectedData.endCol, + injectedData.type + ); + } + ); + + injected->forEachComponentAccess( + [&](const StorageComponentAccess& injectedData) + { + std::unordered_map::const_iterator it; + it = injectedIdToOwnId.find(injectedData.memberEdgeId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownMemberEdgeId = it->second; + + addComponentAccess(ownMemberEdgeId, injectedData.type); + } + ); + + injected->forEachCommentLocation( + [&](const StorageCommentLocation& injectedData) + { + std::unordered_map::const_iterator it; + it = injectedIdToOwnId.find(injectedData.fileNodeId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownFileNodeId = it->second; + + addCommentLocation( + ownFileNodeId, + injectedData.startLine, + injectedData.startCol, + injectedData.endLine, + injectedData.endCol + ); + } + ); + + injected->forEachError( + [&](const StorageError& injectedData) + { + addError( + injectedData.message, + injectedData.fatal, + injectedData.filePath, + injectedData.lineNumber, + injectedData.columnNumber + ); + } + ); + + finishInjection(); } diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index 45ecf74b..e2405b4b 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -1,128 +1,40 @@ #ifndef STORAGE_H #define STORAGE_H -#include -#include +#include +#include -#include "utility/file/FilePath.h" +#include "data/name/NameHierarchy.h" +#include "data/StorageTypes.h" +#include "utility/types.h" -#include "data/access/StorageAccess.h" -#include "data/graph/token_component/TokenComponentAccess.h" -#include "data/location/TokenLocationCollection.h" -#include "data/parser/ParserClient.h" -#include "data/parser/ParseLocation.h" -#include "data/search/SearchIndex.h" -#include "data/HierarchyCache.h" -#include "data/SqliteStorage.h" - -#include "data/parser/ParserClientImpl.h" - -class Storage: public StorageAccess +class Storage { public: - Storage(const FilePath& dbPath); + Storage(); virtual ~Storage(); - FilePath getDbFilePath() const; - Version getVersion() const; + virtual Id addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime) = 0; + virtual Id addNode(int type, const std::string& serializedName, int definitionType) = 0; + virtual Id addEdge(int type, Id sourceId, Id targetId) = 0; + virtual Id addLocalSymbol(const std::string& name) = 0; + virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) = 0; + virtual void addComponentAccess(Id edgeId , int type) = 0; + virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) = 0; + virtual void addError(const std::string& message, bool fatal, const std::string& filePath, uint startLine, uint startCol) = 0; - void init(); - void clear(); - void clearCaches(); + virtual void forEachFile(std::function callback) const = 0; + virtual void forEachNode(std::function callback) const = 0; + virtual void forEachEdge(std::function callback) const = 0; + virtual void forEachLocalSymbol(std::function callback) const = 0; + virtual void forEachSourceLocation(std::function callback) const = 0; + virtual void forEachComponentAccess(std::function callback) const = 0; + virtual void forEachCommentLocation(std::function callback) const = 0; + virtual void forEachError(std::function callback) const = 0; - std::set getDependingFilePaths(const std::set& filePaths); - std::set getDependingFilePaths(const FilePath& filePath); - - void clearFileElements(const std::vector& filePaths); - void removeUnusedNames(); - - std::vector getInfoOnAllFiles() const; - - void logStats() const; - - void startParsing(); - void finishParsing(); - - void injectData(std::shared_ptr injectedStorage); - - // StorageAccess implementation - virtual Id getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const; - virtual Id getIdForEdge( - Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const; - - virtual Id getIdForFirstNode() const; - - virtual NameHierarchy getNameHierarchyForNodeWithId(Id nodeId) const; - virtual Node::NodeType getNodeTypeForNodeWithId(Id nodeId) const; - - virtual std::shared_ptr getFullTextSearchLocations(const std::string& searchTerm) const; - virtual std::vector getAutocompletionMatches(const std::string& query) const; - virtual std::vector getSearchMatchesForTokenIds(const std::vector& elementIds) const; - - virtual std::shared_ptr getGraphForAll() const; - virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const; - - virtual std::vector getActiveTokenIdsForId(Id tokenId, Id* declarationId) const; - - virtual std::vector getNodeIdsForLocationIds(const std::vector& locationIds) const; - virtual std::vector getLocalSymbolIdsForLocationIds(const std::vector& locationIds) const; - - virtual std::vector getTokenIdsForMatches(const std::vector& matches) const; - virtual Id getTokenIdForFileNode(const FilePath& filePath) const; - virtual std::vector getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const; - - virtual std::shared_ptr getTokenLocationsForTokenIds( - const std::vector& tokenIds - ) const; - virtual std::shared_ptr getTokenLocationsForLocationIds( - const std::vector& locationIds - ) const; - virtual std::shared_ptr getTokenLocationsForFile(const std::string& filePath) const; - virtual std::shared_ptr getTokenLocationsForLinesInFile( - const std::string& filePath, uint firstLineNumber, uint lastLineNumber - ) const; - - virtual TokenLocationCollection getErrorTokenLocations(std::vector* errors) const; - virtual std::shared_ptr getCommentLocationsInFile(const FilePath& filePath) const; - - virtual std::shared_ptr getFileContent(const FilePath& filePath) const; - - virtual FileInfo getFileInfoForFilePath(const FilePath& filePath) const; - virtual std::vector getFileInfosForFilePaths(const std::vector& filePaths) const; - - virtual ErrorCountInfo getErrorCount() const; - virtual StorageStats getStorageStats() const; - -private: - Id getFileNodeId(const FilePath& filePath) const; - FilePath getFileNodePath(Id fileId) const; - - Id getLastVisibleParentNodeId(const Id nodeId) const; - std::vector getAllChildNodeIds(const Id nodeId) const; - - void addNodesToGraph(const std::vector& nodeIds, Graph* graph) const; - void addEdgesToGraph(const std::vector& edgeIds, Graph* graph) const; - void addNodesWithChildrenAndEdgesToGraph( - const std::vector& nodeIds, - const std::vector& edgeIds, Graph* graph - ) const; - - void addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const; - void addComponentAccessToGraph(Graph* graph) const; - - void buildSearchIndex(); - void buildHierarchyCache(); - void optimizeFTSTable(); - - void log(std::string type, std::string str, const ParseLocation& location) const; - - SearchIndex m_commandIndex; - SearchIndex m_elementIndex; - - SqliteStorage m_sqliteStorage; - - mutable std::map m_fileNodeIds; - HierarchyCache m_hierarchyCache; + virtual void startInjection(); + virtual void finishInjection(); + void inject(Storage* injected); }; #endif // STORAGE_H diff --git a/src/lib/data/StorageTypes.h b/src/lib/data/StorageTypes.h index d74426c8..f71c3c26 100644 --- a/src/lib/data/StorageTypes.h +++ b/src/lib/data/StorageTypes.h @@ -9,6 +9,13 @@ struct StorageEdge { + StorageEdge() + : id(0) + , type(0) + , sourceNodeId(0) + , targetNodeId(0) + {} + StorageEdge(Id id, int type, Id sourceNodeId, Id targetNodeId) : id(id) , type(type) @@ -45,6 +52,13 @@ struct StorageNode struct StorageFile { + StorageFile() + : id(0) + , name("") + , filePath("") + , modificationTime("") + {} + StorageFile(Id id, const std::string& name, const std::string& filePath, const std::string& modificationTime) : id(id) , name(name) @@ -60,6 +74,11 @@ struct StorageFile struct StorageLocalSymbol { + StorageLocalSymbol() + : id(0) + , name("") + {} + StorageLocalSymbol(Id id, const std::string& name) : id(id) , name(name) @@ -71,6 +90,17 @@ struct StorageLocalSymbol struct StorageSourceLocation { + StorageSourceLocation() + : id(0) + , elementId(0) + , fileNodeId(0) + , startLine(-1) + , startCol(-1) + , endLine(-1) + , endCol(-1) + , type(0) + {} + StorageSourceLocation(Id id, Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) : id(id) , elementId(elementId) @@ -94,6 +124,11 @@ struct StorageSourceLocation struct StorageComponentAccess { + StorageComponentAccess() + : memberEdgeId(0) + , type(0) + {} + StorageComponentAccess(Id memberEdgeId, int type) : memberEdgeId(memberEdgeId) , type(type) @@ -105,6 +140,15 @@ struct StorageComponentAccess struct StorageCommentLocation { + StorageCommentLocation() + : id(0) + , fileNodeId(0) + , startLine(-1) + , startCol(-1) + , endLine(-1) + , endCol(-1) + {} + StorageCommentLocation(Id id, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) : id(id) , fileNodeId(fileNodeId) @@ -124,6 +168,14 @@ struct StorageCommentLocation struct StorageError { + StorageError() + : message("") + , fatal(0) + , filePath("") + , lineNumber(-1) + , columnNumber(-1) + {} + StorageError(const std::string& message, bool fatal, const std::string& filePath, uint lineNumber, uint columnNumber) : message(message) , fatal(fatal) diff --git a/src/lib/data/TaskCleanStorage.cpp b/src/lib/data/TaskCleanStorage.cpp index 9e1afbf7..b6c7430e 100644 --- a/src/lib/data/TaskCleanStorage.cpp +++ b/src/lib/data/TaskCleanStorage.cpp @@ -1,10 +1,10 @@ #include "data/TaskCleanStorage.h" -#include "data/Storage.h" +#include "data/PersistentStorage.h" #include "utility/messaging/type/MessageStatus.h" #include "utility/utility.h" -TaskCleanStorage::TaskCleanStorage(Storage* storage, const std::vector& filePaths) +TaskCleanStorage::TaskCleanStorage(PersistentStorage* storage, const std::vector& filePaths) : m_storage(storage) , m_filePaths(filePaths) , m_fileCount(filePaths.size()) diff --git a/src/lib/data/TaskCleanStorage.h b/src/lib/data/TaskCleanStorage.h index 7242dfd3..1066a908 100644 --- a/src/lib/data/TaskCleanStorage.h +++ b/src/lib/data/TaskCleanStorage.h @@ -7,14 +7,14 @@ #include "utility/scheduling/Task.h" #include "utility/TimePoint.h" -class Storage; +class PersistentStorage; class TaskCleanStorage : public Task { public: TaskCleanStorage( - Storage* storage, + PersistentStorage* storage, const std::vector& filePaths ); @@ -26,7 +26,7 @@ public: virtual void revert(); private: - Storage* m_storage; + PersistentStorage* m_storage; std::vector m_filePaths; const size_t m_fileCount; diff --git a/src/lib/data/access/StorageAccess.h b/src/lib/data/access/StorageAccess.h index 03803bc0..d5fccbb5 100644 --- a/src/lib/data/access/StorageAccess.h +++ b/src/lib/data/access/StorageAccess.h @@ -32,8 +32,6 @@ public: virtual Id getIdForEdge( Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const = 0; - virtual Id getIdForFirstNode() const = 0; - virtual NameHierarchy getNameHierarchyForNodeWithId(Id id) const = 0; virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const = 0; diff --git a/src/lib/data/access/StorageAccessProxy.cpp b/src/lib/data/access/StorageAccessProxy.cpp index 38e7b913..44b1b894 100644 --- a/src/lib/data/access/StorageAccessProxy.cpp +++ b/src/lib/data/access/StorageAccessProxy.cpp @@ -54,16 +54,6 @@ Id StorageAccessProxy::getIdForEdge( return 0; } -Id StorageAccessProxy::getIdForFirstNode() const -{ - if (hasSubject()) - { - return m_subject->getIdForFirstNode(); - } - - return 0; -} - Node::NodeType StorageAccessProxy::getNodeTypeForNodeWithId(Id id) const { if (hasSubject()) diff --git a/src/lib/data/access/StorageAccessProxy.h b/src/lib/data/access/StorageAccessProxy.h index 53c5d590..16f1936d 100644 --- a/src/lib/data/access/StorageAccessProxy.h +++ b/src/lib/data/access/StorageAccessProxy.h @@ -17,8 +17,6 @@ public: virtual Id getIdForEdge( Edge::EdgeType type, const NameHierarchy& fromNameHierarchy, const NameHierarchy& toNameHierarchy) const; - virtual Id getIdForFirstNode() const; - virtual NameHierarchy getNameHierarchyForNodeWithId(Id id) const; virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const; diff --git a/src/lib/data/graph/token_component/TokenComponentAccess.h b/src/lib/data/graph/token_component/TokenComponentAccess.h index 6da1a21e..78a2747e 100644 --- a/src/lib/data/graph/token_component/TokenComponentAccess.h +++ b/src/lib/data/graph/token_component/TokenComponentAccess.h @@ -9,7 +9,7 @@ class TokenComponentAccess : public TokenComponent { public: - enum AccessType : int + enum AccessType : int // todo: use normal numbers here. not 2^x { ACCESS_PUBLIC = 0x1, ACCESS_PROTECTED = 0x2, diff --git a/src/lib/data/parser/ParserClient.h b/src/lib/data/parser/ParserClient.h index 77b82965..0270146d 100644 --- a/src/lib/data/parser/ParserClient.h +++ b/src/lib/data/parser/ParserClient.h @@ -39,9 +39,6 @@ public: ParserClient(); virtual ~ParserClient(); - virtual void startParsing() = 0; - virtual void finishParsing() = 0; - virtual void startParsingFile(const FilePath& filePath) = 0; virtual void finishParsingFile(const FilePath& filePath) = 0; diff --git a/src/lib/data/parser/ParserClientImpl.cpp b/src/lib/data/parser/ParserClientImpl.cpp index 9108a617..f58d1cfa 100644 --- a/src/lib/data/parser/ParserClientImpl.cpp +++ b/src/lib/data/parser/ParserClientImpl.cpp @@ -25,16 +25,9 @@ void ParserClientImpl::resetStorage() m_storage.reset(); } -void ParserClientImpl::startParsing() -{ -} - -void ParserClientImpl::finishParsing() -{ -} - void ParserClientImpl::startParsingFile(const FilePath& filePath) { + m_nodeIdsToMemberEdgeIds.clear(); } void ParserClientImpl::finishParsingFile(const FilePath& filePath) @@ -473,7 +466,7 @@ Id ParserClientImpl::addFile(const std::string& filePath) return 0; } - return m_storage->addFile(filePath); + return m_storage->addFile("", filePath, ""); } Id ParserClientImpl::addNode(Node::NodeType nodeType, NameHierarchy nameHierarchy, DefinitionType definitionType) @@ -483,7 +476,7 @@ Id ParserClientImpl::addNode(Node::NodeType nodeType, NameHierarchy nameHierarch return 0; } - return m_storage->addNode(Node::typeToInt(nodeType), nameHierarchy, definitionTypeToInt(definitionType)); + return m_storage->addNode(Node::typeToInt(nodeType), NameHierarchy::serialize(nameHierarchy), definitionTypeToInt(definitionType)); } Id ParserClientImpl::addEdge(int type, Id sourceId, Id targetId) @@ -498,7 +491,14 @@ Id ParserClientImpl::addEdge(int type, Id sourceId, Id targetId) return 0; } - return m_storage->addEdge(type, sourceId, targetId); + Id edgeId = m_storage->addEdge(type, sourceId, targetId); + + if (type == Edge::EDGE_MEMBER) + { + m_nodeIdsToMemberEdgeIds[targetId] = edgeId; + } + + return edgeId; } Id ParserClientImpl::addLocalSymbol(const std::string& name) @@ -529,7 +529,15 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca return; } - m_storage->addSourceLocation(elementId, location, type); + m_storage->addSourceLocation( + elementId, + addFile(location.filePath.str()), + location.startLineNumber, + location.startColumnNumber, + location.endLineNumber, + location.endColumnNumber, + type + ); } void ParserClientImpl::addComponentAccess(Id nodeId , int type) @@ -539,7 +547,15 @@ void ParserClientImpl::addComponentAccess(Id nodeId , int type) return; } - m_storage->addComponentAccess(nodeId, type); + std::unordered_map::const_iterator it = m_nodeIdsToMemberEdgeIds.find(nodeId); + if (it != m_nodeIdsToMemberEdgeIds.end()) + { + m_storage->addComponentAccess(it->second, type); + } + else + { + LOG_ERROR_STREAM(<< "Cannot assign access" << type << " to node id " << nodeId << " because it's not a child node."); + } } void ParserClientImpl::addCommentLocation(const ParseLocation& location) @@ -549,7 +565,13 @@ void ParserClientImpl::addCommentLocation(const ParseLocation& location) return; } - m_storage->addCommentLocation(location); + m_storage->addCommentLocation( + addFile(location.filePath.str()), + location.startLineNumber, + location.startColumnNumber, + location.endLineNumber, + location.endColumnNumber + ); } void ParserClientImpl::addError(const std::string& message, bool fatal, const ParseLocation& location) @@ -559,7 +581,7 @@ void ParserClientImpl::addError(const std::string& message, bool fatal, const Pa return; } - m_storage->addError(message, fatal, location); + m_storage->addError(message, fatal, location.filePath.str(), location.startLineNumber, location.startColumnNumber); } void ParserClientImpl::log(std::string type, std::string str, const ParseLocation& location) const diff --git a/src/lib/data/parser/ParserClientImpl.h b/src/lib/data/parser/ParserClientImpl.h index a609b973..9ce7a6e3 100644 --- a/src/lib/data/parser/ParserClientImpl.h +++ b/src/lib/data/parser/ParserClientImpl.h @@ -18,9 +18,6 @@ public: void setStorage(std::shared_ptr storage); void resetStorage(); - virtual void startParsing(); - virtual void finishParsing(); - virtual void startParsingFile(const FilePath& filePath); virtual void finishParsingFile(const FilePath& filePath); @@ -105,6 +102,7 @@ private: void log(std::string type, std::string str, const ParseLocation& location) const; std::shared_ptr m_storage; + std::unordered_map m_nodeIdsToMemberEdgeIds; }; #endif // PARSER_CLIENT_IMPL_H diff --git a/src/lib/data/parser/cxx/TaskParseCxx.h b/src/lib/data/parser/cxx/TaskParseCxx.h index b3403523..90657140 100644 --- a/src/lib/data/parser/cxx/TaskParseCxx.h +++ b/src/lib/data/parser/cxx/TaskParseCxx.h @@ -9,7 +9,7 @@ #include "utility/scheduling/Task.h" #include "utility/TimePoint.h" -class Storage; +class PersistentStorage; class FileManager; class CxxParser; @@ -26,7 +26,7 @@ class TaskParseCxx { public: TaskParseCxx( - Storage* storage, + PersistentStorage* storage, const FileManager* fileManager, const Parser::Arguments& arguments, const std::vector& files @@ -42,7 +42,7 @@ public: virtual void revert(); private: - Storage* m_storage; + PersistentStorage* m_storage; std::shared_ptr m_parser; std::shared_ptr m_parserClient; const Parser::Arguments m_arguments; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index dc8f8490..18b0a63d 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -3,8 +3,6 @@ add_files( helper/TestFileManager.cpp helper/TestFileManager.h - helper/TestStorage.cpp - helper/TestStorage.h TestSuiteFixture.cpp TestSuiteFixture.h diff --git a/src/test/StorageTestSuite.h b/src/test/StorageTestSuite.h index bcbec7e5..4fad75a8 100644 --- a/src/test/StorageTestSuite.h +++ b/src/test/StorageTestSuite.h @@ -7,7 +7,7 @@ #include "data/graph/token_component/TokenComponentStatic.h" #include "data/location/TokenLocation.h" #include "data/parser/ParseLocation.h" -#include "data/Storage.h" +#include "data/PersistentStorage.h" #include "data/type/DataType.h" #include "data/type/NamedDataType.h" @@ -26,7 +26,7 @@ public: std::shared_ptr intermetiateStorage = std::make_shared(); Id id = intermetiateStorage->addFile("test.h", "path/to/test.h", "someTime"); - storage.injectData(intermetiateStorage); + storage.inject(intermetiateStorage.get()); TS_ASSERT_EQUALS(storage.getNameHierarchyForNodeWithId(id).getQualifiedNameWithSignature(), "test.h"); TS_ASSERT_EQUALS(storage.getNodeTypeForNodeWithId(id), Node::NODE_FILE); @@ -39,9 +39,9 @@ public: TestStorage storage; std::shared_ptr intermetiateStorage = std::make_shared(); - Id id = intermetiateStorage->addNode(Node::typeToInt(Node::NODE_TYPEDEF), a, true); + Id id = intermetiateStorage->addNode(Node::typeToInt(Node::NODE_TYPEDEF), NameHierarchy::serialize(a), true); - storage.injectData(intermetiateStorage); + storage.inject(intermetiateStorage.get()); Id storedId = storage.getIdForNodeWithNameHierarchy(a); @@ -58,11 +58,11 @@ public: TestStorage storage; std::shared_ptr intermetiateStorage = std::make_shared(); - Id aId = intermetiateStorage->addNode(Node::typeToInt(Node::NODE_STRUCT), a, true); - Id bId = intermetiateStorage->addNode(Node::typeToInt(Node::NODE_FIELD), b, true); + Id aId = intermetiateStorage->addNode(Node::typeToInt(Node::NODE_STRUCT), NameHierarchy::serialize(a), true); + Id bId = intermetiateStorage->addNode(Node::typeToInt(Node::NODE_FIELD), NameHierarchy::serialize(b), true); intermetiateStorage->addEdge(Edge::typeToInt(Edge::EDGE_MEMBER), aId, bId); - storage.injectData(intermetiateStorage); + storage.inject(intermetiateStorage.get()); TS_ASSERT(storage.getIdForEdge(Edge::EDGE_MEMBER, a, b) != 0); } @@ -224,11 +224,11 @@ public: private: class TestStorage - : public Storage + : public PersistentStorage { public: TestStorage() - : Storage("data/test.sqlite") + : PersistentStorage("data/test.sqlite") { clear(); } diff --git a/src/test/helper/TestStorage.cpp b/src/test/helper/TestStorage.cpp deleted file mode 100644 index 60d133e3..00000000 --- a/src/test/helper/TestStorage.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include "TestStorage.h" - -TestStorage::TestStorage() - : Storage("data/test.sqlite") -{ -} diff --git a/src/test/helper/TestStorage.h b/src/test/helper/TestStorage.h deleted file mode 100644 index ab68d1f5..00000000 --- a/src/test/helper/TestStorage.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef TEST_STORAGE_H -#define TEST_STORAGE_H - -#include "data/Storage.h" - -class TestStorage - : public Storage -{ -public: - TestStorage(); - void parseCxxCode(std::string code); -}; - -#endif // TEST_STORAGE_H diff --git a/src/trial/data/parser/cxx/TaskParseCxx.cpp b/src/trial/data/parser/cxx/TaskParseCxx.cpp index 23605717..42f591dc 100644 --- a/src/trial/data/parser/cxx/TaskParseCxx.cpp +++ b/src/trial/data/parser/cxx/TaskParseCxx.cpp @@ -1,10 +1,10 @@ #include "data/parser/cxx/TaskParseCxx.h" -#include "data/Storage.h" +#include "data/PersistentStorage.h" #include "utility/messaging/type/MessageFinishedParsing.h" TaskParseCxx::TaskParseCxx( - Storage* storage, + PersistentStorage* storage, const FileManager* fileManager, const Parser::Arguments& arguments, const std::vector& files