From b9d1f5367c3b9be6a7bb9b6f36a367544dc22a34 Mon Sep 17 00:00:00 2001 From: malte_langkabel Date: Tue, 11 Apr 2017 18:56:15 +0200 Subject: [PATCH] logic: skip saving duplicate data * check for duplicates in intermediate storage * added unique constraints to sqlite tables --- src/lib/data/IntermediateStorage.cpp | 130 +++++++++++++++++++++------ src/lib/data/IntermediateStorage.h | 18 +++- src/lib/data/SqliteStorage.cpp | 8 +- 3 files changed, 127 insertions(+), 29 deletions(-) diff --git a/src/lib/data/IntermediateStorage.cpp b/src/lib/data/IntermediateStorage.cpp index b57e6738..74584506 100644 --- a/src/lib/data/IntermediateStorage.cpp +++ b/src/lib/data/IntermediateStorage.cpp @@ -40,7 +40,7 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName) { std::shared_ptr node = std::make_shared(0, type, serializedName); - std::string serialized = serialize(*(node.get())); + const std::string serialized = serialize(*(node.get())); std::unordered_map::const_iterator it = m_nodeNamesToIds.find(serialized); if (it != m_nodeNamesToIds.end()) { @@ -62,7 +62,15 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName) void IntermediateStorage::addFile(const Id id, const std::string& filePath, const std::string& modificationTime) { - m_files.push_back(StorageFile(id, filePath, modificationTime)); + const StorageFile file(id, filePath, modificationTime); + const std::string serialized = serialize(file); + + if (m_serializedFiles.find(serialized) == m_serializedFiles.end()) + { + m_files.push_back(file); + m_serializedFiles.insert(serialized); + + } } void IntermediateStorage::addSymbol(const Id id, int definitionKind) @@ -74,7 +82,7 @@ Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId) { std::shared_ptr edge = std::make_shared(0, type, sourceId, targetId); - std::string serialized = serialize(*(edge.get())); + const std::string serialized = serialize(*(edge.get())); std::unordered_map::const_iterator it = m_edgeNamesToIds.find(serialized); if (it != m_edgeNamesToIds.end()) { @@ -92,7 +100,7 @@ Id IntermediateStorage::addLocalSymbol(const std::string& name) { std::shared_ptr localSymbol = std::make_shared(0, name); - std::string serialized = serialize(*(localSymbol.get())); + const std::string serialized = serialize(*(localSymbol.get())); std::unordered_map::const_iterator it = m_localSymbolNamesToIds.find(serialized); if (it != m_localSymbolNamesToIds.end()) { @@ -118,7 +126,7 @@ Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint st type ); - std::string serialized = serialize(*(sourceLocation.get())); + const std::string serialized = serialize(*(sourceLocation.get())); std::unordered_map::const_iterator it = m_sourceLocationNamesToIds.find(serialized); if (it != m_sourceLocationNamesToIds.end()) { @@ -134,29 +142,51 @@ Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint st void IntermediateStorage::addOccurrence(Id elementId, Id sourceLocationId) { - m_occurrences.push_back(StorageOccurrence(elementId, sourceLocationId)); + const StorageOccurrence occurrence(elementId, sourceLocationId); + const std::string serialized = serialize(occurrence); + + if (m_serializedOccurrences.find(serialized) == m_serializedOccurrences.end()) + { + m_occurrences.push_back(occurrence); + m_serializedOccurrences.insert(serialized); + } } void IntermediateStorage::addComponentAccess(Id nodeId, int type) { - m_componentAccesses.push_back(StorageComponentAccess(nodeId, type)); + const StorageComponentAccess componentAccess(nodeId, type); + const std::string serialized = serialize(componentAccess); + + if (m_serializedComponentAccesses.find(serialized) == m_serializedComponentAccesses.end()) + { + m_componentAccesses.push_back(componentAccess); + m_serializedComponentAccesses.insert(serialized); + } } void IntermediateStorage::addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) { - m_commentLocations.push_back(StorageCommentLocation( + const StorageCommentLocation commentLocation( 0, fileNodeId, startLine, startCol, endLine, endCol - )); + ); + const std::string serialized = serialize(commentLocation); + + if (m_serializedCommentLocations.find(serialized) == m_serializedCommentLocations.end()) + { + m_commentLocations.push_back(commentLocation); + m_serializedCommentLocations.insert(serialized); + + } } void IntermediateStorage::addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) { - m_errors.push_back(StorageError( + const StorageError error( 0, message, filePath, @@ -164,7 +194,14 @@ void IntermediateStorage::addError(const std::string& message, const FilePath& f startCol, fatal, indexed - )); + ); + const std::string serialized = serialize(error); + + if (m_serializedErrors.find(serialized) == m_serializedErrors.end()) + { + m_errors.push_back(error); + m_serializedErrors.insert(serialized); + } } void IntermediateStorage::forEachNode(std::function callback) const @@ -247,20 +284,25 @@ void IntermediateStorage::forEachError(std::function #include #include +#include #include "data/StorageTypes.h" #include "data/Storage.h" @@ -40,30 +41,43 @@ public: virtual void forEachError(std::function callback) const; private: - std::string serialize(const StorageEdge& edge) const; std::string serialize(const StorageNode& node) const; + std::string serialize(const StorageFile& file) const; + std::string serialize(const StorageEdge& edge) const; std::string serialize(const StorageLocalSymbol& localSymbol) const; std::string serialize(const StorageSourceLocation& sourceLocation) const; + std::string serialize(const StorageOccurrence& occurrence) const; + std::string serialize(const StorageComponentAccess& componentAccess) const; + std::string serialize(const StorageCommentLocation& commentLocation) const; + std::string serialize(const StorageError& error) const; std::unordered_map m_nodeNamesToIds; // this is used to prevent duplicates (unique) std::map> m_nodeIdsToData; + std::unordered_set m_serializedFiles; // this is used to prevent duplicates (unique) std::vector m_files; + std::vector m_symbols; std::unordered_map m_edgeNamesToIds; // this is used to prevent duplicates (unique) std::map> m_edgeIdsToData; - std::unordered_map m_localSymbolNamesToIds; // this is used to prevent duplicates (unique) std::map> m_localSymbolIdsToData; std::unordered_map m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique) std::map> m_sourceLocationIdsToData; + std::unordered_set m_serializedOccurrences; // this is used to prevent duplicates (unique) std::vector m_occurrences; + + std::unordered_set m_serializedComponentAccesses; // this is used to prevent duplicates (unique) std::vector m_componentAccesses; + + std::unordered_set m_serializedCommentLocations; // this is used to prevent duplicates (unique) std::vector m_commentLocations; + + std::unordered_set m_serializedErrors; // this is used to prevent duplicates (unique) std::vector m_errors; Id m_nextId; diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index 35935429..a6b434e7 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -8,7 +8,7 @@ #include "utility/text/TextAccess.h" #include "utility/Version.h" -const size_t SqliteStorage::STORAGE_VERSION = 9; +const size_t SqliteStorage::STORAGE_VERSION = 10; SqliteStorage::SqliteStorage(const FilePath& dbFilePath) : m_dbFilePath(dbFilePath.canonical()) @@ -1287,6 +1287,7 @@ void SqliteStorage::setupTables() "path TEXT, " "modification_time TEXT, " "line_count INTEGER, " + "UNIQUE(path) ON CONFLICT REPLACE," "PRIMARY KEY(id), " "FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);" ); @@ -1323,7 +1324,7 @@ void SqliteStorage::setupTables() ); m_database.execDML( - "CREATE TABLE IF NOT EXISTS occurrence(" // TODO: properly delete this on refresh + "CREATE TABLE IF NOT EXISTS occurrence(" "element_id INTEGER NOT NULL, " "source_location_id INTEGER NOT NULL, " "PRIMARY KEY(element_id, source_location_id), " @@ -1336,6 +1337,7 @@ void SqliteStorage::setupTables() "id INTEGER NOT NULL, " "node_id INTEGER, " "type INTEGER NOT NULL, " + "UNIQUE(node_id) ON CONFLICT REPLACE," "PRIMARY KEY(id), " "FOREIGN KEY(node_id) REFERENCES node(id) ON DELETE CASCADE);" ); @@ -1348,6 +1350,7 @@ void SqliteStorage::setupTables() "start_column INTEGER, " "end_line INTEGER, " "end_column INTEGER, " + "UNIQUE(file_node_id, start_line, start_column, end_line, end_column) ON CONFLICT REPLACE," "PRIMARY KEY(id), " "FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);" ); @@ -1361,6 +1364,7 @@ void SqliteStorage::setupTables() "file_path TEXT, " "line_number INTEGER, " "column_number INTEGER, " + "UNIQUE(message, fatal, file_path, line_number, column_number) ON CONFLICT REPLACE," "PRIMARY KEY(id));" );