ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: skip saving duplicate data

Browse Source 
* check for duplicates in intermediate storage
* added unique constraints to sqlite tables
This commit is contained in:
malte_langkabel
2017-04-11 18:56:15 +02:00
parent 2a6af01ee8
commit b9d1f5367c
3 changed files with 127 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/IntermediateStorage.cpp
+105 -25
View File
@@ -40,7 +40,7 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName)
{
std::shared_ptr<StorageNode> node = std::make_shared<StorageNode>(0, type, serializedName);
std::string serialized = serialize(*(node.get()));
const std::string serialized = serialize(*(node.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_nodeNamesToIds.find(serialized);
if (it != m_nodeNamesToIds.end())
{
@@ -62,7 +62,15 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName)
void IntermediateStorage::addFile(const Id id, const std::string& filePath, const std::string& modificationTime)
{
m_files.push_back(StorageFile(id, filePath, modificationTime));
const StorageFile file(id, filePath, modificationTime);
const std::string serialized = serialize(file);
if (m_serializedFiles.find(serialized) == m_serializedFiles.end())
{
m_files.push_back(file);
m_serializedFiles.insert(serialized);
}
}
void IntermediateStorage::addSymbol(const Id id, int definitionKind)
@@ -74,7 +82,7 @@ Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId)
{
std::shared_ptr<StorageEdge> edge = std::make_shared<StorageEdge>(0, type, sourceId, targetId);
std::string serialized = serialize(*(edge.get()));
const std::string serialized = serialize(*(edge.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_edgeNamesToIds.find(serialized);
if (it != m_edgeNamesToIds.end())
{
@@ -92,7 +100,7 @@ Id IntermediateStorage::addLocalSymbol(const std::string& name)
{
std::shared_ptr<StorageLocalSymbol> localSymbol = std::make_shared<StorageLocalSymbol>(0, name);
std::string serialized = serialize(*(localSymbol.get()));
const std::string serialized = serialize(*(localSymbol.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_localSymbolNamesToIds.find(serialized);
if (it != m_localSymbolNamesToIds.end())
{
@@ -118,7 +126,7 @@ Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint st
type
);
std::string serialized = serialize(*(sourceLocation.get()));
const std::string serialized = serialize(*(sourceLocation.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_sourceLocationNamesToIds.find(serialized);
if (it != m_sourceLocationNamesToIds.end())
{
@@ -134,29 +142,51 @@ Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint st
void IntermediateStorage::addOccurrence(Id elementId, Id sourceLocationId)
{
m_occurrences.push_back(StorageOccurrence(elementId, sourceLocationId));
const StorageOccurrence occurrence(elementId, sourceLocationId);
const std::string serialized = serialize(occurrence);
if (m_serializedOccurrences.find(serialized) == m_serializedOccurrences.end())
{
m_occurrences.push_back(occurrence);
m_serializedOccurrences.insert(serialized);
}
}
void IntermediateStorage::addComponentAccess(Id nodeId, int type)
{
m_componentAccesses.push_back(StorageComponentAccess(nodeId, type));
const StorageComponentAccess componentAccess(nodeId, type);
const std::string serialized = serialize(componentAccess);
if (m_serializedComponentAccesses.find(serialized) == m_serializedComponentAccesses.end())
{
m_componentAccesses.push_back(componentAccess);
m_serializedComponentAccesses.insert(serialized);
}
}
void IntermediateStorage::addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol)
{
m_commentLocations.push_back(StorageCommentLocation(
const StorageCommentLocation commentLocation(
0,
fileNodeId,
startLine,
startCol,
endLine,
endCol
));
);
const std::string serialized = serialize(commentLocation);
if (m_serializedCommentLocations.find(serialized) == m_serializedCommentLocations.end())
{
m_commentLocations.push_back(commentLocation);
m_serializedCommentLocations.insert(serialized);
}
}
void IntermediateStorage::addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed)
{
m_errors.push_back(StorageError(
const StorageError error(
0,
message,
filePath,
@@ -164,7 +194,14 @@ void IntermediateStorage::addError(const std::string& message, const FilePath& f
startCol,
fatal,
indexed
));
);
const std::string serialized = serialize(error);
if (m_serializedErrors.find(serialized) == m_serializedErrors.end())
{
m_errors.push_back(error);
m_serializedErrors.insert(serialized);
}
}
void IntermediateStorage::forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const
@@ -247,20 +284,25 @@ void IntermediateStorage::forEachError(std::function<void(const StorageError& /*
}
}
std::string IntermediateStorage::serialize(const StorageEdge& edge) const
{
return (
std::to_string(edge.type) +
std::to_string(edge.sourceNodeId) +
std::to_string(edge.targetNodeId)
);
}
std::string IntermediateStorage::serialize(const StorageNode& node) const
{
return node.serializedName;
}
std::string IntermediateStorage::serialize(const StorageFile& file) const
{
return file.filePath;
}
std::string IntermediateStorage::serialize(const StorageEdge& edge) const
{
return (
std::to_string(edge.type) + ";" +
std::to_string(edge.sourceNodeId) + ";" +
std::to_string(edge.targetNodeId)
);
}
std::string IntermediateStorage::serialize(const StorageLocalSymbol& localSymbol) const
{
return localSymbol.name;
@@ -269,11 +311,49 @@ std::string IntermediateStorage::serialize(const StorageLocalSymbol& localSymbol
std::string IntermediateStorage::serialize(const StorageSourceLocation& sourceLocation) const
{
return (
std::to_string(sourceLocation.fileNodeId) +
std::to_string(sourceLocation.startLine) +
std::to_string(sourceLocation.startCol) +
std::to_string(sourceLocation.endLine) +
std::to_string(sourceLocation.endCol) +
std::to_string(sourceLocation.fileNodeId) + ";" +
std::to_string(sourceLocation.startLine) + ";" +
std::to_string(sourceLocation.startCol) + ";" +
std::to_string(sourceLocation.endLine) + ";" +
std::to_string(sourceLocation.endCol) + ";" +
std::to_string(sourceLocation.type)
);
}
std::string IntermediateStorage::serialize(const StorageOccurrence& occurrence) const
{
return std::to_string(occurrence.elementId) + ";" + std::to_string(occurrence.sourceLocationId);
}
std::string IntermediateStorage::serialize(const StorageComponentAccess& componentAccess) const
{
return std::to_string(componentAccess.nodeId);
}
std::string IntermediateStorage::serialize(const StorageCommentLocation& commentLocation) const
{
return (
std::to_string(commentLocation.fileNodeId) + ";" +
std::to_string(commentLocation.startLine) + ";" +
std::to_string(commentLocation.startCol) + ";" +
std::to_string(commentLocation.endLine) + ";" +
std::to_string(commentLocation.endCol)
);
}
std::string IntermediateStorage::serialize(const StorageError& error) const
{
return (
error.message + ";" +
std::to_string(error.fatal) + ";" +
error.filePath.str() + ";" +
std::to_string(error.lineNumber) + ";" +
std::to_string(error.columnNumber)
);
}
src/lib/data/IntermediateStorage.h
+16 -2
View File
@@ -4,6 +4,7 @@
#include <memory>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include "data/StorageTypes.h"
#include "data/Storage.h"
@@ -40,30 +41,43 @@ public:
virtual void forEachError(std::function<void(const StorageError& /*data*/)> callback) const;
private:
std::string serialize(const StorageEdge& edge) const;
std::string serialize(const StorageNode& node) const;
std::string serialize(const StorageFile& file) const;
std::string serialize(const StorageEdge& edge) const;
std::string serialize(const StorageLocalSymbol& localSymbol) const;
std::string serialize(const StorageSourceLocation& sourceLocation) const;
std::string serialize(const StorageOccurrence& occurrence) const;
std::string serialize(const StorageComponentAccess& componentAccess) const;
std::string serialize(const StorageCommentLocation& commentLocation) const;
std::string serialize(const StorageError& error) const;
std::unordered_map<std::string, Id> m_nodeNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageNode>> m_nodeIdsToData;
std::unordered_set<std::string> m_serializedFiles; // this is used to prevent duplicates (unique)
std::vector<StorageFile> m_files;
std::vector<StorageSymbol> m_symbols;
std::unordered_map<std::string, Id> m_edgeNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageEdge>> m_edgeIdsToData;
std::unordered_map<std::string, Id> m_localSymbolNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageLocalSymbol>> m_localSymbolIdsToData;
std::unordered_map<std::string, Id> m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageSourceLocation>> m_sourceLocationIdsToData;
std::unordered_set<std::string> m_serializedOccurrences; // this is used to prevent duplicates (unique)
std::vector<StorageOccurrence> m_occurrences;
std::unordered_set<std::string> m_serializedComponentAccesses; // this is used to prevent duplicates (unique)
std::vector<StorageComponentAccess> m_componentAccesses;
std::unordered_set<std::string> m_serializedCommentLocations; // this is used to prevent duplicates (unique)
std::vector<StorageCommentLocation> m_commentLocations;
std::unordered_set<std::string> m_serializedErrors; // this is used to prevent duplicates (unique)
std::vector<StorageError> m_errors;
Id m_nextId;
src/lib/data/SqliteStorage.cpp
+6 -2
View File
@@ -8,7 +8,7 @@
#include "utility/text/TextAccess.h"
#include "utility/Version.h"
const size_t SqliteStorage::STORAGE_VERSION = 9;
const size_t SqliteStorage::STORAGE_VERSION = 10;
SqliteStorage::SqliteStorage(const FilePath& dbFilePath)
: m_dbFilePath(dbFilePath.canonical())
@@ -1287,6 +1287,7 @@ void SqliteStorage::setupTables()
"path TEXT, "
"modification_time TEXT, "
"line_count INTEGER, "
"UNIQUE(path) ON CONFLICT REPLACE,"
"PRIMARY KEY(id), "
"FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);"
);
@@ -1323,7 +1324,7 @@ void SqliteStorage::setupTables()
);
m_database.execDML(
"CREATE TABLE IF NOT EXISTS occurrence(" // TODO: properly delete this on refresh
"CREATE TABLE IF NOT EXISTS occurrence("
"element_id INTEGER NOT NULL, "
"source_location_id INTEGER NOT NULL, "
"PRIMARY KEY(element_id, source_location_id), "
@@ -1336,6 +1337,7 @@ void SqliteStorage::setupTables()
"id INTEGER NOT NULL, "
"node_id INTEGER, "
"type INTEGER NOT NULL, "
"UNIQUE(node_id) ON CONFLICT REPLACE,"
"PRIMARY KEY(id), "
"FOREIGN KEY(node_id) REFERENCES node(id) ON DELETE CASCADE);"
);
@@ -1348,6 +1350,7 @@ void SqliteStorage::setupTables()
"start_column INTEGER, "
"end_line INTEGER, "
"end_column INTEGER, "
"UNIQUE(file_node_id, start_line, start_column, end_line, end_column) ON CONFLICT REPLACE,"
"PRIMARY KEY(id), "
"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
);
@@ -1361,6 +1364,7 @@ void SqliteStorage::setupTables()
"file_path TEXT, "
"line_number INTEGER, "
"column_number INTEGER, "
"UNIQUE(message, fatal, file_path, line_number, column_number) ON CONFLICT REPLACE,"
"PRIMARY KEY(id));"
);