ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: improve database clearing performance

Browse Source 
* added indices that speed up clearing elements
* enclosed clearing in transaction
This commit is contained in:
malte_langkabel
2017-06-29 13:45:14 +02:00
parent db4f18c70d
commit 31b9b74ca8
2 changed files with 28 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/PersistentStorage.cpp
+2 -1
View File
@@ -510,10 +510,11 @@ void PersistentStorage::clearFileElements(const std::vector<FilePath>& filePaths
if (!fileNodeIds.empty())
{
m_sqliteIndexStorage.beginTransaction();
m_sqliteIndexStorage.removeElementsWithLocationInFiles(fileNodeIds, updateStatusCallback);
m_sqliteIndexStorage.removeElements(fileNodeIds);
m_sqliteIndexStorage.removeErrorsInFiles(filePaths);
m_sqliteIndexStorage.commitTransaction();
}
}
src/lib/data/SqliteIndexStorage.cpp
+26 -15
View File
@@ -318,7 +318,7 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector<Id>
if (updateStatusCallback != nullptr)
{
updateStatusCallback(22);
updateStatusCallback(30);
}
// delete all edges originating from element_id_to_clear
@@ -328,19 +328,19 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector<Id>
if (updateStatusCallback != nullptr)
{
updateStatusCallback(23);
updateStatusCallback(31);
}
// remove all edges from element_id_to_clear (they have been cleared by now and we can disregard them)
// remove all non existing ids from element_id_to_clear (they have been cleared by now and we can disregard them)
executeStatement(
"DELETE FROM element_id_to_clear WHERE id IN ("
" SELECT id FROM edge"
"DELETE FROM element_id_to_clear WHERE id NOT IN ("
" SELECT id FROM element"
")"
);
if (updateStatusCallback != nullptr)
{
updateStatusCallback(24);
updateStatusCallback(33);
}
// remove all files from element_id_to_clear (they will be cleared later)
@@ -352,7 +352,7 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector<Id>
if (updateStatusCallback != nullptr)
{
updateStatusCallback(25);
updateStatusCallback(34);
}
// delete source locations from fileIds (this also deletes the respective occurrences)
@@ -362,7 +362,7 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector<Id>
if (updateStatusCallback != nullptr)
{
updateStatusCallback(34);
updateStatusCallback(45);
}
// remove all ids from element_id_to_clear that still have occurrences
@@ -374,7 +374,7 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector<Id>
if (updateStatusCallback != nullptr)
{
updateStatusCallback(35);
updateStatusCallback(59);
}
// remove all ids from element_id_to_clear that still have an edge pointing to them
@@ -386,19 +386,19 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector<Id>
if (updateStatusCallback != nullptr)
{
updateStatusCallback(44);
updateStatusCallback(74);
}
// delete all elements that are still listed in element_id_to_clear
executeStatement(
"DELETE FROM element WHERE id IN ("
" SELECT id FROM element_id_to_clear"
"DELETE FROM element WHERE EXISTS ("
" SELECT * FROM element_id_to_clear WHERE element.id = element_id_to_clear.id"
")"
);
if (updateStatusCallback != nullptr)
{
updateStatusCallback(80);
updateStatusCallback(87);
}
// cleaning up
@@ -724,6 +724,14 @@ std::vector<std::pair<int, SqliteDatabaseIndex>> SqliteIndexStorage::getIndices(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("edge_multipart_index", "edge(type, source_node_id, target_node_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("edge_source_node_id_index", "edge(source_node_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("edge_target_node_id_index", "edge(target_node_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("node_serialized_name_index", "node(serialized_name)")
@@ -744,6 +752,10 @@ std::vector<std::pair<int, SqliteDatabaseIndex>> SqliteIndexStorage::getIndices(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("comment_location_all_data_index", "comment_location(file_node_id, start_line, start_column, end_line, end_column)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_CLEAR,
SqliteDatabaseIndex("comment_location_file_node_id_index", "comment_location(file_node_id)")
));
indices.push_back(std::make_pair(
STORAGE_MODE_WRITE,
SqliteDatabaseIndex("error_all_data_index", "error(message, fatal, file_path, line_number, column_number)")
@@ -845,8 +857,7 @@ void SqliteIndexStorage::setupTables()
"CREATE TABLE IF NOT EXISTS filecontent("
"id INTERGER, "
"content TEXT, "
"FOREIGN KEY(id)"
"REFERENCES file(id)"
"FOREIGN KEY(id) REFERENCES file(id)"
"ON DELETE CASCADE "
"ON UPDATE CASCADE);"
);