From 8d3112d5a94acd10350ba4d3eb9a0993da94a6e9 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Mon, 5 Oct 2015 09:44:27 +0200 Subject: [PATCH] data: increased file clearing performance This change passes all file node ids for clearing to the sqlite storage at once and makes use of ON DELETE CASCADE for everything else, instead of manally deleting all source locations. --- src/lib/Project.cpp | 6 ++--- src/lib/data/SqliteStorage.cpp | 14 +++++------ src/lib/data/SqliteStorage.h | 2 +- src/lib/data/Storage.cpp | 17 +++++++------ src/lib/data/Storage.h | 2 +- src/lib/data/TaskCleanStorage.cpp | 42 +++++++++++++------------------ src/lib/data/TaskCleanStorage.h | 7 +++--- 7 files changed, 43 insertions(+), 47 deletions(-) diff --git a/src/lib/Project.cpp b/src/lib/Project.cpp index 3fd33a54..3b7e33d9 100644 --- a/src/lib/Project.cpp +++ b/src/lib/Project.cpp @@ -105,9 +105,9 @@ void Project::parseCode() std::shared_ptr taskGroup = std::make_shared(); - std::set filesToClean; - utility::append(filesToClean, removedFilePaths); - utility::append(filesToClean, updatedFilePaths); + std::vector filesToClean; + filesToClean.insert(filesToClean.end(), removedFilePaths.begin(), removedFilePaths.end()); + filesToClean.insert(filesToClean.end(), updatedFilePaths.begin(), updatedFilePaths.end()); taskGroup->addTask(std::make_shared(m_storage.get(), filesToClean)); diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index 4c08cf90..77f5c60c 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -160,13 +160,6 @@ void SqliteStorage::removeNameHierarchyElement(Id id) ).c_str()); } -void SqliteStorage::removeElementsWithLocationInFile(Id fileId) -{ - m_database.execDML(( - "DELETE FROM element WHERE id IN (SELECT element_id FROM source_location WHERE source_location.file_node_id == " + std::to_string(fileId) + ");" - ).c_str()); -} - void SqliteStorage::removeFile(Id id) { if (isFile(id)) @@ -181,6 +174,13 @@ void SqliteStorage::removeFile(Id id) } } +void SqliteStorage::removeFiles(const std::vector& fileIds) +{ + m_database.execDML(( + "DELETE FROM element WHERE id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");" + ).c_str()); +} + void SqliteStorage::removeUnusedNameHierarchyElements() { m_database.execDML( diff --git a/src/lib/data/SqliteStorage.h b/src/lib/data/SqliteStorage.h index 5589cf2e..10b0dbd2 100644 --- a/src/lib/data/SqliteStorage.h +++ b/src/lib/data/SqliteStorage.h @@ -40,8 +40,8 @@ public: void removeElement(Id id); void removeNameHierarchyElement(Id id); - void removeElementsWithLocationInFile(Id fileId); void removeFile(Id id); + void removeFiles(const std::vector& fileIds); void removeUnusedNameHierarchyElements(); StorageNode getFirstNode() const; diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index 2deef207..372c3371 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -76,15 +76,18 @@ std::set Storage::getDependingFilePaths(const FilePath& filePath) return dependingFilePaths; } -void Storage::clearFileElement(const FilePath& filePath) +void Storage::clearFileElements(const std::vector& filePaths) { - Id fileId = getFileNodeId(filePath); - if (fileId != 0) + std::vector fileNodeIds; + + for (const FilePath& path : filePaths) { - m_sqliteStorage.beginTransaction(); - m_sqliteStorage.removeElementsWithLocationInFile(fileId); - m_sqliteStorage.removeFile(fileId); - m_sqliteStorage.commitTransaction(); + fileNodeIds.push_back(getFileNodeId(path)); + } + + if (fileNodeIds.size()) + { + m_sqliteStorage.removeFiles(fileNodeIds); } } diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index e7a57b9b..c74cdc0f 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -29,7 +29,7 @@ public: std::set getDependingFilePaths(const std::set& filePaths); std::set getDependingFilePaths(const FilePath& filePath); - void clearFileElement(const FilePath& filePath); + void clearFileElements(const std::vector& filePaths); void removeUnusedNames(); std::vector getInfoOnAllFiles() const; diff --git a/src/lib/data/TaskCleanStorage.cpp b/src/lib/data/TaskCleanStorage.cpp index d1c7374a..47c06201 100644 --- a/src/lib/data/TaskCleanStorage.cpp +++ b/src/lib/data/TaskCleanStorage.cpp @@ -3,14 +3,11 @@ #include "data/Storage.h" #include "utility/messaging/type/MessageStatus.h" -TaskCleanStorage::TaskCleanStorage(Storage* storage, const std::set& filePaths) +TaskCleanStorage::TaskCleanStorage(Storage* storage, const std::vector& filePaths) : m_storage(storage) + , m_filePaths(filePaths) , m_fileCount(filePaths.size()) { - for (const FilePath& p : filePaths) - { - m_filePaths.push(p); - } } void TaskCleanStorage::enter() @@ -20,30 +17,26 @@ void TaskCleanStorage::enter() Task::TaskState TaskCleanStorage::update() { - if (!m_filePaths.size()) + if (m_filePaths.size()) { - if (m_fileCount) - { - MessageStatus("Cleaning up names", false, true).dispatch(); - m_storage->removeUnusedNames(); - } + std::stringstream ss; + ss << "clearing " << m_filePaths.size() << " files (ESC to quit)"; + MessageStatus(ss.str(), false, true).dispatch(); - return Task::STATE_FINISHED; + m_storage->clearFileElements(m_filePaths); + + m_filePaths.clear(); + + return Task::STATE_RUNNING; } - FilePath filePath = m_filePaths.front(); - m_filePaths.pop(); + if (m_fileCount) + { + MessageStatus("Cleaning up names (ESC to quit)", false, true).dispatch(); + m_storage->removeUnusedNames(); + } - std::stringstream ss; - ss << "clearing (ESC to quit): ["; - ss << m_fileCount - m_filePaths.size() - 1 << "/" << m_fileCount << "] "; - ss << filePath.str(); - - MessageStatus(ss.str(), false, true).dispatch(); - - m_storage->clearFileElement(filePath); - - return Task::STATE_RUNNING; + return Task::STATE_FINISHED; } void TaskCleanStorage::exit() @@ -52,6 +45,7 @@ void TaskCleanStorage::exit() ss << "clearing files done, "; ss << std::setprecision(2) << std::fixed << utility::duration(m_start) << " seconds"; MessageStatus(ss.str()).dispatch(); + } void TaskCleanStorage::interrupt() diff --git a/src/lib/data/TaskCleanStorage.h b/src/lib/data/TaskCleanStorage.h index 3f35dfcd..1bbf092f 100644 --- a/src/lib/data/TaskCleanStorage.h +++ b/src/lib/data/TaskCleanStorage.h @@ -1,8 +1,7 @@ #ifndef TASK_CLEAN_STORAGE_H #define TASK_CLEAN_STORAGE_H -#include -#include +#include #include "utility/file/FilePath.h" #include "utility/scheduling/Task.h" @@ -16,7 +15,7 @@ class TaskCleanStorage public: TaskCleanStorage( Storage* storage, - const std::set& filePaths + const std::vector& filePaths ); virtual void enter(); @@ -28,7 +27,7 @@ public: private: Storage* m_storage; - std::queue m_filePaths; + std::vector m_filePaths; const size_t m_fileCount; utility::TimePoint m_start;