From 3dffd55f75333129107bbde2ec6be7670a68b124 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Thu, 29 Sep 2016 14:58:45 +0200 Subject: [PATCH] logic: Fixed refreshing when source paths were removed * Define exit state for TaskRepeatWhileSuccess * Added TaskFinishParsing to end of parsing task sequence * Also measure time spend clearing files in total index time --- src/lib/Application.cpp | 2 +- src/lib/CMakeLists.txt | 6 +- src/lib/Project.cpp | 87 ++++++++++++------- src/lib/data/PersistentStorage.cpp | 15 +--- src/lib/data/PersistentStorage.h | 3 +- src/lib/data/TaskCleanStorage.cpp | 6 ++ src/lib/data/TaskCleanStorage.h | 3 + src/lib/data/TaskFinishParsing.cpp | 74 ++++++++++++++++ src/lib/data/TaskFinishParsing.h | 36 ++++++++ src/lib/data/parser/TaskParseWrapper.cpp | 27 +----- src/lib/data/parser/TaskParseWrapper.h | 3 - .../utility/scheduling/TaskGroupParallel.cpp | 1 - .../scheduling/TaskRepeatWhileSuccess.cpp | 9 +- .../scheduling/TaskRepeatWhileSuccess.h | 3 +- src/lib_gui/qt/window/QtIndexingDialog.cpp | 7 +- 15 files changed, 202 insertions(+), 80 deletions(-) create mode 100644 src/lib/data/TaskFinishParsing.cpp create mode 100644 src/lib/data/TaskFinishParsing.h diff --git a/src/lib/Application.cpp b/src/lib/Application.cpp index 8a6304dd..e842e4b0 100644 --- a/src/lib/Application.cpp +++ b/src/lib/Application.cpp @@ -233,7 +233,7 @@ void Application::handleMessage(MessageLoadProject* message) if (message->forceRefresh) { m_project->setStateSettingsUpdated(); - m_project->refresh(false); + refreshProject(false); } return; diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 2c3698b6..ff58492f 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -186,6 +186,8 @@ add_files( data/StorageStats.h data/TaskCleanStorage.cpp data/TaskCleanStorage.h + data/TaskFinishParsing.cpp + data/TaskFinishParsing.h data/TaskInjectStorage.cpp data/TaskInjectStorage.h @@ -205,10 +207,10 @@ add_files( settings/Settings.h settings/SettingsMigrator.cpp settings/SettingsMigrator.h - + utility/commandline/CommandLineParser.cpp utility/commandline/CommandLineParser.h - + utility/file/FileInfo.cpp utility/file/FileInfo.h utility/file/FileManager.cpp diff --git a/src/lib/Project.cpp b/src/lib/Project.cpp index 48de6903..3314d973 100644 --- a/src/lib/Project.cpp +++ b/src/lib/Project.cpp @@ -7,11 +7,13 @@ #include "data/StorageProvider.h" #include "data/PersistentStorage.h" #include "data/TaskCleanStorage.h" +#include "data/TaskFinishParsing.h" #include "data/TaskInjectStorage.h" #include "settings/ApplicationSettings.h" #include "settings/ProjectSettings.h" #include "utility/file/FileRegister.h" +#include "utility/messaging/type/MessageClearErrorCount.h" #include "utility/messaging/type/MessageFinishedParsing.h" #include "utility/messaging/type/MessageRefresh.h" #include "utility/messaging/type/MessageStatus.h" @@ -225,7 +227,7 @@ void Project::load() if (canLoad) { - m_storage->finishParsing(); + m_storage->buildCaches(); m_storageAccessProxy->setSubject(m_storage.get()); MessageFinishedParsing().dispatch(); @@ -257,26 +259,41 @@ bool Project::buildIndex(bool forceRefresh) std::set updatedFilePaths = m_fileManager.getUpdatedFilePaths(); std::set removedFilePaths = m_fileManager.getRemovedFilePaths(); + std::set filesToClean; + std::set filesToParse; + if (!forceRefresh) { - utility::append(updatedFilePaths, m_storage->getDependingFilePaths(updatedFilePaths)); - utility::append(updatedFilePaths, m_storage->getDependingFilePaths(removedFilePaths)); + std::set dependingFilePaths; + utility::append(dependingFilePaths, m_storage->getDependingFilePaths(updatedFilePaths)); + utility::append(dependingFilePaths, m_storage->getDependingFilePaths(removedFilePaths)); + + for (const FilePath& path : dependingFilePaths) + { + if (removedFilePaths.find(path) == removedFilePaths.end()) + { + updatedFilePaths.insert(path); + } + } + + utility::append(filesToClean, dependingFilePaths); } - std::vector filesToClean; - filesToClean.insert(filesToClean.end(), removedFilePaths.begin(), removedFilePaths.end()); - filesToClean.insert(filesToClean.end(), updatedFilePaths.begin(), updatedFilePaths.end()); - - std::vector filesToParse; - filesToParse.insert(filesToParse.end(), addedFilePaths.begin(), addedFilePaths.end()); - filesToParse.insert(filesToParse.end(), updatedFilePaths.begin(), updatedFilePaths.end()); + utility::append(filesToClean, removedFilePaths); + utility::append(filesToClean, updatedFilePaths); + utility::append(filesToParse, addedFilePaths); + utility::append(filesToParse, updatedFilePaths); if (!filesToClean.size() && !filesToParse.size()) { MessageStatus("Nothing to refresh, all files are up-to-date.").dispatch(); return false; } + else + { + MessageClearErrorCount().dispatch(); + } if (Application::getInstance()->hasGUI()) { @@ -299,37 +316,45 @@ bool Project::buildIndex(bool forceRefresh) if (filesToClean.size()) { - taskSequential->addTask(std::make_shared(m_storage.get(), filesToClean, m_dialogView)); + taskSequential->addTask(std::make_shared( + m_storage.get(), + utility::toVector(filesToClean), + m_dialogView) + ); } const int indexerThreadCount = ApplicationSettings::getInstance()->getIndexerThreadCount(); std::shared_ptr fileRegister = std::make_shared(&m_fileManager, indexerThreadCount > 1); - fileRegister->setFilePaths(filesToParse); - std::shared_ptr taskParserWrapper = std::make_shared( - m_storage.get(), - fileRegister, - m_dialogView - ); - taskSequential->addTask(taskParserWrapper); - - std::shared_ptr taskParallelIndexing = std::make_shared(); - taskParserWrapper->setTask(taskParallelIndexing); - - std::shared_ptr storageProvider = std::make_shared(); - - for (int i = 0; i < indexerThreadCount; i++) + if (filesToParse.size()) { - std::shared_ptr taskRepeat = std::make_shared(); + fileRegister->setFilePaths(utility::toVector(filesToParse)); + + std::shared_ptr taskParserWrapper = std::make_shared( + fileRegister, + m_dialogView + ); + taskSequential->addTask(taskParserWrapper); + + std::shared_ptr taskParallelIndexing = std::make_shared(); + taskParserWrapper->setTask(taskParallelIndexing); + + std::shared_ptr storageProvider = std::make_shared(); + + for (int i = 0; i < indexerThreadCount; i++) + { + std::shared_ptr taskRepeat = std::make_shared(Task::STATE_SUCCESS); + taskParallelIndexing->addTask(taskRepeat); + taskRepeat->setTask(createIndexerTask(storageProvider, fileRegister)); + } + + std::shared_ptr taskRepeat = std::make_shared(Task::STATE_SUCCESS); taskParallelIndexing->addTask(taskRepeat); - taskRepeat->setTask(createIndexerTask(storageProvider, fileRegister)); + taskRepeat->setTask(std::make_shared(storageProvider, m_storage)); } - std::shared_ptr taskRepeat = std::make_shared(); - taskParallelIndexing->addTask(taskRepeat); - taskRepeat->setTask(std::make_shared(storageProvider, m_storage)); - + taskSequential->addTask(std::make_shared(m_storage.get(), fileRegister, m_dialogView)); Task::dispatch(taskSequential); diff --git a/src/lib/data/PersistentStorage.cpp b/src/lib/data/PersistentStorage.cpp index 028618a7..521fefa9 100644 --- a/src/lib/data/PersistentStorage.cpp +++ b/src/lib/data/PersistentStorage.cpp @@ -6,7 +6,6 @@ #include "utility/Cache.h" #include "utility/file/FileSystem.h" #include "utility/logging/logging.h" -#include "utility/messaging/type/MessageClearErrorCount.h" #include "utility/messaging/type/MessageShowErrors.h" #include "utility/messaging/type/MessageStatus.h" #include "utility/text/TextAccess.h" @@ -364,19 +363,12 @@ void PersistentStorage::logStats() const LOG_INFO(ss.str()); } -void PersistentStorage::startParsing() -{ - clearCaches(); - - MessageClearErrorCount().dispatch(); - - m_sqliteStorage.setVersion(); -} - -void PersistentStorage::finishParsing() +void PersistentStorage::buildCaches() { TRACE(); + clearCaches(); + buildSearchIndex(); buildFilePathMaps(); buildHierarchyCache(); @@ -387,6 +379,7 @@ void PersistentStorage::optimizeMemory() TRACE(); m_sqliteStorage.optimizeMemory(); + m_sqliteStorage.setVersion(); } Id PersistentStorage::getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const diff --git a/src/lib/data/PersistentStorage.h b/src/lib/data/PersistentStorage.h index b0906e8b..adbc0bfe 100644 --- a/src/lib/data/PersistentStorage.h +++ b/src/lib/data/PersistentStorage.h @@ -67,8 +67,7 @@ public: void logStats() const; - void startParsing(); - void finishParsing(); + void buildCaches(); void optimizeMemory(); diff --git a/src/lib/data/TaskCleanStorage.cpp b/src/lib/data/TaskCleanStorage.cpp index a292ac07..a591d5f3 100644 --- a/src/lib/data/TaskCleanStorage.cpp +++ b/src/lib/data/TaskCleanStorage.cpp @@ -2,6 +2,8 @@ #include "component/view/DialogView.h" #include "data/PersistentStorage.h" +#include "utility/scheduling/Blackboard.h" +#include "utility/utility.h" TaskCleanStorage::TaskCleanStorage( PersistentStorage* storage, const std::vector& filePaths, DialogView* dialogView @@ -15,6 +17,8 @@ TaskCleanStorage::TaskCleanStorage( void TaskCleanStorage::doEnter(std::shared_ptr blackboard) { m_dialogView->showProgressDialog("Clearing Files", std::to_string(m_filePaths.size()) + " Files"); + + m_start = utility::durationStart(); } Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr blackboard) @@ -28,6 +32,8 @@ Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr blackboar void TaskCleanStorage::doExit(std::shared_ptr blackboard) { + blackboard->set("clear_time", utility::duration(m_start)); + m_dialogView->hideProgressDialog(); } diff --git a/src/lib/data/TaskCleanStorage.h b/src/lib/data/TaskCleanStorage.h index acd57801..735db161 100644 --- a/src/lib/data/TaskCleanStorage.h +++ b/src/lib/data/TaskCleanStorage.h @@ -5,6 +5,7 @@ #include "utility/file/FilePath.h" #include "utility/scheduling/Task.h" +#include "utility/TimePoint.h" class DialogView; class PersistentStorage; @@ -28,6 +29,8 @@ private: PersistentStorage* m_storage; std::vector m_filePaths; DialogView* m_dialogView; + + TimePoint m_start; }; #endif // TASK_CLEAN_STORAGE_H diff --git a/src/lib/data/TaskFinishParsing.cpp b/src/lib/data/TaskFinishParsing.cpp new file mode 100644 index 00000000..dce1cf68 --- /dev/null +++ b/src/lib/data/TaskFinishParsing.cpp @@ -0,0 +1,74 @@ +#include "data/TaskFinishParsing.h" + +#include "component/view/DialogView.h" +#include "data/PersistentStorage.h" +#include "utility/file/FileRegister.h" +#include "utility/messaging/type/MessageFinishedParsing.h" +#include "utility/scheduling/Blackboard.h" +#include "utility/utility.h" + +TaskFinishParsing::TaskFinishParsing( + PersistentStorage* storage, + std::shared_ptr fileRegister, + DialogView* dialogView +) + : m_storage(storage) + , m_fileRegister(fileRegister) + , m_dialogView(dialogView) +{ +} + +TaskFinishParsing::~TaskFinishParsing() +{ +} + +void TaskFinishParsing::doEnter(std::shared_ptr blackboard) +{ +} + +Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr blackboard) +{ + TimePoint start = utility::durationStart(); + + m_dialogView->showProgressDialog("Finish Indexing", "Optimizing database"); + m_storage->optimizeMemory(); + + m_dialogView->showProgressDialog("Finish Indexing", "Building caches"); + m_storage->buildCaches(); + + m_dialogView->hideProgressDialog(); + MessageFinishedParsing().dispatch(); + + float time = utility::duration(start); + + if (blackboard->exists("clear_time")) + { + float clearTime = 0; + blackboard->get("clear_time", clearTime); + time += clearTime; + } + + if (blackboard->exists("index_time")) + { + float indexTime = 0; + blackboard->get("index_time", indexTime); + time += indexTime; + } + + m_dialogView->finishedIndexingDialog( + m_fileRegister->getParsedSourceFilesCount(), + m_fileRegister->getSourceFilesCount(), + time, + m_storage->getErrorCount() + ); + + return STATE_SUCCESS; +} + +void TaskFinishParsing::doExit(std::shared_ptr blackboard) +{ +} + +void TaskFinishParsing::doReset(std::shared_ptr blackboard) +{ +} diff --git a/src/lib/data/TaskFinishParsing.h b/src/lib/data/TaskFinishParsing.h new file mode 100644 index 00000000..87db5a3e --- /dev/null +++ b/src/lib/data/TaskFinishParsing.h @@ -0,0 +1,36 @@ +#ifndef TASK_FINISH_PARSING_H +#define TASK_FINISH_PARSING_H + +#include + +#include "utility/file/FilePath.h" +#include "utility/scheduling/Task.h" + +class DialogView; +class FileRegister; +class PersistentStorage; + +class TaskFinishParsing + : public Task +{ +public: + TaskFinishParsing( + PersistentStorage* storage, + std::shared_ptr fileRegister, + DialogView* dialogView + ); + + virtual ~TaskFinishParsing(); + +private: + virtual void doEnter(std::shared_ptr blackboard); + virtual TaskState doUpdate(std::shared_ptr blackboard); + virtual void doExit(std::shared_ptr blackboard); + virtual void doReset(std::shared_ptr blackboard); + + PersistentStorage* m_storage; + std::shared_ptr m_fileRegister; + DialogView* m_dialogView; +}; + +#endif // TASK_FINISH_PARSING_H diff --git a/src/lib/data/parser/TaskParseWrapper.cpp b/src/lib/data/parser/TaskParseWrapper.cpp index fd07b16a..9799ae32 100644 --- a/src/lib/data/parser/TaskParseWrapper.cpp +++ b/src/lib/data/parser/TaskParseWrapper.cpp @@ -1,19 +1,15 @@ #include "data/parser/TaskParseWrapper.h" #include "component/view/DialogView.h" -#include "data/PersistentStorage.h" #include "utility/file/FileRegister.h" -#include "utility/messaging/type/MessageFinishedParsing.h" #include "utility/scheduling/Blackboard.h" #include "utility/utility.h" TaskParseWrapper::TaskParseWrapper( - PersistentStorage* storage, std::shared_ptr fileRegister, DialogView* dialogView ) - : m_storage(storage) - , m_fileRegister(fileRegister) + : m_fileRegister(fileRegister) , m_dialogView(dialogView) { } @@ -36,7 +32,6 @@ void TaskParseWrapper::doEnter(std::shared_ptr blackboard) m_dialogView->updateIndexingDialog(0, m_fileRegister->getSourceFilesCount(), ""); m_start = utility::durationStart(); - m_storage->startParsing(); } Task::TaskState TaskParseWrapper::doUpdate(std::shared_ptr blackboard) @@ -47,25 +42,7 @@ Task::TaskState TaskParseWrapper::doUpdate(std::shared_ptr blackboar void TaskParseWrapper::doExit(std::shared_ptr blackboard) { blackboard->clear("indexer_count"); - - m_dialogView->showProgressDialog("Finish Indexing", "Optimizing database"); - - m_storage->optimizeMemory(); - - m_dialogView->showProgressDialog("Finish Indexing", "Building caches"); - - m_storage->finishParsing(); - - m_dialogView->hideProgressDialog(); - - MessageFinishedParsing().dispatch(); - - m_dialogView->finishedIndexingDialog( - m_fileRegister->getParsedSourceFilesCount(), - m_fileRegister->getSourceFilesCount(), - utility::duration(m_start), - m_storage->getErrorCount() - ); + blackboard->set("index_time", utility::duration(m_start)); } void TaskParseWrapper::doReset(std::shared_ptr blackboard) diff --git a/src/lib/data/parser/TaskParseWrapper.h b/src/lib/data/parser/TaskParseWrapper.h index 645cfa0a..d3ba8ae3 100644 --- a/src/lib/data/parser/TaskParseWrapper.h +++ b/src/lib/data/parser/TaskParseWrapper.h @@ -12,14 +12,12 @@ class DialogView; class FileRegister; -class PersistentStorage; class TaskParseWrapper : public TaskDecorator { public: TaskParseWrapper( - PersistentStorage* storage, std::shared_ptr fileRegister, DialogView* dialogView ); @@ -33,7 +31,6 @@ private: virtual void doExit(std::shared_ptr blackboard); virtual void doReset(std::shared_ptr blackboard); - PersistentStorage* m_storage; std::shared_ptr m_fileRegister; DialogView* m_dialogView; diff --git a/src/lib/utility/scheduling/TaskGroupParallel.cpp b/src/lib/utility/scheduling/TaskGroupParallel.cpp index fea40557..70f1632c 100644 --- a/src/lib/utility/scheduling/TaskGroupParallel.cpp +++ b/src/lib/utility/scheduling/TaskGroupParallel.cpp @@ -79,7 +79,6 @@ void TaskGroupParallel::processTaskThreaded(std::shared_ptr taskInfo, m_activeTaskCount--; }); - while (true) { TaskState state = taskInfo->taskRunner->update(blackboard); diff --git a/src/lib/utility/scheduling/TaskRepeatWhileSuccess.cpp b/src/lib/utility/scheduling/TaskRepeatWhileSuccess.cpp index 4d866e02..fa468efe 100644 --- a/src/lib/utility/scheduling/TaskRepeatWhileSuccess.cpp +++ b/src/lib/utility/scheduling/TaskRepeatWhileSuccess.cpp @@ -1,6 +1,7 @@ #include "utility/scheduling/TaskRepeatWhileSuccess.h" -TaskRepeatWhileSuccess::TaskRepeatWhileSuccess() +TaskRepeatWhileSuccess::TaskRepeatWhileSuccess(TaskState exitState) + : m_exitState(exitState) { } @@ -22,8 +23,12 @@ Task::TaskState TaskRepeatWhileSuccess::doUpdate(std::shared_ptr bla if (state == Task::STATE_SUCCESS) { - state = Task::STATE_RUNNING; m_taskRunner->reset(); + state = Task::STATE_RUNNING; + } + else if (state == Task::STATE_FAILURE) + { + state = m_exitState; } return state; diff --git a/src/lib/utility/scheduling/TaskRepeatWhileSuccess.h b/src/lib/utility/scheduling/TaskRepeatWhileSuccess.h index 8edcb934..fc66e7f3 100644 --- a/src/lib/utility/scheduling/TaskRepeatWhileSuccess.h +++ b/src/lib/utility/scheduling/TaskRepeatWhileSuccess.h @@ -10,7 +10,7 @@ class TaskRepeatWhileSuccess : public TaskDecorator { public: - TaskRepeatWhileSuccess(); + TaskRepeatWhileSuccess(TaskState exitState); virtual void setTask(std::shared_ptr task); @@ -21,6 +21,7 @@ private: virtual void doReset(std::shared_ptr blackboard); std::shared_ptr m_taskRunner; + const TaskState m_exitState; }; #endif // TASK_REPEAT_WHILE_SUCCESS_H diff --git a/src/lib_gui/qt/window/QtIndexingDialog.cpp b/src/lib_gui/qt/window/QtIndexingDialog.cpp index 4b1adf88..14c2ae87 100644 --- a/src/lib_gui/qt/window/QtIndexingDialog.cpp +++ b/src/lib_gui/qt/window/QtIndexingDialog.cpp @@ -168,7 +168,12 @@ void QtIndexingDialog::updateIndexingProgress(size_t fileCount, size_t totalFile { updateMessage(QString::number(fileCount) + "/" + QString::number(totalFileCount) + " File" + (totalFileCount > 1 ? "s" : "")); - size_t percent = fileCount * 100 / totalFileCount; + size_t percent = 0; + if (totalFileCount > 0) + { + percent = fileCount * 100 / totalFileCount; + } + m_progressBar->showProgress(percent); m_percentLabel->setText(QString::number(percent) + "% Progress"); m_sourcePath = QString::fromStdString(sourcePath);