logic: Fixed refreshing when source paths were removed

* Define exit state for TaskRepeatWhileSuccess
* Added TaskFinishParsing to end of parsing task sequence
* Also measure time spend clearing files in total index time

src/lib/Application.cpp

@@ -233,7 +233,7 @@ void Application::handleMessage(MessageLoadProject* message)
 		if (message->forceRefresh)
 		{
 			m_project->setStateSettingsUpdated();
-			m_project->refresh(false);
+			refreshProject(false);
 		}
 
 		return;

src/lib/CMakeLists.txt

@@ -186,6 +186,8 @@ add_files(
 	data/StorageStats.h
 	data/TaskCleanStorage.cpp
 	data/TaskCleanStorage.h
+	data/TaskFinishParsing.cpp
+	data/TaskFinishParsing.h
 	data/TaskInjectStorage.cpp
 	data/TaskInjectStorage.h
 
@@ -205,10 +207,10 @@ add_files(
 	settings/Settings.h
 	settings/SettingsMigrator.cpp
 	settings/SettingsMigrator.h
-	
+
 	utility/commandline/CommandLineParser.cpp
 	utility/commandline/CommandLineParser.h
-	
+
 	utility/file/FileInfo.cpp
 	utility/file/FileInfo.h
 	utility/file/FileManager.cpp

src/lib/Project.cpp

@@ -7,11 +7,13 @@
 #include "data/StorageProvider.h"
 #include "data/PersistentStorage.h"
 #include "data/TaskCleanStorage.h"
+#include "data/TaskFinishParsing.h"
 #include "data/TaskInjectStorage.h"
 #include "settings/ApplicationSettings.h"
 #include "settings/ProjectSettings.h"
 
 #include "utility/file/FileRegister.h"
+#include "utility/messaging/type/MessageClearErrorCount.h"
 #include "utility/messaging/type/MessageFinishedParsing.h"
 #include "utility/messaging/type/MessageRefresh.h"
 #include "utility/messaging/type/MessageStatus.h"
@@ -225,7 +227,7 @@ void Project::load()
 
 	if (canLoad)
 	{
-		m_storage->finishParsing();
+		m_storage->buildCaches();
 		m_storageAccessProxy->setSubject(m_storage.get());
 
 		MessageFinishedParsing().dispatch();
@@ -257,26 +259,41 @@ bool Project::buildIndex(bool forceRefresh)
 	std::set<FilePath> updatedFilePaths = m_fileManager.getUpdatedFilePaths();
 	std::set<FilePath> removedFilePaths = m_fileManager.getRemovedFilePaths();
 
+	std::set<FilePath> filesToClean;
+	std::set<FilePath> filesToParse;
+
 	if (!forceRefresh)
 	{
-		utility::append(updatedFilePaths, m_storage->getDependingFilePaths(updatedFilePaths));
+		std::set<FilePath> dependingFilePaths;
-		utility::append(updatedFilePaths, m_storage->getDependingFilePaths(removedFilePaths));
+		utility::append(dependingFilePaths, m_storage->getDependingFilePaths(updatedFilePaths));
+		utility::append(dependingFilePaths, m_storage->getDependingFilePaths(removedFilePaths));
+
+		for (const FilePath& path : dependingFilePaths)
+		{
+			if (removedFilePaths.find(path) == removedFilePaths.end())
+			{
+				updatedFilePaths.insert(path);
+			}
+		}
+
+		utility::append(filesToClean, dependingFilePaths);
 	}
 
-	std::vector<FilePath> filesToClean;
+	utility::append(filesToClean, removedFilePaths);
-	filesToClean.insert(filesToClean.end(), removedFilePaths.begin(), removedFilePaths.end());
+	utility::append(filesToClean, updatedFilePaths);
-	filesToClean.insert(filesToClean.end(), updatedFilePaths.begin(), updatedFilePaths.end());
-
-	std::vector<FilePath> filesToParse;
-	filesToParse.insert(filesToParse.end(), addedFilePaths.begin(), addedFilePaths.end());
-	filesToParse.insert(filesToParse.end(), updatedFilePaths.begin(), updatedFilePaths.end());
 
+	utility::append(filesToParse, addedFilePaths);
+	utility::append(filesToParse, updatedFilePaths);
 
 	if (!filesToClean.size() && !filesToParse.size())
 	{
 		MessageStatus("Nothing to refresh, all files are up-to-date.").dispatch();
 		return false;
 	}
+	else
+	{
+		MessageClearErrorCount().dispatch();
+	}
 
 	if (Application::getInstance()->hasGUI())
 	{
@@ -299,37 +316,45 @@ bool Project::buildIndex(bool forceRefresh)
 
 	if (filesToClean.size())
 	{
-		taskSequential->addTask(std::make_shared<TaskCleanStorage>(m_storage.get(), filesToClean, m_dialogView));
+		taskSequential->addTask(std::make_shared<TaskCleanStorage>(
+			m_storage.get(),
+			utility::toVector(filesToClean),
+			m_dialogView)
+		);
 	}
 
 	const int indexerThreadCount = ApplicationSettings::getInstance()->getIndexerThreadCount();
 
 	std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(&m_fileManager, indexerThreadCount > 1);
-	fileRegister->setFilePaths(filesToParse);
 
-	std::shared_ptr<TaskParseWrapper> taskParserWrapper = std::make_shared<TaskParseWrapper>(
+	if (filesToParse.size())
-		m_storage.get(),
-		fileRegister,
-		m_dialogView
-	);
-	taskSequential->addTask(taskParserWrapper);
-
-	std::shared_ptr<TaskGroupParallel> taskParallelIndexing = std::make_shared<TaskGroupParallel>();
-	taskParserWrapper->setTask(taskParallelIndexing);
-
-	std::shared_ptr<StorageProvider> storageProvider = std::make_shared<StorageProvider>();
-
-	for (int i = 0; i < indexerThreadCount; i++)
 	{
-		std::shared_ptr<TaskRepeatWhileSuccess> taskRepeat = std::make_shared<TaskRepeatWhileSuccess>();
+		fileRegister->setFilePaths(utility::toVector(filesToParse));
+
+		std::shared_ptr<TaskParseWrapper> taskParserWrapper = std::make_shared<TaskParseWrapper>(
+			fileRegister,
+			m_dialogView
+		);
+		taskSequential->addTask(taskParserWrapper);
+
+		std::shared_ptr<TaskGroupParallel> taskParallelIndexing = std::make_shared<TaskGroupParallel>();
+		taskParserWrapper->setTask(taskParallelIndexing);
+
+		std::shared_ptr<StorageProvider> storageProvider = std::make_shared<StorageProvider>();
+
+		for (int i = 0; i < indexerThreadCount; i++)
+		{
+			std::shared_ptr<TaskRepeatWhileSuccess> taskRepeat = std::make_shared<TaskRepeatWhileSuccess>(Task::STATE_SUCCESS);
+			taskParallelIndexing->addTask(taskRepeat);
+			taskRepeat->setTask(createIndexerTask(storageProvider, fileRegister));
+		}
+
+		std::shared_ptr<TaskRepeatWhileSuccess> taskRepeat = std::make_shared<TaskRepeatWhileSuccess>(Task::STATE_SUCCESS);
 		taskParallelIndexing->addTask(taskRepeat);
-		taskRepeat->setTask(createIndexerTask(storageProvider, fileRegister));
+		taskRepeat->setTask(std::make_shared<TaskInjectStorage>(storageProvider, m_storage));
 	}
 
-	std::shared_ptr<TaskRepeatWhileSuccess> taskRepeat = std::make_shared<TaskRepeatWhileSuccess>();
+	taskSequential->addTask(std::make_shared<TaskFinishParsing>(m_storage.get(), fileRegister, m_dialogView));
-	taskParallelIndexing->addTask(taskRepeat);
-	taskRepeat->setTask(std::make_shared<TaskInjectStorage>(storageProvider, m_storage));
-
 
 	Task::dispatch(taskSequential);
 

src/lib/data/PersistentStorage.cpp

@@ -6,7 +6,6 @@
 #include "utility/Cache.h"
 #include "utility/file/FileSystem.h"
 #include "utility/logging/logging.h"
-#include "utility/messaging/type/MessageClearErrorCount.h"
 #include "utility/messaging/type/MessageShowErrors.h"
 #include "utility/messaging/type/MessageStatus.h"
 #include "utility/text/TextAccess.h"
@@ -364,19 +363,12 @@ void PersistentStorage::logStats() const
 	LOG_INFO(ss.str());
 }
 
-void PersistentStorage::startParsing()
+void PersistentStorage::buildCaches()
-{
-	clearCaches();
-
-	MessageClearErrorCount().dispatch();
-
-	m_sqliteStorage.setVersion();
-}
-
-void PersistentStorage::finishParsing()
 {
 	TRACE();
 
+	clearCaches();
+
 	buildSearchIndex();
 	buildFilePathMaps();
 	buildHierarchyCache();
@@ -387,6 +379,7 @@ void PersistentStorage::optimizeMemory()
 	TRACE();
 
 	m_sqliteStorage.optimizeMemory();
+	m_sqliteStorage.setVersion();
 }
 
 Id PersistentStorage::getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const

src/lib/data/PersistentStorage.h

@@ -67,8 +67,7 @@ public:
 
 	void logStats() const;
 
-	void startParsing();
+	void buildCaches();
-	void finishParsing();
 
 	void optimizeMemory();
 

src/lib/data/TaskCleanStorage.cpp

@@ -2,6 +2,8 @@
 
 #include "component/view/DialogView.h"
 #include "data/PersistentStorage.h"
+#include "utility/scheduling/Blackboard.h"
+#include "utility/utility.h"
 
 TaskCleanStorage::TaskCleanStorage(
 	PersistentStorage* storage, const std::vector<FilePath>& filePaths, DialogView* dialogView
@@ -15,6 +17,8 @@ TaskCleanStorage::TaskCleanStorage(
 void TaskCleanStorage::doEnter(std::shared_ptr<Blackboard> blackboard)
 {
 	m_dialogView->showProgressDialog("Clearing Files", std::to_string(m_filePaths.size()) + " Files");
+
+	m_start = utility::durationStart();
 }
 
 Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr<Blackboard> blackboard)
@@ -28,6 +32,8 @@ Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr<Blackboard> blackboar
 
 void TaskCleanStorage::doExit(std::shared_ptr<Blackboard> blackboard)
 {
+	blackboard->set("clear_time", utility::duration(m_start));
+
 	m_dialogView->hideProgressDialog();
 }
 

src/lib/data/TaskCleanStorage.h

@@ -5,6 +5,7 @@
 
 #include "utility/file/FilePath.h"
 #include "utility/scheduling/Task.h"
+#include "utility/TimePoint.h"
 
 class DialogView;
 class PersistentStorage;
@@ -28,6 +29,8 @@ private:
 	PersistentStorage* m_storage;
 	std::vector<FilePath> m_filePaths;
 	DialogView* m_dialogView;
+
+	TimePoint m_start;
 };
 
 #endif // TASK_CLEAN_STORAGE_H

src/lib/data/TaskFinishParsing.cpp

@@ -0,0 +1,74 @@
+#include "data/TaskFinishParsing.h"
+
+#include "component/view/DialogView.h"
+#include "data/PersistentStorage.h"
+#include "utility/file/FileRegister.h"
+#include "utility/messaging/type/MessageFinishedParsing.h"
+#include "utility/scheduling/Blackboard.h"
+#include "utility/utility.h"
+
+TaskFinishParsing::TaskFinishParsing(
+	PersistentStorage* storage,
+	std::shared_ptr<FileRegister> fileRegister,
+	DialogView* dialogView
+)
+	: m_storage(storage)
+	, m_fileRegister(fileRegister)
+	, m_dialogView(dialogView)
+{
+}
+
+TaskFinishParsing::~TaskFinishParsing()
+{
+}
+
+void TaskFinishParsing::doEnter(std::shared_ptr<Blackboard> blackboard)
+{
+}
+
+Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboard)
+{
+	TimePoint start = utility::durationStart();
+
+	m_dialogView->showProgressDialog("Finish Indexing", "Optimizing database");
+	m_storage->optimizeMemory();
+
+	m_dialogView->showProgressDialog("Finish Indexing", "Building caches");
+	m_storage->buildCaches();
+
+	m_dialogView->hideProgressDialog();
+	MessageFinishedParsing().dispatch();
+
+	float time = utility::duration(start);
+
+	if (blackboard->exists("clear_time"))
+	{
+		float clearTime = 0;
+		blackboard->get("clear_time", clearTime);
+		time += clearTime;
+	}
+
+	if (blackboard->exists("index_time"))
+	{
+		float indexTime = 0;
+		blackboard->get("index_time", indexTime);
+		time += indexTime;
+	}
+
+	m_dialogView->finishedIndexingDialog(
+		m_fileRegister->getParsedSourceFilesCount(),
+		m_fileRegister->getSourceFilesCount(),
+		time,
+		m_storage->getErrorCount()
+	);
+
+	return STATE_SUCCESS;
+}
+
+void TaskFinishParsing::doExit(std::shared_ptr<Blackboard> blackboard)
+{
+}
+
+void TaskFinishParsing::doReset(std::shared_ptr<Blackboard> blackboard)
+{
+}

src/lib/data/TaskFinishParsing.h

@@ -0,0 +1,36 @@
+#ifndef TASK_FINISH_PARSING_H
+#define TASK_FINISH_PARSING_H
+
+#include <vector>
+
+#include "utility/file/FilePath.h"
+#include "utility/scheduling/Task.h"
+
+class DialogView;
+class FileRegister;
+class PersistentStorage;
+
+class TaskFinishParsing
+	: public Task
+{
+public:
+	TaskFinishParsing(
+		PersistentStorage* storage,
+		std::shared_ptr<FileRegister> fileRegister,
+		DialogView* dialogView
+	);
+
+	virtual ~TaskFinishParsing();
+
+private:
+	virtual void doEnter(std::shared_ptr<Blackboard> blackboard);
+	virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
+	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
+	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
+
+	PersistentStorage* m_storage;
+	std::shared_ptr<FileRegister> m_fileRegister;
+	DialogView* m_dialogView;
+};
+
+#endif // TASK_FINISH_PARSING_H

src/lib/data/parser/TaskParseWrapper.cpp

@@ -1,19 +1,15 @@
 #include "data/parser/TaskParseWrapper.h"
 
 #include "component/view/DialogView.h"
-#include "data/PersistentStorage.h"
 #include "utility/file/FileRegister.h"
-#include "utility/messaging/type/MessageFinishedParsing.h"
 #include "utility/scheduling/Blackboard.h"
 #include "utility/utility.h"
 
 TaskParseWrapper::TaskParseWrapper(
-	PersistentStorage* storage,
 	std::shared_ptr<FileRegister> fileRegister,
 	DialogView* dialogView
 )
-	: m_storage(storage)
+	: m_fileRegister(fileRegister)
-	, m_fileRegister(fileRegister)
 	, m_dialogView(dialogView)
 {
 }
@@ -36,7 +32,6 @@ void TaskParseWrapper::doEnter(std::shared_ptr<Blackboard> blackboard)
 	m_dialogView->updateIndexingDialog(0, m_fileRegister->getSourceFilesCount(), "");
 
 	m_start = utility::durationStart();
-	m_storage->startParsing();
 }
 
 Task::TaskState TaskParseWrapper::doUpdate(std::shared_ptr<Blackboard> blackboard)
@@ -47,25 +42,7 @@ Task::TaskState TaskParseWrapper::doUpdate(std::shared_ptr<Blackboard> blackboar
 void TaskParseWrapper::doExit(std::shared_ptr<Blackboard> blackboard)
 {
 	blackboard->clear("indexer_count");
-
+	blackboard->set("index_time", utility::duration(m_start));
-	m_dialogView->showProgressDialog("Finish Indexing", "Optimizing database");
-
-	m_storage->optimizeMemory();
-
-	m_dialogView->showProgressDialog("Finish Indexing", "Building caches");
-
-	m_storage->finishParsing();
-
-	m_dialogView->hideProgressDialog();
-
-	MessageFinishedParsing().dispatch();
-
-	m_dialogView->finishedIndexingDialog(
-		m_fileRegister->getParsedSourceFilesCount(),
-		m_fileRegister->getSourceFilesCount(),
-		utility::duration(m_start),
-		m_storage->getErrorCount()
-	);
 }
 
 void TaskParseWrapper::doReset(std::shared_ptr<Blackboard> blackboard)

src/lib/data/parser/TaskParseWrapper.h

@@ -12,14 +12,12 @@
 
 class DialogView;
 class FileRegister;
-class PersistentStorage;
 
 class TaskParseWrapper
 	: public TaskDecorator
 {
 public:
 	TaskParseWrapper(
-		PersistentStorage* storage,
 		std::shared_ptr<FileRegister> fileRegister,
 		DialogView* dialogView
 	);
@@ -33,7 +31,6 @@ private:
 	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
 	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
-	PersistentStorage* m_storage;
 	std::shared_ptr<FileRegister> m_fileRegister;
 	DialogView* m_dialogView;
 

src/lib/utility/scheduling/TaskGroupParallel.cpp

@@ -79,7 +79,6 @@ void TaskGroupParallel::processTaskThreaded(std::shared_ptr<TaskInfo> taskInfo,
 		m_activeTaskCount--;
 	});
 
-
 	while (true)
 	{
 		TaskState state = taskInfo->taskRunner->update(blackboard);

src/lib/utility/scheduling/TaskRepeatWhileSuccess.cpp

@@ -1,6 +1,7 @@
 #include "utility/scheduling/TaskRepeatWhileSuccess.h"
 
-TaskRepeatWhileSuccess::TaskRepeatWhileSuccess()
+TaskRepeatWhileSuccess::TaskRepeatWhileSuccess(TaskState exitState)
+	: m_exitState(exitState)
 {
 }
 
@@ -22,8 +23,12 @@ Task::TaskState TaskRepeatWhileSuccess::doUpdate(std::shared_ptr<Blackboard> bla
 
 	if (state == Task::STATE_SUCCESS)
 	{
-		state = Task::STATE_RUNNING;
 		m_taskRunner->reset();
+		state = Task::STATE_RUNNING;
+	}
+	else if (state == Task::STATE_FAILURE)
+	{
+		state = m_exitState;
 	}
 
 	return state;

src/lib/utility/scheduling/TaskRepeatWhileSuccess.h

@@ -10,7 +10,7 @@ class TaskRepeatWhileSuccess
 	: public TaskDecorator
 {
 public:
-	TaskRepeatWhileSuccess();
+	TaskRepeatWhileSuccess(TaskState exitState);
 
 	virtual void setTask(std::shared_ptr<Task> task);
 
@@ -21,6 +21,7 @@ private:
 	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
 	std::shared_ptr<TaskRunner> m_taskRunner;
+	const TaskState m_exitState;
 };
 
 #endif // TASK_REPEAT_WHILE_SUCCESS_H

src/lib_gui/qt/window/QtIndexingDialog.cpp

@@ -168,7 +168,12 @@ void QtIndexingDialog::updateIndexingProgress(size_t fileCount, size_t totalFile
 {
 	updateMessage(QString::number(fileCount) + "/" + QString::number(totalFileCount) + " File" + (totalFileCount > 1 ? "s" : ""));
 
-	size_t percent = fileCount * 100 / totalFileCount;
+	size_t percent = 0;
+	if (totalFileCount > 0)
+	{
+		percent = fileCount * 100 / totalFileCount;
+	}
+
 	m_progressBar->showProgress(percent);
 	m_percentLabel->setText(QString::number(percent) + "% Progress");
 	m_sourcePath = QString::fromStdString(sourcePath);