logic: Added indexing mode to reindex incomplete files (issue #493, #496)

* added radio buttons to start indexing dialog
* don't block task scheduler thread while start indexing dialog is visible
* don't force whole project refresh on settings change
* added help button to start indexing dialog explaining indexing modes
* disable unavailable modes when project needs full refresh
* improved label texts on indexing dialogs
* added --incomplete flag to index command of commandline API
* fixed deletion of window stack elements

src/lib/Application.cpp

@@ -189,7 +189,8 @@ void Application::createAndLoadProject(const FilePath& projectSettingsFilePath)
 		m_storageCache->clear();
 		m_storageCache->setSubject(nullptr);
 
-		m_project = std::make_shared<Project>(std::make_shared<ProjectSettings>(projectSettingsFilePath), m_storageCache.get());
+		m_project = std::make_shared<Project>(
+			std::make_shared<ProjectSettings>(projectSettingsFilePath), m_storageCache.get(), hasGUI());
 
 		if (m_project)
 		{
@@ -220,15 +221,11 @@ void Application::createAndLoadProject(const FilePath& projectSettingsFilePath)
 	}
 }
 
-void Application::refreshProject(bool force)
+void Application::refreshProject(RefreshMode refreshMode)
 {
 	if (m_project && checkSharedMemory())
 	{
-		bool indexing = m_project->refresh(force);
-		if (indexing)
-		{
-			m_storageCache->clear();
-		}
+		m_project->refresh(getDialogView().get(), refreshMode);
 	}
 }
 
@@ -275,22 +272,20 @@ void Application::handleMessage(MessageLoadProject* message)
 
 	if (m_project && projectSettingsFilePath == m_project->getProjectSettingsFilePath())
 	{
-		if (message->forceRefresh && m_hasGUI)
+		if (message->settingsChanged && m_hasGUI)
 		{
 			m_project->setStateSettingsUpdated();
-			refreshProject(false);
+			refreshProject(REFRESH_ALL_FILES);
 		}
-		return;
 	}
-	createAndLoadProject(projectSettingsFilePath);
+	else
+	{
+		createAndLoadProject(projectSettingsFilePath);
 
-	if (message->forceRefresh)
-	{
-		refreshProject(true);
-	}
-	else if (!m_hasGUI)
-	{
-		refreshProject(false);
+		if (message->refreshMode != REFRESH_NONE)
+		{
+			refreshProject(message->refreshMode);
+		}
 	}
 }
 
@@ -311,7 +306,7 @@ void Application::handleMessage(MessageRefresh* message)
 
 	if (!message->uiOnly)
 	{
-		refreshProject(message->all);
+		refreshProject(message->all ? REFRESH_ALL_FILES : REFRESH_UPDATED_FILES);
 	}
 }
 

src/lib/Application.h

@@ -48,7 +48,7 @@ public:
 	const std::shared_ptr<Project> getCurrentProject();
 
 	void createAndLoadProject(const FilePath& projectSettingsFilePath);
-	void refreshProject(bool force);
+	void refreshProject(RefreshMode refreshMode);
 	bool hasGUI();
 
 	int handleDialog(const std::string& message);

src/lib/CMakeLists.txt

@@ -283,6 +283,7 @@ add_files(
 
 	project/Project.cpp
 	project/Project.h
+	project/RefreshInfo.h
 	project/SourceGroupFactory.cpp
 	project/SourceGroupFactory.h
 	project/SourceGroupFactoryModule.cpp

src/lib/component/view/DialogView.cpp

@@ -25,10 +25,9 @@ void DialogView::hideProgressDialog()
 {
 }
 
-DialogView::IndexingOptions DialogView::startIndexingDialog(
-	size_t cleanFileCount, size_t indexFileCount, size_t totalFileCount, DialogView::IndexingOptions options)
+void DialogView::startIndexingDialog(
+	Project* project, const std::vector<RefreshMode>& enabledModes, const RefreshInfo& info)
 {
-	return IndexingOptions();
 }
 
 void DialogView::updateIndexingDialog(

src/lib/component/view/DialogView.h

@@ -5,26 +5,14 @@
 #include <vector>
 
 #include "data/ErrorCountInfo.h"
+#include "project/RefreshInfo.h"
 
+class Project;
 class StorageAccess;
 
 class DialogView
 {
 public:
-	struct IndexingOptions
-	{
-		IndexingOptions()
-			: startIndexing(false)
-			, fullRefreshVisible(false)
-			, fullRefresh(false)
-		{}
-
-		bool startIndexing;
-
-		bool fullRefreshVisible;
-		bool fullRefresh;
-	};
-
 	DialogView(StorageAccess* storageAccess);
 	virtual ~DialogView();
 
@@ -34,8 +22,8 @@ public:
 	virtual void showProgressDialog(const std::string& title, const std::string& message, int progress);
 	virtual void hideProgressDialog();
 
-	virtual IndexingOptions startIndexingDialog(
-		size_t cleanFileCount, size_t indexFileCount, size_t totalFileCount, IndexingOptions options);
+	virtual void startIndexingDialog(
+		Project* project, const std::vector<RefreshMode>& enabledModes, const RefreshInfo& info);
 	virtual void updateIndexingDialog(
 		size_t startedFileCount, size_t finishedFileCount, size_t totalFileCount, std::string sourcePath);
 	virtual void finishedIndexingDialog(

src/lib/data/storage/PersistentStorage.cpp

@@ -345,29 +345,46 @@ void PersistentStorage::clearFileElements(const std::vector<FilePath>& filePaths
 	}
 }
 
-std::vector<FileInfo> PersistentStorage::getInfoOnAllFiles() const
+std::vector<FileInfo> PersistentStorage::getFileInfoForAllFiles() const
 {
 	TRACE();
 
 	std::vector<FileInfo> fileInfos;
-
-	std::vector<StorageFile> storageFiles = m_sqliteIndexStorage.getAll<StorageFile>();
-	for (size_t i = 0; i < storageFiles.size(); i++)
+	for (StorageFile file : m_sqliteIndexStorage.getAll<StorageFile>())
 	{
 		boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time;
-		if (storageFiles[i].modificationTime != "not-a-date-time")
+		if (file.modificationTime != "not-a-date-time")
 		{
-			modificationTime = boost::posix_time::time_from_string(storageFiles[i].modificationTime);
+			modificationTime = boost::posix_time::time_from_string(file.modificationTime);
 		}
-		fileInfos.push_back(FileInfo(
-			FilePath(storageFiles[i].filePath),
-			modificationTime
-		));
+
+		fileInfos.push_back(
+			FileInfo(
+				FilePath(file.filePath),
+				modificationTime
+			)
+		);
 	}
 
 	return fileInfos;
 }
 
+std::set<FilePath> PersistentStorage::getIncompleteFiles() const
+{
+	TRACE();
+
+	std::set<FilePath> incompleteFiles;
+	for (auto p : m_fileNodeComplete)
+	{
+		if (p.second == false)
+		{
+			incompleteFiles.insert(getFileNodePath(p.first));
+		}
+	}
+
+	return incompleteFiles;
+}
+
 void PersistentStorage::buildCaches()
 {
 	TRACE();
@@ -1712,14 +1729,7 @@ TooltipInfo PersistentStorage::getTooltipInfoForTokenIds(const std::vector<Id>&
 
 	if (type.getType() == NodeType::NODE_FILE && m_fileNodePaths.find(node.id) != m_fileNodePaths.end())
 	{
-		bool complete = false;
-		auto it = m_fileNodeComplete.find(node.id);
-		if (it != m_fileNodeComplete.end())
-		{
-			complete = it->second;
-		}
-
-		if (!complete)
+		if (!getFileNodeComplete(node.id))
 		{
 			info.title = "incomplete " + info.title;
 		}
@@ -1960,16 +1970,23 @@ FilePath PersistentStorage::getFileNodePath(Id fileId) const
 	return FilePath();
 }
 
-bool PersistentStorage::getFileNodeComplete(const FilePath& filePath) const
+bool PersistentStorage::getFilePathComplete(const FilePath& filePath) const
 {
 	auto it = m_fileNodeIds.find(filePath);
 	if (it != m_fileNodeIds.end())
 	{
-		auto it2 = m_fileNodeComplete.find(it->second);
-		if (it2 != m_fileNodeComplete.end())
-		{
-			return it2->second;
-		}
+		return getFileNodeComplete(it->second);
+	}
+
+	return false;
+}
+
+bool PersistentStorage::getFileNodeComplete(Id fileId) const
+{
+	auto it = m_fileNodeComplete.find(fileId);
+	if (it != m_fileNodeComplete.end())
+	{
+		return it->second;
 	}
 
 	return false;
@@ -2419,7 +2436,7 @@ void PersistentStorage::addCompleteFlagsToSourceLocationCollection(SourceLocatio
 	collection->forEachSourceLocationFile(
 		[this](std::shared_ptr<SourceLocationFile> file)
 		{
-			file->setIsComplete(getFileNodeComplete(file->getFilePath()));
+			file->setIsComplete(getFilePathComplete(file->getFilePath()));
 		}
 	);
 }

src/lib/data/storage/PersistentStorage.h

@@ -63,7 +63,8 @@ public:
 
 	void clearFileElements(const std::vector<FilePath>& filePaths, std::function<void(int)> updateStatusCallback);
 
-	std::vector<FileInfo> getInfoOnAllFiles() const;
+	std::vector<FileInfo> getFileInfoForAllFiles() const;
+	std::set<FilePath> getIncompleteFiles() const;
 
 	void buildCaches();
 
@@ -148,7 +149,8 @@ private:
 	std::vector<Id> getFileNodeIds(const std::vector<FilePath>& filePaths) const;
 	std::set<Id> getFileNodeIds(const std::set<FilePath>& filePaths) const;
 	FilePath getFileNodePath(Id fileId) const;
-	bool getFileNodeComplete(const FilePath& filePath) const;
+	bool getFilePathComplete(const FilePath& filePath) const;
+	bool getFileNodeComplete(Id fileId) const;
 
 	std::unordered_map<Id, std::set<Id>> getFileIdToIncludingFileIdMap() const;
 	std::unordered_map<Id, std::set<Id>> getFileIdToImportingFileIdMap() const;

src/lib/project/Project.cpp

@@ -1,19 +1,18 @@
 #include "project/Project.h"
 
-#include "Application.h"
 #include "component/view/DialogView.h"
-#include "data/access/StorageAccessProxy.h"
 #include "data/indexer/IndexerCommand.h"
 #include "data/indexer/IndexerCommandList.h"
 #include "data/indexer/TaskBuildIndex.h"
 #include "data/parser/TaskParseWrapper.h"
-#include "data/storage/StorageProvider.h"
 #include "data/storage/PersistentStorage.h"
+#include "data/storage/StorageCache.h"
+#include "data/storage/StorageProvider.h"
 #include "data/TaskCleanStorage.h"
-#include "data/TaskMergeStorages.h"
-#include "data/TaskShowStatusDialog.h"
 #include "data/TaskFinishParsing.h"
 #include "data/TaskInjectStorage.h"
+#include "data/TaskMergeStorages.h"
+#include "data/TaskShowStatusDialog.h"
 #include "project/SourceGroup.h"
 #include "project/SourceGroupFactory.h"
 #include "settings/ApplicationSettings.h"
@@ -37,10 +36,11 @@
 #include "utility/utilityApp.h"
 #include "utility/utilityString.h"
 
-Project::Project(std::shared_ptr<ProjectSettings> settings, StorageAccessProxy* storageAccessProxy)
+Project::Project(std::shared_ptr<ProjectSettings> settings, StorageCache* storageCache, bool hasGUI)
 	: m_settings(settings)
-	, m_storageAccessProxy(storageAccessProxy)
+	, m_storageCache(storageCache)
 	, m_state(PROJECT_STATE_NOT_LOADED)
+	, m_hasGUI(hasGUI)
 {
 }
 
@@ -48,123 +48,6 @@ Project::~Project()
 {
 }
 
-bool Project::refresh(bool forceRefresh)
-{
-	if (m_state == PROJECT_STATE_NOT_LOADED)
-	{
-		return false;
-	}
-
-	bool needsFullRefresh = false;
-	std::string question;
-
-	switch (m_state)
-	{
-		case PROJECT_STATE_EMPTY:
-			needsFullRefresh = true;
-			break;
-
-		case PROJECT_STATE_LOADED:
-			break;
-
-		case PROJECT_STATE_OUTDATED:
-			question =
-				"The project file was changed after the last indexing. The project needs to get fully reindexed to "
-				"reflect the current project state. Do you want to reindex the project?";
-			needsFullRefresh = true;
-			break;
-
-		case PROJECT_STATE_OUTVERSIONED:
-			question =
-				"This project was indexed with a different version of Sourcetrail. It needs to be fully reindexed to be used "
-				"with this version of Sourcetrail. Do you want to reindex the project?";
-			needsFullRefresh = true;
-			break;
-
-		case PROJECT_STATE_SETTINGS_UPDATED:
-			question =
-				"Some settings were changed, the project needs to be fully reindexed. "
-				"Do you want to reindex the project?";
-			needsFullRefresh = true;
-			break;
-
-		case PROJECT_STATE_NEEDS_MIGRATION:
-			question =
-				"This project was created with a different version of Sourcetrail. The project file needs to get updated and "
-				"the project fully reindexed. Do you want to update the project file and reindex the project?";
-			needsFullRefresh = true;
-
-		default:
-			break;
-	}
-
-	std::shared_ptr<DialogView> dialogView = Application::getInstance()->getDialogView();
-
-	if (
-		ApplicationSettings::getInstance()->getLoggingEnabled() &&
-		ApplicationSettings::getInstance()->getVerboseIndexerLoggingEnabled() &&
-		Application::getInstance()->hasGUI()
-		)
-	{
-		std::vector<std::string> options = { "Yes", "No" };
-		int result = dialogView->confirm(
-			"Warning: You are about to index your project with the \"verbose indexer logging\" setting "
-			"enabled. This will cause a significant slowdown in indexing performance. Do you want to proceed?",
-			options
-		);
-
-		if (result == 1)
-		{
-			return false;
-		}
-	}
-
-	if (!forceRefresh && needsFullRefresh && question.size() && Application::getInstance()->hasGUI())
-	{
-		std::vector<std::string> options = { "Yes", "No"};
-		int result = dialogView->confirm(question, options);
-
-		if (result == 1)
-		{
-			return false;
-		}
-	}
-
-	dialogView->showUnknownProgressDialog("Preparing Project", "Processing Files");
-
-	ScopedFunctor dialogHider([&dialogView](){
-		dialogView->hideUnknownProgressDialog();
-	});
-
-
-	if (m_state == PROJECT_STATE_NEEDS_MIGRATION)
-	{
-		m_settings->migrate();
-	}
-
-	m_settings->reload();
-
-	m_sourceGroups = SourceGroupFactory::getInstance()->createSourceGroups(m_settings->getAllSourceGroupSettings());
-	for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)
-	{
-		if (!sourceGroup->prepareRefresh())
-		{
-			return false;
-		}
-	}
-
-	if (requestIndex(forceRefresh, needsFullRefresh))
-	{
-		m_storageAccessProxy->setSubject(m_storage.get());
-
-		m_state = PROJECT_STATE_LOADED;
-
-		return true;
-	}
-
-	return false;
-}
-
 FilePath Project::getProjectSettingsFilePath() const
 {
 	return m_settings->getFilePath();
@@ -190,7 +73,7 @@ void Project::setStateSettingsUpdated()
 
 void Project::load()
 {
-	m_storageAccessProxy->setSubject(nullptr);
+	m_storageCache->setSubject(nullptr);
 
 	bool loadedSettings = m_settings->reload();
 
@@ -246,9 +129,9 @@ void Project::load()
 	{
 		m_storage->setMode(SqliteStorage::STORAGE_MODE_READ);
 		m_storage->buildCaches();
-		m_storageAccessProxy->setSubject(m_storage.get());
+		m_storageCache->setSubject(m_storage.get());
 
-		if (Application::getInstance()->hasGUI())
+		if (m_hasGUI)
 		{
 			MessageFinishedParsing().dispatch();
 		}
@@ -259,171 +142,200 @@ void Project::load()
 		MessageStatus("Project not loaded", false, false).dispatch();
 	}
 
-	if (m_state != PROJECT_STATE_LOADED && Application::getInstance()->hasGUI())
+	if (m_state != PROJECT_STATE_LOADED && m_hasGUI)
 	{
 		MessageRefresh().dispatch();
 	}
 }
 
-bool Project::requestIndex(bool forceRefresh, bool needsFullRefresh)
+void Project::refresh(DialogView* dialogView, RefreshMode refreshMode)
 {
-	std::set<FilePath> allSourceFilePaths;
-	for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)
+	if (m_state == PROJECT_STATE_NOT_LOADED)
+	{
+		return;
+	}
+
+	bool needsFullRefresh = false;
+	bool fullRefresh = false;
+	std::string question;
+
+	switch (m_state)
+	{
+		case PROJECT_STATE_EMPTY:
+			needsFullRefresh = true;
+			break;
+
+		case PROJECT_STATE_LOADED:
+			break;
+
+		case PROJECT_STATE_OUTDATED:
+			question =
+				"The project file was changed after the last indexing. The project needs to get fully reindexed to "
+				"reflect the current project state. Alternatively you can also choose to just reindex updated or "
+				"incomplete files. Do you want to reindex the project?";
+			fullRefresh = true;
+			break;
+
+		case PROJECT_STATE_OUTVERSIONED:
+			question =
+				"This project was indexed with a different version of Sourcetrail. It needs to be fully reindexed to be used "
+				"with this version of Sourcetrail. Do you want to reindex the project?";
+			needsFullRefresh = true;
+			break;
+
+		case PROJECT_STATE_SETTINGS_UPDATED:
+			question =
+				"Some settings were changed, the project should be fully reindexed. Alternatively you can also choose to "
+				"just reindex updated or incomplete files. "
+				"Do you want to reindex the project?";
+			fullRefresh = true;
+			break;
+
+		case PROJECT_STATE_NEEDS_MIGRATION:
+			question =
+				"This project was created with a different version of Sourcetrail. The project file needs to get updated and "
+				"the project fully reindexed. Do you want to update the project file and reindex the project?";
+			needsFullRefresh = true;
+
+		default:
+			break;
+	}
+
+	if (question.size() && m_hasGUI)
+	{
+		std::vector<std::string> options = { "Yes", "No" };
+		int result = dialogView->confirm(question, options);
+
+		if (result == 1)
+		{
+			return;
+		}
+	}
+
+	if (ApplicationSettings::getInstance()->getLoggingEnabled() &&
+		ApplicationSettings::getInstance()->getVerboseIndexerLoggingEnabled() && m_hasGUI)
+	{
+		std::vector<std::string> options = { "Yes", "No" };
+		int result = dialogView->confirm(
+			"Warning: You are about to index your project with the \"verbose indexer logging\" setting "
+			"enabled. This will cause a significant slowdown in indexing performance. Do you want to proceed?",
+			options
+		);
+
+		if (result == 1)
+		{
+			return;
+		}
+	}
+
+	dialogView->showUnknownProgressDialog("Preparing Project", "Processing Files");
+	ScopedFunctor dialogHider([&dialogView](){
+		dialogView->hideUnknownProgressDialog();
+	});
+
+	if (m_state == PROJECT_STATE_NEEDS_MIGRATION)
+	{
+		m_settings->migrate();
+	}
+
+	m_settings->reload();
+
+	m_sourceGroups = SourceGroupFactory::getInstance()->createSourceGroups(m_settings->getAllSourceGroupSettings());
+	for (const std::shared_ptr<SourceGroup>& sourceGroup : m_sourceGroups)
 	{
 		if (!sourceGroup->prepareIndexing())
 		{
-			return false;
+			return;
 		}
+
 		sourceGroup->fetchAllSourceFilePaths();
-		utility::append(allSourceFilePaths, sourceGroup->getAllSourceFilePaths());
 	}
 
-	std::set<FilePath> filesToClean;
-	std::set<FilePath> filesToAdd;
-	if (!needsFullRefresh)
+	if (needsFullRefresh || fullRefresh)
 	{
-		std::set<FilePath> unchangedFilePaths;
-		std::set<FilePath> changedFilePaths;
-		for (const FileInfo& info: m_storage->getInfoOnAllFiles())
-		{
-			if (info.path.exists())
-			{
-				if (didFileChange(info))
-				{
-					changedFilePaths.insert(info.path);
-				}
-				else
-				{
-					unchangedFilePaths.insert(info.path);
-				}
-			}
-			else
-			{
-				// file has been removed
-				changedFilePaths.insert(info.path);
-			}
-		}
-
-		filesToClean = changedFilePaths;
-
-		// handle referencing paths
-		utility::append(filesToClean, m_storage->getReferencing(changedFilePaths));
-
-		// handle referenced paths
-		std::set<FilePath> staticSourceFiles = allSourceFilePaths;
-		for (const FilePath& path: changedFilePaths)
-		{
-			staticSourceFiles.erase(path);
-		}
-
-		const std::set<FilePath> staticReferencedFilePaths = m_storage->getReferenced(staticSourceFiles);
-		const std::set<FilePath> dynamicReferencedFilePaths = m_storage->getReferenced(changedFilePaths);
-
-		for (const FilePath& path : dynamicReferencedFilePaths)
-		{
-			if (staticReferencedFilePaths.find(path) == staticReferencedFilePaths.end() &&
-				staticSourceFiles.find(path) == staticSourceFiles.end())
-			{
-				// file may not be referenced anymore and will be reindexed if still needed
-				filesToClean.insert(path);
-			}
-		}
-
-		for (const FilePath& path: unchangedFilePaths)
-		{
-			staticSourceFiles.erase(path);
-		}
-		filesToAdd = staticSourceFiles;
+		refreshMode = REFRESH_ALL_FILES;
+	}
+	else if (refreshMode == REFRESH_NONE)
+	{
+		refreshMode = REFRESH_UPDATED_FILES;
 	}
 
+	RefreshInfo info = getRefreshInfo(refreshMode);
 
-	std::set<FilePath> staticSourceFilePaths;
-	for (const FilePath& path: allSourceFilePaths)
+	if (m_hasGUI)
 	{
-		if (filesToClean.find(path) == filesToClean.end() && filesToAdd.find(path) == filesToAdd.end())
+		std::vector<RefreshMode> enabledModes = { REFRESH_ALL_FILES };
+		if (!needsFullRefresh)
 		{
-			staticSourceFilePaths.insert(path);
-		}
-	}
-
-	std::set<FilePath> filesToIndex;
-	for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)
-	{
-		sourceGroup->fetchSourceFilePathsToIndex(staticSourceFilePaths);
-		utility::append(filesToIndex, sourceGroup->getSourceFilePathsToIndex());
-	}
-
-	bool fullRefresh = forceRefresh | needsFullRefresh;
-
-	if (Application::getInstance()->hasGUI())
-	{
-		DialogView::IndexingOptions options;
-		options.fullRefreshVisible = !needsFullRefresh;
-		options.fullRefresh = forceRefresh;
-
-		Application::getInstance()->getDialogView()->hideUnknownProgressDialog();
-
-		options = Application::getInstance()->getDialogView()->startIndexingDialog(
-			filesToClean.size(), filesToIndex.size(), allSourceFilePaths.size(), options);
-
-		if (!options.startIndexing)
-		{
-			return false;
+			enabledModes.insert(enabledModes.end(), { REFRESH_UPDATED_FILES, REFRESH_UPDATED_AND_INCOMPLETE_FILES });
 		}
 
-		fullRefresh = options.fullRefresh | needsFullRefresh;
+		dialogView->startIndexingDialog(this, enabledModes, info);
 	}
-
-	if (fullRefresh)
+	else
 	{
-		filesToClean.clear();
-		filesToIndex = allSourceFilePaths;
+		buildIndex(info);
 	}
+}
 
-	if (!filesToClean.size() && !filesToIndex.size())
+RefreshInfo Project::getRefreshInfo(RefreshMode mode) const
+{
+	switch (mode)
 	{
-		if (!Application::getInstance()->hasGUI())
+	case REFRESH_NONE:
+		return RefreshInfo();
+
+	case REFRESH_UPDATED_FILES:
+		return getRefreshInfoForUpdatedFiles();
+
+	case REFRESH_UPDATED_AND_INCOMPLETE_FILES:
+		return getRefreshInfoForIncompleteFiles();
+
+	case REFRESH_ALL_FILES:
+		return getRefreshInfoForAllFiles();
+	}
+}
+
+void Project::buildIndex(const RefreshInfo& info)
+{
+	if (info.mode == REFRESH_NONE || (!info.filesToClear.size() && !info.filesToIndex.size()))
+	{
+		if (!m_hasGUI)
 		{
 			MessageFinishedParsing().dispatch();
 		}
 
 		MessageStatus("Nothing to refresh, all files are up-to-date.").dispatch();
-		return false;
+		return;
 	}
 
-	MessageStatus((fullRefresh ? "Reindexing Project" : "Refreshing Project"), false, true).dispatch();
-
-	buildIndex(filesToIndex, filesToClean, fullRefresh);
-
-	return true;
-}
-
-void Project::buildIndex(
-	const std::set<FilePath>& filesToIndex, const std::set<FilePath>& filesToClean, bool fullRefresh)
-{
+	MessageStatus("Preparing Indexing", false, true).dispatch();
 	MessageClearErrorCount().dispatch();
-	if (fullRefresh)
+
+	if (info.mode == REFRESH_ALL_FILES)
 	{
 		m_storage->clear();
 	}
 
+	m_storageCache->clear();
+
 	m_storage->setProjectSettingsText(TextAccess::createFromFile(getProjectSettingsFilePath())->getText());
 
 	std::shared_ptr<TaskGroupSequence> taskSequential = std::make_shared<TaskGroupSequence>();
 
-	// add task for cleaning the database
-	if (!filesToClean.empty())
+	// add task for clearing the database
+	if (info.filesToClear.size())
 	{
 		taskSequential->addTask(std::make_shared<TaskCleanStorage>(
 			m_storage.get(),
-			utility::toVector(filesToClean)
+			utility::toVector(info.filesToClear)
 		));
 	}
 
 	std::shared_ptr<IndexerCommandList> indexerCommandList = std::make_shared<IndexerCommandList>();
 	for (const std::shared_ptr<SourceGroup>& sourceGroup : m_sourceGroups)
 	{
-		for (const std::shared_ptr<IndexerCommand>& command : sourceGroup->getIndexerCommands(filesToIndex, fullRefresh))
+		for (const std::shared_ptr<IndexerCommand>& command : sourceGroup->getIndexerCommands(info.filesToIndex))
 		{
 			indexerCommandList->addCommand(command);
 		}
@@ -515,9 +427,144 @@ void Project::buildIndex(
 		);
 	}
 
-	taskSequential->addTask(std::make_shared<TaskFinishParsing>(m_storage.get(), m_storageAccessProxy));
+	taskSequential->addTask(std::make_shared<TaskFinishParsing>(m_storage.get(), m_storageCache));
 
 	Task::dispatch(taskSequential);
+
+	m_storageCache->setSubject(m_storage.get());
+	m_state = PROJECT_STATE_LOADED;
+}
+
+std::set<FilePath> Project::getAllSourceFilePaths() const
+{
+	std::set<FilePath> allSourceFilePaths;
+
+	for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)
+	{
+		utility::append(allSourceFilePaths, sourceGroup->getAllSourceFilePaths());
+	}
+
+	return allSourceFilePaths;
+}
+
+RefreshInfo Project::getRefreshInfoForUpdatedFiles() const
+{
+	std::set<FilePath> unchangedFilePaths;
+	std::set<FilePath> changedFilePaths;
+
+	for (const FileInfo& info: m_storage->getFileInfoForAllFiles())
+	{
+		if (info.path.exists())
+		{
+			if (didFileChange(info))
+			{
+				changedFilePaths.insert(info.path);
+			}
+			else
+			{
+				unchangedFilePaths.insert(info.path);
+			}
+		}
+		else // file has been removed
+		{
+			changedFilePaths.insert(info.path);
+		}
+	}
+
+	std::set<FilePath> filesToClear = changedFilePaths;
+
+	// handle referencing paths
+	utility::append(filesToClear, m_storage->getReferencing(changedFilePaths));
+
+	// handle referenced paths
+	std::set<FilePath> allSourceFilePaths = getAllSourceFilePaths();
+	std::set<FilePath> staticSourceFiles = allSourceFilePaths;
+	for (const FilePath& path: changedFilePaths)
+	{
+		staticSourceFiles.erase(path);
+	}
+
+	const std::set<FilePath> staticReferencedFilePaths = m_storage->getReferenced(staticSourceFiles);
+	const std::set<FilePath> dynamicReferencedFilePaths = m_storage->getReferenced(changedFilePaths);
+
+	for (const FilePath& path : dynamicReferencedFilePaths)
+	{
+		if (staticReferencedFilePaths.find(path) == staticReferencedFilePaths.end() &&
+			staticSourceFiles.find(path) == staticSourceFiles.end())
+		{
+			// file may not be referenced anymore and will be reindexed if still needed
+			filesToClear.insert(path);
+		}
+	}
+
+	for (const FilePath& path: unchangedFilePaths)
+	{
+		staticSourceFiles.erase(path);
+	}
+
+	std::set<FilePath> filesToAdd = staticSourceFiles;
+
+	std::set<FilePath> staticSourceFilePaths;
+	for (const FilePath& path: allSourceFilePaths)
+	{
+		if (filesToClear.find(path) == filesToClear.end() && filesToAdd.find(path) == filesToAdd.end())
+		{
+			staticSourceFilePaths.insert(path);
+		}
+	}
+
+	RefreshInfo info;
+	info.mode = REFRESH_UPDATED_FILES;
+	info.filesToClear = filesToClear;
+
+	for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)
+	{
+		utility::append(info.filesToIndex, sourceGroup->getSourceFilePathsToIndex(staticSourceFilePaths));
+	}
+
+	return info;
+}
+
+RefreshInfo Project::getRefreshInfoForIncompleteFiles() const
+{
+	RefreshInfo info = getRefreshInfoForUpdatedFiles();
+	info.mode = REFRESH_UPDATED_AND_INCOMPLETE_FILES;
+
+	std::set<FilePath> incompleteFiles;
+	for (const FilePath& path: m_storage->getIncompleteFiles())
+	{
+		if (info.filesToClear.find(path) == info.filesToClear.end())
+		{
+			incompleteFiles.insert(path);
+		}
+	}
+
+	if (incompleteFiles.size())
+	{
+		utility::append(incompleteFiles, m_storage->getReferencing(incompleteFiles));
+		utility::append(info.filesToClear, incompleteFiles);
+
+		std::set<FilePath> staticSourceFilePaths = getAllSourceFilePaths();
+		for (const FilePath& path: incompleteFiles)
+		{
+			staticSourceFilePaths.erase(path);
+		}
+
+		for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)
+		{
+			utility::append(info.filesToIndex, sourceGroup->getSourceFilePathsToIndex(staticSourceFilePaths));
+		}
+	}
+
+	return info;
+}
+
+RefreshInfo Project::getRefreshInfoForAllFiles() const
+{
+	RefreshInfo info;
+	info.mode = REFRESH_ALL_FILES;
+	info.filesToIndex = getAllSourceFilePaths();
+	return info;
 }
 
 bool Project::hasCxxSourceGroup() const

src/lib/project/Project.h

@@ -6,28 +6,36 @@
 #include <set>
 #include <vector>
 
+#include "project/RefreshInfo.h"
 #include "project/SourceGroup.h"
 
 struct FileInfo;
+class DialogView;
 class FilePath;
 class PersistentStorage;
 class ProjectSettings;
-class StorageAccessProxy;
+class StorageCache;
 
 class Project
 {
 public:
-	Project(std::shared_ptr<ProjectSettings> settings, StorageAccessProxy* storageAccessProxy);
+	Project(std::shared_ptr<ProjectSettings> settings, StorageCache* storageCache, bool hasGUI);
 	virtual ~Project();
 
-	bool refresh(bool forceRefresh);
-
 	FilePath getProjectSettingsFilePath() const;
 	std::string getDescription() const;
 
 	bool settingsEqualExceptNameAndLocation(const ProjectSettings& otherSettings) const;
 	void setStateSettingsUpdated();
 
+	void load();
+
+	void refresh(DialogView* dialogView, RefreshMode refreshMode);
+
+	RefreshInfo getRefreshInfo(RefreshMode mode) const;
+
+	void buildIndex(const RefreshInfo& info);
+
 private:
 	enum ProjectStateType
 	{
@@ -42,23 +50,23 @@ private:
 
 	Project(const Project&);
 
-public: // todo: make private again
-	void load();
+	std::set<FilePath> getAllSourceFilePaths() const;
 
-private:
-	bool requestIndex(bool forceRefresh, bool needsFullRefresh);
-
-	void buildIndex(const std::set<FilePath>& filesToIndex, const std::set<FilePath>& filesToClean, bool fullRefresh);
+	RefreshInfo getRefreshInfoForUpdatedFiles() const;
+	RefreshInfo getRefreshInfoForIncompleteFiles() const;
+	RefreshInfo getRefreshInfoForAllFiles() const;
 
 	bool hasCxxSourceGroup() const;
 	bool didFileChange(const FileInfo& info) const;
 
 	std::shared_ptr<ProjectSettings> m_settings;
-	StorageAccessProxy* const m_storageAccessProxy;
+	StorageCache* const m_storageCache;
 
 	ProjectStateType m_state;
 	std::shared_ptr<PersistentStorage> m_storage;
 	std::vector<std::shared_ptr<SourceGroup>> m_sourceGroups;
+
+	bool m_hasGUI;
 };
 
 #endif // PROJECT_H

src/lib/project/RefreshInfo.h

@@ -0,0 +1,23 @@
+#ifndef REFRESH_INFO_H
+#define REFRESH_INFO_H
+
+#include <set>
+
+#include "utility/file/FilePath.h"
+
+enum RefreshMode
+{
+	REFRESH_NONE,
+	REFRESH_UPDATED_FILES,
+	REFRESH_UPDATED_AND_INCOMPLETE_FILES,
+	REFRESH_ALL_FILES
+};
+
+struct RefreshInfo
+{
+	std::set<FilePath> filesToIndex;
+	std::set<FilePath> filesToClear;
+	RefreshMode mode = REFRESH_NONE;
+};
+
+#endif // REFRESH_INFO_H

src/lib/project/SourceGroup.cpp

@@ -14,11 +14,6 @@ LanguageType SourceGroup::getLanguage() const
 	return getLanguageTypeForSourceGroupType(getType());
 }
 
-bool SourceGroup::prepareRefresh()
-{
-	return true;
-}
-
 bool SourceGroup::prepareIndexing()
 {
 	return true;
@@ -26,7 +21,6 @@ bool SourceGroup::prepareIndexing()
 
 void SourceGroup::fetchAllSourceFilePaths()
 {
-	m_sourceFilePathsToIndex.clear();
 	FileManager fileManager;
 	fileManager.update(
 		getAllSourcePaths(),
@@ -36,15 +30,22 @@ void SourceGroup::fetchAllSourceFilePaths()
 	m_allSourceFilePaths = fileManager.getAllSourceFilePaths();
 }
 
-void SourceGroup::fetchSourceFilePathsToIndex(const std::set<FilePath>& staticSourceFilePaths)
+std::set<FilePath> SourceGroup::getAllSourceFilePaths() const
 {
+	return m_allSourceFilePaths;
+}
+
+std::set<FilePath> SourceGroup::getSourceFilePathsToIndex(const std::set<FilePath>& staticSourceFilePaths)
+{
+	std::set<FilePath> sourceFilePathsToIndex;
 	for (const FilePath& sourceFilePath: m_allSourceFilePaths)
 	{
 		if (staticSourceFilePaths.find(sourceFilePath) == staticSourceFilePaths.end())
 		{
-			m_sourceFilePathsToIndex.insert(sourceFilePath);
+			sourceFilePathsToIndex.insert(sourceFilePath);
 		}
 	}
+	return sourceFilePathsToIndex;
 }
 
 std::set<FilePath> SourceGroup::getIndexedPaths()
@@ -57,16 +58,6 @@ std::set<FilePath> SourceGroup::getExcludedPaths()
 	return findAndAddSymlinkedDirectories(getSourceGroupSettings()->getExcludePathsExpandedAndAbsolute());
 }
 
-std::set<FilePath> SourceGroup::getAllSourceFilePaths() const
-{
-	return m_allSourceFilePaths;
-}
-
-std::set<FilePath> SourceGroup::getSourceFilePathsToIndex() const
-{
-	return m_sourceFilePathsToIndex;
-}
-
 std::set<FilePath> SourceGroup::findAndAddSymlinkedDirectories(const std::vector<FilePath>& paths)
 {
 	std::set<FilePath> resultPaths;

src/lib/project/SourceGroup.h

@@ -7,8 +7,8 @@
 
 #include "settings/LanguageType.h"
 #include "settings/SourceGroupType.h"
+#include "utility/file/FilePath.h"
 
-class FilePath;
 class IndexerCommand;
 class SourceGroupSettings;
 
@@ -20,24 +20,20 @@ public:
 	virtual SourceGroupType getType() const = 0;
 	LanguageType getLanguage() const;
 
-	virtual bool prepareRefresh();
 	virtual bool prepareIndexing();
 
 	void fetchAllSourceFilePaths();
-	void fetchSourceFilePathsToIndex(const std::set<FilePath>& staticSourceFilePaths);
 
 	std::set<FilePath> getAllSourceFilePaths() const;
-	std::set<FilePath> getSourceFilePathsToIndex() const;
+	std::set<FilePath> getSourceFilePathsToIndex(const std::set<FilePath>& staticSourceFilePaths);
 
-	virtual std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(
-		const std::set<FilePath>& filesToIndex, bool fullRefresh) = 0;
+	virtual std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(const std::set<FilePath>& filesToIndex) = 0;
 
 protected:
 	std::set<FilePath> getIndexedPaths();
 	std::set<FilePath> getExcludedPaths();
 
 	std::set<FilePath> m_allSourceFilePaths;
-	std::set<FilePath> m_sourceFilePathsToIndex;
 
 private:
 	virtual std::shared_ptr<SourceGroupSettings> getSourceGroupSettings() = 0;

src/lib/utility/commandline/CommandLineParser.cpp

@@ -249,9 +249,14 @@ License CommandLineParser::getLicense()
 	return m_license;
 }
 
-void CommandLineParser::force()
+void CommandLineParser::fullRefresh()
 {
-	m_force = true;
+	m_refreshMode = REFRESH_ALL_FILES;
+}
+
+void CommandLineParser::incompleteRefresh()
+{
+	m_refreshMode = REFRESH_UPDATED_AND_INCOMPLETE_FILES;
 }
 
 const FilePath& CommandLineParser::getProjectFilePath() const
@@ -259,9 +264,9 @@ const FilePath& CommandLineParser::getProjectFilePath() const
 	return m_projectFile;
 }
 
-bool CommandLineParser::getFullProjectRefresh() const
+RefreshMode CommandLineParser::getRefreshMode() const
 {
-	return m_force;
+	return m_refreshMode;
 }
 
 } // namespace cmd

src/lib/utility/commandline/CommandLineParser.h

@@ -9,6 +9,7 @@
 #include "boost/program_options.hpp"
 
 #include "License.h"
+#include "project/RefreshInfo.h"
 #include "utility/file/FilePath.h"
 
 namespace po = boost::program_options;
@@ -37,7 +38,8 @@ public:
 	bool startedWithLicense();
 	bool hasError();
 
-	void force();
+	void fullRefresh();
+	void incompleteRefresh();
 
 	std::string getError();
 	License getLicense();
@@ -45,7 +47,7 @@ public:
 
 	const FilePath& getProjectFilePath() const;
 	void setProjectFile(const FilePath& filepath);
-	bool getFullProjectRefresh() const;
+	RefreshMode getRefreshMode() const;
 
 private:
 	void addCommand(std::unique_ptr<Command> command);
@@ -57,7 +59,7 @@ private:
 	std::vector<std::shared_ptr<Command>> m_commands;
 
 	const std::string m_version;
-	bool m_force{false};
+	RefreshMode m_refreshMode = REFRESH_UPDATED_FILES;
 	bool m_quit{false};
 	bool m_withoutGUI{false};
 

src/lib/utility/commandline/commands/CommandlineCommandIndex.cpp

@@ -20,6 +20,7 @@ void CommandIndex::setup()
 	po::options_description options("Config Options");
 	options.add_options()
 		("help,h", "Print this help message")
+		("incomplete,i", "Also reindex incomplete files (files with errors)")
 		("full,f", "Index full project (omit to only index new/changed files)")
 		("project-file", po::value<std::string>(), "Project file to index (.srctrlprj)")
 		;
@@ -53,7 +54,11 @@ ReturnStatus CommandIndex::parse(std::vector<std::string>& args)
 
 	if (vm.count("full"))
 	{
-		m_parser->force();
+		m_parser->fullRefresh();
+	}
+	else if (vm.count("incomplete"))
+	{
+		m_parser->incompleteRefresh();
 	}
 
 	if (vm.count("project-file"))

src/lib/utility/messaging/type/MessageLoadProject.h

@@ -1,6 +1,8 @@
 #ifndef MESSAGE_LOAD_PROJECT_H
 #define MESSAGE_LOAD_PROJECT_H
 
+#include "project/RefreshInfo.h"
+
 #include "utility/file/FilePath.h"
 #include "utility/messaging/Message.h"
 
@@ -8,9 +10,10 @@ class MessageLoadProject
 	: public Message<MessageLoadProject>
 {
 public:
-	MessageLoadProject(const FilePath& filePath, bool forceRefresh)
+	MessageLoadProject(const FilePath& filePath, bool settingsChanged = false, RefreshMode refreshMode = REFRESH_NONE)
 		: projectSettingsFilePath(filePath)
-		, forceRefresh(forceRefresh)
+		, settingsChanged(settingsChanged)
+		, refreshMode(refreshMode)
 	{
 	}
 
@@ -21,11 +24,14 @@ public:
 
 	virtual void print(std::ostream& os) const
 	{
-		os << projectSettingsFilePath.str() << ", forceRefresh: " << std::boolalpha << forceRefresh;
+		os << projectSettingsFilePath.str();
+		os << ", settingsChanged: " << std::boolalpha << settingsChanged;
+		os << ", refreshMode: " << refreshMode;
 	}
 
 	const FilePath projectSettingsFilePath;
-	const bool forceRefresh;
+	const bool settingsChanged;
+	const RefreshMode refreshMode;
 };
 
 #endif // MESSAGE_LOAD_PROJECT_H