logic: Fixed intermediate storage fetching starving when indexers are really fast because of other UI updates

src/lib/component/controller/StatusBarController.cpp

@@ -69,7 +69,7 @@ void StatusBarController::handleMessage(MessageShowErrors* message)
 
 void StatusBarController::handleMessage(MessageStatus* message)
 {
-	setStatus(message->status, message->isError, message->showLoader);
+	setStatus(message->status(), message->isError, message->showLoader);
 }
 
 void StatusBarController::setStatus(const std::string& status, bool isError, bool showLoader)

src/lib/component/controller/StatusController.cpp

@@ -35,17 +35,21 @@ void StatusController::handleMessage(MessageShowStatus* message)
 
 void StatusController::handleMessage(MessageStatus* message)
 {
-	if (!message->status.size())
+	if (!message->status().size())
 	{
 		return;
 	}
 
-	std::vector<Status> status;
-	status.push_back(Status(message->status, message->isError));
+	std::vector<Status> stati;
 
-	utility::append(m_status, status);
+	for (const std::string& status : message->stati())
+	{
+		stati.push_back(Status(status, message->isError));
+	}
 
-	addStatus(status);
+	utility::append(m_status, stati);
+
+	addStatus(stati);
 }
 
 void StatusController::handleMessage(MessageStatusFilterChanged* message)

src/lib/data/indexer/TaskBuildIndex.cpp

@@ -41,7 +41,7 @@ TaskBuildIndex::TaskBuildIndex(
 void TaskBuildIndex::doEnter(std::shared_ptr<Blackboard> blackboard)
 {
 	m_indexingFileCount = 0;
-	updateIndexingDialog(blackboard, FilePath());
+	updateIndexingDialog(blackboard, std::vector<FilePath>());
 
 	{
 		std::lock_guard<std::mutex> lock(blackboard->getMutex());
@@ -91,10 +91,9 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
 	if (commandCount != m_lastCommandCount)
 	{
 		std::vector<FilePath> indexingFiles = m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePaths();
-		for (const FilePath& path : indexingFiles)
+		if (indexingFiles.size())
 		{
-			m_indexingFileCount++;
-			updateIndexingDialog(blackboard, path);
+			updateIndexingDialog(blackboard, indexingFiles);
 		}
 
 		m_lastCommandCount = commandCount;
@@ -116,7 +115,7 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
 
 	if (fetchIntermediateStorages(blackboard))
 	{
-		updateIndexingDialog(blackboard, FilePath());
+		updateIndexingDialog(blackboard, std::vector<FilePath>());
 	}
 
 	const int SLEEP_TIME_MS = 50;
@@ -227,37 +226,47 @@ void TaskBuildIndex::runIndexerThread(int processId)
 
 bool TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard)
 {
-	Id finishedProcessId = m_interprocessIndexingStatusManager.getNextFinishedProcessId();
-	if (!finishedProcessId || finishedProcessId > m_interprocessIntermediateStorageManagers.size())
+	int poppedStorageCount = 0;
+
+	TimeStamp t = TimeStamp::now();
+	do
 	{
-		return false;
+		Id finishedProcessId = m_interprocessIndexingStatusManager.getNextFinishedProcessId();
+		if (!finishedProcessId || finishedProcessId > m_interprocessIntermediateStorageManagers.size())
+		{
+			break;
+		}
+
+		std::shared_ptr<InterprocessIntermediateStorageManager> storageManager =
+			m_interprocessIntermediateStorageManagers[finishedProcessId - 1];
+
+		int storageCount = storageManager->getIntermediateStorageCount();
+		if (!storageCount)
+		{
+			break;
+		}
+
+		LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
+		m_storageProvider->insert(storageManager->popIntermediateStorage());
+		poppedStorageCount++;
 	}
+	while (TimeStamp::now().deltaMS(t) < 500); // don't process all storages at once to allow for status updates in-between
 
-	std::shared_ptr<InterprocessIntermediateStorageManager> storageManager =
-		m_interprocessIntermediateStorageManagers[finishedProcessId - 1];
-
-	int storageCount = storageManager->getIntermediateStorageCount();
-	if (!storageCount)
-	{
-		return false;
-	}
-
-	LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
-	m_storageProvider->insert(storageManager->popIntermediateStorage());
-
+	if (poppedStorageCount > 0)
 	{
 		std::lock_guard<std::mutex> lock(blackboard->getMutex());
 
 		int indexedSourceFileCount = 0;
 		blackboard->get("indexed_source_file_count", indexedSourceFileCount);
-		blackboard->set("indexed_source_file_count", indexedSourceFileCount + 1);
+		blackboard->set("indexed_source_file_count", indexedSourceFileCount + poppedStorageCount);
+		return true;
 	}
 
-	return true;
+	return false;
 }
 
 void TaskBuildIndex::updateIndexingDialog(
-	std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath)
+	std::shared_ptr<Blackboard> blackboard, const std::vector<FilePath>& sourcePaths)
 {
 	// TODO: factor in unindexed files...
 	int sourceFileCount = 0;
@@ -268,14 +277,21 @@ void TaskBuildIndex::updateIndexingDialog(
 		blackboard->get("indexed_source_file_count", indexedSourceFileCount);
 	}
 
-	if (!sourcePath.empty())
+	if (sourcePaths.size())
 	{
-		std::stringstream ss;
-		ss << "[" << m_indexingFileCount << "/" << sourceFileCount << "] Indexing file: " << sourcePath.str();
-		MessageStatus(ss.str(), false, true).dispatch();
+		std::vector<std::string> stati;
+		for (const FilePath& path : sourcePaths)
+		{
+			m_indexingFileCount++;
+
+			std::stringstream ss;
+			ss << "[" << m_indexingFileCount << "/" << sourceFileCount << "] Indexing file: " << path.str();
+			stati.push_back(ss.str());
+		}
+		MessageStatus(stati, false, true).dispatch();
 	}
 
 	Application::getInstance()->getDialogView()->updateIndexingDialog(
-		m_indexingFileCount, indexedSourceFileCount, sourceFileCount, sourcePath.str()
+		m_indexingFileCount, indexedSourceFileCount, sourceFileCount, (sourcePaths.size() ? sourcePaths.back().str() : "")
 	);
 }

src/lib/data/indexer/TaskBuildIndex.h

@@ -39,7 +39,7 @@ protected:
 	void runIndexerProcess( int processId, const std::string& logFilePath);
 	void runIndexerThread(int processId);
 	bool fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard);
-	void updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath);
+	void updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const std::vector<FilePath>& sourcePaths);
 
 	static const std::string s_processName;
 

src/lib/utility/messaging/type/MessageStatus.h

@@ -1,6 +1,8 @@
 #ifndef MESSAGE_STATUS_H
 #define MESSAGE_STATUS_H
 
+#include <vector>
+
 #include "utility/messaging/Message.h"
 #include "utility/utilityString.h"
 
@@ -9,9 +11,18 @@ class MessageStatus
 {
 public:
 	MessageStatus(const std::string& status, bool isError = false, bool showLoader = false)
-		: status(utility::replace(status, "\n", " "))
-		, isError(isError)
+		: isError(isError)
 		, showLoader(showLoader)
+	{
+		m_stati.push_back(utility::replace(status, "\n", " "));
+
+		setSendAsTask(false);
+	}
+
+	MessageStatus(const std::vector<std::string>& stati, bool isError = false, bool showLoader = false)
+		: isError(isError)
+		, showLoader(showLoader)
+		, m_stati(stati)
 	{
 		setSendAsTask(false);
 	}
@@ -21,9 +32,32 @@ public:
 		return "MessageStatus";
 	}
 
+	const std::vector<std::string>& stati() const
+	{
+		return m_stati;
+	}
+
+	std::string status() const
+	{
+		if (m_stati.size())
+		{
+			return m_stati[0];
+		}
+
+		return "";
+	}
+
 	virtual void print(std::ostream& os) const
 	{
-		os << status;
+		for (const std::string& status : m_stati)
+		{
+			os << status;
+
+			if (m_stati.size() > 1)
+			{
+				os << " - ";
+			}
+		}
 
 		if (isError)
 		{
@@ -36,9 +70,11 @@ public:
 		}
 	}
 
-	const std::string status;
 	const bool isError;
 	const bool showLoader;
+
+private:
+	std::vector<std::string> m_stati;
 };
 
 #endif // MESSAGE_STATUS_H

src/lib_gui/qt/QtCoreApplication.cpp

@@ -19,5 +19,8 @@ void QtCoreApplication::handleMessage(MessageQuitApplication* message)
 
 void QtCoreApplication::handleMessage(MessageStatus* message)
 {
-	std::cout << message->status << std::endl;
+	for (const std::string& status : message->stati())
+	{
+		std::cout << status << std::endl;
+	}
 }