logic: improved indexer parallelization

* added task that injects into PersistentStorage parallel to indexer tasks.
* added Blackboard for Task classes

src/lib/data/IntermediateStorage.cpp

@@ -29,6 +29,11 @@ void IntermediateStorage::clear()
 	m_nextId = 1;
 }
 
+size_t IntermediateStorage::getSourceLocationCount() const
+{
+	return m_sourceLocations.size();
+}
+
 Id IntermediateStorage::addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime)
 {
 	std::shared_ptr<StorageFile> file = std::make_shared<StorageFile>(0, name, filePath, modificationTime);

src/lib/data/IntermediateStorage.h

@@ -15,6 +15,7 @@ public:
 	virtual ~IntermediateStorage();
 
 	void clear();
+	size_t getSourceLocationCount() const;
 
 	virtual Id addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime);
 	virtual Id addNode(int type, const std::string& serializedName, int definitionType);

src/lib/data/StorageProvider.cpp

@@ -0,0 +1,60 @@
+#include "data/StorageProvider.h"
+
+#include <iostream>
+
+int StorageProvider::getStorageCount() const
+{
+	std::lock_guard<std::mutex> lock(m_storagesMutex);
+	return m_storages.size();
+}
+
+void StorageProvider::pushIndexerTarget(std::shared_ptr<IntermediateStorage> storage)
+{
+	const std::size_t storageSize = storage->getSourceLocationCount();
+	std::list<std::shared_ptr<IntermediateStorage>>::iterator it;
+
+	std::lock_guard<std::mutex> lock(m_storagesMutex);
+	for (it = m_storages.begin(); it != m_storages.end(); it++)
+	{
+		if ((*it)->getSourceLocationCount() < storageSize)
+		{
+			break;
+		}
+	}
+	m_storages.insert(it, storage);
+}
+
+std::shared_ptr<IntermediateStorage> StorageProvider::popIndexerTarget()
+{
+	std::shared_ptr<IntermediateStorage> ret;
+	{
+		std::lock_guard<std::mutex> lock(m_storagesMutex);
+		if (m_storages.size() > 1)
+		{
+			std::list<std::shared_ptr<IntermediateStorage>>::iterator it = m_storages.begin();
+			it++;
+			ret = *it;
+			m_storages.erase(it);
+		}
+		else
+		{
+			ret = std::make_shared<IntermediateStorage>();
+		}
+	}
+	return ret;
+}
+
+std::shared_ptr<IntermediateStorage> StorageProvider::popInjectionSource()
+{
+	std::shared_ptr<IntermediateStorage> ret;
+	{
+		std::lock_guard<std::mutex> lock(m_storagesMutex);
+		if (!m_storages.empty())
+		{
+			ret = m_storages.front();
+			m_storages.pop_front();
+		}
+	}
+
+	return ret;
+}

src/lib/data/StorageProvider.h

@@ -0,0 +1,26 @@
+#ifndef STORAGE_PROVIDER_H
+#define STORAGE_PROVIDER_H
+
+#include <memory>
+#include <mutex>
+#include <list>
+#include "data/IntermediateStorage.h"
+
+class StorageProvider
+{
+public:
+	int getStorageCount() const;
+	void pushIndexerTarget(std::shared_ptr<IntermediateStorage> storage);
+
+	// always returns a usable storage
+	std::shared_ptr<IntermediateStorage> popIndexerTarget();
+
+	// returns empty shared_ptr if no storages available
+	std::shared_ptr<IntermediateStorage> popInjectionSource();
+
+private:
+	std::list<std::shared_ptr<IntermediateStorage>> m_storages; // larger storages are in front
+	mutable std::mutex m_storagesMutex;
+};
+
+#endif // STORAGE_PROVIDER_H

src/lib/data/TaskCleanStorage.cpp

@@ -12,12 +12,12 @@ TaskCleanStorage::TaskCleanStorage(
 {
 }
 
-void TaskCleanStorage::doEnter()
+void TaskCleanStorage::doEnter(std::shared_ptr<Blackboard> blackboard)
 {
 	m_dialogView->showProgressDialog("Clearing Files", std::to_string(m_filePaths.size()) + " Files");
 }
 
-Task::TaskState TaskCleanStorage::doUpdate()
+Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr<Blackboard> blackboard)
 {
 	m_storage->clearFileElements(m_filePaths);
 
@@ -26,11 +26,11 @@ Task::TaskState TaskCleanStorage::doUpdate()
 	return STATE_SUCCESS;
 }
 
-void TaskCleanStorage::doExit()
+void TaskCleanStorage::doExit(std::shared_ptr<Blackboard> blackboard)
 {
 	m_dialogView->hideProgressDialog();
 }
 
-void TaskCleanStorage::doReset()
+void TaskCleanStorage::doReset(std::shared_ptr<Blackboard> blackboard)
 {
 }

src/lib/data/TaskCleanStorage.h

@@ -20,10 +20,10 @@ public:
 	);
 
 private:
-	virtual void doEnter();
-	virtual TaskState doUpdate();
-	virtual void doExit();
-	virtual void doReset();
+	virtual void doEnter(std::shared_ptr<Blackboard> blackboard);
+	virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
+	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
+	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
 	PersistentStorage* m_storage;
 	std::vector<FilePath> m_filePaths;

src/lib/data/TaskInjectStorage.cpp

@@ -1,30 +1,50 @@
 #include "data/TaskInjectStorage.h"
 
 #include "data/Storage.h"
+#include "data/StorageProvider.h"
+#include "utility/scheduling/Blackboard.h"
 
 TaskInjectStorage::TaskInjectStorage(
-	std::shared_ptr<Storage> source,
+	std::shared_ptr<StorageProvider> storageProvider,
 	std::shared_ptr<Storage> target
 )
-	: m_source(source)
+	: m_storageProvider(storageProvider)
 	, m_target(target)
 {
 }
 
-void TaskInjectStorage::doEnter()
+void TaskInjectStorage::doEnter(std::shared_ptr<Blackboard> blackboard)
 {
 }
 
-Task::TaskState TaskInjectStorage::doUpdate()
+Task::TaskState TaskInjectStorage::doUpdate(std::shared_ptr<Blackboard> blackboard)
 {
-	m_target->inject(m_source.get());
-	return STATE_SUCCESS;
+	if (m_storageProvider->getStorageCount() > 0)
+	{
+		std::shared_ptr<IntermediateStorage> source = m_storageProvider->popInjectionSource();
+		if (source)
+		{
+			m_target->inject(source.get());
+			return STATE_SUCCESS;
+		}
+	}
+
+	int indexerCount = 0;
+	if (blackboard->get("indexer_count", indexerCount))
+	{
+		if (indexerCount > 0)
+		{
+			return STATE_SUCCESS;
+		}
+	}
+
+	return STATE_FAILURE;
 }
 
-void TaskInjectStorage::doExit()
+void TaskInjectStorage::doExit(std::shared_ptr<Blackboard> blackboard)
 {
 }
 
-void TaskInjectStorage::doReset()
+void TaskInjectStorage::doReset(std::shared_ptr<Blackboard> blackboard)
 {
 }

src/lib/data/TaskInjectStorage.h

@@ -6,23 +6,24 @@
 #include "utility/scheduling/Task.h"
 
 class Storage;
+class StorageProvider;
 
 class TaskInjectStorage
 	: public Task
 {
 public:
 	TaskInjectStorage(
-		std::shared_ptr<Storage> source,
+		std::shared_ptr<StorageProvider> storageProvider,
 		std::shared_ptr<Storage> target
 	);
 
 private:
-	virtual void doEnter();
-	virtual TaskState doUpdate();
-	virtual void doExit();
-	virtual void doReset();
+	virtual void doEnter(std::shared_ptr<Blackboard> blackboard);
+	virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
+	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
+	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
-	std::shared_ptr<Storage> m_source;
+	std::shared_ptr<StorageProvider> m_storageProvider;
 	std::shared_ptr<Storage> m_target;
 };
 

src/lib/data/TaskRepeatWhileUnparsedSourceFilesAvailable.cpp

@@ -1,47 +0,0 @@
-#include "data/TaskRepeatWhileUnparsedSourceFilesAvailable.h"
-
-#include "utility/file/FileRegister.h"
-
-TaskRepeatWhileUnparsedSourceFilesAvailable::TaskRepeatWhileUnparsedSourceFilesAvailable(
-	std::shared_ptr<FileRegister> fileRegister
-)
-	: m_fileRegister(fileRegister)
-{
-}
-
-void TaskRepeatWhileUnparsedSourceFilesAvailable::setTask(std::shared_ptr<Task> task)
-{
-	if (task)
-	{
-		m_taskRunner = std::make_shared<TaskRunner>(task);
-	}
-}
-
-void TaskRepeatWhileUnparsedSourceFilesAvailable::doEnter()
-{
-}
-
-Task::TaskState TaskRepeatWhileUnparsedSourceFilesAvailable::doUpdate()
-{
-	TaskState state = m_taskRunner->update();
-
-	if (state == Task::STATE_SUCCESS)
-	{
-		if(m_fileRegister->getUnparsedSourceFilePaths().size() != 0)
-		{
-			state = Task::STATE_RUNNING;
-			m_taskRunner->reset();
-		}
-	}
-
-	return state;
-}
-
-void TaskRepeatWhileUnparsedSourceFilesAvailable::doExit()
-{
-}
-
-void TaskRepeatWhileUnparsedSourceFilesAvailable::doReset()
-{
-	m_taskRunner->reset();
-}

src/lib/data/TaskRepeatWhileUnparsedSourceFilesAvailable.h

@@ -1,31 +0,0 @@
-#ifndef TASK_REPEAT_WHILE_UNPARSED_SOURCE_FILES_AVAILABLE_H
-#define TASK_REPEAT_WHILE_UNPARSED_SOURCE_FILES_AVAILABLE_H
-
-#include <vector>
-
-#include "utility/scheduling/TaskDecorator.h"
-#include "utility/scheduling/TaskRunner.h"
-
-class FileRegister;
-
-class TaskRepeatWhileUnparsedSourceFilesAvailable
-	: public TaskDecorator
-{
-public:
-	TaskRepeatWhileUnparsedSourceFilesAvailable(
-		std::shared_ptr<FileRegister> fileRegister
-	);
-
-	virtual void setTask(std::shared_ptr<Task> task);
-
-private:
-	virtual void doEnter();
-	virtual TaskState doUpdate();
-	virtual void doExit();
-	virtual void doReset();
-
-	std::shared_ptr<FileRegister> m_fileRegister;
-	std::shared_ptr<TaskRunner> m_taskRunner;
-};
-
-#endif // TASK_REPEAT_WHILE_UNPARSED_SOURCE_FILES_AVAILABLE_H

src/lib/data/parser/cxx/TaskParseCxx.h

@@ -2,7 +2,6 @@
 #define TASK_PARSE_CXX_H
 
 #include <memory>
-#include <mutex>
 #include <deque>
 
 #include "data/parser/Parser.h"
@@ -15,7 +14,7 @@
 class CxxParser;
 class DialogView;
 class FileRegister;
-class IntermediateStorage;
+class StorageProvider;
 
 namespace clang
 {
@@ -33,22 +32,21 @@ public:
 	static std::vector<FilePath> getSourceFilesFromCDB(const FilePath& compilationDatabasePath);
 
 	TaskParseCxx(
-		std::shared_ptr<IntermediateStorage> storage,
+		std::shared_ptr<StorageProvider> storageProvider,
 		std::shared_ptr<FileRegister> fileRegister,
 		const Parser::Arguments& arguments,
 		DialogView* dialogView
 	);
 
 private:
-	virtual void doEnter();
-	virtual TaskState doUpdate();
-	virtual void doExit();
-	virtual void doReset();
+	virtual void doEnter(std::shared_ptr<Blackboard> blackboard);
+	virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
+	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
+	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
 	virtual void handleMessage(MessageInterruptTasks* message);
 
-	std::shared_ptr<IntermediateStorage> m_storage;
-	std::shared_ptr<std::mutex> m_storageMutex;
+	std::shared_ptr<StorageProvider> m_storageProvider;
 
 	const Parser::Arguments m_arguments;
 	DialogView* m_dialogView;

src/lib/data/parser/cxx/TaskParseWrapper.h

@@ -28,10 +28,10 @@ public:
 	virtual void setTask(std::shared_ptr<Task> task);
 
 private:
-	virtual void doEnter();
-	virtual TaskState doUpdate();
-	virtual void doExit();
-	virtual void doReset();
+	virtual void doEnter(std::shared_ptr<Blackboard> blackboard);
+	virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
+	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
+	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
 	PersistentStorage* m_storage;
 	std::shared_ptr<FileRegister> m_fileRegister;

src/lib/data/parser/java/TaskParseJava.h

@@ -10,7 +10,7 @@
 
 class DialogView;
 class FileRegister;
-class IntermediateStorage;
+class StorageProvider;
 
 class TaskParseJava
 	: public Task
@@ -18,21 +18,21 @@ class TaskParseJava
 {
 public:
 	TaskParseJava(
-		std::shared_ptr<IntermediateStorage> storage,
+		std::shared_ptr<StorageProvider> storageProvider,
 		std::shared_ptr<FileRegister> fileRegister,
 		const Parser::Arguments& arguments,
 		DialogView* dialogView
 	);
 
 private:
-	virtual void doEnter();
-	virtual TaskState doUpdate();
-	virtual void doExit();
-	virtual void doReset();
+	virtual void doEnter(std::shared_ptr<Blackboard> blackboard);
+	virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
+	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
+	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
 	virtual void handleMessage(MessageInterruptTasks* message);
 
-	std::shared_ptr<IntermediateStorage> m_storage;
+	std::shared_ptr<StorageProvider> m_storageProvider;
 	std::shared_ptr<FileRegister> m_fileRegister;
 	Parser::Arguments m_arguments;
 	DialogView* m_dialogView;