logic: multithreaded parsing

* modified the TaskParseCxx to be able to run multiple times in parallel
* made FileRegister threadsafe and changed a lot of its mechanisms
* added TaskGroupParallel that runs all children in parallel
* added TaskParseWrapper that acts as a decorator to execute some code before and after parsing.
* implemented task setup in project with 4 parsing threads
* removed SimpleTask as it was only used as interface for the LambdaTask

src/app/CMakeLists.txt

@@ -2,6 +2,7 @@ add_files(
 	APP_FILES
 
 	data/parser/cxx/TaskParseCxx.cpp
+	data/parser/cxx/TaskParseWrapper.cpp
 
 	utility/commandline/CommandLineParser.cpp
 	utility/commandline/CommandLineParser.h

src/app/data/parser/cxx/TaskParseCxx.cpp

@@ -11,25 +11,6 @@
 #include "utility/messaging/type/MessageStatus.h"
 #include "utility/utility.h"
 
-TaskParseCxx::TaskParseCxx(
-	PersistentStorage* storage,
-	const FileManager* fileManager,
-	const Parser::Arguments& arguments,
-	const std::vector<FilePath>& files
-)
-	: m_storage(storage)
-	, m_arguments(arguments)
-	, m_files(files)
-	, m_isCDB(false)
-{
-	if (arguments.compilationDatabasePath.exists())
-	{
-		m_isCDB = true;
-	}
-	m_parserClient = std::make_shared<ParserClientImpl>();
-	m_parser = std::make_shared<CxxParser>(m_parserClient.get(), fileManager);
-}
-
 std::vector<FilePath> TaskParseCxx::getSourceFilesFromCDB(const FilePath& compilationDatabasePath)
 {
 	std::string error;
@@ -45,52 +26,46 @@ std::vector<FilePath> TaskParseCxx::getSourceFilesFromCDB(const FilePath& compil
 	return filePaths;
 }
 
+TaskParseCxx::TaskParseCxx(
+	PersistentStorage* storage,
+	std::shared_ptr<std::mutex> storageMutex,
+	std::shared_ptr<FileRegister> fileRegister,
+	const Parser::Arguments& arguments
+)
+	: m_storage(storage)
+	, m_storageMutex(storageMutex)
+	, m_arguments(arguments)
+	, m_isCDB(false)
+{
+	if (arguments.compilationDatabasePath.exists())
+	{
+		m_isCDB = true;
+	}
+	m_parserClient = std::make_shared<ParserClientImpl>(); // todo: create one parserclient per file
+	m_parser = std::make_shared<CxxParser>(m_parserClient.get(), fileRegister);
+}
+
 void TaskParseCxx::enter()
 {
-	m_start = utility::durationStart();
-
 	if (m_isCDB)
 	{
 		std::string error;
 		m_cdb = std::shared_ptr<clang::tooling::JSONCompilationDatabase>
 			(clang::tooling::JSONCompilationDatabase::loadFromFile(m_arguments.compilationDatabasePath.str(), error));
 
-		m_parser->setupParsingCDB(m_files, m_arguments);
+		m_parser->setupParsingCDB(m_arguments);
 	}
 	else
 	{
-		m_parser->setupParsing(m_files, m_arguments);
+		m_parser->setupParsing(m_arguments);
 	}
-
-	for (const FilePath& path : m_parser->getFileRegister()->getUnparsedSourceFilePaths())
-	{
-		m_sourcePaths.push_back(path.absolute());
-	}
-
-	m_storage->startParsing();
 }
 
 Task::TaskState TaskParseCxx::update()
 {
-	FilePath sourcePath;
-	bool isSource = false;
-
 	FileRegister* fileRegister = m_parser->getFileRegister();
 
-	if (m_sourcePaths.size())
-	{
-		sourcePath = m_sourcePaths.front();
-		m_sourcePaths.pop_front();
-		isSource = true;
-	}
-	else if (!m_isCDB)
-	{
-		std::vector<FilePath> unparsedHeaders = fileRegister->getUnparsedIncludeFilePaths();
-		if (unparsedHeaders.size())
-		{
-			sourcePath = unparsedHeaders[0];
-		}
-	}
+	FilePath sourcePath = fileRegister->consumeSourceFile();
 
 	if (sourcePath.empty())
 	{
@@ -99,16 +74,15 @@ Task::TaskState TaskParseCxx::update()
 
 	std::stringstream ss;
 	ss << "analyzing files (ESC to quit): [";
-	ss << (m_isCDB ? fileRegister->getParsedSourceFilesCount() : fileRegister->getParsedFilesCount()) + 1 << "/";
-	ss << (m_isCDB ? fileRegister->getSourceFilesCount() : fileRegister->getFilesCount()) << "] ";
+	ss << fileRegister->getParsedSourceFilesCount() << "/";
+	ss << fileRegister->getSourceFilesCount() << "] ";
 	ss << sourcePath.str();
-
 	MessageStatus(ss.str(), false, true).dispatch();
 
 	std::shared_ptr<IntermediateStorage> intermediateStorage = std::make_shared<IntermediateStorage>();
 
 	m_parserClient->setStorage(intermediateStorage);
-	m_parserClient->startParsingFile(sourcePath);
+	m_parserClient->startParsingFile();
 
 	if (m_isCDB)
 	{
@@ -123,37 +97,25 @@ Task::TaskState TaskParseCxx::update()
 		m_parser->runTool(std::vector<std::string>(1, sourcePath.str()));
 	}
 
-	m_parserClient->finishParsingFile(sourcePath);
+	m_parserClient->finishParsingFile();
 	m_parserClient->resetStorage();
 
-	m_storage->inject(intermediateStorage.get());
-
-	if (isSource)
 	{
-		fileRegister->markSourceFileParsed(sourcePath.str());
+		std::lock_guard<std::mutex> lock(*(m_storageMutex.get()));
+		m_storage->inject(intermediateStorage.get());
 	}
 
+	fileRegister->markThreadFilesParsed();
+
 	return Task::STATE_RUNNING;
 }
 
 void TaskParseCxx::exit()
 {
-	MessageStatus("building search index", false, true).dispatch();
-
-	m_storage->finishParsing();
-
-	FileRegister* fileRegister = m_parser->getFileRegister();
-
-	MessageFinishedParsing(
-		(m_isCDB ? fileRegister->getParsedSourceFilesCount() : fileRegister->getParsedFilesCount()),
-		(m_isCDB ? fileRegister->getSourceFilesCount() : fileRegister->getFilesCount()),
-		utility::duration(m_start)
-	).dispatch();
 }
 
 void TaskParseCxx::interrupt()
 {
-	MessageStatus("analyzing files interrupted", false, true).dispatch();
 }
 
 void TaskParseCxx::revert()

src/app/data/parser/cxx/TaskParseWrapper.cpp

@@ -0,0 +1,57 @@
+#include "data/parser/cxx/TaskParseWrapper.h"
+
+#include "data/PersistentStorage.h"
+#include "utility/file/FileRegister.h"
+#include "utility/messaging/type/MessageFinishedParsing.h"
+#include "utility/messaging/type/MessageStatus.h"
+#include "utility/utility.h"
+
+TaskParseWrapper::TaskParseWrapper(
+	std::shared_ptr<Task> child,
+	PersistentStorage* storage,
+	std::shared_ptr<FileRegister> fileRegister
+)
+	: m_child(child)
+	, m_storage(storage)
+	, m_fileRegister(fileRegister)
+{
+}
+
+void TaskParseWrapper::enter()
+{
+	m_start = utility::durationStart();
+	m_storage->startParsing();
+
+	m_child->enter();
+}
+
+Task::TaskState TaskParseWrapper::update()
+{
+	return m_child->update();
+}
+
+void TaskParseWrapper::exit()
+{
+	m_child->exit();
+
+	MessageStatus("building search index", false, true).dispatch();
+
+	m_storage->finishParsing();
+
+	MessageFinishedParsing(
+		m_fileRegister->getParsedSourceFilesCount(),
+		m_fileRegister->getSourceFilesCount(),
+		utility::duration(m_start)
+	).dispatch();
+}
+
+void TaskParseWrapper::interrupt()
+{
+	MessageStatus("analyzing files interrupted", false, true).dispatch();
+	m_child->interrupt();
+}
+
+void TaskParseWrapper::revert()
+{
+	m_child->revert();
+}

src/lib/CMakeLists.txt

@@ -124,6 +124,7 @@ add_files(
 	data/name/NameHierarchy.h
 
 	data/parser/cxx/TaskParseCxx.h
+	data/parser/cxx/TaskParseWrapper.h
 
 	data/parser/ParseLocation.cpp
 	data/parser/ParseLocation.h
@@ -274,12 +275,12 @@ add_files(
 
 	utility/scheduling/LambdaTask.cpp
 	utility/scheduling/LambdaTask.h
-	utility/scheduling/SimpleTask.cpp
-	utility/scheduling/SimpleTask.h
 	utility/scheduling/Task.cpp
 	utility/scheduling/Task.h
 	utility/scheduling/TaskGroup.cpp
 	utility/scheduling/TaskGroup.h
+	utility/scheduling/TaskGroupParallel.cpp
+	utility/scheduling/TaskGroupParallel.h
 	utility/scheduling/TaskGroupSequential.cpp
 	utility/scheduling/TaskGroupSequential.h
 	utility/scheduling/TaskScheduler.cpp

src/lib/Project.cpp

@@ -1,20 +1,23 @@
 #include "Project.h"
 
-#include "utility/file/FileSystem.h"
-#include "utility/logging/logging.h"
-#include "utility/messaging/type/MessageFinishedParsing.h"
-#include "utility/scheduling/TaskGroupSequential.h"
-#include "utility/utility.h"
-#include "utility/Version.h"
-
 #include "data/access/StorageAccessProxy.h"
 #include "data/graph/Token.h"
 #include "data/parser/cxx/TaskParseCxx.h"
+#include "data/parser/cxx/TaskParseWrapper.h"
 #include "data/PersistentStorage.h"
 #include "data/TaskCleanStorage.h"
 #include "settings/ApplicationSettings.h"
 #include "settings/ProjectSettings.h"
 
+#include "utility/file/FileRegister.h"
+#include "utility/file/FileSystem.h"
+#include "utility/logging/logging.h"
+#include "utility/messaging/type/MessageFinishedParsing.h"
+#include "utility/scheduling/TaskGroupSequential.h"
+#include "utility/scheduling/TaskGroupParallel.h"
+#include "utility/utility.h"
+#include "utility/Version.h"
+
 std::shared_ptr<Project> Project::create(StorageAccessProxy* storageAccessProxy)
 {
 	std::shared_ptr<Project> ptr(new Project(storageAccessProxy));
@@ -149,26 +152,42 @@ void Project::parseCode()
 	utility::append(updatedFilePaths, m_storage->getDependingFilePaths(updatedFilePaths));
 	utility::append(updatedFilePaths, m_storage->getDependingFilePaths(removedFilePaths));
 
-	std::shared_ptr<TaskGroupSequential> taskGroup = std::make_shared<TaskGroupSequential>();
+	std::shared_ptr<TaskGroupSequential> taskSequential = std::make_shared<TaskGroupSequential>();
 
 	std::vector<FilePath> filesToClean;
 	filesToClean.insert(filesToClean.end(), removedFilePaths.begin(), removedFilePaths.end());
 	filesToClean.insert(filesToClean.end(), updatedFilePaths.begin(), updatedFilePaths.end());
 
-	taskGroup->addTask(std::make_shared<TaskCleanStorage>(m_storage.get(), filesToClean));
+	taskSequential->addTask(std::make_shared<TaskCleanStorage>(m_storage.get(), filesToClean));
 
 	std::vector<FilePath> filesToParse;
 	filesToParse.insert(filesToParse.end(), addedFilePaths.begin(), addedFilePaths.end());
 	filesToParse.insert(filesToParse.end(), updatedFilePaths.begin(), updatedFilePaths.end());
 
-	taskGroup->addTask(std::make_shared<TaskParseCxx>(
+	std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(&m_fileManager);
+	fileRegister->setFilePaths(filesToParse);
+
+	std::shared_ptr<TaskGroupParallel> taskParallel = std::make_shared<TaskGroupParallel>();
+
+	taskSequential->addTask(std::make_shared<TaskParseWrapper>(
+		taskParallel,
 		m_storage.get(),
-		&m_fileManager,
-		getParserArguments(),
-		filesToParse
+		fileRegister
 	));
 
-	Task::dispatch(taskGroup);
+	std::shared_ptr<std::mutex> storageMutex = std::make_shared<std::mutex>();
+
+	for (int i = 0; i < 4; i++)
+	{
+		taskParallel->addTask(std::make_shared<TaskParseCxx>(
+			m_storage.get(),
+			storageMutex,
+			fileRegister,
+			getParserArguments()
+		));
+	}
+
+	Task::dispatch(taskSequential);
 
 	m_state = PROJECT_LOADED;
 }

src/lib/data/parser/ParserClient.h

@@ -39,8 +39,8 @@ public:
 	ParserClient();
 	virtual ~ParserClient();
 
-	virtual void startParsingFile(const FilePath& filePath) = 0;
-	virtual void finishParsingFile(const FilePath& filePath) = 0;
+	virtual void startParsingFile() = 0;
+	virtual void finishParsingFile() = 0;
 
 	virtual void onError(const ParseLocation& location, const std::string& message, bool fatal) = 0;
 

src/lib/data/parser/ParserClientImpl.cpp

@@ -25,12 +25,12 @@ void ParserClientImpl::resetStorage()
 	m_storage.reset();
 }
 
-void ParserClientImpl::startParsingFile(const FilePath& filePath)
+void ParserClientImpl::startParsingFile()
 {
-	m_nodeIdsToMemberEdgeIds.clear();
+	m_nodeIdsToMemberEdgeIds.clear(); // remove this when one parserclient is created per file
 }
 
-void ParserClientImpl::finishParsingFile(const FilePath& filePath)
+void ParserClientImpl::finishParsingFile()
 {
 }
 

src/lib/data/parser/ParserClientImpl.h

@@ -18,8 +18,8 @@ public:
 	void setStorage(std::shared_ptr<IntermediateStorage> storage);
 	void resetStorage();
 
-	virtual void startParsingFile(const FilePath& filePath);
-	virtual void finishParsingFile(const FilePath& filePath);
+	virtual void startParsingFile();
+	virtual void finishParsingFile();
 
 	virtual void onError(const ParseLocation& location, const std::string& message, bool fatal);
 

src/lib/data/parser/cxx/TaskParseCxx.h

@@ -2,6 +2,7 @@
 #define TASK_PARSE_CXX_H
 
 #include <memory>
+#include <mutex>
 #include <deque>
 
 #include "data/parser/Parser.h"
@@ -10,7 +11,7 @@
 #include "utility/TimePoint.h"
 
 class PersistentStorage;
-class FileManager;
+class FileRegister;
 class CxxParser;
 
 namespace clang
@@ -25,15 +26,15 @@ class TaskParseCxx
 	: public Task
 {
 public:
+	static std::vector<FilePath> getSourceFilesFromCDB(const FilePath& compilationDatabasePath);
+
 	TaskParseCxx(
 		PersistentStorage* storage,
-		const FileManager* fileManager,
-		const Parser::Arguments& arguments,
-		const std::vector<FilePath>& files
+		std::shared_ptr<std::mutex> storageMutex,
+		std::shared_ptr<FileRegister> fileRegister,
+		const Parser::Arguments& arguments
 	);
 
-	static std::vector<FilePath> getSourceFilesFromCDB(const FilePath& compilationDatabasePath);
-
 	virtual void enter();
 	virtual TaskState update();
 	virtual void exit();
@@ -43,14 +44,10 @@ public:
 
 private:
 	PersistentStorage* m_storage;
+	std::shared_ptr<std::mutex> m_storageMutex;
 	std::shared_ptr<CxxParser> m_parser;
 	std::shared_ptr<ParserClientImpl> m_parserClient;
 	const Parser::Arguments m_arguments;
-	const std::vector<FilePath> m_files;
-
-	std::deque<FilePath> m_sourcePaths;
-
-	TimePoint m_start;
 
 	bool m_isCDB;
 	std::shared_ptr<clang::tooling::JSONCompilationDatabase> m_cdb;

src/lib/data/parser/cxx/TaskParseWrapper.h

@@ -0,0 +1,41 @@
+#ifndef TASK_PARSE_WRAPPER_H
+#define TASK_PARSE_WRAPPER_H
+
+#include <memory>
+
+#include "data/parser/Parser.h"
+#include "data/parser/ParserClientImpl.h"
+#include "utility/scheduling/Task.h"
+#include "utility/TimePoint.h"
+
+class PersistentStorage;
+class FileRegister;
+class CxxParser;
+
+class TaskParseWrapper
+	: public Task
+{
+public:
+
+	TaskParseWrapper(
+		std::shared_ptr<Task> child,
+		PersistentStorage* storage,
+		std::shared_ptr<FileRegister> fileRegister
+	);
+
+	virtual void enter();
+	virtual TaskState update();
+	virtual void exit();
+
+	virtual void interrupt();
+	virtual void revert();
+
+private:
+	std::shared_ptr<Task> m_child;
+	PersistentStorage* m_storage;
+	std::shared_ptr<FileRegister> m_fileRegister;
+
+	TimePoint m_start;
+};
+
+#endif // TASK_PARSE_WRAPPER_H

src/lib/utility/file/FileRegister.cpp

@@ -8,13 +8,11 @@ FileRegister::FileRegister(const FileManager* fileManager)
 {
 }
 
-const FileManager* FileRegister::getFileManager() const
-{
-	return m_fileManager;
-}
-
 void FileRegister::setFilePaths(const std::vector<FilePath>& filePaths)
 {
+	std::lock_guard<std::mutex> sourceFileLock(m_sourceFileMutex);
+	std::lock_guard<std::mutex> includeFileLock(m_includeFileMutex);
+
 	m_sourceFilePaths.clear();
 	m_includeFilePaths.clear();
 
@@ -33,96 +31,18 @@ void FileRegister::setFilePaths(const std::vector<FilePath>& filePaths)
 	}
 }
 
+const FileManager* FileRegister::getFileManager() const
+{
+	return m_fileManager;
+}
+
 std::vector<FilePath> FileRegister::getUnparsedSourceFilePaths() const
 {
-	return getUnparsedFilePaths(m_sourceFilePaths);
-}
+	std::lock_guard<std::mutex> lock(m_sourceFileMutex);
 
-std::vector<FilePath> FileRegister::getUnparsedIncludeFilePaths() const
-{
-	return getUnparsedFilePaths(m_includeFilePaths);
-}
-
-bool FileRegister::fileIsParsed(const FilePath& filePath) const
-{
-	std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
-	if (it != m_includeFilePaths.end())
-	{
-		return it->second == STATE_PARSED;
-	}
-
-	it = m_sourceFilePaths.find(filePath);
-	if (it != m_sourceFilePaths.end())
-	{
-		return it->second == STATE_PARSED;
-	}
-
-	return true;
-}
-
-bool FileRegister::includeFileIsParsing(const FilePath& filePath) const
-{
-	std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
-	if (it == m_includeFilePaths.end())
-	{
-		return false;
-	}
-
-	return it->second == STATE_PARSING;
-}
-
-bool FileRegister::includeFileIsParsed(const FilePath& filePath) const
-{
-	std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
-	if (it == m_includeFilePaths.end())
-	{
-		return false;
-	}
-
-	return it->second == STATE_PARSED;
-}
-
-void FileRegister::markSourceFileParsed(const std::string& filePath)
-{
-	std::map<FilePath, ParseState>::iterator it = m_sourceFilePaths.find(FilePath(filePath));
-	if (it == m_sourceFilePaths.end())
-	{
-		return;
-	}
-
-	it->second = STATE_PARSED;
-}
-
-void FileRegister::markIncludeFileParsing(const std::string& filePath)
-{
-	std::map<FilePath, ParseState>::iterator it = m_includeFilePaths.find(FilePath(filePath));
-	if (it == m_includeFilePaths.end())
-	{
-		return;
-	}
-
-	if (it->second != STATE_PARSED)
-	{
-		it->second = STATE_PARSING;
-	}
-}
-
-void FileRegister::markParsingIncludeFilesParsed()
-{
-	for (auto& p : m_includeFilePaths)
-	{
-		if (p.second == STATE_PARSING)
-		{
-			p.second = STATE_PARSED;
-		}
-	}
-}
-
-std::vector<FilePath> FileRegister::getUnparsedFilePaths(const std::map<FilePath, ParseState> filePaths) const
-{
 	std::vector<FilePath> files;
 
-	for (std::pair<FilePath, ParseState>&& p : filePaths)
+	for (std::pair<FilePath, ParseState>&& p : m_sourceFilePaths)
 	{
 		if (p.second == STATE_UNPARSED)
 		{
@@ -133,21 +53,142 @@ std::vector<FilePath> FileRegister::getUnparsedFilePaths(const std::map<FilePath
 	return files;
 }
 
-size_t FileRegister::getFilesCount() const
+bool FileRegister::fileIsParsed(const FilePath& filePath) const
 {
-	return m_sourceFilePaths.size() + m_includeFilePaths.size();
+	return sourceFileIsParsed(filePath) || includeFileIsParsed(filePath);
+}
+
+bool FileRegister::sourceFileIsParsed(const FilePath& filePath) const
+{
+	{
+		std::lock_guard<std::mutex> lock(m_sourceFileMutex);
+
+		std::map<FilePath, ParseState>::const_iterator it = m_sourceFilePaths.find(filePath);
+		if (it == m_sourceFilePaths.end())
+		{
+			return false;
+		}
+
+		if (it->second == STATE_UNPARSED)
+		{
+			return false;
+		}
+		else if (it->second == STATE_PARSED)
+		{
+			return true;
+		}
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(m_threadFileMutex);
+
+		std::map<std::thread::id, std::set<FilePath>>::const_iterator it2 = m_threadParsingFiles.find(std::this_thread::get_id());
+		if (it2 != m_threadParsingFiles.end())
+		{
+			if (it2->second.find(filePath) != it2->second.end())
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+}
+
+bool FileRegister::includeFileIsParsed(const FilePath& filePath) const
+{
+	{
+		std::lock_guard<std::mutex> lock(m_includeFileMutex);
+
+		std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
+		if (it == m_includeFilePaths.end())
+		{
+			return false;
+		}
+
+		if (it->second == STATE_UNPARSED)
+		{
+			return false;
+		}
+		else if (it->second == STATE_PARSED)
+		{
+			return true;
+		}
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(m_threadFileMutex);
+
+		std::map<std::thread::id, std::set<FilePath>>::const_iterator it2 = m_threadParsingFiles.find(std::this_thread::get_id());
+		if (it2 != m_threadParsingFiles.end())
+		{
+			if (it2->second.find(filePath) != it2->second.end())
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+}
+
+FilePath FileRegister::consumeSourceFile()
+{
+	std::lock_guard<std::mutex> lock(m_sourceFileMutex);
+	for (std::map<FilePath, ParseState>::iterator it = m_sourceFilePaths.begin(); it != m_sourceFilePaths.end(); it++)
+	{
+		if (it->second == STATE_UNPARSED)
+		{
+			it->second = STATE_PARSING;
+			m_threadParsingFiles[std::this_thread::get_id()].insert(it->first);
+			return it->first;
+		}
+	}
+	return FilePath();
+}
+
+void FileRegister::markIncludeFileParsing(const FilePath& filePath)
+{
+	std::lock_guard<std::mutex> lock(m_includeFileMutex);
+	std::map<FilePath, ParseState>::iterator it = m_includeFilePaths.find(filePath);
+	if (it != m_includeFilePaths.end())
+	{
+		if (it->second == STATE_UNPARSED)
+		{
+			it->second = STATE_PARSING;
+			m_threadParsingFiles[std::this_thread::get_id()].insert(it->first);
+		}
+	}
+}
+
+void FileRegister::markThreadFilesParsed()
+{
+	std::lock_guard<std::mutex> sourceFileLock(m_sourceFileMutex);
+	std::lock_guard<std::mutex> includeFileLock(m_includeFileMutex);
+	std::lock_guard<std::mutex> threadFileLock(m_threadFileMutex);
+	for (std::set<FilePath>::iterator it = m_threadParsingFiles[std::this_thread::get_id()].begin(); it != m_threadParsingFiles[std::this_thread::get_id()].end(); it++)
+	{
+		std::map<FilePath, ParseState>::iterator it2;
+		it2 = m_sourceFilePaths.find(*it);
+		if (it2 != m_sourceFilePaths.end())
+		{
+			it2->second = STATE_PARSED;
+			continue;
+		}
+
+		it2 = m_includeFilePaths.find(*it);
+		if (it2 != m_includeFilePaths.end())
+		{
+			it2->second = STATE_PARSED;
+		}
+	}
+	m_threadParsingFiles[std::this_thread::get_id()].clear();
 }
 
 size_t FileRegister::getSourceFilesCount() const
 {
+	std::lock_guard<std::mutex> lock(m_sourceFileMutex);
 	return m_sourceFilePaths.size();
 }
 
-size_t FileRegister::getParsedFilesCount() const
-{
-	return getFilesCount() - getUnparsedSourceFilePaths().size() - getUnparsedIncludeFilePaths().size();
-}
-
 size_t FileRegister::getParsedSourceFilesCount() const
 {
 	return getSourceFilesCount() - getUnparsedSourceFilePaths().size();

src/lib/utility/file/FileRegister.h

@@ -2,7 +2,10 @@
 #define FILE_REGISTER_H
 
 #include <map>
+#include <mutex>
+#include <set>
 #include <string>
+#include <thread>
 #include <vector>
 
 #include "utility/file/FilePath.h"
@@ -14,25 +17,22 @@ class FileRegister
 public:
 	explicit FileRegister(const FileManager* fileManager);
 
-	const FileManager* getFileManager() const;
-
 	void setFilePaths(const std::vector<FilePath>& filePaths);
 
+	const FileManager* getFileManager() const;
+
 	std::vector<FilePath> getUnparsedSourceFilePaths() const;
-	std::vector<FilePath> getUnparsedIncludeFilePaths() const;
 
 	bool fileIsParsed(const FilePath& filePath) const;
-
-	bool includeFileIsParsing(const FilePath& filePath) const;
 	bool includeFileIsParsed(const FilePath& filePath) const;
+	bool sourceFileIsParsed(const FilePath& filePath) const;
 
-	void markSourceFileParsed(const std::string& filePath);
-	void markIncludeFileParsing(const std::string& filePath);
-	void markParsingIncludeFilesParsed();
+	FilePath consumeSourceFile();
+
+	void markIncludeFileParsing(const FilePath& filePath);
+	void markThreadFilesParsed();
 
-	size_t getFilesCount() const;
 	size_t getSourceFilesCount() const;
-	size_t getParsedFilesCount() const;
 	size_t getParsedSourceFilesCount() const;
 
 private:
@@ -43,12 +43,16 @@ private:
 		STATE_PARSED
 	};
 
-	std::vector<FilePath> getUnparsedFilePaths(const std::map<FilePath, ParseState> filePaths) const;
-
 	const FileManager* m_fileManager;
 
 	std::map<FilePath, ParseState> m_sourceFilePaths;
 	std::map<FilePath, ParseState> m_includeFilePaths;
+
+	std::map<std::thread::id, std::set<FilePath>> m_threadParsingFiles;
+
+	mutable std::mutex m_sourceFileMutex;
+	mutable std::mutex m_includeFileMutex;
+	mutable std::mutex m_threadFileMutex;
 };
 
 #endif // FILE_REGISTER_H

src/lib/utility/scheduling/LambdaTask.cpp

@@ -9,7 +9,25 @@ LambdaTask::~LambdaTask()
 {
 }
 
-void LambdaTask::perform()
+void LambdaTask::enter()
+{
+}
+
+Task::TaskState LambdaTask::update()
 {
 	m_func();
+
+	return Task::STATE_FINISHED;
+}
+
+void LambdaTask::exit()
+{
+}
+
+void LambdaTask::interrupt()
+{
+}
+
+void LambdaTask::revert()
+{
 }

src/lib/utility/scheduling/LambdaTask.h

@@ -3,16 +3,21 @@
 
 #include <functional>
 
-#include "utility/scheduling/SimpleTask.h"
+#include "utility/scheduling/Task.h"
 
 class LambdaTask
-	: public SimpleTask
+	: public Task
 {
 public:
 	LambdaTask(std::function<void()> func);
 	virtual ~LambdaTask();
 
-	virtual void perform();
+	virtual void enter();
+	virtual TaskState update();
+	virtual void exit();
+
+	virtual void interrupt();
+	virtual void revert();
 
 private:
 	std::function<void()> m_func;

src/lib/utility/scheduling/SimpleTask.cpp

@@ -1,28 +0,0 @@
-#include "utility/scheduling/SimpleTask.h"
-
-void SimpleTask::enter()
-{
-
-}
-
-Task::TaskState SimpleTask::update()
-{
-	perform();
-
-	return Task::STATE_FINISHED;
-}
-
-void SimpleTask::exit()
-{
-
-}
-
-void SimpleTask::interrupt()
-{
-
-}
-
-void SimpleTask::revert()
-{
-
-}

src/lib/utility/scheduling/SimpleTask.h

@@ -1,20 +0,0 @@
-#ifndef SIMPLE_TASK_H
-#define SIMPLE_TASK_H
-
-#include "utility/scheduling/Task.h"
-
-class SimpleTask
-	: public Task
-{
-public:
-	virtual void enter();
-	virtual TaskState update();
-	virtual void exit();
-
-	virtual void interrupt();
-	virtual void revert();
-
-	virtual void perform() = 0;
-};
-
-#endif // SIMPLE_TASK_H

src/lib/utility/scheduling/TaskGroupParallel.cpp

@@ -0,0 +1,75 @@
+#include "utility/scheduling/TaskGroupParallel.h"
+
+#include <thread>
+
+TaskGroupParallel::TaskGroupParallel()
+{
+}
+
+TaskGroupParallel::~TaskGroupParallel()
+{
+}
+
+void TaskGroupParallel::enter()
+{
+	m_interrupt = false;
+	m_running = false;
+	m_activeTaskCount = 0;
+}
+
+Task::TaskState TaskGroupParallel::update()
+{
+	if (!m_running)
+	{
+		for (size_t i = 0; i < m_tasks.size(); i++)
+		{
+			std::thread(&TaskGroupParallel::processTask, this, m_tasks[i]).detach();
+
+			std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
+			m_activeTaskCount++;
+		}
+		m_running = true;
+	}
+
+	int activeTaskCount = 0;
+	{
+		std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
+		activeTaskCount = m_activeTaskCount;
+	}
+
+	if (activeTaskCount == 0)
+	{
+		return (m_interrupt ? STATE_CANCELED : STATE_FINISHED);
+	}
+
+	return Task::STATE_RUNNING;
+}
+
+void TaskGroupParallel::exit()
+{
+}
+
+void TaskGroupParallel::interrupt()
+{
+	m_interrupt = true;
+}
+
+void TaskGroupParallel::revert()
+{
+	m_interrupt = true;
+}
+
+
+void TaskGroupParallel::processTask(std::shared_ptr<Task> task)
+{
+	Task::TaskState state = Task::STATE_NEW;
+	while (state != Task::STATE_FINISHED && state != Task::STATE_CANCELED)
+	{
+		state = task->process(m_interrupt);
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
+		m_activeTaskCount--; // not safe! if exception hits this thread before this point the count is not decremented.
+	}
+}

src/lib/utility/scheduling/TaskGroupParallel.h

@@ -0,0 +1,30 @@
+#ifndef TASK_GROUP_PARALLEL_H
+#define TASK_GROUP_PARALLEL_H
+
+#include "utility/scheduling/TaskGroup.h"
+#include <mutex>
+
+class TaskGroupParallel
+	: public TaskGroup
+{
+public:
+	TaskGroupParallel();
+	virtual ~TaskGroupParallel();
+
+	virtual void enter();
+	virtual TaskState update();
+	virtual void exit();
+
+	virtual void interrupt();
+	virtual void revert();
+
+private:
+	void processTask(std::shared_ptr<Task> task);
+
+	volatile bool m_interrupt;
+	bool m_running;
+	volatile int m_activeTaskCount;
+	std::mutex m_activeTaskCountMutex;
+};
+
+#endif // TASK_GROUP_PARALLEL_H

src/lib_parser/data/parser/cxx/ASTAction.cpp

@@ -29,8 +29,3 @@ bool ASTAction::BeginSourceFileAction(clang::CompilerInstance& compiler, llvm::S
 	preprocessor.addCommentHandler(&m_commentHandler);
 	return true;
 }
-
-void ASTAction::EndSourceFileAction()
-{
-	m_fileRegister->markParsingIncludeFilesParsed();
-}

src/lib_parser/data/parser/cxx/ASTAction.h

@@ -18,7 +18,6 @@ protected:
 	virtual std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(clang::CompilerInstance& compiler, llvm::StringRef inFile);
 
 	virtual bool BeginSourceFileAction(clang::CompilerInstance& compiler, llvm::StringRef filePath);
-	virtual void EndSourceFileAction();
 
 private:
 	ParserClient* m_client;

src/lib_parser/data/parser/cxx/ASTVisitor.cpp

@@ -1503,7 +1503,7 @@ bool ASTVisitor::isLocatedInUnparsedProjectFile(clang::SourceLocation loc)
 			{
                 std::string fileName = fileEntry->getName();
                 FilePath filePath = FilePath(fileName).canonical();
-				ret = m_fileRegister->includeFileIsParsing(filePath.str());
+				ret = m_fileRegister->includeFileIsParsed(filePath.str());
 			}
 		}
 		m_inUnparsedProjectFileMap[fileId] = ret;

src/lib_parser/data/parser/cxx/CxxParser.cpp

@@ -2,7 +2,6 @@
 
 #include "clang/Tooling/Tooling.h"
 
-#include "utility/file/FileManager.h"
 #include "utility/file/FileRegister.h"
 #include "utility/logging/logging.h"
 #include "utility/text/TextAccess.h"
@@ -53,9 +52,9 @@ namespace
 }
 
 
-CxxParser::CxxParser(ParserClient* client, const FileManager* fileManager)
+CxxParser::CxxParser(ParserClient* client, std::shared_ptr<FileRegister> fileRegister)
 	: Parser(client)
-	, m_fileRegister(std::make_shared<FileRegister>(fileManager))
+	, m_fileRegister(fileRegister)
 {
 }
 
@@ -65,29 +64,22 @@ CxxParser::~CxxParser()
 
 void CxxParser::parseFiles(const std::vector<FilePath>& filePaths, const Arguments& arguments)
 {
-	setupParsing(filePaths, arguments);
+	m_fileRegister->setFilePaths(filePaths);
+	setupParsing(arguments);
 
 	std::vector<std::string> sourcePaths;
-	for (const FilePath& path : m_fileRegister->getUnparsedSourceFilePaths())
+	for (const FilePath& path : m_fileRegister->getUnparsedSourceFilePaths()) // filter headers
 	{
 		sourcePaths.push_back(path.absolute().str());
 	}
 
 	runTool(sourcePaths);
-
-	std::vector<FilePath> unparsedHeaders = m_fileRegister->getUnparsedIncludeFilePaths();
-	for (const FilePath& path : unparsedHeaders)
-	{
-		if (!m_fileRegister->includeFileIsParsed(path))
-		{
-			runTool(std::vector<std::string>(1, path.str()));
-		}
-	}
 }
 
 void CxxParser::parseFile(const FilePath& filePath, std::shared_ptr<TextAccess> textAccess, const Arguments& arguments)
 {
-	setupParsing(std::vector<FilePath>(1, filePath), arguments);
+	m_fileRegister->setFilePaths(std::vector<FilePath>(1, filePath));
+	setupParsing(arguments);
 
 	std::vector<std::string> args = getCommandlineArguments(arguments);
 	std::shared_ptr<CxxDiagnosticConsumer> diagnostics = getDiagnostics(arguments);
@@ -190,16 +182,14 @@ std::shared_ptr<CxxDiagnosticConsumer> CxxParser::getDiagnostics(const Arguments
 		llvm::errs(), &*options, m_client, m_fileRegister->getFileManager(), arguments.logErrors);
 }
 
-void CxxParser::setupParsing(const std::vector<FilePath>& filePaths, const Arguments& arguments)
+void CxxParser::setupParsing(const Arguments& arguments)
 {
-	m_fileRegister->setFilePaths(filePaths);
 	m_compilationDatabase = getCompilationDatabase(arguments);
 	m_diagnostics = getDiagnostics(arguments);
 }
 
-void CxxParser::setupParsingCDB(const std::vector<FilePath>& filePaths, const Arguments& arguments)
+void CxxParser::setupParsingCDB(const Arguments& arguments)
 {
-	m_fileRegister->setFilePaths(filePaths);
 	m_diagnostics = getDiagnostics(arguments);
 }
 

src/lib_parser/data/parser/cxx/CxxParser.h

@@ -5,14 +5,14 @@
 #include "data/parser/Parser.h"
 
 class CxxDiagnosticConsumer;
-class FileManager;
+class FileRegister;
 class FileRegister;
 class TaskParseCxx;
 
 class CxxParser: public Parser
 {
 public:
-	CxxParser(ParserClient* client, const FileManager* fileManager);
+	CxxParser(ParserClient* client, std::shared_ptr<FileRegister> fileRegister);
 	~CxxParser();
 
 	// ParserClient implementation
@@ -27,8 +27,8 @@ private:
 	std::shared_ptr<CxxDiagnosticConsumer> getDiagnostics(const Arguments& arguments) const;
 
 	// Accessed by TaskParseCxx
-	void setupParsing(const std::vector<FilePath>& filePaths, const Arguments& arguments);
-	void setupParsingCDB(const std::vector<FilePath>& filePaths, const Arguments& arguments);
+	void setupParsing(const Arguments& arguments);
+	void setupParsingCDB(const Arguments& arguments);
 
 	void runTool(const std::vector<std::string>& files);
 	void runTool(clang::tooling::CompileCommand command, const Arguments& arguments);

src/test/CxxParserTestSuite.h

@@ -2779,8 +2779,9 @@ public:
 	void test_cxx_parser_parses_multiple_files()
 	{
 		TestFileManager fm;
+		std::shared_ptr<FileRegister> fr = std::make_shared<FileRegister>(&fm);
 		TestParserClient client;
-		CxxParser parser(&client, &fm);
+		CxxParser parser(&client, fr);
 
 		std::vector<FilePath> filePaths;
 		filePaths.push_back(FilePath("data/CxxParserTestSuite/header.h"));
@@ -2866,19 +2867,11 @@ private:
 	class TestParserClient: public ParserClient
 	{
 	public:
-		virtual void startParsing()
+		virtual void startParsingFile()
 		{
 		}
 
-		virtual void finishParsing()
-		{
-		}
-
-		virtual void startParsingFile(const FilePath& filePath)
-		{
-		}
-
-		virtual void finishParsingFile(const FilePath& filePath)
+		virtual void finishParsingFile()
 		{
 		}
 
@@ -3109,8 +3102,9 @@ private:
 		m_args.languageStandard = "1z";
 
 		TestFileManager fm;
+		std::shared_ptr<FileRegister> fr = std::make_shared<FileRegister>(&fm);
 		std::shared_ptr<TestParserClient> client = std::make_shared<TestParserClient>();
-		CxxParser parser(client.get(), &fm);
+		CxxParser parser(client.get(), fr);
 		parser.parseFile("input.cc", TextAccess::createFromString(code), m_args);
 		return client;
 	}

src/trial/CMakeLists.txt

@@ -2,6 +2,7 @@ add_files(
 	TRIAL_FILES
 
 	data/parser/cxx/TaskParseCxx.cpp
+	data/parser/cxx/TaskParseWrapper.cpp
 
 	isTrial.cpp
 	main.cpp

src/trial/data/parser/cxx/TaskParseCxx.cpp

@@ -1,16 +1,11 @@
 #include "data/parser/cxx/TaskParseCxx.h"
 
-#include "data/PersistentStorage.h"
-#include "utility/messaging/type/MessageFinishedParsing.h"
-
 TaskParseCxx::TaskParseCxx(
 	PersistentStorage* storage,
-	const FileManager* fileManager,
-	const Parser::Arguments& arguments,
-	const std::vector<FilePath>& files
+	std::shared_ptr<std::mutex> storageMutex,
+	std::shared_ptr<FileRegister> fileRegister,
+	const Parser::Arguments& arguments
 )
-	: m_storage(storage)
-	, m_arguments(arguments)
 {
 }
 
@@ -21,7 +16,6 @@ std::vector<FilePath> TaskParseCxx::getSourceFilesFromCDB(const FilePath& compil
 
 void TaskParseCxx::enter()
 {
-	m_storage->startParsing();
 }
 
 Task::TaskState TaskParseCxx::update()
@@ -31,9 +25,6 @@ Task::TaskState TaskParseCxx::update()
 
 void TaskParseCxx::exit()
 {
-	m_storage->finishParsing();
-
-	MessageFinishedParsing(0, 0, 0).dispatch();
 }
 
 void TaskParseCxx::interrupt()

src/trial/data/parser/cxx/TaskParseWrapper.cpp

@@ -0,0 +1,45 @@
+#include "data/parser/cxx/TaskParseWrapper.h"
+
+#include "data/PersistentStorage.h"
+#include "utility/messaging/type/MessageFinishedParsing.h"
+
+TaskParseWrapper::TaskParseWrapper(
+	std::shared_ptr<Task> child,
+	PersistentStorage* storage,
+	std::shared_ptr<FileRegister> fileRegister
+)
+	: m_child(child)
+	, m_storage(storage)
+{
+}
+
+void TaskParseWrapper::enter()
+{
+	m_storage->startParsing();
+
+	m_child->enter();
+}
+
+Task::TaskState TaskParseWrapper::update()
+{
+	return m_child->update();
+}
+
+void TaskParseWrapper::exit()
+{
+	m_child->exit();
+
+	m_storage->finishParsing();
+
+	MessageFinishedParsing(0, 0, 0).dispatch();
+}
+
+void TaskParseWrapper::interrupt()
+{
+	m_child->interrupt();
+}
+
+void TaskParseWrapper::revert()
+{
+	m_child->revert();
+}