logic: multithreaded parsing

* modified the TaskParseCxx to be able to run multiple times in parallel
* made FileRegister threadsafe and changed a lot of its mechanisms
* added TaskGroupParallel that runs all children in parallel
* added TaskParseWrapper that acts as a decorator to execute some code before and after parsing.
* implemented task setup in project with 4 parsing threads
* removed SimpleTask as it was only used as interface for the LambdaTask
This commit is contained in:
malte_langkabel
2016-05-13 13:23:54 +02:00
parent 245cafdc7b
commit c61efbbcde
27 changed files with 535 additions and 317 deletions
+3 -2
View File
@@ -124,6 +124,7 @@ add_files(
data/name/NameHierarchy.h
data/parser/cxx/TaskParseCxx.h
data/parser/cxx/TaskParseWrapper.h
data/parser/ParseLocation.cpp
data/parser/ParseLocation.h
@@ -274,12 +275,12 @@ add_files(
utility/scheduling/LambdaTask.cpp
utility/scheduling/LambdaTask.h
utility/scheduling/SimpleTask.cpp
utility/scheduling/SimpleTask.h
utility/scheduling/Task.cpp
utility/scheduling/Task.h
utility/scheduling/TaskGroup.cpp
utility/scheduling/TaskGroup.h
utility/scheduling/TaskGroupParallel.cpp
utility/scheduling/TaskGroupParallel.h
utility/scheduling/TaskGroupSequential.cpp
utility/scheduling/TaskGroupSequential.h
utility/scheduling/TaskScheduler.cpp
+33 -14
View File
@@ -1,20 +1,23 @@
#include "Project.h"
#include "utility/file/FileSystem.h"
#include "utility/logging/logging.h"
#include "utility/messaging/type/MessageFinishedParsing.h"
#include "utility/scheduling/TaskGroupSequential.h"
#include "utility/utility.h"
#include "utility/Version.h"
#include "data/access/StorageAccessProxy.h"
#include "data/graph/Token.h"
#include "data/parser/cxx/TaskParseCxx.h"
#include "data/parser/cxx/TaskParseWrapper.h"
#include "data/PersistentStorage.h"
#include "data/TaskCleanStorage.h"
#include "settings/ApplicationSettings.h"
#include "settings/ProjectSettings.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileSystem.h"
#include "utility/logging/logging.h"
#include "utility/messaging/type/MessageFinishedParsing.h"
#include "utility/scheduling/TaskGroupSequential.h"
#include "utility/scheduling/TaskGroupParallel.h"
#include "utility/utility.h"
#include "utility/Version.h"
std::shared_ptr<Project> Project::create(StorageAccessProxy* storageAccessProxy)
{
std::shared_ptr<Project> ptr(new Project(storageAccessProxy));
@@ -149,26 +152,42 @@ void Project::parseCode()
utility::append(updatedFilePaths, m_storage->getDependingFilePaths(updatedFilePaths));
utility::append(updatedFilePaths, m_storage->getDependingFilePaths(removedFilePaths));
std::shared_ptr<TaskGroupSequential> taskGroup = std::make_shared<TaskGroupSequential>();
std::shared_ptr<TaskGroupSequential> taskSequential = std::make_shared<TaskGroupSequential>();
std::vector<FilePath> filesToClean;
filesToClean.insert(filesToClean.end(), removedFilePaths.begin(), removedFilePaths.end());
filesToClean.insert(filesToClean.end(), updatedFilePaths.begin(), updatedFilePaths.end());
taskGroup->addTask(std::make_shared<TaskCleanStorage>(m_storage.get(), filesToClean));
taskSequential->addTask(std::make_shared<TaskCleanStorage>(m_storage.get(), filesToClean));
std::vector<FilePath> filesToParse;
filesToParse.insert(filesToParse.end(), addedFilePaths.begin(), addedFilePaths.end());
filesToParse.insert(filesToParse.end(), updatedFilePaths.begin(), updatedFilePaths.end());
taskGroup->addTask(std::make_shared<TaskParseCxx>(
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(&m_fileManager);
fileRegister->setFilePaths(filesToParse);
std::shared_ptr<TaskGroupParallel> taskParallel = std::make_shared<TaskGroupParallel>();
taskSequential->addTask(std::make_shared<TaskParseWrapper>(
taskParallel,
m_storage.get(),
&m_fileManager,
getParserArguments(),
filesToParse
fileRegister
));
Task::dispatch(taskGroup);
std::shared_ptr<std::mutex> storageMutex = std::make_shared<std::mutex>();
for (int i = 0; i < 4; i++)
{
taskParallel->addTask(std::make_shared<TaskParseCxx>(
m_storage.get(),
storageMutex,
fileRegister,
getParserArguments()
));
}
Task::dispatch(taskSequential);
m_state = PROJECT_LOADED;
}
+2 -2
View File
@@ -39,8 +39,8 @@ public:
ParserClient();
virtual ~ParserClient();
virtual void startParsingFile(const FilePath& filePath) = 0;
virtual void finishParsingFile(const FilePath& filePath) = 0;
virtual void startParsingFile() = 0;
virtual void finishParsingFile() = 0;
virtual void onError(const ParseLocation& location, const std::string& message, bool fatal) = 0;
+3 -3
View File
@@ -25,12 +25,12 @@ void ParserClientImpl::resetStorage()
m_storage.reset();
}
void ParserClientImpl::startParsingFile(const FilePath& filePath)
void ParserClientImpl::startParsingFile()
{
m_nodeIdsToMemberEdgeIds.clear();
m_nodeIdsToMemberEdgeIds.clear(); // remove this when one parserclient is created per file
}
void ParserClientImpl::finishParsingFile(const FilePath& filePath)
void ParserClientImpl::finishParsingFile()
{
}
+2 -2
View File
@@ -18,8 +18,8 @@ public:
void setStorage(std::shared_ptr<IntermediateStorage> storage);
void resetStorage();
virtual void startParsingFile(const FilePath& filePath);
virtual void finishParsingFile(const FilePath& filePath);
virtual void startParsingFile();
virtual void finishParsingFile();
virtual void onError(const ParseLocation& location, const std::string& message, bool fatal);
+8 -11
View File
@@ -2,6 +2,7 @@
#define TASK_PARSE_CXX_H
#include <memory>
#include <mutex>
#include <deque>
#include "data/parser/Parser.h"
@@ -10,7 +11,7 @@
#include "utility/TimePoint.h"
class PersistentStorage;
class FileManager;
class FileRegister;
class CxxParser;
namespace clang
@@ -25,15 +26,15 @@ class TaskParseCxx
: public Task
{
public:
static std::vector<FilePath> getSourceFilesFromCDB(const FilePath& compilationDatabasePath);
TaskParseCxx(
PersistentStorage* storage,
const FileManager* fileManager,
const Parser::Arguments& arguments,
const std::vector<FilePath>& files
std::shared_ptr<std::mutex> storageMutex,
std::shared_ptr<FileRegister> fileRegister,
const Parser::Arguments& arguments
);
static std::vector<FilePath> getSourceFilesFromCDB(const FilePath& compilationDatabasePath);
virtual void enter();
virtual TaskState update();
virtual void exit();
@@ -43,14 +44,10 @@ public:
private:
PersistentStorage* m_storage;
std::shared_ptr<std::mutex> m_storageMutex;
std::shared_ptr<CxxParser> m_parser;
std::shared_ptr<ParserClientImpl> m_parserClient;
const Parser::Arguments m_arguments;
const std::vector<FilePath> m_files;
std::deque<FilePath> m_sourcePaths;
TimePoint m_start;
bool m_isCDB;
std::shared_ptr<clang::tooling::JSONCompilationDatabase> m_cdb;
@@ -0,0 +1,41 @@
#ifndef TASK_PARSE_WRAPPER_H
#define TASK_PARSE_WRAPPER_H
#include <memory>
#include "data/parser/Parser.h"
#include "data/parser/ParserClientImpl.h"
#include "utility/scheduling/Task.h"
#include "utility/TimePoint.h"
class PersistentStorage;
class FileRegister;
class CxxParser;
class TaskParseWrapper
: public Task
{
public:
TaskParseWrapper(
std::shared_ptr<Task> child,
PersistentStorage* storage,
std::shared_ptr<FileRegister> fileRegister
);
virtual void enter();
virtual TaskState update();
virtual void exit();
virtual void interrupt();
virtual void revert();
private:
std::shared_ptr<Task> m_child;
PersistentStorage* m_storage;
std::shared_ptr<FileRegister> m_fileRegister;
TimePoint m_start;
};
#endif // TASK_PARSE_WRAPPER_H
+138 -97
View File
@@ -8,13 +8,11 @@ FileRegister::FileRegister(const FileManager* fileManager)
{
}
const FileManager* FileRegister::getFileManager() const
{
return m_fileManager;
}
void FileRegister::setFilePaths(const std::vector<FilePath>& filePaths)
{
std::lock_guard<std::mutex> sourceFileLock(m_sourceFileMutex);
std::lock_guard<std::mutex> includeFileLock(m_includeFileMutex);
m_sourceFilePaths.clear();
m_includeFilePaths.clear();
@@ -33,96 +31,18 @@ void FileRegister::setFilePaths(const std::vector<FilePath>& filePaths)
}
}
const FileManager* FileRegister::getFileManager() const
{
return m_fileManager;
}
std::vector<FilePath> FileRegister::getUnparsedSourceFilePaths() const
{
return getUnparsedFilePaths(m_sourceFilePaths);
}
std::lock_guard<std::mutex> lock(m_sourceFileMutex);
std::vector<FilePath> FileRegister::getUnparsedIncludeFilePaths() const
{
return getUnparsedFilePaths(m_includeFilePaths);
}
bool FileRegister::fileIsParsed(const FilePath& filePath) const
{
std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
if (it != m_includeFilePaths.end())
{
return it->second == STATE_PARSED;
}
it = m_sourceFilePaths.find(filePath);
if (it != m_sourceFilePaths.end())
{
return it->second == STATE_PARSED;
}
return true;
}
bool FileRegister::includeFileIsParsing(const FilePath& filePath) const
{
std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
if (it == m_includeFilePaths.end())
{
return false;
}
return it->second == STATE_PARSING;
}
bool FileRegister::includeFileIsParsed(const FilePath& filePath) const
{
std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
if (it == m_includeFilePaths.end())
{
return false;
}
return it->second == STATE_PARSED;
}
void FileRegister::markSourceFileParsed(const std::string& filePath)
{
std::map<FilePath, ParseState>::iterator it = m_sourceFilePaths.find(FilePath(filePath));
if (it == m_sourceFilePaths.end())
{
return;
}
it->second = STATE_PARSED;
}
void FileRegister::markIncludeFileParsing(const std::string& filePath)
{
std::map<FilePath, ParseState>::iterator it = m_includeFilePaths.find(FilePath(filePath));
if (it == m_includeFilePaths.end())
{
return;
}
if (it->second != STATE_PARSED)
{
it->second = STATE_PARSING;
}
}
void FileRegister::markParsingIncludeFilesParsed()
{
for (auto& p : m_includeFilePaths)
{
if (p.second == STATE_PARSING)
{
p.second = STATE_PARSED;
}
}
}
std::vector<FilePath> FileRegister::getUnparsedFilePaths(const std::map<FilePath, ParseState> filePaths) const
{
std::vector<FilePath> files;
for (std::pair<FilePath, ParseState>&& p : filePaths)
for (std::pair<FilePath, ParseState>&& p : m_sourceFilePaths)
{
if (p.second == STATE_UNPARSED)
{
@@ -133,21 +53,142 @@ std::vector<FilePath> FileRegister::getUnparsedFilePaths(const std::map<FilePath
return files;
}
size_t FileRegister::getFilesCount() const
bool FileRegister::fileIsParsed(const FilePath& filePath) const
{
return m_sourceFilePaths.size() + m_includeFilePaths.size();
return sourceFileIsParsed(filePath) || includeFileIsParsed(filePath);
}
bool FileRegister::sourceFileIsParsed(const FilePath& filePath) const
{
{
std::lock_guard<std::mutex> lock(m_sourceFileMutex);
std::map<FilePath, ParseState>::const_iterator it = m_sourceFilePaths.find(filePath);
if (it == m_sourceFilePaths.end())
{
return false;
}
if (it->second == STATE_UNPARSED)
{
return false;
}
else if (it->second == STATE_PARSED)
{
return true;
}
}
{
std::lock_guard<std::mutex> lock(m_threadFileMutex);
std::map<std::thread::id, std::set<FilePath>>::const_iterator it2 = m_threadParsingFiles.find(std::this_thread::get_id());
if (it2 != m_threadParsingFiles.end())
{
if (it2->second.find(filePath) != it2->second.end())
{
return false;
}
}
return true;
}
}
bool FileRegister::includeFileIsParsed(const FilePath& filePath) const
{
{
std::lock_guard<std::mutex> lock(m_includeFileMutex);
std::map<FilePath, ParseState>::const_iterator it = m_includeFilePaths.find(filePath);
if (it == m_includeFilePaths.end())
{
return false;
}
if (it->second == STATE_UNPARSED)
{
return false;
}
else if (it->second == STATE_PARSED)
{
return true;
}
}
{
std::lock_guard<std::mutex> lock(m_threadFileMutex);
std::map<std::thread::id, std::set<FilePath>>::const_iterator it2 = m_threadParsingFiles.find(std::this_thread::get_id());
if (it2 != m_threadParsingFiles.end())
{
if (it2->second.find(filePath) != it2->second.end())
{
return false;
}
}
return true;
}
}
FilePath FileRegister::consumeSourceFile()
{
std::lock_guard<std::mutex> lock(m_sourceFileMutex);
for (std::map<FilePath, ParseState>::iterator it = m_sourceFilePaths.begin(); it != m_sourceFilePaths.end(); it++)
{
if (it->second == STATE_UNPARSED)
{
it->second = STATE_PARSING;
m_threadParsingFiles[std::this_thread::get_id()].insert(it->first);
return it->first;
}
}
return FilePath();
}
void FileRegister::markIncludeFileParsing(const FilePath& filePath)
{
std::lock_guard<std::mutex> lock(m_includeFileMutex);
std::map<FilePath, ParseState>::iterator it = m_includeFilePaths.find(filePath);
if (it != m_includeFilePaths.end())
{
if (it->second == STATE_UNPARSED)
{
it->second = STATE_PARSING;
m_threadParsingFiles[std::this_thread::get_id()].insert(it->first);
}
}
}
void FileRegister::markThreadFilesParsed()
{
std::lock_guard<std::mutex> sourceFileLock(m_sourceFileMutex);
std::lock_guard<std::mutex> includeFileLock(m_includeFileMutex);
std::lock_guard<std::mutex> threadFileLock(m_threadFileMutex);
for (std::set<FilePath>::iterator it = m_threadParsingFiles[std::this_thread::get_id()].begin(); it != m_threadParsingFiles[std::this_thread::get_id()].end(); it++)
{
std::map<FilePath, ParseState>::iterator it2;
it2 = m_sourceFilePaths.find(*it);
if (it2 != m_sourceFilePaths.end())
{
it2->second = STATE_PARSED;
continue;
}
it2 = m_includeFilePaths.find(*it);
if (it2 != m_includeFilePaths.end())
{
it2->second = STATE_PARSED;
}
}
m_threadParsingFiles[std::this_thread::get_id()].clear();
}
size_t FileRegister::getSourceFilesCount() const
{
std::lock_guard<std::mutex> lock(m_sourceFileMutex);
return m_sourceFilePaths.size();
}
size_t FileRegister::getParsedFilesCount() const
{
return getFilesCount() - getUnparsedSourceFilePaths().size() - getUnparsedIncludeFilePaths().size();
}
size_t FileRegister::getParsedSourceFilesCount() const
{
return getSourceFilesCount() - getUnparsedSourceFilePaths().size();
+16 -12
View File
@@ -2,7 +2,10 @@
#define FILE_REGISTER_H
#include <map>
#include <mutex>
#include <set>
#include <string>
#include <thread>
#include <vector>
#include "utility/file/FilePath.h"
@@ -14,25 +17,22 @@ class FileRegister
public:
explicit FileRegister(const FileManager* fileManager);
const FileManager* getFileManager() const;
void setFilePaths(const std::vector<FilePath>& filePaths);
const FileManager* getFileManager() const;
std::vector<FilePath> getUnparsedSourceFilePaths() const;
std::vector<FilePath> getUnparsedIncludeFilePaths() const;
bool fileIsParsed(const FilePath& filePath) const;
bool includeFileIsParsing(const FilePath& filePath) const;
bool includeFileIsParsed(const FilePath& filePath) const;
bool sourceFileIsParsed(const FilePath& filePath) const;
void markSourceFileParsed(const std::string& filePath);
void markIncludeFileParsing(const std::string& filePath);
void markParsingIncludeFilesParsed();
FilePath consumeSourceFile();
void markIncludeFileParsing(const FilePath& filePath);
void markThreadFilesParsed();
size_t getFilesCount() const;
size_t getSourceFilesCount() const;
size_t getParsedFilesCount() const;
size_t getParsedSourceFilesCount() const;
private:
@@ -43,12 +43,16 @@ private:
STATE_PARSED
};
std::vector<FilePath> getUnparsedFilePaths(const std::map<FilePath, ParseState> filePaths) const;
const FileManager* m_fileManager;
std::map<FilePath, ParseState> m_sourceFilePaths;
std::map<FilePath, ParseState> m_includeFilePaths;
std::map<std::thread::id, std::set<FilePath>> m_threadParsingFiles;
mutable std::mutex m_sourceFileMutex;
mutable std::mutex m_includeFileMutex;
mutable std::mutex m_threadFileMutex;
};
#endif // FILE_REGISTER_H
+19 -1
View File
@@ -9,7 +9,25 @@ LambdaTask::~LambdaTask()
{
}
void LambdaTask::perform()
void LambdaTask::enter()
{
}
Task::TaskState LambdaTask::update()
{
m_func();
return Task::STATE_FINISHED;
}
void LambdaTask::exit()
{
}
void LambdaTask::interrupt()
{
}
void LambdaTask::revert()
{
}
+8 -3
View File
@@ -3,16 +3,21 @@
#include <functional>
#include "utility/scheduling/SimpleTask.h"
#include "utility/scheduling/Task.h"
class LambdaTask
: public SimpleTask
: public Task
{
public:
LambdaTask(std::function<void()> func);
virtual ~LambdaTask();
virtual void perform();
virtual void enter();
virtual TaskState update();
virtual void exit();
virtual void interrupt();
virtual void revert();
private:
std::function<void()> m_func;
-28
View File
@@ -1,28 +0,0 @@
#include "utility/scheduling/SimpleTask.h"
void SimpleTask::enter()
{
}
Task::TaskState SimpleTask::update()
{
perform();
return Task::STATE_FINISHED;
}
void SimpleTask::exit()
{
}
void SimpleTask::interrupt()
{
}
void SimpleTask::revert()
{
}
-20
View File
@@ -1,20 +0,0 @@
#ifndef SIMPLE_TASK_H
#define SIMPLE_TASK_H
#include "utility/scheduling/Task.h"
class SimpleTask
: public Task
{
public:
virtual void enter();
virtual TaskState update();
virtual void exit();
virtual void interrupt();
virtual void revert();
virtual void perform() = 0;
};
#endif // SIMPLE_TASK_H
@@ -0,0 +1,75 @@
#include "utility/scheduling/TaskGroupParallel.h"
#include <thread>
TaskGroupParallel::TaskGroupParallel()
{
}
TaskGroupParallel::~TaskGroupParallel()
{
}
void TaskGroupParallel::enter()
{
m_interrupt = false;
m_running = false;
m_activeTaskCount = 0;
}
Task::TaskState TaskGroupParallel::update()
{
if (!m_running)
{
for (size_t i = 0; i < m_tasks.size(); i++)
{
std::thread(&TaskGroupParallel::processTask, this, m_tasks[i]).detach();
std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
m_activeTaskCount++;
}
m_running = true;
}
int activeTaskCount = 0;
{
std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
activeTaskCount = m_activeTaskCount;
}
if (activeTaskCount == 0)
{
return (m_interrupt ? STATE_CANCELED : STATE_FINISHED);
}
return Task::STATE_RUNNING;
}
void TaskGroupParallel::exit()
{
}
void TaskGroupParallel::interrupt()
{
m_interrupt = true;
}
void TaskGroupParallel::revert()
{
m_interrupt = true;
}
void TaskGroupParallel::processTask(std::shared_ptr<Task> task)
{
Task::TaskState state = Task::STATE_NEW;
while (state != Task::STATE_FINISHED && state != Task::STATE_CANCELED)
{
state = task->process(m_interrupt);
}
{
std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
m_activeTaskCount--; // not safe! if exception hits this thread before this point the count is not decremented.
}
}
@@ -0,0 +1,30 @@
#ifndef TASK_GROUP_PARALLEL_H
#define TASK_GROUP_PARALLEL_H
#include "utility/scheduling/TaskGroup.h"
#include <mutex>
class TaskGroupParallel
: public TaskGroup
{
public:
TaskGroupParallel();
virtual ~TaskGroupParallel();
virtual void enter();
virtual TaskState update();
virtual void exit();
virtual void interrupt();
virtual void revert();
private:
void processTask(std::shared_ptr<Task> task);
volatile bool m_interrupt;
bool m_running;
volatile int m_activeTaskCount;
std::mutex m_activeTaskCountMutex;
};
#endif // TASK_GROUP_PARALLEL_H