logic: added option to execute custom indexer commands in parallel

This commit is contained in:
mlangkabel
2019-01-21 13:28:51 +01:00
parent 85bd0d39c3
commit 3eb9211520
10 changed files with 289 additions and 60 deletions
+38 -3
View File
@@ -3,17 +3,27 @@
#include <QJsonArray>
#include <QJsonObject>
#include "utilityString.h"
IndexerCommandType IndexerCommandCustom::getStaticIndexerCommandType()
{
return INDEXER_COMMAND_CUSTOM;
}
IndexerCommandCustom::IndexerCommandCustom(
const std::wstring& customCommand,
const FilePath& projectFilePath,
const FilePath& databaseFilePath,
const std::wstring& databaseVersion,
const FilePath& sourceFilePath,
const std::wstring& customCommand
bool runInParallel
)
: IndexerCommand(sourceFilePath)
, m_customCommand(customCommand)
, m_projectFilePath(projectFilePath)
, m_databaseFilePath(databaseFilePath)
, m_databaseVersion(databaseVersion)
, m_runInParallel(runInParallel)
{
}
@@ -29,9 +39,31 @@ size_t IndexerCommandCustom::getByteSize(size_t stringSize) const
return size;
}
const std::wstring& IndexerCommandCustom::getCustomCommand() const
FilePath IndexerCommandCustom::getDatabaseFilePath() const
{
return m_customCommand;
return m_databaseFilePath;
}
void IndexerCommandCustom::setDatabaseFilePath(const FilePath& databaseFilePath)
{
m_databaseFilePath = databaseFilePath;
}
std::wstring IndexerCommandCustom::getCustomCommand() const
{
std::wstring command = m_customCommand;
command = utility::replace(command, L"%{PROJECT_FILE_PATH}", L'\"' + m_projectFilePath.wstr() + L'\"');
command = utility::replace(command, L"%{DATABASE_FILE_PATH}", L'\"' + m_databaseFilePath.wstr() + L'\"');
command = utility::replace(command, L"%{DATABASE_VERSION}", L'\"' + m_databaseVersion + L'\"');
command = utility::replace(command, L"%{SOURCE_FILE_PATH}", L'\"' + getSourceFilePath().wstr() + L'\"');
return command;
}
bool IndexerCommandCustom::getRunInParallel() const
{
return m_runInParallel;
}
QJsonObject IndexerCommandCustom::doSerialize() const
@@ -41,6 +73,9 @@ QJsonObject IndexerCommandCustom::doSerialize() const
{
jsonObject["custom_command"] = QString::fromStdWString(m_customCommand);
}
{
jsonObject["run_in_parallel"] = m_runInParallel;
}
return jsonObject;
}
+16 -2
View File
@@ -12,18 +12,32 @@ class IndexerCommandCustom
public:
static IndexerCommandType getStaticIndexerCommandType();
IndexerCommandCustom(const FilePath& sourceFilePath, const std::wstring& customCommand);
IndexerCommandCustom(
const std::wstring& customCommand,
const FilePath& projectFilePath,
const FilePath& databaseFilePath,
const std::wstring& databaseVersion,
const FilePath& sourceFilePath,
bool runInParallel);
IndexerCommandType getIndexerCommandType() const override;
size_t getByteSize(size_t stringSize) const override;
const std::wstring& getCustomCommand() const;
FilePath getDatabaseFilePath() const;
void setDatabaseFilePath(const FilePath& databaseFilePath);
std::wstring getCustomCommand() const;
bool getRunInParallel() const;
protected:
QJsonObject doSerialize() const override;
private:
std::wstring m_customCommand;
FilePath m_projectFilePath;
FilePath m_databaseFilePath;
std::wstring m_databaseVersion;
bool m_runInParallel;
};
#endif // INDEXER_COMMAND_CXXL_H
@@ -2,6 +2,7 @@
#include "Blackboard.h"
#include "DialogView.h"
#include "FileSystem.h"
#include "IndexerCommandCustom.h"
#include "IndexerCommandProvider.h"
#include "MessageIndexingStatus.h"
@@ -16,12 +17,15 @@ TaskExecuteCustomCommands::TaskExecuteCustomCommands(
std::unique_ptr<IndexerCommandProvider> indexerCommandProvider,
std::shared_ptr<PersistentStorage> storage,
std::shared_ptr<DialogView> dialogView,
int indexerThreadCount,
const FilePath& projectDirectory
)
: m_indexerCommandProvider(std::move(indexerCommandProvider))
, m_storage(storage)
, m_dialogView(dialogView)
, m_indexerThreadCount(indexerThreadCount)
, m_projectDirectory(projectDirectory)
, m_indexerCommandCount(m_indexerCommandProvider->size())
{
}
@@ -29,6 +33,30 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr<Blackboard> blackboard)
{
m_dialogView->hideUnknownProgressDialog();
m_start = utility::durationStart();
if (m_indexerCommandProvider)
{
while (!m_indexerCommandProvider->empty())
{
if (std::shared_ptr<IndexerCommandCustom> indexerCommand =
std::dynamic_pointer_cast<IndexerCommandCustom>(m_indexerCommandProvider->consumeCommand()))
{
if (m_targetDatabaseFilePath.empty())
{
m_targetDatabaseFilePath = indexerCommand->getDatabaseFilePath();
}
if (indexerCommand->getRunInParallel())
{
m_parallelCommands.push_back(indexerCommand);
}
else
{
m_serialCommands.push_back(indexerCommand);
}
}
}
}
}
Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard> blackboard)
@@ -38,45 +66,44 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard>
return STATE_SUCCESS;
}
int sourceFileCount = m_indexerCommandProvider->size();
int indexedSourceFileCount = 0;
m_dialogView->updateCustomIndexingDialog(0, 0, sourceFileCount, { });
m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), { });
while (!m_interrupted && m_indexerCommandProvider->size())
std::vector<std::shared_ptr<std::thread>> indexerThreads;
for (size_t i = 1 /*this method is counting as the first thread*/; i < m_indexerThreadCount; i++)
{
std::shared_ptr<IndexerCommandCustom> indexerCommand =
std::dynamic_pointer_cast<IndexerCommandCustom>(m_indexerCommandProvider->consumeCommand());
if (indexerCommand)
indexerThreads.push_back(std::make_shared<std::thread>(&TaskExecuteCustomCommands::executeParallelIndexerCommands, this, i, blackboard));
}
while (!m_interrupted && !m_serialCommands.empty())
{
std::shared_ptr<IndexerCommandCustom> indexerCommand = m_serialCommands.back();
m_serialCommands.pop_back();
runIndexerCommand(indexerCommand, blackboard);
}
executeParallelIndexerCommands(0, blackboard);
for (std::shared_ptr<std::thread> indexerThread : indexerThreads)
{
indexerThread->join();
}
indexerThreads.clear();
{
PersistentStorage targetStorage(m_targetDatabaseFilePath, FilePath());
targetStorage.setup();
targetStorage.setMode(SqliteIndexStorage::STORAGE_MODE_WRITE);
targetStorage.buildCaches();
for (const FilePath& sourceDatabaseFilePath : m_sourceDatabaseFilePaths)
{
FilePath sourcePath = indexerCommand->getSourceFilePath();
m_dialogView->updateCustomIndexingDialog(indexedSourceFileCount + 1, indexedSourceFileCount, sourceFileCount, { sourcePath });
MessageIndexingStatus(true, indexedSourceFileCount * 100 / sourceFileCount).dispatch();
LOG_INFO_STREAM(<< "Execute command \"" << utility::encodeToUtf8(indexerCommand->getCustomCommand()) << "\"");
m_storage->beforeErrorRecording();
std::wstring processOutput;
int result = utility::executeProcessAndGetExitCode(indexerCommand->getCustomCommand(), {}, m_projectDirectory, -1, &processOutput);
m_storage->afterErrorRecording();
if (processOutput.size() > 3 || result != 0)
{
if (result == 0)
{
LOG_INFO_STREAM(<< "process return 0:\n" << utility::encodeToUtf8(processOutput));
}
else
{
LOG_ERROR_STREAM(<< "process returned " << result << ":\n" << utility::encodeToUtf8(processOutput));
MessageShowStatus().dispatch();
MessageStatus(L"command <" + indexerCommand->getCustomCommand() + L"> returned " +
std::to_wstring(result) + L": " + processOutput, true, false, true).dispatch();
}
PersistentStorage sourceStorage(sourceDatabaseFilePath, FilePath());
sourceStorage.setMode(SqliteIndexStorage::STORAGE_MODE_READ);
sourceStorage.buildCaches();
targetStorage.inject(&sourceStorage);
}
indexedSourceFileCount++;
blackboard->update<int>("indexed_source_file_count", [=](int count) { return count + 1; });
FileSystem::remove(sourceDatabaseFilePath);
}
}
@@ -86,7 +113,7 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard>
void TaskExecuteCustomCommands::doExit(std::shared_ptr<Blackboard> blackboard)
{
m_storage.reset();
float duration = utility::duration(m_start);
const float duration = utility::duration(m_start);
blackboard->update<float>("index_time", [duration](float currentDuration) { return currentDuration + duration; });
}
@@ -102,3 +129,107 @@ void TaskExecuteCustomCommands::handleMessage(MessageIndexingInterrupted* messag
m_dialogView->showUnknownProgressDialog(L"Interrupting Indexing", L"Waiting for running\ncommand to finish");
}
void TaskExecuteCustomCommands::executeParallelIndexerCommands(int threadId, std::shared_ptr<Blackboard> blackboard)
{
while (!m_interrupted)
{
std::shared_ptr<IndexerCommandCustom> indexerCommand;
{
std::lock_guard<std::mutex> lock(m_parallelCommandsMutex);
if (m_parallelCommands.empty())
{
return;
}
indexerCommand = m_parallelCommands.back();
m_parallelCommands.pop_back();
}
if (threadId != 0)
{
FilePath databaseFilePath = indexerCommand->getDatabaseFilePath();
databaseFilePath = databaseFilePath.getParentDirectory().concatenate(databaseFilePath.fileName() + L"_thread" + std::to_wstring(threadId));
bool databaseFilePathKnown = true;
{
std::lock_guard<std::mutex> lock(m_sourceDatabaseFilePathsMutex);
if (m_sourceDatabaseFilePaths.find(databaseFilePath) == m_sourceDatabaseFilePaths.end())
{
m_sourceDatabaseFilePaths.insert(databaseFilePath);
databaseFilePathKnown = false;
}
}
if (!databaseFilePathKnown)
{
if (databaseFilePath.exists())
{
LOG_WARNING(L"Temporary storage \"" + databaseFilePath.wstr() + L"\" already exists on file system. File will be removed to avoid conflicts.");
FileSystem::remove(databaseFilePath);
}
PersistentStorage sourceStorage(databaseFilePath, FilePath());
sourceStorage.setup();
sourceStorage.setMode(SqliteIndexStorage::STORAGE_MODE_WRITE);
sourceStorage.buildCaches();
}
indexerCommand->setDatabaseFilePath(databaseFilePath);
}
runIndexerCommand(indexerCommand, blackboard);
}
}
void TaskExecuteCustomCommands::runIndexerCommand(std::shared_ptr<IndexerCommandCustom> indexerCommand, std::shared_ptr<Blackboard> blackboard)
{
if (indexerCommand)
{
int indexedSourceFileCount = 0;
blackboard->get("indexed_source_file_count", indexedSourceFileCount);
const FilePath sourcePath = indexerCommand->getSourceFilePath();
m_dialogView->updateCustomIndexingDialog(indexedSourceFileCount + 1, indexedSourceFileCount, m_indexerCommandCount, { sourcePath });
MessageIndexingStatus(true, indexedSourceFileCount * 100 / m_indexerCommandCount).dispatch();
const std::wstring command = indexerCommand->getCustomCommand();
LOG_INFO_STREAM(<< "Execute command \"" << utility::encodeToUtf8(command) << "\"");
m_storage->beforeErrorRecording();
std::wstring processOutput;
const int result = utility::executeProcessAndGetExitCode(command, {}, m_projectDirectory, -1, &processOutput);
m_storage->afterErrorRecording();
if (processOutput.size() > 3 || result != 0)
{
if (result == 0)
{
std::wstring message = L"Process returned successfully";
if (processOutput.empty())
{
message += L".";
}
else
{
message += L"with message \"" + processOutput + L"\".";
}
message += L"\n";
LOG_INFO(message);
}
else
{
LOG_ERROR_STREAM(<< "process returned \"" << result << "\" with message:\n" << utility::encodeToUtf8(processOutput));
MessageShowStatus().dispatch();
MessageStatus(L"command <" + indexerCommand->getCustomCommand() + L"> returned " +
std::to_wstring(result) + L": " + processOutput, true, false, true).dispatch();
}
}
indexedSourceFileCount++;
blackboard->update<int>("indexed_source_file_count", [=](int count) { return count + 1; });
}
}
@@ -1,6 +1,7 @@
#ifndef TASK_EXECUTE_CUSTOM_COMMANDS_H
#define TASK_EXECUTE_CUSTOM_COMMANDS_H
#include <set>
#include <vector>
#include "FilePath.h"
@@ -10,6 +11,7 @@
#include "MessageListener.h"
class DialogView;
class IndexerCommandCustom;
class IndexerCommandProvider;
class PersistentStorage;
@@ -22,6 +24,7 @@ public:
std::unique_ptr<IndexerCommandProvider> indexerCommandProvider,
std::shared_ptr<PersistentStorage> storage,
std::shared_ptr<DialogView> dialogView,
int indexerThreadCount,
const FilePath& projectDirectory);
private:
@@ -32,13 +35,24 @@ private:
void handleMessage(MessageIndexingInterrupted* message) override;
void executeParallelIndexerCommands(int threadId, std::shared_ptr<Blackboard> blackboard);
void runIndexerCommand(std::shared_ptr<IndexerCommandCustom> indexerCommand, std::shared_ptr<Blackboard> blackboard);
std::unique_ptr<IndexerCommandProvider> m_indexerCommandProvider;
std::shared_ptr<PersistentStorage> m_storage;
std::shared_ptr<DialogView> m_dialogView;
const int m_indexerThreadCount;
const FilePath m_projectDirectory;
TimeStamp m_start;
bool m_interrupted = false;
int m_indexerCommandCount;
std::vector<std::shared_ptr<IndexerCommandCustom>> m_serialCommands;
std::vector<std::shared_ptr<IndexerCommandCustom>> m_parallelCommands;
std::mutex m_parallelCommandsMutex;
FilePath m_targetDatabaseFilePath;
std::set<FilePath> m_sourceDatabaseFilePaths;
std::mutex m_sourceDatabaseFilePathsMutex;
};
#endif // TASK_EXECUTE_CUSTOM_COMMANDS_H
+17 -11
View File
@@ -516,19 +516,19 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
taskSequential->addTask(std::make_shared<TaskSetValue<bool>>("interrupted_indexing", false));
taskSequential->addTask(std::make_shared<TaskSetValue<float>>("index_time", 0.0f));
if (indexerCommandProvider->size())
int indexerThreadCount = ApplicationSettings::getInstance()->getIndexerThreadCount();
if (indexerThreadCount <= 0)
{
int indexerThreadCount = ApplicationSettings::getInstance()->getIndexerThreadCount();
indexerThreadCount = utility::getIdealThreadCount();
if (indexerThreadCount <= 0)
{
indexerThreadCount = utility::getIdealThreadCount();
if (indexerThreadCount <= 0)
{
indexerThreadCount = 4; // setting to some fallback value
}
indexerThreadCount = 4; // setting to some fallback value
}
}
indexerThreadCount = std::min<int>(indexerThreadCount, indexerCommandProvider->size());
if (!indexerCommandProvider->empty())
{
const int adjustedIndexerThreadCount = std::min<int>(indexerThreadCount, indexerCommandProvider->size());
std::shared_ptr<StorageProvider> storageProvider = std::make_shared<StorageProvider>();
// add tasks for setting some variables on the blackboard that are used during indexing
@@ -556,7 +556,7 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
std::make_shared<TaskDecoratorRepeat>(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS, 25)->addChildTask(
std::make_shared<TaskReturnSuccessIf<bool>>("indexer_command_queue_started", TaskReturnSuccessIf<bool>::CONDITION_EQUALS, false)
),
std::make_shared<TaskBuildIndex>(indexerThreadCount, storageProvider, dialogView, m_appUUID, multiProcess)
std::make_shared<TaskBuildIndex>(adjustedIndexerThreadCount, storageProvider, dialogView, m_appUUID, multiProcess)
)
);
@@ -613,11 +613,17 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
dialogView->hideUnknownProgressDialog();
}
if (customIndexerCommandProvider->size())
if (!customIndexerCommandProvider->empty())
{
const int adjustedIndexerThreadCount = std::min<int>(indexerThreadCount, customIndexerCommandProvider->size());
taskSequential->addTask(
std::make_shared<TaskExecuteCustomCommands>(
std::move(customIndexerCommandProvider), tempStorage, dialogView, getProjectSettingsFilePath().getParentDirectory())
std::move(customIndexerCommandProvider),
tempStorage,
dialogView,
adjustedIndexerThreadCount,
getProjectSettingsFilePath().getParentDirectory())
);
}
+10 -8
View File
@@ -40,20 +40,22 @@ std::set<FilePath> SourceGroupCustomCommand::getAllSourceFilePaths() const
std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexerCommands(const std::set<FilePath>& filesToIndex) const
{
std::wstring customCommand = m_settings->getCustomCommand();
customCommand = utility::replace(customCommand, L"%{PROJECT_FILE_PATH}", L'\"' + m_settings->getProjectSettings()->getProjectFilePath().wstr() + L'\"');
customCommand = utility::replace(customCommand, L"%{DATABASE_FILE_PATH}", L'\"' + m_settings->getProjectSettings()->getTempDBFilePath().wstr() + L'\"');
customCommand = utility::replace(customCommand, L"%{DATABASE_VERSION}", L'\"' + std::to_wstring(SqliteIndexStorage::getStorageVersion()) + L'\"');
const std::wstring customCommand = m_settings->getCustomCommand();
const bool runInParallel = m_settings->getRunInParallel();
std::vector<std::shared_ptr<IndexerCommand>> indexerCommands;
for (const FilePath& sourcePath: getAllSourceFilePaths())
{
if (filesToIndex.find(sourcePath) != filesToIndex.end())
{
std::wstring command = utility::replace(customCommand, L"%{SOURCE_FILE_PATH}", L'\"' + sourcePath.wstr() + L'\"');
indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(sourcePath, command));
indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(
customCommand,
m_settings->getProjectSettings()->getProjectFilePath(),
m_settings->getProjectSettings()->getTempDBFilePath(),
std::to_wstring(SqliteIndexStorage::getStorageVersion()),
sourcePath,
runInParallel
));
}
}
@@ -7,6 +7,7 @@ SourceGroupSettingsCustomCommand::SourceGroupSettingsCustomCommand(
const std::string& id, const ProjectSettings* projectSettings
)
: SourceGroupSettings(id, SOURCE_GROUP_CUSTOM_COMMAND, projectSettings)
, m_runInParallel(false)
{
}
@@ -26,6 +27,7 @@ void SourceGroupSettingsCustomCommand::load(std::shared_ptr<const ConfigManager>
SourceGroupSettingsWithExcludeFilters::load(config, key);
setCustomCommand(config->getValueOrDefault(key + "/custom_command", std::wstring()));
setRunInParallel(config->getValueOrDefault(key + "/run_in_parallel", false));
}
void SourceGroupSettingsCustomCommand::save(std::shared_ptr<ConfigManager> config)
@@ -39,6 +41,7 @@ void SourceGroupSettingsCustomCommand::save(std::shared_ptr<ConfigManager> confi
SourceGroupSettingsWithExcludeFilters::save(config, key);
config->setValue(key + "/custom_command", getCustomCommand());
config->setValue(key + "/run_in_parallel", getRunInParallel());
}
bool SourceGroupSettingsCustomCommand::equals(std::shared_ptr<SourceGroupSettings> other) const
@@ -66,6 +69,16 @@ void SourceGroupSettingsCustomCommand::setCustomCommand(const std::wstring& cust
m_customCommand = customCommand;
}
bool SourceGroupSettingsCustomCommand::getRunInParallel() const
{
return m_runInParallel;
}
void SourceGroupSettingsCustomCommand::setRunInParallel(bool runInParallel)
{
m_runInParallel = runInParallel;
}
std::vector<std::wstring> SourceGroupSettingsCustomCommand::getDefaultSourceExtensions() const
{
return {};
@@ -25,10 +25,14 @@ public:
const std::wstring& getCustomCommand() const;
void setCustomCommand(const std::wstring& customCommand);
bool getRunInParallel() const;
void setRunInParallel(bool runInParallel);
private:
std::vector<std::wstring> getDefaultSourceExtensions() const override;
std::wstring m_customCommand;
bool m_runInParallel;
};
#endif // SOURCE_GROUP_SETTINGS_CUSTOM_COMMAND_H
@@ -1,5 +1,6 @@
#include "QtProjectWizzardContentCustomCommand.h"
#include <QCheckBox>
#include <QLineEdit>
#include <QMessageBox>
#include <boost/filesystem/path.hpp>
@@ -47,10 +48,15 @@ void QtProjectWizzardContentCustomCommand::populate(QGridLayout* layout, int& ro
m_customCommand = new QLineEdit();
m_customCommand->setObjectName("name");
m_customCommand->setAttribute(Qt::WA_MacShowFocusRect, 0);
m_runInParallel = new QCheckBox("Run in Parallel");
layout->setRowMinimumHeight(row, 30);
layout->addWidget(nameLabel, row, QtProjectWizzardWindow::FRONT_COL, Qt::AlignRight);
layout->addWidget(m_customCommand, row, QtProjectWizzardWindow::BACK_COL);
layout->setRowMinimumHeight(row, 30);
row++;
layout->addWidget(m_runInParallel, row, QtProjectWizzardWindow::BACK_COL);
row++;
if (!isInForm())
@@ -63,11 +69,13 @@ void QtProjectWizzardContentCustomCommand::populate(QGridLayout* layout, int& ro
void QtProjectWizzardContentCustomCommand::load()
{
m_customCommand->setText(QString::fromStdWString(m_settings->getCustomCommand()));
m_runInParallel->setChecked(m_settings->getRunInParallel());
}
void QtProjectWizzardContentCustomCommand::save()
{
m_settings->setCustomCommand(m_customCommand->text().toStdWString());
m_settings->setRunInParallel(m_runInParallel->isChecked());
}
bool QtProjectWizzardContentCustomCommand::check()
@@ -3,6 +3,7 @@
#include "QtProjectWizzardContent.h"
class QCheckBox;
class QLineEdit;
class SourceGroupSettingsCustomCommand;
@@ -27,6 +28,7 @@ private:
std::shared_ptr<SourceGroupSettingsCustomCommand> m_settings;
QLineEdit* m_customCommand;
QCheckBox* m_runInParallel;
};
#endif // QT_PROJECT_WIZZARD_CONTENT_CUSTOM_COMMAND_H