From 7008d2abdd4fd9c81c687ee5e681d436c39ae88f Mon Sep 17 00:00:00 2001 From: Malte Langkabel Date: Mon, 29 Jun 2020 19:25:26 +0200 Subject: [PATCH] logic: fix error recording for multi-threaded custom command indexing (issue #1043) (#1049) In multi-threaded custom command indexing errors were read from the wrong storage. --- .../indexer/TaskExecuteCustomCommands.cpp | 58 +++++++++++++++---- .../data/indexer/TaskExecuteCustomCommands.h | 7 ++- src/lib/data/storage/PersistentStorage.cpp | 7 ++- src/lib/data/storage/PersistentStorage.h | 2 + 4 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp index 61b115af..0ad359e4 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp @@ -7,6 +7,8 @@ #include "FileSystem.h" #include "IndexerCommandCustom.h" #include "IndexerCommandProvider.h" +#include "MessageErrorCountClear.h" +#include "MessageErrorCountUpdate.h" #include "MessageIndexingStatus.h" #include "MessageShowStatus.h" #include "MessageStatus.h" @@ -320,7 +322,7 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr { std::shared_ptr indexerCommand = m_serialCommands.back(); m_serialCommands.pop_back(); - runIndexerCommand(indexerCommand, blackboard); + runIndexerCommand(indexerCommand, blackboard, m_storage); } executeParallelIndexerCommands(0, blackboard); @@ -331,6 +333,9 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr } indexerThreads.clear(); + // clear errors here, because otherwise injecting into the main storage will show them twice + MessageErrorCountClear().dispatch(); + { PersistentStorage targetStorage(m_targetDatabaseFilePath, FilePath()); targetStorage.setup(); @@ -386,6 +391,7 @@ void TaskExecuteCustomCommands::handleMessage(MessageIndexingInterrupted* messag void TaskExecuteCustomCommands::executeParallelIndexerCommands( int threadId, std::shared_ptr blackboard) { + std::shared_ptr storage; while (!m_interrupted) { std::shared_ptr indexerCommand; @@ -399,7 +405,11 @@ void TaskExecuteCustomCommands::executeParallelIndexerCommands( m_parallelCommands.pop_back(); } - if (threadId != 0) + if (threadId == 0) + { + storage = m_storage; + } + else { FilePath databaseFilePath = indexerCommand->getDatabaseFilePath(); databaseFilePath = databaseFilePath.getParentDirectory().concatenate( @@ -426,21 +436,23 @@ void TaskExecuteCustomCommands::executeParallelIndexerCommands( L"conflicts."); FileSystem::remove(databaseFilePath); } - PersistentStorage sourceStorage(databaseFilePath, FilePath()); - sourceStorage.setup(); - sourceStorage.setMode(SqliteIndexStorage::STORAGE_MODE_WRITE); - sourceStorage.buildCaches(); + storage = std::make_shared(databaseFilePath, FilePath()); + storage->setup(); + storage->setMode(SqliteIndexStorage::STORAGE_MODE_WRITE); + storage->buildCaches(); } indexerCommand->setDatabaseFilePath(databaseFilePath); } - runIndexerCommand(indexerCommand, blackboard); + runIndexerCommand(indexerCommand, blackboard, storage); } } void TaskExecuteCustomCommands::runIndexerCommand( - std::shared_ptr indexerCommand, std::shared_ptr blackboard) + std::shared_ptr indexerCommand, + std::shared_ptr blackboard, + std::shared_ptr storage) { if (indexerCommand) { @@ -455,15 +467,39 @@ void TaskExecuteCustomCommands::runIndexerCommand( const std::wstring command = indexerCommand->getCustomCommand(); - LOG_INFO_STREAM(<< "Execute command \"" << utility::encodeToUtf8(command) << "\""); + LOG_INFO("Start processing command \"" + utility::encodeToUtf8(command) + "\""); - m_storage->beforeErrorRecording(); + const ErrorCountInfo previousErrorCount = storage ? storage->getErrorCount() + : ErrorCountInfo(); + LOG_INFO("Starting to index"); std::wstring errorMessage; const int result = utility::executeProcessAndGetExitCode( command, {}, m_projectDirectory, -1, true, &errorMessage); + LOG_INFO("Finished indexing"); - m_storage->afterErrorRecording(); + if (storage) + { + std::vector errors = storage->getErrorInfos(); + const ErrorCountInfo currentErrorCount(errors); + if (currentErrorCount.total > previousErrorCount.total) + { + const ErrorCountInfo diff( + currentErrorCount.total - previousErrorCount.total, + currentErrorCount.fatal - previousErrorCount.fatal); + + ErrorCountInfo errorCount; // local copy to release lock early + { + std::lock_guard lock(m_errorCountMutex); + m_errorCount.total += diff.total; + m_errorCount.fatal += diff.fatal; + errorCount = m_errorCount; + } + + errors.erase(errors.begin(), errors.begin() + previousErrorCount.total); + MessageErrorCountUpdate(errorCount, errors).dispatch(); + } + } if (result == 0 && errorMessage.empty()) { diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.h b/src/lib/data/indexer/TaskExecuteCustomCommands.h index 1357300b..84149ac9 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.h +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.h @@ -4,6 +4,7 @@ #include #include +#include "ErrorCountInfo.h" #include "FilePath.h" #include "MessageIndexingInterrupted.h" #include "MessageListener.h" @@ -39,7 +40,9 @@ private: void executeParallelIndexerCommands(int threadId, std::shared_ptr blackboard); void runIndexerCommand( - std::shared_ptr indexerCommand, std::shared_ptr blackboard); + std::shared_ptr indexerCommand, + std::shared_ptr blackboard, + std::shared_ptr storage); std::unique_ptr m_indexerCommandProvider; std::shared_ptr m_storage; @@ -53,6 +56,8 @@ private: std::vector> m_serialCommands; std::vector> m_parallelCommands; std::mutex m_parallelCommandsMutex; + ErrorCountInfo m_errorCount; + std::mutex m_errorCountMutex; FilePath m_targetDatabaseFilePath; bool m_hasPythonCommands; std::set m_sourceDatabaseFilePaths; diff --git a/src/lib/data/storage/PersistentStorage.cpp b/src/lib/data/storage/PersistentStorage.cpp index c595a17f..c987ea2c 100644 --- a/src/lib/data/storage/PersistentStorage.cpp +++ b/src/lib/data/storage/PersistentStorage.cpp @@ -260,6 +260,11 @@ void PersistentStorage::rollbackInjection() afterErrorRecording(); } +const std::vector PersistentStorage::getErrorInfos() const +{ + return m_sqliteIndexStorage.getAllErrorInfos(); +} + void PersistentStorage::beforeErrorRecording() { m_preInjectionErrorCount = m_sqliteIndexStorage.getErrorCount(); @@ -273,7 +278,7 @@ void PersistentStorage::beforeErrorRecording() void PersistentStorage::afterErrorRecording() { - std::vector errors = m_sqliteIndexStorage.getAllErrorInfos(); + std::vector errors = getErrorInfos(); if (m_preInjectionErrorCount < errors.size()) { ErrorCountInfo errorCount(errors); diff --git a/src/lib/data/storage/PersistentStorage.h b/src/lib/data/storage/PersistentStorage.h index e8d315d1..bf95681c 100644 --- a/src/lib/data/storage/PersistentStorage.h +++ b/src/lib/data/storage/PersistentStorage.h @@ -59,6 +59,8 @@ public: void finishInjection() override; void rollbackInjection(); + const std::vector getErrorInfos() const; + void beforeErrorRecording(); void afterErrorRecording();