From 4d6cb464c47098b0f71f1ba8b35726a29bc4eec3 Mon Sep 17 00:00:00 2001 From: mlangkabel Date: Tue, 21 May 2019 17:23:03 +0200 Subject: [PATCH] logic: sort custom indexer commands by indexed file size and name --- .../indexer/TaskExecuteCustomCommands.cpp | 6 ++- .../indexer/TaskFillIndexerCommandQueue.cpp | 48 +++---------------- src/lib/utility/utilityFile.cpp | 39 +++++++++++++++ src/lib/utility/utilityFile.h | 2 + 4 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp index 2c785b03..8c2c00d6 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp @@ -16,6 +16,7 @@ #include "TextAccess.h" #include "utility.h" #include "utilityApp.h" +#include "utilityFile.h" #include "utilityString.h" TaskExecuteCustomCommands::TaskExecuteCustomCommands( @@ -42,7 +43,7 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr blackboard) if (m_indexerCommandProvider) { - for (const FilePath& sourceFilePath : m_indexerCommandProvider->getAllSourceFilePaths()) + for (const FilePath& sourceFilePath : utility::partitionFilePathsBySize(m_indexerCommandProvider->getAllSourceFilePaths(), 2)) { if (std::shared_ptr indexerCommand = std::dynamic_pointer_cast(m_indexerCommandProvider->consumeCommandForSourceFilePath(sourceFilePath))) @@ -67,6 +68,9 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr blackboard) } } } + // reverse because we pull elements from the back of these vectors + std::reverse(m_parallelCommands.begin(), m_parallelCommands.end()); + std::reverse(m_serialCommands.begin(), m_serialCommands.end()); } } diff --git a/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp b/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp index 56d19365..e8ffd7e6 100644 --- a/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp +++ b/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp @@ -1,9 +1,10 @@ #include "TaskFillIndexerCommandQueue.h" -#include "IndexerCommandProvider.h" -#include "FileSystem.h" -#include "logging.h" #include "Blackboard.h" +#include "FileSystem.h" +#include "IndexerCommandProvider.h" +#include "logging.h" +#include "utilityFile.h" TaskFillIndexerCommandsQueue::TaskFillIndexerCommandsQueue( const std::string& appUUID, @@ -18,48 +19,11 @@ TaskFillIndexerCommandsQueue::TaskFillIndexerCommandsQueue( void TaskFillIndexerCommandsQueue::doEnter(std::shared_ptr blackboard) { - typedef std::pair PairType; - std::vector sourceFileSizesToCommands; - { - std::vector allSourceFilePaths; - { - std::lock_guard lock(m_commandsMutex); - allSourceFilePaths = m_indexerCommandProvider->getAllSourceFilePaths(); - } - - for (const FilePath& path : allSourceFilePaths) - { - if (path.exists()) - { - sourceFileSizesToCommands.push_back(std::make_pair(FileSystem::getFileByteSize(path), path)); - } - else - { - sourceFileSizesToCommands.push_back(std::make_pair(1, path)); - } - } - std::sort(sourceFileSizesToCommands.begin(), sourceFileSizesToCommands.end(), [](const PairType& p, const PairType& q) { return p.first > q.first; }); - - if (sourceFileSizesToCommands.size() > 2) - { - std::sort( - sourceFileSizesToCommands.begin(), - sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() / 2, - [](const PairType& p, const PairType& q) { return p.second.wstr() < q.second.wstr(); } - ); - std::sort( - sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() / 2, - sourceFileSizesToCommands.end(), - [](const PairType& p, const PairType& q) { return p.second.wstr() < q.second.wstr(); } - ); - } - } - { std::lock_guard lock(m_commandsMutex); - for (const PairType &pair : sourceFileSizesToCommands) + for (const FilePath& filePath : utility::partitionFilePathsBySize(m_indexerCommandProvider->getAllSourceFilePaths(), 2)) { - m_filePathQueue.emplace(pair.second); + m_filePathQueue.emplace(filePath); } } diff --git a/src/lib/utility/utilityFile.cpp b/src/lib/utility/utilityFile.cpp index 40192b82..95d6c166 100644 --- a/src/lib/utility/utilityFile.cpp +++ b/src/lib/utility/utilityFile.cpp @@ -1,9 +1,48 @@ #include "utilityFile.h" #include "FilePath.h" +#include "FileSystem.h" #include "utility.h" +std::vector utility::partitionFilePathsBySize(std::vector filePaths, int partitionCount) +{ + typedef std::pair PairType; + std::vector sourceFileSizesToCommands; + for (const FilePath& path : filePaths) + { + if (path.exists()) + { + sourceFileSizesToCommands.push_back(std::make_pair(FileSystem::getFileByteSize(path), path)); + } + else + { + sourceFileSizesToCommands.push_back(std::make_pair(1, path)); + } + } + + std::sort(sourceFileSizesToCommands.begin(), sourceFileSizesToCommands.end(), [](const PairType& p, const PairType& q) { return p.first > q.first; }); + + if (0 < partitionCount && partitionCount < sourceFileSizesToCommands.size()) + { + for (int i = 0; i < partitionCount; i++) + { + std::sort( + sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() * i / partitionCount, + sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() * (i + 1) / partitionCount, + [](const PairType& p, const PairType& q) { return p.second.wstr() < q.second.wstr(); } + ); + } + } + + std::vector sortedFilePaths; + for (const PairType &pair : sourceFileSizesToCommands) + { + sortedFilePaths.push_back(pair.second); + } + return sortedFilePaths; +} + std::vector utility::getTopLevelPaths(const std::vector& paths) { return utility::getTopLevelPaths(utility::toSet(paths)); diff --git a/src/lib/utility/utilityFile.h b/src/lib/utility/utilityFile.h index d16828d8..df78b541 100644 --- a/src/lib/utility/utilityFile.h +++ b/src/lib/utility/utilityFile.h @@ -8,6 +8,8 @@ class FilePath; namespace utility { + std::vector partitionFilePathsBySize(std::vector filePaths, int partitionCount = 0); + std::vector getTopLevelPaths(const std::vector& paths); std::vector getTopLevelPaths(const std::set& paths);