From 91969770d56f4dd2d0af90f740ecff664f4c3044 Mon Sep 17 00:00:00 2001
From: malte_langkabel <malte.langkabel@gmail.com>
Date: Wed, 2 Aug 2017 12:03:53 +0200
Subject: [PATCH] logic: sort and shuffle indexer commands for performance

Before indexing the indexer commands are sorted according to byte size of the
indexed source file. Afterwards the list is split into two halves and each half
is shuffled. This ensures that bigger files tend to be indexed first, so that
no indexer thread runs idle at the end of indexing.
---
 src/lib/data/indexer/IndexerCommandList.cpp | 26 +++++++++++++++++++--
 src/lib/utility/file/FileSystem.cpp         |  5 ++++
 src/lib/utility/file/FileSystem.h           |  2 ++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/lib/data/indexer/IndexerCommandList.cpp b/src/lib/data/indexer/IndexerCommandList.cpp
index f42f7b01..45b64166 100644
--- a/src/lib/data/indexer/IndexerCommandList.cpp
+++ b/src/lib/data/indexer/IndexerCommandList.cpp
@@ -3,6 +3,8 @@
 #include <algorithm>
 #include <random>
 
+#include "utility/file/FileSystem.h"
+
 void IndexerCommandList::addCommand(std::shared_ptr<IndexerCommand> command)
 {
 	std::lock_guard<std::mutex> lock(m_commandsMutex);
@@ -16,9 +18,29 @@ size_t IndexerCommandList::size() const
 
 void IndexerCommandList::shuffle()
 {
-	srand(unsigned(time(NULL)));
 	std::lock_guard<std::mutex> lock(m_commandsMutex);
-	std::random_shuffle(m_commands.begin(), m_commands.end());
+
+	typedef std::pair<unsigned long long int, std::shared_ptr<IndexerCommand>> PairType;
+
+	std::vector<PairType> sourceFileSizesToCommands;
+	for (std::shared_ptr<IndexerCommand> command: m_commands)
+	{
+		sourceFileSizesToCommands.push_back(std::make_pair(FileSystem::getFileByteSize(command->getSourceFilePath()), command));
+	}
+	std::sort(sourceFileSizesToCommands.begin(), sourceFileSizesToCommands.end(), [](const PairType& p, const PairType& q){ return p.first > q.first; });
+
+	if (sourceFileSizesToCommands.size() > 2)
+	{
+		srand(unsigned(time(NULL)));
+		std::random_shuffle(sourceFileSizesToCommands.begin(), sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() / 2);
+		std::random_shuffle(sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() / 2, sourceFileSizesToCommands.end());
+	}
+
+	m_commands.clear();
+	for (const PairType& pair: sourceFileSizesToCommands)
+	{
+		m_commands.push_back(pair.second);
+	}
 }
 
 std::shared_ptr<IndexerCommand> IndexerCommandList::consumeCommand()
diff --git a/src/lib/utility/file/FileSystem.cpp b/src/lib/utility/file/FileSystem.cpp
index e8d8751d..9383f372 100644
--- a/src/lib/utility/file/FileSystem.cpp
+++ b/src/lib/utility/file/FileSystem.cpp
@@ -138,6 +138,11 @@ std::vector<FileInfo> FileSystem::getFileInfosFromPaths(
 	return files;
 }
 
+unsigned long long FileSystem::getFileByteSize(const FilePath& filePath)
+{
+	return boost::filesystem::file_size(filePath.path());
+}
+
 TimePoint FileSystem::getLastWriteTime(const FilePath& filePath)
 {
 	boost::posix_time::ptime lastWriteTime;
diff --git a/src/lib/utility/file/FileSystem.h b/src/lib/utility/file/FileSystem.h
index 066ddaa1..689654a0 100644
--- a/src/lib/utility/file/FileSystem.h
+++ b/src/lib/utility/file/FileSystem.h
@@ -18,6 +18,8 @@ public:
 	static std::vector<FileInfo> getFileInfosFromPaths(
 		const std::vector<FilePath>& paths, const std::vector<std::string>& fileExtensions, bool followSymLinks = true);
 
+	static unsigned long long getFileByteSize(const FilePath& filePath);
+
 	static TimePoint getLastWriteTime(const FilePath& filePath);
 	static std::string getTimeStringNow();