logic: sort and shuffle indexer commands for performance

Before indexing the indexer commands are sorted according to byte size of the
indexed source file. Afterwards the list is split into two halves and each half
is shuffled. This ensures that bigger files tend to be indexed first, so that
no indexer thread runs idle at the end of indexing.

src/lib/data/indexer/IndexerCommandList.cpp

@@ -3,6 +3,8 @@
 #include <algorithm>
 #include <random>
 
+#include "utility/file/FileSystem.h"
+
 void IndexerCommandList::addCommand(std::shared_ptr<IndexerCommand> command)
 {
 	std::lock_guard<std::mutex> lock(m_commandsMutex);
@@ -16,9 +18,29 @@ size_t IndexerCommandList::size() const
 
 void IndexerCommandList::shuffle()
 {
-	srand(unsigned(time(NULL)));
 	std::lock_guard<std::mutex> lock(m_commandsMutex);
-	std::random_shuffle(m_commands.begin(), m_commands.end());
+
+	typedef std::pair<unsigned long long int, std::shared_ptr<IndexerCommand>> PairType;
+
+	std::vector<PairType> sourceFileSizesToCommands;
+	for (std::shared_ptr<IndexerCommand> command: m_commands)
+	{
+		sourceFileSizesToCommands.push_back(std::make_pair(FileSystem::getFileByteSize(command->getSourceFilePath()), command));
+	}
+	std::sort(sourceFileSizesToCommands.begin(), sourceFileSizesToCommands.end(), [](const PairType& p, const PairType& q){ return p.first > q.first; });
+
+	if (sourceFileSizesToCommands.size() > 2)
+	{
+		srand(unsigned(time(NULL)));
+		std::random_shuffle(sourceFileSizesToCommands.begin(), sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() / 2);
+		std::random_shuffle(sourceFileSizesToCommands.begin() + sourceFileSizesToCommands.size() / 2, sourceFileSizesToCommands.end());
+	}
+
+	m_commands.clear();
+	for (const PairType& pair: sourceFileSizesToCommands)
+	{
+		m_commands.push_back(pair.second);
+	}
 }
 
 std::shared_ptr<IndexerCommand> IndexerCommandList::consumeCommand()

src/lib/utility/file/FileSystem.cpp

@@ -138,6 +138,11 @@ std::vector<FileInfo> FileSystem::getFileInfosFromPaths(
 	return files;
 }
 
+unsigned long long FileSystem::getFileByteSize(const FilePath& filePath)
+{
+	return boost::filesystem::file_size(filePath.path());
+}
+
 TimePoint FileSystem::getLastWriteTime(const FilePath& filePath)
 {
 	boost::posix_time::ptime lastWriteTime;

src/lib/utility/file/FileSystem.h

@@ -18,6 +18,8 @@ public:
 	static std::vector<FileInfo> getFileInfosFromPaths(
 		const std::vector<FilePath>& paths, const std::vector<std::string>& fileExtensions, bool followSymLinks = true);
 
+	static unsigned long long getFileByteSize(const FilePath& filePath);
+
 	static TimePoint getLastWriteTime(const FilePath& filePath);
 	static std::string getTimeStringNow();