logic: improved performance of initializing the fulltext search index

* implemented using multithreaing for building the fulltext search index
* also: decreased ideal thread count by 1 to keep the machine responsive while indexing and building the index

src/lib/data/fulltextsearch/FullTextSearchIndex.cpp

@@ -4,20 +4,24 @@
 #include "utility/logging/logging.h"
 #include "utility/tracing.h"
 
-void FullTextSearchIndex::addFile(Id fileId, const std::wstring& file)
+void FullTextSearchIndex::addFile(Id fileId, const std::wstring& fileContent)
 {
-	if( file.empty() )
+	if(fileContent.empty())
 	{
 		LOG_ERROR("empty file not added to fulltextsearch index");
 	}
 
-	if ( file.size() >= std::numeric_limits<int>::max() )
+	if (fileContent.size() >= std::numeric_limits<int>::max())
 	{
 		LOG_ERROR("file too big not added to fulltextsearch index");
 	}
 
-	FullTextSearchFile fts_file(fileId, SuffixArray(file));
-	m_files.push_back(fts_file);
+	FullTextSearchFile fts_file(fileId, SuffixArray(fileContent));
+
+	{
+		std::lock_guard<std::mutex> lock(m_filesMutex);
+		m_files.push_back(fts_file);
+	}
 }
 
 std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::wstring& term) const
@@ -26,22 +30,27 @@ std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::
 
 	std::vector<FullTextSearchResult> ret;
 	FullTextSearchResult hit;
-	for (auto& f : m_files)
 	{
-		hit.fileId = f.fileId;
-		hit.positions = f.array.searchForTerm(term);
-		ret.push_back(hit);
+		std::lock_guard<std::mutex> lock(m_filesMutex);
+		for (auto& f : m_files)
+		{
+			hit.fileId = f.fileId;
+			hit.positions = f.array.searchForTerm(term);
+			ret.push_back(hit);
+		}
 	}
 	return ret;
 }
 
 size_t FullTextSearchIndex::fileCount() const
 {
+	std::lock_guard<std::mutex> lock(m_filesMutex);
 	return m_files.size();
 }
 
 void FullTextSearchIndex::clear()
 {
+	std::lock_guard<std::mutex> lock(m_filesMutex);
 	m_files.clear();
 }
 

src/lib/data/fulltextsearch/FullTextSearchIndex.h

@@ -1,6 +1,7 @@
 #ifndef FULLTEXTSEARCH_INDEX_H
 #define FULLTEXTSEARCH_INDEX_H
 
+#include <mutex>
 #include <vector>
 #include <unordered_map>
 
@@ -38,6 +39,7 @@ public:
 	void clear();
 
 private:
+	mutable std::mutex m_filesMutex;
 	std::vector<FullTextSearchFile> m_files;
 };
 

src/lib/data/storage/PersistentStorage.cpp

@@ -25,6 +25,7 @@
 #include "utility/TimeStamp.h"
 #include "utility/tracing.h"
 #include "utility/utility.h"
+#include "utility/utilityApp.h"
 
 PersistentStorage::PersistentStorage(const FilePath& dbPath, const FilePath& bookmarkPath)
 	: m_sqliteIndexStorage(dbPath)
@@ -2741,13 +2742,28 @@ void PersistentStorage::buildFullTextSearchIndex() const
 	m_fullTextSearchCodec = codec.getName();
 
 	m_fullTextSearchIndex.clear();
-	for (StorageFile& file : m_sqliteIndexStorage.getAll<StorageFile>())
-	{
 
-		m_fullTextSearchIndex.addFile(
-			file.id,
-			codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText())
+	std::vector<std::shared_ptr<std::thread>> threads;
+	for (std::vector<StorageFile> part : utility::splitToEqualySizedParts(m_sqliteIndexStorage.getAll<StorageFile>(), utility::getIdealThreadCount()))
+	{
+		std::shared_ptr<std::thread> thread = std::make_shared<std::thread>(
+			[&](const std::vector<StorageFile>& files)
+			{
+				for (const StorageFile& file : files)
+				{
+					m_fullTextSearchIndex.addFile(
+						file.id,
+						codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText())
+					);
+				}
+			},
+			part
 		);
+		threads.push_back(thread);
+	}
+	for (std::shared_ptr<std::thread> thread : threads)
+	{
+		thread->join();
 	}
 }
 

src/lib/utility/utility.h

@@ -28,6 +28,9 @@ namespace utility
 	std::string timeToString(const boost::posix_time::ptime time);
 	std::string timeToString(float seconds);
 
+	template<typename T>
+	std::vector<std::vector<T>> splitToEqualySizedParts(const std::vector<T>& values, const size_t desiredPartCount);
+
 	template<typename T>
 	std::vector<T> concat(const std::vector<T>& a, const std::vector<T>& b);
 
@@ -128,6 +131,27 @@ namespace utility
 	int roundToInt(float n);
 }
 
+template<typename T>
+std::vector<std::vector<T>> utility::splitToEqualySizedParts(const std::vector<T>& values, const size_t desiredPartCount)
+{
+	const size_t partCount = std::max<size_t>(1, std::min(desiredPartCount, values.size()));
+
+	std::vector<std::vector<T>> parts;
+	for (size_t i = 0; i < partCount; i++)
+	{
+		parts.push_back(std::vector<T>());
+	}
+
+	int i = 0;
+	for (const T& value : values)
+	{
+		parts[i % partCount].push_back(value);
+		++i;
+	}
+
+	return parts;
+}
+
 template<typename T>
 std::vector<T> utility::concat(const std::vector<T>& a, const std::vector<T>& b)
 {