logic: improved performance of initializing the fulltext search index

* implemented using multithreaing for building the fulltext search index
* also: decreased ideal thread count by 1 to keep the machine responsive while indexing and building the index

src/lib/data/fulltextsearch/FullTextSearchIndex.cpp

@@ -4,20 +4,24 @@
 #include "utility/logging/logging.h"
 #include "utility/tracing.h"
 
-void FullTextSearchIndex::addFile(Id fileId, const std::wstring& file)
+void FullTextSearchIndex::addFile(Id fileId, const std::wstring& fileContent)
 {
-	if( file.empty() )
+	if(fileContent.empty())
 	{
 		LOG_ERROR("empty file not added to fulltextsearch index");
 	}
 
-	if ( file.size() >= std::numeric_limits<int>::max() )
+	if (fileContent.size() >= std::numeric_limits<int>::max())
 	{
 		LOG_ERROR("file too big not added to fulltextsearch index");
 	}
 
-	FullTextSearchFile fts_file(fileId, SuffixArray(file));
-	m_files.push_back(fts_file);
+	FullTextSearchFile fts_file(fileId, SuffixArray(fileContent));
+
+	{
+		std::lock_guard<std::mutex> lock(m_filesMutex);
+		m_files.push_back(fts_file);
+	}
 }
 
 std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::wstring& term) const
@@ -26,22 +30,27 @@ std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::
 
 	std::vector<FullTextSearchResult> ret;
 	FullTextSearchResult hit;
-	for (auto& f : m_files)
 	{
-		hit.fileId = f.fileId;
-		hit.positions = f.array.searchForTerm(term);
-		ret.push_back(hit);
+		std::lock_guard<std::mutex> lock(m_filesMutex);
+		for (auto& f : m_files)
+		{
+			hit.fileId = f.fileId;
+			hit.positions = f.array.searchForTerm(term);
+			ret.push_back(hit);
+		}
 	}
 	return ret;
 }
 
 size_t FullTextSearchIndex::fileCount() const
 {
+	std::lock_guard<std::mutex> lock(m_filesMutex);
 	return m_files.size();
 }
 
 void FullTextSearchIndex::clear()
 {
+	std::lock_guard<std::mutex> lock(m_filesMutex);
 	m_files.clear();
 }
 

src/lib/data/fulltextsearch/FullTextSearchIndex.h

@@ -1,6 +1,7 @@
 #ifndef FULLTEXTSEARCH_INDEX_H
 #define FULLTEXTSEARCH_INDEX_H
 
+#include <mutex>
 #include <vector>
 #include <unordered_map>
 
@@ -38,6 +39,7 @@ public:
 	void clear();
 
 private:
+	mutable std::mutex m_filesMutex;
 	std::vector<FullTextSearchFile> m_files;
 };
 

src/lib/data/storage/PersistentStorage.cpp

@@ -25,6 +25,7 @@
 #include "utility/TimeStamp.h"
 #include "utility/tracing.h"
 #include "utility/utility.h"
+#include "utility/utilityApp.h"
 
 PersistentStorage::PersistentStorage(const FilePath& dbPath, const FilePath& bookmarkPath)
 	: m_sqliteIndexStorage(dbPath)
@@ -2741,13 +2742,28 @@ void PersistentStorage::buildFullTextSearchIndex() const
 	m_fullTextSearchCodec = codec.getName();
 
 	m_fullTextSearchIndex.clear();
-	for (StorageFile& file : m_sqliteIndexStorage.getAll<StorageFile>())
-	{
 
-		m_fullTextSearchIndex.addFile(
-			file.id,
-			codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText())
+	std::vector<std::shared_ptr<std::thread>> threads;
+	for (std::vector<StorageFile> part : utility::splitToEqualySizedParts(m_sqliteIndexStorage.getAll<StorageFile>(), utility::getIdealThreadCount()))
+	{
+		std::shared_ptr<std::thread> thread = std::make_shared<std::thread>(
+			[&](const std::vector<StorageFile>& files)
+			{
+				for (const StorageFile& file : files)
+				{
+					m_fullTextSearchIndex.addFile(
+						file.id,
+						codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText())
+					);
+				}
+			},
+			part
 		);
+		threads.push_back(thread);
+	}
+	for (std::shared_ptr<std::thread> thread : threads)
+	{
+		thread->join();
 	}
 }
 

src/lib/utility/utility.h

@@ -28,6 +28,9 @@ namespace utility
 	std::string timeToString(const boost::posix_time::ptime time);
 	std::string timeToString(float seconds);
 
+	template<typename T>
+	std::vector<std::vector<T>> splitToEqualySizedParts(const std::vector<T>& values, const size_t desiredPartCount);
+
 	template<typename T>
 	std::vector<T> concat(const std::vector<T>& a, const std::vector<T>& b);
 
@@ -128,6 +131,27 @@ namespace utility
 	int roundToInt(float n);
 }
 
+template<typename T>
+std::vector<std::vector<T>> utility::splitToEqualySizedParts(const std::vector<T>& values, const size_t desiredPartCount)
+{
+	const size_t partCount = std::max<size_t>(1, std::min(desiredPartCount, values.size()));
+
+	std::vector<std::vector<T>> parts;
+	for (size_t i = 0; i < partCount; i++)
+	{
+		parts.push_back(std::vector<T>());
+	}
+
+	int i = 0;
+	for (const T& value : values)
+	{
+		parts[i % partCount].push_back(value);
+		++i;
+	}
+
+	return parts;
+}
+
 template<typename T>
 std::vector<T> utility::concat(const std::vector<T>& a, const std::vector<T>& b)
 {

src/lib_cxx/utility/IncludeProcessing.cpp

@@ -21,28 +21,6 @@ namespace
 			return a.getIncludedFile() < b.getIncludedFile();
 		}
 	};
-
-	std::vector<std::vector<FilePath>> splitToQuantiles(
-		const std::set<FilePath>& sourceFilePaths, 
-		const size_t desiredQuantileCount)
-	{
-		size_t quantileCount = std::max<size_t>(1, std::min(desiredQuantileCount, sourceFilePaths.size()));
-
-		std::vector<std::vector<FilePath>> quantiles;
-		for (size_t i = 0; i < quantileCount; i++)
-		{
-			quantiles.push_back(std::vector<FilePath>());
-		}
-
-		int i = 0;
-		for (const FilePath& sourceFilePath : sourceFilePaths)
-		{
-			quantiles[i % quantileCount].push_back(sourceFilePath);
-			++i;
-		}
-
-		return quantiles;
-	}
 }
 
 std::vector<IncludeDirective> IncludeProcessing::getUnresolvedIncludeDirectives(
@@ -55,14 +33,14 @@ std::vector<IncludeDirective> IncludeProcessing::getUnresolvedIncludeDirectives(
 	std::unordered_set<std::wstring> processedFilePaths;
 	std::set<IncludeDirective, IncludeDirectiveComparator> unresolvedIncludeDirectives;
 
-	std::vector<std::vector<FilePath>> quantiles = splitToQuantiles(sourceFilePaths, desiredQuantileCount);
+	std::vector<std::vector<FilePath>> parts = utility::splitToEqualySizedParts(utility::toVector(sourceFilePaths), desiredQuantileCount);
 
-	for (size_t i = 0; i < quantiles.size(); i++)
+	for (size_t i = 0; i < parts.size(); i++)
 	{
-		progress(float(i) / quantiles.size());
+		progress(float(i) / parts.size());
 
 		const std::vector<IncludeDirective> directives = doGetUnresolvedIncludeDirectives(
-			utility::toSet(quantiles[i]),
+			utility::toSet(parts[i]),
 			processedFilePaths,
 			indexedPaths,
 			headerSearchDirectories
@@ -99,13 +77,13 @@ std::set<FilePath> IncludeProcessing::getHeaderSearchDirectories(
 
 	std::set<FilePath> headerSearchDirectories;
 	std::unordered_set<std::wstring> processedFilePaths;
-	std::vector<std::vector<FilePath>> quantiles = splitToQuantiles(sourceFilePaths, desiredQuantileCount);
+	std::vector<std::vector<FilePath>> parts = utility::splitToEqualySizedParts(utility::toVector(sourceFilePaths), desiredQuantileCount);
 
-	for (size_t i = 0; i < quantiles.size(); i++)
+	for (size_t i = 0; i < parts.size(); i++)
 	{
-		progress(float(i) / quantiles.size());
+		progress(float(i) / parts.size());
 
-		std::set<FilePath> unprocessedFilePaths(quantiles[i].begin(), quantiles[i].end());
+		std::set<FilePath> unprocessedFilePaths(parts[i].begin(), parts[i].end());
 
 		while (!unprocessedFilePaths.empty())
 		{

src/lib_gui/utility/utilityApp.cpp

@@ -156,7 +156,7 @@ OsType utility::getOsType()
 
 int utility::getIdealThreadCount()
 {
-	return QThread::idealThreadCount();
+	return std::max(1, QThread::idealThreadCount() - 1);
 }
 
 bool utility::saveLicense(const License* license)