logic: improved fulltext search performance (issue #680)

* move to multithreaded implementation
* omit unnecessary work

src/lib/data/fulltextsearch/FullTextSearchIndex.cpp

@@ -29,16 +29,21 @@ std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::
 	TRACE();
 
 	std::vector<FullTextSearchResult> ret;
-	FullTextSearchResult hit;
 	{
 		std::lock_guard<std::mutex> lock(m_filesMutex);
 		for (auto& f : m_files)
 		{
+			FullTextSearchResult hit;
 			hit.fileId = f.fileId;
 			hit.positions = f.array.searchForTerm(term);
-			ret.push_back(hit);
+			std::sort(hit.positions.begin(), hit.positions.end());
+			if (!hit.positions.empty())
+			{
+				ret.push_back(hit);
+			}
 		}
 	}
+
 	return ret;
 }
 

src/lib/data/storage/PersistentStorage.cpp

@@ -552,58 +552,75 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getFullTextSearchLo
 		false, true
 	).dispatch();
 
-	const int termLength = searchTerm.length();
-
-	for (const FullTextSearchResult& fileHits : m_fullTextSearchIndex.searchForTerm(searchTerm))
 	{
-		const FilePath filePath = getFileNodePath(fileHits.fileId);
+		std::vector<std::shared_ptr<std::thread>> threads;
-		std::shared_ptr<TextAccess> fileContent = getFileContent(filePath, false);
+		std::mutex collectionMutex;
-
+		for (std::vector<FullTextSearchResult> fileResults : utility::splitToEqualySizedParts(m_fullTextSearchIndex.searchForTerm(searchTerm), utility::getIdealThreadCount()))
-		int charsTotal = 0;
-		int lineNumber = 1;
-		std::wstring line = codec.decode(fileContent->getLine(lineNumber));
-
-		for (int pos : fileHits.positions)
 		{
-			while (charsTotal + (int)line.length() <= pos)
+			std::shared_ptr<std::thread> thread = std::make_shared<std::thread>(
-			{
+				[this, &searchTerm, &caseSensitive, &codec, /*no ref here!*/fileResults, &collection, &collectionMutex]()
-				charsTotal += line.length();
+				{
-				lineNumber++;
+					const int termLength = searchTerm.length();
-				line = codec.decode(fileContent->getLine(lineNumber));
+					for (const FullTextSearchResult& fileResult : fileResults)
-			}
+					{
+						const FilePath filePath = getFileNodePath(fileResult.fileId);
+						std::shared_ptr<TextAccess> fileContent = getFileContent(filePath, false);
 
-			ParseLocation location;
+						int charsTotal = 0;
-			location.startLineNumber = lineNumber;
+						int lineNumber = 1;
-			location.startColumnNumber = pos - charsTotal + 1;
+						std::wstring line = codec.decode(fileContent->getLine(lineNumber));
 
-			if (caseSensitive && line.substr(location.startColumnNumber - 1, termLength) != searchTerm)
+						for (int pos : fileResult.positions)
-			{
+						{
-				continue;
+							while (charsTotal + (int)line.length() <= pos)
-			}
+							{
+								charsTotal += line.length();
+								lineNumber++;
+								line = codec.decode(fileContent->getLine(lineNumber));
+							}
 
-			while ((charsTotal + (int)line.length()) < pos + termLength)
+							ParseLocation location;
-			{
+							location.startLineNumber = lineNumber;
-				charsTotal += line.length();
+							location.startColumnNumber = pos - charsTotal + 1;
-				lineNumber++;
-				line = codec.decode(fileContent->getLine(lineNumber));
-			}
 
-			location.endLineNumber = lineNumber;
+							if (caseSensitive && line.substr(location.startColumnNumber - 1, termLength) != searchTerm)
-			location.endColumnNumber = pos + termLength - charsTotal;
+							{
+								continue;
+							}
+							while ((charsTotal + (int)line.length()) < pos + termLength)
+							{
+								charsTotal += line.length();
+								lineNumber++;
+								line = codec.decode(fileContent->getLine(lineNumber));
+							}
+							location.endLineNumber = lineNumber;
+							location.endColumnNumber = pos + termLength - charsTotal;
 
-			// Set first bit to 1 to avoid collisions
+							{
-			const Id locationId = ~(~Id(0) >> 1) + collection->getSourceLocationCount() + 1;
+								std::lock_guard<std::mutex> lock(collectionMutex);
-
+								// Set first bit to 1 to avoid collisions
-			collection->addSourceLocation(
+								const Id locationId = ~(~Id(0) >> 1) + collection->getSourceLocationCount() + 1;
-				LOCATION_FULLTEXT_SEARCH,
+								collection->addSourceLocation(
-				locationId,
+									LOCATION_FULLTEXT_SEARCH,
-				std::vector<Id>(),
+									locationId,
-				filePath,
+									std::vector<Id>(),
-				location.startLineNumber,
+									filePath,
-				location.startColumnNumber,
+									location.startLineNumber,
-				location.endLineNumber,
+									location.startColumnNumber,
-				location.endColumnNumber
+									location.endLineNumber,
+									location.endColumnNumber
+								);
+							}
+						}
+					}
+				}
 			);
+			threads.push_back(thread);
+		}
+
+		for (std::shared_ptr<std::thread> thread : threads)
+		{
+			thread->join();
 		}
 	}
 

src/lib_utility/utility/TextCodec.cpp

@@ -5,6 +5,9 @@
 TextCodec::TextCodec(const std::string& name)
 	: m_name(name)
 {
+	m_codec = QTextCodec::codecForName(m_name.c_str());
+	m_decoder = std::make_shared<QTextDecoder>(m_codec);
+	m_encoder = std::make_shared<QTextEncoder>(m_codec);
 }
 
 std::string TextCodec::getName() const
@@ -14,8 +17,7 @@ std::string TextCodec::getName() const
 
 bool TextCodec::isValid() const
 {
-	QTextCodec* codec = QTextCodec::codecForName(m_name.c_str());
+	if (m_codec)
-	if (codec)
 	{
 		return true;
 	}
@@ -24,23 +26,18 @@ bool TextCodec::isValid() const
 
 std::wstring TextCodec::decode(const std::string& unicodeString) const
 {
-	QTextCodec* codec = QTextCodec::codecForName(m_name.c_str());
+	if (m_decoder)
-	if (codec)
 	{
-		QTextDecoder decoder(codec);
+		return m_decoder->toUnicode(unicodeString.c_str()).toStdWString();
-		return decoder.toUnicode(unicodeString.c_str()).toStdWString();
 	}
 	return QString::fromStdString(unicodeString).toStdWString();
 }
 
 std::string TextCodec::encode(const std::wstring& string) const
 {
-	QTextCodec* codec = QTextCodec::codecForName(m_name.c_str());
+	if (m_encoder)
-	if (codec)
 	{
-		QTextEncoder encoder(codec);
+		return m_encoder->fromUnicode(QString::fromStdWString(string)).toStdString();
-		return encoder.fromUnicode(QString::fromStdWString(string)).toStdString();
 	}
-
 	return QString::fromStdWString(string).toStdString();
 }

src/lib_utility/utility/TextCodec.h

@@ -2,6 +2,11 @@
 #define TEXT_CODEC_H
 
 #include <string>
+#include <memory>
+
+class QTextCodec;
+class QTextDecoder;
+class QTextEncoder;
 
 class TextCodec
 {
@@ -17,6 +22,9 @@ public:
 
 private:
 	const std::string m_name;
+	QTextCodec* m_codec;
+	std::shared_ptr<QTextDecoder> m_decoder;
+	std::shared_ptr<QTextEncoder> m_encoder;
 };
 
 #endif // TEXT_CODEC_H