diff --git a/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp b/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp index 6020d81a..78f4f535 100644 --- a/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp +++ b/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp @@ -4,20 +4,24 @@ #include "utility/logging/logging.h" #include "utility/tracing.h" -void FullTextSearchIndex::addFile(Id fileId, const std::wstring& file) +void FullTextSearchIndex::addFile(Id fileId, const std::wstring& fileContent) { - if( file.empty() ) + if(fileContent.empty()) { LOG_ERROR("empty file not added to fulltextsearch index"); } - if ( file.size() >= std::numeric_limits::max() ) + if (fileContent.size() >= std::numeric_limits::max()) { LOG_ERROR("file too big not added to fulltextsearch index"); } - FullTextSearchFile fts_file(fileId, SuffixArray(file)); - m_files.push_back(fts_file); + FullTextSearchFile fts_file(fileId, SuffixArray(fileContent)); + + { + std::lock_guard lock(m_filesMutex); + m_files.push_back(fts_file); + } } std::vector FullTextSearchIndex::searchForTerm(const std::wstring& term) const @@ -26,22 +30,27 @@ std::vector FullTextSearchIndex::searchForTerm(const std:: std::vector ret; FullTextSearchResult hit; - for (auto& f : m_files) { - hit.fileId = f.fileId; - hit.positions = f.array.searchForTerm(term); - ret.push_back(hit); + std::lock_guard lock(m_filesMutex); + for (auto& f : m_files) + { + hit.fileId = f.fileId; + hit.positions = f.array.searchForTerm(term); + ret.push_back(hit); + } } return ret; } size_t FullTextSearchIndex::fileCount() const { + std::lock_guard lock(m_filesMutex); return m_files.size(); } void FullTextSearchIndex::clear() { + std::lock_guard lock(m_filesMutex); m_files.clear(); } diff --git a/src/lib/data/fulltextsearch/FullTextSearchIndex.h b/src/lib/data/fulltextsearch/FullTextSearchIndex.h index 49e380af..7f89a9e2 100644 --- a/src/lib/data/fulltextsearch/FullTextSearchIndex.h +++ b/src/lib/data/fulltextsearch/FullTextSearchIndex.h @@ -1,6 +1,7 @@ #ifndef FULLTEXTSEARCH_INDEX_H #define FULLTEXTSEARCH_INDEX_H +#include #include #include @@ -38,6 +39,7 @@ public: void clear(); private: + mutable std::mutex m_filesMutex; std::vector m_files; }; diff --git a/src/lib/data/storage/PersistentStorage.cpp b/src/lib/data/storage/PersistentStorage.cpp index 78cd6085..30b5525e 100644 --- a/src/lib/data/storage/PersistentStorage.cpp +++ b/src/lib/data/storage/PersistentStorage.cpp @@ -25,6 +25,7 @@ #include "utility/TimeStamp.h" #include "utility/tracing.h" #include "utility/utility.h" +#include "utility/utilityApp.h" PersistentStorage::PersistentStorage(const FilePath& dbPath, const FilePath& bookmarkPath) : m_sqliteIndexStorage(dbPath) @@ -2741,13 +2742,28 @@ void PersistentStorage::buildFullTextSearchIndex() const m_fullTextSearchCodec = codec.getName(); m_fullTextSearchIndex.clear(); - for (StorageFile& file : m_sqliteIndexStorage.getAll()) - { - m_fullTextSearchIndex.addFile( - file.id, - codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText()) + std::vector> threads; + for (std::vector part : utility::splitToEqualySizedParts(m_sqliteIndexStorage.getAll(), utility::getIdealThreadCount())) + { + std::shared_ptr thread = std::make_shared( + [&](const std::vector& files) + { + for (const StorageFile& file : files) + { + m_fullTextSearchIndex.addFile( + file.id, + codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText()) + ); + } + }, + part ); + threads.push_back(thread); + } + for (std::shared_ptr thread : threads) + { + thread->join(); } } diff --git a/src/lib/utility/utility.h b/src/lib/utility/utility.h index 12cea58d..075d94bd 100644 --- a/src/lib/utility/utility.h +++ b/src/lib/utility/utility.h @@ -28,6 +28,9 @@ namespace utility std::string timeToString(const boost::posix_time::ptime time); std::string timeToString(float seconds); + template + std::vector> splitToEqualySizedParts(const std::vector& values, const size_t desiredPartCount); + template std::vector concat(const std::vector& a, const std::vector& b); @@ -128,6 +131,27 @@ namespace utility int roundToInt(float n); } +template +std::vector> utility::splitToEqualySizedParts(const std::vector& values, const size_t desiredPartCount) +{ + const size_t partCount = std::max(1, std::min(desiredPartCount, values.size())); + + std::vector> parts; + for (size_t i = 0; i < partCount; i++) + { + parts.push_back(std::vector()); + } + + int i = 0; + for (const T& value : values) + { + parts[i % partCount].push_back(value); + ++i; + } + + return parts; +} + template std::vector utility::concat(const std::vector& a, const std::vector& b) { diff --git a/src/lib_cxx/utility/IncludeProcessing.cpp b/src/lib_cxx/utility/IncludeProcessing.cpp index c98b694d..fb633708 100644 --- a/src/lib_cxx/utility/IncludeProcessing.cpp +++ b/src/lib_cxx/utility/IncludeProcessing.cpp @@ -21,28 +21,6 @@ namespace return a.getIncludedFile() < b.getIncludedFile(); } }; - - std::vector> splitToQuantiles( - const std::set& sourceFilePaths, - const size_t desiredQuantileCount) - { - size_t quantileCount = std::max(1, std::min(desiredQuantileCount, sourceFilePaths.size())); - - std::vector> quantiles; - for (size_t i = 0; i < quantileCount; i++) - { - quantiles.push_back(std::vector()); - } - - int i = 0; - for (const FilePath& sourceFilePath : sourceFilePaths) - { - quantiles[i % quantileCount].push_back(sourceFilePath); - ++i; - } - - return quantiles; - } } std::vector IncludeProcessing::getUnresolvedIncludeDirectives( @@ -55,14 +33,14 @@ std::vector IncludeProcessing::getUnresolvedIncludeDirectives( std::unordered_set processedFilePaths; std::set unresolvedIncludeDirectives; - std::vector> quantiles = splitToQuantiles(sourceFilePaths, desiredQuantileCount); + std::vector> parts = utility::splitToEqualySizedParts(utility::toVector(sourceFilePaths), desiredQuantileCount); - for (size_t i = 0; i < quantiles.size(); i++) + for (size_t i = 0; i < parts.size(); i++) { - progress(float(i) / quantiles.size()); + progress(float(i) / parts.size()); const std::vector directives = doGetUnresolvedIncludeDirectives( - utility::toSet(quantiles[i]), + utility::toSet(parts[i]), processedFilePaths, indexedPaths, headerSearchDirectories @@ -99,13 +77,13 @@ std::set IncludeProcessing::getHeaderSearchDirectories( std::set headerSearchDirectories; std::unordered_set processedFilePaths; - std::vector> quantiles = splitToQuantiles(sourceFilePaths, desiredQuantileCount); + std::vector> parts = utility::splitToEqualySizedParts(utility::toVector(sourceFilePaths), desiredQuantileCount); - for (size_t i = 0; i < quantiles.size(); i++) + for (size_t i = 0; i < parts.size(); i++) { - progress(float(i) / quantiles.size()); + progress(float(i) / parts.size()); - std::set unprocessedFilePaths(quantiles[i].begin(), quantiles[i].end()); + std::set unprocessedFilePaths(parts[i].begin(), parts[i].end()); while (!unprocessedFilePaths.empty()) { diff --git a/src/lib_gui/utility/utilityApp.cpp b/src/lib_gui/utility/utilityApp.cpp index 4d3983f8..a5612b35 100644 --- a/src/lib_gui/utility/utilityApp.cpp +++ b/src/lib_gui/utility/utilityApp.cpp @@ -156,7 +156,7 @@ OsType utility::getOsType() int utility::getIdealThreadCount() { - return QThread::idealThreadCount(); + return std::max(1, QThread::idealThreadCount() - 1); } bool utility::saveLicense(const License* license)