diff --git a/CMakeLists.txt b/CMakeLists.txt index a3c5ed9b..aa6e8188 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ IF(${CMAKE_BINARY_DIR} STREQUAL ${CMAKE_SOURCE_DIR}) MESSAGE(FATAL_ERROR "Please create a separate build directory and call CMake again") ENDIF(${CMAKE_BINARY_DIR} STREQUAL ${CMAKE_SOURCE_DIR}) + # Variables -------------------------------------------------------------------- set(PROJECT_NAME Sourcetrail) @@ -32,6 +33,7 @@ endif() set(APP_PROJECT_NAME "${PROJECT_NAME}") set(LIB_LICENSE_PROJECT_NAME "${PROJECT_NAME}_lib_license") +set(LIB_UTILITY_PROJECT_NAME "${PROJECT_NAME}_lib_utility") set(LIB_GUI_PROJECT_NAME "${PROJECT_NAME}_lib_gui") set(LIB_CXX_PROJECT_NAME "${PROJECT_NAME}_lib_cxx") set(LIB_JAVA_PROJECT_NAME "${PROJECT_NAME}_lib_java") @@ -56,6 +58,7 @@ else () set(PLATFORM_INCLUDE "includesLinux.h") endif () + # Project ---------------------------------------------------------------------- project(${PROJECT_NAME}) @@ -74,7 +77,9 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD 14) set(CMAKE_C_STANDARD 99) + # Settings --------------------------------------------------------------------- + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option -fcolor-diagnostics") endif() @@ -84,6 +89,7 @@ endif() #add_definitions(-fno-omit-frame-pointer) #endif () + # Clang ------------------------------------------------------------------------ if (UNIX AND APPLE) @@ -106,6 +112,7 @@ find_package(CLANG REQUIRED PATHS "${CMAKE_SOURCE_DIR}/cmake" NO_DEFAULT_PATH) add_definitions(${CLANG_DEFINITIONS}) + # Boost ------------------------------------------------------------------------ set(Boost_USE_MULTITHREAD ON) @@ -132,7 +139,8 @@ endif() find_package(Boost 1.64 COMPONENTS system program_options filesystem date_time REQUIRED) -# Botan ------------------------------------------------------------------------- + +# Botan ------------------------------------------------------------------------ set(BOTAN_VERSION "botan-2") @@ -184,7 +192,47 @@ if(NOT BOTAN_INCLUDE_DIR) endif() -# Lib License ------------------------------------------------------------------- +# Qt --------------------------------------------------------------------------- + +# Find the QtWidgets library +if(UNIX) + set(CMAKE_PREFIX_PATH "$ENV{QT_DIR}/") + + if(APPLE) + set(Qt5Widgets_DIR "$ENV{QT_DIR}/cmake/Qt5Widgets/") + set(Qt5PrintSupport_DIR "$ENV{QT_DIR}/cmake/Qt5PrintSupport/") + endif() + +else() + if(${WINXXBITS} STREQUAL "Win32") + set(CMAKE_PREFIX_PATH "$ENV{QT_WIN32_DIR}/") + else() + set(CMAKE_PREFIX_PATH "$ENV{QT_WIN64_DIR}/") + endif() + set(Qt5Widgets_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Widgets") + set(Qt5PrintSupport_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5PrintSupport") + set(Qt5Network_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Network") + set(Qt5WinExtras_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5WinExtras") +endif() + +find_package(Qt5Widgets) +find_package(Qt5PrintSupport) +find_package(Qt5Network) + +if (WIN32) + find_package(Qt5WinExtras) +endif() + +if(Qt5Widgets_FOUND) + MESSAGE(STATUS "Found Qt ${Qt5Widgets_VERSION_STRING}") + # FIX: Qt was built with -reduce-relocations + if (Qt5_POSITION_INDEPENDENT_CODE) + SET(CMAKE_POSITION_INDEPENDENT_CODE ON) + endif() +endif() + + +# Lib License ------------------------------------------------------------------ if (UNIX) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/lib/") @@ -223,6 +271,31 @@ else() endif() +# Lib Utility ------------------------------------------------------------------ + +add_subdirectory(src/lib_utility) + +add_library(${LIB_UTILITY_PROJECT_NAME} ${LIB_UTILITY_FILES}) + +create_source_groups(${LIB_UTILITY_FILES}) + +set_property( + TARGET ${LIB_UTILITY_PROJECT_NAME} + PROPERTY INCLUDE_DIRECTORIES + "${CMAKE_SOURCE_DIR}/src/lib_utility" +) + +target_include_directories(${LIB_UTILITY_PROJECT_NAME} SYSTEM + PUBLIC ${Boost_INCLUDE_DIR} +) + +if(UNIX) + target_link_libraries(${LIB_UTILITY_PROJECT_NAME} ${Boost_LIBRARIES} Qt5::Widgets Qt5::Network) +else() + target_link_libraries(${LIB_UTILITY_PROJECT_NAME} ${Boost_LIBRARIES} Qt5::Widgets Qt5::Network Qt5::WinExtras) +endif() + + # Lib -------------------------------------------------------------------------- add_subdirectory(src/lib) @@ -244,6 +317,7 @@ set_property( "${CMAKE_SOURCE_DIR}/src/external" "${CMAKE_SOURCE_DIR}/src/lib_license" "${CMAKE_SOURCE_DIR}/src/lib_gui" + "${CMAKE_SOURCE_DIR}/src/lib_utility" "${CMAKE_BINARY_DIR}/src/lib_license" "${CMAKE_SOURCE_DIR}/src/lib_cxx" "${CMAKE_SOURCE_DIR}/src/lib_java" @@ -254,10 +328,10 @@ target_include_directories(${LIB_PROJECT_NAME} SYSTEM "${CMAKE_SOURCE_DIR}/src/external" ) -target_link_libraries(${LIB_PROJECT_NAME} ${Boost_LIBRARIES} ${LIB_LICENSE_PROJECT_NAME}) +target_link_libraries(${LIB_PROJECT_NAME} ${LIB_UTILITY_PROJECT_NAME} ${LIB_LICENSE_PROJECT_NAME} ${Boost_LIBRARIES}) -# Lib Cxx ------------------------------------------------------------------- +# Lib Cxx ---------------------------------------------------------------------- add_subdirectory(src/lib_cxx) @@ -287,7 +361,8 @@ if (WIN32) target_link_libraries(${LIB_CXX_PROJECT_NAME} version) endif() -# Lib Java ------------------------------------------------------------------- + +# Lib Java --------------------------------------------------------------------- find_package(JNI) @@ -331,43 +406,6 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON) # Instruct CMake to run moc automatically when needed. set(CMAKE_AUTOMOC ON) -# Find the QtWidgets library -if(UNIX) - set(CMAKE_PREFIX_PATH "$ENV{QT_DIR}/") - - if(APPLE) - set(Qt5Widgets_DIR "$ENV{QT_DIR}/cmake/Qt5Widgets/") - set(Qt5PrintSupport_DIR "$ENV{QT_DIR}/cmake/Qt5PrintSupport/") - endif() - -else() - if(${WINXXBITS} STREQUAL "Win32") - set(CMAKE_PREFIX_PATH "$ENV{QT_WIN32_DIR}/") - else() - set(CMAKE_PREFIX_PATH "$ENV{QT_WIN64_DIR}/") - endif() - set(Qt5Widgets_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Widgets") - set(Qt5PrintSupport_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5PrintSupport") - set(Qt5Network_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Network") - set(Qt5WinExtras_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5WinExtras") -endif() - -find_package(Qt5Widgets) -find_package(Qt5PrintSupport) -find_package(Qt5Network) - -if (WIN32) - find_package(Qt5WinExtras) -endif() - -if(Qt5Widgets_FOUND) - MESSAGE(STATUS "Found Qt ${Qt5Widgets_VERSION_STRING}") - # FIX: Qt was built with -reduce-relocations - if (Qt5_POSITION_INDEPENDENT_CODE) - SET(CMAKE_POSITION_INDEPENDENT_CODE ON) - endif() -endif() - # target for running versionnumber script # workaround for running customcommand (ninja dependency cycle) add_custom_target( @@ -377,9 +415,9 @@ add_custom_target( add_library(${LIB_GUI_PROJECT_NAME} ${LIB_GUI_FILES} ${CMAKE_BINARY_DIR}/src/lib_gui/version.h) if(UNIX) - target_link_libraries(${LIB_GUI_PROJECT_NAME} Qt5::Widgets Qt5::Network) + target_link_libraries(${LIB_GUI_PROJECT_NAME} ${LIB_UTILITY_PROJECT_NAME} Qt5::Widgets Qt5::Network) else() - target_link_libraries(${LIB_GUI_PROJECT_NAME} Qt5::Widgets Qt5::Network Qt5::WinExtras) + target_link_libraries(${LIB_GUI_PROJECT_NAME} ${LIB_UTILITY_PROJECT_NAME} Qt5::Widgets Qt5::Network Qt5::WinExtras) endif() # command for versioning script @@ -400,6 +438,7 @@ set_property( TARGET ${LIB_GUI_PROJECT_NAME} PROPERTY INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/src/lib_gui" + "${CMAKE_SOURCE_DIR}/src/lib_utility" "${CMAKE_SOURCE_DIR}/src/lib" "${CMAKE_SOURCE_DIR}/src/lib_cxx" "${CMAKE_SOURCE_DIR}/src/lib_java" @@ -431,6 +470,7 @@ include(cmake/public_key.cmake) set(CMAKE_AUTOMOC OFF) + # Indexer App ------------------------------------------------------------------ if (UNIX) @@ -484,6 +524,7 @@ set_property( "${CMAKE_BINARY_DIR}/src/lib_license" ) + # App -------------------------------------------------------------------------- if (UNIX) @@ -603,7 +644,8 @@ if (APPLE) endif () -# License Generator -------------------------------------------------------- + +# License Generator ----------------------------------------------------------- add_subdirectory(src/license_generator) @@ -642,6 +684,7 @@ endif() target_link_libraries(${LICENSE_GENERATOR_PROJECT_NAME} ${LIB_LICENSE_PROJECT_NAME} ${Boost_LIBRARIES} ) + # CxxTest ---------------------------------------------------------------------- if (UNIX) @@ -717,6 +760,7 @@ elseif (UNIX) add_dependencies(${TEST_PROJECT_NAME} gen) endif () + # Visual Leak Detector --------------------------------------------------------- if (WIN32) diff --git a/src/lib/component/controller/CodeController.cpp b/src/lib/component/controller/CodeController.cpp index f483c443..affa3f6f 100644 --- a/src/lib/component/controller/CodeController.cpp +++ b/src/lib/component/controller/CodeController.cpp @@ -2,18 +2,17 @@ #include -#include "utility/file/FileInfo.h" -#include "utility/messaging/type/MessageStatus.h" -#include "utility/text/TextAccess.h" -#include "utility/tracing.h" -#include "utility/logging/logging.h" -#include "utility/utilityString.h" - #include "data/access/StorageAccess.h" #include "data/location/SourceLocation.h" #include "data/location/SourceLocationCollection.h" #include "data/location/SourceLocationFile.h" #include "settings/ApplicationSettings.h" +#include "utility/file/FileInfo.h" +#include "utility/logging/logging.h" +#include "utility/messaging/type/MessageStatus.h" +#include "utility/text/TextAccess.h" +#include "utility/tracing.h" +#include "utility/utilityString.h" #include "Application.h" CodeController::CodeController(StorageAccess* storageAccess) @@ -339,7 +338,7 @@ void CodeController::handleMessage(MessageSearchFullText* message) saveOrRestoreViewMode(message); - m_collection = m_storageAccess->getFullTextSearchLocations(utility::encodeToUtf8(message->searchTerm), message->caseSensitive); + m_collection = m_storageAccess->getFullTextSearchLocations(message->searchTerm, message->caseSensitive); CodeView::ScrollParams scrollParams(CodeView::ScrollParams::SCROLL_TO_DEFINITION); getView()->scrollTo(scrollParams); diff --git a/src/lib/data/access/StorageAccess.h b/src/lib/data/access/StorageAccess.h index 48c3a9f6..e5e89183 100644 --- a/src/lib/data/access/StorageAccess.h +++ b/src/lib/data/access/StorageAccess.h @@ -48,7 +48,7 @@ public: virtual StorageEdge getEdgeById(Id edgeId) const = 0; virtual std::shared_ptr getFullTextSearchLocations( - const std::string& searchTerm, bool caseSensitive) const = 0; + const std::wstring& searchTerm, bool caseSensitive) const = 0; virtual std::vector getAutocompletionMatches(const std::wstring& query, NodeTypeSet acceptedNodeTypes) const = 0; virtual std::vector getSearchMatchesForTokenIds(const std::vector& tokenIds) const = 0; diff --git a/src/lib/data/access/StorageAccessProxy.cpp b/src/lib/data/access/StorageAccessProxy.cpp index 88bf8da9..f4c4ef6d 100644 --- a/src/lib/data/access/StorageAccessProxy.cpp +++ b/src/lib/data/access/StorageAccessProxy.cpp @@ -116,7 +116,7 @@ StorageEdge StorageAccessProxy::getEdgeById(Id edgeId) const } std::shared_ptr StorageAccessProxy::getFullTextSearchLocations( - const std::string &searchTerm, bool caseSensitive) const + const std::wstring &searchTerm, bool caseSensitive) const { if (hasSubject()) { diff --git a/src/lib/data/access/StorageAccessProxy.h b/src/lib/data/access/StorageAccessProxy.h index a5e7cd60..d9a1bac5 100644 --- a/src/lib/data/access/StorageAccessProxy.h +++ b/src/lib/data/access/StorageAccessProxy.h @@ -32,7 +32,7 @@ public: virtual StorageEdge getEdgeById(Id edgeId) const override; virtual std::shared_ptr getFullTextSearchLocations( - const std::string& searchTerm, bool caseSensitive) const override; + const std::wstring& searchTerm, bool caseSensitive) const override; virtual std::vector getAutocompletionMatches(const std::wstring& query, NodeTypeSet acceptedNodeTypes) const override; virtual std::vector getSearchMatchesForTokenIds(const std::vector& tokenIds) const override; diff --git a/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp b/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp index b51e96c0..6020d81a 100644 --- a/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp +++ b/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp @@ -4,7 +4,7 @@ #include "utility/logging/logging.h" #include "utility/tracing.h" -void FullTextSearchIndex::addFile(Id fileId, const std::string& file) +void FullTextSearchIndex::addFile(Id fileId, const std::wstring& file) { if( file.empty() ) { @@ -20,7 +20,7 @@ void FullTextSearchIndex::addFile(Id fileId, const std::string& file) m_files.push_back(fts_file); } -std::vector FullTextSearchIndex::searchForTerm(const std::string& term) const +std::vector FullTextSearchIndex::searchForTerm(const std::wstring& term) const { TRACE(); diff --git a/src/lib/data/fulltextsearch/FullTextSearchIndex.h b/src/lib/data/fulltextsearch/FullTextSearchIndex.h index cb90f1bb..49e380af 100644 --- a/src/lib/data/fulltextsearch/FullTextSearchIndex.h +++ b/src/lib/data/fulltextsearch/FullTextSearchIndex.h @@ -30,8 +30,8 @@ struct FullTextSearchFile class FullTextSearchIndex { public: - void addFile(Id fileId, const std::string& file); - std::vector searchForTerm(const std::string& term) const; + void addFile(Id fileId, const std::wstring& file); + std::vector searchForTerm(const std::wstring& term) const; size_t fileCount() const; diff --git a/src/lib/data/fulltextsearch/SuffixArray.cpp b/src/lib/data/fulltextsearch/SuffixArray.cpp index eee8b602..9a3510a9 100644 --- a/src/lib/data/fulltextsearch/SuffixArray.cpp +++ b/src/lib/data/fulltextsearch/SuffixArray.cpp @@ -11,11 +11,10 @@ struct suffix int SuffixArray::cmp(struct suffix a, struct suffix b) { - return (a.rank[0] == b.rank[0])? (a.rank[1] < b.rank[1] ?1: 0): - (a.rank[0] < b.rank[0] ?1: 0); + return (a.rank[0] == b.rank[0]) ? (a.rank[1] < b.rank[1] ? 1 : 0) : (a.rank[0] < b.rank[0] ? 1 : 0); } -SuffixArray::SuffixArray(const std::string& text) +SuffixArray::SuffixArray(const std::wstring& text) : m_text(text) { std::transform(m_text.begin(), m_text.end(), m_text.begin(), ::tolower); @@ -37,7 +36,7 @@ void SuffixArray::printLCP() const std::vector SuffixArray::buildLCP() { - int n = m_array.size(); + const int n = m_array.size(); std::vector lcp(n, 0); std::vector invSuff(n, 0); @@ -75,21 +74,21 @@ std::vector SuffixArray::buildLCP() return lcp; } -std::vector SuffixArray::searchForTerm(const std::string& searchTerm) const +std::vector SuffixArray::searchForTerm(const std::wstring& searchTerm) const { - std::string term = searchTerm; + std::wstring term = searchTerm; std::transform(term.begin(), term.end(), term.begin(), ::tolower); int termLength = term.length(); - int l = 0; - int r = m_text.length()-1; + int l = -1; + int r = m_text.length(); int m; std::vector matches; int compareResult; - while (l+1 < r) + while (l + 1 < r) { - m = (l+r+1)/2; + m = (l + r + 1) / 2; compareResult = term.compare(m_text.substr(m_array[m], termLength)); if( compareResult < 0) { @@ -102,11 +101,11 @@ std::vector SuffixArray::searchForTerm(const std::string& searchTerm) const else { matches.push_back(m_array[m]); - for (int lower = m-1; m_lcp[lower] >= termLength; lower--) + for (int lower = m-1; lower >= 0 && m_lcp[lower] >= termLength; lower--) { matches.push_back(m_array[lower]); } - for (int higher = m+1; m_lcp[higher-1] >= termLength; higher++) + for (int higher = m+1; higher < termLength && m_lcp[higher-1] >= termLength; higher++) { matches.push_back(m_array[higher]); } @@ -129,15 +128,15 @@ std::vector SuffixArray::buildSuffixArray() for (int i = 0; i < n; i++) { s.index = i; - s.rank[0] = m_text[i] - 'a'; - s.rank[1] = ((i+1) < n)? (m_text[i + 1] - 'a'): -1; + s.rank[0] = m_text[i] - L'a'; + s.rank[1] = ((i + 1) < n) ? (m_text[i + 1] - L'a') : -1; suffixes.push_back(s); } std::sort(suffixes.begin(), suffixes.end(), SuffixArray::cmp); std::vector ind (n,0); - for (int k = 4; k < 2*n; k = k*2) + for (int k = 4; k < 2 * n; k = k * 2) { int rank = 0; int prev_rank = suffixes[0].rank[0]; @@ -178,5 +177,3 @@ std::vector SuffixArray::buildSuffixArray() return suffixArr; } - - diff --git a/src/lib/data/fulltextsearch/SuffixArray.h b/src/lib/data/fulltextsearch/SuffixArray.h index 39a59177..bd5d36b5 100644 --- a/src/lib/data/fulltextsearch/SuffixArray.h +++ b/src/lib/data/fulltextsearch/SuffixArray.h @@ -8,8 +8,8 @@ class SuffixArray { public: - SuffixArray(const std::string& text); - std::vector searchForTerm(const std::string& searchTerm) const; + SuffixArray(const std::wstring& text); + std::vector searchForTerm(const std::wstring& searchTerm) const; static int cmp(struct suffix a, struct suffix b); void printArray() const; @@ -30,7 +30,7 @@ private: std::vector buildSuffixArray(); std::vector m_array; std::vector m_lcp; - std::string m_text; + std::wstring m_text; }; #endif // SUFFIX_ARRAY_H diff --git a/src/lib/data/search/SearchIndex.cpp b/src/lib/data/search/SearchIndex.cpp index fa740806..3bbfdc7e 100644 --- a/src/lib/data/search/SearchIndex.cpp +++ b/src/lib/data/search/SearchIndex.cpp @@ -162,7 +162,7 @@ void SearchIndex::searchRecursive( // test if s passes the edge's gate. bool passesGate = true; - for (const char& c : remainingQuery) + for (const wchar_t& c : remainingQuery) { if (currentEdge->gate.find(c) == currentEdge->gate.end()) { diff --git a/src/lib/data/storage/PersistentStorage.cpp b/src/lib/data/storage/PersistentStorage.cpp index 2ee22674..84bc8112 100644 --- a/src/lib/data/storage/PersistentStorage.cpp +++ b/src/lib/data/storage/PersistentStorage.cpp @@ -21,6 +21,7 @@ #include "utility/messaging/type/MessageNewErrors.h" #include "utility/messaging/type/MessageStatus.h" #include "utility/text/TextAccess.h" +#include "utility/TextCodec.h" #include "utility/TimeStamp.h" #include "utility/tracing.h" #include "utility/utility.h" @@ -292,6 +293,7 @@ void PersistentStorage::clearCaches() m_hierarchyCache.clear(); m_fullTextSearchIndex.clear(); + m_fullTextSearchCodec = ""; } std::set PersistentStorage::getReferenced(const std::set& filePaths) @@ -463,25 +465,27 @@ StorageEdge PersistentStorage::getEdgeById(Id edgeId) const } std::shared_ptr PersistentStorage::getFullTextSearchLocations( - const std::string& searchTerm, bool caseSensitive + const std::wstring& searchTerm, bool caseSensitive ) const { TRACE(); + const TextCodec codec(ApplicationSettings::getInstance()->getTextEncoding()); + std::shared_ptr collection = std::make_shared(); - if (!searchTerm.size()) + if (searchTerm.empty()) { return collection; } - if (m_fullTextSearchIndex.fileCount() == 0) + if (m_fullTextSearchCodec != codec.getName()) { MessageStatus(L"Building fulltext search index", false, true).dispatch(); buildFullTextSearchIndex(); } MessageStatus( - std::string("Searching fulltext (case-") + (caseSensitive ? "sensitive" : "insensitive") + "): " + searchTerm, + std::wstring(L"Searching fulltext (case-") + (caseSensitive ? L"sensitive" : L"insensitive") + L"): " + searchTerm, false, true ).dispatch(); @@ -494,7 +498,7 @@ std::shared_ptr PersistentStorage::getFullTextSearchLo int charsTotal = 0; int lineNumber = 1; - std::string line = fileContent->getLine(lineNumber); + std::wstring line = codec.decode(fileContent->getLine(lineNumber)); for (int pos : fileHits.positions) { @@ -502,7 +506,7 @@ std::shared_ptr PersistentStorage::getFullTextSearchLo { charsTotal += line.length(); lineNumber++; - line = fileContent->getLine(lineNumber); + line = codec.decode(fileContent->getLine(lineNumber)); } ParseLocation location; @@ -518,7 +522,7 @@ std::shared_ptr PersistentStorage::getFullTextSearchLo { charsTotal += line.length(); lineNumber++; - line = fileContent->getLine(lineNumber); + line = codec.decode(fileContent->getLine(lineNumber)); } location.endLineNumber = lineNumber; @@ -543,9 +547,9 @@ std::shared_ptr PersistentStorage::getFullTextSearchLo addCompleteFlagsToSourceLocationCollection(collection.get()); MessageStatus( - std::to_string(collection->getSourceLocationCount()) + " results in " + - std::to_string(collection->getSourceLocationFileCount()) + " files for fulltext search (case-" + - (caseSensitive ? "sensitive" : "insensitive") + "): " + searchTerm, + std::to_wstring(collection->getSourceLocationCount()) + L" results in " + + std::to_wstring(collection->getSourceLocationFileCount()) + L" files for fulltext search (case-" + + (caseSensitive ? L"sensitive" : L"insensitive") + L"): " + searchTerm, false, false ).dispatch(); @@ -2580,9 +2584,18 @@ void PersistentStorage::buildFullTextSearchIndex() const { TRACE(); + TextCodec codec(ApplicationSettings::getInstance()->getTextEncoding()); + + m_fullTextSearchCodec = codec.getName(); + + m_fullTextSearchIndex.clear(); for (StorageFile& file : m_sqliteIndexStorage.getAll()) { - m_fullTextSearchIndex.addFile(file.id, m_sqliteIndexStorage.getFileContentById(file.id)->getText()); + + m_fullTextSearchIndex.addFile( + file.id, + codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText()) + ); } } diff --git a/src/lib/data/storage/PersistentStorage.h b/src/lib/data/storage/PersistentStorage.h index f68dc2d1..9fe48a8c 100644 --- a/src/lib/data/storage/PersistentStorage.h +++ b/src/lib/data/storage/PersistentStorage.h @@ -85,7 +85,7 @@ public: virtual StorageEdge getEdgeById(Id edgeId) const override; virtual std::shared_ptr getFullTextSearchLocations( - const std::string& searchTerm, bool caseSensitive) const override; + const std::wstring& searchTerm, bool caseSensitive) const override; virtual std::vector getAutocompletionMatches(const std::wstring& query, NodeTypeSet acceptedNodeTypes) const override; std::vector getAutocompletionSymbolMatches( @@ -187,6 +187,7 @@ private: SearchIndex m_fileIndex; mutable FullTextSearchIndex m_fullTextSearchIndex; + mutable std::string m_fullTextSearchCodec; SqliteIndexStorage m_sqliteIndexStorage; SqliteBookmarkStorage m_sqliteBookmarkStorage; diff --git a/src/lib_gui/qt/element/QtCodeField.cpp b/src/lib_gui/qt/element/QtCodeField.cpp index 4ad177a0..917153c0 100644 --- a/src/lib_gui/qt/element/QtCodeField.cpp +++ b/src/lib_gui/qt/element/QtCodeField.cpp @@ -13,6 +13,7 @@ #include "utility/messaging/type/MessageActivateSourceLocations.h" #include "utility/messaging/type/MessageActivateTokenIds.h" #include "utility/messaging/type/MessageTooltipShow.h" +#include "utility/TextCodec.h" #include "utility/utility.h" std::vector QtCodeField::s_annotationColors; @@ -48,14 +49,14 @@ QtCodeField::QtCodeField( displayCode.pop_back(); } - QTextCodec* codec = QTextCodec::codecForName(ApplicationSettings::getInstance()->getTextEncoding().c_str()); - if (codec) + TextCodec codec(ApplicationSettings::getInstance()->getTextEncoding().c_str()); + if (codec.isValid()) { - QString convertedDisplayCode(codec->toUnicode(displayCode.c_str())); + QString convertedDisplayCode = QString::fromStdWString(codec.decode(displayCode)); setPlainText(convertedDisplayCode); if (displayCode.size() != size_t(convertedDisplayCode.length())) { - LOG_INFO("Converting displayed code to " + codec->name().toStdString() + " resulted in offset of source locations. Correcting this now."); + LOG_INFO("Converting displayed code to " + codec.getName() + " resulted in offset of source locations. Correcting this now."); createMultibyteCharacterLocationCache(); } } diff --git a/src/lib_utility/CMakeLists.txt b/src/lib_utility/CMakeLists.txt new file mode 100644 index 00000000..27381730 --- /dev/null +++ b/src/lib_utility/CMakeLists.txt @@ -0,0 +1,7 @@ + +add_files( + LIB_UTILITY_FILES + + utility/TextCodec.cpp + utility/TextCodec.h +) diff --git a/src/lib_utility/utility/TextCodec.cpp b/src/lib_utility/utility/TextCodec.cpp new file mode 100644 index 00000000..3872ab99 --- /dev/null +++ b/src/lib_utility/utility/TextCodec.cpp @@ -0,0 +1,46 @@ +#include "utility/TextCodec.h" + +#include + +TextCodec::TextCodec(const std::string& name) + : m_name(name) +{ +} + +std::string TextCodec::getName() const +{ + return m_name; +} + +bool TextCodec::isValid() const +{ + QTextCodec* codec = QTextCodec::codecForName(m_name.c_str()); + if (codec) + { + return true; + } + return false; +} + +std::wstring TextCodec::decode(const std::string& unicodeString) const +{ + QTextCodec* codec = QTextCodec::codecForName(m_name.c_str()); + if (codec) + { + QTextDecoder decoder(codec); + return decoder.toUnicode(unicodeString.c_str()).toStdWString(); + } + return QString::fromStdString(unicodeString).toStdWString(); +} + +std::string TextCodec::encode(const std::wstring& string) const +{ + QTextCodec* codec = QTextCodec::codecForName(m_name.c_str()); + if (codec) + { + QTextEncoder encoder(codec); + return encoder.fromUnicode(QString::fromStdWString(string)).toStdString(); + } + + return QString::fromStdWString(string).toStdString(); +} diff --git a/src/lib_utility/utility/TextCodec.h b/src/lib_utility/utility/TextCodec.h new file mode 100644 index 00000000..19e3628a --- /dev/null +++ b/src/lib_utility/utility/TextCodec.h @@ -0,0 +1,22 @@ +#ifndef TEXT_CODEC_H +#define TEXT_CODEC_H + +#include + +class TextCodec +{ +public: + TextCodec(const std::string& name); + + std::string getName() const; + bool isValid() const; + + std::wstring decode(const std::string& unicodeString) const; + + std::string encode(const std::wstring& string) const; + +private: + const std::string m_name; +}; + +#endif // TEXT_CODEC_H