ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: implemented support for non-ascii characters in fulltext search

Browse Source 
* fixed non-ascii characters not working in normal symbol search
* implemented support for non-ascii characters in fulltext search
* fixed fulltext search not working for symbols with lowest and highest character codes in a file
* using wstring in FullTextSearchIndex (causes this index to be dependent on text encoding selected in app settings)
* implemented rebuilding FullTextSearchIndex if used text encoding changed in app settings
* added utility lib
This commit is contained in:
mlangkabel
2018-02-12 20:01:20 +01:00
parent 713754e0e6
commit 919afad03b
16 changed files with 227 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+89 -45
View File
@@ -18,6 +18,7 @@ IF(${CMAKE_BINARY_DIR} STREQUAL ${CMAKE_SOURCE_DIR})
MESSAGE(FATAL_ERROR "Please create a separate build directory and call CMake again")
ENDIF(${CMAKE_BINARY_DIR} STREQUAL ${CMAKE_SOURCE_DIR})
# Variables --------------------------------------------------------------------
set(PROJECT_NAME Sourcetrail)
@@ -32,6 +33,7 @@ endif()
set(APP_PROJECT_NAME "${PROJECT_NAME}")
set(LIB_LICENSE_PROJECT_NAME "${PROJECT_NAME}_lib_license")
set(LIB_UTILITY_PROJECT_NAME "${PROJECT_NAME}_lib_utility")
set(LIB_GUI_PROJECT_NAME "${PROJECT_NAME}_lib_gui")
set(LIB_CXX_PROJECT_NAME "${PROJECT_NAME}_lib_cxx")
set(LIB_JAVA_PROJECT_NAME "${PROJECT_NAME}_lib_java")
@@ -56,6 +58,7 @@ else ()
set(PLATFORM_INCLUDE "includesLinux.h")
endif ()
# Project ----------------------------------------------------------------------
project(${PROJECT_NAME})
@@ -74,7 +77,9 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_C_STANDARD 99)
# Settings ---------------------------------------------------------------------
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option -fcolor-diagnostics")
endif()
@@ -84,6 +89,7 @@ endif()
#add_definitions(-fno-omit-frame-pointer)
#endif ()
# Clang ------------------------------------------------------------------------
if (UNIX AND APPLE)
@@ -106,6 +112,7 @@ find_package(CLANG REQUIRED PATHS "${CMAKE_SOURCE_DIR}/cmake" NO_DEFAULT_PATH)
add_definitions(${CLANG_DEFINITIONS})
# Boost ------------------------------------------------------------------------
set(Boost_USE_MULTITHREAD ON)
@@ -132,7 +139,8 @@ endif()
find_package(Boost 1.64 COMPONENTS system program_options filesystem date_time REQUIRED)
# Botan -------------------------------------------------------------------------
# Botan ------------------------------------------------------------------------
set(BOTAN_VERSION "botan-2")
@@ -184,7 +192,47 @@ if(NOT BOTAN_INCLUDE_DIR)
endif()
# Lib License -------------------------------------------------------------------
# Qt ---------------------------------------------------------------------------
# Find the QtWidgets library
if(UNIX)
set(CMAKE_PREFIX_PATH "$ENV{QT_DIR}/")
if(APPLE)
set(Qt5Widgets_DIR "$ENV{QT_DIR}/cmake/Qt5Widgets/")
set(Qt5PrintSupport_DIR "$ENV{QT_DIR}/cmake/Qt5PrintSupport/")
endif()
else()
if(${WINXXBITS} STREQUAL "Win32")
set(CMAKE_PREFIX_PATH "$ENV{QT_WIN32_DIR}/")
else()
set(CMAKE_PREFIX_PATH "$ENV{QT_WIN64_DIR}/")
endif()
set(Qt5Widgets_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Widgets")
set(Qt5PrintSupport_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5PrintSupport")
set(Qt5Network_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Network")
set(Qt5WinExtras_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5WinExtras")
endif()
find_package(Qt5Widgets)
find_package(Qt5PrintSupport)
find_package(Qt5Network)
if (WIN32)
find_package(Qt5WinExtras)
endif()
if(Qt5Widgets_FOUND)
MESSAGE(STATUS "Found Qt ${Qt5Widgets_VERSION_STRING}")
# FIX: Qt was built with -reduce-relocations
if (Qt5_POSITION_INDEPENDENT_CODE)
SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif()
endif()
# Lib License ------------------------------------------------------------------
if (UNIX)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/lib/")
@@ -223,6 +271,31 @@ else()
endif()
# Lib Utility ------------------------------------------------------------------
add_subdirectory(src/lib_utility)
add_library(${LIB_UTILITY_PROJECT_NAME} ${LIB_UTILITY_FILES})
create_source_groups(${LIB_UTILITY_FILES})
set_property(
TARGET ${LIB_UTILITY_PROJECT_NAME}
PROPERTY INCLUDE_DIRECTORIES
"${CMAKE_SOURCE_DIR}/src/lib_utility"
)
target_include_directories(${LIB_UTILITY_PROJECT_NAME} SYSTEM
PUBLIC ${Boost_INCLUDE_DIR}
)
if(UNIX)
target_link_libraries(${LIB_UTILITY_PROJECT_NAME} ${Boost_LIBRARIES} Qt5::Widgets Qt5::Network)
else()
target_link_libraries(${LIB_UTILITY_PROJECT_NAME} ${Boost_LIBRARIES} Qt5::Widgets Qt5::Network Qt5::WinExtras)
endif()
# Lib --------------------------------------------------------------------------
add_subdirectory(src/lib)
@@ -244,6 +317,7 @@ set_property(
"${CMAKE_SOURCE_DIR}/src/external"
"${CMAKE_SOURCE_DIR}/src/lib_license"
"${CMAKE_SOURCE_DIR}/src/lib_gui"
"${CMAKE_SOURCE_DIR}/src/lib_utility"
"${CMAKE_BINARY_DIR}/src/lib_license"
"${CMAKE_SOURCE_DIR}/src/lib_cxx"
"${CMAKE_SOURCE_DIR}/src/lib_java"
@@ -254,10 +328,10 @@ target_include_directories(${LIB_PROJECT_NAME} SYSTEM
"${CMAKE_SOURCE_DIR}/src/external"
)
target_link_libraries(${LIB_PROJECT_NAME} ${Boost_LIBRARIES} ${LIB_LICENSE_PROJECT_NAME})
target_link_libraries(${LIB_PROJECT_NAME} ${LIB_UTILITY_PROJECT_NAME} ${LIB_LICENSE_PROJECT_NAME} ${Boost_LIBRARIES})
# Lib Cxx -------------------------------------------------------------------
# Lib Cxx ----------------------------------------------------------------------
add_subdirectory(src/lib_cxx)
@@ -287,7 +361,8 @@ if (WIN32)
target_link_libraries(${LIB_CXX_PROJECT_NAME} version)
endif()
# Lib Java -------------------------------------------------------------------
# Lib Java ---------------------------------------------------------------------
find_package(JNI)
@@ -331,43 +406,6 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
# Instruct CMake to run moc automatically when needed.
set(CMAKE_AUTOMOC ON)
# Find the QtWidgets library
if(UNIX)
set(CMAKE_PREFIX_PATH "$ENV{QT_DIR}/")
if(APPLE)
set(Qt5Widgets_DIR "$ENV{QT_DIR}/cmake/Qt5Widgets/")
set(Qt5PrintSupport_DIR "$ENV{QT_DIR}/cmake/Qt5PrintSupport/")
endif()
else()
if(${WINXXBITS} STREQUAL "Win32")
set(CMAKE_PREFIX_PATH "$ENV{QT_WIN32_DIR}/")
else()
set(CMAKE_PREFIX_PATH "$ENV{QT_WIN64_DIR}/")
endif()
set(Qt5Widgets_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Widgets")
set(Qt5PrintSupport_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5PrintSupport")
set(Qt5Network_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5Network")
set(Qt5WinExtras_DIR "${CMAKE_PREFIX_PATH}/lib/cmake/Qt5WinExtras")
endif()
find_package(Qt5Widgets)
find_package(Qt5PrintSupport)
find_package(Qt5Network)
if (WIN32)
find_package(Qt5WinExtras)
endif()
if(Qt5Widgets_FOUND)
MESSAGE(STATUS "Found Qt ${Qt5Widgets_VERSION_STRING}")
# FIX: Qt was built with -reduce-relocations
if (Qt5_POSITION_INDEPENDENT_CODE)
SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif()
endif()
# target for running versionnumber script
# workaround for running customcommand (ninja dependency cycle)
add_custom_target(
@@ -377,9 +415,9 @@ add_custom_target(
add_library(${LIB_GUI_PROJECT_NAME} ${LIB_GUI_FILES} ${CMAKE_BINARY_DIR}/src/lib_gui/version.h)
if(UNIX)
target_link_libraries(${LIB_GUI_PROJECT_NAME} Qt5::Widgets Qt5::Network)
target_link_libraries(${LIB_GUI_PROJECT_NAME} ${LIB_UTILITY_PROJECT_NAME} Qt5::Widgets Qt5::Network)
else()
target_link_libraries(${LIB_GUI_PROJECT_NAME} Qt5::Widgets Qt5::Network Qt5::WinExtras)
target_link_libraries(${LIB_GUI_PROJECT_NAME} ${LIB_UTILITY_PROJECT_NAME} Qt5::Widgets Qt5::Network Qt5::WinExtras)
endif()
# command for versioning script
@@ -400,6 +438,7 @@ set_property(
TARGET ${LIB_GUI_PROJECT_NAME}
PROPERTY INCLUDE_DIRECTORIES
"${CMAKE_SOURCE_DIR}/src/lib_gui"
"${CMAKE_SOURCE_DIR}/src/lib_utility"
"${CMAKE_SOURCE_DIR}/src/lib"
"${CMAKE_SOURCE_DIR}/src/lib_cxx"
"${CMAKE_SOURCE_DIR}/src/lib_java"
@@ -431,6 +470,7 @@ include(cmake/public_key.cmake)
set(CMAKE_AUTOMOC OFF)
# Indexer App ------------------------------------------------------------------
if (UNIX)
@@ -484,6 +524,7 @@ set_property(
"${CMAKE_BINARY_DIR}/src/lib_license"
)
# App --------------------------------------------------------------------------
if (UNIX)
@@ -603,7 +644,8 @@ if (APPLE)
endif ()
# License Generator --------------------------------------------------------
# License Generator -----------------------------------------------------------
add_subdirectory(src/license_generator)
@@ -642,6 +684,7 @@ endif()
target_link_libraries(${LICENSE_GENERATOR_PROJECT_NAME} ${LIB_LICENSE_PROJECT_NAME} ${Boost_LIBRARIES} )
# CxxTest ----------------------------------------------------------------------
if (UNIX)
@@ -717,6 +760,7 @@ elseif (UNIX)
add_dependencies(${TEST_PROJECT_NAME} gen)
endif ()
# Visual Leak Detector ---------------------------------------------------------
if (WIN32)
src/lib/component/controller/CodeController.cpp
+7 -8
View File
@@ -2,18 +2,17 @@
#include <memory>
#include "utility/file/FileInfo.h"
#include "utility/messaging/type/MessageStatus.h"
#include "utility/text/TextAccess.h"
#include "utility/tracing.h"
#include "utility/logging/logging.h"
#include "utility/utilityString.h"
#include "data/access/StorageAccess.h"
#include "data/location/SourceLocation.h"
#include "data/location/SourceLocationCollection.h"
#include "data/location/SourceLocationFile.h"
#include "settings/ApplicationSettings.h"
#include "utility/file/FileInfo.h"
#include "utility/logging/logging.h"
#include "utility/messaging/type/MessageStatus.h"
#include "utility/text/TextAccess.h"
#include "utility/tracing.h"
#include "utility/utilityString.h"
#include "Application.h"
CodeController::CodeController(StorageAccess* storageAccess)
@@ -339,7 +338,7 @@ void CodeController::handleMessage(MessageSearchFullText* message)
saveOrRestoreViewMode(message);
m_collection = m_storageAccess->getFullTextSearchLocations(utility::encodeToUtf8(message->searchTerm), message->caseSensitive);
m_collection = m_storageAccess->getFullTextSearchLocations(message->searchTerm, message->caseSensitive);
CodeView::ScrollParams scrollParams(CodeView::ScrollParams::SCROLL_TO_DEFINITION);
getView()->scrollTo(scrollParams);
src/lib/data/access/StorageAccess.h
+1 -1
View File
@@ -48,7 +48,7 @@ public:
virtual StorageEdge getEdgeById(Id edgeId) const = 0;
virtual std::shared_ptr<SourceLocationCollection> getFullTextSearchLocations(
const std::string& searchTerm, bool caseSensitive) const = 0;
const std::wstring& searchTerm, bool caseSensitive) const = 0;
virtual std::vector<SearchMatch> getAutocompletionMatches(const std::wstring& query, NodeTypeSet acceptedNodeTypes) const = 0;
virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const = 0;
src/lib/data/access/StorageAccessProxy.cpp
+1 -1
View File
@@ -116,7 +116,7 @@ StorageEdge StorageAccessProxy::getEdgeById(Id edgeId) const
}
std::shared_ptr<SourceLocationCollection> StorageAccessProxy::getFullTextSearchLocations(
const std::string &searchTerm, bool caseSensitive) const
const std::wstring &searchTerm, bool caseSensitive) const
{
if (hasSubject())
{
src/lib/data/access/StorageAccessProxy.h
+1 -1
View File
@@ -32,7 +32,7 @@ public:
virtual StorageEdge getEdgeById(Id edgeId) const override;
virtual std::shared_ptr<SourceLocationCollection> getFullTextSearchLocations(
const std::string& searchTerm, bool caseSensitive) const override;
const std::wstring& searchTerm, bool caseSensitive) const override;
virtual std::vector<SearchMatch> getAutocompletionMatches(const std::wstring& query, NodeTypeSet acceptedNodeTypes) const override;
virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const override;
src/lib/data/fulltextsearch/FullTextSearchIndex.cpp
+2 -2
View File
@@ -4,7 +4,7 @@
#include "utility/logging/logging.h"
#include "utility/tracing.h"
void FullTextSearchIndex::addFile(Id fileId, const std::string& file)
void FullTextSearchIndex::addFile(Id fileId, const std::wstring& file)
{
if( file.empty() )
{
@@ -20,7 +20,7 @@ void FullTextSearchIndex::addFile(Id fileId, const std::string& file)
m_files.push_back(fts_file);
}
std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::string& term) const
std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::wstring& term) const
{
TRACE();
src/lib/data/fulltextsearch/FullTextSearchIndex.h
+2 -2
View File
@@ -30,8 +30,8 @@ struct FullTextSearchFile
class FullTextSearchIndex
{
public:
void addFile(Id fileId, const std::string& file);
std::vector<FullTextSearchResult> searchForTerm(const std::string& term) const;
void addFile(Id fileId, const std::wstring& file);
std::vector<FullTextSearchResult> searchForTerm(const std::wstring& term) const;
size_t fileCount() const;
src/lib/data/fulltextsearch/SuffixArray.cpp
+14 -17
View File
@@ -11,11 +11,10 @@ struct suffix
int SuffixArray::cmp(struct suffix a, struct suffix b)
{
return (a.rank[0] == b.rank[0])? (a.rank[1] < b.rank[1] ?1: 0):
(a.rank[0] < b.rank[0] ?1: 0);
return (a.rank[0] == b.rank[0]) ? (a.rank[1] < b.rank[1] ? 1 : 0) : (a.rank[0] < b.rank[0] ? 1 : 0);
}
SuffixArray::SuffixArray(const std::string& text)
SuffixArray::SuffixArray(const std::wstring& text)
: m_text(text)
{
std::transform(m_text.begin(), m_text.end(), m_text.begin(), ::tolower);
@@ -37,7 +36,7 @@ void SuffixArray::printLCP() const
std::vector<int> SuffixArray::buildLCP()
{
int n = m_array.size();
const int n = m_array.size();
std::vector<int> lcp(n, 0);
std::vector<int> invSuff(n, 0);
@@ -75,21 +74,21 @@ std::vector<int> SuffixArray::buildLCP()
return lcp;
}
std::vector<int> SuffixArray::searchForTerm(const std::string& searchTerm) const
std::vector<int> SuffixArray::searchForTerm(const std::wstring& searchTerm) const
{
std::string term = searchTerm;
std::wstring term = searchTerm;
std::transform(term.begin(), term.end(), term.begin(), ::tolower);
int termLength = term.length();
int l = 0;
int r = m_text.length()-1;
int l = -1;
int r = m_text.length();
int m;
std::vector<int> matches;
int compareResult;
while (l+1 < r)
while (l + 1 < r)
{
m = (l+r+1)/2;
m = (l + r + 1) / 2;
compareResult = term.compare(m_text.substr(m_array[m], termLength));
if( compareResult < 0)
{
@@ -102,11 +101,11 @@ std::vector<int> SuffixArray::searchForTerm(const std::string& searchTerm) const
else
{
matches.push_back(m_array[m]);
for (int lower = m-1; m_lcp[lower] >= termLength; lower--)
for (int lower = m-1; lower >= 0 && m_lcp[lower] >= termLength; lower--)
{
matches.push_back(m_array[lower]);
}
for (int higher = m+1; m_lcp[higher-1] >= termLength; higher++)
for (int higher = m+1; higher < termLength && m_lcp[higher-1] >= termLength; higher++)
{
matches.push_back(m_array[higher]);
}
@@ -129,15 +128,15 @@ std::vector<int> SuffixArray::buildSuffixArray()
for (int i = 0; i < n; i++)
{
s.index = i;
s.rank[0] = m_text[i] - 'a';
s.rank[1] = ((i+1) < n)? (m_text[i + 1] - 'a'): -1;
s.rank[0] = m_text[i] - L'a';
s.rank[1] = ((i + 1) < n) ? (m_text[i + 1] - L'a') : -1;
suffixes.push_back(s);
}
std::sort(suffixes.begin(), suffixes.end(), SuffixArray::cmp);
std::vector<int> ind (n,0);
for (int k = 4; k < 2*n; k = k*2)
for (int k = 4; k < 2 * n; k = k * 2)
{
int rank = 0;
int prev_rank = suffixes[0].rank[0];
@@ -178,5 +177,3 @@ std::vector<int> SuffixArray::buildSuffixArray()
return suffixArr;
}
src/lib/data/fulltextsearch/SuffixArray.h
+3 -3
View File
@@ -8,8 +8,8 @@
class SuffixArray
{
public:
SuffixArray(const std::string& text);
std::vector<int> searchForTerm(const std::string& searchTerm) const;
SuffixArray(const std::wstring& text);
std::vector<int> searchForTerm(const std::wstring& searchTerm) const;
static int cmp(struct suffix a, struct suffix b);
void printArray() const;
@@ -30,7 +30,7 @@ private:
std::vector<int> buildSuffixArray();
std::vector<int> m_array;
std::vector<int> m_lcp;
std::string m_text;
std::wstring m_text;
};
#endif // SUFFIX_ARRAY_H
src/lib/data/search/SearchIndex.cpp
+1 -1
View File
@@ -162,7 +162,7 @@ void SearchIndex::searchRecursive(
// test if s passes the edge's gate.
bool passesGate = true;
for (const char& c : remainingQuery)
for (const wchar_t& c : remainingQuery)
{
if (currentEdge->gate.find(c) == currentEdge->gate.end())
{
src/lib/data/storage/PersistentStorage.cpp
+24 -11
View File
@@ -21,6 +21,7 @@
#include "utility/messaging/type/MessageNewErrors.h"
#include "utility/messaging/type/MessageStatus.h"
#include "utility/text/TextAccess.h"
#include "utility/TextCodec.h"
#include "utility/TimeStamp.h"
#include "utility/tracing.h"
#include "utility/utility.h"
@@ -292,6 +293,7 @@ void PersistentStorage::clearCaches()
m_hierarchyCache.clear();
m_fullTextSearchIndex.clear();
m_fullTextSearchCodec = "";
}
std::set<FilePath> PersistentStorage::getReferenced(const std::set<FilePath>& filePaths)
@@ -463,25 +465,27 @@ StorageEdge PersistentStorage::getEdgeById(Id edgeId) const
}
std::shared_ptr<SourceLocationCollection> PersistentStorage::getFullTextSearchLocations(
const std::string& searchTerm, bool caseSensitive
const std::wstring& searchTerm, bool caseSensitive
) const
{
TRACE();
const TextCodec codec(ApplicationSettings::getInstance()->getTextEncoding());
std::shared_ptr<SourceLocationCollection> collection = std::make_shared<SourceLocationCollection>();
if (!searchTerm.size())
if (searchTerm.empty())
{
return collection;
}
if (m_fullTextSearchIndex.fileCount() == 0)
if (m_fullTextSearchCodec != codec.getName())
{
MessageStatus(L"Building fulltext search index", false, true).dispatch();
buildFullTextSearchIndex();
}
MessageStatus(
std::string("Searching fulltext (case-") + (caseSensitive ? "sensitive" : "insensitive") + "): " + searchTerm,
std::wstring(L"Searching fulltext (case-") + (caseSensitive ? L"sensitive" : L"insensitive") + L"): " + searchTerm,
false, true
).dispatch();
@@ -494,7 +498,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getFullTextSearchLo
int charsTotal = 0;
int lineNumber = 1;
std::string line = fileContent->getLine(lineNumber);
std::wstring line = codec.decode(fileContent->getLine(lineNumber));
for (int pos : fileHits.positions)
{
@@ -502,7 +506,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getFullTextSearchLo
{
charsTotal += line.length();
lineNumber++;
line = fileContent->getLine(lineNumber);
line = codec.decode(fileContent->getLine(lineNumber));
}
ParseLocation location;
@@ -518,7 +522,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getFullTextSearchLo
{
charsTotal += line.length();
lineNumber++;
line = fileContent->getLine(lineNumber);
line = codec.decode(fileContent->getLine(lineNumber));
}
location.endLineNumber = lineNumber;
@@ -543,9 +547,9 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getFullTextSearchLo
addCompleteFlagsToSourceLocationCollection(collection.get());
MessageStatus(
std::to_string(collection->getSourceLocationCount()) + " results in " +
std::to_string(collection->getSourceLocationFileCount()) + " files for fulltext search (case-" +
(caseSensitive ? "sensitive" : "insensitive") + "): " + searchTerm,
std::to_wstring(collection->getSourceLocationCount()) + L" results in " +
std::to_wstring(collection->getSourceLocationFileCount()) + L" files for fulltext search (case-" +
(caseSensitive ? L"sensitive" : L"insensitive") + L"): " + searchTerm,
false, false
).dispatch();
@@ -2580,9 +2584,18 @@ void PersistentStorage::buildFullTextSearchIndex() const
{
TRACE();
TextCodec codec(ApplicationSettings::getInstance()->getTextEncoding());
m_fullTextSearchCodec = codec.getName();
m_fullTextSearchIndex.clear();
for (StorageFile& file : m_sqliteIndexStorage.getAll<StorageFile>())
{
m_fullTextSearchIndex.addFile(file.id, m_sqliteIndexStorage.getFileContentById(file.id)->getText());
m_fullTextSearchIndex.addFile(
file.id,
codec.decode(m_sqliteIndexStorage.getFileContentById(file.id)->getText())
);
}
}
src/lib/data/storage/PersistentStorage.h
+2 -1
View File
@@ -85,7 +85,7 @@ public:
virtual StorageEdge getEdgeById(Id edgeId) const override;
virtual std::shared_ptr<SourceLocationCollection> getFullTextSearchLocations(
const std::string& searchTerm, bool caseSensitive) const override;
const std::wstring& searchTerm, bool caseSensitive) const override;
virtual std::vector<SearchMatch> getAutocompletionMatches(const std::wstring& query, NodeTypeSet acceptedNodeTypes) const override;
std::vector<SearchMatch> getAutocompletionSymbolMatches(
@@ -187,6 +187,7 @@ private:
SearchIndex m_fileIndex;
mutable FullTextSearchIndex m_fullTextSearchIndex;
mutable std::string m_fullTextSearchCodec;
SqliteIndexStorage m_sqliteIndexStorage;
SqliteBookmarkStorage m_sqliteBookmarkStorage;
src/lib_gui/qt/element/QtCodeField.cpp
+5 -4
View File
@@ -13,6 +13,7 @@
#include "utility/messaging/type/MessageActivateSourceLocations.h"
#include "utility/messaging/type/MessageActivateTokenIds.h"
#include "utility/messaging/type/MessageTooltipShow.h"
#include "utility/TextCodec.h"
#include "utility/utility.h"
std::vector<QtCodeField::AnnotationColor> QtCodeField::s_annotationColors;
@@ -48,14 +49,14 @@ QtCodeField::QtCodeField(
displayCode.pop_back();
}
QTextCodec* codec = QTextCodec::codecForName(ApplicationSettings::getInstance()->getTextEncoding().c_str());
if (codec)
TextCodec codec(ApplicationSettings::getInstance()->getTextEncoding().c_str());
if (codec.isValid())
{
QString convertedDisplayCode(codec->toUnicode(displayCode.c_str()));
QString convertedDisplayCode = QString::fromStdWString(codec.decode(displayCode));
setPlainText(convertedDisplayCode);
if (displayCode.size() != size_t(convertedDisplayCode.length()))
{
LOG_INFO("Converting displayed code to " + codec->name().toStdString() + " resulted in offset of source locations. Correcting this now.");
LOG_INFO("Converting displayed code to " + codec.getName() + " resulted in offset of source locations. Correcting this now.");
createMultibyteCharacterLocationCache();
}
}
src/lib_utility/CMakeLists.txt
+7
View File
@@ -0,0 +1,7 @@
add_files(
LIB_UTILITY_FILES
utility/TextCodec.cpp
utility/TextCodec.h
)
src/lib_utility/utility/TextCodec.cpp
+46
View File
@@ -0,0 +1,46 @@
#include "utility/TextCodec.h"
#include <QTextCodec>
TextCodec::TextCodec(const std::string& name)
: m_name(name)
{
}
std::string TextCodec::getName() const
{
return m_name;
}
bool TextCodec::isValid() const
{
QTextCodec* codec = QTextCodec::codecForName(m_name.c_str());
if (codec)
{
return true;
}
return false;
}
std::wstring TextCodec::decode(const std::string& unicodeString) const
{
QTextCodec* codec = QTextCodec::codecForName(m_name.c_str());
if (codec)
{
QTextDecoder decoder(codec);
return decoder.toUnicode(unicodeString.c_str()).toStdWString();
}
return QString::fromStdString(unicodeString).toStdWString();
}
std::string TextCodec::encode(const std::wstring& string) const
{
QTextCodec* codec = QTextCodec::codecForName(m_name.c_str());
if (codec)
{
QTextEncoder encoder(codec);
return encoder.fromUnicode(QString::fromStdWString(string)).toStdString();
}
return QString::fromStdWString(string).toStdString();
}
src/lib_utility/utility/TextCodec.h
+22
View File
@@ -0,0 +1,22 @@
#ifndef TEXT_CODEC_H
#define TEXT_CODEC_H
#include <string>
class TextCodec
{
public:
TextCodec(const std::string& name);
std::string getName() const;
bool isValid() const;
std::wstring decode(const std::string& unicodeString) const;
std::string encode(const std::wstring& string) const;
private:
const std::string m_name;
};
#endif // TEXT_CODEC_H