ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: implemented support for non-ascii characters in fulltext search

Browse Source 
* fixed non-ascii characters not working in normal symbol search
* implemented support for non-ascii characters in fulltext search
* fixed fulltext search not working for symbols with lowest and highest character codes in a file
* using wstring in FullTextSearchIndex (causes this index to be dependent on text encoding selected in app settings)
* implemented rebuilding FullTextSearchIndex if used text encoding changed in app settings
* added utility lib
This commit is contained in:
mlangkabel
2018-02-12 20:01:20 +01:00
parent 713754e0e6
commit 919afad03b
16 changed files with 227 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/fulltextsearch/SuffixArray.cpp
+14 -17
View File
@@ -11,11 +11,10 @@ struct suffix
int SuffixArray::cmp(struct suffix a, struct suffix b)
{
return (a.rank[0] == b.rank[0])? (a.rank[1] < b.rank[1] ?1: 0):
(a.rank[0] < b.rank[0] ?1: 0);
return (a.rank[0] == b.rank[0]) ? (a.rank[1] < b.rank[1] ? 1 : 0) : (a.rank[0] < b.rank[0] ? 1 : 0);
}
SuffixArray::SuffixArray(const std::string& text)
SuffixArray::SuffixArray(const std::wstring& text)
: m_text(text)
{
std::transform(m_text.begin(), m_text.end(), m_text.begin(), ::tolower);
@@ -37,7 +36,7 @@ void SuffixArray::printLCP() const
std::vector<int> SuffixArray::buildLCP()
{
int n = m_array.size();
const int n = m_array.size();
std::vector<int> lcp(n, 0);
std::vector<int> invSuff(n, 0);
@@ -75,21 +74,21 @@ std::vector<int> SuffixArray::buildLCP()
return lcp;
}
std::vector<int> SuffixArray::searchForTerm(const std::string& searchTerm) const
std::vector<int> SuffixArray::searchForTerm(const std::wstring& searchTerm) const
{
std::string term = searchTerm;
std::wstring term = searchTerm;
std::transform(term.begin(), term.end(), term.begin(), ::tolower);
int termLength = term.length();
int l = 0;
int r = m_text.length()-1;
int l = -1;
int r = m_text.length();
int m;
std::vector<int> matches;
int compareResult;
while (l+1 < r)
while (l + 1 < r)
{
m = (l+r+1)/2;
m = (l + r + 1) / 2;
compareResult = term.compare(m_text.substr(m_array[m], termLength));
if( compareResult < 0)
{
@@ -102,11 +101,11 @@ std::vector<int> SuffixArray::searchForTerm(const std::string& searchTerm) const
else
{
matches.push_back(m_array[m]);
for (int lower = m-1; m_lcp[lower] >= termLength; lower--)
for (int lower = m-1; lower >= 0 && m_lcp[lower] >= termLength; lower--)
{
matches.push_back(m_array[lower]);
}
for (int higher = m+1; m_lcp[higher-1] >= termLength; higher++)
for (int higher = m+1; higher < termLength && m_lcp[higher-1] >= termLength; higher++)
{
matches.push_back(m_array[higher]);
}
@@ -129,15 +128,15 @@ std::vector<int> SuffixArray::buildSuffixArray()
for (int i = 0; i < n; i++)
{
s.index = i;
s.rank[0] = m_text[i] - 'a';
s.rank[1] = ((i+1) < n)? (m_text[i + 1] - 'a'): -1;
s.rank[0] = m_text[i] - L'a';
s.rank[1] = ((i + 1) < n) ? (m_text[i + 1] - L'a') : -1;
suffixes.push_back(s);
}
std::sort(suffixes.begin(), suffixes.end(), SuffixArray::cmp);
std::vector<int> ind (n,0);
for (int k = 4; k < 2*n; k = k*2)
for (int k = 4; k < 2 * n; k = k * 2)
{
int rank = 0;
int prev_rank = suffixes[0].rank[0];
@@ -178,5 +177,3 @@ std::vector<int> SuffixArray::buildSuffixArray()
return suffixArr;
}