diff --git a/src/lib/data/search/SearchIndex.cpp b/src/lib/data/search/SearchIndex.cpp index ba94e6a3..753e8939 100644 --- a/src/lib/data/search/SearchIndex.cpp +++ b/src/lib/data/search/SearchIndex.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "utility/utility.h" #include "utility/utilityString.h" @@ -107,139 +108,33 @@ void SearchIndex::clear() std::vector SearchIndex::search(const std::string& query, size_t maxResultCount) const { - std::string lowerCaseQuery = utility::toLowerCase(query); + // find paths containing query Path startPath; startPath.node = m_root; + std::vector paths; - search(startPath, lowerCaseQuery, &paths); + searchRecursive(startPath, utility::toLowerCase(query), &paths); - std::set noLetters; - noLetters.insert(' '); - noLetters.insert('.'); - noLetters.insert(','); - noLetters.insert('_'); - noLetters.insert(':'); - noLetters.insert('<'); - noLetters.insert('>'); + // create scored search results + std::multiset searchResults = createScoredResults(paths, maxResultCount * 3); - // scoring paths - std::multiset, bool(*)(const std::pair&, const std::pair&)> scoredPaths( - [](const std::pair& a, const std::pair& b) - { - return a.first > b.first; - } - ); - - for (size_t i = 0; i < paths.size(); i++) + // find best scores + std::map scoresCache; + std::multiset bestResults; + for (const SearchResult& result : searchResults) { - const std::vector& currentIndices = paths[i].indices; - const std::string& currentText = paths[i].text; - - const int unmatchedLetterBonus = -1; - const int consecutiveLetterBonus = 5; - const int camelCaseBonus = 5; - const int noLetterBonus = 3; - const int delayedStartBonus = -3; - const int minDelayedStartBonus = -15; - - int unmatchedLetterScore = 0; - int consecutiveLetterScore = 0; - int camelCaseScore = 0; - int noLetterScore = 0; - - for (size_t j = 0; j < currentIndices.size(); j++) - { - // unmatched and consecutive - if (j > 0) - { - unmatchedLetterScore += (currentIndices[j] - currentIndices[j-1] - 1) * unmatchedLetterBonus; - consecutiveLetterScore += (currentIndices[j] - currentIndices[j-1] == 1 ? consecutiveLetterBonus : 0); - } - - size_t index = currentIndices[j]; - - // camel case - if (isupper(currentText[index])) - { - bool prevIsLower = (index > 0 && islower(currentText[index-1])); - bool nextIsLower = (index + 1 == currentText.size() || islower(currentText[index+1])); - - if (prevIsLower && nextIsLower) - { - camelCaseScore += camelCaseBonus; - } - } - - // after no letter - bool prevIsNoLetter = (index > 0 && noLetters.find(currentText[index-1]) != noLetters.end()); - if (prevIsNoLetter) - { - noLetterScore += noLetterBonus; - } - } - - int leadingStartScore = std::max(int(currentIndices[0]) * delayedStartBonus, minDelayedStartBonus); - - int score = - unmatchedLetterScore + - consecutiveLetterScore + - camelCaseScore + - noLetterScore + - leadingStartScore; - - scoredPaths.insert(std::make_pair(score, paths[i])); + bestResults.insert(bestScoredResult(result, &scoresCache)); } - // preparing results - std::vector searchResults; - for (const std::pair currentResult : scoredPaths) + // narrow down to max result count + auto it = bestResults.end(); + if (maxResultCount && bestResults.size() > maxResultCount) { - if (maxResultCount > 0 && searchResults.size() >= maxResultCount) - { - break; - } - - int currentScore = currentResult.first; - std::vector currentPaths; - currentPaths.push_back(currentResult.second); - - while (currentPaths.size() > 0) - { - std::vector nextPaths; - - for (size_t j = 0; j < currentPaths.size(); j++) - { - Path& currentPath = currentPaths[j]; - if (currentPath.node->elementIds.size() > 0 && (maxResultCount == 0 || searchResults.size() < maxResultCount)) - { - SearchResult result; - result.elementIds = currentPath.node->elementIds; - result.indices = currentPath.indices; - result.text = currentPath.text; - result.score = currentScore; - searchResults.push_back(result); - } - - for (size_t k = 0; k < currentPath.node->edges.size(); k++) - { - Path nextPath; - nextPath.indices = currentPath.indices; - nextPath.node = currentPath.node->edges[k]->target; - nextPath.text = currentPath.text + currentPath.node->edges[k]->s; - nextPaths.push_back(nextPath); - } - } - - currentPaths = nextPaths; - - if (!(maxResultCount == 0 || searchResults.size() < maxResultCount)) - { - break; - } - } + it = bestResults.begin(); + std::advance(it, maxResultCount); } - return searchResults; + return std::vector(bestResults.begin(), it); } void SearchIndex::populateEdgeGate(Edge* e) @@ -257,53 +152,271 @@ void SearchIndex::populateEdgeGate(Edge* e) } } -void SearchIndex::search(const Path& path, const std::string& remainingQuery, std::vector* results) const +void SearchIndex::searchRecursive( + const Path& path, const std::string& remainingQuery, std::vector* results) const { if (remainingQuery.size() == 0) { results->push_back(path); + return; } - else + + for (const Edge* currentEdge : path.node->edges) { - for (size_t i = 0; i < path.node->edges.size(); i++) + // test if s passes the edge's gate. + bool passesGate = true; + for (const char& c : remainingQuery) { - const Edge* currentEdge = path.node->edges[i]; - - // test if s passes the edge's gate. - bool passesGate = true; - for (size_t j = 0; j < remainingQuery.size(); j++) + if (currentEdge->gate.find(c) == currentEdge->gate.end()) { - if (currentEdge->gate.find(tolower(remainingQuery[j])) == currentEdge->gate.end()) + passesGate = false; + break; + } + } + + if (passesGate) + { + // consume characters for edge + const std::string& edgeString = currentEdge->s; + std::vector indices = path.indices; + + size_t j = 0; + for (size_t i = 0; i < edgeString.size() && j < remainingQuery.size(); i++) + { + if (tolower(edgeString[i]) == remainingQuery[j]) { - passesGate = false; - break; + indices.push_back(path.text.size() + i); + j++; } } - if (passesGate) - { - // consume characters for edge - const std::string& edgeString = currentEdge->s; + Path currentPath; + currentPath.node = currentEdge->target; + currentPath.indices = indices; + currentPath.text = path.text + edgeString; - std::vector currentFoundIds = path.indices; - std::string currentRemainingQuery = remainingQuery; - - for (size_t j = 0; j < edgeString.size() && currentRemainingQuery.size() > 0; j++) - { - if (currentRemainingQuery[0] == tolower(edgeString[j])) - { - currentFoundIds.push_back(path.text.size() + j); - currentRemainingQuery = currentRemainingQuery.substr(1); - } - } - - Path currentPath; - currentPath.node = currentEdge->target; - currentPath.indices = currentFoundIds; - currentPath.text = path.text + edgeString; - - search(currentPath, currentRemainingQuery, results); - } + searchRecursive(currentPath, remainingQuery.substr(j), results); } } } + +std::multiset SearchIndex::createScoredResults(const std::vector& paths, size_t maxResultCount) const +{ + // score and order initial paths + std::multiset, bool(*)(const std::pair&, const std::pair&)> scoredPaths( + [](const std::pair& a, const std::pair& b) + { + return a.first > b.first; + } + ); + + for (const Path& path : paths) + { + scoredPaths.insert(std::make_pair(score(path.text, path.indices), path)); + } + + // score paths and subpaths + std::multiset searchResults; + for (const std::pair& p : scoredPaths) + { + std::vector currentPaths; + currentPaths.push_back(p.second); + + while (currentPaths.size()) + { + std::vector nextPaths; + + for (const Path& path : currentPaths) + { + if (path.node->elementIds.size()) + { + SearchResult result; + result.text = path.text; + result.elementIds = path.node->elementIds; + result.indices = path.indices; + result.score = score(path.text, path.indices); + searchResults.insert(result); + + if (maxResultCount && searchResults.size() >= maxResultCount) + { + return searchResults; + } + } + + for (const Edge* edge : path.node->edges) + { + Path nextPath; + nextPath.indices = path.indices; + nextPath.node = edge->target; + nextPath.text = path.text + edge->s; + nextPaths.push_back(nextPath); + } + } + + currentPaths = nextPaths; + } + } + + return searchResults; +} + +SearchResult SearchIndex::bestScoredResult(SearchResult result, std::map* scoresCache) const +{ + if (result.text.size() > 100) + { + return result; + } + + const std::vector& indices = result.indices; + + std::map::const_iterator it = scoresCache->find(result.text.substr(0, indices.back() + 1)); + if (it != scoresCache->end()) + { + result.score = it->second.score; + result.indices = it->second.indices; + } + + int oldScore = result.score; + + bool consecutive = (indices.size() == 1); + for (size_t i = 0; i < indices.size() - 1; i++) + { + if (indices[i + 1] - indices[i] != 1) + { + consecutive = false; + break; + } + } + + if (!consecutive) + { + bestScoredResultRecursive(utility::toLowerCase(result.text), indices, indices.size() - 1, scoresCache, &result); + } + + if (result.score != oldScore || it == scoresCache->end()) + { + scoresCache->emplace(result.text.substr(0, result.indices.back() + 1), result); + } + + return result; +} + +void SearchIndex::bestScoredResultRecursive( + const std::string& lowerText, const std::vector& indices, size_t indicesPos, + std::map* scoresCache, SearchResult* result) const +{ + size_t oldTextPos = indices[indicesPos]; + size_t nextTextPos = (indicesPos + 1 == indices.size() ? result->text.size() : indices[indicesPos + 1]); + for (size_t i = oldTextPos + 1; i < nextTextPos; i++) + { + if (lowerText[i] == lowerText[oldTextPos]) + { + std::vector newIndices = indices; + newIndices[indicesPos] = i; + + int newScore = score(result->text, newIndices); + if (newScore > result->score) + { + result->score = newScore; + result->indices = newIndices; + } + + bestScoredResultRecursive(lowerText, newIndices, indicesPos, scoresCache, result); + break; + } + } + + if (indicesPos + 1 == indices.size()) + { + std::map::const_iterator it = scoresCache->find(result->text.substr(0, indices.back() + 1)); + if (it != scoresCache->end()) + { + result->score = it->second.score; + result->indices = it->second.indices; + return; + } + } + + if (indicesPos > 0) + { + size_t newIndicesPos = indicesPos - 1; + while (newIndicesPos > 0 && indices[newIndicesPos + 1] - indices[newIndicesPos] == 1) + { + newIndicesPos--; + } + + if (indices[newIndicesPos + 1] - indices[newIndicesPos] > 1) + { + bestScoredResultRecursive(lowerText, indices, newIndicesPos, scoresCache, result); + } + } +} + +int SearchIndex::score(const std::string& text, const std::vector& indices) const +{ + const int unmatchedLetterBonus = -1; + const int consecutiveLetterBonus = 5; + const int camelCaseBonus = 4; + const int noLetterBonus = 3; + const int delayedStartBonus = -1; + const int minDelayedStartBonus = -20; + + int unmatchedLetterScore = 0; + int consecutiveLetterScore = 0; + int camelCaseScore = 0; + int noLetterScore = 0; + + static std::set noLetters; + if (!noLetters.size()) + { + noLetters.insert(' '); + noLetters.insert('.'); + noLetters.insert(','); + noLetters.insert('_'); + noLetters.insert(':'); + noLetters.insert('<'); + noLetters.insert('>'); + } + + for (size_t i = 0; i < indices.size(); i++) + { + // unmatched and consecutive + if (i > 0) + { + unmatchedLetterScore += (indices[i] - indices[i - 1] - 1) * unmatchedLetterBonus; + consecutiveLetterScore += (indices[i] - indices[i - 1] == 1 ? consecutiveLetterBonus : 0); + } + + size_t index = indices[i]; + + // camel case + if (isupper(text[index])) + { + bool prevIsLower = (index > 0 && islower(text[index - 1])); + bool nextIsLower = (index + 1 == text.size() || islower(text[index + 1])); + + if (prevIsLower || nextIsLower) + { + camelCaseScore += camelCaseBonus; + } + } + + // after no letter + bool prevIsNoLetter = (index > 0 && noLetters.find(text[index - 1]) != noLetters.end()); + if (prevIsNoLetter) + { + noLetterScore += noLetterBonus; + } + } + + int leadingStartScore = std::max(int(indices[0]) * delayedStartBonus, minDelayedStartBonus); + + int score = + unmatchedLetterScore + + consecutiveLetterScore + + camelCaseScore + + noLetterScore + + leadingStartScore; + + return score; +} diff --git a/src/lib/data/search/SearchIndex.h b/src/lib/data/search/SearchIndex.h index c813ffe6..12936d40 100644 --- a/src/lib/data/search/SearchIndex.h +++ b/src/lib/data/search/SearchIndex.h @@ -17,6 +17,11 @@ struct SearchResult std::set elementIds; std::vector indices; int score; + + bool operator<(const SearchResult& other) const + { + return score > other.score; + } }; class SearchIndex @@ -57,7 +62,14 @@ private: }; void populateEdgeGate(Edge* e); - void search(const Path& path, const std::string& remainingQuery, std::vector* results) const; + void searchRecursive(const Path& path, const std::string& remainingQuery, std::vector* results) const; + + std::multiset createScoredResults(const std::vector& paths, size_t maxResultCount) const; + SearchResult bestScoredResult(SearchResult result, std::map* scoresCache) const; + void bestScoredResultRecursive( + const std::string& lowerText, const std::vector& indices, size_t indicesPos, + std::map* scoresCache, SearchResult* result) const; + int score(const std::string& text, const std::vector& indices) const; std::vector> m_nodes; std::vector> m_edges;