data: calculate best scores for autocompletion matches by moving indices
This commit is contained in:
Eberhard Graether
+269
-156
@@ -2,6 +2,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <ctype.h>
|
||||
#include <iterator>
|
||||
|
||||
#include "utility/utility.h"
|
||||
#include "utility/utilityString.h"
|
||||
@@ -107,139 +108,33 @@ void SearchIndex::clear()
|
||||
|
||||
std::vector<SearchResult> SearchIndex::search(const std::string& query, size_t maxResultCount) const
|
||||
{
|
||||
std::string lowerCaseQuery = utility::toLowerCase(query);
|
||||
// find paths containing query
|
||||
Path startPath;
|
||||
startPath.node = m_root;
|
||||
|
||||
std::vector<Path> paths;
|
||||
search(startPath, lowerCaseQuery, &paths);
|
||||
searchRecursive(startPath, utility::toLowerCase(query), &paths);
|
||||
|
||||
std::set<char> noLetters;
|
||||
noLetters.insert(' ');
|
||||
noLetters.insert('.');
|
||||
noLetters.insert(',');
|
||||
noLetters.insert('_');
|
||||
noLetters.insert(':');
|
||||
noLetters.insert('<');
|
||||
noLetters.insert('>');
|
||||
// create scored search results
|
||||
std::multiset<SearchResult> searchResults = createScoredResults(paths, maxResultCount * 3);
|
||||
|
||||
// scoring paths
|
||||
std::multiset<std::pair<int, Path>, bool(*)(const std::pair<int, Path>&, const std::pair<int, Path>&)> scoredPaths(
|
||||
[](const std::pair<int, Path>& a, const std::pair<int, Path>& b)
|
||||
{
|
||||
return a.first > b.first;
|
||||
}
|
||||
);
|
||||
|
||||
for (size_t i = 0; i < paths.size(); i++)
|
||||
// find best scores
|
||||
std::map<std::string, SearchResult> scoresCache;
|
||||
std::multiset<SearchResult> bestResults;
|
||||
for (const SearchResult& result : searchResults)
|
||||
{
|
||||
const std::vector<size_t>& currentIndices = paths[i].indices;
|
||||
const std::string& currentText = paths[i].text;
|
||||
|
||||
const int unmatchedLetterBonus = -1;
|
||||
const int consecutiveLetterBonus = 5;
|
||||
const int camelCaseBonus = 5;
|
||||
const int noLetterBonus = 3;
|
||||
const int delayedStartBonus = -3;
|
||||
const int minDelayedStartBonus = -15;
|
||||
|
||||
int unmatchedLetterScore = 0;
|
||||
int consecutiveLetterScore = 0;
|
||||
int camelCaseScore = 0;
|
||||
int noLetterScore = 0;
|
||||
|
||||
for (size_t j = 0; j < currentIndices.size(); j++)
|
||||
{
|
||||
// unmatched and consecutive
|
||||
if (j > 0)
|
||||
{
|
||||
unmatchedLetterScore += (currentIndices[j] - currentIndices[j-1] - 1) * unmatchedLetterBonus;
|
||||
consecutiveLetterScore += (currentIndices[j] - currentIndices[j-1] == 1 ? consecutiveLetterBonus : 0);
|
||||
}
|
||||
|
||||
size_t index = currentIndices[j];
|
||||
|
||||
// camel case
|
||||
if (isupper(currentText[index]))
|
||||
{
|
||||
bool prevIsLower = (index > 0 && islower(currentText[index-1]));
|
||||
bool nextIsLower = (index + 1 == currentText.size() || islower(currentText[index+1]));
|
||||
|
||||
if (prevIsLower && nextIsLower)
|
||||
{
|
||||
camelCaseScore += camelCaseBonus;
|
||||
}
|
||||
}
|
||||
|
||||
// after no letter
|
||||
bool prevIsNoLetter = (index > 0 && noLetters.find(currentText[index-1]) != noLetters.end());
|
||||
if (prevIsNoLetter)
|
||||
{
|
||||
noLetterScore += noLetterBonus;
|
||||
}
|
||||
}
|
||||
|
||||
int leadingStartScore = std::max(int(currentIndices[0]) * delayedStartBonus, minDelayedStartBonus);
|
||||
|
||||
int score =
|
||||
unmatchedLetterScore +
|
||||
consecutiveLetterScore +
|
||||
camelCaseScore +
|
||||
noLetterScore +
|
||||
leadingStartScore;
|
||||
|
||||
scoredPaths.insert(std::make_pair(score, paths[i]));
|
||||
bestResults.insert(bestScoredResult(result, &scoresCache));
|
||||
}
|
||||
|
||||
// preparing results
|
||||
std::vector<SearchResult> searchResults;
|
||||
for (const std::pair<int, Path> currentResult : scoredPaths)
|
||||
// narrow down to max result count
|
||||
auto it = bestResults.end();
|
||||
if (maxResultCount && bestResults.size() > maxResultCount)
|
||||
{
|
||||
if (maxResultCount > 0 && searchResults.size() >= maxResultCount)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
int currentScore = currentResult.first;
|
||||
std::vector<Path> currentPaths;
|
||||
currentPaths.push_back(currentResult.second);
|
||||
|
||||
while (currentPaths.size() > 0)
|
||||
{
|
||||
std::vector<Path> nextPaths;
|
||||
|
||||
for (size_t j = 0; j < currentPaths.size(); j++)
|
||||
{
|
||||
Path& currentPath = currentPaths[j];
|
||||
if (currentPath.node->elementIds.size() > 0 && (maxResultCount == 0 || searchResults.size() < maxResultCount))
|
||||
{
|
||||
SearchResult result;
|
||||
result.elementIds = currentPath.node->elementIds;
|
||||
result.indices = currentPath.indices;
|
||||
result.text = currentPath.text;
|
||||
result.score = currentScore;
|
||||
searchResults.push_back(result);
|
||||
}
|
||||
|
||||
for (size_t k = 0; k < currentPath.node->edges.size(); k++)
|
||||
{
|
||||
Path nextPath;
|
||||
nextPath.indices = currentPath.indices;
|
||||
nextPath.node = currentPath.node->edges[k]->target;
|
||||
nextPath.text = currentPath.text + currentPath.node->edges[k]->s;
|
||||
nextPaths.push_back(nextPath);
|
||||
}
|
||||
}
|
||||
|
||||
currentPaths = nextPaths;
|
||||
|
||||
if (!(maxResultCount == 0 || searchResults.size() < maxResultCount))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
it = bestResults.begin();
|
||||
std::advance(it, maxResultCount);
|
||||
}
|
||||
|
||||
return searchResults;
|
||||
return std::vector<SearchResult>(bestResults.begin(), it);
|
||||
}
|
||||
|
||||
void SearchIndex::populateEdgeGate(Edge* e)
|
||||
@@ -257,53 +152,271 @@ void SearchIndex::populateEdgeGate(Edge* e)
|
||||
}
|
||||
}
|
||||
|
||||
void SearchIndex::search(const Path& path, const std::string& remainingQuery, std::vector<SearchIndex::Path>* results) const
|
||||
void SearchIndex::searchRecursive(
|
||||
const Path& path, const std::string& remainingQuery, std::vector<SearchIndex::Path>* results) const
|
||||
{
|
||||
if (remainingQuery.size() == 0)
|
||||
{
|
||||
results->push_back(path);
|
||||
return;
|
||||
}
|
||||
else
|
||||
|
||||
for (const Edge* currentEdge : path.node->edges)
|
||||
{
|
||||
for (size_t i = 0; i < path.node->edges.size(); i++)
|
||||
// test if s passes the edge's gate.
|
||||
bool passesGate = true;
|
||||
for (const char& c : remainingQuery)
|
||||
{
|
||||
const Edge* currentEdge = path.node->edges[i];
|
||||
|
||||
// test if s passes the edge's gate.
|
||||
bool passesGate = true;
|
||||
for (size_t j = 0; j < remainingQuery.size(); j++)
|
||||
if (currentEdge->gate.find(c) == currentEdge->gate.end())
|
||||
{
|
||||
if (currentEdge->gate.find(tolower(remainingQuery[j])) == currentEdge->gate.end())
|
||||
passesGate = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (passesGate)
|
||||
{
|
||||
// consume characters for edge
|
||||
const std::string& edgeString = currentEdge->s;
|
||||
std::vector<size_t> indices = path.indices;
|
||||
|
||||
size_t j = 0;
|
||||
for (size_t i = 0; i < edgeString.size() && j < remainingQuery.size(); i++)
|
||||
{
|
||||
if (tolower(edgeString[i]) == remainingQuery[j])
|
||||
{
|
||||
passesGate = false;
|
||||
break;
|
||||
indices.push_back(path.text.size() + i);
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
if (passesGate)
|
||||
{
|
||||
// consume characters for edge
|
||||
const std::string& edgeString = currentEdge->s;
|
||||
Path currentPath;
|
||||
currentPath.node = currentEdge->target;
|
||||
currentPath.indices = indices;
|
||||
currentPath.text = path.text + edgeString;
|
||||
|
||||
std::vector<size_t> currentFoundIds = path.indices;
|
||||
std::string currentRemainingQuery = remainingQuery;
|
||||
|
||||
for (size_t j = 0; j < edgeString.size() && currentRemainingQuery.size() > 0; j++)
|
||||
{
|
||||
if (currentRemainingQuery[0] == tolower(edgeString[j]))
|
||||
{
|
||||
currentFoundIds.push_back(path.text.size() + j);
|
||||
currentRemainingQuery = currentRemainingQuery.substr(1);
|
||||
}
|
||||
}
|
||||
|
||||
Path currentPath;
|
||||
currentPath.node = currentEdge->target;
|
||||
currentPath.indices = currentFoundIds;
|
||||
currentPath.text = path.text + edgeString;
|
||||
|
||||
search(currentPath, currentRemainingQuery, results);
|
||||
}
|
||||
searchRecursive(currentPath, remainingQuery.substr(j), results);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::multiset<SearchResult> SearchIndex::createScoredResults(const std::vector<Path>& paths, size_t maxResultCount) const
|
||||
{
|
||||
// score and order initial paths
|
||||
std::multiset<std::pair<int, Path>, bool(*)(const std::pair<int, Path>&, const std::pair<int, Path>&)> scoredPaths(
|
||||
[](const std::pair<int, Path>& a, const std::pair<int, Path>& b)
|
||||
{
|
||||
return a.first > b.first;
|
||||
}
|
||||
);
|
||||
|
||||
for (const Path& path : paths)
|
||||
{
|
||||
scoredPaths.insert(std::make_pair(score(path.text, path.indices), path));
|
||||
}
|
||||
|
||||
// score paths and subpaths
|
||||
std::multiset<SearchResult> searchResults;
|
||||
for (const std::pair<int, Path>& p : scoredPaths)
|
||||
{
|
||||
std::vector<Path> currentPaths;
|
||||
currentPaths.push_back(p.second);
|
||||
|
||||
while (currentPaths.size())
|
||||
{
|
||||
std::vector<Path> nextPaths;
|
||||
|
||||
for (const Path& path : currentPaths)
|
||||
{
|
||||
if (path.node->elementIds.size())
|
||||
{
|
||||
SearchResult result;
|
||||
result.text = path.text;
|
||||
result.elementIds = path.node->elementIds;
|
||||
result.indices = path.indices;
|
||||
result.score = score(path.text, path.indices);
|
||||
searchResults.insert(result);
|
||||
|
||||
if (maxResultCount && searchResults.size() >= maxResultCount)
|
||||
{
|
||||
return searchResults;
|
||||
}
|
||||
}
|
||||
|
||||
for (const Edge* edge : path.node->edges)
|
||||
{
|
||||
Path nextPath;
|
||||
nextPath.indices = path.indices;
|
||||
nextPath.node = edge->target;
|
||||
nextPath.text = path.text + edge->s;
|
||||
nextPaths.push_back(nextPath);
|
||||
}
|
||||
}
|
||||
|
||||
currentPaths = nextPaths;
|
||||
}
|
||||
}
|
||||
|
||||
return searchResults;
|
||||
}
|
||||
|
||||
SearchResult SearchIndex::bestScoredResult(SearchResult result, std::map<std::string, SearchResult>* scoresCache) const
|
||||
{
|
||||
if (result.text.size() > 100)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
const std::vector<size_t>& indices = result.indices;
|
||||
|
||||
std::map<std::string, SearchResult>::const_iterator it = scoresCache->find(result.text.substr(0, indices.back() + 1));
|
||||
if (it != scoresCache->end())
|
||||
{
|
||||
result.score = it->second.score;
|
||||
result.indices = it->second.indices;
|
||||
}
|
||||
|
||||
int oldScore = result.score;
|
||||
|
||||
bool consecutive = (indices.size() == 1);
|
||||
for (size_t i = 0; i < indices.size() - 1; i++)
|
||||
{
|
||||
if (indices[i + 1] - indices[i] != 1)
|
||||
{
|
||||
consecutive = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!consecutive)
|
||||
{
|
||||
bestScoredResultRecursive(utility::toLowerCase(result.text), indices, indices.size() - 1, scoresCache, &result);
|
||||
}
|
||||
|
||||
if (result.score != oldScore || it == scoresCache->end())
|
||||
{
|
||||
scoresCache->emplace(result.text.substr(0, result.indices.back() + 1), result);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void SearchIndex::bestScoredResultRecursive(
|
||||
const std::string& lowerText, const std::vector<size_t>& indices, size_t indicesPos,
|
||||
std::map<std::string, SearchResult>* scoresCache, SearchResult* result) const
|
||||
{
|
||||
size_t oldTextPos = indices[indicesPos];
|
||||
size_t nextTextPos = (indicesPos + 1 == indices.size() ? result->text.size() : indices[indicesPos + 1]);
|
||||
for (size_t i = oldTextPos + 1; i < nextTextPos; i++)
|
||||
{
|
||||
if (lowerText[i] == lowerText[oldTextPos])
|
||||
{
|
||||
std::vector<size_t> newIndices = indices;
|
||||
newIndices[indicesPos] = i;
|
||||
|
||||
int newScore = score(result->text, newIndices);
|
||||
if (newScore > result->score)
|
||||
{
|
||||
result->score = newScore;
|
||||
result->indices = newIndices;
|
||||
}
|
||||
|
||||
bestScoredResultRecursive(lowerText, newIndices, indicesPos, scoresCache, result);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (indicesPos + 1 == indices.size())
|
||||
{
|
||||
std::map<std::string, SearchResult>::const_iterator it = scoresCache->find(result->text.substr(0, indices.back() + 1));
|
||||
if (it != scoresCache->end())
|
||||
{
|
||||
result->score = it->second.score;
|
||||
result->indices = it->second.indices;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (indicesPos > 0)
|
||||
{
|
||||
size_t newIndicesPos = indicesPos - 1;
|
||||
while (newIndicesPos > 0 && indices[newIndicesPos + 1] - indices[newIndicesPos] == 1)
|
||||
{
|
||||
newIndicesPos--;
|
||||
}
|
||||
|
||||
if (indices[newIndicesPos + 1] - indices[newIndicesPos] > 1)
|
||||
{
|
||||
bestScoredResultRecursive(lowerText, indices, newIndicesPos, scoresCache, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int SearchIndex::score(const std::string& text, const std::vector<size_t>& indices) const
|
||||
{
|
||||
const int unmatchedLetterBonus = -1;
|
||||
const int consecutiveLetterBonus = 5;
|
||||
const int camelCaseBonus = 4;
|
||||
const int noLetterBonus = 3;
|
||||
const int delayedStartBonus = -1;
|
||||
const int minDelayedStartBonus = -20;
|
||||
|
||||
int unmatchedLetterScore = 0;
|
||||
int consecutiveLetterScore = 0;
|
||||
int camelCaseScore = 0;
|
||||
int noLetterScore = 0;
|
||||
|
||||
static std::set<char> noLetters;
|
||||
if (!noLetters.size())
|
||||
{
|
||||
noLetters.insert(' ');
|
||||
noLetters.insert('.');
|
||||
noLetters.insert(',');
|
||||
noLetters.insert('_');
|
||||
noLetters.insert(':');
|
||||
noLetters.insert('<');
|
||||
noLetters.insert('>');
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < indices.size(); i++)
|
||||
{
|
||||
// unmatched and consecutive
|
||||
if (i > 0)
|
||||
{
|
||||
unmatchedLetterScore += (indices[i] - indices[i - 1] - 1) * unmatchedLetterBonus;
|
||||
consecutiveLetterScore += (indices[i] - indices[i - 1] == 1 ? consecutiveLetterBonus : 0);
|
||||
}
|
||||
|
||||
size_t index = indices[i];
|
||||
|
||||
// camel case
|
||||
if (isupper(text[index]))
|
||||
{
|
||||
bool prevIsLower = (index > 0 && islower(text[index - 1]));
|
||||
bool nextIsLower = (index + 1 == text.size() || islower(text[index + 1]));
|
||||
|
||||
if (prevIsLower || nextIsLower)
|
||||
{
|
||||
camelCaseScore += camelCaseBonus;
|
||||
}
|
||||
}
|
||||
|
||||
// after no letter
|
||||
bool prevIsNoLetter = (index > 0 && noLetters.find(text[index - 1]) != noLetters.end());
|
||||
if (prevIsNoLetter)
|
||||
{
|
||||
noLetterScore += noLetterBonus;
|
||||
}
|
||||
}
|
||||
|
||||
int leadingStartScore = std::max(int(indices[0]) * delayedStartBonus, minDelayedStartBonus);
|
||||
|
||||
int score =
|
||||
unmatchedLetterScore +
|
||||
consecutiveLetterScore +
|
||||
camelCaseScore +
|
||||
noLetterScore +
|
||||
leadingStartScore;
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,11 @@ struct SearchResult
|
||||
std::set<Id> elementIds;
|
||||
std::vector<size_t> indices;
|
||||
int score;
|
||||
|
||||
bool operator<(const SearchResult& other) const
|
||||
{
|
||||
return score > other.score;
|
||||
}
|
||||
};
|
||||
|
||||
class SearchIndex
|
||||
@@ -57,7 +62,14 @@ private:
|
||||
};
|
||||
|
||||
void populateEdgeGate(Edge* e);
|
||||
void search(const Path& path, const std::string& remainingQuery, std::vector<SearchIndex::Path>* results) const;
|
||||
void searchRecursive(const Path& path, const std::string& remainingQuery, std::vector<SearchIndex::Path>* results) const;
|
||||
|
||||
std::multiset<SearchResult> createScoredResults(const std::vector<Path>& paths, size_t maxResultCount) const;
|
||||
SearchResult bestScoredResult(SearchResult result, std::map<std::string, SearchResult>* scoresCache) const;
|
||||
void bestScoredResultRecursive(
|
||||
const std::string& lowerText, const std::vector<size_t>& indices, size_t indicesPos,
|
||||
std::map<std::string, SearchResult>* scoresCache, SearchResult* result) const;
|
||||
int score(const std::string& text, const std::vector<size_t>& indices) const;
|
||||
|
||||
std::vector<std::shared_ptr<Node>> m_nodes;
|
||||
std::vector<std::shared_ptr<Edge>> m_edges;
|
||||
|
||||
Reference in New Issue
Block a user