ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: Improved search result scoring

Browse Source 
* Increase result count relative to query length
* Only run best scoring on shortest 1000 results
* Made NameHierarchy deserializing faster
This commit is contained in:
Eberhard Graether
2018-07-17 14:41:17 +02:00
parent 094bfae8f1
commit 6adcb2b31e
7 changed files with 130 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+2 -1
View File
@@ -27,6 +27,7 @@ set(PROJECT_NAME_LOWER_CASE sourcetrail)
# speed up recompiling on unix with ccache
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
MESSAGE(STATUS "ccache found")
# Support Unix Makefiles and Ninja
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
endif()
@@ -759,7 +760,7 @@ if (WIN32)
COMMAND cd $(ProjectDir)../../bin/test/\n$(OutDir)$(TargetName)$(TargetExt)
COMMENT "Running unittest code"
)
set_target_properties(${TEST_PROJECT_NAME} PROPERTIES COMPILE_FLAGS "/bigobj")
elseif (UNIX)
src/lib/data/NodeType.cpp
+6 -3
View File
@@ -309,12 +309,14 @@ std::string NodeType::getReadableTypeString() const
std::wstring NodeType::getUnderscoredTypeWString() const
{
return utility::decodeFromUtf8(getUnderscoredTypeString());
std::string str = getUnderscoredTypeString();
return std::wstring(str.begin(), str.end());
}
std::wstring NodeType::getReadableTypeWString() const
{
return utility::decodeFromUtf8(getReadableTypeString());
std::string str = getReadableTypeString();
return std::wstring(str.begin(), str.end());
}
int utility::nodeTypeToInt(NodeType::Type type)
@@ -420,7 +422,8 @@ std::string utility::getReadableTypeString(NodeType::Type type)
std::wstring utility::getReadableTypeWString(NodeType::Type type)
{
return utility::decodeFromUtf8(getReadableTypeString(type));
std::string str = getReadableTypeString(type);
return std::wstring(str.begin(), str.end());
}
NodeType::Type utility::getTypeForReadableTypeString(const std::wstring str)
src/lib/data/name/NameElement.cpp
+3 -23
View File
@@ -3,33 +3,13 @@
#include "utility/logging/logging.h"
#include "utility/utilityString.h"
std::wstring NameElement::Signature::serialize(Signature signature)
{
return signature.m_prefix + L"\tp" + signature.m_postfix;
}
NameElement::Signature NameElement::Signature::deserialize(const std::wstring& serialized)
{
if (serialized == L"\tp")
{
return Signature();
}
std::vector<std::wstring> serializedElements = utility::splitToVector(serialized, L"\tp");
if (serializedElements.size() != 2)
{
LOG_ERROR(L"unable to deserialize name signature: " + serialized); // todo: obfuscate serialized!
}
return Signature(serializedElements[0], serializedElements[1]);
}
NameElement::Signature::Signature()
: m_prefix(L"")
, m_postfix(L"")
{
}
NameElement::Signature::Signature(std::wstring prefix, std::wstring postfix)
NameElement::Signature::Signature(const std::wstring& prefix, const std::wstring& postfix)
: m_prefix(prefix)
, m_postfix(postfix)
{
@@ -91,7 +71,7 @@ NameElement::~NameElement()
{
}
std::wstring NameElement::getName() const
const std::wstring& NameElement::getName() const
{
return m_name;
}
@@ -111,7 +91,7 @@ bool NameElement::hasSignature() const
return m_signature.isValid();
}
NameElement::Signature NameElement::getSignature()
const NameElement::Signature& NameElement::getSignature()
{
return m_signature;
}
src/lib/data/name/NameElement.h
+8 -10
View File
@@ -13,11 +13,9 @@ public:
class Signature
{
public:
static std::wstring serialize(Signature signature);
static Signature deserialize(const std::wstring& serialized);
Signature();
Signature(std::wstring prefix, std::wstring postfix);
Signature(const std::wstring& prefix, const std::wstring& postfix);
std::wstring qualifyName(const std::wstring& name) const;
bool isValid() const;
@@ -26,23 +24,23 @@ public:
std::wstring getParameterString() const;
private:
std::wstring m_prefix;
std::wstring m_postfix;
const std::wstring m_prefix;
const std::wstring m_postfix;
};
NameElement(const std::wstring& name);
NameElement(const std::wstring& name, const Signature& signature);
~NameElement();
std::wstring getName() const;
const std::wstring& getName() const;
std::wstring getNameWithSignature() const;
std::wstring getNameWithSignatureParameters() const;
bool hasSignature() const;
Signature getSignature();
const Signature& getSignature();
private:
std::wstring m_name;
Signature m_signature;
const std::wstring m_name;
const Signature m_signature;
};
#endif // NAME_ELEMENT_H
src/lib/data/name/NameHierarchy.cpp
+61 -19
View File
@@ -1,45 +1,87 @@
#include "data/name/NameHierarchy.h"
#include <sstream>
#include "utility/logging/logging.h"
#include "utility/utilityString.h"
namespace
{
const std::wstring META_DELIMITER = L"\tm";
const std::wstring NAME_DELIMITER = L"\tn";
const std::wstring PART_DELIMITER = L"\ts";
const std::wstring SIGNATURE_DELIMITER = L"\tp";
}
std::wstring NameHierarchy::serialize(const NameHierarchy& nameHierarchy)
{
std::wstring serializedName = nameDelimiterTypeToString(nameHierarchy.getDelimiter()) + L"\tm";
std::wstringstream ss;
ss << nameDelimiterTypeToString(nameHierarchy.getDelimiter());
ss << META_DELIMITER;
for (size_t i = 0; i < nameHierarchy.size(); i++)
{
if (i > 0)
{
serializedName += L"\tn";
ss << NAME_DELIMITER;
}
serializedName += nameHierarchy[i]->getName() + L"\ts";
serializedName += NameElement::Signature::serialize(nameHierarchy[i]->getSignature());
ss << nameHierarchy[i]->getName() << PART_DELIMITER;
ss << nameHierarchy[i]->getSignature().getPrefix();
ss << SIGNATURE_DELIMITER;
ss << nameHierarchy[i]->getSignature().getPostfix();
}
return serializedName;
return ss.str();
}
NameHierarchy NameHierarchy::deserialize(const std::wstring& serializedName)
{
std::vector<std::wstring> serializedNameAndMetaElements = utility::splitToVector(serializedName, L"\tm");
if (serializedNameAndMetaElements.size() != 2)
size_t mpos = serializedName.find(META_DELIMITER);
if (mpos == std::wstring::npos)
{
LOG_ERROR(L"unable to deserialize name hierarchy: " + serializedName); // todo: obfuscate serializedName!
return NameHierarchy(NAME_DELIMITER_UNKNOWN);
}
const NameDelimiterType delimiter = stringToNameDelimiterType(serializedNameAndMetaElements[0]);
NameHierarchy nameHierarchy(delimiter);
NameHierarchy nameHierarchy(stringToNameDelimiterType(serializedName.substr(0, mpos)));
std::vector<std::wstring> serializedNameElements = utility::splitToVector(serializedNameAndMetaElements[1], L"\tn");
for (size_t i = 0; i < serializedNameElements.size(); i++)
size_t npos = mpos + META_DELIMITER.size();
while (npos != std::wstring::npos && npos < serializedName.size())
{
std::vector<std::wstring> nameParts = utility::splitToVector(serializedNameElements[i], L"\ts");
if (nameParts.size() != 2)
// name
size_t spos = serializedName.find(PART_DELIMITER, npos);
if (spos == std::wstring::npos)
{
LOG_ERROR(L"unable to deserialize name hierarchy: " + serializedName); // todo: obfuscate serializedName!
return NameHierarchy(delimiter);
return NameHierarchy(NAME_DELIMITER_UNKNOWN);
}
nameHierarchy.push(std::make_shared<NameElement>(nameParts[0], NameElement::Signature::deserialize(nameParts[1])));
std::wstring name = serializedName.substr(npos, spos - npos);
spos += PART_DELIMITER.size();
// signature
size_t ppos = serializedName.find(SIGNATURE_DELIMITER, spos);
if (ppos == std::wstring::npos)
{
LOG_ERROR(L"unable to deserialize name hierarchy: " + serializedName); // todo: obfuscate serializedName!
return NameHierarchy(NAME_DELIMITER_UNKNOWN);
}
std::wstring prefix = serializedName.substr(spos, ppos - spos);
ppos += SIGNATURE_DELIMITER.size();
std::wstring postfix;
npos = serializedName.find(NAME_DELIMITER, ppos);
if (npos == std::wstring::npos)
{
postfix = serializedName.substr(ppos, std::wstring::npos);
}
else
{
postfix = serializedName.substr(ppos, npos - ppos);
npos += NAME_DELIMITER.size();
}
nameHierarchy.push(std::make_shared<NameElement>(name, NameElement::Signature(prefix, postfix)));
}
return nameHierarchy;
@@ -148,16 +190,16 @@ size_t NameHierarchy::size() const
std::wstring NameHierarchy::getQualifiedName() const
{
std::wstring name;
std::wstringstream ss;
for (size_t i = 0; i < m_elements.size(); i++)
{
if (i > 0)
{
name += nameDelimiterTypeToString(m_delimiter);
ss << nameDelimiterTypeToString(m_delimiter);
}
name += m_elements[i]->getName();
ss << m_elements[i]->getName();
}
return name;
return ss.str();
}
std::wstring NameHierarchy::getQualifiedNameWithSignature() const
src/lib/data/search/SearchIndex.cpp
+18 -1
View File
@@ -112,12 +112,29 @@ std::vector<SearchResult> SearchIndex::search(
// create scored search results
std::multiset<SearchResult> searchResults = createScoredResults(paths, acceptedNodeTypes, maxResultCount * 3);
// find maximum length for best scores
std::multiset<size_t> resultLengths;
for (const SearchResult& result : searchResults)
{
resultLengths.insert(result.text.size());
}
size_t maxResultLength = 0;
if (resultLengths.size() > 1000)
{
auto it = resultLengths.begin();
std::advance(it, 1000);
maxResultLength = *it;
}
// find best scores
std::map<std::wstring, SearchResult> scoresCache;
std::multiset<SearchResult> bestResults;
for (const SearchResult& result : searchResults)
{
bestResults.insert(bestScoredResult(result, &scoresCache, maxBestScoredResultsLength));
if (!maxResultLength || result.text.size() <= maxResultLength)
{
bestResults.insert(bestScoredResult(result, &scoresCache, maxBestScoredResultsLength));
}
}
// narrow down to max result count
src/lib/data/storage/PersistentStorage.cpp
+32 -22
View File
@@ -649,15 +649,16 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionMatches(const std::
TRACE();
// search in indices
const size_t maxResultsCount = 100;
const size_t maxResultsCount = std::pow(3, query.size() + 3);
const size_t maxBestScoredResultsLength = 100;
const size_t maxMatchesReturned = 1000;
// create SearchMatches
std::vector<SearchMatch> matches;
if (!acceptedNodeTypes.getWithMatchingRemoved([](const NodeType& type) { return type.isFile(); }).isEmpty())
{
utility::append(matches, getAutocompletionSymbolMatches(query, acceptedNodeTypes, maxResultsCount, maxBestScoredResultsLength));
matches = getAutocompletionSymbolMatches(query, acceptedNodeTypes, maxResultsCount, maxBestScoredResultsLength);
}
if (acceptedNodeTypes.containsMatching([](const NodeType& type) { return type.isFile(); }))
@@ -678,43 +679,55 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionMatches(const std::
SearchIndex::rescoreText(match.name, match.text, match.indices, match.score, maxBestScoredResultsLength);
match.score = newResult.score;
match.indices = newResult.indices;
match.indices = std::move(newResult.indices);
}
matchesMap.emplace(match.name, match);
}
// Score child symbol matches with same score as parent lower
SearchMatch lastMatch;
const SearchMatch* lastMatch = nullptr;
std::set<SearchMatch> matchesSet;
for (const auto& p : matchesMap)
for (auto& p : matchesMap)
{
SearchMatch match = p.second;
if (!lastMatch.name.size() || !utility::isPrefix(lastMatch.name, match.name))
SearchMatch& match = p.second;
if (lastMatch == nullptr || !utility::isPrefix(lastMatch->name, match.name))
{
lastMatch = match;
lastMatch = &match;
}
else if (lastMatch.score == match.score)
else if (lastMatch->score == match.score)
{
if (utility::isPrefix(nameDelimiterTypeToString(match.tokenName.getDelimiter()), match.name.substr(lastMatch.name.size())))
size_t lastSize = lastMatch->name.size();
if (match.name.find(nameDelimiterTypeToString(match.tokenName.getDelimiter()), lastSize) == lastSize)
{
match.score -= 10;
}
else
{
lastMatch = match;
lastMatch = &match;
}
}
matchesSet.insert(match);
}
// for (auto a : matchesSet)
if (matchesSet.size() > maxMatchesReturned)
{
auto it = matchesSet.begin();
std::advance(it, maxMatchesReturned);
matches = std::vector<SearchMatch>(matchesSet.begin(), it);
}
else
{
matches = utility::toVector(matchesSet);
}
// for (auto a : matches)
// {
// std::wcout << a.score << " " << a.name << std::endl;
// }
return utility::toVector(matchesSet);
return matches;
}
std::vector<SearchMatch> PersistentStorage::getAutocompletionSymbolMatches(
@@ -726,7 +739,6 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionSymbolMatches(
// fetch StorageNodes for node ids
std::map<Id, StorageNode> storageNodeMap;
std::map<Id, StorageSymbol> storageSymbolMap;
{
std::vector<Id> elementIds;
@@ -739,11 +751,6 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionSymbolMatches(
{
storageNodeMap[node.id] = node;
}
for (const StorageSymbol& symbol : m_sqliteIndexStorage.getAllByIds<StorageSymbol>(elementIds))
{
storageSymbolMap[symbol.id] = symbol;
}
}
// create SearchMatches
@@ -778,8 +785,11 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionSymbolMatches(
if (name.getQualifiedName() == match.name)
{
const size_t idx = m_hierarchyCache.getIndexOfLastVisibleParentNode(firstNode->id);
match.text = name.getRange(idx, name.size()).getQualifiedName();
match.subtext = name.getRange(0, idx).getQualifiedName();
if (idx != 0)
{
match.text = name.getRange(idx, name.size()).getQualifiedName();
match.subtext = name.getRange(0, idx).getQualifiedName();
}
}
match.tokenName = name;
@@ -789,7 +799,7 @@ std::vector<SearchMatch> PersistentStorage::getAutocompletionSymbolMatches(
match.typeName = match.nodeType.getReadableTypeWString();
match.searchType = SearchMatch::SEARCH_TOKEN;
if (storageSymbolMap.find(firstNode->id) == storageSymbolMap.end())
if (m_symbolDefinitionKinds.find(firstNode->id) == m_symbolDefinitionKinds.end())
{
match.typeName = L"non-indexed " + match.typeName;
}