diff --git a/CMakeLists.txt b/CMakeLists.txt index 6287dbb4..8d9787e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,7 @@ set(PROJECT_NAME_LOWER_CASE sourcetrail) # speed up recompiling on unix with ccache find_program(CCACHE_PROGRAM ccache) if(CCACHE_PROGRAM) + MESSAGE(STATUS "ccache found") # Support Unix Makefiles and Ninja set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") endif() @@ -759,7 +760,7 @@ if (WIN32) COMMAND cd $(ProjectDir)../../bin/test/\n$(OutDir)$(TargetName)$(TargetExt) COMMENT "Running unittest code" ) - + set_target_properties(${TEST_PROJECT_NAME} PROPERTIES COMPILE_FLAGS "/bigobj") elseif (UNIX) diff --git a/src/lib/data/NodeType.cpp b/src/lib/data/NodeType.cpp index 4ece709c..86f6a1ab 100644 --- a/src/lib/data/NodeType.cpp +++ b/src/lib/data/NodeType.cpp @@ -309,12 +309,14 @@ std::string NodeType::getReadableTypeString() const std::wstring NodeType::getUnderscoredTypeWString() const { - return utility::decodeFromUtf8(getUnderscoredTypeString()); + std::string str = getUnderscoredTypeString(); + return std::wstring(str.begin(), str.end()); } std::wstring NodeType::getReadableTypeWString() const { - return utility::decodeFromUtf8(getReadableTypeString()); + std::string str = getReadableTypeString(); + return std::wstring(str.begin(), str.end()); } int utility::nodeTypeToInt(NodeType::Type type) @@ -420,7 +422,8 @@ std::string utility::getReadableTypeString(NodeType::Type type) std::wstring utility::getReadableTypeWString(NodeType::Type type) { - return utility::decodeFromUtf8(getReadableTypeString(type)); + std::string str = getReadableTypeString(type); + return std::wstring(str.begin(), str.end()); } NodeType::Type utility::getTypeForReadableTypeString(const std::wstring str) diff --git a/src/lib/data/name/NameElement.cpp b/src/lib/data/name/NameElement.cpp index 288bee99..c517bcef 100644 --- a/src/lib/data/name/NameElement.cpp +++ b/src/lib/data/name/NameElement.cpp @@ -3,33 +3,13 @@ #include "utility/logging/logging.h" #include "utility/utilityString.h" -std::wstring NameElement::Signature::serialize(Signature signature) -{ - return signature.m_prefix + L"\tp" + signature.m_postfix; -} - -NameElement::Signature NameElement::Signature::deserialize(const std::wstring& serialized) -{ - if (serialized == L"\tp") - { - return Signature(); - } - - std::vector serializedElements = utility::splitToVector(serialized, L"\tp"); - if (serializedElements.size() != 2) - { - LOG_ERROR(L"unable to deserialize name signature: " + serialized); // todo: obfuscate serialized! - } - return Signature(serializedElements[0], serializedElements[1]); -} - NameElement::Signature::Signature() : m_prefix(L"") , m_postfix(L"") { } -NameElement::Signature::Signature(std::wstring prefix, std::wstring postfix) +NameElement::Signature::Signature(const std::wstring& prefix, const std::wstring& postfix) : m_prefix(prefix) , m_postfix(postfix) { @@ -91,7 +71,7 @@ NameElement::~NameElement() { } -std::wstring NameElement::getName() const +const std::wstring& NameElement::getName() const { return m_name; } @@ -111,7 +91,7 @@ bool NameElement::hasSignature() const return m_signature.isValid(); } -NameElement::Signature NameElement::getSignature() +const NameElement::Signature& NameElement::getSignature() { return m_signature; } diff --git a/src/lib/data/name/NameElement.h b/src/lib/data/name/NameElement.h index d7841672..b1955ac6 100644 --- a/src/lib/data/name/NameElement.h +++ b/src/lib/data/name/NameElement.h @@ -13,11 +13,9 @@ public: class Signature { public: - static std::wstring serialize(Signature signature); - static Signature deserialize(const std::wstring& serialized); - Signature(); - Signature(std::wstring prefix, std::wstring postfix); + Signature(const std::wstring& prefix, const std::wstring& postfix); + std::wstring qualifyName(const std::wstring& name) const; bool isValid() const; @@ -26,23 +24,23 @@ public: std::wstring getParameterString() const; private: - std::wstring m_prefix; - std::wstring m_postfix; + const std::wstring m_prefix; + const std::wstring m_postfix; }; NameElement(const std::wstring& name); NameElement(const std::wstring& name, const Signature& signature); ~NameElement(); - std::wstring getName() const; + const std::wstring& getName() const; std::wstring getNameWithSignature() const; std::wstring getNameWithSignatureParameters() const; bool hasSignature() const; - Signature getSignature(); + const Signature& getSignature(); private: - std::wstring m_name; - Signature m_signature; + const std::wstring m_name; + const Signature m_signature; }; #endif // NAME_ELEMENT_H diff --git a/src/lib/data/name/NameHierarchy.cpp b/src/lib/data/name/NameHierarchy.cpp index 719f575d..141a8858 100644 --- a/src/lib/data/name/NameHierarchy.cpp +++ b/src/lib/data/name/NameHierarchy.cpp @@ -1,45 +1,87 @@ #include "data/name/NameHierarchy.h" +#include + #include "utility/logging/logging.h" #include "utility/utilityString.h" +namespace +{ + const std::wstring META_DELIMITER = L"\tm"; + const std::wstring NAME_DELIMITER = L"\tn"; + const std::wstring PART_DELIMITER = L"\ts"; + const std::wstring SIGNATURE_DELIMITER = L"\tp"; +} + std::wstring NameHierarchy::serialize(const NameHierarchy& nameHierarchy) { - std::wstring serializedName = nameDelimiterTypeToString(nameHierarchy.getDelimiter()) + L"\tm"; + std::wstringstream ss; + ss << nameDelimiterTypeToString(nameHierarchy.getDelimiter()); + ss << META_DELIMITER; for (size_t i = 0; i < nameHierarchy.size(); i++) { if (i > 0) { - serializedName += L"\tn"; + ss << NAME_DELIMITER; } - serializedName += nameHierarchy[i]->getName() + L"\ts"; - serializedName += NameElement::Signature::serialize(nameHierarchy[i]->getSignature()); + + ss << nameHierarchy[i]->getName() << PART_DELIMITER; + ss << nameHierarchy[i]->getSignature().getPrefix(); + ss << SIGNATURE_DELIMITER; + ss << nameHierarchy[i]->getSignature().getPostfix(); } - return serializedName; + return ss.str(); } NameHierarchy NameHierarchy::deserialize(const std::wstring& serializedName) { - std::vector serializedNameAndMetaElements = utility::splitToVector(serializedName, L"\tm"); - if (serializedNameAndMetaElements.size() != 2) + size_t mpos = serializedName.find(META_DELIMITER); + if (mpos == std::wstring::npos) { LOG_ERROR(L"unable to deserialize name hierarchy: " + serializedName); // todo: obfuscate serializedName! return NameHierarchy(NAME_DELIMITER_UNKNOWN); } - const NameDelimiterType delimiter = stringToNameDelimiterType(serializedNameAndMetaElements[0]); - NameHierarchy nameHierarchy(delimiter); + NameHierarchy nameHierarchy(stringToNameDelimiterType(serializedName.substr(0, mpos))); - std::vector serializedNameElements = utility::splitToVector(serializedNameAndMetaElements[1], L"\tn"); - for (size_t i = 0; i < serializedNameElements.size(); i++) + size_t npos = mpos + META_DELIMITER.size(); + while (npos != std::wstring::npos && npos < serializedName.size()) { - std::vector nameParts = utility::splitToVector(serializedNameElements[i], L"\ts"); - if (nameParts.size() != 2) + // name + size_t spos = serializedName.find(PART_DELIMITER, npos); + if (spos == std::wstring::npos) { LOG_ERROR(L"unable to deserialize name hierarchy: " + serializedName); // todo: obfuscate serializedName! - return NameHierarchy(delimiter); + return NameHierarchy(NAME_DELIMITER_UNKNOWN); } - nameHierarchy.push(std::make_shared(nameParts[0], NameElement::Signature::deserialize(nameParts[1]))); + + std::wstring name = serializedName.substr(npos, spos - npos); + spos += PART_DELIMITER.size(); + + // signature + size_t ppos = serializedName.find(SIGNATURE_DELIMITER, spos); + if (ppos == std::wstring::npos) + { + LOG_ERROR(L"unable to deserialize name hierarchy: " + serializedName); // todo: obfuscate serializedName! + return NameHierarchy(NAME_DELIMITER_UNKNOWN); + } + + std::wstring prefix = serializedName.substr(spos, ppos - spos); + ppos += SIGNATURE_DELIMITER.size(); + + std::wstring postfix; + npos = serializedName.find(NAME_DELIMITER, ppos); + if (npos == std::wstring::npos) + { + postfix = serializedName.substr(ppos, std::wstring::npos); + } + else + { + postfix = serializedName.substr(ppos, npos - ppos); + npos += NAME_DELIMITER.size(); + } + + nameHierarchy.push(std::make_shared(name, NameElement::Signature(prefix, postfix))); } return nameHierarchy; @@ -148,16 +190,16 @@ size_t NameHierarchy::size() const std::wstring NameHierarchy::getQualifiedName() const { - std::wstring name; + std::wstringstream ss; for (size_t i = 0; i < m_elements.size(); i++) { if (i > 0) { - name += nameDelimiterTypeToString(m_delimiter); + ss << nameDelimiterTypeToString(m_delimiter); } - name += m_elements[i]->getName(); + ss << m_elements[i]->getName(); } - return name; + return ss.str(); } std::wstring NameHierarchy::getQualifiedNameWithSignature() const diff --git a/src/lib/data/search/SearchIndex.cpp b/src/lib/data/search/SearchIndex.cpp index c79c0af7..a3ad56ea 100644 --- a/src/lib/data/search/SearchIndex.cpp +++ b/src/lib/data/search/SearchIndex.cpp @@ -112,12 +112,29 @@ std::vector SearchIndex::search( // create scored search results std::multiset searchResults = createScoredResults(paths, acceptedNodeTypes, maxResultCount * 3); + // find maximum length for best scores + std::multiset resultLengths; + for (const SearchResult& result : searchResults) + { + resultLengths.insert(result.text.size()); + } + size_t maxResultLength = 0; + if (resultLengths.size() > 1000) + { + auto it = resultLengths.begin(); + std::advance(it, 1000); + maxResultLength = *it; + } + // find best scores std::map scoresCache; std::multiset bestResults; for (const SearchResult& result : searchResults) { - bestResults.insert(bestScoredResult(result, &scoresCache, maxBestScoredResultsLength)); + if (!maxResultLength || result.text.size() <= maxResultLength) + { + bestResults.insert(bestScoredResult(result, &scoresCache, maxBestScoredResultsLength)); + } } // narrow down to max result count diff --git a/src/lib/data/storage/PersistentStorage.cpp b/src/lib/data/storage/PersistentStorage.cpp index 7e9b4361..322267a4 100644 --- a/src/lib/data/storage/PersistentStorage.cpp +++ b/src/lib/data/storage/PersistentStorage.cpp @@ -649,15 +649,16 @@ std::vector PersistentStorage::getAutocompletionMatches(const std:: TRACE(); // search in indices - const size_t maxResultsCount = 100; + const size_t maxResultsCount = std::pow(3, query.size() + 3); const size_t maxBestScoredResultsLength = 100; + const size_t maxMatchesReturned = 1000; // create SearchMatches std::vector matches; if (!acceptedNodeTypes.getWithMatchingRemoved([](const NodeType& type) { return type.isFile(); }).isEmpty()) { - utility::append(matches, getAutocompletionSymbolMatches(query, acceptedNodeTypes, maxResultsCount, maxBestScoredResultsLength)); + matches = getAutocompletionSymbolMatches(query, acceptedNodeTypes, maxResultsCount, maxBestScoredResultsLength); } if (acceptedNodeTypes.containsMatching([](const NodeType& type) { return type.isFile(); })) @@ -678,43 +679,55 @@ std::vector PersistentStorage::getAutocompletionMatches(const std:: SearchIndex::rescoreText(match.name, match.text, match.indices, match.score, maxBestScoredResultsLength); match.score = newResult.score; - match.indices = newResult.indices; + match.indices = std::move(newResult.indices); } matchesMap.emplace(match.name, match); } // Score child symbol matches with same score as parent lower - SearchMatch lastMatch; + const SearchMatch* lastMatch = nullptr; std::set matchesSet; - for (const auto& p : matchesMap) + for (auto& p : matchesMap) { - SearchMatch match = p.second; - if (!lastMatch.name.size() || !utility::isPrefix(lastMatch.name, match.name)) + SearchMatch& match = p.second; + if (lastMatch == nullptr || !utility::isPrefix(lastMatch->name, match.name)) { - lastMatch = match; + lastMatch = &match; } - else if (lastMatch.score == match.score) + else if (lastMatch->score == match.score) { - if (utility::isPrefix(nameDelimiterTypeToString(match.tokenName.getDelimiter()), match.name.substr(lastMatch.name.size()))) + size_t lastSize = lastMatch->name.size(); + if (match.name.find(nameDelimiterTypeToString(match.tokenName.getDelimiter()), lastSize) == lastSize) { match.score -= 10; } else { - lastMatch = match; + lastMatch = &match; } } matchesSet.insert(match); } - // for (auto a : matchesSet) + if (matchesSet.size() > maxMatchesReturned) + { + auto it = matchesSet.begin(); + std::advance(it, maxMatchesReturned); + matches = std::vector(matchesSet.begin(), it); + } + else + { + matches = utility::toVector(matchesSet); + } + + // for (auto a : matches) // { // std::wcout << a.score << " " << a.name << std::endl; // } - return utility::toVector(matchesSet); + return matches; } std::vector PersistentStorage::getAutocompletionSymbolMatches( @@ -726,7 +739,6 @@ std::vector PersistentStorage::getAutocompletionSymbolMatches( // fetch StorageNodes for node ids std::map storageNodeMap; - std::map storageSymbolMap; { std::vector elementIds; @@ -739,11 +751,6 @@ std::vector PersistentStorage::getAutocompletionSymbolMatches( { storageNodeMap[node.id] = node; } - - for (const StorageSymbol& symbol : m_sqliteIndexStorage.getAllByIds(elementIds)) - { - storageSymbolMap[symbol.id] = symbol; - } } // create SearchMatches @@ -778,8 +785,11 @@ std::vector PersistentStorage::getAutocompletionSymbolMatches( if (name.getQualifiedName() == match.name) { const size_t idx = m_hierarchyCache.getIndexOfLastVisibleParentNode(firstNode->id); - match.text = name.getRange(idx, name.size()).getQualifiedName(); - match.subtext = name.getRange(0, idx).getQualifiedName(); + if (idx != 0) + { + match.text = name.getRange(idx, name.size()).getQualifiedName(); + match.subtext = name.getRange(0, idx).getQualifiedName(); + } } match.tokenName = name; @@ -789,7 +799,7 @@ std::vector PersistentStorage::getAutocompletionSymbolMatches( match.typeName = match.nodeType.getReadableTypeWString(); match.searchType = SearchMatch::SEARCH_TOKEN; - if (storageSymbolMap.find(firstNode->id) == storageSymbolMap.end()) + if (m_symbolDefinitionKinds.find(firstNode->id) == m_symbolDefinitionKinds.end()) { match.typeName = L"non-indexed " + match.typeName; }