From 2c84bf20ed7fbf7ba0c4dcf4b43b8797ca024d19 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Tue, 28 Oct 2014 22:06:22 +0100 Subject: [PATCH] data: split off inner classes of SearchIndex --- bin/test/data/log/test_log.txt | 6 +- src/app/qt/element/QtSearchBar.cpp | 2 +- src/app/qt/element/QtSearchBar.h | 4 +- src/app/qt/element/QtSmartSearchBox.cpp | 4 +- src/app/qt/element/QtSmartSearchBox.h | 7 +- src/app/qt/view/QtSearchView.cpp | 4 +- src/app/qt/view/QtSearchView.h | 6 +- src/lib/CMakeLists.txt | 11 +- .../component/controller/SearchController.cpp | 9 +- src/lib/component/view/SearchView.h | 4 +- src/lib/data/SearchIndex.cpp | 465 ------------------ src/lib/data/SearchIndex.h | 120 ----- src/lib/data/Storage.cpp | 30 +- src/lib/data/Storage.h | 6 +- src/lib/data/access/GraphAccess.h | 4 +- src/lib/data/access/GraphAccessProxy.cpp | 4 +- src/lib/data/access/GraphAccessProxy.h | 2 +- src/lib/data/graph/StorageGraph.cpp | 12 +- src/lib/data/graph/StorageGraph.h | 10 +- .../token_component/TokenComponentName.cpp | 8 +- .../token_component/TokenComponentName.h | 12 +- src/lib/data/search/SearchIndex.cpp | 84 ++++ src/lib/data/search/SearchIndex.h | 39 ++ src/lib/data/search/SearchMatch.cpp | 55 +++ src/lib/data/search/SearchMatch.h | 24 + src/lib/data/search/SearchNode.cpp | 309 ++++++++++++ src/lib/data/search/SearchNode.h | 69 +++ src/lib/data/search/SearchResult.cpp | 24 + src/lib/data/search/SearchResult.h | 23 + src/test/GraphFilterConductorTestSuite.h | 2 +- src/test/SearchIndexTestSuite.h | 28 +- src/test/StorageGraphTestSuite.h | 6 +- 32 files changed, 723 insertions(+), 670 deletions(-) delete mode 100644 src/lib/data/SearchIndex.cpp delete mode 100644 src/lib/data/SearchIndex.h create mode 100644 src/lib/data/search/SearchIndex.cpp create mode 100644 src/lib/data/search/SearchIndex.h create mode 100644 src/lib/data/search/SearchMatch.cpp create mode 100644 src/lib/data/search/SearchMatch.h create mode 100644 src/lib/data/search/SearchNode.cpp create mode 100644 src/lib/data/search/SearchNode.h create mode 100644 src/lib/data/search/SearchResult.cpp create mode 100644 src/lib/data/search/SearchResult.h diff --git a/bin/test/data/log/test_log.txt b/bin/test/data/log/test_log.txt index c6e80470..3e822d83 100644 --- a/bin/test/data/log/test_log.txt +++ b/bin/test/data/log/test_log.txt @@ -34,17 +34,17 @@ Storage.cpp INFO: function: main Storage.cpp INFO: type usage: main -> B Storage.cpp INFO: call: main -> B::B Storage.cpp INFO: call: main -> A::getCount -SearchIndex.cpp INFO: +SearchMatch.cpp INFO: 1 matches for "main": 474 main ^^^^ -SearchIndex.cpp INFO: +SearchMatch.cpp INFO: 1 matches for "main": 474 main ^^^^ -SearchIndex.cpp INFO: +SearchMatch.cpp INFO: 1 matches for "A::A": 237 A::A ^^^^ diff --git a/src/app/qt/element/QtSearchBar.cpp b/src/app/qt/element/QtSearchBar.cpp index 02997a3c..cf7f8b5a 100644 --- a/src/app/qt/element/QtSearchBar.cpp +++ b/src/app/qt/element/QtSearchBar.cpp @@ -49,7 +49,7 @@ void QtSearchBar::setFocus() m_searchBox->setFocus(); } -void QtSearchBar::setAutocompletionList(const std::vector& autocompletionList) +void QtSearchBar::setAutocompletionList(const std::vector& autocompletionList) { m_searchBox->setAutocompletionList(autocompletionList); } diff --git a/src/app/qt/element/QtSearchBar.h b/src/app/qt/element/QtSearchBar.h index 50f775c1..393da086 100644 --- a/src/app/qt/element/QtSearchBar.h +++ b/src/app/qt/element/QtSearchBar.h @@ -6,7 +6,7 @@ #include #include -#include "data/SearchIndex.h" +#include "data/search/SearchMatch.h" class QPushButton; class QtSmartSearchBox; @@ -22,7 +22,7 @@ public: void setText(const std::string& text); void setFocus(); - void setAutocompletionList(const std::vector& autocompletionList); + void setAutocompletionList(const std::vector& autocompletionList); QAbstractItemView* getCompleterPopup(); diff --git a/src/app/qt/element/QtSmartSearchBox.cpp b/src/app/qt/element/QtSmartSearchBox.cpp index 8e6c46e9..6bc4d469 100644 --- a/src/app/qt/element/QtSmartSearchBox.cpp +++ b/src/app/qt/element/QtSmartSearchBox.cpp @@ -52,7 +52,7 @@ QtSmartSearchBox::~QtSmartSearchBox() { } -void QtSmartSearchBox::setAutocompletionList(const std::vector& autocompletionList) +void QtSmartSearchBox::setAutocompletionList(const std::vector& autocompletionList) { m_matches = autocompletionList; @@ -63,7 +63,7 @@ void QtSmartSearchBox::setAutocompletionList(const std::vector #include #include #include #include -#include "data/SearchIndex.h" +#include "data/search/SearchMatch.h" class QtQueryElement : public QPushButton @@ -37,7 +38,7 @@ public: QtSmartSearchBox(QWidget* parent); virtual ~QtSmartSearchBox(); - void setAutocompletionList(const std::vector& autocompletionList); + void setAutocompletionList(const std::vector& autocompletionList); void setQuery(const std::string& text); void setFocus(); @@ -90,7 +91,7 @@ private: size_t m_cursorIndex; - std::vector m_matches; + std::vector m_matches; bool m_shiftKeyDown; bool m_mousePressed; diff --git a/src/app/qt/view/QtSearchView.cpp b/src/app/qt/view/QtSearchView.cpp index 19d93eaa..dc545f01 100644 --- a/src/app/qt/view/QtSearchView.cpp +++ b/src/app/qt/view/QtSearchView.cpp @@ -43,7 +43,7 @@ void QtSearchView::setFocus() m_setFocusFunctor(); } -void QtSearchView::setAutocompletionList(const std::vector& autocompletionList) +void QtSearchView::setAutocompletionList(const std::vector& autocompletionList) { m_setAutocompletionListFunctor(autocompletionList); } @@ -64,7 +64,7 @@ void QtSearchView::doSetFocus() m_widget->setFocus(); } -void QtSearchView::doSetAutocompletionList(const std::vector& autocompletionList) +void QtSearchView::doSetAutocompletionList(const std::vector& autocompletionList) { m_widget->setAutocompletionList(autocompletionList); setStyleSheet(); diff --git a/src/app/qt/view/QtSearchView.h b/src/app/qt/view/QtSearchView.h index 140efe4a..1bac8359 100644 --- a/src/app/qt/view/QtSearchView.h +++ b/src/app/qt/view/QtSearchView.h @@ -21,20 +21,20 @@ public: // SearchView implementation virtual void setText(const std::string& text); virtual void setFocus(); - virtual void setAutocompletionList(const std::vector& autocompletionList); + virtual void setAutocompletionList(const std::vector& autocompletionList); private: void doRefreshView(); void doSetText(const std::string& text); void doSetFocus(); - void doSetAutocompletionList(const std::vector& autocompletionList); + void doSetAutocompletionList(const std::vector& autocompletionList); void setStyleSheet(); QtThreadedFunctor<> m_refreshViewFunctor; QtThreadedFunctor m_setTextFunctor; QtThreadedFunctor<> m_setFocusFunctor; - QtThreadedFunctor&> m_setAutocompletionListFunctor; + QtThreadedFunctor&> m_setAutocompletionListFunctor; std::shared_ptr m_widget; }; diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index fef79c93..cf9b8524 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -142,6 +142,15 @@ add_files( data/query/QueryTree.cpp data/query/QueryTree.h + data/search/SearchIndex.cpp + data/search/SearchIndex.h + data/search/SearchNode.cpp + data/search/SearchNode.h + data/search/SearchMatch.cpp + data/search/SearchMatch.h + data/search/SearchResult.cpp + data/search/SearchResult.h + data/type/modifier/DataTypeModifier.cpp data/type/modifier/DataTypeModifier.h data/type/modifier/DataTypeModifierArray.cpp @@ -158,8 +167,6 @@ add_files( data/type/DataTypeQualifierList.cpp data/type/DataTypeQualifierList.h - data/SearchIndex.cpp - data/SearchIndex.h data/Storage.cpp data/Storage.h diff --git a/src/lib/component/controller/SearchController.cpp b/src/lib/component/controller/SearchController.cpp index d9247955..03cc2676 100644 --- a/src/lib/component/controller/SearchController.cpp +++ b/src/lib/component/controller/SearchController.cpp @@ -18,10 +18,11 @@ void SearchController::handleMessage(MessageActivateToken* message) { if (!m_ignoreNextMessageActivateToken && message->tokenId) { - std::string name = m_graphAccess->getNameForNodeWithId(message->tokenId); - std::stringstream ss; - ss << '"' << name << ',' << message->tokenId << '"'; - getView()->setText(ss.str()); + SearchMatch match; + match.fullName = m_graphAccess->getNameForNodeWithId(message->tokenId); + match.tokenIds.insert(message->tokenId); + + getView()->setText(match.encodeForQuery()); } m_ignoreNextMessageActivateToken = false; diff --git a/src/lib/component/view/SearchView.h b/src/lib/component/view/SearchView.h index 292d1446..8e3e42f0 100644 --- a/src/lib/component/view/SearchView.h +++ b/src/lib/component/view/SearchView.h @@ -2,7 +2,7 @@ #define SEARCH_VIEW_H #include "component/view/View.h" -#include "data/SearchIndex.h" +#include "data/search/SearchMatch.h" class SearchController; @@ -16,7 +16,7 @@ public: virtual void setText(const std::string& s) = 0; virtual void setFocus() = 0; - virtual void setAutocompletionList(const std::vector& autocompletionList) = 0; + virtual void setAutocompletionList(const std::vector& autocompletionList) = 0; protected: SearchController* getController(); diff --git a/src/lib/data/SearchIndex.cpp b/src/lib/data/SearchIndex.cpp deleted file mode 100644 index c089bbb4..00000000 --- a/src/lib/data/SearchIndex.cpp +++ /dev/null @@ -1,465 +0,0 @@ -#include "data/SearchIndex.h" - -#include -#include - -#include "data/query/QueryCommand.h" -#include "data/query/QueryToken.h" -#include "utility/logging/logging.h" -#include "utility/utilityString.h" - -SearchIndex::SearchResult::SearchResult() -{ -} - -SearchIndex::SearchResult::SearchResult( - size_t weight, - const SearchIndex::SearchNode* node, - const SearchIndex::SearchNode* parent -) - : weight(weight) - , node(node) - , parent(parent) -{ -} - -bool SearchIndex::SearchResult::operator()(const SearchResult& lhs, const SearchResult& rhs) const -{ - if (lhs.weight != rhs.weight) - { - return lhs.weight > rhs.weight; - } - - return lhs.node->getFullName() < rhs.node->getFullName(); -} - -void SearchIndex::SearchMatch::print(std::ostream& ostream) const -{ - ostream << weight << '\t' << fullName << std::endl << '\t'; - size_t i = 0; - for (size_t index : indices) - { - while (i < index) - { - i++; - ostream << ' '; - } - ostream << '^'; - i++; - } - ostream << std::endl; -} - -std::string SearchIndex::SearchMatch::encodeForQuery() const -{ - if (!tokenIds.size()) - { - return QueryCommand::BOUNDARY + fullName + QueryCommand::BOUNDARY; - } - - std::stringstream ss; - ss << QueryToken::BOUNDARY << fullName; - for (Id tokenId : tokenIds) - { - ss << QueryToken::DELIMITER << tokenId; - } - ss << QueryToken::BOUNDARY; - return ss.str(); -} - -SearchIndex::SearchNode::SearchNode(SearchNode* parent, const std::string& name, Id nameId) - : m_parent(parent) - , m_name(name) - , m_nameId(nameId) -{ -} - -SearchIndex::SearchNode::~SearchNode() -{ -} - -const std::string& SearchIndex::SearchNode::getName() const -{ - return m_name; -} - -std::string SearchIndex::SearchNode::getFullName() const -{ - if (m_parent && m_parent->m_nameId) - { - return m_parent->getFullName() + DELIMITER + getName(); - } - else - { - return getName(); - } -} - -Id SearchIndex::SearchNode::getNameId() const -{ - return m_nameId; -} - -Id SearchIndex::SearchNode::getFirstTokenId() const -{ - if (m_tokenIds.size()) - { - return *m_tokenIds.begin(); - } - - return 0; -} - -const std::set& SearchIndex::SearchNode::getTokenIds() const -{ - return m_tokenIds; -} - -void SearchIndex::SearchNode::addTokenId(Id tokenId) -{ - m_tokenIds.insert(tokenId); -} - -SearchIndex::SearchNode* SearchIndex::SearchNode::getParent() const -{ - if (m_parent && m_parent->m_nameId) - { - return m_parent; - } - return nullptr; -} - -std::deque SearchIndex::SearchNode::getParentsWithoutTokenId() -{ - std::deque nodes; - - SearchNode* node = this; - while (node->m_nameId && !node->m_tokenIds.size()) - { - nodes.push_front(node); - node = node->m_parent; - } - - return nodes; -} - -const std::set>& SearchIndex::SearchNode::getChildren() const -{ - return m_nodes; -} - -SearchIndex::SearchResults SearchIndex::SearchNode::runFuzzySearch(const std::string& query, bool recursive) const -{ - SearchResults result; - - if (recursive) - { - for (std::shared_ptr n: m_nodes) - { - FuzzyMap m = n->fuzzyMatchRecursive(query, 0, 0, 0); - for (const std::pair& p : m) - { - result.insert(SearchResult(p.first, p.second, this)); - } - } - } - else - { - std::pair p = fuzzyMatch(query, 0, 0); - size_t pos = p.first; - size_t weight = p.second; - - if (pos == query.size()) - { - result.insert(SearchResult(weight, this, this)); - } - } - - // TODO: Currently all matches are added to the ordered set and get compared by their fullName for alphabetical - // order. This could be improved by limiting the number of items to e.g. 100. - return result; -} - -std::shared_ptr SearchIndex::SearchNode::addNodeRecursive( - std::deque* nameIds, const Dictionary& dictionary -){ - Id nameId = nameIds->front(); - nameIds->pop_front(); - - std::shared_ptr node = getChildWithNameId(nameId); - if (!node) - { - node = std::make_shared(this, dictionary.getWord(nameId), nameId); - m_nodes.insert(node); - } - - if (nameIds->size() > 0) - { - return node->addNodeRecursive(nameIds, dictionary); - } - - return node; -} - -std::shared_ptr SearchIndex::SearchNode::getNodeRecursive(std::deque* nameIds) const -{ - Id nameId = nameIds->front(); - nameIds->pop_front(); - - std::shared_ptr node = getChildWithNameId(nameId); - if (node) - { - if (!nameIds->size()) - { - return node; - } - - return node->getNodeRecursive(nameIds); - } - - return nullptr; -} - -SearchIndex::SearchNode::FuzzyMap SearchIndex::SearchNode::fuzzyMatchRecursive( - const std::string& query, size_t pos, size_t weight, size_t size) const -{ - FuzzyMap result; - std::pair p = fuzzyMatch(query, pos, size); - pos = p.first; - weight += p.second; - - if (pos == query.size()) - { - result.emplace(weight, this); - return result; - } - - for (std::shared_ptr n: m_nodes) - { - FuzzyMap m = n->fuzzyMatchRecursive(query, pos, weight, size + m_name.size() + SearchIndex::DELIMITER.size()); - result.insert(m.begin(), m.end()); - } - - return result; -} - -std::pair SearchIndex::SearchNode::fuzzyMatch( - const std::string query, size_t start, size_t size, std::vector* indices) const -{ - size_t pos = start; - size_t weight = 0; - size_t matchCount = 0; - char lastChar = '\0'; - - size_t ql = query.size(); - size_t ml = m_name.size(); - - if (!query.size()) - { - return std::pair(pos, weight); - } - - if (query[pos] == ':') - { - pos++; - if (indices && size >= 2) - { - indices->push_back(size - 2); - } - - if (pos < ql && query[pos] == ':') - { - pos++; - if (indices && size >= 1) - { - indices->push_back(size - 1); - } - } - } - - for (size_t i = 0; i < ml; i++) - { - char c = m_name[i]; - if (tolower(query[pos]) == tolower(c)) - { - weight += std::max(100 - size - i, 1); - if (matchCount) - { - weight += matchCount * 10; - } - else if (i == 0 || lastChar == '_' || tolower(c) != c) - { - weight += 20; - } - matchCount++; - - pos++; - if (indices) - { - indices->push_back(size + i); - } - - if (pos == ql || query[pos] == ':') - { - break; - } - } - else - { - matchCount = 0; - } - - lastChar = c; - } - - return std::pair(pos, weight); -} - -SearchIndex::SearchMatch SearchIndex::SearchNode::fuzzyMatchData(const std::string& query, const SearchNode* parent) const -{ - SearchMatch data; - data.fullName = getFullName(); - data.tokenIds = m_tokenIds; - data.weight = 0; - - size_t pos = 0; - size_t size = 0; - - std::deque nodes = getNodesToParent(parent); - if (!nodes.size()) - { - nodes.push_back(this); - } - - for (const SearchNode* node : nodes) - { - std::pair p = node->fuzzyMatch(query, pos, size, &data.indices); - pos = p.first; - data.weight += p.second; - size += node->m_name.size() + SearchIndex::DELIMITER.size(); - } - - return data; -} - -std::shared_ptr SearchIndex::SearchNode::getChildWithNameId(Id nameId) const -{ - for (std::shared_ptr n: m_nodes) - { - if (n->m_nameId == nameId) - { - return n; - } - } - - return nullptr; -} - -std::deque SearchIndex::SearchNode::getNodesToParent(const SearchNode* parent) const -{ - std::deque nodes; - - const SearchNode* node = this; - while (node->m_nameId && node != parent) - { - nodes.push_front(node); - node = node->m_parent; - } - - return nodes; -} - - -std::vector SearchIndex::getMatches( - const SearchIndex::SearchResults& searchResults, - const std::string& query -){ - std::vector result; - - for (SearchResultsIterator it = searchResults.begin(); it != searchResults.end(); it++) - { - SearchMatch match = it->node->fuzzyMatchData(query, it->parent); - result.push_back(match); - - if (it->weight != match.weight) - { - LOG_ERROR("Weight between matching and meta data is different."); - } - } - - return result; -} - -void SearchIndex::logMatches(const std::vector& matches, const std::string& query) -{ - std::stringstream ss; - ss << std::endl << matches.size() << " matches for \"" << query << "\":" << std::endl; - for (const SearchIndex::SearchMatch& match : matches) - { - match.print(ss); - } - LOG_INFO(ss.str()); -} - -SearchIndex::SearchIndex() - : m_root(nullptr, DELIMITER, 0) -{ -} - -SearchIndex::~SearchIndex() -{ -} - -void SearchIndex::clear() -{ - m_root.m_nodes.clear(); -} - -Id SearchIndex::getWordId(const std::string& word) -{ - return m_dictionary.getWordId(word); -} - -const std::string& SearchIndex::getWord(Id wordId) const -{ - return m_dictionary.getWord(wordId); -} - -SearchIndex::SearchNode* SearchIndex::addNode(std::vector nameHierarchy) -{ - std::deque nameIds; - for (const std::string& name: nameHierarchy) - { - nameIds.push_back(m_dictionary.getWordId(name)); - } - if (nameIds.size()) - { - return m_root.addNodeRecursive(&nameIds, m_dictionary).get(); - } - - return nullptr; -} - -SearchIndex::SearchNode* SearchIndex::getNode(const std::string& fullName) const -{ - std::deque nameIds = m_dictionary.getWordIdsConst(fullName, DELIMITER); - - if (nameIds.size()) - { - return m_root.getNodeRecursive(&nameIds).get(); - } - - return nullptr; -} - -SearchIndex::SearchResults SearchIndex::runFuzzySearch(const std::string& query) const -{ - return m_root.runFuzzySearch(query, true); -} - -std::vector SearchIndex::runFuzzySearchAndGetMatches(const std::string& query) const -{ - return getMatches(runFuzzySearch(query), query); -} - -const std::string SearchIndex::DELIMITER = "::"; diff --git a/src/lib/data/SearchIndex.h b/src/lib/data/SearchIndex.h deleted file mode 100644 index 4cdcd98e..00000000 --- a/src/lib/data/SearchIndex.h +++ /dev/null @@ -1,120 +0,0 @@ -#ifndef SEARCH_INDEX_H -#define SEARCH_INDEX_H - -#include -#include -#include -#include -#include -#include - -#include "utility/text/Dictionary.h" -#include "utility/types.h" - -class SearchIndex -{ -public: - class SearchNode; - - struct SearchResult - { - SearchResult(); - SearchResult(size_t weight, const SearchIndex::SearchNode* node, const SearchIndex::SearchNode* parent); - - bool operator()(const SearchResult& lhs, const SearchResult& rhs) const; - - size_t weight; - const SearchIndex::SearchNode* node; - const SearchIndex::SearchNode* parent; - }; - - typedef std::set SearchResults; - typedef SearchResults::const_iterator SearchResultsIterator; - - struct SearchMatch - { - void print(std::ostream& ostream) const; - - std::string encodeForQuery() const; - - std::string fullName; - std::set tokenIds; - std::vector indices; - size_t weight; - }; - - class SearchNode - { - public: - typedef std::multimap FuzzyMap; - typedef FuzzyMap::const_iterator FuzzyMapIterator; - - SearchNode(SearchNode* parent, const std::string& name, Id nameId); - ~SearchNode(); - - const std::string& getName() const; - std::string getFullName() const; - - Id getNameId() const; - - Id getFirstTokenId() const; - const std::set& getTokenIds() const; - void addTokenId(Id tokenId); - - SearchNode* getParent() const; - std::deque getParentsWithoutTokenId(); - - const std::set>& getChildren() const; - - SearchResults runFuzzySearch(const std::string& query, bool recursive) const; - - private: - // Accessed by SearchIndex - std::shared_ptr addNodeRecursive(std::deque* nameIds, const Dictionary& dictionary); - std::shared_ptr getNodeRecursive(std::deque* nameIds) const; - - friend class SearchIndex; - - FuzzyMap fuzzyMatchRecursive(const std::string& query, size_t pos, size_t weight, size_t size) const; - std::pair fuzzyMatch( - const std::string query, size_t start, size_t size, std::vector* indices = nullptr) const; - SearchMatch fuzzyMatchData(const std::string& query, const SearchNode* parent) const; - - std::shared_ptr getChildWithNameId(Id nameId) const; - std::deque getNodesToParent(const SearchNode* parent) const; - - std::set> m_nodes; - SearchNode* m_parent; - - std::set m_tokenIds; - - const std::string& m_name; - const Id m_nameId; - }; - - static std::vector getMatches(const SearchResults& searchResults, const std::string& query); - - static void logMatches(const std::vector& matches, const std::string& query); - - SearchIndex(); - virtual ~SearchIndex(); - - void clear(); - - Id getWordId(const std::string& word); - const std::string& getWord(Id wordId) const; - - SearchNode* addNode(std::vector nameHierarchy); - SearchNode* getNode(const std::string& fullName) const; - - SearchResults runFuzzySearch(const std::string& query) const; - std::vector runFuzzySearchAndGetMatches(const std::string& query) const; - - static const std::string DELIMITER; - -private: - SearchNode m_root; - Dictionary m_dictionary; -}; - -#endif // SEARCH_INDEX_H diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index 057b6346..f54196cd 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -325,7 +325,7 @@ Id Storage::onTypeUsageParsed(const ParseTypeUsage& type, const ParseFunction& f Id Storage::getIdForNodeWithName(const std::string& fullName) const { - SearchIndex::SearchNode* node = m_tokenIndex.getNode(fullName); + SearchNode* node = m_tokenIndex.getNode(fullName); if (node) { return node->getFirstTokenId(); @@ -352,10 +352,10 @@ std::string Storage::getNameForNodeWithId(Id id) const } } -std::vector Storage::getAutocompletionMatches( +std::vector Storage::getAutocompletionMatches( const std::string& query, const std::string& word) const { - SearchIndex::SearchResults tokenResults; + SearchResults tokenResults; bool usedSubquery = false; if (query.size()) @@ -370,12 +370,12 @@ std::vector Storage::getAutocompletionMatches( if (word.size()) { - SearchIndex::SearchResults filterResults = m_filterIndex.runFuzzySearch(word); + SearchResults filterResults = m_filterIndex.runFuzzySearch(word); tokenResults.insert(filterResults.begin(), filterResults.end()); } - std::vector matches = SearchIndex::getMatches(tokenResults, word); - SearchIndex::logMatches(matches, word); + std::vector matches = SearchIndex::getMatches(tokenResults, word); + SearchMatch::log(matches, word); return matches; } @@ -588,7 +588,7 @@ const SearchIndex& Storage::getSearchIndex() const Node* Storage::addNodeHierarchy(Node::NodeType type, std::vector nameHierarchy) { - SearchIndex::SearchNode* searchNode = m_tokenIndex.addNode(nameHierarchy); + SearchNode* searchNode = m_tokenIndex.addNode(nameHierarchy); if (!searchNode) { LOG_ERROR("No SearchNode"); @@ -600,7 +600,7 @@ Node* Storage::addNodeHierarchy(Node::NodeType type, std::vector na Node* Storage::addNodeHierarchyWithDistinctSignature(Node::NodeType type, const ParseFunction& function) { - SearchIndex::SearchNode* searchNode = m_tokenIndex.addNode(function.nameHierarchy); + SearchNode* searchNode = m_tokenIndex.addNode(function.nameHierarchy); if (!searchNode) { LOG_ERROR("No SearchNode"); @@ -728,7 +728,7 @@ TokenLocation* Storage::addTokenLocation(Token* token, const ParseLocation& loc, bool Storage::getSubQuerySearchResults( const std::string& query, const std::string& word, - SearchIndex::SearchResults* results + SearchResults* results ) const { std::string q = query; bool returnChilds = false; @@ -757,7 +757,7 @@ bool Storage::getSubQuerySearchResults( GraphFilterConductor conductor; conductor.filter(&tree, &m_graph, &graph); - std::vector searchNodes; + std::vector searchNodes; graph.forEachNode( [&searchNodes](Node* node) { @@ -769,23 +769,23 @@ bool Storage::getSubQuerySearchResults( } ); - for (const SearchIndex::SearchNode* node : searchNodes) + for (const SearchNode* node : searchNodes) { if (word.size()) { - SearchIndex::SearchResults res = node->runFuzzySearch(word, returnChilds); + SearchResults res = node->runFuzzySearch(word, returnChilds); results->insert(res.begin(), res.end()); } else if (returnChilds) { - for (const std::shared_ptr& child : node->getChildren()) + for (const std::shared_ptr& child : node->getChildren()) { - results->insert(SearchIndex::SearchResult(0, child.get(), child.get())); + results->insert(SearchResult(0, child.get(), child.get())); } } else if (searchNodes.size() > 1) { - results->insert(SearchIndex::SearchResult(0, node, node)); + results->insert(SearchResult(0, node, node)); } } diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index 6f6b94dd..bca8a46a 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -11,7 +11,7 @@ #include "data/graph/token_component/TokenComponentAccess.h" #include "data/location/TokenLocationCollection.h" #include "data/parser/ParserClient.h" -#include "data/SearchIndex.h" +#include "data/search/SearchIndex.h" class Storage : public ParserClient @@ -71,7 +71,7 @@ public: // GraphAccess implementation virtual Id getIdForNodeWithName(const std::string& fullName) const; virtual std::string getNameForNodeWithId(Id id) const; - virtual std::vector getAutocompletionMatches( + virtual std::vector getAutocompletionMatches( const std::string& query, const std::string& word) const; virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const; @@ -110,7 +110,7 @@ private: TokenLocation* addTokenLocation(Token* token, const ParseLocation& location, bool isScope = false); bool getSubQuerySearchResults( - const std::string& query, const std::string& word, SearchIndex::SearchResults* results) const; + const std::string& query, const std::string& word, SearchResults* results) const; void log(std::string type, std::string str, const ParseLocation& location) const; diff --git a/src/lib/data/access/GraphAccess.h b/src/lib/data/access/GraphAccess.h index a31d4eae..10f60479 100644 --- a/src/lib/data/access/GraphAccess.h +++ b/src/lib/data/access/GraphAccess.h @@ -6,7 +6,7 @@ #include #include "data/graph/Graph.h" -#include "data/SearchIndex.h" +#include "data/search/SearchMatch.h" #include "utility/types.h" class GraphAccess @@ -16,7 +16,7 @@ public: virtual Id getIdForNodeWithName(const std::string& name) const = 0; virtual std::string getNameForNodeWithId(Id id) const = 0; - virtual std::vector getAutocompletionMatches( + virtual std::vector getAutocompletionMatches( const std::string& query, const std::string& word) const = 0; virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const = 0; diff --git a/src/lib/data/access/GraphAccessProxy.cpp b/src/lib/data/access/GraphAccessProxy.cpp index 05ccecf9..4521afcc 100644 --- a/src/lib/data/access/GraphAccessProxy.cpp +++ b/src/lib/data/access/GraphAccessProxy.cpp @@ -47,7 +47,7 @@ std::string GraphAccessProxy::getNameForNodeWithId(Id id) const return ""; } -std::vector GraphAccessProxy::getAutocompletionMatches( +std::vector GraphAccessProxy::getAutocompletionMatches( const std::string& query, const std::string& word ) const { @@ -56,7 +56,7 @@ std::vector GraphAccessProxy::getAutocompletionMatches return m_subject->getAutocompletionMatches(query, word); } - return std::vector(); + return std::vector(); } std::shared_ptr GraphAccessProxy::getGraphForActiveTokenIds(const std::vector& tokenIds) const diff --git a/src/lib/data/access/GraphAccessProxy.h b/src/lib/data/access/GraphAccessProxy.h index 9157f442..3b2968fb 100644 --- a/src/lib/data/access/GraphAccessProxy.h +++ b/src/lib/data/access/GraphAccessProxy.h @@ -15,7 +15,7 @@ public: // GraphAccess implementation virtual Id getIdForNodeWithName(const std::string& name) const; virtual std::string getNameForNodeWithId(Id id) const; - virtual std::vector getAutocompletionMatches( + virtual std::vector getAutocompletionMatches( const std::string& query, const std::string& word) const; virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const; diff --git a/src/lib/data/graph/StorageGraph.cpp b/src/lib/data/graph/StorageGraph.cpp index 3a7ef422..d43ba326 100644 --- a/src/lib/data/graph/StorageGraph.cpp +++ b/src/lib/data/graph/StorageGraph.cpp @@ -12,7 +12,7 @@ StorageGraph::~StorageGraph() { } -Node* StorageGraph::createNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode) +Node* StorageGraph::createNodeHierarchy(Node::NodeType type, SearchNode* searchNode) { Node* node = getNodeById(searchNode->getFirstTokenId()); if (!node) @@ -29,7 +29,7 @@ Node* StorageGraph::createNodeHierarchy(Node::NodeType type, SearchIndex::Search } Node* StorageGraph::createNodeHierarchyWithDistinctSignature( - Node::NodeType type, SearchIndex::SearchNode* searchNode, std::shared_ptr signature + Node::NodeType type, SearchNode* searchNode, std::shared_ptr signature ){ Node* node = getNodeById(searchNode->getFirstTokenId()); if (!node) @@ -90,9 +90,9 @@ Edge* StorageGraph::createEdge(Edge::EdgeType type, Node* from, Node* to) return insertEdge(type, from, to); } -Node* StorageGraph::insertNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode) +Node* StorageGraph::insertNodeHierarchy(Node::NodeType type, SearchNode* searchNode) { - std::deque searchNodes = searchNode->getParentsWithoutTokenId(); + std::deque searchNodes = searchNode->getParentsWithoutTokenId(); if (!searchNodes.size()) { @@ -101,7 +101,7 @@ Node* StorageGraph::insertNodeHierarchy(Node::NodeType type, SearchIndex::Search } Node* parentNode = nullptr; - SearchIndex::SearchNode* parentSearchNode = searchNodes.front()->getParent(); + SearchNode* parentSearchNode = searchNodes.front()->getParent(); if (parentSearchNode) { parentNode = getNodeById(parentSearchNode->getFirstTokenId()); @@ -118,7 +118,7 @@ Node* StorageGraph::insertNodeHierarchy(Node::NodeType type, SearchIndex::Search return parentNode; } -Node* StorageGraph::insertNode(Node::NodeType type, Node* parentNode, SearchIndex::SearchNode* searchNode) +Node* StorageGraph::insertNode(Node::NodeType type, Node* parentNode, SearchNode* searchNode) { std::shared_ptr node = std::make_shared(type, std::make_shared(searchNode)); diff --git a/src/lib/data/graph/StorageGraph.h b/src/lib/data/graph/StorageGraph.h index d02d950f..f7915993 100644 --- a/src/lib/data/graph/StorageGraph.h +++ b/src/lib/data/graph/StorageGraph.h @@ -3,7 +3,7 @@ #include "data/graph/Graph.h" #include "data/graph/token_component/TokenComponentSignature.h" -#include "data/SearchIndex.h" +#include "data/search/SearchNode.h" class StorageGraph : public Graph @@ -12,14 +12,14 @@ public: StorageGraph(); virtual ~StorageGraph(); - Node* createNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode); + Node* createNodeHierarchy(Node::NodeType type, SearchNode* searchNode); Node* createNodeHierarchyWithDistinctSignature( - Node::NodeType type, SearchIndex::SearchNode* searchNode, std::shared_ptr signature); + Node::NodeType type, SearchNode* searchNode, std::shared_ptr signature); Edge* createEdge(Edge::EdgeType type, Node* from, Node* to); private: - Node* insertNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode); - Node* insertNode(Node::NodeType type, Node* parentNode, SearchIndex::SearchNode* searchNode); + Node* insertNodeHierarchy(Node::NodeType type, SearchNode* searchNode); + Node* insertNode(Node::NodeType type, Node* parentNode, SearchNode* searchNode); Edge* insertEdge(Edge::EdgeType type, Node* from, Node* to); }; diff --git a/src/lib/data/graph/token_component/TokenComponentName.cpp b/src/lib/data/graph/token_component/TokenComponentName.cpp index 8978e8cc..296b52ef 100644 --- a/src/lib/data/graph/token_component/TokenComponentName.cpp +++ b/src/lib/data/graph/token_component/TokenComponentName.cpp @@ -2,6 +2,8 @@ #include "utility/utilityString.h" +#include "data/search/SearchIndex.h" + TokenComponentName::TokenComponentName() { } @@ -16,7 +18,7 @@ std::shared_ptr TokenComponentName::copyComponentName() cons } -TokenComponentNameReferenced::TokenComponentNameReferenced(const SearchIndex::SearchNode* searchNode) +TokenComponentNameReferenced::TokenComponentNameReferenced(const SearchNode* searchNode) : m_searchNode(searchNode) { } @@ -40,7 +42,7 @@ std::string TokenComponentNameReferenced::getFullName() const return m_searchNode->getFullName(); } -const SearchIndex::SearchNode* TokenComponentNameReferenced::getSearchNode() const +const SearchNode* TokenComponentNameReferenced::getSearchNode() const { return m_searchNode; } @@ -69,7 +71,7 @@ std::string TokenComponentNameCached::getFullName() const return m_fullName; } -const SearchIndex::SearchNode* TokenComponentNameCached::getSearchNode() const +const SearchNode* TokenComponentNameCached::getSearchNode() const { return nullptr; } diff --git a/src/lib/data/graph/token_component/TokenComponentName.h b/src/lib/data/graph/token_component/TokenComponentName.h index e6514f33..a5d48aaf 100644 --- a/src/lib/data/graph/token_component/TokenComponentName.h +++ b/src/lib/data/graph/token_component/TokenComponentName.h @@ -4,7 +4,7 @@ #include #include "data/graph/token_component/TokenComponent.h" -#include "data/SearchIndex.h" +#include "data/search/SearchNode.h" class TokenComponentName : public TokenComponent @@ -18,7 +18,7 @@ public: virtual std::string getName() const = 0; virtual std::string getFullName() const = 0; - virtual const SearchIndex::SearchNode* getSearchNode() const = 0; + virtual const SearchNode* getSearchNode() const = 0; }; @@ -26,7 +26,7 @@ class TokenComponentNameReferenced : public TokenComponentName { public: - TokenComponentNameReferenced(const SearchIndex::SearchNode* searchNode); + TokenComponentNameReferenced(const SearchNode* searchNode); virtual ~TokenComponentNameReferenced(); virtual std::shared_ptr copy() const; @@ -34,10 +34,10 @@ public: virtual std::string getName() const; virtual std::string getFullName() const; - virtual const SearchIndex::SearchNode* getSearchNode() const; + virtual const SearchNode* getSearchNode() const; private: - const SearchIndex::SearchNode* m_searchNode; + const SearchNode* m_searchNode; }; @@ -53,7 +53,7 @@ public: virtual std::string getName() const; virtual std::string getFullName() const; - virtual const SearchIndex::SearchNode* getSearchNode() const; + virtual const SearchNode* getSearchNode() const; private: const std::string m_fullName; diff --git a/src/lib/data/search/SearchIndex.cpp b/src/lib/data/search/SearchIndex.cpp new file mode 100644 index 00000000..d5f74cfb --- /dev/null +++ b/src/lib/data/search/SearchIndex.cpp @@ -0,0 +1,84 @@ +#include "data/search/SearchIndex.h" + +#include +#include + +#include "data/search/SearchMatch.h" + +std::vector SearchIndex::getMatches( + const SearchResults& searchResults, + const std::string& query +){ + std::vector result; + + for (SearchResultsIterator it = searchResults.begin(); it != searchResults.end(); it++) + { + SearchMatch match = it->node->fuzzyMatchData(query, it->parent); + result.push_back(match); + } + + return result; +} + +SearchIndex::SearchIndex() + : m_root(nullptr, DELIMITER, 0) +{ +} + +SearchIndex::~SearchIndex() +{ +} + +void SearchIndex::clear() +{ + m_root.m_nodes.clear(); +} + +Id SearchIndex::getWordId(const std::string& word) +{ + return m_dictionary.getWordId(word); +} + +const std::string& SearchIndex::getWord(Id wordId) const +{ + return m_dictionary.getWord(wordId); +} + +SearchNode* SearchIndex::addNode(std::vector nameHierarchy) +{ + std::deque nameIds; + for (const std::string& name: nameHierarchy) + { + nameIds.push_back(m_dictionary.getWordId(name)); + } + if (nameIds.size()) + { + return m_root.addNodeRecursive(&nameIds, m_dictionary).get(); + } + + return nullptr; +} + +SearchNode* SearchIndex::getNode(const std::string& fullName) const +{ + std::deque nameIds = m_dictionary.getWordIdsConst(fullName, DELIMITER); + + if (nameIds.size()) + { + return m_root.getNodeRecursive(&nameIds).get(); + } + + return nullptr; +} + +SearchResults SearchIndex::runFuzzySearch(const std::string& query) const +{ + return m_root.runFuzzySearch(query, true); +} + +std::vector SearchIndex::runFuzzySearchAndGetMatches(const std::string& query) const +{ + return getMatches(runFuzzySearch(query), query); +} + +const std::string SearchIndex::DELIMITER = "::"; diff --git a/src/lib/data/search/SearchIndex.h b/src/lib/data/search/SearchIndex.h new file mode 100644 index 00000000..2e41300a --- /dev/null +++ b/src/lib/data/search/SearchIndex.h @@ -0,0 +1,39 @@ +#ifndef SEARCH_INDEX_H +#define SEARCH_INDEX_H + +#include +#include +#include + +#include "utility/text/Dictionary.h" +#include "utility/types.h" + +#include "data/search/SearchNode.h" + +class SearchIndex +{ +public: + static std::vector getMatches(const SearchResults& searchResults, const std::string& query); + + SearchIndex(); + virtual ~SearchIndex(); + + void clear(); + + Id getWordId(const std::string& word); + const std::string& getWord(Id wordId) const; + + SearchNode* addNode(std::vector nameHierarchy); + SearchNode* getNode(const std::string& fullName) const; + + SearchResults runFuzzySearch(const std::string& query) const; + std::vector runFuzzySearchAndGetMatches(const std::string& query) const; + + static const std::string DELIMITER; + +private: + SearchNode m_root; + Dictionary m_dictionary; +}; + +#endif // SEARCH_INDEX_H diff --git a/src/lib/data/search/SearchMatch.cpp b/src/lib/data/search/SearchMatch.cpp new file mode 100644 index 00000000..d2f9e16c --- /dev/null +++ b/src/lib/data/search/SearchMatch.cpp @@ -0,0 +1,55 @@ +#include "data/search/SearchMatch.h" + +#include + +#include "utility/logging/logging.h" + +#include "data/query/QueryCommand.h" +#include "data/query/QueryToken.h" + +void SearchMatch::log(const std::vector& matches, const std::string& query) +{ + std::stringstream ss; + ss << std::endl << matches.size() << " matches for \"" << query << "\":" << std::endl; + + for (const SearchMatch& match : matches) + { + match.print(ss); + } + + LOG_INFO(ss.str()); +} + +void SearchMatch::print(std::ostream& ostream) const +{ + ostream << weight << '\t' << fullName << std::endl << '\t'; + size_t i = 0; + for (size_t index : indices) + { + while (i < index) + { + i++; + ostream << ' '; + } + ostream << '^'; + i++; + } + ostream << std::endl; +} + +std::string SearchMatch::encodeForQuery() const +{ + if (!tokenIds.size()) + { + return QueryCommand::BOUNDARY + fullName + QueryCommand::BOUNDARY; + } + + std::stringstream ss; + ss << QueryToken::BOUNDARY << fullName; + for (Id tokenId : tokenIds) + { + ss << QueryToken::DELIMITER << tokenId; + } + ss << QueryToken::BOUNDARY; + return ss.str(); +} diff --git a/src/lib/data/search/SearchMatch.h b/src/lib/data/search/SearchMatch.h new file mode 100644 index 00000000..0fc8fde4 --- /dev/null +++ b/src/lib/data/search/SearchMatch.h @@ -0,0 +1,24 @@ +#ifndef SEARCH_MATCH_H +#define SEARCH_MATCH_H + +#include +#include +#include +#include + +#include "utility/types.h" + +struct SearchMatch +{ + static void log(const std::vector& matches, const std::string& query); + void print(std::ostream& ostream) const; + + std::string encodeForQuery() const; + + std::string fullName; + std::set tokenIds; + std::vector indices; + size_t weight; +}; + +#endif // SEARCH_MATCH_H diff --git a/src/lib/data/search/SearchNode.cpp b/src/lib/data/search/SearchNode.cpp new file mode 100644 index 00000000..2880c372 --- /dev/null +++ b/src/lib/data/search/SearchNode.cpp @@ -0,0 +1,309 @@ +#include "data/search/SearchNode.h" + +#include "utility/text/Dictionary.h" + +#include "data/search/SearchIndex.h" +#include "data/search/SearchMatch.h" +#include "data/search/SearchResult.h" + +SearchNode::SearchNode(SearchNode* parent, const std::string& name, Id nameId) + : m_parent(parent) + , m_name(name) + , m_nameId(nameId) +{ +} + +SearchNode::~SearchNode() +{ +} + +const std::string& SearchNode::getName() const +{ + return m_name; +} + +std::string SearchNode::getFullName() const +{ + if (m_parent && m_parent->m_nameId) + { + return m_parent->getFullName() + SearchIndex::DELIMITER + getName(); + } + else + { + return getName(); + } +} + +Id SearchNode::getNameId() const +{ + return m_nameId; +} + +Id SearchNode::getFirstTokenId() const +{ + if (m_tokenIds.size()) + { + return *m_tokenIds.begin(); + } + + return 0; +} + +const std::set& SearchNode::getTokenIds() const +{ + return m_tokenIds; +} + +void SearchNode::addTokenId(Id tokenId) +{ + m_tokenIds.insert(tokenId); +} + +SearchNode* SearchNode::getParent() const +{ + if (m_parent && m_parent->m_nameId) + { + return m_parent; + } + return nullptr; +} + +std::deque SearchNode::getParentsWithoutTokenId() +{ + std::deque nodes; + + SearchNode* node = this; + while (node->m_nameId && !node->m_tokenIds.size()) + { + nodes.push_front(node); + node = node->m_parent; + } + + return nodes; +} + +const std::set>& SearchNode::getChildren() const +{ + return m_nodes; +} + +SearchResults SearchNode::runFuzzySearch(const std::string& query, bool recursive) const +{ + SearchResults result; + + if (recursive) + { + for (std::shared_ptr n: m_nodes) + { + FuzzyMap m = n->fuzzyMatchRecursive(query, 0, 0, 0); + for (const std::pair& p : m) + { + result.insert(SearchResult(p.first, p.second, this)); + } + } + } + else + { + std::pair p = fuzzyMatch(query, 0, 0); + size_t pos = p.first; + size_t weight = p.second; + + if (pos == query.size()) + { + result.insert(SearchResult(weight, this, this)); + } + } + + // TODO: Currently all matches are added to the ordered set and get compared by their fullName for alphabetical + // order. This could be improved by limiting the number of items to e.g. 100. + return result; +} + +std::shared_ptr SearchNode::addNodeRecursive( + std::deque* nameIds, const Dictionary& dictionary +){ + Id nameId = nameIds->front(); + nameIds->pop_front(); + + std::shared_ptr node = getChildWithNameId(nameId); + if (!node) + { + node = std::make_shared(this, dictionary.getWord(nameId), nameId); + m_nodes.insert(node); + } + + if (nameIds->size() > 0) + { + return node->addNodeRecursive(nameIds, dictionary); + } + + return node; +} + +std::shared_ptr SearchNode::getNodeRecursive(std::deque* nameIds) const +{ + Id nameId = nameIds->front(); + nameIds->pop_front(); + + std::shared_ptr node = getChildWithNameId(nameId); + if (node) + { + if (!nameIds->size()) + { + return node; + } + + return node->getNodeRecursive(nameIds); + } + + return nullptr; +} + +SearchMatch SearchNode::fuzzyMatchData(const std::string& query, const SearchNode* parent) const +{ + SearchMatch data; + data.fullName = getFullName(); + data.tokenIds = m_tokenIds; + data.weight = 0; + + size_t pos = 0; + size_t size = 0; + + std::deque nodes = getNodesToParent(parent); + if (!nodes.size()) + { + nodes.push_back(this); + } + + for (const SearchNode* node : nodes) + { + std::pair p = node->fuzzyMatch(query, pos, size, &data.indices); + pos = p.first; + data.weight += p.second; + size += node->m_name.size() + SearchIndex::DELIMITER.size(); + } + + return data; +} + +SearchNode::FuzzyMap SearchNode::fuzzyMatchRecursive( + const std::string& query, size_t pos, size_t weight, size_t size) const +{ + FuzzyMap result; + std::pair p = fuzzyMatch(query, pos, size); + pos = p.first; + weight += p.second; + + if (pos == query.size()) + { + result.emplace(weight, this); + return result; + } + + for (std::shared_ptr n: m_nodes) + { + FuzzyMap m = n->fuzzyMatchRecursive(query, pos, weight, size + m_name.size() + SearchIndex::DELIMITER.size()); + result.insert(m.begin(), m.end()); + } + + return result; +} + +std::pair SearchNode::fuzzyMatch( + const std::string query, size_t start, size_t size, std::vector* indices) const +{ + size_t pos = start; + size_t weight = 0; + size_t matchCount = 0; + char lastChar = '\0'; + + size_t ql = query.size(); + size_t ml = m_name.size(); + + if (!query.size()) + { + return std::pair(pos, weight); + } + + if (query[pos] == ':') + { + pos++; + if (indices && size >= 2) + { + indices->push_back(size - 2); + } + + if (pos < ql && query[pos] == ':') + { + pos++; + if (indices && size >= 1) + { + indices->push_back(size - 1); + } + } + } + + for (size_t i = 0; i < ml; i++) + { + char c = m_name[i]; + if (tolower(query[pos]) == tolower(c)) + { + weight += std::max(100 - size - i, 1); + if (matchCount) + { + weight += matchCount * 10; + } + else if (i == 0 || lastChar == '_' || tolower(c) != c) + { + weight += 20; + } + matchCount++; + + pos++; + if (indices) + { + indices->push_back(size + i); + } + + if (pos == ql || query[pos] == ':') + { + break; + } + } + else + { + matchCount = 0; + } + + lastChar = c; + } + + return std::pair(pos, weight); +} + +std::shared_ptr SearchNode::getChildWithNameId(Id nameId) const +{ + for (std::shared_ptr n: m_nodes) + { + if (n->m_nameId == nameId) + { + return n; + } + } + + return nullptr; +} + +std::deque SearchNode::getNodesToParent(const SearchNode* parent) const +{ + std::deque nodes; + + const SearchNode* node = this; + while (node->m_nameId && node != parent) + { + nodes.push_front(node); + node = node->m_parent; + } + + return nodes; +} diff --git a/src/lib/data/search/SearchNode.h b/src/lib/data/search/SearchNode.h new file mode 100644 index 00000000..34837b53 --- /dev/null +++ b/src/lib/data/search/SearchNode.h @@ -0,0 +1,69 @@ +#ifndef SEARCH_NODE_H +#define SEARCH_NODE_H + +#include +#include +#include +#include +#include +#include + +#include "utility/types.h" + +#include "data/search/SearchResult.h" + +class Dictionary; +class SearchIndex; +struct SearchMatch; + +class SearchNode +{ +public: + SearchNode(SearchNode* parent, const std::string& name, Id nameId); + ~SearchNode(); + + const std::string& getName() const; + std::string getFullName() const; + + Id getNameId() const; + + Id getFirstTokenId() const; + const std::set& getTokenIds() const; + void addTokenId(Id tokenId); + + SearchNode* getParent() const; + std::deque getParentsWithoutTokenId(); + + const std::set>& getChildren() const; + + SearchResults runFuzzySearch(const std::string& query, bool recursive) const; + +private: + typedef std::multimap FuzzyMap; + typedef FuzzyMap::const_iterator FuzzyMapIterator; + + // Accessed by SearchIndex + std::shared_ptr addNodeRecursive(std::deque* nameIds, const Dictionary& dictionary); + std::shared_ptr getNodeRecursive(std::deque* nameIds) const; + + SearchMatch fuzzyMatchData(const std::string& query, const SearchNode* parent) const; + + friend class SearchIndex; + + FuzzyMap fuzzyMatchRecursive(const std::string& query, size_t pos, size_t weight, size_t size) const; + std::pair fuzzyMatch( + const std::string query, size_t start, size_t size, std::vector* indices = nullptr) const; + + std::shared_ptr getChildWithNameId(Id nameId) const; + std::deque getNodesToParent(const SearchNode* parent) const; + + std::set> m_nodes; + SearchNode* m_parent; + + std::set m_tokenIds; + + const std::string& m_name; + const Id m_nameId; +}; + +#endif // SEARCH_NODE_H diff --git a/src/lib/data/search/SearchResult.cpp b/src/lib/data/search/SearchResult.cpp new file mode 100644 index 00000000..6e36fcc9 --- /dev/null +++ b/src/lib/data/search/SearchResult.cpp @@ -0,0 +1,24 @@ +#include "data/search/SearchResult.h" + +#include "data/search/SearchNode.h" + +SearchResult::SearchResult() +{ +} + +SearchResult::SearchResult(size_t weight, const SearchNode* node, const SearchNode* parent) + : weight(weight) + , node(node) + , parent(parent) +{ +} + +bool SearchResult::operator()(const SearchResult& lhs, const SearchResult& rhs) const +{ + if (lhs.weight != rhs.weight) + { + return lhs.weight > rhs.weight; + } + + return lhs.node->getFullName() < rhs.node->getFullName(); +} diff --git a/src/lib/data/search/SearchResult.h b/src/lib/data/search/SearchResult.h new file mode 100644 index 00000000..89c3adc6 --- /dev/null +++ b/src/lib/data/search/SearchResult.h @@ -0,0 +1,23 @@ +#ifndef SEARCH_RESULT_H +#define SEARCH_RESULT_H + +#include + +class SearchNode; + +struct SearchResult +{ + SearchResult(); + SearchResult(size_t weight, const SearchNode* node, const SearchNode* parent); + + bool operator()(const SearchResult& lhs, const SearchResult& rhs) const; + + size_t weight; + const SearchNode* node; + const SearchNode* parent; +}; + +typedef std::set SearchResults; +typedef SearchResults::const_iterator SearchResultsIterator; + +#endif // SEARCH_RESULT_H diff --git a/src/test/GraphFilterConductorTestSuite.h b/src/test/GraphFilterConductorTestSuite.h index 31e7f241..6aa93969 100644 --- a/src/test/GraphFilterConductorTestSuite.h +++ b/src/test/GraphFilterConductorTestSuite.h @@ -165,7 +165,7 @@ private: { createTestStorage(); - std::vector matches = m_storage->getAutocompletionMatches("", name); + std::vector matches = m_storage->getAutocompletionMatches("", name); if (matches.size() && matches[0].fullName == name) { return matches[0].tokenIds; diff --git a/src/test/SearchIndexTestSuite.h b/src/test/SearchIndexTestSuite.h index 4e0cbb0c..d1e7188b 100644 --- a/src/test/SearchIndexTestSuite.h +++ b/src/test/SearchIndexTestSuite.h @@ -1,6 +1,6 @@ #include "cxxtest/TestSuite.h" -#include "data/SearchIndex.h" +#include "data/search/SearchIndex.h" #include "utility/utilityString.h" class SearchIndexTestSuite : public CxxTest::TestSuite @@ -9,7 +9,7 @@ public: void test_add_node() { SearchIndex index; - SearchIndex::SearchNode* node = index.addNode(utility::splitToVector("util", "::")); + SearchNode* node = index.addNode(utility::splitToVector("util", "::")); TS_ASSERT(node); TS_ASSERT_EQUALS("util", node->getName()); @@ -24,7 +24,7 @@ public: { SearchIndex index; index.addNode(utility::splitToVector("util", "::")); - SearchIndex::SearchNode* node = index.getNode("util"); + SearchNode* node = index.getNode("util"); TS_ASSERT(node); TS_ASSERT_EQUALS("util", node->getName()); @@ -41,7 +41,7 @@ public: void test_add_hierarchy_node() { SearchIndex index; - SearchIndex::SearchNode* node = index.addNode(utility::splitToVector("util::math::pow", "::")); + SearchNode* node = index.addNode(utility::splitToVector("util::math::pow", "::")); TS_ASSERT(node); TS_ASSERT_EQUALS("pow", node->getName()); @@ -60,8 +60,8 @@ public: void test_reuse_hierarchy_node() { SearchIndex index; - SearchIndex::SearchNode* node1 = index.addNode(utility::splitToVector("math::pow", "::")); - SearchIndex::SearchNode* node2 = index.addNode(utility::splitToVector("math::floor", "::")); + SearchNode* node1 = index.addNode(utility::splitToVector("math::pow", "::")); + SearchNode* node2 = index.addNode(utility::splitToVector("math::floor", "::")); TS_ASSERT(node1); TS_ASSERT(node2); @@ -92,7 +92,7 @@ public: index.addNode(utility::splitToVector("math", "::")); index.addNode(utility::splitToVector("string", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("u"); + std::vector matches = index.runFuzzySearchAndGetMatches("u"); TS_ASSERT_EQUALS(1, matches.size()); TS_ASSERT_EQUALS("util", matches[0].fullName); @@ -117,7 +117,7 @@ public: index.addNode(utility::splitToVector("util", "::")); index.addNode(utility::splitToVector("MATH", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("MATH", matches[0].fullName); @@ -137,7 +137,7 @@ public: index.addNode(utility::splitToVector("math", "::")); index.addNode(utility::splitToVector("string", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(3, matches.size()); TS_ASSERT_EQUALS("string", matches[0].fullName); @@ -160,7 +160,7 @@ public: index.addNode(utility::splitToVector("uTil", "::")); index.addNode(utility::splitToVector("string", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("uTil", matches[0].fullName); @@ -173,7 +173,7 @@ public: index.addNode(utility::splitToVector("oaabbcc", "::")); index.addNode(utility::splitToVector("ocbaabc", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("abc"); + std::vector matches = index.runFuzzySearchAndGetMatches("abc"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("ocbaabc", matches[0].fullName); @@ -187,7 +187,7 @@ public: index.addNode(utility::splitToVector("util::math::floor", "::")); index.addNode(utility::splitToVector("util::string::concat", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(1, matches.size()); TS_ASSERT_EQUALS("util", matches[0].fullName); @@ -206,7 +206,7 @@ public: index.addNode(utility::splitToVector("util::math::floor", "::")); index.addNode(utility::splitToVector("util::string::concat", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("u:i"); + std::vector matches = index.runFuzzySearchAndGetMatches("u:i"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("util::string", matches[0].fullName); @@ -224,7 +224,7 @@ public: index.addNode(utility::splitToVector("abc::dfe::ghi", "::")); index.addNode(utility::splitToVector("abc::hgi", "::")); - std::vector matches = index.runFuzzySearchAndGetMatches("g"); + std::vector matches = index.runFuzzySearchAndGetMatches("g"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("abc::dfe::ghi", matches[0].fullName); diff --git a/src/test/StorageGraphTestSuite.h b/src/test/StorageGraphTestSuite.h index 45de4a44..0b6c7751 100644 --- a/src/test/StorageGraphTestSuite.h +++ b/src/test/StorageGraphTestSuite.h @@ -1,7 +1,7 @@ #include "cxxtest/TestSuite.h" #include "data/graph/StorageGraph.h" -#include "data/SearchIndex.h" +#include "data/search/SearchIndex.h" class StorageGraphTestSuite : public CxxTest::TestSuite { @@ -295,14 +295,14 @@ private: public: Node* createNodeHierarchy(Node::NodeType type, const std::string& name) { - SearchIndex::SearchNode* searchNode = m_index.addNode(utility::splitToVector(name, "::")); + SearchNode* searchNode = m_index.addNode(utility::splitToVector(name, "::")); return StorageGraph::createNodeHierarchy(type, searchNode); } Node* createNodeHierarchyWithDistinctSignature( Node::NodeType type, const std::string& name, Id signatureId ){ - SearchIndex::SearchNode* searchNode = m_index.addNode(utility::splitToVector(name, "::")); + SearchNode* searchNode = m_index.addNode(utility::splitToVector(name, "::")); std::shared_ptr signature = std::make_shared(signatureId); return StorageGraph::createNodeHierarchyWithDistinctSignature(type, searchNode, signature); }