From 86f4629b0d0e68a27b255249db4c7a2facf915e7 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Tue, 28 Oct 2014 20:21:19 +0100 Subject: [PATCH] logic: preselecting SearchNodes for autocompletion by prefiltering the Graph with the entered Query This change only shows matches in the autocompletion list that are prefiltered by the already entered query. The autocompletion now also shows up after an operator of type . or > has been entered to show an alphabetical list of possible tokens. --- bin/test/data/log/test_log.txt | 38 ++--- src/app/qt/element/QtSmartSearchBox.cpp | 44 +++-- src/app/qt/element/QtSmartSearchBox.h | 5 +- .../component/controller/SearchController.cpp | 9 +- src/lib/data/SearchIndex.cpp | 156 +++++++++++------- src/lib/data/SearchIndex.h | 31 +++- src/lib/data/Storage.cpp | 127 +++++++++++--- src/lib/data/Storage.h | 12 +- src/lib/data/access/GraphAccess.h | 3 +- src/lib/data/access/GraphAccessProxy.cpp | 8 +- src/lib/data/access/GraphAccessProxy.h | 3 +- src/lib/data/graph/FilterableGraph.cpp | 5 + src/lib/data/graph/FilterableGraph.h | 2 + src/lib/data/graph/Node.cpp | 10 ++ src/lib/data/graph/Node.h | 1 + src/lib/data/graph/SubGraph.cpp | 10 ++ src/lib/data/graph/SubGraph.h | 3 + .../token_component/TokenComponentName.cpp | 9 + .../token_component/TokenComponentName.h | 6 + src/lib/data/query/QueryOperator.cpp | 4 +- .../type/MessageSearchAutocomplete.h | 4 +- src/lib/utility/utilityString.cpp | 20 ++- src/lib/utility/utilityString.h | 2 + src/test/GraphFilterConductorTestSuite.h | 8 +- src/test/QueryTreeTestSuite.h | 12 +- src/test/SearchIndexTestSuite.h | 63 +++---- src/test/UtilityStringTestSuite.h | 10 ++ 27 files changed, 407 insertions(+), 198 deletions(-) diff --git a/bin/test/data/log/test_log.txt b/bin/test/data/log/test_log.txt index bc8a2184..c6e80470 100644 --- a/bin/test/data/log/test_log.txt +++ b/bin/test/data/log/test_log.txt @@ -1,4 +1,22 @@ ConfigManager.cpp ERROR: value path/to/nowhere is not present in config. +Token.cpp ERROR: Location Id was not referenced by this Token. +Node.cpp WARNING: Cannot change NodeType after it was already set from namespace to class +Edge.cpp ERROR: Nodes are not plain copies. +Storage.cpp INFO: class: A +Storage.cpp INFO: method: A::A +Storage.cpp INFO: global usage: A::A -> A::count +Storage.cpp INFO: method: A::getCount +Storage.cpp INFO: global usage: A::getCount -> A::count +Storage.cpp INFO: method: A::process +Storage.cpp INFO: field: A::count +Storage.cpp INFO: class: B +Storage.cpp INFO: inheritance: B : A +Storage.cpp INFO: method: B::process +Storage.cpp INFO: type usage: B::process -> int +Storage.cpp INFO: function: main +Storage.cpp INFO: type usage: main -> B +Storage.cpp INFO: call: main -> B::B +Storage.cpp INFO: call: main -> A::getCount Storage.cpp INFO: class: A Storage.cpp INFO: method: A::A Storage.cpp INFO: global usage: A::A -> A::count @@ -31,24 +49,6 @@ SearchIndex.cpp INFO: 237 A::A ^^^^ -Storage.cpp INFO: class: A -Storage.cpp INFO: method: A::A -Storage.cpp INFO: global usage: A::A -> A::count -Storage.cpp INFO: method: A::getCount -Storage.cpp INFO: global usage: A::getCount -> A::count -Storage.cpp INFO: method: A::process -Storage.cpp INFO: field: A::count -Storage.cpp INFO: class: B -Storage.cpp INFO: inheritance: B : A -Storage.cpp INFO: method: B::process -Storage.cpp INFO: type usage: B::process -> int -Storage.cpp INFO: function: main -Storage.cpp INFO: type usage: main -> B -Storage.cpp INFO: call: main -> B::B -Storage.cpp INFO: call: main -> A::getCount -Token.cpp ERROR: Location Id was not referenced by this Token. -Node.cpp WARNING: Cannot change NodeType after it was already set from namespace to class -Edge.cpp ERROR: Nodes are not plain copies. Settings.cpp WARNING: File for Settings not found. ConfigManager.cpp ERROR: value Bool is not present in config. ConfigManager.cpp ERROR: value Int is not present in config. @@ -64,7 +64,6 @@ ConfigManager.cpp ERROR: value Int is not present in config. ConfigManager.cpp ERROR: value Float is not present in config. ConfigManager.cpp ERROR: value String is not present in config. ConfigManager.cpp ERROR: value NewBool is not present in config. -Graph.cpp ERROR: Can't remove member edge, without removing the child node. Storage.cpp INFO: typedef: type -> int Storage.cpp INFO: class: Class Storage.cpp INFO: struct: Struct @@ -104,6 +103,7 @@ Storage.cpp INFO: global usage: isTrue -> global Storage.cpp INFO: function: isTrue Storage.cpp INFO: struct: Struct Storage.cpp INFO: type usage: isTrue -> Struct +Graph.cpp ERROR: Can't remove member edge, without removing the child node. TextAccess.cpp WARNING: Index 'firstLine' has to be lower or equal index 'lastLine', is 3 > 2 TextAccess.cpp WARNING: Tried to access index 10. Maximum index is 8 TextAccess.cpp WARNING: Tried to access index 10. Maximum index is 8 diff --git a/src/app/qt/element/QtSmartSearchBox.cpp b/src/app/qt/element/QtSmartSearchBox.cpp index 08c40855..8e6c46e9 100644 --- a/src/app/qt/element/QtSmartSearchBox.cpp +++ b/src/app/qt/element/QtSmartSearchBox.cpp @@ -159,7 +159,7 @@ void QtSmartSearchBox::keyPressEvent(QKeyEvent* event) selectAllElementsWith(false); layoutElements(); } - else if (cursorPosition() == text().size() && (text().size() || m_cursorIndex < static_cast(m_elements.size()))) + else if (cursorPosition() == text().size() && (text().size() || m_cursorIndex < m_elements.size())) { if (!editTextToElement()) { @@ -179,7 +179,7 @@ void QtSmartSearchBox::keyPressEvent(QKeyEvent* event) } else if (event->matches(QKeySequence::SelectNextChar)) { - if (cursorPosition() == text().size() && m_cursorIndex < static_cast(m_elements.size())) + if (cursorPosition() == text().size() && m_cursorIndex < m_elements.size()) { editTextToElement(); m_elements[m_cursorIndex]->setChecked(!m_elements[m_cursorIndex]->isChecked()); @@ -194,7 +194,7 @@ void QtSmartSearchBox::keyPressEvent(QKeyEvent* event) } else if (event->matches(QKeySequence::MoveToEndOfLine)) { - if (m_cursorIndex < static_cast(m_elements.size())) + if (m_cursorIndex < m_elements.size()) { editTextToElement(); moveCursorTo(m_elements.size()); @@ -343,9 +343,9 @@ void QtSmartSearchBox::onTextEdited(const QString& text) layoutElements(); } - if (token.size()) + if (token.size() || m_elements.size()) { - MessageSearchAutocomplete(token).dispatch(); + requestAutoCompletions(); } } @@ -398,7 +398,7 @@ void QtSmartSearchBox::onElementSelected(QtQueryElement* element) return; } - int idx = 0; + size_t idx = 0; bool checked = element->isChecked(); for (size_t i = 0; i < m_elements.size(); i++) { @@ -503,7 +503,7 @@ void QtSmartSearchBox::editElement(QtQueryElement* element) setEditText(QString::fromStdString(token)); updateElements(); - MessageSearchAutocomplete(token).dispatch(); + requestAutoCompletions(); } void QtSmartSearchBox::updateElements() @@ -514,6 +514,8 @@ void QtSmartSearchBox::updateElements() { std::string name = QueryTree::getTokenName(token); + name = utility::replace(name, "&", "&&"); + std::shared_ptr element = std::make_shared(QString::fromStdString(name), this); m_elements.push_back(element); @@ -538,7 +540,7 @@ void QtSmartSearchBox::layoutElements() for (size_t i = 0; i <= m_elements.size(); i++) { - if (!hasSelected && static_cast(i) == m_cursorIndex) + if (!hasSelected && i == m_cursorIndex) { int left, top, right, bottom; getTextMargins(&left, &top, &right, &bottom); @@ -597,10 +599,10 @@ void QtSmartSearchBox::selectAllElementsWith(bool selected) } } -void QtSmartSearchBox::selectElementsTo(int idx, bool selected) +void QtSmartSearchBox::selectElementsTo(size_t idx, bool selected) { - int low = idx < m_cursorIndex ? idx : m_cursorIndex; - int hi = idx > m_cursorIndex ? idx + 1 : m_cursorIndex; + size_t low = idx < m_cursorIndex ? idx : m_cursorIndex; + size_t hi = idx > m_cursorIndex ? idx + 1 : m_cursorIndex; while (low < hi) { @@ -630,13 +632,13 @@ void QtSmartSearchBox::deleteSelectedElements() return; } - for (int i = m_elements.size() - 1; i >= 0; i--) + for (size_t i = m_elements.size(); i > 0; i--) { - if (m_elements[i]->isChecked()) + if (m_elements[i - 1]->isChecked()) { - m_tokens.erase(m_tokens.begin() + i); + m_tokens.erase(m_tokens.begin() + (i - 1)); - if (i < m_cursorIndex) + if ((i - 1) < m_cursorIndex) { m_cursorIndex--; } @@ -667,3 +669,15 @@ void QtSmartSearchBox::clearLineEdit() completer()->popup()->hide(); } } + +void QtSmartSearchBox::requestAutoCompletions() const +{ + std::string query; + + for (size_t i = 0; i < m_tokens.size() && i < m_cursorIndex; i++) + { + query += m_tokens[i]; + } + + MessageSearchAutocomplete(query, text().toStdString()).dispatch(); +} diff --git a/src/app/qt/element/QtSmartSearchBox.h b/src/app/qt/element/QtSmartSearchBox.h index 928c55ed..f56c6f64 100644 --- a/src/app/qt/element/QtSmartSearchBox.h +++ b/src/app/qt/element/QtSmartSearchBox.h @@ -75,11 +75,12 @@ private: std::string getSelectedString() const; void selectAllElementsWith(bool selected); - void selectElementsTo(int idx, bool selected); + void selectElementsTo(size_t idx, bool selected); void deleteSelectedElements(); void updatePlaceholder(); void clearLineEdit(); + void requestAutoCompletions() const; bool m_allowTextChange; QString m_oldText; @@ -87,7 +88,7 @@ private: std::deque m_tokens; std::vector> m_elements; - int m_cursorIndex; + size_t m_cursorIndex; std::vector m_matches; diff --git a/src/lib/component/controller/SearchController.cpp b/src/lib/component/controller/SearchController.cpp index 0d738935..d9247955 100644 --- a/src/lib/component/controller/SearchController.cpp +++ b/src/lib/component/controller/SearchController.cpp @@ -18,7 +18,10 @@ void SearchController::handleMessage(MessageActivateToken* message) { if (!m_ignoreNextMessageActivateToken && message->tokenId) { - getView()->setText(m_graphAccess->getNameForNodeWithId(message->tokenId)); + std::string name = m_graphAccess->getNameForNodeWithId(message->tokenId); + std::stringstream ss; + ss << '"' << name << ',' << message->tokenId << '"'; + getView()->setText(ss.str()); } m_ignoreNextMessageActivateToken = false; @@ -55,8 +58,8 @@ void SearchController::handleMessage(MessageSearch* message) void SearchController::handleMessage(MessageSearchAutocomplete* message) { - LOG_INFO("autocomplete string: \"" + message->query + "\""); - getView()->setAutocompletionList(m_graphAccess->getAutocompletionMatches(message->query)); + LOG_INFO("autocomplete string: \"" + message->word + "\""); + getView()->setAutocompletionList(m_graphAccess->getAutocompletionMatches(message->query, message->word)); } SearchView* SearchController::getView() diff --git a/src/lib/data/SearchIndex.cpp b/src/lib/data/SearchIndex.cpp index 11ef6d29..c089bbb4 100644 --- a/src/lib/data/SearchIndex.cpp +++ b/src/lib/data/SearchIndex.cpp @@ -8,17 +8,29 @@ #include "utility/logging/logging.h" #include "utility/utilityString.h" -namespace +SearchIndex::SearchResult::SearchResult() { - bool fncomp(const SearchIndex::SearchNode::FuzzySetPair& lhs, const SearchIndex::SearchNode::FuzzySetPair& rhs) - { - if (lhs.first != rhs.first) - { - return lhs.first > rhs.first; - } +} - return lhs.second->getFullName() < rhs.second->getFullName(); +SearchIndex::SearchResult::SearchResult( + size_t weight, + const SearchIndex::SearchNode* node, + const SearchIndex::SearchNode* parent +) + : weight(weight) + , node(node) + , parent(parent) +{ +} + +bool SearchIndex::SearchResult::operator()(const SearchResult& lhs, const SearchResult& rhs) const +{ + if (lhs.weight != rhs.weight) + { + return lhs.weight > rhs.weight; } + + return lhs.node->getFullName() < rhs.node->getFullName(); } void SearchIndex::SearchMatch::print(std::ostream& ostream) const @@ -131,6 +143,43 @@ std::deque SearchIndex::SearchNode::getParentsWithoutT return nodes; } +const std::set>& SearchIndex::SearchNode::getChildren() const +{ + return m_nodes; +} + +SearchIndex::SearchResults SearchIndex::SearchNode::runFuzzySearch(const std::string& query, bool recursive) const +{ + SearchResults result; + + if (recursive) + { + for (std::shared_ptr n: m_nodes) + { + FuzzyMap m = n->fuzzyMatchRecursive(query, 0, 0, 0); + for (const std::pair& p : m) + { + result.insert(SearchResult(p.first, p.second, this)); + } + } + } + else + { + std::pair p = fuzzyMatch(query, 0, 0); + size_t pos = p.first; + size_t weight = p.second; + + if (pos == query.size()) + { + result.insert(SearchResult(weight, this, this)); + } + } + + // TODO: Currently all matches are added to the ordered set and get compared by their fullName for alphabetical + // order. This could be improved by limiting the number of items to e.g. 100. + return result; +} + std::shared_ptr SearchIndex::SearchNode::addNodeRecursive( std::deque* nameIds, const Dictionary& dictionary ){ @@ -171,54 +220,15 @@ std::shared_ptr SearchIndex::SearchNode::getNodeRecursi return nullptr; } -std::vector SearchIndex::SearchNode::findFuzzyMatches(const std::string& query) const -{ - std::vector result; - - if (!query.size()) - { - return result; - } - - // TODO: Currently all matches are added to the ordered set and get compared by their fullName for alphabetical - // order. This should be avoided e.g. by only returning a subset of the best 100 matches in alphabetical order. - FuzzySet ordered(&fncomp); - for (std::shared_ptr n: m_nodes) - { - FuzzyMap m = n->fuzzyMatchRecursive(query, 0, 0, 0); - ordered.insert(m.begin(), m.end()); - } - - for (FuzzySetIterator it = ordered.begin(); it != ordered.end(); it++) - { - SearchMatch match = it->second->fuzzyMatchData(query, this); - result.push_back(match); - - if (it->first != match.weight) - { - LOG_ERROR("Weight between matching and meta data is different."); - } - } - - return result; -} - SearchIndex::SearchNode::FuzzyMap SearchIndex::SearchNode::fuzzyMatchRecursive( const std::string& query, size_t pos, size_t weight, size_t size) const { FuzzyMap result; - - size_t length = query.size(); - if (pos == length) - { - return result; - } - std::pair p = fuzzyMatch(query, pos, size); pos = p.first; weight += p.second; - if (pos == length) + if (pos == query.size()) { result.emplace(weight, this); return result; @@ -244,6 +254,11 @@ std::pair SearchIndex::SearchNode::fuzzyMatch( size_t ql = query.size(); size_t ml = m_name.size(); + if (!query.size()) + { + return std::pair(pos, weight); + } + if (query[pos] == ':') { pos++; @@ -311,6 +326,11 @@ SearchIndex::SearchMatch SearchIndex::SearchNode::fuzzyMatchData(const std::stri size_t size = 0; std::deque nodes = getNodesToParent(parent); + if (!nodes.size()) + { + nodes.push_back(this); + } + for (const SearchNode* node : nodes) { std::pair p = node->fuzzyMatch(query, pos, size, &data.indices); @@ -350,6 +370,26 @@ std::deque SearchIndex::SearchNode::getNodesToPa } +std::vector SearchIndex::getMatches( + const SearchIndex::SearchResults& searchResults, + const std::string& query +){ + std::vector result; + + for (SearchResultsIterator it = searchResults.begin(); it != searchResults.end(); it++) + { + SearchMatch match = it->node->fuzzyMatchData(query, it->parent); + result.push_back(match); + + if (it->weight != match.weight) + { + LOG_ERROR("Weight between matching and meta data is different."); + } + } + + return result; +} + void SearchIndex::logMatches(const std::vector& matches, const std::string& query) { std::stringstream ss; @@ -412,22 +452,14 @@ SearchIndex::SearchNode* SearchIndex::getNode(const std::string& fullName) const return nullptr; } -std::vector SearchIndex::findFuzzyMatches(const std::string& query) const +SearchIndex::SearchResults SearchIndex::runFuzzySearch(const std::string& query) const { - std::deque names = utility::split(query, '\"'); + return m_root.runFuzzySearch(query, true); +} - if (names.size() == 3 && names.at(0).size() == 0) - { - SearchNode* node = getNode(names.at(1)); - if (!node) - { - LOG_ERROR_STREAM(<< "Couldn't find node with name " << names.at(1) << " in the SearchIndex."); - } - - return node->findFuzzyMatches(names.at(2)); - } - - return m_root.findFuzzyMatches(query); +std::vector SearchIndex::runFuzzySearchAndGetMatches(const std::string& query) const +{ + return getMatches(runFuzzySearch(query), query); } const std::string SearchIndex::DELIMITER = "::"; diff --git a/src/lib/data/SearchIndex.h b/src/lib/data/SearchIndex.h index a9953f3d..4cdcd98e 100644 --- a/src/lib/data/SearchIndex.h +++ b/src/lib/data/SearchIndex.h @@ -16,6 +16,21 @@ class SearchIndex public: class SearchNode; + struct SearchResult + { + SearchResult(); + SearchResult(size_t weight, const SearchIndex::SearchNode* node, const SearchIndex::SearchNode* parent); + + bool operator()(const SearchResult& lhs, const SearchResult& rhs) const; + + size_t weight; + const SearchIndex::SearchNode* node; + const SearchIndex::SearchNode* parent; + }; + + typedef std::set SearchResults; + typedef SearchResults::const_iterator SearchResultsIterator; + struct SearchMatch { void print(std::ostream& ostream) const; @@ -34,10 +49,6 @@ public: typedef std::multimap FuzzyMap; typedef FuzzyMap::const_iterator FuzzyMapIterator; - typedef std::pair FuzzySetPair; - typedef std::multiset FuzzySet; - typedef FuzzySet::const_iterator FuzzySetIterator; - SearchNode(SearchNode* parent, const std::string& name, Id nameId); ~SearchNode(); @@ -53,11 +64,14 @@ public: SearchNode* getParent() const; std::deque getParentsWithoutTokenId(); + const std::set>& getChildren() const; + + SearchResults runFuzzySearch(const std::string& query, bool recursive) const; + private: // Accessed by SearchIndex std::shared_ptr addNodeRecursive(std::deque* nameIds, const Dictionary& dictionary); std::shared_ptr getNodeRecursive(std::deque* nameIds) const; - std::vector findFuzzyMatches(const std::string& query) const; friend class SearchIndex; @@ -78,7 +92,9 @@ public: const Id m_nameId; }; - static void logMatches(const std::vector& matches, const std::string& query); + static std::vector getMatches(const SearchResults& searchResults, const std::string& query); + + static void logMatches(const std::vector& matches, const std::string& query); SearchIndex(); virtual ~SearchIndex(); @@ -91,7 +107,8 @@ public: SearchNode* addNode(std::vector nameHierarchy); SearchNode* getNode(const std::string& fullName) const; - std::vector findFuzzyMatches(const std::string& query) const; + SearchResults runFuzzySearch(const std::string& query) const; + std::vector runFuzzySearchAndGetMatches(const std::string& query) const; static const std::string DELIMITER; diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index a15b2b4e..057b6346 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -1,10 +1,12 @@ #include "data/Storage.h" +#include "utility/logging/logging.h" #include "utility/utilityString.h" #include "data/graph/filter/GraphFilterConductor.h" #include "data/graph/token_component/TokenComponentConst.h" #include "data/graph/token_component/TokenComponentDataType.h" +#include "data/graph/token_component/TokenComponentName.h" #include "data/graph/token_component/TokenComponentStatic.h" #include "data/graph/SubGraph.h" #include "data/location/TokenLocation.h" @@ -17,11 +19,13 @@ #include "data/query/QueryCommand.h" #include "data/query/QueryTree.h" #include "data/type/DataType.h" -#include "utility/logging/logging.h" Storage::Storage() { - initSearchIndex(); + for (const std::pair& p : QueryCommand::getCommandTypeMap()) + { + m_filterIndex.addNode(std::vector({ p.first })); + } } Storage::~Storage() @@ -32,9 +36,7 @@ void Storage::clear() { m_graph.clear(); m_locationCollection.clear(); - - m_index.clear(); - initSearchIndex(); + m_tokenIndex.clear(); } void Storage::logGraph() const @@ -323,7 +325,7 @@ Id Storage::onTypeUsageParsed(const ParseTypeUsage& type, const ParseFunction& f Id Storage::getIdForNodeWithName(const std::string& fullName) const { - SearchIndex::SearchNode* node = m_index.getNode(fullName); + SearchIndex::SearchNode* node = m_tokenIndex.getNode(fullName); if (node) { return node->getFirstTokenId(); @@ -350,10 +352,30 @@ std::string Storage::getNameForNodeWithId(Id id) const } } -std::vector Storage::getAutocompletionMatches(const std::string& query) const +std::vector Storage::getAutocompletionMatches( + const std::string& query, const std::string& word) const { - std::vector matches = m_index.findFuzzyMatches(query); - SearchIndex::logMatches(matches, query); + SearchIndex::SearchResults tokenResults; + + bool usedSubquery = false; + if (query.size()) + { + usedSubquery = getSubQuerySearchResults(query, word, &tokenResults); + } + + if (!usedSubquery && word.size()) + { + tokenResults = m_tokenIndex.runFuzzySearch(word); + } + + if (word.size()) + { + SearchIndex::SearchResults filterResults = m_filterIndex.runFuzzySearch(word); + tokenResults.insert(filterResults.begin(), filterResults.end()); + } + + std::vector matches = SearchIndex::getMatches(tokenResults, word); + SearchIndex::logMatches(matches, word); return matches; } @@ -411,7 +433,7 @@ std::vector Storage::getActiveTokenIdsForId(Id tokenId, Id& declarationId) c ret.push_back(node->getId()); } else - { + { node = dynamic_cast(token); declarationId = node->getId(); } @@ -561,22 +583,12 @@ const TokenLocationCollection& Storage::getTokenLocationCollection() const const SearchIndex& Storage::getSearchIndex() const { - return m_index; -} - -void Storage::initSearchIndex() -{ - for (const std::pair& p : QueryCommand::getCommandTypeMap()) - { - std::vector nodeHierarchy; - nodeHierarchy.push_back(p.first); - m_index.addNode(nodeHierarchy); - } + return m_tokenIndex; } Node* Storage::addNodeHierarchy(Node::NodeType type, std::vector nameHierarchy) { - SearchIndex::SearchNode* searchNode = m_index.addNode(nameHierarchy); + SearchIndex::SearchNode* searchNode = m_tokenIndex.addNode(nameHierarchy); if (!searchNode) { LOG_ERROR("No SearchNode"); @@ -588,7 +600,7 @@ Node* Storage::addNodeHierarchy(Node::NodeType type, std::vector na Node* Storage::addNodeHierarchyWithDistinctSignature(Node::NodeType type, const ParseFunction& function) { - SearchIndex::SearchNode* searchNode = m_index.addNode(function.nameHierarchy); + SearchIndex::SearchNode* searchNode = m_tokenIndex.addNode(function.nameHierarchy); if (!searchNode) { LOG_ERROR("No SearchNode"); @@ -596,7 +608,7 @@ Node* Storage::addNodeHierarchyWithDistinctSignature(Node::NodeType type, const } // TODO: Instead of saving the whole signature string, the signature should be just a set of wordIds. - Id signatureId = m_index.getWordId(ParserClient::functionSignatureStr(function)); + Id signatureId = m_tokenIndex.getWordId(ParserClient::functionSignatureStr(function)); std::shared_ptr signature = std::make_shared(signatureId); return m_graph.createNodeHierarchyWithDistinctSignature(type, searchNode, signature); @@ -713,6 +725,73 @@ TokenLocation* Storage::addTokenLocation(Token* token, const ParseLocation& loc, return location; } +bool Storage::getSubQuerySearchResults( + const std::string& query, + const std::string& word, + SearchIndex::SearchResults* results +) const { + std::string q = query; + bool returnChilds = false; + + if (QueryOperator::getOperatorType(q.back()) == QueryOperator::OPERATOR_SUB) + { + q.pop_back(); + } + else if (QueryOperator::getOperatorType(q.back()) == QueryOperator::OPERATOR_HAS) + { + q.pop_back(); + returnChilds = true; + } + else if (QueryOperator::getOperatorType(q.back()) != QueryOperator::OPERATOR_NONE) + { + return false; + } + + QueryTree tree(q); + if (!tree.isValid()) + { + return false; + } + + SubGraph graph; + GraphFilterConductor conductor; + conductor.filter(&tree, &m_graph, &graph); + + std::vector searchNodes; + graph.forEachNode( + [&searchNodes](Node* node) + { + const TokenComponentName* nameComponent = node->getTokenComponentName(); + if (nameComponent) + { + searchNodes.push_back(nameComponent->getSearchNode()); + } + } + ); + + for (const SearchIndex::SearchNode* node : searchNodes) + { + if (word.size()) + { + SearchIndex::SearchResults res = node->runFuzzySearch(word, returnChilds); + results->insert(res.begin(), res.end()); + } + else if (returnChilds) + { + for (const std::shared_ptr& child : node->getChildren()) + { + results->insert(SearchIndex::SearchResult(0, child.get(), child.get())); + } + } + else if (searchNodes.size() > 1) + { + results->insert(SearchIndex::SearchResult(0, node, node)); + } + } + + return true; +} + void Storage::log(std::string type, std::string str, const ParseLocation& location) const { LOG_INFO_STREAM( diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index 3d548389..6f6b94dd 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -71,7 +71,8 @@ public: // GraphAccess implementation virtual Id getIdForNodeWithName(const std::string& fullName) const; virtual std::string getNameForNodeWithId(Id id) const; - virtual std::vector getAutocompletionMatches(const std::string& query) const; + virtual std::vector getAutocompletionMatches( + const std::string& query, const std::string& word) const; virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const; @@ -93,8 +94,6 @@ protected: const SearchIndex& getSearchIndex() const; private: - void initSearchIndex(); - Node* addNodeHierarchy(Node::NodeType type, std::vector nameHierarchy); Node* addNodeHierarchy(Node::NodeType type, const std::string& fullName); @@ -110,11 +109,16 @@ private: Edge* addTypeEdge(Node* node, Edge::EdgeType edgeType, const ParseTypeUsage& typeUsage); TokenLocation* addTokenLocation(Token* token, const ParseLocation& location, bool isScope = false); + bool getSubQuerySearchResults( + const std::string& query, const std::string& word, SearchIndex::SearchResults* results) const; + void log(std::string type, std::string str, const ParseLocation& location) const; StorageGraph m_graph; TokenLocationCollection m_locationCollection; - SearchIndex m_index; + + SearchIndex m_tokenIndex; + SearchIndex m_filterIndex; }; #endif // STORAGE_H diff --git a/src/lib/data/access/GraphAccess.h b/src/lib/data/access/GraphAccess.h index 003cbf10..a31d4eae 100644 --- a/src/lib/data/access/GraphAccess.h +++ b/src/lib/data/access/GraphAccess.h @@ -16,7 +16,8 @@ public: virtual Id getIdForNodeWithName(const std::string& name) const = 0; virtual std::string getNameForNodeWithId(Id id) const = 0; - virtual std::vector getAutocompletionMatches(const std::string& query) const = 0; + virtual std::vector getAutocompletionMatches( + const std::string& query, const std::string& word) const = 0; virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const = 0; diff --git a/src/lib/data/access/GraphAccessProxy.cpp b/src/lib/data/access/GraphAccessProxy.cpp index 77318df8..05ccecf9 100644 --- a/src/lib/data/access/GraphAccessProxy.cpp +++ b/src/lib/data/access/GraphAccessProxy.cpp @@ -47,11 +47,13 @@ std::string GraphAccessProxy::getNameForNodeWithId(Id id) const return ""; } -std::vector GraphAccessProxy::getAutocompletionMatches(const std::string& query) const -{ +std::vector GraphAccessProxy::getAutocompletionMatches( + const std::string& query, + const std::string& word +) const { if (hasSubject()) { - return m_subject->getAutocompletionMatches(query); + return m_subject->getAutocompletionMatches(query, word); } return std::vector(); diff --git a/src/lib/data/access/GraphAccessProxy.h b/src/lib/data/access/GraphAccessProxy.h index 3c9893cc..9157f442 100644 --- a/src/lib/data/access/GraphAccessProxy.h +++ b/src/lib/data/access/GraphAccessProxy.h @@ -15,7 +15,8 @@ public: // GraphAccess implementation virtual Id getIdForNodeWithName(const std::string& name) const; virtual std::string getNameForNodeWithId(Id id) const; - virtual std::vector getAutocompletionMatches(const std::string& query) const; + virtual std::vector getAutocompletionMatches( + const std::string& query, const std::string& word) const; virtual std::shared_ptr getGraphForActiveTokenIds(const std::vector& tokenIds) const; diff --git a/src/lib/data/graph/FilterableGraph.cpp b/src/lib/data/graph/FilterableGraph.cpp index 9e66ce43..282d6e4b 100644 --- a/src/lib/data/graph/FilterableGraph.cpp +++ b/src/lib/data/graph/FilterableGraph.cpp @@ -11,6 +11,11 @@ FilterableGraph::~FilterableGraph() { } +size_t FilterableGraph::size() const +{ + return getNodeCount() + getEdgeCount(); +} + Token* FilterableGraph::getTokenById(Id id) const { Token* token = getNodeById(id); diff --git a/src/lib/data/graph/FilterableGraph.h b/src/lib/data/graph/FilterableGraph.h index e1c51f60..1808aacb 100644 --- a/src/lib/data/graph/FilterableGraph.h +++ b/src/lib/data/graph/FilterableGraph.h @@ -34,6 +34,8 @@ public: virtual Node* getNodeById(Id id) const = 0; virtual Edge* getEdgeById(Id id) const = 0; + size_t size() const; + Token* getTokenById(Id id) const; void print(std::ostream& ostream) const; diff --git a/src/lib/data/graph/Node.cpp b/src/lib/data/graph/Node.cpp index aa9cfe39..9ca37efb 100644 --- a/src/lib/data/graph/Node.cpp +++ b/src/lib/data/graph/Node.cpp @@ -64,6 +64,16 @@ std::string Node::getFullName() const return m_nameComponent->getFullName(); } +const TokenComponentName* Node::getTokenComponentName() const +{ + if (m_nameComponent) + { + return m_nameComponent.get(); + } + + return nullptr; +} + const std::vector& Node::getEdges() const { return m_edges; diff --git a/src/lib/data/graph/Node.h b/src/lib/data/graph/Node.h index 8f0cc580..605dff6e 100644 --- a/src/lib/data/graph/Node.h +++ b/src/lib/data/graph/Node.h @@ -47,6 +47,7 @@ public: std::string getName() const; std::string getFullName() const; + const TokenComponentName* getTokenComponentName() const; const std::vector& getEdges() const; diff --git a/src/lib/data/graph/SubGraph.cpp b/src/lib/data/graph/SubGraph.cpp index ca12f582..1bdd1a49 100644 --- a/src/lib/data/graph/SubGraph.cpp +++ b/src/lib/data/graph/SubGraph.cpp @@ -91,6 +91,16 @@ Edge* SubGraph::getEdgeById(Id id) const return nullptr; } +const std::map& SubGraph::getNodes() const +{ + return m_nodes; +} + +const std::map& SubGraph::getEdges() const +{ + return m_edges; +} + std::vector SubGraph::getTokenIds() const { std::vector ids; diff --git a/src/lib/data/graph/SubGraph.h b/src/lib/data/graph/SubGraph.h index 65ecb2ab..757e94a6 100644 --- a/src/lib/data/graph/SubGraph.h +++ b/src/lib/data/graph/SubGraph.h @@ -37,6 +37,9 @@ public: virtual Node* getNodeById(Id id) const; virtual Edge* getEdgeById(Id id) const; + const std::map& getNodes() const; + const std::map& getEdges() const; + std::vector getTokenIds() const; void subtract(const SubGraph& other); diff --git a/src/lib/data/graph/token_component/TokenComponentName.cpp b/src/lib/data/graph/token_component/TokenComponentName.cpp index 0ebd12be..8978e8cc 100644 --- a/src/lib/data/graph/token_component/TokenComponentName.cpp +++ b/src/lib/data/graph/token_component/TokenComponentName.cpp @@ -40,6 +40,10 @@ std::string TokenComponentNameReferenced::getFullName() const return m_searchNode->getFullName(); } +const SearchIndex::SearchNode* TokenComponentNameReferenced::getSearchNode() const +{ + return m_searchNode; +} TokenComponentNameCached::TokenComponentNameCached(const std::string& fullName) : m_fullName(fullName) @@ -64,3 +68,8 @@ std::string TokenComponentNameCached::getFullName() const { return m_fullName; } + +const SearchIndex::SearchNode* TokenComponentNameCached::getSearchNode() const +{ + return nullptr; +} diff --git a/src/lib/data/graph/token_component/TokenComponentName.h b/src/lib/data/graph/token_component/TokenComponentName.h index b581b39f..e6514f33 100644 --- a/src/lib/data/graph/token_component/TokenComponentName.h +++ b/src/lib/data/graph/token_component/TokenComponentName.h @@ -17,6 +17,8 @@ public: virtual std::string getName() const = 0; virtual std::string getFullName() const = 0; + + virtual const SearchIndex::SearchNode* getSearchNode() const = 0; }; @@ -32,6 +34,8 @@ public: virtual std::string getName() const; virtual std::string getFullName() const; + virtual const SearchIndex::SearchNode* getSearchNode() const; + private: const SearchIndex::SearchNode* m_searchNode; }; @@ -49,6 +53,8 @@ public: virtual std::string getName() const; virtual std::string getFullName() const; + virtual const SearchIndex::SearchNode* getSearchNode() const; + private: const std::string m_fullName; }; diff --git a/src/lib/data/query/QueryOperator.cpp b/src/lib/data/query/QueryOperator.cpp index d778bfd4..a9d16f63 100644 --- a/src/lib/data/query/QueryOperator.cpp +++ b/src/lib/data/query/QueryOperator.cpp @@ -15,8 +15,8 @@ const std::map& QueryOperator::getOperatorTyp operatorMap.emplace(' ', OPERATOR_NONE); operatorMap.emplace('!', OPERATOR_NOT); - operatorMap.emplace('.', OPERATOR_HAS); - operatorMap.emplace('>', OPERATOR_SUB); + operatorMap.emplace('>', OPERATOR_HAS); + operatorMap.emplace('.', OPERATOR_SUB); operatorMap.emplace('&', OPERATOR_AND); operatorMap.emplace('|', OPERATOR_OR); diff --git a/src/lib/utility/messaging/type/MessageSearchAutocomplete.h b/src/lib/utility/messaging/type/MessageSearchAutocomplete.h index 53e31a78..00106034 100644 --- a/src/lib/utility/messaging/type/MessageSearchAutocomplete.h +++ b/src/lib/utility/messaging/type/MessageSearchAutocomplete.h @@ -7,8 +7,9 @@ class MessageSearchAutocomplete: public Message { public: - MessageSearchAutocomplete(const std::string& query) + MessageSearchAutocomplete(const std::string& query, const std::string& word) : query(query) + , word(word) { } @@ -18,6 +19,7 @@ public: } const std::string query; + const std::string word; }; #endif // MESSAGE_SEARCH_AUTOCOMPLETE_H diff --git a/src/lib/utility/utilityString.cpp b/src/lib/utility/utilityString.cpp index 1d7738da..a64f7c08 100644 --- a/src/lib/utility/utilityString.cpp +++ b/src/lib/utility/utilityString.cpp @@ -120,7 +120,7 @@ namespace utility { if (a.size() == b.size()) { - for (int i = 0; i < a.size(); i++) + for (size_t i = 0; i < a.size(); i++) { if (tolower(a[i]) != tolower(b[i])) { @@ -131,4 +131,22 @@ namespace utility } return false; } + + std::string replace(std::string str, const std::string& from, const std::string& to) + { + size_t pos = 0; + + if (from.size() == 0) + { + return str; + } + + while ((pos = str.find(from, pos)) != std::string::npos) + { + str.replace(pos, from.length(), to); + pos += to.length(); + } + + return str; + } } diff --git a/src/lib/utility/utilityString.h b/src/lib/utility/utilityString.h index fa635393..53892c18 100644 --- a/src/lib/utility/utilityString.h +++ b/src/lib/utility/utilityString.h @@ -35,6 +35,8 @@ namespace utility bool equalsCaseInsensitive(const std::string& a, const std::string& b); + std::string replace(std::string str, const std::string& from, const std::string& to); + template ContainerType split(const std::string& str, const std::string& delimiter) diff --git a/src/test/GraphFilterConductorTestSuite.h b/src/test/GraphFilterConductorTestSuite.h index 856d605e..31e7f241 100644 --- a/src/test/GraphFilterConductorTestSuite.h +++ b/src/test/GraphFilterConductorTestSuite.h @@ -108,7 +108,7 @@ public: void test_operator_sub() { TS_ASSERT_EQUALS( - printedFilteredTestGraph("'class'>'base'"), + printedFilteredTestGraph("'class'.'base'"), "1 nodes: class:A\n" "0 edges:\n" @@ -118,7 +118,7 @@ public: void test_operator_has() { TS_ASSERT_EQUALS( - printedFilteredTestGraph("\"A\".'field'"), + printedFilteredTestGraph("\"A\">'field'"), "1 nodes: field:A::count\n" "0 edges:\n" @@ -138,7 +138,7 @@ public: void test_operator_group() { TS_ASSERT_EQUALS( - printedFilteredTestGraph("('static'|'const')>'public'"), + printedFilteredTestGraph("('static'|'const').'public'"), "1 nodes: method:A::getCount\n" "0 edges:\n" @@ -165,7 +165,7 @@ private: { createTestStorage(); - std::vector matches = m_storage->getAutocompletionMatches(name); + std::vector matches = m_storage->getAutocompletionMatches("", name); if (matches.size() && matches[0].fullName == name) { return matches[0].tokenIds; diff --git a/src/test/QueryTreeTestSuite.h b/src/test/QueryTreeTestSuite.h index 10767868..a80bbed5 100644 --- a/src/test/QueryTreeTestSuite.h +++ b/src/test/QueryTreeTestSuite.h @@ -75,7 +75,7 @@ public: "A \" INVALID\n" " \"A\"\n" - "> IMPLICIT\n" + ". IMPLICIT\n" " \"\"\n" ); } @@ -276,7 +276,7 @@ public: "\"A\" ( \"B\" )\n" " \"A\"\n" - "> IMPLICIT\n" + ". IMPLICIT\n" " (\"B\")\n" ); } @@ -298,13 +298,13 @@ public: void test_operator_precedence_has_before_sub() { TS_ASSERT_EQUALS( - printedQueryTree("'namespace'.'class'>'base'"), + printedQueryTree("'namespace'>'class'.'base'"), - "'namespace' . 'class' > 'base'\n" + "'namespace' > 'class' . 'base'\n" " 'namespace'\n" - " .\n" + " >\n" " 'class'\n" - ">\n" + ".\n" " 'base'\n" ); } diff --git a/src/test/SearchIndexTestSuite.h b/src/test/SearchIndexTestSuite.h index 16b6b917..4e0cbb0c 100644 --- a/src/test/SearchIndexTestSuite.h +++ b/src/test/SearchIndexTestSuite.h @@ -6,14 +6,6 @@ class SearchIndexTestSuite : public CxxTest::TestSuite { public: - - // void clear(); - - // SearchNode* addNode(const std::string& fullName); - // SearchNode* getNode(const std::string& fullName) const; - - // std::vector findFuzzyMatches(const std::string& query) const; - void test_add_node() { SearchIndex index; @@ -100,7 +92,7 @@ public: index.addNode(utility::splitToVector("math", "::")); index.addNode(utility::splitToVector("string", "::")); - std::vector matches = index.findFuzzyMatches("u"); + std::vector matches = index.runFuzzySearchAndGetMatches("u"); TS_ASSERT_EQUALS(1, matches.size()); TS_ASSERT_EQUALS("util", matches[0].fullName); @@ -108,8 +100,15 @@ public: TS_ASSERT_EQUALS(1, matches[0].indices.size()); TS_ASSERT_EQUALS(0, matches[0].indices[0]); - matches = index.findFuzzyMatches(""); - TS_ASSERT_EQUALS(0, matches.size()); + matches = index.runFuzzySearchAndGetMatches(""); + TS_ASSERT_EQUALS(3, matches.size()); + TS_ASSERT_EQUALS("math", matches[0].fullName); + TS_ASSERT_EQUALS("string", matches[1].fullName); + TS_ASSERT_EQUALS("util", matches[2].fullName); + + TS_ASSERT_EQUALS(0, matches[0].weight); + TS_ASSERT_EQUALS(0, matches[1].weight); + TS_ASSERT_EQUALS(0, matches[2].weight); } void test_fuzzy_matching_is_case_insensitive() @@ -118,13 +117,13 @@ public: index.addNode(utility::splitToVector("util", "::")); index.addNode(utility::splitToVector("MATH", "::")); - std::vector matches = index.findFuzzyMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("MATH", matches[0].fullName); TS_ASSERT_EQUALS("util", matches[1].fullName); - matches = index.findFuzzyMatches("T"); + matches = index.runFuzzySearchAndGetMatches("T"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("MATH", matches[0].fullName); @@ -138,7 +137,7 @@ public: index.addNode(utility::splitToVector("math", "::")); index.addNode(utility::splitToVector("string", "::")); - std::vector matches = index.findFuzzyMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(3, matches.size()); TS_ASSERT_EQUALS("string", matches[0].fullName); @@ -161,7 +160,7 @@ public: index.addNode(utility::splitToVector("uTil", "::")); index.addNode(utility::splitToVector("string", "::")); - std::vector matches = index.findFuzzyMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("uTil", matches[0].fullName); @@ -174,7 +173,7 @@ public: index.addNode(utility::splitToVector("oaabbcc", "::")); index.addNode(utility::splitToVector("ocbaabc", "::")); - std::vector matches = index.findFuzzyMatches("abc"); + std::vector matches = index.runFuzzySearchAndGetMatches("abc"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("ocbaabc", matches[0].fullName); @@ -188,12 +187,12 @@ public: index.addNode(utility::splitToVector("util::math::floor", "::")); index.addNode(utility::splitToVector("util::string::concat", "::")); - std::vector matches = index.findFuzzyMatches("t"); + std::vector matches = index.runFuzzySearchAndGetMatches("t"); TS_ASSERT_EQUALS(1, matches.size()); TS_ASSERT_EQUALS("util", matches[0].fullName); - matches = index.findFuzzyMatches("uml"); + matches = index.runFuzzySearchAndGetMatches("uml"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("util::math::floor", matches[0].fullName); @@ -207,13 +206,13 @@ public: index.addNode(utility::splitToVector("util::math::floor", "::")); index.addNode(utility::splitToVector("util::string::concat", "::")); - std::vector matches = index.findFuzzyMatches("u:i"); + std::vector matches = index.runFuzzySearchAndGetMatches("u:i"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("util::string", matches[0].fullName); TS_ASSERT_EQUALS("util::math::ceil", matches[1].fullName); - matches = index.findFuzzyMatches("u:t:i"); + matches = index.runFuzzySearchAndGetMatches("u:t:i"); TS_ASSERT_EQUALS(1, matches.size()); TS_ASSERT_EQUALS("util::math::ceil", matches[0].fullName); @@ -225,32 +224,10 @@ public: index.addNode(utility::splitToVector("abc::dfe::ghi", "::")); index.addNode(utility::splitToVector("abc::hgi", "::")); - std::vector matches = index.findFuzzyMatches("g"); + std::vector matches = index.runFuzzySearchAndGetMatches("g"); TS_ASSERT_EQUALS(2, matches.size()); TS_ASSERT_EQUALS("abc::dfe::ghi", matches[0].fullName); TS_ASSERT_EQUALS("abc::hgi", matches[1].fullName); } - - void test_fuzzy_matching_with_defined_start_node() - { - SearchIndex index; - index.addNode(utility::splitToVector("math::ceil", "::")); - index.addNode(utility::splitToVector("math::floor", "::")); - index.addNode(utility::splitToVector("string::concat", "::")); - - std::vector matches = index.findFuzzyMatches("\"math\"c"); - - TS_ASSERT_EQUALS(1, matches.size()); - TS_ASSERT_EQUALS("math::ceil", matches[0].fullName); - - matches = index.findFuzzyMatches("\"mathc"); - TS_ASSERT_EQUALS(0, matches.size()); - - matches = index.findFuzzyMatches("math\"c"); - TS_ASSERT_EQUALS(0, matches.size()); - - matches = index.findFuzzyMatches("\"mat\"h\"c"); - TS_ASSERT_EQUALS(0, matches.size()); - } }; diff --git a/src/test/UtilityStringTestSuite.h b/src/test/UtilityStringTestSuite.h index 87531c15..67a9e291 100644 --- a/src/test/UtilityStringTestSuite.h +++ b/src/test/UtilityStringTestSuite.h @@ -236,4 +236,14 @@ public: TS_ASSERT(!utility::equalsCaseInsensitive(foo, foo2)); } + void test_replace() + { + TS_ASSERT_EQUALS("fubar", utility::replace("foobar", "oo", "u")); + TS_ASSERT_EQUALS("fuuuubar", utility::replace("foobar", "o", "uu")); + TS_ASSERT_EQUALS("bar", utility::replace("foobar", "foo", "")); + TS_ASSERT_EQUALS("foobar", utility::replace("foobar", "", "i")); + TS_ASSERT_EQUALS("foobar", utility::replace("foobar", "", "")); + TS_ASSERT_EQUALS("", utility::replace("", "foo", "bar")); + TS_ASSERT_EQUALS("foobar", utility::replace("foobar", "ba", "ba")); + } };