diff --git a/src/lib/data/SearchIndex.cpp b/src/lib/data/SearchIndex.cpp index 06599565..29d008a8 100644 --- a/src/lib/data/SearchIndex.cpp +++ b/src/lib/data/SearchIndex.cpp @@ -48,11 +48,6 @@ SearchIndex::SearchNode::~SearchNode() { } -void SearchIndex::SearchNode::clear() -{ - m_nodes.clear(); -} - const std::string& SearchIndex::SearchNode::getName() const { return m_name; @@ -85,6 +80,11 @@ Id SearchIndex::SearchNode::getFirstTokenId() const return 0; } +const std::set& SearchIndex::SearchNode::getTokenIds() const +{ + return m_tokenIds; +} + void SearchIndex::SearchNode::addTokenId(Id tokenId) { m_tokenIds.insert(tokenId); @@ -166,17 +166,14 @@ std::vector SearchIndex::SearchNode::findFuzzyMatches( FuzzySet ordered(&fncomp); for (std::shared_ptr n: m_nodes) { - FuzzyMap m = n->fuzzyMatches(query, 0, 0, 0); + FuzzyMap m = n->fuzzyMatchRecursive(query, 0, 0, 0); ordered.insert(m.begin(), m.end()); } - std::stringstream ss; - ss << std::endl << ordered.size() << " matches for \"" << query << "\":" << std::endl; for (FuzzySetIterator it = ordered.begin(); it != ordered.end(); it++) { SearchMatch match = it->second->fuzzyMatchData(query, this); result.push_back(match); - match.print(ss); if (it->first != match.weight) { @@ -184,12 +181,10 @@ std::vector SearchIndex::SearchNode::findFuzzyMatches( } } - LOG_INFO(ss.str()); - return result; } -SearchIndex::SearchNode::FuzzyMap SearchIndex::SearchNode::fuzzyMatches( +SearchIndex::SearchNode::FuzzyMap SearchIndex::SearchNode::fuzzyMatchRecursive( const std::string& query, size_t pos, size_t weight, size_t size) const { FuzzyMap result; @@ -212,7 +207,7 @@ SearchIndex::SearchNode::FuzzyMap SearchIndex::SearchNode::fuzzyMatches( for (std::shared_ptr n: m_nodes) { - FuzzyMap m = n->fuzzyMatches(query, pos, weight, size + m_name.size() + SearchIndex::DELIMITER.size()); + FuzzyMap m = n->fuzzyMatchRecursive(query, pos, weight, size + m_name.size() + SearchIndex::DELIMITER.size()); result.insert(m.begin(), m.end()); } @@ -325,15 +320,9 @@ std::deque SearchIndex::SearchNode::getNodesToPa std::deque nodes; const SearchNode* node = this; - while (node->m_nameId) + while (node->m_nameId && node != parent) { nodes.push_front(node); - - if (node == parent) - { - break; - } - node = node->m_parent; } @@ -341,6 +330,17 @@ std::deque SearchIndex::SearchNode::getNodesToPa } +void SearchIndex::logMatches(const std::vector& matches, const std::string& query) +{ + std::stringstream ss; + ss << std::endl << matches.size() << " matches for \"" << query << "\":" << std::endl; + for (const SearchIndex::SearchMatch& match : matches) + { + match.print(ss); + } + LOG_INFO(ss.str()); +} + SearchIndex::SearchIndex() : m_root(nullptr, DELIMITER, 0) { @@ -352,7 +352,7 @@ SearchIndex::~SearchIndex() void SearchIndex::clear() { - m_root.clear(); + m_root.m_nodes.clear(); } SearchIndex::SearchNode* SearchIndex::addNode(const std::string& fullName) @@ -379,32 +379,22 @@ SearchIndex::SearchNode* SearchIndex::getNode(const std::string& fullName) const return nullptr; } -std::vector SearchIndex::findFuzzyMatches(const std::string& query) const +std::vector SearchIndex::findFuzzyMatches(const std::string& query) const { - std::vector matches; - std::vector pieces = utility::split>(query, '\"'); + std::vector names = utility::split>(query, '\"'); - if (pieces.size() == 3 && pieces[0].size() == 0) + if (names.size() == 3 && names[0].size() == 0) { - SearchNode* node = getNode(pieces[1]); + SearchNode* node = getNode(names[1]); if (!node) { - LOG_ERROR_STREAM(<< "Couldn't find node with name " << pieces[1] << " in the SearchIndex."); + LOG_ERROR_STREAM(<< "Couldn't find node with name " << names[1] << " in the SearchIndex."); } - matches = node->findFuzzyMatches(pieces[2]); - } - else - { - matches = m_root.findFuzzyMatches(query); + return node->findFuzzyMatches(names[2]); } - std::vector names; - for (const SearchMatch& match : matches) - { - names.push_back(match.node->getFullName()); - } - return names; + return m_root.findFuzzyMatches(query); } const std::string SearchIndex::DELIMITER = "::"; diff --git a/src/lib/data/SearchIndex.h b/src/lib/data/SearchIndex.h index b596ad2d..5299fd53 100644 --- a/src/lib/data/SearchIndex.h +++ b/src/lib/data/SearchIndex.h @@ -37,26 +37,27 @@ public: SearchNode(SearchNode* parent, const std::string& name, Id nameId); ~SearchNode(); - void clear(); - const std::string& getName() const; std::string getFullName() const; Id getNameId() const; Id getFirstTokenId() const; + const std::set& getTokenIds() const; void addTokenId(Id tokenId); SearchNode* getParent() const; std::deque getParentsWithoutTokenId(); + private: + // Accessed by SearchIndex std::shared_ptr addNodeRecursive(std::deque* nameIds); std::shared_ptr getNodeRecursive(std::deque* nameIds) const; - std::vector findFuzzyMatches(const std::string& query) const; - private: - FuzzyMap fuzzyMatches(const std::string& query, size_t pos, size_t weight, size_t size) const; + friend class SearchIndex; + + FuzzyMap fuzzyMatchRecursive(const std::string& query, size_t pos, size_t weight, size_t size) const; std::pair fuzzyMatch( const std::string query, size_t start, size_t size, std::vector* indices = nullptr) const; SearchMatch fuzzyMatchData(const std::string& query, const SearchNode* parent) const; @@ -73,6 +74,8 @@ public: const Id m_nameId; }; + static void logMatches(const std::vector& matches, const std::string& query); + SearchIndex(); virtual ~SearchIndex(); @@ -81,7 +84,7 @@ public: SearchNode* addNode(const std::string& fullName); SearchNode* getNode(const std::string& fullName) const; - std::vector findFuzzyMatches(const std::string& query) const; + std::vector findFuzzyMatches(const std::string& query) const; static const std::string DELIMITER; diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index 63246410..05543395 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -338,7 +338,13 @@ std::string Storage::getNameForNodeWithId(Id id) const std::vector Storage::getNamesForNodesWithNamePrefix(const std::string& prefix) const { - return m_index.findFuzzyMatches(prefix); + std::vector names; + std::vector matches = m_index.findFuzzyMatches(prefix); + for (const SearchIndex::SearchMatch& match : matches) + { + names.push_back(match.node->getFullName()); + } + return names; } std::vector Storage::getIdsOfNeighbours(const Id id) const @@ -526,7 +532,7 @@ std::vector Storage::getTokenIdsForQuery(std::string query) const LOG_INFO_STREAM(<< '\n' << tree << '\n' << outGraph); - m_index.findFuzzyMatches(query); + SearchIndex::logMatches(m_index.findFuzzyMatches(query), query); return outGraph.getTokenIds(); } diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 49f6f333..9329a30c 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -19,6 +19,7 @@ add_files( MessageQueueTestSuite.h QueryTreeTestSuite.h SettingsTestSuite.h + SearchIndexTestSuite.h StorageTestSuite.h StorageGraphTestSuite.h TextAccessTestSuite.h diff --git a/src/test/SearchIndexTestSuite.h b/src/test/SearchIndexTestSuite.h new file mode 100644 index 00000000..1d86581d --- /dev/null +++ b/src/test/SearchIndexTestSuite.h @@ -0,0 +1,255 @@ +#include "cxxtest/TestSuite.h" + +#include "data/SearchIndex.h" + +class SearchIndexTestSuite : public CxxTest::TestSuite +{ +public: + + // void clear(); + + // SearchNode* addNode(const std::string& fullName); + // SearchNode* getNode(const std::string& fullName) const; + + // std::vector findFuzzyMatches(const std::string& query) const; + + void test_add_node() + { + SearchIndex index; + SearchIndex::SearchNode* node = index.addNode("util"); + + TS_ASSERT(node); + TS_ASSERT_EQUALS("util", node->getName()); + TS_ASSERT_EQUALS("util", node->getFullName()); + + TS_ASSERT(node->getNameId()); + TS_ASSERT(!node->getFirstTokenId()); + TS_ASSERT(!node->getParent()); + } + + void test_get_node() + { + SearchIndex index; + index.addNode("util"); + SearchIndex::SearchNode* node = index.getNode("util"); + + TS_ASSERT(node); + TS_ASSERT_EQUALS("util", node->getName()); + TS_ASSERT_EQUALS("util", node->getFullName()); + + TS_ASSERT(node->getNameId()); + TS_ASSERT(!node->getFirstTokenId()); + TS_ASSERT(!node->getParent()); + + node = index.getNode("math"); + TS_ASSERT(!node); + } + + void test_add_hierarchy_node() + { + SearchIndex index; + SearchIndex::SearchNode* node = index.addNode("util::math::pow"); + + TS_ASSERT(node); + TS_ASSERT_EQUALS("pow", node->getName()); + TS_ASSERT_EQUALS("util::math::pow", node->getFullName()); + + TS_ASSERT(node->getNameId()); + TS_ASSERT(!node->getFirstTokenId()); + + TS_ASSERT(node->getParent()); + TS_ASSERT_EQUALS("math", node->getParent()->getName()); + + TS_ASSERT(node->getParent()->getParent()); + TS_ASSERT_EQUALS("util", node->getParent()->getParent()->getName()); + } + + void test_reuse_hierarchy_node() + { + SearchIndex index; + SearchIndex::SearchNode* node = index.addNode("math::pow"); + SearchIndex::SearchNode* node2 = index.addNode("math::floor"); + + TS_ASSERT(node); + TS_ASSERT(node2); + + TS_ASSERT_EQUALS("pow", node->getName()); + TS_ASSERT_EQUALS("floor", node2->getName()); + + TS_ASSERT_EQUALS(node->getParent(), node2->getParent()); + } + + void test_clear() + { + SearchIndex index; + index.addNode("math"); + index.addNode("string"); + + TS_ASSERT(index.getNode("math")); + + index.clear(); + + TS_ASSERT(!index.getNode("math")); + } + + void test_fuzzy_matching() + { + SearchIndex index; + index.addNode("util"); + index.addNode("math"); + index.addNode("string"); + + std::vector matches = index.findFuzzyMatches("u"); + + TS_ASSERT_EQUALS(1, matches.size()); + TS_ASSERT_EQUALS("util", matches[0].node->getName()); + + TS_ASSERT_EQUALS(1, matches[0].indices.size()); + TS_ASSERT_EQUALS(0, matches[0].indices[0]); + + matches = index.findFuzzyMatches(""); + TS_ASSERT_EQUALS(0, matches.size()); + } + + void test_fuzzy_matching_is_case_insensitive() + { + SearchIndex index; + index.addNode("util"); + index.addNode("MATH"); + + std::vector matches = index.findFuzzyMatches("t"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("MATH", matches[0].node->getName()); + TS_ASSERT_EQUALS("util", matches[1].node->getName()); + + matches = index.findFuzzyMatches("T"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("MATH", matches[0].node->getName()); + TS_ASSERT_EQUALS("util", matches[1].node->getName()); + } + + void test_fuzzy_matching_wheighs_by_distance_and_alphabet() + { + SearchIndex index; + index.addNode("util"); + index.addNode("math"); + index.addNode("string"); + + std::vector matches = index.findFuzzyMatches("t"); + + TS_ASSERT_EQUALS(3, matches.size()); + TS_ASSERT_EQUALS("string", matches[0].node->getName()); + TS_ASSERT_EQUALS("util", matches[1].node->getName()); + TS_ASSERT_EQUALS("math", matches[2].node->getName()); + + TS_ASSERT_EQUALS(1, matches[0].indices.size()); + TS_ASSERT_EQUALS(1, matches[0].indices[0]); + + TS_ASSERT_EQUALS(1, matches[1].indices.size()); + TS_ASSERT_EQUALS(1, matches[1].indices[0]); + + TS_ASSERT_EQUALS(1, matches[2].indices.size()); + TS_ASSERT_EQUALS(2, matches[2].indices[0]); + } + + void test_fuzzy_matching_wheighs_higher_by_uppercase() + { + SearchIndex index; + index.addNode("uTil"); + index.addNode("string"); + + std::vector matches = index.findFuzzyMatches("t"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("uTil", matches[0].node->getName()); + TS_ASSERT_EQUALS("string", matches[1].node->getName()); + } + + void test_fuzzy_matching_wheighs_higher_on_consecutive_letters() + { + SearchIndex index; + index.addNode("oaabbcc"); + index.addNode("ocbaabc"); + + std::vector matches = index.findFuzzyMatches("abc"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("ocbaabc", matches[0].node->getName()); + TS_ASSERT_EQUALS("oaabbcc", matches[1].node->getName()); + } + + void test_fuzzy_matching_in_hierarchy() + { + SearchIndex index; + index.addNode("util::math::ceil"); + index.addNode("util::math::floor"); + index.addNode("util::string::concat"); + + std::vector matches = index.findFuzzyMatches("t"); + + TS_ASSERT_EQUALS(1, matches.size()); + TS_ASSERT_EQUALS("util", matches[0].node->getName()); + + matches = index.findFuzzyMatches("uml"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("floor", matches[0].node->getName()); + TS_ASSERT_EQUALS("ceil", matches[1].node->getName()); + } + + void test_fuzzy_matching_in_hierarchy_respects_collin() + { + SearchIndex index; + index.addNode("util::math::ceil"); + index.addNode("util::math::floor"); + index.addNode("util::string::concat"); + + std::vector matches = index.findFuzzyMatches("u:i"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("string", matches[0].node->getName()); + TS_ASSERT_EQUALS("ceil", matches[1].node->getName()); + + matches = index.findFuzzyMatches("u:t:i"); + + TS_ASSERT_EQUALS(1, matches.size()); + TS_ASSERT_EQUALS("ceil", matches[0].node->getName()); + } + + void test_fuzzy_matching_in_hierarchy_weighs_front_letters_higher() + { + SearchIndex index; + index.addNode("abc::dfe::ghi"); + index.addNode("abc::hgi"); + + std::vector matches = index.findFuzzyMatches("g"); + + TS_ASSERT_EQUALS(2, matches.size()); + TS_ASSERT_EQUALS("ghi", matches[0].node->getName()); + TS_ASSERT_EQUALS("hgi", matches[1].node->getName()); + } + + void test_fuzzy_matching_with_defined_start_node() + { + SearchIndex index; + index.addNode("math::ceil"); + index.addNode("math::floor"); + index.addNode("string::concat"); + + std::vector matches = index.findFuzzyMatches("\"math\"c"); + + TS_ASSERT_EQUALS(1, matches.size()); + TS_ASSERT_EQUALS("ceil", matches[0].node->getName()); + + matches = index.findFuzzyMatches("\"mathc"); + TS_ASSERT_EQUALS(0, matches.size()); + + matches = index.findFuzzyMatches("math\"c"); + TS_ASSERT_EQUALS(0, matches.size()); + + matches = index.findFuzzyMatches("\"mat\"h\"c"); + TS_ASSERT_EQUALS(0, matches.size()); + } +};