data: added SearchIndex for fuzzy name search and rewrote name handling in the Storage

This change stores each Token name in the separate singleton class Dictionary. The Dictionary gives each saved word an
Id and thereby avoids duplicated names. E.g if the constructor method "Graph::Graph" is stored then the word "Graph"
only appears once in memory.

The class SearchIndex is now responsible for the name hierarchy and is instantiated by the Storage. The SearchIndex
builds the name hierarchy using SearchNodes, each holding a Dictionary string reference of the name it holds. E.g if the
names "math::ceil" and "math::floor" are added to the SearchIndex then 3 nodes get created, the SearchNode "math" will
hold the two childs "ceil" and "floor".

The hierarchical graph creation functionality got split off from Graph into the new subclass StorageGraph. The
StorageGraph creates nodes with a passed SearchNode pointer of the name it represents in the SearchIndex. Thereby the
StorageGraph reuses the hierarchical information in the SearchIndex and can create nodes much quicker by avoiding node
searches and name comparisions.

The name information is now stored in the Nodes via the TokenComponentName class, which is subclassed into
TokenComponentNameReferenced and TokenComponentNameCached. The StorageClass creates nodes with the component
TokenComponentNameReferenced, which holds a pointer to the SearchNode instance holding the name. This allows for
retrieving the full name of the node, without using other Nodes int the Graph, which might not be present. If the Node
is copied then the component changes to a TokenComponentNameCached, which holds the full name as a string, so the
memory in the Storage doesn't have to be accessed anymore.

TokenComponentSignature is now only holding an Id of the signature string saved in the Dictionary, which speeds up the
signature comparison. A follow-up will change saving the whole signature as string to reusing the wordIds it is
consisting of.

Lastly the SearchIndex holds basic fuzzy search functionality. A passed query gets compared down the SearchNode
hierarchy as long as matches for each letter are found. Matches must contain all letters of the query. The search is
case-insensitive. If letters are found in front positions, next to each other or written in uppercase they are weighed
higher in the match ranking. The character ':' is also interpreted and found, although the '::' delimiter is not stored.

E.g. the query "m:l" used on the example above will return both "math::floor" and "math::ceil", but "floor" is ranked
higher because the 'l' appears closer to the start.

src/lib/CMakeLists.txt

@@ -87,6 +87,8 @@ add_files(
 	data/graph/token_component/TokenComponentConst.h
 	data/graph/token_component/TokenComponentDataType.cpp
 	data/graph/token_component/TokenComponentDataType.h
+	data/graph/token_component/TokenComponentName.cpp
+	data/graph/token_component/TokenComponentName.h
 	data/graph/token_component/TokenComponentSignature.cpp
 	data/graph/token_component/TokenComponentSignature.h
 	data/graph/token_component/TokenComponentStatic.cpp
@@ -100,6 +102,8 @@ add_files(
 	data/graph/Graph.h
 	data/graph/Node.cpp
 	data/graph/Node.h
+	data/graph/StorageGraph.cpp
+	data/graph/StorageGraph.h
 	data/graph/SubGraph.cpp
 	data/graph/SubGraph.h
 	data/graph/Token.cpp
@@ -154,8 +158,6 @@ add_files(
 	data/type/DataTypeQualifierList.cpp
 	data/type/DataTypeQualifierList.h
 
-	data/ElementIndex.cpp
-	data/ElementIndex.h
 	data/SearchIndex.cpp
 	data/SearchIndex.h
 	data/Storage.cpp
@@ -196,6 +198,8 @@ add_files(
 	utility/messaging/MessageQueue.cpp
 	utility/messaging/MessageQueue.h
 
+	utility/text/Dictionary.cpp
+	utility/text/Dictionary.h
 	utility/text/TextAccess.cpp
 	utility/text/TextAccess.h
 

src/lib/component/controller/SearchController.cpp

@@ -49,7 +49,7 @@ void SearchController::handleMessage(MessageFind* message)
 
 void SearchController::handleMessage(MessageFinishedParsing* message)
 {
-	getView()->setAutocompletionList(m_graphAccess->getNamesForNodesWithNamePrefix(""));
+	getView()->setAutocompletionList(m_graphAccess->getNamesForNodesWithNamePrefix(":"));
 }
 
 void SearchController::handleMessage(MessageRefresh* message)

src/lib/data/ElementIndex.cpp

@@ -1,9 +0,0 @@
-#include "data/ElementIndex.h"
-
-ElementIndex::ElementIndex()
-{
-}
-
-ElementIndex::~ElementIndex()
-{
-}

src/lib/data/ElementIndex.h

@@ -1,11 +0,0 @@
-#ifndef ELEMENT_INDEX_H
-#define ELEMENT_INDEX_H
-
-class ElementIndex
-{
-public:
-	ElementIndex();
-	virtual ~ElementIndex();
-};
-
-#endif // ELEMENT_INDEX_H

src/lib/data/SearchIndex.cpp

@@ -1,9 +1,410 @@
 #include "data/SearchIndex.h"
 
+#include <algorithm>
+#include <cctype>
+
+#include "utility/logging/logging.h"
+#include "utility/text/Dictionary.h"
+#include "utility/utilityString.h"
+
+namespace
+{
+	bool fncomp(const SearchIndex::SearchNode::FuzzySetPair& lhs, const SearchIndex::SearchNode::FuzzySetPair& rhs)
+	{
+		if (lhs.first != rhs.first)
+		{
+			return lhs.first > rhs.first;
+		}
+
+		return lhs.second->getFullName() < rhs.second->getFullName();
+	}
+}
+
+void SearchIndex::SearchMatch::print(std::ostream& ostream) const
+{
+	ostream << weight << '\t' << node->getFullName() << std::endl << '\t';
+	size_t i = 0;
+	for (size_t index : indices)
+	{
+		while (i < index)
+		{
+			i++;
+			ostream << ' ';
+		}
+		ostream << '^';
+		i++;
+	}
+	ostream << std::endl;
+}
+
+SearchIndex::SearchNode::SearchNode(SearchNode* parent, const std::string& name, Id nameId)
+	: m_parent(parent)
+	, m_name(name)
+	, m_nameId(nameId)
+{
+}
+
+SearchIndex::SearchNode::~SearchNode()
+{
+}
+
+void SearchIndex::SearchNode::clear()
+{
+	m_nodes.clear();
+}
+
+const std::string& SearchIndex::SearchNode::getName() const
+{
+	return m_name;
+}
+
+std::string SearchIndex::SearchNode::getFullName() const
+{
+	if (m_parent && m_parent->m_nameId)
+	{
+		return m_parent->getFullName() + DELIMITER + getName();
+	}
+	else
+	{
+		return getName();
+	}
+}
+
+Id SearchIndex::SearchNode::getNameId() const
+{
+	return m_nameId;
+}
+
+Id SearchIndex::SearchNode::getFirstTokenId() const
+{
+	if (m_tokenIds.size())
+	{
+		return *m_tokenIds.begin();
+	}
+
+	return 0;
+}
+
+void SearchIndex::SearchNode::addTokenId(Id tokenId)
+{
+	m_tokenIds.insert(tokenId);
+}
+
+SearchIndex::SearchNode* SearchIndex::SearchNode::getParent() const
+{
+	if (m_parent && m_parent->m_nameId)
+	{
+		return m_parent;
+	}
+	return nullptr;
+}
+
+std::deque<SearchIndex::SearchNode*> SearchIndex::SearchNode::getParentsWithoutTokenId()
+{
+	std::deque<SearchNode*> nodes;
+
+	SearchNode* node = this;
+	while (node->m_nameId && !node->m_tokenIds.size())
+	{
+		nodes.push_front(node);
+		node = node->m_parent;
+	}
+
+	return nodes;
+}
+
+std::shared_ptr<SearchIndex::SearchNode> SearchIndex::SearchNode::addNodeRecursive(std::deque<Id>* nameIds)
+{
+	Id nameId = nameIds->front();
+	nameIds->pop_front();
+
+	std::shared_ptr<SearchNode> node = getChildWithNameId(nameId);
+	if (!node)
+	{
+		node = std::make_shared<SearchNode>(this, Dictionary::getInstance()->getWord(nameId), nameId);
+		m_nodes.insert(node);
+	}
+
+	if (nameIds->size())
+	{
+		return node->addNodeRecursive(nameIds);
+	}
+
+	return node;
+}
+
+std::shared_ptr<SearchIndex::SearchNode> SearchIndex::SearchNode::getNodeRecursive(std::deque<Id>* nameIds) const
+{
+	Id nameId = nameIds->front();
+	nameIds->pop_front();
+
+	std::shared_ptr<SearchNode> node = getChildWithNameId(nameId);
+	if (node)
+	{
+		if (!nameIds->size())
+		{
+			return node;
+		}
+
+		return node->getNodeRecursive(nameIds);
+	}
+
+	return nullptr;
+}
+
+std::vector<SearchIndex::SearchMatch> SearchIndex::SearchNode::findFuzzyMatches(const std::string& query) const
+{
+	std::vector<SearchIndex::SearchMatch> result;
+
+	if (!query.size())
+	{
+		return result;
+	}
+
+	// TODO: Currently all matches are added to the ordered set and get compared by their fullName for alphabetical
+	// order. This should be avoided e.g. by only returning a subset of the best 100 matches in alphabetical order.
+	FuzzySet ordered(&fncomp);
+	for (std::shared_ptr<SearchNode> n: m_nodes)
+	{
+		FuzzyMap m = n->fuzzyMatches(query, 0, 0, 0);
+		ordered.insert(m.begin(), m.end());
+	}
+
+	std::stringstream ss;
+	ss << std::endl << ordered.size() << " matches for \"" << query << "\":" << std::endl;
+	for (FuzzySetIterator it = ordered.begin(); it != ordered.end(); it++)
+	{
+		SearchMatch match = it->second->fuzzyMatchData(query, this);
+		result.push_back(match);
+		match.print(ss);
+
+		if (it->first != match.weight)
+		{
+			LOG_ERROR("Weight between matching and meta data is different.");
+		}
+	}
+
+	LOG_INFO(ss.str());
+
+	return result;
+}
+
+SearchIndex::SearchNode::FuzzyMap SearchIndex::SearchNode::fuzzyMatches(
+	const std::string& query, size_t pos, size_t weight, size_t size) const
+{
+	FuzzyMap result;
+
+	size_t length = query.size();
+	if (pos == length)
+	{
+		return result;
+	}
+
+	std::pair<size_t, size_t> p = fuzzyMatch(query, pos, size);
+	pos = p.first;
+	weight += p.second;
+
+	if (pos == length)
+	{
+		result.emplace(weight, this);
+		return result;
+	}
+
+	for (std::shared_ptr<SearchNode> n: m_nodes)
+	{
+		FuzzyMap m = n->fuzzyMatches(query, pos, weight, size + m_name.size() + SearchIndex::DELIMITER.size());
+		result.insert(m.begin(), m.end());
+	}
+
+	return result;
+}
+
+std::pair<size_t, size_t> SearchIndex::SearchNode::fuzzyMatch(
+	const std::string query, size_t start, size_t size, std::vector<size_t>* indices) const
+{
+	size_t pos = start;
+	size_t weight = 0;
+	size_t matchCount = 0;
+	char lastChar = '\0';
+
+	size_t ql = query.size();
+	size_t ml = m_name.size();
+
+	if (query[pos] == ':')
+	{
+		pos++;
+		if (indices && size >= 2)
+		{
+			indices->push_back(size - 2);
+		}
+
+		if (pos < ql && query[pos] == ':')
+		{
+			pos++;
+			if (indices && size >= 1)
+			{
+				indices->push_back(size - 1);
+			}
+		}
+	}
+
+	for (size_t i = 0; i < ml; i++)
+	{
+		char c = m_name[i];
+		if (tolower(query[pos]) == tolower(c))
+		{
+			weight += std::max<size_t>(100 - size - i, 1);
+			if (matchCount)
+			{
+				weight += matchCount * 10;
+			}
+			else if (i == 0 || lastChar == '_' || tolower(c) != c)
+			{
+				weight += 20;
+			}
+			matchCount++;
+
+			pos++;
+			if (indices)
+			{
+				indices->push_back(size + i);
+			}
+
+			if (pos == ql || query[pos] == ':')
+			{
+				break;
+			}
+		}
+		else
+		{
+			matchCount = 0;
+		}
+
+		lastChar = c;
+	}
+
+	return std::pair<size_t, size_t>(pos, weight);
+}
+
+SearchIndex::SearchMatch SearchIndex::SearchNode::fuzzyMatchData(const std::string& query, const SearchNode* parent) const
+{
+	SearchMatch data;
+	data.node = this;
+	data.weight = 0;
+
+	size_t pos = 0;
+	size_t size = 0;
+
+	std::deque<const SearchNode*> nodes = getNodesToParent(parent);
+	for (const SearchNode* node : nodes)
+	{
+		std::pair<size_t, size_t> p = node->fuzzyMatch(query, pos, size, &data.indices);
+		pos = p.first;
+		data.weight += p.second;
+		size += node->m_name.size() + SearchIndex::DELIMITER.size();
+	}
+
+	return data;
+}
+
+std::shared_ptr<SearchIndex::SearchNode> SearchIndex::SearchNode::getChildWithNameId(Id nameId) const
+{
+	for (std::shared_ptr<SearchNode> n: m_nodes)
+	{
+		if (n->m_nameId == nameId)
+		{
+			return n;
+		}
+	}
+
+	return nullptr;
+}
+
+std::deque<const SearchIndex::SearchNode*> SearchIndex::SearchNode::getNodesToParent(const SearchNode* parent) const
+{
+	std::deque<const SearchIndex::SearchNode*> nodes;
+
+	const SearchNode* node = this;
+	while (node->m_nameId)
+	{
+		nodes.push_front(node);
+
+		if (node == parent)
+		{
+			break;
+		}
+
+		node = node->m_parent;
+	}
+
+	return nodes;
+}
+
+
 SearchIndex::SearchIndex()
+	: m_root(nullptr, DELIMITER, 0)
 {
 }
 
 SearchIndex::~SearchIndex()
 {
 }
+
+void SearchIndex::clear()
+{
+	m_root.clear();
+}
+
+SearchIndex::SearchNode* SearchIndex::addNode(const std::string& fullName)
+{
+	std::deque<Id> nameIds = Dictionary::getInstance()->getWordIds(fullName, DELIMITER);
+
+	if (nameIds.size())
+	{
+		return m_root.addNodeRecursive(&nameIds).get();
+	}
+
+	return nullptr;
+}
+
+SearchIndex::SearchNode* SearchIndex::getNode(const std::string& fullName) const
+{
+	std::deque<Id> nameIds = Dictionary::getInstance()->getWordIds(fullName, DELIMITER);
+
+	if (nameIds.size())
+	{
+		return m_root.getNodeRecursive(&nameIds).get();
+	}
+
+	return nullptr;
+}
+
+std::vector<std::string> SearchIndex::findFuzzyMatches(const std::string& query) const
+{
+	std::vector<SearchMatch> matches;
+	std::vector<std::string> pieces = utility::split<std::vector<std::string>>(query, '\"');
+
+	if (pieces.size() == 3 && pieces[0].size() == 0)
+	{
+		SearchNode* node = getNode(pieces[1]);
+		if (!node)
+		{
+			LOG_ERROR_STREAM(<< "Couldn't find node with name " << pieces[1] << " in the SearchIndex.");
+		}
+
+		matches = node->findFuzzyMatches(pieces[2]);
+	}
+	else
+	{
+		matches = m_root.findFuzzyMatches(query);
+	}
+
+	std::vector<std::string> names;
+	for (const SearchMatch& match : matches)
+	{
+		names.push_back(match.node->getFullName());
+	}
+	return names;
+}
+
+const std::string SearchIndex::DELIMITER = "::";

src/lib/data/SearchIndex.h

@@ -1,11 +1,92 @@
 #ifndef SEARCH_INDEX_H
 #define SEARCH_INDEX_H
 
+#include <deque>
+#include <map>
+#include <memory>
+#include <ostream>
+#include <set>
+#include <vector>
+
+#include "utility/types.h"
+
 class SearchIndex
 {
 public:
+	class SearchNode;
+
+	struct SearchMatch
+	{
+		void print(std::ostream& ostream) const;
+
+		const SearchIndex::SearchNode* node;
+		std::vector<size_t> indices;
+		size_t weight;
+	};
+
+	class SearchNode
+	{
+	public:
+		typedef std::multimap<size_t, const SearchIndex::SearchNode*> FuzzyMap;
+		typedef FuzzyMap::const_iterator FuzzyMapIterator;
+
+		typedef std::pair<size_t, const SearchIndex::SearchNode*> FuzzySetPair;
+		typedef std::multiset<FuzzySetPair, bool(*)(const FuzzySetPair&, const FuzzySetPair&)> FuzzySet;
+		typedef FuzzySet::const_iterator FuzzySetIterator;
+
+		SearchNode(SearchNode* parent, const std::string& name, Id nameId);
+		~SearchNode();
+
+		void clear();
+
+		const std::string& getName() const;
+		std::string getFullName() const;
+
+		Id getNameId() const;
+
+		Id getFirstTokenId() const;
+		void addTokenId(Id tokenId);
+
+		SearchNode* getParent() const;
+		std::deque<SearchIndex::SearchNode*> getParentsWithoutTokenId();
+
+		std::shared_ptr<SearchNode> addNodeRecursive(std::deque<Id>* nameIds);
+		std::shared_ptr<SearchNode> getNodeRecursive(std::deque<Id>* nameIds) const;
+
+		std::vector<SearchIndex::SearchMatch> findFuzzyMatches(const std::string& query) const;
+
+	private:
+		FuzzyMap fuzzyMatches(const std::string& query, size_t pos, size_t weight, size_t size) const;
+		std::pair<size_t, size_t> fuzzyMatch(
+			const std::string query, size_t start, size_t size, std::vector<size_t>* indices = nullptr) const;
+		SearchMatch fuzzyMatchData(const std::string& query, const SearchNode* parent) const;
+
+		std::shared_ptr<SearchIndex::SearchNode> getChildWithNameId(Id nameId) const;
+		std::deque<const SearchNode*> getNodesToParent(const SearchNode* parent) const;
+
+		std::set<std::shared_ptr<SearchNode>> m_nodes;
+		SearchNode* m_parent;
+
+		std::set<Id> m_tokenIds;
+
+		const std::string& m_name;
+		const Id m_nameId;
+	};
+
 	SearchIndex();
 	virtual ~SearchIndex();
+
+	void clear();
+
+	SearchNode* addNode(const std::string& fullName);
+	SearchNode* getNode(const std::string& fullName) const;
+
+	std::vector<std::string> findFuzzyMatches(const std::string& query) const;
+
+	static const std::string DELIMITER;
+
+private:
+	SearchNode m_root;
 };
 
 #endif // SEARCH_INDEX_H

src/lib/data/Storage.cpp

@@ -12,13 +12,14 @@
 #include "data/parser/ParseLocation.h"
 #include "data/parser/ParseTypeUsage.h"
 #include "data/parser/ParseVariable.h"
+#include "data/query/QueryCommand.h"
 #include "data/query/QueryTree.h"
 #include "data/type/DataType.h"
 #include "utility/logging/logging.h"
-#include "utility/utilityString.h"
 
 Storage::Storage()
 {
+	initSearchIndex();
 }
 
 Storage::~Storage()
@@ -29,6 +30,9 @@ void Storage::clear()
 {
 	m_graph.clear();
 	m_locationCollection.clear();
+
+	m_index.clear();
+	initSearchIndex();
 }
 
 void Storage::logGraph() const
@@ -47,7 +51,7 @@ Id Storage::onTypedefParsed(
 ){
 	log("typedef", fullName + " -> " + underlyingType.dataType.getFullTypeName(), location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_TYPEDEF, fullName);
+	Node* node = addNodeHierarchy(Node::NODE_TYPEDEF, fullName);
 	addAccess(node, access);
 	addTokenLocation(node, location);
 	addTypeEdge(node, Edge::EDGE_TYPEDEF_OF, underlyingType);
@@ -60,7 +64,7 @@ Id Storage::onClassParsed(
 ){
 	log("class", fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_CLASS, fullName);
+	Node* node = addNodeHierarchy(Node::NODE_CLASS, fullName);
 	addAccess(node, access);
 	addTokenLocation(node, location);
 	addTokenLocation(node, scopeLocation, true);
@@ -73,7 +77,7 @@ Id Storage::onStructParsed(
 ){
 	log("struct", fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_STRUCT, fullName);
+	Node* node = addNodeHierarchy(Node::NODE_STRUCT, fullName);
 	addAccess(node, access);
 	addTokenLocation(node, location);
 	addTokenLocation(node, scopeLocation, true);
@@ -85,7 +89,7 @@ Id Storage::onGlobalVariableParsed(const ParseLocation& location, const ParseVar
 {
 	log("global", variable.fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_GLOBAL_VARIABLE, variable.fullName);
+	Node* node = addNodeHierarchy(Node::NODE_GLOBAL_VARIABLE, variable.fullName);
 
 	if (variable.isStatic)
 	{
@@ -102,7 +106,7 @@ Id Storage::onFieldParsed(const ParseLocation& location, const ParseVariable& va
 {
 	log("field", variable.fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_FIELD, variable.fullName);
+	Node* node = addNodeHierarchy(Node::NODE_FIELD, variable.fullName);
 
 	if (!node->getMemberEdge())
 	{
@@ -131,9 +135,7 @@ Id Storage::onFunctionParsed(
 ){
 	log("function", function.fullName, location);
 
-	Node* node = m_graph.createNodeHierarchyWithDistinctSignature(
-		Node::NODE_FUNCTION, function.fullName, ParserClient::functionSignatureStr(function)
-	);
+	Node* node = addNodeHierarchyWithDistinctSignature(Node::NODE_FUNCTION, function);
 
 	addTokenLocation(node, location);
 	addTokenLocation(node, scopeLocation, true);
@@ -153,9 +155,7 @@ Id Storage::onMethodParsed(
 ){
 	log("method", method.fullName, location);
 
-	Node* node = m_graph.createNodeHierarchyWithDistinctSignature(
-		Node::NODE_METHOD, method.fullName, ParserClient::functionSignatureStr(method)
-	);
+	Node* node = addNodeHierarchyWithDistinctSignature(Node::NODE_METHOD, method);
 
 	if (!node->getMemberEdge())
 	{
@@ -196,7 +196,7 @@ Id Storage::onNamespaceParsed(
 ){
 	log("namespace", fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_NAMESPACE, fullName);
+	Node* node = addNodeHierarchy(Node::NODE_NAMESPACE, fullName);
 	addTokenLocation(node, location);
 	addTokenLocation(node, scopeLocation, true);
 
@@ -208,7 +208,7 @@ Id Storage::onEnumParsed(
 ){
 	log("enum", fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_ENUM, fullName);
+	Node* node = addNodeHierarchy(Node::NODE_ENUM, fullName);
 	addAccess(node, access);
 	addTokenLocation(node, location);
 	addTokenLocation(node, scopeLocation, true);
@@ -220,7 +220,7 @@ Id Storage::onEnumFieldParsed(const ParseLocation& location, const std::string&
 {
 	log("enum field", fullName, location);
 
-	Node* node = m_graph.createNodeHierarchy(Node::NODE_FIELD, fullName);
+	Node* node = addNodeHierarchy(Node::NODE_FIELD, fullName);
 	addTokenLocation(node, location);
 
 	return node->getId();
@@ -231,8 +231,8 @@ Id Storage::onInheritanceParsed(
 ){
 	log("inheritance", fullName + " : " + baseName, location);
 
-	Node* node = m_graph.createNodeHierarchy(fullName);
-	Node* baseNode = m_graph.createNodeHierarchy(baseName);
+	Node* node = addNodeHierarchy(Node::NODE_UNDEFINED_TYPE, fullName);
+	Node* baseNode = addNodeHierarchy(Node::NODE_UNDEFINED_TYPE, baseName);
 
 	Edge* edge = m_graph.createEdge(Edge::EDGE_INHERITANCE, node, baseNode);
 	edge->addComponentAccess(std::make_shared<TokenComponentAccess>(convertAccessType(access)));
@@ -246,10 +246,8 @@ Id Storage::onCallParsed(const ParseLocation& location, const ParseFunction& cal
 {
 	log("call", caller.fullName + " -> " + callee.fullName, location);
 
-	Node* callerNode =
-		m_graph.createNodeHierarchyWithDistinctSignature(caller.fullName, ParserClient::functionSignatureStr(caller));
-	Node* calleeNode =
-		m_graph.createNodeHierarchyWithDistinctSignature(callee.fullName, ParserClient::functionSignatureStr(callee));
+	Node* callerNode = addNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, caller);
+	Node* calleeNode = addNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, callee);
 
 	Edge* edge = m_graph.createEdge(Edge::EDGE_CALL, callerNode, calleeNode);
 
@@ -262,10 +260,8 @@ Id Storage::onCallParsed(const ParseLocation& location, const ParseVariable& cal
 {
 	log("call", caller.fullName + " -> " + callee.fullName, location);
 
-	Node* callerNode =
-		m_graph.createNodeHierarchy(caller.fullName);
-	Node* calleeNode =
-		m_graph.createNodeHierarchyWithDistinctSignature(callee.fullName, ParserClient::functionSignatureStr(callee));
+	Node* callerNode = addNodeHierarchy(Node::NODE_UNDEFINED, caller.fullName);
+	Node* calleeNode = addNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, callee);
 
 	Edge* edge = m_graph.createEdge(Edge::EDGE_CALL, callerNode, calleeNode);
 
@@ -278,9 +274,8 @@ Id Storage::onFieldUsageParsed(const ParseLocation& location, const ParseFunctio
 {
 	log("field usage", user.fullName + " -> " + usedName, location);
 
-	Node* userNode =
-		m_graph.createNodeHierarchyWithDistinctSignature(user.fullName, ParserClient::functionSignatureStr(user));
-	Node* usedNode = m_graph.createNodeHierarchy(usedName);
+	Node* userNode = addNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, user);
+	Node* usedNode = addNodeHierarchy(Node::NODE_UNDEFINED_VARIABLE, usedName);
 
 	Edge* edge = m_graph.createEdge(Edge::EDGE_USAGE, userNode, usedNode);
 	addTokenLocation(edge, location);
@@ -293,9 +288,8 @@ Id Storage::onGlobalVariableUsageParsed(
 ){
 	log("global usage", user.fullName + " -> " + usedName, location);
 
-	Node* userNode =
-		m_graph.createNodeHierarchyWithDistinctSignature(user.fullName, ParserClient::functionSignatureStr(user));;
-	Node* usedNode = m_graph.createNodeHierarchy(usedName);
+	Node* userNode = addNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, user);
+	Node* usedNode = addNodeHierarchy(Node::NODE_UNDEFINED_VARIABLE, usedName);
 
 	Edge* edge = m_graph.createEdge(Edge::EDGE_USAGE, userNode, usedNode);
 	addTokenLocation(edge, location);
@@ -307,8 +301,7 @@ Id Storage::onTypeUsageParsed(const ParseTypeUsage& type, const ParseFunction& f
 {
 	log("type usage", function.fullName + " -> " + type.dataType.getRawTypeName(), type.location);
 
-	Node* functionNode =
-		m_graph.createNodeHierarchyWithDistinctSignature(function.fullName, ParserClient::functionSignatureStr(function));
+	Node* functionNode = addNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, function);
 	Edge* edge = addTypeEdge(functionNode, Edge::EDGE_TYPE_USAGE, type);
 
 	return edge->getId();
@@ -316,8 +309,12 @@ Id Storage::onTypeUsageParsed(const ParseTypeUsage& type, const ParseFunction& f
 
 Id Storage::getIdForNodeWithName(const std::string& fullName) const
 {
-	Node* node = m_graph.getNode(fullName);
-	return (node ? node->getId() : 0);
+	SearchIndex::SearchNode* node = m_index.getNode(fullName);
+	if (node)
+	{
+		return node->getFirstTokenId();
+	}
+	return 0;
 }
 
 std::string Storage::getNameForNodeWithId(Id id) const
@@ -341,16 +338,7 @@ std::string Storage::getNameForNodeWithId(Id id) const
 
 std::vector<std::string> Storage::getNamesForNodesWithNamePrefix(const std::string& prefix) const
 {
-	std::vector<std::string> names;
-	m_graph.forEachNode([&](Node* node){
-		const std::string& nodeName = node->getFullName();
-		if (utility::isPrefix(prefix, nodeName))
-		{
-			names.push_back(nodeName);
-		}
-	});
-
-	return names;
+	return m_index.findFuzzyMatches(prefix);
 }
 
 std::vector<Id> Storage::getIdsOfNeighbours(const Id id) const
@@ -538,6 +526,8 @@ std::vector<Id> Storage::getTokenIdsForQuery(std::string query) const
 
 	LOG_INFO_STREAM(<< '\n' << tree << '\n' << outGraph);
 
+	m_index.findFuzzyMatches(query);
+
 	return outGraph.getTokenIds();
 }
 
@@ -651,6 +641,38 @@ std::vector<TokenLocation*> Storage::getTokenLocationsForId(Id tokenId) const
 	return result;
 }
 
+void Storage::initSearchIndex()
+{
+	for (const std::pair<std::string, QueryCommand::CommandType>& p : QueryCommand::getCommandTypeMap())
+	{
+		m_index.addNode(p.first);
+	}
+}
+
+Node* Storage::addNodeHierarchy(Node::NodeType type, const std::string& fullName)
+{
+	SearchIndex::SearchNode* searchNode = m_index.addNode(fullName);
+	if (!searchNode)
+	{
+		LOG_ERROR("No SearchNode");
+		return nullptr;
+	}
+
+	return m_graph.createNodeHierarchy(type, searchNode);
+}
+
+Node* Storage::addNodeHierarchyWithDistinctSignature(Node::NodeType type, const ParseFunction& function)
+{
+	SearchIndex::SearchNode* searchNode = m_index.addNode(function.fullName);
+	if (!searchNode)
+	{
+		LOG_ERROR("No SearchNode");
+		return nullptr;
+	}
+
+	return m_graph.createNodeHierarchyWithDistinctSignature(type, searchNode, ParserClient::functionSignatureStr(function));
+}
+
 TokenComponentAccess::AccessType Storage::convertAccessType(ParserClient::AccessType access) const
 {
 	switch (access)
@@ -704,7 +726,7 @@ TokenComponentAbstraction* Storage::addAbstraction(Node* node, ParserClient::Abs
 
 Edge* Storage::addTypeEdge(Node* node, Edge::EdgeType edgeType, const DataType& type)
 {
-	Node* typeNode = m_graph.createNodeHierarchy(type.getRawTypeName());
+	Node* typeNode = addNodeHierarchy(Node::NODE_UNDEFINED_TYPE, type.getRawTypeName());
 	Edge* edge = m_graph.createEdge(edgeType, node, typeNode);
 
 	// FIXME: When a function uses the same type multiple times then we still only use one edge to save this,

src/lib/data/Storage.h

@@ -6,11 +6,12 @@
 
 #include "data/access/GraphAccess.h"
 #include "data/access/LocationAccess.h"
-#include "data/graph/Graph.h"
+#include "data/graph/StorageGraph.h"
 #include "data/graph/token_component/TokenComponentAbstraction.h"
 #include "data/graph/token_component/TokenComponentAccess.h"
 #include "data/location/TokenLocationCollection.h"
 #include "data/parser/ParserClient.h"
+#include "data/SearchIndex.h"
 
 class Storage
 	: public ParserClient
@@ -99,6 +100,11 @@ protected:
 	std::vector<TokenLocation*> getTokenLocationsForId(Id tokenId) const;
 
 private:
+	void initSearchIndex();
+
+	Node* addNodeHierarchy(Node::NodeType type, const std::string& fullName);
+	Node* addNodeHierarchyWithDistinctSignature(Node::NodeType type, const ParseFunction& function);
+
 	TokenComponentAccess::AccessType convertAccessType(ParserClient::AccessType access) const;
 	TokenComponentAccess* addAccess(Node* node, ParserClient::AccessType access);
 
@@ -113,8 +119,9 @@ private:
 
 	std::vector<std::tuple<Id, Id, Id>> getEdgesOfTypeOfNode(const Id id, const Edge::EdgeType type) const;
 
-	Graph m_graph;
+	StorageGraph m_graph;
 	TokenLocationCollection m_locationCollection;
+	SearchIndex m_index;
 };
 
 #endif // STORAGE_H

src/lib/data/graph/Edge.cpp

@@ -164,8 +164,9 @@ std::ostream& operator<<(std::ostream& ostream, const Edge& edge)
 
 bool Edge::checkType() const
 {
-	Node::NodeTypeMask typeMask = Node::NODE_UNDEFINED | Node::NODE_CLASS | Node::NODE_STRUCT | Node::NODE_ENUM | Node::NODE_TYPEDEF;
-	Node::NodeTypeMask variableMask = Node::NODE_UNDEFINED | Node::NODE_GLOBAL_VARIABLE | Node::NODE_FIELD;
+	Node::NodeTypeMask complexTypeMask = Node::NODE_UNDEFINED_TYPE | Node::NODE_CLASS | Node::NODE_STRUCT;
+	Node::NodeTypeMask typeMask = Node::NODE_UNDEFINED | Node::NODE_ENUM | Node::NODE_TYPEDEF | complexTypeMask;
+	Node::NodeTypeMask variableMask = Node::NODE_UNDEFINED | Node::NODE_UNDEFINED_VARIABLE | Node::NODE_GLOBAL_VARIABLE | Node::NODE_FIELD;
 	Node::NodeTypeMask functionMask = Node::NODE_UNDEFINED_FUNCTION | Node::NODE_FUNCTION | Node::NODE_METHOD;
 
 	switch (m_type)
@@ -196,7 +197,7 @@ bool Edge::checkType() const
 		return true;
 
 	case EDGE_INHERITANCE:
-		if (!m_from->isType(Node::NODE_CLASS) || !m_to->isType(Node::NODE_UNDEFINED | Node::NODE_CLASS))
+		if (!m_from->isType(complexTypeMask) || !m_to->isType(complexTypeMask))
 		{
 			break;
 		}

src/lib/data/graph/Graph.cpp

@@ -1,8 +1,6 @@
 #include "data/graph/Graph.h"
 
-#include "data/graph/token_component/TokenComponentSignature.h"
 #include "utility/logging/logging.h"
-#include "utility/utilityString.h"
 
 Graph::Graph()
 {
@@ -14,28 +12,6 @@ Graph::~Graph()
 	m_nodes.clear();
 }
 
-Graph& Graph::operator=(const Graph& other)
-{
-	if (&other != this)
-	{
-		other.forEachNode(
-			[this](Node* node)
-			{
-				addNodeAsPlainCopy(node);
-			}
-		);
-
-		other.forEachEdge(
-			[this](Edge* edge)
-			{
-				addEdgeAsPlainCopy(edge);
-			}
-		);
-	}
-
-	return *this;
-}
-
 void Graph::copy(const FilterableGraph* other)
 {
 	clear();
@@ -109,29 +85,6 @@ const std::map<Id, std::shared_ptr<Edge>>& Graph::getEdges() const
 	return m_edges;
 }
 
-Node* Graph::getNode(const std::string& fullName) const
-{
-	std::deque<std::string> names = utility::split<std::deque<std::string>>(fullName, DELIMITER);
-	Node* node = getLastValidNode(&names);
-
-	if (node && !names.size())
-	{
-		return node;
-	}
-
-	return nullptr;
-}
-
-Edge* Graph::getEdge(Edge::EdgeType type, Node* from, Node* to) const
-{
-	return from->findEdgeOfType(type,
-		[to](Edge* e)
-		{
-			return e->getTo() == to;
-		}
-	);
-}
-
 Node* Graph::getNodeById(Id id) const
 {
 	std::map<Id, std::shared_ptr<Node>>::const_iterator it = m_nodes.find(id);
@@ -162,95 +115,6 @@ Token* Graph::getTokenById(Id id) const
 	return token;
 }
 
-Node* Graph::createNodeHierarchy(const std::string& fullName)
-{
-	return createNodeHierarchy(Node::NODE_UNDEFINED, fullName);
-}
-
-Node* Graph::createNodeHierarchy(Node::NodeType type, const std::string& fullName)
-{
-	std::deque<std::string> names = utility::split<std::deque<std::string>>(fullName, DELIMITER);
-	Node* node = getLastValidNode(&names);
-	if (node && !names.size())
-	{
-		if (type != Node::NODE_UNDEFINED)
-		{
-			node->setType(type);
-		}
-		return node;
-	}
-
-	return insertNodeHierarchy(type, names, node);
-}
-
-Node* Graph::createNodeHierarchyWithDistinctSignature(const std::string& fullName, const std::string& signature)
-{
-	return createNodeHierarchyWithDistinctSignature(Node::NODE_UNDEFINED_FUNCTION, fullName, signature);
-}
-
-Node* Graph::createNodeHierarchyWithDistinctSignature(
-	Node::NodeType type, const std::string& fullName, const std::string& signature
-){
-	std::deque<std::string> names = utility::split<std::deque<std::string>>(fullName, DELIMITER);
-	Node* node = getLastValidNode(&names);
-	if (node && !names.size())
-	{
-		TokenComponentSignature* sigComponent = node->getComponent<TokenComponentSignature>();
-		if (sigComponent && sigComponent->getSignature() == signature)
-		{
-			if (type != Node::NODE_UNDEFINED && type != Node::NODE_UNDEFINED_FUNCTION)
-			{
-				node->setType(type);
-			}
-			return node;
-		}
-
-		Node* parentNode = node->getParentNode();
-		const std::string& name = node->getName();
-
-		std::function<bool(Node*)> findSignature =
-			[&name, &signature](Node* n)
-			{
-				TokenComponentSignature* c = n->getComponent<TokenComponentSignature>();
-				return n->getName() == name && c && c->getSignature() == signature;
-			};
-
-		if (parentNode)
-		{
-			node = parentNode->findChildNode(findSignature);
-		}
-		else
-		{
-			node = findNode(findSignature);
-		}
-
-		if (node)
-		{
-			return node;
-		}
-
-		node = insertNode(type, name, parentNode);
-	}
-	else
-	{
-		node = insertNodeHierarchy(type, names, node);
-	}
-
-	node->addComponentSignature(std::make_shared<TokenComponentSignature>(signature));
-	return node;
-}
-
-Edge* Graph::createEdge(Edge::EdgeType type, Node* from, Node* to)
-{
-	Edge* edge = getEdge(type, from, to);
-	if (edge)
-	{
-		return edge;
-	}
-
-	return insertEdge(type, from, to);
-}
-
 void Graph::removeNode(Node* node)
 {
 	std::map<Id, std::shared_ptr<Node>>::const_iterator it = m_nodes.find(node->getId());
@@ -380,81 +244,6 @@ Edge* Graph::addEdgeAsPlainCopy(Edge* edge)
 	return copy.get();
 }
 
-const std::string Graph::DELIMITER = "::";
-
-Node* Graph::getLastValidNode(std::deque<std::string>* names) const
-{
-	const std::string& name = names->front();
-
-	Node* node = findNode(
-		[&name](Node* n)
-		{
-			return n->getName() == name && n->getParentNode() == nullptr;
-		}
-	);
-
-	if (!node)
-	{
-		return nullptr;
-	}
-
-	names->pop_front();
-
-	while (names->size())
-	{
-		const std::string& name = names->front();
-		Node* childNode = node->findChildNode(
-			[&name](Node* n)
-			{
-				return n->getName() == name;
-			}
-		);
-
-		if (!childNode)
-		{
-			break;
-		}
-
-		node = childNode;
-		names->pop_front();
-	}
-
-	return node;
-}
-
-Node* Graph::insertNodeHierarchy(Node::NodeType type, std::deque<std::string> names, Node* parentNode)
-{
-	while (names.size())
-	{
-		parentNode = insertNode(names.size() == 1 ? type : Node::NODE_UNDEFINED, names.front(), parentNode);
-		names.pop_front();
-	}
-
-	return parentNode;
-}
-
-Node* Graph::insertNode(Node::NodeType type, const std::string& name, Node* parentNode)
-{
-	std::shared_ptr<Node> nodePtr = std::make_shared<Node>(type, name);
-	m_nodes.emplace(nodePtr->getId(), nodePtr);
-
-	Node* node = nodePtr.get();
-
-	if (parentNode)
-	{
-		createEdge(Edge::EDGE_MEMBER, parentNode, node);
-	}
-
-	return node;
-}
-
-Edge* Graph::insertEdge(Edge::EdgeType type, Node* from, Node* to)
-{
-	std::shared_ptr<Edge> edgePtr = std::make_shared<Edge>(type, from, to);
-	m_edges.emplace(edgePtr->getId(), edgePtr);
-	return edgePtr.get();
-}
-
 void Graph::removeEdgeInternal(Edge* edge)
 {
 	std::map<Id, std::shared_ptr<Edge> >::const_iterator it = m_edges.find(edge->getId());

src/lib/data/graph/Graph.h

@@ -15,7 +15,6 @@ class Graph
 public:
 	Graph();
 	virtual ~Graph();
-	Graph& operator=(const Graph& other);
 
 	// FilterableGraph implementation
 	virtual void copy(const FilterableGraph* other);
@@ -36,23 +35,10 @@ public:
 	const std::map<Id, std::shared_ptr<Node>>& getNodes() const;
 	const std::map<Id, std::shared_ptr<Edge>>& getEdges() const;
 
-	Node* getNode(const std::string& fullName) const;
-	Edge* getEdge(Edge::EdgeType type, Node* from, Node* to) const;
-
 	Node* getNodeById(Id id) const;
 	Edge* getEdgeById(Id id) const;
 	Token* getTokenById(Id id) const;
 
-	Node* createNodeHierarchy(const std::string& fullName);
-	Node* createNodeHierarchy(Node::NodeType type, const std::string& fullName);
-
-	Node* createNodeHierarchyWithDistinctSignature(const std::string& fullName, const std::string& signature);
-	Node* createNodeHierarchyWithDistinctSignature(
-		Node::NodeType type, const std::string& fullName, const std::string& signature
-	);
-
-	Edge* createEdge(Edge::EdgeType type, Node* from, Node* to);
-
 	void removeNode(Node* node);
 	void removeEdge(Edge* edge);
 
@@ -63,17 +49,15 @@ public:
 	Node* addNodeAsPlainCopy(Node* node);
 	Edge* addEdgeAsPlainCopy(Edge* edge);
 
-private:
-	static const std::string DELIMITER;
-
-	Node* getLastValidNode(std::deque<std::string>* names) const;
-	Node* insertNodeHierarchy(Node::NodeType type, std::deque<std::string> names, Node* parentNode);
-	Node* insertNode(Node::NodeType type, const std::string& name, Node* parentNode);
-	Edge* insertEdge(Edge::EdgeType type, Node* from, Node* to);
-	void removeEdgeInternal(Edge* edge);
-
+protected:
 	std::map<Id, std::shared_ptr<Node>> m_nodes;
 	std::map<Id, std::shared_ptr<Edge>> m_edges;
+
+private:
+	Graph(const Graph&);
+	void operator=(const Graph&);
+
+	void removeEdgeInternal(Edge* edge);
 };
 
 std::ostream& operator<<(std::ostream& ostream, const Graph& graph);

src/lib/data/graph/Node.cpp

@@ -4,20 +4,27 @@
 
 #include "data/graph/token_component/TokenComponentAbstraction.h"
 #include "data/graph/token_component/TokenComponentConst.h"
+#include "data/graph/token_component/TokenComponentName.h"
 #include "data/graph/token_component/TokenComponentStatic.h"
 #include "data/graph/token_component/TokenComponentSignature.h"
 #include "utility/logging/logging.h"
 
 Node::Node(NodeType type, const std::string& name)
 	: m_type(type)
-	, m_name(name)
+	, m_nameComponent(std::make_shared<TokenComponentNameCached>(name))
+{
+}
+
+Node::Node(NodeType type, std::shared_ptr<TokenComponentName> nameComponent)
+	: m_type(type)
+	, m_nameComponent(nameComponent)
 {
 }
 
 Node::Node(const Node& other)
 	: Token(other)
 	, m_type(other.m_type)
-	, m_name(other.m_name)
+	, m_nameComponent(other.m_nameComponent->copyComponentName())
 {
 }
 
@@ -32,7 +39,7 @@ Node::NodeType Node::getType() const
 
 void Node::setType(NodeType type)
 {
-	if (!isType(type | NODE_UNDEFINED | NODE_UNDEFINED_FUNCTION))
+	if (!isType(type | NODE_UNDEFINED | NODE_UNDEFINED_FUNCTION | NODE_UNDEFINED_VARIABLE | NODE_UNDEFINED_TYPE))
 	{
 		LOG_WARNING(
 			"Cannot change NodeType after it was already set from " + getTypeString() + " to " + getTypeString(type)
@@ -49,20 +56,12 @@ bool Node::isType(NodeTypeMask mask) const
 
 const std::string& Node::getName() const
 {
-	return m_name;
+	return m_nameComponent->getName();
 }
 
 std::string Node::getFullName() const
 {
-	Node* parent = getParentNode();
-	if (parent)
-	{
-		return parent->getFullName() + "::" + m_name;
-	}
-	else
-	{
-		return m_name;
-	}
+	return m_nameComponent->getFullName();
 }
 
 const std::vector<Edge*>& Node::getEdges() const
@@ -269,6 +268,10 @@ std::string Node::getTypeString(NodeType type) const
 		return "undefined";
 	case NODE_UNDEFINED_FUNCTION:
 		return "undefined_function";
+	case NODE_UNDEFINED_VARIABLE:
+		return "undefined_variable";
+	case NODE_UNDEFINED_TYPE:
+		return "undefined_type";
 	case NODE_CLASS:
 		return "class";
 	case NODE_STRUCT:

src/lib/data/graph/Node.h

@@ -11,6 +11,7 @@
 
 class TokenComponentAbstraction;
 class TokenComponentConst;
+class TokenComponentName;
 class TokenComponentStatic;
 class TokenComponentSignature;
 
@@ -22,18 +23,21 @@ public:
 	{
 		NODE_UNDEFINED          = 0x1,
 		NODE_UNDEFINED_FUNCTION = 0x2,
-		NODE_CLASS              = 0x4,
-		NODE_STRUCT             = 0x8,
-		NODE_GLOBAL_VARIABLE    = 0x10,
-		NODE_FIELD              = 0x20,
-		NODE_FUNCTION           = 0x40,
-		NODE_METHOD             = 0x80,
-		NODE_NAMESPACE          = 0x100,
-		NODE_ENUM               = 0x200,
-		NODE_TYPEDEF            = 0x400
+		NODE_UNDEFINED_VARIABLE = 0x4,
+		NODE_UNDEFINED_TYPE     = 0x8,
+		NODE_STRUCT             = 0x10,
+		NODE_CLASS              = 0x20,
+		NODE_GLOBAL_VARIABLE    = 0x40,
+		NODE_FIELD              = 0x80,
+		NODE_FUNCTION           = 0x100,
+		NODE_METHOD             = 0x200,
+		NODE_NAMESPACE          = 0x400,
+		NODE_ENUM               = 0x800,
+		NODE_TYPEDEF            = 0x1000
 	};
 
 	Node(NodeType type, const std::string& name);
+	Node(NodeType type, std::shared_ptr<TokenComponentName> nameComponent);
 	Node(const Node& other);
 	virtual ~Node();
 
@@ -79,10 +83,10 @@ public:
 private:
 	void operator=(const Node&);
 
-	NodeType m_type;
-	std::string m_name;
-
 	std::vector<Edge*> m_edges;
+
+	NodeType m_type;
+	std::shared_ptr<TokenComponentName> m_nameComponent;
 };
 
 std::ostream& operator<<(std::ostream& ostream, const Node& node);

src/lib/data/graph/StorageGraph.cpp

@@ -0,0 +1,145 @@
+#include "data/graph/StorageGraph.h"
+
+#include "data/graph/token_component/TokenComponentName.h"
+#include "data/graph/token_component/TokenComponentSignature.h"
+#include "utility/logging/logging.h"
+#include "utility/utilityString.h"
+
+StorageGraph::StorageGraph()
+{
+}
+
+StorageGraph::~StorageGraph()
+{
+}
+
+Node* StorageGraph::createNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode)
+{
+	Node* node = getNodeById(searchNode->getFirstTokenId());
+	if (!node)
+	{
+		return insertNodeHierarchy(type, searchNode);
+	}
+
+	if (node->getType() < type)
+	{
+		node->setType(type);
+	}
+
+	return node;
+}
+
+Node* StorageGraph::createNodeHierarchyWithDistinctSignature(
+	Node::NodeType type, SearchIndex::SearchNode* searchNode, const std::string& signature
+){
+	Node* node = getNodeById(searchNode->getFirstTokenId());
+	std::shared_ptr<TokenComponentSignature> sigPtr = TokenComponentSignature::create(signature);
+
+	if (!node)
+	{
+		node = insertNodeHierarchy(type, searchNode);
+	}
+	else
+	{
+		std::function<bool(Node*)> findSignature =
+			[sigPtr](Node* n)
+			{
+				TokenComponentSignature* c = n->getComponent<TokenComponentSignature>();
+				return c && *c == *sigPtr.get();
+			};
+
+		Node* parentNode = node->getParentNode();
+		if (parentNode)
+		{
+			node = parentNode->findChildNode(findSignature);
+		}
+		else
+		{
+			node = findNode(findSignature);
+		}
+
+		if (!node)
+		{
+			node = insertNode(type, parentNode, searchNode);
+		}
+		else
+		{
+			if (node->getType() < type)
+			{
+				node->setType(type);
+			}
+			return node;
+		}
+	}
+
+	node->addComponentSignature(sigPtr);
+	return node;
+}
+
+Edge* StorageGraph::createEdge(Edge::EdgeType type, Node* from, Node* to)
+{
+	Edge* edge = from->findEdgeOfType(type,
+		[to](Edge* e)
+		{
+			return e->getTo() == to;
+		}
+	);
+
+	if (edge)
+	{
+		return edge;
+	}
+
+	return insertEdge(type, from, to);
+}
+
+Node* StorageGraph::insertNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode)
+{
+	std::deque<SearchIndex::SearchNode*> searchNodes = searchNode->getParentsWithoutTokenId();
+
+	if (!searchNodes.size())
+	{
+		LOG_ERROR("There are no nodes without a set tokenId so this method shouldn't have been called.");
+		return nullptr;
+	}
+
+	Node* parentNode = nullptr;
+	SearchIndex::SearchNode* parentSearchNode = searchNodes.front()->getParent();
+	if (parentSearchNode)
+	{
+		parentNode = getNodeById(parentSearchNode->getFirstTokenId());
+	}
+
+	while (searchNodes.size())
+	{
+		searchNode = searchNodes.front();
+		searchNodes.pop_front();
+
+		parentNode = insertNode(searchNodes.size() ? Node::NODE_UNDEFINED : type, parentNode, searchNode);
+	}
+
+	return parentNode;
+}
+
+Node* StorageGraph::insertNode(Node::NodeType type, Node* parentNode, SearchIndex::SearchNode* searchNode)
+{
+	std::shared_ptr<Node> node =
+		std::make_shared<Node>(type, std::make_shared<TokenComponentNameReferenced>(searchNode));
+	m_nodes.emplace(node->getId(), node);
+
+	searchNode->addTokenId(node->getId());
+
+	if (parentNode)
+	{
+		createEdge(Edge::EDGE_MEMBER, parentNode, node.get());
+	}
+
+	return node.get();
+}
+
+Edge* StorageGraph::insertEdge(Edge::EdgeType type, Node* from, Node* to)
+{
+	std::shared_ptr<Edge> edgePtr = std::make_shared<Edge>(type, from, to);
+	m_edges.emplace(edgePtr->getId(), edgePtr);
+	return edgePtr.get();
+}

src/lib/data/graph/StorageGraph.h

@@ -0,0 +1,25 @@
+#ifndef STORAGE_GRAPH_H
+#define STORAGE_GRAPH_H
+
+#include "data/graph/Graph.h"
+#include "data/SearchIndex.h"
+
+class StorageGraph
+	: public Graph
+{
+public:
+	StorageGraph();
+	virtual ~StorageGraph();
+
+	Node* createNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode);
+	Node* createNodeHierarchyWithDistinctSignature(
+		Node::NodeType type, SearchIndex::SearchNode* searchNode, const std::string& signature);
+	Edge* createEdge(Edge::EdgeType type, Node* from, Node* to);
+
+private:
+	Node* insertNodeHierarchy(Node::NodeType type, SearchIndex::SearchNode* searchNode);
+	Node* insertNode(Node::NodeType type, Node* parentNode, SearchIndex::SearchNode* searchNode);
+	Edge* insertEdge(Edge::EdgeType type, Node* from, Node* to);
+};
+
+#endif // STORAGE_GRAPH_H

src/lib/data/graph/token_component/TokenComponentName.cpp

@@ -0,0 +1,66 @@
+#include "data/graph/token_component/TokenComponentName.h"
+
+#include "utility/utilityString.h"
+
+TokenComponentName::TokenComponentName()
+{
+}
+
+TokenComponentName::~TokenComponentName()
+{
+}
+
+std::shared_ptr<TokenComponentName> TokenComponentName::copyComponentName() const
+{
+	return std::dynamic_pointer_cast<TokenComponentName>(copy());
+}
+
+
+TokenComponentNameReferenced::TokenComponentNameReferenced(const SearchIndex::SearchNode* searchNode)
+	: m_searchNode(searchNode)
+{
+}
+
+TokenComponentNameReferenced::~TokenComponentNameReferenced()
+{
+}
+
+std::shared_ptr<TokenComponent> TokenComponentNameReferenced::copy() const
+{
+	return std::make_shared<TokenComponentNameCached>(getFullName());
+}
+
+const std::string& TokenComponentNameReferenced::getName() const
+{
+	return m_searchNode->getName();
+}
+
+std::string TokenComponentNameReferenced::getFullName() const
+{
+	return m_searchNode->getFullName();
+}
+
+
+TokenComponentNameCached::TokenComponentNameCached(const std::string& fullName)
+	: m_fullName(fullName)
+{
+}
+
+TokenComponentNameCached::~TokenComponentNameCached()
+{
+}
+
+std::shared_ptr<TokenComponent> TokenComponentNameCached::copy() const
+{
+	return std::make_shared<TokenComponentNameCached>(m_fullName);
+}
+
+const std::string& TokenComponentNameCached::getName() const
+{
+	return utility::split<std::deque<std::string>>(m_fullName, SearchIndex::DELIMITER).back();
+}
+
+std::string TokenComponentNameCached::getFullName() const
+{
+	return m_fullName;
+}

src/lib/data/graph/token_component/TokenComponentName.h

@@ -0,0 +1,56 @@
+#ifndef TOKEN_COMPONENT_NAME_H
+#define TOKEN_COMPONENT_NAME_H
+
+#include <string>
+
+#include "data/graph/token_component/TokenComponent.h"
+#include "data/SearchIndex.h"
+
+class TokenComponentName
+	: public TokenComponent
+{
+public:
+	TokenComponentName();
+	virtual ~TokenComponentName();
+
+	std::shared_ptr<TokenComponentName> copyComponentName() const;
+
+	virtual const std::string& getName() const = 0;
+	virtual std::string getFullName() const = 0;
+};
+
+
+class TokenComponentNameReferenced
+	: public TokenComponentName
+{
+public:
+	TokenComponentNameReferenced(const SearchIndex::SearchNode* searchNode);
+	virtual ~TokenComponentNameReferenced();
+
+	virtual std::shared_ptr<TokenComponent> copy() const;
+
+	virtual const std::string& getName() const;
+	virtual std::string getFullName() const;
+
+private:
+	const SearchIndex::SearchNode* m_searchNode;
+};
+
+
+class TokenComponentNameCached
+	: public TokenComponentName
+{
+public:
+	TokenComponentNameCached(const std::string& fullName);
+	virtual ~TokenComponentNameCached();
+
+	virtual std::shared_ptr<TokenComponent> copy() const;
+
+	virtual const std::string& getName() const;
+	virtual std::string getFullName() const;
+
+private:
+	const std::string m_fullName;
+};
+
+#endif // TOKEN_COMPONENT_NAME_H

src/lib/data/graph/token_component/TokenComponentSignature.cpp

@@ -1,8 +1,11 @@
 #include "data/graph/token_component/TokenComponentSignature.h"
 
-TokenComponentSignature::TokenComponentSignature(std::string signature)
-	: m_signature(signature)
+#include "utility/text/Dictionary.h"
+
+std::shared_ptr<TokenComponentSignature> TokenComponentSignature::create(const std::string& signature)
 {
+	return std::shared_ptr<TokenComponentSignature>(
+		new TokenComponentSignature(Dictionary::getInstance()->getWordId(signature)));
 }
 
 TokenComponentSignature::~TokenComponentSignature()
@@ -16,5 +19,15 @@ std::shared_ptr<TokenComponent> TokenComponentSignature::copy() const
 
 const std::string& TokenComponentSignature::getSignature() const
 {
-	return m_signature;
+	return Dictionary::getInstance()->getWord(m_wordId);
+}
+
+bool TokenComponentSignature::operator==(const TokenComponentSignature& other) const
+{
+	return m_wordId == other.m_wordId;
+}
+
+TokenComponentSignature::TokenComponentSignature(Id wordId)
+	: m_wordId(wordId)
+{
 }

src/lib/data/graph/token_component/TokenComponentSignature.h

@@ -1,23 +1,30 @@
 #ifndef TOKEN_COMPONENT_SIGNATURE_H
 #define TOKEN_COMPONENT_SIGNATURE_H
 
+#include <memory>
 #include <string>
 
 #include "data/graph/token_component/TokenComponent.h"
+#include "utility/types.h"
 
 class TokenComponentSignature
 	: public TokenComponent
 {
 public:
-	TokenComponentSignature(std::string signature);
+	static std::shared_ptr<TokenComponentSignature> create(const std::string& signature);
+
 	virtual ~TokenComponentSignature();
 
 	virtual std::shared_ptr<TokenComponent> copy() const;
 
 	const std::string& getSignature() const;
 
+	bool operator==(const TokenComponentSignature& other) const;
+
 private:
-	const std::string m_signature;
+	TokenComponentSignature(Id wordId);
+
+	const Id m_wordId;
 };
 
 #endif // TOKEN_COMPONENT_SIGNATURE_H

src/lib/data/query/QueryCommand.cpp

@@ -1,52 +1,5 @@
 #include "data/query/QueryCommand.h"
 
-QueryCommand::QueryCommand(const std::string& name)
-	: m_type(COMMAND_INVALID)
-	, m_name(name)
-{
-	std::map<std::string, CommandType> commandMap = getCommandTypeMap();
-	std::map<std::string, CommandType>::iterator it = commandMap.find(name);
-
-	if (it != commandMap.end())
-	{
-		m_type = it->second;
-	}
-}
-
-QueryCommand::~QueryCommand()
-{
-}
-
-bool QueryCommand::isCommand() const
-{
-	return true;
-}
-
-bool QueryCommand::isOperator() const
-{
-	return false;
-}
-
-bool QueryCommand::isToken() const
-{
-	return false;
-}
-
-bool QueryCommand::isComplete() const
-{
-	return m_type != COMMAND_INVALID;
-}
-
-void QueryCommand::print(std::ostream& ostream) const
-{
-	ostream << m_name;
-}
-
-QueryCommand::CommandType QueryCommand::getType() const
-{
-	return m_type;
-}
-
 std::map<std::string, QueryCommand::CommandType> QueryCommand::getCommandTypeMap()
 {
 	static std::map<std::string, CommandType> commandMap;
@@ -94,3 +47,50 @@ std::map<std::string, QueryCommand::CommandType> QueryCommand::getCommandTypeMap
 
 	return commandMap;
 }
+
+QueryCommand::QueryCommand(const std::string& name)
+	: m_type(COMMAND_INVALID)
+	, m_name(name)
+{
+	std::map<std::string, CommandType> commandMap = getCommandTypeMap();
+	std::map<std::string, CommandType>::iterator it = commandMap.find(name);
+
+	if (it != commandMap.end())
+	{
+		m_type = it->second;
+	}
+}
+
+QueryCommand::~QueryCommand()
+{
+}
+
+bool QueryCommand::isCommand() const
+{
+	return true;
+}
+
+bool QueryCommand::isOperator() const
+{
+	return false;
+}
+
+bool QueryCommand::isToken() const
+{
+	return false;
+}
+
+bool QueryCommand::isComplete() const
+{
+	return m_type != COMMAND_INVALID;
+}
+
+void QueryCommand::print(std::ostream& ostream) const
+{
+	ostream << m_name;
+}
+
+QueryCommand::CommandType QueryCommand::getType() const
+{
+	return m_type;
+}

src/lib/data/query/QueryCommand.h

@@ -44,6 +44,8 @@ public:
 		COMMAND_SUB_CLASS
 	};
 
+	static std::map<std::string, CommandType> getCommandTypeMap();
+
 	QueryCommand(const std::string& name);
 	~QueryCommand();
 
@@ -58,8 +60,6 @@ public:
 	CommandType getType() const;
 
 private:
-	static std::map<std::string, CommandType> getCommandTypeMap();
-
 	CommandType m_type;
 	const std::string m_name;
 };

src/lib/data/query/QueryTree.cpp

@@ -97,7 +97,11 @@ std::shared_ptr<QueryNode> QueryTree::buildTree(std::deque<std::string>& tokens,
 			std::shared_ptr<QueryNode> rightNode = buildTree(tokens, nullptr);
 			std::shared_ptr<QueryOperator> rightOperatorNode = std::dynamic_pointer_cast<QueryOperator>(rightNode);
 
-			if (rightNode->isOperator() && !rightNode->isGroup() &&
+			if (!rightNode)
+			{
+				m_valid = false;
+			}
+			else if (rightNode->isOperator() && !rightNode->isGroup() &&
 				operatorNode->lowerPrecedence(*rightOperatorNode.get()))
 			{
 				operatorNode->setRight(rightOperatorNode->getLeft());

src/lib/utility/text/Dictionary.cpp

@@ -0,0 +1,81 @@
+#include "utility/text/Dictionary.h"
+
+#include "utility/utilityString.h"
+
+std::shared_ptr<Dictionary> Dictionary::getInstance()
+{
+	std::lock_guard<std::mutex> lockGuard(s_instanceMutex);
+	if (!s_instance)
+	{
+		s_instance = std::shared_ptr<Dictionary>(new Dictionary());
+	}
+	return s_instance;
+}
+
+Dictionary::~Dictionary()
+{
+}
+
+Id Dictionary::getWordId(const std::string& word)
+{
+	for (std::unordered_map<Id, std::string>::const_iterator it = m_words.begin(); it != m_words.end(); it++)
+	{
+		if (it->second == word)
+		{
+			return it->first;
+		}
+	}
+
+	m_words.emplace(++s_nextId, word);
+	return s_nextId;
+}
+
+std::deque<Id> Dictionary::getWordIds(const std::string& wordList, const std::string& delimiter)
+{
+	std::deque<std::string> words = utility::split<std::deque<std::string>>(wordList, delimiter);
+	std::deque<Id> ids;
+
+	for (const std::string& word: words)
+	{
+		ids.push_back(getWordId(word));
+	}
+
+	return ids;
+}
+
+const std::string& Dictionary::getWord(Id id) const
+{
+	std::unordered_map<Id, std::string>::const_iterator it = m_words.find(id);
+
+	if (it != m_words.end())
+	{
+		return it->second;
+	}
+
+	return m_emptyWord;
+}
+
+std::string Dictionary::getWord(const std::deque<Id> ids, const std::string& delimiter) const
+{
+	std::string word;
+
+	for (std::deque<Id>::const_iterator it = ids.begin(); it != ids.end(); it++)
+	{
+		if (it != ids.begin())
+		{
+			word += delimiter;
+		}
+
+		word += getWord(*it);
+	}
+
+	return word;
+}
+
+Dictionary::Dictionary()
+{
+}
+
+std::shared_ptr<Dictionary> Dictionary::s_instance;
+std::mutex Dictionary::s_instanceMutex;
+Id Dictionary::s_nextId = 0;

src/lib/utility/text/Dictionary.h

@@ -0,0 +1,38 @@
+#ifndef DICTIONARY_H
+#define DICTIONARY_H
+
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+
+#include "utility/types.h"
+
+class Dictionary
+{
+public:
+	static std::shared_ptr<Dictionary> getInstance();
+	~Dictionary();
+
+	Id getWordId(const std::string& word);
+	std::deque<Id> getWordIds(const std::string& wordList, const std::string& delimiter);
+
+	// Note: References to values in an unordered_map don't change on rehashing so they can be saved and used elsewhere.
+	const std::string& getWord(Id id) const;
+	std::string getWord(const std::deque<Id> ids, const std::string& delimiter) const;
+
+private:
+	Dictionary();
+	Dictionary(const Dictionary&);
+	void operator=(const Dictionary&);
+
+	static std::shared_ptr<Dictionary> s_instance;
+	static std::mutex s_instanceMutex;
+	static Id s_nextId;
+
+	std::unordered_map<Id, std::string> m_words;
+	std::string m_emptyWord;
+};
+
+#endif // DICTIONARY_H