ui/logic/data: refreshing only source files that have been updated

This change adds the refresh component that allows for refreshing the source code via UI or shortcut. If automatic
refreshing is activated via the UI, the code is refreshed anytime the window gets focus. The contents of all the
updated source files get removed from Storage, before reparsing them. File dependencies are not respected yet.

src/lib/data/Storage.cpp

@@ -1,7 +1,5 @@
 #include "data/Storage.h"
 
-#include <iostream>
-
 #include "utility/logging/logging.h"
 #include "utility/utilityString.h"
 
@@ -38,6 +36,66 @@ void Storage::clear()
 	m_graph.clear();
 	m_locationCollection.clear();
 	m_tokenIndex.clear();
+
+	m_errorMessages.clear();
+	m_errorLocationCollection.clear();
+}
+
+void Storage::clearFileData(const std::vector<std::string>& filePaths)
+{
+	for (const std::string& filePath : filePaths)
+	{
+		TokenLocationFile* errorFile = m_errorLocationCollection.findTokenLocationFileByPath(filePath);
+		if (errorFile)
+		{
+			m_errorLocationCollection.removeTokenLocationFile(errorFile);
+		}
+
+		TokenLocationFile* file = m_locationCollection.findTokenLocationFileByPath(filePath);
+		if (!file)
+		{
+			continue;
+		}
+
+		file->forEachTokenLocation(
+			[&](TokenLocation* location)
+			{
+				if (location->isEndTokenLocation())
+				{
+					return;
+				}
+
+				Token* token = m_graph.getTokenById(location->getTokenId());
+				if (!token)
+				{
+					return;
+				}
+
+				token->removeLocationId(location->getId());
+				if (token->getLocationIds().size())
+				{
+					return;
+				}
+
+				if (token->isEdge())
+				{
+					Edge* edge = dynamic_cast<Edge*>(token);
+					Node* from = edge->getFrom();
+					Node* to = edge->getTo();
+
+					m_graph.removeEdge(edge);
+					removeNodeIfUnreferenced(from);
+					removeNodeIfUnreferenced(to);
+				}
+				else
+				{
+					removeNodeIfUnreferenced(dynamic_cast<Node*>(token));
+				}
+			}
+		);
+
+		m_locationCollection.removeTokenLocationFile(file);
+	}
 }
 
 void Storage::logGraph() const
@@ -52,7 +110,7 @@ void Storage::logLocations() const
 
 size_t Storage::getErrorCount() const
 {
-	return m_errorMessages.size();
+	return m_errorLocationCollection.getTokenLocationCount();
 }
 
 void Storage::onError(const ParseLocation& location, const std::string& message)
@@ -64,15 +122,38 @@ void Storage::onError(const ParseLocation& location, const std::string& message)
 		return;
 	}
 
-	Id errorId = m_errorMessages.size();
+	bool duplicate = false;
+	std::string filePath = location.filePath;
+	TokenLocationFile* file = m_errorLocationCollection.findTokenLocationFileByPath(filePath);
 
-	TokenLocation* loc = m_errorLocationCollection.addTokenLocation(
-		errorId, location.filePath,
-		location.startLineNumber, location.startColumnNumber,
-		location.endLineNumber, location.endColumnNumber
-	);
+	if (file)
+	{
+		file->forEachTokenLocation(
+			[&](TokenLocation* loc)
+			{
+				if (loc->isStartTokenLocation() &&
+					loc->getLineNumber() == location.startLineNumber &&
+					loc->getColumnNumber() == location.startColumnNumber &&
+					m_errorMessages[loc->getTokenId()] == message)
+				{
+					duplicate = true;
+				}
+			}
+		);
+	}
 
-	m_errorMessages.push_back(message);
+	if (!duplicate)
+	{
+		Id errorId = m_errorMessages.size();
+
+		TokenLocation* loc = m_errorLocationCollection.addTokenLocation(
+			errorId, filePath,
+			location.startLineNumber, location.startColumnNumber,
+			location.endLineNumber, location.endColumnNumber
+		);
+
+		m_errorMessages.push_back(message);
+	}
 }
 
 Id Storage::onTypedefParsed(
@@ -1030,6 +1111,20 @@ bool Storage::getSubQuerySearchResults(
 	return true;
 }
 
+void Storage::removeNodeIfUnreferenced(Node* node)
+{
+	Id tokenId = node->getId();
+	SearchNode* searchNode = m_tokenIndex.getNode(node->getTokenComponentName()->getSearchNode());
+
+	bool removed = m_graph.removeNodeIfUnreferencedRecursive(node);
+
+	if (removed && searchNode)
+	{
+		searchNode->removeTokenId(tokenId);
+		m_tokenIndex.removeNodeIfUnreferencedRecursive(searchNode);
+	}
+}
+
 void Storage::log(std::string type, std::string str, const ParseLocation& location) const
 {
 	LOG_INFO_STREAM(

src/lib/data/Storage.h

@@ -23,6 +23,7 @@ public:
 	virtual ~Storage();
 
 	void clear();
+	void clearFileData(const std::vector<std::string>& filePaths);
 
 	void logGraph() const;
 	void logLocations() const;
@@ -125,7 +126,6 @@ protected:
 	const SearchIndex& getSearchIndex() const;
 
 private:
-
 	Node* addNodeHierarchy(Node::NodeType type, std::vector<std::string> nameHierarchy);
 	Node* addNodeHierarchyWithDistinctSignature(Node::NodeType type, const ParseFunction& function);
 
@@ -141,6 +141,8 @@ private:
 	bool getSubQuerySearchResults(
 		const std::string& query, const std::string& word, SearchResults* results) const;
 
+	void removeNodeIfUnreferenced(Node* node);
+
 	void log(std::string type, std::string str, const ParseLocation& location) const;
 
 	StorageGraph m_graph;

src/lib/data/graph/Graph.cpp

@@ -114,13 +114,16 @@ void Graph::removeNode(Node* node)
 		return;
 	}
 
-	node->forEachEdgeOfType(Edge::EDGE_MEMBER, [this, node](Edge* e)
-	{
-		if (node == e->getFrom())
+	node->forEachEdgeOfType(
+		Edge::EDGE_MEMBER,
+		[this, node](Edge* e)
 		{
-			this->removeNode(e->getTo());
+			if (node == e->getFrom())
+			{
+				this->removeNode(e->getTo());
+			}
 		}
-	});
+	);
 
 	node->forEachEdge(
 		[this](Edge* e)
@@ -131,7 +134,7 @@ void Graph::removeNode(Node* node)
 
 	if (node->getEdges().size())
 	{
-		LOG_ERROR("Node has still edges.");
+		LOG_ERROR("Node still has edges.");
 	}
 
 	m_nodes.erase(it);
@@ -154,6 +157,25 @@ void Graph::removeEdge(Edge* edge)
 	m_edges.erase(it);
 }
 
+bool Graph::removeNodeIfUnreferencedRecursive(Node* node)
+{
+	if (!node->hasReferences())
+	{
+		Node* parent = node->getParentNode();
+
+		removeNode(node);
+
+		if (parent)
+		{
+			removeNodeIfUnreferencedRecursive(parent);
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
 Node* Graph::findNode(std::function<bool(Node*)> func) const
 {
 	std::map<Id, std::shared_ptr<Node>>::const_iterator it = find_if(m_nodes.begin(), m_nodes.end(),

src/lib/data/graph/Graph.h

@@ -40,6 +40,7 @@ public:
 
 	void removeNode(Node* node);
 	void removeEdge(Edge* edge);
+	bool removeNodeIfUnreferencedRecursive(Node* node);
 
 	Node* findNode(std::function<bool(Node*)> func) const;
 	Edge* findEdge(std::function<bool(Edge*)> func) const;

src/lib/data/graph/Node.cpp

@@ -203,6 +203,37 @@ void Node::forEachChildNode(std::function<void(Node*)> func) const
 	);
 }
 
+bool Node::hasReferences() const
+{
+	if (getLocationIds().size() > 0)
+	{
+		return true;
+	}
+
+	bool hasChildrenWithReferences = false;
+	size_t childNodeCount = 0;
+
+	forEachEdgeOfType(
+		Edge::EDGE_MEMBER,
+		[&](Edge* edge)
+		{
+			childNodeCount++;
+
+			if (!hasChildrenWithReferences && edge->getTo() != this && edge->getTo()->hasReferences())
+			{
+				hasChildrenWithReferences = true;
+			}
+		}
+	);
+
+	if (hasChildrenWithReferences || getEdges().size() > childNodeCount)
+	{
+		return true;
+	}
+
+	return false;
+}
+
 bool Node::isNode() const
 {
 	return true;

src/lib/data/graph/Node.h

@@ -68,6 +68,8 @@ public:
 	void forEachEdgeOfType(Edge::EdgeType type, std::function<void(Edge*)> func) const;
 	void forEachChildNode(std::function<void(Node*)> func) const;
 
+	bool hasReferences() const;
+
 	// Token implementation.
 	virtual bool isNode() const;
 	virtual bool isEdge() const;

src/lib/data/location/TokenLocationCollection.cpp

@@ -1,9 +1,13 @@
 #include "data/location/TokenLocationCollection.h"
 
+#include <algorithm>
+
+#include "utility/FileSystem.h"
+#include "utility/logging/logging.h"
+
 #include "data/location/TokenLocation.h"
 #include "data/location/TokenLocationFile.h"
 #include "data/location/TokenLocationLine.h"
-#include "utility/logging/logging.h"
 
 TokenLocationCollection::TokenLocationCollection()
 {
@@ -85,7 +89,13 @@ TokenLocation* TokenLocationCollection::findTokenLocationById(Id id) const
 
 TokenLocationFile* TokenLocationCollection::findTokenLocationFileByPath(const std::string& filePath) const
 {
-	std::map<std::string, std::shared_ptr<TokenLocationFile> >::const_iterator it = m_files.find(filePath);
+	std::map<std::string, std::shared_ptr<TokenLocationFile>>::const_iterator it =
+		find_if(m_files.begin(), m_files.end(),
+			[&](const std::pair<std::string, std::shared_ptr<TokenLocationFile>>& p)
+			{
+				return FileSystem::equivalent(p.first, filePath);
+			}
+		);
 
 	if (it != m_files.end())
 	{
@@ -119,6 +129,18 @@ void TokenLocationCollection::forEachTokenLocation(std::function<void(TokenLocat
 	}
 }
 
+void TokenLocationCollection::removeTokenLocationFile(TokenLocationFile* file)
+{
+	file->forEachTokenLocation(
+		[&](TokenLocation* location)
+		{
+			m_locations.erase(location->getId());
+		}
+	);
+
+	m_files.erase(file->getFilePath());
+}
+
 TokenLocation* TokenLocationCollection::addTokenLocationAsPlainCopy(const TokenLocation* location)
 {
 	const std::string& filePath = location->getTokenLocationLine()->getTokenLocationFile()->getFilePath();

src/lib/data/location/TokenLocationCollection.h

@@ -41,6 +41,8 @@ public:
 	void forEachTokenLocationLine(std::function<void(TokenLocationLine*)> func) const;
 	void forEachTokenLocation(std::function<void(TokenLocation*)> func) const;
 
+	void removeTokenLocationFile(TokenLocationFile* file);
+
 	TokenLocation* addTokenLocationAsPlainCopy(const TokenLocation* location);
 
 	void clear();

src/lib/data/search/SearchIndex.cpp

@@ -3,6 +3,8 @@
 #include <algorithm>
 #include <cctype>
 
+#include "utility/logging/logging.h"
+
 #include "data/search/SearchMatch.h"
 
 std::vector<SearchMatch> SearchIndex::getMatches(
@@ -34,6 +36,11 @@ void SearchIndex::clear()
 	m_root.m_nodes.clear();
 }
 
+size_t SearchIndex::getNodeCount() const
+{
+	return m_root.getNodeCount() - 1;
+}
+
 Id SearchIndex::getWordId(const std::string& word)
 {
 	return m_dictionary.getWordId(word);
@@ -71,6 +78,50 @@ SearchNode* SearchIndex::getNode(const std::string& fullName) const
 	return nullptr;
 }
 
+SearchNode* SearchIndex::getNode(const SearchNode* searchNode) const
+{
+	std::deque<Id> nameIds = searchNode->getNameIdsRecursive();
+
+	if (nameIds.size())
+	{
+		return m_root.getNodeRecursive(&nameIds).get();
+	}
+
+	return nullptr;
+}
+
+void SearchIndex::removeNode(SearchNode* searchNode)
+{
+	SearchNode* parent = searchNode->getParent();
+
+	if (!parent)
+	{
+		LOG_ERROR_STREAM(<< "SearchNode to be removed has no parent: " << searchNode->getFullName());
+		return;
+	}
+
+	parent->removeSearchNode(searchNode);
+}
+
+bool SearchIndex::removeNodeIfUnreferencedRecursive(SearchNode* searchNode)
+{
+	if (!searchNode->hasTokenIdsRecursive())
+	{
+		SearchNode* parent = searchNode->getParent();
+
+		removeNode(searchNode);
+
+		if (parent && parent != &m_root)
+		{
+			removeNodeIfUnreferencedRecursive(parent);
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
 SearchResults SearchIndex::runFuzzySearch(const std::string& query) const
 {
 	return m_root.runFuzzySearch(query);

src/lib/data/search/SearchIndex.h

@@ -20,11 +20,17 @@ public:
 
 	void clear();
 
+	size_t getNodeCount() const;
+
 	Id getWordId(const std::string& word);
 	const std::string& getWord(Id wordId) const;
 
 	SearchNode* addNode(std::vector<std::string> nameHierarchy);
 	SearchNode* getNode(const std::string& fullName) const;
+	SearchNode* getNode(const SearchNode* searchNode) const;
+
+	void removeNode(SearchNode* searchNode);
+	bool removeNodeIfUnreferencedRecursive(SearchNode* searchNode);
 
 	SearchResults runFuzzySearch(const std::string& query) const;
 	std::vector<SearchMatch> runFuzzySearchAndGetMatches(const std::string& query) const;

src/lib/data/search/SearchNode.cpp

@@ -19,6 +19,18 @@ SearchNode::~SearchNode()
 {
 }
 
+size_t SearchNode::getNodeCount() const
+{
+	size_t count = 1;
+
+	for (std::shared_ptr<SearchNode> n: m_nodes)
+	{
+		count += n->getNodeCount();
+	}
+
+	return count;
+}
+
 const std::string& SearchNode::getName() const
 {
 	return m_name;
@@ -28,7 +40,7 @@ std::vector<std::string> SearchNode::getNameHierarchy() const
 {
 	std::vector<std::string> nameHierarchy;
 	const SearchNode* parent = getParent();
-	if (parent)
+	if (parent && parent->m_nameId)
 	{
 		nameHierarchy = parent->getNameHierarchy();
 	}
@@ -53,6 +65,22 @@ Id SearchNode::getNameId() const
 	return m_nameId;
 }
 
+std::deque<Id> SearchNode::getNameIdsRecursive() const
+{
+	std::deque<Id> ids;
+
+	ids.push_front(m_nameId);
+
+	SearchNode* parent = m_parent;
+	while (parent && parent->m_nameId)
+	{
+		ids.push_front(parent->getNameId());
+		parent = parent->getParent();
+	}
+
+	return ids;
+}
+
 Id SearchNode::getFirstTokenId() const
 {
 	if (m_tokenIds.size())
@@ -68,14 +96,37 @@ const std::set<Id>& SearchNode::getTokenIds() const
 	return m_tokenIds;
 }
 
+bool SearchNode::hasTokenIdsRecursive() const
+{
+	if (m_tokenIds.size())
+	{
+		return true;
+	}
+
+	for (std::shared_ptr<SearchNode> n: m_nodes)
+	{
+		if (n->hasTokenIdsRecursive())
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
 void SearchNode::addTokenId(Id tokenId)
 {
 	m_tokenIds.insert(tokenId);
 }
 
+void SearchNode::removeTokenId(Id tokenId)
+{
+	m_tokenIds.erase(tokenId);
+}
+
 SearchNode* SearchNode::getParent() const
 {
-	if (m_parent && m_parent->m_nameId)
+	if (m_parent)
 	{
 		return m_parent;
 	}
@@ -183,6 +234,18 @@ std::shared_ptr<SearchNode> SearchNode::getNodeRecursive(std::deque<Id>* nameIds
 	return nullptr;
 }
 
+void SearchNode::removeSearchNode(SearchNode* node)
+{
+	for (std::set<std::shared_ptr<SearchNode>>::iterator it = m_nodes.begin(); it != m_nodes.end(); it++)
+	{
+		if ((*it)->m_nameId == node->m_nameId)
+		{
+			m_nodes.erase(it);
+			return;
+		}
+	}
+}
+
 SearchMatch SearchNode::fuzzyMatchData(const std::string& query, const SearchNode* parent) const
 {
 	SearchMatch data;

src/lib/data/search/SearchNode.h

@@ -22,15 +22,21 @@ public:
 	SearchNode(SearchNode* parent, const std::string& name, Id nameId);
 	~SearchNode();
 
+	size_t getNodeCount() const;
+
 	const std::string& getName() const;
 	std::vector<std::string> getNameHierarchy() const;
 	std::string getFullName() const;
 
 	Id getNameId() const;
+	std::deque<Id> getNameIdsRecursive() const;
 
 	Id getFirstTokenId() const;
 	const std::set<Id>& getTokenIds() const;
+	bool hasTokenIdsRecursive() const;
+
 	void addTokenId(Id tokenId);
+	void removeTokenId(Id tokenId);
 
 	SearchNode* getParent() const;
 	std::deque<SearchNode*> getParentsWithoutTokenId();
@@ -46,11 +52,12 @@ private:
 	typedef std::multimap<size_t, const SearchNode*> FuzzyMap;
 	typedef FuzzyMap::const_iterator FuzzyMapIterator;
 
-
 	// Accessed by SearchIndex
 	std::shared_ptr<SearchNode> addNodeRecursive(std::deque<Id>* nameIds, const Dictionary& dictionary);
 	std::shared_ptr<SearchNode> getNodeRecursive(std::deque<Id>* nameIds) const;
 
+	void removeSearchNode(SearchNode* node);
+
 	SearchMatch fuzzyMatchData(const std::string& query, const SearchNode* parent) const;
 
 	friend class SearchIndex;