logic: record and display include edges for non-indexed files

* non-indexed files are shown in graph view and in code view
* non-indexed files are ignored in search view, overview, stats and refresh dialog

src/lib/component/controller/CodeController.cpp

@@ -46,7 +46,7 @@ void CodeController::handleMessage(MessageActivateAll* message)
 
 	statsSnippet.reduced = true;
 
-	statsSnippet.locationFile = std::make_shared<SourceLocationFile>(FilePath(), true, true);
+	statsSnippet.locationFile = std::make_shared<SourceLocationFile>(FilePath(), true, true, true);
 
 	std::vector<std::string> description = getProjectDescription(statsSnippet.locationFile.get());
 

src/lib/component/controller/GraphController.cpp

@@ -172,6 +172,30 @@ void GraphController::handleMessage(MessageActivateTrail* message)
 	std::shared_ptr<Graph> graph = m_storageAccess->getGraphForTrail(
 		message->originId, message->targetId, message->trailType, message->depth);
 
+	// remove non-indexed files from include graph if indexed file is origin
+	if (message->trailType & Edge::EDGE_INCLUDE)
+	{
+		Node* fileNode = graph->getNodeById(message->originId ? message->originId : message->targetId);
+		if (fileNode && fileNode->isDefined())
+		{
+			std::vector<Node*> nodesToRemove;
+			graph->forEachNode(
+				[&nodesToRemove](Node* node)
+				{
+					if (!node->isDefined())
+					{
+						nodesToRemove.push_back(node);
+					}
+				}
+			);
+
+			for (Node* node : nodesToRemove)
+			{
+				graph->removeNode(node);
+			}
+		}
+	}
+
 	if (graph->getNodeCount() > 1000)
 	{
 		int r = Application::getInstance()->handleDialog(

src/lib/component/view/GraphViewStyle.cpp

@@ -697,6 +697,9 @@ GraphViewStyle::EdgeStyle GraphViewStyle::getStyleForEdgeType(
 
 	case Edge::EDGE_INCLUDE:
 		style.zValue = isActive ? 2 : -3;
+	case Edge::EDGE_MACRO_USAGE:
+		style.originOffset.y = 0;
+		style.targetOffset.y = 0;
 	default:
 		break;
 	}

src/lib/data/access/StorageAccessProxy.cpp

@@ -256,7 +256,7 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsForFil
 		return m_subject->getSourceLocationsForFile(filePath);
 	}
 
-	return std::make_shared<SourceLocationFile>(FilePath(), false, false);
+	return std::make_shared<SourceLocationFile>(FilePath(), false, false, false);
 }
 
 std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsForLinesInFile(
@@ -268,7 +268,7 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsForLin
 		return m_subject->getSourceLocationsForLinesInFile(filePath, startLine, endLine);
 	}
 
-	return std::make_shared<SourceLocationFile>(FilePath(), false, false);
+	return std::make_shared<SourceLocationFile>(FilePath(), false, false, false);
 }
 
 std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsOfTypeInFile(
@@ -280,7 +280,7 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsOfType
 		return m_subject->getSourceLocationsOfTypeInFile(filePath, type);
 	}
 
-	return std::make_shared<SourceLocationFile>(FilePath(), false, false);
+	return std::make_shared<SourceLocationFile>(FilePath(), false, false, false);
 }
 
 std::shared_ptr<SourceLocationFile> StorageAccessProxy::getCommentLocationsInFile(const FilePath& filePath) const
@@ -290,7 +290,7 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getCommentLocationsInFil
 		return m_subject->getCommentLocationsInFile(filePath);
 	}
 
-	return std::make_shared<SourceLocationFile>(FilePath(), false, false);
+	return std::make_shared<SourceLocationFile>(FilePath(), false, false, false);
 }
 
 std::shared_ptr<TextAccess> StorageAccessProxy::getFileContent(const FilePath& filePath) const

src/lib/data/graph/Graph.cpp

@@ -41,7 +41,7 @@ void Graph::forEachToken(std::function<void(Token*)> func) const
 	forEachEdge(func);
 }
 
-Node* Graph::createNode(Id id, NodeType type, const NameHierarchy& nameHierarchy, bool defined)
+Node* Graph::createNode(Id id, NodeType type, const NameHierarchy& nameHierarchy, DefinitionKind definitionKind)
 {
 	Node* n = getNodeById(id);
 	if (n)
@@ -49,7 +49,7 @@ Node* Graph::createNode(Id id, NodeType type, const NameHierarchy& nameHierarchy
 		return n;
 	}
 
-	std::shared_ptr<Node> node = std::make_shared<Node>(id, type, nameHierarchy, defined);
+	std::shared_ptr<Node> node = std::make_shared<Node>(id, type, nameHierarchy, definitionKind);
 	m_nodes.emplace(node->getId(), node);
 	return node.get();
 }
@@ -122,24 +122,36 @@ void Graph::removeNode(Node* node)
 		return;
 	}
 
+	std::vector<Node*> childNodesToRemove;
 	node->forEachEdgeOfType(
 		Edge::EDGE_MEMBER,
-		[this, node](Edge* e)
+		[node, &childNodesToRemove](Edge* e)
 		{
 			if (node == e->getFrom())
 			{
-				this->removeNode(e->getTo());
+				childNodesToRemove.push_back(e->getTo());
 			}
 		}
 	);
 
+	for (Node* childNode : childNodesToRemove)
+	{
+		removeNode(childNode);
+	}
+
+	std::vector<Edge*> edgesToRemove;
 	node->forEachEdge(
-		[this](Edge* e)
+		[&edgesToRemove](Edge* e)
 		{
-			this->removeEdgeInternal(e);
+			edgesToRemove.push_back(e);
 		}
 	);
 
+	for (Edge* edge : edgesToRemove)
+	{
+		removeEdgeInternal(edge);
+	}
+
 	if (node->getEdgeCount())
 	{
 		LOG_ERROR("Node still has edges.");
@@ -349,11 +361,10 @@ void Graph::printBasic(std::wostream& ostream) const
 
 void Graph::removeEdgeInternal(Edge* edge)
 {
-	std::map<Id, std::shared_ptr<Edge> >::const_iterator it = m_edges.find(edge->getId());
+	std::map<Id, std::shared_ptr<Edge>>::const_iterator it = m_edges.find(edge->getId());
 	if (it != m_edges.end() && it->second.get() == edge)
 	{
 		m_edges.erase(it);
-		return;
 	}
 }
 

src/lib/data/graph/Graph.h

@@ -27,7 +27,7 @@ public:
 	void forEachEdge(std::function<void(Edge*)> func) const;
 	void forEachToken(std::function<void(Token*)> func) const;
 
-	Node* createNode(Id id, NodeType type, const NameHierarchy& nameHierarchy, bool defined);
+	Node* createNode(Id id, NodeType type, const NameHierarchy& nameHierarchy, DefinitionKind definitionKind);
 	Edge* createEdge(Id id, Edge::EdgeType type, Node* from, Node* to);
 
 	size_t getNodeCount() const;

src/lib/data/graph/Node.cpp

@@ -9,13 +9,11 @@
 #include "data/graph/token_component/TokenComponentConst.h"
 #include "data/graph/token_component/TokenComponentStatic.h"
 
-Node::Node(Id id, NodeType type, const NameHierarchy& nameHierarchy, bool defined)
+Node::Node(Id id, NodeType type, const NameHierarchy& nameHierarchy, DefinitionKind definitionKind)
 	: Token(id)
 	, m_type(type)
 	, m_nameHierarchy(nameHierarchy)
-	, m_defined(defined)
-	, m_implicit(false)
-	, m_explicit(false)
+	, m_definitionKind(definitionKind)
 	, m_childCount(0)
 {
 }
@@ -24,9 +22,7 @@ Node::Node(const Node& other)
 	: Token(other)
 	, m_type(other.m_type)
 	, m_nameHierarchy(other.m_nameHierarchy)
-	, m_defined(other.m_defined)
-	, m_implicit(other.m_implicit)
-	, m_explicit(other.m_explicit)
+	, m_definitionKind(other.m_definitionKind)
 	, m_childCount(other.m_childCount)
 {
 }
@@ -75,32 +71,17 @@ NameHierarchy Node::getNameHierarchy() const
 
 bool Node::isDefined() const
 {
-	return m_defined;
-}
-
-void Node::setDefined(bool defined)
-{
-	m_defined = defined;
+	return m_definitionKind != DEFINITION_NONE;
 }
 
 bool Node::isImplicit() const
 {
-	return m_implicit;
-}
-
-void Node::setImplicit(bool implicit)
-{
-	m_implicit = implicit;
+	return m_definitionKind == DEFINITION_IMPLICIT;
 }
 
 bool Node::isExplicit() const
 {
-	return m_explicit;
-}
-
-void Node::setExplicit(bool bExplicit)
-{
-	m_explicit = bExplicit;
+	return m_definitionKind == DEFINITION_EXPLICIT;
 }
 
 size_t Node::getChildCount() const

src/lib/data/graph/Node.h

@@ -6,6 +6,7 @@
 #include <map>
 #include <string>
 
+#include "data/DefinitionKind.h"
 #include "data/graph/Edge.h"
 #include "data/graph/Token.h"
 #include "data/name/NameHierarchy.h"
@@ -15,7 +16,7 @@ class Node
 	: public Token
 {
 public:
-	Node(Id id, NodeType type, const NameHierarchy& nameHierarchy, bool defined);
+	Node(Id id, NodeType type, const NameHierarchy& nameHierarchy, DefinitionKind definitionKind);
 	Node(const Node& other);
 	virtual ~Node();
 
@@ -28,13 +29,8 @@ public:
 	NameHierarchy getNameHierarchy() const;
 
 	bool isDefined() const;
-	void setDefined(bool defined);
-
 	bool isImplicit() const;
-	void setImplicit(bool implicit);
-
 	bool isExplicit() const;
-	void setExplicit(bool bExplicit);
 
 	size_t getChildCount() const;
 	void setChildCount(size_t childCount);
@@ -73,9 +69,7 @@ private:
 
 	NodeType m_type;
 	NameHierarchy m_nameHierarchy;
-	bool m_defined;
-	bool m_implicit;
-	bool m_explicit;
+	DefinitionKind m_definitionKind;
 
 	size_t m_childCount;
 };

src/lib/data/graph/token_component/TokenComponentFilePath.cpp

@@ -1,7 +1,8 @@
 #include "data/graph/token_component/TokenComponentFilePath.h"
 
-TokenComponentFilePath::TokenComponentFilePath(const FilePath& path)
+TokenComponentFilePath::TokenComponentFilePath(const FilePath& path, bool complete)
 	: m_path(path)
+	, m_complete(complete)
 {
 }
 
@@ -18,3 +19,8 @@ const FilePath& TokenComponentFilePath::getFilePath() const
 {
 	return m_path;
 }
+
+bool TokenComponentFilePath::isComplete() const
+{
+	return m_complete;
+}

src/lib/data/graph/token_component/TokenComponentFilePath.h

@@ -9,15 +9,17 @@ class TokenComponentFilePath
 	: public TokenComponent
 {
 public:
-	TokenComponentFilePath(const FilePath& path);
+	TokenComponentFilePath(const FilePath& path, bool complete);
 	virtual ~TokenComponentFilePath();
 
 	virtual std::shared_ptr<TokenComponent> copy() const;
 
 	const FilePath& getFilePath() const;
+	bool isComplete() const;
 
 private:
 	const FilePath m_path;
+	const bool m_complete;
 };
 
 #endif // TOKEN_COMPONENT_FILE_PATH_H

src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h

@@ -67,28 +67,30 @@ inline StorageNode fromShared(const SharedStorageNode& node)
 struct SharedStorageFile
 {
 	SharedStorageFile(
-		Id id, const std::string& filePath, const std::string& modificationTime, bool complete, SharedMemory::Allocator* allocator
+		Id id, const std::string& filePath, const std::string& modificationTime, bool indexed, bool complete, SharedMemory::Allocator* allocator
 	)
 		: id(id)
 		, filePath(filePath.c_str(), allocator)
 		, modificationTime(modificationTime.c_str(), allocator)
+		, indexed(indexed)
 		, complete(complete)
 	{}
 
 	Id id;
 	SharedMemory::String filePath;
 	SharedMemory::String modificationTime;
+	bool indexed;
 	bool complete;
 };
 
 inline SharedStorageFile toShared(const StorageFile& file, SharedMemory::Allocator* allocator)
 {
-	return SharedStorageFile(file.id, utility::encodeToUtf8(file.filePath), file.modificationTime, file.complete, allocator);
+	return SharedStorageFile(file.id, utility::encodeToUtf8(file.filePath), file.modificationTime, file.indexed, file.complete, allocator);
 }
 
 inline StorageFile fromShared(const SharedStorageFile& file)
 {
-	return StorageFile(file.id, utility::decodeFromUtf8(file.filePath.c_str()), file.modificationTime.c_str(), file.complete);
+	return StorageFile(file.id, utility::decodeFromUtf8(file.filePath.c_str()), file.modificationTime.c_str(), file.indexed, file.complete);
 }
 
 

src/lib/data/location/SourceLocationCollection.cpp

@@ -113,7 +113,7 @@ SourceLocationFile* SourceLocationCollection::createSourceLocationFile(const Fil
 		return file;
 	}
 
-	std::shared_ptr<SourceLocationFile> filePtr = std::make_shared<SourceLocationFile>(filePath, false, false);
+	std::shared_ptr<SourceLocationFile> filePtr = std::make_shared<SourceLocationFile>(filePath, false, false, false);
 	m_files.emplace(filePath, filePtr);
 	return filePtr.get();
 }

src/lib/data/location/SourceLocationFile.cpp

@@ -1,9 +1,10 @@
 #include "data/location/SourceLocationFile.h"
 
-SourceLocationFile::SourceLocationFile(const FilePath& filePath, bool isWhole, bool isComplete)
+SourceLocationFile::SourceLocationFile(const FilePath& filePath, bool isWhole, bool isComplete, bool isIndexed)
 	: m_filePath(filePath)
 	, m_isWhole(isWhole)
 	, m_isComplete(isComplete)
+	, m_isIndexed(isIndexed)
 {
 }
 
@@ -36,6 +37,16 @@ bool SourceLocationFile::isComplete() const
 	return m_isComplete;
 }
 
+void SourceLocationFile::setIsIndexed(bool isIndexed)
+{
+	m_isIndexed = isIndexed;
+}
+
+bool SourceLocationFile::isIndexed() const
+{
+	return m_isIndexed;
+}
+
 const std::multiset<std::shared_ptr<SourceLocation>, SourceLocationFile::LocationComp>& SourceLocationFile::getSourceLocations() const
 {
 	return m_locations;
@@ -167,9 +178,11 @@ void SourceLocationFile::forEachEndSourceLocation(std::function<void(SourceLocat
 	}
 }
 
-std::shared_ptr<SourceLocationFile> SourceLocationFile::getFilteredByLines(size_t firstLineNumber, size_t lastLineNumber) const
+std::shared_ptr<SourceLocationFile> SourceLocationFile::getFilteredByLines(
+	size_t firstLineNumber, size_t lastLineNumber) const
 {
-	std::shared_ptr<SourceLocationFile> ret = std::make_shared<SourceLocationFile>(getFilePath(), false, isComplete());
+	std::shared_ptr<SourceLocationFile> ret =
+		std::make_shared<SourceLocationFile>(getFilePath(), false, isComplete(), isIndexed());
 
 	for (const std::shared_ptr<SourceLocation>& location : m_locations)
 	{
@@ -184,7 +197,8 @@ std::shared_ptr<SourceLocationFile> SourceLocationFile::getFilteredByLines(size_
 
 std::shared_ptr<SourceLocationFile> SourceLocationFile::getFilteredByType(LocationType type) const
 {
-	std::shared_ptr<SourceLocationFile> ret = std::make_shared<SourceLocationFile>(getFilePath(), false, isComplete());
+	std::shared_ptr<SourceLocationFile> ret =
+		std::make_shared<SourceLocationFile>(getFilePath(), false, isComplete(), isIndexed());
 
 	for (const std::shared_ptr<SourceLocation>& location : m_locations)
 	{

src/lib/data/location/SourceLocationFile.h

@@ -23,7 +23,7 @@ public:
 		}
 	};
 
-	SourceLocationFile(const FilePath& filePath, bool isWhole, bool isComplete);
+	SourceLocationFile(const FilePath& filePath, bool isWhole, bool isComplete, bool isIndexed);
 	virtual ~SourceLocationFile();
 
 	const FilePath& getFilePath() const;
@@ -34,6 +34,9 @@ public:
 	void setIsComplete(bool isComplete);
 	bool isComplete() const;
 
+	void setIsIndexed(bool isIndexed);
+	bool isIndexed() const;
+
 	const std::multiset<std::shared_ptr<SourceLocation>, LocationComp>& getSourceLocations() const;
 
 	size_t getSourceLocationCount() const;
@@ -60,6 +63,7 @@ private:
 	const FilePath m_filePath;
 	bool m_isWhole;
 	bool m_isComplete;
+	bool m_isIndexed;
 
 	std::multiset<std::shared_ptr<SourceLocation>, LocationComp> m_locations;
 	std::map<Id, SourceLocation*> m_locationIndex;

src/lib/data/parser/ParserClient.h

@@ -51,9 +51,9 @@ public:
 
 	void recordError(
 		const ParseLocation& location, const std::wstring& message, bool fatal, bool indexed);
-	
+
 	virtual void recordLocalSymbol(const std::wstring& name, const ParseLocation& location) = 0;
-	virtual void recordFile(const FileInfo& fileInfo) = 0;
+	virtual void recordFile(const FileInfo& fileInfo, bool indexed) = 0;
 	virtual void recordComment(const ParseLocation& location) = 0;
 
 	bool hasFatalErrors() const;

src/lib/data/parser/ParserClientImpl.cpp

@@ -79,10 +79,10 @@ void ParserClientImpl::recordLocalSymbol(const std::wstring& name, const ParseLo
 	addSourceLocation(localSymbolId, location, locationTypeToInt(LOCATION_LOCAL_SYMBOL));
 }
 
-void ParserClientImpl::recordFile(const FileInfo& fileInfo)
+void ParserClientImpl::recordFile(const FileInfo& fileInfo, bool indexed)
 {
 	const Id nodeId = addNodeHierarchy(NameHierarchy(fileInfo.path.wstr(), NAME_DELIMITER_FILE), NodeType::NODE_FILE);
-	addFile(nodeId, fileInfo.path, fileInfo.lastWriteTime.toString());
+	addFile(nodeId, fileInfo.path, fileInfo.lastWriteTime.toString(), indexed);
 }
 
 void ParserClientImpl::recordComment(const ParseLocation& location)
@@ -225,14 +225,12 @@ Id ParserClientImpl::addNode(NodeType nodeType, const NameHierarchy& nameHierarc
 	return m_storage->addNode(StorageNodeData(utility::nodeTypeToInt(nodeType.getType()), NameHierarchy::serialize(nameHierarchy)));
 }
 
-void ParserClientImpl::addFile(Id id, const FilePath& filePath, const std::string& modificationTime)
+void ParserClientImpl::addFile(Id id, const FilePath& filePath, const std::string& modificationTime, bool indexed)
 {
-	if (!m_storage)
+	if (m_storage)
 	{
-		return;
+		m_storage->addFile(StorageFile(id, filePath.wstr(), modificationTime, indexed, true));
 	}
-
-	m_storage->addFile(StorageFile(id, filePath.wstr(), modificationTime, true));
 }
 
 void ParserClientImpl::addSymbol(Id id, DefinitionKind definitionKind)

src/lib/data/parser/ParserClientImpl.h

@@ -40,7 +40,7 @@ public:
 		const NameHierarchy& qualifierName, const ParseLocation& location) override;
 	
 	virtual void recordLocalSymbol(const std::wstring& name, const ParseLocation& location) override;
-	virtual void recordFile(const FileInfo& fileInfo) override;
+	virtual void recordFile(const FileInfo& fileInfo, bool indexed) override;
 	virtual void recordComment(const ParseLocation& location) override;
 
 private:
@@ -53,7 +53,7 @@ private:
 	Id addNodeHierarchy(const NameHierarchy& nameHierarchy, NodeType nodeType = NodeType::NODE_SYMBOL);
 
 	Id addNode(NodeType nodeType, const NameHierarchy& nameHierarchy);
-	void addFile(Id id, const FilePath& filePath, const std::string& modificationTime);
+	void addFile(Id id, const FilePath& filePath, const std::string& modificationTime, bool indexed);
 	void addSymbol(Id id, DefinitionKind definitionKind);
 	Id addEdge(int type, Id sourceId, Id targetId);
 	Id addLocalSymbol(const std::wstring& name);

src/lib/data/storage/PersistentStorage.cpp

@@ -83,10 +83,17 @@ void PersistentStorage::addFile(const StorageFile& data)
 	{
 		m_sqliteIndexStorage.addFile(data);
 	}
-
-	if (!storedFile.complete && data.complete)
+	else
 	{
-		m_sqliteIndexStorage.setFileComplete(data.complete, storedFile.id);
+		if (!storedFile.indexed && data.indexed)
+		{
+			m_sqliteIndexStorage.setFileIndexed(storedFile.id, data.indexed);
+		}
+
+		if (!storedFile.complete && data.complete)
+		{
+			m_sqliteIndexStorage.setFileComplete(storedFile.id, data.complete);
+		}
 	}
 }
 
@@ -290,6 +297,7 @@ void PersistentStorage::clearCaches()
 	m_fileNodeIds.clear();
 	m_fileNodePaths.clear();
 	m_fileNodeComplete.clear();
+	m_fileNodeIndexed.clear();
 	m_symbolDefinitionKinds.clear();
 
 	m_hierarchyCache.clear();
@@ -336,13 +344,18 @@ void PersistentStorage::clearFileElements(const std::vector<FilePath>& filePaths
 	}
 }
 
-std::vector<FileInfo> PersistentStorage::getFileInfoForAllFiles() const
+std::vector<FileInfo> PersistentStorage::getFileInfoForAllIndexedFiles() const
 {
 	TRACE();
 
 	std::vector<FileInfo> fileInfos;
 	for (StorageFile file : m_sqliteIndexStorage.getAll<StorageFile>())
 	{
+		if (!file.indexed)
+		{
+			continue;
+		}
+
 		boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time;
 		if (file.modificationTime != "not-a-date-time")
 		{
@@ -376,6 +389,17 @@ std::set<FilePath> PersistentStorage::getIncompleteFiles() const
 	return incompleteFiles;
 }
 
+bool PersistentStorage::getFilePathIndexed(const FilePath& path) const
+{
+	Id fileId = getFileNodeId(path);
+	if (fileId)
+	{
+		return getFileNodeIndexed(fileId);
+	}
+
+	return false;
+}
+
 void PersistentStorage::buildCaches()
 {
 	TRACE();
@@ -863,8 +887,6 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForAll() const
 {
 	TRACE();
 
-	std::shared_ptr<Graph> graph = std::make_shared<Graph>();
-
 	std::vector<Id> tokenIds;
 	for (StorageNode& node: m_sqliteIndexStorage.getAll<StorageNode>())
 	{
@@ -879,11 +901,15 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForAll() const
 		}
 	}
 
-	for (const auto& p : m_fileNodePaths)
+	for (const auto& p : m_fileNodeIndexed)
 	{
-		tokenIds.push_back(p.first);
+		if (p.second)
+		{
+			tokenIds.push_back(p.first);
+		}
 	}
 
+	std::shared_ptr<Graph> graph = std::make_shared<Graph>();
 	addNodesToGraph(tokenIds, graph.get(), false);
 
 	return graph;
@@ -1296,7 +1322,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getSourceLocationsF
 	std::shared_ptr<SourceLocationCollection> collection = std::make_shared<SourceLocationCollection>();
 	for (const FilePath& path : filePaths)
 	{
-		collection->addSourceLocationFile(std::make_shared<SourceLocationFile>(path, true, false));
+		collection->addSourceLocationFile(std::make_shared<SourceLocationFile>(path, true, false, false));
 	}
 
 	if (nonFileIds.size())
@@ -1421,7 +1447,7 @@ std::shared_ptr<SourceLocationFile> PersistentStorage::getCommentLocationsInFile
 {
 	TRACE();
 
-	const std::shared_ptr<SourceLocationFile> file = std::make_shared<SourceLocationFile>(filePath, false, false);
+	const std::shared_ptr<SourceLocationFile> file = std::make_shared<SourceLocationFile>(filePath, false, false, false);
 
 	const std::vector<StorageCommentLocation> storageLocations = m_sqliteIndexStorage.getCommentLocationsInFile(filePath);
 	for (size_t i = 0; i < storageLocations.size(); i++)
@@ -1836,8 +1862,13 @@ TooltipInfo PersistentStorage::getTooltipInfoForTokenIds(const std::vector<Id>&
 		}
 	}
 
-	if (type.isFile() && m_fileNodePaths.find(node.id) != m_fileNodePaths.end())
+	if (type.isFile())
 	{
+		if (!getFileNodeIndexed(node.id))
+		{
+			info.title = L"non-indexed " + info.title;
+		}
+
 		if (!getFileNodeComplete(node.id))
 		{
 			info.title = L"incomplete " + info.title;
@@ -1884,7 +1915,7 @@ TooltipSnippet PersistentStorage::getTooltipSnippetForNode(const StorageNode& no
 	TooltipSnippet snippet;
 	snippet.code = nameHierarchy.getQualifiedNameWithSignature();
 	snippet.locationFile = std::make_shared<SourceLocationFile>(
-		FilePath(nameHierarchy.getDelimiter() == NAME_DELIMITER_JAVA ? L"main.java" : L"main.cpp"), true, true);
+		FilePath(nameHierarchy.getDelimiter() == NAME_DELIMITER_JAVA ? L"main.java" : L"main.cpp"), true, true, true);
 
 	if (nameHierarchy.hasSignature())
 	{
@@ -1994,7 +2025,7 @@ TooltipInfo PersistentStorage::getTooltipInfoForSourceLocationIdsAndLocalSymbolI
 			const NameHierarchy nameHierarchy = NameHierarchy::deserialize(node.serializedName);
 			snippet.code = nameHierarchy.getQualifiedName();
 			snippet.locationFile = std::make_shared<SourceLocationFile>(
-				FilePath(nameHierarchy.getDelimiter() == NAME_DELIMITER_JAVA ? L"main.java" : L"main.cpp"), true, true);
+				FilePath(nameHierarchy.getDelimiter() == NAME_DELIMITER_JAVA ? L"main.java" : L"main.cpp"), true, true, true);
 
 			snippet.locationFile->addSourceLocation(
 				LOCATION_TOKEN, 0, std::vector<Id>(1, node.id), 1, 1, 1, snippet.code.size());
@@ -2013,7 +2044,7 @@ TooltipInfo PersistentStorage::getTooltipInfoForSourceLocationIdsAndLocalSymbolI
 		TooltipSnippet snippet;
 
 		snippet.code = L"local symbol";
-		snippet.locationFile = std::make_shared<SourceLocationFile>(FilePath(L"main.cpp"), true, true);
+		snippet.locationFile = std::make_shared<SourceLocationFile>(FilePath(L"main.cpp"), true, true, true);
 		snippet.locationFile->addSourceLocation(
 			LOCATION_LOCAL_SYMBOL, 0, std::vector<Id>(1, id), 1, 1, 1, snippet.code.size());
 
@@ -2081,21 +2112,21 @@ FilePath PersistentStorage::getFileNodePath(Id fileId) const
 	return FilePath();
 }
 
-bool PersistentStorage::getFilePathComplete(const FilePath& filePath) const
+bool PersistentStorage::getFileNodeComplete(Id fileId) const
 {
-	auto it = m_fileNodeIds.find(filePath);
-	if (it != m_fileNodeIds.end())
+	auto it = m_fileNodeComplete.find(fileId);
+	if (it != m_fileNodeComplete.end())
 	{
-		return getFileNodeComplete(it->second);
+		return it->second;
 	}
 
 	return false;
 }
 
-bool PersistentStorage::getFileNodeComplete(Id fileId) const
+bool PersistentStorage::getFileNodeIndexed(Id fileId) const
 {
-	auto it = m_fileNodeComplete.find(fileId);
-	if (it != m_fileNodeComplete.end())
+	auto it = m_fileNodeIndexed.find(fileId);
+	if (it != m_fileNodeIndexed.end())
 	{
 		return it->second;
 	}
@@ -2299,21 +2330,16 @@ void PersistentStorage::addNodesToGraph(const std::vector<Id>& newNodeIds, Graph
 		{
 			const FilePath filePath(NameHierarchy::deserialize(storageNode.serializedName).getRawName());
 
-			bool defined = false;
-			auto it = m_fileNodeComplete.find(storageNode.id);
-			if (it != m_fileNodeComplete.end())
-			{
-				defined = it->second;
-			}
+			bool complete = getFileNodeComplete(storageNode.id);
+			bool indexed = getFileNodeIndexed(storageNode.id);
 
 			Node* node = graph->createNode(
 				storageNode.id,
 				type,
 				NameHierarchy(filePath.fileName(), NAME_DELIMITER_FILE),
-				defined
+				indexed ? DEFINITION_EXPLICIT : DEFINITION_NONE
 			);
-			node->addComponent(std::make_shared<TokenComponentFilePath>(filePath));
-			node->setExplicit(defined);
+			node->addComponent(std::make_shared<TokenComponentFilePath>(filePath, complete));
 		}
 		else
 		{
@@ -2326,21 +2352,7 @@ void PersistentStorage::addNodesToGraph(const std::vector<Id>& newNodeIds, Graph
 				defKind = it->second;
 			}
 
-			Node* node = graph->createNode(
-				storageNode.id,
-				type,
-				nameHierarchy,
-				defKind != DEFINITION_NONE
-			);
-
-			if (defKind == DEFINITION_IMPLICIT)
-			{
-				node->setImplicit(true);
-			}
-			else if (defKind == DEFINITION_EXPLICIT)
-			{
-				node->setExplicit(true);
-			}
+			Node* node = graph->createNode(storageNode.id, type, nameHierarchy, defKind);
 
 			if (addChildCount)
 			{
@@ -2609,7 +2621,9 @@ void PersistentStorage::addCompleteFlagsToSourceLocationCollection(SourceLocatio
 	collection->forEachSourceLocationFile(
 		[this](std::shared_ptr<SourceLocationFile> file)
 		{
-			file->setIsComplete(getFilePathComplete(file->getFilePath()));
+			Id fileId = getFileNodeId(file->getFilePath());
+			file->setIsComplete(getFileNodeComplete(fileId));
+			file->setIsIndexed(getFileNodeIndexed(fileId));
 		}
 	);
 }
@@ -2691,6 +2705,7 @@ void PersistentStorage::buildFilePathMaps()
 		m_fileNodeIds.emplace(path, file.id);
 		m_fileNodePaths.emplace(file.id, path);
 		m_fileNodeComplete.emplace(file.id, file.complete);
+		m_fileNodeIndexed.emplace(file.id, file.indexed);
 
 		if (!m_hasJavaFiles && path.extension() == L".java")
 		{
@@ -2715,6 +2730,12 @@ void PersistentStorage::buildSearchIndex()
 		NodeType type = utility::intToType(node.type);
 		if (type.isFile())
 		{
+			bool indexed = getFileNodeIndexed(node.id);
+			if (!indexed)
+			{
+				continue;
+			}
+
 			auto it = m_fileNodePaths.find(node.id);
 			if (it != m_fileNodePaths.end())
 			{

src/lib/data/storage/PersistentStorage.h

@@ -63,8 +63,9 @@ public:
 
 	void clearFileElements(const std::vector<FilePath>& filePaths, std::function<void(int)> updateStatusCallback);
 
-	std::vector<FileInfo> getFileInfoForAllFiles() const;
+	std::vector<FileInfo> getFileInfoForAllIndexedFiles() const;
 	std::set<FilePath> getIncompleteFiles() const;
+	bool getFilePathIndexed(const FilePath& path) const;
 
 	void buildCaches();
 
@@ -153,8 +154,8 @@ private:
 	std::vector<Id> getFileNodeIds(const std::vector<FilePath>& filePaths) const;
 	std::set<Id> getFileNodeIds(const std::set<FilePath>& filePaths) const;
 	FilePath getFileNodePath(Id fileId) const;
-	bool getFilePathComplete(const FilePath& filePath) const;
 	bool getFileNodeComplete(Id fileId) const;
+	bool getFileNodeIndexed(Id fileId) const;
 
 	std::unordered_map<Id, std::set<Id>> getFileIdToIncludingFileIdMap() const;
 	std::unordered_map<Id, std::set<Id>> getFileIdToIncludedFileIdMap() const;
@@ -201,6 +202,7 @@ private:
 	std::map<FilePath, Id> m_fileNodeIds;
 	std::map<Id, FilePath> m_fileNodePaths;
 	std::map<Id, bool> m_fileNodeComplete;
+	std::map<Id, bool> m_fileNodeIndexed;
 
 	std::map<Id, DefinitionKind> m_symbolDefinitionKinds;
 	std::map<Id, Id> m_memberEdgeIdOrderMap;

src/lib/data/storage/Storage.cpp

@@ -50,7 +50,7 @@ void Storage::inject(Storage* injected)
 			if (it != injectedIdToOwnId.end())
 			{
 				const Id ownId = it->second;
-				addFile(StorageFile(ownId, injectedData.filePath, injectedData.modificationTime, injectedData.complete));
+				addFile(StorageFile(ownId, injectedData.filePath, injectedData.modificationTime, injectedData.indexed, injectedData.complete));
 			}
 		}
 	);

src/lib/data/storage/sqlite/SqliteIndexStorage.cpp

@@ -9,7 +9,7 @@
 #include "data/location/SourceLocationCollection.h"
 #include "data/location/SourceLocationFile.h"
 
-const size_t SqliteIndexStorage::s_storageVersion = 15;
+const size_t SqliteIndexStorage::s_storageVersion = 16;
 
 SqliteIndexStorage::SqliteIndexStorage(const FilePath& dbFilePath)
 	: SqliteStorage(dbFilePath.getCanonical())
@@ -68,21 +68,27 @@ void SqliteIndexStorage::addFile(const StorageFile& data)
 		return;
 	}
 
-	std::shared_ptr<TextAccess> content = TextAccess::createFromFile(FilePath(data.filePath));
-	const int lineCount = content->getLineCount();
+	std::shared_ptr<TextAccess> content;
+	int lineCount = 0;
+	if (data.indexed)
+	{
+		content = TextAccess::createFromFile(FilePath(data.filePath));
+		lineCount = content->getLineCount();
+	}
 
 	bool success = false;
 	{
 		m_insertFileStmt.bind(1, int(data.id));
 		m_insertFileStmt.bind(2, utility::encodeToUtf8(data.filePath).c_str());
 		m_insertFileStmt.bind(3, data.modificationTime.c_str());
-		m_insertFileStmt.bind(4, data.complete);
-		m_insertFileStmt.bind(5, lineCount);
+		m_insertFileStmt.bind(4, data.indexed);
+		m_insertFileStmt.bind(5, data.complete);
+		m_insertFileStmt.bind(6, lineCount);
 		success = executeStatement(m_insertFileStmt);
 		m_insertFileStmt.reset();
 	}
 
-	if (success)
+	if (success && content)
 	{
 		m_insertFileContentStmt.bind(1, int(data.id));
 		m_insertFileContentStmt.bind(2, content->getText().c_str());
@@ -632,7 +638,14 @@ std::shared_ptr<TextAccess> SqliteIndexStorage::getFileContentByPath(const std::
 	return TextAccess::createFromFile(FilePath(filePath));
 }
 
-void SqliteIndexStorage::setFileComplete(bool complete, Id fileId)
+void SqliteIndexStorage::setFileIndexed(Id fileId, bool indexed)
+{
+	executeStatement(
+		"UPDATE file SET indexed = " + std::to_string(indexed) + " WHERE id == " + std::to_string(fileId) + ";"
+	);
+}
+
+void SqliteIndexStorage::setFileComplete(Id fileId, bool complete)
 {
 	executeStatement(
 		"UPDATE file SET complete = " + std::to_string(complete) + " WHERE id == " + std::to_string(fileId) + ";"
@@ -649,7 +662,7 @@ void SqliteIndexStorage::setNodeType(int type, Id nodeId)
 std::shared_ptr<SourceLocationFile> SqliteIndexStorage::getSourceLocationsForFile(
 	const FilePath& filePath, const std::string& query) const
 {
-	std::shared_ptr<SourceLocationFile> ret = std::make_shared<SourceLocationFile>(filePath, true, false);
+	std::shared_ptr<SourceLocationFile> ret = std::make_shared<SourceLocationFile>(filePath, true, false, false);
 
 	const StorageFile file = getFileByPath(filePath.wstr());
 	if (file.id == 0) // early out
@@ -658,6 +671,7 @@ std::shared_ptr<SourceLocationFile> SqliteIndexStorage::getSourceLocationsForFil
 	}
 
 	ret->setIsComplete(file.complete);
+	ret->setIsIndexed(file.indexed);
 
 	std::vector<Id> sourceLocationIds;
 	std::unordered_map<Id, StorageSourceLocation> sourceLocationIdToData;
@@ -802,12 +816,12 @@ int SqliteIndexStorage::getEdgeCount() const
 
 int SqliteIndexStorage::getFileCount() const
 {
-	return executeStatementScalar("SELECT COUNT(*) FROM file;", 0);
+	return executeStatementScalar("SELECT COUNT(*) FROM file WHERE indexed = 1;", 0);
 }
 
 int SqliteIndexStorage::getCompletedFileCount() const
 {
-	return executeStatementScalar("SELECT COUNT(*) FROM file WHERE complete = 1;", 0);
+	return executeStatementScalar("SELECT COUNT(*) FROM file WHERE indexed = 1 AND complete = 1;", 0);
 }
 
 int SqliteIndexStorage::getFileLineSum() const
@@ -954,6 +968,7 @@ void SqliteIndexStorage::setupTables()
 				"id INTEGER NOT NULL, "
 				"path TEXT, "
 				"modification_time TEXT, "
+				"indexed INTEGER, "
 				"complete INTEGER, "
 				"line_count INTEGER, "
 				"PRIMARY KEY(id), "
@@ -1057,7 +1072,7 @@ void SqliteIndexStorage::setupPrecompiledStatements()
 			"INSERT INTO symbol(id, definition_kind) VALUES(?, ?);"
 		);
 		m_insertFileStmt = m_database.compileStatement(
-			"INSERT INTO file(id, path, modification_time, complete, line_count) VALUES(?, ?, ?, ?, ?);"
+			"INSERT INTO file(id, path, modification_time, indexed, complete, line_count) VALUES(?, ?, ?, ?, ?, ?);"
 		);
 		m_insertFileContentStmt = m_database.compileStatement(
 			"INSERT INTO filecontent(id, content) VALUES(?, ?);"
@@ -1198,7 +1213,7 @@ template <>
 std::vector<StorageFile> SqliteIndexStorage::doGetAll<StorageFile>(const std::string& query) const
 {
 	CppSQLite3Query q = executeQuery(
-		"SELECT id, path, modification_time, complete FROM file " + query + ";"
+		"SELECT id, path, modification_time, indexed, complete FROM file " + query + ";"
 	);
 
 	std::vector<StorageFile> files;
@@ -1207,11 +1222,12 @@ std::vector<StorageFile> SqliteIndexStorage::doGetAll<StorageFile>(const std::st
 		const Id id							= q.getIntField(0, 0);
 		const std::string filePath			= q.getStringField(1, "");
 		const std::string modificationTime	= q.getStringField(2, "");
-		const bool complete					= q.getIntField(3, 0);
+		const bool indexed					= q.getIntField(3, 0);
+		const bool complete					= q.getIntField(4, 0);
 
 		if (id != 0)
 		{
-			files.push_back(StorageFile(id, utility::decodeFromUtf8(filePath), modificationTime, complete));
+			files.push_back(StorageFile(id, utility::decodeFromUtf8(filePath), modificationTime, indexed, complete));
 		}
 		q.nextRow();
 	}

src/lib/data/storage/sqlite/SqliteIndexStorage.h

@@ -87,7 +87,8 @@ public:
 	std::shared_ptr<TextAccess> getFileContentByPath(const std::wstring& filePath) const;
 	std::shared_ptr<TextAccess> getFileContentById(Id fileId) const;
 
-	void setFileComplete(bool complete, Id fileId);
+	void setFileIndexed(Id fileId, bool indexed);
+	void setFileComplete(Id fileId, bool complete);
 	void setNodeType(int type, Id nodeId);
 
 	std::shared_ptr<SourceLocationFile> getSourceLocationsForFile(

src/lib/data/storage/type/StorageFile.h

@@ -11,19 +11,22 @@ struct StorageFile
 		: id(0)
 		, filePath(L"")
 		, modificationTime("")
+		, indexed(true)
 		, complete(true)
 	{}
 
-	StorageFile(Id id, const std::wstring& filePath, const std::string& modificationTime, bool complete)
+	StorageFile(Id id, const std::wstring& filePath, const std::string& modificationTime, bool indexed, bool complete)
 		: id(id)
 		, filePath(filePath)
 		, modificationTime(modificationTime)
+		, indexed(indexed)
 		, complete(complete)
 	{}
 
 	Id id;
 	std::wstring filePath;
 	std::string modificationTime;
+	bool indexed;
 	bool complete;
 };
 

src/lib/project/Project.cpp

@@ -352,11 +352,11 @@ void Project::buildIndex(const RefreshInfo& info, DialogView* dialogView)
 	{
 		m_storage->clear();
 	}
-	else if (info.filesToClear.size())
+	else if (info.filesToClear.size() || info.nonIndexedFilesToClear.size())
 	{
 		taskSequential->addTask(std::make_shared<TaskCleanStorage>(
 			m_storage.get(),
-			utility::toVector(info.filesToClear)
+			utility::toVector(utility::concat(info.filesToClear, info.nonIndexedFilesToClear))
 		));
 	}
 
@@ -497,7 +497,7 @@ RefreshInfo Project::getRefreshInfoForUpdatedFiles() const
 	{
 		std::set<FilePath> alreadyIndexedPaths;
 
-		const std::vector<FileInfo> fileInfos = m_storage->getFileInfoForAllFiles();
+		const std::vector<FileInfo> fileInfos = m_storage->getFileInfoForAllIndexedFiles();
 
 		for (const std::shared_ptr<SourceGroup>& sourceGroup : m_sourceGroups)
 		{
@@ -635,12 +635,20 @@ RefreshInfo Project::getRefreshInfoForIncompleteFiles() const
 	if (!incompleteFiles.empty())
 	{
 		utility::append(incompleteFiles, m_storage->getReferencing(incompleteFiles));
-		utility::append(info.filesToClear, incompleteFiles);
 
 		std::set<FilePath> staticSourceFilePaths = getAllSourceFilePaths();
 		for (const FilePath& path: incompleteFiles)
 		{
 			staticSourceFilePaths.erase(path);
+
+			if (m_storage->getFilePathIndexed(path))
+			{
+				info.filesToClear.insert(path);
+			}
+			else
+			{
+				info.nonIndexedFilesToClear.insert(path);
+			}
 		}
 
 		for (const std::shared_ptr<SourceGroup>& sourceGroup: m_sourceGroups)

src/lib/project/RefreshInfo.h

@@ -17,6 +17,8 @@ struct RefreshInfo
 {
 	std::set<FilePath> filesToIndex;
 	std::set<FilePath> filesToClear;
+	std::set<FilePath> nonIndexedFilesToClear;
+
 	RefreshMode mode = REFRESH_NONE;
 };