data: Added forEach<StorageType>() methods to SqliteIndexStorage for significantly lower memory use of queries

* use in local index building during refresh
* use in read cache building on project load
* use in some storage access requests
* also slightly improves speed

src/lib/data/storage/PersistentStorage.cpp

@@ -344,21 +344,19 @@ std::vector<FileInfo> PersistentStorage::getFileInfoForAllFiles() const
 	TRACE();
 
 	std::vector<FileInfo> fileInfos;
-	for (StorageFile file : m_sqliteIndexStorage.getAll<StorageFile>())
-	{
-		boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time;
-		if (file.modificationTime != "not-a-date-time")
-		{
-			modificationTime = boost::posix_time::time_from_string(file.modificationTime);
-		}
 
-		fileInfos.push_back(
-			FileInfo(
-				FilePath(file.filePath),
-				modificationTime
-			)
-		);
-	}
+	m_sqliteIndexStorage.forEach<StorageFile>(
+		[&](StorageFile&& file)
+		{
+			boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time;
+			if (file.modificationTime != "not-a-date-time")
+			{
+				modificationTime = boost::posix_time::time_from_string(file.modificationTime);
+			}
+
+			fileInfos.emplace_back(FilePath(file.filePath), modificationTime);
+		}
+	);
 
 	return fileInfos;
 }
@@ -906,18 +904,21 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForAll() const
 	TRACE();
 
 	std::vector<Id> tokenIds;
-	for (const StorageNode& node: m_sqliteIndexStorage.getAll<StorageNode>())
-	{
-		auto it = m_symbolDefinitionKinds.find(node.id);
-		if (it != m_symbolDefinitionKinds.end() && it->second == DEFINITION_EXPLICIT &&
-			(
-				NodeType(NodeType::intToType(node.type)).isPackage() ||
-				!m_hierarchyCache.isChildOfVisibleNodeOrInvisible(node.id)
-			)
-		){
-			tokenIds.push_back(node.id);
+
+	m_sqliteIndexStorage.forEach<StorageNode>(
+		[&](StorageNode&& node)
+		{
+			auto it = m_symbolDefinitionKinds.find(node.id);
+			if (it != m_symbolDefinitionKinds.end() && it->second == DEFINITION_EXPLICIT &&
+				(
+					NodeType(NodeType::intToType(node.type)).isPackage() ||
+					!m_hierarchyCache.isChildOfVisibleNodeOrInvisible(node.id)
+				)
+			){
+				tokenIds.push_back(node.id);
+			}
 		}
-	}
+	);
 
 	for (const auto& p : m_fileNodeIndexed)
 	{
@@ -937,20 +938,21 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForNodeTypes(NodeTypeSet nodeT
 {
 	TRACE();
 
-	std::shared_ptr<Graph> graph = std::make_shared<Graph>();
-
 	std::vector<Id> tokenIds;
-	for (StorageNode& node: m_sqliteIndexStorage.getAll<StorageNode>())
-	{
-		if (nodeTypes.contains(NodeType::intToType(node.type)))
+
+	m_sqliteIndexStorage.forEach<StorageNode>(
+		[&](StorageNode&& node)
 		{
-			auto it = m_symbolDefinitionKinds.find(node.id);
-			if (it != m_symbolDefinitionKinds.end() && it->second == DEFINITION_EXPLICIT)
+			if (nodeTypes.contains(NodeType::intToType(node.type)))
 			{
-				tokenIds.push_back(node.id);
+				auto it = m_symbolDefinitionKinds.find(node.id);
+				if (it != m_symbolDefinitionKinds.end() && it->second == DEFINITION_EXPLICIT)
+				{
+					tokenIds.push_back(node.id);
+				}
 			}
 		}
-	}
+	);
 
 	if (nodeTypes.containsMatching([](const NodeType& type) { return type.isFile(); }))
 	{
@@ -960,6 +962,7 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForNodeTypes(NodeTypeSet nodeT
 		}
 	}
 
+	std::shared_ptr<Graph> graph = std::make_shared<Graph>();
 	addNodesWithParentsAndEdgesToGraph(tokenIds, std::vector<Id>(), graph.get(), false);
 
 	return graph;
@@ -2267,20 +2270,30 @@ bool PersistentStorage::getFileNodeIndexed(Id fileId) const
 std::unordered_map<Id, std::set<Id>> PersistentStorage::getFileIdToIncludingFileIdMap() const
 {
 	std::unordered_map<Id, std::set<Id>> fileIdToIncludingFileIdMap;
-	for (const StorageEdge& includeEdge : m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_INCLUDE)))
-	{
-		fileIdToIncludingFileIdMap[includeEdge.targetNodeId].insert(includeEdge.sourceNodeId);
-	}
+
+	m_sqliteIndexStorage.forEachOfType<StorageEdge>(
+		Edge::typeToInt(Edge::EDGE_INCLUDE),
+		[&fileIdToIncludingFileIdMap](StorageEdge&& edge)
+		{
+			fileIdToIncludingFileIdMap[edge.targetNodeId].insert(edge.sourceNodeId);
+		}
+	);
+
 	return fileIdToIncludingFileIdMap;
 }
 
 std::unordered_map<Id, std::set<Id>> PersistentStorage::getFileIdToIncludedFileIdMap() const
 {
 	std::unordered_map<Id, std::set<Id>> fileIdToIncludingFileIdMap;
-	for (const StorageEdge& includeEdge : m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_INCLUDE)))
-	{
-		fileIdToIncludingFileIdMap[includeEdge.sourceNodeId].insert(includeEdge.targetNodeId);
-	}
+
+	m_sqliteIndexStorage.forEachOfType<StorageEdge>(
+		Edge::typeToInt(Edge::EDGE_INCLUDE),
+		[&fileIdToIncludingFileIdMap](StorageEdge&& edge)
+		{
+			fileIdToIncludingFileIdMap[edge.sourceNodeId].insert(edge.targetNodeId);
+		}
+	);
+
 	return fileIdToIncludingFileIdMap;
 }
 
@@ -2291,11 +2304,14 @@ std::unordered_map<Id, std::set<Id>> PersistentStorage::getFileIdToImportingFile
 		std::vector<Id> importedElementIds;
 		std::map<Id, std::set<Id>> elementIdToImportingFileIds;
 
-		for (const StorageEdge& importEdge : m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_IMPORT)))
-		{
-			importedElementIds.push_back(importEdge.targetNodeId);
-			elementIdToImportingFileIds[importEdge.targetNodeId].insert(importEdge.sourceNodeId);
-		}
+		m_sqliteIndexStorage.forEachOfType<StorageEdge>(
+			Edge::typeToInt(Edge::EDGE_IMPORT),
+			[&importedElementIds, &elementIdToImportingFileIds](StorageEdge&& edge)
+			{
+				importedElementIds.push_back(edge.targetNodeId);
+				elementIdToImportingFileIds[edge.targetNodeId].insert(edge.sourceNodeId);
+			}
+		);
 
 		std::unordered_map<Id, Id> importedElementIdToFileNodeId;
 		{
@@ -2828,26 +2844,30 @@ void PersistentStorage::buildFilePathMaps()
 {
 	TRACE();
 
-	for (const StorageFile& file: m_sqliteIndexStorage.getAll<StorageFile>())
-	{
-		const FilePath path(file.filePath);
-
-		m_fileNodeIds.emplace(path, file.id);
-		m_lowerCasefileNodeIds.emplace(path.getLowerCase(), file.id);
-		m_fileNodePaths.emplace(file.id, path);
-		m_fileNodeComplete.emplace(file.id, file.complete);
-		m_fileNodeIndexed.emplace(file.id, file.indexed);
-
-		if (!m_hasJavaFiles && path.extension() == L".java")
+	m_sqliteIndexStorage.forEach<StorageFile>(
+		[&](StorageFile&& file)
 		{
-			m_hasJavaFiles = true;
-		}
-	}
+			const FilePath path(file.filePath);
 
-	for (StorageSymbol& symbol : m_sqliteIndexStorage.getAll<StorageSymbol>())
-	{
-		m_symbolDefinitionKinds.emplace(symbol.id, intToDefinitionKind(symbol.definitionKind));
-	}
+			m_fileNodeIds.emplace(path, file.id);
+			m_lowerCasefileNodeIds.emplace(path.getLowerCase(), file.id);
+			m_fileNodePaths.emplace(file.id, path);
+			m_fileNodeComplete.emplace(file.id, file.complete);
+			m_fileNodeIndexed.emplace(file.id, file.indexed);
+
+			if (!m_hasJavaFiles && path.extension() == L".java")
+			{
+				m_hasJavaFiles = true;
+			}
+		}
+	);
+
+	m_sqliteIndexStorage.forEach<StorageSymbol>(
+		[&](StorageSymbol&& symbol)
+		{
+			m_symbolDefinitionKinds.emplace(symbol.id, intToDefinitionKind(symbol.definitionKind));
+		}
+	);
 }
 
 void PersistentStorage::buildSearchIndex()
@@ -2856,52 +2876,54 @@ void PersistentStorage::buildSearchIndex()
 
 	const FilePath dbPath = getIndexDbFilePath();
 
-	for (const StorageNode& node : m_sqliteIndexStorage.getAll<StorageNode>())
-	{
-		const NodeType type = NodeType::intToType(node.type);
-		if (type.isFile())
+	m_sqliteIndexStorage.forEach<StorageNode>(
+		[&](StorageNode&& node)
 		{
-			bool indexed = getFileNodeIndexed(node.id);
-			if (!indexed)
+			const NodeType type = NodeType::intToType(node.type);
+			if (type.isFile())
 			{
-				continue;
-			}
-
-			auto it = m_fileNodePaths.find(node.id);
-			if (it != m_fileNodePaths.end())
-			{
-				FilePath filePath(it->second);
-
-				if (filePath.exists())
+				bool indexed = getFileNodeIndexed(node.id);
+				if (!indexed)
 				{
-					filePath.makeRelativeTo(dbPath);
+					return;
 				}
 
-				m_fileIndex.addNode(node.id, filePath.wstr(), type);
-			}
-		}
-		else
-		{
-			auto it = m_symbolDefinitionKinds.find(node.id);
-			const DefinitionKind defKind = (it != m_symbolDefinitionKinds.end() ? it->second : DEFINITION_NONE);
-			if (defKind != DEFINITION_IMPLICIT)
-			{
-				const NameHierarchy nameHierarchy = NameHierarchy::deserialize(node.serializedName);
-
-				// we don't use the signature here, so elements with the same signature share the same node.
-				std::wstring name = nameHierarchy.getQualifiedName();
-
-				// replace template arguments with .. to avoid clutter in search results and have different
-				// template specializations share the same node.
-				if (defKind == DEFINITION_NONE && nameHierarchy.getDelimiter() == NAME_DELIMITER_CXX)
+				auto it = m_fileNodePaths.find(node.id);
+				if (it != m_fileNodePaths.end())
 				{
-					name = utility::replaceBetween(name, L'<', L'>', L"..");
-				}
+					FilePath filePath(it->second);
 
-				m_symbolIndex.addNode(node.id, std::move(name), type);
+					if (filePath.exists())
+					{
+						filePath.makeRelativeTo(dbPath);
+					}
+
+					m_fileIndex.addNode(node.id, filePath.wstr(), type);
+				}
+			}
+			else
+			{
+				auto it = m_symbolDefinitionKinds.find(node.id);
+				const DefinitionKind defKind = (it != m_symbolDefinitionKinds.end() ? it->second : DEFINITION_NONE);
+				if (defKind != DEFINITION_IMPLICIT)
+				{
+					const NameHierarchy nameHierarchy = NameHierarchy::deserialize(node.serializedName);
+
+					// we don't use the signature here, so elements with the same signature share the same node.
+					std::wstring name = nameHierarchy.getQualifiedName();
+
+					// replace template arguments with .. to avoid clutter in search results and have different
+					// template specializations share the same node.
+					if (defKind == DEFINITION_NONE && nameHierarchy.getDelimiter() == NAME_DELIMITER_CXX)
+					{
+						name = utility::replaceBetween(name, L'<', L'>', L"..");
+					}
+
+					m_symbolIndex.addNode(node.id, std::move(name), type);
+				}
 			}
 		}
-	}
+	);
 
 	m_symbolIndex.finishSetup();
 	m_fileIndex.finishSetup();
@@ -2963,11 +2985,14 @@ void PersistentStorage::buildMemberEdgeIdOrderMap()
 	std::vector<Id> childNodeIds;
 	std::unordered_map<Id, Id> childIdToMemberEdgeIdMap;
 
-	for (const StorageEdge& edge : m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER)))
-	{
-		childNodeIds.push_back(edge.targetNodeId);
-		childIdToMemberEdgeIdMap.emplace(edge.targetNodeId, edge.id);
-	}
+	m_sqliteIndexStorage.forEachOfType<StorageEdge>(
+		Edge::typeToInt(Edge::EDGE_MEMBER),
+		[&childNodeIds, &childIdToMemberEdgeIdMap](StorageEdge&& edge)
+		{
+			childNodeIds.push_back(edge.targetNodeId);
+			childIdToMemberEdgeIdMap.emplace(edge.targetNodeId, edge.id);
+		}
+	);
 
 	std::vector<Id> locationIds;
 	std::unordered_map<Id, Id> locationIdToElementIdMap;
@@ -3028,29 +3053,37 @@ void PersistentStorage::buildHierarchyCache()
 {
 	TRACE();
 
-	const std::vector<StorageEdge> memberEdges = m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER));
-
 	std::vector<Id> sourceNodeIds;
-	for (const StorageEdge& edge : memberEdges)
-	{
-		sourceNodeIds.push_back(edge.sourceNodeId);
-	}
+	std::vector<StorageEdge> memberEdges;
 
-	std::map<Id, NodeType> sourceNodeTypeMap;
-	for (const StorageNode& node : m_sqliteIndexStorage.getAllByIds<StorageNode>(sourceNodeIds))
-	{
-		sourceNodeTypeMap.emplace(node.id, NodeType::intToType(node.type));
-	}
+	m_sqliteIndexStorage.forEachOfType<StorageEdge>(
+		Edge::typeToInt(Edge::EDGE_MEMBER),
+		[&sourceNodeIds, &memberEdges](StorageEdge&& edge)
+		{
+			sourceNodeIds.push_back(edge.sourceNodeId);
+			memberEdges.emplace_back(edge);
+		}
+	);
+
+	std::set<Id> invisibleParentSourceNodeIds;
+
+	m_sqliteIndexStorage.forEachByIds<StorageNode>(
+		sourceNodeIds,
+		[&invisibleParentSourceNodeIds](StorageNode&& node)
+		{
+			if (!NodeType(NodeType::intToType(node.type)).isVisibleAsParentInGraph())
+			{
+				invisibleParentSourceNodeIds.insert(node.id);
+			}
+		}
+	);
 
 	for (const StorageEdge& edge : memberEdges)
 	{
 		bool sourceIsVisible = true;
+		if (invisibleParentSourceNodeIds.find(edge.sourceNodeId) != invisibleParentSourceNodeIds.end())
 		{
-			std::map<Id, NodeType>::const_iterator it = sourceNodeTypeMap.find(edge.sourceNodeId);
-			if (it != sourceNodeTypeMap.end())
-			{
-				sourceIsVisible = it->second.isVisibleAsParentInGraph();
-			}
+			sourceIsVisible = false;
 		}
 
 		bool sourceIsImplicit = false;
@@ -3071,9 +3104,11 @@ void PersistentStorage::buildHierarchyCache()
 			edge.id, edge.sourceNodeId, edge.targetNodeId, sourceIsVisible, sourceIsImplicit, targetIsImplicit);
 	}
 
-	std::vector<StorageEdge> inheritanceEdges = m_sqliteIndexStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_INHERITANCE));
-	for (const StorageEdge& edge : inheritanceEdges)
-	{
-		m_hierarchyCache.createInheritance(edge.id, edge.sourceNodeId, edge.targetNodeId);
-	}
+	m_sqliteIndexStorage.forEachOfType<StorageEdge>(
+		Edge::typeToInt(Edge::EDGE_INHERITANCE),
+		[this](StorageEdge&& edge)
+		{
+			m_hierarchyCache.createInheritance(edge.id, edge.sourceNodeId, edge.targetNodeId);
+		}
+	);
 }

src/lib/data/storage/sqlite/SqliteIndexStorage.cpp

@@ -97,20 +97,22 @@ std::vector<Id> SqliteIndexStorage::addNodes(const std::vector<StorageNode>& nod
 {
 	if (m_tempNodeNameIndex.empty() && m_tempWNodeNameIndex.empty())
 	{
-		for (const StorageNode& node : getAll<StorageNode>())
-		{
-			std::string name = utility::encodeToUtf8(node.serializedName);
-			if (name.size() != node.serializedName.size())
+		forEach<StorageNode>(
+			[this](StorageNode&& node)
 			{
-				m_tempWNodeNameIndex.add(node.serializedName, node.id);
-			}
-			else
-			{
-				m_tempNodeNameIndex.add(name, node.id);
-			}
+				std::string name = utility::encodeToUtf8(node.serializedName);
+				if (name.size() != node.serializedName.size())
+				{
+					m_tempWNodeNameIndex.add(node.serializedName, node.id);
+				}
+				else
+				{
+					m_tempNodeNameIndex.add(name, node.id);
+				}
 
-			m_tempNodeTypes.emplace(node.id, node.type);
-		}
+				m_tempNodeTypes.emplace(node.id, node.type);
+			}
+		);
 	}
 
 	std::vector<Id> nodeIds(nodes.size(), 0);
@@ -234,10 +236,12 @@ std::vector<Id> SqliteIndexStorage::addEdges(const std::vector<StorageEdge>& edg
 {
 	if (m_tempEdgeIndex.empty())
 	{
-		for (const StorageEdge& edge : getAll<StorageEdge>())
-		{
-			m_tempEdgeIndex.emplace(StorageEdgeData(edge.type, edge.sourceNodeId, edge.targetNodeId), edge.id);
-		}
+		forEach<StorageEdge>(
+			[this](StorageEdge&& edge)
+			{
+				m_tempEdgeIndex.emplace(StorageEdgeData(edge.type, edge.sourceNodeId, edge.targetNodeId), edge.id);
+			}
+		);
 	}
 
 	std::vector<Id> edgeIds(edges.size(), 0);
@@ -278,20 +282,20 @@ Id SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data)
 
 std::vector<Id> SqliteIndexStorage::addLocalSymbols(const std::set<StorageLocalSymbol>& symbols)
 {
-	std::wstring name;
-	uint32_t line;
-	uint32_t col;
-
 	if (m_tempLocalSymbolIndex.empty())
 	{
-		for (const StorageLocalSymbol& localSymbol : getAll<StorageLocalSymbol>())
-		{
-			std::tie(name, line, col) = splitLocalSymbolName(localSymbol.name);
-			if (name.size())
+		forEach<StorageLocalSymbol>(
+			[this](StorageLocalSymbol&& localSymbol)
 			{
-				m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), localSymbol.id);
+				std::wstring name;
+				uint32_t line, col;
+				std::tie(name, line, col) = splitLocalSymbolName(localSymbol.name);
+				if (name.size())
+				{
+					m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), localSymbol.id);
+				}
 			}
-		}
+		);
 	}
 
 	std::vector<Id> symbolIds(symbols.size(), 0);
@@ -300,6 +304,8 @@ std::vector<Id> SqliteIndexStorage::addLocalSymbols(const std::set<StorageLocalS
 	for (size_t i = 0; i < symbols.size(); i++)
 	{
 		const StorageLocalSymbol& data = *it;
+		std::wstring name;
+		uint32_t line, col;
 		std::tie(name, line, col) = splitLocalSymbolName(data.name);
 		if (name.size())
 		{
@@ -346,13 +352,15 @@ std::vector<Id> SqliteIndexStorage::addSourceLocations(const std::vector<Storage
 {
 	if (m_tempSourceLocationIndices.empty())
 	{
-		for (const StorageSourceLocation& loc : getAll<StorageSourceLocation>())
-		{
-			std::map<TempSourceLocation, uint32_t>& index = m_tempSourceLocationIndices[loc.fileNodeId];
-			index.emplace(
-				TempSourceLocation(loc.startLine, loc.endLine - loc.startLine, loc.startCol, loc.endCol, loc.type),
-				loc.id);
-		}
+		forEach<StorageSourceLocation>(
+			[this](StorageSourceLocation&& loc)
+			{
+				std::map<TempSourceLocation, uint32_t>& index = m_tempSourceLocationIndices[loc.fileNodeId];
+				index.emplace(
+					TempSourceLocation(loc.startLine, loc.endLine - loc.startLine, loc.startCol, loc.endCol, loc.type),
+					loc.id);
+			}
+		);
 	}
 
 	std::vector<Id> locationIds(locations.size(), 0);
@@ -1293,13 +1301,12 @@ void SqliteIndexStorage::setupPrecompiledStatements()
 }
 
 template <>
-std::vector<StorageEdge> SqliteIndexStorage::doGetAll<StorageEdge>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageEdge>(const std::string& query, std::function<void(StorageEdge&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT id, type, source_node_id, target_node_id FROM edge " + query + ";"
 	);
 
-	std::vector<StorageEdge> edges;
 	while (!q.eof())
 	{
 		const Id id = q.getIntField(0, 0);
@@ -1309,22 +1316,20 @@ std::vector<StorageEdge> SqliteIndexStorage::doGetAll<StorageEdge>(const std::st
 
 		if (id != 0 && type != -1)
 		{
-			edges.emplace_back(id, type, sourceId, targetId);
+			func(StorageEdge(id, type, sourceId, targetId));
 		}
 
 		q.nextRow();
 	}
-	return edges;
 }
 
 template <>
-std::vector<StorageNode> SqliteIndexStorage::doGetAll<StorageNode>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageNode>(const std::string& query, std::function<void(StorageNode&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT id, type, serialized_name FROM node " + query + ";"
 	);
 
-	std::vector<StorageNode> nodes;
 	while (!q.eof())
 	{
 		const Id id = q.getIntField(0, 0);
@@ -1333,22 +1338,20 @@ std::vector<StorageNode> SqliteIndexStorage::doGetAll<StorageNode>(const std::st
 
 		if (id != 0 && type != -1)
 		{
-			nodes.emplace_back(id, type, utility::decodeFromUtf8(serializedName));
+			func(StorageNode(id, type, utility::decodeFromUtf8(serializedName)));
 		}
 
 		q.nextRow();
 	}
-	return nodes;
 }
 
 template <>
-std::vector<StorageSymbol> SqliteIndexStorage::doGetAll<StorageSymbol>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageSymbol>(const std::string& query, std::function<void(StorageSymbol&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT id, definition_kind FROM symbol " + query + ";"
 	);
 
-	std::vector<StorageSymbol> symbols;
 	while (!q.eof())
 	{
 		const Id id = q.getIntField(0, 0);
@@ -1356,22 +1359,20 @@ std::vector<StorageSymbol> SqliteIndexStorage::doGetAll<StorageSymbol>(const std
 
 		if (id != 0)
 		{
-			symbols.emplace_back(id, definitionKind);
+			func(StorageSymbol(id, definitionKind));
 		}
 
 		q.nextRow();
 	}
-	return symbols;
 }
 
 template <>
-std::vector<StorageFile> SqliteIndexStorage::doGetAll<StorageFile>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageFile>(const std::string& query, std::function<void(StorageFile&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT id, path, modification_time, indexed, complete FROM file " + query + ";"
 	);
 
-	std::vector<StorageFile> files;
 	while (!q.eof())
 	{
 		const Id id							= q.getIntField(0, 0);
@@ -1382,23 +1383,19 @@ std::vector<StorageFile> SqliteIndexStorage::doGetAll<StorageFile>(const std::st
 
 		if (id != 0)
 		{
-			files.emplace_back(id, utility::decodeFromUtf8(filePath), modificationTime, indexed, complete);
+			func(StorageFile(id, utility::decodeFromUtf8(filePath), modificationTime, indexed, complete));
 		}
 		q.nextRow();
 	}
-
-	return files;
 }
 
 template <>
-std::vector<StorageLocalSymbol> SqliteIndexStorage::doGetAll<StorageLocalSymbol>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageLocalSymbol>(const std::string& query, std::function<void(StorageLocalSymbol&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT id, name FROM local_symbol " + query + ";"
 	);
 
-	std::vector<StorageLocalSymbol> localSymbols;
-
 	while (!q.eof())
 	{
 		const Id id = q.getIntField(0, 0);
@@ -1406,23 +1403,20 @@ std::vector<StorageLocalSymbol> SqliteIndexStorage::doGetAll<StorageLocalSymbol>
 
 		if (id != 0)
 		{
-			localSymbols.emplace_back(id, utility::decodeFromUtf8(name));
+			func(StorageLocalSymbol(id, utility::decodeFromUtf8(name)));
 		}
 
 		q.nextRow();
 	}
-	return localSymbols;
 }
 
 template <>
-std::vector<StorageSourceLocation> SqliteIndexStorage::doGetAll<StorageSourceLocation>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageSourceLocation>(const std::string& query, std::function<void(StorageSourceLocation&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location " + query + ";"
 	);
 
-	std::vector<StorageSourceLocation> sourceLocations;
-
 	while (!q.eof())
 	{
 		const Id id					= q.getIntField(0, 0);
@@ -1436,23 +1430,20 @@ std::vector<StorageSourceLocation> SqliteIndexStorage::doGetAll<StorageSourceLoc
 		if (id != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 &&
 			endColNumber != -1 && type != -1)
 		{
-			sourceLocations.emplace_back(id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type);
+			func(StorageSourceLocation(id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type));
 		}
 
 		q.nextRow();
 	}
-	return sourceLocations;
 }
 
 template <>
-std::vector<StorageOccurrence> SqliteIndexStorage::doGetAll<StorageOccurrence>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageOccurrence>(const std::string& query, std::function<void(StorageOccurrence&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT element_id, source_location_id FROM occurrence " + query + ";"
 	);
 
-	std::vector<StorageOccurrence> occurrences;
-
 	while (!q.eof())
 	{
 		const Id elementId			= q.getIntField(0, 0);
@@ -1460,23 +1451,20 @@ std::vector<StorageOccurrence> SqliteIndexStorage::doGetAll<StorageOccurrence>(c
 
 		if (elementId != 0 && sourceLocationId != 0)
 		{
-			occurrences.emplace_back(elementId, sourceLocationId);
+			func(StorageOccurrence(elementId, sourceLocationId));
 		}
 
 		q.nextRow();
 	}
-	return occurrences;
 }
 
 template <>
-std::vector<StorageComponentAccess> SqliteIndexStorage::doGetAll<StorageComponentAccess>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageComponentAccess>(const std::string& query, std::function<void(StorageComponentAccess&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT node_id, type FROM component_access " + query + ";"
 	);
 
-	std::vector<StorageComponentAccess> componentAccesses;
-
 	while (!q.eof())
 	{
 		const Id nodeId	= q.getIntField(0, 0);
@@ -1484,22 +1472,20 @@ std::vector<StorageComponentAccess> SqliteIndexStorage::doGetAll<StorageComponen
 
 		if (nodeId != 0 && type != -1)
 		{
-			componentAccesses.emplace_back(nodeId, type);
+			func(StorageComponentAccess(nodeId, type));
 		}
 
 		q.nextRow();
 	}
-	return componentAccesses;
 }
 
 template <>
-std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::string& query) const
+void SqliteIndexStorage::forEach<StorageError>(const std::string& query, std::function<void(StorageError&&)> func) const
 {
 	CppSQLite3Query q = executeQuery(
 		"SELECT message, fatal, indexed, file_path, line_number, column_number, translation_unit FROM error " + query + ";"
 	);
 
-	std::vector<StorageError> errors;
 	Id id = 1;
 	while (!q.eof())
 	{
@@ -1513,15 +1499,13 @@ std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::
 
 		if (lineNumber != -1 && columnNumber != -1)
 		{
-			errors.emplace_back(
+			func(StorageError(
 				id, utility::decodeFromUtf8(message), utility::decodeFromUtf8(filePath), lineNumber, columnNumber,
 				utility::decodeFromUtf8(translationUnit), fatal, indexed
-			);
+			));
 			id++;
 		}
 
 		q.nextRow();
 	}
-
-	return errors;
 }

src/lib/data/storage/sqlite/SqliteIndexStorage.h

@@ -145,6 +145,27 @@ public:
 		return std::vector<ResultType>();
 	}
 
+	template <typename StorageType>
+	void forEach(std::function<void(StorageType&&)> func) const
+	{
+		forEach("", func);
+	}
+
+	template <typename StorageType>
+	void forEachOfType(int type, std::function<void(StorageType&&)> func) const
+	{
+		forEach("WHERE type == " + std::to_string(type), func);
+	}
+
+	template <typename StorageType>
+	void forEachByIds(const std::vector<Id> ids, std::function<void(StorageType&&)> func) const
+	{
+		if (ids.size())
+		{
+			forEach("WHERE id IN (" + utility::join(utility::toStrings(ids), ',') + ")", func);
+		}
+	}
+
 	int getNodeCount() const;
 	int getEdgeCount() const;
 	int getFileCount() const;
@@ -204,7 +225,18 @@ private:
 	virtual void setupPrecompiledStatements();
 
 	template <typename ResultType>
-	std::vector<ResultType> doGetAll(const std::string& query) const;
+	std::vector<ResultType> doGetAll(const std::string& query) const
+	{
+		std::vector<ResultType> elements;
+		forEach<ResultType>(
+			query,
+			[&elements](ResultType&& element)
+			{
+				elements.emplace_back(element);
+			}
+		);
+		return elements;
+	}
 
 	template <typename ResultType>
 	ResultType doGetFirst(const std::string& query) const
@@ -217,6 +249,9 @@ private:
 		return ResultType();
 	}
 
+	template <typename StorageType>
+	void forEach(const std::string& query, std::function<void(StorageType&&)> func) const;
+
 	LowMemoryStringMap<std::string, uint32_t, 0> m_tempNodeNameIndex;
 	LowMemoryStringMap<std::wstring, uint32_t, 0> m_tempWNodeNameIndex;
 	std::map<uint32_t, int> m_tempNodeTypes;
@@ -309,22 +344,22 @@ template<typename StorageType>
 const size_t SqliteIndexStorage::InsertBatchStatement<StorageType>::BATCH_SIZES[4] = { 100, 27, 8, 1 };
 
 template <>
-std::vector<StorageEdge> SqliteIndexStorage::doGetAll<StorageEdge>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageEdge>(const std::string& query, std::function<void(StorageEdge&&)> func) const;
 template <>
-std::vector<StorageNode> SqliteIndexStorage::doGetAll<StorageNode>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageNode>(const std::string& query, std::function<void(StorageNode&&)> func) const;
 template <>
-std::vector<StorageSymbol> SqliteIndexStorage::doGetAll<StorageSymbol>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageSymbol>(const std::string& query, std::function<void(StorageSymbol&&)> func) const;
 template <>
-std::vector<StorageFile> SqliteIndexStorage::doGetAll<StorageFile>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageFile>(const std::string& query, std::function<void(StorageFile&&)> func) const;
 template <>
-std::vector<StorageLocalSymbol> SqliteIndexStorage::doGetAll<StorageLocalSymbol>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageLocalSymbol>(const std::string& query, std::function<void(StorageLocalSymbol&&)> func) const;
 template <>
-std::vector<StorageSourceLocation> SqliteIndexStorage::doGetAll<StorageSourceLocation>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageSourceLocation>(const std::string& query, std::function<void(StorageSourceLocation&&)> func) const;
 template <>
-std::vector<StorageOccurrence> SqliteIndexStorage::doGetAll<StorageOccurrence>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageOccurrence>(const std::string& query, std::function<void(StorageOccurrence&&)> func) const;
 template <>
-std::vector<StorageComponentAccess> SqliteIndexStorage::doGetAll<StorageComponentAccess>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageComponentAccess>(const std::string& query, std::function<void(StorageComponentAccess&&)> func) const;
 template <>
-std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::string& query) const;
+void SqliteIndexStorage::forEach<StorageError>(const std::string& query, std::function<void(StorageError&&)> func) const;
 
 #endif // SQLITE_INDEX_STORAGE_H