data: Improved injection performance with batched insertions

* add InsertBatchStatement inserting any number as multiples
* switched Storage injection interface to containers

src/lib/data/storage/IntermediateStorage.cpp

@@ -121,11 +121,27 @@ std::pair<Id, bool> IntermediateStorage::addNode(const StorageNodeData& nodeData
 	return std::make_pair(nodeId, true);
 }
 
+std::vector<Id> IntermediateStorage::addNodes(const std::vector<StorageNode>& nodes)
+{
+	std::vector<Id> nodeIds;
+	nodeIds.reserve(nodes.size());
+	for (const StorageNode& node : nodes)
+	{
+		nodeIds.emplace_back(addNode(node).first);
+	}
+	return nodeIds;
+}
+
 void IntermediateStorage::addSymbol(const StorageSymbol& symbol)
 {
 	m_symbols.push_back(symbol);
 }
 
+void IntermediateStorage::addSymbols(const std::vector<StorageSymbol>& symbols)
+{
+	m_symbols.insert(m_symbols.end(), symbols.begin(), symbols.end());
+}
+
 void IntermediateStorage::addFile(const StorageFile& file)
 {
 	auto it = m_filesIndex.find(file);
@@ -164,6 +180,17 @@ Id IntermediateStorage::addEdge(const StorageEdgeData& edgeData)
 	return edgeId;
 }
 
+std::vector<Id> IntermediateStorage::addEdges(const std::vector<StorageEdge>& edges)
+{
+	std::vector<Id> edgeIds;
+	edgeIds.reserve(edges.size());
+	for (const StorageEdge& edge : edges)
+	{
+		edgeIds.emplace_back(addEdge(edge));
+	}
+	return edgeIds;
+}
+
 Id IntermediateStorage::addLocalSymbol(const StorageLocalSymbolData& localSymbolData)
 {
 	auto it = m_localSymbols.find(StorageLocalSymbol(0, localSymbolData));
@@ -177,6 +204,17 @@ Id IntermediateStorage::addLocalSymbol(const StorageLocalSymbolData& localSymbol
 	return localSymbolId;
 }
 
+std::vector<Id> IntermediateStorage::addLocalSymbols(const std::set<StorageLocalSymbol>& symbols)
+{
+	std::vector<Id> symbolIds;
+	symbolIds.reserve(symbols.size());
+	for (const StorageLocalSymbol& symbol : symbols)
+	{
+		symbolIds.emplace_back(addLocalSymbol(symbol));
+	}
+	return symbolIds;
+}
+
 Id IntermediateStorage::addSourceLocation(const StorageSourceLocationData& sourceLocationData)
 {
 	auto it = m_sourceLocations.find(StorageSourceLocation(0, sourceLocationData));
@@ -190,6 +228,17 @@ Id IntermediateStorage::addSourceLocation(const StorageSourceLocationData& sourc
 	return sourceLocationId;
 }
 
+std::vector<Id> IntermediateStorage::addSourceLocations(const std::vector<StorageSourceLocation>& locations)
+{
+	std::vector<Id> locationIds;
+	locationIds.reserve(locations.size());
+	for (const StorageSourceLocation& location : locations)
+	{
+		locationIds.emplace_back(addSourceLocation(location));
+	}
+	return locationIds;
+}
+
 void IntermediateStorage::addOccurrence(const StorageOccurrence& occurrence)
 {
 	m_occurrences.emplace(occurrence);
@@ -205,6 +254,11 @@ void IntermediateStorage::addComponentAccess(const StorageComponentAccess& compo
 	m_componentAccesses.emplace(componentAccess);
 }
 
+void IntermediateStorage::addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses)
+{
+	m_componentAccesses.insert(componentAccesses.begin(), componentAccesses.end());
+}
+
 void IntermediateStorage::addCommentLocation(const StorageCommentLocationData& commentLocationData)
 {
 	m_commentLocations.emplace(commentLocationData);
@@ -219,86 +273,6 @@ void IntermediateStorage::addError(const StorageErrorData& errorData)
 	}
 }
 
-void IntermediateStorage::forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const
-{
-	for (const StorageNode& node : m_nodes)
-	{
-		callback(node);
-	}
-}
-
-void IntermediateStorage::forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const
-{
-	for (const StorageFile& file : m_files)
-	{
-		callback(file);
-	}
-}
-
-void IntermediateStorage::forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const
-{
-	for (const StorageSymbol& symbol : m_symbols)
-	{
-		callback(symbol);
-	}
-}
-
-void IntermediateStorage::forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const
-{
-	for (const StorageEdge& edge : m_edges)
-	{
-		callback(edge);
-	}
-}
-
-void IntermediateStorage::forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const
-{
-	for (const StorageLocalSymbol& localSymbol : m_localSymbols)
-	{
-		callback(localSymbol);
-	}
-}
-
-void IntermediateStorage::forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const
-{
-	for (const StorageSourceLocation& sourceLocation : m_sourceLocations)
-	{
-		callback(sourceLocation);
-	}
-}
-
-void IntermediateStorage::forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const
-{
-	for (const StorageOccurrence& occurrence : m_occurrences)
-	{
-		callback(occurrence);
-	}
-}
-
-void IntermediateStorage::forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const
-{
-	for (const StorageComponentAccess& componentAccess : m_componentAccesses)
-	{
-		callback(componentAccess);
-	}
-}
-
-void IntermediateStorage::forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const
-{
-	for (const StorageCommentLocationData& commentLocation : m_commentLocations)
-	{
-		callback(commentLocation);
-	}
-}
-
-void IntermediateStorage::forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const
-{
-	for (const StorageErrorData& error : m_errors)
-	{
-		callback(error);
-	}
-}
-
 const std::vector<StorageNode>& IntermediateStorage::getStorageNodes() const
 {
 	return m_nodes;

src/lib/data/storage/IntermediateStorage.h

@@ -31,40 +31,33 @@ public:
 	void setFilesWithErrorsIncomplete();
 
 	std::pair<Id, bool> addNode(const StorageNodeData& nodeData) override;
+	std::vector<Id> addNodes(const std::vector<StorageNode>& nodes) override;
 	void addSymbol(const StorageSymbol& symbol) override;
+	void addSymbols(const std::vector<StorageSymbol>& symbols) override;
 	void addFile(const StorageFile& file) override;
 	Id addEdge(const StorageEdgeData& edgeData) override;
+	std::vector<Id> addEdges(const std::vector<StorageEdge>& edges) override;
 	Id addLocalSymbol(const StorageLocalSymbolData& localSymbolData) override;
+	std::vector<Id> addLocalSymbols(const std::set<StorageLocalSymbol>& symbols) override;
 	Id addSourceLocation(const StorageSourceLocationData& sourceLocationData) override;
+	std::vector<Id> addSourceLocations(const std::vector<StorageSourceLocation>& locations) override;
 	void addOccurrence(const StorageOccurrence& occurrence) override;
 	void addOccurrences(const std::vector<StorageOccurrence>& occurrences) override;
 	void addComponentAccess(const StorageComponentAccess& componentAccess) override;
+	void addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses) override;
 	void addCommentLocation(const StorageCommentLocationData& commentLocationData) override;
 	void addError(const StorageErrorData& errorData) override;
 
-	void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const override;
-	void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const override;
-	void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const override;
-	void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const override;
-	void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const override;
-	void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const override;
-	void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const override;
-	void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const override;
-	void forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const override;
-	void forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const override;
-
-	// for conversion to and from 'SharedIntermediateStorage'
-
-	const std::vector<StorageNode>& getStorageNodes() const;
-	const std::vector<StorageFile>& getStorageFiles() const;
-	const std::vector<StorageSymbol>& getStorageSymbols() const;
-	const std::vector<StorageEdge>& getStorageEdges() const;
-	const std::set<StorageLocalSymbol>& getStorageLocalSymbols() const;
-	const std::set<StorageSourceLocation>& getStorageSourceLocations() const;
-	const std::set<StorageOccurrence>& getStorageOccurrences() const;
-	const std::set<StorageComponentAccess>& getComponentAccesses() const;
-	const std::set<StorageCommentLocationData>& getCommentLocations() const;
-	const std::vector<StorageErrorData>& getErrors() const;
+	const std::vector<StorageNode>& getStorageNodes() const override;
+	const std::vector<StorageFile>& getStorageFiles() const override;
+	const std::vector<StorageSymbol>& getStorageSymbols() const override;
+	const std::vector<StorageEdge>& getStorageEdges() const override;
+	const std::set<StorageLocalSymbol>& getStorageLocalSymbols() const override;
+	const std::set<StorageSourceLocation>& getStorageSourceLocations() const override;
+	const std::set<StorageOccurrence>& getStorageOccurrences() const override;
+	const std::set<StorageComponentAccess>& getComponentAccesses() const override;
+	const std::set<StorageCommentLocationData>& getCommentLocations() const override;
+	const std::vector<StorageErrorData>& getErrors() const override;
 
 	void setStorageNodes(std::vector<StorageNode> storageNodes);
 	void setStorageFiles(std::vector<StorageFile> storageFiles);

src/lib/data/storage/PersistentStorage.cpp

@@ -48,7 +48,12 @@ PersistentStorage::PersistentStorage(const FilePath& dbPath, const FilePath& boo
 
 std::pair<Id, bool> PersistentStorage::addNode(const StorageNodeData& data)
 {
-	return std::make_pair(m_sqliteIndexStorage.addNode(data).id, true);
+	return std::make_pair(m_sqliteIndexStorage.addNode(data), true);
+}
+
+std::vector<Id> PersistentStorage::addNodes(const std::vector<StorageNode>& nodes)
+{
+	return m_sqliteIndexStorage.addNodes(nodes);
 }
 
 void PersistentStorage::addSymbol(const StorageSymbol& data)
@@ -56,6 +61,11 @@ void PersistentStorage::addSymbol(const StorageSymbol& data)
 	m_sqliteIndexStorage.addSymbol(data);
 }
 
+void PersistentStorage::addSymbols(const std::vector<StorageSymbol>& symbols)
+{
+	m_sqliteIndexStorage.addSymbols(symbols);
+}
+
 void PersistentStorage::addFile(const StorageFile& data)
 {
 	const StorageFile storedFile = m_sqliteIndexStorage.getFirstById<StorageFile>(data.id);
@@ -80,17 +90,32 @@ void PersistentStorage::addFile(const StorageFile& data)
 
 Id PersistentStorage::addEdge(const StorageEdgeData& data)
 {
-	return m_sqliteIndexStorage.addEdge(data).id;
+	return m_sqliteIndexStorage.addEdge(data);
+}
+
+std::vector<Id> PersistentStorage::addEdges(const std::vector<StorageEdge>& edges)
+{
+	return m_sqliteIndexStorage.addEdges(edges);
 }
 
 Id PersistentStorage::addLocalSymbol(const StorageLocalSymbolData& data)
 {
-	return m_sqliteIndexStorage.addLocalSymbol(data).id;
+	return m_sqliteIndexStorage.addLocalSymbol(data);
+}
+
+std::vector<Id> PersistentStorage::addLocalSymbols(const std::set<StorageLocalSymbol>& symbols)
+{
+	return m_sqliteIndexStorage.addLocalSymbols(symbols);
 }
 
 Id PersistentStorage::addSourceLocation(const StorageSourceLocationData& data)
 {
-	return m_sqliteIndexStorage.addSourceLocation(data).id;
+	return m_sqliteIndexStorage.addSourceLocation(data);
+}
+
+std::vector<Id> PersistentStorage::addSourceLocations(const std::vector<StorageSourceLocation>& locations)
+{
+	return m_sqliteIndexStorage.addSourceLocations(locations);
 }
 
 void PersistentStorage::addOccurrence(const StorageOccurrence& data)
@@ -108,6 +133,11 @@ void PersistentStorage::addComponentAccess(const StorageComponentAccess& compone
 	m_sqliteIndexStorage.addComponentAccess(componentAccess);
 }
 
+void PersistentStorage::addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses)
+{
+	m_sqliteIndexStorage.addComponentAccesses(componentAccesses);
+}
+
 void PersistentStorage::addCommentLocation(const StorageCommentLocationData& data)
 {
 	m_sqliteIndexStorage.addCommentLocation(data);
@@ -118,84 +148,64 @@ void PersistentStorage::addError(const StorageErrorData& data)
 	m_sqliteIndexStorage.addError(data);
 }
 
-void PersistentStorage::forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const
+const std::vector<StorageNode>& PersistentStorage::getStorageNodes() const
 {
-	for (StorageNode& node: m_sqliteIndexStorage.getAll<StorageNode>())
-	{
-		callback(node);
-	}
+	return m_storageData.nodes = m_sqliteIndexStorage.getAll<StorageNode>();
 }
 
-void PersistentStorage::forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const
+const std::vector<StorageFile>& PersistentStorage::getStorageFiles() const
 {
-	for (StorageFile& file: m_sqliteIndexStorage.getAll<StorageFile>())
-	{
-		callback(file);
-	}
+	return m_storageData.files = m_sqliteIndexStorage.getAll<StorageFile>();
 }
 
-void PersistentStorage::forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const
+const std::vector<StorageSymbol>& PersistentStorage::getStorageSymbols() const
 {
-	for (StorageSymbol& symbol: m_sqliteIndexStorage.getAll<StorageSymbol>())
-	{
-		callback(symbol);
-	}
+	return m_storageData.symbols = m_sqliteIndexStorage.getAll<StorageSymbol>();
 }
 
-void PersistentStorage::forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const
+const std::vector<StorageEdge>& PersistentStorage::getStorageEdges() const
 {
-	for (StorageEdge& edge: m_sqliteIndexStorage.getAll<StorageEdge>())
-	{
-		callback(edge);
-	}
+	return m_storageData.edges = m_sqliteIndexStorage.getAll<StorageEdge>();
 }
 
-void PersistentStorage::forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const
+const std::set<StorageLocalSymbol>& PersistentStorage::getStorageLocalSymbols() const
 {
-	for (StorageLocalSymbol& localSymbol: m_sqliteIndexStorage.getAll<StorageLocalSymbol>())
-	{
-		callback(localSymbol);
-	}
+	return m_storageData.locals = utility::toSet(m_sqliteIndexStorage.getAll<StorageLocalSymbol>());
 }
 
-void PersistentStorage::forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const
+const std::set<StorageSourceLocation>& PersistentStorage::getStorageSourceLocations() const
 {
-	for (StorageSourceLocation& sourceLocation: m_sqliteIndexStorage.getAll<StorageSourceLocation>())
-	{
-		callback(sourceLocation);
-	}
+	return m_storageData.locations = utility::toSet(m_sqliteIndexStorage.getAll<StorageSourceLocation>());
 }
 
-void PersistentStorage::forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const
+const std::set<StorageOccurrence>& PersistentStorage::getStorageOccurrences() const
 {
-	for (StorageOccurrence& occurrence: m_sqliteIndexStorage.getAll<StorageOccurrence>())
-	{
-		callback(occurrence);
-	}
+	return m_storageData.occurrences = utility::toSet(m_sqliteIndexStorage.getAll<StorageOccurrence>());
 }
 
-void PersistentStorage::forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const
+const std::set<StorageComponentAccess>& PersistentStorage::getComponentAccesses() const
 {
-	for (StorageComponentAccess& componentAccess: m_sqliteIndexStorage.getAll<StorageComponentAccess>())
-	{
-		callback(componentAccess);
-	}
+	return m_storageData.accesses = utility::toSet(m_sqliteIndexStorage.getAll<StorageComponentAccess>());
 }
 
-void PersistentStorage::forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const
+const std::set<StorageCommentLocationData>& PersistentStorage::getCommentLocations() const
 {
-	for (StorageCommentLocation& commentLocation: m_sqliteIndexStorage.getAll<StorageCommentLocation>())
+	std::set<StorageCommentLocationData> comments;
+	for (const StorageCommentLocation& comment : m_sqliteIndexStorage.getAll<StorageCommentLocation>())
 	{
-		callback(commentLocation);
+		comments.emplace(comment);
 	}
+	return m_storageData.comments = comments;
 }
 
-void PersistentStorage::forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const
+const std::vector<StorageErrorData>& PersistentStorage::getErrors() const
 {
-	for (StorageError& error: m_sqliteIndexStorage.getAll<StorageError>())
+	std::vector<StorageErrorData> errors;
+	for (const StorageError& error : m_sqliteIndexStorage.getAll<StorageError>())
 	{
-		callback(error);
+		errors.emplace_back(error);
 	}
+	return m_storageData.errors = errors;
 }
 
 void PersistentStorage::startInjection()

src/lib/data/storage/PersistentStorage.h

@@ -20,27 +20,33 @@ public:
 	PersistentStorage(const FilePath& dbPath, const FilePath& bookmarkPath);
 
 	std::pair<Id, bool> addNode(const StorageNodeData& data) override;
+	std::vector<Id> addNodes(const std::vector<StorageNode>& nodes) override;
 	void addSymbol(const StorageSymbol& data) override;
+	void addSymbols(const std::vector<StorageSymbol>& symbols) override;
 	void addFile(const StorageFile& data) override;
 	Id addEdge(const StorageEdgeData& data) override;
+	std::vector<Id> addEdges(const std::vector<StorageEdge>& edges) override;
 	Id addLocalSymbol(const StorageLocalSymbolData& data) override;
+	std::vector<Id> addLocalSymbols(const std::set<StorageLocalSymbol>& symbols) override;
 	Id addSourceLocation(const StorageSourceLocationData& data) override;
+	std::vector<Id> addSourceLocations(const std::vector<StorageSourceLocation>& locations) override;
 	void addOccurrence(const StorageOccurrence& data) override;
 	void addOccurrences(const std::vector<StorageOccurrence>& occurrences) override;
 	void addComponentAccess(const StorageComponentAccess& componentAccess) override;
+	void addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses) override;
 	void addCommentLocation(const StorageCommentLocationData& data) override;
 	void addError(const StorageErrorData& data) override;
 
-	void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const override;
-	void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const override;
-	void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const override;
-	void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const override;
-	void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const override;
-	void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const override;
-	void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const override;
-	void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const override;
-	void forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const override;
-	void forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const override;
+	const std::vector<StorageNode>& getStorageNodes() const override;
+	const std::vector<StorageFile>& getStorageFiles() const override;
+	const std::vector<StorageSymbol>& getStorageSymbols() const override;
+	const std::vector<StorageEdge>& getStorageEdges() const override;
+	const std::set<StorageLocalSymbol>& getStorageLocalSymbols() const override;
+	const std::set<StorageSourceLocation>& getStorageSourceLocations() const override;
+	const std::set<StorageOccurrence>& getStorageOccurrences() const override;
+	const std::set<StorageComponentAccess>& getComponentAccesses() const override;
+	const std::set<StorageCommentLocationData>& getCommentLocations() const override;
+	const std::vector<StorageErrorData>& getErrors() const override;
 
 	void startInjection() override;
 	void finishInjection() override;
@@ -153,6 +159,19 @@ public:
 		const std::vector<Id>& locationIds, const std::vector<Id>& localSymbolIds) const override;
 
 private:
+	mutable struct {
+		std::vector<StorageNode> nodes;
+		std::vector<StorageFile> files;
+		std::vector<StorageSymbol> symbols;
+		std::vector<StorageEdge> edges;
+		std::set<StorageLocalSymbol> locals;
+		std::set<StorageSourceLocation> locations;
+		std::set<StorageOccurrence> occurrences;
+		std::set<StorageComponentAccess> accesses;
+		std::set<StorageCommentLocationData> comments;
+		std::vector<StorageErrorData> errors;
+	} m_storageData;
+
 	Id getFileNodeId(const FilePath& filePath) const;
 	std::vector<Id> getFileNodeIds(const std::vector<FilePath>& filePaths) const;
 	std::set<Id> getFileNodeIds(const std::set<FilePath>& filePaths) const;

src/lib/data/storage/Storage.cpp

@@ -2,6 +2,7 @@
 
 #include <unordered_map>
 
+#include "logging.h"
 #include "StorageCommentLocation.h"
 #include "StorageComponentAccess.h"
 #include "StorageEdge.h"
@@ -22,181 +23,263 @@ void Storage::inject(Storage* injected)
 {
 	std::lock_guard<std::mutex> lock(m_dataMutex);
 
+	std::map<Id, Id> injectedIdToOwnElementId;
+	std::map<Id, Id> injectedIdToOwnSourceLocationId;
+
 	TRACE();
 	startInjection();
 
-	injected->forEachError(
-		[&](const StorageErrorData& injectedData)
+	{
+		// TRACE("inject errors");
+
+		for (const StorageErrorData& error : injected->getErrors())
 		{
-			addError(injectedData);
+			addError(error);
 		}
-	);
+	}
 
-	std::unordered_map<Id, Id> injectedIdToOwnElementId;
+	{
+		// TRACE("inject nodes");
 
-	injected->forEachNode(
-		[&](const StorageNode& injectedData)
+		const std::vector<StorageNode>& nodes = injected->getStorageNodes();
+
+		std::vector<Id> nodeIds = addNodes(nodes);
+
+		for (size_t i = 0; i < nodes.size(); i++)
 		{
-			const Id ownId = addNode(injectedData).first;
-			if (ownId != 0)
+			if (nodeIds[i])
 			{
-				injectedIdToOwnElementId[injectedData.id] = ownId;
+				injectedIdToOwnElementId.emplace(nodes[i].id, nodeIds[i]);
 			}
 		}
-	);
+	}
 
-	injected->forEachFile(
-		[&](const StorageFile& injectedData)
+	{
+		// TRACE("inject files");
+
+		for (const StorageFile& file : injected->getStorageFiles())
 		{
-			std::unordered_map<Id, Id>::const_iterator it;
-			it = injectedIdToOwnElementId.find(injectedData.id);
+			auto it = injectedIdToOwnElementId.find(file.id);
 			if (it != injectedIdToOwnElementId.end())
 			{
-				const Id ownId = it->second;
-				addFile(StorageFile(ownId, injectedData.filePath, injectedData.modificationTime, injectedData.indexed, injectedData.complete));
+				addFile(StorageFile(
+					it->second,
+					file.filePath,
+					file.modificationTime,
+					file.indexed,
+					file.complete
+				));
 			}
 		}
-	);
+	}
 
-	injected->forEachSymbol(
-		[&](const StorageSymbol& injectedData)
+	{
+		// TRACE("inject symbols");
+
+		std::vector<StorageSymbol> symbols = injected->getStorageSymbols();
+		for (size_t i = 0; i < symbols.size(); i++)
 		{
-			std::unordered_map<Id, Id>::const_iterator it;
-			it = injectedIdToOwnElementId.find(injectedData.id);
+			auto it = injectedIdToOwnElementId.find(symbols[i].id);
 			if (it != injectedIdToOwnElementId.end())
 			{
-				const Id ownId = it->second;
-				addSymbol(StorageSymbol(ownId, injectedData.definitionKind));
+				symbols[i].id = it->second;
+			}
+			else
+			{
+				LOG_WARNING("New symbol id could not be found.");
+				symbols.erase(symbols.begin() + i);
+				i--;
 			}
 		}
-	);
 
-	injected->forEachEdge(
-		[&](const StorageEdge& injectedData)
+		addSymbols(symbols);
+	}
+
+	{
+		// TRACE("inject edges");
+
+		std::vector<StorageEdge> edges = injected->getStorageEdges();
+		for (size_t i = 0; i < edges.size(); i++)
 		{
-			std::unordered_map<Id, Id>::const_iterator it;
-			it = injectedIdToOwnElementId.find(injectedData.sourceNodeId);
-			if (it == injectedIdToOwnElementId.end())
+			StorageEdge& edge = edges[i];
+			size_t updateCount = 0;
+
+			auto it = injectedIdToOwnElementId.find(edge.sourceNodeId);
+			if (it != injectedIdToOwnElementId.end())
 			{
-				return;
+				edge.sourceNodeId = it->second;
+				updateCount++;
 			}
-			const Id ownSourceId = it->second;
 
-			it = injectedIdToOwnElementId.find(injectedData.targetNodeId);
-			if (it == injectedIdToOwnElementId.end())
+			it = injectedIdToOwnElementId.find(edge.targetNodeId);
+			if (it != injectedIdToOwnElementId.end())
 			{
-				return;
+				edge.targetNodeId = it->second;
+				updateCount++;
 			}
-			const Id ownTargetId = it->second;
 
-			const Id ownId = addEdge(StorageEdgeData(injectedData.type, ownSourceId, ownTargetId));
-
-			if (ownId != 0)
+			if (updateCount != 2)
 			{
-				injectedIdToOwnElementId[injectedData.id] = ownId;
+				LOG_WARNING("New edge source or target id could not be found.");
+				edges.erase(edges.begin() + i);
+				i--;
 			}
 		}
-	);
 
-	injected->forEachLocalSymbol(
-		[&](const StorageLocalSymbol& injectedData)
+		std::vector<Id> edgeIds = addEdges(edges);
+
+		if (edges.size() == edgeIds.size())
 		{
-			const Id ownId = addLocalSymbol(injectedData);
-			if (ownId != 0)
+			for (size_t i = 0; i < edgeIds.size(); i++)
 			{
-				injectedIdToOwnElementId[injectedData.id] = ownId;
+				if (edgeIds[i])
+				{
+					injectedIdToOwnElementId.emplace(edges[i].id, edgeIds[i]);
+				}
 			}
 		}
-	);
-
-	std::unordered_map<Id, Id> injectedIdToOwnSourceLocationId;
-
-	injected->forEachSourceLocation(
-		[&](const StorageSourceLocation& injectedData)
+		else
 		{
-			std::unordered_map<Id, Id>::const_iterator it;
-			it = injectedIdToOwnElementId.find(injectedData.fileNodeId);
+			LOG_ERROR("Returned edge ids don't match injected count.");
+		}
+	}
+
+	{
+		// TRACE("inject local symbols");
+
+		const std::set<StorageLocalSymbol>& symbols = injected->getStorageLocalSymbols();
+		std::vector<Id> symbolIds = addLocalSymbols(symbols);
+
+		auto it = symbols.begin();
+		for (size_t i = 0; i < symbols.size(); i++)
+		{
+			if (symbolIds[i])
+			{
+				injectedIdToOwnElementId.emplace(it->id, symbolIds[i]);
+			}
+			it++;
+		}
+	}
+
+	{
+		// TRACE("inject locations");
+
+		const std::set<StorageSourceLocation>& oldLocations = injected->getStorageSourceLocations();
+		std::vector<StorageSourceLocation> locations;
+		locations.reserve(oldLocations.size());
+
+		for (const StorageSourceLocation& location : oldLocations)
+		{
+			auto it = injectedIdToOwnElementId.find(location.fileNodeId);
 			if (it != injectedIdToOwnElementId.end())
 			{
 				const Id ownFileNodeId = it->second;
-
-				const Id ownId = addSourceLocation(StorageSourceLocationData(
+				locations.emplace_back(
+					location.id,
 					ownFileNodeId,
-					injectedData.startLine,
-					injectedData.startCol,
-					injectedData.endLine,
-					injectedData.endCol,
-					injectedData.type
-				));
-				if (ownId != 0)
+					location.startLine,
+					location.startCol,
+					location.endLine,
+					location.endCol,
+					location.type
+				);
+			}
+		}
+
+		std::vector<Id> locationIds = addSourceLocations(locations);
+
+		if (locations.size() == locationIds.size())
+		{
+			for (size_t i = 0; i < locationIds.size(); i++)
+			{
+				if (locationIds[i])
 				{
-					injectedIdToOwnSourceLocationId[injectedData.id] = ownId;
+					injectedIdToOwnSourceLocationId.emplace(locations[i].id, locationIds[i]);
 				}
 			}
 		}
-	);
+		else
+		{
+			LOG_ERROR("Returned source locations ids don't match injected count.");
+		}
+	}
 
 	{
+		// TRACE("inject occurrences");
+
+		const std::set<StorageOccurrence>& oldOccurences = injected->getStorageOccurrences();
+
 		std::vector<StorageOccurrence> occurrences;
-		injected->forEachOccurrence(
-			[&](const StorageOccurrence& injectedData)
+		occurrences.reserve(oldOccurences.size());
+
+		for (const StorageOccurrence& occurrence : oldOccurences)
+		{
+			Id elementId = 0;
+			Id sourceLocationId = 0;
+
+			auto it = injectedIdToOwnElementId.find(occurrence.elementId);
+			if (it != injectedIdToOwnElementId.end())
 			{
-				std::unordered_map<Id, Id>::const_iterator it;
-				it = injectedIdToOwnElementId.find(injectedData.elementId);
-				if (it == injectedIdToOwnElementId.end())
-				{
-					return;
-				}
-				const Id ownElementId = it->second;
-
-				it = injectedIdToOwnSourceLocationId.find(injectedData.sourceLocationId);
-				if (it == injectedIdToOwnSourceLocationId.end())
-				{
-					return;
-				}
-				const Id ownSourceLocationId = it->second;
-
-				occurrences.emplace_back(ownElementId, ownSourceLocationId);
+				elementId = it->second;
 			}
-		);
+
+			it = injectedIdToOwnSourceLocationId.find(occurrence.sourceLocationId);
+			if (it != injectedIdToOwnSourceLocationId.end())
+			{
+				sourceLocationId = it->second;
+			}
+
+			if (elementId && sourceLocationId)
+			{
+				occurrences.emplace_back(elementId, sourceLocationId);
+			}
+			else
+			{
+				LOG_WARNING("New occurrence element or location id could not be found.");
+			}
+		}
+
 		addOccurrences(occurrences);
 	}
 
-	injected->forEachComponentAccess(
-		[&](const StorageComponentAccess& injectedData)
+	{
+		// TRACE("inject accesses");
+
+		const std::set<StorageComponentAccess>& oldAccesses = injected->getComponentAccesses();
+		std::vector<StorageComponentAccess> accesses;
+		accesses.reserve(oldAccesses.size());
+
+		for (const StorageComponentAccess& access : oldAccesses)
 		{
-			std::unordered_map<Id, Id>::const_iterator it;
-			it = injectedIdToOwnElementId.find(injectedData.nodeId);
-			if (it == injectedIdToOwnElementId.end())
+			auto it = injectedIdToOwnElementId.find(access.nodeId);
+			if (it != injectedIdToOwnElementId.end())
 			{
-				return;
+				accesses.emplace_back(it->second, access.type);
 			}
-			const Id ownNodeId = it->second;
-
-			addComponentAccess(StorageComponentAccess(ownNodeId, injectedData.type));
 		}
-	);
 
-	injected->forEachCommentLocation(
-		[&](const StorageCommentLocationData& injectedData)
+		addComponentAccesses(accesses);
+	}
+
+	{
+		// TRACE("inject comments");
+
+		for (const StorageCommentLocationData& location : injected->getCommentLocations())
 		{
-			std::unordered_map<Id, Id>::const_iterator it;
-			it = injectedIdToOwnElementId.find(injectedData.fileNodeId);
-			if (it == injectedIdToOwnElementId.end())
+			auto it = injectedIdToOwnElementId.find(location.fileNodeId);
+			if (it != injectedIdToOwnElementId.end())
 			{
-				return;
+				const Id ownFileNodeId = it->second;
+				addCommentLocation(StorageCommentLocationData(
+					ownFileNodeId,
+					location.startLine,
+					location.startCol,
+					location.endLine,
+					location.endCol
+				));
 			}
-			const Id ownFileNodeId = it->second;
-
-			addCommentLocation(StorageCommentLocationData(
-				ownFileNodeId,
-				injectedData.startLine,
-				injectedData.startCol,
-				injectedData.endLine,
-				injectedData.endCol
-			));
 		}
-	);
+	}
 
 	finishInjection();
 }

src/lib/data/storage/Storage.h

@@ -3,6 +3,7 @@
 
 #include <functional>
 #include <mutex>
+#include <set>
 #include <string>
 
 #include "StorageCommentLocation.h"
@@ -24,27 +25,33 @@ public:
 	virtual ~Storage() = default;
 
 	virtual std::pair<Id, bool> addNode(const StorageNodeData& data) = 0;
+	virtual std::vector<Id> addNodes(const std::vector<StorageNode>& nodes) = 0;
 	virtual void addSymbol(const StorageSymbol& data) = 0;
+	virtual void addSymbols(const std::vector<StorageSymbol>& symbols) = 0;
 	virtual void addFile(const StorageFile& data) = 0;
 	virtual Id addEdge(const StorageEdgeData& data) = 0;
+	virtual std::vector<Id> addEdges(const std::vector<StorageEdge>& edges) = 0;
 	virtual Id addLocalSymbol(const StorageLocalSymbolData& data) = 0;
+	virtual std::vector<Id> addLocalSymbols(const std::set<StorageLocalSymbol>& symbols) = 0;
 	virtual Id addSourceLocation(const StorageSourceLocationData& data) = 0;
+	virtual std::vector<Id> addSourceLocations(const std::vector<StorageSourceLocation>& locations) = 0;
 	virtual void addOccurrence(const StorageOccurrence& data) = 0;
 	virtual void addOccurrences(const std::vector<StorageOccurrence>& occurrences) = 0;
 	virtual void addComponentAccess(const StorageComponentAccess& componentAccess) = 0;
+	virtual void addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses) = 0;
 	virtual void addCommentLocation(const StorageCommentLocationData& data) = 0;
 	virtual void addError(const StorageErrorData& data) = 0;
 
-	virtual void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const = 0;
-	virtual void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const = 0;
-	virtual void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const = 0;
-	virtual void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const = 0;
-	virtual void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const = 0;
-	virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const = 0;
-	virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const = 0;
-	virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const = 0;
-	virtual void forEachCommentLocation(std::function<void(const StorageCommentLocationData& /*data*/)> callback) const = 0;
-	virtual void forEachError(std::function<void(const StorageErrorData& /*data*/)> callback) const = 0;
+	virtual const std::vector<StorageNode>& getStorageNodes() const = 0;
+	virtual const std::vector<StorageFile>& getStorageFiles() const = 0;
+	virtual const std::vector<StorageSymbol>& getStorageSymbols() const = 0;
+	virtual const std::vector<StorageEdge>& getStorageEdges() const = 0;
+	virtual const std::set<StorageLocalSymbol>& getStorageLocalSymbols() const = 0;
+	virtual const std::set<StorageSourceLocation>& getStorageSourceLocations() const = 0;
+	virtual const std::set<StorageOccurrence>& getStorageOccurrences() const = 0;
+	virtual const std::set<StorageComponentAccess>& getComponentAccesses() const = 0;
+	virtual const std::set<StorageCommentLocationData>& getCommentLocations() const = 0;
+	virtual const std::vector<StorageErrorData>& getErrors() const = 0;
 
 	void inject(Storage* injected);
 

src/lib/data/storage/sqlite/SqliteIndexStorage.cpp

@@ -1,5 +1,6 @@
 #include "SqliteIndexStorage.h"
 
+#include <sstream>
 #include <unordered_map>
 
 #include "FileSystem.h"
@@ -12,7 +13,6 @@
 
 const size_t SqliteIndexStorage::s_storageVersion = 19;
 
-
 namespace
 {
 	std::tuple<std::wstring, uint32_t, uint32_t> splitLocalSymbolName(const std::wstring& name)
@@ -87,7 +87,13 @@ void SqliteIndexStorage::setProjectSettingsText(std::string text)
 	insertOrUpdateMetaValue("project_settings", text);
 }
 
-StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data)
+Id SqliteIndexStorage::addNode(const StorageNodeData& data)
+{
+	std::vector<Id> ids = addNodes({ StorageNode(0, data) });
+	return ids.size() ? ids[0] : 0;
+}
+
+std::vector<Id> SqliteIndexStorage::addNodes(const std::vector<StorageNode>& nodes)
 {
 	if (m_tempNodeNameIndex.empty() && m_tempWNodeNameIndex.empty())
 	{
@@ -107,70 +113,78 @@ StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data)
 		}
 	}
 
-	std::string name = utility::encodeToUtf8(data.serializedName);
+	std::vector<Id> nodeIds(nodes.size(), 0);
+	std::vector<StorageNode> nodesToInsert;
+	for (size_t i = 0; i < nodes.size(); i++)
 	{
-		Id nodeId;
-		if (name.size() != data.serializedName.size())
+		const StorageNodeData& data = nodes[i];
+		std::string name = utility::encodeToUtf8(data.serializedName);
 		{
-			nodeId = m_tempWNodeNameIndex.find(data.serializedName);
-		}
-		else
-		{
-			nodeId = m_tempNodeNameIndex.find(name);
-		}
-
-		if (nodeId)
-		{
-			auto it = m_tempNodeTypes.find(nodeId);
-			if (it != m_tempNodeTypes.end() && it->second < data.type)
+			Id nodeId;
+			if (name.size() != data.serializedName.size())
 			{
-				setNodeType(data.type, nodeId);
-				m_tempNodeTypes[nodeId] = data.type;
+				nodeId = m_tempWNodeNameIndex.find(data.serializedName);
+			}
+			else
+			{
+				nodeId = m_tempNodeNameIndex.find(name);
+			}
+
+			if (nodeId)
+			{
+				auto it = m_tempNodeTypes.find(nodeId);
+				if (it != m_tempNodeTypes.end() && it->second < data.type)
+				{
+					setNodeType(data.type, nodeId);
+					m_tempNodeTypes[nodeId] = data.type;
+				}
+
+				nodeIds[i] = nodeId;
+			}
+			else
+			{
+				executeStatement(m_insertElementStmt);
+				Id id = m_database.lastRowId();
+
+				nodesToInsert.emplace_back(id, data);
+				nodeIds[i] = id;
+
+				if (name.size() != data.serializedName.size())
+				{
+					m_tempWNodeNameIndex.add(data.serializedName, id);
+				}
+				else
+				{
+					m_tempNodeNameIndex.add(name, id);
+				}
+				m_tempNodeTypes.emplace(id, data.type);
 			}
-			return StorageNode(nodeId, data);
 		}
 	}
 
-	Id id = 0;
+	if (nodesToInsert.size())
 	{
-		executeStatement(m_insertElementStmt);
-		id = m_database.lastRowId();
-		m_insertElementStmt.reset();
-	}
-	{
-		m_inserNodeStmt.bind(1, int(id));
-		m_inserNodeStmt.bind(2, data.type);
-		m_inserNodeStmt.bind(3, name.c_str());
-		executeStatement(m_inserNodeStmt);
-		m_inserNodeStmt.reset();
+		m_insertNodeBatchStatement.execute(nodesToInsert, this);
 	}
 
-	if (name.size() != data.serializedName.size())
-	{
-		m_tempWNodeNameIndex.add(data.serializedName, id);
-	}
-	else
-	{
-		m_tempNodeNameIndex.add(name, id);
-	}
-	m_tempNodeTypes.emplace(id, data.type);
-
-	return StorageNode(id, data);
+	return nodeIds;
 }
 
-void SqliteIndexStorage::addSymbol(const StorageSymbol& data)
+bool SqliteIndexStorage::addSymbol(const StorageSymbol& data)
 {
-	m_insertSymbolStmt.bind(1, int(data.id));
-	m_insertSymbolStmt.bind(2, data.definitionKind);
-	executeStatement(m_insertSymbolStmt);
-	m_insertSymbolStmt.reset();
+	return addSymbols({ data });
 }
 
-void SqliteIndexStorage::addFile(const StorageFile& data)
+bool SqliteIndexStorage::addSymbols(const std::vector<StorageSymbol>& symbols)
+{
+	return m_insertSymbolBatchStatement.execute(symbols, this);
+}
+
+bool SqliteIndexStorage::addFile(const StorageFile& data)
 {
 	if (getFileByPath(data.filePath).id != 0)
 	{
-		return;
+		return false;
 	}
 
 	FilePath filePath(data.filePath);
@@ -198,19 +212,25 @@ void SqliteIndexStorage::addFile(const StorageFile& data)
 		m_insertFileStmt.bind(5, data.complete);
 		m_insertFileStmt.bind(6, lineCount);
 		success = executeStatement(m_insertFileStmt);
-		m_insertFileStmt.reset();
 	}
 
 	if (success && content)
 	{
 		m_insertFileContentStmt.bind(1, int(data.id));
 		m_insertFileContentStmt.bind(2, content->getText().c_str());
-		executeStatement(m_insertFileContentStmt);
-		m_insertFileContentStmt.reset();
+		success = executeStatement(m_insertFileContentStmt);
 	}
+
+	return success;
 }
 
-StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data)
+Id SqliteIndexStorage::addEdge(const StorageEdgeData& data)
+{
+	std::vector<Id> ids = addEdges({ StorageEdge(0, data) });
+	return ids.size() ? ids[0] : 0;
+}
+
+std::vector<Id> SqliteIndexStorage::addEdges(const std::vector<StorageEdge>& edges)
 {
 	if (m_tempEdgeIndex.empty())
 	{
@@ -220,35 +240,43 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data)
 		}
 	}
 
+	std::vector<Id> edgeIds(edges.size(), 0);
+	std::vector<StorageEdge> edgesToInsert;
+	for (size_t i = 0; i < edges.size(); i++)
 	{
+		const StorageEdge& data = edges[i];
 		std::map<StorageEdgeData, Id>::const_iterator it = m_tempEdgeIndex.find(data);
 		if (it != m_tempEdgeIndex.end())
 		{
-			return StorageEdge(it->second, data);
+			edgeIds[i] = it->second;
+		}
+		else
+		{
+			executeStatement(m_insertElementStmt);
+			Id id = m_database.lastRowId();
+
+			edgeIds[i] = id;
+			edgesToInsert.emplace_back(id, data);
+
+			m_tempEdgeIndex.emplace(data, id);
 		}
 	}
 
-	Id id = 0;
+	if (edgesToInsert.size())
 	{
-		executeStatement(m_insertElementStmt);
-		id = m_database.lastRowId();
-		m_insertElementStmt.reset();
-	}
-	{
-		m_insertEdgeStmt.bind(1, int(id));
-		m_insertEdgeStmt.bind(2, data.type);
-		m_insertEdgeStmt.bind(3, int(data.sourceNodeId));
-		m_insertEdgeStmt.bind(4, int(data.targetNodeId));
-		executeStatement(m_insertEdgeStmt);
-		m_insertEdgeStmt.reset();
+		m_insertEdgeBatchStatement.execute(edgesToInsert, this);
 	}
 
-	m_tempEdgeIndex.emplace(data, id);
-
-	return StorageEdge(id, data);
+	return edgeIds;
 }
 
-StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data)
+Id SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolData& data)
+{
+	std::vector<Id> ids = addLocalSymbols({ StorageLocalSymbol(0, data) });
+	return ids.size() ? ids[0] : 0;
+}
+
+std::vector<Id> SqliteIndexStorage::addLocalSymbols(const std::set<StorageLocalSymbol>& symbols)
 {
 	std::wstring name;
 	uint32_t line;
@@ -266,42 +294,55 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa
 		}
 	}
 
-	std::tie(name, line, col) = splitLocalSymbolName(data.name);
-	if (name.size())
+	std::vector<Id> symbolIds(symbols.size(), 0);
+	std::vector<StorageLocalSymbol> symbolsToInsert;
+	auto it = symbols.begin();
+	for (size_t i = 0; i < symbols.size(); i++)
 	{
-		auto it = m_tempLocalSymbolIndex.find(name);
-		if (it != m_tempLocalSymbolIndex.end())
+		const StorageLocalSymbol& data = *it;
+		std::tie(name, line, col) = splitLocalSymbolName(data.name);
+		if (name.size())
 		{
-			auto it2 = it->second.find(std::make_pair(line, col));
-			if (it2 != it->second.end())
+			auto it = m_tempLocalSymbolIndex.find(name);
+			if (it != m_tempLocalSymbolIndex.end())
 			{
-				return StorageLocalSymbol(it2->second, data);
+				auto it2 = it->second.find(std::make_pair(line, col));
+				if (it2 != it->second.end())
+				{
+					symbolIds[i] = it2->second;
+				}
 			}
 		}
+
+		if (!symbolIds[i])
+		{
+			executeStatement(m_insertElementStmt);
+			Id id = m_database.lastRowId();
+
+			symbolIds[i] = id;
+			symbolsToInsert.emplace_back(id, data);
+
+			m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id);
+		}
+
+		it++;
 	}
 
-	Id id = 0;
+	if (symbolsToInsert.size())
 	{
-		executeStatement(m_insertElementStmt);
-		id = m_database.lastRowId();
-		m_insertElementStmt.reset();
-	}
-	{
-		m_inserLocalSymbolStmt.bind(1, int(id));
-		m_inserLocalSymbolStmt.bind(2, utility::encodeToUtf8(data.name).c_str());
-		executeStatement(m_inserLocalSymbolStmt);
-		m_inserLocalSymbolStmt.reset();
+		m_insertLocalSymbolBatchStatement.execute(symbolsToInsert, this);
 	}
 
-	if (name.size())
-	{
-		m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id);
-	}
-
-	return StorageLocalSymbol(id, data);
+	return symbolIds;
 }
 
-StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceLocationData& data)
+Id SqliteIndexStorage::addSourceLocation(const StorageSourceLocationData& data)
+{
+	std::vector<Id> ids = addSourceLocations({ StorageSourceLocation(0, data) });
+	return ids.size() ? ids[0] : 0;
+}
+
+std::vector<Id> SqliteIndexStorage::addSourceLocations(const std::vector<StorageSourceLocation>& locations)
 {
 	if (m_tempSourceLocationIndices.empty())
 	{
@@ -314,98 +355,59 @@ StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceL
 		}
 	}
 
-	const TempSourceLocation tempLoc(data.startLine, data.endLine - data.startLine, data.startCol, data.endCol, data.type);
+	std::vector<Id> locationIds(locations.size(), 0);
+	std::vector<StorageSourceLocationData> locationsToInsert;
+	size_t lastRowId = executeStatementScalar("SELECT MAX(rowid) from source_location", 0);
 
-	std::map<TempSourceLocation, Id>& index = m_tempSourceLocationIndices[data.fileNodeId];
+	for (size_t i = 0; i < locations.size(); i++)
 	{
+		const StorageSourceLocation& data = locations[i];
+		const TempSourceLocation tempLoc(data.startLine, data.endLine - data.startLine, data.startCol, data.endCol, data.type);
+
+		std::map<TempSourceLocation, Id>& index = m_tempSourceLocationIndices[data.fileNodeId];
 		std::map<TempSourceLocation, Id>::const_iterator it = index.find(tempLoc);
 		if (it != index.end())
 		{
-			return StorageSourceLocation(it->second, data);
+			locationIds[i] = it->second;
+		}
+		else
+		{
+			executeStatement(m_insertElementStmt);
+			Id id = lastRowId + 1 + locationsToInsert.size();
+
+			locationIds[i] = id;
+			index.emplace(tempLoc, id);
+
+			locationsToInsert.emplace_back(data);
 		}
 	}
 
-	Id id = 0;
-	m_insertSourceLocationStmt.bind(1, int(data.fileNodeId));
-	m_insertSourceLocationStmt.bind(2, int(data.startLine));
-	m_insertSourceLocationStmt.bind(3, int(data.startCol));
-	m_insertSourceLocationStmt.bind(4, int(data.endLine));
-	m_insertSourceLocationStmt.bind(5, int(data.endCol));
-	m_insertSourceLocationStmt.bind(6, data.type);
-
-	const bool success = executeStatement(m_insertSourceLocationStmt);
-	if (success)
+	if (locationsToInsert.size())
 	{
-		id = m_database.lastRowId();
-		index.emplace(tempLoc, id);
+		m_insertSourceLocationBatchStatement.execute(locationsToInsert, this);
 	}
 
-	m_insertSourceLocationStmt.reset();
-
-	return StorageSourceLocation(id, data);
+	return locationIds;
 }
 
 bool SqliteIndexStorage::addOccurrence(const StorageOccurrence& data)
 {
-	m_insertOccurrenceStmt.bind(1, int(data.elementId));
-	m_insertOccurrenceStmt.bind(2, int(data.sourceLocationId));
-	const bool success = executeStatement(m_insertOccurrenceStmt);
-	m_insertOccurrenceStmt.reset();
-	return success;
+	return addOccurrences({ data });
 }
 
 bool SqliteIndexStorage::addOccurrences(const std::vector<StorageOccurrence>& occurrences)
 {
-	size_t i = 0;
-	while (occurrences.size() - i >= 100)
-	{
-		for (size_t j = 0; j < 100; j++)
-		{
-			m_insert100OccurrencesStmt.bind((j * 2) + 1, int(occurrences[i + j].elementId));
-			m_insert100OccurrencesStmt.bind((j * 2) + 2, int(occurrences[i + j].sourceLocationId));
-		}
-
-		const bool success = executeStatement(m_insert100OccurrencesStmt);
-		m_insert100OccurrencesStmt.reset();
-
-		if (!success)
-		{
-			return false;
-		}
-
-		i += 100;
-	}
-
-	if (i < occurrences.size())
-	{
-		std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES";
-		{
-			bool isFirst = true;
-			while (i < occurrences.size())
-			{
-				if (!isFirst)
-				{
-					stmt += ",";
-				}
-				isFirst = false;
-				stmt += "(" + std::to_string(occurrences[i].elementId) + "," + std::to_string(occurrences[i].sourceLocationId) + ")";
-				i++;
-			}
-			stmt += ";";
-		}
-		return executeStatement(stmt);
-	}
-
-	return true;
+	return m_insertOccurenceBatchStatement.execute(occurrences, this);
 }
 
 bool SqliteIndexStorage::addComponentAccess(const StorageComponentAccess& componentAccess)
 {
-	m_insertComponentAccessStmt.bind(1, int(componentAccess.nodeId));
-	m_insertComponentAccessStmt.bind(2, componentAccess.type);
-	const bool success = executeStatement(m_insertComponentAccessStmt);
-	m_insertComponentAccessStmt.reset();
-	return success;
+	return addComponentAccesses({ componentAccess });
+}
+
+bool SqliteIndexStorage::addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses)
+{
+	return m_insertComponentAccessBatchStatement.execute(componentAccesses, this);
 }
 
 StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageCommentLocationData& data)
@@ -423,7 +425,6 @@ StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageComme
 		{
 			id = checkQuery.getIntField(0, 0);
 		}
-
 		m_checkCommentLocationExistsStmt.reset();
 	}
 
@@ -440,8 +441,6 @@ StorageCommentLocation SqliteIndexStorage::addCommentLocation(const StorageComme
 		{
 			id = m_database.lastRowId();
 		}
-
-		m_insertCommentLocationStmt.reset();
 	}
 
 	return StorageCommentLocation(id, data);
@@ -464,7 +463,6 @@ StorageError SqliteIndexStorage::addError(const StorageErrorData& data)
 		{
 			id = checkQuery.getIntField(0, -1);
 		}
-
 		m_checkErrorExistsStmt.reset();
 	}
 
@@ -483,8 +481,6 @@ StorageError SqliteIndexStorage::addError(const StorageErrorData& data)
 		{
 			id = m_database.lastRowId();
 		}
-
-		m_insertErrorStmt.reset();
 	}
 
 	return StorageError(id, data);
@@ -787,6 +783,8 @@ StorageNode SqliteIndexStorage::getNodeBySerializedName(const std::wstring& seri
 		}
 	}
 
+	stmt.reset();
+
 	return StorageNode();
 }
 
@@ -1252,50 +1250,93 @@ void SqliteIndexStorage::setupPrecompiledStatements()
 {
 	try
 	{
+		m_insertNodeBatchStatement.compile(
+			"INSERT INTO node(id, type, serialized_name) VALUES",
+			3,
+			[](CppSQLite3Statement& stmt, const StorageNode& node, size_t index)
+			{
+				stmt.bind(index * 3 + 1, int(node.id));
+				stmt.bind(index * 3 + 2, int(node.type));
+				stmt.bind(index * 3 + 3, utility::encodeToUtf8(node.serializedName).c_str());
+			},
+			m_database
+		);
+		m_insertEdgeBatchStatement.compile(
+			"INSERT INTO edge(id, type, source_node_id, target_node_id) VALUES",
+			4,
+			[](CppSQLite3Statement& stmt, const StorageEdge& edge, size_t index)
+			{
+				stmt.bind(index * 4 + 1, int(edge.id));
+				stmt.bind(index * 4 + 2, int(edge.type));
+				stmt.bind(index * 4 + 3, int(edge.sourceNodeId));
+				stmt.bind(index * 4 + 4, int(edge.targetNodeId));
+			},
+			m_database
+		);
+		m_insertSymbolBatchStatement.compile(
+			"INSERT OR IGNORE INTO symbol(id, definition_kind) VALUES",
+			2,
+			[](CppSQLite3Statement& stmt, const StorageSymbol& symbol, size_t index)
+			{
+				stmt.bind(index * 2 + 1, int(symbol.id));
+				stmt.bind(index * 2 + 2, int(symbol.definitionKind));
+			},
+			m_database
+		);
+		m_insertLocalSymbolBatchStatement.compile(
+			"INSERT INTO local_symbol(id, name) VALUES",
+			2,
+			[](CppSQLite3Statement& stmt, const StorageLocalSymbol& symbol, size_t index)
+			{
+				stmt.bind(index * 2 + 1, int(symbol.id));
+				stmt.bind(index * 2 + 2, utility::encodeToUtf8(symbol.name).c_str());
+			},
+			m_database
+		);
+		m_insertSourceLocationBatchStatement.compile(
+			"INSERT INTO source_location(file_node_id, start_line, start_column, end_line, end_column, type) VALUES",
+			6,
+			[](CppSQLite3Statement& stmt, const StorageSourceLocationData& location, size_t index)
+			{
+				stmt.bind(index * 6 + 1, int(location.fileNodeId));
+				stmt.bind(index * 6 + 2, int(location.startLine));
+				stmt.bind(index * 6 + 3, int(location.startCol));
+				stmt.bind(index * 6 + 4, int(location.endLine));
+				stmt.bind(index * 6 + 5, int(location.endCol));
+				stmt.bind(index * 6 + 6, int(location.type));
+			},
+			m_database
+		);
+		m_insertOccurenceBatchStatement.compile(
+			"INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES",
+			2,
+			[](CppSQLite3Statement& stmt, const StorageOccurrence& occurrence, size_t index)
+			{
+				stmt.bind(index * 2 + 1, int(occurrence.elementId));
+				stmt.bind(index * 2 + 2, int(occurrence.sourceLocationId));
+			},
+			m_database
+		);
+		m_insertComponentAccessBatchStatement.compile(
+			"INSERT OR IGNORE INTO component_access(node_id, type) VALUES",
+			2,
+			[](CppSQLite3Statement& stmt, const StorageComponentAccess& componentAccess, size_t index)
+			{
+				stmt.bind(index * 2 + 1, int(componentAccess.nodeId));
+				stmt.bind(index * 2 + 2, int(componentAccess.type));
+			},
+			m_database
+		);
+
 		m_insertElementStmt = m_database.compileStatement(
 			"INSERT INTO element(id) VALUES(NULL);"
 		);
-		m_insertEdgeStmt = m_database.compileStatement(
-			"INSERT INTO edge(id, type, source_node_id, target_node_id) VALUES(?, ?, ?, ?);"
-		);
-		m_inserNodeStmt = m_database.compileStatement(
-			"INSERT INTO node(id, type, serialized_name) VALUES(?, ?, ?);"
-		);
-		m_insertSymbolStmt = m_database.compileStatement(
-			"INSERT OR IGNORE INTO symbol(id, definition_kind) VALUES(?, ?);"
-		);
 		m_insertFileStmt = m_database.compileStatement(
 			"INSERT INTO file(id, path, modification_time, indexed, complete, line_count) VALUES(?, ?, ?, ?, ?, ?);"
 		);
 		m_insertFileContentStmt = m_database.compileStatement(
 			"INSERT INTO filecontent(id, content) VALUES(?, ?);"
 		);
-		m_inserLocalSymbolStmt = m_database.compileStatement(
-			"INSERT INTO local_symbol(id, name) VALUES(?, ?);"
-		);
-		m_insertSourceLocationStmt = m_database.compileStatement(
-			"INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) "
-			"VALUES(NULL, ?, ?, ?, ?, ?, ?);"
-		);
-		m_insertOccurrenceStmt = m_database.compileStatement(
-			"INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES(?, ?);"
-		);
-		{
-			std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES";
-			for (int i = 0; i < 100; i++)
-			{
-				if (i != 0)
-				{
-					stmt += ",";
-				}
-				stmt += "(?, ?)";
-			}
-			stmt += ";";
-			m_insert100OccurrencesStmt = m_database.compileStatement(stmt.c_str());
-		}
-		m_insertComponentAccessStmt = m_database.compileStatement(
-			"INSERT OR IGNORE INTO component_access(node_id, type) VALUES(?, ?);"
-		);
 		m_checkCommentLocationExistsStmt = m_database.compileStatement(
 			"SELECT id FROM comment_location WHERE "
 			"file_node_id = ? AND "

src/lib/data/storage/sqlite/SqliteIndexStorage.h

@@ -49,15 +49,21 @@ public:
 	std::string getProjectSettingsText() const;
 	void setProjectSettingsText(std::string text);
 
-	StorageNode addNode(const StorageNodeData& data);
-	void addSymbol(const StorageSymbol& data);
-	void addFile(const StorageFile& data);
-	StorageEdge addEdge(const StorageEdgeData& data);
-	StorageLocalSymbol addLocalSymbol(const StorageLocalSymbolData& data);
-	StorageSourceLocation addSourceLocation(const StorageSourceLocationData& data);
+	Id addNode(const StorageNodeData& data);
+	std::vector<Id> addNodes(const std::vector<StorageNode>& nodes);
+	bool addSymbol(const StorageSymbol& data);
+	bool addSymbols(const std::vector<StorageSymbol>& symbols);
+	bool addFile(const StorageFile& data);
+	Id addEdge(const StorageEdgeData& data);
+	std::vector<Id> addEdges(const std::vector<StorageEdge>& edges);
+	Id addLocalSymbol(const StorageLocalSymbolData& data);
+	std::vector<Id> addLocalSymbols(const std::set<StorageLocalSymbol>& symbols);
+	Id addSourceLocation(const StorageSourceLocationData& data);
+	std::vector<Id> addSourceLocations(const std::vector<StorageSourceLocation>& locations);
 	bool addOccurrence(const StorageOccurrence& data);
 	bool addOccurrences(const std::vector<StorageOccurrence>& occurrences);
 	bool addComponentAccess(const StorageComponentAccess& componentAccess);
+	bool addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses);
 	StorageCommentLocation addCommentLocation(const StorageCommentLocationData& data);
 	StorageError addError(const StorageErrorData& data);
 
@@ -223,23 +229,92 @@ private:
 	std::map<std::wstring, std::map<std::pair<uint32_t, uint32_t>, Id>> m_tempLocalSymbolIndex;
 	std::map<Id, std::map<TempSourceLocation, Id>> m_tempSourceLocationIndices;
 
+	template <typename StorageType>
+	class InsertBatchStatement
+	{
+	public:
+		void compile(
+			const std::string header,
+			size_t valueCount,
+			std::function<void(CppSQLite3Statement& stmt, const StorageType&, size_t)> bindValuesFunc,
+			CppSQLite3DB& database)
+		{
+			m_bindValuesFunc = bindValuesFunc;
+
+			std::string valueStr = '(' + utility::join(std::vector<std::string>(valueCount, "?"), ',') + ')';
+
+			for (size_t j = 0; j < 4; j++)
+			{
+				std::stringstream stmt;
+				stmt << header;
+
+				for (size_t i = 0; i < BATCH_SIZES[j]; i++)
+				{
+					if (i != 0)
+					{
+						stmt << ',';
+					}
+					stmt << valueStr;
+				}
+				stmt << ';';
+
+				m_stmt[j] = database.compileStatement(stmt.str().c_str());
+			}
+		}
+
+		bool execute(const std::vector<StorageType>& types, SqliteIndexStorage* storage)
+		{
+			size_t i = 0;
+			for (size_t x = 0; x < 4; x++)
+			{
+				while (types.size() - i >= BATCH_SIZES[x])
+				{
+					for (size_t j = 0; j < BATCH_SIZES[x]; j++)
+					{
+						m_bindValuesFunc(m_stmt[x], types[i + j], j);
+					}
+
+					const bool success = storage->executeStatement(m_stmt[x]);
+					if (!success)
+					{
+						return false;
+					}
+
+					i += BATCH_SIZES[x];
+				}
+			}
+
+			return true;
+		}
+
+	private:
+		static const size_t BATCH_SIZES[4];
+
+		CppSQLite3Statement m_stmt[4];
+
+		std::function<void(CppSQLite3Statement& stmt, const StorageType&, size_t)> m_bindValuesFunc;
+	};
+
+	InsertBatchStatement<StorageNode> m_insertNodeBatchStatement;
+	InsertBatchStatement<StorageEdge> m_insertEdgeBatchStatement;
+	InsertBatchStatement<StorageSymbol> m_insertSymbolBatchStatement;
+	InsertBatchStatement<StorageLocalSymbol> m_insertLocalSymbolBatchStatement;
+	InsertBatchStatement<StorageSourceLocationData> m_insertSourceLocationBatchStatement;
+	InsertBatchStatement<StorageOccurrence> m_insertOccurenceBatchStatement;
+	InsertBatchStatement<StorageComponentAccess> m_insertComponentAccessBatchStatement;
+
 	CppSQLite3Statement m_insertElementStmt;
-	CppSQLite3Statement m_insertEdgeStmt;
-	CppSQLite3Statement m_inserNodeStmt;
-	CppSQLite3Statement m_insertSymbolStmt;
 	CppSQLite3Statement m_insertFileStmt;
 	CppSQLite3Statement m_insertFileContentStmt;
-	CppSQLite3Statement m_inserLocalSymbolStmt;
-	CppSQLite3Statement m_insertSourceLocationStmt;
-	CppSQLite3Statement m_insertOccurrenceStmt;
-	CppSQLite3Statement m_insert100OccurrencesStmt;
-	CppSQLite3Statement m_insertComponentAccessStmt;
 	CppSQLite3Statement m_checkCommentLocationExistsStmt;
 	CppSQLite3Statement m_insertCommentLocationStmt;
 	CppSQLite3Statement m_checkErrorExistsStmt;
 	CppSQLite3Statement m_insertErrorStmt;
 };
 
+template<typename StorageType>
+const size_t SqliteIndexStorage::InsertBatchStatement<StorageType>::BATCH_SIZES[4] = { 100, 27, 8, 1 };
+
 template <>
 std::vector<StorageEdge> SqliteIndexStorage::doGetAll<StorageEdge>(const std::string& query) const;
 template <>

src/lib/data/storage/sqlite/SqliteStorage.cpp

@@ -182,6 +182,8 @@ bool SqliteStorage::executeStatement(CppSQLite3Statement& statement) const
 		LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage());
 		return false;
 	}
+
+	statement.reset();
 	return true;
 }
 
@@ -222,6 +224,7 @@ int SqliteStorage::executeStatementScalar(CppSQLite3Statement& statement, const
 	{
 		LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage());
 	}
+
 	return ret;
 }
 

src/test/SqliteIndexStorageTestSuite.h

@@ -32,7 +32,7 @@ public:
 			SqliteIndexStorage storage(databasePath);
 			storage.setup();
 			storage.beginTransaction();
-			int nodeId = storage.addNode(StorageNodeData(0, L"a")).id;
+			int nodeId = storage.addNode(StorageNodeData(0, L"a"));
 			storage.removeElement(nodeId);
 			storage.commitTransaction();
 			nodeCount = storage.getNodeCount();
@@ -50,8 +50,8 @@ public:
 			SqliteIndexStorage storage(databasePath);
 			storage.setup();
 			storage.beginTransaction();
-			int sourceNodeId = storage.addNode(StorageNodeData(0, L"a")).id;
-			int targetNodeId = storage.addNode(StorageNodeData(0, L"b")).id;
+			int sourceNodeId = storage.addNode(StorageNodeData(0, L"a"));
+			int targetNodeId = storage.addNode(StorageNodeData(0, L"b"));
 			storage.addEdge(StorageEdgeData(0, sourceNodeId, targetNodeId));
 			storage.commitTransaction();
 			edgeCount = storage.getEdgeCount();
@@ -69,9 +69,9 @@ public:
 			SqliteIndexStorage storage(databasePath);
 			storage.setup();
 			storage.beginTransaction();
-			int sourceNodeId = storage.addNode(StorageNodeData(0, L"a")).id;
-			int targetNodeId = storage.addNode(StorageNodeData(0, L"b")).id;
-			int edgeId = storage.addEdge(StorageEdgeData(0, sourceNodeId, targetNodeId)).id;
+			int sourceNodeId = storage.addNode(StorageNodeData(0, L"a"));
+			int targetNodeId = storage.addNode(StorageNodeData(0, L"b"));
+			int edgeId = storage.addEdge(StorageEdgeData(0, sourceNodeId, targetNodeId));
 			storage.removeElement(edgeId);
 			storage.commitTransaction();
 			edgeCount = storage.getEdgeCount();

src/test/StorageTestSuite.h

@@ -69,14 +69,13 @@ public:
 
 		const Id sourceId = storage.getNodeIdForNameHierarchy(a);
 		const Id targetId = storage.getNodeIdForNameHierarchy(b);
-		storage.forEachEdge([&](const StorageEdge& edge)
+		for (auto edge : storage.getStorageEdges())
+		{
+			if (edge.sourceNodeId == sourceId && edge.targetNodeId == targetId && edge.type == Edge::typeToInt(Edge::EDGE_MEMBER))
 			{
-				if (edge.sourceNodeId == sourceId && edge.targetNodeId == targetId && edge.type == Edge::typeToInt(Edge::EDGE_MEMBER))
-				{
-					foundEdge = true;
-				}
+				foundEdge = true;
 			}
-		);
+		}
 		TS_ASSERT(foundEdge);
 	}