data: reduced size of generated .coatidb files

* added occurrence table to get rid of duplicate source locations
* reverted storage provider

src/lib/data/IntermediateStorage.cpp

@@ -22,7 +22,9 @@ void IntermediateStorage::clear()
 	m_edgeIdsToData.clear();
 	m_localSymbolNamesToIds.clear();
 	m_localSymbolIdsToData.clear();
-	m_sourceLocations.clear();
+	m_sourceLocationNamesToIds.clear();
+	m_sourceLocationIdsToData.clear();
+	m_occurrences.clear();
 	m_componentAccesses.clear();
 	m_commentLocations.clear();
 	m_errors.clear();
@@ -31,7 +33,7 @@ void IntermediateStorage::clear()
 
 size_t IntermediateStorage::getSourceLocationCount() const
 {
-	return m_sourceLocations.size();
+	return m_sourceLocationNamesToIds.size();
 }
 
 Id IntermediateStorage::addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime)
@@ -129,18 +131,35 @@ Id IntermediateStorage::addLocalSymbol(const std::string& name)
 	return id;
 }
 
-void IntermediateStorage::addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
+Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
 {
-	m_sourceLocations.push_back(StorageSourceLocation(
+	std::shared_ptr<StorageSourceLocation> sourceLocation = std::make_shared<StorageSourceLocation>(
 		0,
-		elementId,
 		fileNodeId,
 		startLine,
 		startCol,
 		endLine,
 		endCol,
 		type
-	));
+	);
+
+	std::string serialized = serialize(*(sourceLocation.get()));
+	std::unordered_map<std::string, Id>::const_iterator it = m_sourceLocationNamesToIds.find(serialized);
+	if (it != m_sourceLocationNamesToIds.end())
+	{
+		return it->second;
+	}
+
+	Id id = m_nextId++;
+	m_sourceLocationNamesToIds[serialized] = id;
+	m_sourceLocationIdsToData[id] = sourceLocation;
+
+	return id;
+}
+
+void IntermediateStorage::addOccurrence(Id elementId, Id sourceLocationId)
+{
+	m_occurrences.push_back(StorageOccurrence(elementId, sourceLocationId));
 }
 
 void IntermediateStorage::addComponentAccess(Id nodeId, int type)
@@ -205,9 +224,17 @@ void IntermediateStorage::forEachLocalSymbol(std::function<void(const Id /*id*/,
 	}
 }
 
-void IntermediateStorage::forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const
+void IntermediateStorage::forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const
 {
-	for (std::vector<StorageSourceLocation>::const_iterator it = m_sourceLocations.begin(); it != m_sourceLocations.end(); it++)
+	for (std::map<Id, std::shared_ptr<StorageSourceLocation>>::const_iterator it = m_sourceLocationIdsToData.begin(); it != m_sourceLocationIdsToData.end(); it++)
+	{
+		callback(it->first, *(it->second.get()));
+	}
+}
+
+void IntermediateStorage::forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const
+{
+	for (std::vector<StorageOccurrence>::const_iterator it = m_occurrences.begin(); it != m_occurrences.end(); it++)
 	{
 		callback(*it);
 	}
@@ -237,7 +264,7 @@ void IntermediateStorage::forEachError(std::function<void(const StorageError& /*
 	}
 }
 
-std::string IntermediateStorage::serialize(const StorageEdge& edge)
+std::string IntermediateStorage::serialize(const StorageEdge& edge) const
 {
 	return (
 		std::to_string(edge.type) +
@@ -246,17 +273,29 @@ std::string IntermediateStorage::serialize(const StorageEdge& edge)
 	);
 }
 
-std::string IntermediateStorage::serialize(const StorageNode& node)
+std::string IntermediateStorage::serialize(const StorageNode& node) const
 {
 	return node.serializedName;
 }
 
-std::string IntermediateStorage::serialize(const StorageFile& file)
+std::string IntermediateStorage::serialize(const StorageFile& file) const
 {
 	return file.filePath;
 }
 
-std::string IntermediateStorage::serialize(const StorageLocalSymbol& localSymbol)
+std::string IntermediateStorage::serialize(const StorageLocalSymbol& localSymbol) const
 {
 	return localSymbol.name;
 }
+
+std::string IntermediateStorage::serialize(const StorageSourceLocation& sourceLocation) const
+{
+	return (
+		std::to_string(sourceLocation.fileNodeId) +
+		std::to_string(sourceLocation.startLine) +
+		std::to_string(sourceLocation.startCol) +
+		std::to_string(sourceLocation.endLine) +
+		std::to_string(sourceLocation.endCol) +
+		std::to_string(sourceLocation.type)
+	);
+}

src/lib/data/IntermediateStorage.h

@@ -21,7 +21,8 @@ public:
 	virtual Id addNode(int type, const std::string& serializedName, int definitionType);
 	virtual Id addEdge(int type, Id sourceId, Id targetId);
 	virtual Id addLocalSymbol(const std::string& name);
-	virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type);
+	virtual Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type);
+	virtual void addOccurrence(Id elementId, Id sourceLocationId);
 	virtual void addComponentAccess(Id nodeId , int type);
 	virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol);
 	virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed);
@@ -30,16 +31,18 @@ public:
 	virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const;
 	virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const;
 	virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const;
-	virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const;
+	virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const;
+	virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const;
 	virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const;
 	virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const;
 	virtual void forEachError(std::function<void(const StorageError& /*data*/)> callback) const;
 
 private:
-	std::string serialize(const StorageEdge& edge);
-	std::string serialize(const StorageNode& node);
-	std::string serialize(const StorageFile& file);
-	std::string serialize(const StorageLocalSymbol& localSymbol);
+	std::string serialize(const StorageEdge& edge) const;
+	std::string serialize(const StorageNode& node) const;
+	std::string serialize(const StorageFile& file) const;
+	std::string serialize(const StorageLocalSymbol& localSymbol) const;
+	std::string serialize(const StorageSourceLocation& sourceLocation) const;
 
 	std::unordered_map<std::string, Id> m_fileNamesToIds; // this is used to prevent duplicates (unique)
 	std::unordered_map<Id, std::shared_ptr<StorageFile>> m_fileIdsToData;
@@ -54,7 +57,10 @@ private:
 	std::unordered_map<std::string, Id> m_localSymbolNamesToIds; // this is used to prevent duplicates (unique)
 	std::map<Id, std::shared_ptr<StorageLocalSymbol>> m_localSymbolIdsToData;
 
-	std::vector<StorageSourceLocation> m_sourceLocations;
+	std::unordered_map<std::string, Id> m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique)
+	std::map<Id, std::shared_ptr<StorageSourceLocation>> m_sourceLocationIdsToData;
+
+	std::vector<StorageOccurrence> m_occurrences;
 	std::vector<StorageComponentAccess> m_componentAccesses;
 	std::vector<StorageCommentLocation> m_commentLocations;
 	std::vector<StorageError> m_errors;

src/lib/data/PersistentStorage.cpp

@@ -105,18 +105,27 @@ Id PersistentStorage::addLocalSymbol(const std::string& name)
 	return localSymbolId;
 }
 
-void PersistentStorage::addSourceLocation(
-	Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
+Id PersistentStorage::addSourceLocation(
+	Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
 {
-	m_sqliteStorage.addSourceLocation(
-		elementId,
-		fileNodeId,
-		startLine,
-		startCol,
-		endLine,
-		endCol,
-		type
-	);
+	Id sourceLocationId = m_sqliteStorage.getSourceLocationByAll(fileNodeId, startLine, startCol, endLine, endCol, type).id;
+	if (sourceLocationId == 0)
+	{
+		sourceLocationId = m_sqliteStorage.addSourceLocation(
+			fileNodeId,
+			startLine,
+			startCol,
+			endLine,
+			endCol,
+			type
+		);
+	}
+	return sourceLocationId;
+}
+
+void PersistentStorage::addOccurrence(Id elementId, Id sourceLocationId)
+{
+	m_sqliteStorage.addOccurrence(elementId, sourceLocationId);
 }
 
 void PersistentStorage::addComponentAccess(Id nodeId , int type)
@@ -184,11 +193,19 @@ void PersistentStorage::forEachLocalSymbol(std::function<void(
 	}
 }
 
-void PersistentStorage::forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const
+void PersistentStorage::forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const
 {
 	for (StorageSourceLocation& sourceLocation: m_sqliteStorage.getAllSourceLocations())
 	{
-		callback(sourceLocation);
+		callback(sourceLocation.id, sourceLocation);
+	}
+}
+
+void PersistentStorage::forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const
+{
+	for (StorageOccurrence& occurrence: m_sqliteStorage.getAllOccurrences())
+	{
+		callback(occurrence);
 	}
 }
 
@@ -784,9 +801,9 @@ std::vector<Id> PersistentStorage::getNodeIdsForLocationIds(const std::vector<Id
 	std::set<Id> nodeIds;
 	std::set<Id> implicitNodeIds;
 
-	for (Id locationId : locationIds)
+	for (const StorageOccurrence& occurrence: m_sqliteStorage.getOccurrencesForLocationIds(locationIds))
 	{
-		Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId);
+		const Id elementId = occurrence.elementId;
 
 		StorageEdge edge = m_sqliteStorage.getEdgeById(elementId);
 		if (edge.id != 0) // here we test if location is an edge.
@@ -827,9 +844,9 @@ std::vector<Id> PersistentStorage::getLocalSymbolIdsForLocationIds(const std::ve
 {
 	std::set<Id> localSymbolIds;
 
-	for (Id locationId : locationIds)
+	for (const StorageOccurrence& occurrence: m_sqliteStorage.getOccurrencesForLocationIds(locationIds))
 	{
-		Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId);
+		Id elementId = occurrence.elementId;
 
 		if (m_sqliteStorage.getNodeById(elementId).id == 0 && m_sqliteStorage.getEdgeById(elementId).id == 0)
 		{
@@ -898,24 +915,21 @@ std::shared_ptr<TokenLocationCollection> PersistentStorage::getTokenLocationsFor
 		collection->addTokenLocationFile(m_sqliteStorage.getTokenLocationsForFile(storageFile.filePath));
 	}
 
-	std::vector<StorageSourceLocation> locations = m_sqliteStorage.getTokenLocationsForElementIds(nonFileIds);
-	for (size_t i = 0; i < locations.size(); i++)
+	for (const std::pair<StorageSourceLocation, Id>& e: m_sqliteStorage.getSourceLocationsAndElementIdsForElementIds(nonFileIds))
 	{
-		const StorageSourceLocation& location = locations[i];
-
 		TokenLocation* loc = collection->addTokenLocation(
-			location.id,
-			location.elementId,
-			getFileNodePath(location.fileNodeId),
-			location.startLine,
-			location.startCol,
-			location.endLine,
-			location.endCol
+			e.first.id,
+			e.second,
+			getFileNodePath(e.first.fileNodeId),
+			e.first.startLine,
+			e.first.startCol,
+			e.first.endLine,
+			e.first.endCol
 		);
 
 		if (loc)
 		{
-			loc->setType(intToLocationType(location.type));
+			loc->setType(intToLocationType(e.first.type));
 		}
 	}
 
@@ -933,15 +947,18 @@ std::shared_ptr<TokenLocationCollection> PersistentStorage::getTokenLocationsFor
 	for (size_t i = 0; i < locationIds.size(); i++)
 	{
 		StorageSourceLocation location = m_sqliteStorage.getSourceLocationById(locationIds[i]);
-		collection->addTokenLocation(
-			location.id,
-			location.elementId,
-			m_sqliteStorage.getFileById(location.fileNodeId).filePath, // TODO: optimize: only once per file!
-			location.startLine,
-			location.startCol,
-			location.endLine,
-			location.endCol
-		)->setType(intToLocationType(location.type));
+		for (const StorageOccurrence& occurrences: m_sqliteStorage.getOccurrencesForLocationId(locationIds[i]))
+		{
+			collection->addTokenLocation(
+				location.id,
+				occurrences.elementId,
+				m_sqliteStorage.getFileById(location.fileNodeId).filePath, // TODO: optimize: only once per file!
+				location.startLine,
+				location.startCol,
+				location.endLine,
+				location.endCol
+			)->setType(intToLocationType(location.type));
+		}
 	}
 
 	return collection;
@@ -1132,7 +1149,7 @@ std::set<FilePath> PersistentStorage::getDependingFilePathsForImports(const std:
 
 	for (const StorageEdge& importEdge: m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_IMPORT)))
 	{
-		for (const StorageSourceLocation& importedElementLoc: m_sqliteStorage.getTokenLocationsForElementId(importEdge.targetNodeId))
+		for (const StorageSourceLocation& importedElementLoc: m_sqliteStorage.getSourceLocationsForElementId(importEdge.targetNodeId))
 		{
 			fileIdToDependingFileIds[importedElementLoc.fileNodeId].insert(importEdge.sourceNodeId);
 		}

src/lib/data/PersistentStorage.h

@@ -31,7 +31,8 @@ public:
 	virtual Id addNode(int type, const std::string& serializedName, int definitionType);
 	virtual Id addEdge(int type, Id sourceId, Id targetId);
 	virtual Id addLocalSymbol(const std::string& name);
-	virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type);
+	virtual Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type);
+	virtual void addOccurrence(Id elementId, Id sourceLocationId);
 	virtual void addComponentAccess(Id nodeId , int type);
 	virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol);
 	virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed);
@@ -40,7 +41,8 @@ public:
 	virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const;
 	virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const;
 	virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const;
-	virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const;
+	virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const;
+	virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const;
 	virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const;
 	virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const;
 	virtual void forEachError(std::function<void(const StorageError& /*data*/)> callback) const;

src/lib/data/SqliteStorage.cpp

@@ -11,7 +11,7 @@
 #include "utility/utilityString.h"
 #include "utility/Version.h"
 
-const size_t SqliteStorage::STORAGE_VERSION = 5;
+const size_t SqliteStorage::STORAGE_VERSION = 6;
 
 SqliteStorage::SqliteStorage(const FilePath& dbFilePath)
 	: m_dbFilePath(dbFilePath)
@@ -62,6 +62,18 @@ void SqliteStorage::rollbackTransaction()
 	m_database.execDML("ROLLBACK TRANSACTION;");
 }
 
+void SqliteStorage::optimizeMemory() const
+{
+	try
+	{
+		m_database.execDML("VACUUM;");
+	}
+	catch(CppSQLite3Exception e)
+	{
+		LOG_ERROR(e.errorMessage());
+	}
+}
+
 FilePath SqliteStorage::getDbFilePath() const
 {
 	return m_dbFilePath;
@@ -181,11 +193,11 @@ Id SqliteStorage::addLocalSymbol(const std::string& name)
 }
 
 Id SqliteStorage::addSourceLocation(
-	Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
+	Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
 {
 	m_database.execDML((
-		"INSERT INTO source_location(id, element_id, file_node_id, start_line, start_column, end_line, end_column, type) "
-		"VALUES(NULL, " + std::to_string(elementId) + ", " + std::to_string(fileNodeId) + ", "
+		"INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) "
+		"VALUES(NULL, " + std::to_string(fileNodeId) + ", "
 		+ std::to_string(startLine) + ", " + std::to_string(startCol) + ", "
 		+ std::to_string(endLine) + ", " + std::to_string(endCol) + ", " + std::to_string(type) + ");"
 	).c_str());
@@ -193,6 +205,23 @@ Id SqliteStorage::addSourceLocation(
 	return m_database.lastRowId();
 }
 
+bool SqliteStorage::addOccurrence(Id elementId, Id sourceLocationId)
+{
+	try
+	{
+		m_database.execDML((
+			"INSERT INTO occurrence(element_id, source_location_id) "
+			"VALUES(" + std::to_string(elementId) + ", "
+			+ std::to_string(sourceLocationId) + ");"
+		).c_str());
+	}
+	catch (CppSQLite3Exception& e)
+	{
+		return false;
+	}
+	return true;
+}
+
 Id SqliteStorage::addComponentAccess(Id nodeId, int type)
 {
 	m_database.execDML((
@@ -266,32 +295,18 @@ void SqliteStorage::removeElements(const std::vector<Id>& ids)
 	).c_str());
 }
 
-void SqliteStorage::removeElementsWithLocationInFiles(const std::vector<Id>& fileIds)
+void SqliteStorage::removeElementsWithLocationInFiles(const std::vector<Id>& fileIds) // TODO: make one single clearFiles method
 {
-	CppSQLite3Query q = m_database.execQuery((
-		"SELECT id, element_id FROM source_location WHERE file_node_id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");"
+	m_database.execDML((
+		"DELETE FROM source_location WHERE file_node_id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");"
 	).c_str());
 
-	std::vector<Id> sourceLocationIds;
-	std::vector<Id> elementIds;
-	while (!q.eof())
-	{
-		sourceLocationIds.push_back(q.getIntField(0, 0));
-		elementIds.push_back(q.getIntField(1, 0));
-		q.nextRow();
-	}
-
-	m_database.execDML((
-		"DELETE FROM source_location WHERE id IN (" + utility::join(utility::toStrings(sourceLocationIds), ',') + ");"
-	).c_str());
-
-	m_database.execDML((
+	m_database.execDML(
 		"DELETE FROM element WHERE "
-			"element.id IN (" + utility::join(utility::toStrings(elementIds), ',') + ") " // skip all elements that dont have a matching id.
-			"AND element.id NOT IN ("
-				"SELECT source_location.element_id FROM source_location WHERE source_location.element_id == element.id LIMIT 1"
+			"element.id NOT IN ("
+				"SELECT occurrence.element_id FROM occurrence WHERE occurrence.element_id == element.id LIMIT 1"
 			");" // delete all elements that dont have a source location. This query is executed for each element that passed the first test.
-	).c_str());
+	);
 }
 
 void SqliteStorage::removeErrorsInFiles(const std::vector<FilePath>& filePaths)
@@ -378,18 +393,6 @@ std::vector<StorageEdge> SqliteStorage::getEdgesByTargetType(Id targetId, int ty
 	return getAll<StorageEdge>("WHERE target_node_id == " + std::to_string(targetId) + " AND type == " + std::to_string(type));
 }
 
-void SqliteStorage::optimizeMemory() const
-{
-	try
-	{
-		m_database.execDML("VACUUM;");
-	}
-	catch(CppSQLite3Exception e)
-	{
-		LOG_ERROR(e.errorMessage());
-	}
-}
-
 StorageNode SqliteStorage::getNodeById(Id id) const
 {
 	if (id != 0)
@@ -466,7 +469,6 @@ std::shared_ptr<TextAccess> SqliteStorage::getFileContentById(Id fileId) const
 		LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage());
 	}
 
-
 	return TextAccess::createFromString("");
 }
 
@@ -515,6 +517,18 @@ StorageSourceLocation SqliteStorage::getSourceLocationById(const Id id) const
 	);
 }
 
+StorageSourceLocation SqliteStorage::getSourceLocationByAll(const Id fileNodeId, const uint startLine, const uint startCol, const uint endLine, const uint endCol, const int type) const
+{
+	return getFirst<StorageSourceLocation>(
+		"WHERE file_node_id == " + std::to_string(fileNodeId) +
+		" AND start_line == " + std::to_string(startLine) +
+		" AND start_column == " + std::to_string(startCol) +
+		" AND end_line == " + std::to_string(endLine) +
+		" AND end_column == " + std::to_string(endCol) +
+		" AND type == " + std::to_string(type) + ";"
+	);
+}
+
 std::shared_ptr<TokenLocationFile> SqliteStorage::getTokenLocationsForFile(const FilePath& filePath) const
 {
 	std::shared_ptr<TokenLocationFile> ret = std::make_shared<TokenLocationFile>(filePath);
@@ -525,17 +539,17 @@ std::shared_ptr<TokenLocationFile> SqliteStorage::getTokenLocationsForFile(const
 		return ret;
 	}
 
-	for (StorageSourceLocation& location: getAll<StorageSourceLocation>("WHERE file_node_id == " + std::to_string(fileNodeId)))
+	for (std::pair<StorageSourceLocation, Id> e: getAllSourceLocationsAndElementIds("WHERE source_location.file_node_id == " + std::to_string(fileNodeId)))
 	{
 		TokenLocation* loc = ret->addTokenLocation(
-			location.id,
-			location.elementId,
-			location.startLine,
-			location.startCol,
-			location.endLine,
-			location.endCol
+			e.first.id,
+			e.second,
+			e.first.startLine,
+			e.first.startCol,
+			e.first.endLine,
+			e.first.endCol
 		);
-		loc->setType(intToLocationType(location.type));
+		loc->setType(intToLocationType(e.first.type));
 	}
 
 	ret->isWholeCopy = true;
@@ -543,27 +557,78 @@ std::shared_ptr<TokenLocationFile> SqliteStorage::getTokenLocationsForFile(const
 	return ret;
 }
 
-std::vector<StorageSourceLocation> SqliteStorage::getTokenLocationsForElementId(const Id elementId) const
+std::vector<StorageSourceLocation> SqliteStorage::getSourceLocationsForElementId(const Id elementId) const
 {
 	std::vector<Id> elementIds {elementId};
-	return getTokenLocationsForElementIds(elementIds);
+	std::vector<StorageSourceLocation> ret;
+	for (std::pair<StorageSourceLocation, Id> e: getSourceLocationsAndElementIdsForElementIds(elementIds))
+	{
+		ret.push_back(e.first);
+	}
+	return ret;
 }
 
-std::vector<StorageSourceLocation> SqliteStorage::getTokenLocationsForElementIds(const std::vector<Id> elementIds) const
+std::vector<std::pair<StorageSourceLocation, Id>> SqliteStorage::getSourceLocationsAndElementIdsForElementIds(const std::vector<Id> elementIds) const
 {
-	return getAll<StorageSourceLocation>("WHERE element_id IN (" + utility::join(utility::toStrings(elementIds), ',') + ")");
+	return getAllSourceLocationsAndElementIds("WHERE occurrence.element_id IN (" + utility::join(utility::toStrings(elementIds), ',') + ")");
 }
 
-Id SqliteStorage::getElementIdByLocationId(Id locationId) const
+std::vector<std::pair<StorageSourceLocation, Id>> SqliteStorage::getAllSourceLocationsAndElementIds(const std::string& query) const
 {
 	CppSQLite3Query q = m_database.execQuery((
-		"SELECT element_id FROM source_location WHERE id == " + std::to_string(locationId) + " LIMIT 1;"
+		"SELECT "
+			"source_location.id, "
+			"source_location.file_node_id, "
+			"source_location.start_line, "
+			"source_location.start_column, "
+			"source_location.end_line, "
+			"source_location.end_column, "
+			"source_location.type, "
+			"occurrence.element_id "
+		"FROM source_location "
+		"INNER JOIN occurrence ON occurrence.source_location_id = source_location.id " + query + ";"
 	).c_str());
-	if (!q.eof())
+
+	std::vector<std::pair<StorageSourceLocation, Id>> ret;
+	while (!q.eof())
 	{
-		return q.getIntField(0, 0);
+		const Id id				= q.getIntField(0, 0);
+		const Id  fileNodeId	= q.getIntField(1, 0);
+		const int startLine		= q.getIntField(2, -1);
+		const int startColumn	= q.getIntField(3, -1);
+		const int endLine		= q.getIntField(4, -1);
+		const int endColumn		= q.getIntField(5, -1);
+		const int type			= q.getIntField(6, -1);
+		const Id elementId		= q.getIntField(7, 0);
+
+		if (id != 0 && fileNodeId != 0 && startLine != -1 && startColumn != -1 && endLine != -1 && endColumn != -1 && type != -1 && elementId != 0)
+		{
+			ret.push_back(std::make_pair(
+				StorageSourceLocation(
+					id,
+					fileNodeId,
+					startLine,
+					startColumn,
+					endLine,
+					endColumn,
+					type),
+				elementId
+			));
+		}
+		q.nextRow();
 	}
-	return 0;
+	return ret;
+}
+
+std::vector<StorageOccurrence> SqliteStorage::getOccurrencesForLocationId(Id locationId) const
+{
+	std::vector<Id> locationIds {locationId};
+	return getOccurrencesForLocationIds(locationIds);
+}
+
+std::vector<StorageOccurrence> SqliteStorage::getOccurrencesForLocationIds(const std::vector<Id>& locationIds) const
+{
+	return getAll<StorageOccurrence>("WHERE source_location_id IN (" + utility::join(utility::toStrings(locationIds), ',') + ")");
 }
 
 StorageComponentAccess SqliteStorage::getComponentAccessByNodeId(Id nodeId) const
@@ -607,6 +672,11 @@ std::vector<StorageSourceLocation> SqliteStorage::getAllSourceLocations() const
 	return getAll<StorageSourceLocation>("");
 }
 
+std::vector<StorageOccurrence> SqliteStorage::getAllOccurrences() const
+{
+	return getAll<StorageOccurrence>("");
+}
+
 std::vector<StorageComponentAccess> SqliteStorage::getAllComponentAccesses() const
 {
 	return getAll<StorageComponentAccess>("");
@@ -652,6 +722,7 @@ void SqliteStorage::clearTables()
 	m_database.execDML("DROP TABLE IF EXISTS main.error;");
 	m_database.execDML("DROP TABLE IF EXISTS main.comment_location;");
 	m_database.execDML("DROP TABLE IF EXISTS main.component_access;");
+	m_database.execDML("DROP TABLE IF EXISTS main.occurrence;");
 	m_database.execDML("DROP TABLE IF EXISTS main.source_location;");
 	m_database.execDML("DROP TABLE IF EXISTS main.local_symbol;");
 	m_database.execDML("DROP TABLE IF EXISTS main.filecontent;");
@@ -746,7 +817,6 @@ void SqliteStorage::setupTables()
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS source_location("
 				"id INTEGER NOT NULL, "
-				"element_id INTEGER, "
 				"file_node_id INTEGER, "
 				"start_line INTEGER, "
 				"start_column INTEGER, "
@@ -754,12 +824,20 @@ void SqliteStorage::setupTables()
 				"end_column INTEGER, "
 				"type INTEGER, "
 				"PRIMARY KEY(id), "
-				"FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, "
 				"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
 		);
 
-		SqliteIndex("source_location_element_id_index", "source_location(element_id)").createOnDatabase(m_database);
 		SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database);
+		SqliteIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)").createOnDatabase(m_database);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS occurrence(" // TODO: properly delete this on refresh
+				"element_id INTEGER NOT NULL, "
+				"source_location_id INTEGER NOT NULL, "
+				"PRIMARY KEY(element_id, source_location_id), "
+				"FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, "
+				"FOREIGN KEY(source_location_id) REFERENCES source_location(id) ON DELETE CASCADE);"
+		);
 
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS component_access("
@@ -983,25 +1061,24 @@ template <>
 std::vector<StorageSourceLocation> SqliteStorage::getAll<StorageSourceLocation>(const std::string& query) const
 {
 	CppSQLite3Query q = m_database.execQuery((
-		"SELECT id, element_id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location " + query + ";"
-	).c_str());
+		"SELECT id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location " + query + ";"
+		).c_str());
 
 	std::vector<StorageSourceLocation> sourceLocations;
 
 	while (!q.eof())
 	{
 		const Id id					= q.getIntField(0, 0);
-		const Id elementId			= q.getIntField(1, 0);
-		const Id fileNodeId			= q.getIntField(2, 0);
-		const int startLineNumber	= q.getIntField(3, -1);
-		const int startColNumber	= q.getIntField(4, -1);
-		const int endLineNumber		= q.getIntField(5, -1);
-		const int endColNumber		= q.getIntField(6, -1);
-		const int type				= q.getIntField(7, -1);
+		const Id fileNodeId			= q.getIntField(1, 0);
+		const int startLineNumber	= q.getIntField(2, -1);
+		const int startColNumber	= q.getIntField(3, -1);
+		const int endLineNumber		= q.getIntField(4, -1);
+		const int endColNumber		= q.getIntField(5, -1);
+		const int type				= q.getIntField(6, -1);
 
-		if (id != 0 && elementId != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1 && type != -1)
+		if (id != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1 && type != -1)
 		{
-			sourceLocations.push_back(StorageSourceLocation(id, elementId, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type));
+			sourceLocations.push_back(StorageSourceLocation(id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type));
 		}
 
 		q.nextRow();
@@ -1009,6 +1086,30 @@ std::vector<StorageSourceLocation> SqliteStorage::getAll<StorageSourceLocation>(
 	return sourceLocations;
 }
 
+template <>
+std::vector<StorageOccurrence> SqliteStorage::getAll<StorageOccurrence>(const std::string& query) const
+{
+	CppSQLite3Query q = m_database.execQuery((
+		"SELECT element_id, source_location_id FROM occurrence " + query + ";"
+		).c_str());
+
+	std::vector<StorageOccurrence> occurrences;
+
+	while (!q.eof())
+	{
+		const Id elementId			= q.getIntField(0, 0);
+		const Id sourceLocationId	= q.getIntField(1, 0);
+
+		if (elementId != 0 && sourceLocationId != 0)
+		{
+			occurrences.push_back(StorageOccurrence(elementId, sourceLocationId));
+		}
+
+		q.nextRow();
+	}
+	return occurrences;
+}
+
 template <>
 std::vector<StorageComponentAccess> SqliteStorage::getAll<StorageComponentAccess>(const std::string& query) const
 {

src/lib/data/SqliteStorage.h

@@ -31,6 +31,8 @@ public:
 	void commitTransaction();
 	void rollbackTransaction();
 
+	void optimizeMemory() const;
+
 	FilePath getDbFilePath() const;
 
 	bool isEmpty() const;
@@ -44,7 +46,8 @@ public:
 	Id addNode(int type, const std::string& serializedName, int definitionType);
 	Id addFile(const std::string& serializedName, const std::string& filePath, const std::string& modificationTime);
 	Id addLocalSymbol(const std::string& name);
-	Id addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type);
+	Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type);
+	bool addOccurrence(Id elementId, Id sourceLocationId);
 	Id addComponentAccess(Id nodeId, int type);
 	Id addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol);
 	Id addError(const std::string& message, const FilePath& filePath, uint lineNumber, uint columnNumber, bool fatal, bool indexed);
@@ -90,17 +93,18 @@ public:
 	void setNodeDefinitionType(int definitionType, Id nodeId);
 
 	StorageSourceLocation getSourceLocationById(const Id id) const;
+	StorageSourceLocation getSourceLocationByAll(const Id fileNodeId, const uint startLine, const uint startCol, const uint endLine, const uint endCol, const int type) const;
 	std::shared_ptr<TokenLocationFile> getTokenLocationsForFile(const FilePath& filePath) const;
-	std::vector<StorageSourceLocation> getTokenLocationsForElementId(const Id elementId) const;
-	std::vector<StorageSourceLocation> getTokenLocationsForElementIds(const std::vector<Id> elementIds) const;
+	std::vector<StorageSourceLocation> getSourceLocationsForElementId(const Id elementId) const;
+	std::vector<std::pair<StorageSourceLocation, Id>> getSourceLocationsAndElementIdsForElementIds(const std::vector<Id> elementIds) const;
+	std::vector<std::pair<StorageSourceLocation, Id>> getAllSourceLocationsAndElementIds(const std::string& query) const;
 
-	Id getElementIdByLocationId(Id locationId) const;
+	std::vector<StorageOccurrence> getOccurrencesForLocationId(Id locationId) const;
+	std::vector<StorageOccurrence> getOccurrencesForLocationIds(const std::vector<Id>& locationIds) const;
 
 	StorageComponentAccess getComponentAccessByNodeId(Id memberEdgeId) const;
 	std::vector<StorageComponentAccess> getComponentAccessesByNodeIds(const std::vector<Id>& memberEdgeIds) const;
 
-	void optimizeMemory() const;
-
 	std::vector<ParseLocation> getFullTextSearch(const std::string& searchTerm) const;
 
 	std::vector<StorageCommentLocation> getCommentLocationsInFile(const FilePath& filePath) const;
@@ -110,6 +114,7 @@ public:
 	std::vector<StorageEdge> getAllEdges() const;
 	std::vector<StorageLocalSymbol> getAllLocalSymbols() const;
 	std::vector<StorageSourceLocation> getAllSourceLocations() const;
+	std::vector<StorageOccurrence> getAllOccurrences() const;
 	std::vector<StorageComponentAccess> getAllComponentAccesses() const;
 	std::vector<StorageCommentLocation> getAllCommentLocations() const;
 	std::vector<StorageError> getAllErrors() const;
@@ -166,6 +171,8 @@ std::vector<StorageLocalSymbol> SqliteStorage::getAll<StorageLocalSymbol>(const
 template <>
 std::vector<StorageSourceLocation> SqliteStorage::getAll<StorageSourceLocation>(const std::string& query) const;
 template <>
+std::vector<StorageOccurrence> SqliteStorage::getAll<StorageOccurrence>(const std::string& query) const;
+template <>
 std::vector<StorageComponentAccess> SqliteStorage::getAll<StorageComponentAccess>(const std::string& query) const;
 template <>
 std::vector<StorageCommentLocation> SqliteStorage::getAll<StorageCommentLocation>(const std::string& query) const;

src/lib/data/Storage.cpp

@@ -90,7 +90,33 @@ void Storage::inject(Storage* injected)
 	);
 
 	injected->forEachSourceLocation(
-		[&](const StorageSourceLocation& injectedData)
+		[&](const Id injectedId, const StorageSourceLocation& injectedData)
+		{
+			std::unordered_map<Id, Id>::const_iterator it;
+			it = injectedIdToOwnId.find(injectedData.fileNodeId);
+			if (it == injectedIdToOwnId.end())
+			{
+				return;
+			}
+			Id ownFileNodeId = it->second;
+
+			Id ownId = addSourceLocation(
+				ownFileNodeId,
+				injectedData.startLine,
+				injectedData.startCol,
+				injectedData.endLine,
+				injectedData.endCol,
+				injectedData.type
+			);
+			if (ownId != 0)
+			{
+				injectedIdToOwnId[injectedId] = ownId;
+			}
+		}
+	);
+
+	injected->forEachOccurrence(
+		[&](const StorageOccurrence& injectedData)
 		{
 			std::unordered_map<Id, Id>::const_iterator it;
 			it = injectedIdToOwnId.find(injectedData.elementId);
@@ -100,22 +126,14 @@ void Storage::inject(Storage* injected)
 			}
 			Id ownElementId = it->second;
 
-			it = injectedIdToOwnId.find(injectedData.fileNodeId);
+			it = injectedIdToOwnId.find(injectedData.sourceLocationId);
 			if (it == injectedIdToOwnId.end())
 			{
 				return;
 			}
-			Id ownFileNodeId = it->second;
+			Id ownSourceLocationId = it->second;
 
-			addSourceLocation(
-				ownElementId,
-				ownFileNodeId,
-				injectedData.startLine,
-				injectedData.startCol,
-				injectedData.endLine,
-				injectedData.endCol,
-				injectedData.type
-			);
+			addOccurrence(ownElementId, ownSourceLocationId);
 		}
 	);
 

src/lib/data/Storage.h

@@ -19,7 +19,8 @@ public:
 	virtual Id addNode(int type, const std::string& serializedName, int definitionType) = 0;
 	virtual Id addEdge(int type, Id sourceId, Id targetId) = 0;
 	virtual Id addLocalSymbol(const std::string& name) = 0;
-	virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) = 0;
+	virtual Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) = 0;
+	virtual void addOccurrence(Id elementId, Id sourceLocationId) = 0;
 	virtual void addComponentAccess(Id nodeId , int type) = 0;
 	virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) = 0;
 	virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) = 0;
@@ -28,7 +29,8 @@ public:
 	virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const = 0;
 	virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const = 0;
 	virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const = 0;
-	virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const = 0;
+	virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const = 0;
+	virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const = 0;
 	virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const = 0;
 	virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const = 0;
 	virtual void forEachError(std::function<void(const StorageError& /*data*/)> callback) const = 0;

src/lib/data/StorageProvider.cpp

@@ -1,7 +1,5 @@
 #include "data/StorageProvider.h"
 
-#include <iostream>
-
 int StorageProvider::getStorageCount() const
 {
 	std::lock_guard<std::mutex> lock(m_storagesMutex);
@@ -10,13 +8,38 @@ int StorageProvider::getStorageCount() const
 
 void StorageProvider::pushIndexerTarget(std::shared_ptr<IntermediateStorage> storage)
 {
+	const std::size_t storageSize = storage->getSourceLocationCount();
+	std::list<std::shared_ptr<IntermediateStorage>>::iterator it;
+
 	std::lock_guard<std::mutex> lock(m_storagesMutex);
-	m_storages.push_back(storage);
+	for (it = m_storages.begin(); it != m_storages.end(); it++)
+	{
+		if ((*it)->getSourceLocationCount() < storageSize)
+		{
+			break;
+		}
+	}
+	m_storages.insert(it, storage);
 }
 
 std::shared_ptr<IntermediateStorage> StorageProvider::popIndexerTarget()
 {
-	return std::make_shared<IntermediateStorage>();;
+	std::shared_ptr<IntermediateStorage> ret;
+	{
+		std::lock_guard<std::mutex> lock(m_storagesMutex);
+		if (m_storages.size() > 1)
+		{
+			std::list<std::shared_ptr<IntermediateStorage>>::iterator it = m_storages.begin();
+			it++;
+			ret = *it;
+			m_storages.erase(it);
+		}
+		else
+		{
+			ret = std::make_shared<IntermediateStorage>();
+		}
+	}
+	return ret;
 }
 
 std::shared_ptr<IntermediateStorage> StorageProvider::popInjectionSource()
@@ -33,3 +56,6 @@ std::shared_ptr<IntermediateStorage> StorageProvider::popInjectionSource()
 
 	return ret;
 }
+
+
+

src/lib/data/StorageTypes.h

@@ -93,7 +93,6 @@ struct StorageSourceLocation
 {
 	StorageSourceLocation()
 		: id(0)
-		, elementId(0)
 		, fileNodeId(0)
 		, startLine(-1)
 		, startCol(-1)
@@ -102,9 +101,8 @@ struct StorageSourceLocation
 		, type(0)
 	{}
 
-	StorageSourceLocation(Id id, Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
+	StorageSourceLocation(Id id, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
 		: id(id)
-		, elementId(elementId)
 		, fileNodeId(fileNodeId)
 		, startLine(startLine)
 		, startCol(startCol)
@@ -114,7 +112,6 @@ struct StorageSourceLocation
 	{}
 
 	Id id;
-	Id elementId;
 	Id fileNodeId;
 	uint startLine;
 	uint startCol;
@@ -123,6 +120,22 @@ struct StorageSourceLocation
 	int type;
 };
 
+struct StorageOccurrence
+{
+	StorageOccurrence()
+		: elementId(0)
+		, sourceLocationId(0)
+	{}
+
+	StorageOccurrence(Id elementId, Id sourceLocationId)
+		: elementId(elementId)
+		, sourceLocationId(sourceLocationId)
+	{}
+
+	Id elementId;
+	Id sourceLocationId;
+};
+
 struct StorageComponentAccess
 {
 	StorageComponentAccess()

src/lib/data/parser/ParserClientImpl.cpp

@@ -293,8 +293,7 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca
 		return;
 	}
 
-	m_storage->addSourceLocation(
-		elementId,
+	Id sourceLocationId = m_storage->addSourceLocation(
 		addFile(location.filePath.str()),
 		location.startLineNumber,
 		location.startColumnNumber,
@@ -302,6 +301,11 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca
 		location.endColumnNumber,
 		type
 	);
+
+	m_storage->addOccurrence(
+		elementId,
+		sourceLocationId
+	);
 }
 
 void ParserClientImpl::addComponentAccess(Id nodeId , int type)