data: reducing database size by discarding indices

* added storage modes to facilitate adding and removing sqlite indices as needed.

src/lib/Project.cpp

@@ -221,6 +221,7 @@ void Project::load()
 
 	if (canLoad)
 	{
+		m_storage->setMode(SqliteStorage::STORAGE_MODE_READ);
 		m_storage->buildCaches();
 		m_storageAccessProxy->setSubject(m_storage.get());
 
@@ -324,6 +325,7 @@ bool Project::buildIndex(bool forceRefresh)
 		fileRegister->setFilePaths(utility::toVector(filesToParse));
 
 		std::shared_ptr<TaskParseWrapper> taskParserWrapper = std::make_shared<TaskParseWrapper>(
+			m_storage.get(),
 			fileRegister,
 			m_dialogView
 		);

src/lib/data/PersistentStorage.cpp

@@ -252,6 +252,11 @@ void PersistentStorage::finishInjection()
 	}
 }
 
+void PersistentStorage::setMode(const SqliteStorage::StorageModeType mode)
+{
+	m_sqliteStorage.setMode(mode);
+}
+
 FilePath PersistentStorage::getDbFilePath() const
 {
 	return m_sqliteStorage.getDbFilePath();

src/lib/data/PersistentStorage.h

@@ -50,6 +50,8 @@ public:
 	virtual void startInjection();
 	virtual void finishInjection();
 
+	void setMode(const SqliteStorage::StorageModeType mode);
+
 	FilePath getDbFilePath() const;
 
 	bool isEmpty() const;

src/lib/data/SqliteStorage.cpp

@@ -4,7 +4,6 @@
 #include "data/location/TokenLocation.h"
 #include "data/DefinitionType.h"
 #include "data/parser/ParseLocation.h"
-#include "data/SqliteIndex.h"
 #include "utility/logging/logging.h"
 #include "utility/text/TextAccess.h"
 #include "utility/utility.h"
@@ -19,6 +18,15 @@ SqliteStorage::SqliteStorage(const FilePath& dbFilePath)
 	m_database.open(m_dbFilePath.str().c_str());
 
 	m_database.execDML("PRAGMA foreign_keys=ON;");
+
+	m_mode = STORAGE_MODE_UNKNOWN;
+
+	m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE, SqliteIndex("edge_multipart_index", "edge(type, source_node_id, target_node_id)")));
+	m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR, SqliteIndex("node_serialized_name_index", "node(serialized_name)")));
+	m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE, SqliteIndex("local_symbol_name_index", "local_symbol(name)")));
+	m_indices.push_back(std::make_pair(STORAGE_MODE_READ | STORAGE_MODE_CLEAR, SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)")));
+	m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE, SqliteIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)")));
+	m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR, SqliteIndex("component_access_node_id_index", "component_access(node_id)")));
 }
 
 SqliteStorage::~SqliteStorage()
@@ -37,6 +45,7 @@ void SqliteStorage::setup()
 {
 	m_database.execDML("PRAGMA foreign_keys=ON;");
 	setupTables();
+	m_mode = STORAGE_MODE_UNKNOWN;
 }
 
 void SqliteStorage::clear()
@@ -47,6 +56,28 @@ void SqliteStorage::clear()
 	setup();
 }
 
+void SqliteStorage::setMode(const StorageModeType mode)
+{
+	if (mode == m_mode)
+	{
+		return;
+	}
+
+	for (size_t i = 0; i < m_indices.size(); i++)
+	{
+		if (m_indices[i].first & mode)
+		{
+			m_indices[i].second.createOnDatabase(m_database);
+		}
+		else
+		{
+			m_indices[i].second.removeFromDatabase(m_database);
+		}
+	}
+
+	m_mode = mode;
+}
+
 void SqliteStorage::beginTransaction()
 {
 	m_database.execDML("BEGIN TRANSACTION;");
@@ -295,7 +326,7 @@ void SqliteStorage::removeElements(const std::vector<Id>& ids)
 	).c_str());
 }
 
-void SqliteStorage::removeElementsWithLocationInFiles(const std::vector<Id>& fileIds) // TODO: make one single clearFiles method
+void SqliteStorage::removeElementsWithLocationInFiles(const std::vector<Id>& fileIds)
 {
 	m_database.execDML((
 		"DELETE FROM source_location WHERE file_node_id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");"
@@ -763,11 +794,6 @@ void SqliteStorage::setupTables()
 				"FOREIGN KEY(target_node_id) REFERENCES node(id) ON DELETE CASCADE);"
 		);
 
-		// TODO: move to createIndexesForAnalysis() or prepareForAnalysis
-		m_database.execDML( // used for checking for duplicates during code analysis
-			"CREATE INDEX IF NOT EXISTS edge_multipart_index ON edge(type, source_node_id, target_node_id);"
-		);
-
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS node("
 				"id INTEGER NOT NULL, "
@@ -778,10 +804,6 @@ void SqliteStorage::setupTables()
 				"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
 		);
 
-		m_database.execDML(
-			"CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);"
-		);
-
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS file("
 				"id INTEGER NOT NULL, "
@@ -810,10 +832,6 @@ void SqliteStorage::setupTables()
 				"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
 		);
 
-		m_database.execDML(
-			"CREATE INDEX IF NOT EXISTS local_symbol_name_index ON local_symbol(name);"
-		);
-
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS source_location("
 				"id INTEGER NOT NULL, "
@@ -827,9 +845,6 @@ void SqliteStorage::setupTables()
 				"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
 		);
 
-		SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database);
-		SqliteIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)").createOnDatabase(m_database);
-
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS occurrence(" // TODO: properly delete this on refresh
 				"element_id INTEGER NOT NULL, "
@@ -848,8 +863,6 @@ void SqliteStorage::setupTables()
 				"FOREIGN KEY(node_id) REFERENCES node(id) ON DELETE CASCADE);"
 		);
 
-		SqliteIndex("component_access_node_id_index", "component_access(node_id)").createOnDatabase(m_database);
-
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS comment_location("
 				"id INTEGER NOT NULL, "

src/lib/data/SqliteStorage.h

@@ -13,6 +13,7 @@
 #include "data/location/TokenLocationCollection.h"
 #include "data/name/NameHierarchy.h"
 #include "data/StorageTypes.h"
+#include "data/SqliteIndex.h"
 
 class TextAccess;
 class Version;
@@ -21,12 +22,22 @@ struct ParseLocation;
 class SqliteStorage
 {
 public:
+	enum StorageModeType
+	{
+		STORAGE_MODE_UNKNOWN = 0,
+		STORAGE_MODE_READ = 1,
+		STORAGE_MODE_WRITE = 2,
+		STORAGE_MODE_CLEAR = 4,
+	};
+
 	SqliteStorage(const FilePath& dbFilePath);
 	~SqliteStorage();
 
 	void setup();
 	void clear();
 
+	void setMode(const StorageModeType mode);
+
 	void beginTransaction();
 	void commitTransaction();
 	void rollbackTransaction();
@@ -105,8 +116,6 @@ public:
 	StorageComponentAccess getComponentAccessByNodeId(Id memberEdgeId) const;
 	std::vector<StorageComponentAccess> getComponentAccessesByNodeIds(const std::vector<Id>& memberEdgeIds) const;
 
-	std::vector<ParseLocation> getFullTextSearch(const std::string& searchTerm) const;
-
 	std::vector<StorageCommentLocation> getCommentLocationsInFile(const FilePath& filePath) const;
 
 	std::vector<StorageFile> getAllFiles() const;
@@ -158,6 +167,9 @@ private:
 
 	mutable CppSQLite3DB m_database;
 	FilePath m_dbFilePath;
+
+	StorageModeType m_mode;
+	std::vector<std::pair<int, SqliteIndex>> m_indices;
 };
 
 template <>

src/lib/data/TaskCleanStorage.cpp

@@ -19,6 +19,11 @@ void TaskCleanStorage::doEnter(std::shared_ptr<Blackboard> blackboard)
 	m_dialogView->showProgressDialog("Clearing Files", std::to_string(m_filePaths.size()) + " Files");
 
 	m_start = utility::durationStart();
+
+	if (!m_filePaths.empty())
+	{
+		m_storage->setMode(SqliteStorage::STORAGE_MODE_CLEAR);
+	}
 }
 
 Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr<Blackboard> blackboard)

src/lib/data/TaskFinishParsing.cpp

@@ -26,6 +26,7 @@ TaskFinishParsing::~TaskFinishParsing()
 
 void TaskFinishParsing::doEnter(std::shared_ptr<Blackboard> blackboard)
 {
+	m_storage->setMode(SqliteStorage::STORAGE_MODE_READ);
 }
 
 Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboard)

src/lib/data/parser/TaskParseWrapper.cpp

@@ -1,15 +1,18 @@
 #include "data/parser/TaskParseWrapper.h"
 
 #include "component/view/DialogView.h"
+#include "data/PersistentStorage.h"
 #include "utility/file/FileRegister.h"
 #include "utility/scheduling/Blackboard.h"
 #include "utility/utility.h"
 
 TaskParseWrapper::TaskParseWrapper(
+	PersistentStorage* storage,
 	std::shared_ptr<FileRegister> fileRegister,
 	DialogView* dialogView
 )
-	: m_fileRegister(fileRegister)
+	: m_storage(storage)
+	, m_fileRegister(fileRegister)
 	, m_dialogView(dialogView)
 {
 }
@@ -29,9 +32,16 @@ void TaskParseWrapper::setTask(std::shared_ptr<Task> task)
 void TaskParseWrapper::doEnter(std::shared_ptr<Blackboard> blackboard)
 {
 	blackboard->set("indexer_count", 0);
-	m_dialogView->updateIndexingDialog(0, m_fileRegister->getSourceFilesCount(), "");
+
+	const size_t sourceFileCount = m_fileRegister->getSourceFilesCount();
+	m_dialogView->updateIndexingDialog(0, sourceFileCount, "");
 
 	m_start = utility::durationStart();
+
+	if (sourceFileCount > 0)
+	{
+		m_storage->setMode(SqliteStorage::STORAGE_MODE_WRITE);
+	}
 }
 
 Task::TaskState TaskParseWrapper::doUpdate(std::shared_ptr<Blackboard> blackboard)

src/lib/data/parser/TaskParseWrapper.h

@@ -12,12 +12,14 @@
 
 class DialogView;
 class FileRegister;
+class PersistentStorage;
 
 class TaskParseWrapper
 	: public TaskDecorator
 {
 public:
 	TaskParseWrapper(
+		PersistentStorage* storage,
 		std::shared_ptr<FileRegister> fileRegister,
 		DialogView* dialogView
 	);
@@ -31,6 +33,7 @@ private:
 	virtual void doExit(std::shared_ptr<Blackboard> blackboard);
 	virtual void doReset(std::shared_ptr<Blackboard> blackboard);
 
+	PersistentStorage* m_storage;
 	std::shared_ptr<FileRegister> m_fileRegister;
 	DialogView* m_dialogView;