ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: reducing database size by discarding indices

Browse Source 
* added storage modes to facilitate adding and removing sqlite indices as needed.
This commit is contained in:
malte_langkabel
2016-11-04 14:29:20 +01:00
parent e7c9a1340e
commit 3f27c89965
9 changed files with 77 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/Project.cpp
+2
View File
@@ -221,6 +221,7 @@ void Project::load()
if (canLoad)
{
m_storage->setMode(SqliteStorage::STORAGE_MODE_READ);
m_storage->buildCaches();
m_storageAccessProxy->setSubject(m_storage.get());
@@ -324,6 +325,7 @@ bool Project::buildIndex(bool forceRefresh)
fileRegister->setFilePaths(utility::toVector(filesToParse));
std::shared_ptr<TaskParseWrapper> taskParserWrapper = std::make_shared<TaskParseWrapper>(
m_storage.get(),
fileRegister,
m_dialogView
);
src/lib/data/PersistentStorage.cpp
+5
View File
@@ -252,6 +252,11 @@ void PersistentStorage::finishInjection()
}
}
void PersistentStorage::setMode(const SqliteStorage::StorageModeType mode)
{
m_sqliteStorage.setMode(mode);
}
FilePath PersistentStorage::getDbFilePath() const
{
return m_sqliteStorage.getDbFilePath();
src/lib/data/PersistentStorage.h
+2
View File
@@ -50,6 +50,8 @@ public:
virtual void startInjection();
virtual void finishInjection();
void setMode(const SqliteStorage::StorageModeType mode);
FilePath getDbFilePath() const;
bool isEmpty() const;
src/lib/data/SqliteStorage.cpp
+33 -20
View File
@@ -4,7 +4,6 @@
#include "data/location/TokenLocation.h"
#include "data/DefinitionType.h"
#include "data/parser/ParseLocation.h"
#include "data/SqliteIndex.h"
#include "utility/logging/logging.h"
#include "utility/text/TextAccess.h"
#include "utility/utility.h"
@@ -19,6 +18,15 @@ SqliteStorage::SqliteStorage(const FilePath& dbFilePath)
m_database.open(m_dbFilePath.str().c_str());
m_database.execDML("PRAGMA foreign_keys=ON;");
m_mode = STORAGE_MODE_UNKNOWN;
m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE, SqliteIndex("edge_multipart_index", "edge(type, source_node_id, target_node_id)")));
m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR, SqliteIndex("node_serialized_name_index", "node(serialized_name)")));
m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE, SqliteIndex("local_symbol_name_index", "local_symbol(name)")));
m_indices.push_back(std::make_pair(STORAGE_MODE_READ | STORAGE_MODE_CLEAR, SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)")));
m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE, SqliteIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)")));
m_indices.push_back(std::make_pair(STORAGE_MODE_WRITE | STORAGE_MODE_READ | STORAGE_MODE_CLEAR, SqliteIndex("component_access_node_id_index", "component_access(node_id)")));
}
SqliteStorage::~SqliteStorage()
@@ -37,6 +45,7 @@ void SqliteStorage::setup()
{
m_database.execDML("PRAGMA foreign_keys=ON;");
setupTables();
m_mode = STORAGE_MODE_UNKNOWN;
}
void SqliteStorage::clear()
@@ -47,6 +56,28 @@ void SqliteStorage::clear()
setup();
}
void SqliteStorage::setMode(const StorageModeType mode)
{
if (mode == m_mode)
{
return;
}
for (size_t i = 0; i < m_indices.size(); i++)
{
if (m_indices[i].first & mode)
{
m_indices[i].second.createOnDatabase(m_database);
}
else
{
m_indices[i].second.removeFromDatabase(m_database);
}
}
m_mode = mode;
}
void SqliteStorage::beginTransaction()
{
m_database.execDML("BEGIN TRANSACTION;");
@@ -295,7 +326,7 @@ void SqliteStorage::removeElements(const std::vector<Id>& ids)
).c_str());
}
void SqliteStorage::removeElementsWithLocationInFiles(const std::vector<Id>& fileIds) // TODO: make one single clearFiles method
void SqliteStorage::removeElementsWithLocationInFiles(const std::vector<Id>& fileIds)
{
m_database.execDML((
"DELETE FROM source_location WHERE file_node_id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");"
@@ -763,11 +794,6 @@ void SqliteStorage::setupTables()
"FOREIGN KEY(target_node_id) REFERENCES node(id) ON DELETE CASCADE);"
);
// TODO: move to createIndexesForAnalysis() or prepareForAnalysis
m_database.execDML( // used for checking for duplicates during code analysis
"CREATE INDEX IF NOT EXISTS edge_multipart_index ON edge(type, source_node_id, target_node_id);"
);
m_database.execDML(
"CREATE TABLE IF NOT EXISTS node("
"id INTEGER NOT NULL, "
@@ -778,10 +804,6 @@ void SqliteStorage::setupTables()
"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
);
m_database.execDML(
"CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);"
);
m_database.execDML(
"CREATE TABLE IF NOT EXISTS file("
"id INTEGER NOT NULL, "
@@ -810,10 +832,6 @@ void SqliteStorage::setupTables()
"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
);
m_database.execDML(
"CREATE INDEX IF NOT EXISTS local_symbol_name_index ON local_symbol(name);"
);
m_database.execDML(
"CREATE TABLE IF NOT EXISTS source_location("
"id INTEGER NOT NULL, "
@@ -827,9 +845,6 @@ void SqliteStorage::setupTables()
"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
);
SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database);
SqliteIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)").createOnDatabase(m_database);
m_database.execDML(
"CREATE TABLE IF NOT EXISTS occurrence(" // TODO: properly delete this on refresh
"element_id INTEGER NOT NULL, "
@@ -848,8 +863,6 @@ void SqliteStorage::setupTables()
"FOREIGN KEY(node_id) REFERENCES node(id) ON DELETE CASCADE);"
);
SqliteIndex("component_access_node_id_index", "component_access(node_id)").createOnDatabase(m_database);
m_database.execDML(
"CREATE TABLE IF NOT EXISTS comment_location("
"id INTEGER NOT NULL, "
src/lib/data/SqliteStorage.h
+14 -2
View File
@@ -13,6 +13,7 @@
#include "data/location/TokenLocationCollection.h"
#include "data/name/NameHierarchy.h"
#include "data/StorageTypes.h"
#include "data/SqliteIndex.h"
class TextAccess;
class Version;
@@ -21,12 +22,22 @@ struct ParseLocation;
class SqliteStorage
{
public:
enum StorageModeType
{
STORAGE_MODE_UNKNOWN = 0,
STORAGE_MODE_READ = 1,
STORAGE_MODE_WRITE = 2,
STORAGE_MODE_CLEAR = 4,
};
SqliteStorage(const FilePath& dbFilePath);
~SqliteStorage();
void setup();
void clear();
void setMode(const StorageModeType mode);
void beginTransaction();
void commitTransaction();
void rollbackTransaction();
@@ -105,8 +116,6 @@ public:
StorageComponentAccess getComponentAccessByNodeId(Id memberEdgeId) const;
std::vector<StorageComponentAccess> getComponentAccessesByNodeIds(const std::vector<Id>& memberEdgeIds) const;
std::vector<ParseLocation> getFullTextSearch(const std::string& searchTerm) const;
std::vector<StorageCommentLocation> getCommentLocationsInFile(const FilePath& filePath) const;
std::vector<StorageFile> getAllFiles() const;
@@ -158,6 +167,9 @@ private:
mutable CppSQLite3DB m_database;
FilePath m_dbFilePath;
StorageModeType m_mode;
std::vector<std::pair<int, SqliteIndex>> m_indices;
};
template <>
src/lib/data/TaskCleanStorage.cpp
+5
View File
@@ -19,6 +19,11 @@ void TaskCleanStorage::doEnter(std::shared_ptr<Blackboard> blackboard)
m_dialogView->showProgressDialog("Clearing Files", std::to_string(m_filePaths.size()) + " Files");
m_start = utility::durationStart();
if (!m_filePaths.empty())
{
m_storage->setMode(SqliteStorage::STORAGE_MODE_CLEAR);
}
}
Task::TaskState TaskCleanStorage::doUpdate(std::shared_ptr<Blackboard> blackboard)
src/lib/data/TaskFinishParsing.cpp
+1
View File
@@ -26,6 +26,7 @@ TaskFinishParsing::~TaskFinishParsing()
void TaskFinishParsing::doEnter(std::shared_ptr<Blackboard> blackboard)
{
m_storage->setMode(SqliteStorage::STORAGE_MODE_READ);
}
Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboard)
src/lib/data/parser/TaskParseWrapper.cpp
+12 -2
View File
@@ -1,15 +1,18 @@
#include "data/parser/TaskParseWrapper.h"
#include "component/view/DialogView.h"
#include "data/PersistentStorage.h"
#include "utility/file/FileRegister.h"
#include "utility/scheduling/Blackboard.h"
#include "utility/utility.h"
TaskParseWrapper::TaskParseWrapper(
PersistentStorage* storage,
std::shared_ptr<FileRegister> fileRegister,
DialogView* dialogView
)
: m_fileRegister(fileRegister)
: m_storage(storage)
, m_fileRegister(fileRegister)
, m_dialogView(dialogView)
{
}
@@ -29,9 +32,16 @@ void TaskParseWrapper::setTask(std::shared_ptr<Task> task)
void TaskParseWrapper::doEnter(std::shared_ptr<Blackboard> blackboard)
{
blackboard->set("indexer_count", 0);
m_dialogView->updateIndexingDialog(0, m_fileRegister->getSourceFilesCount(), "");
const size_t sourceFileCount = m_fileRegister->getSourceFilesCount();
m_dialogView->updateIndexingDialog(0, sourceFileCount, "");
m_start = utility::durationStart();
if (sourceFileCount > 0)
{
m_storage->setMode(SqliteStorage::STORAGE_MODE_WRITE);
}
}
Task::TaskState TaskParseWrapper::doUpdate(std::shared_ptr<Blackboard> blackboard)
src/lib/data/parser/TaskParseWrapper.h
+3
View File
@@ -12,12 +12,14 @@
class DialogView;
class FileRegister;
class PersistentStorage;
class TaskParseWrapper
: public TaskDecorator
{
public:
TaskParseWrapper(
PersistentStorage* storage,
std::shared_ptr<FileRegister> fileRegister,
DialogView* dialogView
);
@@ -31,6 +33,7 @@ private:
virtual void doExit(std::shared_ptr<Blackboard> blackboard);
virtual void doReset(std::shared_ptr<Blackboard> blackboard);
PersistentStorage* m_storage;
std::shared_ptr<FileRegister> m_fileRegister;
DialogView* m_dialogView;