From e7c9a1340e2bd57064c1de946031877bbc1dd775 Mon Sep 17 00:00:00 2001 From: malte_langkabel Date: Fri, 4 Nov 2016 08:43:10 +0100 Subject: [PATCH] data: reduced size of generated .coatidb files * added occurrence table to get rid of duplicate source locations * reverted storage provider --- src/lib/data/IntermediateStorage.cpp | 63 ++++-- src/lib/data/IntermediateStorage.h | 20 +- src/lib/data/PersistentStorage.cpp | 95 +++++---- src/lib/data/PersistentStorage.h | 6 +- src/lib/data/SqliteStorage.cpp | 239 ++++++++++++++++------- src/lib/data/SqliteStorage.h | 19 +- src/lib/data/Storage.cpp | 42 ++-- src/lib/data/Storage.h | 6 +- src/lib/data/StorageProvider.cpp | 34 +++- src/lib/data/StorageTypes.h | 21 +- src/lib/data/parser/ParserClientImpl.cpp | 8 +- 11 files changed, 394 insertions(+), 159 deletions(-) diff --git a/src/lib/data/IntermediateStorage.cpp b/src/lib/data/IntermediateStorage.cpp index 2b6ebaa5..6ec50fd0 100644 --- a/src/lib/data/IntermediateStorage.cpp +++ b/src/lib/data/IntermediateStorage.cpp @@ -22,7 +22,9 @@ void IntermediateStorage::clear() m_edgeIdsToData.clear(); m_localSymbolNamesToIds.clear(); m_localSymbolIdsToData.clear(); - m_sourceLocations.clear(); + m_sourceLocationNamesToIds.clear(); + m_sourceLocationIdsToData.clear(); + m_occurrences.clear(); m_componentAccesses.clear(); m_commentLocations.clear(); m_errors.clear(); @@ -31,7 +33,7 @@ void IntermediateStorage::clear() size_t IntermediateStorage::getSourceLocationCount() const { - return m_sourceLocations.size(); + return m_sourceLocationNamesToIds.size(); } Id IntermediateStorage::addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime) @@ -129,18 +131,35 @@ Id IntermediateStorage::addLocalSymbol(const std::string& name) return id; } -void IntermediateStorage::addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) +Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) { - m_sourceLocations.push_back(StorageSourceLocation( + std::shared_ptr sourceLocation = std::make_shared( 0, - elementId, fileNodeId, startLine, startCol, endLine, endCol, type - )); + ); + + std::string serialized = serialize(*(sourceLocation.get())); + std::unordered_map::const_iterator it = m_sourceLocationNamesToIds.find(serialized); + if (it != m_sourceLocationNamesToIds.end()) + { + return it->second; + } + + Id id = m_nextId++; + m_sourceLocationNamesToIds[serialized] = id; + m_sourceLocationIdsToData[id] = sourceLocation; + + return id; +} + +void IntermediateStorage::addOccurrence(Id elementId, Id sourceLocationId) +{ + m_occurrences.push_back(StorageOccurrence(elementId, sourceLocationId)); } void IntermediateStorage::addComponentAccess(Id nodeId, int type) @@ -205,9 +224,17 @@ void IntermediateStorage::forEachLocalSymbol(std::function callback) const +void IntermediateStorage::forEachSourceLocation(std::function callback) const { - for (std::vector::const_iterator it = m_sourceLocations.begin(); it != m_sourceLocations.end(); it++) + for (std::map>::const_iterator it = m_sourceLocationIdsToData.begin(); it != m_sourceLocationIdsToData.end(); it++) + { + callback(it->first, *(it->second.get())); + } +} + +void IntermediateStorage::forEachOccurrence(std::function callback) const +{ + for (std::vector::const_iterator it = m_occurrences.begin(); it != m_occurrences.end(); it++) { callback(*it); } @@ -237,7 +264,7 @@ void IntermediateStorage::forEachError(std::function callback) const; virtual void forEachEdge(std::function callback) const; virtual void forEachLocalSymbol(std::function callback) const; - virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachOccurrence(std::function callback) const; virtual void forEachComponentAccess(std::function callback) const; virtual void forEachCommentLocation(std::function callback) const; virtual void forEachError(std::function callback) const; private: - std::string serialize(const StorageEdge& edge); - std::string serialize(const StorageNode& node); - std::string serialize(const StorageFile& file); - std::string serialize(const StorageLocalSymbol& localSymbol); + std::string serialize(const StorageEdge& edge) const; + std::string serialize(const StorageNode& node) const; + std::string serialize(const StorageFile& file) const; + std::string serialize(const StorageLocalSymbol& localSymbol) const; + std::string serialize(const StorageSourceLocation& sourceLocation) const; std::unordered_map m_fileNamesToIds; // this is used to prevent duplicates (unique) std::unordered_map> m_fileIdsToData; @@ -54,7 +57,10 @@ private: std::unordered_map m_localSymbolNamesToIds; // this is used to prevent duplicates (unique) std::map> m_localSymbolIdsToData; - std::vector m_sourceLocations; + std::unordered_map m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique) + std::map> m_sourceLocationIdsToData; + + std::vector m_occurrences; std::vector m_componentAccesses; std::vector m_commentLocations; std::vector m_errors; diff --git a/src/lib/data/PersistentStorage.cpp b/src/lib/data/PersistentStorage.cpp index 4331a5dc..7f719022 100644 --- a/src/lib/data/PersistentStorage.cpp +++ b/src/lib/data/PersistentStorage.cpp @@ -105,18 +105,27 @@ Id PersistentStorage::addLocalSymbol(const std::string& name) return localSymbolId; } -void PersistentStorage::addSourceLocation( - Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) +Id PersistentStorage::addSourceLocation( + Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) { - m_sqliteStorage.addSourceLocation( - elementId, - fileNodeId, - startLine, - startCol, - endLine, - endCol, - type - ); + Id sourceLocationId = m_sqliteStorage.getSourceLocationByAll(fileNodeId, startLine, startCol, endLine, endCol, type).id; + if (sourceLocationId == 0) + { + sourceLocationId = m_sqliteStorage.addSourceLocation( + fileNodeId, + startLine, + startCol, + endLine, + endCol, + type + ); + } + return sourceLocationId; +} + +void PersistentStorage::addOccurrence(Id elementId, Id sourceLocationId) +{ + m_sqliteStorage.addOccurrence(elementId, sourceLocationId); } void PersistentStorage::addComponentAccess(Id nodeId , int type) @@ -184,11 +193,19 @@ void PersistentStorage::forEachLocalSymbol(std::function callback) const +void PersistentStorage::forEachSourceLocation(std::function callback) const { for (StorageSourceLocation& sourceLocation: m_sqliteStorage.getAllSourceLocations()) { - callback(sourceLocation); + callback(sourceLocation.id, sourceLocation); + } +} + +void PersistentStorage::forEachOccurrence(std::function callback) const +{ + for (StorageOccurrence& occurrence: m_sqliteStorage.getAllOccurrences()) + { + callback(occurrence); } } @@ -784,9 +801,9 @@ std::vector PersistentStorage::getNodeIdsForLocationIds(const std::vector nodeIds; std::set implicitNodeIds; - for (Id locationId : locationIds) + for (const StorageOccurrence& occurrence: m_sqliteStorage.getOccurrencesForLocationIds(locationIds)) { - Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId); + const Id elementId = occurrence.elementId; StorageEdge edge = m_sqliteStorage.getEdgeById(elementId); if (edge.id != 0) // here we test if location is an edge. @@ -827,9 +844,9 @@ std::vector PersistentStorage::getLocalSymbolIdsForLocationIds(const std::ve { std::set localSymbolIds; - for (Id locationId : locationIds) + for (const StorageOccurrence& occurrence: m_sqliteStorage.getOccurrencesForLocationIds(locationIds)) { - Id elementId = m_sqliteStorage.getElementIdByLocationId(locationId); + Id elementId = occurrence.elementId; if (m_sqliteStorage.getNodeById(elementId).id == 0 && m_sqliteStorage.getEdgeById(elementId).id == 0) { @@ -898,24 +915,21 @@ std::shared_ptr PersistentStorage::getTokenLocationsFor collection->addTokenLocationFile(m_sqliteStorage.getTokenLocationsForFile(storageFile.filePath)); } - std::vector locations = m_sqliteStorage.getTokenLocationsForElementIds(nonFileIds); - for (size_t i = 0; i < locations.size(); i++) + for (const std::pair& e: m_sqliteStorage.getSourceLocationsAndElementIdsForElementIds(nonFileIds)) { - const StorageSourceLocation& location = locations[i]; - TokenLocation* loc = collection->addTokenLocation( - location.id, - location.elementId, - getFileNodePath(location.fileNodeId), - location.startLine, - location.startCol, - location.endLine, - location.endCol + e.first.id, + e.second, + getFileNodePath(e.first.fileNodeId), + e.first.startLine, + e.first.startCol, + e.first.endLine, + e.first.endCol ); if (loc) { - loc->setType(intToLocationType(location.type)); + loc->setType(intToLocationType(e.first.type)); } } @@ -933,15 +947,18 @@ std::shared_ptr PersistentStorage::getTokenLocationsFor for (size_t i = 0; i < locationIds.size(); i++) { StorageSourceLocation location = m_sqliteStorage.getSourceLocationById(locationIds[i]); - collection->addTokenLocation( - location.id, - location.elementId, - m_sqliteStorage.getFileById(location.fileNodeId).filePath, // TODO: optimize: only once per file! - location.startLine, - location.startCol, - location.endLine, - location.endCol - )->setType(intToLocationType(location.type)); + for (const StorageOccurrence& occurrences: m_sqliteStorage.getOccurrencesForLocationId(locationIds[i])) + { + collection->addTokenLocation( + location.id, + occurrences.elementId, + m_sqliteStorage.getFileById(location.fileNodeId).filePath, // TODO: optimize: only once per file! + location.startLine, + location.startCol, + location.endLine, + location.endCol + )->setType(intToLocationType(location.type)); + } } return collection; @@ -1132,7 +1149,7 @@ std::set PersistentStorage::getDependingFilePathsForImports(const std: for (const StorageEdge& importEdge: m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_IMPORT))) { - for (const StorageSourceLocation& importedElementLoc: m_sqliteStorage.getTokenLocationsForElementId(importEdge.targetNodeId)) + for (const StorageSourceLocation& importedElementLoc: m_sqliteStorage.getSourceLocationsForElementId(importEdge.targetNodeId)) { fileIdToDependingFileIds[importedElementLoc.fileNodeId].insert(importEdge.sourceNodeId); } diff --git a/src/lib/data/PersistentStorage.h b/src/lib/data/PersistentStorage.h index 74aa0e8e..a3657c43 100644 --- a/src/lib/data/PersistentStorage.h +++ b/src/lib/data/PersistentStorage.h @@ -31,7 +31,8 @@ public: virtual Id addNode(int type, const std::string& serializedName, int definitionType); virtual Id addEdge(int type, Id sourceId, Id targetId); virtual Id addLocalSymbol(const std::string& name); - virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); + virtual Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); + virtual void addOccurrence(Id elementId, Id sourceLocationId); virtual void addComponentAccess(Id nodeId , int type); virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol); virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed); @@ -40,7 +41,8 @@ public: virtual void forEachNode(std::function callback) const; virtual void forEachEdge(std::function callback) const; virtual void forEachLocalSymbol(std::function callback) const; - virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachOccurrence(std::function callback) const; virtual void forEachComponentAccess(std::function callback) const; virtual void forEachCommentLocation(std::function callback) const; virtual void forEachError(std::function callback) const; diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index 7e6b4a5a..4f73f456 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -11,7 +11,7 @@ #include "utility/utilityString.h" #include "utility/Version.h" -const size_t SqliteStorage::STORAGE_VERSION = 5; +const size_t SqliteStorage::STORAGE_VERSION = 6; SqliteStorage::SqliteStorage(const FilePath& dbFilePath) : m_dbFilePath(dbFilePath) @@ -62,6 +62,18 @@ void SqliteStorage::rollbackTransaction() m_database.execDML("ROLLBACK TRANSACTION;"); } +void SqliteStorage::optimizeMemory() const +{ + try + { + m_database.execDML("VACUUM;"); + } + catch(CppSQLite3Exception e) + { + LOG_ERROR(e.errorMessage()); + } +} + FilePath SqliteStorage::getDbFilePath() const { return m_dbFilePath; @@ -181,11 +193,11 @@ Id SqliteStorage::addLocalSymbol(const std::string& name) } Id SqliteStorage::addSourceLocation( - Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) + Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) { m_database.execDML(( - "INSERT INTO source_location(id, element_id, file_node_id, start_line, start_column, end_line, end_column, type) " - "VALUES(NULL, " + std::to_string(elementId) + ", " + std::to_string(fileNodeId) + ", " + "INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) " + "VALUES(NULL, " + std::to_string(fileNodeId) + ", " + std::to_string(startLine) + ", " + std::to_string(startCol) + ", " + std::to_string(endLine) + ", " + std::to_string(endCol) + ", " + std::to_string(type) + ");" ).c_str()); @@ -193,6 +205,23 @@ Id SqliteStorage::addSourceLocation( return m_database.lastRowId(); } +bool SqliteStorage::addOccurrence(Id elementId, Id sourceLocationId) +{ + try + { + m_database.execDML(( + "INSERT INTO occurrence(element_id, source_location_id) " + "VALUES(" + std::to_string(elementId) + ", " + + std::to_string(sourceLocationId) + ");" + ).c_str()); + } + catch (CppSQLite3Exception& e) + { + return false; + } + return true; +} + Id SqliteStorage::addComponentAccess(Id nodeId, int type) { m_database.execDML(( @@ -266,32 +295,18 @@ void SqliteStorage::removeElements(const std::vector& ids) ).c_str()); } -void SqliteStorage::removeElementsWithLocationInFiles(const std::vector& fileIds) +void SqliteStorage::removeElementsWithLocationInFiles(const std::vector& fileIds) // TODO: make one single clearFiles method { - CppSQLite3Query q = m_database.execQuery(( - "SELECT id, element_id FROM source_location WHERE file_node_id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");" + m_database.execDML(( + "DELETE FROM source_location WHERE file_node_id IN (" + utility::join(utility::toStrings(fileIds), ',') + ");" ).c_str()); - std::vector sourceLocationIds; - std::vector elementIds; - while (!q.eof()) - { - sourceLocationIds.push_back(q.getIntField(0, 0)); - elementIds.push_back(q.getIntField(1, 0)); - q.nextRow(); - } - - m_database.execDML(( - "DELETE FROM source_location WHERE id IN (" + utility::join(utility::toStrings(sourceLocationIds), ',') + ");" - ).c_str()); - - m_database.execDML(( + m_database.execDML( "DELETE FROM element WHERE " - "element.id IN (" + utility::join(utility::toStrings(elementIds), ',') + ") " // skip all elements that dont have a matching id. - "AND element.id NOT IN (" - "SELECT source_location.element_id FROM source_location WHERE source_location.element_id == element.id LIMIT 1" + "element.id NOT IN (" + "SELECT occurrence.element_id FROM occurrence WHERE occurrence.element_id == element.id LIMIT 1" ");" // delete all elements that dont have a source location. This query is executed for each element that passed the first test. - ).c_str()); + ); } void SqliteStorage::removeErrorsInFiles(const std::vector& filePaths) @@ -378,18 +393,6 @@ std::vector SqliteStorage::getEdgesByTargetType(Id targetId, int ty return getAll("WHERE target_node_id == " + std::to_string(targetId) + " AND type == " + std::to_string(type)); } -void SqliteStorage::optimizeMemory() const -{ - try - { - m_database.execDML("VACUUM;"); - } - catch(CppSQLite3Exception e) - { - LOG_ERROR(e.errorMessage()); - } -} - StorageNode SqliteStorage::getNodeById(Id id) const { if (id != 0) @@ -466,7 +469,6 @@ std::shared_ptr SqliteStorage::getFileContentById(Id fileId) const LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage()); } - return TextAccess::createFromString(""); } @@ -515,6 +517,18 @@ StorageSourceLocation SqliteStorage::getSourceLocationById(const Id id) const ); } +StorageSourceLocation SqliteStorage::getSourceLocationByAll(const Id fileNodeId, const uint startLine, const uint startCol, const uint endLine, const uint endCol, const int type) const +{ + return getFirst( + "WHERE file_node_id == " + std::to_string(fileNodeId) + + " AND start_line == " + std::to_string(startLine) + + " AND start_column == " + std::to_string(startCol) + + " AND end_line == " + std::to_string(endLine) + + " AND end_column == " + std::to_string(endCol) + + " AND type == " + std::to_string(type) + ";" + ); +} + std::shared_ptr SqliteStorage::getTokenLocationsForFile(const FilePath& filePath) const { std::shared_ptr ret = std::make_shared(filePath); @@ -525,17 +539,17 @@ std::shared_ptr SqliteStorage::getTokenLocationsForFile(const return ret; } - for (StorageSourceLocation& location: getAll("WHERE file_node_id == " + std::to_string(fileNodeId))) + for (std::pair e: getAllSourceLocationsAndElementIds("WHERE source_location.file_node_id == " + std::to_string(fileNodeId))) { TokenLocation* loc = ret->addTokenLocation( - location.id, - location.elementId, - location.startLine, - location.startCol, - location.endLine, - location.endCol + e.first.id, + e.second, + e.first.startLine, + e.first.startCol, + e.first.endLine, + e.first.endCol ); - loc->setType(intToLocationType(location.type)); + loc->setType(intToLocationType(e.first.type)); } ret->isWholeCopy = true; @@ -543,27 +557,78 @@ std::shared_ptr SqliteStorage::getTokenLocationsForFile(const return ret; } -std::vector SqliteStorage::getTokenLocationsForElementId(const Id elementId) const +std::vector SqliteStorage::getSourceLocationsForElementId(const Id elementId) const { std::vector elementIds {elementId}; - return getTokenLocationsForElementIds(elementIds); + std::vector ret; + for (std::pair e: getSourceLocationsAndElementIdsForElementIds(elementIds)) + { + ret.push_back(e.first); + } + return ret; } -std::vector SqliteStorage::getTokenLocationsForElementIds(const std::vector elementIds) const +std::vector> SqliteStorage::getSourceLocationsAndElementIdsForElementIds(const std::vector elementIds) const { - return getAll("WHERE element_id IN (" + utility::join(utility::toStrings(elementIds), ',') + ")"); + return getAllSourceLocationsAndElementIds("WHERE occurrence.element_id IN (" + utility::join(utility::toStrings(elementIds), ',') + ")"); } -Id SqliteStorage::getElementIdByLocationId(Id locationId) const +std::vector> SqliteStorage::getAllSourceLocationsAndElementIds(const std::string& query) const { CppSQLite3Query q = m_database.execQuery(( - "SELECT element_id FROM source_location WHERE id == " + std::to_string(locationId) + " LIMIT 1;" + "SELECT " + "source_location.id, " + "source_location.file_node_id, " + "source_location.start_line, " + "source_location.start_column, " + "source_location.end_line, " + "source_location.end_column, " + "source_location.type, " + "occurrence.element_id " + "FROM source_location " + "INNER JOIN occurrence ON occurrence.source_location_id = source_location.id " + query + ";" ).c_str()); - if (!q.eof()) + + std::vector> ret; + while (!q.eof()) { - return q.getIntField(0, 0); + const Id id = q.getIntField(0, 0); + const Id fileNodeId = q.getIntField(1, 0); + const int startLine = q.getIntField(2, -1); + const int startColumn = q.getIntField(3, -1); + const int endLine = q.getIntField(4, -1); + const int endColumn = q.getIntField(5, -1); + const int type = q.getIntField(6, -1); + const Id elementId = q.getIntField(7, 0); + + if (id != 0 && fileNodeId != 0 && startLine != -1 && startColumn != -1 && endLine != -1 && endColumn != -1 && type != -1 && elementId != 0) + { + ret.push_back(std::make_pair( + StorageSourceLocation( + id, + fileNodeId, + startLine, + startColumn, + endLine, + endColumn, + type), + elementId + )); + } + q.nextRow(); } - return 0; + return ret; +} + +std::vector SqliteStorage::getOccurrencesForLocationId(Id locationId) const +{ + std::vector locationIds {locationId}; + return getOccurrencesForLocationIds(locationIds); +} + +std::vector SqliteStorage::getOccurrencesForLocationIds(const std::vector& locationIds) const +{ + return getAll("WHERE source_location_id IN (" + utility::join(utility::toStrings(locationIds), ',') + ")"); } StorageComponentAccess SqliteStorage::getComponentAccessByNodeId(Id nodeId) const @@ -607,6 +672,11 @@ std::vector SqliteStorage::getAllSourceLocations() const return getAll(""); } +std::vector SqliteStorage::getAllOccurrences() const +{ + return getAll(""); +} + std::vector SqliteStorage::getAllComponentAccesses() const { return getAll(""); @@ -652,6 +722,7 @@ void SqliteStorage::clearTables() m_database.execDML("DROP TABLE IF EXISTS main.error;"); m_database.execDML("DROP TABLE IF EXISTS main.comment_location;"); m_database.execDML("DROP TABLE IF EXISTS main.component_access;"); + m_database.execDML("DROP TABLE IF EXISTS main.occurrence;"); m_database.execDML("DROP TABLE IF EXISTS main.source_location;"); m_database.execDML("DROP TABLE IF EXISTS main.local_symbol;"); m_database.execDML("DROP TABLE IF EXISTS main.filecontent;"); @@ -746,7 +817,6 @@ void SqliteStorage::setupTables() m_database.execDML( "CREATE TABLE IF NOT EXISTS source_location(" "id INTEGER NOT NULL, " - "element_id INTEGER, " "file_node_id INTEGER, " "start_line INTEGER, " "start_column INTEGER, " @@ -754,12 +824,20 @@ void SqliteStorage::setupTables() "end_column INTEGER, " "type INTEGER, " "PRIMARY KEY(id), " - "FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, " "FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);" ); - SqliteIndex("source_location_element_id_index", "source_location(element_id)").createOnDatabase(m_database); SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database); + SqliteIndex("source_location_all_data_index", "source_location(file_node_id, start_line, start_column, end_line, end_column, type)").createOnDatabase(m_database); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS occurrence(" // TODO: properly delete this on refresh + "element_id INTEGER NOT NULL, " + "source_location_id INTEGER NOT NULL, " + "PRIMARY KEY(element_id, source_location_id), " + "FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, " + "FOREIGN KEY(source_location_id) REFERENCES source_location(id) ON DELETE CASCADE);" + ); m_database.execDML( "CREATE TABLE IF NOT EXISTS component_access(" @@ -983,25 +1061,24 @@ template <> std::vector SqliteStorage::getAll(const std::string& query) const { CppSQLite3Query q = m_database.execQuery(( - "SELECT id, element_id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location " + query + ";" - ).c_str()); + "SELECT id, file_node_id, start_line, start_column, end_line, end_column, type FROM source_location " + query + ";" + ).c_str()); std::vector sourceLocations; while (!q.eof()) { const Id id = q.getIntField(0, 0); - const Id elementId = q.getIntField(1, 0); - const Id fileNodeId = q.getIntField(2, 0); - const int startLineNumber = q.getIntField(3, -1); - const int startColNumber = q.getIntField(4, -1); - const int endLineNumber = q.getIntField(5, -1); - const int endColNumber = q.getIntField(6, -1); - const int type = q.getIntField(7, -1); + const Id fileNodeId = q.getIntField(1, 0); + const int startLineNumber = q.getIntField(2, -1); + const int startColNumber = q.getIntField(3, -1); + const int endLineNumber = q.getIntField(4, -1); + const int endColNumber = q.getIntField(5, -1); + const int type = q.getIntField(6, -1); - if (id != 0 && elementId != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1 && type != -1) + if (id != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1 && type != -1) { - sourceLocations.push_back(StorageSourceLocation(id, elementId, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type)); + sourceLocations.push_back(StorageSourceLocation(id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber, type)); } q.nextRow(); @@ -1009,6 +1086,30 @@ std::vector SqliteStorage::getAll( return sourceLocations; } +template <> +std::vector SqliteStorage::getAll(const std::string& query) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT element_id, source_location_id FROM occurrence " + query + ";" + ).c_str()); + + std::vector occurrences; + + while (!q.eof()) + { + const Id elementId = q.getIntField(0, 0); + const Id sourceLocationId = q.getIntField(1, 0); + + if (elementId != 0 && sourceLocationId != 0) + { + occurrences.push_back(StorageOccurrence(elementId, sourceLocationId)); + } + + q.nextRow(); + } + return occurrences; +} + template <> std::vector SqliteStorage::getAll(const std::string& query) const { diff --git a/src/lib/data/SqliteStorage.h b/src/lib/data/SqliteStorage.h index 025bb85c..3602cac4 100644 --- a/src/lib/data/SqliteStorage.h +++ b/src/lib/data/SqliteStorage.h @@ -31,6 +31,8 @@ public: void commitTransaction(); void rollbackTransaction(); + void optimizeMemory() const; + FilePath getDbFilePath() const; bool isEmpty() const; @@ -44,7 +46,8 @@ public: Id addNode(int type, const std::string& serializedName, int definitionType); Id addFile(const std::string& serializedName, const std::string& filePath, const std::string& modificationTime); Id addLocalSymbol(const std::string& name); - Id addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); + Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type); + bool addOccurrence(Id elementId, Id sourceLocationId); Id addComponentAccess(Id nodeId, int type); Id addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol); Id addError(const std::string& message, const FilePath& filePath, uint lineNumber, uint columnNumber, bool fatal, bool indexed); @@ -90,17 +93,18 @@ public: void setNodeDefinitionType(int definitionType, Id nodeId); StorageSourceLocation getSourceLocationById(const Id id) const; + StorageSourceLocation getSourceLocationByAll(const Id fileNodeId, const uint startLine, const uint startCol, const uint endLine, const uint endCol, const int type) const; std::shared_ptr getTokenLocationsForFile(const FilePath& filePath) const; - std::vector getTokenLocationsForElementId(const Id elementId) const; - std::vector getTokenLocationsForElementIds(const std::vector elementIds) const; + std::vector getSourceLocationsForElementId(const Id elementId) const; + std::vector> getSourceLocationsAndElementIdsForElementIds(const std::vector elementIds) const; + std::vector> getAllSourceLocationsAndElementIds(const std::string& query) const; - Id getElementIdByLocationId(Id locationId) const; + std::vector getOccurrencesForLocationId(Id locationId) const; + std::vector getOccurrencesForLocationIds(const std::vector& locationIds) const; StorageComponentAccess getComponentAccessByNodeId(Id memberEdgeId) const; std::vector getComponentAccessesByNodeIds(const std::vector& memberEdgeIds) const; - void optimizeMemory() const; - std::vector getFullTextSearch(const std::string& searchTerm) const; std::vector getCommentLocationsInFile(const FilePath& filePath) const; @@ -110,6 +114,7 @@ public: std::vector getAllEdges() const; std::vector getAllLocalSymbols() const; std::vector getAllSourceLocations() const; + std::vector getAllOccurrences() const; std::vector getAllComponentAccesses() const; std::vector getAllCommentLocations() const; std::vector getAllErrors() const; @@ -166,6 +171,8 @@ std::vector SqliteStorage::getAll(const template <> std::vector SqliteStorage::getAll(const std::string& query) const; template <> +std::vector SqliteStorage::getAll(const std::string& query) const; +template <> std::vector SqliteStorage::getAll(const std::string& query) const; template <> std::vector SqliteStorage::getAll(const std::string& query) const; diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index 1c5620cc..bcc13ec2 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -90,7 +90,33 @@ void Storage::inject(Storage* injected) ); injected->forEachSourceLocation( - [&](const StorageSourceLocation& injectedData) + [&](const Id injectedId, const StorageSourceLocation& injectedData) + { + std::unordered_map::const_iterator it; + it = injectedIdToOwnId.find(injectedData.fileNodeId); + if (it == injectedIdToOwnId.end()) + { + return; + } + Id ownFileNodeId = it->second; + + Id ownId = addSourceLocation( + ownFileNodeId, + injectedData.startLine, + injectedData.startCol, + injectedData.endLine, + injectedData.endCol, + injectedData.type + ); + if (ownId != 0) + { + injectedIdToOwnId[injectedId] = ownId; + } + } + ); + + injected->forEachOccurrence( + [&](const StorageOccurrence& injectedData) { std::unordered_map::const_iterator it; it = injectedIdToOwnId.find(injectedData.elementId); @@ -100,22 +126,14 @@ void Storage::inject(Storage* injected) } Id ownElementId = it->second; - it = injectedIdToOwnId.find(injectedData.fileNodeId); + it = injectedIdToOwnId.find(injectedData.sourceLocationId); if (it == injectedIdToOwnId.end()) { return; } - Id ownFileNodeId = it->second; + Id ownSourceLocationId = it->second; - addSourceLocation( - ownElementId, - ownFileNodeId, - injectedData.startLine, - injectedData.startCol, - injectedData.endLine, - injectedData.endCol, - injectedData.type - ); + addOccurrence(ownElementId, ownSourceLocationId); } ); diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index c0bb646e..6f16c539 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -19,7 +19,8 @@ public: virtual Id addNode(int type, const std::string& serializedName, int definitionType) = 0; virtual Id addEdge(int type, Id sourceId, Id targetId) = 0; virtual Id addLocalSymbol(const std::string& name) = 0; - virtual void addSourceLocation(Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) = 0; + virtual Id addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) = 0; + virtual void addOccurrence(Id elementId, Id sourceLocationId) = 0; virtual void addComponentAccess(Id nodeId , int type) = 0; virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) = 0; virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) = 0; @@ -28,7 +29,8 @@ public: virtual void forEachNode(std::function callback) const = 0; virtual void forEachEdge(std::function callback) const = 0; virtual void forEachLocalSymbol(std::function callback) const = 0; - virtual void forEachSourceLocation(std::function callback) const = 0; + virtual void forEachSourceLocation(std::function callback) const = 0; + virtual void forEachOccurrence(std::function callback) const = 0; virtual void forEachComponentAccess(std::function callback) const = 0; virtual void forEachCommentLocation(std::function callback) const = 0; virtual void forEachError(std::function callback) const = 0; diff --git a/src/lib/data/StorageProvider.cpp b/src/lib/data/StorageProvider.cpp index 6bec370b..5a9fca0c 100644 --- a/src/lib/data/StorageProvider.cpp +++ b/src/lib/data/StorageProvider.cpp @@ -1,7 +1,5 @@ #include "data/StorageProvider.h" -#include - int StorageProvider::getStorageCount() const { std::lock_guard lock(m_storagesMutex); @@ -10,13 +8,38 @@ int StorageProvider::getStorageCount() const void StorageProvider::pushIndexerTarget(std::shared_ptr storage) { + const std::size_t storageSize = storage->getSourceLocationCount(); + std::list>::iterator it; + std::lock_guard lock(m_storagesMutex); - m_storages.push_back(storage); + for (it = m_storages.begin(); it != m_storages.end(); it++) + { + if ((*it)->getSourceLocationCount() < storageSize) + { + break; + } + } + m_storages.insert(it, storage); } std::shared_ptr StorageProvider::popIndexerTarget() { - return std::make_shared();; + std::shared_ptr ret; + { + std::lock_guard lock(m_storagesMutex); + if (m_storages.size() > 1) + { + std::list>::iterator it = m_storages.begin(); + it++; + ret = *it; + m_storages.erase(it); + } + else + { + ret = std::make_shared(); + } + } + return ret; } std::shared_ptr StorageProvider::popInjectionSource() @@ -33,3 +56,6 @@ std::shared_ptr StorageProvider::popInjectionSource() return ret; } + + + diff --git a/src/lib/data/StorageTypes.h b/src/lib/data/StorageTypes.h index 80aaf464..29056633 100644 --- a/src/lib/data/StorageTypes.h +++ b/src/lib/data/StorageTypes.h @@ -93,7 +93,6 @@ struct StorageSourceLocation { StorageSourceLocation() : id(0) - , elementId(0) , fileNodeId(0) , startLine(-1) , startCol(-1) @@ -102,9 +101,8 @@ struct StorageSourceLocation , type(0) {} - StorageSourceLocation(Id id, Id elementId, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) + StorageSourceLocation(Id id, Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) : id(id) - , elementId(elementId) , fileNodeId(fileNodeId) , startLine(startLine) , startCol(startCol) @@ -114,7 +112,6 @@ struct StorageSourceLocation {} Id id; - Id elementId; Id fileNodeId; uint startLine; uint startCol; @@ -123,6 +120,22 @@ struct StorageSourceLocation int type; }; +struct StorageOccurrence +{ + StorageOccurrence() + : elementId(0) + , sourceLocationId(0) + {} + + StorageOccurrence(Id elementId, Id sourceLocationId) + : elementId(elementId) + , sourceLocationId(sourceLocationId) + {} + + Id elementId; + Id sourceLocationId; +}; + struct StorageComponentAccess { StorageComponentAccess() diff --git a/src/lib/data/parser/ParserClientImpl.cpp b/src/lib/data/parser/ParserClientImpl.cpp index 040be879..fdb5e0a5 100644 --- a/src/lib/data/parser/ParserClientImpl.cpp +++ b/src/lib/data/parser/ParserClientImpl.cpp @@ -293,8 +293,7 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca return; } - m_storage->addSourceLocation( - elementId, + Id sourceLocationId = m_storage->addSourceLocation( addFile(location.filePath.str()), location.startLineNumber, location.startColumnNumber, @@ -302,6 +301,11 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca location.endColumnNumber, type ); + + m_storage->addOccurrence( + elementId, + sourceLocationId + ); } void ParserClientImpl::addComponentAccess(Id nodeId , int type)