diff --git a/src/lib/data/storage/Storage.cpp b/src/lib/data/storage/Storage.cpp index 3b53693b..5bc40f2a 100644 --- a/src/lib/data/storage/Storage.cpp +++ b/src/lib/data/storage/Storage.cpp @@ -156,12 +156,7 @@ void Storage::inject(Storage* injected) } const Id ownSourceLocationId = it->second; - occurrences.push_back(StorageOccurrence(ownElementId, ownSourceLocationId)); - if (occurrences.size() >= 100) - { - addOccurrences(occurrences); - occurrences.clear(); - } + occurrences.emplace_back(ownElementId, ownSourceLocationId); } ); addOccurrences(occurrences); diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp index 11145861..f7cad798 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp @@ -9,27 +9,6 @@ #include "data/location/SourceLocationCollection.h" #include "data/location/SourceLocationFile.h" -namespace -{ - std::string serialize(const StorageEdgeData& data) - { - return ( - std::to_string(data.sourceNodeId) + "," + - std::to_string(data.targetNodeId) + "," + - std::to_string(data.type) - ); - } - - std::string serialize(const StorageSourceLocationData& loc) - { - return ( - std::to_string(loc.startLine) + "," + std::to_string(loc.startCol) + "," + - std::to_string(loc.endLine) + "," + std::to_string(loc.endCol) + "," + - std::to_string(loc.type) - ); - } -} - const size_t SqliteIndexStorage::s_storageVersion = 19; SqliteIndexStorage::SqliteIndexStorage(const FilePath& dbFilePath) @@ -79,17 +58,17 @@ StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data) { for (const StorageNode& node : getAll()) { - m_tempNodeIndex[node.serializedName] = std::make_pair(node.id, node.type); + m_tempNodeIndex.emplace(utility::encodeToUtf8(node.serializedName), std::make_pair(node.id, node.type)); } } + std::string name = utility::encodeToUtf8(data.serializedName); { - std::map>::iterator it = m_tempNodeIndex.find(data.serializedName); + std::map>::const_iterator it = m_tempNodeIndex.find(name); if (it != m_tempNodeIndex.end()) { if (it->second.second < data.type) { - it->second.second = data.type; setNodeType(data.type, it->second.first); } return StorageNode(it->second.first, data); @@ -105,12 +84,12 @@ StorageNode SqliteIndexStorage::addNode(const StorageNodeData& data) { m_inserNodeStmt.bind(1, int(id)); m_inserNodeStmt.bind(2, data.type); - m_inserNodeStmt.bind(3, utility::encodeToUtf8(data.serializedName).c_str()); + m_inserNodeStmt.bind(3, name.c_str()); executeStatement(m_inserNodeStmt); m_inserNodeStmt.reset(); } - m_tempNodeIndex[data.serializedName] = std::make_pair(id, data.type); + m_tempNodeIndex.emplace(name, std::make_pair(id, data.type)); return StorageNode(id, data); } @@ -165,13 +144,12 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data) { for (const StorageEdge& edge : getAll()) { - m_tempEdgeIndex[serialize(edge)] = edge.id; + m_tempEdgeIndex.emplace(StorageEdgeData(edge.type, edge.sourceNodeId, edge.targetNodeId), edge.id); } } - const std::string serialized = serialize(data); { - std::map::const_iterator it = m_tempEdgeIndex.find(serialized); + std::map::const_iterator it = m_tempEdgeIndex.find(data); if (it != m_tempEdgeIndex.end()) { return StorageEdge(it->second, data); @@ -193,7 +171,7 @@ StorageEdge SqliteIndexStorage::addEdge(const StorageEdgeData& data) m_insertEdgeStmt.reset(); } - m_tempEdgeIndex[serialized] = id; + m_tempEdgeIndex.emplace(data, id); return StorageEdge(id, data); } @@ -204,12 +182,13 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa { for (const StorageLocalSymbol& localSymbol : getAll()) { - m_tempLocalSymbolIndex[localSymbol.name] = localSymbol.id; + m_tempLocalSymbolIndex.emplace(utility::encodeToUtf8(localSymbol.name), localSymbol.id); } } + std::string name = utility::encodeToUtf8(data.name); { - std::map::const_iterator it = m_tempLocalSymbolIndex.find(data.name); + std::map::const_iterator it = m_tempLocalSymbolIndex.find(name); if (it != m_tempLocalSymbolIndex.end()) { return StorageLocalSymbol(it->second, data); @@ -224,12 +203,12 @@ StorageLocalSymbol SqliteIndexStorage::addLocalSymbol(const StorageLocalSymbolDa } { m_inserLocalSymbolStmt.bind(1, int(id)); - m_inserLocalSymbolStmt.bind(2, utility::encodeToUtf8(data.name).c_str()); + m_inserLocalSymbolStmt.bind(2, name.c_str()); executeStatement(m_inserLocalSymbolStmt); m_inserLocalSymbolStmt.reset(); } - m_tempLocalSymbolIndex[data.name] = id; + m_tempLocalSymbolIndex.emplace(name, id); return StorageLocalSymbol(id, data); } @@ -238,16 +217,20 @@ StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceL { if (m_tempSourceLocationIndices.empty()) { - for (const StorageSourceLocation& sourceLocation : getAll()) + for (const StorageSourceLocation& loc : getAll()) { - m_tempSourceLocationIndices[sourceLocation.fileNodeId][serialize(sourceLocation)] = sourceLocation.id; + std::map& index = m_tempSourceLocationIndices[loc.fileNodeId]; + index.emplace( + TempSourceLocation(loc.startLine, loc.endLine - loc.startLine, loc.startCol, loc.endCol, loc.type), + loc.id); } } - const std::string serialized = serialize(data); - std::map& index = m_tempSourceLocationIndices[data.fileNodeId]; + const TempSourceLocation tempLoc(data.startLine, data.endLine - data.startLine, data.startCol, data.endCol, data.type); + + std::map& index = m_tempSourceLocationIndices[data.fileNodeId]; { - std::map::const_iterator it = index.find(serialized); + std::map::const_iterator it = index.find(tempLoc); if (it != index.end()) { return StorageSourceLocation(it->second, data); @@ -266,7 +249,7 @@ StorageSourceLocation SqliteIndexStorage::addSourceLocation(const StorageSourceL if (success) { id = m_database.lastRowId(); - index[serialized] = id; + index.emplace(tempLoc, id); } m_insertSourceLocationStmt.reset(); @@ -285,35 +268,46 @@ bool SqliteIndexStorage::addOccurrence(const StorageOccurrence& data) bool SqliteIndexStorage::addOccurrences(const std::vector& occurrences) { - if (occurrences.size() == 100) + size_t i = 0; + while (occurrences.size() - i >= 100) { - for (int i = 0; i < occurrences.size(); i++) + for (size_t j = 0; j < 100; j++) { - m_insert100OccurrencesStmt.bind((i * 2) + 1, int(occurrences[i].elementId)); - m_insert100OccurrencesStmt.bind((i * 2) + 2, int(occurrences[i].sourceLocationId)); + m_insert100OccurrencesStmt.bind((j * 2) + 1, int(occurrences[i + j].elementId)); + m_insert100OccurrencesStmt.bind((j * 2) + 2, int(occurrences[i + j].sourceLocationId)); } + const bool success = executeStatement(m_insert100OccurrencesStmt); m_insert100OccurrencesStmt.reset(); - return success; + + if (!success) + { + return false; + } + + i += 100; } - else if (!occurrences.empty()) + + if (i < occurrences.size()) { std::string stmt = "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES"; { bool isFirst = true; - for (const StorageOccurrence& occurrence : occurrences) + while (i < occurrences.size()) { if (!isFirst) { stmt += ","; } isFirst = false; - stmt += "(" + std::to_string(occurrence.elementId) + "," + std::to_string(occurrence.sourceLocationId) + ")"; + stmt += "(" + std::to_string(occurrences[i].elementId) + "," + std::to_string(occurrences[i].sourceLocationId) + ")"; + i++; } stmt += ";"; } return executeStatement(stmt); } + return true; } @@ -422,7 +416,8 @@ void SqliteIndexStorage::removeElements(const std::vector& ids) ); } -void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector& fileIds, std::function updateStatusCallback) +void SqliteIndexStorage::removeElementsWithLocationInFiles( + const std::vector& fileIds, std::function updateStatusCallback) { if (updateStatusCallback != nullptr) { @@ -467,7 +462,8 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector // delete all edges in element_id_to_clear executeStatement( - "DELETE FROM element WHERE element.id IN (SELECT element_id_to_clear.id FROM element_id_to_clear INNER JOIN edge ON (element_id_to_clear.id = edge.id))" + "DELETE FROM element WHERE element.id IN " + " (SELECT element_id_to_clear.id FROM element_id_to_clear INNER JOIN edge ON (element_id_to_clear.id = edge.id))" ); if (updateStatusCallback != nullptr) @@ -522,7 +518,8 @@ void SqliteIndexStorage::removeElementsWithLocationInFiles(const std::vector // remove all ids from element_id_to_clear that still have occurrences executeStatement( "DELETE FROM element_id_to_clear WHERE id IN (" - " SELECT element_id_to_clear.id FROM element_id_to_clear INNER JOIN occurrence ON element_id_to_clear.id = occurrence.element_id" + " SELECT element_id_to_clear.id FROM element_id_to_clear INNER JOIN occurrence ON " + " element_id_to_clear.id = occurrence.element_id" ")" ); @@ -654,7 +651,8 @@ std::vector SqliteIndexStorage::getEdgesBySourceType(Id sourceId, i std::vector SqliteIndexStorage::getEdgesBySourcesType(const std::vector& sourceIds, int type) const { - return doGetAll("WHERE source_node_id IN (" + utility::join(utility::toStrings(sourceIds), ',') + ") AND type == " + std::to_string(type)); + return doGetAll("WHERE source_node_id IN (" + utility::join(utility::toStrings(sourceIds), ',') + ")" + " AND type == " + std::to_string(type)); } std::vector SqliteIndexStorage::getEdgesByTargetType(Id targetId, int type) const @@ -664,7 +662,8 @@ std::vector SqliteIndexStorage::getEdgesByTargetType(Id targetId, i std::vector SqliteIndexStorage::getEdgesByTargetsType(const std::vector& targetIds, int type) const { - return doGetAll("WHERE target_node_id IN (" + utility::join(utility::toStrings(targetIds), ',') + ") AND type == " + std::to_string(type)); + return doGetAll("WHERE target_node_id IN (" + utility::join(utility::toStrings(targetIds), ',') + ")" + " AND type == " + std::to_string(type)); } StorageNode SqliteIndexStorage::getNodeById(Id id) const @@ -980,7 +979,8 @@ std::vector> SqliteIndexStorage::getIndices( )); indices.push_back(std::make_pair( STORAGE_MODE_WRITE, - SqliteDatabaseIndex("comment_location_all_data_index", "comment_location(file_node_id, start_line, start_column, end_line, end_column)") + SqliteDatabaseIndex("comment_location_all_data_index", + "comment_location(file_node_id, start_line, start_column, end_line, end_column)") )); indices.push_back(std::make_pair( STORAGE_MODE_CLEAR, @@ -1186,7 +1186,8 @@ void SqliteIndexStorage::setupPrecompiledStatements() "INSERT INTO local_symbol(id, name) VALUES(?, ?);" ); m_insertSourceLocationStmt = m_database.compileStatement( - "INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) VALUES(NULL, ?, ?, ?, ?, ?, ?);" + "INSERT INTO source_location(id, file_node_id, start_line, start_column, end_line, end_column, type) " + "VALUES(NULL, ?, ?, ?, ?, ?, ?);" ); m_insertOccurrenceStmt = m_database.compileStatement( "INSERT OR IGNORE INTO occurrence(element_id, source_location_id) VALUES(?, ?);" @@ -1217,7 +1218,8 @@ void SqliteIndexStorage::setupPrecompiledStatements() "LIMIT 1;" ); m_insertCommentLocationStmt = m_database.compileStatement( - "INSERT INTO comment_location(id, file_node_id, start_line, start_column, end_line, end_column) VALUES(NULL, ?, ?, ?, ?, ?);" + "INSERT INTO comment_location(id, file_node_id, start_line, start_column, end_line, end_column) " + "VALUES(NULL, ?, ?, ?, ?, ?);" ); m_checkErrorExistsStmt = m_database.compileStatement( "SELECT id FROM error WHERE " @@ -1229,7 +1231,8 @@ void SqliteIndexStorage::setupPrecompiledStatements() "LIMIT 1;" ); m_insertErrorStmt = m_database.compileStatement( - "INSERT INTO error(message, fatal, indexed, file_path, line_number, column_number, translation_unit) VALUES(?, ?, ?, ?, ?, ?, ?);" + "INSERT INTO error(message, fatal, indexed, file_path, line_number, column_number, translation_unit) " + "VALUES(?, ?, ?, ?, ?, ?, ?);" ); } catch (CppSQLite3Exception& e) @@ -1383,7 +1386,8 @@ std::vector SqliteIndexStorage::doGetAll SqliteIndexStorage::doGetAll> getIndices() const; virtual void clearTables(); @@ -174,10 +215,11 @@ private: return ResultType(); } - std::map> m_tempNodeIndex; - std::map m_tempEdgeIndex; - std::map m_tempLocalSymbolIndex; - std::map> m_tempSourceLocationIndices; + + std::map> m_tempNodeIndex; + std::map m_tempEdgeIndex; + std::map m_tempLocalSymbolIndex; + std::map> m_tempSourceLocationIndices; CppSQLite3Statement m_insertElementStmt; CppSQLite3Statement m_insertEdgeStmt; diff --git a/src/lib/data/storage/type/StorageEdge.h b/src/lib/data/storage/type/StorageEdge.h index bf38505f..01b35cb1 100644 --- a/src/lib/data/storage/type/StorageEdge.h +++ b/src/lib/data/storage/type/StorageEdge.h @@ -17,6 +17,22 @@ struct StorageEdgeData , targetNodeId(targetNodeId) {} + bool operator<(const StorageEdgeData& other) const + { + if (type != other.type) + { + return type < other.type; + } + else if (sourceNodeId != other.sourceNodeId) + { + return sourceNodeId < other.sourceNodeId; + } + else + { + return targetNodeId < other.targetNodeId; + } + } + int type; Id sourceNodeId; Id targetNodeId;