From 6ea248b5002dacfadf411b41d17570f982e56be9 Mon Sep 17 00:00:00 2001 From: malte_langkabel Date: Mon, 23 Jan 2017 10:36:42 +0100 Subject: [PATCH] logic: clearing unused includes * refresh also clears referenced (included or imported) files if they are not part of the project's source files and are not referenced by any other unchanged file anymore. --- src/lib/Project.cpp | 43 +++++-- src/lib/data/PersistentStorage.cpp | 199 ++++++++++++++++++----------- src/lib/data/PersistentStorage.h | 17 ++- src/lib/data/SqliteStorage.cpp | 7 +- src/lib/data/SqliteStorage.h | 3 +- src/lib/utility/utility.h | 12 ++ 6 files changed, 192 insertions(+), 89 deletions(-) diff --git a/src/lib/Project.cpp b/src/lib/Project.cpp index e8601252..892a828b 100644 --- a/src/lib/Project.cpp +++ b/src/lib/Project.cpp @@ -247,31 +247,52 @@ bool Project::requestIndex(bool forceRefresh, bool needsFullRefresh) return false; } - FileManager::FileSets fileSets = m_fileManager.fetchFilePaths(m_storage->getInfoOnAllFiles()); + const FileManager::FileSets fileSets = m_fileManager.fetchFilePaths(m_storage->getInfoOnAllFiles()); std::set filesToClean; std::set filesToIndex; if (!needsFullRefresh) { - std::set dependingFilePaths; - utility::append(dependingFilePaths, m_storage->getDependingFilePaths(fileSets.updatedFiles)); - utility::append(dependingFilePaths, m_storage->getDependingFilePaths(fileSets.removedFiles)); + utility::append(filesToClean, fileSets.removedFiles); + utility::append(filesToClean, fileSets.updatedFiles); + utility::append(filesToIndex, fileSets.addedFiles); + utility::append(filesToIndex, fileSets.updatedFiles); - for (const FilePath& path : dependingFilePaths) + // handle referencing paths + const std::set referencingFilePaths = m_storage->getReferencing(utility::concat( + fileSets.updatedFiles, fileSets.removedFiles + )); + + for (const FilePath& path : referencingFilePaths) { if (fileSets.removedFiles.find(path) == fileSets.removedFiles.end()) { - fileSets.updatedFiles.insert(path); + filesToClean.insert(path); + filesToIndex.insert(path); } } - utility::append(filesToClean, fileSets.removedFiles); - utility::append(filesToClean, fileSets.updatedFiles); - utility::append(filesToClean, dependingFilePaths); + // handle referenced paths + std::set staticSourceFiles = fileSets.allFiles; + for (const FilePath& path : fileSets.updatedFiles) + { + staticSourceFiles.erase(path); + } + const std::set staticReferencedFilePaths = m_storage->getReferenced(staticSourceFiles); + const std::set dynamicReferencedFilePaths = m_storage->getReferenced(utility::concat( + fileSets.updatedFiles, fileSets.removedFiles + )); - utility::append(filesToIndex, fileSets.addedFiles); - utility::append(filesToIndex, fileSets.updatedFiles); + for (const FilePath& path : dynamicReferencedFilePaths) + { + if (staticReferencedFilePaths.find(path) == staticReferencedFilePaths.end() && + staticSourceFiles.find(path) == staticSourceFiles.end()) + { + // file may not be referenced anymore and will be reindexed if still needed + filesToClean.insert(path); + } + } } bool fullRefresh = forceRefresh | needsFullRefresh; diff --git a/src/lib/data/PersistentStorage.cpp b/src/lib/data/PersistentStorage.cpp index 89a2a7b7..cb7f1328 100644 --- a/src/lib/data/PersistentStorage.cpp +++ b/src/lib/data/PersistentStorage.cpp @@ -300,32 +300,35 @@ void PersistentStorage::clearCaches() m_fullTextSearchIndex.clear(); } -std::set PersistentStorage::getDependingFilePaths(const std::set& filePaths) +std::set PersistentStorage::getReferenced(const std::set& filePaths) { TRACE(); - std::set dependingFilePaths; + std::set referenced; - std::set dependingFilePathsForIncludes = getDependingFilePathsForIncludes(filePaths); - dependingFilePaths.insert(dependingFilePathsForIncludes.begin(), dependingFilePathsForIncludes.end()); + utility::append(referenced, getReferencedByIncludes(filePaths)); + utility::append(referenced, getReferencedByImports(filePaths)); - std::set dependingFilePathsForImports = getDependingFilePathsForImports(filePaths); - dependingFilePaths.insert(dependingFilePathsForImports.begin(), dependingFilePathsForImports.end()); + return referenced; +} - return dependingFilePaths; +std::set PersistentStorage::getReferencing(const std::set& filePaths) +{ + TRACE(); + std::set referencing; + + utility::append(referencing, getReferencingByIncludes(filePaths)); + utility::append(referencing, getReferencingByImports(filePaths)); + + return referencing; } void PersistentStorage::clearFileElements(const std::vector& filePaths) { TRACE(); - std::vector fileNodeIds; + const std::vector fileNodeIds = getFileNodeIds(filePaths); - for (const FilePath& path : filePaths) - { - fileNodeIds.push_back(getFileNodeId(path)); - } - - if (fileNodeIds.size()) + if (!fileNodeIds.empty()) { m_sqliteStorage.removeElementsWithLocationInFiles(fileNodeIds); m_sqliteStorage.removeElements(fileNodeIds); @@ -1232,6 +1235,26 @@ Id PersistentStorage::getFileNodeId(const FilePath& filePath) const return 0; } +std::vector PersistentStorage::getFileNodeIds(const std::vector& filePaths) const +{ + std::vector ids; + for (const FilePath& path : filePaths) + { + ids.push_back(getFileNodeId(path)); + } + return ids; +} + +std::set PersistentStorage::getFileNodeIds(const std::set& filePaths) const +{ + std::set ids; + for (const FilePath& path : filePaths) + { + ids.insert(getFileNodeId(path)); + } + return ids; +} + FilePath PersistentStorage::getFileNodePath(Id fileId) const { if (fileId == 0) @@ -1250,42 +1273,19 @@ FilePath PersistentStorage::getFileNodePath(Id fileId) const return FilePath(); } -std::set PersistentStorage::getDependingFilePathsForIncludes(const std::set& filePaths) +std::unordered_map> PersistentStorage::getFileIdToIncludingFileIdMap() const { - std::set dependingFilePaths; - std::set processedFileNodeIds; - - std::vector fileNodeIds; - for (const FilePath& filePath : filePaths) + std::unordered_map> fileIdToIncludingFileIdMap; + for (const StorageEdge& includeEdge : m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_INCLUDE))) { - fileNodeIds.push_back(getFileNodeId(filePath)); + fileIdToIncludingFileIdMap[includeEdge.targetNodeId].insert(includeEdge.sourceNodeId); } - - while (fileNodeIds.size()) - { - std::vector incomingEdges = - m_sqliteStorage.getEdgesByTargetType(fileNodeIds, Edge::typeToInt(Edge::EDGE_INCLUDE)); - - processedFileNodeIds.insert(fileNodeIds.begin(), fileNodeIds.end()); - fileNodeIds.clear(); - - for (const StorageEdge& incomingEdge: incomingEdges) - { - dependingFilePaths.insert(getFileNodePath(incomingEdge.sourceNodeId)); - - if (processedFileNodeIds.find(incomingEdge.sourceNodeId) == processedFileNodeIds.end()) - { - fileNodeIds.push_back(incomingEdge.sourceNodeId); - } - } - } - - return dependingFilePaths; + return fileIdToIncludingFileIdMap; } -std::set PersistentStorage::getDependingFilePathsForImports(const std::set& filePaths) +std::unordered_map> PersistentStorage::getFileIdToImportingFileIdMap() const { - std::unordered_map> fileIdToDependingFileIds; + std::unordered_map> fileIdToImportingFileIdMap; { std::vector importedElementIds; std::map> elementIdToImportingFileIds; @@ -1296,11 +1296,6 @@ std::set PersistentStorage::getDependingFilePathsForImports(const std: elementIdToImportingFileIds[importEdge.targetNodeId].insert(importEdge.sourceNodeId); } - if (!importedElementIds.size()) - { - return std::set(); - } - std::unordered_map importedElementIdToFileNodeId; { std::vector importedSourceLocationIds; @@ -1326,49 +1321,107 @@ std::set PersistentStorage::getDependingFilePathsForImports(const std: auto importedFileIt = importedElementIdToFileNodeId.find(it.first); if (importedFileIt != importedElementIdToFileNodeId.end()) { - fileIdToDependingFileIds[importedFileIt->second].insert(it.second.begin(), it.second.end()); + fileIdToImportingFileIdMap[importedFileIt->second].insert(it.second.begin(), it.second.end()); } } } + return fileIdToImportingFileIdMap; +} - std::set dependingFileNodeIds; +std::set PersistentStorage::getReferenced(const std::set& ids, std::unordered_map> idToReferencingIdMap) const +{ + std::unordered_map> idToReferencedIdMap; + for (auto it: idToReferencingIdMap) { - std::set working; - for (const FilePath& filePath: filePaths) + for (Id referencingId: it.second) { - working.insert(getFileNodeId(filePath)); + idToReferencedIdMap[referencingId].insert(it.first); } + } - std::set tempWorking; - while (working.size() > 0) + return getReferencing(ids, idToReferencedIdMap); +} + +std::set PersistentStorage::getReferencing(const std::set& ids, std::unordered_map> idToReferencingIdMap) const +{ + std::set referencingIds; + + std::set processingIds = ids; + std::set processedIds; + + while (!processingIds.empty()) + { + std::set tempIds = processingIds; + utility::append(processedIds, processingIds); + processingIds.clear(); + + for (Id id: tempIds) { - for (Id id: working) + utility::append(referencingIds, idToReferencingIdMap[id]); + for (Id referencingId: idToReferencingIdMap[id]) { - auto it = fileIdToDependingFileIds.find(id); - if (it != fileIdToDependingFileIds.end()) + if (processedIds.find(referencingId) == processedIds.end()) { - for (Id dependingFileNodeId: it->second) - { - bool inserted = dependingFileNodeIds.insert(dependingFileNodeId).second; - if (inserted) - { - tempWorking.insert(dependingFileNodeId); - } - } + processingIds.insert(referencingId); } } - working = tempWorking; - tempWorking.clear(); } } - std::set dependingFilePaths; - for (Id id: dependingFileNodeIds) + return referencingIds; + +} + +std::set PersistentStorage::getReferencedByIncludes(const std::set& filePaths) +{ + std::set ids = getReferenced(getFileNodeIds(filePaths), getFileIdToIncludingFileIdMap()); + + std::set paths; + for (Id id: ids) { - dependingFilePaths.insert(getFileNodePath(id)); + paths.insert(getFileNodePath(id)); } - return dependingFilePaths; + return paths; +} + +std::set PersistentStorage::getReferencedByImports(const std::set& filePaths) +{ + std::set ids = getReferenced(getFileNodeIds(filePaths), getFileIdToImportingFileIdMap()); + + std::set paths; + for (Id id: ids) + { + paths.insert(getFileNodePath(id)); + } + + return paths; +} + +std::set PersistentStorage::getReferencingByIncludes(const std::set& filePaths) +{ + std::set ids = getReferencing(getFileNodeIds(filePaths), getFileIdToIncludingFileIdMap()); + + std::set paths; + for (Id id: ids) + { + paths.insert(getFileNodePath(id)); + } + + return paths; +} + +std::set PersistentStorage::getReferencingByImports(const std::set& filePaths) +{ + std::set ids = getReferencing(getFileNodeIds(filePaths), getFileIdToImportingFileIdMap()); + + std::set paths; + for (Id id: ids) + { + paths.insert(getFileNodePath(id)); + } + + return paths; } Id PersistentStorage::getLastVisibleParentNodeId(const Id nodeId) const diff --git a/src/lib/data/PersistentStorage.h b/src/lib/data/PersistentStorage.h index 081a0d10..7befaaa2 100644 --- a/src/lib/data/PersistentStorage.h +++ b/src/lib/data/PersistentStorage.h @@ -65,7 +65,8 @@ public: void clear(); void clearCaches(); - std::set getDependingFilePaths(const std::set& filePaths); + std::set getReferenced(const std::set& filePaths); + std::set getReferencing(const std::set& filePaths); void clearFileElements(const std::vector& filePaths); @@ -129,10 +130,20 @@ public: private: Id getFileNodeId(const FilePath& filePath) const; + std::vector getFileNodeIds(const std::vector& filePaths) const; + std::set getFileNodeIds(const std::set& filePaths) const; FilePath getFileNodePath(Id fileId) const; - std::set getDependingFilePathsForIncludes(const std::set& filePaths); - std::set getDependingFilePathsForImports(const std::set& filePaths); + std::unordered_map> getFileIdToIncludingFileIdMap() const; + std::unordered_map> getFileIdToImportingFileIdMap() const; + std::set getReferenced(const std::set& filePaths, std::unordered_map> idToReferencingIdMap) const; + std::set getReferencing(const std::set& filePaths, std::unordered_map> idToReferencingIdMap) const; + + std::set getReferencedByIncludes(const std::set& filePaths); + std::set getReferencedByImports(const std::set& filePaths); + + std::set getReferencingByIncludes(const std::set& filePaths); + std::set getReferencingByImports(const std::set& filePaths); Id getLastVisibleParentNodeId(const Id nodeId) const; std::vector getAllChildNodeIds(const Id nodeId) const; diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index efeeb774..926dfec0 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -489,12 +489,17 @@ std::vector SqliteStorage::getEdgesBySourceType(Id sourceId, int ty return doGetAll("WHERE source_node_id == " + std::to_string(sourceId) + " AND type == " + std::to_string(type)); } +std::vector SqliteStorage::getEdgesBySourcesType(const std::vector& sourceIds, int type) const +{ + return doGetAll("WHERE source_node_id IN (" + utility::join(utility::toStrings(sourceIds), ',') + ") AND type == " + std::to_string(type)); +} + std::vector SqliteStorage::getEdgesByTargetType(Id targetId, int type) const { return doGetAll("WHERE target_node_id == " + std::to_string(targetId) + " AND type == " + std::to_string(type)); } -std::vector SqliteStorage::getEdgesByTargetType(const std::vector& targetIds, int type) const +std::vector SqliteStorage::getEdgesByTargetsType(const std::vector& targetIds, int type) const { return doGetAll("WHERE target_node_id IN (" + utility::join(utility::toStrings(targetIds), ',') + ") AND type == " + std::to_string(type)); } diff --git a/src/lib/data/SqliteStorage.h b/src/lib/data/SqliteStorage.h index 36cb5552..48182bc2 100644 --- a/src/lib/data/SqliteStorage.h +++ b/src/lib/data/SqliteStorage.h @@ -87,8 +87,9 @@ public: std::vector getEdgesByType(int type) const; std::vector getEdgesBySourceType(Id sourceId, int type) const; + std::vector getEdgesBySourcesType(const std::vector& sourceIds, int type) const; std::vector getEdgesByTargetType(Id targetId, int type) const; - std::vector getEdgesByTargetType(const std::vector& targetIds, int type) const; + std::vector getEdgesByTargetsType(const std::vector& targetIds, int type) const; StorageNode getNodeBySerializedName(const std::string& serializedName) const; diff --git a/src/lib/utility/utility.h b/src/lib/utility/utility.h index f128e4b7..3fac358c 100644 --- a/src/lib/utility/utility.h +++ b/src/lib/utility/utility.h @@ -26,6 +26,9 @@ namespace utility template std::vector concat(const std::vector& a, const std::vector& b); + template + std::set concat(const std::set& a, const std::set& b); + template void append(std::vector& a, const std::vector& b); @@ -87,6 +90,15 @@ std::vector utility::concat(const std::vector& a, const std::vector& b) return r; } +template +std::set utility::concat(const std::set& a, const std::set& b) +{ + std::set r; + append(r, a); + append(r, b); + return r; +} + template void utility::append(std::vector& a, const std::vector& b) {