ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: clearing unused includes

Browse Source 
* refresh also clears referenced (included or imported) files if they are not part of the project's source files and are not referenced by any other unchanged file anymore.
This commit is contained in:
malte_langkabel
2017-01-23 10:36:42 +01:00
parent 0e5f65b475
commit 6ea248b500
6 changed files with 192 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/Project.cpp
+32 -11
View File
@@ -247,31 +247,52 @@ bool Project::requestIndex(bool forceRefresh, bool needsFullRefresh)
return false;
}
FileManager::FileSets fileSets = m_fileManager.fetchFilePaths(m_storage->getInfoOnAllFiles());
const FileManager::FileSets fileSets = m_fileManager.fetchFilePaths(m_storage->getInfoOnAllFiles());
std::set<FilePath> filesToClean;
std::set<FilePath> filesToIndex;
if (!needsFullRefresh)
{
std::set<FilePath> dependingFilePaths;
utility::append(dependingFilePaths, m_storage->getDependingFilePaths(fileSets.updatedFiles));
utility::append(dependingFilePaths, m_storage->getDependingFilePaths(fileSets.removedFiles));
utility::append(filesToClean, fileSets.removedFiles);
utility::append(filesToClean, fileSets.updatedFiles);
utility::append(filesToIndex, fileSets.addedFiles);
utility::append(filesToIndex, fileSets.updatedFiles);
for (const FilePath& path : dependingFilePaths)
// handle referencing paths
const std::set<FilePath> referencingFilePaths = m_storage->getReferencing(utility::concat(
fileSets.updatedFiles, fileSets.removedFiles
));
for (const FilePath& path : referencingFilePaths)
{
if (fileSets.removedFiles.find(path) == fileSets.removedFiles.end())
{
fileSets.updatedFiles.insert(path);
filesToClean.insert(path);
filesToIndex.insert(path);
}
}
utility::append(filesToClean, fileSets.removedFiles);
utility::append(filesToClean, fileSets.updatedFiles);
utility::append(filesToClean, dependingFilePaths);
// handle referenced paths
std::set<FilePath> staticSourceFiles = fileSets.allFiles;
for (const FilePath& path : fileSets.updatedFiles)
{
staticSourceFiles.erase(path);
}
const std::set<FilePath> staticReferencedFilePaths = m_storage->getReferenced(staticSourceFiles);
const std::set<FilePath> dynamicReferencedFilePaths = m_storage->getReferenced(utility::concat(
fileSets.updatedFiles, fileSets.removedFiles
));
utility::append(filesToIndex, fileSets.addedFiles);
utility::append(filesToIndex, fileSets.updatedFiles);
for (const FilePath& path : dynamicReferencedFilePaths)
{
if (staticReferencedFilePaths.find(path) == staticReferencedFilePaths.end() &&
staticSourceFiles.find(path) == staticSourceFiles.end())
{
// file may not be referenced anymore and will be reindexed if still needed
filesToClean.insert(path);
}
}
}
bool fullRefresh = forceRefresh | needsFullRefresh;
src/lib/data/PersistentStorage.cpp
+126 -73
View File
@@ -300,32 +300,35 @@ void PersistentStorage::clearCaches()
m_fullTextSearchIndex.clear();
}
std::set<FilePath> PersistentStorage::getDependingFilePaths(const std::set<FilePath>& filePaths)
std::set<FilePath> PersistentStorage::getReferenced(const std::set<FilePath>& filePaths)
{
TRACE();
std::set<FilePath> dependingFilePaths;
std::set<FilePath> referenced;
std::set<FilePath> dependingFilePathsForIncludes = getDependingFilePathsForIncludes(filePaths);
dependingFilePaths.insert(dependingFilePathsForIncludes.begin(), dependingFilePathsForIncludes.end());
utility::append(referenced, getReferencedByIncludes(filePaths));
utility::append(referenced, getReferencedByImports(filePaths));
std::set<FilePath> dependingFilePathsForImports = getDependingFilePathsForImports(filePaths);
dependingFilePaths.insert(dependingFilePathsForImports.begin(), dependingFilePathsForImports.end());
return referenced;
}
return dependingFilePaths;
std::set<FilePath> PersistentStorage::getReferencing(const std::set<FilePath>& filePaths)
{
TRACE();
std::set<FilePath> referencing;
utility::append(referencing, getReferencingByIncludes(filePaths));
utility::append(referencing, getReferencingByImports(filePaths));
return referencing;
}
void PersistentStorage::clearFileElements(const std::vector<FilePath>& filePaths)
{
TRACE();
std::vector<Id> fileNodeIds;
const std::vector<Id> fileNodeIds = getFileNodeIds(filePaths);
for (const FilePath& path : filePaths)
{
fileNodeIds.push_back(getFileNodeId(path));
}
if (fileNodeIds.size())
if (!fileNodeIds.empty())
{
m_sqliteStorage.removeElementsWithLocationInFiles(fileNodeIds);
m_sqliteStorage.removeElements(fileNodeIds);
@@ -1232,6 +1235,26 @@ Id PersistentStorage::getFileNodeId(const FilePath& filePath) const
return 0;
}
std::vector<Id> PersistentStorage::getFileNodeIds(const std::vector<FilePath>& filePaths) const
{
std::vector<Id> ids;
for (const FilePath& path : filePaths)
{
ids.push_back(getFileNodeId(path));
}
return ids;
}
std::set<Id> PersistentStorage::getFileNodeIds(const std::set<FilePath>& filePaths) const
{
std::set<Id> ids;
for (const FilePath& path : filePaths)
{
ids.insert(getFileNodeId(path));
}
return ids;
}
FilePath PersistentStorage::getFileNodePath(Id fileId) const
{
if (fileId == 0)
@@ -1250,42 +1273,19 @@ FilePath PersistentStorage::getFileNodePath(Id fileId) const
return FilePath();
}
std::set<FilePath> PersistentStorage::getDependingFilePathsForIncludes(const std::set<FilePath>& filePaths)
std::unordered_map<Id, std::set<Id>> PersistentStorage::getFileIdToIncludingFileIdMap() const
{
std::set<FilePath> dependingFilePaths;
std::set<Id> processedFileNodeIds;
std::vector<Id> fileNodeIds;
for (const FilePath& filePath : filePaths)
std::unordered_map<Id, std::set<Id>> fileIdToIncludingFileIdMap;
for (const StorageEdge& includeEdge : m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_INCLUDE)))
{
fileNodeIds.push_back(getFileNodeId(filePath));
fileIdToIncludingFileIdMap[includeEdge.targetNodeId].insert(includeEdge.sourceNodeId);
}
while (fileNodeIds.size())
{
std::vector<StorageEdge> incomingEdges =
m_sqliteStorage.getEdgesByTargetType(fileNodeIds, Edge::typeToInt(Edge::EDGE_INCLUDE));
processedFileNodeIds.insert(fileNodeIds.begin(), fileNodeIds.end());
fileNodeIds.clear();
for (const StorageEdge& incomingEdge: incomingEdges)
{
dependingFilePaths.insert(getFileNodePath(incomingEdge.sourceNodeId));
if (processedFileNodeIds.find(incomingEdge.sourceNodeId) == processedFileNodeIds.end())
{
fileNodeIds.push_back(incomingEdge.sourceNodeId);
}
}
}
return dependingFilePaths;
return fileIdToIncludingFileIdMap;
}
std::set<FilePath> PersistentStorage::getDependingFilePathsForImports(const std::set<FilePath>& filePaths)
std::unordered_map<Id, std::set<Id>> PersistentStorage::getFileIdToImportingFileIdMap() const
{
std::unordered_map<Id, std::set<Id>> fileIdToDependingFileIds;
std::unordered_map<Id, std::set<Id>> fileIdToImportingFileIdMap;
{
std::vector<Id> importedElementIds;
std::map<Id, std::set<Id>> elementIdToImportingFileIds;
@@ -1296,11 +1296,6 @@ std::set<FilePath> PersistentStorage::getDependingFilePathsForImports(const std:
elementIdToImportingFileIds[importEdge.targetNodeId].insert(importEdge.sourceNodeId);
}
if (!importedElementIds.size())
{
return std::set<FilePath>();
}
std::unordered_map<Id, Id> importedElementIdToFileNodeId;
{
std::vector<Id> importedSourceLocationIds;
@@ -1326,49 +1321,107 @@ std::set<FilePath> PersistentStorage::getDependingFilePathsForImports(const std:
auto importedFileIt = importedElementIdToFileNodeId.find(it.first);
if (importedFileIt != importedElementIdToFileNodeId.end())
{
fileIdToDependingFileIds[importedFileIt->second].insert(it.second.begin(), it.second.end());
fileIdToImportingFileIdMap[importedFileIt->second].insert(it.second.begin(), it.second.end());
}
}
}
return fileIdToImportingFileIdMap;
}
std::set<Id> dependingFileNodeIds;
std::set<Id> PersistentStorage::getReferenced(const std::set<Id>& ids, std::unordered_map<Id, std::set<Id>> idToReferencingIdMap) const
{
std::unordered_map<Id, std::set<Id>> idToReferencedIdMap;
for (auto it: idToReferencingIdMap)
{
std::set<Id> working;
for (const FilePath& filePath: filePaths)
for (Id referencingId: it.second)
{
working.insert(getFileNodeId(filePath));
idToReferencedIdMap[referencingId].insert(it.first);
}
}
std::set<Id> tempWorking;
while (working.size() > 0)
return getReferencing(ids, idToReferencedIdMap);
}
std::set<Id> PersistentStorage::getReferencing(const std::set<Id>& ids, std::unordered_map<Id, std::set<Id>> idToReferencingIdMap) const
{
std::set<Id> referencingIds;
std::set<Id> processingIds = ids;
std::set<Id> processedIds;
while (!processingIds.empty())
{
std::set<Id> tempIds = processingIds;
utility::append(processedIds, processingIds);
processingIds.clear();
for (Id id: tempIds)
{
for (Id id: working)
utility::append(referencingIds, idToReferencingIdMap[id]);
for (Id referencingId: idToReferencingIdMap[id])
{
auto it = fileIdToDependingFileIds.find(id);
if (it != fileIdToDependingFileIds.end())
if (processedIds.find(referencingId) == processedIds.end())
{
for (Id dependingFileNodeId: it->second)
{
bool inserted = dependingFileNodeIds.insert(dependingFileNodeId).second;
if (inserted)
{
tempWorking.insert(dependingFileNodeId);
}
}
processingIds.insert(referencingId);
}
}
working = tempWorking;
tempWorking.clear();
}
}
std::set<FilePath> dependingFilePaths;
for (Id id: dependingFileNodeIds)
return referencingIds;
}
std::set<FilePath> PersistentStorage::getReferencedByIncludes(const std::set<FilePath>& filePaths)
{
std::set<Id> ids = getReferenced(getFileNodeIds(filePaths), getFileIdToIncludingFileIdMap());
std::set<FilePath> paths;
for (Id id: ids)
{
dependingFilePaths.insert(getFileNodePath(id));
paths.insert(getFileNodePath(id));
}
return dependingFilePaths;
return paths;
}
std::set<FilePath> PersistentStorage::getReferencedByImports(const std::set<FilePath>& filePaths)
{
std::set<Id> ids = getReferenced(getFileNodeIds(filePaths), getFileIdToImportingFileIdMap());
std::set<FilePath> paths;
for (Id id: ids)
{
paths.insert(getFileNodePath(id));
}
return paths;
}
std::set<FilePath> PersistentStorage::getReferencingByIncludes(const std::set<FilePath>& filePaths)
{
std::set<Id> ids = getReferencing(getFileNodeIds(filePaths), getFileIdToIncludingFileIdMap());
std::set<FilePath> paths;
for (Id id: ids)
{
paths.insert(getFileNodePath(id));
}
return paths;
}
std::set<FilePath> PersistentStorage::getReferencingByImports(const std::set<FilePath>& filePaths)
{
std::set<Id> ids = getReferencing(getFileNodeIds(filePaths), getFileIdToImportingFileIdMap());
std::set<FilePath> paths;
for (Id id: ids)
{
paths.insert(getFileNodePath(id));
}
return paths;
}
Id PersistentStorage::getLastVisibleParentNodeId(const Id nodeId) const
src/lib/data/PersistentStorage.h
+14 -3
View File
@@ -65,7 +65,8 @@ public:
void clear();
void clearCaches();
std::set<FilePath> getDependingFilePaths(const std::set<FilePath>& filePaths);
std::set<FilePath> getReferenced(const std::set<FilePath>& filePaths);
std::set<FilePath> getReferencing(const std::set<FilePath>& filePaths);
void clearFileElements(const std::vector<FilePath>& filePaths);
@@ -129,10 +130,20 @@ public:
private:
Id getFileNodeId(const FilePath& filePath) const;
std::vector<Id> getFileNodeIds(const std::vector<FilePath>& filePaths) const;
std::set<Id> getFileNodeIds(const std::set<FilePath>& filePaths) const;
FilePath getFileNodePath(Id fileId) const;
std::set<FilePath> getDependingFilePathsForIncludes(const std::set<FilePath>& filePaths);
std::set<FilePath> getDependingFilePathsForImports(const std::set<FilePath>& filePaths);
std::unordered_map<Id, std::set<Id>> getFileIdToIncludingFileIdMap() const;
std::unordered_map<Id, std::set<Id>> getFileIdToImportingFileIdMap() const;
std::set<Id> getReferenced(const std::set<Id>& filePaths, std::unordered_map<Id, std::set<Id>> idToReferencingIdMap) const;
std::set<Id> getReferencing(const std::set<Id>& filePaths, std::unordered_map<Id, std::set<Id>> idToReferencingIdMap) const;
std::set<FilePath> getReferencedByIncludes(const std::set<FilePath>& filePaths);
std::set<FilePath> getReferencedByImports(const std::set<FilePath>& filePaths);
std::set<FilePath> getReferencingByIncludes(const std::set<FilePath>& filePaths);
std::set<FilePath> getReferencingByImports(const std::set<FilePath>& filePaths);
Id getLastVisibleParentNodeId(const Id nodeId) const;
std::vector<Id> getAllChildNodeIds(const Id nodeId) const;
src/lib/data/SqliteStorage.cpp
+6 -1
View File
@@ -489,12 +489,17 @@ std::vector<StorageEdge> SqliteStorage::getEdgesBySourceType(Id sourceId, int ty
return doGetAll<StorageEdge>("WHERE source_node_id == " + std::to_string(sourceId) + " AND type == " + std::to_string(type));
}
std::vector<StorageEdge> SqliteStorage::getEdgesBySourcesType(const std::vector<Id>& sourceIds, int type) const
{
return doGetAll<StorageEdge>("WHERE source_node_id IN (" + utility::join(utility::toStrings(sourceIds), ',') + ") AND type == " + std::to_string(type));
}
std::vector<StorageEdge> SqliteStorage::getEdgesByTargetType(Id targetId, int type) const
{
return doGetAll<StorageEdge>("WHERE target_node_id == " + std::to_string(targetId) + " AND type == " + std::to_string(type));
}
std::vector<StorageEdge> SqliteStorage::getEdgesByTargetType(const std::vector<Id>& targetIds, int type) const
std::vector<StorageEdge> SqliteStorage::getEdgesByTargetsType(const std::vector<Id>& targetIds, int type) const
{
return doGetAll<StorageEdge>("WHERE target_node_id IN (" + utility::join(utility::toStrings(targetIds), ',') + ") AND type == " + std::to_string(type));
}
src/lib/data/SqliteStorage.h
+2 -1
View File
@@ -87,8 +87,9 @@ public:
std::vector<StorageEdge> getEdgesByType(int type) const;
std::vector<StorageEdge> getEdgesBySourceType(Id sourceId, int type) const;
std::vector<StorageEdge> getEdgesBySourcesType(const std::vector<Id>& sourceIds, int type) const;
std::vector<StorageEdge> getEdgesByTargetType(Id targetId, int type) const;
std::vector<StorageEdge> getEdgesByTargetType(const std::vector<Id>& targetIds, int type) const;
std::vector<StorageEdge> getEdgesByTargetsType(const std::vector<Id>& targetIds, int type) const;
StorageNode getNodeBySerializedName(const std::string& serializedName) const;
src/lib/utility/utility.h
+12
View File
@@ -26,6 +26,9 @@ namespace utility
template<typename T>
std::vector<T> concat(const std::vector<T>& a, const std::vector<T>& b);
template<typename T>
std::set<T> concat(const std::set<T>& a, const std::set<T>& b);
template<typename T>
void append(std::vector<T>& a, const std::vector<T>& b);
@@ -87,6 +90,15 @@ std::vector<T> utility::concat(const std::vector<T>& a, const std::vector<T>& b)
return r;
}
template<typename T>
std::set<T> utility::concat(const std::set<T>& a, const std::set<T>& b)
{
std::set<T> r;
append(r, a);
append(r, b);
return r;
}
template<typename T>
void utility::append(std::vector<T>& a, const std::vector<T>& b)
{