ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: shallow python indexing (issue #725)

Browse Source
This commit is contained in:
mlangkabel
2019-11-06 19:50:47 +01:00
parent 73f3be72e1
commit 9d556599ab
37 changed files with 284 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/component/view/DialogView.cpp
+2 -2
View File
@@ -42,7 +42,7 @@ void DialogView::hideProgressDialog()
}
void DialogView::startIndexingDialog(
Project* project, const std::vector<RefreshMode>& enabledModes, const RefreshMode initialMode,
Project* project, const std::vector<RefreshMode>& enabledModes, const RefreshMode initialMode, bool enabledShallowOption, bool shallow,
std::function<void(const RefreshInfo& info)> onStartIndexing, std::function<void()> onCancelIndexing)
{
}
@@ -59,7 +59,7 @@ void DialogView::updateCustomIndexingDialog(
DatabasePolicy DialogView::finishedIndexingDialog(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, ErrorCountInfo errorInfo, bool interrupted)
float time, ErrorCountInfo errorInfo, bool interrupted, bool shallow)
{
return DATABASE_POLICY_KEEP; // used in non-gui mode
}
src/lib/component/view/DialogView.h
+3 -2
View File
@@ -15,6 +15,7 @@ enum DatabasePolicy
{
DATABASE_POLICY_KEEP,
DATABASE_POLICY_DISCARD,
DATABASE_POLICY_REFRESH,
DATABASE_POLICY_UNKNOWN
};
@@ -45,7 +46,7 @@ public:
virtual void hideProgressDialog();
virtual void startIndexingDialog(
Project* project, const std::vector<RefreshMode>& enabledModes, const RefreshMode initialMode,
Project* project, const std::vector<RefreshMode>& enabledModes, const RefreshMode initialMode, bool enabledShallowOption, bool shallow,
std::function<void(const RefreshInfo& info)> onStartIndexing, std::function<void()> onCancelIndexing);
virtual void updateIndexingDialog(
size_t startedFileCount, size_t finishedFileCount, size_t totalFileCount, const std::vector<FilePath>& sourcePaths);
@@ -53,7 +54,7 @@ public:
size_t startedFileCount, size_t finishedFileCount, size_t totalFileCount, const std::vector<FilePath>& sourcePaths);
virtual DatabasePolicy finishedIndexingDialog(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, ErrorCountInfo errorInfo, bool interrupted);
float time, ErrorCountInfo errorInfo, bool interrupted, bool shallow);
int confirm(const std::wstring& message);
virtual int confirm(const std::wstring& message, const std::vector<std::wstring>& options);
src/lib/data/TaskFinishParsing.cpp
+10 -1
View File
@@ -61,6 +61,9 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboa
bool interruptedIndexing = false;
blackboard->get("interrupted_indexing", interruptedIndexing);
bool shallowIndexing = false;
blackboard->get("shallow_indexing", shallowIndexing);
ErrorCountInfo errorInfo = m_storage->getErrorCount();
std::wstring status;
@@ -82,7 +85,8 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboa
stats.fileCount,
time,
errorInfo,
interruptedIndexing
interruptedIndexing,
shallowIndexing
);
MessageIndexingStatus(false).dispatch();
@@ -95,6 +99,11 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboa
{
blackboard->set("discard_database", true);
}
else if (policy == DATABASE_POLICY_REFRESH)
{
blackboard->set("keep_database", true);
blackboard->set("refresh_database", true);
}
return STATE_SUCCESS;
}
src/lib/data/indexer/TaskExecuteCustomCommands.cpp
+19 -14
View File
@@ -260,7 +260,7 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
std::vector<Id> unsolvedLocationIds;
for (const StorageSourceLocation location : storage.getStorageSourceLocations())
{
if (intToLocationType(location.type) == LOCATION_UNSOLVED)
if (intToLocationType(location.type) == LOCATION_UNSOLVED) // FIXME: this doesn't catch unsolved qualifiers -> convert Qualifier location type to qualifier edge
{
unsolvedLocationIds.push_back(location.id);
}
@@ -286,9 +286,9 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
storage.setMode(SqliteIndexStorage::STORAGE_MODE_READ);
std::vector<DataToInsert> dataToInsert;
std::set<Id> elementsToDelete;
std::vector<StorageOccurrence> occurrencesToDelete;
locationCollection->forEachSourceLocationFile(
[&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr<SourceLocationFile> locationFile)
[&nodeNameToStorageNodes, &storage, &dataToInsert, &occurrencesToDelete](std::shared_ptr<SourceLocationFile> locationFile)
{
const FilePath filePath = locationFile->getFilePath();
if (filePath.empty())
@@ -305,7 +305,7 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
if (textAccess)
{
locationFile->forEachStartSourceLocation(
[textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc)
[textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &occurrencesToDelete](const SourceLocation* startLoc)
{
if (!startLoc)
{
@@ -319,23 +319,20 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
const std::wstring token = utility::decodeFromUtf8(textAccess->getLine(startLoc->getLineNumber()).substr(startLoc->getColumnNumber() - 1, endLoc->getColumnNumber() - startLoc->getColumnNumber() + 1));
for (const Id tokenId : startLoc->getTokenIds())
for (const Id elementId : startLoc->getTokenIds())
{
const StorageEdge edge = storage.getEdgeById(tokenId);
const StorageEdge edge = storage.getEdgeById(elementId);
if (edge.id != 0)
{
for (const StorageNode& targetNode : nodeNameToStorageNodes[token])
{
if (Edge::intToType(edge.type) == Edge::EDGE_CALL &&
(
NodeType::intToType(targetNode.type) != NodeType::NODE_FUNCTION ||
NodeType::intToType(targetNode.type) != NodeType::NODE_METHOD
)
){
if (Edge::intToType(edge.type) == Edge::EDGE_INHERITANCE &&
NodeType::intToType(targetNode.type) != NodeType::NODE_CLASS)
{
continue;
}
dataToInsert.push_back({ StorageEdgeData(edge.type, edge.sourceNodeId, targetNode.id) , startLoc->getLocationId() });
elementsToDelete.insert(edge.id);
occurrencesToDelete.push_back(StorageOccurrence(edge.id, startLoc->getLocationId()));
}
}
}
@@ -364,7 +361,15 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
storage.addOccurrence(StorageOccurrence(ambiguousEdgeIds[i], dataToInsert[i].sourceLocationId));
}
storage.setMode(SqliteIndexStorage::STORAGE_MODE_CLEAR);
storage.removeElements(utility::toVector(elementsToDelete));
storage.removeOccurrences(occurrencesToDelete);
std::set<Id> edgeIds;
for (const StorageOccurrence& occurrence : occurrencesToDelete)
{
edgeIds.insert(occurrence.elementId);
}
storage.removeElementsWithoutOccurrences(utility::toVector(edgeIds));
storage.finishInjection();
LOG_INFO("Finished Python post processing.");
}
src/lib/data/storage/PersistentStorage.cpp
+15
View File
@@ -165,6 +165,21 @@ void PersistentStorage::removeElements(const std::vector<Id>& ids)
m_sqliteIndexStorage.removeElements(ids);
}
void PersistentStorage::removeOccurrence(const StorageOccurrence& occurrence)
{
m_sqliteIndexStorage.removeOccurrence(occurrence);
}
void PersistentStorage::removeOccurrences(const std::vector<StorageOccurrence>& occurrences)
{
m_sqliteIndexStorage.removeOccurrences(occurrences);
}
void PersistentStorage::removeElementsWithoutOccurrences(const std::vector<Id>& elementIds)
{
m_sqliteIndexStorage.removeElementsWithoutOccurrences(elementIds);
}
const std::vector<StorageNode>& PersistentStorage::getStorageNodes() const
{
return m_storageData.nodes = m_sqliteIndexStorage.getAll<StorageNode>();
src/lib/data/storage/PersistentStorage.h
+3
View File
@@ -40,6 +40,9 @@ public:
void removeElement(const Id id);
void removeElements(const std::vector<Id>& ids);
void removeOccurrence(const StorageOccurrence& occurrence);
void removeOccurrences(const std::vector<StorageOccurrence>& occurrences);
void removeElementsWithoutOccurrences(const std::vector<Id>& elementIds);
const std::vector<StorageNode>& getStorageNodes() const override;
const std::vector<StorageFile>& getStorageFiles() const override;
src/lib/data/storage/sqlite/SqliteIndexStorage.cpp
+22
View File
@@ -477,6 +477,28 @@ void SqliteIndexStorage::removeElements(const std::vector<Id>& ids)
);
}
void SqliteIndexStorage::removeOccurrence(const StorageOccurrence& occurrence)
{
executeStatement(
"DELETE FROM occurrence WHERE element_id = " + std::to_string(occurrence.elementId) + " AND source_location_id = " + std::to_string(occurrence.sourceLocationId) + ";"
);
}
void SqliteIndexStorage::removeOccurrences(const std::vector<StorageOccurrence>& occurrences)
{
for (const StorageOccurrence& occurrence : occurrences)
{
removeOccurrence(occurrence);
}
}
void SqliteIndexStorage::removeElementsWithoutOccurrences(const std::vector<Id>& elementIds)
{
executeStatement(
"DELETE FROM element WHERE id IN (" + utility::join(utility::toStrings(elementIds), ',') + ") AND id NOT IN (SELECT element_id FROM occurrence);"
);
}
void SqliteIndexStorage::removeElementsWithLocationInFiles(
const std::vector<Id>& fileIds, std::function<void(int)> updateStatusCallback)
{
src/lib/data/storage/sqlite/SqliteIndexStorage.h
+3
View File
@@ -72,6 +72,9 @@ public:
void removeElement(Id id);
void removeElements(const std::vector<Id>& ids);
void removeOccurrence(const StorageOccurrence& occurrence);
void removeOccurrences(const std::vector<StorageOccurrence>& occurrences);
void removeElementsWithoutOccurrences(const std::vector<Id>& elementIds);
void removeElementsWithLocationInFiles(const std::vector<Id>& fileIds, std::function<void(int)> updateStatusCallback);
void removeAllErrors();
src/lib/project/Project.cpp
+45 -10
View File
@@ -26,6 +26,7 @@
#include "FileSystem.h"
#include "MessageErrorCountClear.h"
#include "MessageIndexingFinished.h"
#include "MessageIndexingShowDialog.h"
#include "MessageIndexingStarted.h"
#include "MessageIndexingStatus.h"
#include "MessageRefresh.h"
@@ -73,14 +74,14 @@ bool Project::isLoaded() const
{
switch (m_state)
{
case PROJECT_STATE_EMPTY:
case PROJECT_STATE_LOADED:
case PROJECT_STATE_OUTDATED:
case PROJECT_STATE_NEEDS_MIGRATION:
return true;
case PROJECT_STATE_EMPTY:
case PROJECT_STATE_LOADED:
case PROJECT_STATE_OUTDATED:
case PROJECT_STATE_NEEDS_MIGRATION:
return true;
default:
break;
default:
break;
}
return false;
@@ -354,6 +355,18 @@ void Project::refresh(RefreshMode refreshMode, std::shared_ptr<DialogView> dialo
refreshMode = REFRESH_UPDATED_FILES;
}
bool allowsShallowIndexing = false;
for (const std::shared_ptr<SourceGroup>& sourceGroup : m_sourceGroups)
{
if (sourceGroup->getStatus() == SOURCE_GROUP_STATUS_ENABLED && sourceGroup->allowsShallowIndexing())
{
allowsShallowIndexing = true;
break;
}
}
const bool useShallowIndexing = allowsShallowIndexing && (!isLoaded() || m_state == PROJECT_STATE_EMPTY);
if (m_hasGUI)
{
std::vector<RefreshMode> enabledModes = { REFRESH_ALL_FILES };
@@ -362,7 +375,7 @@ void Project::refresh(RefreshMode refreshMode, std::shared_ptr<DialogView> dialo
enabledModes.insert(enabledModes.end(), { REFRESH_UPDATED_FILES, REFRESH_UPDATED_AND_INCOMPLETE_FILES });
}
dialogView->startIndexingDialog(this, enabledModes, refreshMode,
dialogView->startIndexingDialog(this, enabledModes, refreshMode, allowsShallowIndexing, useShallowIndexing,
[this, dialogView](const RefreshInfo& info)
{
buildIndex(info, dialogView);
@@ -453,7 +466,9 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
}
else
{
const bool shallow = info.shallow;
info = getRefreshInfo(REFRESH_ALL_FILES);
info.shallow = shallow;
}
}
@@ -476,6 +491,7 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
if (info.mode != REFRESH_ALL_FILES)
{
// store the indexed data into the temp db but keep the current state to allow browsing while indexing
FileSystem::copyFile(indexDbFilePath, tempIndexDbFilePath);
}
@@ -507,17 +523,18 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
if (sourceGroup->getType() == SOURCE_GROUP_CUSTOM_COMMAND ||
sourceGroup->getType() == SOURCE_GROUP_PYTHON_EMPTY)
{
customIndexerCommandProvider->addProvider(sourceGroup->getIndexerCommandProvider(info.filesToIndex));
customIndexerCommandProvider->addProvider(sourceGroup->getIndexerCommandProvider(info));
}
else
{
indexerCommandProvider->addProvider(sourceGroup->getIndexerCommandProvider(info.filesToIndex));
indexerCommandProvider->addProvider(sourceGroup->getIndexerCommandProvider(info));
}
}
}
size_t sourceFileCount = indexerCommandProvider->size() + customIndexerCommandProvider->size();
taskSequential->addTask(std::make_shared<TaskSetValue<bool>>("shallow_indexing", info.shallow));
taskSequential->addTask(std::make_shared<TaskSetValue<int>>("source_file_count", sourceFileCount));
taskSequential->addTask(std::make_shared<TaskSetValue<int>>("indexed_source_file_count", 0));
taskSequential->addTask(std::make_shared<TaskSetValue<bool>>("interrupted_indexing", false));
@@ -670,6 +687,24 @@ void Project::buildIndex(RefreshInfo info, std::shared_ptr<DialogView> dialogVie
MessageIndexingFinished().dispatch();
}));
taskSequential->addTask(std::make_shared<TaskGroupSelector>()->addChildTasks(
std::make_shared<TaskGroupSequence>()->addChildTasks(
std::make_shared<TaskFindKeyOnBlackboard>("refresh_database"),
std::make_shared<TaskLambda>([dialogView, this]() {
Task::dispatch(TabId::app(), std::make_shared<TaskLambda>([dialogView, this]() {
MessageIndexingShowDialog().dispatch();
MessageRefresh().refreshAll().dispatch();
}));
})
),
std::make_shared<TaskGroupSequence>()->addChildTasks(
std::make_shared<TaskLambda>([this]() {
Task::dispatch(TabId::app(), std::make_shared<TaskLambda>([this]() {
}));
})
)
));
taskSequential->setIsBackgroundTask(true);
Task::dispatch(TabId::app(), taskSequential);
src/lib/project/RefreshInfo.h
+1
View File
@@ -20,6 +20,7 @@ struct RefreshInfo
std::set<FilePath> nonIndexedFilesToClear;
RefreshMode mode = REFRESH_NONE;
bool shallow = true;
};
#endif // REFRESH_INFO_H
src/lib/project/SourceGroup.cpp
+7 -2
View File
@@ -7,9 +7,9 @@
#include "SourceGroupSettings.h"
#include "TaskLambda.h"
std::shared_ptr<IndexerCommandProvider> SourceGroup::getIndexerCommandProvider(const std::set<FilePath>& filesToIndex) const
std::shared_ptr<IndexerCommandProvider> SourceGroup::getIndexerCommandProvider(const RefreshInfo& info) const
{
return std::make_shared<MemoryIndexerCommandProvider>(getIndexerCommands(filesToIndex));
return std::make_shared<MemoryIndexerCommandProvider>(getIndexerCommands(info));
}
std::shared_ptr<Task> SourceGroup::getPreIndexTask(
@@ -43,6 +43,11 @@ bool SourceGroup::allowsPartialClearing() const
return true;
}
bool SourceGroup::allowsShallowIndexing() const
{
return false;
}
std::set<FilePath> SourceGroup::filterToContainedSourceFilePath(const std::set<FilePath>& sourceFilePaths) const
{
std::set<FilePath> filteredSourceFilePaths;
src/lib/project/SourceGroup.h
+5 -2
View File
@@ -18,6 +18,8 @@ class SourceGroupSettings;
class StorageProvider;
class Task;
struct RefreshInfo;
class SourceGroup
{
public:
@@ -25,11 +27,12 @@ public:
virtual bool prepareIndexing();
virtual bool allowsPartialClearing() const;
virtual bool allowsShallowIndexing() const;
virtual std::set<FilePath> filterToContainedFilePaths(const std::set<FilePath>& filePaths) const = 0;
virtual std::set<FilePath> getAllSourceFilePaths() const = 0;
virtual std::shared_ptr<IndexerCommandProvider> getIndexerCommandProvider(const std::set<FilePath>& filesToIndex) const;
virtual std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(const std::set<FilePath>& filesToIndex) const = 0;
virtual std::shared_ptr<IndexerCommandProvider> getIndexerCommandProvider(const RefreshInfo& info) const;
virtual std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(const RefreshInfo& info) const = 0;
virtual std::shared_ptr<Task> getPreIndexTask(
std::shared_ptr<StorageProvider> storageProvider, std::shared_ptr<DialogView> dialogView) const;
src/lib/project/SourceGroupCustomCommand.cpp
+3 -2
View File
@@ -3,6 +3,7 @@
#include "FileManager.h"
#include "IndexerCommandCustom.h"
#include "ProjectSettings.h"
#include "RefreshInfo.h"
#include "SourceGroupSettingsCustomCommand.h"
#include "SqliteIndexStorage.h"
#include "utility.h"
@@ -38,7 +39,7 @@ std::set<FilePath> SourceGroupCustomCommand::getAllSourceFilePaths() const
return fileManager.getAllSourceFilePaths();
}
std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexerCommands(const std::set<FilePath>& filesToIndex) const
std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexerCommands(const RefreshInfo& info) const
{
const std::wstring customCommand = m_settings->getCustomCommand();
const bool runInParallel = m_settings->getRunInParallel();
@@ -46,7 +47,7 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexe
std::vector<std::shared_ptr<IndexerCommand>> indexerCommands;
for (const FilePath& sourcePath: getAllSourceFilePaths())
{
if (filesToIndex.find(sourcePath) != filesToIndex.end())
if (info.filesToIndex.find(sourcePath) != info.filesToIndex.end())
{
indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(
customCommand,
src/lib/project/SourceGroupCustomCommand.h
+1 -1
View File
@@ -18,7 +18,7 @@ public:
std::set<FilePath> filterToContainedFilePaths(const std::set<FilePath>& filePaths) const override;
std::set<FilePath> getAllSourceFilePaths() const override;
std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(const std::set<FilePath>& filesToIndex) const override;
std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(const RefreshInfo& info) const override;
private:
std::shared_ptr<SourceGroupSettings> getSourceGroupSettings() override;