ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

<module>: fixed storage cache to accept local symbols names of python indexer

Browse Source
This commit is contained in:
mlangkabel
2019-05-21 17:10:55 +02:00
parent f74789e210
commit 0fb7fe719a
6 changed files with 46 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/TaskExecuteCustomCommands.cpp
+20 -8
View File
@@ -42,10 +42,10 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr<Blackboard> blackboard)
if (m_indexerCommandProvider)
{
while (!m_indexerCommandProvider->empty())
for (const FilePath& sourceFilePath : m_indexerCommandProvider->getAllSourceFilePaths())
{
if (std::shared_ptr<IndexerCommandCustom> indexerCommand =
std::dynamic_pointer_cast<IndexerCommandCustom>(m_indexerCommandProvider->consumeCommand()))
std::dynamic_pointer_cast<IndexerCommandCustom>(m_indexerCommandProvider->consumeCommandForSourceFilePath(sourceFilePath)))
{
if (m_targetDatabaseFilePath.empty())
{
@@ -79,7 +79,6 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard>
m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), {});
std::vector<std::shared_ptr<std::thread>> indexerThreads;
for (size_t i = 1 /*this method is counting as the first thread*/; i < m_indexerThreadCount; i++)
{
@@ -119,6 +118,8 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard>
if (m_hasPythonCommands && ApplicationSettings::getInstance()->getPythonPostProcessingEnabled())
{
targetStorage.clearCaches();
targetStorage.buildCaches();
runPythonPostProcessing(targetStorage);
}
}
@@ -286,13 +287,25 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
std::vector<DataToInsert> dataToInsert;
std::set<Id> elementsToDelete;
locationCollection->forEachSourceLocationFile([&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr<SourceLocationFile> locationFile)
locationCollection->forEachSourceLocationFile(
[&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr<SourceLocationFile> locationFile)
{
std::shared_ptr<TextAccess> textAccess = TextAccess::createFromFile(locationFile->getFilePath());
const FilePath filePath = locationFile->getFilePath();
if (filePath.empty())
{
return;
}
if (!filePath.exists())
{
LOG_WARNING(L"Skipping post processing for non-existing file: " + filePath.wstr());
return;
}
std::shared_ptr<TextAccess> textAccess = TextAccess::createFromFile(filePath);
if (textAccess)
{
locationFile->forEachStartSourceLocation([textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc)
locationFile->forEachStartSourceLocation(
[textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc)
{
if (!startLoc)
{
@@ -353,12 +366,11 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora
storage.removeElements(utility::toVector(elementsToDelete));
storage.finishInjection();
LOG_INFO("Finished Python post processing.");
}
else
{
LOG_ERROR("Error occurred while running Python post processing. Rolling back all changes.");
storage.rollbackInjection();
}
LOG_INFO("Finished Python post processing.");
}
src/lib/data/indexer/TaskExecuteCustomCommands.h
+1 -1
View File
@@ -37,8 +37,8 @@ private:
void executeParallelIndexerCommands(int threadId, std::shared_ptr<Blackboard> blackboard);
void runIndexerCommand(std::shared_ptr<IndexerCommandCustom> indexerCommand, std::shared_ptr<Blackboard> blackboard);
public:
static void runPythonPostProcessing(PersistentStorage& storage);
private:
std::unique_ptr<IndexerCommandProvider> m_indexerCommandProvider;
std::shared_ptr<PersistentStorage> m_storage;
src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp
-1
View File
@@ -27,7 +27,6 @@ void TaskFillIndexerCommandsQueue::doEnter(std::shared_ptr<Blackboard> blackboar
allSourceFilePaths = m_indexerCommandProvider->getAllSourceFilePaths();
}
for (const FilePath& path : allSourceFilePaths)
{
if (path.exists())
src/lib/data/storage/Storage.cpp
+7 -3
View File
@@ -219,13 +219,17 @@ void Storage::inject(Storage* injected)
sourceLocationId = it->second;
}
if (elementId && sourceLocationId)
if (!elementId)
{
occurrences.emplace_back(elementId, sourceLocationId);
LOG_WARNING("New occurrence element id could not be found.");
}
else if (!sourceLocationId)
{
LOG_WARNING("New occurrence location id could not be found.");
}
else
{
LOG_WARNING("New occurrence element or location id could not be found.");
occurrences.emplace_back(elementId, sourceLocationId);
}
}
src/lib/data/storage/sqlite/SqliteIndexStorage.cpp
+17 -33
View File
@@ -15,31 +15,17 @@ const size_t SqliteIndexStorage::s_storageVersion = 24;
namespace
{
std::tuple<std::wstring, uint32_t, uint32_t> splitLocalSymbolName(const std::wstring& name)
std::pair<std::wstring, std::wstring> splitLocalSymbolName(const std::wstring& name)
{
const std::tuple<std::wstring, uint32_t, uint32_t> res = std::make_tuple(L"", 0, 0);
size_t pos = name.find_last_of(L'<');
if (pos == std::wstring::npos)
if (pos == std::wstring::npos || name.back() != L'>')
{
return res;
return std::make_pair(L"", L"");
}
size_t pos2 = name.find(L':', pos + 1);
if (pos2 == std::wstring::npos)
{
return res;
}
if (name.back() != L'>')
{
return res;
}
return std::tuple<std::wstring, uint32_t, uint32_t>(
return std::make_pair(
name.substr(0, pos),
std::stoi(name.substr(pos + 1, pos2 - pos - 1)),
std::stoi(name.substr(pos2 + 1, name.size() - pos2 - 2))
name.substr(pos + 1, name.size() - pos - 2)
);
}
}
@@ -293,12 +279,10 @@ std::vector<Id> SqliteIndexStorage::addLocalSymbols(const std::set<StorageLocalS
forEach<StorageLocalSymbol>(
[this](StorageLocalSymbol&& localSymbol)
{
std::wstring name;
uint32_t line, col;
std::tie(name, line, col) = splitLocalSymbolName(localSymbol.name);
if (name.size())
std::pair<std::wstring, std::wstring> name = splitLocalSymbolName(localSymbol.name);
if (name.second.size())
{
m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), localSymbol.id);
m_tempLocalSymbolIndex[name.first].emplace(name.second, localSymbol.id);
}
}
);
@@ -310,15 +294,13 @@ std::vector<Id> SqliteIndexStorage::addLocalSymbols(const std::set<StorageLocalS
for (size_t i = 0; i < symbols.size(); i++)
{
const StorageLocalSymbol& data = *it;
std::wstring name;
uint32_t line, col;
std::tie(name, line, col) = splitLocalSymbolName(data.name);
if (name.size())
std::pair<std::wstring, std::wstring> name = splitLocalSymbolName(data.name);
if (name.second.size())
{
auto it = m_tempLocalSymbolIndex.find(name);
auto it = m_tempLocalSymbolIndex.find(name.first);
if (it != m_tempLocalSymbolIndex.end())
{
auto it2 = it->second.find(std::make_pair(line, col));
auto it2 = it->second.find(name.second);
if (it2 != it->second.end())
{
symbolIds[i] = it2->second;
@@ -329,12 +311,14 @@ std::vector<Id> SqliteIndexStorage::addLocalSymbols(const std::set<StorageLocalS
if (!symbolIds[i])
{
executeStatement(m_insertElementStmt);
Id id = m_database.lastRowId();
const Id id = m_database.lastRowId();
symbolIds[i] = id;
symbolsToInsert.emplace_back(id, data);
m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), id);
if (name.second.size())
{
m_tempLocalSymbolIndex[name.first].emplace(name.second, id);
}
}
it++;
src/lib/data/storage/sqlite/SqliteIndexStorage.h
+1 -1
View File
@@ -264,7 +264,7 @@ private:
LowMemoryStringMap<std::wstring, uint32_t, 0> m_tempWNodeNameIndex;
std::map<uint32_t, int> m_tempNodeTypes;
std::map<StorageEdgeData, uint32_t> m_tempEdgeIndex;
std::map<std::wstring, std::map<std::pair<uint32_t, uint32_t>, uint32_t>> m_tempLocalSymbolIndex;
std::map<std::wstring, std::map<std::wstring, uint32_t>> m_tempLocalSymbolIndex;
std::map<uint32_t, std::map<TempSourceLocation, uint32_t>> m_tempSourceLocationIndices;
template <typename StorageType>