diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp index 71d2191d..2c785b03 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp @@ -42,10 +42,10 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr blackboard) if (m_indexerCommandProvider) { - while (!m_indexerCommandProvider->empty()) + for (const FilePath& sourceFilePath : m_indexerCommandProvider->getAllSourceFilePaths()) { if (std::shared_ptr indexerCommand = - std::dynamic_pointer_cast(m_indexerCommandProvider->consumeCommand())) + std::dynamic_pointer_cast(m_indexerCommandProvider->consumeCommandForSourceFilePath(sourceFilePath))) { if (m_targetDatabaseFilePath.empty()) { @@ -79,7 +79,6 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), {}); - std::vector> indexerThreads; for (size_t i = 1 /*this method is counting as the first thread*/; i < m_indexerThreadCount; i++) { @@ -119,6 +118,8 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr if (m_hasPythonCommands && ApplicationSettings::getInstance()->getPythonPostProcessingEnabled()) { + targetStorage.clearCaches(); + targetStorage.buildCaches(); runPythonPostProcessing(targetStorage); } } @@ -286,13 +287,25 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora std::vector dataToInsert; std::set elementsToDelete; - locationCollection->forEachSourceLocationFile([&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr locationFile) + locationCollection->forEachSourceLocationFile( + [&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr locationFile) { - std::shared_ptr textAccess = TextAccess::createFromFile(locationFile->getFilePath()); + const FilePath filePath = locationFile->getFilePath(); + if (filePath.empty()) + { + return; + } + if (!filePath.exists()) + { + LOG_WARNING(L"Skipping post processing for non-existing file: " + filePath.wstr()); + return; + } + std::shared_ptr textAccess = TextAccess::createFromFile(filePath); if (textAccess) { - locationFile->forEachStartSourceLocation([textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc) + locationFile->forEachStartSourceLocation( + [textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc) { if (!startLoc) { @@ -353,12 +366,11 @@ void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& stora storage.removeElements(utility::toVector(elementsToDelete)); storage.finishInjection(); + LOG_INFO("Finished Python post processing."); } else { LOG_ERROR("Error occurred while running Python post processing. Rolling back all changes."); storage.rollbackInjection(); } - - LOG_INFO("Finished Python post processing."); } diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.h b/src/lib/data/indexer/TaskExecuteCustomCommands.h index edeac9b7..9d4ed08f 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.h +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.h @@ -37,8 +37,8 @@ private: void executeParallelIndexerCommands(int threadId, std::shared_ptr blackboard); void runIndexerCommand(std::shared_ptr indexerCommand, std::shared_ptr blackboard); -public: static void runPythonPostProcessing(PersistentStorage& storage); + private: std::unique_ptr m_indexerCommandProvider; std::shared_ptr m_storage; diff --git a/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp b/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp index b9f46604..56d19365 100644 --- a/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp +++ b/src/lib/data/indexer/TaskFillIndexerCommandQueue.cpp @@ -27,7 +27,6 @@ void TaskFillIndexerCommandsQueue::doEnter(std::shared_ptr blackboar allSourceFilePaths = m_indexerCommandProvider->getAllSourceFilePaths(); } - for (const FilePath& path : allSourceFilePaths) { if (path.exists()) diff --git a/src/lib/data/storage/Storage.cpp b/src/lib/data/storage/Storage.cpp index 299ead7c..081d787f 100644 --- a/src/lib/data/storage/Storage.cpp +++ b/src/lib/data/storage/Storage.cpp @@ -219,13 +219,17 @@ void Storage::inject(Storage* injected) sourceLocationId = it->second; } - if (elementId && sourceLocationId) + if (!elementId) { - occurrences.emplace_back(elementId, sourceLocationId); + LOG_WARNING("New occurrence element id could not be found."); + } + else if (!sourceLocationId) + { + LOG_WARNING("New occurrence location id could not be found."); } else { - LOG_WARNING("New occurrence element or location id could not be found."); + occurrences.emplace_back(elementId, sourceLocationId); } } diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp index a68fdd2d..b176500c 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp @@ -15,31 +15,17 @@ const size_t SqliteIndexStorage::s_storageVersion = 24; namespace { - std::tuple splitLocalSymbolName(const std::wstring& name) + std::pair splitLocalSymbolName(const std::wstring& name) { - const std::tuple res = std::make_tuple(L"", 0, 0); - size_t pos = name.find_last_of(L'<'); - if (pos == std::wstring::npos) + if (pos == std::wstring::npos || name.back() != L'>') { - return res; + return std::make_pair(L"", L""); } - size_t pos2 = name.find(L':', pos + 1); - if (pos2 == std::wstring::npos) - { - return res; - } - - if (name.back() != L'>') - { - return res; - } - - return std::tuple( + return std::make_pair( name.substr(0, pos), - std::stoi(name.substr(pos + 1, pos2 - pos - 1)), - std::stoi(name.substr(pos2 + 1, name.size() - pos2 - 2)) + name.substr(pos + 1, name.size() - pos - 2) ); } } @@ -293,12 +279,10 @@ std::vector SqliteIndexStorage::addLocalSymbols(const std::set( [this](StorageLocalSymbol&& localSymbol) { - std::wstring name; - uint32_t line, col; - std::tie(name, line, col) = splitLocalSymbolName(localSymbol.name); - if (name.size()) + std::pair name = splitLocalSymbolName(localSymbol.name); + if (name.second.size()) { - m_tempLocalSymbolIndex[name].emplace(std::make_pair(line, col), localSymbol.id); + m_tempLocalSymbolIndex[name.first].emplace(name.second, localSymbol.id); } } ); @@ -310,15 +294,13 @@ std::vector SqliteIndexStorage::addLocalSymbols(const std::set name = splitLocalSymbolName(data.name); + if (name.second.size()) { - auto it = m_tempLocalSymbolIndex.find(name); + auto it = m_tempLocalSymbolIndex.find(name.first); if (it != m_tempLocalSymbolIndex.end()) { - auto it2 = it->second.find(std::make_pair(line, col)); + auto it2 = it->second.find(name.second); if (it2 != it->second.end()) { symbolIds[i] = it2->second; @@ -329,12 +311,14 @@ std::vector SqliteIndexStorage::addLocalSymbols(const std::set m_tempWNodeNameIndex; std::map m_tempNodeTypes; std::map m_tempEdgeIndex; - std::map, uint32_t>> m_tempLocalSymbolIndex; + std::map> m_tempLocalSymbolIndex; std::map> m_tempSourceLocationIndices; template