diff --git a/src/lib/data/IntermediateStorage.cpp b/src/lib/data/IntermediateStorage.cpp index 067f65fe..53919741 100644 --- a/src/lib/data/IntermediateStorage.cpp +++ b/src/lib/data/IntermediateStorage.cpp @@ -17,9 +17,11 @@ IntermediateStorage::~IntermediateStorage() void IntermediateStorage::clear() { m_nodes.clear(); + m_nodesInOrder.clear(); m_files.clear(); m_symbols.clear(); m_edges.clear(); + m_edgesInOrder.clear(); m_localSymbols.clear(); m_sourceLocations.clear(); m_occurrences.clear(); @@ -118,6 +120,7 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName) const Id id = m_nextId++; node.id = id; m_nodes[serialized] = node; + m_nodesInOrder.push_back(node); return id; } @@ -153,6 +156,7 @@ Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId) const Id id = m_nextId++; edge.id = id; m_edges[serialized] = edge; + m_edgesInOrder.push_back(edge); return id; } @@ -265,9 +269,9 @@ void IntermediateStorage::addError( void IntermediateStorage::forEachNode(std::function callback) const { - for (std::unordered_map::const_iterator it = m_nodes.begin(); it != m_nodes.end(); it++) + for (const StorageNode& node : m_nodesInOrder) { - callback(it->second); + callback(node); } } @@ -289,9 +293,9 @@ void IntermediateStorage::forEachSymbol(std::function callback) const { - for (std::unordered_map::const_iterator it = m_edges.begin(); it != m_edges.end(); it++) + for (const StorageEdge& edge : m_edgesInOrder) { - callback(it->second); + callback(edge); } } @@ -347,13 +351,7 @@ void IntermediateStorage::forEachError(std::function IntermediateStorage::getStorageNodes() const { - std::vector nodes; - nodes.reserve(m_nodes.size()); - for (auto it: m_nodes) - { - nodes.push_back(it.second); - } - return nodes; + return m_nodesInOrder; } std::vector IntermediateStorage::getStorageFiles() const @@ -368,13 +366,7 @@ std::vector IntermediateStorage::getStorageSymbols() const std::vector IntermediateStorage::getStorageEdges() const { - std::vector edges; - edges.reserve(m_edges.size()); - for (auto it: m_edges) - { - edges.push_back(it.second); - } - return edges; + return m_edgesInOrder; } std::vector IntermediateStorage::getStorageLocalSymbols() const @@ -422,9 +414,11 @@ std::vector IntermediateStorage::getErrors() const void IntermediateStorage::setStorageNodes(const std::vector& storageNodes) { m_nodes.clear(); + m_nodesInOrder.clear(); for (const StorageNode& storageNode: storageNodes) { m_nodes[serialize(storageNode)] = storageNode; + m_nodesInOrder.push_back(storageNode); } } @@ -441,9 +435,11 @@ void IntermediateStorage::setStorageSymbols(const std::vector& st void IntermediateStorage::setStorageEdges(const std::vector& storageEdges) { m_edges.clear(); + m_edgesInOrder.clear(); for (const StorageEdge& storageEdge: storageEdges) { m_edges[serialize(storageEdge)] = storageEdge; + m_edgesInOrder.push_back(storageEdge); } } diff --git a/src/lib/data/IntermediateStorage.h b/src/lib/data/IntermediateStorage.h index 74c961bb..5bee1034 100644 --- a/src/lib/data/IntermediateStorage.h +++ b/src/lib/data/IntermediateStorage.h @@ -85,6 +85,7 @@ private: std::string serialize(const StorageError& error) const; std::unordered_map m_nodes; + std::vector m_nodesInOrder; std::unordered_set m_serializedFiles; // this is used to prevent duplicates (unique) std::vector m_files; @@ -92,6 +93,7 @@ private: std::vector m_symbols; std::unordered_map m_edges; + std::vector m_edgesInOrder; std::unordered_map m_localSymbols; diff --git a/src/lib/data/indexer/TaskBuildIndex.cpp b/src/lib/data/indexer/TaskBuildIndex.cpp index 80745871..927e6178 100644 --- a/src/lib/data/indexer/TaskBuildIndex.cpp +++ b/src/lib/data/indexer/TaskBuildIndex.cpp @@ -94,7 +94,12 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr blackboard) size_t commandCount = m_interprocessIndexerCommandManager.indexerCommandCount(); if (commandCount != m_lastCommandCount) { - updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath()); + std::vector indexingFiles = m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePaths(); + for (const FilePath& path : indexingFiles) + { + updateIndexingDialog(blackboard, path); + } + m_lastCommandCount = commandCount; } @@ -114,7 +119,7 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr blackboard) fetchIntermediateStorages(blackboard); - const int SLEEP_TIME_MS = 100; + const int SLEEP_TIME_MS = 50; std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_MS)); return STATE_RUNNING; @@ -129,7 +134,7 @@ void TaskBuildIndex::doExit(std::shared_ptr blackboard) } m_processThreads.clear(); - fetchIntermediateStorages(blackboard); + while (fetchIntermediateStorages(blackboard)); std::vector crashedFiles = m_interprocessIndexingStatusManager.getCrashedSourceFilePaths(); if (crashedFiles.size()) @@ -220,30 +225,35 @@ void TaskBuildIndex::runIndexerThread(int processId) } } -void TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr blackboard) +bool TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr blackboard) { - int newlyIndexedCount = 0; - - for (std::shared_ptr storageManager: m_interprocessIntermediateStorageManagers) + Id finishedProcessId = m_interprocessIndexingStatusManager.getNextFinishedProcessId(); + if (!finishedProcessId || finishedProcessId > m_interprocessIntermediateStorageManagers.size()) { - while (int storageCount = storageManager->getIntermediateStorageCount()) - { - LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount); - m_storageProvider->insert(storageManager->popIntermediateStorage()); - ++newlyIndexedCount; - - updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath()); - } + return false; } - if (newlyIndexedCount > 0) + std::shared_ptr storageManager = + m_interprocessIntermediateStorageManagers[finishedProcessId - 1]; + + int storageCount = storageManager->getIntermediateStorageCount(); + if (!storageCount) + { + return false; + } + + LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount); + m_storageProvider->insert(storageManager->popIntermediateStorage()); + { std::lock_guard lock(blackboard->getMutex()); int indexedSourceFileCount = 0; blackboard->get("indexed_source_file_count", indexedSourceFileCount); - blackboard->set("indexed_source_file_count", indexedSourceFileCount + newlyIndexedCount); + blackboard->set("indexed_source_file_count", indexedSourceFileCount + 1); } + + return true; } void TaskBuildIndex::updateIndexingDialog(std::shared_ptr blackboard, const FilePath& sourcePath) diff --git a/src/lib/data/indexer/TaskBuildIndex.h b/src/lib/data/indexer/TaskBuildIndex.h index c6c25d9b..db52e1fc 100644 --- a/src/lib/data/indexer/TaskBuildIndex.h +++ b/src/lib/data/indexer/TaskBuildIndex.h @@ -40,7 +40,7 @@ protected: void runIndexerProcess( int processId, const std::string& logFilePath); void runIndexerThread(int processId); - void fetchIntermediateStorages(std::shared_ptr blackboard); + bool fetchIntermediateStorages(std::shared_ptr blackboard); void updateIndexingDialog(std::shared_ptr blackboard, const FilePath& sourcePath); static const std::string s_processName; diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp index 6d9504fd..ec0aceaa 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp +++ b/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp @@ -33,7 +33,7 @@ void InterprocessIndexer::work() LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str()); LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1)); - m_interprocessIndexingStatusManager.setCurrentlyIndexedSourceFilePath(indexerCommand->getSourceFilePath()); + m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath()); FileRegisterStateData data; data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles()); @@ -48,7 +48,7 @@ void InterprocessIndexer::work() m_interprocessIntermediateStorageManager.pushIntermediateStorage(result); - m_interprocessIndexingStatusManager.clearCurrentlyIndexedSourceFilePath(); + m_interprocessIndexingStatusManager.finishIndexingSourceFile(); } } catch (boost::interprocess::interprocess_exception& e) diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp index 071a5fad..41d94f76 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp +++ b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp @@ -4,10 +4,11 @@ const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_"; -const char* InterprocessIndexingStatusManager::s_lastFileKeyName = "last_file"; +const char* InterprocessIndexingStatusManager::s_indexingFilesKeyName = "indexing_files"; const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files"; const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files"; const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files"; +const char* InterprocessIndexingStatusManager::s_finishedProcessIdsKeyName = "finished_process_ids"; InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner) : BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner) @@ -18,14 +19,17 @@ InterprocessIndexingStatusManager::~InterprocessIndexingStatusManager() { } -void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const FilePath& filePath) +void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath& filePath) { SharedMemory::ScopedAccess access(&m_sharedMemory); - SharedMemory::String* strPtr = access.accessValueWithAllocator(s_lastFileKeyName); - if (strPtr) + SharedMemory::Queue* indexingFilesPtr = + access.accessValueWithAllocator>(s_indexingFilesKeyName); + if (indexingFilesPtr) { - *strPtr = filePath.str().c_str(); + SharedMemory::String fileStr(access.getAllocator()); + fileStr = filePath.str().c_str(); + indexingFilesPtr->push_back(fileStr); } SharedMemory::Map* currentFilesPtr = @@ -52,7 +56,7 @@ void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const } } -void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath() +void InterprocessIndexingStatusManager::finishIndexingSourceFile() { SharedMemory::ScopedAccess access(&m_sharedMemory); @@ -62,19 +66,49 @@ void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath() { currentFilesPtr->erase(currentFilesPtr->find(getProcessId()), currentFilesPtr->end()); } + + SharedMemory::Queue* finishedProcessIdsPtr = + access.accessValueWithAllocator>(s_finishedProcessIdsKeyName); + if (finishedProcessIdsPtr) + { + finishedProcessIdsPtr->push_back(m_processId); + } } -FilePath InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePath() +Id InterprocessIndexingStatusManager::getNextFinishedProcessId() { SharedMemory::ScopedAccess access(&m_sharedMemory); - SharedMemory::String* strPtr = access.accessValueWithAllocator(s_lastFileKeyName); - if (strPtr) + SharedMemory::Queue* finishedProcessIdsPtr = + access.accessValueWithAllocator>(s_finishedProcessIdsKeyName); + if (finishedProcessIdsPtr && finishedProcessIdsPtr->size()) { - return FilePath(strPtr->c_str()); + Id processId = finishedProcessIdsPtr->front(); + finishedProcessIdsPtr->pop_front(); + return processId; } - return FilePath(); + return 0; +} + +std::vector InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePaths() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + std::vector indexingFiles; + + SharedMemory::Queue* indexingFilesPtr = + access.accessValueWithAllocator>(s_indexingFilesKeyName); + if (indexingFilesPtr) + { + while (indexingFilesPtr->size()) + { + indexingFiles.push_back(FilePath(indexingFilesPtr->front().c_str())); + indexingFilesPtr->pop_front(); + } + } + + return indexingFiles; } std::vector InterprocessIndexingStatusManager::getCrashedSourceFilePaths() diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h index b8b83e38..794cbf2f 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h +++ b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h @@ -13,10 +13,12 @@ public: InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner); virtual ~InterprocessIndexingStatusManager(); - void setCurrentlyIndexedSourceFilePath(const FilePath& filePath); - void clearCurrentlyIndexedSourceFilePath(); - FilePath getCurrentlyIndexedSourceFilePath(); + void startIndexingSourceFile(const FilePath& filePath); + void finishIndexingSourceFile(); + Id getNextFinishedProcessId(); + + std::vector getCurrentlyIndexedSourceFilePaths(); std::vector getCrashedSourceFilePaths(); std::set getIndexedFiles(); @@ -25,10 +27,11 @@ public: private: static const char* s_sharedMemoryNamePrefix; - static const char* s_lastFileKeyName; + static const char* s_indexingFilesKeyName; static const char* s_currentFilesKeyName; static const char* s_crashedFilesKeyName; static const char* s_indexedFilesKeyName; + static const char* s_finishedProcessIdsKeyName; }; #endif // INTERPROCESS_INDEXING_STATUS_MANAGER_H