data: Fixes in indexing

* Show status update for every indexed translation unit and only once
* Save nodes in same order they were indexed

src/lib/data/IntermediateStorage.cpp

@@ -17,9 +17,11 @@ IntermediateStorage::~IntermediateStorage()
 void IntermediateStorage::clear()
 {
 	m_nodes.clear();
+	m_nodesInOrder.clear();
 	m_files.clear();
 	m_symbols.clear();
 	m_edges.clear();
+	m_edgesInOrder.clear();
 	m_localSymbols.clear();
 	m_sourceLocations.clear();
 	m_occurrences.clear();
@@ -118,6 +120,7 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName)
 	const Id id = m_nextId++;
 	node.id = id;
 	m_nodes[serialized] = node;
+	m_nodesInOrder.push_back(node);
 	return id;
 }
 
@@ -153,6 +156,7 @@ Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId)
 	const Id id = m_nextId++;
 	edge.id = id;
 	m_edges[serialized] = edge;
+	m_edgesInOrder.push_back(edge);
 	return id;
 }
 
@@ -265,9 +269,9 @@ void IntermediateStorage::addError(
 
 void IntermediateStorage::forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const
 {
-	for (std::unordered_map<std::string, StorageNode>::const_iterator it = m_nodes.begin(); it != m_nodes.end(); it++)
+	for (const StorageNode& node : m_nodesInOrder)
 	{
-		callback(it->second);
+		callback(node);
 	}
 }
 
@@ -289,9 +293,9 @@ void IntermediateStorage::forEachSymbol(std::function<void(const StorageSymbol&
 
 void IntermediateStorage::forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const
 {
-	for (std::unordered_map<std::string, StorageEdge>::const_iterator it = m_edges.begin(); it != m_edges.end(); it++)
+	for (const StorageEdge& edge : m_edgesInOrder)
 	{
-		callback(it->second);
+		callback(edge);
 	}
 }
 
@@ -347,13 +351,7 @@ void IntermediateStorage::forEachError(std::function<void(const StorageError& /*
 
 std::vector<StorageNode> IntermediateStorage::getStorageNodes() const
 {
-	std::vector<StorageNode> nodes;
+	return m_nodesInOrder;
-	nodes.reserve(m_nodes.size());
-	for (auto it: m_nodes)
-	{
-		nodes.push_back(it.second);
-	}
-	return nodes;
 }
 
 std::vector<StorageFile> IntermediateStorage::getStorageFiles() const
@@ -368,13 +366,7 @@ std::vector<StorageSymbol> IntermediateStorage::getStorageSymbols() const
 
 std::vector<StorageEdge> IntermediateStorage::getStorageEdges() const
 {
-	std::vector<StorageEdge> edges;
+	return m_edgesInOrder;
-	edges.reserve(m_edges.size());
-	for (auto it: m_edges)
-	{
-		edges.push_back(it.second);
-	}
-	return edges;
 }
 
 std::vector<StorageLocalSymbol> IntermediateStorage::getStorageLocalSymbols() const
@@ -422,9 +414,11 @@ std::vector<StorageError> IntermediateStorage::getErrors() const
 void IntermediateStorage::setStorageNodes(const std::vector<StorageNode>& storageNodes)
 {
 	m_nodes.clear();
+	m_nodesInOrder.clear();
 	for (const StorageNode& storageNode: storageNodes)
 	{
 		m_nodes[serialize(storageNode)] = storageNode;
+		m_nodesInOrder.push_back(storageNode);
 	}
 }
 
@@ -441,9 +435,11 @@ void IntermediateStorage::setStorageSymbols(const std::vector<StorageSymbol>& st
 void IntermediateStorage::setStorageEdges(const std::vector<StorageEdge>& storageEdges)
 {
 	m_edges.clear();
+	m_edgesInOrder.clear();
 	for (const StorageEdge& storageEdge: storageEdges)
 	{
 		m_edges[serialize(storageEdge)] = storageEdge;
+		m_edgesInOrder.push_back(storageEdge);
 	}
 }
 

src/lib/data/IntermediateStorage.h

@@ -85,6 +85,7 @@ private:
 	std::string serialize(const StorageError& error) const;
 
 	std::unordered_map<std::string, StorageNode> m_nodes;
+	std::vector<StorageNode> m_nodesInOrder;
 
 	std::unordered_set<std::string> m_serializedFiles; // this is used to prevent duplicates (unique)
 	std::vector<StorageFile> m_files;
@@ -92,6 +93,7 @@ private:
 	std::vector<StorageSymbol> m_symbols;
 
 	std::unordered_map<std::string, StorageEdge> m_edges;
+	std::vector<StorageEdge> m_edgesInOrder;
 
 	std::unordered_map<std::string, StorageLocalSymbol> m_localSymbols;
 

src/lib/data/indexer/TaskBuildIndex.cpp

@@ -94,7 +94,12 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
 	size_t commandCount = m_interprocessIndexerCommandManager.indexerCommandCount();
 	if (commandCount != m_lastCommandCount)
 	{
-		updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath());
+		std::vector<FilePath> indexingFiles = m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePaths();
+		for (const FilePath& path : indexingFiles)
+		{
+			updateIndexingDialog(blackboard, path);
+		}
+
 		m_lastCommandCount = commandCount;
 	}
 
@@ -114,7 +119,7 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
 
 	fetchIntermediateStorages(blackboard);
 
-	const int SLEEP_TIME_MS = 100;
+	const int SLEEP_TIME_MS = 50;
 	std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_MS));
 
 	return STATE_RUNNING;
@@ -129,7 +134,7 @@ void TaskBuildIndex::doExit(std::shared_ptr<Blackboard> blackboard)
 	}
 	m_processThreads.clear();
 
-	fetchIntermediateStorages(blackboard);
+	while (fetchIntermediateStorages(blackboard));
 
 	std::vector<FilePath> crashedFiles = m_interprocessIndexingStatusManager.getCrashedSourceFilePaths();
 	if (crashedFiles.size())
@@ -220,30 +225,35 @@ void TaskBuildIndex::runIndexerThread(int processId)
 	}
 }
 
-void TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard)
+bool TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard)
 {
-	int newlyIndexedCount = 0;
+	Id finishedProcessId = m_interprocessIndexingStatusManager.getNextFinishedProcessId();
-
+	if (!finishedProcessId || finishedProcessId > m_interprocessIntermediateStorageManagers.size())
-	for (std::shared_ptr<InterprocessIntermediateStorageManager> storageManager: m_interprocessIntermediateStorageManagers)
 	{
-		while (int storageCount = storageManager->getIntermediateStorageCount())
+		return false;
-		{
-			LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
-			m_storageProvider->insert(storageManager->popIntermediateStorage());
-			++newlyIndexedCount;
-
-			updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath());
-		}
 	}
 
-	if (newlyIndexedCount > 0)
+	std::shared_ptr<InterprocessIntermediateStorageManager> storageManager =
+		m_interprocessIntermediateStorageManagers[finishedProcessId - 1];
+
+	int storageCount = storageManager->getIntermediateStorageCount();
+	if (!storageCount)
+	{
+		return false;
+	}
+
+	LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
+	m_storageProvider->insert(storageManager->popIntermediateStorage());
+
 	{
 		std::lock_guard<std::mutex> lock(blackboard->getMutex());
 
 		int indexedSourceFileCount = 0;
 		blackboard->get("indexed_source_file_count", indexedSourceFileCount);
-		blackboard->set("indexed_source_file_count", indexedSourceFileCount + newlyIndexedCount);
+		blackboard->set("indexed_source_file_count", indexedSourceFileCount + 1);
 	}
+
+	return true;
 }
 
 void TaskBuildIndex::updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath)

src/lib/data/indexer/TaskBuildIndex.h

@@ -40,7 +40,7 @@ protected:
 
 	void runIndexerProcess( int processId, const std::string& logFilePath);
 	void runIndexerThread(int processId);
-	void fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard);
+	bool fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard);
 	void updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath);
 
 	static const std::string s_processName;

src/lib/data/indexer/interprocess/InterprocessIndexer.cpp

@@ -33,7 +33,7 @@ void InterprocessIndexer::work()
 			LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str());
 			LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
 
-			m_interprocessIndexingStatusManager.setCurrentlyIndexedSourceFilePath(indexerCommand->getSourceFilePath());
+			m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
 
 			FileRegisterStateData data;
 			data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles());
@@ -48,7 +48,7 @@ void InterprocessIndexer::work()
 
 			m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
 
-			m_interprocessIndexingStatusManager.clearCurrentlyIndexedSourceFilePath();
+			m_interprocessIndexingStatusManager.finishIndexingSourceFile();
 		}
 	}
 	catch (boost::interprocess::interprocess_exception& e)

src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp

@@ -4,10 +4,11 @@
 
 const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_";
 
-const char* InterprocessIndexingStatusManager::s_lastFileKeyName = "last_file";
+const char* InterprocessIndexingStatusManager::s_indexingFilesKeyName = "indexing_files";
 const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files";
 const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files";
 const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files";
+const char* InterprocessIndexingStatusManager::s_finishedProcessIdsKeyName = "finished_process_ids";
 
 InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner)
 	: BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner)
@@ -18,14 +19,17 @@ InterprocessIndexingStatusManager::~InterprocessIndexingStatusManager()
 {
 }
 
-void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const FilePath& filePath)
+void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath& filePath)
 {
 	SharedMemory::ScopedAccess access(&m_sharedMemory);
 
-	SharedMemory::String* strPtr = access.accessValueWithAllocator<SharedMemory::String>(s_lastFileKeyName);
+	SharedMemory::Queue<SharedMemory::String>* indexingFilesPtr =
-	if (strPtr)
+		access.accessValueWithAllocator<SharedMemory::Queue<SharedMemory::String>>(s_indexingFilesKeyName);
+	if (indexingFilesPtr)
 	{
-		*strPtr = filePath.str().c_str();
+		SharedMemory::String fileStr(access.getAllocator());
+		fileStr = filePath.str().c_str();
+		indexingFilesPtr->push_back(fileStr);
 	}
 
 	SharedMemory::Map<Id, SharedMemory::String>* currentFilesPtr =
@@ -52,7 +56,7 @@ void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const
 	}
 }
 
-void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath()
+void InterprocessIndexingStatusManager::finishIndexingSourceFile()
 {
 	SharedMemory::ScopedAccess access(&m_sharedMemory);
 
@@ -62,19 +66,49 @@ void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath()
 	{
 		currentFilesPtr->erase(currentFilesPtr->find(getProcessId()), currentFilesPtr->end());
 	}
+
+	SharedMemory::Queue<Id>* finishedProcessIdsPtr =
+		access.accessValueWithAllocator<SharedMemory::Queue<Id>>(s_finishedProcessIdsKeyName);
+	if (finishedProcessIdsPtr)
+	{
+		finishedProcessIdsPtr->push_back(m_processId);
+	}
 }
 
-FilePath InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePath()
+Id InterprocessIndexingStatusManager::getNextFinishedProcessId()
 {
 	SharedMemory::ScopedAccess access(&m_sharedMemory);
 
-	SharedMemory::String* strPtr = access.accessValueWithAllocator<SharedMemory::String>(s_lastFileKeyName);
+	SharedMemory::Queue<Id>* finishedProcessIdsPtr =
-	if (strPtr)
+		access.accessValueWithAllocator<SharedMemory::Queue<Id>>(s_finishedProcessIdsKeyName);
+	if (finishedProcessIdsPtr && finishedProcessIdsPtr->size())
 	{
-		return FilePath(strPtr->c_str());
+		Id processId = finishedProcessIdsPtr->front();
+		finishedProcessIdsPtr->pop_front();
+		return processId;
 	}
 
-	return FilePath();
+	return 0;
+}
+
+std::vector<FilePath> InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePaths()
+{
+	SharedMemory::ScopedAccess access(&m_sharedMemory);
+
+	std::vector<FilePath> indexingFiles;
+
+	SharedMemory::Queue<SharedMemory::String>* indexingFilesPtr =
+		access.accessValueWithAllocator<SharedMemory::Queue<SharedMemory::String>>(s_indexingFilesKeyName);
+	if (indexingFilesPtr)
+	{
+		while (indexingFilesPtr->size())
+		{
+			indexingFiles.push_back(FilePath(indexingFilesPtr->front().c_str()));
+			indexingFilesPtr->pop_front();
+		}
+	}
+
+	return indexingFiles;
 }
 
 std::vector<FilePath> InterprocessIndexingStatusManager::getCrashedSourceFilePaths()

src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h

@@ -13,10 +13,12 @@ public:
 	InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner);
 	virtual ~InterprocessIndexingStatusManager();
 
-	void setCurrentlyIndexedSourceFilePath(const FilePath& filePath);
+	void startIndexingSourceFile(const FilePath& filePath);
-	void clearCurrentlyIndexedSourceFilePath();
+	void finishIndexingSourceFile();
-	FilePath getCurrentlyIndexedSourceFilePath();
 
+	Id getNextFinishedProcessId();
+
+	std::vector<FilePath> getCurrentlyIndexedSourceFilePaths();
 	std::vector<FilePath> getCrashedSourceFilePaths();
 
 	std::set<FilePath> getIndexedFiles();
@@ -25,10 +27,11 @@ public:
 private:
 	static const char* s_sharedMemoryNamePrefix;
 
-	static const char* s_lastFileKeyName;
+	static const char* s_indexingFilesKeyName;
 	static const char* s_currentFilesKeyName;
 	static const char* s_crashedFilesKeyName;
 	static const char* s_indexedFilesKeyName;
+	static const char* s_finishedProcessIdsKeyName;
 };
 
 #endif // INTERPROCESS_INDEXING_STATUS_MANAGER_H