ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: Fixes in indexing

Browse Source 
* Show status update for every indexed translation unit and only once
* Save nodes in same order they were indexed
This commit is contained in:
Eberhard Graether
2017-05-09 22:55:49 +02:00
parent f4caacacf2
commit 3fdd24f038
7 changed files with 98 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/IntermediateStorage.cpp
+14 -18
View File
@@ -17,9 +17,11 @@ IntermediateStorage::~IntermediateStorage()
void IntermediateStorage::clear()
{
m_nodes.clear();
m_nodesInOrder.clear();
m_files.clear();
m_symbols.clear();
m_edges.clear();
m_edgesInOrder.clear();
m_localSymbols.clear();
m_sourceLocations.clear();
m_occurrences.clear();
@@ -118,6 +120,7 @@ Id IntermediateStorage::addNode(int type, const std::string& serializedName)
const Id id = m_nextId++;
node.id = id;
m_nodes[serialized] = node;
m_nodesInOrder.push_back(node);
return id;
}
@@ -153,6 +156,7 @@ Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId)
const Id id = m_nextId++;
edge.id = id;
m_edges[serialized] = edge;
m_edgesInOrder.push_back(edge);
return id;
}
@@ -265,9 +269,9 @@ void IntermediateStorage::addError(
void IntermediateStorage::forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const
{
for (std::unordered_map<std::string, StorageNode>::const_iterator it = m_nodes.begin(); it != m_nodes.end(); it++)
for (const StorageNode& node : m_nodesInOrder)
{
callback(it->second);
callback(node);
}
}
@@ -289,9 +293,9 @@ void IntermediateStorage::forEachSymbol(std::function<void(const StorageSymbol&
void IntermediateStorage::forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const
{
for (std::unordered_map<std::string, StorageEdge>::const_iterator it = m_edges.begin(); it != m_edges.end(); it++)
for (const StorageEdge& edge : m_edgesInOrder)
{
callback(it->second);
callback(edge);
}
}
@@ -347,13 +351,7 @@ void IntermediateStorage::forEachError(std::function<void(const StorageError& /*
std::vector<StorageNode> IntermediateStorage::getStorageNodes() const
{
std::vector<StorageNode> nodes;
nodes.reserve(m_nodes.size());
for (auto it: m_nodes)
{
nodes.push_back(it.second);
}
return nodes;
return m_nodesInOrder;
}
std::vector<StorageFile> IntermediateStorage::getStorageFiles() const
@@ -368,13 +366,7 @@ std::vector<StorageSymbol> IntermediateStorage::getStorageSymbols() const
std::vector<StorageEdge> IntermediateStorage::getStorageEdges() const
{
std::vector<StorageEdge> edges;
edges.reserve(m_edges.size());
for (auto it: m_edges)
{
edges.push_back(it.second);
}
return edges;
return m_edgesInOrder;
}
std::vector<StorageLocalSymbol> IntermediateStorage::getStorageLocalSymbols() const
@@ -422,9 +414,11 @@ std::vector<StorageError> IntermediateStorage::getErrors() const
void IntermediateStorage::setStorageNodes(const std::vector<StorageNode>& storageNodes)
{
m_nodes.clear();
m_nodesInOrder.clear();
for (const StorageNode& storageNode: storageNodes)
{
m_nodes[serialize(storageNode)] = storageNode;
m_nodesInOrder.push_back(storageNode);
}
}
@@ -441,9 +435,11 @@ void IntermediateStorage::setStorageSymbols(const std::vector<StorageSymbol>& st
void IntermediateStorage::setStorageEdges(const std::vector<StorageEdge>& storageEdges)
{
m_edges.clear();
m_edgesInOrder.clear();
for (const StorageEdge& storageEdge: storageEdges)
{
m_edges[serialize(storageEdge)] = storageEdge;
m_edgesInOrder.push_back(storageEdge);
}
}
src/lib/data/IntermediateStorage.h
+2
View File
@@ -85,6 +85,7 @@ private:
std::string serialize(const StorageError& error) const;
std::unordered_map<std::string, StorageNode> m_nodes;
std::vector<StorageNode> m_nodesInOrder;
std::unordered_set<std::string> m_serializedFiles; // this is used to prevent duplicates (unique)
std::vector<StorageFile> m_files;
@@ -92,6 +93,7 @@ private:
std::vector<StorageSymbol> m_symbols;
std::unordered_map<std::string, StorageEdge> m_edges;
std::vector<StorageEdge> m_edgesInOrder;
std::unordered_map<std::string, StorageLocalSymbol> m_localSymbols;
src/lib/data/indexer/TaskBuildIndex.cpp
+27 -17
View File
@@ -94,7 +94,12 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
size_t commandCount = m_interprocessIndexerCommandManager.indexerCommandCount();
if (commandCount != m_lastCommandCount)
{
updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath());
std::vector<FilePath> indexingFiles = m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePaths();
for (const FilePath& path : indexingFiles)
{
updateIndexingDialog(blackboard, path);
}
m_lastCommandCount = commandCount;
}
@@ -114,7 +119,7 @@ Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
fetchIntermediateStorages(blackboard);
const int SLEEP_TIME_MS = 100;
const int SLEEP_TIME_MS = 50;
std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_MS));
return STATE_RUNNING;
@@ -129,7 +134,7 @@ void TaskBuildIndex::doExit(std::shared_ptr<Blackboard> blackboard)
}
m_processThreads.clear();
fetchIntermediateStorages(blackboard);
while (fetchIntermediateStorages(blackboard));
std::vector<FilePath> crashedFiles = m_interprocessIndexingStatusManager.getCrashedSourceFilePaths();
if (crashedFiles.size())
@@ -220,30 +225,35 @@ void TaskBuildIndex::runIndexerThread(int processId)
}
}
void TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard)
bool TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard)
{
int newlyIndexedCount = 0;
for (std::shared_ptr<InterprocessIntermediateStorageManager> storageManager: m_interprocessIntermediateStorageManagers)
Id finishedProcessId = m_interprocessIndexingStatusManager.getNextFinishedProcessId();
if (!finishedProcessId || finishedProcessId > m_interprocessIntermediateStorageManagers.size())
{
while (int storageCount = storageManager->getIntermediateStorageCount())
{
LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
m_storageProvider->insert(storageManager->popIntermediateStorage());
++newlyIndexedCount;
updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath());
}
return false;
}
if (newlyIndexedCount > 0)
std::shared_ptr<InterprocessIntermediateStorageManager> storageManager =
m_interprocessIntermediateStorageManagers[finishedProcessId - 1];
int storageCount = storageManager->getIntermediateStorageCount();
if (!storageCount)
{
return false;
}
LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
m_storageProvider->insert(storageManager->popIntermediateStorage());
{
std::lock_guard<std::mutex> lock(blackboard->getMutex());
int indexedSourceFileCount = 0;
blackboard->get("indexed_source_file_count", indexedSourceFileCount);
blackboard->set("indexed_source_file_count", indexedSourceFileCount + newlyIndexedCount);
blackboard->set("indexed_source_file_count", indexedSourceFileCount + 1);
}
return true;
}
void TaskBuildIndex::updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath)
src/lib/data/indexer/TaskBuildIndex.h
+1 -1
View File
@@ -40,7 +40,7 @@ protected:
void runIndexerProcess( int processId, const std::string& logFilePath);
void runIndexerThread(int processId);
void fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard);
bool fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard);
void updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath);
static const std::string s_processName;
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+2 -2
View File
@@ -33,7 +33,7 @@ void InterprocessIndexer::work()
LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str());
LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
m_interprocessIndexingStatusManager.setCurrentlyIndexedSourceFilePath(indexerCommand->getSourceFilePath());
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles());
@@ -48,7 +48,7 @@ void InterprocessIndexer::work()
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
m_interprocessIndexingStatusManager.clearCurrentlyIndexedSourceFilePath();
m_interprocessIndexingStatusManager.finishIndexingSourceFile();
}
}
catch (boost::interprocess::interprocess_exception& e)
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
+45 -11
View File
@@ -4,10 +4,11 @@
const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_";
const char* InterprocessIndexingStatusManager::s_lastFileKeyName = "last_file";
const char* InterprocessIndexingStatusManager::s_indexingFilesKeyName = "indexing_files";
const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files";
const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files";
const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files";
const char* InterprocessIndexingStatusManager::s_finishedProcessIdsKeyName = "finished_process_ids";
InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner)
: BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner)
@@ -18,14 +19,17 @@ InterprocessIndexingStatusManager::~InterprocessIndexingStatusManager()
{
}
void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const FilePath& filePath)
void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath& filePath)
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::String* strPtr = access.accessValueWithAllocator<SharedMemory::String>(s_lastFileKeyName);
if (strPtr)
SharedMemory::Queue<SharedMemory::String>* indexingFilesPtr =
access.accessValueWithAllocator<SharedMemory::Queue<SharedMemory::String>>(s_indexingFilesKeyName);
if (indexingFilesPtr)
{
*strPtr = filePath.str().c_str();
SharedMemory::String fileStr(access.getAllocator());
fileStr = filePath.str().c_str();
indexingFilesPtr->push_back(fileStr);
}
SharedMemory::Map<Id, SharedMemory::String>* currentFilesPtr =
@@ -52,7 +56,7 @@ void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const
}
}
void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath()
void InterprocessIndexingStatusManager::finishIndexingSourceFile()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
@@ -62,19 +66,49 @@ void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath()
{
currentFilesPtr->erase(currentFilesPtr->find(getProcessId()), currentFilesPtr->end());
}
SharedMemory::Queue<Id>* finishedProcessIdsPtr =
access.accessValueWithAllocator<SharedMemory::Queue<Id>>(s_finishedProcessIdsKeyName);
if (finishedProcessIdsPtr)
{
finishedProcessIdsPtr->push_back(m_processId);
}
}
FilePath InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePath()
Id InterprocessIndexingStatusManager::getNextFinishedProcessId()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::String* strPtr = access.accessValueWithAllocator<SharedMemory::String>(s_lastFileKeyName);
if (strPtr)
SharedMemory::Queue<Id>* finishedProcessIdsPtr =
access.accessValueWithAllocator<SharedMemory::Queue<Id>>(s_finishedProcessIdsKeyName);
if (finishedProcessIdsPtr && finishedProcessIdsPtr->size())
{
return FilePath(strPtr->c_str());
Id processId = finishedProcessIdsPtr->front();
finishedProcessIdsPtr->pop_front();
return processId;
}
return FilePath();
return 0;
}
std::vector<FilePath> InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePaths()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
std::vector<FilePath> indexingFiles;
SharedMemory::Queue<SharedMemory::String>* indexingFilesPtr =
access.accessValueWithAllocator<SharedMemory::Queue<SharedMemory::String>>(s_indexingFilesKeyName);
if (indexingFilesPtr)
{
while (indexingFilesPtr->size())
{
indexingFiles.push_back(FilePath(indexingFilesPtr->front().c_str()));
indexingFilesPtr->pop_front();
}
}
return indexingFiles;
}
std::vector<FilePath> InterprocessIndexingStatusManager::getCrashedSourceFilePaths()
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h
+7 -4
View File
@@ -13,10 +13,12 @@ public:
InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner);
virtual ~InterprocessIndexingStatusManager();
void setCurrentlyIndexedSourceFilePath(const FilePath& filePath);
void clearCurrentlyIndexedSourceFilePath();
FilePath getCurrentlyIndexedSourceFilePath();
void startIndexingSourceFile(const FilePath& filePath);
void finishIndexingSourceFile();
Id getNextFinishedProcessId();
std::vector<FilePath> getCurrentlyIndexedSourceFilePaths();
std::vector<FilePath> getCrashedSourceFilePaths();
std::set<FilePath> getIndexedFiles();
@@ -25,10 +27,11 @@ public:
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_lastFileKeyName;
static const char* s_indexingFilesKeyName;
static const char* s_currentFilesKeyName;
static const char* s_crashedFilesKeyName;
static const char* s_indexedFilesKeyName;
static const char* s_finishedProcessIdsKeyName;
};
#endif // INTERPROCESS_INDEXING_STATUS_MANAGER_H