ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: apply some shared memory fixes

Browse Source 
* corrections in estimating size of intermediate indexer commands
* avoided clearing and refilling the indexed files list every time files get added
* changed memory growth policy of InterprocessIndexingStatusManager to double allocated memory when it gets full
* added potential to grow memory when adding crashed file paths
This commit is contained in:
malte_langkabel
2017-06-19 16:24:01 +02:00
parent fba8dadee7
commit e6c5ffa97a
10 changed files with 100 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+15 -4
View File
@@ -25,14 +25,15 @@ void InterprocessIndexer::work()
{
try
{
LOG_INFO_STREAM(<< m_processId << " Starting to index");
LOG_INFO_STREAM(<< m_processId << " starting up indexer");
std::shared_ptr<IndexerBase> indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules();
while (std::shared_ptr<IndexerCommand> indexerCommand = m_interprocessIndexerCommandManager.popIndexerCommand())
{
LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str());
LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
LOG_INFO_STREAM(<< m_processId << " fetched indexer command for \"" << indexerCommand->getSourceFilePath().str() << "\"");
LOG_INFO_STREAM(<< m_processId << " indexer commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
@@ -42,13 +43,19 @@ void InterprocessIndexer::work()
data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath()
);
LOG_INFO_STREAM(<< m_processId << " starting to index current file");
std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status");
m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory");
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
LOG_INFO_STREAM(<< m_processId << " finalizing indexer status for current file");
m_interprocessIndexingStatusManager.finishIndexingSourceFile();
LOG_INFO_STREAM(<< m_processId << " all done");
}
}
catch (boost::interprocess::interprocess_exception& e)
@@ -61,6 +68,10 @@ void InterprocessIndexer::work()
LOG_ERROR(e.what());
throw e;
}
catch (...)
{
LOG_ERROR("something went wrong while running the indexer");
}
LOG_INFO_STREAM(<< "Finished indexing");
LOG_INFO_STREAM(<< "shutting down indexer");
}