ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: Improvements in indexing to avoid OOM (issue #471)

Browse Source 
* indexer processes wait if already too many storages queued
* storage fetcher waits if already too many storages queued
* increased size estimation for shared memory of indexer commands
* Added MessageFilterNewErrors to combine messages for new errors

bug id = 471
This commit is contained in:
Eberhard Graether
2017-10-12 17:15:46 +02:00
parent 1e38ce53c4
commit af626143a5
10 changed files with 74 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+14
View File
@@ -33,6 +33,20 @@ void InterprocessIndexer::work()
LOG_INFO_STREAM(<< m_processId << " fetched indexer command for \"" << indexerCommand->getSourceFilePath().str() << "\"");
LOG_INFO_STREAM(<< m_processId << " indexer commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
while (true)
{
size_t storageCount = m_interprocessIntermediateStorageManager.getIntermediateStorageCount();
if (storageCount < 10)
{
break;
}
LOG_INFO_STREAM(<< m_processId << " waits, too many intermediate storages: " << storageCount);
const int SLEEP_TIME_MS = 200;
std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_MS));
}
LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp
+1 -1
View File
@@ -19,7 +19,7 @@ InterprocessIndexerCommandManager::~InterprocessIndexerCommandManager()
void InterprocessIndexerCommandManager::setIndexerCommands(
const std::vector<std::shared_ptr<IndexerCommand>>& indexerCommands)
{
const unsigned int overestimationMultiplier = 2;
const unsigned int overestimationMultiplier = 3;
size_t estimatedSize = 1048576; /* 1 MB */
for (auto& command : indexerCommands)
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
+2 -2
View File
@@ -39,7 +39,7 @@ void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath&
SharedMemory::Map<Id, SharedMemory::String>::iterator it = currentFilesPtr->find(getProcessId());
if (it != currentFilesPtr->end())
{
const int overestimationMultiplier = 2;
const int overestimationMultiplier = 3;
const std::string crashedFilePath = it->second.c_str();
size_t estimatedSize = 262144 + sizeof(std::string) + crashedFilePath.size();
@@ -186,7 +186,7 @@ std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
{
const unsigned int overestimationMultiplier = 2;
const unsigned int overestimationMultiplier = 3;
SharedMemory::ScopedAccess access(&m_sharedMemory);
src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp
+1 -1
View File
@@ -28,7 +28,7 @@ void InterprocessIntermediateStorageManager::pushIntermediateStorage(
const std::shared_ptr<IntermediateStorage>& intermediateStorage)
{
const unsigned int overestimationMultiplier = 3;
size_t size = (intermediateStorage->getByteSize() + sizeof(SharedIntermediateStorage))* overestimationMultiplier + 1048576/* 1 MB */;
size_t size = (intermediateStorage->getByteSize() + sizeof(SharedIntermediateStorage)) * overestimationMultiplier + 1048576/* 1 MB */;
SharedMemory::ScopedAccess access(&m_sharedMemory);