ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: apply some shared memory fixes

Browse Source 
* corrections in estimating size of intermediate indexer commands
* avoided clearing and refilling the indexed files list every time files get added
* changed memory growth policy of InterprocessIndexingStatusManager to double allocated memory when it gets full
* added potential to grow memory when adding crashed file paths
This commit is contained in:
malte_langkabel
2017-06-19 16:24:01 +02:00
parent fba8dadee7
commit e6c5ffa97a
10 changed files with 100 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/IndexerCommand.cpp
+2 -2
View File
@@ -19,12 +19,12 @@ size_t IndexerCommand::getByteSize() const
for (auto i : m_indexedPaths)
{
size += i.str().size();
size += sizeof(std::string) + i.str().size();
}
for (auto i : m_excludedPaths)
{
size += i.str().size();
size += sizeof(std::string) + i.str().size();
}
return size;
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+15 -4
View File
@@ -25,14 +25,15 @@ void InterprocessIndexer::work()
{
try
{
LOG_INFO_STREAM(<< m_processId << " Starting to index");
LOG_INFO_STREAM(<< m_processId << " starting up indexer");
std::shared_ptr<IndexerBase> indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules();
while (std::shared_ptr<IndexerCommand> indexerCommand = m_interprocessIndexerCommandManager.popIndexerCommand())
{
LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str());
LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
LOG_INFO_STREAM(<< m_processId << " fetched indexer command for \"" << indexerCommand->getSourceFilePath().str() << "\"");
LOG_INFO_STREAM(<< m_processId << " indexer commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
@@ -42,13 +43,19 @@ void InterprocessIndexer::work()
data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath()
);
LOG_INFO_STREAM(<< m_processId << " starting to index current file");
std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status");
m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory");
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
LOG_INFO_STREAM(<< m_processId << " finalizing indexer status for current file");
m_interprocessIndexingStatusManager.finishIndexingSourceFile();
LOG_INFO_STREAM(<< m_processId << " all done");
}
}
catch (boost::interprocess::interprocess_exception& e)
@@ -61,6 +68,10 @@ void InterprocessIndexer::work()
LOG_ERROR(e.what());
throw e;
}
catch (...)
{
LOG_ERROR("something went wrong while running the indexer");
}
LOG_INFO_STREAM(<< "Finished indexing");
LOG_INFO_STREAM(<< "shutting down indexer");
}
src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp
+10 -9
View File
@@ -19,26 +19,27 @@ InterprocessIndexerCommandManager::~InterprocessIndexerCommandManager()
void InterprocessIndexerCommandManager::setIndexerCommands(
const std::vector<std::shared_ptr<IndexerCommand>>& indexerCommands)
{
const unsigned int overestimationMultiplier = 3;
const unsigned int overestimationMultiplier = 2;
size_t size = 1000;
size_t estimatedSize = 1048576; /* 1 MB */
for (auto command : indexerCommands)
{
size += command->getByteSize() + sizeof(SharedIndexerCommand);
estimatedSize += command->getByteSize() + sizeof(SharedIndexerCommand);
}
size *= overestimationMultiplier;
estimatedSize *= overestimationMultiplier;
SharedMemory::ScopedAccess access(&m_sharedMemory);
size_t freeMemory = access.getFreeMemorySize();
if (freeMemory <= size)
if (freeMemory < estimatedSize)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << size << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (estimatedSize - freeMemory));
access.growMemory(size - freeMemory);
access.growMemory(estimatedSize - freeMemory);
LOG_INFO("growing memory succeeded");
}
SharedMemory::Queue<SharedIndexerCommand>* queue =
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
+55 -18
View File
@@ -39,6 +39,28 @@ void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath&
SharedMemory::Map<Id, SharedMemory::String>::iterator it = currentFilesPtr->find(getProcessId());
if (it != currentFilesPtr->end())
{
const int overestimationMultiplier = 2;
const std::string crashedFilePath = it->second.c_str();
size_t estimatedSize = 262144 + sizeof(std::string) + crashedFilePath.size();
estimatedSize *= overestimationMultiplier;
while (access.getFreeMemorySize() < estimatedSize)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
access.growMemory(access.getMemorySize());
LOG_INFO("growing memory succeeded");
currentFilesPtr = access.accessValueWithAllocator<SharedMemory::Map<Id, SharedMemory::String>>(s_currentFilesKeyName);
if (!currentFilesPtr)
{
return;
}
}
SharedMemory::Vector<SharedMemory::String>* crashedFilesPtr =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_crashedFilesKeyName);
@@ -164,43 +186,58 @@ std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
{
const unsigned int overestimationMultiplier = 3;
const unsigned int overestimationMultiplier = 2;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* files =
SharedMemory::Vector<SharedMemory::String>* indexedFiles =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!files)
if (!indexedFiles)
{
return;
}
for (auto file : *files)
std::set<std::string> oldFiles;
for (auto indexedFile : *indexedFiles)
{
filePaths.insert(FilePath(file.c_str()));
oldFiles.insert(indexedFile.c_str());
}
files->clear();
size_t size = 1000;
for (auto path : filePaths)
std::set<std::string> newFiles;
for (const FilePath& filePath : filePaths)
{
size += sizeof(std::string) + path.str().size();
if (oldFiles.find(filePath.str()) == oldFiles.end())
{
newFiles.insert(filePath.str());
}
}
size *= overestimationMultiplier;
size_t freeMemory = access.getFreeMemorySize();
if (freeMemory <= size)
size_t estimatedSize = 262144;
for (auto newFile : newFiles)
{
estimatedSize += sizeof(std::string) + newFile.size();
}
estimatedSize *= overestimationMultiplier;
while (access.getFreeMemorySize() < estimatedSize)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << size << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
access.growMemory(size - freeMemory);
<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
access.growMemory(access.getMemorySize());
LOG_INFO("growing memory succeeded");
indexedFiles = access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
}
for (auto path : filePaths)
for (const std::string& newFile: newFiles)
{
files->push_back(SharedMemory::String(path.str().c_str(), access.getAllocator()));
indexedFiles->push_back(SharedMemory::String(newFile.c_str(), access.getAllocator()));
}
LOG_INFO(access.logString());
src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp
+3 -1
View File
@@ -33,13 +33,15 @@ void InterprocessIntermediateStorageManager::pushIntermediateStorage(
SharedMemory::ScopedAccess access(&m_sharedMemory);
size_t freeMemory = access.getFreeMemorySize();
if (freeMemory <= size)
if (freeMemory < size)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << size << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
access.growMemory(size - freeMemory);
LOG_INFO("growing memory succeeded");
}
SharedMemory::Queue<SharedIntermediateStorage>* queue =
src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp
+8
View File
@@ -198,6 +198,7 @@ void SharedIndexerCommand::setLanguageStandard(const std::string& languageStanda
std::vector<std::string> SharedIndexerCommand::getCompilerFlags() const
{
std::vector<std::string> result;
result.reserve(m_compilerFlags.size());
for (unsigned int i = 0; i < m_compilerFlags.size(); i++)
{
@@ -210,6 +211,7 @@ std::vector<std::string> SharedIndexerCommand::getCompilerFlags() const
void SharedIndexerCommand::setCompilerFlags(const std::vector<std::string>& compilerFlags)
{
m_compilerFlags.clear();
m_compilerFlags.reserve(compilerFlags.size());
for (unsigned int i = 0; i < compilerFlags.size(); i++)
{
@@ -222,6 +224,7 @@ void SharedIndexerCommand::setCompilerFlags(const std::vector<std::string>& comp
std::vector<FilePath> SharedIndexerCommand::getSystemHeaderSearchPaths() const
{
std::vector<FilePath> result;
result.reserve(m_systemHeaderSearchPaths.size());
for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++)
{
@@ -234,6 +237,7 @@ std::vector<FilePath> SharedIndexerCommand::getSystemHeaderSearchPaths() const
void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector<FilePath>& filePaths)
{
m_systemHeaderSearchPaths.clear();
m_systemHeaderSearchPaths.reserve(filePaths.size());
for (unsigned int i = 0; i < filePaths.size(); i++)
{
@@ -246,6 +250,7 @@ void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector<FilePath
std::vector<FilePath> SharedIndexerCommand::getFrameworkSearchhPaths() const
{
std::vector<FilePath> result;
result.reserve(m_frameworkSearchPaths.size());
for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++)
{
@@ -258,6 +263,7 @@ std::vector<FilePath> SharedIndexerCommand::getFrameworkSearchhPaths() const
void SharedIndexerCommand::setFrameworkSearchhPaths(const std::vector<FilePath>& searchPaths)
{
m_frameworkSearchPaths.clear();
m_frameworkSearchPaths.reserve(searchPaths.size());
for (unsigned int i = 0; i < searchPaths.size(); i++)
{
@@ -280,6 +286,7 @@ void SharedIndexerCommand::setPreprocessorOnly(bool preprocessorOnly)
std::vector<FilePath> SharedIndexerCommand::getClassPaths() const
{
std::vector<FilePath> result;
result.reserve(m_classPaths.size());
for (unsigned int i = 0; i < m_classPaths.size(); i++)
{
@@ -292,6 +299,7 @@ std::vector<FilePath> SharedIndexerCommand::getClassPaths() const
void SharedIndexerCommand::setClassPaths(const std::vector<FilePath>& classPaths)
{
m_classPaths.clear();
m_classPaths.reserve(classPaths.size());
for (unsigned int i = 0; i < classPaths.size(); i++)
{
src/lib_cxx/data/indexer/IndexerCommandCxx.cpp
+3 -3
View File
@@ -26,17 +26,17 @@ size_t IndexerCommandCxx::getByteSize() const
for (auto i : m_systemHeaderSearchPaths)
{
size += i.str().size();
size += sizeof(std::string) + i.str().size();
}
for (auto i : m_frameworkSearchPaths)
{
size += i.str().size();
size += sizeof(std::string) + i.str().size();
}
for (auto i : m_compilerFlags)
{
size += i.size();
size += sizeof(std::string) + i.size();
}
return size;
src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp
+1 -1
View File
@@ -64,7 +64,7 @@ IndexerCommandType IndexerCommandCxxCdb::getIndexerCommandType() const
size_t IndexerCommandCxxCdb::getByteSize() const
{
return IndexerCommandCxx::getByteSize() + sizeof(*this) + m_workingDirectory.str().size();
return IndexerCommandCxx::getByteSize() + sizeof(std::string) + m_workingDirectory.str().size();
}
FilePath IndexerCommandCxxCdb::getWorkingDirectory() const
src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp
+1 -1
View File
@@ -30,7 +30,7 @@ IndexerCommandType IndexerCommandCxxManual::getIndexerCommandType() const
size_t IndexerCommandCxxManual::getByteSize() const
{
return IndexerCommandCxx::getByteSize() + sizeof(*this);
return IndexerCommandCxx::getByteSize() + sizeof(std::string) + m_languageStandard.size();
}
std::string IndexerCommandCxxManual::getLanguageStandard() const
src/lib_java/data/indexer/IndexerCommandJava.cpp
+2 -2
View File
@@ -27,11 +27,11 @@ IndexerCommandType IndexerCommandJava::getIndexerCommandType() const
size_t IndexerCommandJava::getByteSize() const
{
size_t size = IndexerCommand::getByteSize() + sizeof(*this);
size_t size = IndexerCommand::getByteSize();
for (auto i : m_classPath)
{
size += i.str().size();
size += sizeof(std::string) + i.str().size();
}
return size;