logic: apply some shared memory fixes

* corrections in estimating size of intermediate indexer commands
* avoided clearing and refilling the indexed files list every time files get added
* changed memory growth policy of InterprocessIndexingStatusManager to double allocated memory when it gets full
* added potential to grow memory when adding crashed file paths

src/lib/data/indexer/IndexerCommand.cpp

@@ -19,12 +19,12 @@ size_t IndexerCommand::getByteSize() const
 
 	for (auto i : m_indexedPaths)
 	{
-		size += i.str().size();
+		size += sizeof(std::string) + i.str().size();
 	}
 
 	for (auto i : m_excludedPaths)
 	{
-		size += i.str().size();
+		size += sizeof(std::string) + i.str().size();
 	}
 
 	return size;

src/lib/data/indexer/interprocess/InterprocessIndexer.cpp

@@ -25,14 +25,15 @@ void InterprocessIndexer::work()
 {
 	try
 	{
-		LOG_INFO_STREAM(<< m_processId << " Starting to index");
+		LOG_INFO_STREAM(<< m_processId << " starting up indexer");
 		std::shared_ptr<IndexerBase> indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules();
 
 		while (std::shared_ptr<IndexerCommand> indexerCommand = m_interprocessIndexerCommandManager.popIndexerCommand())
 		{
-			LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str());
+			LOG_INFO_STREAM(<< m_processId << " fetched indexer command for \"" << indexerCommand->getSourceFilePath().str() << "\"");
-			LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
+			LOG_INFO_STREAM(<< m_processId << " indexer commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
 
+			LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
 			m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
 
 			FileRegisterStateData data;
@@ -42,13 +43,19 @@ void InterprocessIndexer::work()
 				data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath()
 			);
 
+			LOG_INFO_STREAM(<< m_processId << " starting to index current file");
 			std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
 
+			LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status");
 			m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
 
+			LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory");
 			m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
 
+			LOG_INFO_STREAM(<< m_processId << " finalizing indexer status for current file");
 			m_interprocessIndexingStatusManager.finishIndexingSourceFile();
+
+			LOG_INFO_STREAM(<< m_processId << " all done");
 		}
 	}
 	catch (boost::interprocess::interprocess_exception& e)
@@ -61,6 +68,10 @@ void InterprocessIndexer::work()
 		LOG_ERROR(e.what());
 		throw e;
 	}
+	catch (...)
+	{
+		LOG_ERROR("something went wrong while running the indexer");
+	}
 
-	LOG_INFO_STREAM(<< "Finished indexing");
+	LOG_INFO_STREAM(<< "shutting down indexer");
 }

src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp

@@ -19,26 +19,27 @@ InterprocessIndexerCommandManager::~InterprocessIndexerCommandManager()
 void InterprocessIndexerCommandManager::setIndexerCommands(
 	const std::vector<std::shared_ptr<IndexerCommand>>& indexerCommands)
 {
-	const unsigned int overestimationMultiplier = 3;
+	const unsigned int overestimationMultiplier = 2;
 
-	size_t size = 1000;
+	size_t estimatedSize = 1048576; /* 1 MB */
 	for (auto command : indexerCommands)
 	{
-		size += command->getByteSize() + sizeof(SharedIndexerCommand);
+		estimatedSize += command->getByteSize() + sizeof(SharedIndexerCommand);
 	}
-	size *= overestimationMultiplier;
+	estimatedSize *= overestimationMultiplier;
-
 
 	SharedMemory::ScopedAccess access(&m_sharedMemory);
 
 	size_t freeMemory = access.getFreeMemorySize();
-	if (freeMemory <= size)
+	if (freeMemory < estimatedSize)
 	{
 		LOG_INFO_STREAM(
-			<< "grow memory - est: " << size << " size: " << access.getMemorySize()
+			<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
-			<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
+			<< " free: " << access.getFreeMemorySize() << " alloc: " << (estimatedSize - freeMemory));
 
-		access.growMemory(size - freeMemory);
+		access.growMemory(estimatedSize - freeMemory);
+
+		LOG_INFO("growing memory succeeded");
 	}
 
 	SharedMemory::Queue<SharedIndexerCommand>* queue =

src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp

@@ -39,6 +39,28 @@ void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath&
 		SharedMemory::Map<Id, SharedMemory::String>::iterator it = currentFilesPtr->find(getProcessId());
 		if (it != currentFilesPtr->end())
 		{
+			const int overestimationMultiplier = 2;
+			const std::string crashedFilePath = it->second.c_str();
+
+			size_t estimatedSize = 262144 + sizeof(std::string) + crashedFilePath.size();
+			estimatedSize *= overestimationMultiplier;
+
+			while (access.getFreeMemorySize() < estimatedSize)
+			{
+				LOG_INFO_STREAM(
+					<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
+					<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
+				access.growMemory(access.getMemorySize());
+
+				LOG_INFO("growing memory succeeded");
+
+				currentFilesPtr = access.accessValueWithAllocator<SharedMemory::Map<Id, SharedMemory::String>>(s_currentFilesKeyName);
+				if (!currentFilesPtr)
+				{
+					return;
+				}
+			}
+
 			SharedMemory::Vector<SharedMemory::String>* crashedFilesPtr =
 				access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_crashedFilesKeyName);
 
@@ -164,43 +186,58 @@ std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
 
 void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
 {
-	const unsigned int overestimationMultiplier = 3;
+	const unsigned int overestimationMultiplier = 2;
 
 	SharedMemory::ScopedAccess access(&m_sharedMemory);
 
-	SharedMemory::Vector<SharedMemory::String>* files =
+	SharedMemory::Vector<SharedMemory::String>* indexedFiles =
 		access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
-	if (!files)
+	if (!indexedFiles)
 	{
 		return;
 	}
 
-	for (auto file : *files)
+	std::set<std::string> oldFiles;
+	for (auto indexedFile : *indexedFiles)
 	{
-		filePaths.insert(FilePath(file.c_str()));
+		oldFiles.insert(indexedFile.c_str());
 	}
-	files->clear();
 
-
+	std::set<std::string> newFiles;
-	size_t size = 1000;
+	for (const FilePath& filePath : filePaths)
-	for (auto path : filePaths)
 	{
-		size += sizeof(std::string) + path.str().size();
+		if (oldFiles.find(filePath.str()) == oldFiles.end())
+		{
+			newFiles.insert(filePath.str());
+		}
 	}
-	size *= overestimationMultiplier;
 
-	size_t freeMemory = access.getFreeMemorySize();
+	size_t estimatedSize = 262144;
-	if (freeMemory <= size)
+	for (auto newFile : newFiles)
+	{
+		estimatedSize += sizeof(std::string) + newFile.size();
+	}
+	estimatedSize *= overestimationMultiplier;
+
+	while (access.getFreeMemorySize() < estimatedSize)
 	{
 		LOG_INFO_STREAM(
-			<< "grow memory - est: " << size << " size: " << access.getMemorySize()
+			<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
-			<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
+			<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
-		access.growMemory(size - freeMemory);
+		access.growMemory(access.getMemorySize());
+
+		LOG_INFO("growing memory succeeded");
+
+		indexedFiles = access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
+		if (!indexedFiles)
+		{
+			return;
+		}
 	}
 
-	for (auto path : filePaths)
+	for (const std::string& newFile: newFiles)
 	{
-		files->push_back(SharedMemory::String(path.str().c_str(), access.getAllocator()));
+		indexedFiles->push_back(SharedMemory::String(newFile.c_str(), access.getAllocator()));
 	}
 
 	LOG_INFO(access.logString());

src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp

@@ -33,13 +33,15 @@ void InterprocessIntermediateStorageManager::pushIntermediateStorage(
 	SharedMemory::ScopedAccess access(&m_sharedMemory);
 
 	size_t freeMemory = access.getFreeMemorySize();
-	if (freeMemory <= size)
+	if (freeMemory < size)
 	{
 		LOG_INFO_STREAM(
 			<< "grow memory - est: " << size << " size: " << access.getMemorySize()
 			<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
 
 		access.growMemory(size - freeMemory);
+
+		LOG_INFO("growing memory succeeded");
 	}
 
 	SharedMemory::Queue<SharedIntermediateStorage>* queue =

src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp

@@ -198,6 +198,7 @@ void SharedIndexerCommand::setLanguageStandard(const std::string& languageStanda
 std::vector<std::string> SharedIndexerCommand::getCompilerFlags() const
 {
 	std::vector<std::string> result;
+	result.reserve(m_compilerFlags.size());
 
 	for (unsigned int i = 0; i <  m_compilerFlags.size(); i++)
 	{
@@ -210,6 +211,7 @@ std::vector<std::string> SharedIndexerCommand::getCompilerFlags() const
 void SharedIndexerCommand::setCompilerFlags(const std::vector<std::string>& compilerFlags)
 {
 	m_compilerFlags.clear();
+	m_compilerFlags.reserve(compilerFlags.size());
 
 	for (unsigned int i = 0; i < compilerFlags.size(); i++)
 	{
@@ -222,6 +224,7 @@ void SharedIndexerCommand::setCompilerFlags(const std::vector<std::string>& comp
 std::vector<FilePath> SharedIndexerCommand::getSystemHeaderSearchPaths() const
 {
 	std::vector<FilePath> result;
+	result.reserve(m_systemHeaderSearchPaths.size());
 
 	for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++)
 	{
@@ -234,6 +237,7 @@ std::vector<FilePath> SharedIndexerCommand::getSystemHeaderSearchPaths() const
 void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector<FilePath>& filePaths)
 {
 	m_systemHeaderSearchPaths.clear();
+	m_systemHeaderSearchPaths.reserve(filePaths.size());
 
 	for (unsigned int i = 0; i < filePaths.size(); i++)
 	{
@@ -246,6 +250,7 @@ void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector<FilePath
 std::vector<FilePath> SharedIndexerCommand::getFrameworkSearchhPaths() const
 {
 	std::vector<FilePath> result;
+	result.reserve(m_frameworkSearchPaths.size());
 
 	for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++)
 	{
@@ -258,6 +263,7 @@ std::vector<FilePath> SharedIndexerCommand::getFrameworkSearchhPaths() const
 void SharedIndexerCommand::setFrameworkSearchhPaths(const std::vector<FilePath>& searchPaths)
 {
 	m_frameworkSearchPaths.clear();
+	m_frameworkSearchPaths.reserve(searchPaths.size());
 
 	for (unsigned int i = 0; i < searchPaths.size(); i++)
 	{
@@ -280,6 +286,7 @@ void SharedIndexerCommand::setPreprocessorOnly(bool preprocessorOnly)
 std::vector<FilePath> SharedIndexerCommand::getClassPaths() const
 {
 	std::vector<FilePath> result;
+	result.reserve(m_classPaths.size());
 
 	for (unsigned int i = 0; i < m_classPaths.size(); i++)
 	{
@@ -292,6 +299,7 @@ std::vector<FilePath> SharedIndexerCommand::getClassPaths() const
 void SharedIndexerCommand::setClassPaths(const std::vector<FilePath>& classPaths)
 {
 	m_classPaths.clear();
+	m_classPaths.reserve(classPaths.size());
 
 	for (unsigned int i = 0; i < classPaths.size(); i++)
 	{

src/lib_cxx/data/indexer/IndexerCommandCxx.cpp

@@ -26,17 +26,17 @@ size_t IndexerCommandCxx::getByteSize() const
 
 	for (auto i : m_systemHeaderSearchPaths)
 	{
-		size += i.str().size();
+		size += sizeof(std::string) + i.str().size();
 	}
 
 	for (auto i : m_frameworkSearchPaths)
 	{
-		size += i.str().size();
+		size += sizeof(std::string) + i.str().size();
 	}
 
 	for (auto i : m_compilerFlags)
 	{
-		size += i.size();
+		size += sizeof(std::string) + i.size();
 	}
 
 	return size;

src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp

@@ -64,7 +64,7 @@ IndexerCommandType IndexerCommandCxxCdb::getIndexerCommandType() const
 
 size_t IndexerCommandCxxCdb::getByteSize() const
 {
-	return IndexerCommandCxx::getByteSize() + sizeof(*this) + m_workingDirectory.str().size();
+	return IndexerCommandCxx::getByteSize() + sizeof(std::string) + m_workingDirectory.str().size();
 }
 
 FilePath IndexerCommandCxxCdb::getWorkingDirectory() const

src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp

@@ -30,7 +30,7 @@ IndexerCommandType IndexerCommandCxxManual::getIndexerCommandType() const
 
 size_t IndexerCommandCxxManual::getByteSize() const
 {
-	return IndexerCommandCxx::getByteSize() + sizeof(*this);
+	return IndexerCommandCxx::getByteSize() + sizeof(std::string) + m_languageStandard.size();
 }
 
 std::string IndexerCommandCxxManual::getLanguageStandard() const

src/lib_java/data/indexer/IndexerCommandJava.cpp

@@ -27,11 +27,11 @@ IndexerCommandType IndexerCommandJava::getIndexerCommandType() const
 
 size_t IndexerCommandJava::getByteSize() const
 {
-	size_t size = IndexerCommand::getByteSize() + sizeof(*this);
+	size_t size = IndexerCommand::getByteSize();
 
 	for (auto i : m_classPath)
 	{
-		size += i.str().size();
+		size += sizeof(std::string) + i.str().size();
 	}
 
 	return size;