diff --git a/src/lib/data/indexer/IndexerCommand.cpp b/src/lib/data/indexer/IndexerCommand.cpp index 490bf51f..9c9d5a59 100644 --- a/src/lib/data/indexer/IndexerCommand.cpp +++ b/src/lib/data/indexer/IndexerCommand.cpp @@ -19,12 +19,12 @@ size_t IndexerCommand::getByteSize() const for (auto i : m_indexedPaths) { - size += i.str().size(); + size += sizeof(std::string) + i.str().size(); } for (auto i : m_excludedPaths) { - size += i.str().size(); + size += sizeof(std::string) + i.str().size(); } return size; diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp index 05c9bec9..5cbfa323 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp +++ b/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp @@ -25,14 +25,15 @@ void InterprocessIndexer::work() { try { - LOG_INFO_STREAM(<< m_processId << " Starting to index"); + LOG_INFO_STREAM(<< m_processId << " starting up indexer"); std::shared_ptr indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules(); while (std::shared_ptr indexerCommand = m_interprocessIndexerCommandManager.popIndexerCommand()) { - LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str()); - LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1)); + LOG_INFO_STREAM(<< m_processId << " fetched indexer command for \"" << indexerCommand->getSourceFilePath().str() << "\""); + LOG_INFO_STREAM(<< m_processId << " indexer commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1)); + LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath"); m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath()); FileRegisterStateData data; @@ -42,13 +43,19 @@ void InterprocessIndexer::work() data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath() ); + LOG_INFO_STREAM(<< m_processId << " starting to index current file"); std::shared_ptr result = indexer->index(indexerCommand, fileRegister); + LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status"); m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles()); + LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory"); m_interprocessIntermediateStorageManager.pushIntermediateStorage(result); + LOG_INFO_STREAM(<< m_processId << " finalizing indexer status for current file"); m_interprocessIndexingStatusManager.finishIndexingSourceFile(); + + LOG_INFO_STREAM(<< m_processId << " all done"); } } catch (boost::interprocess::interprocess_exception& e) @@ -61,6 +68,10 @@ void InterprocessIndexer::work() LOG_ERROR(e.what()); throw e; } + catch (...) + { + LOG_ERROR("something went wrong while running the indexer"); + } - LOG_INFO_STREAM(<< "Finished indexing"); + LOG_INFO_STREAM(<< "shutting down indexer"); } diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp index 07c0ff84..63f77051 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp +++ b/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp @@ -19,26 +19,27 @@ InterprocessIndexerCommandManager::~InterprocessIndexerCommandManager() void InterprocessIndexerCommandManager::setIndexerCommands( const std::vector>& indexerCommands) { - const unsigned int overestimationMultiplier = 3; + const unsigned int overestimationMultiplier = 2; - size_t size = 1000; + size_t estimatedSize = 1048576; /* 1 MB */ for (auto command : indexerCommands) { - size += command->getByteSize() + sizeof(SharedIndexerCommand); + estimatedSize += command->getByteSize() + sizeof(SharedIndexerCommand); } - size *= overestimationMultiplier; - + estimatedSize *= overestimationMultiplier; SharedMemory::ScopedAccess access(&m_sharedMemory); size_t freeMemory = access.getFreeMemorySize(); - if (freeMemory <= size) + if (freeMemory < estimatedSize) { LOG_INFO_STREAM( - << "grow memory - est: " << size << " size: " << access.getMemorySize() - << " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory)); + << "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize() + << " free: " << access.getFreeMemorySize() << " alloc: " << (estimatedSize - freeMemory)); - access.growMemory(size - freeMemory); + access.growMemory(estimatedSize - freeMemory); + + LOG_INFO("growing memory succeeded"); } SharedMemory::Queue* queue = diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp index 41d94f76..69583b35 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp +++ b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp @@ -39,6 +39,28 @@ void InterprocessIndexingStatusManager::startIndexingSourceFile(const FilePath& SharedMemory::Map::iterator it = currentFilesPtr->find(getProcessId()); if (it != currentFilesPtr->end()) { + const int overestimationMultiplier = 2; + const std::string crashedFilePath = it->second.c_str(); + + size_t estimatedSize = 262144 + sizeof(std::string) + crashedFilePath.size(); + estimatedSize *= overestimationMultiplier; + + while (access.getFreeMemorySize() < estimatedSize) + { + LOG_INFO_STREAM( + << "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize() + << " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize())); + access.growMemory(access.getMemorySize()); + + LOG_INFO("growing memory succeeded"); + + currentFilesPtr = access.accessValueWithAllocator>(s_currentFilesKeyName); + if (!currentFilesPtr) + { + return; + } + } + SharedMemory::Vector* crashedFilesPtr = access.accessValueWithAllocator>(s_crashedFilesKeyName); @@ -164,43 +186,58 @@ std::set InterprocessIndexingStatusManager::getIndexedFiles() void InterprocessIndexingStatusManager::addIndexedFiles(std::set filePaths) { - const unsigned int overestimationMultiplier = 3; + const unsigned int overestimationMultiplier = 2; SharedMemory::ScopedAccess access(&m_sharedMemory); - SharedMemory::Vector* files = + SharedMemory::Vector* indexedFiles = access.accessValueWithAllocator>(s_indexedFilesKeyName); - if (!files) + if (!indexedFiles) { return; } - for (auto file : *files) + std::set oldFiles; + for (auto indexedFile : *indexedFiles) { - filePaths.insert(FilePath(file.c_str())); + oldFiles.insert(indexedFile.c_str()); } - files->clear(); - - size_t size = 1000; - for (auto path : filePaths) + std::set newFiles; + for (const FilePath& filePath : filePaths) { - size += sizeof(std::string) + path.str().size(); + if (oldFiles.find(filePath.str()) == oldFiles.end()) + { + newFiles.insert(filePath.str()); + } } - size *= overestimationMultiplier; - size_t freeMemory = access.getFreeMemorySize(); - if (freeMemory <= size) + size_t estimatedSize = 262144; + for (auto newFile : newFiles) + { + estimatedSize += sizeof(std::string) + newFile.size(); + } + estimatedSize *= overestimationMultiplier; + + while (access.getFreeMemorySize() < estimatedSize) { LOG_INFO_STREAM( - << "grow memory - est: " << size << " size: " << access.getMemorySize() - << " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory)); - access.growMemory(size - freeMemory); + << "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize() + << " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize())); + access.growMemory(access.getMemorySize()); + + LOG_INFO("growing memory succeeded"); + + indexedFiles = access.accessValueWithAllocator>(s_indexedFilesKeyName); + if (!indexedFiles) + { + return; + } } - for (auto path : filePaths) + for (const std::string& newFile: newFiles) { - files->push_back(SharedMemory::String(path.str().c_str(), access.getAllocator())); + indexedFiles->push_back(SharedMemory::String(newFile.c_str(), access.getAllocator())); } LOG_INFO(access.logString()); diff --git a/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp index 4960b372..4e7ba6b6 100644 --- a/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp +++ b/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp @@ -33,13 +33,15 @@ void InterprocessIntermediateStorageManager::pushIntermediateStorage( SharedMemory::ScopedAccess access(&m_sharedMemory); size_t freeMemory = access.getFreeMemorySize(); - if (freeMemory <= size) + if (freeMemory < size) { LOG_INFO_STREAM( << "grow memory - est: " << size << " size: " << access.getMemorySize() << " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory)); access.growMemory(size - freeMemory); + + LOG_INFO("growing memory succeeded"); } SharedMemory::Queue* queue = diff --git a/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp b/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp index 56ac64cc..c6960bbf 100644 --- a/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp +++ b/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp @@ -198,6 +198,7 @@ void SharedIndexerCommand::setLanguageStandard(const std::string& languageStanda std::vector SharedIndexerCommand::getCompilerFlags() const { std::vector result; + result.reserve(m_compilerFlags.size()); for (unsigned int i = 0; i < m_compilerFlags.size(); i++) { @@ -210,6 +211,7 @@ std::vector SharedIndexerCommand::getCompilerFlags() const void SharedIndexerCommand::setCompilerFlags(const std::vector& compilerFlags) { m_compilerFlags.clear(); + m_compilerFlags.reserve(compilerFlags.size()); for (unsigned int i = 0; i < compilerFlags.size(); i++) { @@ -222,6 +224,7 @@ void SharedIndexerCommand::setCompilerFlags(const std::vector& comp std::vector SharedIndexerCommand::getSystemHeaderSearchPaths() const { std::vector result; + result.reserve(m_systemHeaderSearchPaths.size()); for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++) { @@ -234,6 +237,7 @@ std::vector SharedIndexerCommand::getSystemHeaderSearchPaths() const void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector& filePaths) { m_systemHeaderSearchPaths.clear(); + m_systemHeaderSearchPaths.reserve(filePaths.size()); for (unsigned int i = 0; i < filePaths.size(); i++) { @@ -246,6 +250,7 @@ void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector SharedIndexerCommand::getFrameworkSearchhPaths() const { std::vector result; + result.reserve(m_frameworkSearchPaths.size()); for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++) { @@ -258,6 +263,7 @@ std::vector SharedIndexerCommand::getFrameworkSearchhPaths() const void SharedIndexerCommand::setFrameworkSearchhPaths(const std::vector& searchPaths) { m_frameworkSearchPaths.clear(); + m_frameworkSearchPaths.reserve(searchPaths.size()); for (unsigned int i = 0; i < searchPaths.size(); i++) { @@ -280,6 +286,7 @@ void SharedIndexerCommand::setPreprocessorOnly(bool preprocessorOnly) std::vector SharedIndexerCommand::getClassPaths() const { std::vector result; + result.reserve(m_classPaths.size()); for (unsigned int i = 0; i < m_classPaths.size(); i++) { @@ -292,6 +299,7 @@ std::vector SharedIndexerCommand::getClassPaths() const void SharedIndexerCommand::setClassPaths(const std::vector& classPaths) { m_classPaths.clear(); + m_classPaths.reserve(classPaths.size()); for (unsigned int i = 0; i < classPaths.size(); i++) { diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp b/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp index b04355a4..130c952b 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp +++ b/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp @@ -26,17 +26,17 @@ size_t IndexerCommandCxx::getByteSize() const for (auto i : m_systemHeaderSearchPaths) { - size += i.str().size(); + size += sizeof(std::string) + i.str().size(); } for (auto i : m_frameworkSearchPaths) { - size += i.str().size(); + size += sizeof(std::string) + i.str().size(); } for (auto i : m_compilerFlags) { - size += i.size(); + size += sizeof(std::string) + i.size(); } return size; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp b/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp index e04ded9a..426acb03 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp +++ b/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp @@ -64,7 +64,7 @@ IndexerCommandType IndexerCommandCxxCdb::getIndexerCommandType() const size_t IndexerCommandCxxCdb::getByteSize() const { - return IndexerCommandCxx::getByteSize() + sizeof(*this) + m_workingDirectory.str().size(); + return IndexerCommandCxx::getByteSize() + sizeof(std::string) + m_workingDirectory.str().size(); } FilePath IndexerCommandCxxCdb::getWorkingDirectory() const diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp b/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp index 369552f3..a48dc633 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp +++ b/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp @@ -30,7 +30,7 @@ IndexerCommandType IndexerCommandCxxManual::getIndexerCommandType() const size_t IndexerCommandCxxManual::getByteSize() const { - return IndexerCommandCxx::getByteSize() + sizeof(*this); + return IndexerCommandCxx::getByteSize() + sizeof(std::string) + m_languageStandard.size(); } std::string IndexerCommandCxxManual::getLanguageStandard() const diff --git a/src/lib_java/data/indexer/IndexerCommandJava.cpp b/src/lib_java/data/indexer/IndexerCommandJava.cpp index 162552ea..34930d4f 100644 --- a/src/lib_java/data/indexer/IndexerCommandJava.cpp +++ b/src/lib_java/data/indexer/IndexerCommandJava.cpp @@ -27,11 +27,11 @@ IndexerCommandType IndexerCommandJava::getIndexerCommandType() const size_t IndexerCommandJava::getByteSize() const { - size_t size = IndexerCommand::getByteSize() + sizeof(*this); + size_t size = IndexerCommand::getByteSize(); for (auto i : m_classPath) { - size += i.str().size(); + size += sizeof(std::string) + i.str().size(); } return size;