From 706119ebccb2dc4b7267aa654858518a08b41178 Mon Sep 17 00:00:00 2001 From: Eberhard Graether Date: Thu, 4 May 2017 15:50:59 +0200 Subject: [PATCH] data: indexer processes * TaskBuildIndex starts seperate indexer processes and communicates via shared memory * indexer processes get restarted if they fail * indexer processes are killed when the app closes or crashes * added utility class SharedMemory to allocate and access shared memory, providing basic data structures * added SharedMemoryGarbageCollector for keeping track of running instances and cleaning up shared memory in case of a crash * show errors for source files that crashed during indexing * added basic exception handling to task scheduling * added option to turn off processes and use threads in preferences, but still use shared memory fortune cookie message = Good news from someone dear is coming soon. --- CMakeLists.txt | 75 ++++- bin/app/user/ApplicationSettings_template.xml | 2 +- cmake/LinuxPackage.cmake | 6 + deployment/windows/wixSetup/installDir.wxs | 47 ++- deployment/windows/wixSetup/sourcetrail.wxs | 19 +- script/build.sh | 12 +- setup/MacOSX/bundle_install.sh.in | 12 +- src/app/main.cpp | 12 +- src/indexer/CMakeLists.txt | 5 + src/indexer/main.cpp | 88 +++++ src/lib/Application.cpp | 21 +- src/lib/Application.h | 3 + src/lib/CMakeLists.txt | 33 +- src/lib/component/view/DialogView.cpp | 6 +- src/lib/component/view/DialogView.h | 4 +- src/lib/data/IntermediateStorage.cpp | 302 +++++++++++++---- src/lib/data/IntermediateStorage.h | 50 ++- src/lib/data/PersistentStorage.cpp | 18 +- src/lib/data/PersistentStorage.h | 8 +- src/lib/data/Storage.cpp | 16 +- src/lib/data/Storage.h | 8 +- src/lib/data/StorageTypes.h | 3 - src/lib/data/TaskFinishParsing.cpp | 19 +- src/lib/data/TaskFinishParsing.h | 1 + src/lib/data/indexer/Indexer.h | 4 +- src/lib/data/indexer/IndexerBase.h | 4 +- src/lib/data/indexer/IndexerCommand.cpp | 32 +- src/lib/data/indexer/IndexerCommand.h | 6 +- src/lib/data/indexer/IndexerCommandList.cpp | 6 + src/lib/data/indexer/IndexerCommandList.h | 2 + src/lib/data/indexer/IndexerComposite.cpp | 3 +- src/lib/data/indexer/IndexerComposite.h | 5 +- src/lib/data/indexer/TaskBuildIndex.cpp | 245 ++++++++++---- src/lib/data/indexer/TaskBuildIndex.h | 33 +- .../BaseInterprocessDataManager.cpp | 23 ++ .../BaseInterprocessDataManager.h | 29 ++ .../interprocess/InterprocessIndexer.cpp | 66 ++++ .../interprocess/InterprocessIndexer.h | 25 ++ .../InterprocessIndexerCommandManager.cpp | 105 ++++++ .../InterprocessIndexerCommandManager.h | 27 ++ .../InterprocessIndexingStatusManager.cpp | 173 ++++++++++ .../InterprocessIndexingStatusManager.h | 34 ++ ...InterprocessIntermediateStorageManager.cpp | 115 +++++++ .../InterprocessIntermediateStorageManager.h | 26 ++ .../shared_types/SharedIndexerCommand.cpp | 312 ++++++++++++++++++ .../shared_types/SharedIndexerCommand.h | 81 +++++ .../SharedIntermediateStorage.cpp | 251 ++++++++++++++ .../shared_types/SharedIntermediateStorage.h | 64 ++++ .../shared_types/SharedStorageTypes.h | 152 +++++++++ src/lib/data/parser/ParserClient.cpp | 11 - src/lib/data/parser/ParserClient.h | 4 - src/lib/project/Project.cpp | 27 +- src/lib/settings/ApplicationSettings.cpp | 8 +- src/lib/settings/ApplicationSettings.h | 4 +- src/lib/utility/TimePoint.cpp | 5 + src/lib/utility/TimePoint.h | 1 + src/lib/utility/UUIDUtility.cpp | 6 +- src/lib/utility/UUIDUtility.h | 8 +- .../utility/file/FileRegisterStateData.cpp | 26 +- src/lib/utility/file/FileRegisterStateData.h | 7 +- .../interprocess/InterprocessDataManager.cpp | 110 ------ .../interprocess/InterprocessDataManager.h | 38 --- .../InterprocessProcessManager.cpp | 65 ---- .../interprocess/InterprocessProcessManager.h | 27 -- .../interprocess/InterprocessUtility.h | 15 - .../utility/interprocess/SharedContainer.h | 95 ------ src/lib/utility/interprocess/SharedMap.h | 125 ------- src/lib/utility/interprocess/SharedMemory.cpp | 145 ++++++++ src/lib/utility/interprocess/SharedMemory.h | 116 +++++++ .../SharedMemoryGarbageCollector.cpp | 242 ++++++++++++++ .../SharedMemoryGarbageCollector.h | 49 +++ .../interprocess/SharedParserArguments.cpp | 173 ---------- .../interprocess/SharedParserArguments.h | 67 ---- src/lib/utility/interprocess/SharedQueue.h | 128 ------- .../utility/interprocess/SharedUUIDManager.h | 47 --- src/lib/utility/logging/FileLogger.cpp | 32 +- src/lib/utility/logging/FileLogger.h | 7 +- .../utility/scheduling/TaskGroupParallel.cpp | 18 +- .../utility/scheduling/TaskGroupParallel.h | 7 +- src/lib/utility/scheduling/TaskRunner.cpp | 25 +- src/lib/utility/scheduling/TaskScheduler.cpp | 20 +- src/lib/utility/scheduling/TaskScheduler.h | 3 + .../data/indexer/IndexerCommandCxx.cpp | 22 ++ src/lib_cxx/data/indexer/IndexerCommandCxx.h | 1 + .../data/indexer/IndexerCommandCxxCdb.cpp | 5 + .../data/indexer/IndexerCommandCxxCdb.h | 1 + .../data/indexer/IndexerCommandCxxManual.cpp | 5 + .../data/indexer/IndexerCommandCxxManual.h | 2 + src/lib_cxx/data/parser/cxx/ASTConsumer.cpp | 5 +- src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp | 2 +- src/lib_cxx/data/parser/cxx/CxxParser.h | 1 - src/lib_gui/platform_includes/includesMac.h | 3 +- .../qt/element/QtCodeFileTitleButton.cpp | 7 +- src/lib_gui/qt/view/QtDialogView.cpp | 15 +- src/lib_gui/qt/view/QtDialogView.h | 4 +- src/lib_gui/qt/window/QtIndexingDialog.cpp | 4 +- src/lib_gui/qt/window/QtIndexingDialog.h | 3 +- .../QtProjectWizzardContentPreferences.cpp | 11 +- .../QtProjectWizzardContentPreferences.h | 2 +- src/lib_gui/utility/utilityApp.cpp | 59 +++- src/lib_gui/utility/utilityApp.h | 7 +- .../data/indexer/IndexerCommandJava.cpp | 12 + .../data/indexer/IndexerCommandJava.h | 3 +- src/lib_java/data/parser/java/JavaParser.cpp | 39 ++- src/lib_java/data/parser/java/JavaParser.h | 6 +- src/lib_java/project/SourceGroupJava.cpp | 42 +-- src/test/CMakeLists.txt | 1 + src/test/JavaParserTestSuite.h | 1 - src/test/SharedMemoryTestSuite.h | 95 ++++++ 109 files changed, 3351 insertions(+), 1261 deletions(-) create mode 100644 src/indexer/CMakeLists.txt create mode 100644 src/indexer/main.cpp create mode 100644 src/lib/data/indexer/interprocess/BaseInterprocessDataManager.cpp create mode 100644 src/lib/data/indexer/interprocess/BaseInterprocessDataManager.h create mode 100644 src/lib/data/indexer/interprocess/InterprocessIndexer.cpp create mode 100644 src/lib/data/indexer/interprocess/InterprocessIndexer.h create mode 100644 src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp create mode 100644 src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.h create mode 100644 src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp create mode 100644 src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h create mode 100644 src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp create mode 100644 src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.h create mode 100644 src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp create mode 100644 src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.h create mode 100644 src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp create mode 100644 src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.h create mode 100644 src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h delete mode 100644 src/lib/utility/interprocess/InterprocessDataManager.cpp delete mode 100644 src/lib/utility/interprocess/InterprocessDataManager.h delete mode 100644 src/lib/utility/interprocess/InterprocessProcessManager.cpp delete mode 100644 src/lib/utility/interprocess/InterprocessProcessManager.h delete mode 100644 src/lib/utility/interprocess/InterprocessUtility.h delete mode 100644 src/lib/utility/interprocess/SharedContainer.h delete mode 100644 src/lib/utility/interprocess/SharedMap.h create mode 100644 src/lib/utility/interprocess/SharedMemory.cpp create mode 100644 src/lib/utility/interprocess/SharedMemory.h create mode 100644 src/lib/utility/interprocess/SharedMemoryGarbageCollector.cpp create mode 100644 src/lib/utility/interprocess/SharedMemoryGarbageCollector.h delete mode 100644 src/lib/utility/interprocess/SharedParserArguments.cpp delete mode 100644 src/lib/utility/interprocess/SharedParserArguments.h delete mode 100644 src/lib/utility/interprocess/SharedQueue.h delete mode 100644 src/lib/utility/interprocess/SharedUUIDManager.h create mode 100644 src/test/SharedMemoryTestSuite.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dcafb736..b3d8df46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ include(cmake/setWinXxBits.cmake) # Variables -------------------------------------------------------------------- set(PROJECT_NAME Sourcetrail) +set(PROJECT_NAME_LOWER_CASE sourcetrail) set(APP_PROJECT_NAME "${PROJECT_NAME}") set(LIB_LICENSE_PROJECT_NAME "${PROJECT_NAME}_lib_license") @@ -24,6 +25,8 @@ set(LIB_PROJECT_NAME "${PROJECT_NAME}_lib") set(LICENSE_GENERATOR_PROJECT_NAME "${PROJECT_NAME}_license_generator") set(TEST_PROJECT_NAME "${PROJECT_NAME}_test") +set (APP_INDEXER_NAME "${PROJECT_NAME_LOWER_CASE}_indexer") + if (WIN32) set(PLATFORM_INCLUDE "includesWindows.h") # this adds the library path for VLD, must be done before the project target is created @@ -221,6 +224,8 @@ set_property( "${CMAKE_SOURCE_DIR}/src/lib_license" "${CMAKE_SOURCE_DIR}/src/lib_gui" "${CMAKE_BINARY_DIR}/src/lib_license" + "${CMAKE_SOURCE_DIR}/src/lib_cxx" + "${CMAKE_SOURCE_DIR}/src/lib_java" ) target_include_directories(${LIB_PROJECT_NAME} SYSTEM @@ -398,6 +403,60 @@ include(cmake/publicKey.cmake) set(CMAKE_AUTOMOC OFF) + +# Indexer App ------------------------------------------------------------------ + +if (UNIX) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/app/") +else () + foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} ) + string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG ) + set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} "${CMAKE_BINARY_DIR}/${OUTPUTCONFIG}/app/") + endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES ) +endif () + +add_subdirectory(src/indexer) + +# Find includes in corresponding build directories +set(CMAKE_INCLUDE_CURRENT_DIR ON) + +add_executable(${APP_INDEXER_NAME} ${INDEXER_FILES}) + +if (WIN32) + # hide the console when running a release build. + set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_DEBUG "/SUBSYSTEM:CONSOLE /DEBUG:FASTLINK") + set_target_properties(${APP_INDEXER_NAME} PROPERTIES COMPILE_DEFINITIONS_DEBUG "_CONSOLE") + set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_RELWITHDEBINFO "/SUBSYSTEM:CONSOLE") + set_target_properties(${APP_INDEXER_NAME} PROPERTIES COMPILE_DEFINITIONS_RELWITHDEBINFO "_CONSOLE") + set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_RELEASE "/ENTRY:\"mainCRTStartup\" /SUBSYSTEM:WINDOWS /DEBUG") + set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_MINSIZEREL "/SUBSYSTEM:WINDOWS") + + # generate pdb for release build + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") +endif () + +create_source_groups(${INDEXER_FILES}) + +target_link_libraries(${APP_INDEXER_NAME} ${LIB_GUI_PROJECT_NAME} ${LIB_CXX_PROJECT_NAME} ${LIB_JAVA_PROJECT_NAME} ${LIB_PROJECT_NAME}) + +if (APPLE) + find_library(CORE_FOUNDATION CoreFoundation) + target_link_libraries(${APP_INDEXER_NAME} ${CORE_FOUNDATION}) +endif () + +set_property( + TARGET ${APP_INDEXER_NAME} + PROPERTY INCLUDE_DIRECTORIES + "${CMAKE_SOURCE_DIR}/src/app" + "${CMAKE_SOURCE_DIR}/src/lib" + "${CMAKE_SOURCE_DIR}/src/lib_gui" + "${CMAKE_SOURCE_DIR}/src/lib_license" + "${CMAKE_SOURCE_DIR}/src/lib_cxx" + "${CMAKE_SOURCE_DIR}/src/lib_java" + "${CMAKE_BINARY_DIR}/src/lib_gui" + "${CMAKE_BINARY_DIR}/src/lib_license" +) + # App -------------------------------------------------------------------------- if (UNIX) @@ -418,7 +477,7 @@ if (WIN32) file(WRITE ${CMAKE_BINARY_DIR}/Sourcetrail.rc "// Icon with lowest ID value placed first to ensure application icon\n" "// remains consistent on all systems.\n" - "IDI_ICON1 ICON \"${CMAKE_BINARY_DIR}/sourcetrail.ico\"\n" + "IDI_ICON1 ICON \"${CMAKE_BINARY_DIR}/Sourcetrail.ico\"\n" ) add_executable(${APP_PROJECT_NAME} ${APP_FILES} ${CMAKE_BINARY_DIR}/Sourcetrail.rc) @@ -469,6 +528,8 @@ if (APPLE) target_link_libraries(${APP_PROJECT_NAME} ${CORE_FOUNDATION}) endif () +add_dependencies(${APP_PROJECT_NAME} ${APP_INDEXER_NAME}) + # MacOSX Bundle ---------------------------------------------------------------- @@ -476,6 +537,7 @@ if (APPLE) set(MACOSX_BUNDLE_NAME ${PROJECT_NAME}) set(MACOSX_BINARY_NAME ${APP_PROJECT_NAME}) + set(MACOSX_INDEXER_BINARY_NAME ${APP_INDEXER_NAME}) set(MACOSX_DYNAMIC_LIBRARIES "") string(REPLACE ";" " " MACOSX_DYNAMIC_LIBRARIES "${MACOSX_DYNAMIC_LIBRARIES}") @@ -625,7 +687,7 @@ if (WIN32) include_directories($ENV{VLD_DIR}/include) endif () -if(UNIX AND NOT APPLE) +if (UNIX) # symlinks for data message(STATUS "create symlink: " "${CMAKE_SOURCE_DIR}/bin/app/data ->" @@ -641,6 +703,9 @@ if(UNIX AND NOT APPLE) "${CMAKE_SOURCE_DIR}/bin/app/user" "${CMAKE_BINARY_DIR}/app/user" ) - # Linux package - include(cmake/LinuxPackage.cmake) -endif () + + if (UNIX AND NOT APPLE) + # Linux package + include(cmake/LinuxPackage.cmake) + endif () +endif () diff --git a/bin/app/user/ApplicationSettings_template.xml b/bin/app/user/ApplicationSettings_template.xml index a64eaa05..8f647fd0 100644 --- a/bin/app/user/ApplicationSettings_template.xml +++ b/bin/app/user/ApplicationSettings_template.xml @@ -32,7 +32,7 @@ - + diff --git a/cmake/LinuxPackage.cmake b/cmake/LinuxPackage.cmake index 92f91c25..12680c39 100644 --- a/cmake/LinuxPackage.cmake +++ b/cmake/LinuxPackage.cmake @@ -158,6 +158,12 @@ INSTALL(TARGETS COMPONENT FULL ) +INSTALL(TARGETS + ${APP_INDEXER_NAME} + DESTINATION Sourcetrail + COMPONENT FULL +) + # SET(CPACK_GENERATOR "DEB;TGZ") SET(CPACK_GENERATOR "TGZ") SET(CPACK_ARCHIVE_COMPONENT_INSTALL "ON") diff --git a/deployment/windows/wixSetup/installDir.wxs b/deployment/windows/wixSetup/installDir.wxs index 7db68371..797b79b1 100644 --- a/deployment/windows/wixSetup/installDir.wxs +++ b/deployment/windows/wixSetup/installDir.wxs @@ -29,6 +29,10 @@ + + + + @@ -323,6 +327,12 @@ + + + + + + @@ -335,12 +345,6 @@ - - - - - - @@ -383,8 +387,14 @@ - - + + + + + + + + @@ -414,12 +424,33 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/deployment/windows/wixSetup/sourcetrail.wxs b/deployment/windows/wixSetup/sourcetrail.wxs index ff12769c..a5439926 100644 --- a/deployment/windows/wixSetup/sourcetrail.wxs +++ b/deployment/windows/wixSetup/sourcetrail.wxs @@ -166,11 +166,13 @@ - + + + @@ -253,12 +255,12 @@ + + - - @@ -272,7 +274,9 @@ - + + + @@ -281,8 +285,15 @@ + + + + + + + diff --git a/script/build.sh b/script/build.sh index 0263c138..0a3c5008 100755 --- a/script/build.sh +++ b/script/build.sh @@ -19,28 +19,28 @@ then if [ "$2" = "test" ] then #echo "release test" - cd bin/test && ../../build/Release/test/Sourcetrail_test + cd build/Release/test/ && ./Sourcetrail_test elif [ "$2" = "keygen" ] then #echo "release keygen" - cd bin/license_generator && ../../build/Release/license_generator/Sourcetrail_license_generator + cd build/Release/license_generator && ./Sourcetrail_license_generator else #echo "release app" - cd bin/app && ../../build/Release/app/Sourcetrail + cd build/Release/app && ./Sourcetrail fi elif [ "$1" = "debug" ] || [ "$1" = "d" ] then if [ "$2" = "test" ] then #echo "debug test" - cd bin/test && ../../build/Debug/test/Sourcetrail_test + cd build/Debug/test && ./Sourcetrail_test elif [ "$2" = "keygen" ] then #echo "debug keygen" - cd bin/license_generator && ../../build/Debug/license_generator/Sourcetrail_license_generator + cd build/Debug/license_generator && ./Sourcetrail_license_generator else #echo "debug app" - cd bin/app && ../../build/Debug/app/Sourcetrail + cd build/Debug/app && ./Sourcetrail fi else echo "no arguments: first argument 'release' or 'debug', second argument 'test' for tests" diff --git a/setup/MacOSX/bundle_install.sh.in b/setup/MacOSX/bundle_install.sh.in index 4e0b4441..93bae3a3 100755 --- a/setup/MacOSX/bundle_install.sh.in +++ b/setup/MacOSX/bundle_install.sh.in @@ -11,6 +11,7 @@ echo -e $INFO "Creating app bundle for MacOSX." BUNDLE_NAME="@MACOSX_BUNDLE_NAME@" APP_NAME="@MACOSX_BINARY_NAME@" +INDEXER_NAME="@MACOSX_INDEXER_BINARY_NAME@" DYNLIB_PATHS=(@MACOSX_DYNAMIC_LIBRARIES@) VERSION_STRING=$(git describe --long) @@ -49,6 +50,7 @@ PLUGIN_DIR=$BUNDLE_PATH/Contents/PlugIns FRAMEWORK_DIR=$BUNDLE_PATH/Contents/Frameworks RES_DIR=$BUNDLE_PATH/Contents/Resources APP_PATH=$BIN_DIR/$APP_NAME +INDEXER_PATH=$RES_DIR/$INDEXER_NAME echo -e $INFO "Creating bundle folders." mkdir -p $BIN_DIR @@ -76,6 +78,8 @@ cp $APP_NAME $APP_PATH cp bundle_info.plist $BUNDLE_PATH/Contents/Info.plist +cp $INDEXER_NAME $INDEXER_PATH + mkdir -p $RES_DIR/data cp -R ../../../bin/app/data/color_schemes $RES_DIR/data/color_schemes @@ -95,6 +99,7 @@ do install_name_tool -change $LIB_NAME @executable_path/../lib/$LIB_NAME $APP_PATH install_name_tool -change @rpath/$LIB_NAME @executable_path/../lib/$LIB_NAME $APP_PATH + install_name_tool -change @rpath/$LIB_NAME @executable_path/../lib/$LIB_NAME $INDEXER_PATH echo "lib" $LIB_PATH $LIB_NAME done @@ -122,6 +127,7 @@ do FRAMEWORK_SUB_PATH=$(echo $FRAMEWORK_BIN_PATH | grep -o "\w*.framework\S*") install_name_tool -id @executable_path/../Frameworks/$FRAMEWORK_SUB_PATH $FRAMEWORK_DIR/$FRAMEWORK_SUB_PATH install_name_tool -change @rpath/$FRAMEWORK_SUB_PATH @executable_path/../Frameworks/$FRAMEWORK_SUB_PATH $APP_PATH + install_name_tool -change @rpath/$FRAMEWORK_SUB_PATH @executable_path/../Frameworks/$FRAMEWORK_SUB_PATH $INDEXER_PATH echo "framework" $FRAMEWORK_BIN_PATH $FRAMEWORK_PATH done @@ -160,10 +166,12 @@ otool -L $FRAMEWORK_DIR/QtNetwork.framework/Versions/5/QtNetwork otool -L $PLUGIN_DIR/platforms/libqcocoa.dylib otool -L $PLUGIN_DIR/imageformats/libqgif.dylib - echo -e $INFO "App dependencies" otool -L $APP_PATH +echo -e $INFO "Indexer dependencies" +otool -L $INDEXER_PATH + echo -e $INFO "Create icon" ICON=../../../bin/app/data/gui/icon/logo_1024_1024.png ICON_SET=$RES_DIR/icon.iconset @@ -207,6 +215,6 @@ iconutil -c icns -o $RES_DIR/project.icns $ICON_SET # upx --best $APP_PATH echo -e $INFO "create DMG" -hdiutil create ${PACKAGE_DIR}.dmg -volname ${PACKAGE_DIR} -srcfolder ${PACKAGE_DIR} +# hdiutil create ${PACKAGE_DIR}.dmg -volname ${PACKAGE_DIR} -srcfolder ${PACKAGE_DIR} echo -e $SUCCESS "Installed bundle successfully!" diff --git a/src/app/main.cpp b/src/app/main.cpp index 2cff9301..a7f1f117 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -1,13 +1,11 @@ -#include "Application.h" +#include "includes.h" // defines 'void setup(int argc, char *argv[])' +#include "Application.h" #include "data/indexer/IndexerFactory.h" #include "data/indexer/IndexerFactoryModuleJava.h" #include "data/indexer/IndexerFactoryModuleCxxCdb.h" #include "data/indexer/IndexerFactoryModuleCxxManual.h" - -#include "includes.h" // defines 'void setup(int argc, char *argv[])' #include "LicenseChecker.h" - #include "project/SourceGroupFactory.h" #include "project/SourceGroupFactoryModuleC.h" #include "project/SourceGroupFactoryModuleCpp.h" @@ -31,6 +29,7 @@ #include "utility/ResourcePaths.h" #include "utility/ScopedFunctor.h" #include "utility/UserPaths.h" +#include "utility/utilityApp.h" #include "utility/utilityPathDetection.h" #include "utility/Version.h" #include "version.h" @@ -40,12 +39,13 @@ void setupLogging() LogManager* logManager = LogManager::getInstance().get(); std::shared_ptr consoleLogger = std::make_shared(); - consoleLogger->setLogLevel(Logger::LOG_WARNINGS | Logger::LOG_ERRORS); + // consoleLogger->setLogLevel(Logger::LOG_WARNINGS | Logger::LOG_ERRORS); + consoleLogger->setLogLevel(Logger::LOG_ALL); logManager->addLogger(consoleLogger); std::shared_ptr fileLogger = std::make_shared(); - fileLogger->setFileName(LoggerUtility::generateDatedFileName("log")); fileLogger->setLogDirectory(UserPaths::getLogPath()); + fileLogger->setFileName(LoggerUtility::generateDatedFileName("log")); fileLogger->setLogLevel(Logger::LOG_ALL); logManager->addLogger(fileLogger); } diff --git a/src/indexer/CMakeLists.txt b/src/indexer/CMakeLists.txt new file mode 100644 index 00000000..4de95018 --- /dev/null +++ b/src/indexer/CMakeLists.txt @@ -0,0 +1,5 @@ +add_files( + INDEXER_FILES + + main.cpp +) diff --git a/src/indexer/main.cpp b/src/indexer/main.cpp new file mode 100644 index 00000000..38eff6b2 --- /dev/null +++ b/src/indexer/main.cpp @@ -0,0 +1,88 @@ +#include "includes.h" + +#include "utility/AppPath.h" +#include "utility/logging/ConsoleLogger.h" +#include "utility/logging/FileLogger.h" +#include "utility/logging/logging.h" +#include "utility/logging/LogManager.h" + +#include "data/indexer/IndexerFactory.h" +#include "data/indexer/IndexerFactoryModuleJava.h" +#include "data/indexer/IndexerFactoryModuleCxxCdb.h" +#include "data/indexer/IndexerFactoryModuleCxxManual.h" +#include "data/indexer/interprocess/InterprocessIndexer.h" +#include "qt/QtCoreApplication.h" +#include "settings/ApplicationSettings.h" + +void setupLogging(const std::string logFilePath) +{ + LogManager* logManager = LogManager::getInstance().get(); + + std::shared_ptr consoleLogger = std::make_shared(); + // consoleLogger->setLogLevel(Logger::LOG_WARNINGS | Logger::LOG_ERRORS); + consoleLogger->setLogLevel(Logger::LOG_ALL); + logManager->addLogger(consoleLogger); + + std::shared_ptr fileLogger = std::make_shared(); + fileLogger->setLogFilePath(FilePath(logFilePath)); + fileLogger->setLogLevel(Logger::LOG_ALL); + logManager->addLogger(fileLogger); +} + +int main(int argc, char *argv[]) +{ + QCoreApplication qtApp(argc, argv); + + int processId = -1; + std::string instanceUuid; + std::string appPath; + std::string userDataPath; + std::string logFilePath; + + if (argc >= 2) + { + processId = std::stoi(argv[1]); + } + + if (argc >= 3) + { + instanceUuid = argv[2]; + } + + if (argc >= 4) + { + appPath = argv[3]; + } + + if (argc >= 5) + { + userDataPath = argv[4]; + } + + if (argc >= 6) + { + logFilePath = argv[5]; + } + + AppPath::setAppPath(appPath); + UserPaths::setUserDataPath(FilePath(userDataPath)); + + setupLogging(logFilePath); + + ApplicationSettings* appSettings = ApplicationSettings::getInstance().get(); + appSettings->load(FilePath(UserPaths::getAppSettingsPath())); + LogManager::getInstance()->setLoggingEnabled(appSettings->getLoggingEnabled()); + + LOG_INFO("appPath: " + appPath); + LOG_INFO("userDataPath: " + userDataPath); + + IndexerFactory::getInstance()->addModule(std::make_shared()); + IndexerFactory::getInstance()->addModule(std::make_shared()); + IndexerFactory::getInstance()->addModule(std::make_shared()); + + InterprocessIndexer indexer(instanceUuid, processId); + indexer.work(); + +// qtApp.quit(); + return 0; +} diff --git a/src/lib/Application.cpp b/src/lib/Application.cpp index 1c9426f7..722f9e16 100644 --- a/src/lib/Application.cpp +++ b/src/lib/Application.cpp @@ -1,5 +1,6 @@ #include "Application.h" +#include "utility/interprocess/SharedMemoryGarbageCollector.h" #include "utility/logging/logging.h" #include "utility/logging/LogManager.h" #include "utility/messaging/MessageQueue.h" @@ -8,6 +9,7 @@ #include "utility/scheduling/TaskScheduler.h" #include "utility/tracing.h" #include "utility/UserPaths.h" +#include "utility/UUIDUtility.h" #include "utility/Version.h" #include "component/view/DialogView.h" @@ -22,10 +24,14 @@ #include "settings/ProjectSettings.h" #include "settings/ColorScheme.h" +std::shared_ptr Application::s_instance; +std::string Application::s_uuid; + void Application::createInstance( const Version& version, ViewFactory* viewFactory, NetworkFactory* networkFactory ){ Version::setApplicationVersion(version); + SharedMemoryGarbageCollector::createInstance()->run(Application::getUUID()); loadSettings(); TaskScheduler::getInstance(); @@ -69,6 +75,16 @@ void Application::destroyInstance() s_instance.reset(); } +std::string Application::getUUID() +{ + if (!s_uuid.size()) + { + s_uuid = UUIDUtility::UUIDtoString(UUIDUtility::getUUID()); + } + + return s_uuid; +} + void Application::loadSettings() { MessageStatus("Load settings: " + UserPaths::getAppSettingsPath().str()).dispatch(); @@ -87,8 +103,6 @@ void Application::loadStyle(const FilePath& colorSchemePath) GraphViewStyle::loadStyleSettings(); } -std::shared_ptr Application::s_instance; - Application::Application(bool withGUI) : m_hasGUI(withGUI) , m_isInTrial(true) @@ -100,10 +114,13 @@ Application::~Application() { MessageQueue::getInstance()->stopMessageLoop(); TaskScheduler::getInstance()->stopSchedulerLoop(); + if (m_hasGUI) { m_mainView->saveLayout(); } + + SharedMemoryGarbageCollector::getInstance()->stop(); } const std::shared_ptr Application::getCurrentProject() diff --git a/src/lib/Application.h b/src/lib/Application.h index 868eacc2..216daf48 100644 --- a/src/lib/Application.h +++ b/src/lib/Application.h @@ -34,6 +34,8 @@ public: static std::shared_ptr getInstance(); static void destroyInstance(); + static std::string getUUID(); + static void loadSettings(); static void loadStyle(const FilePath& colorSchemePath); @@ -53,6 +55,7 @@ public: private: static std::shared_ptr s_instance; + static std::string s_uuid; Application(bool withGUI=true); diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 39f3fe4f..78312cf7 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -138,6 +138,23 @@ add_files( data/graph/Token.cpp data/graph/Token.h + data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp + data/indexer/interprocess/shared_types/SharedIndexerCommand.h + data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp + data/indexer/interprocess/shared_types/SharedIntermediateStorage.h + data/indexer/interprocess/shared_types/SharedStorageTypes.h + + data/indexer/interprocess/BaseInterprocessDataManager.cpp + data/indexer/interprocess/BaseInterprocessDataManager.h + data/indexer/interprocess/InterprocessIndexer.cpp + data/indexer/interprocess/InterprocessIndexer.h + data/indexer/interprocess/InterprocessIndexerCommandManager.cpp + data/indexer/interprocess/InterprocessIndexerCommandManager.h + data/indexer/interprocess/InterprocessIndexingStatusManager.cpp + data/indexer/interprocess/InterprocessIndexingStatusManager.h + data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp + data/indexer/interprocess/InterprocessIntermediateStorageManager.h + data/indexer/Indexer.h data/indexer/IndexerBase.cpp data/indexer/IndexerBase.h @@ -283,18 +300,10 @@ add_files( utility/file/FileSystem.cpp utility/file/FileSystem.h - utility/interprocess/InterprocessDataManager.cpp - utility/interprocess/InterprocessDataManager.h - utility/interprocess/InterprocessProcessManager.cpp - utility/interprocess/InterprocessProcessManager.h - utility/interprocess/InterprocessUtility.h - utility/interprocess/SharedContainer.h - utility/interprocess/SharedMap.h - utility/interprocess/SharedParserArguments.cpp - utility/interprocess/SharedParserArguments.h - utility/interprocess/SharedQueue.h - utility/interprocess/SharedUUIDManager.cpp - utility/interprocess/SharedUUIDManager.h + utility/interprocess/SharedMemory.cpp + utility/interprocess/SharedMemory.h + utility/interprocess/SharedMemoryGarbageCollector.cpp + utility/interprocess/SharedMemoryGarbageCollector.h utility/logging/ConsoleLogger.cpp utility/logging/ConsoleLogger.h diff --git a/src/lib/component/view/DialogView.cpp b/src/lib/component/view/DialogView.cpp index 32fc1dfa..2afb89de 100644 --- a/src/lib/component/view/DialogView.cpp +++ b/src/lib/component/view/DialogView.cpp @@ -37,7 +37,11 @@ void DialogView::updateIndexingDialog(size_t fileCount, size_t totalFileCount, s void DialogView::finishedIndexingDialog( size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, - float time, ErrorCountInfo errorInfo) + float time, ErrorCountInfo errorInfo, bool interrupted) +{ +} + +void DialogView::hideDialogs() { } diff --git a/src/lib/component/view/DialogView.h b/src/lib/component/view/DialogView.h index e9a171b1..7dc62ee6 100644 --- a/src/lib/component/view/DialogView.h +++ b/src/lib/component/view/DialogView.h @@ -44,7 +44,9 @@ public: virtual void updateIndexingDialog(size_t fileCount, size_t totalFileCount, std::string sourcePath); virtual void finishedIndexingDialog( size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, - float time, ErrorCountInfo errorInfo); + float time, ErrorCountInfo errorInfo, bool interrupted); + + virtual void hideDialogs(); int confirm(const std::string& message); virtual int confirm(const std::string& message, const std::vector& options); diff --git a/src/lib/data/IntermediateStorage.cpp b/src/lib/data/IntermediateStorage.cpp index ba4539ed..067f65fe 100644 --- a/src/lib/data/IntermediateStorage.cpp +++ b/src/lib/data/IntermediateStorage.cpp @@ -16,16 +16,12 @@ IntermediateStorage::~IntermediateStorage() void IntermediateStorage::clear() { - m_nodeNamesToIds.clear(); - m_nodeIdsToData.clear(); + m_nodes.clear(); m_files.clear(); m_symbols.clear(); - m_edgeNamesToIds.clear(); - m_edgeIdsToData.clear(); - m_localSymbolNamesToIds.clear(); - m_localSymbolIdsToData.clear(); - m_sourceLocationNamesToIds.clear(); - m_sourceLocationIdsToData.clear(); + m_edges.clear(); + m_localSymbols.clear(); + m_sourceLocations.clear(); m_occurrences.clear(); m_componentAccesses.clear(); m_commentLocations.clear(); @@ -33,9 +29,49 @@ void IntermediateStorage::clear() m_nextId = 1; } +size_t IntermediateStorage::getByteSize() const +{ + unsigned int byteSize = 0; + + for (const StorageFile& storageFile: getStorageFiles()) + { + byteSize += sizeof(StorageFile); + byteSize += storageFile.filePath.size(); + byteSize += storageFile.modificationTime.size(); + } + + for (const StorageError& storageError: getErrors()) + { + byteSize += sizeof(StorageError); + byteSize += storageError.filePath.str().size(); + byteSize += storageError.message.size(); + } + + for (const StorageNode& storageNode: getStorageNodes()) + { + byteSize += sizeof(StorageNode); + byteSize += storageNode.serializedName.size(); + } + + for (const StorageLocalSymbol& storageLocalSymbol: getStorageLocalSymbols()) + { + byteSize += sizeof(StorageLocalSymbol); + byteSize += storageLocalSymbol.name.size(); + } + + byteSize += sizeof(StorageEdge) * getStorageEdges().size(); + byteSize += sizeof(StorageCommentLocation) * getCommentLocations().size(); + byteSize += sizeof(StorageComponentAccess) * getComponentAccesses().size(); + byteSize += sizeof(StorageOccurrence) * getStorageOccurrences().size(); + byteSize += sizeof(StorageSymbol) * getStorageSymbols().size(); + byteSize += sizeof(StorageSourceLocation) * getStorageSourceLocations().size(); + + return byteSize; +} + size_t IntermediateStorage::getSourceLocationCount() const { - return m_sourceLocationNamesToIds.size(); + return m_sourceLocations.size(); } void IntermediateStorage::setAllFilesIncomplete() @@ -65,25 +101,23 @@ void IntermediateStorage::setFilesWithErrorsIncomplete() Id IntermediateStorage::addNode(int type, const std::string& serializedName) { - std::shared_ptr node = std::make_shared(0, type, serializedName); + StorageNode node(0, type, serializedName); - const std::string serialized = serialize(*(node.get())); - std::unordered_map::const_iterator it = m_nodeNamesToIds.find(serialized); - if (it != m_nodeNamesToIds.end()) + const std::string serialized = serialize(node); + std::unordered_map::iterator it = m_nodes.find(serialized); + if (it != m_nodes.end()) { - std::map>::const_iterator it2 = m_nodeIdsToData.find(it->second); - std::shared_ptr storedNode = it2->second; - if (storedNode->type < type) + StorageNode& storedNode = it->second; + if (storedNode.type < type) { - storedNode->type = type; + storedNode.type = type; } - return it->second; + return storedNode.id; } const Id id = m_nextId++; - m_nodeNamesToIds[serialized] = id; - m_nodeIdsToData[id] = node; - + node.id = id; + m_nodes[serialized] = node; return id; } @@ -107,43 +141,41 @@ void IntermediateStorage::addSymbol(const Id id, int definitionKind) Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId) { - std::shared_ptr edge = std::make_shared(0, type, sourceId, targetId); + StorageEdge edge = StorageEdge(0, type, sourceId, targetId); - const std::string serialized = serialize(*(edge.get())); - std::unordered_map::const_iterator it = m_edgeNamesToIds.find(serialized); - if (it != m_edgeNamesToIds.end()) + const std::string serialized = serialize(edge); + std::unordered_map::const_iterator it = m_edges.find(serialized); + if (it != m_edges.end()) { - return it->second; + return it->second.id; } const Id id = m_nextId++; - m_edgeNamesToIds[serialized] = id; - m_edgeIdsToData[id] = edge; - + edge.id = id; + m_edges[serialized] = edge; return id; } Id IntermediateStorage::addLocalSymbol(const std::string& name) { - std::shared_ptr localSymbol = std::make_shared(0, name); + StorageLocalSymbol localSymbol = StorageLocalSymbol(0, name); - const std::string serialized = serialize(*(localSymbol.get())); - std::unordered_map::const_iterator it = m_localSymbolNamesToIds.find(serialized); - if (it != m_localSymbolNamesToIds.end()) + const std::string serialized = serialize(localSymbol); + std::unordered_map::const_iterator it = m_localSymbols.find(serialized); + if (it != m_localSymbols.end()) { - return it->second; + return it->second.id; } const Id id = m_nextId++; - m_localSymbolNamesToIds[serialized] = id; - m_localSymbolIdsToData[id] = localSymbol; - + localSymbol.id = id; + m_localSymbols[serialized] = localSymbol; return id; } Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type) { - std::shared_ptr sourceLocation = std::make_shared( + StorageSourceLocation sourceLocation = StorageSourceLocation( 0, fileNodeId, startLine, @@ -153,17 +185,16 @@ Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint st type ); - const std::string serialized = serialize(*(sourceLocation.get())); - std::unordered_map::const_iterator it = m_sourceLocationNamesToIds.find(serialized); - if (it != m_sourceLocationNamesToIds.end()) + const std::string serialized = serialize(sourceLocation); + std::unordered_map::const_iterator it = m_sourceLocations.find(serialized); + if (it != m_sourceLocations.end()) { - return it->second; + return it->second.id; } const Id id = m_nextId++; - m_sourceLocationNamesToIds[serialized] = id; - m_sourceLocationIdsToData[id] = sourceLocation; - + sourceLocation.id = id; + m_sourceLocations[serialized] = sourceLocation; return id; } @@ -211,7 +242,8 @@ void IntermediateStorage::addCommentLocation(Id fileNodeId, uint startLine, uint } } -void IntermediateStorage::addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) +void IntermediateStorage::addError( + const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) { const StorageError error( 0, @@ -231,11 +263,11 @@ void IntermediateStorage::addError(const std::string& message, const FilePath& f } } -void IntermediateStorage::forEachNode(std::function callback) const +void IntermediateStorage::forEachNode(std::function callback) const { - for (std::map>::const_iterator it = m_nodeIdsToData.begin(); it != m_nodeIdsToData.end(); it++) + for (std::unordered_map::const_iterator it = m_nodes.begin(); it != m_nodes.end(); it++) { - callback(it->first, *(it->second.get())); + callback(it->second); } } @@ -255,27 +287,29 @@ void IntermediateStorage::forEachSymbol(std::function callback) const +void IntermediateStorage::forEachEdge(std::function callback) const { - for (std::map>::const_iterator it = m_edgeIdsToData.begin(); it != m_edgeIdsToData.end(); it++) + for (std::unordered_map::const_iterator it = m_edges.begin(); it != m_edges.end(); it++) { - callback(it->first, *(it->second.get())); + callback(it->second); } } -void IntermediateStorage::forEachLocalSymbol(std::function callback) const +void IntermediateStorage::forEachLocalSymbol(std::function callback) const { - for (std::map>::const_iterator it = m_localSymbolIdsToData.begin(); it != m_localSymbolIdsToData.end(); it++) + for (std::unordered_map::const_iterator it = m_localSymbols.begin(); + it != m_localSymbols.end(); it++) { - callback(it->first, *(it->second.get())); + callback(it->second); } } -void IntermediateStorage::forEachSourceLocation(std::function callback) const +void IntermediateStorage::forEachSourceLocation(std::function callback) const { - for (std::map>::const_iterator it = m_sourceLocationIdsToData.begin(); it != m_sourceLocationIdsToData.end(); it++) + for (std::unordered_map::const_iterator it = m_sourceLocations.begin(); + it != m_sourceLocations.end(); it++) { - callback(it->first, *(it->second.get())); + callback(it->second); } } @@ -311,6 +345,156 @@ void IntermediateStorage::forEachError(std::function IntermediateStorage::getStorageNodes() const +{ + std::vector nodes; + nodes.reserve(m_nodes.size()); + for (auto it: m_nodes) + { + nodes.push_back(it.second); + } + return nodes; +} + +std::vector IntermediateStorage::getStorageFiles() const +{ + return m_files; +} + +std::vector IntermediateStorage::getStorageSymbols() const +{ + return m_symbols; +} + +std::vector IntermediateStorage::getStorageEdges() const +{ + std::vector edges; + edges.reserve(m_edges.size()); + for (auto it: m_edges) + { + edges.push_back(it.second); + } + return edges; +} + +std::vector IntermediateStorage::getStorageLocalSymbols() const +{ + std::vector localSymbol; + localSymbol.reserve(m_localSymbols.size()); + for (auto it: m_localSymbols) + { + localSymbol.push_back(it.second); + } + return localSymbol; +} + +std::vector IntermediateStorage::getStorageSourceLocations() const +{ + std::vector sourceLocations; + sourceLocations.reserve(m_sourceLocations.size()); + for (auto it: m_sourceLocations) + { + sourceLocations.push_back(it.second); + } + return sourceLocations; +} + +std::vector IntermediateStorage::getStorageOccurrences() const +{ + return m_occurrences; +} + +std::vector IntermediateStorage::getComponentAccesses() const +{ + return m_componentAccesses; +} + +std::vector IntermediateStorage::getCommentLocations() const +{ + return m_commentLocations; +} + +std::vector IntermediateStorage::getErrors() const +{ + return m_errors; +} + +void IntermediateStorage::setStorageNodes(const std::vector& storageNodes) +{ + m_nodes.clear(); + for (const StorageNode& storageNode: storageNodes) + { + m_nodes[serialize(storageNode)] = storageNode; + } +} + +void IntermediateStorage::setStorageFiles(const std::vector& storageFiles) +{ + m_files = storageFiles; +} + +void IntermediateStorage::setStorageSymbols(const std::vector& storageSymbols) +{ + m_symbols = storageSymbols; +} + +void IntermediateStorage::setStorageEdges(const std::vector& storageEdges) +{ + m_edges.clear(); + for (const StorageEdge& storageEdge: storageEdges) + { + m_edges[serialize(storageEdge)] = storageEdge; + } +} + +void IntermediateStorage::setStorageLocalSymbols(const std::vector& storageLocalSymbols) +{ + m_localSymbols.clear(); + for (const StorageLocalSymbol& storageLocalSymbol: storageLocalSymbols) + { + m_localSymbols[serialize(storageLocalSymbol)] = storageLocalSymbol; + } +} + +void IntermediateStorage::setStorageSourceLocations(const std::vector& storageSourceLocations) +{ + m_sourceLocations.clear(); + for (const StorageSourceLocation& storageSourceLocation: storageSourceLocations) + { + m_sourceLocations[serialize(storageSourceLocation)] = storageSourceLocation; + } +} + +void IntermediateStorage::setStorageOccurrences(const std::vector& storageOccurrences) +{ + m_occurrences = storageOccurrences; +} + +void IntermediateStorage::setComponentAccesses(const std::vector& componentAccesses) +{ + m_componentAccesses = componentAccesses; +} + +void IntermediateStorage::setCommentLocations(const std::vector& commentLocations) +{ + m_commentLocations = commentLocations; +} + +void IntermediateStorage::setErrors(const std::vector& errors) +{ + m_errors = errors; +} + +Id IntermediateStorage::getNextId() const +{ + return m_nextId; +} + +void IntermediateStorage::setNextId(const Id nextId) +{ + m_nextId = nextId; +} + std::string IntermediateStorage::serialize(const StorageNode& node) const { return node.serializedName; @@ -327,7 +511,7 @@ std::string IntermediateStorage::serialize(const StorageEdge& edge) const std::to_string(edge.type) + ";" + std::to_string(edge.sourceNodeId) + ";" + std::to_string(edge.targetNodeId) - ); + ); } std::string IntermediateStorage::serialize(const StorageLocalSymbol& localSymbol) const diff --git a/src/lib/data/IntermediateStorage.h b/src/lib/data/IntermediateStorage.h index 43b4127f..74c961bb 100644 --- a/src/lib/data/IntermediateStorage.h +++ b/src/lib/data/IntermediateStorage.h @@ -16,6 +16,8 @@ public: virtual ~IntermediateStorage(); void clear(); + + size_t getByteSize() const; size_t getSourceLocationCount() const; void setAllFilesIncomplete(); @@ -32,17 +34,45 @@ public: virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol); virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed); - virtual void forEachNode(std::function callback) const; + virtual void forEachNode(std::function callback) const; virtual void forEachFile(std::function callback) const; virtual void forEachSymbol(std::function callback) const; - virtual void forEachEdge(std::function callback) const; - virtual void forEachLocalSymbol(std::function callback) const; - virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachEdge(std::function callback) const; + virtual void forEachLocalSymbol(std::function callback) const; + virtual void forEachSourceLocation(std::function callback) const; virtual void forEachOccurrence(std::function callback) const; virtual void forEachComponentAccess(std::function callback) const; virtual void forEachCommentLocation(std::function callback) const; virtual void forEachError(std::function callback) const; + + // for conversion to and from 'SharedIntermediateStorage' + + std::vector getStorageNodes() const; + std::vector getStorageFiles() const; + std::vector getStorageSymbols() const; + std::vector getStorageEdges() const; + std::vector getStorageLocalSymbols() const; + std::vector getStorageSourceLocations() const; + std::vector getStorageOccurrences() const; + std::vector getComponentAccesses() const; + std::vector getCommentLocations() const; + std::vector getErrors() const; + + void setStorageNodes(const std::vector& storageNodes); + void setStorageFiles(const std::vector& storageFiles); + void setStorageSymbols(const std::vector& storageSymbols); + void setStorageEdges(const std::vector& storageEdges); + void setStorageLocalSymbols(const std::vector& storageLocalSymbols); + void setStorageSourceLocations(const std::vector& storageSourceLocations); + void setStorageOccurrences(const std::vector& storageOccurrences); + void setComponentAccesses(const std::vector& componentAccesses); + void setCommentLocations(const std::vector& commentLocations); + void setErrors(const std::vector& errors); + + Id getNextId() const; + void setNextId(const Id nextId); + private: std::string serialize(const StorageNode& node) const; std::string serialize(const StorageFile& file) const; @@ -54,22 +84,18 @@ private: std::string serialize(const StorageCommentLocation& commentLocation) const; std::string serialize(const StorageError& error) const; - std::unordered_map m_nodeNamesToIds; // this is used to prevent duplicates (unique) - std::map> m_nodeIdsToData; + std::unordered_map m_nodes; std::unordered_set m_serializedFiles; // this is used to prevent duplicates (unique) std::vector m_files; std::vector m_symbols; - std::unordered_map m_edgeNamesToIds; // this is used to prevent duplicates (unique) - std::map> m_edgeIdsToData; + std::unordered_map m_edges; - std::unordered_map m_localSymbolNamesToIds; // this is used to prevent duplicates (unique) - std::map> m_localSymbolIdsToData; + std::unordered_map m_localSymbols; - std::unordered_map m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique) - std::map> m_sourceLocationIdsToData; + std::unordered_map m_sourceLocations; std::unordered_set m_serializedOccurrences; // this is used to prevent duplicates (unique) std::vector m_occurrences; diff --git a/src/lib/data/PersistentStorage.cpp b/src/lib/data/PersistentStorage.cpp index 80cf49cb..e235f3d1 100644 --- a/src/lib/data/PersistentStorage.cpp +++ b/src/lib/data/PersistentStorage.cpp @@ -324,11 +324,11 @@ std::vector PersistentStorage::getAllBookmarkCategories() cons return categories; } -void PersistentStorage::forEachNode(std::function callback) const +void PersistentStorage::forEachNode(std::function callback) const { for (StorageNode& node: m_sqliteIndexStorage.getAll()) { - callback(node.id, node); + callback(node); } } @@ -348,29 +348,27 @@ void PersistentStorage::forEachSymbol(std::function callback) const +void PersistentStorage::forEachEdge(std::function callback) const { for (StorageEdge& edge: m_sqliteIndexStorage.getAll()) { - callback(edge.id, edge); + callback(edge); } } -void PersistentStorage::forEachLocalSymbol(std::function callback) const +void PersistentStorage::forEachLocalSymbol(std::function callback) const { for (StorageLocalSymbol& localSymbol: m_sqliteIndexStorage.getAll()) { - callback(localSymbol.id, localSymbol); + callback(localSymbol); } } -void PersistentStorage::forEachSourceLocation( - std::function callback) const +void PersistentStorage::forEachSourceLocation(std::function callback) const { for (StorageSourceLocation& sourceLocation: m_sqliteIndexStorage.getAll()) { - callback(sourceLocation.id, sourceLocation); + callback(sourceLocation); } } diff --git a/src/lib/data/PersistentStorage.h b/src/lib/data/PersistentStorage.h index a8258413..f3f3f285 100644 --- a/src/lib/data/PersistentStorage.h +++ b/src/lib/data/PersistentStorage.h @@ -50,12 +50,12 @@ public: virtual std::vector getAllBookmarkCategories() const; - virtual void forEachNode(std::function callback) const; + virtual void forEachNode(std::function callback) const; virtual void forEachFile(std::function callback) const; virtual void forEachSymbol(std::function callback) const; - virtual void forEachEdge(std::function callback) const; - virtual void forEachLocalSymbol(std::function callback) const; - virtual void forEachSourceLocation(std::function callback) const; + virtual void forEachEdge(std::function callback) const; + virtual void forEachLocalSymbol(std::function callback) const; + virtual void forEachSourceLocation(std::function callback) const; virtual void forEachOccurrence(std::function callback) const; virtual void forEachComponentAccess(std::function callback) const; virtual void forEachCommentLocation(std::function callback) const; diff --git a/src/lib/data/Storage.cpp b/src/lib/data/Storage.cpp index 655afdf2..0df2fae7 100644 --- a/src/lib/data/Storage.cpp +++ b/src/lib/data/Storage.cpp @@ -25,12 +25,12 @@ void Storage::inject(Storage* injected) std::unordered_map injectedIdToOwnId; injected->forEachNode( - [&](Id injectedId, const StorageNode& injectedData) + [&](const StorageNode& injectedData) { const Id ownId = addNode(injectedData.type, injectedData.serializedName); if (ownId != 0) { - injectedIdToOwnId[injectedId] = ownId; + injectedIdToOwnId[injectedData.id] = ownId; } } ); @@ -66,7 +66,7 @@ void Storage::inject(Storage* injected) ); injected->forEachEdge( - [&](Id injectedId, const StorageEdge& injectedData) + [&](const StorageEdge& injectedData) { std::unordered_map::const_iterator it; it = injectedIdToOwnId.find(injectedData.sourceNodeId); @@ -87,24 +87,24 @@ void Storage::inject(Storage* injected) if (ownId != 0) { - injectedIdToOwnId[injectedId] = ownId; + injectedIdToOwnId[injectedData.id] = ownId; } } ); injected->forEachLocalSymbol( - [&](const Id injectedId, const StorageLocalSymbol& injectedData) + [&](const StorageLocalSymbol& injectedData) { const Id ownId = addLocalSymbol(injectedData.name); if (ownId != 0) { - injectedIdToOwnId[injectedId] = ownId; + injectedIdToOwnId[injectedData.id] = ownId; } } ); injected->forEachSourceLocation( - [&](const Id injectedId, const StorageSourceLocation& injectedData) + [&](const StorageSourceLocation& injectedData) { std::unordered_map::const_iterator it; it = injectedIdToOwnId.find(injectedData.fileNodeId); @@ -124,7 +124,7 @@ void Storage::inject(Storage* injected) ); if (ownId != 0) { - injectedIdToOwnId[injectedId] = ownId; + injectedIdToOwnId[injectedData.id] = ownId; } } ); diff --git a/src/lib/data/Storage.h b/src/lib/data/Storage.h index b90806cd..6658e971 100644 --- a/src/lib/data/Storage.h +++ b/src/lib/data/Storage.h @@ -26,12 +26,12 @@ public: virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) = 0; virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) = 0; - virtual void forEachNode(std::function callback) const = 0; + virtual void forEachNode(std::function callback) const = 0; virtual void forEachFile(std::function callback) const = 0; virtual void forEachSymbol(std::function callback) const = 0; - virtual void forEachEdge(std::function callback) const = 0; - virtual void forEachLocalSymbol(std::function callback) const = 0; - virtual void forEachSourceLocation(std::function callback) const = 0; + virtual void forEachEdge(std::function callback) const = 0; + virtual void forEachLocalSymbol(std::function callback) const = 0; + virtual void forEachSourceLocation(std::function callback) const = 0; virtual void forEachOccurrence(std::function callback) const = 0; virtual void forEachComponentAccess(std::function callback) const = 0; virtual void forEachCommentLocation(std::function callback) const = 0; diff --git a/src/lib/data/StorageTypes.h b/src/lib/data/StorageTypes.h index 9a9f8a21..03878fe4 100644 --- a/src/lib/data/StorageTypes.h +++ b/src/lib/data/StorageTypes.h @@ -238,9 +238,6 @@ struct StorageError }; - - - struct StorageBookmarkCategory { StorageBookmarkCategory() diff --git a/src/lib/data/TaskFinishParsing.cpp b/src/lib/data/TaskFinishParsing.cpp index 3f4d70a8..b1d5c9b7 100644 --- a/src/lib/data/TaskFinishParsing.cpp +++ b/src/lib/data/TaskFinishParsing.cpp @@ -4,6 +4,7 @@ #include "data/PersistentStorage.h" #include "utility/file/FileRegister.h" #include "utility/messaging/type/MessageFinishedParsing.h" +#include "utility/messaging/type/MessageStatus.h" #include "utility/scheduling/Blackboard.h" #include "utility/utility.h" #include "Application.h" @@ -63,6 +64,9 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr blackboa int sourceFileCount = 0; blackboard->get("source_file_count", sourceFileCount); + bool interruptedIndexing = false; + blackboard->get("interrupted_indexing", interruptedIndexing); + StorageStats stats = m_storageAccess->getStorageStats(); dialogView->finishedIndexingDialog( indexedSourceFileCount, @@ -70,7 +74,8 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr blackboa stats.completedFileCount, stats.fileCount, time, - m_storageAccess->getErrorCount() + m_storageAccess->getErrorCount(), + interruptedIndexing ); return STATE_SUCCESS; @@ -83,3 +88,15 @@ void TaskFinishParsing::doExit(std::shared_ptr blackboard) void TaskFinishParsing::doReset(std::shared_ptr blackboard) { } + +void TaskFinishParsing::terminate() +{ + Application* app = Application::getInstance().get(); + if (app) + { + app->getDialogView()->hideDialogs(); + } + + MessageStatus("An unknown exception was thrown during indexing.", true, false).dispatch(); + MessageFinishedParsing().dispatch(); +} diff --git a/src/lib/data/TaskFinishParsing.h b/src/lib/data/TaskFinishParsing.h index f3996240..177d1f07 100644 --- a/src/lib/data/TaskFinishParsing.h +++ b/src/lib/data/TaskFinishParsing.h @@ -26,6 +26,7 @@ private: virtual TaskState doUpdate(std::shared_ptr blackboard); virtual void doExit(std::shared_ptr blackboard); virtual void doReset(std::shared_ptr blackboard); + virtual void terminate(); PersistentStorage* m_storage; StorageAccess* m_storageAccess; diff --git a/src/lib/data/indexer/Indexer.h b/src/lib/data/indexer/Indexer.h index 1b73e112..58cb74b3 100644 --- a/src/lib/data/indexer/Indexer.h +++ b/src/lib/data/indexer/Indexer.h @@ -18,7 +18,8 @@ public: virtual std::string getKindString() const; virtual std::shared_ptr index( - std::shared_ptr indexerCommand, std::shared_ptr fileRegister); + std::shared_ptr indexerCommand, + std::shared_ptr fileRegister); }; template @@ -46,7 +47,6 @@ std::shared_ptr Indexer::in } std::shared_ptr parserClient = std::make_shared(); - parserClient->setCancelOnFatalErrors(indexerCommand->cancelOnFatalErrors()); std::shared_ptr parser = std::make_shared(parserClient, fileRegister); diff --git a/src/lib/data/indexer/IndexerBase.h b/src/lib/data/indexer/IndexerBase.h index 014f3652..23d608f3 100644 --- a/src/lib/data/indexer/IndexerBase.h +++ b/src/lib/data/indexer/IndexerBase.h @@ -16,7 +16,9 @@ public: virtual std::string getKindString() const = 0; - virtual std::shared_ptr index(std::shared_ptr indexerCommand, std::shared_ptr fileRegister) = 0; + virtual std::shared_ptr index( + std::shared_ptr indexerCommand, + std::shared_ptr fileRegister) = 0; virtual void interrupt(); diff --git a/src/lib/data/indexer/IndexerCommand.cpp b/src/lib/data/indexer/IndexerCommand.cpp index 1d13174c..0f7fc229 100644 --- a/src/lib/data/indexer/IndexerCommand.cpp +++ b/src/lib/data/indexer/IndexerCommand.cpp @@ -1,10 +1,11 @@ #include "data/indexer/IndexerCommand.h" -IndexerCommand::IndexerCommand(const FilePath& sourceFilePath, const std::set& indexedPaths, const std::set& excludedPaths) +IndexerCommand::IndexerCommand( + const FilePath& sourceFilePath, const std::set& indexedPaths, const std::set& excludedPaths +) : m_sourceFilePath(sourceFilePath) , m_indexedPaths(indexedPaths) , m_excludedPaths(excludedPaths) - , m_cancelOnFatalErrors(false) { } @@ -12,6 +13,23 @@ IndexerCommand::~IndexerCommand() { } +size_t IndexerCommand::getByteSize() const +{ + size_t size = m_sourceFilePath.str().size(); + + for (auto i : m_indexedPaths) + { + size += i.str().size(); + } + + for (auto i : m_excludedPaths) + { + size += i.str().size(); + } + + return size; +} + FilePath IndexerCommand::getSourceFilePath() const { return m_sourceFilePath; @@ -26,13 +44,3 @@ std::set IndexerCommand::getExcludedPath() const { return m_excludedPaths; } - -bool IndexerCommand::cancelOnFatalErrors() const -{ - return m_cancelOnFatalErrors; -} - -void IndexerCommand::setCancelOnFatalErrors(bool cancelOnFatalErrors) -{ - m_cancelOnFatalErrors = cancelOnFatalErrors; -} diff --git a/src/lib/data/indexer/IndexerCommand.h b/src/lib/data/indexer/IndexerCommand.h index 9a2ac43f..e136b0f3 100644 --- a/src/lib/data/indexer/IndexerCommand.h +++ b/src/lib/data/indexer/IndexerCommand.h @@ -13,14 +13,12 @@ public: virtual ~IndexerCommand(); virtual std::string getKindString() const = 0; + virtual size_t getByteSize() const; FilePath getSourceFilePath() const; std::set getIndexedPaths() const; std::set getExcludedPath() const; - bool cancelOnFatalErrors() const; - void setCancelOnFatalErrors(bool cancelOnFatalErrors); - virtual bool preprocessorOnly() const = 0; virtual void setPreprocessorOnly(bool preprocessorOnly) = 0; @@ -28,8 +26,6 @@ private: FilePath m_sourceFilePath; std::set m_indexedPaths; std::set m_excludedPaths; - - bool m_cancelOnFatalErrors; }; #endif // INDEXER_COMMAND_H diff --git a/src/lib/data/indexer/IndexerCommandList.cpp b/src/lib/data/indexer/IndexerCommandList.cpp index 74f35c23..f42f7b01 100644 --- a/src/lib/data/indexer/IndexerCommandList.cpp +++ b/src/lib/data/indexer/IndexerCommandList.cpp @@ -32,3 +32,9 @@ std::shared_ptr IndexerCommandList::consumeCommand() } return ret; } + +std::vector> IndexerCommandList::getAllCommands() +{ + std::lock_guard lock(m_commandsMutex); + return std::vector>(m_commands.begin(), m_commands.end()); +} diff --git a/src/lib/data/indexer/IndexerCommandList.h b/src/lib/data/indexer/IndexerCommandList.h index 91b771e0..745959bf 100644 --- a/src/lib/data/indexer/IndexerCommandList.h +++ b/src/lib/data/indexer/IndexerCommandList.h @@ -18,6 +18,8 @@ public: std::shared_ptr consumeCommand(); + std::vector> getAllCommands(); + private: std::deque> m_commands; std::mutex m_commandsMutex; diff --git a/src/lib/data/indexer/IndexerComposite.cpp b/src/lib/data/indexer/IndexerComposite.cpp index 73f5a45e..f400db41 100644 --- a/src/lib/data/indexer/IndexerComposite.cpp +++ b/src/lib/data/indexer/IndexerComposite.cpp @@ -15,7 +15,8 @@ void IndexerComposite::addIndexer(std::shared_ptr indexer) m_indexers.emplace(indexer->getKindString(), indexer); } -std::shared_ptr IndexerComposite::index(std::shared_ptr indexerCommand, std::shared_ptr fileRegister) +std::shared_ptr IndexerComposite::index( + std::shared_ptr indexerCommand, std::shared_ptr fileRegister) { auto it = m_indexers.find(indexerCommand->getKindString()); if (it != m_indexers.end()) diff --git a/src/lib/data/indexer/IndexerComposite.h b/src/lib/data/indexer/IndexerComposite.h index f22d8532..b096ae73 100644 --- a/src/lib/data/indexer/IndexerComposite.h +++ b/src/lib/data/indexer/IndexerComposite.h @@ -15,7 +15,10 @@ public: void addIndexer(std::shared_ptr indexer); - virtual std::shared_ptr index(std::shared_ptr indexerCommand, std::shared_ptr fileRegister); + virtual std::shared_ptr index( + std::shared_ptr indexerCommand, + std::shared_ptr fileRegister + ); virtual void interrupt(); diff --git a/src/lib/data/indexer/TaskBuildIndex.cpp b/src/lib/data/indexer/TaskBuildIndex.cpp index c0fc4297..aada2e9f 100644 --- a/src/lib/data/indexer/TaskBuildIndex.cpp +++ b/src/lib/data/indexer/TaskBuildIndex.cpp @@ -1,16 +1,30 @@ #include "data/indexer/TaskBuildIndex.h" +#include "utility/AppPath.h" +#include "utility/file/FileRegister.h" +#include "utility/file/FileRegisterStateData.h" +#include "utility/logging/FileLogger.h" +#include "utility/scheduling/Blackboard.h" +#include "utility/UserPaths.h" +#include "utility/utilityApp.h" + +#include "Application.h" +#include "component/view/DialogView.h" #include "data/indexer/IndexerFactory.h" #include "data/indexer/IndexerCommandList.h" #include "data/indexer/IndexerComposite.h" +#include "data/indexer/interprocess/InterprocessIndexer.h" #include "data/StorageProvider.h" -#include "component/view/DialogView.h" -#include "utility/file/FileRegister.h" -#include "utility/file/FileRegisterStateData.h" -#include "utility/scheduling/Blackboard.h" -#include "Application.h" +#include "settings/ApplicationSettings.h" + +#if _WIN32 +const std::string TaskBuildIndex::s_processName("sourcetrail_indexer.exe"); +#else +const std::string TaskBuildIndex::s_processName("sourcetrail_indexer"); +#endif TaskBuildIndex::TaskBuildIndex( + unsigned int processCount, std::shared_ptr indexerCommandList, std::shared_ptr storageProvider, std::shared_ptr fileRegisterStateData @@ -18,89 +32,206 @@ TaskBuildIndex::TaskBuildIndex( : m_indexerCommandList(indexerCommandList) , m_storageProvider(storageProvider) , m_fileRegisterStateData(fileRegisterStateData) + , m_interprocessIndexerCommandManager(Application::getUUID(), 0, true) + , m_interprocessIndexingStatusManager(Application::getUUID(), 0, true) + , m_processCount(processCount) + , m_interrupted(false) + , m_lastCommandCount(0) { - m_indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules(); } void TaskBuildIndex::doEnter(std::shared_ptr blackboard) { - std::lock_guard lock(blackboard->getMutex()); - - int indexerCount = 0; - if (blackboard->get("indexer_count", indexerCount)) { - indexerCount++; - blackboard->set("indexer_count", indexerCount); + std::lock_guard lock(blackboard->getMutex()); + blackboard->set("indexer_count", (int)m_processCount); + } + + // move indexer commands to shared memory + m_lastCommandCount = m_indexerCommandList->size(); + m_interprocessIndexerCommandManager.setIndexerCommands(m_indexerCommandList->getAllCommands()); + + std::string logFilePath; + Logger* logger = LogManager::getInstance()->getLoggerByType("FileLogger"); + if (logger) + { + logFilePath = dynamic_cast(logger)->getLogFilePath().str(); + } + + bool multiProcess = ApplicationSettings::getInstance()->getMultiProcessIndexingEnabled(); + + // start indexer processes + for (unsigned int i = 0; i < m_processCount; i++) + { + const int processId = i + 1; // 0 remains reserved for the main process + + m_interprocessIntermediateStorageManagers.push_back( + std::make_shared(Application::getUUID(), processId, true) + ); + + if (multiProcess) + { + m_processThreads.push_back(new std::thread(&TaskBuildIndex::runIndexerProcess, this, processId, logFilePath)); + } + else + { + m_processThreads.push_back(new std::thread(&TaskBuildIndex::runIndexerThread, this, processId)); + } } } Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr blackboard) { - std::shared_ptr indexerCommand = m_indexerCommandList->consumeCommand(); + size_t commandCount = m_interprocessIndexerCommandManager.indexerCommandCount(); + if (commandCount != m_lastCommandCount) + { + updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath()); + m_lastCommandCount = commandCount; + } - if (!indexerCommand) + if (commandCount == 0) { return STATE_FAILURE; } - else + else if (m_interrupted) { - { - std::lock_guard lock(blackboard->getMutex()); + std::lock_guard lock(blackboard->getMutex()); + blackboard->set("interrupted_indexing", true); - int sourceFileCount = 0; - blackboard->get("source_file_count", sourceFileCount); - - int indexedSourceFileCount = 0; - blackboard->get("indexed_source_file_count", indexedSourceFileCount); - - if (std::shared_ptr dialogView = Application::getInstance()->getDialogView()) - { - dialogView->updateIndexingDialog( - indexedSourceFileCount, sourceFileCount, indexerCommand->getSourceFilePath().str() - ); - } - } - - // file register only copies the DileRegisterStateData - std::shared_ptr fileRegister = std::make_shared( - *(m_fileRegisterStateData.get()), indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath() - ); - - std::shared_ptr storage = m_indexer->index(indexerCommand, fileRegister); - if (storage) - { - // only write back the changes made to FileRegisterStateData if the indexer actually succeeded - m_fileRegisterStateData->inject(fileRegister->getStateData()); - - m_storageProvider->insert(storage); - - std::lock_guard lock(blackboard->getMutex()); - int indexedSourceFileCount = 0; - blackboard->get("indexed_source_file_count", indexedSourceFileCount); - blackboard->set("indexed_source_file_count", indexedSourceFileCount + 1); - } + // clear indexer commands, this causes the indexer processes to return when finished with respective current indexer commands + m_interprocessIndexerCommandManager.clearIndexerCommands(); + return STATE_FAILURE; } - return (m_indexer->interrupted() ? STATE_FAILURE : STATE_SUCCESS); + fetchIntermediateStorages(blackboard); + + const int SLEEP_TIME_MS = 100; + std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_MS)); + + return STATE_RUNNING; } void TaskBuildIndex::doExit(std::shared_ptr blackboard) { - std::lock_guard lock(blackboard->getMutex()); - - int indexerCount = 0; - if (blackboard->get("indexer_count", indexerCount)) + for (auto processThread : m_processThreads) { - indexerCount--; - blackboard->set("indexer_count", indexerCount); + processThread->join(); + delete processThread; } + m_processThreads.clear(); + + fetchIntermediateStorages(blackboard); + + std::vector crashedFiles = m_interprocessIndexingStatusManager.getCrashedSourceFilePaths(); + if (crashedFiles.size()) + { + std::shared_ptr is = std::make_shared(); + for (auto path : crashedFiles) + { + is->addError("The translation unit threw an exception during indexing. Please check if the source file " + "conforms to the specified language standard and all necessary options are defined within your project " + "setup.", path, 1, 1, true, true); + LOG_INFO_STREAM(<< "crashed translation unit: " << path.str()); + } + m_storageProvider->insert(is); + } + + std::lock_guard lock(blackboard->getMutex()); + blackboard->set("indexer_count", 0); } void TaskBuildIndex::doReset(std::shared_ptr blackboard) { } +void TaskBuildIndex::terminate() +{ + m_interrupted = true; + utility::killRunningProcesses(); +} + void TaskBuildIndex::handleMessage(MessageInterruptTasks* message) { - m_indexer->interrupt(); + m_interrupted = true; +} + +void TaskBuildIndex::runIndexerProcess(int processId, const std::string& logFilePath) +{ + FilePath indexerProcessPath(AppPath::getAppPath() + s_processName); + if (!indexerProcessPath.exists()) + { + m_interrupted = true; + LOG_ERROR("Cannot start indexer process because executable is missing at \"" + indexerProcessPath.str() + "\""); + return; + } + + std::string command = indexerProcessPath.str(); + command += " " + std::to_string(processId); + command += " " + Application::getUUID(); + command += " \"" + AppPath::getAppPath() + "\""; + command += " \"" + UserPaths::getUserDataPath().str() + "\""; + + if (logFilePath.size()) + { + command += " \"" + logFilePath + "\""; + } + + int result = 1; + while (result != 0 && !m_interrupted) + { + result = utility::executeProcessAndGetExitCode(command.c_str(), "", -1); + + LOG_INFO_STREAM(<< "Indexer process " << processId << " returned with " + std::to_string(result)); + } +} + +void TaskBuildIndex::runIndexerThread(int processId) +{ + InterprocessIndexer indexer(Application::getUUID(), processId); + indexer.work(); +} + +void TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr blackboard) +{ + int newlyIndexedCount = 0; + + for (std::shared_ptr storageManager: m_interprocessIntermediateStorageManagers) + { + while (int storageCount = storageManager->getIntermediateStorageCount()) + { + LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount); + m_storageProvider->insert(storageManager->popIntermediateStorage()); + ++newlyIndexedCount; + + updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath()); + } + } + + if (newlyIndexedCount > 0) + { + std::lock_guard lock(blackboard->getMutex()); + + int indexedSourceFileCount = 0; + blackboard->get("indexed_source_file_count", indexedSourceFileCount); + blackboard->set("indexed_source_file_count", indexedSourceFileCount + newlyIndexedCount); + } +} + +void TaskBuildIndex::updateIndexingDialog(std::shared_ptr blackboard, const FilePath& sourcePath) +{ + // TODO: factor in unindexed files... + int sourceFileCount = 0; + int indexedSourceFileCount = 0; + { + std::lock_guard lock(blackboard->getMutex()); + blackboard->get("source_file_count", sourceFileCount); + blackboard->get("indexed_source_file_count", indexedSourceFileCount); + } + + if (std::shared_ptr dialogView = Application::getInstance()->getDialogView()) + { + dialogView->updateIndexingDialog( + indexedSourceFileCount, sourceFileCount, sourcePath.str() + ); + } } diff --git a/src/lib/data/indexer/TaskBuildIndex.h b/src/lib/data/indexer/TaskBuildIndex.h index 7f2a4c5b..c9347c54 100644 --- a/src/lib/data/indexer/TaskBuildIndex.h +++ b/src/lib/data/indexer/TaskBuildIndex.h @@ -1,17 +1,20 @@ #ifndef TASK_BUILD_INDEX_H #define TASK_BUILD_INDEX_H -#include "data/parser/Parser.h" -#include "utility/scheduling/Task.h" -#include "utility/messaging/type/MessageInterruptTasks.h" +#include "utility/file/FileRegisterStateData.h" #include "utility/messaging/MessageListener.h" +#include "utility/messaging/type/MessageInterruptTasks.h" +#include "utility/scheduling/Task.h" + +#include "data/indexer/interprocess/InterprocessIndexerCommandManager.h" +#include "data/indexer/interprocess/InterprocessIndexingStatusManager.h" +#include "data/indexer/interprocess/InterprocessIntermediateStorageManager.h" +#include "data/parser/Parser.h" -class CxxParser; class DialogView; class FileRegisterStateData; class StorageProvider; class IndexerCommandList; -class IndexerBase; class TaskBuildIndex : public Task @@ -19,6 +22,7 @@ class TaskBuildIndex { public: TaskBuildIndex( + unsigned int processCount, std::shared_ptr indexerCommandList, std::shared_ptr storageProvider, std::shared_ptr fileRegisterStateData @@ -29,14 +33,31 @@ protected: virtual TaskState doUpdate(std::shared_ptr blackboard); virtual void doExit(std::shared_ptr blackboard); virtual void doReset(std::shared_ptr blackboard); + virtual void terminate(); virtual void handleMessage(MessageInterruptTasks* message); + void runIndexerProcess( int processId, const std::string& logFilePath); + void runIndexerThread(int processId); + void fetchIntermediateStorages(std::shared_ptr blackboard); + void updateIndexingDialog(std::shared_ptr blackboard, const FilePath& sourcePath); + + static const std::string s_processName; + std::shared_ptr m_indexerCommandList; std::shared_ptr m_storageProvider; std::shared_ptr m_fileRegisterStateData; - std::shared_ptr m_indexer; + InterprocessIndexerCommandManager m_interprocessIndexerCommandManager; + InterprocessIndexingStatusManager m_interprocessIndexingStatusManager; + + unsigned int m_processCount; + bool m_interrupted; + size_t m_lastCommandCount; + + // store as plain pointers to avoid deallocation issues when closing app during indexing + std::vector m_processThreads; + std::vector> m_interprocessIntermediateStorageManagers; }; #endif // TASK_PARSE_H diff --git a/src/lib/data/indexer/interprocess/BaseInterprocessDataManager.cpp b/src/lib/data/indexer/interprocess/BaseInterprocessDataManager.cpp new file mode 100644 index 00000000..1132036c --- /dev/null +++ b/src/lib/data/indexer/interprocess/BaseInterprocessDataManager.cpp @@ -0,0 +1,23 @@ +#include "BaseInterprocessDataManager.h" + +BaseInterprocessDataManager::BaseInterprocessDataManager( + const std::string& sharedMemoryName, + size_t initialSharedMemorySize, + const std::string& instanceUuid, + Id processId, + bool isOwner +) + : m_sharedMemory(sharedMemoryName, initialSharedMemorySize, isOwner ? SharedMemory::CREATE_AND_DELETE : SharedMemory::OPEN_ONLY) + , m_instanceUuid(instanceUuid) + , m_processId(processId) +{ +} + +BaseInterprocessDataManager::~BaseInterprocessDataManager() +{ +} + +Id BaseInterprocessDataManager::getProcessId() const +{ + return m_processId; +} diff --git a/src/lib/data/indexer/interprocess/BaseInterprocessDataManager.h b/src/lib/data/indexer/interprocess/BaseInterprocessDataManager.h new file mode 100644 index 00000000..c1d13f60 --- /dev/null +++ b/src/lib/data/indexer/interprocess/BaseInterprocessDataManager.h @@ -0,0 +1,29 @@ +#ifndef BASE_INTERPROCESS_DATA_MANAGER_H +#define BASE_INTERPROCESS_DATA_MANAGER_H + +#include + +#include "utility/types.h" +#include "utility/interprocess/SharedMemory.h" + +class BaseInterprocessDataManager +{ +public: + BaseInterprocessDataManager( + const std::string& sharedMemoryName, + size_t initialSharedMemorySize, + const std::string& instanceUuid, + Id processId, + bool isOwner); + virtual ~BaseInterprocessDataManager(); + + Id getProcessId() const; + +protected: + SharedMemory m_sharedMemory; + + const std::string m_instanceUuid; + const Id m_processId; +}; + +#endif // BASE_INTERPROCESS_DATA_MANAGER_H diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp new file mode 100644 index 00000000..6d9504fd --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIndexer.cpp @@ -0,0 +1,66 @@ +#include "InterprocessIndexer.h" + +#include "utility/file/FileRegister.h" +#include "utility/file/FileRegisterStateData.h" +#include "utility/logging/logging.h" + +#include "data/indexer/IndexerComposite.h" +#include "data/indexer/IndexerCommand.h" +#include "data/indexer/IndexerFactory.h" + +InterprocessIndexer::InterprocessIndexer(const std::string& uuid, Id processId) + : m_interprocessIndexerCommandManager(uuid, processId, false) + , m_interprocessIndexingStatusManager(uuid, processId, false) + , m_interprocessIntermediateStorageManager(uuid, processId, false) + , m_uuid(uuid) + , m_processId(processId) +{ +} + +InterprocessIndexer::~InterprocessIndexer() +{ +} + +void InterprocessIndexer::work() +{ + try + { + LOG_INFO_STREAM(<< m_processId << " Starting to index"); + std::shared_ptr indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules(); + + while (std::shared_ptr indexerCommand = m_interprocessIndexerCommandManager.popIndexerCommand()) + { + LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str()); + LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1)); + + m_interprocessIndexingStatusManager.setCurrentlyIndexedSourceFilePath(indexerCommand->getSourceFilePath()); + + FileRegisterStateData data; + data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles()); + + std::shared_ptr fileRegister = std::make_shared( + data, indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath() + ); + + std::shared_ptr result = indexer->index(indexerCommand, fileRegister); + + m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles()); + + m_interprocessIntermediateStorageManager.pushIntermediateStorage(result); + + m_interprocessIndexingStatusManager.clearCurrentlyIndexedSourceFilePath(); + } + } + catch (boost::interprocess::interprocess_exception& e) + { + LOG_ERROR(e.what()); + throw e; + } + catch (std::exception& e) + { + LOG_ERROR(e.what()); + throw e; + } + + LOG_INFO_STREAM(<< "Finished indexing"); +} diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexer.h b/src/lib/data/indexer/interprocess/InterprocessIndexer.h new file mode 100644 index 00000000..d4e04fb6 --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIndexer.h @@ -0,0 +1,25 @@ +#ifndef INTERPROCESS_INDEXER_H +#define INTERPROCESS_INDEXER_H + +#include "InterprocessIndexerCommandManager.h" +#include "InterprocessIndexingStatusManager.h" +#include "InterprocessIntermediateStorageManager.h" + +class InterprocessIndexer +{ +public: + InterprocessIndexer(const std::string& uuid, Id processId); + ~InterprocessIndexer(); + + void work(); + +private: + InterprocessIndexerCommandManager m_interprocessIndexerCommandManager; + InterprocessIndexingStatusManager m_interprocessIndexingStatusManager; + InterprocessIntermediateStorageManager m_interprocessIntermediateStorageManager; + + const std::string m_uuid; + const Id m_processId; +}; + +#endif // INTERPROCESS_INDEXER_H diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp new file mode 100644 index 00000000..07c0ff84 --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp @@ -0,0 +1,105 @@ +#include "InterprocessIndexerCommandManager.h" + +#include "data/indexer/IndexerCommand.h" +#include "utility/logging/logging.h" + +const char* InterprocessIndexerCommandManager::s_sharedMemoryNamePrefix = "icmd_"; + +const char* InterprocessIndexerCommandManager::s_indexerCommandsKeyName = "indexer_commands"; + +InterprocessIndexerCommandManager::InterprocessIndexerCommandManager(const std::string& instanceUuid, Id processId, bool isOwner) + : BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner) +{ +} + +InterprocessIndexerCommandManager::~InterprocessIndexerCommandManager() +{ +} + +void InterprocessIndexerCommandManager::setIndexerCommands( + const std::vector>& indexerCommands) +{ + const unsigned int overestimationMultiplier = 3; + + size_t size = 1000; + for (auto command : indexerCommands) + { + size += command->getByteSize() + sizeof(SharedIndexerCommand); + } + size *= overestimationMultiplier; + + + SharedMemory::ScopedAccess access(&m_sharedMemory); + + size_t freeMemory = access.getFreeMemorySize(); + if (freeMemory <= size) + { + LOG_INFO_STREAM( + << "grow memory - est: " << size << " size: " << access.getMemorySize() + << " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory)); + + access.growMemory(size - freeMemory); + } + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_indexerCommandsKeyName); + if (!queue) + { + return; + } + + for (auto command : indexerCommands) + { + queue->push_back(SharedIndexerCommand(access.getAllocator())); + SharedIndexerCommand& sharedCommand = queue->back(); + sharedCommand.fromLocal(command.get()); + } + + LOG_INFO(access.logString()); +} + +std::shared_ptr InterprocessIndexerCommandManager::popIndexerCommand() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_indexerCommandsKeyName); + if (!queue || !queue->size()) + { + return nullptr; + } + + std::shared_ptr command = SharedIndexerCommand::fromShared(queue->front()); + + queue->pop_front(); + + return command; +} + +void InterprocessIndexerCommandManager::clearIndexerCommands() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_indexerCommandsKeyName); + if (!queue) + { + return; + } + + queue->clear(); +} + +size_t InterprocessIndexerCommandManager::indexerCommandCount() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_indexerCommandsKeyName); + if (!queue) + { + return 0; + } + + return queue->size(); +} diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.h b/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.h new file mode 100644 index 00000000..8078ecef --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.h @@ -0,0 +1,27 @@ +#ifndef INTERPROCESS_INDEXER_COMMAND_MANAGER_H +#define INTERPROCESS_INDEXER_COMMAND_MANAGER_H + +#include "BaseInterprocessDataManager.h" +#include "shared_types/SharedIndexerCommand.h" + +class IndexerCommand; + +class InterprocessIndexerCommandManager + : public BaseInterprocessDataManager +{ +public: + InterprocessIndexerCommandManager(const std::string& instanceUuid, Id processId, bool isOwner); + virtual ~InterprocessIndexerCommandManager(); + + void setIndexerCommands(const std::vector>& indexerCommands); + std::shared_ptr popIndexerCommand(); + + void clearIndexerCommands(); + size_t indexerCommandCount(); + +private: + static const char* s_sharedMemoryNamePrefix; + static const char* s_indexerCommandsKeyName; +}; + +#endif // INTERPROCESS_INDEXER_COMMAND_MANAGER_H diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp new file mode 100644 index 00000000..071a5fad --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp @@ -0,0 +1,173 @@ +#include "InterprocessIndexingStatusManager.h" + +#include "utility/logging/logging.h" + +const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_"; + +const char* InterprocessIndexingStatusManager::s_lastFileKeyName = "last_file"; +const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files"; +const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files"; +const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files"; + +InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner) + : BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner) +{ +} + +InterprocessIndexingStatusManager::~InterprocessIndexingStatusManager() +{ +} + +void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const FilePath& filePath) +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::String* strPtr = access.accessValueWithAllocator(s_lastFileKeyName); + if (strPtr) + { + *strPtr = filePath.str().c_str(); + } + + SharedMemory::Map* currentFilesPtr = + access.accessValueWithAllocator>(s_currentFilesKeyName); + if (currentFilesPtr) + { + SharedMemory::Map::iterator it = currentFilesPtr->find(getProcessId()); + if (it != currentFilesPtr->end()) + { + SharedMemory::Vector* crashedFilesPtr = + access.accessValueWithAllocator>(s_crashedFilesKeyName); + + if (crashedFilesPtr) + { + crashedFilesPtr->push_back(it->second); + } + } + + SharedMemory::String str(access.getAllocator()); + str = filePath.str().c_str(); + + it = currentFilesPtr->insert(std::pair(getProcessId(), str)).first; + it->second = str; + } +} + +void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Map* currentFilesPtr = + access.accessValueWithAllocator>(s_currentFilesKeyName); + if (currentFilesPtr) + { + currentFilesPtr->erase(currentFilesPtr->find(getProcessId()), currentFilesPtr->end()); + } +} + +FilePath InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePath() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::String* strPtr = access.accessValueWithAllocator(s_lastFileKeyName); + if (strPtr) + { + return FilePath(strPtr->c_str()); + } + + return FilePath(); +} + +std::vector InterprocessIndexingStatusManager::getCrashedSourceFilePaths() +{ + std::vector crashedFiles; + + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Vector* crashedFilesPtr = + access.accessValueWithAllocator>(s_crashedFilesKeyName); + + if (crashedFilesPtr) + { + for (size_t i = 0; i < crashedFilesPtr->size(); i++) + { + crashedFiles.push_back(FilePath(crashedFilesPtr->at(i).c_str())); + } + } + + SharedMemory::Map* currentFilesPtr = + access.accessValueWithAllocator>(s_currentFilesKeyName); + if (currentFilesPtr) + { + for (SharedMemory::Map::iterator it = currentFilesPtr->begin(); it != currentFilesPtr->end(); it++) + { + crashedFiles.push_back(FilePath(it->second.c_str())); + } + } + + return crashedFiles; +} + +std::set InterprocessIndexingStatusManager::getIndexedFiles() +{ + std::set result; + + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Vector* files = + access.accessValueWithAllocator>(s_indexedFilesKeyName); + if (!files) + { + return result; + } + + for (auto file : *files) + { + result.insert(FilePath(file.c_str())); + } + + return result; +} + +void InterprocessIndexingStatusManager::addIndexedFiles(std::set filePaths) +{ + const unsigned int overestimationMultiplier = 3; + + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Vector* files = + access.accessValueWithAllocator>(s_indexedFilesKeyName); + if (!files) + { + return; + } + + for (auto file : *files) + { + filePaths.insert(FilePath(file.c_str())); + } + files->clear(); + + + size_t size = 1000; + for (auto path : filePaths) + { + size += sizeof(std::string) + path.str().size(); + } + size *= overestimationMultiplier; + + size_t freeMemory = access.getFreeMemorySize(); + if (freeMemory <= size) + { + LOG_INFO_STREAM( + << "grow memory - est: " << size << " size: " << access.getMemorySize() + << " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory)); + access.growMemory(size - freeMemory); + } + + for (auto path : filePaths) + { + files->push_back(SharedMemory::String(path.str().c_str(), access.getAllocator())); + } + + LOG_INFO(access.logString()); +} diff --git a/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h new file mode 100644 index 00000000..b8b83e38 --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h @@ -0,0 +1,34 @@ +#ifndef INTERPROCESS_INDEXING_STATUS_MANAGER_H +#define INTERPROCESS_INDEXING_STATUS_MANAGER_H + +#include + +#include "utility/file/FilePath.h" +#include "BaseInterprocessDataManager.h" + +class InterprocessIndexingStatusManager + : public BaseInterprocessDataManager +{ +public: + InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner); + virtual ~InterprocessIndexingStatusManager(); + + void setCurrentlyIndexedSourceFilePath(const FilePath& filePath); + void clearCurrentlyIndexedSourceFilePath(); + FilePath getCurrentlyIndexedSourceFilePath(); + + std::vector getCrashedSourceFilePaths(); + + std::set getIndexedFiles(); + void addIndexedFiles(std::set filePaths); + +private: + static const char* s_sharedMemoryNamePrefix; + + static const char* s_lastFileKeyName; + static const char* s_currentFilesKeyName; + static const char* s_crashedFilesKeyName; + static const char* s_indexedFilesKeyName; +}; + +#endif // INTERPROCESS_INDEXING_STATUS_MANAGER_H diff --git a/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp b/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp new file mode 100644 index 00000000..0d0ad701 --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp @@ -0,0 +1,115 @@ +#include "InterprocessIntermediateStorageManager.h" + +#include "data/IntermediateStorage.h" +#include "utility/logging/logging.h" + +const char* InterprocessIntermediateStorageManager::s_sharedMemoryNamePrefix = "iist_"; + +const char* InterprocessIntermediateStorageManager::s_intermediatStoragesKeyName = "intermediate_storages"; + +InterprocessIntermediateStorageManager::InterprocessIntermediateStorageManager( + const std::string& instanceUuid, Id processId, bool isOwner +) + : BaseInterprocessDataManager( + s_sharedMemoryNamePrefix + std::to_string(processId) + "_" + instanceUuid, + 1048576 /* 1 MB */, + instanceUuid, + processId, + isOwner) +{ +} + +InterprocessIntermediateStorageManager::~InterprocessIntermediateStorageManager() +{ +} + +void InterprocessIntermediateStorageManager::pushIntermediateStorage( + const std::shared_ptr& intermediateStorage) +{ + const unsigned int overestimationMultiplier = 3; + size_t size = intermediateStorage->getByteSize() * overestimationMultiplier; + + SharedMemory::ScopedAccess access(&m_sharedMemory); + + size_t freeMemory = access.getFreeMemorySize(); + if (freeMemory <= size) + { + LOG_INFO_STREAM( + << "grow memory - est: " << size << " size: " << access.getMemorySize() + << " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory)); + + access.growMemory(size - freeMemory); + } + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_intermediatStoragesKeyName); + if (!queue) + { + return; + } + + queue->push_back(SharedIntermediateStorage(access.getAllocator())); + SharedIntermediateStorage& storage = queue->back(); + + storage.setStorageNodes(intermediateStorage->getStorageNodes()); + storage.setStorageFiles(intermediateStorage->getStorageFiles()); + storage.setStorageSymbols(intermediateStorage->getStorageSymbols()); + storage.setStorageEdges(intermediateStorage->getStorageEdges()); + storage.setStorageLocalSymbols(intermediateStorage->getStorageLocalSymbols()); + storage.setStorageSourceLocations(intermediateStorage->getStorageSourceLocations()); + storage.setStorageOccurrences(intermediateStorage->getStorageOccurrences()); + storage.setStorageComponentAccesses(intermediateStorage->getComponentAccesses()); + storage.setStorageCommentLocations(intermediateStorage->getCommentLocations()); + storage.setStorageErrors(intermediateStorage->getErrors()); + + storage.setNextId(intermediateStorage->getNextId()); + + LOG_INFO(access.logString()); +} + +std::shared_ptr InterprocessIntermediateStorageManager::popIntermediateStorage() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_intermediatStoragesKeyName); + if (!queue || !queue->size()) + { + return nullptr; + } + + SharedIntermediateStorage& sharedIntermediateStorage = queue->front(); + + std::shared_ptr storage = std::make_shared(); + + storage->setStorageNodes(sharedIntermediateStorage.getStorageNodes()); + storage->setStorageFiles(sharedIntermediateStorage.getStorageFiles()); + storage->setStorageSymbols(sharedIntermediateStorage.getStorageSymbols()); + storage->setStorageEdges(sharedIntermediateStorage.getStorageEdges()); + storage->setStorageLocalSymbols(sharedIntermediateStorage.getStorageLocalSymbols()); + storage->setStorageSourceLocations(sharedIntermediateStorage.getStorageSourceLocations()); + storage->setStorageOccurrences(sharedIntermediateStorage.getStorageOccurrences()); + storage->setComponentAccesses(sharedIntermediateStorage.getStorageComponentAccesses()); + storage->setCommentLocations(sharedIntermediateStorage.getStorageCommentLocations()); + storage->setErrors(sharedIntermediateStorage.getStorageErrors()); + + storage->setNextId(sharedIntermediateStorage.getNextId()); + + queue->pop_front(); + + return storage; +} + +size_t InterprocessIntermediateStorageManager::getIntermediateStorageCount() +{ + SharedMemory::ScopedAccess access(&m_sharedMemory); + + SharedMemory::Queue* queue = + access.accessValueWithAllocator>(s_intermediatStoragesKeyName); + if (!queue) + { + return 0; + } + + return queue->size(); +} diff --git a/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.h b/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.h new file mode 100644 index 00000000..7d723f7a --- /dev/null +++ b/src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.h @@ -0,0 +1,26 @@ +#ifndef INTERPROCESS_INTERMEDIATE_STORAGE_MANAGER_H +#define INTERPROCESS_INTERMEDIATE_STORAGE_MANAGER_H + +#include "BaseInterprocessDataManager.h" +#include "shared_types/SharedIntermediateStorage.h" + +class IntermediateStorage; + +class InterprocessIntermediateStorageManager + : public BaseInterprocessDataManager +{ +public: + InterprocessIntermediateStorageManager(const std::string& instanceUuid, Id processId, bool isOwner); + virtual ~InterprocessIntermediateStorageManager(); + + void pushIntermediateStorage(const std::shared_ptr& intermediateStorage); + std::shared_ptr popIntermediateStorage(); + + size_t getIntermediateStorageCount(); + +private: + static const char* s_sharedMemoryNamePrefix; + static const char* s_intermediatStoragesKeyName; +}; + +#endif // INTERPROCESS_INTERMEDIATE_STORAGE_MANAGER_H diff --git a/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp b/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp new file mode 100644 index 00000000..614458d8 --- /dev/null +++ b/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp @@ -0,0 +1,312 @@ +#include "SharedIndexerCommand.h" + +#include "data/indexer/IndexerCommandCxxCdb.h" +#include "data/indexer/IndexerCommandCxxManual.h" +#include "data/indexer/IndexerCommandJava.h" + +#include "utility/logging/logging.h" + +void SharedIndexerCommand::fromLocal(IndexerCommand* indexerCommand) +{ + setSourceFilePath(indexerCommand->getSourceFilePath()); + setIndexedPaths(indexerCommand->getIndexedPaths()); + setExcludedPaths(indexerCommand->getExcludedPath()); + + if (dynamic_cast(indexerCommand) != NULL) + { + IndexerCommandCxxCdb* cmd = dynamic_cast(indexerCommand); + + setType(CXX_CDB); + setWorkingDirectory(cmd->getWorkingDirectory()); + setCompilerFlags(cmd->getCompilerFlags()); + setSystemHeaderSearchPaths(cmd->getSystemHeaderSearchPaths()); + setFrameworkSearchhPaths(cmd->getFrameworkSearchPaths()); + setPreprocessorOnly(cmd->preprocessorOnly()); + } + else if (dynamic_cast(indexerCommand) != NULL) + { + IndexerCommandCxxManual* cmd = dynamic_cast(indexerCommand); + + setType(CXX_MANUAL); + setLanguageStandard(cmd->getLanguageStandard()); + setCompilerFlags(cmd->getCompilerFlags()); + setSystemHeaderSearchPaths(cmd->getSystemHeaderSearchPaths()); + setFrameworkSearchhPaths(cmd->getFrameworkSearchPaths()); + setPreprocessorOnly(cmd->preprocessorOnly()); + } + else if (dynamic_cast(indexerCommand) != NULL) + { + IndexerCommandJava* cmd = dynamic_cast(indexerCommand); + + setType(JAVA); + setClassPaths(cmd->getClassPath()); + } + else + { + LOG_ERROR_STREAM(<< "Trying to push unhandled type of IndexerCommand for file: " + << indexerCommand->getSourceFilePath().str() << ". Type string is: " << indexerCommand->getKindString() + << ". It will be ignored."); + } +} + +std::shared_ptr SharedIndexerCommand::fromShared(const SharedIndexerCommand& indexerCommand) +{ + if (indexerCommand.getType() == CXX_CDB) + { + std::shared_ptr command = std::make_shared( + indexerCommand.getSourceFilePath(), + indexerCommand.getIndexedPaths(), + indexerCommand.getExcludedPaths(), + indexerCommand.getWorkingDirectory(), + indexerCommand.getCompilerFlags(), + indexerCommand.getSystemHeaderSearchPaths(), + indexerCommand.getFrameworkSearchhPaths() + ); + command->setPreprocessorOnly(indexerCommand.preprocessorOnly()); + return command; + } + else if (indexerCommand.getType() == CXX_MANUAL) + { + std::shared_ptr command = std::make_shared( + indexerCommand.getSourceFilePath(), + indexerCommand.getIndexedPaths(), + indexerCommand.getExcludedPaths(), + indexerCommand.getLanguageStandard(), + indexerCommand.getSystemHeaderSearchPaths(), + indexerCommand.getFrameworkSearchhPaths(), + indexerCommand.getCompilerFlags() + ); + command->setPreprocessorOnly(indexerCommand.preprocessorOnly()); + return command; + } + else if (indexerCommand.getType() == JAVA) + { + return std::make_shared( + indexerCommand.getSourceFilePath(), + indexerCommand.getIndexedPaths(), + indexerCommand.getExcludedPaths(), + indexerCommand.getClassPaths() + ); + } + else + { + LOG_ERROR_STREAM(<< "Cannot convert shared IndexerCommand for file: " + << indexerCommand.getSourceFilePath().str() << ". The type is unknown."); + } + + return nullptr; +} + + +SharedIndexerCommand::SharedIndexerCommand(SharedMemory::Allocator* allocator) + : m_type(Type::UNKNOWN) + , m_sourceFilePath("", allocator) + , m_indexedPaths(allocator) + , m_excludedPaths(allocator) + , m_workingDirectory("", allocator) + , m_languageStandard("", allocator) + , m_compilerFlags(allocator) + , m_systemHeaderSearchPaths(allocator) + , m_frameworkSearchPaths(allocator) + , m_preprocessorOnly(false) + , m_classPaths(allocator) +{ +} + +SharedIndexerCommand::~SharedIndexerCommand() +{ +} + +FilePath SharedIndexerCommand::getSourceFilePath() const +{ + return FilePath(m_sourceFilePath.c_str()); +} + +void SharedIndexerCommand::setSourceFilePath(const FilePath& filePath) +{ + m_sourceFilePath = filePath.str().c_str(); +} + +std::set SharedIndexerCommand::getIndexedPaths() const +{ + std::set result; + + for (unsigned int i = 0; i < m_indexedPaths.size(); i++) + { + result.insert(FilePath(m_indexedPaths[i].c_str())); + } + + return result; +} + +void SharedIndexerCommand::setIndexedPaths(const std::set& indexedPaths) +{ + m_indexedPaths.clear(); + + for (std::set::iterator it = indexedPaths.begin(); it != indexedPaths.end(); it++) + { + SharedMemory::String path(m_indexedPaths.get_allocator()); + path = (*it).str().c_str(); + m_indexedPaths.push_back(path); + } +} + +std::set SharedIndexerCommand::getExcludedPaths() const +{ + std::set result; + + for (unsigned int i = 0; i < m_excludedPaths.size(); i++) + { + result.insert(FilePath(m_excludedPaths[i].c_str())); + } + + return result; +} + +void SharedIndexerCommand::setExcludedPaths(const std::set& excludedPaths) +{ + m_excludedPaths.clear(); + + for (std::set::iterator it = excludedPaths.begin(); it != excludedPaths.end(); it++) + { + SharedMemory::String path(m_excludedPaths.get_allocator()); + path = (*it).str().c_str(); + m_excludedPaths.push_back(path); + } +} + +FilePath SharedIndexerCommand::getWorkingDirectory() const +{ + return FilePath(m_workingDirectory.c_str()); +} + +void SharedIndexerCommand::setWorkingDirectory(const FilePath& workingDirectory) +{ + m_workingDirectory = workingDirectory.str().c_str(); +} + +std::string SharedIndexerCommand::getLanguageStandard() const +{ + return m_languageStandard.c_str(); +} + +void SharedIndexerCommand::setLanguageStandard(const std::string& languageStandard) +{ + m_languageStandard = languageStandard.c_str(); +} + +std::vector SharedIndexerCommand::getCompilerFlags() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_compilerFlags.size(); i++) + { + result.push_back(m_compilerFlags[i].c_str()); + } + + return result; +} + +void SharedIndexerCommand::setCompilerFlags(const std::vector& compilerFlags) +{ + m_compilerFlags.clear(); + + for (unsigned int i = 0; i < compilerFlags.size(); i++) + { + SharedMemory::String path(m_compilerFlags.get_allocator()); + path = compilerFlags[i].c_str(); + m_compilerFlags.push_back(path); + } +} + +std::vector SharedIndexerCommand::getSystemHeaderSearchPaths() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++) + { + result.push_back(FilePath(m_systemHeaderSearchPaths[i].c_str())); + } + + return result; +} + +void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector& filePaths) +{ + m_systemHeaderSearchPaths.clear(); + + for (unsigned int i = 0; i < filePaths.size(); i++) + { + SharedMemory::String path(m_systemHeaderSearchPaths.get_allocator()); + path = filePaths[i].str().c_str(); + m_systemHeaderSearchPaths.push_back(path); + } +} + +std::vector SharedIndexerCommand::getFrameworkSearchhPaths() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++) + { + result.push_back(FilePath(m_frameworkSearchPaths[i].c_str())); + } + + return result; +} + +void SharedIndexerCommand::setFrameworkSearchhPaths(const std::vector& searchPaths) +{ + m_frameworkSearchPaths.clear(); + + for (unsigned int i = 0; i < searchPaths.size(); i++) + { + SharedMemory::String path(m_frameworkSearchPaths.get_allocator()); + path = searchPaths[i].str().c_str(); + m_frameworkSearchPaths.push_back(path); + } +} + +bool SharedIndexerCommand::preprocessorOnly() const +{ + return m_preprocessorOnly; +} + +void SharedIndexerCommand::setPreprocessorOnly(bool preprocessorOnly) +{ + m_preprocessorOnly = preprocessorOnly; +} + +std::vector SharedIndexerCommand::getClassPaths() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_classPaths.size(); i++) + { + result.push_back(FilePath(m_classPaths[i].c_str())); + } + + return result; +} + +void SharedIndexerCommand::setClassPaths(const std::vector& classPaths) +{ + m_classPaths.clear(); + + for (unsigned int i = 0; i < classPaths.size(); i++) + { + SharedMemory::String path(m_classPaths.get_allocator()); + path = classPaths[i].str().c_str(); + m_classPaths.push_back(path); + } +} + +SharedIndexerCommand::Type SharedIndexerCommand::getType() const +{ + return m_type; +} + +void SharedIndexerCommand::setType(const SharedIndexerCommand::Type type) +{ + m_type = type; +} diff --git a/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.h b/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.h new file mode 100644 index 00000000..2794ad7f --- /dev/null +++ b/src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.h @@ -0,0 +1,81 @@ +#ifndef SHARED_INDEXER_COMMAND_H +#define SHARED_INDEXER_COMMAND_H + +#include + +#include "utility/file/FilePath.h" +#include "utility/interprocess/SharedMemory.h" + +class IndexerCommand; + +class SharedIndexerCommand +{ +public: + void fromLocal(IndexerCommand* indexerCommand); + static std::shared_ptr fromShared(const SharedIndexerCommand& indexerCommand); + + SharedIndexerCommand(SharedMemory::Allocator* allocator); + ~SharedIndexerCommand(); + + FilePath getSourceFilePath() const; + void setSourceFilePath(const FilePath& filePath); + + std::set getIndexedPaths() const; + void setIndexedPaths(const std::set& indexedPaths); + + std::set getExcludedPaths() const; + void setExcludedPaths(const std::set& excludedPaths); + + FilePath getWorkingDirectory() const; + void setWorkingDirectory(const FilePath& workingDirectory); + + std::string getLanguageStandard() const; + void setLanguageStandard(const std::string& languageStandard); + + std::vector getCompilerFlags() const; + void setCompilerFlags(const std::vector& compilerFlags); + + std::vector getSystemHeaderSearchPaths() const; + void setSystemHeaderSearchPaths(const std::vector& filePaths); + + std::vector getFrameworkSearchhPaths() const; + void setFrameworkSearchhPaths(const std::vector& searchPaths); + + bool preprocessorOnly() const; + void setPreprocessorOnly(bool preprocessorOnly); + + std::vector getClassPaths() const; + void setClassPaths(const std::vector& classPaths); + +private: + enum Type + { + UNKNOWN = 0, + CXX_CDB, + CXX_MANUAL, + JAVA + }; + + Type getType() const; + void setType(const Type type); + + Type m_type; + + // indexer command + SharedMemory::String m_sourceFilePath; + SharedMemory::Vector m_indexedPaths; + SharedMemory::Vector m_excludedPaths; + + // cxx + SharedMemory::String m_workingDirectory; + SharedMemory::String m_languageStandard; + SharedMemory::Vector m_compilerFlags; + SharedMemory::Vector m_systemHeaderSearchPaths; + SharedMemory::Vector m_frameworkSearchPaths; + bool m_preprocessorOnly; + + // java + SharedMemory::Vector m_classPaths; +}; + +#endif // SHARED_INDEXER_COMMAND_H diff --git a/src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp b/src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp new file mode 100644 index 00000000..15bc1d8f --- /dev/null +++ b/src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp @@ -0,0 +1,251 @@ +#include "SharedIntermediateStorage.h" + +SharedIntermediateStorage::SharedIntermediateStorage(SharedMemory::Allocator* allocator) + : m_storageFiles(allocator) + , m_storageSymbols(allocator) + , m_storageOccurrences(allocator) + , m_storageComponentAccesses(allocator) + , m_storageCommentLocations(allocator) + , m_storageNodes(allocator) + , m_storageEdges(allocator) + , m_storageLocalSymbols(allocator) + , m_storageSourceLocations(allocator) + , m_storageErrors(allocator) + , m_allocator(allocator) + , m_nextId(1) +{ +} + +SharedIntermediateStorage::~SharedIntermediateStorage() +{ +} + +std::vector SharedIntermediateStorage::getStorageFiles() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageFiles.size(); i++) + { + result.push_back(fromShared(m_storageFiles[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageFiles(const std::vector& storageFiles) +{ + m_storageFiles.clear(); + + for (unsigned int i = 0; i < storageFiles.size(); i++) + { + m_storageFiles.push_back(toShared(storageFiles[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageNodes() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageNodes.size(); i++) + { + result.push_back(fromShared(m_storageNodes[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageNodes(const std::vector& storageNodes) +{ + m_storageNodes.clear(); + + for (unsigned int i = 0; i < storageNodes.size(); i++) + { + m_storageNodes.push_back(toShared(storageNodes[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageSymbols() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageSymbols.size(); i++) + { + result.push_back(fromShared(m_storageSymbols[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageSymbols(const std::vector& storageSymbols) +{ + m_storageSymbols.clear(); + + for (unsigned int i = 0; i < storageSymbols.size(); i++) + { + m_storageSymbols.push_back(toShared(storageSymbols[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageEdges() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageEdges.size(); i++) + { + result.push_back(fromShared(m_storageEdges[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageEdges(const std::vector& storageEdges) +{ + m_storageEdges.clear(); + + for (unsigned int i = 0; i < storageEdges.size(); i++) + { + m_storageEdges.push_back(toShared(storageEdges[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageLocalSymbols() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageLocalSymbols.size(); i++) + { + result.push_back(fromShared(m_storageLocalSymbols[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageLocalSymbols(const std::vector& storageLocalSymbols) +{ + m_storageLocalSymbols.clear(); + + for (unsigned int i = 0; i < storageLocalSymbols.size(); i++) + { + m_storageLocalSymbols.push_back(toShared(storageLocalSymbols[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageSourceLocations() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageSourceLocations.size(); i++) + { + result.push_back(fromShared(m_storageSourceLocations[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageSourceLocations(const std::vector& storageSourceLocations) +{ + m_storageSourceLocations.clear(); + + for (unsigned int i = 0; i < storageSourceLocations.size(); i++) + { + m_storageSourceLocations.push_back(toShared(storageSourceLocations[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageOccurrences() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageOccurrences.size(); i++) + { + result.push_back(fromShared(m_storageOccurrences[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageOccurrences(const std::vector& storageOccurences) +{ + m_storageOccurrences.clear(); + + for (unsigned int i = 0; i < storageOccurences.size(); i++) + { + m_storageOccurrences.push_back(toShared(storageOccurences[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageComponentAccesses() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageComponentAccesses.size(); i++) + { + result.push_back(fromShared(m_storageComponentAccesses[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageComponentAccesses(const std::vector& storageComponentAccesses) +{ + m_storageComponentAccesses.clear(); + + for (unsigned int i = 0; i < storageComponentAccesses.size(); i++) + { + m_storageComponentAccesses.push_back(toShared(storageComponentAccesses[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageCommentLocations() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageCommentLocations.size(); i++) + { + result.push_back(fromShared(m_storageCommentLocations[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageCommentLocations(const std::vector& commentLocations) +{ + m_storageCommentLocations.clear(); + + for (unsigned int i = 0; i < commentLocations.size(); i++) + { + m_storageCommentLocations.push_back(toShared(commentLocations[i], m_allocator)); + } +} + +std::vector SharedIntermediateStorage::getStorageErrors() const +{ + std::vector result; + + for (unsigned int i = 0; i < m_storageErrors.size(); i++) + { + result.push_back(fromShared(m_storageErrors[i])); + } + + return result; +} + +void SharedIntermediateStorage::setStorageErrors(const std::vector& errors) +{ + m_storageErrors.clear(); + + for (unsigned int i = 0; i < errors.size(); i++) + { + m_storageErrors.push_back(toShared(errors[i], m_allocator)); + } +} + +Id SharedIntermediateStorage::getNextId() const +{ + return m_nextId; +} + +void SharedIntermediateStorage::setNextId(const Id nextId) +{ + m_nextId = nextId; +} diff --git a/src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.h b/src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.h new file mode 100644 index 00000000..0ad59509 --- /dev/null +++ b/src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.h @@ -0,0 +1,64 @@ +#ifndef SHARED_INTERMEDIATE_STORAGE_H +#define SHARED_INTERMEDIATE_STORAGE_H + +#include "data/StorageTypes.h" +#include "SharedStorageTypes.h" +#include "utility/interprocess/SharedMemory.h" + +class SharedIntermediateStorage +{ +public: + SharedIntermediateStorage(SharedMemory::Allocator* allocator); + ~SharedIntermediateStorage(); + + std::vector getStorageNodes() const; + void setStorageNodes(const std::vector& storageNodes); + + std::vector getStorageFiles() const; + void setStorageFiles(const std::vector& storageFiles); + + std::vector getStorageSymbols() const; + void setStorageSymbols(const std::vector& storageSymbols); + + std::vector getStorageEdges() const; + void setStorageEdges(const std::vector& storageEdges); + + std::vector getStorageLocalSymbols() const; + void setStorageLocalSymbols(const std::vector& storageLocalSymbols); + + std::vector getStorageSourceLocations() const; + void setStorageSourceLocations(const std::vector& storageSourceLocations); + + std::vector getStorageOccurrences() const; + void setStorageOccurrences(const std::vector& storageOccurences); + + std::vector getStorageComponentAccesses() const; + void setStorageComponentAccesses(const std::vector& storageComponentAccesses); + + std::vector getStorageCommentLocations() const; + void setStorageCommentLocations(const std::vector& commentLocations); + + std::vector getStorageErrors() const; + void setStorageErrors(const std::vector& errors); + + Id getNextId() const; + void setNextId(const Id nextId); + +private: + SharedMemory::Vector m_storageFiles; + SharedMemory::Vector m_storageSymbols; + SharedMemory::Vector m_storageOccurrences; + SharedMemory::Vector m_storageComponentAccesses; + SharedMemory::Vector m_storageCommentLocations; + SharedMemory::Vector m_storageNodes; + SharedMemory::Vector m_storageEdges; + SharedMemory::Vector m_storageLocalSymbols; + SharedMemory::Vector m_storageSourceLocations; + SharedMemory::Vector m_storageErrors; + + SharedMemory::Allocator* m_allocator; + + int m_nextId; +}; + +#endif // SHARED_INTERMEDIATE_STORAGE_H diff --git a/src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h b/src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h new file mode 100644 index 00000000..ec5abf0f --- /dev/null +++ b/src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h @@ -0,0 +1,152 @@ +#ifndef SHARED_STORAGE_TYPES_H +#define SHARED_STORAGE_TYPES_H + +#include "utility/types.h" +#include "utility/interprocess/SharedMemory.h" + +// macro creating SharedStorageType from StorageType +// - arguments: StorageType & SharedStorageType +// - defines: conversion functions toShared() & fromShared() + +#define CONVERT_STORAGE_TYPE_TO_SHARED_TYPE(__type__, __shared_type__) \ + typedef __type__ __shared_type__; \ +\ + inline const __shared_type__& toShared(const __type__& instance, SharedMemory::Allocator* allocator) \ + { \ + return instance; \ + } \ +\ + inline const __type__& fromShared(const __shared_type__& instance) \ + { \ + return instance; \ + } + +CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageEdge, SharedStorageEdge ) +CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageSymbol, SharedStorageSymbol ) +CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageSourceLocation, SharedStorageSourceLocation ) +CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageOccurrence, SharedStorageOccurrence ) +CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageComponentAccess, SharedStorageComponentAccess ) +CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageCommentLocation, SharedStorageCommentLocation ) + + +struct SharedStorageNode +{ + SharedStorageNode(Id id, int type, const std::string& serializedName, SharedMemory::Allocator* allocator) + : id(id) + , type(type) + , serializedName(serializedName.c_str(), allocator) + {} + + Id id; + int type; + SharedMemory::String serializedName; +}; + +inline SharedStorageNode toShared(const StorageNode& node, SharedMemory::Allocator* allocator) +{ + return SharedStorageNode(node.id, node.type, node.serializedName, allocator); +} + +inline StorageNode fromShared(const SharedStorageNode& node) +{ + return StorageNode(node.id, node.type, node.serializedName.c_str()); +} + + +struct SharedStorageFile +{ + SharedStorageFile( + Id id, const std::string& filePath, const std::string& modificationTime, bool complete, SharedMemory::Allocator* allocator + ) + : id(id) + , filePath(filePath.c_str(), allocator) + , modificationTime(modificationTime.c_str(), allocator) + , complete(complete) + {} + + Id id; + SharedMemory::String filePath; + SharedMemory::String modificationTime; + bool complete; +}; + +inline SharedStorageFile toShared(const StorageFile& file, SharedMemory::Allocator* allocator) +{ + return SharedStorageFile(file.id, file.filePath, file.modificationTime, file.complete, allocator); +} + +inline StorageFile fromShared(const SharedStorageFile& file) +{ + return StorageFile(file.id, file.filePath.c_str(), file.modificationTime.c_str(), file.complete); +} + + +struct SharedStorageLocalSymbol +{ + SharedStorageLocalSymbol(Id id, const std::string& name, SharedMemory::Allocator* allocator) + : id(id) + , name(name.c_str(), allocator) + {} + + Id id; + SharedMemory::String name; +}; + +inline SharedStorageLocalSymbol toShared(const StorageLocalSymbol& symbol, SharedMemory::Allocator* allocator) +{ + return SharedStorageLocalSymbol(symbol.id, symbol.name, allocator); +} + +inline StorageLocalSymbol fromShared(const SharedStorageLocalSymbol& symbol) +{ + return StorageLocalSymbol(symbol.id, symbol.name.c_str()); +} + + +struct SharedStorageError +{ + SharedStorageError( + Id id, + const std::string& message, + const std::string& filePath, + uint lineNumber, + uint columnNumber, + bool fatal, + bool indexed, + SharedMemory::Allocator* allocator + ) + : id(id) + , message(message.c_str(), allocator) + , filePath(filePath.c_str(), allocator) + , lineNumber(lineNumber) + , columnNumber(columnNumber) + , fatal(fatal) + , indexed(indexed) + {} + + Id id; + SharedMemory::String message; + + SharedMemory::String filePath; + uint lineNumber; + uint columnNumber; + + bool fatal; + bool indexed; +}; + +inline SharedStorageError toShared(const StorageError& error, SharedMemory::Allocator* allocator) +{ + return SharedStorageError( + error.id, error.message, error.filePath.str(), + error.lineNumber, error.columnNumber, error.fatal, error.indexed, allocator); +} + +inline StorageError fromShared(const SharedStorageError& error) +{ + return StorageError( + error.id, error.message.c_str(), FilePath(error.filePath.c_str()), + error.lineNumber, error.columnNumber, error.fatal, error.indexed); +} + +#endif // SHARED_STORAGE_TYPES_H diff --git a/src/lib/data/parser/ParserClient.cpp b/src/lib/data/parser/ParserClient.cpp index 125cf5fa..ffffffcd 100644 --- a/src/lib/data/parser/ParserClient.cpp +++ b/src/lib/data/parser/ParserClient.cpp @@ -73,7 +73,6 @@ std::string ParserClient::addLocationSuffix( ParserClient::ParserClient() : m_hasFatalErrors(false) - , m_cancelOnFatalErrors(false) { } @@ -95,13 +94,3 @@ bool ParserClient::hasFatalErrors() const { return m_hasFatalErrors; } - -bool ParserClient::cancelOnFatalErrors() const -{ - return m_cancelOnFatalErrors; -} - -void ParserClient::setCancelOnFatalErrors(bool cancelOnFatalErrors) -{ - m_cancelOnFatalErrors = cancelOnFatalErrors; -} diff --git a/src/lib/data/parser/ParserClient.h b/src/lib/data/parser/ParserClient.h index f623903b..73d601b3 100644 --- a/src/lib/data/parser/ParserClient.h +++ b/src/lib/data/parser/ParserClient.h @@ -55,12 +55,8 @@ public: bool hasFatalErrors() const; - bool cancelOnFatalErrors() const; - void setCancelOnFatalErrors(bool cancelOnFatalErrors); - protected: bool m_hasFatalErrors; - bool m_cancelOnFatalErrors; }; #endif // PARSER_CLIENT_H diff --git a/src/lib/project/Project.cpp b/src/lib/project/Project.cpp index e983c8e1..8d950112 100644 --- a/src/lib/project/Project.cpp +++ b/src/lib/project/Project.cpp @@ -47,6 +47,7 @@ Project::Project(std::shared_ptr settings, StorageAccessProxy* { } + Project::~Project() { } @@ -418,7 +419,6 @@ bool Project::requestIndex(bool forceRefresh, bool needsFullRefresh) void Project::buildIndex(const std::set& filesToClean, bool fullRefresh, bool preprocessorOnly) { MessageClearErrorCount().dispatch(); - if (fullRefresh) { m_storage->clear(); @@ -433,20 +433,16 @@ void Project::buildIndex(const std::set& filesToClean, bool fullRefres { taskSequential->addTask(std::make_shared( m_storage.get(), - utility::toVector(filesToClean)) - ); + utility::toVector(filesToClean) + )); } std::shared_ptr indexerCommandList = std::make_shared(); - bool cancelIndexingOnFatalErrors = ApplicationSettings::getInstance()->getCancelIndexingOnFatalErrors(); - for (std::shared_ptr sourceGroup: m_sourceGroups) { for (std::shared_ptr command: sourceGroup->getIndexerCommands(fullRefresh)) { - command->setCancelOnFatalErrors(cancelIndexingOnFatalErrors); command->setPreprocessorOnly(preprocessorOnly); - indexerCommandList->addCommand(command); } } @@ -463,36 +459,33 @@ void Project::buildIndex(const std::set& filesToClean, bool fullRefres } } + indexerThreadCount = std::min(indexerThreadCount, indexerCommandList->size()); if (indexerThreadCount > 1) { indexerCommandList->shuffle(); } std::shared_ptr fileRegisterStateData = std::make_shared(); - std::shared_ptr storageProvider = std::make_shared(); - // add tasks for setting some variables on the blackboard that are used during indexing taskSequential->addTask(std::make_shared>("source_file_count", indexerCommandList->size())); taskSequential->addTask(std::make_shared>("indexed_source_file_count", 0)); taskSequential->addTask(std::make_shared>("indexer_count", 0)); std::shared_ptr taskParserWrapper = std::make_shared(m_storage.get()); - taskSequential->addTask(taskParserWrapper); + taskSequential->addTask(taskParserWrapper); std::shared_ptr taskParallelIndexing = std::make_shared(); taskParserWrapper->setTask(taskParallelIndexing); - - // add tasks for indexing and merging - for (int i = 0; i < indexerThreadCount && size_t(i) < indexerCommandList->size(); i++) + // add task for indexing + if (indexerThreadCount > 0) { taskParallelIndexing->addChildTasks( std::make_shared(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS)->addChildTask( - std::make_shared(indexerCommandList, storageProvider, fileRegisterStateData) + std::make_shared(indexerThreadCount, indexerCommandList, storageProvider, fileRegisterStateData) ) ); } - // add task for merging the intermediate storages taskParallelIndexing->addTask( std::make_shared()->addChildTasks( @@ -507,13 +500,12 @@ void Project::buildIndex(const std::set& filesToClean, bool fullRefres ) ) ); - // add task for injecting the intermediate storages into the persistent storage taskParallelIndexing->addTask( std::make_shared()->addChildTasks( std::make_shared(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS)->addChildTask( std::make_shared>("indexer_count", TaskReturnSuccessWhile::CONDITION_EQUALS, 0) - ), + ), std::make_shared(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS)->addChildTask( std::make_shared()->addChildTasks( // stopping when indexer count is zero, regardless wether there are still storages left to insert. @@ -527,7 +519,6 @@ void Project::buildIndex(const std::set& filesToClean, bool fullRefres ) ) ); - // add task that notifies the user of what's going on taskSequential->addTask( // we don't need to hide this dialog again, because it's overridden by other dialogs later on. std::make_shared("Finish Indexing", "Saving\nRemaining Data") diff --git a/src/lib/settings/ApplicationSettings.cpp b/src/lib/settings/ApplicationSettings.cpp index a7829e52..2ee9e0bd 100644 --- a/src/lib/settings/ApplicationSettings.cpp +++ b/src/lib/settings/ApplicationSettings.cpp @@ -246,14 +246,14 @@ void ApplicationSettings::setIndexerThreadCount(const int count) setValue("indexing/indexer_thread_count", count); } -bool ApplicationSettings::getCancelIndexingOnFatalErrors() const +bool ApplicationSettings::getMultiProcessIndexingEnabled() const { - return getValue("indexing/cancel_on_fatal_errors", true); + return getValue("indexing/multi_process_indexing", true); } -void ApplicationSettings::setCancelIndexingOnFatalErrors(bool enabled) +void ApplicationSettings::setMultiProcessIndexingEnabled(bool enabled) { - setValue("indexing/cancel_on_fatal_errors", enabled); + setValue("indexing/multi_process_indexing", enabled); } std::string ApplicationSettings::getJavaPath() const diff --git a/src/lib/settings/ApplicationSettings.h b/src/lib/settings/ApplicationSettings.h index 41de1b3c..3453cd59 100644 --- a/src/lib/settings/ApplicationSettings.h +++ b/src/lib/settings/ApplicationSettings.h @@ -73,8 +73,8 @@ public: int getIndexerThreadCount() const; void setIndexerThreadCount(const int count); - bool getCancelIndexingOnFatalErrors() const; - void setCancelIndexingOnFatalErrors(bool enabled); + bool getMultiProcessIndexingEnabled() const; + void setMultiProcessIndexingEnabled(bool enabled); std::string getJavaPath() const; void setJavaPath(const std::string path); diff --git a/src/lib/utility/TimePoint.cpp b/src/lib/utility/TimePoint.cpp index 8365b2e2..3e087708 100644 --- a/src/lib/utility/TimePoint.cpp +++ b/src/lib/utility/TimePoint.cpp @@ -59,6 +59,11 @@ size_t TimePoint::deltaMS(const TimePoint& other) const return (m_time - other.m_time).total_milliseconds(); } +size_t TimePoint::deltaS(const TimePoint& other) const +{ + return (m_time - other.m_time).total_seconds(); +} + bool TimePoint::isSameDay(const TimePoint& other) const { if (m_time.date().day() == other.m_time.date().day() && diff --git a/src/lib/utility/TimePoint.h b/src/lib/utility/TimePoint.h index 4cab32a8..2ec31a3d 100644 --- a/src/lib/utility/TimePoint.h +++ b/src/lib/utility/TimePoint.h @@ -28,6 +28,7 @@ public: inline float operator-(const TimePoint& rhs){ return deltaMS(rhs) / 1000.0f; } size_t deltaMS(const TimePoint& other) const; + size_t deltaS(const TimePoint& other) const; bool isSameDay(const TimePoint& other) const; size_t deltaDays(const TimePoint& other) const; // days are counted beginning at 00:00, so a tp of 1.1.2017 23:59 is 1 day ago if it's the 2.1.2017 00:01 diff --git a/src/lib/utility/UUIDUtility.cpp b/src/lib/utility/UUIDUtility.cpp index 2349e4e4..4b6d3404 100644 --- a/src/lib/utility/UUIDUtility.cpp +++ b/src/lib/utility/UUIDUtility.cpp @@ -1,15 +1,15 @@ #include "UUIDUtility.h" +#include #include #include -#include -UUID UUIDUtility::getUUID() +boost::uuids::uuid UUIDUtility::getUUID() { return boost::uuids::random_generator()(); } -std::string UUIDUtility::UUIDtoString(const UUID& uuid) +std::string UUIDUtility::UUIDtoString(const boost::uuids::uuid& uuid) { return boost::lexical_cast(uuid); } diff --git a/src/lib/utility/UUIDUtility.h b/src/lib/utility/UUIDUtility.h index c3a4dbd0..ebc0f010 100644 --- a/src/lib/utility/UUIDUtility.h +++ b/src/lib/utility/UUIDUtility.h @@ -1,16 +1,14 @@ #ifndef UUID_UTILITY_H #define UUID_UTILITY_H -#include #include - -typedef boost::uuids::uuid UUID; +#include class UUIDUtility { public: - static UUID getUUID(); - static std::string UUIDtoString(const UUID& uuid); + static boost::uuids::uuid getUUID(); + static std::string UUIDtoString(const boost::uuids::uuid& uuid); static std::string getUUIDString(); }; diff --git a/src/lib/utility/file/FileRegisterStateData.cpp b/src/lib/utility/file/FileRegisterStateData.cpp index 4a90f987..6a1477ba 100644 --- a/src/lib/utility/file/FileRegisterStateData.cpp +++ b/src/lib/utility/file/FileRegisterStateData.cpp @@ -13,8 +13,6 @@ FileRegisterStateData::FileRegisterStateData(const FileRegisterStateData& o) void FileRegisterStateData::inject(const FileRegisterStateData& o) { - std::lock_guard oLock(o.m_filePathsMutex); - std::lock_guard thisLock(m_filePathsMutex); for (const auto& it: o.m_filePaths) { if (it.second == STATE_INDEXED) @@ -26,13 +24,11 @@ void FileRegisterStateData::inject(const FileRegisterStateData& o) void FileRegisterStateData::markFileIndexing(const FilePath& filePath) { - std::lock_guard lock(m_filePathsMutex); m_filePaths[filePath] = STATE_INDEXING; } void FileRegisterStateData::markIndexingFilesIndexed() { - std::lock_guard lock(m_filePathsMutex); for (auto& it: m_filePaths) { if (it.second == STATE_INDEXING) @@ -44,7 +40,6 @@ void FileRegisterStateData::markIndexingFilesIndexed() bool FileRegisterStateData::fileIsIndexed(const FilePath& filePath) const { - std::lock_guard lock(m_filePathsMutex); auto it = m_filePaths.find(filePath); if (it != m_filePaths.end()) { @@ -53,3 +48,24 @@ bool FileRegisterStateData::fileIsIndexed(const FilePath& filePath) const return false; } + +void FileRegisterStateData::setIndexedFiles(const std::set& filePaths) +{ + for (auto path : filePaths) + { + m_filePaths[path] = STATE_INDEXED; + } +} + +std::set FileRegisterStateData::getIndexedFiles() const +{ + std::set paths; + for (auto& it : m_filePaths) + { + if (it.second == STATE_INDEXED) + { + paths.insert(it.first); + } + } + return paths; +} diff --git a/src/lib/utility/file/FileRegisterStateData.h b/src/lib/utility/file/FileRegisterStateData.h index 2ac35ae8..4b739bf3 100644 --- a/src/lib/utility/file/FileRegisterStateData.h +++ b/src/lib/utility/file/FileRegisterStateData.h @@ -1,8 +1,9 @@ #ifndef FILE_REGISTER_STATE_DATA_H #define FILE_REGISTER_STATE_DATA_H -#include #include +#include +#include #include "utility/file/FilePath.h" @@ -18,6 +19,9 @@ public: void markIndexingFilesIndexed(); bool fileIsIndexed(const FilePath& filePath) const; + void setIndexedFiles(const std::set& filePaths); + std::set getIndexedFiles() const; + private: enum IndexingState { @@ -27,7 +31,6 @@ private: }; std::map m_filePaths; - mutable std::mutex m_filePathsMutex; }; #endif // FILE_REGISTER_STATE_DATA_H diff --git a/src/lib/utility/interprocess/InterprocessDataManager.cpp b/src/lib/utility/interprocess/InterprocessDataManager.cpp deleted file mode 100644 index 57776c85..00000000 --- a/src/lib/utility/interprocess/InterprocessDataManager.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include "InterprocessDataManager.h" - -#include - -#include "utility/logging/logging.h" -#include "SharedUUIDManager.h" - -#include "InterprocessUtility.h" -#include "SharedQueue.h" - -std::string InterprocessDataManager::m_sharedArgumentQueueName = "sourcetrail_parser_arguments"; - -InterprocessDataManager::InterprocessDataManager(const bool isOwner) - : m_isOwner(isOwner) - , m_initialized(false) -{ -} - -InterprocessDataManager::~InterprocessDataManager() -{ - SharedUUIDManager::getInstance()->removeUUIDsForInstance( - SharedUUIDManager::getInstance()->getInstanceUUID()); -} - -void InterprocessDataManager::initialize() -{ - try - { - m_sharedArgumentQueueName = UUIDUtility::getUUIDString() + "_" + m_sharedArgumentQueueName; - - std::string uuid = SharedUUIDManager::getInstance()->getNewUUID(); - m_parserArguments.initialize(m_isOwner, uuid.c_str()); - - m_initialized = true; - } - catch (std::exception& e) - { - LOG_ERROR_STREAM(<< e.what()); - } -} - -//void InterprocessDataManager::pushParserArguments(const Parser::Arguments& arguments) -//{ -// IF_INITIALIZED() -// { -// SharedParserArguments::VoidAllocator allocator(m_parserArguments.getSegmentManager()); -// SharedParserArguments args(allocator); -// -// args.setCompilationDatabasePath(arguments.compilationDatabasePath.str()); -// args.setCompilerFlags(arguments.compilerFlags); -// args.setFrameworkSearchPaths(arguments.frameworkSearchPaths); -// args.setHeaderSearchPaths(arguments.headerSearchPaths); -// args.setJavaClassPaths(arguments.javaClassPaths); -// args.setLanguage(arguments.language); -// args.setLanguageStandard(arguments.languageStandard); -// args.setLogErrors(arguments.logErrors); -// args.setSystemHeaderSearchPaths(arguments.systemHeaderSearchPaths); -// -// m_parserArguments.pushValue(args); -// } -//} -// -//Parser::Arguments InterprocessDataManager::popParserArguments() -//{ -// IF_INITIALIZED(Parser::Arguments()) -// { -// if (m_parserArguments.size() > 0) -// { -// SharedParserArguments args = m_parserArguments.popValue(); -// -// Parser::Arguments result; -// -// result.compilationDatabasePath = FilePath(args.getCompilationDatabasePath()); -// result.compilerFlags = args.getCompilerFlags(); -// result.frameworkSearchPaths = args.getFrameworkSearchPaths(); -// result.headerSearchPaths = args.getHeaderSearchPaths(); -// result.javaClassPaths = args.getJavaClassPaths(); -// result.language = args.getLanguage(); -// result.languageStandard = args.getLanguageStandard(); -// result.logErrors = args.getLogErrors(); -// result.systemHeaderSearchPaths = args.getSystemHeaderSearchPaths(); -// -// return result; -// } -// } -//} - -unsigned int InterprocessDataManager::parserArgumentCount() const -{ - IF_INITIALIZED(0) - { - return m_parserArguments.size(); - } -} - -void InterprocessDataManager::cleanSharedMemory() -{ - std::vector instances = SharedUUIDManager::getInstance()->getStoredInstanceUUIDs(); - - for (unsigned int i = 0; i < instances.size(); i++) - { - std::vector uuids = SharedUUIDManager::getInstance()->getUUIDsForInstance(instances[i]); - - if (uuids.size() == 1) // uuid list has to equal the count of used containers - { - std::string parserArgumentMemName = SharedQueue::getMemoryNamePrefix() + uuids[0]; - boost::interprocess::shared_memory_object::remove(parserArgumentMemName.c_str()); - } - } -} diff --git a/src/lib/utility/interprocess/InterprocessDataManager.h b/src/lib/utility/interprocess/InterprocessDataManager.h deleted file mode 100644 index d95aee8d..00000000 --- a/src/lib/utility/interprocess/InterprocessDataManager.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef INTERPROCESS_DATA_MANAGER_H -#define INTERPROCESS_DATA_MANAGER_H - -#include "data/parser/Parser.h" - -#include "SharedQueue.h" - -#include "SharedParserArguments.h" - -class InterprocessDataManager -{ -public: - InterprocessDataManager(const bool isOwner = false); - ~InterprocessDataManager(); - - void initialize(); - - // TODO: use IndexerCommands here and rename SharedParserArguments to SharedIndexerCommand - // TODO: either make one SharedIndexerCommand that stores everything or make - // SharedIndexerCommandJava, SharedIndexerCommandCxxManual, ..., each having a separate datastructure here - - //void pushParserArguments(const Parser::Arguments& arguments); - //Parser::Arguments popParserArguments(); - - unsigned int parserArgumentCount() const; - -private: - void cleanSharedMemory(); - - static std::string m_sharedArgumentQueueName; - - bool m_isOwner; - bool m_initialized; - - SharedQueue m_parserArguments; -}; - -#endif // INTERPROCESS_DATA_MANAGER_H diff --git a/src/lib/utility/interprocess/InterprocessProcessManager.cpp b/src/lib/utility/interprocess/InterprocessProcessManager.cpp deleted file mode 100644 index 9590fec9..00000000 --- a/src/lib/utility/interprocess/InterprocessProcessManager.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "InterprocessProcessManager.h" - -#include - -#include "utility/logging/logging.h" - -InterprocessProcessManager::InterprocessProcessManager() - : m_processName("") - , m_processCount(1) -{ -} - -InterprocessProcessManager::~InterprocessProcessManager() -{ -} - -void InterprocessProcessManager::setProcessName(const std::string& processName) -{ - m_processName = processName; -} - -std::string InterprocessProcessManager::getProcessName() const -{ - return m_processName; -} - -void InterprocessProcessManager::setProcessCount(const unsigned int processCount) -{ - m_processCount = processCount; -} - -unsigned int InterprocessProcessManager::getProcessCount() const -{ - return m_processCount; -} - -void InterprocessProcessManager::runProcesses() -{ - std::vector processThreads; - - for (unsigned int i = 0; i < m_processCount; i++) - { - std::thread* thread = new std::thread(&InterprocessProcessManager::runProcess, this); - processThreads.push_back(thread); - } - - for (unsigned int i = 0; i < m_processCount; i++) - { - processThreads[i]->join(); - - delete processThreads[i]; - } -} - -void InterprocessProcessManager::runProcess() -{ - int result = 1; - - while (result != 0) - { - result = system(m_processName.c_str()); - - LOG_INFO_STREAM(<< "Process returned with " << std::to_string(result)); - } -} diff --git a/src/lib/utility/interprocess/InterprocessProcessManager.h b/src/lib/utility/interprocess/InterprocessProcessManager.h deleted file mode 100644 index 35aa5223..00000000 --- a/src/lib/utility/interprocess/InterprocessProcessManager.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef INTERPROCESS_PROCESS_MANAGER_H -#define INTERPROCESS_PROCESS_MANAGER_H - -#include - -class InterprocessProcessManager -{ -public: - InterprocessProcessManager(); - ~InterprocessProcessManager(); - - void setProcessName(const std::string& processName); - std::string getProcessName() const; - - void setProcessCount(const unsigned int processCount); - unsigned int getProcessCount() const; - - void runProcesses(); - -private: - void runProcess(); - - std::string m_processName; - unsigned int m_processCount; -}; - -#endif // INTERPROCESS_PROCESS_MANAGER_H diff --git a/src/lib/utility/interprocess/InterprocessUtility.h b/src/lib/utility/interprocess/InterprocessUtility.h deleted file mode 100644 index 106c061e..00000000 --- a/src/lib/utility/interprocess/InterprocessUtility.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef INTERPROCESS_UTILITY_H -#define INTERPROCESS_UTILITY_H - -// requires a bool field 'm_initialized' to be defined -// if 'm_initialized' is false, the value '__retVal__' will be returned -// '__retVal__' is not required -#define IF_INITIALIZED(__retVal__) \ - if(m_initialized == false) \ - { \ - LOG_ERROR_STREAM(<< "not initialized"); \ - return __retVal__; \ - } \ - else \ - -#endif // INTERPROCESS_UTILITY_H diff --git a/src/lib/utility/interprocess/SharedContainer.h b/src/lib/utility/interprocess/SharedContainer.h deleted file mode 100644 index d61d974e..00000000 --- a/src/lib/utility/interprocess/SharedContainer.h +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef SHARED_CONTAINER_H -#define SHARED_CONTAINER_H - -#include - -#include - -#include "utility/logging/logging.h" - -#include "InterprocessUtility.h" - -class SharedContainer -{ -public: - static std::string getMemoryNamePrefix(); - static std::string getContainerNamePrefix(); - - SharedContainer(); - virtual ~SharedContainer(); - - virtual bool initialize(const bool isOwner, const std::string& containerName) = 0; - -protected: - static const std::string m_memoryNamePrefix; - static const std::string m_containerNamePrefix; - - bool initializeSharedMemory(const bool isOwner); - - boost::interprocess::managed_shared_memory m_sharedMemory; - - bool m_isOwner; - bool m_initialized; - - std::string m_memoryName; - std::string m_containerName; -}; - -const std::string SharedContainer::m_memoryNamePrefix("sourcetrail_memory_"); - -const std::string SharedContainer::m_containerNamePrefix("sourcetrail_container_"); - -std::string SharedContainer::getMemoryNamePrefix() -{ - return m_memoryNamePrefix; -} - -std::string SharedContainer::getContainerNamePrefix() -{ - return m_containerNamePrefix; -} - -SharedContainer::SharedContainer() - : m_isOwner(false) - , m_initialized(false) - , m_memoryName(m_memoryNamePrefix) - , m_containerName(m_containerNamePrefix) -{ -} - -SharedContainer::~SharedContainer() -{ -} - -bool SharedContainer::initializeSharedMemory(const bool isOwner) -{ - bool initialized = false; - - try - { - if (isOwner) - { - boost::interprocess::shared_memory_object::remove(m_memoryName.c_str()); - - m_sharedMemory = boost::interprocess::managed_shared_memory(boost::interprocess::create_only, - m_memoryName.c_str(), - 65536); - } - else - { - m_sharedMemory = boost::interprocess::managed_shared_memory(boost::interprocess::open_only, - m_memoryName.c_str()); - } - - initialized = true; - } - catch (std::exception& e) - { - LOG_ERROR(e.what()); - } - - return initialized; -} - - -#endif // SHARED_CONTAINER_H diff --git a/src/lib/utility/interprocess/SharedMap.h b/src/lib/utility/interprocess/SharedMap.h deleted file mode 100644 index 0d859463..00000000 --- a/src/lib/utility/interprocess/SharedMap.h +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef SHARED_DICTIONARY_H -#define SHARED_DICTIONARY_H - -#include - -#include -#include - -#include "SharedContainer.h" - -template -class SharedMap : public SharedContainer -{ -public: - SharedMap(); - virtual ~SharedMap(); - - virtual void initialize(const bool isOwner, const std::string& mapName); - - Value& operator[](const Key& key); - - unsigned int size() const; - - void clear(); - - boost::interprocess::managed_shared_memory::segment_manager* getSegmentManager() const; - -private: - typedef std::pair ValueType; - typedef boost::interprocess::allocator ShmemAllocator; - typedef boost::interprocess::map, ShmemAllocator> ShmemMap; - - ShmemMap* m_map; -}; - -template -SharedMap::SharedMap() - : m_map(NULL) -{ -} - -template -SharedMap::~SharedMap() -{ - if (m_map != NULL && m_initialized && m_isOwner) - { - m_sharedMemory.destroy(m_containerName.c_str()); - boost::interprocess::shared_memory_object::remove(m_memoryName.c_str()); - - delete m_map; - } -} - -template -void SharedMap::initialize(const bool isOwner, const std::string& mapName) -{ - m_isOwner = isOwner; - m_containerName = m_containerNamePrefix + dequeName; - m_memoryName = m_memoryNamePrefix + m_containerName; - - if (initializeSharedMemory(m_isOwner)) - { - try - { - if (m_isOwner) - { - m_sharedMemory.destroy(m_containerName.c_str()); - - const ShmemAllocator allocator(m_sharedMemory.get_segment_manager()); - - m_map = m_sharedMemory.construct(m_containerName.c_str())(allocator); - } - else - { - m_map = m_sharedMemory.find(m_containerName.c_str()).first; - } - - m_initialized = true; - } - catch (std::exception& e) - { - LOG_ERROR_STREAM(<< e.what()); - } - } - - return m_initialized; -} - -template -Value& SharedMap::operator[](const Key& key) -{ - IF_INITIALIZED(NULL) - { - return m_map->at(key); - } -} - -template -unsigned int SharedMap::size() const -{ - IF_INITIALIZED(0) - { - return m_map->size(); - } -} - -template -void SharedMap::clear() -{ - IF_INITIALIZED() - { - m_map->clear(); - } -} - -template -boost::interprocess::managed_shared_memory::segment_manager* SharedMap::getSegmentManager() const -{ - IF_INITIALIZED(NULL) - { - return m_sharedMemory.get_segment_manager(); - } -} - -#endif // SHARED_DICTIONARY_H diff --git a/src/lib/utility/interprocess/SharedMemory.cpp b/src/lib/utility/interprocess/SharedMemory.cpp new file mode 100644 index 00000000..ad5f870d --- /dev/null +++ b/src/lib/utility/interprocess/SharedMemory.cpp @@ -0,0 +1,145 @@ +#include "utility/interprocess/SharedMemory.h" + +#include "utility/interprocess/SharedMemoryGarbageCollector.h" +#include "utility/logging/logging.h" + +const char* SharedMemory::s_memoryNamePrefix = "srctrlmem_"; +const char* SharedMemory::s_mutexNamePrefix = "srctrlmtx_"; + +SharedMemory::ScopedAccess::ScopedAccess(SharedMemory* memory) + : boost::interprocess::scoped_lock(memory->getMutex()) + , m_memory(boost::interprocess::open_only, memory->getMemoryName().c_str()) + , m_memoryName(memory->getMemoryName()) +{ +} + +SharedMemory::ScopedAccess::~ScopedAccess() +{ +} + +SharedMemory::Allocator* SharedMemory::ScopedAccess::getAllocator() +{ + return m_memory.get_segment_manager(); +} + +size_t SharedMemory::ScopedAccess::getMemorySize() const +{ + return m_memory.get_size(); +} + +size_t SharedMemory::ScopedAccess::getFreeMemorySize() const +{ + return m_memory.get_free_memory(); +} + +void SharedMemory::ScopedAccess::growMemory(size_t size) +{ + m_memory = boost::interprocess::managed_shared_memory(); + + boost::interprocess::managed_shared_memory::grow(m_memoryName.c_str(), size); + + m_memory = boost::interprocess::managed_shared_memory(boost::interprocess::open_only, m_memoryName.c_str()); +} + +std::string SharedMemory::ScopedAccess::logString() const +{ + std::string log = m_memoryName + " -"; + log += " size: " + std::to_string(getMemorySize()); + log += " free: " + std::to_string(getFreeMemorySize()); + log += " used: " + std::to_string(getMemorySize() - getFreeMemorySize()); + log += " pct: " + std::to_string(100 - int(float(getFreeMemorySize()) / getMemorySize() * 100)); + return log; +} + + +std::string SharedMemory::checkName(const std::string& name) +{ + return name.size() > 18 ? name.substr(0, 18) : name; +} + +void SharedMemory::deleteSharedMemory(const std::string& name) +{ + boost::interprocess::shared_memory_object::remove((s_memoryNamePrefix + name).c_str()); + boost::interprocess::named_mutex::remove((s_mutexNamePrefix + name).c_str()); +} + +SharedMemory::SharedMemory(const std::string& name, size_t initialMemorySize, AccessMode mode) + : m_name(checkName(name)) + , m_mode(mode) +{ + bool unlockMutex = true; + + switch (mode) + { + case CREATE_AND_DELETE: + { + SharedMemoryGarbageCollector* collector = SharedMemoryGarbageCollector::getInstance(); + if (collector) + { + collector->registerSharedMemory(m_name); + } + } + + deleteSharedMemory(m_name); + + boost::interprocess::managed_shared_memory( + boost::interprocess::create_only, getMemoryName().c_str(), initialMemorySize); + boost::interprocess::named_mutex(boost::interprocess::create_only, getMutexName().c_str()); + break; + + case OPEN_ONLY: + boost::interprocess::managed_shared_memory( + boost::interprocess::open_only, getMemoryName().c_str()); + boost::interprocess::named_mutex(boost::interprocess::open_only, getMutexName().c_str()); + unlockMutex = false; + break; + + case OPEN_OR_CREATE: + boost::interprocess::managed_shared_memory( + boost::interprocess::open_or_create, getMemoryName().c_str(), initialMemorySize); + boost::interprocess::named_mutex(boost::interprocess::open_or_create, getMutexName().c_str()); + break; + } + + if (unlockMutex) + { + boost::interprocess::named_mutex mutex(boost::interprocess::open_only, getMutexName().c_str()); + mutex.try_lock(); + mutex.unlock(); + } +} + +SharedMemory::~SharedMemory() +{ + if (m_mode == CREATE_AND_DELETE) + { + SharedMemoryGarbageCollector* collector = SharedMemoryGarbageCollector::getInstance(); + if (collector) + { + collector->unregisterSharedMemory(m_name); + } + + deleteSharedMemory(m_name); + } +} + +std::string SharedMemory::getMemoryName() const +{ + return s_memoryNamePrefix + m_name; +} + +std::string SharedMemory::getMutexName() const +{ + return s_mutexNamePrefix + m_name; +} + +boost::interprocess::named_mutex& SharedMemory::getMutex() +{ + if (!m_mutex) + { + m_mutex = std::make_shared( + boost::interprocess::open_only, getMutexName().c_str()); + } + + return *m_mutex.get(); +} diff --git a/src/lib/utility/interprocess/SharedMemory.h b/src/lib/utility/interprocess/SharedMemory.h new file mode 100644 index 00000000..c444730c --- /dev/null +++ b/src/lib/utility/interprocess/SharedMemory.h @@ -0,0 +1,116 @@ +#ifndef SHARED_MEMORY_H +#define SHARED_MEMORY_H + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +class SharedMemory +{ +public: + enum AccessMode + { + CREATE_AND_DELETE, + OPEN_ONLY, + OPEN_OR_CREATE + }; + + using Allocator = boost::interprocess::managed_shared_memory::segment_manager; + + using String = boost::interprocess::basic_string, + boost::interprocess::allocator>; + + template + using Vector = boost::interprocess::vector>; + + template + using Queue = boost::interprocess::deque>; + + template + using Map = boost::interprocess::map, + boost::interprocess::allocator, Allocator>>; + + template + using Set = boost::interprocess::set, boost::interprocess::allocator>; + + // Names addressing shared memory objects longer than 29 characters can throw and exception + static std::string checkName(const std::string& name); + static void deleteSharedMemory(const std::string& name); + + SharedMemory(const std::string& name, size_t initialMemorySize, AccessMode mode); + ~SharedMemory(); + + class ScopedAccess + : public boost::interprocess::scoped_lock + { + public: + ScopedAccess(SharedMemory* memory); + ~ScopedAccess(); + + Allocator* getAllocator(); + + size_t getMemorySize() const; + size_t getFreeMemorySize() const; + + void growMemory(size_t size); + + template + T* accessValue(const std::string& key) + { + return m_memory.find_or_construct(key.c_str())(); + } + + template + T* accessValues(const std::string& key, size_t count) + { + return m_memory.find_or_construct(key.c_str())[count](); + } + + template + T* accessValueWithAllocator(const std::string& key) + { + return m_memory.find_or_construct(key.c_str())(getAllocator()); + } + + template + T* accessValuesWithAllocator(const std::string& key, size_t count) + { + return m_memory.find_or_construct(key.c_str())[count](getAllocator()); + } + + template + void destroyValue(const std::string& key) + { + m_memory.destroy(key.c_str()); + } + + std::string logString() const; + + private: + boost::interprocess::managed_shared_memory m_memory; + std::string m_memoryName; + }; + +private: + static const char* s_memoryNamePrefix; + static const char* s_mutexNamePrefix; + + std::string getMemoryName() const; + std::string getMutexName() const; + + boost::interprocess::named_mutex& getMutex(); + + std::shared_ptr m_mutex; + std::string m_name; + AccessMode m_mode; +}; + + +#endif // SHARED_MEMORY_H diff --git a/src/lib/utility/interprocess/SharedMemoryGarbageCollector.cpp b/src/lib/utility/interprocess/SharedMemoryGarbageCollector.cpp new file mode 100644 index 00000000..ea485475 --- /dev/null +++ b/src/lib/utility/interprocess/SharedMemoryGarbageCollector.cpp @@ -0,0 +1,242 @@ +#include "utility/interprocess/SharedMemoryGarbageCollector.h" + +#include + +#include "utility/logging/logging.h" +#include "utility/TimePoint.h" + +std::string SharedMemoryGarbageCollector::s_memoryName = "garbage_collector"; +std::string SharedMemoryGarbageCollector::s_instancesKeyName = "running_instances"; +std::string SharedMemoryGarbageCollector::s_timeStampsKeyName = "memory_to_timestamps"; + +const size_t SharedMemoryGarbageCollector::s_updateIntervalSeconds = 1; +const size_t SharedMemoryGarbageCollector::s_deleteThresholdSeconds = 10; + +std::shared_ptr SharedMemoryGarbageCollector::s_instance; + +SharedMemoryGarbageCollector* SharedMemoryGarbageCollector::createInstance() +{ + if (!s_instance) + { + s_instance = std::shared_ptr(new SharedMemoryGarbageCollector()); + } + + return s_instance.get(); +} + +SharedMemoryGarbageCollector* SharedMemoryGarbageCollector::getInstance() +{ + return s_instance.get(); +} + +SharedMemoryGarbageCollector::SharedMemoryGarbageCollector() + : m_memory(s_memoryName, 65536 /* 64 kB */, SharedMemory::OPEN_OR_CREATE) + , m_loopIsRunning(false) +{ +} + +SharedMemoryGarbageCollector::~SharedMemoryGarbageCollector() +{ +} + +void SharedMemoryGarbageCollector::run(const std::string& uuid) +{ + LOG_INFO_STREAM(<< "start shared memory garbage collection"); + + m_uuid = uuid; + + std::thread( + [this]() + { + m_loopIsRunning = true; + + while (m_loopIsRunning) + { + update(); + + std::this_thread::sleep_for(std::chrono::seconds(s_updateIntervalSeconds)); + } + } + ).detach(); +} + +void SharedMemoryGarbageCollector::stop() +{ + LOG_INFO_STREAM(<< "stop shared memory garbage collection"); + + m_loopIsRunning = false; + + { + std::lock_guard lock(m_sharedMemoryNamesMutex); + m_removedSharedMemoryNames.insert(m_sharedMemoryNames.begin(), m_sharedMemoryNames.end()); + } + + update(); + + m_sharedMemoryNames.clear(); + m_removedSharedMemoryNames.clear(); + + + SharedMemory::ScopedAccess access(&m_memory); + + SharedMemory::Map* instances = + access.accessValueWithAllocator>(s_instancesKeyName); + + if (!instances) + { + return; + } + + SharedMemory::String i(access.getAllocator()); + i = m_uuid.c_str(); + + SharedMemory::Map::iterator itt = instances->find(i); + if (itt != instances->end()) + { + instances->erase(itt); + } + + bool otherRunningInstances = false; + TimePoint now = TimePoint::now(); + for (SharedMemory::Map::iterator it = instances->begin(); + it != instances->end(); it++) + { + TimePoint timestamp = TimePoint(std::string(it->second.c_str())); + if (now.deltaS(timestamp) <= s_deleteThresholdSeconds) + { + otherRunningInstances = true; + LOG_INFO_STREAM(<< "currently running instance: " << it->first.c_str()); + } + } + + if (!otherRunningInstances) + { + LOG_INFO_STREAM(<< "delete garbage collector memory: " << s_memoryName); + SharedMemory::deleteSharedMemory(s_memoryName); + } +} + +void SharedMemoryGarbageCollector::registerSharedMemory(const std::string& sharedMemoryName) +{ + { + std::lock_guard lock(m_sharedMemoryNamesMutex); + m_sharedMemoryNames.insert(sharedMemoryName); + } + + update(); +} + +void SharedMemoryGarbageCollector::unregisterSharedMemory(const std::string& sharedMemoryName) +{ + { + std::lock_guard lock(m_sharedMemoryNamesMutex); + size_t removedCount = m_sharedMemoryNames.erase(sharedMemoryName); + + if (removedCount > 0) + { + m_removedSharedMemoryNames.insert(sharedMemoryName); + } + } + + update(); +} + +void SharedMemoryGarbageCollector::update() +{ + std::lock_guard lock(m_sharedMemoryNamesMutex); + + SharedMemory::ScopedAccess access(&m_memory); + + if (access.getFreeMemorySize() * 2 < access.getMemorySize()) + { + access.growMemory(access.getMemorySize()); + LOG_INFO(access.logString()); + } + + SharedMemory::String t(access.getAllocator()); + t = TimePoint::now().toString().c_str(); + + // update instances + { + SharedMemory::Map* instances = + access.accessValueWithAllocator>(s_instancesKeyName); + + if (!instances) + { + return; + } + + SharedMemory::String i(access.getAllocator()); + i = m_uuid.c_str(); + + SharedMemory::Map::iterator it = instances->find(i); + if (it != instances->end()) + { + it->second = t; + } + else + { + instances->insert(std::pair(i, t)); + } + } + + // update shared memories + { + SharedMemory::Map* timeStamps = + access.accessValueWithAllocator>(s_timeStampsKeyName); + + if (!timeStamps) + { + return; + } + + // remove deleted shared memories + for (const std::string& name : m_removedSharedMemoryNames) + { + SharedMemory::String n(access.getAllocator()); + n = name.c_str(); + + SharedMemory::Map::iterator it = timeStamps->find(n); + if (it != timeStamps->end()) + { + timeStamps->erase(it); + } + } + m_removedSharedMemoryNames.clear(); + + // add or update shared memories + for (const std::string& name : m_sharedMemoryNames) + { + SharedMemory::String n(access.getAllocator()); + n = name.c_str(); + + SharedMemory::Map::iterator it = timeStamps->find(n); + if (it != timeStamps->end()) + { + it->second = t; + } + else + { + timeStamps->insert(std::pair(n, t)); + } + } + + // delete old shared memories + TimePoint now = TimePoint::now(); + for (SharedMemory::Map::iterator it = timeStamps->begin(); + it != timeStamps->end();) + { + TimePoint timestamp = TimePoint(std::string(it->second.c_str())); + if (now.deltaS(timestamp) > s_deleteThresholdSeconds) + { + LOG_INFO_STREAM(<< "collect garbage: " << it->first.c_str()); + SharedMemory::deleteSharedMemory(it->first.c_str()); + timeStamps->erase(it++); + } + else + { + it++; + } + } + } +} diff --git a/src/lib/utility/interprocess/SharedMemoryGarbageCollector.h b/src/lib/utility/interprocess/SharedMemoryGarbageCollector.h new file mode 100644 index 00000000..e857360c --- /dev/null +++ b/src/lib/utility/interprocess/SharedMemoryGarbageCollector.h @@ -0,0 +1,49 @@ +#ifndef SHARED_MEMORY_GARBAGE_COLLECTOR_H +#define SHARED_MEMORY_GARBAGE_COLLECTOR_H + +#include +#include +#include + +#include "utility/interprocess/SharedMemory.h" + +#include + +class SharedMemoryGarbageCollector +{ +public: + static SharedMemoryGarbageCollector* createInstance(); + static SharedMemoryGarbageCollector* getInstance(); + + SharedMemoryGarbageCollector(); + ~SharedMemoryGarbageCollector(); + + void run(const std::string& uuid); + void stop(); + + void registerSharedMemory(const std::string& sharedMemoryName); + void unregisterSharedMemory(const std::string& sharedMemoryName); + +private: + void update(); + + static std::string s_memoryName; + static std::string s_instancesKeyName; + static std::string s_timeStampsKeyName; + + static const size_t s_updateIntervalSeconds; + static const size_t s_deleteThresholdSeconds; + + static std::shared_ptr s_instance; + + SharedMemory m_memory; + volatile bool m_loopIsRunning; + + std::string m_uuid; + + std::mutex m_sharedMemoryNamesMutex; + std::set m_sharedMemoryNames; + std::set m_removedSharedMemoryNames; +}; + +#endif // SHARED_MEMORY_GARBAGE_COLLECTOR_H diff --git a/src/lib/utility/interprocess/SharedParserArguments.cpp b/src/lib/utility/interprocess/SharedParserArguments.cpp deleted file mode 100644 index 0d90dd60..00000000 --- a/src/lib/utility/interprocess/SharedParserArguments.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include "SharedParserArguments.h" - -SharedParserArguments::SharedParserArguments(const VoidAllocator& allocator) - : m_logErrors(false) - , m_language("", allocator) - , m_languageStandard("", allocator) - , m_compilationDatabasePath("", allocator) - , m_javaClassPaths(allocator) - , m_headerSearchPaths(allocator) - , m_systemHeaderSearchPaths(allocator) - , m_frameworkSearchPaths(allocator) - , m_compilerFlags(allocator) -{ -} - -SharedParserArguments::~SharedParserArguments() -{ -} - -void SharedParserArguments::setLogErrors(const bool logErrors) -{ - m_logErrors = logErrors; -} - -bool SharedParserArguments::getLogErrors() const -{ - return m_logErrors; -} - -void SharedParserArguments::setLanguage(const std::string& language) -{ - m_language = language.c_str(); -} - -std::string SharedParserArguments::getLanguage() const -{ - return std::string(m_language.c_str()); -} - -void SharedParserArguments::setLanguageStandard(const std::string& languageStandard) -{ - m_languageStandard = languageStandard.c_str(); -} - -std::string SharedParserArguments::getLanguageStandard() const -{ - return std::string(m_languageStandard.c_str()); -} - -void SharedParserArguments::setCompilationDatabasePath(const std::string& compilationDatabasePath) -{ - m_compilationDatabasePath = compilationDatabasePath.c_str(); -} - -std::string SharedParserArguments::getCompilationDatabasePath() const -{ - return std::string(m_compilationDatabasePath.c_str()); -} - -void SharedParserArguments::setJavaClassPaths(std::vector javaClassPaths) -{ - m_javaClassPaths.clear(); - for (unsigned int i = 0; i < javaClassPaths.size(); i++) - { - SharedString path(m_javaClassPaths.get_allocator()); - path = javaClassPaths[i].str().c_str(); - m_javaClassPaths.push_back(path); - } -} - -std::vector SharedParserArguments::getJavaClassPaths() const -{ - std::vector result; - - for (unsigned int i = 0; i < m_javaClassPaths.size(); i++) - { - result.push_back(FilePath(m_javaClassPaths[i].c_str())); - } - - return result; -} - -void SharedParserArguments::setHeaderSearchPaths(std::vector headerSearchPaths) -{ - m_headerSearchPaths.clear(); - for (unsigned int i = 0; i < headerSearchPaths.size(); i++) - { - SharedString path(m_headerSearchPaths.get_allocator()); - path = headerSearchPaths[i].str().c_str(); - m_headerSearchPaths.push_back(path); - } -} - -std::vector SharedParserArguments::getHeaderSearchPaths() const -{ - std::vector result; - - for (unsigned int i = 0; i < m_headerSearchPaths.size(); i++) - { - result.push_back(FilePath(m_headerSearchPaths[i].c_str())); - } - - return result; -} - -void SharedParserArguments::setSystemHeaderSearchPaths(std::vector systemHeaderSearchPaths) -{ - m_systemHeaderSearchPaths.clear(); - for (unsigned int i = 0; i < systemHeaderSearchPaths.size(); i++) - { - SharedString path(m_systemHeaderSearchPaths.get_allocator()); - path = systemHeaderSearchPaths[i].str().c_str(); - m_systemHeaderSearchPaths.push_back(path); - } -} - -std::vector SharedParserArguments::getSystemHeaderSearchPaths() const -{ - std::vector result; - - for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++) - { - result.push_back(FilePath(m_systemHeaderSearchPaths[i].c_str())); - } - - return result; -} - -void SharedParserArguments::setFrameworkSearchPaths(std::vector frameworkSearchPaths) -{ - m_frameworkSearchPaths.clear(); - for (unsigned int i = 0; i < frameworkSearchPaths.size(); i++) - { - SharedString path(m_frameworkSearchPaths.get_allocator()); - path = frameworkSearchPaths[i].str().c_str(); - m_frameworkSearchPaths.push_back(path); - } -} - -std::vector SharedParserArguments::getFrameworkSearchPaths() const -{ - std::vector result; - - for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++) - { - result.push_back(FilePath(m_frameworkSearchPaths[i].c_str())); - } - - return result; -} - -void SharedParserArguments::setCompilerFlags(std::vector compilerFlags) -{ - m_compilerFlags.clear(); - for (unsigned int i = 0; i < compilerFlags.size(); i++) - { - SharedString path(m_compilerFlags.get_allocator()); - path = compilerFlags[i].c_str(); - m_compilerFlags.push_back(path); - } -} - -std::vector SharedParserArguments::getCompilerFlags() const -{ - std::vector result; - - for (unsigned int i = 0; i < m_compilerFlags.size(); i++) - { - result.push_back(m_compilerFlags[i].c_str()); - } - - return result; -} diff --git a/src/lib/utility/interprocess/SharedParserArguments.h b/src/lib/utility/interprocess/SharedParserArguments.h deleted file mode 100644 index 7aac3f4e..00000000 --- a/src/lib/utility/interprocess/SharedParserArguments.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef SHARED_PARSER_ARGUMENTS_H -#define SHARED_PARSER_ARGUMENTS_H - -#include -#include -#include -#include - -#include "utility/file/FilePath.h" - -class SharedParserArguments -{ -public: - typedef boost::interprocess::allocator VoidAllocator; - - SharedParserArguments(const VoidAllocator& allocator); - ~SharedParserArguments(); - - void setLogErrors(const bool logErrors); - bool getLogErrors() const; - - - void setLanguage(const std::string& language); - std::string getLanguage() const; - - - void setLanguageStandard(const std::string& languageStandard); - std::string getLanguageStandard() const; - - void setCompilationDatabasePath(const std::string& compilationDatabasePath); - std::string getCompilationDatabasePath() const; - - void setJavaClassPaths(std::vector javaClassPaths); - std::vector getJavaClassPaths() const; - - void setHeaderSearchPaths(std::vector headerSearchPaths); - std::vector getHeaderSearchPaths() const; - - void setSystemHeaderSearchPaths(std::vector systemHeaderSearchPaths); - std::vector getSystemHeaderSearchPaths() const; - - void setFrameworkSearchPaths(std::vector frameworkSearchPaths); - std::vector getFrameworkSearchPaths() const; - - void setCompilerFlags(std::vector compilerFlags); - std::vector getCompilerFlags() const; - -private: - typedef boost::interprocess::allocator CharAllocator; - typedef boost::interprocess::basic_string, CharAllocator> SharedString; - typedef boost::interprocess::allocator StringAllocator; - typedef boost::interprocess::vector SharedStringVector; - - bool m_logErrors; - - SharedString m_language; - SharedString m_languageStandard; - SharedString m_compilationDatabasePath; - - SharedStringVector m_javaClassPaths; - SharedStringVector m_headerSearchPaths; - SharedStringVector m_systemHeaderSearchPaths; - SharedStringVector m_frameworkSearchPaths; - SharedStringVector m_compilerFlags; -}; - -#endif // SHARED_PARSER_ARGUMENTS_H diff --git a/src/lib/utility/interprocess/SharedQueue.h b/src/lib/utility/interprocess/SharedQueue.h deleted file mode 100644 index 3046eb5a..00000000 --- a/src/lib/utility/interprocess/SharedQueue.h +++ /dev/null @@ -1,128 +0,0 @@ -#ifndef SHARED_QUEUE_H -#define SHARED_QUEUE_H - -#include -#include - -#include "SharedContainer.h" - -template -class SharedQueue : public SharedContainer -{ -public: - SharedQueue(); - virtual ~SharedQueue(); - - virtual bool initialize(const bool isOwner, const std::string& dequeName); - - void pushValue(const T& val); - T popValue(); - - unsigned int size() const; - - boost::interprocess::managed_shared_memory::segment_manager* getSegmentManager() const; - -private: - typedef boost::interprocess::allocator ShmemAllocator; - typedef boost::interprocess::deque ShmemDeque; - - ShmemDeque* m_deque; -}; - -template -SharedQueue::SharedQueue() - : m_deque(NULL) -{ -} - -template -SharedQueue::~SharedQueue() -{ - if (m_deque != NULL && m_initialized && m_isOwner) - { - m_sharedMemory.destroy(m_containerName.c_str()); - boost::interprocess::shared_memory_object::remove(m_memoryName.c_str()); - - delete m_deque; - } -} - -template -bool SharedQueue::initialize(const bool isOwner, const std::string& dequeName) -{ - m_isOwner = isOwner; - m_containerName = m_containerNamePrefix + dequeName; - m_memoryName = m_memoryNamePrefix + m_containerName; - - if (initializeSharedMemory(m_isOwner)) - { - try - { - if (isOwner) - { - m_sharedMemory.destroy(m_containerName.c_str()); - - const ShmemAllocator allocator(m_sharedMemory.get_segment_manager()); - - m_deque = m_sharedMemory.construct(m_containerName.c_str())(allocator); - } - else - { - m_deque = m_sharedMemory.find(m_containerName.c_str()).first; - } - - m_initialized = true; - } - catch (std::exception& e) - { - LOG_ERROR(e.what()); - } - } - - return m_initialized; -} - -template -void SharedQueue::pushValue(const T& val) -{ - IF_INITIALIZED() - { - m_deque->push_back(val); - } -} - -template -T SharedQueue::popValue() -{ - if (m_initialized) - { - T val = m_deque->front(); - m_deque->pop_front(); - - return val; - } - else - { - throw(std::runtime_error("Deque was not initialized")); - } -} - -template -unsigned int SharedQueue::size() const -{ - IF_INITIALIZED(0) - { - return m_deque->size(); - } -} - -template -boost::interprocess::managed_shared_memory::segment_manager* SharedQueue::getSegmentManager() const -{ - IF_INITIALIZED(NULL) - { - return m_sharedMemory.get_segment_manager(); - } -} - -#endif // SHARED_QUEUE_H diff --git a/src/lib/utility/interprocess/SharedUUIDManager.h b/src/lib/utility/interprocess/SharedUUIDManager.h deleted file mode 100644 index 6c8f0bb6..00000000 --- a/src/lib/utility/interprocess/SharedUUIDManager.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef SHARED_UUID_MANAGER_H -#define SHARED_UUID_MANAGER_H - -#include -#include - -#include "utility/UUIDUtility.h" - -#include "utility/ConfigManager.h" - -class SharedUUIDManager -{ -public: - static std::shared_ptr getInstance(); - - ~SharedUUIDManager(); - - void setFilePath(const std::string& filePath); - - std::string getInstanceUUID() const; - std::string getNewUUID(); - - std::vector getUUIDsForInstance(const std::string& instanceUUID); - void removeUUIDsForInstance(const std::string& instanceUUID); - - std::vector getStoredInstanceUUIDs() const; - void removeInstanceUUID(const std::string& instanceUUID); - -private: - SharedUUIDManager(); - - void saveInstanceUUID(); - - void refreshUUIDs(); - - static const std::string m_fileName; - static const std::string m_instanceUUIDsKey; - static std::shared_ptr m_instance; - - std::string m_filePath; - - const UUID m_instanceUUID; - - std::shared_ptr m_uuids; -}; - -#endif // SHARED_UUID_MANAGER_H diff --git a/src/lib/utility/logging/FileLogger.cpp b/src/lib/utility/logging/FileLogger.cpp index 0689b95d..d77085cc 100644 --- a/src/lib/utility/logging/FileLogger.cpp +++ b/src/lib/utility/logging/FileLogger.cpp @@ -8,7 +8,7 @@ FileLogger::FileLogger() : Logger("FileLogger") - , m_logFileName() + , m_logFileName("log") , m_logDirectory("user/log/") , m_maxLogLineCount(0) , m_maxLogFileCount(0) @@ -22,6 +22,17 @@ FileLogger::~FileLogger() { } +FilePath FileLogger::getLogFilePath() const +{ + return m_currentLogFilePath; +} + +void FileLogger::setLogFilePath(const FilePath& filePath) +{ + m_currentLogFilePath = filePath; + m_logFileName = ""; +} + void FileLogger::setLogDirectory(const FilePath& filePath) { m_logDirectory = filePath; @@ -66,12 +77,17 @@ void FileLogger::setMaxLogFileCount(unsigned int fileCount) void FileLogger::updateLogFileName() { + if (!m_logFileName.size()) + { + return; + } + bool fileChanged = false; - m_currentLogFileName = m_logFileName; + std::string currentLogFilePath = m_logDirectory.str() + m_logFileName; if (m_maxLogFileCount > 0) { - m_currentLogFileName += "_"; + currentLogFilePath += "_"; if (m_currentLogLineCount >= m_maxLogLineCount) { m_currentLogLineCount = 0; @@ -83,21 +99,23 @@ void FileLogger::updateLogFileName() } fileChanged = true; } - m_currentLogFileName += std::to_string(m_currentLogFileCount); + currentLogFilePath += std::to_string(m_currentLogFileCount); } - m_currentLogFileName += ".txt"; + currentLogFilePath += ".txt"; + + m_currentLogFilePath = FilePath(currentLogFilePath); if (fileChanged) { - FileSystem::remove(m_logDirectory.concat(FilePath(m_currentLogFileName))); + FileSystem::remove(m_currentLogFilePath); } } void FileLogger::logMessage(const std::string& type, const LogMessage& message) { std::ofstream fileStream; - fileStream.open(m_logDirectory.concat(FilePath(m_currentLogFileName)).str(), std::ios::app); + fileStream.open(m_currentLogFilePath.str(), std::ios::app); fileStream << message.getTimeString("%H:%M:%S") << " | "; fileStream << message.threadId << " | "; diff --git a/src/lib/utility/logging/FileLogger.h b/src/lib/utility/logging/FileLogger.h index 6e20e4ef..854e26cd 100644 --- a/src/lib/utility/logging/FileLogger.h +++ b/src/lib/utility/logging/FileLogger.h @@ -13,6 +13,9 @@ public: FileLogger(); virtual ~FileLogger(); + FilePath getLogFilePath() const; + void setLogFilePath(const FilePath& filePath); + void setLogDirectory(const FilePath& filePath); void setFileName(const std::string& fileName); void setMaxLogLineCount(unsigned int logCount); @@ -30,12 +33,12 @@ private: std::string m_logFileName; FilePath m_logDirectory; + FilePath m_currentLogFilePath; + unsigned int m_maxLogLineCount; unsigned int m_maxLogFileCount; unsigned int m_currentLogLineCount; unsigned int m_currentLogFileCount; - - std::string m_currentLogFileName; }; #endif // FILE_LOGGER_H diff --git a/src/lib/utility/scheduling/TaskGroupParallel.cpp b/src/lib/utility/scheduling/TaskGroupParallel.cpp index 966ccbba..e07fc553 100644 --- a/src/lib/utility/scheduling/TaskGroupParallel.cpp +++ b/src/lib/utility/scheduling/TaskGroupParallel.cpp @@ -7,6 +7,7 @@ TaskGroupParallel::TaskGroupParallel() : m_needsToStartThreads(true) + , m_activeTaskCountMutex(std::make_shared()) { } @@ -30,7 +31,8 @@ void TaskGroupParallel::doEnter(std::shared_ptr blackboard) for (size_t i = 0; i < m_tasks.size(); i++) { m_tasks[i]->active = true; - m_tasks[i]->thread = std::make_shared(&TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard); + m_tasks[i]->thread = std::make_shared( + &TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard, m_activeTaskCountMutex); } } } @@ -65,12 +67,13 @@ void TaskGroupParallel::doReset(std::shared_ptr blackboard) if (!m_tasks[i]->active) { { - std::lock_guard lock(m_activeTaskCountMutex); + std::lock_guard lock(*m_activeTaskCountMutex.get()); m_activeTaskCount++; } m_tasks[i]->thread->join(); m_tasks[i]->active = true; - m_tasks[i]->thread = std::make_shared(&TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard); + m_tasks[i]->thread = std::make_shared( + &TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard, m_activeTaskCountMutex); } } } @@ -88,10 +91,13 @@ void TaskGroupParallel::doTerminate() } } -void TaskGroupParallel::processTaskThreaded(std::shared_ptr taskInfo, std::shared_ptr blackboard) +void TaskGroupParallel::processTaskThreaded( + std::shared_ptr taskInfo, + std::shared_ptr blackboard, + std::shared_ptr activeTaskCountMutex) { ScopedFunctor functor([&](){ - std::lock_guard lock(m_activeTaskCountMutex); + std::lock_guard lock(*activeTaskCountMutex.get()); m_activeTaskCount--; }); @@ -113,6 +119,6 @@ void TaskGroupParallel::processTaskThreaded(std::shared_ptr taskInfo, int TaskGroupParallel::getActiveTaskCount() const { - std::lock_guard lock(m_activeTaskCountMutex); + std::lock_guard lock(*m_activeTaskCountMutex.get()); return m_activeTaskCount; } diff --git a/src/lib/utility/scheduling/TaskGroupParallel.h b/src/lib/utility/scheduling/TaskGroupParallel.h index b4ba7036..98099fb3 100644 --- a/src/lib/utility/scheduling/TaskGroupParallel.h +++ b/src/lib/utility/scheduling/TaskGroupParallel.h @@ -35,7 +35,10 @@ private: virtual void doReset(std::shared_ptr blackboard); virtual void doTerminate(); - void processTaskThreaded(std::shared_ptr taskInfo, std::shared_ptr blackboard); + void processTaskThreaded( + std::shared_ptr taskInfo, + std::shared_ptr blackboard, + std::shared_ptr activeTaskCountMutex); int getActiveTaskCount() const; std::vector> m_tasks; @@ -43,7 +46,7 @@ private: volatile bool m_taskFailed; volatile int m_activeTaskCount; - mutable std::mutex m_activeTaskCountMutex; + mutable std::shared_ptr m_activeTaskCountMutex; }; #endif // TASK_GROUP_PARALLEL_H diff --git a/src/lib/utility/scheduling/TaskRunner.cpp b/src/lib/utility/scheduling/TaskRunner.cpp index 341bb287..e1f7a54b 100644 --- a/src/lib/utility/scheduling/TaskRunner.cpp +++ b/src/lib/utility/scheduling/TaskRunner.cpp @@ -1,5 +1,8 @@ #include "utility/scheduling/TaskRunner.h" +#include "utility/logging/logging.h" +#include "utility/scheduling/TaskScheduler.h" + TaskRunner::TaskRunner(std::shared_ptr task) : m_task(task) , m_reset(false) @@ -12,13 +15,27 @@ TaskRunner::~TaskRunner() Task::TaskState TaskRunner::update(std::shared_ptr blackboard) { - if (m_reset) + try { - m_task->reset(blackboard); - m_reset = false; + if (m_reset) + { + m_task->reset(blackboard); + m_reset = false; + } + + return m_task->update(blackboard); + } + catch (std::exception& e) + { + LOG_ERROR(e.what()); + } + catch (...) + { + LOG_ERROR("Unknown exception thrown during task running"); } - return m_task->update(blackboard); + TaskScheduler::getInstance()->terminateRunningTasks(); + return Task::STATE_FAILURE; } void TaskRunner::reset() diff --git a/src/lib/utility/scheduling/TaskScheduler.cpp b/src/lib/utility/scheduling/TaskScheduler.cpp index e4561d77..b6d5079b 100644 --- a/src/lib/utility/scheduling/TaskScheduler.cpp +++ b/src/lib/utility/scheduling/TaskScheduler.cpp @@ -126,11 +126,17 @@ bool TaskScheduler::hasTasksQueued() const return m_taskRunners.size(); } +void TaskScheduler::terminateRunningTasks() +{ + m_terminateRunningTasks = true; +} + std::shared_ptr TaskScheduler::s_instance; TaskScheduler::TaskScheduler() : m_loopIsRunning(false) , m_threadIsRunning(false) + , m_terminateRunningTasks(false) { } @@ -151,23 +157,25 @@ void TaskScheduler::processTasks() std::shared_ptr blackboard = std::make_shared(); while (true) { - if (runner->update(blackboard) != Task::STATE_RUNNING) - { - break; - } - { std::lock_guard lock(m_loopMutex); - if (!m_loopIsRunning) + if (!m_loopIsRunning || m_terminateRunningTasks) { runner->terminate(); break; } } + + if (runner->update(blackboard) != Task::STATE_RUNNING) + { + break; + } } } m_taskRunners.pop_front(); } + + m_terminateRunningTasks = false; } diff --git a/src/lib/utility/scheduling/TaskScheduler.h b/src/lib/utility/scheduling/TaskScheduler.h index 8b96550d..0d7ceb04 100644 --- a/src/lib/utility/scheduling/TaskScheduler.h +++ b/src/lib/utility/scheduling/TaskScheduler.h @@ -25,6 +25,8 @@ public: bool loopIsRunning() const; bool hasTasksQueued() const; + void terminateRunningTasks(); + private: static std::shared_ptr s_instance; @@ -37,6 +39,7 @@ private: bool m_loopIsRunning; bool m_threadIsRunning; + bool m_terminateRunningTasks; std::deque> m_taskRunners; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp b/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp index 59755f1e..b04355a4 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp +++ b/src/lib_cxx/data/indexer/IndexerCommandCxx.cpp @@ -20,6 +20,28 @@ IndexerCommandCxx::~IndexerCommandCxx() { } +size_t IndexerCommandCxx::getByteSize() const +{ + size_t size = IndexerCommand::getByteSize(); + + for (auto i : m_systemHeaderSearchPaths) + { + size += i.str().size(); + } + + for (auto i : m_frameworkSearchPaths) + { + size += i.str().size(); + } + + for (auto i : m_compilerFlags) + { + size += i.size(); + } + + return size; +} + std::vector IndexerCommandCxx::getSystemHeaderSearchPaths() const { return m_systemHeaderSearchPaths; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxx.h b/src/lib_cxx/data/indexer/IndexerCommandCxx.h index 321e4208..d136d698 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxx.h +++ b/src/lib_cxx/data/indexer/IndexerCommandCxx.h @@ -21,6 +21,7 @@ public: const std::vector& compilerFlags); virtual ~IndexerCommandCxx(); + virtual size_t getByteSize() const override; std::vector getSystemHeaderSearchPaths() const; std::vector getFrameworkSearchPaths() const; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp b/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp index 2c040725..ff57bca0 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp +++ b/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp @@ -49,6 +49,11 @@ std::string IndexerCommandCxxCdb::getKindString() const return getIndexerKindString(); } +size_t IndexerCommandCxxCdb::getByteSize() const +{ + return IndexerCommandCxx::getByteSize() + sizeof(*this) + m_workingDirectory.str().size(); +} + FilePath IndexerCommandCxxCdb::getWorkingDirectory() const { return m_workingDirectory; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.h b/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.h index fcfed9eb..eb144598 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.h +++ b/src/lib_cxx/data/indexer/IndexerCommandCxxCdb.h @@ -31,6 +31,7 @@ public: virtual ~IndexerCommandCxxCdb(); virtual std::string getKindString() const; + virtual size_t getByteSize() const; FilePath getWorkingDirectory() const; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp b/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp index 524b3de1..5b48f11d 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp +++ b/src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp @@ -28,6 +28,11 @@ std::string IndexerCommandCxxManual::getKindString() const return getIndexerKindString(); } +size_t IndexerCommandCxxManual::getByteSize() const +{ + return IndexerCommandCxx::getByteSize() + sizeof(*this); +} + std::string IndexerCommandCxxManual::getLanguageStandard() const { return m_languageStandard; diff --git a/src/lib_cxx/data/indexer/IndexerCommandCxxManual.h b/src/lib_cxx/data/indexer/IndexerCommandCxxManual.h index 824bf6e7..24b50385 100644 --- a/src/lib_cxx/data/indexer/IndexerCommandCxxManual.h +++ b/src/lib_cxx/data/indexer/IndexerCommandCxxManual.h @@ -19,9 +19,11 @@ public: const std::vector& systemHeaderSearchPaths, const std::vector& frameworkSearchPaths, const std::vector& compilerFlags); + virtual ~IndexerCommandCxxManual(); virtual std::string getKindString() const; + virtual size_t getByteSize() const; std::string getLanguageStandard() const; diff --git a/src/lib_cxx/data/parser/cxx/ASTConsumer.cpp b/src/lib_cxx/data/parser/cxx/ASTConsumer.cpp index 9d3e7cce..b8789b74 100644 --- a/src/lib_cxx/data/parser/cxx/ASTConsumer.cpp +++ b/src/lib_cxx/data/parser/cxx/ASTConsumer.cpp @@ -1,4 +1,5 @@ #include "data/parser/cxx/ASTConsumer.h" + #include "data/parser/cxx/CxxAstVisitor.h" #include "data/parser/cxx/CxxVerboseAstVisitor.h" #include "settings/ApplicationSettings.h" @@ -11,7 +12,9 @@ ASTConsumer::ASTConsumer( std::shared_ptr canonicalFilePathCache ) { - if (ApplicationSettings::getInstance()->getLoggingEnabled() && ApplicationSettings::getInstance()->getVerboseIndexerLoggingEnabled()) + ApplicationSettings* appSettings = ApplicationSettings::getInstance().get(); + + if (appSettings->getLoggingEnabled() && appSettings->getVerboseIndexerLoggingEnabled()) { m_visitor = std::make_shared(context, preprocessor, client, fileRegister, canonicalFilePathCache); } diff --git a/src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp b/src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp index 3b227aa2..a4e34e30 100644 --- a/src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp +++ b/src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp @@ -299,7 +299,7 @@ bool CxxAstVisitor::TraverseTemplateTemplateParmDecl(clang::TemplateTemplateParm bool CxxAstVisitor::VisitTranslationUnitDecl(clang::TranslationUnitDecl *d) { - return !m_client->cancelOnFatalErrors() || !m_client->hasFatalErrors(); + return true; } bool CxxAstVisitor::TraverseNestedNameSpecifierLoc(clang::NestedNameSpecifierLoc loc) diff --git a/src/lib_cxx/data/parser/cxx/CxxParser.h b/src/lib_cxx/data/parser/cxx/CxxParser.h index 6b6aa7f0..871dc07e 100644 --- a/src/lib_cxx/data/parser/cxx/CxxParser.h +++ b/src/lib_cxx/data/parser/cxx/CxxParser.h @@ -22,7 +22,6 @@ public: void buildIndex(std::shared_ptr indexerCommand); void buildIndex(const std::string& fileName, std::shared_ptr fileContent); - private: std::vector getCommandlineArgumentsEssential( const std::vector& compilerFlags, diff --git a/src/lib_gui/platform_includes/includesMac.h b/src/lib_gui/platform_includes/includesMac.h index eb0f713f..3b22fad7 100644 --- a/src/lib_gui/platform_includes/includesMac.h +++ b/src/lib_gui/platform_includes/includesMac.h @@ -8,6 +8,7 @@ #include "qt/utility/utilityQt.h" #include "utility/AppPath.h" +#include "utility/file/FilePath.h" #include "utility/ResourcePaths.h" #include "utility/UserPaths.h" @@ -91,7 +92,7 @@ void setupApp(int argc, char *argv[]) if (!appIsMacBundle) { - UserPaths::setUserDataPath(FilePath(path.absolute().str() + "/user/")); + UserPaths::setUserDataPath(FilePath("./user/")); } else { diff --git a/src/lib_gui/qt/element/QtCodeFileTitleButton.cpp b/src/lib_gui/qt/element/QtCodeFileTitleButton.cpp index 69a9d28e..88c04455 100644 --- a/src/lib_gui/qt/element/QtCodeFileTitleButton.cpp +++ b/src/lib_gui/qt/element/QtCodeFileTitleButton.cpp @@ -66,9 +66,12 @@ void QtCodeFileTitleButton::setIsComplete(bool isComplete) if (!isComplete) { + FilePath hatchingFilePath(ResourcePaths::getGuiPath().str() + "code_view/images/pattern_" + + ColorScheme::getInstance()->getColor("code/file/title/hatching") + ".png" + ); + setStyleSheet(( - "background-image: url(" + ResourcePaths::getGuiPath().str() + "code_view/images/pattern_" + - ColorScheme::getInstance()->getColor("code/file/title/hatching") + ".png);" + "background-image: url(" + hatchingFilePath.str() + ");" ).c_str()); } else diff --git a/src/lib_gui/qt/view/QtDialogView.cpp b/src/lib_gui/qt/view/QtDialogView.cpp index c42fac74..89002698 100644 --- a/src/lib_gui/qt/view/QtDialogView.cpp +++ b/src/lib_gui/qt/view/QtDialogView.cpp @@ -179,7 +179,7 @@ void QtDialogView::updateIndexingDialog(size_t fileCount, size_t totalFileCount, void QtDialogView::finishedIndexingDialog( size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, - float time, ErrorCountInfo errorInfo) + float time, ErrorCountInfo errorInfo, bool interrupted) { std::stringstream ss; ss << "Finished indexing: "; @@ -198,7 +198,7 @@ void QtDialogView::finishedIndexingDialog( m_windowStack.clearWindows(); QtIndexingDialog* window = createWindow(); - window->setupReport(indexedFileCount, totalIndexedFileCount, completedFileCount, totalFileCount, time); + window->setupReport(indexedFileCount, totalIndexedFileCount, completedFileCount, totalFileCount, time, interrupted); window->updateErrorCount(errorInfo.total, errorInfo.fatal); setUIBlocked(false); @@ -206,6 +206,17 @@ void QtDialogView::finishedIndexingDialog( ); } +void QtDialogView::hideDialogs() +{ + m_onQtThread( + [=]() + { + m_windowStack.clearWindows(); + setUIBlocked(false); + } + ); +} + int QtDialogView::confirm(const std::string& message, const std::vector& options) { int result = -1; diff --git a/src/lib_gui/qt/view/QtDialogView.h b/src/lib_gui/qt/view/QtDialogView.h index ec65830f..d1c60919 100644 --- a/src/lib_gui/qt/view/QtDialogView.h +++ b/src/lib_gui/qt/view/QtDialogView.h @@ -39,7 +39,9 @@ public: virtual void updateIndexingDialog(size_t fileCount, size_t totalFileCount, std::string sourcePath) override; virtual void finishedIndexingDialog( size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, - float time, ErrorCountInfo errorInfo) override; + float time, ErrorCountInfo errorInfo, bool interrupted) override; + + virtual void hideDialogs() override; int confirm(const std::string& message, const std::vector& options) override; diff --git a/src/lib_gui/qt/window/QtIndexingDialog.cpp b/src/lib_gui/qt/window/QtIndexingDialog.cpp index 1e8261e7..6699c515 100644 --- a/src/lib_gui/qt/window/QtIndexingDialog.cpp +++ b/src/lib_gui/qt/window/QtIndexingDialog.cpp @@ -153,7 +153,7 @@ void QtIndexingDialog::setupIndexing() void QtIndexingDialog::setupReport( size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, - float time) + float time, bool interrupted) { QBoxLayout* layout = createLayout(); @@ -182,7 +182,7 @@ void QtIndexingDialog::setupReport( m_sizeHint = QSize(400, 280); - if (indexedFileCount != totalIndexedFileCount) + if (interrupted) { updateTitle("Interrupted Indexing"); } diff --git a/src/lib_gui/qt/window/QtIndexingDialog.h b/src/lib_gui/qt/window/QtIndexingDialog.h index 193b1e43..84bceddc 100644 --- a/src/lib_gui/qt/window/QtIndexingDialog.h +++ b/src/lib_gui/qt/window/QtIndexingDialog.h @@ -33,7 +33,8 @@ public: DialogView::IndexingOptions options, std::function callback); void setupIndexing(); void setupReport( - size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, float time); + size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, + float time, bool interrupted); void setupUnknownProgress(); void setupProgress(); diff --git a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp index 4461f184..e8bff69f 100644 --- a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp +++ b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp @@ -118,9 +118,10 @@ void QtProjectWizzardContentPreferences::populate(QGridLayout* layout, int& row) addHelpButton("Number of parallel threads used to index your projects.\nWhen setting this to 0 Sourcetrail tries to use the ideal thread count for your computer.", layout, row); row++; - // cancel indexing on fatal errors - m_cancelIndexingOnFatalErrors = addCheckBox("Cancel on Fatals", "Cancel indexing on Fatal errors.", - "Cancel indexing of translation units with fatal errors, which result in partly indexed files.", layout, row); + // multi process indexing + m_multiProcessIndexing = addCheckBox("Multi process indexing", "Use processes instead of threads for indexing.", + "Using processes instead of threads prevents the application from crashing on unforseen exceptions during indexing.", + layout, row); addGap(layout, row); @@ -260,7 +261,7 @@ void QtProjectWizzardContentPreferences::load() m_threads->setCurrentIndex(appSettings->getIndexerThreadCount()); // index and value are the same indexerThreadsChanges(m_threads->currentIndex()); - m_cancelIndexingOnFatalErrors->setChecked(appSettings->getCancelIndexingOnFatalErrors()); + m_multiProcessIndexing->setChecked(appSettings->getMultiProcessIndexingEnabled()); if (m_javaPath) { @@ -304,7 +305,7 @@ void QtProjectWizzardContentPreferences::save() if (pluginPort) appSettings->setPluginPort(pluginPort); appSettings->setIndexerThreadCount(m_threads->currentIndex()); // index and value are the same - appSettings->setCancelIndexingOnFatalErrors(m_cancelIndexingOnFatalErrors->isChecked()); + appSettings->setMultiProcessIndexingEnabled(m_multiProcessIndexing->isChecked()); if (m_javaPath) { diff --git a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h index 2df1d1a7..b45bbc0b 100644 --- a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h +++ b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h @@ -69,7 +69,7 @@ private: QComboBox* m_threads; QLabel* m_threadsInfoLabel; - QCheckBox* m_cancelIndexingOnFatalErrors; + QCheckBox* m_multiProcessIndexing; std::shared_ptr m_javaPathDetector; std::shared_ptr m_mavenPathDetector; diff --git a/src/lib_gui/utility/utilityApp.cpp b/src/lib_gui/utility/utilityApp.cpp index 55e21bb2..8163219e 100644 --- a/src/lib_gui/utility/utilityApp.cpp +++ b/src/lib_gui/utility/utilityApp.cpp @@ -6,7 +6,15 @@ #include "utility/utilityString.h" -std::string utility::executeProcess(const std::string& command, const std::string& workingDirectory) +#include + +namespace utility +{ + std::mutex s_runningProcessesMutex; + std::set s_runningProcesses; +} + +std::string utility::executeProcess(const std::string& command, const std::string& workingDirectory, int timeout) { QProcess process; process.setProcessChannelMode(QProcess::MergedChannels); @@ -16,8 +24,18 @@ std::string utility::executeProcess(const std::string& command, const std::strin process.setWorkingDirectory(workingDirectory.c_str()); } - process.start(command.c_str()); - process.waitForFinished(); + { + std::lock_guard lock(s_runningProcessesMutex); + process.start(command.c_str()); + s_runningProcesses.insert(&process); + } + + process.waitForFinished(timeout); + { + std::lock_guard lock(s_runningProcessesMutex); + s_runningProcesses.erase(&process); + } + std::string processoutput = process.readAll().toStdString(); process.close(); processoutput = utility::trim(processoutput); @@ -25,6 +43,41 @@ std::string utility::executeProcess(const std::string& command, const std::strin return processoutput; } +int utility::executeProcessAndGetExitCode(const std::string& command, const std::string& workingDirectory, int timeout) +{ + QProcess process; + + if (!workingDirectory.empty()) + { + process.setWorkingDirectory(workingDirectory.c_str()); + } + + { + std::lock_guard lock(s_runningProcessesMutex); + process.start(command.c_str()); + s_runningProcesses.insert(&process); + } + + process.waitForFinished(timeout); + { + std::lock_guard lock(s_runningProcessesMutex); + s_runningProcesses.erase(&process); + } + + int exitCode = process.exitCode(); + process.close(); + return exitCode; +} + +void utility::killRunningProcesses() +{ + std::lock_guard lock(s_runningProcessesMutex); + for (QProcess* process : s_runningProcesses) + { + process->kill(); + } +} + ApplicationArchitectureType utility::getApplicationArchitectureType() { #ifdef Q_PROCESSOR_X86_64 diff --git a/src/lib_gui/utility/utilityApp.h b/src/lib_gui/utility/utilityApp.h index 7978bffa..25a8ba18 100644 --- a/src/lib_gui/utility/utilityApp.h +++ b/src/lib_gui/utility/utilityApp.h @@ -1,13 +1,18 @@ #ifndef UTILITY_APP_H #define UTILITY_APP_H +#include +#include #include #include "utility/ApplicationArchitectureType.h" namespace utility { - std::string executeProcess(const std::string& command, const std::string& workingDirectory = ""); + std::string executeProcess(const std::string& command, const std::string& workingDirectory = "", int timeout = 30000); + int executeProcessAndGetExitCode(const std::string& command, const std::string& workingDirectory = "", int timeout = 30000); + + void killRunningProcesses(); ApplicationArchitectureType getApplicationArchitectureType(); diff --git a/src/lib_java/data/indexer/IndexerCommandJava.cpp b/src/lib_java/data/indexer/IndexerCommandJava.cpp index 0f2bdecf..4346b540 100644 --- a/src/lib_java/data/indexer/IndexerCommandJava.cpp +++ b/src/lib_java/data/indexer/IndexerCommandJava.cpp @@ -25,6 +25,18 @@ std::string IndexerCommandJava::getKindString() const return getIndexerKindString(); } +size_t IndexerCommandJava::getByteSize() const +{ + size_t size = IndexerCommand::getByteSize() + sizeof(*this); + + for (auto i : m_classPath) + { + size += i.str().size(); + } + + return size; +} + std::vector IndexerCommandJava::getClassPath() const { return m_classPath; diff --git a/src/lib_java/data/indexer/IndexerCommandJava.h b/src/lib_java/data/indexer/IndexerCommandJava.h index 6a4db9fb..2f945478 100644 --- a/src/lib_java/data/indexer/IndexerCommandJava.h +++ b/src/lib_java/data/indexer/IndexerCommandJava.h @@ -20,7 +20,8 @@ public: const std::vector& classPath); virtual ~IndexerCommandJava(); - std::string getKindString() const override; + virtual std::string getKindString() const override; + virtual size_t getByteSize() const override; std::vector getClassPath() const; diff --git a/src/lib_java/data/parser/java/JavaParser.cpp b/src/lib_java/data/parser/java/JavaParser.cpp index 1e1fdee6..4d3b7b35 100644 --- a/src/lib_java/data/parser/java/JavaParser.cpp +++ b/src/lib_java/data/parser/java/JavaParser.cpp @@ -9,12 +9,46 @@ #include "settings/ApplicationSettings.h" #include "utility/file/FileSystem.h" #include "utility/text/TextAccess.h" +#include "utility/ResourcePaths.h" #include "utility/utilityString.h" +std::string JavaParser::prepareJavaEnvironment() +{ + std::string errorString; + + if (!JavaEnvironmentFactory::getInstance()) + { +#ifdef _WIN32 + const std::string separator = ";"; +#else + const std::string separator = ":"; +#endif + JavaEnvironmentFactory::createInstance( + ResourcePaths::getJavaPath().str() + "guava-18.0.jar" + separator + + ResourcePaths::getJavaPath().str() + "java-indexer.jar" + separator + + ResourcePaths::getJavaPath().str() + "javaparser-core.jar" + separator + + ResourcePaths::getJavaPath().str() + "javaslang-2.0.3.jar" + separator + + ResourcePaths::getJavaPath().str() + "javassist-3.19.0-GA.jar" + separator + + ResourcePaths::getJavaPath().str() + "java-symbol-solver-core.jar" + separator + + ResourcePaths::getJavaPath().str() + "java-symbol-solver-logic.jar" + separator + + ResourcePaths::getJavaPath().str() + "java-symbol-solver-model.jar", + errorString + ); + } + + return errorString; +} + JavaParser::JavaParser(std::shared_ptr client, std::shared_ptr fileRegister) : Parser(client) , m_id(s_nextParserId++) { + const std::string errorString = prepareJavaEnvironment(); + if (errorString.size() > 0) + { + LOG_ERROR(errorString); + } + std::shared_ptr factory = JavaEnvironmentFactory::getInstance(); if (factory) { @@ -63,7 +97,10 @@ void JavaParser::buildIndex(const FilePath& filePath, std::shared_ptr textAccess) +void JavaParser::buildIndex( + const FilePath& sourceFilePath, + const std::string& classPath, + std::shared_ptr textAccess) { if (m_javaEnvironment) { diff --git a/src/lib_java/data/parser/java/JavaParser.h b/src/lib_java/data/parser/java/JavaParser.h index 8130bc85..5ade75a8 100644 --- a/src/lib_java/data/parser/java/JavaParser.h +++ b/src/lib_java/data/parser/java/JavaParser.h @@ -31,6 +31,9 @@ class FileRegister; class JavaParser: public Parser { public: + // returns: error message + static std::string prepareJavaEnvironment(); + JavaParser(std::shared_ptr client, std::shared_ptr fileRegister); ~JavaParser(); @@ -38,7 +41,8 @@ public: void buildIndex(const FilePath& filePath, std::shared_ptr textAccess); private: - void buildIndex(const FilePath& sourceFilePath, const std::string& classPath, std::shared_ptr textAccess); + void buildIndex( + const FilePath& sourceFilePath, const std::string& classPath, std::shared_ptr textAccess); // This macro makes available a variable T, the passed-in t. blablabla TODO: write somethign real here #define MAKE_PARAMS_0() diff --git a/src/lib_java/project/SourceGroupJava.cpp b/src/lib_java/project/SourceGroupJava.cpp index 41c5ea82..5ad5610a 100644 --- a/src/lib_java/project/SourceGroupJava.cpp +++ b/src/lib_java/project/SourceGroupJava.cpp @@ -1,20 +1,22 @@ #include "project/SourceGroupJava.h" +#include "utility/file/FileRegister.h" +#include "utility/file/FileSystem.h" +#include "utility/messaging/type/MessageStatus.h" +#include "utility/ResourcePaths.h" +#include "utility/text/TextAccess.h" +#include "utility/ScopedFunctor.h" +#include "utility/utility.h" +#include "utility/utilityMaven.h" +#include "utility/utilityString.h" + #include "Application.h" #include "component/view/DialogView.h" #include "data/indexer/IndexerCommandJava.h" #include "data/parser/java/JavaEnvironmentFactory.h" #include "data/parser/java/JavaEnvironment.h" +#include "data/parser/java/JavaParser.h" #include "settings/ApplicationSettings.h" -#include "utility/file/FileRegister.h" -#include "utility/file/FileSystem.h" -#include "utility/messaging/type/MessageStatus.h" -#include "utility/text/TextAccess.h" -#include "utility/ScopedFunctor.h" -#include "utility/ResourcePaths.h" -#include "utility/utilityMaven.h" -#include "utility/utilityString.h" -#include "utility/utility.h" SourceGroupJava::SourceGroupJava(std::shared_ptr settings) : m_settings(settings) @@ -105,27 +107,7 @@ std::vector> SourceGroupJava::getIndexerCommands bool SourceGroupJava::prepareJavaEnvironment() { - std::string errorString; - if (!JavaEnvironmentFactory::getInstance()) - { -#ifdef _WIN32 - const std::string separator = ";"; -#else - const std::string separator = ":"; -#endif - JavaEnvironmentFactory::createInstance( - ResourcePaths::getJavaPath().str() + "guava-18.0.jar" + separator + - ResourcePaths::getJavaPath().str() + "java-indexer.jar" + separator + - ResourcePaths::getJavaPath().str() + "javaparser-core.jar" + separator + - ResourcePaths::getJavaPath().str() + "javaslang-2.0.3.jar" + separator + - ResourcePaths::getJavaPath().str() + "javassist-3.19.0-GA.jar" + separator + - ResourcePaths::getJavaPath().str() + "java-symbol-solver-core.jar" + separator + - ResourcePaths::getJavaPath().str() + "java-symbol-solver-logic.jar" + separator + - ResourcePaths::getJavaPath().str() + "java-symbol-solver-model.jar", - errorString - ); - } - + const std::string errorString = JavaParser::prepareJavaEnvironment(); if (errorString.size() > 0) { LOG_ERROR(errorString); diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index b97e0694..5c0ff56c 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -23,6 +23,7 @@ add_files( SettingsTestSuite.h SettingsMigratorTestSuite.h SearchIndexTestSuite.h + SharedMemoryTestSuite.h SourceLocationCollectionTestSuite.h SqliteBookmarkStorageTestSuite.h SqliteIndexStorageTestSuite.h diff --git a/src/test/JavaParserTestSuite.h b/src/test/JavaParserTestSuite.h index 0ff3a341..b6535280 100644 --- a/src/test/JavaParserTestSuite.h +++ b/src/test/JavaParserTestSuite.h @@ -12,7 +12,6 @@ #include "helper/TestParserClient.h" - class JavaParserTestSuite: public CxxTest::TestSuite { public: diff --git a/src/test/SharedMemoryTestSuite.h b/src/test/SharedMemoryTestSuite.h new file mode 100644 index 00000000..fc0439b3 --- /dev/null +++ b/src/test/SharedMemoryTestSuite.h @@ -0,0 +1,95 @@ +#include + +#include "utility/interprocess/SharedMemory.h" + +class SharedMemoryTestSuite : public CxxTest::TestSuite +{ +public: + void test_shared_memory(void) + { + SharedMemory memory("memory", 1000, SharedMemory::CREATE_AND_DELETE); + + { + SharedMemory::ScopedAccess access(&memory); + + TS_ASSERT_EQUALS(access.getMemorySize(), 1000); + *access.accessValue("count") = 0; + } + + std::vector> threads; + for (unsigned int i = 0; i < 4; i++) + { + threads.push_back(std::make_shared( + []() + { + SharedMemory memory("memory", 0, SharedMemory::OPEN_ONLY); + + SharedMemory::ScopedAccess access(&memory); + + if (access.getMemorySize() < 5000) + { + access.growMemory(5000 - access.getMemorySize()); + } + + int* count = access.accessValue("count"); + *count += 1; + + SharedMemory::String* str = access.accessValueWithAllocator("string"); + str->append("hi"); + + SharedMemory::Vector* nums = access.accessValueWithAllocator>("nums"); + nums->push_back(nums->size()); + + SharedMemory::Vector* strings = + access.accessValueWithAllocator>("strings"); + strings->push_back(SharedMemory::String("ho", access.getAllocator())); + + SharedMemory::Map* vals = + access.accessValueWithAllocator>("vals"); + vals->emplace(vals->size(), vals->size() * vals->size()); + } + )); + } + + for (auto thread : threads) + { + thread->join(); + } + threads.clear(); + + { + SharedMemory::ScopedAccess access(&memory); + + TS_ASSERT_EQUALS(access.getMemorySize(), 5000); + TS_ASSERT_EQUALS(*access.accessValue("count"), 4); + TS_ASSERT_EQUALS(access.accessValueWithAllocator("string")->c_str(), "hihihihi"); + + SharedMemory::Vector* nums = access.accessValueWithAllocator>("nums"); + TS_ASSERT_EQUALS(nums->size(), 4); + for (int i : *nums) + { + TS_ASSERT_EQUALS(i, i); + } + + SharedMemory::Vector* strings = + access.accessValueWithAllocator>("strings"); + TS_ASSERT_EQUALS(strings->size(), 4); + for (SharedMemory::String str : *strings) + { + TS_ASSERT_EQUALS(str, "ho"); + } + + SharedMemory::Map* vals = + access.accessValueWithAllocator>("vals"); + TS_ASSERT_EQUALS(vals->size(), 4); + + size_t i = 0; + for (auto val : *vals) + { + TS_ASSERT_EQUALS(val.first, i); + TS_ASSERT_EQUALS(val.second, i * i); + i++; + } + } + } +};