ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: indexer processes

Browse Source 
* TaskBuildIndex starts seperate indexer processes and communicates via shared memory
* indexer processes get restarted if they fail
* indexer processes are killed when the app closes or crashes
* added utility class SharedMemory to allocate and access shared memory, providing basic data structures
* added SharedMemoryGarbageCollector for keeping track of running instances and cleaning up shared memory in case of a crash
* show errors for source files that crashed during indexing
* added basic exception handling to task scheduling
* added option to turn off processes and use threads in preferences, but still use shared memory

fortune cookie message = Good news from someone dear is coming soon.
This commit is contained in:
Eberhard Graether
2017-05-04 15:50:59 +02:00
parent 9beea94fd8
commit 706119ebcc
109 changed files with 3351 additions and 1261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+70 -5
View File
@@ -14,6 +14,7 @@ include(cmake/setWinXxBits.cmake)
# Variables --------------------------------------------------------------------
set(PROJECT_NAME Sourcetrail)
set(PROJECT_NAME_LOWER_CASE sourcetrail)
set(APP_PROJECT_NAME "${PROJECT_NAME}")
set(LIB_LICENSE_PROJECT_NAME "${PROJECT_NAME}_lib_license")
@@ -24,6 +25,8 @@ set(LIB_PROJECT_NAME "${PROJECT_NAME}_lib")
set(LICENSE_GENERATOR_PROJECT_NAME "${PROJECT_NAME}_license_generator")
set(TEST_PROJECT_NAME "${PROJECT_NAME}_test")
set (APP_INDEXER_NAME "${PROJECT_NAME_LOWER_CASE}_indexer")
if (WIN32)
set(PLATFORM_INCLUDE "includesWindows.h")
# this adds the library path for VLD, must be done before the project target is created
@@ -221,6 +224,8 @@ set_property(
"${CMAKE_SOURCE_DIR}/src/lib_license"
"${CMAKE_SOURCE_DIR}/src/lib_gui"
"${CMAKE_BINARY_DIR}/src/lib_license"
"${CMAKE_SOURCE_DIR}/src/lib_cxx"
"${CMAKE_SOURCE_DIR}/src/lib_java"
)
target_include_directories(${LIB_PROJECT_NAME} SYSTEM
@@ -398,6 +403,60 @@ include(cmake/publicKey.cmake)
set(CMAKE_AUTOMOC OFF)
# Indexer App ------------------------------------------------------------------
if (UNIX)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/app/")
else ()
foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} )
string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG )
set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} "${CMAKE_BINARY_DIR}/${OUTPUTCONFIG}/app/")
endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES )
endif ()
add_subdirectory(src/indexer)
# Find includes in corresponding build directories
set(CMAKE_INCLUDE_CURRENT_DIR ON)
add_executable(${APP_INDEXER_NAME} ${INDEXER_FILES})
if (WIN32)
# hide the console when running a release build.
set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_DEBUG "/SUBSYSTEM:CONSOLE /DEBUG:FASTLINK")
set_target_properties(${APP_INDEXER_NAME} PROPERTIES COMPILE_DEFINITIONS_DEBUG "_CONSOLE")
set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_RELWITHDEBINFO "/SUBSYSTEM:CONSOLE")
set_target_properties(${APP_INDEXER_NAME} PROPERTIES COMPILE_DEFINITIONS_RELWITHDEBINFO "_CONSOLE")
set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_RELEASE "/ENTRY:\"mainCRTStartup\" /SUBSYSTEM:WINDOWS /DEBUG")
set_target_properties(${APP_INDEXER_NAME} PROPERTIES LINK_FLAGS_MINSIZEREL "/SUBSYSTEM:WINDOWS")
# generate pdb for release build
set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi")
endif ()
create_source_groups(${INDEXER_FILES})
target_link_libraries(${APP_INDEXER_NAME} ${LIB_GUI_PROJECT_NAME} ${LIB_CXX_PROJECT_NAME} ${LIB_JAVA_PROJECT_NAME} ${LIB_PROJECT_NAME})
if (APPLE)
find_library(CORE_FOUNDATION CoreFoundation)
target_link_libraries(${APP_INDEXER_NAME} ${CORE_FOUNDATION})
endif ()
set_property(
TARGET ${APP_INDEXER_NAME}
PROPERTY INCLUDE_DIRECTORIES
"${CMAKE_SOURCE_DIR}/src/app"
"${CMAKE_SOURCE_DIR}/src/lib"
"${CMAKE_SOURCE_DIR}/src/lib_gui"
"${CMAKE_SOURCE_DIR}/src/lib_license"
"${CMAKE_SOURCE_DIR}/src/lib_cxx"
"${CMAKE_SOURCE_DIR}/src/lib_java"
"${CMAKE_BINARY_DIR}/src/lib_gui"
"${CMAKE_BINARY_DIR}/src/lib_license"
)
# App --------------------------------------------------------------------------
if (UNIX)
@@ -418,7 +477,7 @@ if (WIN32)
file(WRITE ${CMAKE_BINARY_DIR}/Sourcetrail.rc
"// Icon with lowest ID value placed first to ensure application icon\n"
"// remains consistent on all systems.\n"
"IDI_ICON1 ICON \"${CMAKE_BINARY_DIR}/sourcetrail.ico\"\n"
"IDI_ICON1 ICON \"${CMAKE_BINARY_DIR}/Sourcetrail.ico\"\n"
)
add_executable(${APP_PROJECT_NAME} ${APP_FILES} ${CMAKE_BINARY_DIR}/Sourcetrail.rc)
@@ -469,6 +528,8 @@ if (APPLE)
target_link_libraries(${APP_PROJECT_NAME} ${CORE_FOUNDATION})
endif ()
add_dependencies(${APP_PROJECT_NAME} ${APP_INDEXER_NAME})
# MacOSX Bundle ----------------------------------------------------------------
@@ -476,6 +537,7 @@ if (APPLE)
set(MACOSX_BUNDLE_NAME ${PROJECT_NAME})
set(MACOSX_BINARY_NAME ${APP_PROJECT_NAME})
set(MACOSX_INDEXER_BINARY_NAME ${APP_INDEXER_NAME})
set(MACOSX_DYNAMIC_LIBRARIES "")
string(REPLACE ";" " " MACOSX_DYNAMIC_LIBRARIES "${MACOSX_DYNAMIC_LIBRARIES}")
@@ -625,7 +687,7 @@ if (WIN32)
include_directories($ENV{VLD_DIR}/include)
endif ()
if(UNIX AND NOT APPLE)
if (UNIX)
# symlinks for data
message(STATUS "create symlink: "
"${CMAKE_SOURCE_DIR}/bin/app/data ->"
@@ -641,6 +703,9 @@ if(UNIX AND NOT APPLE)
"${CMAKE_SOURCE_DIR}/bin/app/user"
"${CMAKE_BINARY_DIR}/app/user"
)
# Linux package
include(cmake/LinuxPackage.cmake)
endif ()
if (UNIX AND NOT APPLE)
# Linux package
include(cmake/LinuxPackage.cmake)
endif ()
endif ()
bin/app/user/ApplicationSettings_template.xml
+1 -1
View File
@@ -32,7 +32,7 @@
<indexing>
<indexer_thread_count><!-- INTEGER: number of threads indexing the source code --></indexer_thread_count>
<cancel_on_fatal_errors><!-- BOOL: cancel indexing in translation units with fatal errors --></cancel_on_fatal_errors>
<multi_process_indexing><!-- BOOL: use different processes instead of threads during indexing --></multi_process_indexing>
<cxx>
<compiler_flags>
cmake/LinuxPackage.cmake
+6
View File
@@ -158,6 +158,12 @@ INSTALL(TARGETS
COMPONENT FULL
)
INSTALL(TARGETS
${APP_INDEXER_NAME}
DESTINATION Sourcetrail
COMPONENT FULL
)
# SET(CPACK_GENERATOR "DEB;TGZ")
SET(CPACK_GENERATOR "TGZ")
SET(CPACK_ARCHIVE_COMPONENT_INSTALL "ON")
deployment/windows/wixSetup/installDir.wxs
+39 -8
View File
@@ -29,6 +29,10 @@
</ProgId>
</Component>
<Component Id='SourcetrailIndexerExe' Guid='*' Win64="$(var.Win64)">
<File Id='SourcetrailIndexer' Name='sourcetrail_indexer.exe' DiskId='1' Source='$(var.PlatformSourceFolder)/sourcetrail_indexer.exe' KeyPath='yes' />
</Component>
<Component Id='Qt5CoreDll' Guid='*' Win64="$(var.Win64)">
<File Id='qt5coreDLL' Name='Qt5Core.dll' DiskId='1' Source='$(var.PlatformSourceFolder)/Qt5Core.dll' KeyPath='yes' />
</Component>
@@ -323,6 +327,12 @@
<Component Id='BookmarkViewArrowDownPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewArrowDown' Name='arrow_down.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/arrow_down.png' KeyPath='yes' />
</Component>
<Component Id='BookmarkViewArrowLineDownPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewArrowLineDown' Name='arrow_line_down.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/arrow_line_down.png' KeyPath='yes' />
</Component>
<Component Id='BookmarkViewArrowLineUpPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewArrowLineUp' Name='arrow_line_up.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/arrow_line_up.png' KeyPath='yes' />
</Component>
<Component Id='BookmarkViewArrowRightPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewArrowRight' Name='arrow_right.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/arrow_right.png' KeyPath='yes' />
</Component>
@@ -335,12 +345,6 @@
<Component Id='BookmarkViewBookmarkEditIconPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewBookmarkEditIcon' Name='bookmark_edit_icon.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/bookmark_edit_icon.png' KeyPath='yes' />
</Component>
<Component Id='BookmarkViewBookmarkIconPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewBookmarkIcon' Name='bookmark_icon.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/bookmark_icon.png' KeyPath='yes' />
</Component>
<Component Id='BookmarkViewBookmarkIcon2Png' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewBookmarkIcon2' Name='bookmark_icon_2.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/bookmark_icon_2.png' KeyPath='yes' />
</Component>
<Component Id='BookmarkViewBookmarkListIconPng' Guid='*' Win64="$(var.Win64)">
<File Id='BookmarkViewBookmarkListIcon' Name='bookmark_list_icon.png' DiskId='1' Source='./../../../bin/app/data/gui/bookmark_view/images/bookmark_list_icon.png' KeyPath='yes' />
</Component>
@@ -383,8 +387,14 @@
<Component Id='CodeViewMinimizeInactivePng' Guid='*' Win64="$(var.Win64)">
<File Id='CodeViewMinimizeInactive' Name='minimize_inactive.png' DiskId='1' Source='./../../../bin/app/data/gui/code_view/images/minimize_inactive.png' KeyPath='yes' />
</Component>
<Component Id='CodeViewPatternPng' Guid='*' Win64="$(var.Win64)">
<File Id='CodeViewPattern' Name='pattern.png' DiskId='1' Source='./../../../bin/app/data/gui/code_view/images/pattern.png' KeyPath='yes' />
<Component Id='CodeViewPatternBrightPng' Guid='*' Win64="$(var.Win64)">
<File Id='CodeViewPatternBright' Name='pattern_bright.png' DiskId='1' Source='./../../../bin/app/data/gui/code_view/images/pattern_bright.png' KeyPath='yes' />
</Component>
<Component Id='CodeViewPatternDarkPng' Guid='*' Win64="$(var.Win64)">
<File Id='CodeViewPatternDark' Name='pattern_dark.png' DiskId='1' Source='./../../../bin/app/data/gui/code_view/images/pattern_dark.png' KeyPath='yes' />
</Component>
<Component Id='CodeViewPatternGreyPng' Guid='*' Win64="$(var.Win64)">
<File Id='CodeViewPatternGrey' Name='pattern_grey.png' DiskId='1' Source='./../../../bin/app/data/gui/code_view/images/pattern_grey.png' KeyPath='yes' />
</Component>
<Component Id='CodeViewSnippetActivePng' Guid='*' Win64="$(var.Win64)">
<File Id='CodeViewSnippetActive' Name='snippet_active.png' DiskId='1' Source='./../../../bin/app/data/gui/code_view/images/snippet_active.png' KeyPath='yes' />
@@ -414,12 +424,33 @@
<Component Id='BundlePng' Guid='*' Win64="$(var.Win64)">
<File Id='Bundle' Name='bundle.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/bundle.png' KeyPath='yes' />
</Component>
<Component Id='DefaultPng' Guid='*' Win64="$(var.Win64)">
<File Id='Default' Name='default.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/default.png' KeyPath='yes' />
</Component>
<Component Id='EnumPng' Guid='*' Win64="$(var.Win64)">
<File Id='Enum' Name='enum.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/enum.png' KeyPath='yes' />
</Component>
<Component Id='FilePng' Guid='*' Win64="$(var.Win64)">
<File Id='File' Name='file.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/file.png' KeyPath='yes' />
</Component>
<Component Id='GraphPng' Guid='*' Win64="$(var.Win64)">
<File Id='Graph' Name='graph.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/graph.png' KeyPath='yes' />
</Component>
<Component Id='GraphArrowPng' Guid='*' Win64="$(var.Win64)">
<File Id='GraphArrow' Name='graph_arrow.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/graph_arrow.png' KeyPath='yes' />
</Component>
<Component Id='GraphDownPng' Guid='*' Win64="$(var.Win64)">
<File Id='GraphDown' Name='graph_down.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/graph_down.png' KeyPath='yes' />
</Component>
<Component Id='GraphLeftPng' Guid='*' Win64="$(var.Win64)">
<File Id='GraphLeft' Name='graph_left.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/graph_left.png' KeyPath='yes' />
</Component>
<Component Id='GraphRightPng' Guid='*' Win64="$(var.Win64)">
<File Id='GraphRight' Name='graph_right.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/graph_right.png' KeyPath='yes' />
</Component>
<Component Id='GraphUpPng' Guid='*' Win64="$(var.Win64)">
<File Id='GraphUp' Name='graph_up.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/graph_up.png' KeyPath='yes' />
</Component>
<Component Id='MacroPng' Guid='*' Win64="$(var.Win64)">
<File Id='Macro' Name='macro.png' DiskId='1' Source='./../../../bin/app/data/gui/graph_view/images/macro.png' KeyPath='yes' />
</Component>
deployment/windows/wixSetup/sourcetrail.wxs
+15 -4
View File
@@ -166,11 +166,13 @@
<!-- Application Folder Stuff -->
<ComponentRef Id='MainExecutable'/>
<ComponentRef Id='ApplicationShortcut'/>
<ComponentRef Id='DesktopShortcut'/>
<ComponentRef Id='UninstallShortcut'/>
<ComponentRef Id='SourcetrailIndexerExe'/>
<ComponentRef Id='Qt5CoreDll'/>
<ComponentRef Id='Qt5GuiDll'/>
<ComponentRef Id='Qt5NetworkDll'/>
@@ -253,12 +255,12 @@
<ComponentRef Id='BookmarkViewCss'/>
<ComponentRef Id='BookmarkViewArrowDownPng'/>
<ComponentRef Id='BookmarkViewArrowLineDownPng'/>
<ComponentRef Id='BookmarkViewArrowLineUpPng'/>
<ComponentRef Id='BookmarkViewArrowRightPng'/>
<ComponentRef Id='BookmarkViewBookmarkActivePng'/>
<ComponentRef Id='BookmarkViewBookmarkDeleteIconPng'/>
<ComponentRef Id='BookmarkViewBookmarkEditIconPng'/>
<ComponentRef Id='BookmarkViewBookmarkIconPng'/>
<ComponentRef Id='BookmarkViewBookmarkIcon2Png'/>
<ComponentRef Id='BookmarkViewBookmarkListIconPng'/>
<ComponentRef Id='BookmarkViewEditBookmarkIconPng'/>
@@ -272,7 +274,9 @@
<ComponentRef Id='CodeViewMaximizeInactivePng'/>
<ComponentRef Id='CodeViewMinimizeActivePng'/>
<ComponentRef Id='CodeViewMinimizeInactivePng'/>
<ComponentRef Id='CodeViewPatternPng'/>
<ComponentRef Id='CodeViewPatternBrightPng'/>
<ComponentRef Id='CodeViewPatternDarkPng'/>
<ComponentRef Id='CodeViewPatternGreyPng'/>
<ComponentRef Id='CodeViewSnippetActivePng'/>
<ComponentRef Id='CodeViewSnippetInactivePng'/>
@@ -281,8 +285,15 @@
<ComponentRef Id='GraphViewCss'/>
<ComponentRef Id='ArrowGraphPng'/>
<ComponentRef Id='BundlePng'/>
<ComponentRef Id='DefaultPng'/>
<ComponentRef Id='EnumPng'/>
<ComponentRef Id='FilePng'/>
<ComponentRef Id='GraphPng'/>
<ComponentRef Id='GraphArrowPng'/>
<ComponentRef Id='GraphDownPng'/>
<ComponentRef Id='GraphLeftPng'/>
<ComponentRef Id='GraphRightPng'/>
<ComponentRef Id='GraphUpPng'/>
<ComponentRef Id='MacroPng'/>
<ComponentRef Id='NamespacePng'/>
<ComponentRef Id='GraphPatternPng'/>
script/build.sh
+6 -6
View File
@@ -19,28 +19,28 @@ then
if [ "$2" = "test" ]
then
#echo "release test"
cd bin/test && ../../build/Release/test/Sourcetrail_test
cd build/Release/test/ && ./Sourcetrail_test
elif [ "$2" = "keygen" ]
then
#echo "release keygen"
cd bin/license_generator && ../../build/Release/license_generator/Sourcetrail_license_generator
cd build/Release/license_generator && ./Sourcetrail_license_generator
else
#echo "release app"
cd bin/app && ../../build/Release/app/Sourcetrail
cd build/Release/app && ./Sourcetrail
fi
elif [ "$1" = "debug" ] || [ "$1" = "d" ]
then
if [ "$2" = "test" ]
then
#echo "debug test"
cd bin/test && ../../build/Debug/test/Sourcetrail_test
cd build/Debug/test && ./Sourcetrail_test
elif [ "$2" = "keygen" ]
then
#echo "debug keygen"
cd bin/license_generator && ../../build/Debug/license_generator/Sourcetrail_license_generator
cd build/Debug/license_generator && ./Sourcetrail_license_generator
else
#echo "debug app"
cd bin/app && ../../build/Debug/app/Sourcetrail
cd build/Debug/app && ./Sourcetrail
fi
else
echo "no arguments: first argument 'release' or 'debug', second argument 'test' for tests"
setup/MacOSX/bundle_install.sh.in
+10 -2
View File
@@ -11,6 +11,7 @@ echo -e $INFO "Creating app bundle for MacOSX."
BUNDLE_NAME="@MACOSX_BUNDLE_NAME@"
APP_NAME="@MACOSX_BINARY_NAME@"
INDEXER_NAME="@MACOSX_INDEXER_BINARY_NAME@"
DYNLIB_PATHS=(@MACOSX_DYNAMIC_LIBRARIES@)
VERSION_STRING=$(git describe --long)
@@ -49,6 +50,7 @@ PLUGIN_DIR=$BUNDLE_PATH/Contents/PlugIns
FRAMEWORK_DIR=$BUNDLE_PATH/Contents/Frameworks
RES_DIR=$BUNDLE_PATH/Contents/Resources
APP_PATH=$BIN_DIR/$APP_NAME
INDEXER_PATH=$RES_DIR/$INDEXER_NAME
echo -e $INFO "Creating bundle folders."
mkdir -p $BIN_DIR
@@ -76,6 +78,8 @@ cp $APP_NAME $APP_PATH
cp bundle_info.plist $BUNDLE_PATH/Contents/Info.plist
cp $INDEXER_NAME $INDEXER_PATH
mkdir -p $RES_DIR/data
cp -R ../../../bin/app/data/color_schemes $RES_DIR/data/color_schemes
@@ -95,6 +99,7 @@ do
install_name_tool -change $LIB_NAME @executable_path/../lib/$LIB_NAME $APP_PATH
install_name_tool -change @rpath/$LIB_NAME @executable_path/../lib/$LIB_NAME $APP_PATH
install_name_tool -change @rpath/$LIB_NAME @executable_path/../lib/$LIB_NAME $INDEXER_PATH
echo "lib" $LIB_PATH $LIB_NAME
done
@@ -122,6 +127,7 @@ do
FRAMEWORK_SUB_PATH=$(echo $FRAMEWORK_BIN_PATH | grep -o "\w*.framework\S*")
install_name_tool -id @executable_path/../Frameworks/$FRAMEWORK_SUB_PATH $FRAMEWORK_DIR/$FRAMEWORK_SUB_PATH
install_name_tool -change @rpath/$FRAMEWORK_SUB_PATH @executable_path/../Frameworks/$FRAMEWORK_SUB_PATH $APP_PATH
install_name_tool -change @rpath/$FRAMEWORK_SUB_PATH @executable_path/../Frameworks/$FRAMEWORK_SUB_PATH $INDEXER_PATH
echo "framework" $FRAMEWORK_BIN_PATH $FRAMEWORK_PATH
done
@@ -160,10 +166,12 @@ otool -L $FRAMEWORK_DIR/QtNetwork.framework/Versions/5/QtNetwork
otool -L $PLUGIN_DIR/platforms/libqcocoa.dylib
otool -L $PLUGIN_DIR/imageformats/libqgif.dylib
echo -e $INFO "App dependencies"
otool -L $APP_PATH
echo -e $INFO "Indexer dependencies"
otool -L $INDEXER_PATH
echo -e $INFO "Create icon"
ICON=../../../bin/app/data/gui/icon/logo_1024_1024.png
ICON_SET=$RES_DIR/icon.iconset
@@ -207,6 +215,6 @@ iconutil -c icns -o $RES_DIR/project.icns $ICON_SET
# upx --best $APP_PATH
echo -e $INFO "create DMG"
hdiutil create ${PACKAGE_DIR}.dmg -volname ${PACKAGE_DIR} -srcfolder ${PACKAGE_DIR}
# hdiutil create ${PACKAGE_DIR}.dmg -volname ${PACKAGE_DIR} -srcfolder ${PACKAGE_DIR}
echo -e $SUCCESS "Installed bundle successfully!"
src/app/main.cpp
+6 -6
View File
@@ -1,13 +1,11 @@
#include "Application.h"
#include "includes.h" // defines 'void setup(int argc, char *argv[])'
#include "Application.h"
#include "data/indexer/IndexerFactory.h"
#include "data/indexer/IndexerFactoryModuleJava.h"
#include "data/indexer/IndexerFactoryModuleCxxCdb.h"
#include "data/indexer/IndexerFactoryModuleCxxManual.h"
#include "includes.h" // defines 'void setup(int argc, char *argv[])'
#include "LicenseChecker.h"
#include "project/SourceGroupFactory.h"
#include "project/SourceGroupFactoryModuleC.h"
#include "project/SourceGroupFactoryModuleCpp.h"
@@ -31,6 +29,7 @@
#include "utility/ResourcePaths.h"
#include "utility/ScopedFunctor.h"
#include "utility/UserPaths.h"
#include "utility/utilityApp.h"
#include "utility/utilityPathDetection.h"
#include "utility/Version.h"
#include "version.h"
@@ -40,12 +39,13 @@ void setupLogging()
LogManager* logManager = LogManager::getInstance().get();
std::shared_ptr<ConsoleLogger> consoleLogger = std::make_shared<ConsoleLogger>();
consoleLogger->setLogLevel(Logger::LOG_WARNINGS | Logger::LOG_ERRORS);
// consoleLogger->setLogLevel(Logger::LOG_WARNINGS | Logger::LOG_ERRORS);
consoleLogger->setLogLevel(Logger::LOG_ALL);
logManager->addLogger(consoleLogger);
std::shared_ptr<FileLogger> fileLogger = std::make_shared<FileLogger>();
fileLogger->setFileName(LoggerUtility::generateDatedFileName("log"));
fileLogger->setLogDirectory(UserPaths::getLogPath());
fileLogger->setFileName(LoggerUtility::generateDatedFileName("log"));
fileLogger->setLogLevel(Logger::LOG_ALL);
logManager->addLogger(fileLogger);
}
src/indexer/CMakeLists.txt
+5
View File
@@ -0,0 +1,5 @@
add_files(
INDEXER_FILES
main.cpp
)
src/indexer/main.cpp
+88
View File
@@ -0,0 +1,88 @@
#include "includes.h"
#include "utility/AppPath.h"
#include "utility/logging/ConsoleLogger.h"
#include "utility/logging/FileLogger.h"
#include "utility/logging/logging.h"
#include "utility/logging/LogManager.h"
#include "data/indexer/IndexerFactory.h"
#include "data/indexer/IndexerFactoryModuleJava.h"
#include "data/indexer/IndexerFactoryModuleCxxCdb.h"
#include "data/indexer/IndexerFactoryModuleCxxManual.h"
#include "data/indexer/interprocess/InterprocessIndexer.h"
#include "qt/QtCoreApplication.h"
#include "settings/ApplicationSettings.h"
void setupLogging(const std::string logFilePath)
{
LogManager* logManager = LogManager::getInstance().get();
std::shared_ptr<ConsoleLogger> consoleLogger = std::make_shared<ConsoleLogger>();
// consoleLogger->setLogLevel(Logger::LOG_WARNINGS | Logger::LOG_ERRORS);
consoleLogger->setLogLevel(Logger::LOG_ALL);
logManager->addLogger(consoleLogger);
std::shared_ptr<FileLogger> fileLogger = std::make_shared<FileLogger>();
fileLogger->setLogFilePath(FilePath(logFilePath));
fileLogger->setLogLevel(Logger::LOG_ALL);
logManager->addLogger(fileLogger);
}
int main(int argc, char *argv[])
{
QCoreApplication qtApp(argc, argv);
int processId = -1;
std::string instanceUuid;
std::string appPath;
std::string userDataPath;
std::string logFilePath;
if (argc >= 2)
{
processId = std::stoi(argv[1]);
}
if (argc >= 3)
{
instanceUuid = argv[2];
}
if (argc >= 4)
{
appPath = argv[3];
}
if (argc >= 5)
{
userDataPath = argv[4];
}
if (argc >= 6)
{
logFilePath = argv[5];
}
AppPath::setAppPath(appPath);
UserPaths::setUserDataPath(FilePath(userDataPath));
setupLogging(logFilePath);
ApplicationSettings* appSettings = ApplicationSettings::getInstance().get();
appSettings->load(FilePath(UserPaths::getAppSettingsPath()));
LogManager::getInstance()->setLoggingEnabled(appSettings->getLoggingEnabled());
LOG_INFO("appPath: " + appPath);
LOG_INFO("userDataPath: " + userDataPath);
IndexerFactory::getInstance()->addModule(std::make_shared<IndexerFactoryModuleJava>());
IndexerFactory::getInstance()->addModule(std::make_shared<IndexerFactoryModuleCxxCdb>());
IndexerFactory::getInstance()->addModule(std::make_shared<IndexerFactoryModuleCxxManual>());
InterprocessIndexer indexer(instanceUuid, processId);
indexer.work();
// qtApp.quit();
return 0;
}
src/lib/Application.cpp
+19 -2
View File
@@ -1,5 +1,6 @@
#include "Application.h"
#include "utility/interprocess/SharedMemoryGarbageCollector.h"
#include "utility/logging/logging.h"
#include "utility/logging/LogManager.h"
#include "utility/messaging/MessageQueue.h"
@@ -8,6 +9,7 @@
#include "utility/scheduling/TaskScheduler.h"
#include "utility/tracing.h"
#include "utility/UserPaths.h"
#include "utility/UUIDUtility.h"
#include "utility/Version.h"
#include "component/view/DialogView.h"
@@ -22,10 +24,14 @@
#include "settings/ProjectSettings.h"
#include "settings/ColorScheme.h"
std::shared_ptr<Application> Application::s_instance;
std::string Application::s_uuid;
void Application::createInstance(
const Version& version, ViewFactory* viewFactory, NetworkFactory* networkFactory
){
Version::setApplicationVersion(version);
SharedMemoryGarbageCollector::createInstance()->run(Application::getUUID());
loadSettings();
TaskScheduler::getInstance();
@@ -69,6 +75,16 @@ void Application::destroyInstance()
s_instance.reset();
}
std::string Application::getUUID()
{
if (!s_uuid.size())
{
s_uuid = UUIDUtility::UUIDtoString(UUIDUtility::getUUID());
}
return s_uuid;
}
void Application::loadSettings()
{
MessageStatus("Load settings: " + UserPaths::getAppSettingsPath().str()).dispatch();
@@ -87,8 +103,6 @@ void Application::loadStyle(const FilePath& colorSchemePath)
GraphViewStyle::loadStyleSettings();
}
std::shared_ptr<Application> Application::s_instance;
Application::Application(bool withGUI)
: m_hasGUI(withGUI)
, m_isInTrial(true)
@@ -100,10 +114,13 @@ Application::~Application()
{
MessageQueue::getInstance()->stopMessageLoop();
TaskScheduler::getInstance()->stopSchedulerLoop();
if (m_hasGUI)
{
m_mainView->saveLayout();
}
SharedMemoryGarbageCollector::getInstance()->stop();
}
const std::shared_ptr<Project> Application::getCurrentProject()
src/lib/Application.h
+3
View File
@@ -34,6 +34,8 @@ public:
static std::shared_ptr<Application> getInstance();
static void destroyInstance();
static std::string getUUID();
static void loadSettings();
static void loadStyle(const FilePath& colorSchemePath);
@@ -53,6 +55,7 @@ public:
private:
static std::shared_ptr<Application> s_instance;
static std::string s_uuid;
Application(bool withGUI=true);
src/lib/CMakeLists.txt
+21 -12
View File
@@ -138,6 +138,23 @@ add_files(
data/graph/Token.cpp
data/graph/Token.h
data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp
data/indexer/interprocess/shared_types/SharedIndexerCommand.h
data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp
data/indexer/interprocess/shared_types/SharedIntermediateStorage.h
data/indexer/interprocess/shared_types/SharedStorageTypes.h
data/indexer/interprocess/BaseInterprocessDataManager.cpp
data/indexer/interprocess/BaseInterprocessDataManager.h
data/indexer/interprocess/InterprocessIndexer.cpp
data/indexer/interprocess/InterprocessIndexer.h
data/indexer/interprocess/InterprocessIndexerCommandManager.cpp
data/indexer/interprocess/InterprocessIndexerCommandManager.h
data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
data/indexer/interprocess/InterprocessIndexingStatusManager.h
data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp
data/indexer/interprocess/InterprocessIntermediateStorageManager.h
data/indexer/Indexer.h
data/indexer/IndexerBase.cpp
data/indexer/IndexerBase.h
@@ -283,18 +300,10 @@ add_files(
utility/file/FileSystem.cpp
utility/file/FileSystem.h
utility/interprocess/InterprocessDataManager.cpp
utility/interprocess/InterprocessDataManager.h
utility/interprocess/InterprocessProcessManager.cpp
utility/interprocess/InterprocessProcessManager.h
utility/interprocess/InterprocessUtility.h
utility/interprocess/SharedContainer.h
utility/interprocess/SharedMap.h
utility/interprocess/SharedParserArguments.cpp
utility/interprocess/SharedParserArguments.h
utility/interprocess/SharedQueue.h
utility/interprocess/SharedUUIDManager.cpp
utility/interprocess/SharedUUIDManager.h
utility/interprocess/SharedMemory.cpp
utility/interprocess/SharedMemory.h
utility/interprocess/SharedMemoryGarbageCollector.cpp
utility/interprocess/SharedMemoryGarbageCollector.h
utility/logging/ConsoleLogger.cpp
utility/logging/ConsoleLogger.h
src/lib/component/view/DialogView.cpp
+5 -1
View File
@@ -37,7 +37,11 @@ void DialogView::updateIndexingDialog(size_t fileCount, size_t totalFileCount, s
void DialogView::finishedIndexingDialog(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, ErrorCountInfo errorInfo)
float time, ErrorCountInfo errorInfo, bool interrupted)
{
}
void DialogView::hideDialogs()
{
}
src/lib/component/view/DialogView.h
+3 -1
View File
@@ -44,7 +44,9 @@ public:
virtual void updateIndexingDialog(size_t fileCount, size_t totalFileCount, std::string sourcePath);
virtual void finishedIndexingDialog(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, ErrorCountInfo errorInfo);
float time, ErrorCountInfo errorInfo, bool interrupted);
virtual void hideDialogs();
int confirm(const std::string& message);
virtual int confirm(const std::string& message, const std::vector<std::string>& options);
src/lib/data/IntermediateStorage.cpp
+243 -59
View File
@@ -16,16 +16,12 @@ IntermediateStorage::~IntermediateStorage()
void IntermediateStorage::clear()
{
m_nodeNamesToIds.clear();
m_nodeIdsToData.clear();
m_nodes.clear();
m_files.clear();
m_symbols.clear();
m_edgeNamesToIds.clear();
m_edgeIdsToData.clear();
m_localSymbolNamesToIds.clear();
m_localSymbolIdsToData.clear();
m_sourceLocationNamesToIds.clear();
m_sourceLocationIdsToData.clear();
m_edges.clear();
m_localSymbols.clear();
m_sourceLocations.clear();
m_occurrences.clear();
m_componentAccesses.clear();
m_commentLocations.clear();
@@ -33,9 +29,49 @@ void IntermediateStorage::clear()
m_nextId = 1;
}
size_t IntermediateStorage::getByteSize() const
{
unsigned int byteSize = 0;
for (const StorageFile& storageFile: getStorageFiles())
{
byteSize += sizeof(StorageFile);
byteSize += storageFile.filePath.size();
byteSize += storageFile.modificationTime.size();
}
for (const StorageError& storageError: getErrors())
{
byteSize += sizeof(StorageError);
byteSize += storageError.filePath.str().size();
byteSize += storageError.message.size();
}
for (const StorageNode& storageNode: getStorageNodes())
{
byteSize += sizeof(StorageNode);
byteSize += storageNode.serializedName.size();
}
for (const StorageLocalSymbol& storageLocalSymbol: getStorageLocalSymbols())
{
byteSize += sizeof(StorageLocalSymbol);
byteSize += storageLocalSymbol.name.size();
}
byteSize += sizeof(StorageEdge) * getStorageEdges().size();
byteSize += sizeof(StorageCommentLocation) * getCommentLocations().size();
byteSize += sizeof(StorageComponentAccess) * getComponentAccesses().size();
byteSize += sizeof(StorageOccurrence) * getStorageOccurrences().size();
byteSize += sizeof(StorageSymbol) * getStorageSymbols().size();
byteSize += sizeof(StorageSourceLocation) * getStorageSourceLocations().size();
return byteSize;
}
size_t IntermediateStorage::getSourceLocationCount() const
{
return m_sourceLocationNamesToIds.size();
return m_sourceLocations.size();
}
void IntermediateStorage::setAllFilesIncomplete()
@@ -65,25 +101,23 @@ void IntermediateStorage::setFilesWithErrorsIncomplete()
Id IntermediateStorage::addNode(int type, const std::string& serializedName)
{
std::shared_ptr<StorageNode> node = std::make_shared<StorageNode>(0, type, serializedName);
StorageNode node(0, type, serializedName);
const std::string serialized = serialize(*(node.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_nodeNamesToIds.find(serialized);
if (it != m_nodeNamesToIds.end())
const std::string serialized = serialize(node);
std::unordered_map<std::string, StorageNode>::iterator it = m_nodes.find(serialized);
if (it != m_nodes.end())
{
std::map<Id, std::shared_ptr<StorageNode>>::const_iterator it2 = m_nodeIdsToData.find(it->second);
std::shared_ptr<StorageNode> storedNode = it2->second;
if (storedNode->type < type)
StorageNode& storedNode = it->second;
if (storedNode.type < type)
{
storedNode->type = type;
storedNode.type = type;
}
return it->second;
return storedNode.id;
}
const Id id = m_nextId++;
m_nodeNamesToIds[serialized] = id;
m_nodeIdsToData[id] = node;
node.id = id;
m_nodes[serialized] = node;
return id;
}
@@ -107,43 +141,41 @@ void IntermediateStorage::addSymbol(const Id id, int definitionKind)
Id IntermediateStorage::addEdge(int type, Id sourceId, Id targetId)
{
std::shared_ptr<StorageEdge> edge = std::make_shared<StorageEdge>(0, type, sourceId, targetId);
StorageEdge edge = StorageEdge(0, type, sourceId, targetId);
const std::string serialized = serialize(*(edge.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_edgeNamesToIds.find(serialized);
if (it != m_edgeNamesToIds.end())
const std::string serialized = serialize(edge);
std::unordered_map<std::string, StorageEdge>::const_iterator it = m_edges.find(serialized);
if (it != m_edges.end())
{
return it->second;
return it->second.id;
}
const Id id = m_nextId++;
m_edgeNamesToIds[serialized] = id;
m_edgeIdsToData[id] = edge;
edge.id = id;
m_edges[serialized] = edge;
return id;
}
Id IntermediateStorage::addLocalSymbol(const std::string& name)
{
std::shared_ptr<StorageLocalSymbol> localSymbol = std::make_shared<StorageLocalSymbol>(0, name);
StorageLocalSymbol localSymbol = StorageLocalSymbol(0, name);
const std::string serialized = serialize(*(localSymbol.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_localSymbolNamesToIds.find(serialized);
if (it != m_localSymbolNamesToIds.end())
const std::string serialized = serialize(localSymbol);
std::unordered_map<std::string, StorageLocalSymbol>::const_iterator it = m_localSymbols.find(serialized);
if (it != m_localSymbols.end())
{
return it->second;
return it->second.id;
}
const Id id = m_nextId++;
m_localSymbolNamesToIds[serialized] = id;
m_localSymbolIdsToData[id] = localSymbol;
localSymbol.id = id;
m_localSymbols[serialized] = localSymbol;
return id;
}
Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol, int type)
{
std::shared_ptr<StorageSourceLocation> sourceLocation = std::make_shared<StorageSourceLocation>(
StorageSourceLocation sourceLocation = StorageSourceLocation(
0,
fileNodeId,
startLine,
@@ -153,17 +185,16 @@ Id IntermediateStorage::addSourceLocation(Id fileNodeId, uint startLine, uint st
type
);
const std::string serialized = serialize(*(sourceLocation.get()));
std::unordered_map<std::string, Id>::const_iterator it = m_sourceLocationNamesToIds.find(serialized);
if (it != m_sourceLocationNamesToIds.end())
const std::string serialized = serialize(sourceLocation);
std::unordered_map<std::string, StorageSourceLocation>::const_iterator it = m_sourceLocations.find(serialized);
if (it != m_sourceLocations.end())
{
return it->second;
return it->second.id;
}
const Id id = m_nextId++;
m_sourceLocationNamesToIds[serialized] = id;
m_sourceLocationIdsToData[id] = sourceLocation;
sourceLocation.id = id;
m_sourceLocations[serialized] = sourceLocation;
return id;
}
@@ -211,7 +242,8 @@ void IntermediateStorage::addCommentLocation(Id fileNodeId, uint startLine, uint
}
}
void IntermediateStorage::addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed)
void IntermediateStorage::addError(
const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed)
{
const StorageError error(
0,
@@ -231,11 +263,11 @@ void IntermediateStorage::addError(const std::string& message, const FilePath& f
}
}
void IntermediateStorage::forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const
void IntermediateStorage::forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const
{
for (std::map<Id, std::shared_ptr<StorageNode>>::const_iterator it = m_nodeIdsToData.begin(); it != m_nodeIdsToData.end(); it++)
for (std::unordered_map<std::string, StorageNode>::const_iterator it = m_nodes.begin(); it != m_nodes.end(); it++)
{
callback(it->first, *(it->second.get()));
callback(it->second);
}
}
@@ -255,27 +287,29 @@ void IntermediateStorage::forEachSymbol(std::function<void(const StorageSymbol&
}
}
void IntermediateStorage::forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const
void IntermediateStorage::forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const
{
for (std::map<Id, std::shared_ptr<StorageEdge>>::const_iterator it = m_edgeIdsToData.begin(); it != m_edgeIdsToData.end(); it++)
for (std::unordered_map<std::string, StorageEdge>::const_iterator it = m_edges.begin(); it != m_edges.end(); it++)
{
callback(it->first, *(it->second.get()));
callback(it->second);
}
}
void IntermediateStorage::forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const
void IntermediateStorage::forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const
{
for (std::map<Id, std::shared_ptr<StorageLocalSymbol>>::const_iterator it = m_localSymbolIdsToData.begin(); it != m_localSymbolIdsToData.end(); it++)
for (std::unordered_map<std::string, StorageLocalSymbol>::const_iterator it = m_localSymbols.begin();
it != m_localSymbols.end(); it++)
{
callback(it->first, *(it->second.get()));
callback(it->second);
}
}
void IntermediateStorage::forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const
void IntermediateStorage::forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const
{
for (std::map<Id, std::shared_ptr<StorageSourceLocation>>::const_iterator it = m_sourceLocationIdsToData.begin(); it != m_sourceLocationIdsToData.end(); it++)
for (std::unordered_map<std::string, StorageSourceLocation>::const_iterator it = m_sourceLocations.begin();
it != m_sourceLocations.end(); it++)
{
callback(it->first, *(it->second.get()));
callback(it->second);
}
}
@@ -311,6 +345,156 @@ void IntermediateStorage::forEachError(std::function<void(const StorageError& /*
}
}
std::vector<StorageNode> IntermediateStorage::getStorageNodes() const
{
std::vector<StorageNode> nodes;
nodes.reserve(m_nodes.size());
for (auto it: m_nodes)
{
nodes.push_back(it.second);
}
return nodes;
}
std::vector<StorageFile> IntermediateStorage::getStorageFiles() const
{
return m_files;
}
std::vector<StorageSymbol> IntermediateStorage::getStorageSymbols() const
{
return m_symbols;
}
std::vector<StorageEdge> IntermediateStorage::getStorageEdges() const
{
std::vector<StorageEdge> edges;
edges.reserve(m_edges.size());
for (auto it: m_edges)
{
edges.push_back(it.second);
}
return edges;
}
std::vector<StorageLocalSymbol> IntermediateStorage::getStorageLocalSymbols() const
{
std::vector<StorageLocalSymbol> localSymbol;
localSymbol.reserve(m_localSymbols.size());
for (auto it: m_localSymbols)
{
localSymbol.push_back(it.second);
}
return localSymbol;
}
std::vector<StorageSourceLocation> IntermediateStorage::getStorageSourceLocations() const
{
std::vector<StorageSourceLocation> sourceLocations;
sourceLocations.reserve(m_sourceLocations.size());
for (auto it: m_sourceLocations)
{
sourceLocations.push_back(it.second);
}
return sourceLocations;
}
std::vector<StorageOccurrence> IntermediateStorage::getStorageOccurrences() const
{
return m_occurrences;
}
std::vector<StorageComponentAccess> IntermediateStorage::getComponentAccesses() const
{
return m_componentAccesses;
}
std::vector<StorageCommentLocation> IntermediateStorage::getCommentLocations() const
{
return m_commentLocations;
}
std::vector<StorageError> IntermediateStorage::getErrors() const
{
return m_errors;
}
void IntermediateStorage::setStorageNodes(const std::vector<StorageNode>& storageNodes)
{
m_nodes.clear();
for (const StorageNode& storageNode: storageNodes)
{
m_nodes[serialize(storageNode)] = storageNode;
}
}
void IntermediateStorage::setStorageFiles(const std::vector<StorageFile>& storageFiles)
{
m_files = storageFiles;
}
void IntermediateStorage::setStorageSymbols(const std::vector<StorageSymbol>& storageSymbols)
{
m_symbols = storageSymbols;
}
void IntermediateStorage::setStorageEdges(const std::vector<StorageEdge>& storageEdges)
{
m_edges.clear();
for (const StorageEdge& storageEdge: storageEdges)
{
m_edges[serialize(storageEdge)] = storageEdge;
}
}
void IntermediateStorage::setStorageLocalSymbols(const std::vector<StorageLocalSymbol>& storageLocalSymbols)
{
m_localSymbols.clear();
for (const StorageLocalSymbol& storageLocalSymbol: storageLocalSymbols)
{
m_localSymbols[serialize(storageLocalSymbol)] = storageLocalSymbol;
}
}
void IntermediateStorage::setStorageSourceLocations(const std::vector<StorageSourceLocation>& storageSourceLocations)
{
m_sourceLocations.clear();
for (const StorageSourceLocation& storageSourceLocation: storageSourceLocations)
{
m_sourceLocations[serialize(storageSourceLocation)] = storageSourceLocation;
}
}
void IntermediateStorage::setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurrences)
{
m_occurrences = storageOccurrences;
}
void IntermediateStorage::setComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses)
{
m_componentAccesses = componentAccesses;
}
void IntermediateStorage::setCommentLocations(const std::vector<StorageCommentLocation>& commentLocations)
{
m_commentLocations = commentLocations;
}
void IntermediateStorage::setErrors(const std::vector<StorageError>& errors)
{
m_errors = errors;
}
Id IntermediateStorage::getNextId() const
{
return m_nextId;
}
void IntermediateStorage::setNextId(const Id nextId)
{
m_nextId = nextId;
}
std::string IntermediateStorage::serialize(const StorageNode& node) const
{
return node.serializedName;
@@ -327,7 +511,7 @@ std::string IntermediateStorage::serialize(const StorageEdge& edge) const
std::to_string(edge.type) + ";" +
std::to_string(edge.sourceNodeId) + ";" +
std::to_string(edge.targetNodeId)
);
);
}
std::string IntermediateStorage::serialize(const StorageLocalSymbol& localSymbol) const
src/lib/data/IntermediateStorage.h
+38 -12
View File
@@ -16,6 +16,8 @@ public:
virtual ~IntermediateStorage();
void clear();
size_t getByteSize() const;
size_t getSourceLocationCount() const;
void setAllFilesIncomplete();
@@ -32,17 +34,45 @@ public:
virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol);
virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed);
virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const;
virtual void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const;
virtual void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const;
virtual void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const;
virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const;
virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const;
virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const;
virtual void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const;
virtual void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const;
virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const;
virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const;
virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const;
virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const;
virtual void forEachError(std::function<void(const StorageError& /*data*/)> callback) const;
// for conversion to and from 'SharedIntermediateStorage'
std::vector<StorageNode> getStorageNodes() const;
std::vector<StorageFile> getStorageFiles() const;
std::vector<StorageSymbol> getStorageSymbols() const;
std::vector<StorageEdge> getStorageEdges() const;
std::vector<StorageLocalSymbol> getStorageLocalSymbols() const;
std::vector<StorageSourceLocation> getStorageSourceLocations() const;
std::vector<StorageOccurrence> getStorageOccurrences() const;
std::vector<StorageComponentAccess> getComponentAccesses() const;
std::vector<StorageCommentLocation> getCommentLocations() const;
std::vector<StorageError> getErrors() const;
void setStorageNodes(const std::vector<StorageNode>& storageNodes);
void setStorageFiles(const std::vector<StorageFile>& storageFiles);
void setStorageSymbols(const std::vector<StorageSymbol>& storageSymbols);
void setStorageEdges(const std::vector<StorageEdge>& storageEdges);
void setStorageLocalSymbols(const std::vector<StorageLocalSymbol>& storageLocalSymbols);
void setStorageSourceLocations(const std::vector<StorageSourceLocation>& storageSourceLocations);
void setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurrences);
void setComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses);
void setCommentLocations(const std::vector<StorageCommentLocation>& commentLocations);
void setErrors(const std::vector<StorageError>& errors);
Id getNextId() const;
void setNextId(const Id nextId);
private:
std::string serialize(const StorageNode& node) const;
std::string serialize(const StorageFile& file) const;
@@ -54,22 +84,18 @@ private:
std::string serialize(const StorageCommentLocation& commentLocation) const;
std::string serialize(const StorageError& error) const;
std::unordered_map<std::string, Id> m_nodeNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageNode>> m_nodeIdsToData;
std::unordered_map<std::string, StorageNode> m_nodes;
std::unordered_set<std::string> m_serializedFiles; // this is used to prevent duplicates (unique)
std::vector<StorageFile> m_files;
std::vector<StorageSymbol> m_symbols;
std::unordered_map<std::string, Id> m_edgeNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageEdge>> m_edgeIdsToData;
std::unordered_map<std::string, StorageEdge> m_edges;
std::unordered_map<std::string, Id> m_localSymbolNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageLocalSymbol>> m_localSymbolIdsToData;
std::unordered_map<std::string, StorageLocalSymbol> m_localSymbols;
std::unordered_map<std::string, Id> m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageSourceLocation>> m_sourceLocationIdsToData;
std::unordered_map<std::string, StorageSourceLocation> m_sourceLocations;
std::unordered_set<std::string> m_serializedOccurrences; // this is used to prevent duplicates (unique)
std::vector<StorageOccurrence> m_occurrences;
src/lib/data/PersistentStorage.cpp
+8 -10
View File
@@ -324,11 +324,11 @@ std::vector<BookmarkCategory> PersistentStorage::getAllBookmarkCategories() cons
return categories;
}
void PersistentStorage::forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const
void PersistentStorage::forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const
{
for (StorageNode& node: m_sqliteIndexStorage.getAll<StorageNode>())
{
callback(node.id, node);
callback(node);
}
}
@@ -348,29 +348,27 @@ void PersistentStorage::forEachSymbol(std::function<void(const StorageSymbol& /*
}
}
void PersistentStorage::forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const
void PersistentStorage::forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const
{
for (StorageEdge& edge: m_sqliteIndexStorage.getAll<StorageEdge>())
{
callback(edge.id, edge);
callback(edge);
}
}
void PersistentStorage::forEachLocalSymbol(std::function<void(
const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const
void PersistentStorage::forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const
{
for (StorageLocalSymbol& localSymbol: m_sqliteIndexStorage.getAll<StorageLocalSymbol>())
{
callback(localSymbol.id, localSymbol);
callback(localSymbol);
}
}
void PersistentStorage::forEachSourceLocation(
std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const
void PersistentStorage::forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const
{
for (StorageSourceLocation& sourceLocation: m_sqliteIndexStorage.getAll<StorageSourceLocation>())
{
callback(sourceLocation.id, sourceLocation);
callback(sourceLocation);
}
}
src/lib/data/PersistentStorage.h
+4 -4
View File
@@ -50,12 +50,12 @@ public:
virtual std::vector<BookmarkCategory> getAllBookmarkCategories() const;
virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const;
virtual void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const;
virtual void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const;
virtual void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const;
virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const;
virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const;
virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const;
virtual void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const;
virtual void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const;
virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const;
virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const;
virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const;
virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const;
src/lib/data/Storage.cpp
+8 -8
View File
@@ -25,12 +25,12 @@ void Storage::inject(Storage* injected)
std::unordered_map<Id, Id> injectedIdToOwnId;
injected->forEachNode(
[&](Id injectedId, const StorageNode& injectedData)
[&](const StorageNode& injectedData)
{
const Id ownId = addNode(injectedData.type, injectedData.serializedName);
if (ownId != 0)
{
injectedIdToOwnId[injectedId] = ownId;
injectedIdToOwnId[injectedData.id] = ownId;
}
}
);
@@ -66,7 +66,7 @@ void Storage::inject(Storage* injected)
);
injected->forEachEdge(
[&](Id injectedId, const StorageEdge& injectedData)
[&](const StorageEdge& injectedData)
{
std::unordered_map<Id, Id>::const_iterator it;
it = injectedIdToOwnId.find(injectedData.sourceNodeId);
@@ -87,24 +87,24 @@ void Storage::inject(Storage* injected)
if (ownId != 0)
{
injectedIdToOwnId[injectedId] = ownId;
injectedIdToOwnId[injectedData.id] = ownId;
}
}
);
injected->forEachLocalSymbol(
[&](const Id injectedId, const StorageLocalSymbol& injectedData)
[&](const StorageLocalSymbol& injectedData)
{
const Id ownId = addLocalSymbol(injectedData.name);
if (ownId != 0)
{
injectedIdToOwnId[injectedId] = ownId;
injectedIdToOwnId[injectedData.id] = ownId;
}
}
);
injected->forEachSourceLocation(
[&](const Id injectedId, const StorageSourceLocation& injectedData)
[&](const StorageSourceLocation& injectedData)
{
std::unordered_map<Id, Id>::const_iterator it;
it = injectedIdToOwnId.find(injectedData.fileNodeId);
@@ -124,7 +124,7 @@ void Storage::inject(Storage* injected)
);
if (ownId != 0)
{
injectedIdToOwnId[injectedId] = ownId;
injectedIdToOwnId[injectedData.id] = ownId;
}
}
);
src/lib/data/Storage.h
+4 -4
View File
@@ -26,12 +26,12 @@ public:
virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol) = 0;
virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed) = 0;
virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const = 0;
virtual void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const = 0;
virtual void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const = 0;
virtual void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const = 0;
virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const = 0;
virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const = 0;
virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const = 0;
virtual void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const = 0;
virtual void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const = 0;
virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const = 0;
virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const = 0;
virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const = 0;
virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const = 0;
src/lib/data/StorageTypes.h
-3
View File
@@ -238,9 +238,6 @@ struct StorageError
};
struct StorageBookmarkCategory
{
StorageBookmarkCategory()
src/lib/data/TaskFinishParsing.cpp
+18 -1
View File
@@ -4,6 +4,7 @@
#include "data/PersistentStorage.h"
#include "utility/file/FileRegister.h"
#include "utility/messaging/type/MessageFinishedParsing.h"
#include "utility/messaging/type/MessageStatus.h"
#include "utility/scheduling/Blackboard.h"
#include "utility/utility.h"
#include "Application.h"
@@ -63,6 +64,9 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboa
int sourceFileCount = 0;
blackboard->get("source_file_count", sourceFileCount);
bool interruptedIndexing = false;
blackboard->get("interrupted_indexing", interruptedIndexing);
StorageStats stats = m_storageAccess->getStorageStats();
dialogView->finishedIndexingDialog(
indexedSourceFileCount,
@@ -70,7 +74,8 @@ Task::TaskState TaskFinishParsing::doUpdate(std::shared_ptr<Blackboard> blackboa
stats.completedFileCount,
stats.fileCount,
time,
m_storageAccess->getErrorCount()
m_storageAccess->getErrorCount(),
interruptedIndexing
);
return STATE_SUCCESS;
@@ -83,3 +88,15 @@ void TaskFinishParsing::doExit(std::shared_ptr<Blackboard> blackboard)
void TaskFinishParsing::doReset(std::shared_ptr<Blackboard> blackboard)
{
}
void TaskFinishParsing::terminate()
{
Application* app = Application::getInstance().get();
if (app)
{
app->getDialogView()->hideDialogs();
}
MessageStatus("An unknown exception was thrown during indexing.", true, false).dispatch();
MessageFinishedParsing().dispatch();
}
src/lib/data/TaskFinishParsing.h
+1
View File
@@ -26,6 +26,7 @@ private:
virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
virtual void doExit(std::shared_ptr<Blackboard> blackboard);
virtual void doReset(std::shared_ptr<Blackboard> blackboard);
virtual void terminate();
PersistentStorage* m_storage;
StorageAccess* m_storageAccess;
src/lib/data/indexer/Indexer.h
+2 -2
View File
@@ -18,7 +18,8 @@ public:
virtual std::string getKindString() const;
virtual std::shared_ptr<IntermediateStorage> index(
std::shared_ptr<IndexerCommand> indexerCommand, std::shared_ptr<FileRegister> fileRegister);
std::shared_ptr<IndexerCommand> indexerCommand,
std::shared_ptr<FileRegister> fileRegister);
};
template <typename IndexerCommandType, typename ParserType>
@@ -46,7 +47,6 @@ std::shared_ptr<IntermediateStorage> Indexer<IndexerCommandType, ParserType>::in
}
std::shared_ptr<ParserClientImpl> parserClient = std::make_shared<ParserClientImpl>();
parserClient->setCancelOnFatalErrors(indexerCommand->cancelOnFatalErrors());
std::shared_ptr<ParserType> parser = std::make_shared<ParserType>(parserClient, fileRegister);
src/lib/data/indexer/IndexerBase.h
+3 -1
View File
@@ -16,7 +16,9 @@ public:
virtual std::string getKindString() const = 0;
virtual std::shared_ptr<IntermediateStorage> index(std::shared_ptr<IndexerCommand> indexerCommand, std::shared_ptr<FileRegister> fileRegister) = 0;
virtual std::shared_ptr<IntermediateStorage> index(
std::shared_ptr<IndexerCommand> indexerCommand,
std::shared_ptr<FileRegister> fileRegister) = 0;
virtual void interrupt();
src/lib/data/indexer/IndexerCommand.cpp
+20 -12
View File
@@ -1,10 +1,11 @@
#include "data/indexer/IndexerCommand.h"
IndexerCommand::IndexerCommand(const FilePath& sourceFilePath, const std::set<FilePath>& indexedPaths, const std::set<FilePath>& excludedPaths)
IndexerCommand::IndexerCommand(
const FilePath& sourceFilePath, const std::set<FilePath>& indexedPaths, const std::set<FilePath>& excludedPaths
)
: m_sourceFilePath(sourceFilePath)
, m_indexedPaths(indexedPaths)
, m_excludedPaths(excludedPaths)
, m_cancelOnFatalErrors(false)
{
}
@@ -12,6 +13,23 @@ IndexerCommand::~IndexerCommand()
{
}
size_t IndexerCommand::getByteSize() const
{
size_t size = m_sourceFilePath.str().size();
for (auto i : m_indexedPaths)
{
size += i.str().size();
}
for (auto i : m_excludedPaths)
{
size += i.str().size();
}
return size;
}
FilePath IndexerCommand::getSourceFilePath() const
{
return m_sourceFilePath;
@@ -26,13 +44,3 @@ std::set<FilePath> IndexerCommand::getExcludedPath() const
{
return m_excludedPaths;
}
bool IndexerCommand::cancelOnFatalErrors() const
{
return m_cancelOnFatalErrors;
}
void IndexerCommand::setCancelOnFatalErrors(bool cancelOnFatalErrors)
{
m_cancelOnFatalErrors = cancelOnFatalErrors;
}
src/lib/data/indexer/IndexerCommand.h
+1 -5
View File
@@ -13,14 +13,12 @@ public:
virtual ~IndexerCommand();
virtual std::string getKindString() const = 0;
virtual size_t getByteSize() const;
FilePath getSourceFilePath() const;
std::set<FilePath> getIndexedPaths() const;
std::set<FilePath> getExcludedPath() const;
bool cancelOnFatalErrors() const;
void setCancelOnFatalErrors(bool cancelOnFatalErrors);
virtual bool preprocessorOnly() const = 0;
virtual void setPreprocessorOnly(bool preprocessorOnly) = 0;
@@ -28,8 +26,6 @@ private:
FilePath m_sourceFilePath;
std::set<FilePath> m_indexedPaths;
std::set<FilePath> m_excludedPaths;
bool m_cancelOnFatalErrors;
};
#endif // INDEXER_COMMAND_H
src/lib/data/indexer/IndexerCommandList.cpp
+6
View File
@@ -32,3 +32,9 @@ std::shared_ptr<IndexerCommand> IndexerCommandList::consumeCommand()
}
return ret;
}
std::vector<std::shared_ptr<IndexerCommand>> IndexerCommandList::getAllCommands()
{
std::lock_guard<std::mutex> lock(m_commandsMutex);
return std::vector<std::shared_ptr<IndexerCommand>>(m_commands.begin(), m_commands.end());
}
src/lib/data/indexer/IndexerCommandList.h
+2
View File
@@ -18,6 +18,8 @@ public:
std::shared_ptr<IndexerCommand> consumeCommand();
std::vector<std::shared_ptr<IndexerCommand>> getAllCommands();
private:
std::deque<std::shared_ptr<IndexerCommand>> m_commands;
std::mutex m_commandsMutex;
src/lib/data/indexer/IndexerComposite.cpp
+2 -1
View File
@@ -15,7 +15,8 @@ void IndexerComposite::addIndexer(std::shared_ptr<IndexerBase> indexer)
m_indexers.emplace(indexer->getKindString(), indexer);
}
std::shared_ptr<IntermediateStorage> IndexerComposite::index(std::shared_ptr<IndexerCommand> indexerCommand, std::shared_ptr<FileRegister> fileRegister)
std::shared_ptr<IntermediateStorage> IndexerComposite::index(
std::shared_ptr<IndexerCommand> indexerCommand, std::shared_ptr<FileRegister> fileRegister)
{
auto it = m_indexers.find(indexerCommand->getKindString());
if (it != m_indexers.end())
src/lib/data/indexer/IndexerComposite.h
+4 -1
View File
@@ -15,7 +15,10 @@ public:
void addIndexer(std::shared_ptr<IndexerBase> indexer);
virtual std::shared_ptr<IntermediateStorage> index(std::shared_ptr<IndexerCommand> indexerCommand, std::shared_ptr<FileRegister> fileRegister);
virtual std::shared_ptr<IntermediateStorage> index(
std::shared_ptr<IndexerCommand> indexerCommand,
std::shared_ptr<FileRegister> fileRegister
);
virtual void interrupt();
src/lib/data/indexer/TaskBuildIndex.cpp
+188 -57
View File
@@ -1,16 +1,30 @@
#include "data/indexer/TaskBuildIndex.h"
#include "utility/AppPath.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/logging/FileLogger.h"
#include "utility/scheduling/Blackboard.h"
#include "utility/UserPaths.h"
#include "utility/utilityApp.h"
#include "Application.h"
#include "component/view/DialogView.h"
#include "data/indexer/IndexerFactory.h"
#include "data/indexer/IndexerCommandList.h"
#include "data/indexer/IndexerComposite.h"
#include "data/indexer/interprocess/InterprocessIndexer.h"
#include "data/StorageProvider.h"
#include "component/view/DialogView.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/scheduling/Blackboard.h"
#include "Application.h"
#include "settings/ApplicationSettings.h"
#if _WIN32
const std::string TaskBuildIndex::s_processName("sourcetrail_indexer.exe");
#else
const std::string TaskBuildIndex::s_processName("sourcetrail_indexer");
#endif
TaskBuildIndex::TaskBuildIndex(
unsigned int processCount,
std::shared_ptr<IndexerCommandList> indexerCommandList,
std::shared_ptr<StorageProvider> storageProvider,
std::shared_ptr<FileRegisterStateData> fileRegisterStateData
@@ -18,89 +32,206 @@ TaskBuildIndex::TaskBuildIndex(
: m_indexerCommandList(indexerCommandList)
, m_storageProvider(storageProvider)
, m_fileRegisterStateData(fileRegisterStateData)
, m_interprocessIndexerCommandManager(Application::getUUID(), 0, true)
, m_interprocessIndexingStatusManager(Application::getUUID(), 0, true)
, m_processCount(processCount)
, m_interrupted(false)
, m_lastCommandCount(0)
{
m_indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules();
}
void TaskBuildIndex::doEnter(std::shared_ptr<Blackboard> blackboard)
{
std::lock_guard<std::mutex> lock(blackboard->getMutex());
int indexerCount = 0;
if (blackboard->get("indexer_count", indexerCount))
{
indexerCount++;
blackboard->set("indexer_count", indexerCount);
std::lock_guard<std::mutex> lock(blackboard->getMutex());
blackboard->set("indexer_count", (int)m_processCount);
}
// move indexer commands to shared memory
m_lastCommandCount = m_indexerCommandList->size();
m_interprocessIndexerCommandManager.setIndexerCommands(m_indexerCommandList->getAllCommands());
std::string logFilePath;
Logger* logger = LogManager::getInstance()->getLoggerByType("FileLogger");
if (logger)
{
logFilePath = dynamic_cast<FileLogger*>(logger)->getLogFilePath().str();
}
bool multiProcess = ApplicationSettings::getInstance()->getMultiProcessIndexingEnabled();
// start indexer processes
for (unsigned int i = 0; i < m_processCount; i++)
{
const int processId = i + 1; // 0 remains reserved for the main process
m_interprocessIntermediateStorageManagers.push_back(
std::make_shared<InterprocessIntermediateStorageManager>(Application::getUUID(), processId, true)
);
if (multiProcess)
{
m_processThreads.push_back(new std::thread(&TaskBuildIndex::runIndexerProcess, this, processId, logFilePath));
}
else
{
m_processThreads.push_back(new std::thread(&TaskBuildIndex::runIndexerThread, this, processId));
}
}
}
Task::TaskState TaskBuildIndex::doUpdate(std::shared_ptr<Blackboard> blackboard)
{
std::shared_ptr<IndexerCommand> indexerCommand = m_indexerCommandList->consumeCommand();
size_t commandCount = m_interprocessIndexerCommandManager.indexerCommandCount();
if (commandCount != m_lastCommandCount)
{
updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath());
m_lastCommandCount = commandCount;
}
if (!indexerCommand)
if (commandCount == 0)
{
return STATE_FAILURE;
}
else
else if (m_interrupted)
{
{
std::lock_guard<std::mutex> lock(blackboard->getMutex());
std::lock_guard<std::mutex> lock(blackboard->getMutex());
blackboard->set("interrupted_indexing", true);
int sourceFileCount = 0;
blackboard->get("source_file_count", sourceFileCount);
int indexedSourceFileCount = 0;
blackboard->get("indexed_source_file_count", indexedSourceFileCount);
if (std::shared_ptr<DialogView> dialogView = Application::getInstance()->getDialogView())
{
dialogView->updateIndexingDialog(
indexedSourceFileCount, sourceFileCount, indexerCommand->getSourceFilePath().str()
);
}
}
// file register only copies the DileRegisterStateData
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
*(m_fileRegisterStateData.get()), indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath()
);
std::shared_ptr<IntermediateStorage> storage = m_indexer->index(indexerCommand, fileRegister);
if (storage)
{
// only write back the changes made to FileRegisterStateData if the indexer actually succeeded
m_fileRegisterStateData->inject(fileRegister->getStateData());
m_storageProvider->insert(storage);
std::lock_guard<std::mutex> lock(blackboard->getMutex());
int indexedSourceFileCount = 0;
blackboard->get("indexed_source_file_count", indexedSourceFileCount);
blackboard->set("indexed_source_file_count", indexedSourceFileCount + 1);
}
// clear indexer commands, this causes the indexer processes to return when finished with respective current indexer commands
m_interprocessIndexerCommandManager.clearIndexerCommands();
return STATE_FAILURE;
}
return (m_indexer->interrupted() ? STATE_FAILURE : STATE_SUCCESS);
fetchIntermediateStorages(blackboard);
const int SLEEP_TIME_MS = 100;
std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME_MS));
return STATE_RUNNING;
}
void TaskBuildIndex::doExit(std::shared_ptr<Blackboard> blackboard)
{
std::lock_guard<std::mutex> lock(blackboard->getMutex());
int indexerCount = 0;
if (blackboard->get("indexer_count", indexerCount))
for (auto processThread : m_processThreads)
{
indexerCount--;
blackboard->set("indexer_count", indexerCount);
processThread->join();
delete processThread;
}
m_processThreads.clear();
fetchIntermediateStorages(blackboard);
std::vector<FilePath> crashedFiles = m_interprocessIndexingStatusManager.getCrashedSourceFilePaths();
if (crashedFiles.size())
{
std::shared_ptr<IntermediateStorage> is = std::make_shared<IntermediateStorage>();
for (auto path : crashedFiles)
{
is->addError("The translation unit threw an exception during indexing. Please check if the source file "
"conforms to the specified language standard and all necessary options are defined within your project "
"setup.", path, 1, 1, true, true);
LOG_INFO_STREAM(<< "crashed translation unit: " << path.str());
}
m_storageProvider->insert(is);
}
std::lock_guard<std::mutex> lock(blackboard->getMutex());
blackboard->set("indexer_count", 0);
}
void TaskBuildIndex::doReset(std::shared_ptr<Blackboard> blackboard)
{
}
void TaskBuildIndex::terminate()
{
m_interrupted = true;
utility::killRunningProcesses();
}
void TaskBuildIndex::handleMessage(MessageInterruptTasks* message)
{
m_indexer->interrupt();
m_interrupted = true;
}
void TaskBuildIndex::runIndexerProcess(int processId, const std::string& logFilePath)
{
FilePath indexerProcessPath(AppPath::getAppPath() + s_processName);
if (!indexerProcessPath.exists())
{
m_interrupted = true;
LOG_ERROR("Cannot start indexer process because executable is missing at \"" + indexerProcessPath.str() + "\"");
return;
}
std::string command = indexerProcessPath.str();
command += " " + std::to_string(processId);
command += " " + Application::getUUID();
command += " \"" + AppPath::getAppPath() + "\"";
command += " \"" + UserPaths::getUserDataPath().str() + "\"";
if (logFilePath.size())
{
command += " \"" + logFilePath + "\"";
}
int result = 1;
while (result != 0 && !m_interrupted)
{
result = utility::executeProcessAndGetExitCode(command.c_str(), "", -1);
LOG_INFO_STREAM(<< "Indexer process " << processId << " returned with " + std::to_string(result));
}
}
void TaskBuildIndex::runIndexerThread(int processId)
{
InterprocessIndexer indexer(Application::getUUID(), processId);
indexer.work();
}
void TaskBuildIndex::fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard)
{
int newlyIndexedCount = 0;
for (std::shared_ptr<InterprocessIntermediateStorageManager> storageManager: m_interprocessIntermediateStorageManagers)
{
while (int storageCount = storageManager->getIntermediateStorageCount())
{
LOG_INFO_STREAM(<< storageManager->getProcessId() << " - storage count: " << storageCount);
m_storageProvider->insert(storageManager->popIntermediateStorage());
++newlyIndexedCount;
updateIndexingDialog(blackboard, m_interprocessIndexingStatusManager.getCurrentlyIndexedSourceFilePath());
}
}
if (newlyIndexedCount > 0)
{
std::lock_guard<std::mutex> lock(blackboard->getMutex());
int indexedSourceFileCount = 0;
blackboard->get("indexed_source_file_count", indexedSourceFileCount);
blackboard->set("indexed_source_file_count", indexedSourceFileCount + newlyIndexedCount);
}
}
void TaskBuildIndex::updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath)
{
// TODO: factor in unindexed files...
int sourceFileCount = 0;
int indexedSourceFileCount = 0;
{
std::lock_guard<std::mutex> lock(blackboard->getMutex());
blackboard->get("source_file_count", sourceFileCount);
blackboard->get("indexed_source_file_count", indexedSourceFileCount);
}
if (std::shared_ptr<DialogView> dialogView = Application::getInstance()->getDialogView())
{
dialogView->updateIndexingDialog(
indexedSourceFileCount, sourceFileCount, sourcePath.str()
);
}
}
src/lib/data/indexer/TaskBuildIndex.h
+27 -6
View File
@@ -1,17 +1,20 @@
#ifndef TASK_BUILD_INDEX_H
#define TASK_BUILD_INDEX_H
#include "data/parser/Parser.h"
#include "utility/scheduling/Task.h"
#include "utility/messaging/type/MessageInterruptTasks.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/messaging/MessageListener.h"
#include "utility/messaging/type/MessageInterruptTasks.h"
#include "utility/scheduling/Task.h"
#include "data/indexer/interprocess/InterprocessIndexerCommandManager.h"
#include "data/indexer/interprocess/InterprocessIndexingStatusManager.h"
#include "data/indexer/interprocess/InterprocessIntermediateStorageManager.h"
#include "data/parser/Parser.h"
class CxxParser;
class DialogView;
class FileRegisterStateData;
class StorageProvider;
class IndexerCommandList;
class IndexerBase;
class TaskBuildIndex
: public Task
@@ -19,6 +22,7 @@ class TaskBuildIndex
{
public:
TaskBuildIndex(
unsigned int processCount,
std::shared_ptr<IndexerCommandList> indexerCommandList,
std::shared_ptr<StorageProvider> storageProvider,
std::shared_ptr<FileRegisterStateData> fileRegisterStateData
@@ -29,14 +33,31 @@ protected:
virtual TaskState doUpdate(std::shared_ptr<Blackboard> blackboard);
virtual void doExit(std::shared_ptr<Blackboard> blackboard);
virtual void doReset(std::shared_ptr<Blackboard> blackboard);
virtual void terminate();
virtual void handleMessage(MessageInterruptTasks* message);
void runIndexerProcess( int processId, const std::string& logFilePath);
void runIndexerThread(int processId);
void fetchIntermediateStorages(std::shared_ptr<Blackboard> blackboard);
void updateIndexingDialog(std::shared_ptr<Blackboard> blackboard, const FilePath& sourcePath);
static const std::string s_processName;
std::shared_ptr<IndexerCommandList> m_indexerCommandList;
std::shared_ptr<StorageProvider> m_storageProvider;
std::shared_ptr<FileRegisterStateData> m_fileRegisterStateData;
std::shared_ptr<IndexerBase> m_indexer;
InterprocessIndexerCommandManager m_interprocessIndexerCommandManager;
InterprocessIndexingStatusManager m_interprocessIndexingStatusManager;
unsigned int m_processCount;
bool m_interrupted;
size_t m_lastCommandCount;
// store as plain pointers to avoid deallocation issues when closing app during indexing
std::vector<std::thread*> m_processThreads;
std::vector<std::shared_ptr<InterprocessIntermediateStorageManager>> m_interprocessIntermediateStorageManagers;
};
#endif // TASK_PARSE_H
src/lib/data/indexer/interprocess/BaseInterprocessDataManager.cpp
+23
View File
@@ -0,0 +1,23 @@
#include "BaseInterprocessDataManager.h"
BaseInterprocessDataManager::BaseInterprocessDataManager(
const std::string& sharedMemoryName,
size_t initialSharedMemorySize,
const std::string& instanceUuid,
Id processId,
bool isOwner
)
: m_sharedMemory(sharedMemoryName, initialSharedMemorySize, isOwner ? SharedMemory::CREATE_AND_DELETE : SharedMemory::OPEN_ONLY)
, m_instanceUuid(instanceUuid)
, m_processId(processId)
{
}
BaseInterprocessDataManager::~BaseInterprocessDataManager()
{
}
Id BaseInterprocessDataManager::getProcessId() const
{
return m_processId;
}
src/lib/data/indexer/interprocess/BaseInterprocessDataManager.h
+29
View File
@@ -0,0 +1,29 @@
#ifndef BASE_INTERPROCESS_DATA_MANAGER_H
#define BASE_INTERPROCESS_DATA_MANAGER_H
#include <string>
#include "utility/types.h"
#include "utility/interprocess/SharedMemory.h"
class BaseInterprocessDataManager
{
public:
BaseInterprocessDataManager(
const std::string& sharedMemoryName,
size_t initialSharedMemorySize,
const std::string& instanceUuid,
Id processId,
bool isOwner);
virtual ~BaseInterprocessDataManager();
Id getProcessId() const;
protected:
SharedMemory m_sharedMemory;
const std::string m_instanceUuid;
const Id m_processId;
};
#endif // BASE_INTERPROCESS_DATA_MANAGER_H
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+66
View File
@@ -0,0 +1,66 @@
#include "InterprocessIndexer.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/logging/logging.h"
#include "data/indexer/IndexerComposite.h"
#include "data/indexer/IndexerCommand.h"
#include "data/indexer/IndexerFactory.h"
InterprocessIndexer::InterprocessIndexer(const std::string& uuid, Id processId)
: m_interprocessIndexerCommandManager(uuid, processId, false)
, m_interprocessIndexingStatusManager(uuid, processId, false)
, m_interprocessIntermediateStorageManager(uuid, processId, false)
, m_uuid(uuid)
, m_processId(processId)
{
}
InterprocessIndexer::~InterprocessIndexer()
{
}
void InterprocessIndexer::work()
{
try
{
LOG_INFO_STREAM(<< m_processId << " Starting to index");
std::shared_ptr<IndexerBase> indexer = IndexerFactory::getInstance()->createCompositeIndexerForAllRegisteredModules();
while (std::shared_ptr<IndexerCommand> indexerCommand = m_interprocessIndexerCommandManager.popIndexerCommand())
{
LOG_INFO_STREAM(<< m_processId << " Indexing " << indexerCommand->getSourceFilePath().str());
LOG_INFO_STREAM(<< m_processId << " Commands left: " << (m_interprocessIndexerCommandManager.indexerCommandCount() + 1));
m_interprocessIndexingStatusManager.setCurrentlyIndexedSourceFilePath(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles());
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
data, indexerCommand->getIndexedPaths(), indexerCommand->getExcludedPath()
);
std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
m_interprocessIndexingStatusManager.clearCurrentlyIndexedSourceFilePath();
}
}
catch (boost::interprocess::interprocess_exception& e)
{
LOG_ERROR(e.what());
throw e;
}
catch (std::exception& e)
{
LOG_ERROR(e.what());
throw e;
}
LOG_INFO_STREAM(<< "Finished indexing");
}
src/lib/data/indexer/interprocess/InterprocessIndexer.h
+25
View File
@@ -0,0 +1,25 @@
#ifndef INTERPROCESS_INDEXER_H
#define INTERPROCESS_INDEXER_H
#include "InterprocessIndexerCommandManager.h"
#include "InterprocessIndexingStatusManager.h"
#include "InterprocessIntermediateStorageManager.h"
class InterprocessIndexer
{
public:
InterprocessIndexer(const std::string& uuid, Id processId);
~InterprocessIndexer();
void work();
private:
InterprocessIndexerCommandManager m_interprocessIndexerCommandManager;
InterprocessIndexingStatusManager m_interprocessIndexingStatusManager;
InterprocessIntermediateStorageManager m_interprocessIntermediateStorageManager;
const std::string m_uuid;
const Id m_processId;
};
#endif // INTERPROCESS_INDEXER_H
src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.cpp
+105
View File
@@ -0,0 +1,105 @@
#include "InterprocessIndexerCommandManager.h"
#include "data/indexer/IndexerCommand.h"
#include "utility/logging/logging.h"
const char* InterprocessIndexerCommandManager::s_sharedMemoryNamePrefix = "icmd_";
const char* InterprocessIndexerCommandManager::s_indexerCommandsKeyName = "indexer_commands";
InterprocessIndexerCommandManager::InterprocessIndexerCommandManager(const std::string& instanceUuid, Id processId, bool isOwner)
: BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner)
{
}
InterprocessIndexerCommandManager::~InterprocessIndexerCommandManager()
{
}
void InterprocessIndexerCommandManager::setIndexerCommands(
const std::vector<std::shared_ptr<IndexerCommand>>& indexerCommands)
{
const unsigned int overestimationMultiplier = 3;
size_t size = 1000;
for (auto command : indexerCommands)
{
size += command->getByteSize() + sizeof(SharedIndexerCommand);
}
size *= overestimationMultiplier;
SharedMemory::ScopedAccess access(&m_sharedMemory);
size_t freeMemory = access.getFreeMemorySize();
if (freeMemory <= size)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << size << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
access.growMemory(size - freeMemory);
}
SharedMemory::Queue<SharedIndexerCommand>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIndexerCommand>>(s_indexerCommandsKeyName);
if (!queue)
{
return;
}
for (auto command : indexerCommands)
{
queue->push_back(SharedIndexerCommand(access.getAllocator()));
SharedIndexerCommand& sharedCommand = queue->back();
sharedCommand.fromLocal(command.get());
}
LOG_INFO(access.logString());
}
std::shared_ptr<IndexerCommand> InterprocessIndexerCommandManager::popIndexerCommand()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Queue<SharedIndexerCommand>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIndexerCommand>>(s_indexerCommandsKeyName);
if (!queue || !queue->size())
{
return nullptr;
}
std::shared_ptr<IndexerCommand> command = SharedIndexerCommand::fromShared(queue->front());
queue->pop_front();
return command;
}
void InterprocessIndexerCommandManager::clearIndexerCommands()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Queue<SharedIndexerCommand>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIndexerCommand>>(s_indexerCommandsKeyName);
if (!queue)
{
return;
}
queue->clear();
}
size_t InterprocessIndexerCommandManager::indexerCommandCount()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Queue<SharedIndexerCommand>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIndexerCommand>>(s_indexerCommandsKeyName);
if (!queue)
{
return 0;
}
return queue->size();
}
src/lib/data/indexer/interprocess/InterprocessIndexerCommandManager.h
+27
View File
@@ -0,0 +1,27 @@
#ifndef INTERPROCESS_INDEXER_COMMAND_MANAGER_H
#define INTERPROCESS_INDEXER_COMMAND_MANAGER_H
#include "BaseInterprocessDataManager.h"
#include "shared_types/SharedIndexerCommand.h"
class IndexerCommand;
class InterprocessIndexerCommandManager
: public BaseInterprocessDataManager
{
public:
InterprocessIndexerCommandManager(const std::string& instanceUuid, Id processId, bool isOwner);
virtual ~InterprocessIndexerCommandManager();
void setIndexerCommands(const std::vector<std::shared_ptr<IndexerCommand>>& indexerCommands);
std::shared_ptr<IndexerCommand> popIndexerCommand();
void clearIndexerCommands();
size_t indexerCommandCount();
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_indexerCommandsKeyName;
};
#endif // INTERPROCESS_INDEXER_COMMAND_MANAGER_H
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
+173
View File
@@ -0,0 +1,173 @@
#include "InterprocessIndexingStatusManager.h"
#include "utility/logging/logging.h"
const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_";
const char* InterprocessIndexingStatusManager::s_lastFileKeyName = "last_file";
const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files";
const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files";
const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files";
InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner)
: BaseInterprocessDataManager(s_sharedMemoryNamePrefix + instanceUuid, 1048576 /* 1 MB */, instanceUuid, processId, isOwner)
{
}
InterprocessIndexingStatusManager::~InterprocessIndexingStatusManager()
{
}
void InterprocessIndexingStatusManager::setCurrentlyIndexedSourceFilePath(const FilePath& filePath)
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::String* strPtr = access.accessValueWithAllocator<SharedMemory::String>(s_lastFileKeyName);
if (strPtr)
{
*strPtr = filePath.str().c_str();
}
SharedMemory::Map<Id, SharedMemory::String>* currentFilesPtr =
access.accessValueWithAllocator<SharedMemory::Map<Id, SharedMemory::String>>(s_currentFilesKeyName);
if (currentFilesPtr)
{
SharedMemory::Map<Id, SharedMemory::String>::iterator it = currentFilesPtr->find(getProcessId());
if (it != currentFilesPtr->end())
{
SharedMemory::Vector<SharedMemory::String>* crashedFilesPtr =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_crashedFilesKeyName);
if (crashedFilesPtr)
{
crashedFilesPtr->push_back(it->second);
}
}
SharedMemory::String str(access.getAllocator());
str = filePath.str().c_str();
it = currentFilesPtr->insert(std::pair<Id, SharedMemory::String>(getProcessId(), str)).first;
it->second = str;
}
}
void InterprocessIndexingStatusManager::clearCurrentlyIndexedSourceFilePath()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Map<Id, SharedMemory::String>* currentFilesPtr =
access.accessValueWithAllocator<SharedMemory::Map<Id, SharedMemory::String>>(s_currentFilesKeyName);
if (currentFilesPtr)
{
currentFilesPtr->erase(currentFilesPtr->find(getProcessId()), currentFilesPtr->end());
}
}
FilePath InterprocessIndexingStatusManager::getCurrentlyIndexedSourceFilePath()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::String* strPtr = access.accessValueWithAllocator<SharedMemory::String>(s_lastFileKeyName);
if (strPtr)
{
return FilePath(strPtr->c_str());
}
return FilePath();
}
std::vector<FilePath> InterprocessIndexingStatusManager::getCrashedSourceFilePaths()
{
std::vector<FilePath> crashedFiles;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* crashedFilesPtr =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_crashedFilesKeyName);
if (crashedFilesPtr)
{
for (size_t i = 0; i < crashedFilesPtr->size(); i++)
{
crashedFiles.push_back(FilePath(crashedFilesPtr->at(i).c_str()));
}
}
SharedMemory::Map<Id, SharedMemory::String>* currentFilesPtr =
access.accessValueWithAllocator<SharedMemory::Map<Id, SharedMemory::String>>(s_currentFilesKeyName);
if (currentFilesPtr)
{
for (SharedMemory::Map<Id, SharedMemory::String>::iterator it = currentFilesPtr->begin(); it != currentFilesPtr->end(); it++)
{
crashedFiles.push_back(FilePath(it->second.c_str()));
}
}
return crashedFiles;
}
std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
{
std::set<FilePath> result;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* files =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!files)
{
return result;
}
for (auto file : *files)
{
result.insert(FilePath(file.c_str()));
}
return result;
}
void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
{
const unsigned int overestimationMultiplier = 3;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* files =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!files)
{
return;
}
for (auto file : *files)
{
filePaths.insert(FilePath(file.c_str()));
}
files->clear();
size_t size = 1000;
for (auto path : filePaths)
{
size += sizeof(std::string) + path.str().size();
}
size *= overestimationMultiplier;
size_t freeMemory = access.getFreeMemorySize();
if (freeMemory <= size)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << size << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
access.growMemory(size - freeMemory);
}
for (auto path : filePaths)
{
files->push_back(SharedMemory::String(path.str().c_str(), access.getAllocator()));
}
LOG_INFO(access.logString());
}
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h
+34
View File
@@ -0,0 +1,34 @@
#ifndef INTERPROCESS_INDEXING_STATUS_MANAGER_H
#define INTERPROCESS_INDEXING_STATUS_MANAGER_H
#include <set>
#include "utility/file/FilePath.h"
#include "BaseInterprocessDataManager.h"
class InterprocessIndexingStatusManager
: public BaseInterprocessDataManager
{
public:
InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner);
virtual ~InterprocessIndexingStatusManager();
void setCurrentlyIndexedSourceFilePath(const FilePath& filePath);
void clearCurrentlyIndexedSourceFilePath();
FilePath getCurrentlyIndexedSourceFilePath();
std::vector<FilePath> getCrashedSourceFilePaths();
std::set<FilePath> getIndexedFiles();
void addIndexedFiles(std::set<FilePath> filePaths);
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_lastFileKeyName;
static const char* s_currentFilesKeyName;
static const char* s_crashedFilesKeyName;
static const char* s_indexedFilesKeyName;
};
#endif // INTERPROCESS_INDEXING_STATUS_MANAGER_H
src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.cpp
+115
View File
@@ -0,0 +1,115 @@
#include "InterprocessIntermediateStorageManager.h"
#include "data/IntermediateStorage.h"
#include "utility/logging/logging.h"
const char* InterprocessIntermediateStorageManager::s_sharedMemoryNamePrefix = "iist_";
const char* InterprocessIntermediateStorageManager::s_intermediatStoragesKeyName = "intermediate_storages";
InterprocessIntermediateStorageManager::InterprocessIntermediateStorageManager(
const std::string& instanceUuid, Id processId, bool isOwner
)
: BaseInterprocessDataManager(
s_sharedMemoryNamePrefix + std::to_string(processId) + "_" + instanceUuid,
1048576 /* 1 MB */,
instanceUuid,
processId,
isOwner)
{
}
InterprocessIntermediateStorageManager::~InterprocessIntermediateStorageManager()
{
}
void InterprocessIntermediateStorageManager::pushIntermediateStorage(
const std::shared_ptr<IntermediateStorage>& intermediateStorage)
{
const unsigned int overestimationMultiplier = 3;
size_t size = intermediateStorage->getByteSize() * overestimationMultiplier;
SharedMemory::ScopedAccess access(&m_sharedMemory);
size_t freeMemory = access.getFreeMemorySize();
if (freeMemory <= size)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << size << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (size - freeMemory));
access.growMemory(size - freeMemory);
}
SharedMemory::Queue<SharedIntermediateStorage>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIntermediateStorage>>(s_intermediatStoragesKeyName);
if (!queue)
{
return;
}
queue->push_back(SharedIntermediateStorage(access.getAllocator()));
SharedIntermediateStorage& storage = queue->back();
storage.setStorageNodes(intermediateStorage->getStorageNodes());
storage.setStorageFiles(intermediateStorage->getStorageFiles());
storage.setStorageSymbols(intermediateStorage->getStorageSymbols());
storage.setStorageEdges(intermediateStorage->getStorageEdges());
storage.setStorageLocalSymbols(intermediateStorage->getStorageLocalSymbols());
storage.setStorageSourceLocations(intermediateStorage->getStorageSourceLocations());
storage.setStorageOccurrences(intermediateStorage->getStorageOccurrences());
storage.setStorageComponentAccesses(intermediateStorage->getComponentAccesses());
storage.setStorageCommentLocations(intermediateStorage->getCommentLocations());
storage.setStorageErrors(intermediateStorage->getErrors());
storage.setNextId(intermediateStorage->getNextId());
LOG_INFO(access.logString());
}
std::shared_ptr<IntermediateStorage> InterprocessIntermediateStorageManager::popIntermediateStorage()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Queue<SharedIntermediateStorage>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIntermediateStorage>>(s_intermediatStoragesKeyName);
if (!queue || !queue->size())
{
return nullptr;
}
SharedIntermediateStorage& sharedIntermediateStorage = queue->front();
std::shared_ptr<IntermediateStorage> storage = std::make_shared<IntermediateStorage>();
storage->setStorageNodes(sharedIntermediateStorage.getStorageNodes());
storage->setStorageFiles(sharedIntermediateStorage.getStorageFiles());
storage->setStorageSymbols(sharedIntermediateStorage.getStorageSymbols());
storage->setStorageEdges(sharedIntermediateStorage.getStorageEdges());
storage->setStorageLocalSymbols(sharedIntermediateStorage.getStorageLocalSymbols());
storage->setStorageSourceLocations(sharedIntermediateStorage.getStorageSourceLocations());
storage->setStorageOccurrences(sharedIntermediateStorage.getStorageOccurrences());
storage->setComponentAccesses(sharedIntermediateStorage.getStorageComponentAccesses());
storage->setCommentLocations(sharedIntermediateStorage.getStorageCommentLocations());
storage->setErrors(sharedIntermediateStorage.getStorageErrors());
storage->setNextId(sharedIntermediateStorage.getNextId());
queue->pop_front();
return storage;
}
size_t InterprocessIntermediateStorageManager::getIntermediateStorageCount()
{
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Queue<SharedIntermediateStorage>* queue =
access.accessValueWithAllocator<SharedMemory::Queue<SharedIntermediateStorage>>(s_intermediatStoragesKeyName);
if (!queue)
{
return 0;
}
return queue->size();
}
src/lib/data/indexer/interprocess/InterprocessIntermediateStorageManager.h
+26
View File
@@ -0,0 +1,26 @@
#ifndef INTERPROCESS_INTERMEDIATE_STORAGE_MANAGER_H
#define INTERPROCESS_INTERMEDIATE_STORAGE_MANAGER_H
#include "BaseInterprocessDataManager.h"
#include "shared_types/SharedIntermediateStorage.h"
class IntermediateStorage;
class InterprocessIntermediateStorageManager
: public BaseInterprocessDataManager
{
public:
InterprocessIntermediateStorageManager(const std::string& instanceUuid, Id processId, bool isOwner);
virtual ~InterprocessIntermediateStorageManager();
void pushIntermediateStorage(const std::shared_ptr<IntermediateStorage>& intermediateStorage);
std::shared_ptr<IntermediateStorage> popIntermediateStorage();
size_t getIntermediateStorageCount();
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_intermediatStoragesKeyName;
};
#endif // INTERPROCESS_INTERMEDIATE_STORAGE_MANAGER_H
src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp
+312
View File
@@ -0,0 +1,312 @@
#include "SharedIndexerCommand.h"
#include "data/indexer/IndexerCommandCxxCdb.h"
#include "data/indexer/IndexerCommandCxxManual.h"
#include "data/indexer/IndexerCommandJava.h"
#include "utility/logging/logging.h"
void SharedIndexerCommand::fromLocal(IndexerCommand* indexerCommand)
{
setSourceFilePath(indexerCommand->getSourceFilePath());
setIndexedPaths(indexerCommand->getIndexedPaths());
setExcludedPaths(indexerCommand->getExcludedPath());
if (dynamic_cast<IndexerCommandCxxCdb*>(indexerCommand) != NULL)
{
IndexerCommandCxxCdb* cmd = dynamic_cast<IndexerCommandCxxCdb*>(indexerCommand);
setType(CXX_CDB);
setWorkingDirectory(cmd->getWorkingDirectory());
setCompilerFlags(cmd->getCompilerFlags());
setSystemHeaderSearchPaths(cmd->getSystemHeaderSearchPaths());
setFrameworkSearchhPaths(cmd->getFrameworkSearchPaths());
setPreprocessorOnly(cmd->preprocessorOnly());
}
else if (dynamic_cast<IndexerCommandCxxManual*>(indexerCommand) != NULL)
{
IndexerCommandCxxManual* cmd = dynamic_cast<IndexerCommandCxxManual*>(indexerCommand);
setType(CXX_MANUAL);
setLanguageStandard(cmd->getLanguageStandard());
setCompilerFlags(cmd->getCompilerFlags());
setSystemHeaderSearchPaths(cmd->getSystemHeaderSearchPaths());
setFrameworkSearchhPaths(cmd->getFrameworkSearchPaths());
setPreprocessorOnly(cmd->preprocessorOnly());
}
else if (dynamic_cast<IndexerCommandJava*>(indexerCommand) != NULL)
{
IndexerCommandJava* cmd = dynamic_cast<IndexerCommandJava*>(indexerCommand);
setType(JAVA);
setClassPaths(cmd->getClassPath());
}
else
{
LOG_ERROR_STREAM(<< "Trying to push unhandled type of IndexerCommand for file: "
<< indexerCommand->getSourceFilePath().str() << ". Type string is: " << indexerCommand->getKindString()
<< ". It will be ignored.");
}
}
std::shared_ptr<IndexerCommand> SharedIndexerCommand::fromShared(const SharedIndexerCommand& indexerCommand)
{
if (indexerCommand.getType() == CXX_CDB)
{
std::shared_ptr<IndexerCommand> command = std::make_shared<IndexerCommandCxxCdb>(
indexerCommand.getSourceFilePath(),
indexerCommand.getIndexedPaths(),
indexerCommand.getExcludedPaths(),
indexerCommand.getWorkingDirectory(),
indexerCommand.getCompilerFlags(),
indexerCommand.getSystemHeaderSearchPaths(),
indexerCommand.getFrameworkSearchhPaths()
);
command->setPreprocessorOnly(indexerCommand.preprocessorOnly());
return command;
}
else if (indexerCommand.getType() == CXX_MANUAL)
{
std::shared_ptr<IndexerCommand> command = std::make_shared<IndexerCommandCxxManual>(
indexerCommand.getSourceFilePath(),
indexerCommand.getIndexedPaths(),
indexerCommand.getExcludedPaths(),
indexerCommand.getLanguageStandard(),
indexerCommand.getSystemHeaderSearchPaths(),
indexerCommand.getFrameworkSearchhPaths(),
indexerCommand.getCompilerFlags()
);
command->setPreprocessorOnly(indexerCommand.preprocessorOnly());
return command;
}
else if (indexerCommand.getType() == JAVA)
{
return std::make_shared<IndexerCommandJava>(
indexerCommand.getSourceFilePath(),
indexerCommand.getIndexedPaths(),
indexerCommand.getExcludedPaths(),
indexerCommand.getClassPaths()
);
}
else
{
LOG_ERROR_STREAM(<< "Cannot convert shared IndexerCommand for file: "
<< indexerCommand.getSourceFilePath().str() << ". The type is unknown.");
}
return nullptr;
}
SharedIndexerCommand::SharedIndexerCommand(SharedMemory::Allocator* allocator)
: m_type(Type::UNKNOWN)
, m_sourceFilePath("", allocator)
, m_indexedPaths(allocator)
, m_excludedPaths(allocator)
, m_workingDirectory("", allocator)
, m_languageStandard("", allocator)
, m_compilerFlags(allocator)
, m_systemHeaderSearchPaths(allocator)
, m_frameworkSearchPaths(allocator)
, m_preprocessorOnly(false)
, m_classPaths(allocator)
{
}
SharedIndexerCommand::~SharedIndexerCommand()
{
}
FilePath SharedIndexerCommand::getSourceFilePath() const
{
return FilePath(m_sourceFilePath.c_str());
}
void SharedIndexerCommand::setSourceFilePath(const FilePath& filePath)
{
m_sourceFilePath = filePath.str().c_str();
}
std::set<FilePath> SharedIndexerCommand::getIndexedPaths() const
{
std::set<FilePath> result;
for (unsigned int i = 0; i < m_indexedPaths.size(); i++)
{
result.insert(FilePath(m_indexedPaths[i].c_str()));
}
return result;
}
void SharedIndexerCommand::setIndexedPaths(const std::set<FilePath>& indexedPaths)
{
m_indexedPaths.clear();
for (std::set<FilePath>::iterator it = indexedPaths.begin(); it != indexedPaths.end(); it++)
{
SharedMemory::String path(m_indexedPaths.get_allocator());
path = (*it).str().c_str();
m_indexedPaths.push_back(path);
}
}
std::set<FilePath> SharedIndexerCommand::getExcludedPaths() const
{
std::set<FilePath> result;
for (unsigned int i = 0; i < m_excludedPaths.size(); i++)
{
result.insert(FilePath(m_excludedPaths[i].c_str()));
}
return result;
}
void SharedIndexerCommand::setExcludedPaths(const std::set<FilePath>& excludedPaths)
{
m_excludedPaths.clear();
for (std::set<FilePath>::iterator it = excludedPaths.begin(); it != excludedPaths.end(); it++)
{
SharedMemory::String path(m_excludedPaths.get_allocator());
path = (*it).str().c_str();
m_excludedPaths.push_back(path);
}
}
FilePath SharedIndexerCommand::getWorkingDirectory() const
{
return FilePath(m_workingDirectory.c_str());
}
void SharedIndexerCommand::setWorkingDirectory(const FilePath& workingDirectory)
{
m_workingDirectory = workingDirectory.str().c_str();
}
std::string SharedIndexerCommand::getLanguageStandard() const
{
return m_languageStandard.c_str();
}
void SharedIndexerCommand::setLanguageStandard(const std::string& languageStandard)
{
m_languageStandard = languageStandard.c_str();
}
std::vector<std::string> SharedIndexerCommand::getCompilerFlags() const
{
std::vector<std::string> result;
for (unsigned int i = 0; i < m_compilerFlags.size(); i++)
{
result.push_back(m_compilerFlags[i].c_str());
}
return result;
}
void SharedIndexerCommand::setCompilerFlags(const std::vector<std::string>& compilerFlags)
{
m_compilerFlags.clear();
for (unsigned int i = 0; i < compilerFlags.size(); i++)
{
SharedMemory::String path(m_compilerFlags.get_allocator());
path = compilerFlags[i].c_str();
m_compilerFlags.push_back(path);
}
}
std::vector<FilePath> SharedIndexerCommand::getSystemHeaderSearchPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++)
{
result.push_back(FilePath(m_systemHeaderSearchPaths[i].c_str()));
}
return result;
}
void SharedIndexerCommand::setSystemHeaderSearchPaths(const std::vector<FilePath>& filePaths)
{
m_systemHeaderSearchPaths.clear();
for (unsigned int i = 0; i < filePaths.size(); i++)
{
SharedMemory::String path(m_systemHeaderSearchPaths.get_allocator());
path = filePaths[i].str().c_str();
m_systemHeaderSearchPaths.push_back(path);
}
}
std::vector<FilePath> SharedIndexerCommand::getFrameworkSearchhPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++)
{
result.push_back(FilePath(m_frameworkSearchPaths[i].c_str()));
}
return result;
}
void SharedIndexerCommand::setFrameworkSearchhPaths(const std::vector<FilePath>& searchPaths)
{
m_frameworkSearchPaths.clear();
for (unsigned int i = 0; i < searchPaths.size(); i++)
{
SharedMemory::String path(m_frameworkSearchPaths.get_allocator());
path = searchPaths[i].str().c_str();
m_frameworkSearchPaths.push_back(path);
}
}
bool SharedIndexerCommand::preprocessorOnly() const
{
return m_preprocessorOnly;
}
void SharedIndexerCommand::setPreprocessorOnly(bool preprocessorOnly)
{
m_preprocessorOnly = preprocessorOnly;
}
std::vector<FilePath> SharedIndexerCommand::getClassPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_classPaths.size(); i++)
{
result.push_back(FilePath(m_classPaths[i].c_str()));
}
return result;
}
void SharedIndexerCommand::setClassPaths(const std::vector<FilePath>& classPaths)
{
m_classPaths.clear();
for (unsigned int i = 0; i < classPaths.size(); i++)
{
SharedMemory::String path(m_classPaths.get_allocator());
path = classPaths[i].str().c_str();
m_classPaths.push_back(path);
}
}
SharedIndexerCommand::Type SharedIndexerCommand::getType() const
{
return m_type;
}
void SharedIndexerCommand::setType(const SharedIndexerCommand::Type type)
{
m_type = type;
}
src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.h
+81
View File
@@ -0,0 +1,81 @@
#ifndef SHARED_INDEXER_COMMAND_H
#define SHARED_INDEXER_COMMAND_H
#include <set>
#include "utility/file/FilePath.h"
#include "utility/interprocess/SharedMemory.h"
class IndexerCommand;
class SharedIndexerCommand
{
public:
void fromLocal(IndexerCommand* indexerCommand);
static std::shared_ptr<IndexerCommand> fromShared(const SharedIndexerCommand& indexerCommand);
SharedIndexerCommand(SharedMemory::Allocator* allocator);
~SharedIndexerCommand();
FilePath getSourceFilePath() const;
void setSourceFilePath(const FilePath& filePath);
std::set<FilePath> getIndexedPaths() const;
void setIndexedPaths(const std::set<FilePath>& indexedPaths);
std::set<FilePath> getExcludedPaths() const;
void setExcludedPaths(const std::set<FilePath>& excludedPaths);
FilePath getWorkingDirectory() const;
void setWorkingDirectory(const FilePath& workingDirectory);
std::string getLanguageStandard() const;
void setLanguageStandard(const std::string& languageStandard);
std::vector<std::string> getCompilerFlags() const;
void setCompilerFlags(const std::vector<std::string>& compilerFlags);
std::vector<FilePath> getSystemHeaderSearchPaths() const;
void setSystemHeaderSearchPaths(const std::vector<FilePath>& filePaths);
std::vector<FilePath> getFrameworkSearchhPaths() const;
void setFrameworkSearchhPaths(const std::vector<FilePath>& searchPaths);
bool preprocessorOnly() const;
void setPreprocessorOnly(bool preprocessorOnly);
std::vector<FilePath> getClassPaths() const;
void setClassPaths(const std::vector<FilePath>& classPaths);
private:
enum Type
{
UNKNOWN = 0,
CXX_CDB,
CXX_MANUAL,
JAVA
};
Type getType() const;
void setType(const Type type);
Type m_type;
// indexer command
SharedMemory::String m_sourceFilePath;
SharedMemory::Vector<SharedMemory::String> m_indexedPaths;
SharedMemory::Vector<SharedMemory::String> m_excludedPaths;
// cxx
SharedMemory::String m_workingDirectory;
SharedMemory::String m_languageStandard;
SharedMemory::Vector<SharedMemory::String> m_compilerFlags;
SharedMemory::Vector<SharedMemory::String> m_systemHeaderSearchPaths;
SharedMemory::Vector<SharedMemory::String> m_frameworkSearchPaths;
bool m_preprocessorOnly;
// java
SharedMemory::Vector<SharedMemory::String> m_classPaths;
};
#endif // SHARED_INDEXER_COMMAND_H
src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.cpp
+251
View File
@@ -0,0 +1,251 @@
#include "SharedIntermediateStorage.h"
SharedIntermediateStorage::SharedIntermediateStorage(SharedMemory::Allocator* allocator)
: m_storageFiles(allocator)
, m_storageSymbols(allocator)
, m_storageOccurrences(allocator)
, m_storageComponentAccesses(allocator)
, m_storageCommentLocations(allocator)
, m_storageNodes(allocator)
, m_storageEdges(allocator)
, m_storageLocalSymbols(allocator)
, m_storageSourceLocations(allocator)
, m_storageErrors(allocator)
, m_allocator(allocator)
, m_nextId(1)
{
}
SharedIntermediateStorage::~SharedIntermediateStorage()
{
}
std::vector<StorageFile> SharedIntermediateStorage::getStorageFiles() const
{
std::vector<StorageFile> result;
for (unsigned int i = 0; i < m_storageFiles.size(); i++)
{
result.push_back(fromShared(m_storageFiles[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageFiles(const std::vector<StorageFile>& storageFiles)
{
m_storageFiles.clear();
for (unsigned int i = 0; i < storageFiles.size(); i++)
{
m_storageFiles.push_back(toShared(storageFiles[i], m_allocator));
}
}
std::vector<StorageNode> SharedIntermediateStorage::getStorageNodes() const
{
std::vector<StorageNode> result;
for (unsigned int i = 0; i < m_storageNodes.size(); i++)
{
result.push_back(fromShared(m_storageNodes[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageNodes(const std::vector<StorageNode>& storageNodes)
{
m_storageNodes.clear();
for (unsigned int i = 0; i < storageNodes.size(); i++)
{
m_storageNodes.push_back(toShared(storageNodes[i], m_allocator));
}
}
std::vector<StorageSymbol> SharedIntermediateStorage::getStorageSymbols() const
{
std::vector<StorageSymbol> result;
for (unsigned int i = 0; i < m_storageSymbols.size(); i++)
{
result.push_back(fromShared(m_storageSymbols[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageSymbols(const std::vector<StorageSymbol>& storageSymbols)
{
m_storageSymbols.clear();
for (unsigned int i = 0; i < storageSymbols.size(); i++)
{
m_storageSymbols.push_back(toShared(storageSymbols[i], m_allocator));
}
}
std::vector<StorageEdge> SharedIntermediateStorage::getStorageEdges() const
{
std::vector<StorageEdge> result;
for (unsigned int i = 0; i < m_storageEdges.size(); i++)
{
result.push_back(fromShared(m_storageEdges[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageEdges(const std::vector<StorageEdge>& storageEdges)
{
m_storageEdges.clear();
for (unsigned int i = 0; i < storageEdges.size(); i++)
{
m_storageEdges.push_back(toShared(storageEdges[i], m_allocator));
}
}
std::vector<StorageLocalSymbol> SharedIntermediateStorage::getStorageLocalSymbols() const
{
std::vector<StorageLocalSymbol> result;
for (unsigned int i = 0; i < m_storageLocalSymbols.size(); i++)
{
result.push_back(fromShared(m_storageLocalSymbols[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageLocalSymbols(const std::vector<StorageLocalSymbol>& storageLocalSymbols)
{
m_storageLocalSymbols.clear();
for (unsigned int i = 0; i < storageLocalSymbols.size(); i++)
{
m_storageLocalSymbols.push_back(toShared(storageLocalSymbols[i], m_allocator));
}
}
std::vector<StorageSourceLocation> SharedIntermediateStorage::getStorageSourceLocations() const
{
std::vector<StorageSourceLocation> result;
for (unsigned int i = 0; i < m_storageSourceLocations.size(); i++)
{
result.push_back(fromShared(m_storageSourceLocations[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageSourceLocations(const std::vector<StorageSourceLocation>& storageSourceLocations)
{
m_storageSourceLocations.clear();
for (unsigned int i = 0; i < storageSourceLocations.size(); i++)
{
m_storageSourceLocations.push_back(toShared(storageSourceLocations[i], m_allocator));
}
}
std::vector<StorageOccurrence> SharedIntermediateStorage::getStorageOccurrences() const
{
std::vector<StorageOccurrence> result;
for (unsigned int i = 0; i < m_storageOccurrences.size(); i++)
{
result.push_back(fromShared(m_storageOccurrences[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurences)
{
m_storageOccurrences.clear();
for (unsigned int i = 0; i < storageOccurences.size(); i++)
{
m_storageOccurrences.push_back(toShared(storageOccurences[i], m_allocator));
}
}
std::vector<StorageComponentAccess> SharedIntermediateStorage::getStorageComponentAccesses() const
{
std::vector<StorageComponentAccess> result;
for (unsigned int i = 0; i < m_storageComponentAccesses.size(); i++)
{
result.push_back(fromShared(m_storageComponentAccesses[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageComponentAccesses(const std::vector<StorageComponentAccess>& storageComponentAccesses)
{
m_storageComponentAccesses.clear();
for (unsigned int i = 0; i < storageComponentAccesses.size(); i++)
{
m_storageComponentAccesses.push_back(toShared(storageComponentAccesses[i], m_allocator));
}
}
std::vector<StorageCommentLocation> SharedIntermediateStorage::getStorageCommentLocations() const
{
std::vector<StorageCommentLocation> result;
for (unsigned int i = 0; i < m_storageCommentLocations.size(); i++)
{
result.push_back(fromShared(m_storageCommentLocations[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageCommentLocations(const std::vector<StorageCommentLocation>& commentLocations)
{
m_storageCommentLocations.clear();
for (unsigned int i = 0; i < commentLocations.size(); i++)
{
m_storageCommentLocations.push_back(toShared(commentLocations[i], m_allocator));
}
}
std::vector<StorageError> SharedIntermediateStorage::getStorageErrors() const
{
std::vector<StorageError> result;
for (unsigned int i = 0; i < m_storageErrors.size(); i++)
{
result.push_back(fromShared(m_storageErrors[i]));
}
return result;
}
void SharedIntermediateStorage::setStorageErrors(const std::vector<StorageError>& errors)
{
m_storageErrors.clear();
for (unsigned int i = 0; i < errors.size(); i++)
{
m_storageErrors.push_back(toShared(errors[i], m_allocator));
}
}
Id SharedIntermediateStorage::getNextId() const
{
return m_nextId;
}
void SharedIntermediateStorage::setNextId(const Id nextId)
{
m_nextId = nextId;
}
src/lib/data/indexer/interprocess/shared_types/SharedIntermediateStorage.h
+64
View File
@@ -0,0 +1,64 @@
#ifndef SHARED_INTERMEDIATE_STORAGE_H
#define SHARED_INTERMEDIATE_STORAGE_H
#include "data/StorageTypes.h"
#include "SharedStorageTypes.h"
#include "utility/interprocess/SharedMemory.h"
class SharedIntermediateStorage
{
public:
SharedIntermediateStorage(SharedMemory::Allocator* allocator);
~SharedIntermediateStorage();
std::vector<StorageNode> getStorageNodes() const;
void setStorageNodes(const std::vector<StorageNode>& storageNodes);
std::vector<StorageFile> getStorageFiles() const;
void setStorageFiles(const std::vector<StorageFile>& storageFiles);
std::vector<StorageSymbol> getStorageSymbols() const;
void setStorageSymbols(const std::vector<StorageSymbol>& storageSymbols);
std::vector<StorageEdge> getStorageEdges() const;
void setStorageEdges(const std::vector<StorageEdge>& storageEdges);
std::vector<StorageLocalSymbol> getStorageLocalSymbols() const;
void setStorageLocalSymbols(const std::vector<StorageLocalSymbol>& storageLocalSymbols);
std::vector<StorageSourceLocation> getStorageSourceLocations() const;
void setStorageSourceLocations(const std::vector<StorageSourceLocation>& storageSourceLocations);
std::vector<StorageOccurrence> getStorageOccurrences() const;
void setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurences);
std::vector<StorageComponentAccess> getStorageComponentAccesses() const;
void setStorageComponentAccesses(const std::vector<StorageComponentAccess>& storageComponentAccesses);
std::vector<StorageCommentLocation> getStorageCommentLocations() const;
void setStorageCommentLocations(const std::vector<StorageCommentLocation>& commentLocations);
std::vector<StorageError> getStorageErrors() const;
void setStorageErrors(const std::vector<StorageError>& errors);
Id getNextId() const;
void setNextId(const Id nextId);
private:
SharedMemory::Vector<SharedStorageFile> m_storageFiles;
SharedMemory::Vector<SharedStorageSymbol> m_storageSymbols;
SharedMemory::Vector<SharedStorageOccurrence> m_storageOccurrences;
SharedMemory::Vector<SharedStorageComponentAccess> m_storageComponentAccesses;
SharedMemory::Vector<SharedStorageCommentLocation> m_storageCommentLocations;
SharedMemory::Vector<SharedStorageNode> m_storageNodes;
SharedMemory::Vector<SharedStorageEdge> m_storageEdges;
SharedMemory::Vector<SharedStorageLocalSymbol> m_storageLocalSymbols;
SharedMemory::Vector<SharedStorageSourceLocation> m_storageSourceLocations;
SharedMemory::Vector<SharedStorageError> m_storageErrors;
SharedMemory::Allocator* m_allocator;
int m_nextId;
};
#endif // SHARED_INTERMEDIATE_STORAGE_H
src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h
+152
View File
@@ -0,0 +1,152 @@
#ifndef SHARED_STORAGE_TYPES_H
#define SHARED_STORAGE_TYPES_H
#include "utility/types.h"
#include "utility/interprocess/SharedMemory.h"
// macro creating SharedStorageType from StorageType
// - arguments: StorageType & SharedStorageType
// - defines: conversion functions toShared() & fromShared()
#define CONVERT_STORAGE_TYPE_TO_SHARED_TYPE(__type__, __shared_type__) \
typedef __type__ __shared_type__; \
\
inline const __shared_type__& toShared(const __type__& instance, SharedMemory::Allocator* allocator) \
{ \
return instance; \
} \
\
inline const __type__& fromShared(const __shared_type__& instance) \
{ \
return instance; \
}
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageEdge, SharedStorageEdge )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageSymbol, SharedStorageSymbol )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageSourceLocation, SharedStorageSourceLocation )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageOccurrence, SharedStorageOccurrence )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageComponentAccess, SharedStorageComponentAccess )
CONVERT_STORAGE_TYPE_TO_SHARED_TYPE( StorageCommentLocation, SharedStorageCommentLocation )
struct SharedStorageNode
{
SharedStorageNode(Id id, int type, const std::string& serializedName, SharedMemory::Allocator* allocator)
: id(id)
, type(type)
, serializedName(serializedName.c_str(), allocator)
{}
Id id;
int type;
SharedMemory::String serializedName;
};
inline SharedStorageNode toShared(const StorageNode& node, SharedMemory::Allocator* allocator)
{
return SharedStorageNode(node.id, node.type, node.serializedName, allocator);
}
inline StorageNode fromShared(const SharedStorageNode& node)
{
return StorageNode(node.id, node.type, node.serializedName.c_str());
}
struct SharedStorageFile
{
SharedStorageFile(
Id id, const std::string& filePath, const std::string& modificationTime, bool complete, SharedMemory::Allocator* allocator
)
: id(id)
, filePath(filePath.c_str(), allocator)
, modificationTime(modificationTime.c_str(), allocator)
, complete(complete)
{}
Id id;
SharedMemory::String filePath;
SharedMemory::String modificationTime;
bool complete;
};
inline SharedStorageFile toShared(const StorageFile& file, SharedMemory::Allocator* allocator)
{
return SharedStorageFile(file.id, file.filePath, file.modificationTime, file.complete, allocator);
}
inline StorageFile fromShared(const SharedStorageFile& file)
{
return StorageFile(file.id, file.filePath.c_str(), file.modificationTime.c_str(), file.complete);
}
struct SharedStorageLocalSymbol
{
SharedStorageLocalSymbol(Id id, const std::string& name, SharedMemory::Allocator* allocator)
: id(id)
, name(name.c_str(), allocator)
{}
Id id;
SharedMemory::String name;
};
inline SharedStorageLocalSymbol toShared(const StorageLocalSymbol& symbol, SharedMemory::Allocator* allocator)
{
return SharedStorageLocalSymbol(symbol.id, symbol.name, allocator);
}
inline StorageLocalSymbol fromShared(const SharedStorageLocalSymbol& symbol)
{
return StorageLocalSymbol(symbol.id, symbol.name.c_str());
}
struct SharedStorageError
{
SharedStorageError(
Id id,
const std::string& message,
const std::string& filePath,
uint lineNumber,
uint columnNumber,
bool fatal,
bool indexed,
SharedMemory::Allocator* allocator
)
: id(id)
, message(message.c_str(), allocator)
, filePath(filePath.c_str(), allocator)
, lineNumber(lineNumber)
, columnNumber(columnNumber)
, fatal(fatal)
, indexed(indexed)
{}
Id id;
SharedMemory::String message;
SharedMemory::String filePath;
uint lineNumber;
uint columnNumber;
bool fatal;
bool indexed;
};
inline SharedStorageError toShared(const StorageError& error, SharedMemory::Allocator* allocator)
{
return SharedStorageError(
error.id, error.message, error.filePath.str(),
error.lineNumber, error.columnNumber, error.fatal, error.indexed, allocator);
}
inline StorageError fromShared(const SharedStorageError& error)
{
return StorageError(
error.id, error.message.c_str(), FilePath(error.filePath.c_str()),
error.lineNumber, error.columnNumber, error.fatal, error.indexed);
}
#endif // SHARED_STORAGE_TYPES_H
src/lib/data/parser/ParserClient.cpp
-11
View File
@@ -73,7 +73,6 @@ std::string ParserClient::addLocationSuffix(
ParserClient::ParserClient()
: m_hasFatalErrors(false)
, m_cancelOnFatalErrors(false)
{
}
@@ -95,13 +94,3 @@ bool ParserClient::hasFatalErrors() const
{
return m_hasFatalErrors;
}
bool ParserClient::cancelOnFatalErrors() const
{
return m_cancelOnFatalErrors;
}
void ParserClient::setCancelOnFatalErrors(bool cancelOnFatalErrors)
{
m_cancelOnFatalErrors = cancelOnFatalErrors;
}
src/lib/data/parser/ParserClient.h
-4
View File
@@ -55,12 +55,8 @@ public:
bool hasFatalErrors() const;
bool cancelOnFatalErrors() const;
void setCancelOnFatalErrors(bool cancelOnFatalErrors);
protected:
bool m_hasFatalErrors;
bool m_cancelOnFatalErrors;
};
#endif // PARSER_CLIENT_H
src/lib/project/Project.cpp
+9 -18
View File
@@ -47,6 +47,7 @@ Project::Project(std::shared_ptr<ProjectSettings> settings, StorageAccessProxy*
{
}
Project::~Project()
{
}
@@ -418,7 +419,6 @@ bool Project::requestIndex(bool forceRefresh, bool needsFullRefresh)
void Project::buildIndex(const std::set<FilePath>& filesToClean, bool fullRefresh, bool preprocessorOnly)
{
MessageClearErrorCount().dispatch();
if (fullRefresh)
{
m_storage->clear();
@@ -433,20 +433,16 @@ void Project::buildIndex(const std::set<FilePath>& filesToClean, bool fullRefres
{
taskSequential->addTask(std::make_shared<TaskCleanStorage>(
m_storage.get(),
utility::toVector(filesToClean))
);
utility::toVector(filesToClean)
));
}
std::shared_ptr<IndexerCommandList> indexerCommandList = std::make_shared<IndexerCommandList>();
bool cancelIndexingOnFatalErrors = ApplicationSettings::getInstance()->getCancelIndexingOnFatalErrors();
for (std::shared_ptr<SourceGroup> sourceGroup: m_sourceGroups)
{
for (std::shared_ptr<IndexerCommand> command: sourceGroup->getIndexerCommands(fullRefresh))
{
command->setCancelOnFatalErrors(cancelIndexingOnFatalErrors);
command->setPreprocessorOnly(preprocessorOnly);
indexerCommandList->addCommand(command);
}
}
@@ -463,36 +459,33 @@ void Project::buildIndex(const std::set<FilePath>& filesToClean, bool fullRefres
}
}
indexerThreadCount = std::min<int>(indexerThreadCount, indexerCommandList->size());
if (indexerThreadCount > 1)
{
indexerCommandList->shuffle();
}
std::shared_ptr<FileRegisterStateData> fileRegisterStateData = std::make_shared<FileRegisterStateData>();
std::shared_ptr<StorageProvider> storageProvider = std::make_shared<StorageProvider>();
// add tasks for setting some variables on the blackboard that are used during indexing
taskSequential->addTask(std::make_shared<TaskSetValue<int>>("source_file_count", indexerCommandList->size()));
taskSequential->addTask(std::make_shared<TaskSetValue<int>>("indexed_source_file_count", 0));
taskSequential->addTask(std::make_shared<TaskSetValue<int>>("indexer_count", 0));
std::shared_ptr<TaskParseWrapper> taskParserWrapper = std::make_shared<TaskParseWrapper>(m_storage.get());
taskSequential->addTask(taskParserWrapper);
taskSequential->addTask(taskParserWrapper);
std::shared_ptr<TaskGroupParallel> taskParallelIndexing = std::make_shared<TaskGroupParallel>();
taskParserWrapper->setTask(taskParallelIndexing);
// add tasks for indexing and merging
for (int i = 0; i < indexerThreadCount && size_t(i) < indexerCommandList->size(); i++)
// add task for indexing
if (indexerThreadCount > 0)
{
taskParallelIndexing->addChildTasks(
std::make_shared<TaskDecoratorRepeat>(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS)->addChildTask(
std::make_shared<TaskBuildIndex>(indexerCommandList, storageProvider, fileRegisterStateData)
std::make_shared<TaskBuildIndex>(indexerThreadCount, indexerCommandList, storageProvider, fileRegisterStateData)
)
);
}
// add task for merging the intermediate storages
taskParallelIndexing->addTask(
std::make_shared<TaskGroupSequence>()->addChildTasks(
@@ -507,13 +500,12 @@ void Project::buildIndex(const std::set<FilePath>& filesToClean, bool fullRefres
)
)
);
// add task for injecting the intermediate storages into the persistent storage
taskParallelIndexing->addTask(
std::make_shared<TaskGroupSequence>()->addChildTasks(
std::make_shared<TaskDecoratorRepeat>(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS)->addChildTask(
std::make_shared<TaskReturnSuccessWhile<int>>("indexer_count", TaskReturnSuccessWhile<int>::CONDITION_EQUALS, 0)
),
),
std::make_shared<TaskDecoratorRepeat>(TaskDecoratorRepeat::CONDITION_WHILE_SUCCESS, Task::STATE_SUCCESS)->addChildTask(
std::make_shared<TaskGroupSequence>()->addChildTasks(
// stopping when indexer count is zero, regardless wether there are still storages left to insert.
@@ -527,7 +519,6 @@ void Project::buildIndex(const std::set<FilePath>& filesToClean, bool fullRefres
)
)
);
// add task that notifies the user of what's going on
taskSequential->addTask( // we don't need to hide this dialog again, because it's overridden by other dialogs later on.
std::make_shared<TaskShowStatusDialog>("Finish Indexing", "Saving\nRemaining Data")
src/lib/settings/ApplicationSettings.cpp
+4 -4
View File
@@ -246,14 +246,14 @@ void ApplicationSettings::setIndexerThreadCount(const int count)
setValue<int>("indexing/indexer_thread_count", count);
}
bool ApplicationSettings::getCancelIndexingOnFatalErrors() const
bool ApplicationSettings::getMultiProcessIndexingEnabled() const
{
return getValue<bool>("indexing/cancel_on_fatal_errors", true);
return getValue<bool>("indexing/multi_process_indexing", true);
}
void ApplicationSettings::setCancelIndexingOnFatalErrors(bool enabled)
void ApplicationSettings::setMultiProcessIndexingEnabled(bool enabled)
{
setValue<bool>("indexing/cancel_on_fatal_errors", enabled);
setValue<bool>("indexing/multi_process_indexing", enabled);
}
std::string ApplicationSettings::getJavaPath() const
src/lib/settings/ApplicationSettings.h
+2 -2
View File
@@ -73,8 +73,8 @@ public:
int getIndexerThreadCount() const;
void setIndexerThreadCount(const int count);
bool getCancelIndexingOnFatalErrors() const;
void setCancelIndexingOnFatalErrors(bool enabled);
bool getMultiProcessIndexingEnabled() const;
void setMultiProcessIndexingEnabled(bool enabled);
std::string getJavaPath() const;
void setJavaPath(const std::string path);
src/lib/utility/TimePoint.cpp
+5
View File
@@ -59,6 +59,11 @@ size_t TimePoint::deltaMS(const TimePoint& other) const
return (m_time - other.m_time).total_milliseconds();
}
size_t TimePoint::deltaS(const TimePoint& other) const
{
return (m_time - other.m_time).total_seconds();
}
bool TimePoint::isSameDay(const TimePoint& other) const
{
if (m_time.date().day() == other.m_time.date().day() &&
src/lib/utility/TimePoint.h
+1
View File
@@ -28,6 +28,7 @@ public:
inline float operator-(const TimePoint& rhs){ return deltaMS(rhs) / 1000.0f; }
size_t deltaMS(const TimePoint& other) const;
size_t deltaS(const TimePoint& other) const;
bool isSameDay(const TimePoint& other) const;
size_t deltaDays(const TimePoint& other) const; // days are counted beginning at 00:00, so a tp of 1.1.2017 23:59 is 1 day ago if it's the 2.1.2017 00:01
src/lib/utility/UUIDUtility.cpp
+3 -3
View File
@@ -1,15 +1,15 @@
#include "UUIDUtility.h"
#include <boost/lexical_cast.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <boost/lexical_cast.hpp>
UUID UUIDUtility::getUUID()
boost::uuids::uuid UUIDUtility::getUUID()
{
return boost::uuids::random_generator()();
}
std::string UUIDUtility::UUIDtoString(const UUID& uuid)
std::string UUIDUtility::UUIDtoString(const boost::uuids::uuid& uuid)
{
return boost::lexical_cast<std::string>(uuid);
}
src/lib/utility/UUIDUtility.h
+3 -5
View File
@@ -1,16 +1,14 @@
#ifndef UUID_UTILITY_H
#define UUID_UTILITY_H
#include <string>
#include <boost/uuid/uuid.hpp>
typedef boost::uuids::uuid UUID;
#include <string>
class UUIDUtility
{
public:
static UUID getUUID();
static std::string UUIDtoString(const UUID& uuid);
static boost::uuids::uuid getUUID();
static std::string UUIDtoString(const boost::uuids::uuid& uuid);
static std::string getUUIDString();
};
src/lib/utility/file/FileRegisterStateData.cpp
+21 -5
View File
@@ -13,8 +13,6 @@ FileRegisterStateData::FileRegisterStateData(const FileRegisterStateData& o)
void FileRegisterStateData::inject(const FileRegisterStateData& o)
{
std::lock_guard<std::mutex> oLock(o.m_filePathsMutex);
std::lock_guard<std::mutex> thisLock(m_filePathsMutex);
for (const auto& it: o.m_filePaths)
{
if (it.second == STATE_INDEXED)
@@ -26,13 +24,11 @@ void FileRegisterStateData::inject(const FileRegisterStateData& o)
void FileRegisterStateData::markFileIndexing(const FilePath& filePath)
{
std::lock_guard<std::mutex> lock(m_filePathsMutex);
m_filePaths[filePath] = STATE_INDEXING;
}
void FileRegisterStateData::markIndexingFilesIndexed()
{
std::lock_guard<std::mutex> lock(m_filePathsMutex);
for (auto& it: m_filePaths)
{
if (it.second == STATE_INDEXING)
@@ -44,7 +40,6 @@ void FileRegisterStateData::markIndexingFilesIndexed()
bool FileRegisterStateData::fileIsIndexed(const FilePath& filePath) const
{
std::lock_guard<std::mutex> lock(m_filePathsMutex);
auto it = m_filePaths.find(filePath);
if (it != m_filePaths.end())
{
@@ -53,3 +48,24 @@ bool FileRegisterStateData::fileIsIndexed(const FilePath& filePath) const
return false;
}
void FileRegisterStateData::setIndexedFiles(const std::set<FilePath>& filePaths)
{
for (auto path : filePaths)
{
m_filePaths[path] = STATE_INDEXED;
}
}
std::set<FilePath> FileRegisterStateData::getIndexedFiles() const
{
std::set<FilePath> paths;
for (auto& it : m_filePaths)
{
if (it.second == STATE_INDEXED)
{
paths.insert(it.first);
}
}
return paths;
}
src/lib/utility/file/FileRegisterStateData.h
+5 -2
View File
@@ -1,8 +1,9 @@
#ifndef FILE_REGISTER_STATE_DATA_H
#define FILE_REGISTER_STATE_DATA_H
#include <mutex>
#include <map>
#include <mutex>
#include <set>
#include "utility/file/FilePath.h"
@@ -18,6 +19,9 @@ public:
void markIndexingFilesIndexed();
bool fileIsIndexed(const FilePath& filePath) const;
void setIndexedFiles(const std::set<FilePath>& filePaths);
std::set<FilePath> getIndexedFiles() const;
private:
enum IndexingState
{
@@ -27,7 +31,6 @@ private:
};
std::map<FilePath, IndexingState> m_filePaths;
mutable std::mutex m_filePathsMutex;
};
#endif // FILE_REGISTER_STATE_DATA_H
src/lib/utility/interprocess/InterprocessDataManager.cpp
-110
View File
@@ -1,110 +0,0 @@
#include "InterprocessDataManager.h"
#include <boost/interprocess/managed_shared_memory.hpp>
#include "utility/logging/logging.h"
#include "SharedUUIDManager.h"
#include "InterprocessUtility.h"
#include "SharedQueue.h"
std::string InterprocessDataManager::m_sharedArgumentQueueName = "sourcetrail_parser_arguments";
InterprocessDataManager::InterprocessDataManager(const bool isOwner)
: m_isOwner(isOwner)
, m_initialized(false)
{
}
InterprocessDataManager::~InterprocessDataManager()
{
SharedUUIDManager::getInstance()->removeUUIDsForInstance(
SharedUUIDManager::getInstance()->getInstanceUUID());
}
void InterprocessDataManager::initialize()
{
try
{
m_sharedArgumentQueueName = UUIDUtility::getUUIDString() + "_" + m_sharedArgumentQueueName;
std::string uuid = SharedUUIDManager::getInstance()->getNewUUID();
m_parserArguments.initialize(m_isOwner, uuid.c_str());
m_initialized = true;
}
catch (std::exception& e)
{
LOG_ERROR_STREAM(<< e.what());
}
}
//void InterprocessDataManager::pushParserArguments(const Parser::Arguments& arguments)
//{
// IF_INITIALIZED()
// {
// SharedParserArguments::VoidAllocator allocator(m_parserArguments.getSegmentManager());
// SharedParserArguments args(allocator);
//
// args.setCompilationDatabasePath(arguments.compilationDatabasePath.str());
// args.setCompilerFlags(arguments.compilerFlags);
// args.setFrameworkSearchPaths(arguments.frameworkSearchPaths);
// args.setHeaderSearchPaths(arguments.headerSearchPaths);
// args.setJavaClassPaths(arguments.javaClassPaths);
// args.setLanguage(arguments.language);
// args.setLanguageStandard(arguments.languageStandard);
// args.setLogErrors(arguments.logErrors);
// args.setSystemHeaderSearchPaths(arguments.systemHeaderSearchPaths);
//
// m_parserArguments.pushValue(args);
// }
//}
//
//Parser::Arguments InterprocessDataManager::popParserArguments()
//{
// IF_INITIALIZED(Parser::Arguments())
// {
// if (m_parserArguments.size() > 0)
// {
// SharedParserArguments args = m_parserArguments.popValue();
//
// Parser::Arguments result;
//
// result.compilationDatabasePath = FilePath(args.getCompilationDatabasePath());
// result.compilerFlags = args.getCompilerFlags();
// result.frameworkSearchPaths = args.getFrameworkSearchPaths();
// result.headerSearchPaths = args.getHeaderSearchPaths();
// result.javaClassPaths = args.getJavaClassPaths();
// result.language = args.getLanguage();
// result.languageStandard = args.getLanguageStandard();
// result.logErrors = args.getLogErrors();
// result.systemHeaderSearchPaths = args.getSystemHeaderSearchPaths();
//
// return result;
// }
// }
//}
unsigned int InterprocessDataManager::parserArgumentCount() const
{
IF_INITIALIZED(0)
{
return m_parserArguments.size();
}
}
void InterprocessDataManager::cleanSharedMemory()
{
std::vector<std::string> instances = SharedUUIDManager::getInstance()->getStoredInstanceUUIDs();
for (unsigned int i = 0; i < instances.size(); i++)
{
std::vector<std::string> uuids = SharedUUIDManager::getInstance()->getUUIDsForInstance(instances[i]);
if (uuids.size() == 1) // uuid list has to equal the count of used containers
{
std::string parserArgumentMemName = SharedQueue<SharedParserArguments>::getMemoryNamePrefix() + uuids[0];
boost::interprocess::shared_memory_object::remove(parserArgumentMemName.c_str());
}
}
}
src/lib/utility/interprocess/InterprocessDataManager.h
-38
View File
@@ -1,38 +0,0 @@
#ifndef INTERPROCESS_DATA_MANAGER_H
#define INTERPROCESS_DATA_MANAGER_H
#include "data/parser/Parser.h"
#include "SharedQueue.h"
#include "SharedParserArguments.h"
class InterprocessDataManager
{
public:
InterprocessDataManager(const bool isOwner = false);
~InterprocessDataManager();
void initialize();
// TODO: use IndexerCommands here and rename SharedParserArguments to SharedIndexerCommand
// TODO: either make one SharedIndexerCommand that stores everything or make
// SharedIndexerCommandJava, SharedIndexerCommandCxxManual, ..., each having a separate datastructure here
//void pushParserArguments(const Parser::Arguments& arguments);
//Parser::Arguments popParserArguments();
unsigned int parserArgumentCount() const;
private:
void cleanSharedMemory();
static std::string m_sharedArgumentQueueName;
bool m_isOwner;
bool m_initialized;
SharedQueue<SharedParserArguments> m_parserArguments;
};
#endif // INTERPROCESS_DATA_MANAGER_H
src/lib/utility/interprocess/InterprocessProcessManager.cpp
-65
View File
@@ -1,65 +0,0 @@
#include "InterprocessProcessManager.h"
#include <thread>
#include "utility/logging/logging.h"
InterprocessProcessManager::InterprocessProcessManager()
: m_processName("")
, m_processCount(1)
{
}
InterprocessProcessManager::~InterprocessProcessManager()
{
}
void InterprocessProcessManager::setProcessName(const std::string& processName)
{
m_processName = processName;
}
std::string InterprocessProcessManager::getProcessName() const
{
return m_processName;
}
void InterprocessProcessManager::setProcessCount(const unsigned int processCount)
{
m_processCount = processCount;
}
unsigned int InterprocessProcessManager::getProcessCount() const
{
return m_processCount;
}
void InterprocessProcessManager::runProcesses()
{
std::vector<std::thread*> processThreads;
for (unsigned int i = 0; i < m_processCount; i++)
{
std::thread* thread = new std::thread(&InterprocessProcessManager::runProcess, this);
processThreads.push_back(thread);
}
for (unsigned int i = 0; i < m_processCount; i++)
{
processThreads[i]->join();
delete processThreads[i];
}
}
void InterprocessProcessManager::runProcess()
{
int result = 1;
while (result != 0)
{
result = system(m_processName.c_str());
LOG_INFO_STREAM(<< "Process returned with " << std::to_string(result));
}
}
src/lib/utility/interprocess/InterprocessProcessManager.h
-27
View File
@@ -1,27 +0,0 @@
#ifndef INTERPROCESS_PROCESS_MANAGER_H
#define INTERPROCESS_PROCESS_MANAGER_H
#include <string>
class InterprocessProcessManager
{
public:
InterprocessProcessManager();
~InterprocessProcessManager();
void setProcessName(const std::string& processName);
std::string getProcessName() const;
void setProcessCount(const unsigned int processCount);
unsigned int getProcessCount() const;
void runProcesses();
private:
void runProcess();
std::string m_processName;
unsigned int m_processCount;
};
#endif // INTERPROCESS_PROCESS_MANAGER_H
src/lib/utility/interprocess/InterprocessUtility.h
-15
View File
@@ -1,15 +0,0 @@
#ifndef INTERPROCESS_UTILITY_H
#define INTERPROCESS_UTILITY_H
// requires a bool field 'm_initialized' to be defined
// if 'm_initialized' is false, the value '__retVal__' will be returned
// '__retVal__' is not required
#define IF_INITIALIZED(__retVal__) \
if(m_initialized == false) \
{ \
LOG_ERROR_STREAM(<< "not initialized"); \
return __retVal__; \
} \
else \
#endif // INTERPROCESS_UTILITY_H
src/lib/utility/interprocess/SharedContainer.h
-95
View File
@@ -1,95 +0,0 @@
#ifndef SHARED_CONTAINER_H
#define SHARED_CONTAINER_H
#include <string>
#include <boost/interprocess/managed_shared_memory.hpp>
#include "utility/logging/logging.h"
#include "InterprocessUtility.h"
class SharedContainer
{
public:
static std::string getMemoryNamePrefix();
static std::string getContainerNamePrefix();
SharedContainer();
virtual ~SharedContainer();
virtual bool initialize(const bool isOwner, const std::string& containerName) = 0;
protected:
static const std::string m_memoryNamePrefix;
static const std::string m_containerNamePrefix;
bool initializeSharedMemory(const bool isOwner);
boost::interprocess::managed_shared_memory m_sharedMemory;
bool m_isOwner;
bool m_initialized;
std::string m_memoryName;
std::string m_containerName;
};
const std::string SharedContainer::m_memoryNamePrefix("sourcetrail_memory_");
const std::string SharedContainer::m_containerNamePrefix("sourcetrail_container_");
std::string SharedContainer::getMemoryNamePrefix()
{
return m_memoryNamePrefix;
}
std::string SharedContainer::getContainerNamePrefix()
{
return m_containerNamePrefix;
}
SharedContainer::SharedContainer()
: m_isOwner(false)
, m_initialized(false)
, m_memoryName(m_memoryNamePrefix)
, m_containerName(m_containerNamePrefix)
{
}
SharedContainer::~SharedContainer()
{
}
bool SharedContainer::initializeSharedMemory(const bool isOwner)
{
bool initialized = false;
try
{
if (isOwner)
{
boost::interprocess::shared_memory_object::remove(m_memoryName.c_str());
m_sharedMemory = boost::interprocess::managed_shared_memory(boost::interprocess::create_only,
m_memoryName.c_str(),
65536);
}
else
{
m_sharedMemory = boost::interprocess::managed_shared_memory(boost::interprocess::open_only,
m_memoryName.c_str());
}
initialized = true;
}
catch (std::exception& e)
{
LOG_ERROR(e.what());
}
return initialized;
}
#endif // SHARED_CONTAINER_H
src/lib/utility/interprocess/SharedMap.h
-125
View File
@@ -1,125 +0,0 @@
#ifndef SHARED_DICTIONARY_H
#define SHARED_DICTIONARY_H
#include <string>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/containers/map.hpp>
#include "SharedContainer.h"
template<typename Key, typename Value>
class SharedMap : public SharedContainer
{
public:
SharedMap();
virtual ~SharedMap();
virtual void initialize(const bool isOwner, const std::string& mapName);
Value& operator[](const Key& key);
unsigned int size() const;
void clear();
boost::interprocess::managed_shared_memory::segment_manager* getSegmentManager() const;
private:
typedef std::pair<Key, Value> ValueType;
typedef boost::interprocess::allocator<ValueType, boost::interprocess::managed_shared_memory::segment_manager> ShmemAllocator;
typedef boost::interprocess::map<Key, Value, std::less<Key>, ShmemAllocator> ShmemMap;
ShmemMap* m_map;
};
template<typename Key, typename Value>
SharedMap<Key, Value>::SharedMap()
: m_map(NULL)
{
}
template<typename Key, typename Value>
SharedMap<Key, Value>::~SharedMap()
{
if (m_map != NULL && m_initialized && m_isOwner)
{
m_sharedMemory.destroy<ShmemDeque>(m_containerName.c_str());
boost::interprocess::shared_memory_object::remove(m_memoryName.c_str());
delete m_map;
}
}
template<typename Key, typename Value>
void SharedMap<Key, Value>::initialize(const bool isOwner, const std::string& mapName)
{
m_isOwner = isOwner;
m_containerName = m_containerNamePrefix + dequeName;
m_memoryName = m_memoryNamePrefix + m_containerName;
if (initializeSharedMemory(m_isOwner))
{
try
{
if (m_isOwner)
{
m_sharedMemory.destroy<ShmemDeque>(m_containerName.c_str());
const ShmemAllocator allocator(m_sharedMemory.get_segment_manager());
m_map = m_sharedMemory.construct<ShmemMap>(m_containerName.c_str())(allocator);
}
else
{
m_map = m_sharedMemory.find<ShmemMap>(m_containerName.c_str()).first;
}
m_initialized = true;
}
catch (std::exception& e)
{
LOG_ERROR_STREAM(<< e.what());
}
}
return m_initialized;
}
template<typename Key, typename Value>
Value& SharedMap<Key, Value>::operator[](const Key& key)
{
IF_INITIALIZED(NULL)
{
return m_map->at(key);
}
}
template<typename Key, typename Value>
unsigned int SharedMap<Key, Value>::size() const
{
IF_INITIALIZED(0)
{
return m_map->size();
}
}
template<typename Key, typename Value>
void SharedMap<Key, Value>::clear()
{
IF_INITIALIZED()
{
m_map->clear();
}
}
template<typename Key, typename Value>
boost::interprocess::managed_shared_memory::segment_manager* SharedMap<Key, Value>::getSegmentManager() const
{
IF_INITIALIZED(NULL)
{
return m_sharedMemory.get_segment_manager();
}
}
#endif // SHARED_DICTIONARY_H
src/lib/utility/interprocess/SharedMemory.cpp
+145
View File
@@ -0,0 +1,145 @@
#include "utility/interprocess/SharedMemory.h"
#include "utility/interprocess/SharedMemoryGarbageCollector.h"
#include "utility/logging/logging.h"
const char* SharedMemory::s_memoryNamePrefix = "srctrlmem_";
const char* SharedMemory::s_mutexNamePrefix = "srctrlmtx_";
SharedMemory::ScopedAccess::ScopedAccess(SharedMemory* memory)
: boost::interprocess::scoped_lock<boost::interprocess::named_mutex>(memory->getMutex())
, m_memory(boost::interprocess::open_only, memory->getMemoryName().c_str())
, m_memoryName(memory->getMemoryName())
{
}
SharedMemory::ScopedAccess::~ScopedAccess()
{
}
SharedMemory::Allocator* SharedMemory::ScopedAccess::getAllocator()
{
return m_memory.get_segment_manager();
}
size_t SharedMemory::ScopedAccess::getMemorySize() const
{
return m_memory.get_size();
}
size_t SharedMemory::ScopedAccess::getFreeMemorySize() const
{
return m_memory.get_free_memory();
}
void SharedMemory::ScopedAccess::growMemory(size_t size)
{
m_memory = boost::interprocess::managed_shared_memory();
boost::interprocess::managed_shared_memory::grow(m_memoryName.c_str(), size);
m_memory = boost::interprocess::managed_shared_memory(boost::interprocess::open_only, m_memoryName.c_str());
}
std::string SharedMemory::ScopedAccess::logString() const
{
std::string log = m_memoryName + " -";
log += " size: " + std::to_string(getMemorySize());
log += " free: " + std::to_string(getFreeMemorySize());
log += " used: " + std::to_string(getMemorySize() - getFreeMemorySize());
log += " pct: " + std::to_string(100 - int(float(getFreeMemorySize()) / getMemorySize() * 100));
return log;
}
std::string SharedMemory::checkName(const std::string& name)
{
return name.size() > 18 ? name.substr(0, 18) : name;
}
void SharedMemory::deleteSharedMemory(const std::string& name)
{
boost::interprocess::shared_memory_object::remove((s_memoryNamePrefix + name).c_str());
boost::interprocess::named_mutex::remove((s_mutexNamePrefix + name).c_str());
}
SharedMemory::SharedMemory(const std::string& name, size_t initialMemorySize, AccessMode mode)
: m_name(checkName(name))
, m_mode(mode)
{
bool unlockMutex = true;
switch (mode)
{
case CREATE_AND_DELETE:
{
SharedMemoryGarbageCollector* collector = SharedMemoryGarbageCollector::getInstance();
if (collector)
{
collector->registerSharedMemory(m_name);
}
}
deleteSharedMemory(m_name);
boost::interprocess::managed_shared_memory(
boost::interprocess::create_only, getMemoryName().c_str(), initialMemorySize);
boost::interprocess::named_mutex(boost::interprocess::create_only, getMutexName().c_str());
break;
case OPEN_ONLY:
boost::interprocess::managed_shared_memory(
boost::interprocess::open_only, getMemoryName().c_str());
boost::interprocess::named_mutex(boost::interprocess::open_only, getMutexName().c_str());
unlockMutex = false;
break;
case OPEN_OR_CREATE:
boost::interprocess::managed_shared_memory(
boost::interprocess::open_or_create, getMemoryName().c_str(), initialMemorySize);
boost::interprocess::named_mutex(boost::interprocess::open_or_create, getMutexName().c_str());
break;
}
if (unlockMutex)
{
boost::interprocess::named_mutex mutex(boost::interprocess::open_only, getMutexName().c_str());
mutex.try_lock();
mutex.unlock();
}
}
SharedMemory::~SharedMemory()
{
if (m_mode == CREATE_AND_DELETE)
{
SharedMemoryGarbageCollector* collector = SharedMemoryGarbageCollector::getInstance();
if (collector)
{
collector->unregisterSharedMemory(m_name);
}
deleteSharedMemory(m_name);
}
}
std::string SharedMemory::getMemoryName() const
{
return s_memoryNamePrefix + m_name;
}
std::string SharedMemory::getMutexName() const
{
return s_mutexNamePrefix + m_name;
}
boost::interprocess::named_mutex& SharedMemory::getMutex()
{
if (!m_mutex)
{
m_mutex = std::make_shared<boost::interprocess::named_mutex>(
boost::interprocess::open_only, getMutexName().c_str());
}
return *m_mutex.get();
}
src/lib/utility/interprocess/SharedMemory.h
+116
View File
@@ -0,0 +1,116 @@
#ifndef SHARED_MEMORY_H
#define SHARED_MEMORY_H
#include <string>
#include <boost/interprocess/containers/deque.hpp>
#include <boost/interprocess/containers/map.hpp>
#include <boost/interprocess/containers/set.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <boost/interprocess/containers/vector.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/sync/named_mutex.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
class SharedMemory
{
public:
enum AccessMode
{
CREATE_AND_DELETE,
OPEN_ONLY,
OPEN_OR_CREATE
};
using Allocator = boost::interprocess::managed_shared_memory::segment_manager;
using String = boost::interprocess::basic_string<char, std::char_traits<char>,
boost::interprocess::allocator<char, Allocator>>;
template <typename T>
using Vector = boost::interprocess::vector<T, boost::interprocess::allocator<T, Allocator>>;
template <typename T>
using Queue = boost::interprocess::deque<T, boost::interprocess::allocator<T, Allocator>>;
template <typename T, typename T2>
using Map = boost::interprocess::map<T, T2, std::less<T>,
boost::interprocess::allocator<std::pair<const T, T2>, Allocator>>;
template <typename T>
using Set = boost::interprocess::set<T, std::less<T>, boost::interprocess::allocator<T, Allocator>>;
// Names addressing shared memory objects longer than 29 characters can throw and exception
static std::string checkName(const std::string& name);
static void deleteSharedMemory(const std::string& name);
SharedMemory(const std::string& name, size_t initialMemorySize, AccessMode mode);
~SharedMemory();
class ScopedAccess
: public boost::interprocess::scoped_lock<boost::interprocess::named_mutex>
{
public:
ScopedAccess(SharedMemory* memory);
~ScopedAccess();
Allocator* getAllocator();
size_t getMemorySize() const;
size_t getFreeMemorySize() const;
void growMemory(size_t size);
template <typename T>
T* accessValue(const std::string& key)
{
return m_memory.find_or_construct<T>(key.c_str())();
}
template <typename T>
T* accessValues(const std::string& key, size_t count)
{
return m_memory.find_or_construct<T>(key.c_str())[count]();
}
template <typename T>
T* accessValueWithAllocator(const std::string& key)
{
return m_memory.find_or_construct<T>(key.c_str())(getAllocator());
}
template <typename T>
T* accessValuesWithAllocator(const std::string& key, size_t count)
{
return m_memory.find_or_construct<T>(key.c_str())[count](getAllocator());
}
template <typename T>
void destroyValue(const std::string& key)
{
m_memory.destroy<T>(key.c_str());
}
std::string logString() const;
private:
boost::interprocess::managed_shared_memory m_memory;
std::string m_memoryName;
};
private:
static const char* s_memoryNamePrefix;
static const char* s_mutexNamePrefix;
std::string getMemoryName() const;
std::string getMutexName() const;
boost::interprocess::named_mutex& getMutex();
std::shared_ptr<boost::interprocess::named_mutex> m_mutex;
std::string m_name;
AccessMode m_mode;
};
#endif // SHARED_MEMORY_H
src/lib/utility/interprocess/SharedMemoryGarbageCollector.cpp
+242
View File
@@ -0,0 +1,242 @@
#include "utility/interprocess/SharedMemoryGarbageCollector.h"
#include <thread>
#include "utility/logging/logging.h"
#include "utility/TimePoint.h"
std::string SharedMemoryGarbageCollector::s_memoryName = "garbage_collector";
std::string SharedMemoryGarbageCollector::s_instancesKeyName = "running_instances";
std::string SharedMemoryGarbageCollector::s_timeStampsKeyName = "memory_to_timestamps";
const size_t SharedMemoryGarbageCollector::s_updateIntervalSeconds = 1;
const size_t SharedMemoryGarbageCollector::s_deleteThresholdSeconds = 10;
std::shared_ptr<SharedMemoryGarbageCollector> SharedMemoryGarbageCollector::s_instance;
SharedMemoryGarbageCollector* SharedMemoryGarbageCollector::createInstance()
{
if (!s_instance)
{
s_instance = std::shared_ptr<SharedMemoryGarbageCollector>(new SharedMemoryGarbageCollector());
}
return s_instance.get();
}
SharedMemoryGarbageCollector* SharedMemoryGarbageCollector::getInstance()
{
return s_instance.get();
}
SharedMemoryGarbageCollector::SharedMemoryGarbageCollector()
: m_memory(s_memoryName, 65536 /* 64 kB */, SharedMemory::OPEN_OR_CREATE)
, m_loopIsRunning(false)
{
}
SharedMemoryGarbageCollector::~SharedMemoryGarbageCollector()
{
}
void SharedMemoryGarbageCollector::run(const std::string& uuid)
{
LOG_INFO_STREAM(<< "start shared memory garbage collection");
m_uuid = uuid;
std::thread(
[this]()
{
m_loopIsRunning = true;
while (m_loopIsRunning)
{
update();
std::this_thread::sleep_for(std::chrono::seconds(s_updateIntervalSeconds));
}
}
).detach();
}
void SharedMemoryGarbageCollector::stop()
{
LOG_INFO_STREAM(<< "stop shared memory garbage collection");
m_loopIsRunning = false;
{
std::lock_guard<std::mutex> lock(m_sharedMemoryNamesMutex);
m_removedSharedMemoryNames.insert(m_sharedMemoryNames.begin(), m_sharedMemoryNames.end());
}
update();
m_sharedMemoryNames.clear();
m_removedSharedMemoryNames.clear();
SharedMemory::ScopedAccess access(&m_memory);
SharedMemory::Map<SharedMemory::String, SharedMemory::String>* instances =
access.accessValueWithAllocator<SharedMemory::Map<SharedMemory::String, SharedMemory::String>>(s_instancesKeyName);
if (!instances)
{
return;
}
SharedMemory::String i(access.getAllocator());
i = m_uuid.c_str();
SharedMemory::Map<SharedMemory::String, SharedMemory::String>::iterator itt = instances->find(i);
if (itt != instances->end())
{
instances->erase(itt);
}
bool otherRunningInstances = false;
TimePoint now = TimePoint::now();
for (SharedMemory::Map<SharedMemory::String, SharedMemory::String>::iterator it = instances->begin();
it != instances->end(); it++)
{
TimePoint timestamp = TimePoint(std::string(it->second.c_str()));
if (now.deltaS(timestamp) <= s_deleteThresholdSeconds)
{
otherRunningInstances = true;
LOG_INFO_STREAM(<< "currently running instance: " << it->first.c_str());
}
}
if (!otherRunningInstances)
{
LOG_INFO_STREAM(<< "delete garbage collector memory: " << s_memoryName);
SharedMemory::deleteSharedMemory(s_memoryName);
}
}
void SharedMemoryGarbageCollector::registerSharedMemory(const std::string& sharedMemoryName)
{
{
std::lock_guard<std::mutex> lock(m_sharedMemoryNamesMutex);
m_sharedMemoryNames.insert(sharedMemoryName);
}
update();
}
void SharedMemoryGarbageCollector::unregisterSharedMemory(const std::string& sharedMemoryName)
{
{
std::lock_guard<std::mutex> lock(m_sharedMemoryNamesMutex);
size_t removedCount = m_sharedMemoryNames.erase(sharedMemoryName);
if (removedCount > 0)
{
m_removedSharedMemoryNames.insert(sharedMemoryName);
}
}
update();
}
void SharedMemoryGarbageCollector::update()
{
std::lock_guard<std::mutex> lock(m_sharedMemoryNamesMutex);
SharedMemory::ScopedAccess access(&m_memory);
if (access.getFreeMemorySize() * 2 < access.getMemorySize())
{
access.growMemory(access.getMemorySize());
LOG_INFO(access.logString());
}
SharedMemory::String t(access.getAllocator());
t = TimePoint::now().toString().c_str();
// update instances
{
SharedMemory::Map<SharedMemory::String, SharedMemory::String>* instances =
access.accessValueWithAllocator<SharedMemory::Map<SharedMemory::String, SharedMemory::String>>(s_instancesKeyName);
if (!instances)
{
return;
}
SharedMemory::String i(access.getAllocator());
i = m_uuid.c_str();
SharedMemory::Map<SharedMemory::String, SharedMemory::String>::iterator it = instances->find(i);
if (it != instances->end())
{
it->second = t;
}
else
{
instances->insert(std::pair<SharedMemory::String, SharedMemory::String>(i, t));
}
}
// update shared memories
{
SharedMemory::Map<SharedMemory::String, SharedMemory::String>* timeStamps =
access.accessValueWithAllocator<SharedMemory::Map<SharedMemory::String, SharedMemory::String>>(s_timeStampsKeyName);
if (!timeStamps)
{
return;
}
// remove deleted shared memories
for (const std::string& name : m_removedSharedMemoryNames)
{
SharedMemory::String n(access.getAllocator());
n = name.c_str();
SharedMemory::Map<SharedMemory::String, SharedMemory::String>::iterator it = timeStamps->find(n);
if (it != timeStamps->end())
{
timeStamps->erase(it);
}
}
m_removedSharedMemoryNames.clear();
// add or update shared memories
for (const std::string& name : m_sharedMemoryNames)
{
SharedMemory::String n(access.getAllocator());
n = name.c_str();
SharedMemory::Map<SharedMemory::String, SharedMemory::String>::iterator it = timeStamps->find(n);
if (it != timeStamps->end())
{
it->second = t;
}
else
{
timeStamps->insert(std::pair<SharedMemory::String, SharedMemory::String>(n, t));
}
}
// delete old shared memories
TimePoint now = TimePoint::now();
for (SharedMemory::Map<SharedMemory::String, SharedMemory::String>::iterator it = timeStamps->begin();
it != timeStamps->end();)
{
TimePoint timestamp = TimePoint(std::string(it->second.c_str()));
if (now.deltaS(timestamp) > s_deleteThresholdSeconds)
{
LOG_INFO_STREAM(<< "collect garbage: " << it->first.c_str());
SharedMemory::deleteSharedMemory(it->first.c_str());
timeStamps->erase(it++);
}
else
{
it++;
}
}
}
}
src/lib/utility/interprocess/SharedMemoryGarbageCollector.h
+49
View File
@@ -0,0 +1,49 @@
#ifndef SHARED_MEMORY_GARBAGE_COLLECTOR_H
#define SHARED_MEMORY_GARBAGE_COLLECTOR_H
#include <mutex>
#include <string>
#include <set>
#include "utility/interprocess/SharedMemory.h"
#include <mutex>
class SharedMemoryGarbageCollector
{
public:
static SharedMemoryGarbageCollector* createInstance();
static SharedMemoryGarbageCollector* getInstance();
SharedMemoryGarbageCollector();
~SharedMemoryGarbageCollector();
void run(const std::string& uuid);
void stop();
void registerSharedMemory(const std::string& sharedMemoryName);
void unregisterSharedMemory(const std::string& sharedMemoryName);
private:
void update();
static std::string s_memoryName;
static std::string s_instancesKeyName;
static std::string s_timeStampsKeyName;
static const size_t s_updateIntervalSeconds;
static const size_t s_deleteThresholdSeconds;
static std::shared_ptr<SharedMemoryGarbageCollector> s_instance;
SharedMemory m_memory;
volatile bool m_loopIsRunning;
std::string m_uuid;
std::mutex m_sharedMemoryNamesMutex;
std::set<std::string> m_sharedMemoryNames;
std::set<std::string> m_removedSharedMemoryNames;
};
#endif // SHARED_MEMORY_GARBAGE_COLLECTOR_H
src/lib/utility/interprocess/SharedParserArguments.cpp
-173
View File
@@ -1,173 +0,0 @@
#include "SharedParserArguments.h"
SharedParserArguments::SharedParserArguments(const VoidAllocator& allocator)
: m_logErrors(false)
, m_language("", allocator)
, m_languageStandard("", allocator)
, m_compilationDatabasePath("", allocator)
, m_javaClassPaths(allocator)
, m_headerSearchPaths(allocator)
, m_systemHeaderSearchPaths(allocator)
, m_frameworkSearchPaths(allocator)
, m_compilerFlags(allocator)
{
}
SharedParserArguments::~SharedParserArguments()
{
}
void SharedParserArguments::setLogErrors(const bool logErrors)
{
m_logErrors = logErrors;
}
bool SharedParserArguments::getLogErrors() const
{
return m_logErrors;
}
void SharedParserArguments::setLanguage(const std::string& language)
{
m_language = language.c_str();
}
std::string SharedParserArguments::getLanguage() const
{
return std::string(m_language.c_str());
}
void SharedParserArguments::setLanguageStandard(const std::string& languageStandard)
{
m_languageStandard = languageStandard.c_str();
}
std::string SharedParserArguments::getLanguageStandard() const
{
return std::string(m_languageStandard.c_str());
}
void SharedParserArguments::setCompilationDatabasePath(const std::string& compilationDatabasePath)
{
m_compilationDatabasePath = compilationDatabasePath.c_str();
}
std::string SharedParserArguments::getCompilationDatabasePath() const
{
return std::string(m_compilationDatabasePath.c_str());
}
void SharedParserArguments::setJavaClassPaths(std::vector<FilePath> javaClassPaths)
{
m_javaClassPaths.clear();
for (unsigned int i = 0; i < javaClassPaths.size(); i++)
{
SharedString path(m_javaClassPaths.get_allocator());
path = javaClassPaths[i].str().c_str();
m_javaClassPaths.push_back(path);
}
}
std::vector<FilePath> SharedParserArguments::getJavaClassPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_javaClassPaths.size(); i++)
{
result.push_back(FilePath(m_javaClassPaths[i].c_str()));
}
return result;
}
void SharedParserArguments::setHeaderSearchPaths(std::vector<FilePath> headerSearchPaths)
{
m_headerSearchPaths.clear();
for (unsigned int i = 0; i < headerSearchPaths.size(); i++)
{
SharedString path(m_headerSearchPaths.get_allocator());
path = headerSearchPaths[i].str().c_str();
m_headerSearchPaths.push_back(path);
}
}
std::vector<FilePath> SharedParserArguments::getHeaderSearchPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_headerSearchPaths.size(); i++)
{
result.push_back(FilePath(m_headerSearchPaths[i].c_str()));
}
return result;
}
void SharedParserArguments::setSystemHeaderSearchPaths(std::vector<FilePath> systemHeaderSearchPaths)
{
m_systemHeaderSearchPaths.clear();
for (unsigned int i = 0; i < systemHeaderSearchPaths.size(); i++)
{
SharedString path(m_systemHeaderSearchPaths.get_allocator());
path = systemHeaderSearchPaths[i].str().c_str();
m_systemHeaderSearchPaths.push_back(path);
}
}
std::vector<FilePath> SharedParserArguments::getSystemHeaderSearchPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_systemHeaderSearchPaths.size(); i++)
{
result.push_back(FilePath(m_systemHeaderSearchPaths[i].c_str()));
}
return result;
}
void SharedParserArguments::setFrameworkSearchPaths(std::vector<FilePath> frameworkSearchPaths)
{
m_frameworkSearchPaths.clear();
for (unsigned int i = 0; i < frameworkSearchPaths.size(); i++)
{
SharedString path(m_frameworkSearchPaths.get_allocator());
path = frameworkSearchPaths[i].str().c_str();
m_frameworkSearchPaths.push_back(path);
}
}
std::vector<FilePath> SharedParserArguments::getFrameworkSearchPaths() const
{
std::vector<FilePath> result;
for (unsigned int i = 0; i < m_frameworkSearchPaths.size(); i++)
{
result.push_back(FilePath(m_frameworkSearchPaths[i].c_str()));
}
return result;
}
void SharedParserArguments::setCompilerFlags(std::vector<std::string> compilerFlags)
{
m_compilerFlags.clear();
for (unsigned int i = 0; i < compilerFlags.size(); i++)
{
SharedString path(m_compilerFlags.get_allocator());
path = compilerFlags[i].c_str();
m_compilerFlags.push_back(path);
}
}
std::vector<std::string> SharedParserArguments::getCompilerFlags() const
{
std::vector<std::string> result;
for (unsigned int i = 0; i < m_compilerFlags.size(); i++)
{
result.push_back(m_compilerFlags[i].c_str());
}
return result;
}
src/lib/utility/interprocess/SharedParserArguments.h
-67
View File
@@ -1,67 +0,0 @@
#ifndef SHARED_PARSER_ARGUMENTS_H
#define SHARED_PARSER_ARGUMENTS_H
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <boost/interprocess/containers/vector.hpp>
#include "utility/file/FilePath.h"
class SharedParserArguments
{
public:
typedef boost::interprocess::allocator<void, boost::interprocess::managed_shared_memory::segment_manager> VoidAllocator;
SharedParserArguments(const VoidAllocator& allocator);
~SharedParserArguments();
void setLogErrors(const bool logErrors);
bool getLogErrors() const;
void setLanguage(const std::string& language);
std::string getLanguage() const;
void setLanguageStandard(const std::string& languageStandard);
std::string getLanguageStandard() const;
void setCompilationDatabasePath(const std::string& compilationDatabasePath);
std::string getCompilationDatabasePath() const;
void setJavaClassPaths(std::vector<FilePath> javaClassPaths);
std::vector<FilePath> getJavaClassPaths() const;
void setHeaderSearchPaths(std::vector<FilePath> headerSearchPaths);
std::vector<FilePath> getHeaderSearchPaths() const;
void setSystemHeaderSearchPaths(std::vector<FilePath> systemHeaderSearchPaths);
std::vector<FilePath> getSystemHeaderSearchPaths() const;
void setFrameworkSearchPaths(std::vector<FilePath> frameworkSearchPaths);
std::vector<FilePath> getFrameworkSearchPaths() const;
void setCompilerFlags(std::vector<std::string> compilerFlags);
std::vector<std::string> getCompilerFlags() const;
private:
typedef boost::interprocess::allocator<char, boost::interprocess::managed_shared_memory::segment_manager> CharAllocator;
typedef boost::interprocess::basic_string<char, std::char_traits<char>, CharAllocator> SharedString;
typedef boost::interprocess::allocator<SharedString, boost::interprocess::managed_shared_memory::segment_manager> StringAllocator;
typedef boost::interprocess::vector<SharedString, StringAllocator> SharedStringVector;
bool m_logErrors;
SharedString m_language;
SharedString m_languageStandard;
SharedString m_compilationDatabasePath;
SharedStringVector m_javaClassPaths;
SharedStringVector m_headerSearchPaths;
SharedStringVector m_systemHeaderSearchPaths;
SharedStringVector m_frameworkSearchPaths;
SharedStringVector m_compilerFlags;
};
#endif // SHARED_PARSER_ARGUMENTS_H
src/lib/utility/interprocess/SharedQueue.h
-128
View File
@@ -1,128 +0,0 @@
#ifndef SHARED_QUEUE_H
#define SHARED_QUEUE_H
#include <boost/interprocess/containers/deque.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include "SharedContainer.h"
template<typename T>
class SharedQueue : public SharedContainer
{
public:
SharedQueue();
virtual ~SharedQueue();
virtual bool initialize(const bool isOwner, const std::string& dequeName);
void pushValue(const T& val);
T popValue();
unsigned int size() const;
boost::interprocess::managed_shared_memory::segment_manager* getSegmentManager() const;
private:
typedef boost::interprocess::allocator<T, boost::interprocess::managed_shared_memory::segment_manager> ShmemAllocator;
typedef boost::interprocess::deque<T, ShmemAllocator> ShmemDeque;
ShmemDeque* m_deque;
};
template<typename T>
SharedQueue<T>::SharedQueue()
: m_deque(NULL)
{
}
template<typename T>
SharedQueue<T>::~SharedQueue()
{
if (m_deque != NULL && m_initialized && m_isOwner)
{
m_sharedMemory.destroy<ShmemDeque>(m_containerName.c_str());
boost::interprocess::shared_memory_object::remove(m_memoryName.c_str());
delete m_deque;
}
}
template<typename T>
bool SharedQueue<T>::initialize(const bool isOwner, const std::string& dequeName)
{
m_isOwner = isOwner;
m_containerName = m_containerNamePrefix + dequeName;
m_memoryName = m_memoryNamePrefix + m_containerName;
if (initializeSharedMemory(m_isOwner))
{
try
{
if (isOwner)
{
m_sharedMemory.destroy<ShmemDeque>(m_containerName.c_str());
const ShmemAllocator allocator(m_sharedMemory.get_segment_manager());
m_deque = m_sharedMemory.construct<ShmemDeque>(m_containerName.c_str())(allocator);
}
else
{
m_deque = m_sharedMemory.find<ShmemDeque>(m_containerName.c_str()).first;
}
m_initialized = true;
}
catch (std::exception& e)
{
LOG_ERROR(e.what());
}
}
return m_initialized;
}
template<typename T>
void SharedQueue<T>::pushValue(const T& val)
{
IF_INITIALIZED()
{
m_deque->push_back(val);
}
}
template<typename T>
T SharedQueue<T>::popValue()
{
if (m_initialized)
{
T val = m_deque->front();
m_deque->pop_front();
return val;
}
else
{
throw(std::runtime_error("Deque was not initialized"));
}
}
template<typename T>
unsigned int SharedQueue<T>::size() const
{
IF_INITIALIZED(0)
{
return m_deque->size();
}
}
template<typename T>
boost::interprocess::managed_shared_memory::segment_manager* SharedQueue<T>::getSegmentManager() const
{
IF_INITIALIZED(NULL)
{
return m_sharedMemory.get_segment_manager();
}
}
#endif // SHARED_QUEUE_H
src/lib/utility/interprocess/SharedUUIDManager.h
-47
View File
@@ -1,47 +0,0 @@
#ifndef SHARED_UUID_MANAGER_H
#define SHARED_UUID_MANAGER_H
#include <string>
#include <memory>
#include "utility/UUIDUtility.h"
#include "utility/ConfigManager.h"
class SharedUUIDManager
{
public:
static std::shared_ptr<SharedUUIDManager> getInstance();
~SharedUUIDManager();
void setFilePath(const std::string& filePath);
std::string getInstanceUUID() const;
std::string getNewUUID();
std::vector<std::string> getUUIDsForInstance(const std::string& instanceUUID);
void removeUUIDsForInstance(const std::string& instanceUUID);
std::vector<std::string> getStoredInstanceUUIDs() const;
void removeInstanceUUID(const std::string& instanceUUID);
private:
SharedUUIDManager();
void saveInstanceUUID();
void refreshUUIDs();
static const std::string m_fileName;
static const std::string m_instanceUUIDsKey;
static std::shared_ptr<SharedUUIDManager> m_instance;
std::string m_filePath;
const UUID m_instanceUUID;
std::shared_ptr<ConfigManager> m_uuids;
};
#endif // SHARED_UUID_MANAGER_H
src/lib/utility/logging/FileLogger.cpp
+25 -7
View File
@@ -8,7 +8,7 @@
FileLogger::FileLogger()
: Logger("FileLogger")
, m_logFileName()
, m_logFileName("log")
, m_logDirectory("user/log/")
, m_maxLogLineCount(0)
, m_maxLogFileCount(0)
@@ -22,6 +22,17 @@ FileLogger::~FileLogger()
{
}
FilePath FileLogger::getLogFilePath() const
{
return m_currentLogFilePath;
}
void FileLogger::setLogFilePath(const FilePath& filePath)
{
m_currentLogFilePath = filePath;
m_logFileName = "";
}
void FileLogger::setLogDirectory(const FilePath& filePath)
{
m_logDirectory = filePath;
@@ -66,12 +77,17 @@ void FileLogger::setMaxLogFileCount(unsigned int fileCount)
void FileLogger::updateLogFileName()
{
if (!m_logFileName.size())
{
return;
}
bool fileChanged = false;
m_currentLogFileName = m_logFileName;
std::string currentLogFilePath = m_logDirectory.str() + m_logFileName;
if (m_maxLogFileCount > 0)
{
m_currentLogFileName += "_";
currentLogFilePath += "_";
if (m_currentLogLineCount >= m_maxLogLineCount)
{
m_currentLogLineCount = 0;
@@ -83,21 +99,23 @@ void FileLogger::updateLogFileName()
}
fileChanged = true;
}
m_currentLogFileName += std::to_string(m_currentLogFileCount);
currentLogFilePath += std::to_string(m_currentLogFileCount);
}
m_currentLogFileName += ".txt";
currentLogFilePath += ".txt";
m_currentLogFilePath = FilePath(currentLogFilePath);
if (fileChanged)
{
FileSystem::remove(m_logDirectory.concat(FilePath(m_currentLogFileName)));
FileSystem::remove(m_currentLogFilePath);
}
}
void FileLogger::logMessage(const std::string& type, const LogMessage& message)
{
std::ofstream fileStream;
fileStream.open(m_logDirectory.concat(FilePath(m_currentLogFileName)).str(), std::ios::app);
fileStream.open(m_currentLogFilePath.str(), std::ios::app);
fileStream << message.getTimeString("%H:%M:%S") << " | ";
fileStream << message.threadId << " | ";
src/lib/utility/logging/FileLogger.h
+5 -2
View File
@@ -13,6 +13,9 @@ public:
FileLogger();
virtual ~FileLogger();
FilePath getLogFilePath() const;
void setLogFilePath(const FilePath& filePath);
void setLogDirectory(const FilePath& filePath);
void setFileName(const std::string& fileName);
void setMaxLogLineCount(unsigned int logCount);
@@ -30,12 +33,12 @@ private:
std::string m_logFileName;
FilePath m_logDirectory;
FilePath m_currentLogFilePath;
unsigned int m_maxLogLineCount;
unsigned int m_maxLogFileCount;
unsigned int m_currentLogLineCount;
unsigned int m_currentLogFileCount;
std::string m_currentLogFileName;
};
#endif // FILE_LOGGER_H
src/lib/utility/scheduling/TaskGroupParallel.cpp
+12 -6
View File
@@ -7,6 +7,7 @@
TaskGroupParallel::TaskGroupParallel()
: m_needsToStartThreads(true)
, m_activeTaskCountMutex(std::make_shared<std::mutex>())
{
}
@@ -30,7 +31,8 @@ void TaskGroupParallel::doEnter(std::shared_ptr<Blackboard> blackboard)
for (size_t i = 0; i < m_tasks.size(); i++)
{
m_tasks[i]->active = true;
m_tasks[i]->thread = std::make_shared<std::thread>(&TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard);
m_tasks[i]->thread = std::make_shared<std::thread>(
&TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard, m_activeTaskCountMutex);
}
}
}
@@ -65,12 +67,13 @@ void TaskGroupParallel::doReset(std::shared_ptr<Blackboard> blackboard)
if (!m_tasks[i]->active)
{
{
std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
std::lock_guard<std::mutex> lock(*m_activeTaskCountMutex.get());
m_activeTaskCount++;
}
m_tasks[i]->thread->join();
m_tasks[i]->active = true;
m_tasks[i]->thread = std::make_shared<std::thread>(&TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard);
m_tasks[i]->thread = std::make_shared<std::thread>(
&TaskGroupParallel::processTaskThreaded, this, m_tasks[i], blackboard, m_activeTaskCountMutex);
}
}
}
@@ -88,10 +91,13 @@ void TaskGroupParallel::doTerminate()
}
}
void TaskGroupParallel::processTaskThreaded(std::shared_ptr<TaskInfo> taskInfo, std::shared_ptr<Blackboard> blackboard)
void TaskGroupParallel::processTaskThreaded(
std::shared_ptr<TaskInfo> taskInfo,
std::shared_ptr<Blackboard> blackboard,
std::shared_ptr<std::mutex> activeTaskCountMutex)
{
ScopedFunctor functor([&](){
std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
std::lock_guard<std::mutex> lock(*activeTaskCountMutex.get());
m_activeTaskCount--;
});
@@ -113,6 +119,6 @@ void TaskGroupParallel::processTaskThreaded(std::shared_ptr<TaskInfo> taskInfo,
int TaskGroupParallel::getActiveTaskCount() const
{
std::lock_guard<std::mutex> lock(m_activeTaskCountMutex);
std::lock_guard<std::mutex> lock(*m_activeTaskCountMutex.get());
return m_activeTaskCount;
}
src/lib/utility/scheduling/TaskGroupParallel.h
+5 -2
View File
@@ -35,7 +35,10 @@ private:
virtual void doReset(std::shared_ptr<Blackboard> blackboard);
virtual void doTerminate();
void processTaskThreaded(std::shared_ptr<TaskInfo> taskInfo, std::shared_ptr<Blackboard> blackboard);
void processTaskThreaded(
std::shared_ptr<TaskInfo> taskInfo,
std::shared_ptr<Blackboard> blackboard,
std::shared_ptr<std::mutex> activeTaskCountMutex);
int getActiveTaskCount() const;
std::vector<std::shared_ptr<TaskInfo>> m_tasks;
@@ -43,7 +46,7 @@ private:
volatile bool m_taskFailed;
volatile int m_activeTaskCount;
mutable std::mutex m_activeTaskCountMutex;
mutable std::shared_ptr<std::mutex> m_activeTaskCountMutex;
};
#endif // TASK_GROUP_PARALLEL_H
src/lib/utility/scheduling/TaskRunner.cpp
+21 -4
View File
@@ -1,5 +1,8 @@
#include "utility/scheduling/TaskRunner.h"
#include "utility/logging/logging.h"
#include "utility/scheduling/TaskScheduler.h"
TaskRunner::TaskRunner(std::shared_ptr<Task> task)
: m_task(task)
, m_reset(false)
@@ -12,13 +15,27 @@ TaskRunner::~TaskRunner()
Task::TaskState TaskRunner::update(std::shared_ptr<Blackboard> blackboard)
{
if (m_reset)
try
{
m_task->reset(blackboard);
m_reset = false;
if (m_reset)
{
m_task->reset(blackboard);
m_reset = false;
}
return m_task->update(blackboard);
}
catch (std::exception& e)
{
LOG_ERROR(e.what());
}
catch (...)
{
LOG_ERROR("Unknown exception thrown during task running");
}
return m_task->update(blackboard);
TaskScheduler::getInstance()->terminateRunningTasks();
return Task::STATE_FAILURE;
}
void TaskRunner::reset()
src/lib/utility/scheduling/TaskScheduler.cpp
+14 -6
View File
@@ -126,11 +126,17 @@ bool TaskScheduler::hasTasksQueued() const
return m_taskRunners.size();
}
void TaskScheduler::terminateRunningTasks()
{
m_terminateRunningTasks = true;
}
std::shared_ptr<TaskScheduler> TaskScheduler::s_instance;
TaskScheduler::TaskScheduler()
: m_loopIsRunning(false)
, m_threadIsRunning(false)
, m_terminateRunningTasks(false)
{
}
@@ -151,23 +157,25 @@ void TaskScheduler::processTasks()
std::shared_ptr<Blackboard> blackboard = std::make_shared<Blackboard>();
while (true)
{
if (runner->update(blackboard) != Task::STATE_RUNNING)
{
break;
}
{
std::lock_guard<std::mutex> lock(m_loopMutex);
if (!m_loopIsRunning)
if (!m_loopIsRunning || m_terminateRunningTasks)
{
runner->terminate();
break;
}
}
if (runner->update(blackboard) != Task::STATE_RUNNING)
{
break;
}
}
}
m_taskRunners.pop_front();
}
m_terminateRunningTasks = false;
}
src/lib/utility/scheduling/TaskScheduler.h
+3
View File
@@ -25,6 +25,8 @@ public:
bool loopIsRunning() const;
bool hasTasksQueued() const;
void terminateRunningTasks();
private:
static std::shared_ptr<TaskScheduler> s_instance;
@@ -37,6 +39,7 @@ private:
bool m_loopIsRunning;
bool m_threadIsRunning;
bool m_terminateRunningTasks;
std::deque<std::shared_ptr<TaskRunner>> m_taskRunners;
src/lib_cxx/data/indexer/IndexerCommandCxx.cpp
+22
View File
@@ -20,6 +20,28 @@ IndexerCommandCxx::~IndexerCommandCxx()
{
}
size_t IndexerCommandCxx::getByteSize() const
{
size_t size = IndexerCommand::getByteSize();
for (auto i : m_systemHeaderSearchPaths)
{
size += i.str().size();
}
for (auto i : m_frameworkSearchPaths)
{
size += i.str().size();
}
for (auto i : m_compilerFlags)
{
size += i.size();
}
return size;
}
std::vector<FilePath> IndexerCommandCxx::getSystemHeaderSearchPaths() const
{
return m_systemHeaderSearchPaths;
src/lib_cxx/data/indexer/IndexerCommandCxx.h
+1
View File
@@ -21,6 +21,7 @@ public:
const std::vector<std::string>& compilerFlags);
virtual ~IndexerCommandCxx();
virtual size_t getByteSize() const override;
std::vector<FilePath> getSystemHeaderSearchPaths() const;
std::vector<FilePath> getFrameworkSearchPaths() const;
src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp
+5
View File
@@ -49,6 +49,11 @@ std::string IndexerCommandCxxCdb::getKindString() const
return getIndexerKindString();
}
size_t IndexerCommandCxxCdb::getByteSize() const
{
return IndexerCommandCxx::getByteSize() + sizeof(*this) + m_workingDirectory.str().size();
}
FilePath IndexerCommandCxxCdb::getWorkingDirectory() const
{
return m_workingDirectory;
src/lib_cxx/data/indexer/IndexerCommandCxxCdb.h
+1
View File
@@ -31,6 +31,7 @@ public:
virtual ~IndexerCommandCxxCdb();
virtual std::string getKindString() const;
virtual size_t getByteSize() const;
FilePath getWorkingDirectory() const;
src/lib_cxx/data/indexer/IndexerCommandCxxManual.cpp
+5
View File
@@ -28,6 +28,11 @@ std::string IndexerCommandCxxManual::getKindString() const
return getIndexerKindString();
}
size_t IndexerCommandCxxManual::getByteSize() const
{
return IndexerCommandCxx::getByteSize() + sizeof(*this);
}
std::string IndexerCommandCxxManual::getLanguageStandard() const
{
return m_languageStandard;
src/lib_cxx/data/indexer/IndexerCommandCxxManual.h
+2
View File
@@ -19,9 +19,11 @@ public:
const std::vector<FilePath>& systemHeaderSearchPaths,
const std::vector<FilePath>& frameworkSearchPaths,
const std::vector<std::string>& compilerFlags);
virtual ~IndexerCommandCxxManual();
virtual std::string getKindString() const;
virtual size_t getByteSize() const;
std::string getLanguageStandard() const;
src/lib_cxx/data/parser/cxx/ASTConsumer.cpp
+4 -1
View File
@@ -1,4 +1,5 @@
#include "data/parser/cxx/ASTConsumer.h"
#include "data/parser/cxx/CxxAstVisitor.h"
#include "data/parser/cxx/CxxVerboseAstVisitor.h"
#include "settings/ApplicationSettings.h"
@@ -11,7 +12,9 @@ ASTConsumer::ASTConsumer(
std::shared_ptr<FilePathCache> canonicalFilePathCache
)
{
if (ApplicationSettings::getInstance()->getLoggingEnabled() && ApplicationSettings::getInstance()->getVerboseIndexerLoggingEnabled())
ApplicationSettings* appSettings = ApplicationSettings::getInstance().get();
if (appSettings->getLoggingEnabled() && appSettings->getVerboseIndexerLoggingEnabled())
{
m_visitor = std::make_shared<CxxVerboseAstVisitor>(context, preprocessor, client, fileRegister, canonicalFilePathCache);
}
src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp
+1 -1
View File
@@ -299,7 +299,7 @@ bool CxxAstVisitor::TraverseTemplateTemplateParmDecl(clang::TemplateTemplateParm
bool CxxAstVisitor::VisitTranslationUnitDecl(clang::TranslationUnitDecl *d)
{
return !m_client->cancelOnFatalErrors() || !m_client->hasFatalErrors();
return true;
}
bool CxxAstVisitor::TraverseNestedNameSpecifierLoc(clang::NestedNameSpecifierLoc loc)
src/lib_cxx/data/parser/cxx/CxxParser.h
-1
View File
@@ -22,7 +22,6 @@ public:
void buildIndex(std::shared_ptr<IndexerCommandCxxManual> indexerCommand);
void buildIndex(const std::string& fileName, std::shared_ptr<TextAccess> fileContent);
private:
std::vector<std::string> getCommandlineArgumentsEssential(
const std::vector<std::string>& compilerFlags,
src/lib_gui/platform_includes/includesMac.h
+2 -1
View File
@@ -8,6 +8,7 @@
#include "qt/utility/utilityQt.h"
#include "utility/AppPath.h"
#include "utility/file/FilePath.h"
#include "utility/ResourcePaths.h"
#include "utility/UserPaths.h"
@@ -91,7 +92,7 @@ void setupApp(int argc, char *argv[])
if (!appIsMacBundle)
{
UserPaths::setUserDataPath(FilePath(path.absolute().str() + "/user/"));
UserPaths::setUserDataPath(FilePath("./user/"));
}
else
{
src/lib_gui/qt/element/QtCodeFileTitleButton.cpp
+5 -2
View File
@@ -66,9 +66,12 @@ void QtCodeFileTitleButton::setIsComplete(bool isComplete)
if (!isComplete)
{
FilePath hatchingFilePath(ResourcePaths::getGuiPath().str() + "code_view/images/pattern_" +
ColorScheme::getInstance()->getColor("code/file/title/hatching") + ".png"
);
setStyleSheet((
"background-image: url(" + ResourcePaths::getGuiPath().str() + "code_view/images/pattern_" +
ColorScheme::getInstance()->getColor("code/file/title/hatching") + ".png);"
"background-image: url(" + hatchingFilePath.str() + ");"
).c_str());
}
else
src/lib_gui/qt/view/QtDialogView.cpp
+13 -2
View File
@@ -179,7 +179,7 @@ void QtDialogView::updateIndexingDialog(size_t fileCount, size_t totalFileCount,
void QtDialogView::finishedIndexingDialog(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, ErrorCountInfo errorInfo)
float time, ErrorCountInfo errorInfo, bool interrupted)
{
std::stringstream ss;
ss << "Finished indexing: ";
@@ -198,7 +198,7 @@ void QtDialogView::finishedIndexingDialog(
m_windowStack.clearWindows();
QtIndexingDialog* window = createWindow<QtIndexingDialog>();
window->setupReport(indexedFileCount, totalIndexedFileCount, completedFileCount, totalFileCount, time);
window->setupReport(indexedFileCount, totalIndexedFileCount, completedFileCount, totalFileCount, time, interrupted);
window->updateErrorCount(errorInfo.total, errorInfo.fatal);
setUIBlocked(false);
@@ -206,6 +206,17 @@ void QtDialogView::finishedIndexingDialog(
);
}
void QtDialogView::hideDialogs()
{
m_onQtThread(
[=]()
{
m_windowStack.clearWindows();
setUIBlocked(false);
}
);
}
int QtDialogView::confirm(const std::string& message, const std::vector<std::string>& options)
{
int result = -1;
src/lib_gui/qt/view/QtDialogView.h
+3 -1
View File
@@ -39,7 +39,9 @@ public:
virtual void updateIndexingDialog(size_t fileCount, size_t totalFileCount, std::string sourcePath) override;
virtual void finishedIndexingDialog(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, ErrorCountInfo errorInfo) override;
float time, ErrorCountInfo errorInfo, bool interrupted) override;
virtual void hideDialogs() override;
int confirm(const std::string& message, const std::vector<std::string>& options) override;
src/lib_gui/qt/window/QtIndexingDialog.cpp
+2 -2
View File
@@ -153,7 +153,7 @@ void QtIndexingDialog::setupIndexing()
void QtIndexingDialog::setupReport(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time)
float time, bool interrupted)
{
QBoxLayout* layout = createLayout();
@@ -182,7 +182,7 @@ void QtIndexingDialog::setupReport(
m_sizeHint = QSize(400, 280);
if (indexedFileCount != totalIndexedFileCount)
if (interrupted)
{
updateTitle("Interrupted Indexing");
}
src/lib_gui/qt/window/QtIndexingDialog.h
+2 -1
View File
@@ -33,7 +33,8 @@ public:
DialogView::IndexingOptions options, std::function<void(DialogView::IndexingOptions)> callback);
void setupIndexing();
void setupReport(
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, float time);
size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
float time, bool interrupted);
void setupUnknownProgress();
void setupProgress();
src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp
+6 -5
View File
@@ -118,9 +118,10 @@ void QtProjectWizzardContentPreferences::populate(QGridLayout* layout, int& row)
addHelpButton("Number of parallel threads used to index your projects.\nWhen setting this to 0 Sourcetrail tries to use the ideal thread count for your computer.", layout, row);
row++;
// cancel indexing on fatal errors
m_cancelIndexingOnFatalErrors = addCheckBox("Cancel on Fatals", "Cancel indexing on Fatal errors.",
"Cancel indexing of translation units with fatal errors, which result in partly indexed files.", layout, row);
// multi process indexing
m_multiProcessIndexing = addCheckBox("Multi process indexing", "Use processes instead of threads for indexing.",
"Using processes instead of threads prevents the application from crashing on unforseen exceptions during indexing.",
layout, row);
addGap(layout, row);
@@ -260,7 +261,7 @@ void QtProjectWizzardContentPreferences::load()
m_threads->setCurrentIndex(appSettings->getIndexerThreadCount()); // index and value are the same
indexerThreadsChanges(m_threads->currentIndex());
m_cancelIndexingOnFatalErrors->setChecked(appSettings->getCancelIndexingOnFatalErrors());
m_multiProcessIndexing->setChecked(appSettings->getMultiProcessIndexingEnabled());
if (m_javaPath)
{
@@ -304,7 +305,7 @@ void QtProjectWizzardContentPreferences::save()
if (pluginPort) appSettings->setPluginPort(pluginPort);
appSettings->setIndexerThreadCount(m_threads->currentIndex()); // index and value are the same
appSettings->setCancelIndexingOnFatalErrors(m_cancelIndexingOnFatalErrors->isChecked());
appSettings->setMultiProcessIndexingEnabled(m_multiProcessIndexing->isChecked());
if (m_javaPath)
{
src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h
+1 -1
View File
@@ -69,7 +69,7 @@ private:
QComboBox* m_threads;
QLabel* m_threadsInfoLabel;
QCheckBox* m_cancelIndexingOnFatalErrors;
QCheckBox* m_multiProcessIndexing;
std::shared_ptr<CombinedPathDetector> m_javaPathDetector;
std::shared_ptr<CombinedPathDetector> m_mavenPathDetector;
src/lib_gui/utility/utilityApp.cpp
+56 -3
View File
@@ -6,7 +6,15 @@
#include "utility/utilityString.h"
std::string utility::executeProcess(const std::string& command, const std::string& workingDirectory)
#include <iostream>
namespace utility
{
std::mutex s_runningProcessesMutex;
std::set<QProcess*> s_runningProcesses;
}
std::string utility::executeProcess(const std::string& command, const std::string& workingDirectory, int timeout)
{
QProcess process;
process.setProcessChannelMode(QProcess::MergedChannels);
@@ -16,8 +24,18 @@ std::string utility::executeProcess(const std::string& command, const std::strin
process.setWorkingDirectory(workingDirectory.c_str());
}
process.start(command.c_str());
process.waitForFinished();
{
std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
process.start(command.c_str());
s_runningProcesses.insert(&process);
}
process.waitForFinished(timeout);
{
std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
s_runningProcesses.erase(&process);
}
std::string processoutput = process.readAll().toStdString();
process.close();
processoutput = utility::trim(processoutput);
@@ -25,6 +43,41 @@ std::string utility::executeProcess(const std::string& command, const std::strin
return processoutput;
}
int utility::executeProcessAndGetExitCode(const std::string& command, const std::string& workingDirectory, int timeout)
{
QProcess process;
if (!workingDirectory.empty())
{
process.setWorkingDirectory(workingDirectory.c_str());
}
{
std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
process.start(command.c_str());
s_runningProcesses.insert(&process);
}
process.waitForFinished(timeout);
{
std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
s_runningProcesses.erase(&process);
}
int exitCode = process.exitCode();
process.close();
return exitCode;
}
void utility::killRunningProcesses()
{
std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
for (QProcess* process : s_runningProcesses)
{
process->kill();
}
}
ApplicationArchitectureType utility::getApplicationArchitectureType()
{
#ifdef Q_PROCESSOR_X86_64

Some files were not shown because too many files have changed in this diff Show More