data: indexer processes

* TaskBuildIndex starts seperate indexer processes and communicates via shared memory
* indexer processes get restarted if they fail
* indexer processes are killed when the app closes or crashes
* added utility class SharedMemory to allocate and access shared memory, providing basic data structures
* added SharedMemoryGarbageCollector for keeping track of running instances and cleaning up shared memory in case of a crash
* show errors for source files that crashed during indexing
* added basic exception handling to task scheduling
* added option to turn off processes and use threads in preferences, but still use shared memory

fortune cookie message = Good news from someone dear is coming soon.

src/lib_gui/platform_includes/includesMac.h

@@ -8,6 +8,7 @@
 
 #include "qt/utility/utilityQt.h"
 #include "utility/AppPath.h"
+#include "utility/file/FilePath.h"
 #include "utility/ResourcePaths.h"
 #include "utility/UserPaths.h"
 
@@ -91,7 +92,7 @@ void setupApp(int argc, char *argv[])
 
 	if (!appIsMacBundle)
 	{
-		UserPaths::setUserDataPath(FilePath(path.absolute().str() + "/user/"));
+		UserPaths::setUserDataPath(FilePath("./user/"));
 	}
 	else
 	{

src/lib_gui/qt/element/QtCodeFileTitleButton.cpp

@@ -66,9 +66,12 @@ void QtCodeFileTitleButton::setIsComplete(bool isComplete)
 
 	if (!isComplete)
 	{
+		FilePath hatchingFilePath(ResourcePaths::getGuiPath().str() + "code_view/images/pattern_" +
+			ColorScheme::getInstance()->getColor("code/file/title/hatching") + ".png"
+		);
+
 		setStyleSheet((
-			"background-image: url(" + ResourcePaths::getGuiPath().str() + "code_view/images/pattern_" +
-			ColorScheme::getInstance()->getColor("code/file/title/hatching") + ".png);"
+			"background-image: url(" + hatchingFilePath.str() + ");"
 		).c_str());
 	}
 	else

src/lib_gui/qt/view/QtDialogView.cpp

@@ -179,7 +179,7 @@ void QtDialogView::updateIndexingDialog(size_t fileCount, size_t totalFileCount,
 
 void QtDialogView::finishedIndexingDialog(
 	size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
-	float time, ErrorCountInfo errorInfo)
+	float time, ErrorCountInfo errorInfo, bool interrupted)
 {
 	std::stringstream ss;
 	ss << "Finished indexing: ";
@@ -198,7 +198,7 @@ void QtDialogView::finishedIndexingDialog(
 			m_windowStack.clearWindows();
 
 			QtIndexingDialog* window = createWindow<QtIndexingDialog>();
-			window->setupReport(indexedFileCount, totalIndexedFileCount, completedFileCount, totalFileCount, time);
+			window->setupReport(indexedFileCount, totalIndexedFileCount, completedFileCount, totalFileCount, time, interrupted);
 			window->updateErrorCount(errorInfo.total, errorInfo.fatal);
 
 			setUIBlocked(false);
@@ -206,6 +206,17 @@ void QtDialogView::finishedIndexingDialog(
 	);
 }
 
+void QtDialogView::hideDialogs()
+{
+	m_onQtThread(
+		[=]()
+		{
+			m_windowStack.clearWindows();
+			setUIBlocked(false);
+		}
+	);
+}
+
 int QtDialogView::confirm(const std::string& message, const std::vector<std::string>& options)
 {
 	int result = -1;

src/lib_gui/qt/view/QtDialogView.h

@@ -39,7 +39,9 @@ public:
 	virtual void updateIndexingDialog(size_t fileCount, size_t totalFileCount, std::string sourcePath) override;
 	virtual void finishedIndexingDialog(
 		size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
-		float time, ErrorCountInfo errorInfo) override;
+		float time, ErrorCountInfo errorInfo, bool interrupted) override;
+
+	virtual void hideDialogs() override;
 
 	int confirm(const std::string& message, const std::vector<std::string>& options) override;
 

src/lib_gui/qt/window/QtIndexingDialog.cpp

@@ -153,7 +153,7 @@ void QtIndexingDialog::setupIndexing()
 
 void QtIndexingDialog::setupReport(
 	size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
-	float time)
+	float time, bool interrupted)
 {
 	QBoxLayout* layout = createLayout();
 
@@ -182,7 +182,7 @@ void QtIndexingDialog::setupReport(
 
 	m_sizeHint = QSize(400, 280);
 
-	if (indexedFileCount != totalIndexedFileCount)
+	if (interrupted)
 	{
 		updateTitle("Interrupted Indexing");
 	}

src/lib_gui/qt/window/QtIndexingDialog.h

@@ -33,7 +33,8 @@ public:
 		DialogView::IndexingOptions options, std::function<void(DialogView::IndexingOptions)> callback);
 	void setupIndexing();
 	void setupReport(
-		size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount, float time);
+		size_t indexedFileCount, size_t totalIndexedFileCount, size_t completedFileCount, size_t totalFileCount,
+		float time, bool interrupted);
 
 	void setupUnknownProgress();
 	void setupProgress();

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp

@@ -118,9 +118,10 @@ void QtProjectWizzardContentPreferences::populate(QGridLayout* layout, int& row)
 	addHelpButton("Number of parallel threads used to index your projects.\nWhen setting this to 0 Sourcetrail tries to use the ideal thread count for your computer.", layout, row);
 	row++;
 
-	// cancel indexing on fatal errors
-	m_cancelIndexingOnFatalErrors = addCheckBox("Cancel on Fatals", "Cancel indexing on Fatal errors.",
-		"Cancel indexing of translation units with fatal errors, which result in partly indexed files.", layout, row);
+	// multi process indexing
+	m_multiProcessIndexing = addCheckBox("Multi process indexing", "Use processes instead of threads for indexing.",
+		"Using processes instead of threads prevents the application from crashing on unforseen exceptions during indexing.",
+		layout, row);
 
 	addGap(layout, row);
 
@@ -260,7 +261,7 @@ void QtProjectWizzardContentPreferences::load()
 
 	m_threads->setCurrentIndex(appSettings->getIndexerThreadCount()); // index and value are the same
 	indexerThreadsChanges(m_threads->currentIndex());
-	m_cancelIndexingOnFatalErrors->setChecked(appSettings->getCancelIndexingOnFatalErrors());
+	m_multiProcessIndexing->setChecked(appSettings->getMultiProcessIndexingEnabled());
 
 	if (m_javaPath)
 	{
@@ -304,7 +305,7 @@ void QtProjectWizzardContentPreferences::save()
 	if (pluginPort) appSettings->setPluginPort(pluginPort);
 
 	appSettings->setIndexerThreadCount(m_threads->currentIndex()); // index and value are the same
-	appSettings->setCancelIndexingOnFatalErrors(m_cancelIndexingOnFatalErrors->isChecked());
+	appSettings->setMultiProcessIndexingEnabled(m_multiProcessIndexing->isChecked());
 
 	if (m_javaPath)
 	{

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h

@@ -69,7 +69,7 @@ private:
 	QComboBox* m_threads;
 	QLabel* m_threadsInfoLabel;
 
-	QCheckBox* m_cancelIndexingOnFatalErrors;
+	QCheckBox* m_multiProcessIndexing;
 
 	std::shared_ptr<CombinedPathDetector> m_javaPathDetector;
 	std::shared_ptr<CombinedPathDetector> m_mavenPathDetector;

src/lib_gui/utility/utilityApp.cpp

@@ -6,7 +6,15 @@
 
 #include "utility/utilityString.h"
 
-std::string utility::executeProcess(const std::string& command, const std::string& workingDirectory)
+#include <iostream>
+
+namespace utility
+{
+	std::mutex s_runningProcessesMutex;
+	std::set<QProcess*> s_runningProcesses;
+}
+
+std::string utility::executeProcess(const std::string& command, const std::string& workingDirectory, int timeout)
 {
 	QProcess process;
 	process.setProcessChannelMode(QProcess::MergedChannels);
@@ -16,8 +24,18 @@ std::string utility::executeProcess(const std::string& command, const std::strin
 		process.setWorkingDirectory(workingDirectory.c_str());
 	}
 
-	process.start(command.c_str());
-	process.waitForFinished();
+	{
+		std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
+		process.start(command.c_str());
+		s_runningProcesses.insert(&process);
+	}
+
+	process.waitForFinished(timeout);
+	{
+		std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
+		s_runningProcesses.erase(&process);
+	}
+
 	std::string processoutput = process.readAll().toStdString();
 	process.close();
 	processoutput = utility::trim(processoutput);
@@ -25,6 +43,41 @@ std::string utility::executeProcess(const std::string& command, const std::strin
 	return processoutput;
 }
 
+int utility::executeProcessAndGetExitCode(const std::string& command, const std::string& workingDirectory, int timeout)
+{
+	QProcess process;
+
+	if (!workingDirectory.empty())
+	{
+		process.setWorkingDirectory(workingDirectory.c_str());
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
+		process.start(command.c_str());
+		s_runningProcesses.insert(&process);
+	}
+
+	process.waitForFinished(timeout);
+	{
+		std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
+		s_runningProcesses.erase(&process);
+	}
+
+	int exitCode = process.exitCode();
+	process.close();
+	return exitCode;
+}
+
+void utility::killRunningProcesses()
+{
+	std::lock_guard<std::mutex> lock(s_runningProcessesMutex);
+	for (QProcess* process : s_runningProcesses)
+	{
+		process->kill();
+	}
+}
+
 ApplicationArchitectureType utility::getApplicationArchitectureType()
 {
 #ifdef Q_PROCESSOR_X86_64

src/lib_gui/utility/utilityApp.h

@@ -1,13 +1,18 @@
 #ifndef UTILITY_APP_H
 #define UTILITY_APP_H
 
+#include <mutex>
+#include <set>
 #include <string>
 
 #include "utility/ApplicationArchitectureType.h"
 
 namespace utility
 {
-	std::string executeProcess(const std::string& command, const std::string& workingDirectory = "");
+	std::string executeProcess(const std::string& command, const std::string& workingDirectory = "", int timeout = 30000);
+	int executeProcessAndGetExitCode(const std::string& command, const std::string& workingDirectory = "", int timeout = 30000);
+
+	void killRunningProcesses();
 
 	ApplicationArchitectureType getApplicationArchitectureType();