split command line args for IndexerCommandCustom usages

* this seems to be required when using the new QProcess API.
* this also required to remove quotes that were added to path args because otherwise the Python indexer would mistake absolute paths for relative paths.

bin/test/data/SourceGroupTestSuite/custom_command/expected_output/output_windows.txt

@@ -1,6 +1,15 @@
-SourceFilePath: "src/a.txt"
-	Custom Command: "echo "Hello World""
-SourceFilePath: "src/b.txt"
-	Custom Command: "echo "Hello World""
-SourceFilePath: "src/included/a.txt"
-	Custom Command: "echo "Hello World""
+IndexerCommandCustom
+	SourceFilePath: "src/a.txt"
+	Custom Command: "echo"
+	Arguments:
+		"Hello World"
+IndexerCommandCustom
+	SourceFilePath: "src/b.txt"
+	Custom Command: "echo"
+	Arguments:
+		"Hello World"
+IndexerCommandCustom
+	SourceFilePath: "src/included/a.txt"
+	Custom Command: "echo"
+	Arguments:
+		"Hello World"

src/lib/data/indexer/IndexerCommandCustom.cpp

@@ -11,7 +11,8 @@ IndexerCommandType IndexerCommandCustom::getStaticIndexerCommandType()
 }
 
 IndexerCommandCustom::IndexerCommandCustom(
-	const std::wstring& customCommand,
+	const std::wstring& command,
+	const std::vector<std::wstring>& arguments,
 	const FilePath& projectFilePath,
 	const FilePath& databaseFilePath,
 	const std::wstring& databaseVersion,
@@ -19,7 +20,8 @@ IndexerCommandCustom::IndexerCommandCustom(
 	bool runInParallel)
 	: IndexerCommand(sourceFilePath)
 	, m_type(getStaticIndexerCommandType())
-	, m_customCommand(customCommand)
+	, m_command(command)
+	, m_arguments(arguments)
 	, m_projectFilePath(projectFilePath)
 	, m_databaseFilePath(databaseFilePath)
 	, m_databaseVersion(databaseVersion)
@@ -29,7 +31,8 @@ IndexerCommandCustom::IndexerCommandCustom(
 
 IndexerCommandCustom::IndexerCommandCustom(
 	IndexerCommandType type,
-	const std::wstring& customCommand,
+	const std::wstring& command,
+	const std::vector<std::wstring>& arguments,
 	const FilePath& projectFilePath,
 	const FilePath& databaseFilePath,
 	const std::wstring& databaseVersion,
@@ -37,7 +40,8 @@ IndexerCommandCustom::IndexerCommandCustom(
 	bool runInParallel)
 	: IndexerCommand(sourceFilePath)
 	, m_type(type)
-	, m_customCommand(customCommand)
+	, m_command(command)
+	, m_arguments(arguments)
 	, m_projectFilePath(projectFilePath)
 	, m_databaseFilePath(databaseFilePath)
 	, m_databaseVersion(databaseVersion)
@@ -67,19 +71,19 @@ void IndexerCommandCustom::setDatabaseFilePath(const FilePath& databaseFilePath)
 	m_databaseFilePath = databaseFilePath;
 }
 
-std::wstring IndexerCommandCustom::getCustomCommand() const
+std::wstring IndexerCommandCustom::getCommand() const
 {
-	std::wstring command = m_customCommand;
+	return replaceVariables(m_command);
+}
 
-	command = utility::replace(
-		command, L"%{PROJECT_FILE_PATH}", L'\"' + m_projectFilePath.wstr() + L'\"');
-	command = utility::replace(
-		command, L"%{DATABASE_FILE_PATH}", L'\"' + m_databaseFilePath.wstr() + L'\"');
-	command = utility::replace(command, L"%{DATABASE_VERSION}", L'\"' + m_databaseVersion + L'\"');
-	command = utility::replace(
-		command, L"%{SOURCE_FILE_PATH}", L'\"' + getSourceFilePath().wstr() + L'\"');
-
-	return command;
+std::vector<std::wstring> IndexerCommandCustom::getArguments() const
+{
+	std::vector<std::wstring> args;
+	for (const std::wstring& argument: m_arguments)
+	{
+		args.push_back(replaceVariables(argument));
+	}
+	return args;
 }
 
 bool IndexerCommandCustom::getRunInParallel() const
@@ -92,7 +96,15 @@ QJsonObject IndexerCommandCustom::doSerialize() const
 	QJsonObject jsonObject = IndexerCommand::doSerialize();
 
 	{
-		jsonObject["custom_command"] = QString::fromStdWString(m_customCommand);
+		jsonObject["command"] = QString::fromStdWString(m_command);
+	}
+	{
+		QJsonArray argumentsArray;
+		for (const std::wstring& argument: m_arguments)
+		{
+			argumentsArray.append(QString::fromStdWString(argument));
+		}
+		jsonObject["arguments"] = argumentsArray;
 	}
 	{
 		jsonObject["run_in_parallel"] = m_runInParallel;
@@ -100,3 +112,12 @@ QJsonObject IndexerCommandCustom::doSerialize() const
 
 	return jsonObject;
 }
+
+std::wstring IndexerCommandCustom::replaceVariables(std::wstring s) const
+{
+	s = utility::replace(s, L"%{PROJECT_FILE_PATH}", m_projectFilePath.wstr());
+	s = utility::replace(s, L"%{DATABASE_FILE_PATH}", m_databaseFilePath.wstr());
+	s = utility::replace(s, L"%{DATABASE_VERSION}", m_databaseVersion);
+	s = utility::replace(s, L"%{SOURCE_FILE_PATH}", getSourceFilePath().wstr());
+	return s;
+}

src/lib/data/indexer/IndexerCommandCustom.h

@@ -12,7 +12,8 @@ public:
 	static IndexerCommandType getStaticIndexerCommandType();
 
 	IndexerCommandCustom(
-		const std::wstring& customCommand,
+		const std::wstring& command,
+		const std::vector<std::wstring>& arguments,
 		const FilePath& projectFilePath,
 		const FilePath& databaseFilePath,
 		const std::wstring& databaseVersion,
@@ -21,7 +22,8 @@ public:
 
 	IndexerCommandCustom(
 		IndexerCommandType type,
-		const std::wstring& customCommand,
+		const std::wstring& command,
+		const std::vector<std::wstring>& arguments,
 		const FilePath& projectFilePath,
 		const FilePath& databaseFilePath,
 		const std::wstring& databaseVersion,
@@ -34,15 +36,19 @@ public:
 	FilePath getDatabaseFilePath() const;
 	void setDatabaseFilePath(const FilePath& databaseFilePath);
 
-	std::wstring getCustomCommand() const;
+	std::wstring getCommand() const;
+	std::vector<std::wstring> getArguments() const;
 	bool getRunInParallel() const;
 
 protected:
 	QJsonObject doSerialize() const override;
 
 private:
+	std::wstring replaceVariables(std::wstring s) const;
+
 	IndexerCommandType m_type;
-	std::wstring m_customCommand;
+	std::wstring m_command;
+	std::vector<std::wstring> m_arguments;
 	FilePath m_projectFilePath;
 	FilePath m_databaseFilePath;
 	std::wstring m_databaseVersion;

src/lib/data/indexer/TaskExecuteCustomCommands.cpp

@@ -465,9 +465,12 @@ void TaskExecuteCustomCommands::runIndexerCommand(
 			indexedSourceFileCount + 1, indexedSourceFileCount, m_indexerCommandCount, {sourcePath});
 		MessageIndexingStatus(true, indexedSourceFileCount * 100 / m_indexerCommandCount).dispatch();
 
-		const std::wstring command = indexerCommand->getCustomCommand();
+		const std::wstring command = indexerCommand->getCommand();
+		const std::vector<std::wstring> arguments = indexerCommand->getArguments();
 
-		LOG_INFO("Start processing command \"" + utility::encodeToUtf8(command) + "\"");
+		LOG_INFO(
+			"Start processing command \"" +
+			utility::encodeToUtf8(command + L" " + utility::join(arguments, L" ")) + "\"");
 
 		const ErrorCountInfo previousErrorCount = storage ? storage->getErrorCount()
 														  : ErrorCountInfo();
@@ -475,7 +478,7 @@ void TaskExecuteCustomCommands::runIndexerCommand(
 		LOG_INFO("Starting to index");
 		std::wstring errorMessage;
 		const int result = utility::executeProcessAndGetExitCode(
-			command, {}, m_projectDirectory, -1, true, &errorMessage);
+			command, arguments, m_projectDirectory, -1, true, &errorMessage);
 		LOG_INFO("Finished indexing");
 
 		if (storage)
@@ -508,8 +511,9 @@ void TaskExecuteCustomCommands::runIndexerCommand(
 		}
 		else
 		{
-			std::wstring statusText = L"command \"" + indexerCommand->getCustomCommand() +
-				L"\" returned";
+			std::wstring statusText = L"command \"" + indexerCommand->getCommand() + L" " +
+				utility::join(arguments, L" ") + L"\" returned";
+
 			if (result != 0)
 			{
 				statusText += L" code \"" + std::to_wstring(result) + L"\"";

src/lib/project/SourceGroupCustomCommand.cpp

@@ -42,7 +42,67 @@ std::set<FilePath> SourceGroupCustomCommand::getAllSourceFilePaths() const
 std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexerCommands(
 	const RefreshInfo& info) const
 {
-	const std::wstring customCommand = m_settings->getCustomCommand();
+	std::vector<std::wstring> parts;
+	{
+		const std::wstring customCommand = m_settings->getCustomCommand();
+
+		std::wstring tmp;
+		int quoteCount = 0;
+		bool inQuote = false;
+		// handle quoting. tokens can be surrounded by double quotes
+		// "hello world". three consecutive double quotes represent
+		// the quote character itself.
+		for (int i = 0; i < customCommand.size(); ++i)
+		{
+			if (customCommand.at(i) == L'"')
+			{
+				++quoteCount;
+				if (quoteCount == 3)	// third consecutive quote
+				{
+					quoteCount = 0;
+					tmp += customCommand.at(i);
+				}
+				continue;
+			}
+			if (quoteCount)
+			{
+				if (quoteCount == 1)
+				{
+					inQuote = !inQuote;
+				}
+				quoteCount = 0;
+			}
+			if (!inQuote && customCommand.at(i) == L' ')
+			{
+				if (!tmp.empty())
+				{
+					parts.push_back(tmp);
+					tmp.clear();
+				}
+			}
+			else
+			{
+				tmp += customCommand.at(i);
+			}
+		}
+		if (!tmp.empty())
+		{
+			parts.push_back(tmp);
+		}
+	}
+
+	if (parts.empty())
+	{
+		return {};
+	}
+
+	const std::wstring command = parts.front();
+	std::vector<std::wstring> args;
+	for (size_t i = 1; i < parts.size(); i++)
+	{
+		args.push_back(parts[i]);
+	}
+
 	const bool runInParallel = m_settings->getRunInParallel();
 
 	std::vector<std::shared_ptr<IndexerCommand>> indexerCommands;
@@ -51,7 +111,8 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexe
 		if (info.filesToIndex.find(sourcePath) != info.filesToIndex.end())
 		{
 			indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(
-				customCommand,
+				command,
+				args,
 				m_settings->getProjectSettings()->getProjectFilePath(),
 				m_settings->getProjectSettings()->getTempDBFilePath(),
 				std::to_wstring(SqliteIndexStorage::getStorageVersion()),

src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSelect.cpp

@@ -24,12 +24,13 @@ void QtProjectWizardContentSelect::populate(QGridLayout* layout, int& row)
 {
 	std::string pythonIndexerVersion = " ";
 	{
-		std::string str = utility::executeProcess(
+		std::string str =
+			utility::executeProcess(
 				ResourcePaths::getPythonPath().wstr().append(L"SourcetrailPythonIndexer"),
-							  std::vector<std::wstring>{L"--version"},
-							  FilePath(),
-							  5000)
-							  .second;
+				{L"--version"},
+				FilePath(),
+				5000)
+				.second;
 		std::regex regex(
 			"v\\d*\\.db\\d*\\.p\\d*");	  // "\\d" matches any digit; "\\." matches the "." character
 		std::smatch matches;

src/lib_gui/qt/project_wizard/content/path/QtProjectWizardContentPathPythonEnvironment.cpp

@@ -64,12 +64,11 @@ void QtProjectWizardContentPathPythonEnvironment::onTextChanged(const QString& t
 		std::thread([=]() {
 			std::pair<int, std::string> out = utility::executeProcess(
 				ResourcePaths::getPythonPath().wstr().append(L"SourcetrailPythonIndexer"),
-				std::vector<std::wstring>{
-						L"check-environment",
-						L"--environment-path " + utility::getExpandedAndAbsolutePath(
-						FilePath(text.toStdWString()), m_settings->getProjectDirectoryPath())
-						.wstr()
-						},
+				{L"check-environment",
+				 L"--environment-path " +
+					 utility::getExpandedAndAbsolutePath(
+						 FilePath(text.toStdWString()), m_settings->getProjectDirectoryPath())
+						 .wstr()},
 				FilePath(),
 				5000);
 			m_onQtThread([=]() {

src/lib_gui/utility/path_detector/cxx_header/CxxVs15HeaderPathDetector.cpp

@@ -20,10 +20,12 @@ std::vector<FilePath> CxxVs15HeaderPathDetector::doGetPaths() const
 				.expandEnvironmentVariables();
 		if (!expandedPaths.empty())
 		{
-			const std::string output =
-				utility::executeProcess(
-					expandedPaths[0].wstr(), std::vector<std::wstring> {L"-latest", L"-property installationPath"}, FilePath(), 10000)
-					.second;
+			const std::string output = utility::executeProcess(
+										   expandedPaths[0].wstr(),
+										   {L"-latest", L"-property installationPath"},
+										   FilePath(),
+										   10000)
+										   .second;
 
 			const FilePath vsInstallPath(output);
 			if (vsInstallPath.exists())

src/lib_gui/utility/path_detector/cxx_header/utilityCxxHeaderDetection.cpp

@@ -14,7 +14,7 @@ std::vector<std::string> getCxxHeaderPaths(const std::string& compilerName)
 	std::string command = compilerName + " -x c++ -v -E /dev/null";
 	std::string clangOutput = utility::executeProcess(
 								  utility::decodeFromUtf8(compilerName),
-								  std::vector<std::wstring> {L"-x c++", L"-v", L"-E /dev/null"})
+								  {L"-x c++", L"-v", L"-E /dev/null"})
 								  .second;
 	std::string standardHeaders = utility::substrBetween<std::string>(
 		clangOutput, "#include <...> search starts here:\n", "\nEnd of search list");

src/lib_gui/utility/path_detector/java_runtime/JavaPathDetectorLinux.cpp

@@ -54,7 +54,7 @@ std::vector<FilePath> JavaPathDetectorLinux::doGetPaths() const
 
 FilePath JavaPathDetectorLinux::getJavaInPath() const
 {
-	std::string output = utility::executeProcess(L"which", std::vector<std::wstring>{L"java"}).second;
+	std::string output = utility::executeProcess(L"which", {L"java"}).second;
 
 	if (!output.empty())
 	{
@@ -72,7 +72,7 @@ FilePath JavaPathDetectorLinux::getJavaInPath() const
 
 FilePath JavaPathDetectorLinux::readLink(const FilePath& path) const
 {
-	FilePath javaPath(utility::executeProcess(L"readlink", std::vector<std::wstring>{L"-f " + path.wstr()}).second);
+	FilePath javaPath(utility::executeProcess(L"readlink", {L"-f " + path.wstr()}).second);
 	if (!javaPath.empty())
 	{
 		return javaPath;
@@ -113,7 +113,7 @@ FilePath JavaPathDetectorLinux::getJavaInJavaHome() const
 
 bool JavaPathDetectorLinux::checkVersion(const FilePath& path) const
 {
-	std::string output = utility::executeProcess(path.wstr(), std::vector<std::wstring>{L"-version"}).second;
+	std::string output = utility::executeProcess(path.wstr(), {L"-version"}).second;
 
 	return output.find(m_javaVersion) != std::string::npos;
 }

src/lib_gui/utility/path_detector/java_runtime/JavaPathDetectorMac.cpp

@@ -14,7 +14,7 @@ std::vector<FilePath> JavaPathDetectorMac::doGetPaths() const
 	std::vector<FilePath> paths;
 	FilePath javaPath;
 
-	std::string output = utility::executeProcess(L"/usr/libexec/java_home", std::vector<std::wstring>{}).second;
+	const std::string output = utility::executeProcess(L"/usr/libexec/java_home", {}).second;
 
 	if (!output.empty())
 	{

src/lib_gui/utility/path_detector/maven_executable/MavenPathDetectorUnix.cpp

@@ -7,7 +7,7 @@ MavenPathDetectorUnix::MavenPathDetectorUnix(): PathDetector("Maven for Unix") {
 
 std::vector<FilePath> MavenPathDetectorUnix::doGetPaths() const
 {
-	FilePath mavenPath(utility::executeProcess(L"which", std::vector<std::wstring>{L"mvn"}).second);
+	FilePath mavenPath(utility::executeProcess(L"which", {L"mvn"}).second);
 
 	std::vector<FilePath> paths;
 	if (mavenPath.exists())

src/lib_gui/utility/path_detector/maven_executable/MavenPathDetectorWindows.cpp

@@ -7,9 +7,7 @@ MavenPathDetectorWindows::MavenPathDetectorWindows(): PathDetector("Maven for Wi
 
 std::vector<FilePath> MavenPathDetectorWindows::doGetPaths() const
 {
-	FilePath mavenPath(
-		utility::executeProcess(L"cmd", std::vector<std::wstring> {L"\"/c where mvn.cmd && exit\""})
-			.second);
+	FilePath mavenPath(utility::executeProcess(L"cmd", {L"\"/c where mvn.cmd && exit\""}).second);
 
 	std::vector<FilePath> paths;
 	if (mavenPath.exists())

src/lib_gui/utility/utilityApp.cpp

@@ -208,13 +208,24 @@ int utility::executeProcessAndGetExitCode(
 					*errorMessage = L"An unknown error occurred while executing process.";
 					break;
 				}
-			};
+			}
 		});
 
 	QObject::connect(
 		&process,
 		static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
-		[&finished](int exitCode, QProcess::ExitStatus exitStatus) { finished = true; });
+		[&finished, errorMessage](int exitCode, QProcess::ExitStatus exitStatus) {
+			finished = true;
+			if (errorMessage != nullptr)
+			{
+				switch (exitStatus)
+				{
+				case QProcess::CrashExit:
+					*errorMessage = L"Process crashed.";
+					break;
+				}
+			}
+		});
 
 
 	if (!workingDirectory.empty())

src/lib_python/project/SourceGroupPythonEmpty.cpp

@@ -49,25 +49,27 @@ std::set<FilePath> SourceGroupPythonEmpty::getAllSourceFilePaths() const
 std::vector<std::shared_ptr<IndexerCommand>> SourceGroupPythonEmpty::getIndexerCommands(
 	const RefreshInfo& info) const
 {
-	std::wstring args = L"";
+	std::vector<std::wstring> args;
+	args.push_back(L"index");
 
-	args += L" --source-file-path=%{SOURCE_FILE_PATH}";
-	args += L" --database-file-path=%{DATABASE_FILE_PATH}";
+	args.push_back(L"--source-file-path=%{SOURCE_FILE_PATH}");
+	args.push_back(L"--database-file-path=%{DATABASE_FILE_PATH}");
 
 	if (!m_settings->getEnvironmentPath().empty())
 	{
-		args += L" --environment-path=\"" +
-			m_settings->getEnvironmentPathExpandedAndAbsolute().wstr() + L"\"";
+		args.push_back(
+			L"--environment-path=\"" + m_settings->getEnvironmentPathExpandedAndAbsolute().wstr() +
+			L"\"");
 	}
 
 	if (ApplicationSettings::getInstance()->getVerboseIndexerLoggingEnabled())
 	{
-		args += L" --verbose";
+		args.push_back(L"--verbose");
 	}
 
 	if (info.shallow)
 	{
-		args += L" --shallow";
+		args.push_back(L"--shallow");
 	}
 
 	std::vector<std::shared_ptr<IndexerCommand>> indexerCommands;
@@ -77,8 +79,8 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupPythonEmpty::getIndexerC
 		{
 			indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(
 				INDEXER_COMMAND_PYTHON,
-				L"\"" + ResourcePaths::getPythonPath().wstr() +
-					L"SourcetrailPythonIndexer\" index" + args,
+				L"\"" + ResourcePaths::getPythonPath().wstr() + L"SourcetrailPythonIndexer\"",
+				args,
 				m_settings->getProjectSettings()->getProjectFilePath(),
 				m_settings->getProjectSettings()->getTempDBFilePath(),
 				std::to_wstring(SqliteIndexStorage::getStorageVersion()),

src/test/PythonIndexerTestSuite.cpp

@@ -56,16 +56,18 @@ std::shared_ptr<TestStorage> parseCode(std::string code)
 		const std::set<FilePathFilter> includeFilters;
 		const FilePath workingDirectory(L".");
 
-		std::wstring args = L"";
-		args += L" --source-file-path=%{SOURCE_FILE_PATH}";
-		args += L" --database-file-path=%{DATABASE_FILE_PATH}";
-		args += L" --shallow";
+		std::vector<std::wstring> args;
+		args.push_back(L"index");
+		args.push_back(L"--source-file-path=%{SOURCE_FILE_PATH}");
+		args.push_back(L"--database-file-path=%{DATABASE_FILE_PATH}");
+		args.push_back(L"--shallow");
 
 		std::shared_ptr<IndexerCommandCustom> indexerCommand = std::make_shared<IndexerCommandCustom>(
 			INDEXER_COMMAND_PYTHON,
 			L"\"" +
 				FilePath("../app").getConcatenated(ResourcePaths::getPythonPath()).makeAbsolute().wstr() +
-				L"SourcetrailPythonIndexer\" index" + args,
+				L"SourcetrailPythonIndexer\"",
+			args,
 			rootPath,
 			tempDbPath,
 			std::to_wstring(SqliteIndexStorage::getStorageVersion()),
@@ -74,7 +76,12 @@ std::shared_ptr<TestStorage> parseCode(std::string code)
 
 		std::wstring errorMessage;
 		const int result = utility::executeProcessAndGetExitCode(
-			indexerCommand->getCustomCommand(), {}, rootPath, -1, true, &errorMessage);
+			indexerCommand->getCommand(),
+			indexerCommand->getArguments(),
+			rootPath,
+			-1,
+			true,
+			&errorMessage);
 
 		REQUIRE(result == 0);
 		REQUIRE(errorMessage.empty());

src/test/SourceGroupTestSuite.cpp

@@ -148,9 +148,15 @@ std::wstring indexerCommandCustomToString(
 	std::shared_ptr<const IndexerCommandCustom> indexerCommand, const FilePath& baseDirectory)
 {
 	std::wstring result;
-	result += L"SourceFilePath: \"" +
+	result += L"IndexerCommandCustom\n";
+	result += L"\tSourceFilePath: \"" +
 		indexerCommand->getSourceFilePath().getRelativeTo(baseDirectory).wstr() + L"\"\n";
-	result += L"\tCustom Command: \"" + indexerCommand->getCustomCommand() + L"\"\n";
+	result += L"\tCustom Command: \"" + indexerCommand->getCommand() + L"\"\n";
+	result += L"\tArguments:\n";
+	for (const std::wstring& argument: indexerCommand->getArguments())
+	{
+		result += L"\t\t\"" + argument + L"\"\n";
+	}
 	return result;
 }