src: switch from Qt to boost implementation of executeProcess (#1145)

* changed implementation of executeProcess
* unified all different implementations of executeProcess
* separated arg list from command for all calls of executeProcess (this also required to remove quotes that were added to path args because otherwise the Python indexer would mistake absolute paths for relative paths.)
* update expected output for custom command tests

src/lib/app/paths/ResourcePaths.cpp

@@ -1,6 +1,7 @@
 #include "ResourcePaths.h"
 
 #include "AppPath.h"
+#include "utilityApp.h"
 
 FilePath ResourcePaths::getColorSchemesPath()
 {
@@ -37,7 +38,7 @@ FilePath ResourcePaths::getJavaPath()
 	return AppPath::getSharedDataPath().concatenate(L"data/java/");
 }
 
-FilePath ResourcePaths::getPythonPath()
+FilePath ResourcePaths::getPythonDirectoryPath()
 {
 	return AppPath::getSharedDataPath().concatenate(L"data/python/");
 }
@@ -46,3 +47,12 @@ FilePath ResourcePaths::getCxxCompilerHeaderPath()
 {
 	return AppPath::getSharedDataPath().concatenate(L"data/cxx/include/").getCanonical();
 }
+
+FilePath ResourcePaths::getPythonIndexerFilePath()
+{
+	if (utility::getOsType() == OS_WINDOWS)
+	{
+		return getPythonDirectoryPath().concatenate(L"SourcetrailPythonIndexer.exe");
+	}
+	return getPythonDirectoryPath().concatenate(L"SourcetrailPythonIndexer");
+}

src/lib/app/paths/ResourcePaths.h

@@ -15,8 +15,9 @@ public:
 	static FilePath getGuiPath();
 	static FilePath getLicensePath();
 	static FilePath getJavaPath();
-	static FilePath getPythonPath();
+	static FilePath getPythonDirectoryPath();
 	static FilePath getCxxCompilerHeaderPath();
+	static FilePath getPythonIndexerFilePath();
 };
 
 #endif	  // RESOURCE_PATHS_H

src/lib/data/indexer/IndexerCommandCustom.cpp

@@ -11,7 +11,8 @@ IndexerCommandType IndexerCommandCustom::getStaticIndexerCommandType()
 }
 
 IndexerCommandCustom::IndexerCommandCustom(
-	const std::wstring& customCommand,
+	const std::wstring& command,
+	const std::vector<std::wstring>& arguments,
 	const FilePath& projectFilePath,
 	const FilePath& databaseFilePath,
 	const std::wstring& databaseVersion,
@@ -19,7 +20,8 @@ IndexerCommandCustom::IndexerCommandCustom(
 	bool runInParallel)
 	: IndexerCommand(sourceFilePath)
 	, m_type(getStaticIndexerCommandType())
-	, m_customCommand(customCommand)
+	, m_command(command)
+	, m_arguments(arguments)
 	, m_projectFilePath(projectFilePath)
 	, m_databaseFilePath(databaseFilePath)
 	, m_databaseVersion(databaseVersion)
@@ -29,7 +31,8 @@ IndexerCommandCustom::IndexerCommandCustom(
 
 IndexerCommandCustom::IndexerCommandCustom(
 	IndexerCommandType type,
-	const std::wstring& customCommand,
+	const std::wstring& command,
+	const std::vector<std::wstring>& arguments,
 	const FilePath& projectFilePath,
 	const FilePath& databaseFilePath,
 	const std::wstring& databaseVersion,
@@ -37,7 +40,8 @@ IndexerCommandCustom::IndexerCommandCustom(
 	bool runInParallel)
 	: IndexerCommand(sourceFilePath)
 	, m_type(type)
-	, m_customCommand(customCommand)
+	, m_command(command)
+	, m_arguments(arguments)
 	, m_projectFilePath(projectFilePath)
 	, m_databaseFilePath(databaseFilePath)
 	, m_databaseVersion(databaseVersion)
@@ -67,19 +71,19 @@ void IndexerCommandCustom::setDatabaseFilePath(const FilePath& databaseFilePath)
 	m_databaseFilePath = databaseFilePath;
 }
 
-std::wstring IndexerCommandCustom::getCustomCommand() const
+std::wstring IndexerCommandCustom::getCommand() const
 {
-	std::wstring command = m_customCommand;
+	return replaceVariables(m_command);
+}
 
-	command = utility::replace(
-		command, L"%{PROJECT_FILE_PATH}", L'\"' + m_projectFilePath.wstr() + L'\"');
-	command = utility::replace(
-		command, L"%{DATABASE_FILE_PATH}", L'\"' + m_databaseFilePath.wstr() + L'\"');
-	command = utility::replace(command, L"%{DATABASE_VERSION}", L'\"' + m_databaseVersion + L'\"');
-	command = utility::replace(
-		command, L"%{SOURCE_FILE_PATH}", L'\"' + getSourceFilePath().wstr() + L'\"');
-
-	return command;
+std::vector<std::wstring> IndexerCommandCustom::getArguments() const
+{
+	std::vector<std::wstring> args;
+	for (const std::wstring& argument: m_arguments)
+	{
+		args.push_back(replaceVariables(argument));
+	}
+	return args;
 }
 
 bool IndexerCommandCustom::getRunInParallel() const
@@ -92,7 +96,15 @@ QJsonObject IndexerCommandCustom::doSerialize() const
 	QJsonObject jsonObject = IndexerCommand::doSerialize();
 
 	{
-		jsonObject["custom_command"] = QString::fromStdWString(m_customCommand);
+		jsonObject["command"] = QString::fromStdWString(m_command);
+	}
+	{
+		QJsonArray argumentsArray;
+		for (const std::wstring& argument: m_arguments)
+		{
+			argumentsArray.append(QString::fromStdWString(argument));
+		}
+		jsonObject["arguments"] = argumentsArray;
 	}
 	{
 		jsonObject["run_in_parallel"] = m_runInParallel;
@@ -100,3 +112,12 @@ QJsonObject IndexerCommandCustom::doSerialize() const
 
 	return jsonObject;
 }
+
+std::wstring IndexerCommandCustom::replaceVariables(std::wstring s) const
+{
+	s = utility::replace(s, L"%{PROJECT_FILE_PATH}", m_projectFilePath.wstr());
+	s = utility::replace(s, L"%{DATABASE_FILE_PATH}", m_databaseFilePath.wstr());
+	s = utility::replace(s, L"%{DATABASE_VERSION}", m_databaseVersion);
+	s = utility::replace(s, L"%{SOURCE_FILE_PATH}", getSourceFilePath().wstr());
+	return s;
+}

src/lib/data/indexer/IndexerCommandCustom.h

@@ -12,7 +12,8 @@ public:
 	static IndexerCommandType getStaticIndexerCommandType();
 
 	IndexerCommandCustom(
-		const std::wstring& customCommand,
+		const std::wstring& command,
+		const std::vector<std::wstring>& arguments,
 		const FilePath& projectFilePath,
 		const FilePath& databaseFilePath,
 		const std::wstring& databaseVersion,
@@ -21,7 +22,8 @@ public:
 
 	IndexerCommandCustom(
 		IndexerCommandType type,
-		const std::wstring& customCommand,
+		const std::wstring& command,
+		const std::vector<std::wstring>& arguments,
 		const FilePath& projectFilePath,
 		const FilePath& databaseFilePath,
 		const std::wstring& databaseVersion,
@@ -34,15 +36,19 @@ public:
 	FilePath getDatabaseFilePath() const;
 	void setDatabaseFilePath(const FilePath& databaseFilePath);
 
-	std::wstring getCustomCommand() const;
+	std::wstring getCommand() const;
+	std::vector<std::wstring> getArguments() const;
 	bool getRunInParallel() const;
 
 protected:
 	QJsonObject doSerialize() const override;
 
 private:
+	std::wstring replaceVariables(std::wstring s) const;
+
 	IndexerCommandType m_type;
-	std::wstring m_customCommand;
+	std::wstring m_command;
+	std::vector<std::wstring> m_arguments;
 	FilePath m_projectFilePath;
 	FilePath m_databaseFilePath;
 	std::wstring m_databaseVersion;

src/lib/data/indexer/TaskBuildIndex.cpp

@@ -188,22 +188,23 @@ void TaskBuildIndex::runIndexerProcess(int processId, const std::wstring& logFil
 		return;
 	}
 
-	const std::wstring commandPath = L"\"" + indexerProcessPath.wstr() + L"\"";
 	std::vector<std::wstring> commandArguments;
 	commandArguments.push_back(std::to_wstring(processId));
 	commandArguments.push_back(utility::decodeFromUtf8(m_appUUID));
-	commandArguments.push_back(L"\"" + AppPath::getSharedDataPath().getAbsolute().wstr() + L"\"");
-	commandArguments.push_back(L"\"" + UserPaths::getUserDataPath().getAbsolute().wstr() + L"\"");
+	commandArguments.push_back(AppPath::getSharedDataPath().getAbsolute().wstr());
+	commandArguments.push_back(UserPaths::getUserDataPath().getAbsolute().wstr());
 
 	if (!logFilePath.empty())
 	{
-		commandArguments.push_back(L"\"" + logFilePath + L"\"");
+		commandArguments.push_back(logFilePath);
 	}
 
 	int result = 1;
 	while ((!m_indexerCommandQueueStopped || result != 0) && !m_interrupted)
 	{
-		result = utility::executeProcessAndGetExitCode(commandPath, commandArguments, FilePath(), -1);
+		result = utility::executeProcess(
+					 indexerProcessPath.wstr(), commandArguments, FilePath(), false, -1)
+					 .exitCode;
 
 		LOG_INFO_STREAM(<< "Indexer process " << processId << " returned with " + std::to_string(result));
 	}

src/lib/data/indexer/TaskExecuteCustomCommands.cpp

@@ -465,17 +465,19 @@ void TaskExecuteCustomCommands::runIndexerCommand(
 			indexedSourceFileCount + 1, indexedSourceFileCount, m_indexerCommandCount, {sourcePath});
 		MessageIndexingStatus(true, indexedSourceFileCount * 100 / m_indexerCommandCount).dispatch();
 
-		const std::wstring command = indexerCommand->getCustomCommand();
+		const std::wstring command = indexerCommand->getCommand();
+		const std::vector<std::wstring> arguments = indexerCommand->getArguments();
 
-		LOG_INFO("Start processing command \"" + utility::encodeToUtf8(command) + "\"");
+		LOG_INFO(
+			"Start processing command \"" +
+			utility::encodeToUtf8(command + L" " + utility::join(arguments, L" ")) + "\"");
 
 		const ErrorCountInfo previousErrorCount = storage ? storage->getErrorCount()
 														  : ErrorCountInfo();
 
 		LOG_INFO("Starting to index");
-		std::wstring errorMessage;
-		const int result = utility::executeProcessAndGetExitCode(
-			command, {}, m_projectDirectory, -1, true, &errorMessage);
+		const utility::ProcessOutput out = utility::executeProcess(
+			command, arguments, m_projectDirectory, false, -1, true);
 		LOG_INFO("Finished indexing");
 
 		if (storage)
@@ -501,22 +503,22 @@ void TaskExecuteCustomCommands::runIndexerCommand(
 			}
 		}
 
-		if (result == 0 && errorMessage.empty())
+		if (out.exitCode == 0 && out.error.empty())
 		{
 			std::wstring message = L"Process returned successfully.\n";
 			LOG_INFO(message);
 		}
 		else
 		{
-			std::wstring statusText = L"command \"" + indexerCommand->getCustomCommand() +
-				L"\" returned";
-			if (result != 0)
+			std::wstring statusText = L"command \"" + indexerCommand->getCommand() + L" " +
+				utility::join(arguments, L" ") + L"\" returned";
+			if (out.exitCode != 0)
 			{
-				statusText += L" code \"" + std::to_wstring(result) + L"\"";
+				statusText += L" code \"" + std::to_wstring(out.exitCode) + L"\"";
 			}
-			if (!errorMessage.empty())
+			if (!out.error.empty())
 			{
-				statusText += L" with message \"" + errorMessage + L"\"";
+				statusText += L" with message \"" + out.error + L"\"";
 			}
 			statusText += L".";
 

src/lib/project/SourceGroupCustomCommand.cpp

@@ -42,7 +42,6 @@ std::set<FilePath> SourceGroupCustomCommand::getAllSourceFilePaths() const
 std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexerCommands(
 	const RefreshInfo& info) const
 {
-	const std::wstring customCommand = m_settings->getCustomCommand();
 	const bool runInParallel = m_settings->getRunInParallel();
 
 	std::vector<std::shared_ptr<IndexerCommand>> indexerCommands;
@@ -51,7 +50,8 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCustomCommand::getIndexe
 		if (info.filesToIndex.find(sourcePath) != info.filesToIndex.end())
 		{
 			indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(
-				customCommand,
+				m_settings->getCustomCommand(),
+				std::vector<std::wstring> {},
 				m_settings->getProjectSettings()->getProjectFilePath(),
 				m_settings->getProjectSettings()->getTempDBFilePath(),
 				std::to_wstring(SqliteIndexStorage::getStorageVersion()),