logic : Fixed no indexed information saved for compilation database referencing certain compiler (issue #531)

* changed order of compiler flags for CDB indexer commands so that manually added flags are appended to the flags form the compile command.
* removed usage of clang::tooling::FixedCompilationDatabase and used CompilationDatabaseSingle for empty and cdb indexer commands.
* removed reference to a specific clang tool (e.g. "cl" or "cl.exe") from compile command and prepended "clang-tool" before running the indexer.

src/lib/data/indexer/interprocess/shared_types/SharedIndexerCommand.cpp

@@ -27,6 +27,7 @@ void SharedIndexerCommand::fromLocal(IndexerCommand* indexerCommand)
 		IndexerCommandCxxEmpty* cmd = dynamic_cast<IndexerCommandCxxEmpty*>(indexerCommand);
 
 		setType(CXX_EMPTY);
+		setWorkingDirectory(cmd->getWorkingDirectory());
 		setLanguageStandard(cmd->getLanguageStandard());
 		setCompilerFlags(cmd->getCompilerFlags());
 		setSystemHeaderSearchPaths(cmd->getSystemHeaderSearchPaths());
@@ -69,6 +70,7 @@ std::shared_ptr<IndexerCommand> SharedIndexerCommand::fromShared(const SharedInd
 			indexerCommand.getSourceFilePath(),
 			indexerCommand.getIndexedPaths(),
 			indexerCommand.getExcludedPaths(),
+			indexerCommand.getWorkingDirectory(),
 			indexerCommand.getLanguageStandard(),
 			indexerCommand.getSystemHeaderSearchPaths(),
 			indexerCommand.getFrameworkSearchhPaths(),

src/lib_cxx/data/indexer/IndexerCommandCxx.cpp

@@ -4,11 +4,13 @@ IndexerCommandCxx::IndexerCommandCxx(
 	const FilePath& sourceFilePath,
 	const std::set<FilePath>& indexedPaths,
 	const std::set<FilePath>& excludedPaths,
+	const FilePath& workingDirectory,
 	const std::vector<FilePath>& systemHeaderSearchPaths,
 	const std::vector<FilePath>& frameworkSearchPaths,
 	const std::vector<std::string>& compilerFlags
 )
 	: IndexerCommand(sourceFilePath, indexedPaths, excludedPaths)
+	, m_workingDirectory(workingDirectory)
 	, m_systemHeaderSearchPaths(systemHeaderSearchPaths)
 	, m_frameworkSearchPaths(frameworkSearchPaths)
 	, m_compilerFlags(compilerFlags)
@@ -51,3 +53,8 @@ std::vector<std::string> IndexerCommandCxx::getCompilerFlags() const
 {
 	return m_compilerFlags;
 }
+
+FilePath IndexerCommandCxx::getWorkingDirectory() const
+{
+	return m_workingDirectory;
+}

src/lib_cxx/data/indexer/IndexerCommandCxx.h

@@ -16,6 +16,7 @@ public:
 		const FilePath& sourceFilePath,
 		const std::set<FilePath>& indexedPaths,
 		const std::set<FilePath>& excludedPaths,
+		const FilePath& workingDirectory,
 		const std::vector<FilePath>& systemHeaderSearchPaths,
 		const std::vector<FilePath>& frameworkSearchPaths,
 		const std::vector<std::string>& compilerFlags);
@@ -25,8 +26,11 @@ public:
 	std::vector<FilePath> getSystemHeaderSearchPaths() const;
 	std::vector<FilePath> getFrameworkSearchPaths() const;
 	std::vector<std::string> getCompilerFlags() const;
+	FilePath getWorkingDirectory() const;
+
 
 private:
+	FilePath m_workingDirectory;
 	std::vector<FilePath> m_systemHeaderSearchPaths;
 	std::vector<FilePath> m_frameworkSearchPaths;
 	std::vector<std::string> m_compilerFlags;

src/lib_cxx/data/indexer/IndexerCommandCxxCdb.cpp

@@ -48,8 +48,7 @@ IndexerCommandCxxCdb::IndexerCommandCxxCdb(
 	const std::vector<FilePath>& systemHeaderSearchPaths,
 	const std::vector<FilePath>& frameworkSearchPaths
 )
-	: IndexerCommandCxx(sourceFilePath, indexedPaths, excludedPaths, systemHeaderSearchPaths, frameworkSearchPaths, compilerFlags)
-	, m_workingDirectory(workingDirectory)
+	: IndexerCommandCxx(sourceFilePath, indexedPaths, excludedPaths, workingDirectory, systemHeaderSearchPaths, frameworkSearchPaths, compilerFlags)
 {
 }
 
@@ -57,13 +56,3 @@ IndexerCommandType IndexerCommandCxxCdb::getIndexerCommandType() const
 {
 	return getStaticIndexerCommandType();
 }
-
-size_t IndexerCommandCxxCdb::getByteSize(size_t stringSize) const
-{
-	return IndexerCommandCxx::getByteSize(stringSize) + m_workingDirectory.str().size();
-}
-
-FilePath IndexerCommandCxxCdb::getWorkingDirectory() const
-{
-	return m_workingDirectory;
-}

src/lib_cxx/data/indexer/IndexerCommandCxxCdb.h

@@ -30,12 +30,6 @@ public:
 		const std::vector<FilePath>& frameworkSearchPaths);
 
 	virtual IndexerCommandType getIndexerCommandType() const override;
-	virtual size_t getByteSize(size_t stringSize) const override;
-
-	FilePath getWorkingDirectory() const;
-
-private:
-	FilePath m_workingDirectory;
 };
 
 #endif // INDEXER_COMMAND_CXX_CDB_H

src/lib_cxx/data/indexer/IndexerCommandCxxEmpty.cpp

@@ -9,12 +9,13 @@ IndexerCommandCxxEmpty::IndexerCommandCxxEmpty(
 	const FilePath& sourceFilePath,
 	const std::set<FilePath>& indexedPaths,
 	const std::set<FilePath>& excludedPaths,
+	const FilePath& workingDirectory,
 	const std::string& languageStandard,
 	const std::vector<FilePath>& systemHeaderSearchPaths,
 	const std::vector<FilePath>& frameworkSearchPaths,
 	const std::vector<std::string>& compilerFlags
 )
-	: IndexerCommandCxx(sourceFilePath, indexedPaths, excludedPaths, systemHeaderSearchPaths, frameworkSearchPaths, compilerFlags)
+	: IndexerCommandCxx(sourceFilePath, indexedPaths, excludedPaths, workingDirectory, systemHeaderSearchPaths, frameworkSearchPaths, compilerFlags)
 	, m_languageStandard(languageStandard)
 {
 }

src/lib_cxx/data/indexer/IndexerCommandCxxEmpty.h

@@ -15,6 +15,7 @@ public:
 		const FilePath& sourceFilePath,
 		const std::set<FilePath>& indexedPaths,
 		const std::set<FilePath>& excludedPaths,
+		const FilePath& workingDirectory,
 		const std::string& languageStandard,
 		const std::vector<FilePath>& systemHeaderSearchPaths,
 		const std::vector<FilePath>& frameworkSearchPaths,

src/lib_cxx/data/parser/cxx/CxxParser.cpp

@@ -14,21 +14,23 @@
 #include "utility/file/FileRegister.h"
 #include "utility/logging/logging.h"
 #include "utility/text/TextAccess.h"
+#include "utility/utilityString.h"
+#include "utility/utility.h"
 
 namespace
 {
-	static std::vector<std::string> getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs, llvm::StringRef FileName)
+	std::vector<std::string> prependSyntaxOnlyToolArgs(const std::vector<std::string>& args)
 	{
-		std::vector<std::string> Args;
-		Args.push_back("clang-tool");
-		Args.push_back("-fsyntax-only");
-		Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
-		Args.push_back(FileName.str());
-		return Args;
+		return utility::concat({ "clang-tool", "-fsyntax-only" }, args);
+	}
+
+	std::vector<std::string> appendFilePath(const std::vector<std::string>& args, llvm::StringRef fileName)
+	{
+		return utility::concat(args, { fileName.str() });
 	}
 
 	// custom implementation of clang::runToolOnCodeWithArgs which also sets our custon DiagnosticConsumer
-	static bool runToolOnCodeWithArgs(
+	bool runToolOnCodeWithArgs(
 		clang::DiagnosticConsumer* DiagConsumer,
 		clang::FrontendAction *ToolAction,
 		const llvm::Twine &Code,
@@ -40,7 +42,7 @@ namespace
 		llvm::SmallString<16> FileNameStorage;
 		llvm::StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
 		llvm::IntrusiveRefCntPtr<clang::FileManager> Files(new clang::FileManager(clang::FileSystemOptions()));
-		clang::tooling::ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), ToolAction, Files.get());
+		clang::tooling::ToolInvocation Invocation(prependSyntaxOnlyToolArgs(appendFilePath(Args, FileNameRef)), ToolAction, Files.get());
 
 		llvm::SmallString<1024> CodeStorage;
 		Invocation.mapVirtualFile(FileNameRef, Code.toNullTerminatedStringRef(CodeStorage));
@@ -73,14 +75,15 @@ void CxxParser::buildIndex(std::shared_ptr<IndexerCommandCxxCdb> indexerCommand)
 	clang::tooling::CompileCommand compileCommand;
 	compileCommand.Filename = indexerCommand->getSourceFilePath().str();
 	compileCommand.Directory = indexerCommand->getWorkingDirectory().str();
-	compileCommand.CommandLine = indexerCommand->getCompilerFlags();
+	compileCommand.CommandLine = utility::concat(indexerCommand->getCompilerFlags(), getCommandlineArgumentsEssential(
+		std::vector<std::string>(), indexerCommand->getSystemHeaderSearchPaths(), indexerCommand->getFrameworkSearchPaths()
+	));
 
+	if (!utility::isPrefix("-", compileCommand.CommandLine.front()))
 	{
-		std::vector<std::string> args = getCommandlineArgumentsEssential(
-			std::vector<std::string>(), indexerCommand->getSystemHeaderSearchPaths(), indexerCommand->getFrameworkSearchPaths()
-		);
-		compileCommand.CommandLine.insert(compileCommand.CommandLine.end(), args.begin(), args.end());
+		compileCommand.CommandLine.erase(compileCommand.CommandLine.begin());
 	}
+	compileCommand.CommandLine = prependSyntaxOnlyToolArgs(compileCommand.CommandLine);
 
 	CxxCompilationDatabaseSingle compilationDatabase(compileCommand);
 	runTool(&compilationDatabase, indexerCommand->getSourceFilePath());
@@ -88,9 +91,13 @@ void CxxParser::buildIndex(std::shared_ptr<IndexerCommandCxxCdb> indexerCommand)
 
 void CxxParser::buildIndex(std::shared_ptr<IndexerCommandCxxEmpty> indexerCommand)
 {
-	std::shared_ptr<clang::tooling::CompilationDatabase> compilationDatabase = getCompilationDatabase(indexerCommand);
+	clang::tooling::CompileCommand compileCommand;
+	compileCommand.Filename = indexerCommand->getSourceFilePath().str();
+	compileCommand.Directory = indexerCommand->getWorkingDirectory().str();
+	compileCommand.CommandLine = prependSyntaxOnlyToolArgs(appendFilePath(getCommandlineArguments(indexerCommand), indexerCommand->getSourceFilePath().str()));
 
-	runTool(compilationDatabase.get(), indexerCommand->getSourceFilePath());
+	CxxCompilationDatabaseSingle compilationDatabase(compileCommand);
+	runTool(&compilationDatabase, indexerCommand->getSourceFilePath());
 }
 
 void CxxParser::buildIndex(const std::string& fileName, std::shared_ptr<TextAccess> fileContent, std::vector<std::string> compilerFlags)
@@ -179,37 +186,6 @@ std::vector<std::string> CxxParser::getCommandlineArguments(std::shared_ptr<Inde
 	return args;
 }
 
-std::shared_ptr<clang::tooling::FixedCompilationDatabase> CxxParser::getCompilationDatabase(
-	std::shared_ptr<IndexerCommandCxxEmpty> indexerCommand
-) const {
-	// Commandline flags passed to the programm. Everything after '--' will be interpreted by the ClangTool.
-	std::vector<std::string> args = getCommandlineArguments(indexerCommand);
-	args.insert(args.begin(), "app");
-	args.insert(args.begin() + 1, "--");
-
-	int argc = args.size();
-	const char** argv = new const char*[argc];
-	for (size_t i = 0; i < args.size(); i++)
-	{
-		argv[i] = args[i].c_str();
-	}
-
-	std::string errorMessage;
-	std::shared_ptr<clang::tooling::FixedCompilationDatabase> compilationDatabase(
-		clang::tooling::FixedCompilationDatabase::loadFromCommandLine(argc, argv, errorMessage)
-	);
-
-	delete[] argv;
-
-	if (!compilationDatabase)
-	{
-		LOG_ERROR("Failed to load compilation database");
-		return nullptr;
-	}
-
-	return compilationDatabase;
-}
-
 std::shared_ptr<CxxDiagnosticConsumer> CxxParser::getDiagnostics(std::shared_ptr<CanonicalFilePathCache> canonicalFilePathCache, bool logErrors) const
 {
 	llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> options = new clang::DiagnosticOptions();

src/lib_cxx/data/parser/cxx/CxxParser.h

@@ -36,7 +36,6 @@ private:
 		const std::vector<FilePath>& systemHeaderSearchPaths,
 		const std::vector<FilePath>& frameworkSearchPaths) const;
 	std::vector<std::string> getCommandlineArguments(std::shared_ptr<IndexerCommandCxxEmpty> indexerCommand) const;
-	std::shared_ptr<clang::tooling::FixedCompilationDatabase> getCompilationDatabase(std::shared_ptr<IndexerCommandCxxEmpty> indexerCommand) const;
 
 	std::shared_ptr<CxxDiagnosticConsumer> getDiagnostics(std::shared_ptr<CanonicalFilePathCache> canonicalFilePathCache, bool logErrors) const;
 

src/lib_cxx/project/SourceGroupCxxCdb.cpp

@@ -68,8 +68,7 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCxxCdb::getIndexerComman
 	utility::append(frameworkSearchPaths, m_settings->getFrameworkSearchPathsExpandedAndAbsolute());
 	utility::append(frameworkSearchPaths, appSettings->getFrameworkSearchPathsExpanded());
 
-	std::vector<std::string> compilerFlags;
-	utility::append(compilerFlags, m_settings->getCompilerFlags());
+	const std::vector<std::string> compilerFlags = m_settings->getCompilerFlags();
 
 	std::set<FilePath> indexedPaths = getIndexedPaths();
 	std::set<FilePath> excludedPaths = getExcludedPaths();
@@ -108,15 +107,12 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCxxCdb::getIndexerComman
 			if (filesToIndex.find(sourcePath) != filesToIndex.end() &&
 				sourceFilePaths.find(sourcePath) != sourceFilePaths.end())
 			{
-				std::vector<std::string> currentCompilerFlags = compilerFlags;
-				currentCompilerFlags.insert(currentCompilerFlags.end(), command.CommandLine.begin(), command.CommandLine.end());
-
 				indexerCommands.push_back(std::make_shared<IndexerCommandCxxCdb>(
 					sourcePath,
 					indexedPaths,
 					excludedPaths,
 					FilePath(command.Directory),
-					currentCompilerFlags,
+					utility::concat(command.CommandLine, compilerFlags),
 					systemHeaderSearchPaths,
 					frameworkSearchPaths
 				));

src/lib_cxx/project/SourceGroupCxxEmpty.cpp

@@ -63,6 +63,7 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCxxEmpty::getIndexerComm
 				sourcePath,
 				indexedPaths,
 				excludedPaths,
+				m_settings->getProjectDirectoryPath(),
 				m_settings->getStandard(),
 				systemHeaderSearchPaths,
 				frameworkSearchPaths,

src/test/CxxIndexSampleProjectsTestSuite.h

@@ -122,8 +122,9 @@ private:
 
 	std::shared_ptr<TextAccess> parseCode(const FilePath& sourceFilePath, const FilePath& projectDataSrcRoot)
 	{
-		std::set<FilePath> indexedPaths = { projectDataSrcRoot };
-		std::set<FilePath> excludedPaths = { };
+		const std::set<FilePath> indexedPaths = { projectDataSrcRoot };
+		const std::set<FilePath> excludedPaths = {};
+		const FilePath workingDirectory(".");
 
 		std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
 			FileRegisterStateData(),
@@ -140,6 +141,7 @@ private:
 			sourceFilePath, 
 			indexedPaths,
 			excludedPaths,
+			workingDirectory,
 			"c++1z", 
 			utility::concat(std::vector<FilePath> { projectDataSrcRoot }, ApplicationSettings::getInstance()->getHeaderSearchPathsExpanded()),
 			ApplicationSettings::getInstance()->getFrameworkSearchPathsExpanded(), 

src/test/CxxParserTestSuite.h

@@ -3990,13 +3990,15 @@ public:
 
 	void test_cxx_parser_parses_multiple_files()
 	{
-		std::set<FilePath> indexedPaths;
-		indexedPaths.insert(FilePath("data/CxxParserTestSuite/"));
+		const std::set<FilePath> indexedPaths = { FilePath("data/CxxParserTestSuite/") };
+		const std::set<FilePath> excludedPaths;
+		const FilePath workingDirectory(".");
 
 		std::shared_ptr<IndexerCommandCxxEmpty> indexerCommand = std::make_shared<IndexerCommandCxxEmpty>(
 			FilePath("data/CxxParserTestSuite/code.cpp"),
 			indexedPaths,
-			std::set<FilePath>(),
+			excludedPaths,
+			workingDirectory,
 			"c++1z",
 			std::vector<FilePath>(),
 			std::vector<FilePath>(),