logic: fixed indexed header paths for compilation database project screw up the processing order of includes (issue #571)

* prevented indexed header paths for CDB to be appended to compile command
* fixed exists check for relative paths in QtSelectPathsDialog
* indexed header paths will automatically be updated when a new compilation database is picked

src/lib/project/SourceGroup.h

@@ -26,7 +26,7 @@ public:
 	virtual bool prepareIndexing();
 	void fetchAllSourceFilePaths();
 
-	std::set<FilePath> getIndexedPaths() const;
+	virtual std::set<FilePath> getIndexedPaths() const;
 	std::set<FilePathFilter> getExcludeFilters() const;
 	std::set<FilePath> getAllSourceFilePaths() const;
 	std::set<FilePath> getSourceFilePathsToIndex(const std::set<FilePath>& staticSourceFilePaths) const;
@@ -35,12 +35,13 @@ public:
 
 	std::set<FilePath> m_allSourceFilePaths;
 
+protected:
+	std::set<FilePath> findAndAddSymlinkedDirectories(const std::vector<FilePath>& paths) const;
+
 private:
 	virtual std::shared_ptr<SourceGroupSettings> getSourceGroupSettings() = 0;
 	virtual std::shared_ptr<const SourceGroupSettings> getSourceGroupSettings() const = 0;
 	virtual std::vector<FilePath> getAllSourcePaths() const = 0;
-
-	std::set<FilePath> findAndAddSymlinkedDirectories(const std::vector<FilePath>& paths) const;
 };
 
 #endif // SOURCE_GROUP_H

src/lib/settings/ProjectSettings.cpp

@@ -12,7 +12,7 @@
 #include "utility/utilityString.h"
 #include "utility/utilityUuid.h"
 
-const size_t ProjectSettings::VERSION = 5;
+const size_t ProjectSettings::VERSION = 6;
 const wchar_t PROJECT_FILE_EXTENSION[] = L".srctrlprj";
 
 LanguageType ProjectSettings::getLanguageOfProject(const FilePath& filePath)
@@ -320,6 +320,15 @@ SettingsMigrator ProjectSettings::getMigrations() const
 		migrator.addMigration(5, std::make_shared<SettingsMigrationMoveKey>(key + "/exclude_paths/exclude_path", key + "/exclude_filters/exclude_filter"));
 	}
 
+	for (std::shared_ptr<SourceGroupSettings> sourceGroupSettings : getAllSourceGroupSettings())
+	{
+		if (sourceGroupSettings->getType() == SOURCE_GROUP_CXX_CDB)
+		{
+			const std::string key = SourceGroupSettings::s_keyPrefix + sourceGroupSettings->getId();
+			migrator.addMigration(6, std::make_shared<SettingsMigrationMoveKey>(key + "/source_paths/source_path", key + "/indexed_header_paths/indexed_header_path"));
+		}
+	}
+
 	return migrator;
 }
 

src/lib/settings/SourceGroupSettingsCxxCdb.cpp

@@ -20,6 +20,7 @@ void SourceGroupSettingsCxxCdb::load(std::shared_ptr<const ConfigManager> config
 	const std::string key = s_keyPrefix + getId();
 
 	setCompilationDatabasePath(FilePath(getValue<std::wstring>(key + "/build_file_path/compilation_db_path", L"", config)));
+	setIndexedHeaderPaths(getPathValues(key + "/indexed_header_paths/indexed_header_path", config));
 }
 
 void SourceGroupSettingsCxxCdb::save(std::shared_ptr<ConfigManager> config)
@@ -29,6 +30,7 @@ void SourceGroupSettingsCxxCdb::save(std::shared_ptr<ConfigManager> config)
 	const std::string key = s_keyPrefix + getId();
 
 	setValue(key + "/build_file_path/compilation_db_path", getCompilationDatabasePath().wstr(), config);
+	setPathValues(key + "/indexed_header_paths/indexed_header_path", getIndexedHeaderPaths(), config);
 }
 
 bool SourceGroupSettingsCxxCdb::equals(std::shared_ptr<SourceGroupSettings> other) const
@@ -56,3 +58,19 @@ void SourceGroupSettingsCxxCdb::setCompilationDatabasePath(const FilePath& compi
 {
 	m_compilationDatabasePath = compilationDatabasePath;
 }
+
+std::vector<FilePath> SourceGroupSettingsCxxCdb::getIndexedHeaderPaths() const
+{
+	return m_indexedHeaderPaths;
+}
+
+std::vector<FilePath> SourceGroupSettingsCxxCdb::getIndexedHeaderPathsExpandedAndAbsolute() const
+{
+	return m_projectSettings->makePathsExpandedAndAbsolute(getIndexedHeaderPaths());
+}
+
+void SourceGroupSettingsCxxCdb::setIndexedHeaderPaths(const std::vector<FilePath>& indexedHeaderPaths)
+{
+	m_indexedHeaderPaths = indexedHeaderPaths;
+}
+

src/lib/settings/SourceGroupSettingsCxxCdb.h

@@ -19,8 +19,13 @@ public:
 	FilePath getCompilationDatabasePathExpandedAndAbsolute() const;
 	void setCompilationDatabasePath(const FilePath& compilationDatabasePath);
 
+	std::vector<FilePath> getIndexedHeaderPaths() const;
+	std::vector<FilePath> getIndexedHeaderPathsExpandedAndAbsolute() const;
+	void setIndexedHeaderPaths(const std::vector<FilePath>& indexedHeaderPaths);
+
 private:
 	FilePath m_compilationDatabasePath;
+	std::vector<FilePath> m_indexedHeaderPaths;
 };
 
 #endif // SOURCE_GROUP_SETTINGS_CXX_CDB_H

src/lib_cxx/project/SourceGroupCxxCdb.cpp

@@ -45,6 +45,12 @@ bool SourceGroupCxxCdb::prepareIndexing()
 	return true;
 }
 
+std::set<FilePath> SourceGroupCxxCdb::getIndexedPaths() const
+{
+	return findAndAddSymlinkedDirectories(m_settings->getIndexedHeaderPathsExpandedAndAbsolute());
+}
+
+
 std::vector<std::shared_ptr<IndexerCommand>> SourceGroupCxxCdb::getIndexerCommands(const std::set<FilePath>& filesToIndex) const
 {
 	std::shared_ptr<ApplicationSettings> appSettings = ApplicationSettings::getInstance();

src/lib_cxx/project/SourceGroupCxxCdb.h

@@ -18,6 +18,8 @@ public:
 
 	virtual bool prepareIndexing() override;
 
+	virtual std::set<FilePath> getIndexedPaths() const override;
+
 	virtual std::vector<std::shared_ptr<IndexerCommand>> getIndexerCommands(const std::set<FilePath>& filesToIndex) const override;
 
 private:

src/lib_cxx/utility/CompilationDatabase.cpp

@@ -39,10 +39,16 @@ std::vector<FilePath> utility::CompilationDatabase::getFrameworkHeaderPaths() co
 void utility::CompilationDatabase::init()
 {
 	std::string error;
-	std::shared_ptr<clang::tooling::JSONCompilationDatabase> cdb = std::shared_ptr<clang::tooling::JSONCompilationDatabase>(
+	std::shared_ptr<clang::tooling::JSONCompilationDatabase> cdb(
 		clang::tooling::JSONCompilationDatabase::loadFromFile(utility::encodeToUtf8(m_filePath.wstr()), error, clang::tooling::JSONCommandLineSyntax::AutoDetect)
 	);
 
+	if (!cdb)
+	{
+		LOG_ERROR(L"Loading compilation database from file \"" + m_filePath.wstr() + L"\" failed with error: " + utility::decodeFromUtf8(error));
+		return;
+	}
+
 	std::vector<clang::tooling::CompileCommand> commands = cdb->getAllCompileCommands();
 	std::set<FilePath> frameworkHeaders;
 	std::set<FilePath> systemHeaders;

src/lib_gui/qt/window/QtSelectPathsDialog.cpp

@@ -7,6 +7,7 @@
 #include <QPushButton>
 
 #include "utility/file/FilePath.h"
+#include "utility/utility.h"
 
 QtSelectPathsDialog::QtSelectPathsDialog(const QString& title, const QString& description, QWidget* parent)
 	: QtTextEditDialog(title, description, parent)
@@ -28,11 +29,11 @@ std::vector<FilePath> QtSelectPathsDialog::getPathsList() const
 	return checkedPaths;
 }
 
-void QtSelectPathsDialog::setPathsList(const std::vector<FilePath>& paths, const std::vector<FilePath>& checkedPaths)
+void QtSelectPathsDialog::setPathsList(const std::vector<FilePath>& paths, const std::vector<FilePath>& checkedPaths, const FilePath& rootPathForRelativePaths)
 {
 	std::set<FilePath> checked(checkedPaths.begin(), checkedPaths.end());
 
-	for (const FilePath& s : paths)
+	for (FilePath s : utility::unique(utility::concat(paths, checkedPaths)))
 	{
 		QListWidgetItem* item = new QListWidgetItem(QString::fromStdWString(s.wstr()), m_list);
 		item->setFlags(item->flags() | Qt::ItemIsUserCheckable); // set checkable flag
@@ -46,6 +47,11 @@ void QtSelectPathsDialog::setPathsList(const std::vector<FilePath>& paths, const
 			item->setCheckState(Qt::Checked);
 		}
 
+		if (!s.isAbsolute())
+		{
+			s = rootPathForRelativePaths.getConcatenated(s);
+		}
+
 		if (!s.exists())
 		{
 			item->setTextColor(Qt::red);

src/lib_gui/qt/window/QtSelectPathsDialog.h

@@ -14,7 +14,7 @@ public:
 	QtSelectPathsDialog(const QString& title, const QString& description, QWidget* parent = 0);
 
 	std::vector<FilePath> getPathsList() const;
-	void setPathsList(const std::vector<FilePath>& paths, const std::vector<FilePath>& checkedPaths);
+	void setPathsList(const std::vector<FilePath>& paths, const std::vector<FilePath>& checkedPaths, const FilePath& rootPathForRelativePaths);
 
 	virtual void populateWindow(QWidget* widget) override;
 	virtual void windowReady() override;

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPath.cpp

@@ -9,6 +9,7 @@
 #include "Application.h"
 #include "qt/element/QtLocationPicker.h"
 #include "qt/view/QtDialogView.h"
+#include "qt/window/project_wizzard/QtProjectWizzardContentPaths.h"
 #include "settings/ApplicationSettings.h"
 #include "settings/SourceGroupSettingsCxxCdb.h"
 #include "settings/SourceGroupSettingsJavaMaven.h"
@@ -164,6 +165,12 @@ void QtProjectWizzardContentPathCDB::save()
 void QtProjectWizzardContentPathCDB::pickedCDBPath()
 {
 	m_window->saveContent();
+
+	if (std::shared_ptr<SourceGroupSettingsCxxCdb> cdbSettings = std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings))
+	{
+		cdbSettings->setIndexedHeaderPaths(QtProjectWizzardContentPathsCDBHeader::getIndexedPathsDerivedFromCDB(cdbSettings));
+	}
+
 	m_window->loadContent();
 }
 

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPaths.cpp

@@ -253,25 +253,52 @@ QString QtProjectWizzardContentPathsSource::getFileNamesDescription() const
 }
 
 
-std::vector<FilePath> QtProjectWizzardContentPathsCDBHeader::getTopLevelHeaderSearchPaths(
+std::vector<FilePath> QtProjectWizzardContentPathsCDBHeader::getIndexedPathsDerivedFromCDB(
 	std::shared_ptr<SourceGroupSettingsCxxCdb> settings)
 {
+	const FilePath projectPath = settings->getProjectDirectoryPath();
 	const FilePath cdbPath = settings->getCompilationDatabasePathExpandedAndAbsolute();
-	if (!cdbPath.exists())
+
+	std::set<FilePath> indexedHeaderPaths;
+	if (!cdbPath.empty() && cdbPath.exists())
 	{
-		LOG_WARNING("Unable to fetch top level header search directories. The provided Compilation Database path does not exist.");
+		for (const FilePath& path : IndexerCommandCxxCdb::getSourceFilesFromCDB(cdbPath))
-		return std::vector<FilePath>();
+		{
+			indexedHeaderPaths.insert(path.getParentDirectory());
+		}
 	}
-	const std::vector<FilePath> sourcePaths = settings->getSourcePaths();
+	else
-	return utility::getTopLevelPaths(utility::unique(utility::concat(
+	{
-		sourcePaths, utility::CompilationDatabase(cdbPath).getAllHeaderPaths()
+		LOG_WARNING("Unable to fetch indexed header paths. The provided Compilation Database path does not exist.");
-	)));
+	}
+
+	for (const FilePath& path : utility::CompilationDatabase(cdbPath).getAllHeaderPaths())
+	{
+		if (path.exists() && projectPath.contains(path))
+		{
+			indexedHeaderPaths.insert(path);
+		}
+	}
+
+	std::vector<FilePath> rootPaths;
+
+	FilePath lastPath;
+	for (const FilePath& path : indexedHeaderPaths)
+	{
+		if (lastPath.empty() || !lastPath.contains(path)) // don't add subdirectories of already added paths
+		{
+			lastPath = path;
+			rootPaths.push_back(path.getRelativeTo(projectPath));
+		}
+	}
+
+	return rootPaths;
 }
 
 QtProjectWizzardContentPathsCDBHeader::QtProjectWizzardContentPathsCDBHeader(
 	std::shared_ptr<SourceGroupSettings> settings, QtProjectWizzardWindow* window
 )
-	: QtProjectWizzardContentPathsSource(settings, window)
+	: QtProjectWizzardContentPaths(settings, window, QtPathListBox::SELECTION_POLICY_FILES_AND_DIRECTORIES)
 {
 	m_showFilesString = "";
 
@@ -305,47 +332,23 @@ void QtProjectWizzardContentPathsCDBHeader::populate(QGridLayout* layout, int& r
 
 void QtProjectWizzardContentPathsCDBHeader::load()
 {
-	if (m_settings->getSourcePaths().empty())
+	if (std::shared_ptr<SourceGroupSettingsCxxCdb> cdbSettings = std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings))
 	{
-		std::shared_ptr<SourceGroupSettingsCxxCdb> cdbSettings =
+		if (cdbSettings->getIndexedHeaderPaths().empty())
-			std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings);
-		std::set<FilePath> sourcePaths;
-
-		const FilePath projectPath = m_settings->getProjectDirectoryPath();
-		const FilePath cdbPath = cdbSettings->getCompilationDatabasePathExpandedAndAbsolute();
-
-		if (!cdbPath.empty() && cdbPath.exists())
 		{
-			for (const FilePath& path : IndexerCommandCxxCdb::getSourceFilesFromCDB(cdbPath))
+			cdbSettings->setIndexedHeaderPaths(getIndexedPathsDerivedFromCDB(cdbSettings));
-			{
-				sourcePaths.insert(path.getParentDirectory());
-			}
 		}
 
-		for (const FilePath& path : getTopLevelHeaderSearchPaths(cdbSettings))
+		m_list->setPaths(cdbSettings->getIndexedHeaderPaths());
-		{
-			if (path.exists() && projectPath.contains(path))
-			{
-				sourcePaths.insert(path);
-			}
-		}
-
-		std::vector<FilePath> rootPaths;
-
-		FilePath lastPath;
-		for (const FilePath& path : sourcePaths)
-		{
-			if (lastPath.empty() || !lastPath.contains(path)) // don't add subdirectories of already added paths
-			{
-				lastPath = path;
-				rootPaths.push_back(path.getRelativeTo(projectPath));
-			}
-		}
-
-		m_settings->setSourcePaths(rootPaths);
 	}
+}
 
-	QtProjectWizzardContentPathsSource::load();
+void QtProjectWizzardContentPathsCDBHeader::save()
+{
+	if (std::shared_ptr<SourceGroupSettingsCxxCdb> cdbSettings = std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings))
+	{
+		cdbSettings->setIndexedHeaderPaths(m_list->getPathsAsDisplayed());
+	}
 }
 
 bool QtProjectWizzardContentPathsCDBHeader::check()
@@ -375,30 +378,33 @@ void QtProjectWizzardContentPathsCDBHeader::buttonClicked()
 
 	if (!m_filesDialog)
 	{
-		const FilePath cdbPath =
+		if (std::shared_ptr<SourceGroupSettingsCxxCdb> cdbSettings = std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings))
-			std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings)->getCompilationDatabasePathExpandedAndAbsolute();
-		if (!cdbPath.exists())
 		{
-			QMessageBox msgBox;
+			const FilePath cdbPath = cdbSettings->getCompilationDatabasePathExpandedAndAbsolute();
-			msgBox.setText("The provided Compilation Database path does not exist.");
+			if (!cdbPath.exists())
-			msgBox.setDetailedText(QString::fromStdWString(cdbPath.wstr()));
+			{
-			msgBox.exec();
+				QMessageBox msgBox;
-			return;
+				msgBox.setText("The provided Compilation Database path does not exist.");
+				msgBox.setDetailedText(QString::fromStdWString(cdbPath.wstr()));
+				msgBox.exec();
+				return;
+			}
+
+			m_filesDialog = std::make_shared<QtSelectPathsDialog>(
+				"Select from Include Paths",
+				"The list contains all Include Paths found in the Compilation Database. Red paths do not exist. Select the "
+				"paths containing the header files you want to index with Sourcetrail.");
+			m_filesDialog->setup();
+
+			connect(m_filesDialog.get(), &QtSelectPathsDialog::finished, this, &QtProjectWizzardContentPathsCDBHeader::savedFilesDialog);
+			connect(m_filesDialog.get(), &QtSelectPathsDialog::canceled, this, &QtProjectWizzardContentPathsCDBHeader::closedFilesDialog);
+
+			dynamic_cast<QtSelectPathsDialog*>(m_filesDialog.get())->setPathsList(
+				getIndexedPathsDerivedFromCDB(cdbSettings),
+				cdbSettings->getIndexedHeaderPaths(),
+				m_settings->getProjectDirectoryPath()
+			);
 		}
-
-		m_filesDialog = std::make_shared<QtSelectPathsDialog>(
-			"Select from Include Paths",
-			"The list contains all Include Paths found in the Compilation Database. Red paths do not exist. Select the "
-			"paths containing the header files you want to index with Sourcetrail.");
-		m_filesDialog->setup();
-
-		connect(m_filesDialog.get(), &QtSelectPathsDialog::finished, this, &QtProjectWizzardContentPathsCDBHeader::savedFilesDialog);
-		connect(m_filesDialog.get(), &QtSelectPathsDialog::canceled, this, &QtProjectWizzardContentPathsCDBHeader::closedFilesDialog);
-
-		dynamic_cast<QtSelectPathsDialog*>(m_filesDialog.get())->setPathsList(
-			getTopLevelHeaderSearchPaths(std::dynamic_pointer_cast<SourceGroupSettingsCxxCdb>(m_settings)),
-			m_settings->getSourcePaths()
-		);
 	}
 
 	m_filesDialog->showWindow();
@@ -407,9 +413,7 @@ void QtProjectWizzardContentPathsCDBHeader::buttonClicked()
 
 void QtProjectWizzardContentPathsCDBHeader::savedFilesDialog()
 {
-	// TODO: extend instead of replace
 	m_list->setPaths(dynamic_cast<QtSelectPathsDialog*>(m_filesDialog.get())->getPathsList());
-
 	closedFilesDialog();
 }
 

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPaths.h

@@ -79,19 +79,22 @@ public:
 	virtual QString getFileNamesDescription() const override;
 };
 
+
 class QtProjectWizzardContentPathsCDBHeader
-	: public QtProjectWizzardContentPathsSource
+	: public QtProjectWizzardContentPaths
 {
 	Q_OBJECT
 
 public:
-	static std::vector<FilePath> getTopLevelHeaderSearchPaths(std::shared_ptr<SourceGroupSettingsCxxCdb> settings);
+	static std::vector<FilePath> getIndexedPathsDerivedFromCDB(std::shared_ptr<SourceGroupSettingsCxxCdb> settings);
 
 	QtProjectWizzardContentPathsCDBHeader(std::shared_ptr<SourceGroupSettings> settings, QtProjectWizzardWindow* window);
 
 	virtual void populate(QGridLayout* layout, int& row) override;
 
+	// QtProjectWizzardContent implementation
 	virtual void load() override;
+	virtual void save() override;
 
 	virtual bool check() override;