logic: improved include detection

* also search for include paths of include directives extracted from header search paths
* extracted logic to gather unsolved includes
* extracted some code to filetree class
This commit is contained in:
mlangkabel
2018-01-23 21:35:53 +01:00
parent 56cac267a2
commit 7c863ddf00
13 changed files with 269 additions and 148 deletions
+2
View File
@@ -356,6 +356,8 @@ add_files(
utility/file/FileRegisterStateData.h
utility/file/FileSystem.cpp
utility/file/FileSystem.h
utility/file/FileTree.cpp
utility/file/FileTree.h
utility/interprocess/SharedMemory.cpp
utility/interprocess/SharedMemory.h
+86
View File
@@ -0,0 +1,86 @@
#include "utility/file/FileTree.h"
#include "utility/file/FileSystem.h"
FileTree::FileTree(const FilePath& rootPath)
: m_rootPath(rootPath.getAbsolute().makeCanonical())
{
if (m_rootPath.exists())
{
if (m_rootPath.isDirectory())
{
for (const FilePath& filePath : FileSystem::getFilePathsFromDirectory(m_rootPath))
{
m_files[filePath.fileName()].insert(filePath);
}
}
else
{
m_files[m_rootPath.fileName()].insert(m_rootPath);
}
}
}
FilePath FileTree::getAbsoluteRootPathForRelativeFilePath(const FilePath& relativeFilePath)
{
std::vector<FilePath> rootPaths = doGetAbsoluteRootPathsForRelativeFilePath(relativeFilePath, false);
if (!rootPaths.empty())
{
return rootPaths.front();
}
return FilePath();
}
std::vector<FilePath> FileTree::getAbsoluteRootPathsForRelativeFilePath(const FilePath& relativeFilePath)
{
return doGetAbsoluteRootPathsForRelativeFilePath(relativeFilePath, true);
}
std::vector<FilePath> FileTree::doGetAbsoluteRootPathsForRelativeFilePath(const FilePath& relativeFilePath, bool allowMultipleResults)
{
std::vector<FilePath> rootPaths;
std::unordered_map<std::string, std::set<FilePath>>::const_iterator it = m_files.find(relativeFilePath.fileName());
if (it != m_files.end())
{
for (FilePath existingFilePath : it->second)
{
existingFilePath = existingFilePath.getParentDirectory();
bool ok = true;
{
FilePath temp = relativeFilePath.getParentDirectory();
while (!temp.empty())
{
if (temp.fileName() == "..")
{
std::vector<FilePath> subDirectories = FileSystem::getDirectSubDirectories(existingFilePath);
if (!subDirectories.empty())
{
existingFilePath = subDirectories.front();
}
else
{
ok = false;
break;
}
}
else
{
existingFilePath = existingFilePath.getParentDirectory();
}
temp = temp.getParentDirectory();
}
}
if (ok)
{
rootPaths.push_back(existingFilePath);
if (!allowMultipleResults)
{
break;
}
}
}
}
return rootPaths;
}
+25
View File
@@ -0,0 +1,25 @@
#ifndef FILE_TREE_H
#define FILE_TREE_H
#include <string>
#include <set>
#include <unordered_map>
#include "utility/file/FilePath.h"
class FileTree
{
public:
FileTree(const FilePath& rootPath);
FilePath getAbsoluteRootPathForRelativeFilePath(const FilePath& relativeFilePath);
std::vector<FilePath> getAbsoluteRootPathsForRelativeFilePath(const FilePath& relativeFilePath);
private:
std::vector<FilePath> doGetAbsoluteRootPathsForRelativeFilePath(const FilePath& relativeFilePath, bool allowMultipleResults);
FilePath m_rootPath;
std::unordered_map<std::string, std::set<FilePath>> m_files;
};
#endif // FILE_TREE_H
+72 -115
View File
@@ -5,7 +5,7 @@
#include "utility/IncludeDirective.h"
#include "utility/file/FilePath.h"
#include "utility/file/FileSystem.h"
#include "utility/file/FileTree.h"
#include "utility/text/TextAccess.h"
#include "utility/utility.h"
#include "utility/utilityString.h"
@@ -57,47 +57,15 @@ std::vector<IncludeDirective> IncludeProcessing::getUnresolvedIncludeDirectives(
for (size_t i = 0; i < quantiles.size(); i++)
{
const std::vector<FilePath>& quantile = quantiles[i];
progress(float(i) / quantiles.size());
std::set<FilePath> unprocessedFilePaths(quantile.begin(), quantile.end());
while (!unprocessedFilePaths.empty())
{
std::transform(
unprocessedFilePaths.begin(), unprocessedFilePaths.end(),
std::inserter(processedFilePaths, processedFilePaths.begin()),
[](const FilePath& p){ return p.str(); }
);
std::set<FilePath> unprocessedFilePathsForNextIteration;
for (const FilePath& filePath: unprocessedFilePaths)
{
for (const IncludeDirective& includeDirective: getIncludeDirectives(filePath))
{
const FilePath resolvedIncludePath = resolveIncludeDirective(includeDirective, headerSearchDirectories).makeCanonical();
if (resolvedIncludePath.empty())
{
unresolvedIncludeDirectives.insert(includeDirective);
}
else if (processedFilePaths.find(resolvedIncludePath.str()) == processedFilePaths.end())
{
for (const FilePath& indexedPath: indexedPaths)
{
if (indexedPath.contains(resolvedIncludePath))
{
unprocessedFilePathsForNextIteration.insert(resolvedIncludePath);
break;
}
}
}
}
}
unprocessedFilePaths = unprocessedFilePathsForNextIteration;
}
const std::vector<IncludeDirective> directives = doGetUnresolvedIncludeDirectives(
utility::toSet(quantiles[i]),
processedFilePaths,
indexedPaths,
headerSearchDirectories
);
std::copy(directives.begin(), directives.end(), std::inserter(unresolvedIncludeDirectives, unresolvedIncludeDirectives.end()));
}
std::vector<IncludeDirective> ret;
@@ -115,25 +83,16 @@ std::vector<IncludeDirective> IncludeProcessing::getUnresolvedIncludeDirectives(
std::set<FilePath> IncludeProcessing::getHeaderSearchDirectories(
const std::set<FilePath>& sourceFilePaths,
const std::set<FilePath>& searchedPaths,
const std::set<FilePath>& currentHeaderSearchDirectories,
const size_t desiredQuantileCount, std::function<void(float)> progress
)
{
progress(0.0f);
std::map<std::string, std::set<FilePath>> existingFilePaths;
std::vector<std::shared_ptr<FileTree>> existingFileTrees;
for (const FilePath& searchedPath : searchedPaths)
{
if (searchedPath.isDirectory())
{
for (const FilePath& filePath : FileSystem::getFilePathsFromDirectory(searchedPath.getCanonical()))
{
existingFilePaths[filePath.fileName()].insert(filePath);
}
}
else
{
existingFilePaths[searchedPath.fileName()].insert(searchedPath);
}
existingFileTrees.push_back(std::make_shared<FileTree>(searchedPath));
}
std::set<FilePath> headerSearchDirectories;
@@ -142,18 +101,16 @@ std::set<FilePath> IncludeProcessing::getHeaderSearchDirectories(
for (size_t i = 0; i < quantiles.size(); i++)
{
const std::vector<FilePath>& quantile = quantiles[i];
progress(float(i) / quantiles.size());
std::set<FilePath> unprocessedFilePaths(quantile.begin(), quantile.end());
std::set<FilePath> unprocessedFilePaths(quantiles[i].begin(), quantiles[i].end());
while (!unprocessedFilePaths.empty())
{
std::transform(
unprocessedFilePaths.begin(), unprocessedFilePaths.end(),
std::inserter(processedFilePaths, processedFilePaths.begin()),
[](const FilePath& p) { return p.str(); }
[](const FilePath& p) { return p.getAbsolute().str(); }
);
std::set<FilePath> unprocessedFilePathsForNextIteration;
@@ -164,61 +121,20 @@ std::set<FilePath> IncludeProcessing::getHeaderSearchDirectories(
{
const FilePath includedFilePath = includeDirective.getIncludedFile();
FilePath foundIncludedPath;
if (includedFilePath.isAbsolute())
FilePath foundIncludedPath = resolveIncludeDirective(includeDirective, currentHeaderSearchDirectories);
if (foundIncludedPath.empty())
{
foundIncludedPath = includedFilePath;
}
else
{
const FilePath relativeToIncludingPath = includeDirective.getIncludingFile().getParentDirectory().concatenate(includedFilePath);
if (relativeToIncludingPath.exists())
for (std::shared_ptr<FileTree> existingFileTree : existingFileTrees)
{
foundIncludedPath = relativeToIncludingPath;
}
else
{
std::map<std::string, std::set<FilePath>>::const_iterator it = existingFilePaths.find(includedFilePath.fileName());
if (it != existingFilePaths.end())
// TODO: handle the case where a file can be found by two different paths
const FilePath rootPath = existingFileTree->getAbsoluteRootPathForRelativeFilePath(includedFilePath);
if (!rootPath.empty())
{
// TODO: handle the case where a file can be found by two different paths
for (FilePath existingFilePath : it->second)
foundIncludedPath = rootPath.getConcatenated(includedFilePath);
if (foundIncludedPath.exists())
{
existingFilePath = existingFilePath.getParentDirectory();
bool ok = true;
{
FilePath tempIncludedFilePath = includedFilePath.getParentDirectory();
while (!tempIncludedFilePath.empty())
{
if (tempIncludedFilePath.fileName() == "..")
{
std::vector<FilePath> subDirectories = FileSystem::getDirectSubDirectories(existingFilePath);
if (!subDirectories.empty())
{
existingFilePath = subDirectories.front();
}
else
{
ok = false;
break;
}
}
else
{
existingFilePath = existingFilePath.getParentDirectory();
}
tempIncludedFilePath = tempIncludedFilePath.getParentDirectory();
}
}
if (ok)
{
foundIncludedPath = existingFilePath.getConcatenated(includedFilePath);
if (foundIncludedPath.exists())
{
headerSearchDirectories.insert(existingFilePath);
break;
}
}
headerSearchDirectories.insert(rootPath);
break;
}
}
}
@@ -227,14 +143,7 @@ std::set<FilePath> IncludeProcessing::getHeaderSearchDirectories(
{
if (processedFilePaths.find(foundIncludedPath.str()) == processedFilePaths.end())
{
for (const FilePath& searchedPath : searchedPaths)
{
if (searchedPath.contains(foundIncludedPath))
{
unprocessedFilePathsForNextIteration.insert(foundIncludedPath);
break;
}
}
unprocessedFilePathsForNextIteration.insert(foundIncludedPath);
}
}
}
@@ -291,6 +200,54 @@ std::vector<IncludeDirective> IncludeProcessing::getIncludeDirectives(std::share
return includeDirectives;
}
std::vector<IncludeDirective> IncludeProcessing::doGetUnresolvedIncludeDirectives(
std::set<FilePath> filePathsToProcess,
std::unordered_set<std::string>& processedFilePaths,
const std::set<FilePath>& indexedPaths,
const std::set<FilePath>& headerSearchDirectories
)
{
std::vector<IncludeDirective> unresolvedIncludeDirectives;
while (!filePathsToProcess.empty())
{
std::transform(
filePathsToProcess.begin(), filePathsToProcess.end(),
std::inserter(processedFilePaths, processedFilePaths.begin()),
[](const FilePath& p) { return p.getAbsolute().makeCanonical().str(); }
);
std::set<FilePath> filePathsToProcessForNextIteration;
for (const FilePath& filePath : filePathsToProcess)
{
for (const IncludeDirective& includeDirective : getIncludeDirectives(filePath))
{
const FilePath resolvedIncludePath = resolveIncludeDirective(includeDirective, headerSearchDirectories).makeCanonical();
if (resolvedIncludePath.empty())
{
unresolvedIncludeDirectives.push_back(includeDirective);
}
else if (processedFilePaths.find(resolvedIncludePath.str()) == processedFilePaths.end())
{
for (const FilePath& indexedPath : indexedPaths)
{
if (indexedPath.contains(resolvedIncludePath))
{
filePathsToProcessForNextIteration.insert(resolvedIncludePath);
break;
}
}
}
}
}
filePathsToProcess.clear();
filePathsToProcess.swap(filePathsToProcessForNextIteration);
}
return unresolvedIncludeDirectives;
}
FilePath IncludeProcessing::resolveIncludeDirective(
const IncludeDirective& includeDirective,
const std::set<FilePath>& headerSearchDirectories
+9
View File
@@ -3,6 +3,7 @@
#include <memory>
#include <set>
#include <unordered_set>
#include <vector>
#include "utility/OrderedCache.h"
@@ -24,6 +25,7 @@ public:
static std::set<FilePath> getHeaderSearchDirectories(
const std::set<FilePath>& sourceFilePaths,
const std::set<FilePath>& searchedPaths,
const std::set<FilePath>& currentHeaderSearchDirectories,
const size_t desiredQuantileCount, std::function<void(float)> progress
);
@@ -32,6 +34,13 @@ public:
static std::vector<IncludeDirective> getIncludeDirectives(std::shared_ptr<TextAccess> textAccess);
private:
static std::vector<IncludeDirective> doGetUnresolvedIncludeDirectives(
std::set<FilePath> filePathsToProcess,
std::unordered_set<std::string>& processedFilePaths,
const std::set<FilePath>& indexedPaths,
const std::set<FilePath>& headerSearchDirectories
);
static FilePath resolveIncludeDirective(
const IncludeDirective& includeDirective,
const std::set<FilePath>& headerSearchDirectories
@@ -589,7 +589,6 @@ void QtProjectWizzardContentPathsHeaderSearch::finishedSelectDetectIncludesRootP
{
// TODO: regard Force Includes here, too!
const std::vector<FilePath> searchedPaths = m_settings->makePathsExpandedAndAbsolute(m_pathsDialog->getPaths());
closedPathsDialog();
std::thread([=]()
@@ -599,7 +598,7 @@ void QtProjectWizzardContentPathsHeaderSearch::finishedSelectDetectIncludesRootP
std::shared_ptr<QtDialogView> dialogView = std::dynamic_pointer_cast<QtDialogView>(Application::getInstance()->getDialogView());
std::set<FilePath> sourceFilePaths;
std::vector<FilePath> headerSearchPaths;
{
dialogView->setParentWindow(m_window);
dialogView->showUnknownProgressDialog("Processing", "Gathering Source Files");
@@ -614,6 +613,13 @@ void QtProjectWizzardContentPathsHeaderSearch::finishedSelectDetectIncludesRootP
m_settings->getSourceExtensions()
);
sourceFilePaths = fileManager.getAllSourceFilePaths();
headerSearchPaths = ApplicationSettings::getInstance()->getHeaderSearchPathsExpanded();
if (std::shared_ptr<SourceGroupSettingsCxx> cxxSettings =
std::dynamic_pointer_cast<SourceGroupSettingsCxx>(m_settings))
{
utility::append(headerSearchPaths, cxxSettings->getHeaderSearchPathsExpandedAndAbsolute());
}
}
{
dialogView->setParentWindow(m_window);
@@ -624,6 +630,7 @@ void QtProjectWizzardContentPathsHeaderSearch::finishedSelectDetectIncludesRootP
detectedHeaderSearchPaths = IncludeProcessing::getHeaderSearchDirectories(
sourceFilePaths,
utility::toSet(searchedPaths),
utility::toSet(headerSearchPaths),
log2(sourceFilePaths.size()),
[&](const float progress)
{
+62 -31
View File
@@ -3,6 +3,7 @@
#include "utility/text/TextAccess.h"
#include "utility/IncludeDirective.h"
#include "utility/IncludeProcessing.h"
#include "utility/utility.h"
class CxxIncludeProcessingTestSuite: public CxxTest::TestSuite
{
@@ -71,48 +72,78 @@ public:
void test_header_search_path_detection_does_not_find_path_relative_to_including_file()
{
std::set<FilePath> headerSearchDirectories = IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_does_not_find_path_relative_to_including_file/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_does_not_find_path_relative_to_including_file") },
1, [&](float) {}
);
std::vector<FilePath> headerSearchDirectories = utility::toVector(IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_does_not_find_path_relative_to_including_file/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_does_not_find_path_relative_to_including_file") },
{ },
1, [](float) {}
));
TS_ASSERT(headerSearchDirectories.empty());
}
void test_header_search_path_detection_finds_path_inside_sub_directory()
{
std::set<FilePath> headerSearchDirectories = IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_inside_sub_directory/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_inside_sub_directory") },
1, [&](float) {}
);
std::vector<FilePath> headerSearchDirectories = utility::toVector(IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_inside_sub_directory/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_inside_sub_directory") },
{},
1, [](float) {}
));
TS_ASSERT(!headerSearchDirectories.empty());
if (!headerSearchDirectories.empty())
{
TS_ASSERT_EQUALS(
"CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_inside_sub_directory/include",
headerSearchDirectories.begin()->getRelativeTo(FilePath("data").getAbsolute()).str()
);
}
TS_ASSERT(utility::containsElement<FilePath>(
headerSearchDirectories,
FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_inside_sub_directory/include").makeAbsolute()
));
}
void test_header_search_path_detection_finds_path_relative_to_sub_directory()
{
std::set<FilePath> headerSearchDirectories = IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_relative_to_sub_directory/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_relative_to_sub_directory") },
1, [&](float) {}
);
std::vector<FilePath> headerSearchDirectories = utility::toVector(IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_relative_to_sub_directory/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_relative_to_sub_directory") },
{},
1, [](float) {}
));
TS_ASSERT(!headerSearchDirectories.empty());
if (!headerSearchDirectories.empty())
{
TS_ASSERT_EQUALS(
"CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_relative_to_sub_directory/include",
headerSearchDirectories.begin()->getRelativeTo(FilePath("data").getAbsolute()).str()
);
}
TS_ASSERT(utility::containsElement<FilePath>(
headerSearchDirectories,
FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_relative_to_sub_directory/include").makeAbsolute()
));
}
void test_header_search_path_detection_finds_path_included_in_header_search_path()
{
std::vector<FilePath> headerSearchDirectories = utility::toVector(IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_header_search_path/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_header_search_path/include_b") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_header_search_path/include_a") },
1, [](float) {}
));
TS_ASSERT(utility::containsElement<FilePath>(
headerSearchDirectories,
FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_header_search_path/include_b").makeAbsolute()
));
}
void test_header_search_path_detection_finds_path_included_in_future_header_search_path()
{
std::vector<FilePath> headerSearchDirectories = utility::toVector(IncludeProcessing::getHeaderSearchDirectories(
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_future_header_search_path/a.cpp") },
{ FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_future_header_search_path") },
{ },
1, [](float) {}
));
TS_ASSERT(utility::containsElement<FilePath>(
headerSearchDirectories,
FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_future_header_search_path/include_a").makeAbsolute()
));
TS_ASSERT(utility::containsElement<FilePath>(
headerSearchDirectories,
FilePath("data/CxxIncludeProcessingTestSuite/test_header_search_path_detection_finds_path_included_in_future_header_search_path/include_b").makeAbsolute()
));
}
};