ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: removed C/C++ "unparsed" check from indexer

Browse Source 
Cxx AstVisitor re-visits previously or simultaneously indexed files because the "visible" content may have changed due to other preprocessor defines etc.
This commit is contained in:
mlangkabel
2018-04-24 13:25:06 +02:00
parent c5d0362df5
commit ca3e2f48ee
18 changed files with 13 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/CMakeLists.txt
-2
View File
@@ -358,8 +358,6 @@ add_files(
utility/file/FilePathFilter.h
utility/file/FileRegister.cpp
utility/file/FileRegister.h
utility/file/FileRegisterStateData.cpp
utility/file/FileRegisterStateData.h
utility/file/FileSystem.cpp
utility/file/FileSystem.h
utility/file/FileTree.cpp
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+1 -8
View File
@@ -1,7 +1,6 @@
#include "InterprocessIndexer.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/logging/logging.h"
#include "data/indexer/IndexerCommand.h"
@@ -50,19 +49,13 @@ void InterprocessIndexer::work()
LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles());
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
);
LOG_INFO_STREAM(<< m_processId << " starting to index current file");
std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status");
m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory");
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
-81
View File
@@ -8,7 +8,6 @@ const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_
const char* InterprocessIndexingStatusManager::s_indexingFilesKeyName = "indexing_files";
const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files";
const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files";
const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files";
const char* InterprocessIndexingStatusManager::s_finishedProcessIdsKeyName = "finished_process_ids";
InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner)
@@ -163,83 +162,3 @@ std::vector<FilePath> InterprocessIndexingStatusManager::getCrashedSourceFilePat
return crashedFiles;
}
std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
{
std::set<FilePath> result;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* files =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!files)
{
return result;
}
for (auto& file : *files)
{
result.insert(FilePath(utility::decodeFromUtf8(file.c_str())));
}
return result;
}
void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
{
const unsigned int overestimationMultiplier = 3;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* indexedFiles =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
std::set<std::string> oldFiles;
for (auto& indexedFile : *indexedFiles)
{
oldFiles.insert(indexedFile.c_str());
}
std::set<std::string> newFiles;
for (const FilePath& filePath : filePaths)
{
if (oldFiles.find(utility::encodeToUtf8(filePath.wstr())) == oldFiles.end())
{
newFiles.insert(utility::encodeToUtf8(filePath.wstr()));
}
}
size_t estimatedSize = 262144;
for (auto& newFile : newFiles)
{
estimatedSize += sizeof(SharedMemory::String) + newFile.size();
}
estimatedSize *= overestimationMultiplier;
while (access.getFreeMemorySize() < estimatedSize)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
access.growMemory(access.getMemorySize());
LOG_INFO("growing memory succeeded");
indexedFiles = access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
}
for (const std::string& newFile: newFiles)
{
indexedFiles->push_back(SharedMemory::String(newFile.c_str(), access.getAllocator()));
}
LOG_INFO(access.logString());
}
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h
-4
View File
@@ -21,16 +21,12 @@ public:
std::vector<FilePath> getCurrentlyIndexedSourceFilePaths();
std::vector<FilePath> getCrashedSourceFilePaths();
std::set<FilePath> getIndexedFiles();
void addIndexedFiles(std::set<FilePath> filePaths);
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_indexingFilesKeyName;
static const char* s_currentFilesKeyName;
static const char* s_crashedFilesKeyName;
static const char* s_indexedFilesKeyName;
static const char* s_finishedProcessIdsKeyName;
};
src/lib/utility/file/FileRegister.cpp
+1 -24
View File
@@ -4,13 +4,11 @@
#include "utility/file/FilePathFilter.h"
FileRegister::FileRegister(
const FileRegisterStateData& stateData,
const FilePath& currentPath,
const std::set<FilePath>& indexedPaths,
const std::set<FilePathFilter>& excludeFilters
)
: m_stateData(stateData)
, m_currentPath(currentPath)
: m_currentPath(currentPath)
, m_indexedPaths(indexedPaths)
, m_excludeFilters(excludeFilters)
, m_hasFilePathCache(
@@ -64,31 +62,10 @@ FileRegister::FileRegister(
{
}
FileRegister::~FileRegister()
{
}
const FileRegisterStateData& FileRegister::getStateData() const
{
return m_stateData;
}
void FileRegister::markFileIndexing(const FilePath& filePath)
{
m_stateData.markFileIndexing(filePath);
}
void FileRegister::markIndexingFilesIndexed()
{
m_stateData.markIndexingFilesIndexed();
}
bool FileRegister::fileIsIndexed(const FilePath& filePath) const
{
return m_stateData.fileIsIndexed(filePath);
}
bool FileRegister::hasFilePath(const FilePath& filePath) const
{
return m_hasFilePathCache.getValue(filePath.wstr());
src/lib/utility/file/FileRegister.h
+1 -8
View File
@@ -3,7 +3,7 @@
#include <set>
#include "utility/file/FileRegisterStateData.h"
#include "utility/file/FilePath.h"
#include "utility/UnorderedCache.h"
class FilePathFilter;
@@ -12,22 +12,15 @@ class FileRegister
{
public:
FileRegister(
const FileRegisterStateData& stateData,
const FilePath& currentPath,
const std::set<FilePath>& indexedPaths,
const std::set<FilePathFilter>& excludeFilters
);
virtual ~FileRegister();
const FileRegisterStateData& getStateData() const;
void markFileIndexing(const FilePath& filePath);
void markIndexingFilesIndexed();
virtual bool fileIsIndexed(const FilePath& filePath) const;
virtual bool hasFilePath(const FilePath& filePath) const;
private:
FileRegisterStateData m_stateData;
const FilePath& m_currentPath;
const std::set<FilePath> m_indexedPaths;
const std::set<FilePathFilter> m_excludeFilters;
src/lib/utility/file/FileRegisterStateData.cpp
-71
View File
@@ -1,71 +0,0 @@
#include "utility/file/FileRegisterStateData.h"
#include "utility/file/FilePath.h"
FileRegisterStateData::FileRegisterStateData()
{
}
FileRegisterStateData::FileRegisterStateData(const FileRegisterStateData& o)
{
this->inject(o);
}
void FileRegisterStateData::inject(const FileRegisterStateData& o)
{
for (const auto& it: o.m_filePaths)
{
if (it.second == STATE_INDEXED)
{
m_filePaths[it.first] = STATE_INDEXED;
}
}
}
void FileRegisterStateData::markFileIndexing(const FilePath& filePath)
{
m_filePaths[filePath] = STATE_INDEXING;
}
void FileRegisterStateData::markIndexingFilesIndexed()
{
for (auto& it: m_filePaths)
{
if (it.second == STATE_INDEXING)
{
it.second = STATE_INDEXED;
}
}
}
bool FileRegisterStateData::fileIsIndexed(const FilePath& filePath) const
{
auto it = m_filePaths.find(filePath);
if (it != m_filePaths.end())
{
return it->second == STATE_INDEXED;
}
return false;
}
void FileRegisterStateData::setIndexedFiles(const std::set<FilePath>& filePaths)
{
for (auto& path : filePaths)
{
m_filePaths[path] = STATE_INDEXED;
}
}
std::set<FilePath> FileRegisterStateData::getIndexedFiles() const
{
std::set<FilePath> paths;
for (auto& it : m_filePaths)
{
if (it.second == STATE_INDEXED)
{
paths.insert(it.first);
}
}
return paths;
}
src/lib/utility/file/FileRegisterStateData.h
-36
View File
@@ -1,36 +0,0 @@
#ifndef FILE_REGISTER_STATE_DATA_H
#define FILE_REGISTER_STATE_DATA_H
#include <map>
#include <mutex>
#include <set>
#include "utility/file/FilePath.h"
class FileRegisterStateData
{
public:
FileRegisterStateData();
FileRegisterStateData(const FileRegisterStateData& o);
void inject(const FileRegisterStateData& o);
void markFileIndexing(const FilePath& filePath);
void markIndexingFilesIndexed();
bool fileIsIndexed(const FilePath& filePath) const;
void setIndexedFiles(const std::set<FilePath>& filePaths);
std::set<FilePath> getIndexedFiles() const;
private:
enum IndexingState
{
STATE_NON_INDEXED,
STATE_INDEXING,
STATE_INDEXED
};
std::map<FilePath, IndexingState> m_filePaths;
};
#endif // FILE_REGISTER_STATE_DATA_H
src/lib_cxx/data/indexer/IndexerCxx.h
-1
View File
@@ -41,7 +41,6 @@ std::shared_ptr<IntermediateStorage> IndexerCxx<IndexerCommandType, ParserType>:
else
{
storage->setFilesWithErrorsIncomplete();
fileRegister->markIndexingFilesIndexed();
}
if (IndexerBase::interrupted())
src/lib_cxx/data/parser/cxx/CommentHandler.cpp
+1 -1
View File
@@ -36,7 +36,7 @@ bool CommentHandler::HandleComment(clang::Preprocessor& preprocessor, clang::Sou
if (fileEntry != nullptr && fileEntry->isValid())
{
FilePath filePath = m_canonicalFilePathCache->getCanonicalFilePath(fileEntry);
if (m_fileRegister->hasFilePath(filePath) && !m_fileRegister->fileIsIndexed(filePath))
if (m_fileRegister->hasFilePath(filePath))
{
m_client->recordComment(ParseLocation(
filePath,
src/lib_cxx/data/parser/cxx/CxxAstVisitor.cpp
-42
View File
@@ -789,48 +789,6 @@ ParseLocation CxxAstVisitor::getParseLocation(const clang::SourceRange& sourceRa
return parseLocation;
}
bool CxxAstVisitor::isLocatedInUnparsedProjectFile(clang::SourceLocation loc)
{
const clang::SourceManager& sourceManager = m_astContext->getSourceManager();
clang::FileID fileId;
if (loc.isValid())
{
if (sourceManager.isWrittenInMainFile(loc))
{
return true;
}
fileId = sourceManager.getFileID(loc);
}
if (fileId.isValid())
{
auto it = m_inUnparsedProjectFileMap.find(fileId);
if (it != m_inUnparsedProjectFileMap.end())
{
return it->second;
}
bool ret = false;
const clang::FileEntry* fileEntry = sourceManager.getFileEntryForID(fileId);
if (fileEntry != nullptr && fileEntry->isValid())
{
FilePath filePath = getCanonicalFilePathCache()->getCanonicalFilePath(fileEntry);
if (m_fileRegister->hasFilePath(filePath))
{
ret = !(m_fileRegister->fileIsIndexed(filePath));
}
}
m_inUnparsedProjectFileMap[fileId] = ret;
return ret;
}
return false;
}
bool CxxAstVisitor::isLocatedInProjectFile(clang::SourceLocation loc)
{
const clang::SourceManager& sourceManager = m_astContext->getSourceManager();
src/lib_cxx/data/parser/cxx/CxxAstVisitorComponentIndexer.cpp
+3 -11
View File
@@ -801,9 +801,7 @@ bool CxxAstVisitorComponentIndexer::shouldVisitDecl(const clang::Decl* decl)
loc = decl->getLocation();
}
bool declIsImplicit = utility::isImplicit(decl);
if ((declIsImplicit && getAstVisitor()->isLocatedInProjectFile(loc)) ||
(!declIsImplicit && getAstVisitor()->isLocatedInUnparsedProjectFile(loc)))
if (getAstVisitor()->isLocatedInProjectFile(loc))
{
return true;
}
@@ -813,22 +811,16 @@ bool CxxAstVisitorComponentIndexer::shouldVisitDecl(const clang::Decl* decl)
bool CxxAstVisitorComponentIndexer::shouldVisitReference(const clang::SourceLocation& referenceLocation, const clang::Decl* contextDecl)
{
bool declIsImplicit = true; // default value is "true" to make sure that everything that should be visited gets visited.
if (contextDecl)
{
declIsImplicit = utility::isImplicit(contextDecl);
}
clang::SourceLocation loc = m_astContext->getSourceManager().getExpansionLoc(referenceLocation);
if (loc.isInvalid())
{
loc = referenceLocation;
}
if ((declIsImplicit && getAstVisitor()->isLocatedInProjectFile(loc)) ||
(!declIsImplicit && getAstVisitor()->isLocatedInUnparsedProjectFile(loc)))
if (getAstVisitor()->isLocatedInProjectFile(loc))
{
return true;
}
return false;
}
src/lib_cxx/data/parser/cxx/PreprocessorCallbacks.cpp
+5 -14
View File
@@ -35,20 +35,11 @@ void PreprocessorCallbacks::FileChanged(
if (fileEntry != nullptr && fileEntry->isValid())
{
m_currentPath = m_canonicalFilePathCache->getCanonicalFilePath(fileEntry);
}
if (!m_currentPath.empty())
{
bool hasFilePath = m_fileRegister->hasFilePath(m_currentPath);
m_client->recordFile(FileSystem::getFileInfoForPath(m_currentPath), hasFilePath); // todo: fix for tests
if (hasFilePath && !m_fileRegister->fileIsIndexed(m_currentPath))
if (!m_currentPath.empty())
{
if (reason == EnterFile)
{
m_fileRegister->markFileIndexing(m_currentPath);
}
bool hasFilePath = m_fileRegister->hasFilePath(m_currentPath);
m_client->recordFile(FileSystem::getFileInfoForPath(m_currentPath), hasFilePath); // todo: fix for tests
}
}
}
@@ -75,7 +66,7 @@ void PreprocessorCallbacks::InclusionDirective(
void PreprocessorCallbacks::MacroDefined(const clang::Token& macroNameToken, const clang::MacroDirective* macroDirective)
{
if (!m_currentPath.empty() && m_fileRegister->hasFilePath(m_currentPath) && !m_fileRegister->fileIsIndexed(m_currentPath) /*TODO: remove this last check if indexed isn't important anymore*/)
if (!m_currentPath.empty() && m_fileRegister->hasFilePath(m_currentPath))
{
// ignore builtin macros
if (m_sourceManager.getSpellingLoc(macroNameToken.getLocation()).printToString(m_sourceManager)[0] == '<')
@@ -128,7 +119,7 @@ void PreprocessorCallbacks::MacroExpands(
void PreprocessorCallbacks::onMacroUsage(const clang::Token& macroNameToken)
{
if (!m_currentPath.empty() && m_fileRegister->hasFilePath(m_currentPath) && !m_fileRegister->fileIsIndexed(m_currentPath) /*TODO: remove this last check if indexed isn't important anymore*/ && isLocatedInProjectFile(macroNameToken.getLocation()))
if (!m_currentPath.empty() && m_fileRegister->hasFilePath(m_currentPath) && isLocatedInProjectFile(macroNameToken.getLocation()))
{
const ParseLocation loc = getParseLocation(macroNameToken);
src/lib_java/data/indexer/IndexerJava.cpp
-1
View File
@@ -32,7 +32,6 @@ std::shared_ptr<IntermediateStorage> IndexerJava::doIndex(
else
{
storage->setFilesWithErrorsIncomplete();
fileRegister->markIndexingFilesIndexed();
}
if (interrupted())
src/test/CxxIndexSampleProjectsTestSuite.h
-1
View File
@@ -127,7 +127,6 @@ private:
const FilePath workingDirectory(L".");
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
FileRegisterStateData(),
sourceFilePath,
indexedPaths,
excludedFilters
src/test/JavaIndexSampleProjectsTestSuite.h
-1
View File
@@ -251,7 +251,6 @@ private:
std::set<FilePathFilter> excludeFilters = { };
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
FileRegisterStateData(),
sourceFilePath,
indexedPaths,
excludeFilters
src/test/helper/TestFileRegister.cpp
+1 -6
View File
@@ -3,7 +3,7 @@
#include "utility/file/FilePathFilter.h"
TestFileRegister::TestFileRegister()
: FileRegister(FileRegisterStateData(), FilePath(), std::set<FilePath>(), { FilePathFilter(L"") })
: FileRegister(FilePath(), std::set<FilePath>(), { FilePathFilter(L"") })
{
}
@@ -11,11 +11,6 @@ TestFileRegister::~TestFileRegister()
{
}
bool TestFileRegister::fileIsIndexed(const FilePath& filePath) const
{
return false;
}
bool TestFileRegister::hasFilePath(const FilePath& filePath) const
{
return true;
src/test/helper/TestFileRegister.h
-2
View File
@@ -9,8 +9,6 @@ class TestFileRegister
public:
TestFileRegister();
virtual ~TestFileRegister();
virtual bool fileIsIndexed(const FilePath& filePath) const;
virtual bool hasFilePath(const FilePath& filePath) const;
};