ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: removed C/C++ "unparsed" check from indexer

Browse Source 
Cxx AstVisitor re-visits previously or simultaneously indexed files because the "visible" content may have changed due to other preprocessor defines etc.
This commit is contained in:
mlangkabel
2018-04-24 13:25:06 +02:00
parent c5d0362df5
commit ca3e2f48ee
18 changed files with 13 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/CMakeLists.txt
-2
View File
@@ -358,8 +358,6 @@ add_files(
utility/file/FilePathFilter.h
utility/file/FileRegister.cpp
utility/file/FileRegister.h
utility/file/FileRegisterStateData.cpp
utility/file/FileRegisterStateData.h
utility/file/FileSystem.cpp
utility/file/FileSystem.h
utility/file/FileTree.cpp
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+1 -8
View File
@@ -1,7 +1,6 @@
#include "InterprocessIndexer.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/logging/logging.h"
#include "data/indexer/IndexerCommand.h"
@@ -50,19 +49,13 @@ void InterprocessIndexer::work()
LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles());
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
);
LOG_INFO_STREAM(<< m_processId << " starting to index current file");
std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status");
m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory");
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
-81
View File
@@ -8,7 +8,6 @@ const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_
const char* InterprocessIndexingStatusManager::s_indexingFilesKeyName = "indexing_files";
const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files";
const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files";
const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files";
const char* InterprocessIndexingStatusManager::s_finishedProcessIdsKeyName = "finished_process_ids";
InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner)
@@ -163,83 +162,3 @@ std::vector<FilePath> InterprocessIndexingStatusManager::getCrashedSourceFilePat
return crashedFiles;
}
std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
{
std::set<FilePath> result;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* files =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!files)
{
return result;
}
for (auto& file : *files)
{
result.insert(FilePath(utility::decodeFromUtf8(file.c_str())));
}
return result;
}
void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
{
const unsigned int overestimationMultiplier = 3;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* indexedFiles =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
std::set<std::string> oldFiles;
for (auto& indexedFile : *indexedFiles)
{
oldFiles.insert(indexedFile.c_str());
}
std::set<std::string> newFiles;
for (const FilePath& filePath : filePaths)
{
if (oldFiles.find(utility::encodeToUtf8(filePath.wstr())) == oldFiles.end())
{
newFiles.insert(utility::encodeToUtf8(filePath.wstr()));
}
}
size_t estimatedSize = 262144;
for (auto& newFile : newFiles)
{
estimatedSize += sizeof(SharedMemory::String) + newFile.size();
}
estimatedSize *= overestimationMultiplier;
while (access.getFreeMemorySize() < estimatedSize)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
access.growMemory(access.getMemorySize());
LOG_INFO("growing memory succeeded");
indexedFiles = access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
}
for (const std::string& newFile: newFiles)
{
indexedFiles->push_back(SharedMemory::String(newFile.c_str(), access.getAllocator()));
}
LOG_INFO(access.logString());
}
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h
-4
View File
@@ -21,16 +21,12 @@ public:
std::vector<FilePath> getCurrentlyIndexedSourceFilePaths();
std::vector<FilePath> getCrashedSourceFilePaths();
std::set<FilePath> getIndexedFiles();
void addIndexedFiles(std::set<FilePath> filePaths);
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_indexingFilesKeyName;
static const char* s_currentFilesKeyName;
static const char* s_crashedFilesKeyName;
static const char* s_indexedFilesKeyName;
static const char* s_finishedProcessIdsKeyName;
};
src/lib/utility/file/FileRegister.cpp
+1 -24
View File
@@ -4,13 +4,11 @@
#include "utility/file/FilePathFilter.h"
FileRegister::FileRegister(
const FileRegisterStateData& stateData,
const FilePath& currentPath,
const std::set<FilePath>& indexedPaths,
const std::set<FilePathFilter>& excludeFilters
)
: m_stateData(stateData)
, m_currentPath(currentPath)
: m_currentPath(currentPath)
, m_indexedPaths(indexedPaths)
, m_excludeFilters(excludeFilters)
, m_hasFilePathCache(
@@ -64,31 +62,10 @@ FileRegister::FileRegister(
{
}
FileRegister::~FileRegister()
{
}
const FileRegisterStateData& FileRegister::getStateData() const
{
return m_stateData;
}
void FileRegister::markFileIndexing(const FilePath& filePath)
{
m_stateData.markFileIndexing(filePath);
}
void FileRegister::markIndexingFilesIndexed()
{
m_stateData.markIndexingFilesIndexed();
}
bool FileRegister::fileIsIndexed(const FilePath& filePath) const
{
return m_stateData.fileIsIndexed(filePath);
}
bool FileRegister::hasFilePath(const FilePath& filePath) const
{
return m_hasFilePathCache.getValue(filePath.wstr());
src/lib/utility/file/FileRegister.h
+1 -8
View File
@@ -3,7 +3,7 @@
#include <set>
#include "utility/file/FileRegisterStateData.h"
#include "utility/file/FilePath.h"
#include "utility/UnorderedCache.h"
class FilePathFilter;
@@ -12,22 +12,15 @@ class FileRegister
{
public:
FileRegister(
const FileRegisterStateData& stateData,
const FilePath& currentPath,
const std::set<FilePath>& indexedPaths,
const std::set<FilePathFilter>& excludeFilters
);
virtual ~FileRegister();
const FileRegisterStateData& getStateData() const;
void markFileIndexing(const FilePath& filePath);
void markIndexingFilesIndexed();
virtual bool fileIsIndexed(const FilePath& filePath) const;
virtual bool hasFilePath(const FilePath& filePath) const;
private:
FileRegisterStateData m_stateData;
const FilePath& m_currentPath;
const std::set<FilePath> m_indexedPaths;
const std::set<FilePathFilter> m_excludeFilters;
src/lib/utility/file/FileRegisterStateData.cpp
-71
View File
@@ -1,71 +0,0 @@
#include "utility/file/FileRegisterStateData.h"
#include "utility/file/FilePath.h"
FileRegisterStateData::FileRegisterStateData()
{
}
FileRegisterStateData::FileRegisterStateData(const FileRegisterStateData& o)
{
this->inject(o);
}
void FileRegisterStateData::inject(const FileRegisterStateData& o)
{
for (const auto& it: o.m_filePaths)
{
if (it.second == STATE_INDEXED)
{
m_filePaths[it.first] = STATE_INDEXED;
}
}
}
void FileRegisterStateData::markFileIndexing(const FilePath& filePath)
{
m_filePaths[filePath] = STATE_INDEXING;
}
void FileRegisterStateData::markIndexingFilesIndexed()
{
for (auto& it: m_filePaths)
{
if (it.second == STATE_INDEXING)
{
it.second = STATE_INDEXED;
}
}
}
bool FileRegisterStateData::fileIsIndexed(const FilePath& filePath) const
{
auto it = m_filePaths.find(filePath);
if (it != m_filePaths.end())
{
return it->second == STATE_INDEXED;
}
return false;
}
void FileRegisterStateData::setIndexedFiles(const std::set<FilePath>& filePaths)
{
for (auto& path : filePaths)
{
m_filePaths[path] = STATE_INDEXED;
}
}
std::set<FilePath> FileRegisterStateData::getIndexedFiles() const
{
std::set<FilePath> paths;
for (auto& it : m_filePaths)
{
if (it.second == STATE_INDEXED)
{
paths.insert(it.first);
}
}
return paths;
}
src/lib/utility/file/FileRegisterStateData.h
-36
View File
@@ -1,36 +0,0 @@
#ifndef FILE_REGISTER_STATE_DATA_H
#define FILE_REGISTER_STATE_DATA_H
#include <map>
#include <mutex>
#include <set>
#include "utility/file/FilePath.h"
class FileRegisterStateData
{
public:
FileRegisterStateData();
FileRegisterStateData(const FileRegisterStateData& o);
void inject(const FileRegisterStateData& o);
void markFileIndexing(const FilePath& filePath);
void markIndexingFilesIndexed();
bool fileIsIndexed(const FilePath& filePath) const;
void setIndexedFiles(const std::set<FilePath>& filePaths);
std::set<FilePath> getIndexedFiles() const;
private:
enum IndexingState
{
STATE_NON_INDEXED,
STATE_INDEXING,
STATE_INDEXED
};
std::map<FilePath, IndexingState> m_filePaths;
};
#endif // FILE_REGISTER_STATE_DATA_H