ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: removed C/C++ "unparsed" check from indexer

Browse Source 
Cxx AstVisitor re-visits previously or simultaneously indexed files because the "visible" content may have changed due to other preprocessor defines etc.
This commit is contained in:
mlangkabel
2018-04-24 13:25:06 +02:00
parent c5d0362df5
commit ca3e2f48ee
18 changed files with 13 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/interprocess/InterprocessIndexer.cpp
+1 -8
View File
@@ -1,7 +1,6 @@
#include "InterprocessIndexer.h"
#include "utility/file/FileRegister.h"
#include "utility/file/FileRegisterStateData.h"
#include "utility/logging/logging.h"
#include "data/indexer/IndexerCommand.h"
@@ -50,19 +49,13 @@ void InterprocessIndexer::work()
LOG_INFO_STREAM(<< m_processId << " updating indexer status with currently indexed filepath");
m_interprocessIndexingStatusManager.startIndexingSourceFile(indexerCommand->getSourceFilePath());
FileRegisterStateData data;
data.setIndexedFiles(m_interprocessIndexingStatusManager.getIndexedFiles());
std::shared_ptr<FileRegister> fileRegister = std::make_shared<FileRegister>(
data, indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
);
LOG_INFO_STREAM(<< m_processId << " starting to index current file");
std::shared_ptr<IntermediateStorage> result = indexer->index(indexerCommand, fileRegister);
LOG_INFO_STREAM(<< m_processId << " finished indexing current file, updating indexer status");
m_interprocessIndexingStatusManager.addIndexedFiles(fileRegister->getStateData().getIndexedFiles());
LOG_INFO_STREAM(<< m_processId << " pushing index to shared memory");
m_interprocessIntermediateStorageManager.pushIntermediateStorage(result);
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.cpp
-81
View File
@@ -8,7 +8,6 @@ const char* InterprocessIndexingStatusManager::s_sharedMemoryNamePrefix = "ists_
const char* InterprocessIndexingStatusManager::s_indexingFilesKeyName = "indexing_files";
const char* InterprocessIndexingStatusManager::s_currentFilesKeyName = "current_files";
const char* InterprocessIndexingStatusManager::s_crashedFilesKeyName = "crashed_files";
const char* InterprocessIndexingStatusManager::s_indexedFilesKeyName = "indexed_files";
const char* InterprocessIndexingStatusManager::s_finishedProcessIdsKeyName = "finished_process_ids";
InterprocessIndexingStatusManager::InterprocessIndexingStatusManager(const std::string& instanceUuid, Id processId, bool isOwner)
@@ -163,83 +162,3 @@ std::vector<FilePath> InterprocessIndexingStatusManager::getCrashedSourceFilePat
return crashedFiles;
}
std::set<FilePath> InterprocessIndexingStatusManager::getIndexedFiles()
{
std::set<FilePath> result;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* files =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!files)
{
return result;
}
for (auto& file : *files)
{
result.insert(FilePath(utility::decodeFromUtf8(file.c_str())));
}
return result;
}
void InterprocessIndexingStatusManager::addIndexedFiles(std::set<FilePath> filePaths)
{
const unsigned int overestimationMultiplier = 3;
SharedMemory::ScopedAccess access(&m_sharedMemory);
SharedMemory::Vector<SharedMemory::String>* indexedFiles =
access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
std::set<std::string> oldFiles;
for (auto& indexedFile : *indexedFiles)
{
oldFiles.insert(indexedFile.c_str());
}
std::set<std::string> newFiles;
for (const FilePath& filePath : filePaths)
{
if (oldFiles.find(utility::encodeToUtf8(filePath.wstr())) == oldFiles.end())
{
newFiles.insert(utility::encodeToUtf8(filePath.wstr()));
}
}
size_t estimatedSize = 262144;
for (auto& newFile : newFiles)
{
estimatedSize += sizeof(SharedMemory::String) + newFile.size();
}
estimatedSize *= overestimationMultiplier;
while (access.getFreeMemorySize() < estimatedSize)
{
LOG_INFO_STREAM(
<< "grow memory - est: " << estimatedSize << " size: " << access.getMemorySize()
<< " free: " << access.getFreeMemorySize() << " alloc: " << (access.getMemorySize()));
access.growMemory(access.getMemorySize());
LOG_INFO("growing memory succeeded");
indexedFiles = access.accessValueWithAllocator<SharedMemory::Vector<SharedMemory::String>>(s_indexedFilesKeyName);
if (!indexedFiles)
{
return;
}
}
for (const std::string& newFile: newFiles)
{
indexedFiles->push_back(SharedMemory::String(newFile.c_str(), access.getAllocator()));
}
LOG_INFO(access.logString());
}
src/lib/data/indexer/interprocess/InterprocessIndexingStatusManager.h
-4
View File
@@ -21,16 +21,12 @@ public:
std::vector<FilePath> getCurrentlyIndexedSourceFilePaths();
std::vector<FilePath> getCrashedSourceFilePaths();
std::set<FilePath> getIndexedFiles();
void addIndexedFiles(std::set<FilePath> filePaths);
private:
static const char* s_sharedMemoryNamePrefix;
static const char* s_indexingFilesKeyName;
static const char* s_currentFilesKeyName;
static const char* s_crashedFilesKeyName;
static const char* s_indexedFilesKeyName;
static const char* s_finishedProcessIdsKeyName;
};