ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: Request file modification time not until storing to index for faster indexing performance

Browse Source
This commit is contained in:
Eberhard Graether
2018-08-27 17:05:38 +02:00
parent e8a4830229
commit e3e8fa1547
10 changed files with 36 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h
+3 -5
View File
@@ -67,30 +67,28 @@ inline StorageNode fromShared(const SharedStorageNode& node)
struct SharedStorageFile
{
SharedStorageFile(
Id id, const std::string& filePath, const std::string& modificationTime, bool indexed, bool complete, SharedMemory::Allocator* allocator
Id id, const std::string& filePath, bool indexed, bool complete, SharedMemory::Allocator* allocator
)
: id(id)
, filePath(filePath.c_str(), allocator)
, modificationTime(modificationTime.c_str(), allocator)
, indexed(indexed)
, complete(complete)
{}
Id id;
SharedMemory::String filePath;
SharedMemory::String modificationTime;
bool indexed;
bool complete;
};
inline SharedStorageFile toShared(const StorageFile& file, SharedMemory::Allocator* allocator)
{
return SharedStorageFile(file.id, utility::encodeToUtf8(file.filePath), file.modificationTime, file.indexed, file.complete, allocator);
return SharedStorageFile(file.id, utility::encodeToUtf8(file.filePath), file.indexed, file.complete, allocator);
}
inline StorageFile fromShared(const SharedStorageFile& file)
{
return StorageFile(file.id, utility::decodeFromUtf8(file.filePath.c_str()), file.modificationTime.c_str(), file.indexed, file.complete);
return StorageFile(file.id, utility::decodeFromUtf8(file.filePath.c_str()), file.indexed, file.complete);
}
src/lib/data/parser/ParserClient.h
+1 -1
View File
@@ -64,7 +64,7 @@ public:
const ParseLocation& errorLocation, const std::wstring& message, bool fatal, bool indexed, const FilePath& translationUnit);
virtual void recordLocalSymbol(const std::wstring& name, const ParseLocation& location) = 0;
virtual void recordFile(const FileInfo& fileInfo, bool indexed) = 0;
virtual void recordFile(const FilePath& filePath, bool indexed) = 0;
virtual void recordComment(const ParseLocation& location) = 0;
bool hasFatalErrors() const;
src/lib/data/parser/ParserClientImpl.cpp
+5 -5
View File
@@ -85,10 +85,10 @@ void ParserClientImpl::recordLocalSymbol(const std::wstring& name, const ParseLo
addSourceLocation(localSymbolId, location, locationTypeToInt(LOCATION_LOCAL_SYMBOL));
}
void ParserClientImpl::recordFile(const FileInfo& fileInfo, bool indexed)
void ParserClientImpl::recordFile(const FilePath& filePath, bool indexed)
{
const Id nodeId = addNodeHierarchy(NameHierarchy(fileInfo.path.wstr(), NAME_DELIMITER_FILE), NodeType::NODE_FILE);
addFile(nodeId, fileInfo.path, fileInfo.lastWriteTime.toString(), indexed);
const Id nodeId = addNodeHierarchy(NameHierarchy(filePath.wstr(), NAME_DELIMITER_FILE), NodeType::NODE_FILE);
addFile(nodeId, filePath, indexed);
}
void ParserClientImpl::recordComment(const ParseLocation& location)
@@ -235,11 +235,11 @@ Id ParserClientImpl::addNode(NodeType nodeType, const NameHierarchy& nameHierarc
return m_storage->addNode(StorageNodeData(NodeType::typeToInt(nodeType.getType()), NameHierarchy::serialize(nameHierarchy)));
}
void ParserClientImpl::addFile(Id id, const FilePath& filePath, const std::string& modificationTime, bool indexed)
void ParserClientImpl::addFile(Id id, const FilePath& filePath, bool indexed)
{
if (m_storage)
{
m_storage->addFile(StorageFile(id, filePath.wstr(), modificationTime, indexed, true));
m_storage->addFile(StorageFile(id, filePath.wstr(), indexed, true));
}
}
src/lib/data/parser/ParserClientImpl.h
+2 -2
View File
@@ -44,7 +44,7 @@ public:
const NameHierarchy& qualifierName, const ParseLocation& location) override;
void recordLocalSymbol(const std::wstring& name, const ParseLocation& location) override;
void recordFile(const FileInfo& fileInfo, bool indexed) override;
void recordFile(const FilePath& filePath, bool indexed) override;
void recordComment(const ParseLocation& location) override;
private:
@@ -57,7 +57,7 @@ private:
Id addNodeHierarchy(const NameHierarchy& nameHierarchy, NodeType nodeType = NodeType::NODE_SYMBOL);
Id addNode(NodeType nodeType, const NameHierarchy& nameHierarchy);
void addFile(Id id, const FilePath& filePath, const std::string& modificationTime, bool indexed);
void addFile(Id id, const FilePath& filePath, bool indexed);
void addSymbol(Id id, DefinitionKind definitionKind);
Id addEdge(int type, Id sourceId, Id targetId);
Id addLocalSymbol(const std::wstring& name);
src/lib/data/storage/sqlite/SqliteIndexStorage.cpp
+11 -2
View File
@@ -2,6 +2,7 @@
#include <unordered_map>
#include "utility/file/FileSystem.h"
#include "utility/logging/logging.h"
#include "utility/text/TextAccess.h"
#include "utility/utilityString.h"
@@ -109,11 +110,19 @@ void SqliteIndexStorage::addFile(const StorageFile& data)
return;
}
FilePath filePath(data.filePath);
std::string modificationTime(data.modificationTime);
if (modificationTime.empty())
{
modificationTime = FileSystem::getFileInfoForPath(filePath).lastWriteTime.toString();
}
std::shared_ptr<TextAccess> content;
int lineCount = 0;
if (data.indexed)
{
content = TextAccess::createFromFile(FilePath(data.filePath));
content = TextAccess::createFromFile(filePath);
lineCount = content->getLineCount();
}
@@ -121,7 +130,7 @@ void SqliteIndexStorage::addFile(const StorageFile& data)
{
m_insertFileStmt.bind(1, int(data.id));
m_insertFileStmt.bind(2, utility::encodeToUtf8(data.filePath).c_str());
m_insertFileStmt.bind(3, data.modificationTime.c_str());
m_insertFileStmt.bind(3, modificationTime.c_str());
m_insertFileStmt.bind(4, data.indexed);
m_insertFileStmt.bind(5, data.complete);
m_insertFileStmt.bind(6, lineCount);
src/lib/data/storage/type/StorageFile.h
+8
View File
@@ -15,6 +15,14 @@ struct StorageFile
, complete(true)
{}
StorageFile(Id id, std::wstring filePath, bool indexed, bool complete)
: id(id)
, filePath(std::move(filePath))
, modificationTime("")
, indexed(indexed)
, complete(complete)
{}
StorageFile(Id id, std::wstring filePath, std::string modificationTime, bool indexed, bool complete)
: id(id)
, filePath(std::move(filePath))
src/lib_cxx/data/parser/cxx/PreprocessorCallbacks.cpp
+1 -2
View File
@@ -9,7 +9,6 @@
#include "data/parser/ParserClient.h"
#include "data/parser/ParseLocation.h"
#include "utility/file/FileSystem.h"
#include "utility/utilityString.h"
PreprocessorCallbacks::PreprocessorCallbacks(
@@ -36,7 +35,7 @@ void PreprocessorCallbacks::FileChanged(
if (m_fileWasRecorded.find(fileId) == m_fileWasRecorded.end())
{
m_client->recordFile(FileSystem::getFileInfoForPath(m_currentPath), m_currentPathIsProjectFile); // todo: fix for tests
m_client->recordFile(m_currentPath, m_currentPathIsProjectFile); // todo: fix for tests
m_fileWasRecorded.insert(fileId);
}
}
src/lib_java/data/parser/java/JavaParser.cpp
+1 -2
View File
@@ -8,7 +8,6 @@
#include "data/parser/ReferenceKind.h"
#include "data/parser/ParserClient.h"
#include "settings/ApplicationSettings.h"
#include "utility/file/FileSystem.h"
#include "utility/text/TextAccess.h"
#include "utility/ResourcePaths.h"
#include "utility/utilityJava.h"
@@ -101,7 +100,7 @@ void JavaParser::buildIndex(
{
m_currentFilePath = sourceFilePath;
m_client->recordFile(FileSystem::getFileInfoForPath(sourceFilePath), true);
m_client->recordFile(sourceFilePath, true);
// remove tabs because they screw with javaparser's location resolver
std::string fileContent = utility::replace(textAccess->getText(), "\t", " ");
src/test/helper/DumpParserClient.h
+2 -2
View File
@@ -92,9 +92,9 @@ public:
recordLine(L"LOCAL_SYMBOL: " + addLocationSuffix(name + L" [" + location.filePath.fileName(), location) + L"]\n");
}
void recordFile(const FileInfo& fileInfo, bool indexed) override
void recordFile(const FilePath& filePath, bool indexed) override
{
recordLine(L"FILE: " + fileInfo.path.fileName() + (indexed ? L"" : L" non-indexed") + L"\n");
recordLine(L"FILE: " + filePath.fileName() + (indexed ? L"" : L" non-indexed") + L"\n");
}
void recordComment(const ParseLocation& location) override
src/test/helper/TestParserClient.h
+2 -2
View File
@@ -129,9 +129,9 @@ public:
localSymbols.push_back(addLocationSuffix(name, location));
}
void recordFile(const FileInfo& fileInfo, bool indexed) override
void recordFile(const FilePath& filePath, bool indexed) override
{
files.insert(fileInfo.path.wstr());
files.insert(filePath.wstr());
}
void recordComment(const ParseLocation& location) override