data: improved ParserClientImpl recording performance

* Removed storage checks and single use methods
* cache file ids in ParserClientImpl to avoid lookup for every source location
* NameHierarchy::serialize with range to avoid copies when adding nodes
* removed duplicate code in IndexerCxx and IndexerJava, moved to Indexer
This commit is contained in:
Eberhard Graether
2018-09-23 17:52:05 +02:00
parent 7c039a2fb8
commit 34cdde604e
10 changed files with 149 additions and 259 deletions
+44 -3
View File
@@ -5,27 +5,49 @@
#include "IndexerBase.h"
#include "IndexerCommand.h"
#include "IndexerStateInfo.h"
#include "logging.h"
#include "ParserClientImpl.h"
template <typename T>
class Indexer
: public IndexerBase
{
public:
Indexer();
IndexerCommandType getSupportedIndexerCommandType() const override;
std::shared_ptr<IntermediateStorage> index(std::shared_ptr<IndexerCommand> indexerCommand) override;
void interrupt() override;
private:
virtual std::shared_ptr<IntermediateStorage> doIndex(std::shared_ptr<T> indexerCommand) = 0;
virtual void doIndex(
std::shared_ptr<T> indexerCommand,
std::shared_ptr<ParserClientImpl> parserClient,
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo) = 0;
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo;
};
template <typename T>
Indexer<T>::Indexer()
: m_indexerStateInfo(std::make_shared<IndexerStateInfo>())
{
m_indexerStateInfo->indexingInterrupted = false;
}
template <typename T>
IndexerCommandType Indexer<T>::getSupportedIndexerCommandType() const
{
return T::getStaticIndexerCommandType();
}
template <typename T>
void Indexer<T>::interrupt()
{
m_indexerStateInfo->indexingInterrupted = true;
}
template <typename T>
std::shared_ptr<IntermediateStorage> Indexer<T>::index(std::shared_ptr<IndexerCommand> indexerCommand)
{
@@ -35,10 +57,29 @@ std::shared_ptr<IntermediateStorage> Indexer<T>::index(std::shared_ptr<IndexerCo
LOG_ERROR("Trying to process " + indexerCommandTypeToString(indexerCommand->getIndexerCommandType()) +
" indexer command with indexer that supports \"" + indexerCommandTypeToString(getSupportedIndexerCommandType()) + "\".");
return std::shared_ptr<IntermediateStorage>();
return nullptr;
}
return doIndex(castCommand);
std::shared_ptr<IntermediateStorage> storage = std::make_shared<IntermediateStorage>();
std::shared_ptr<ParserClientImpl> parserClient = std::make_shared<ParserClientImpl>(storage.get());
doIndex(castCommand, parserClient, m_indexerStateInfo);
if (parserClient->hasFatalErrors())
{
storage->setAllFilesIncomplete();
}
else
{
storage->setFilesWithErrorsIncomplete();
}
if (m_indexerStateInfo->indexingInterrupted)
{
return nullptr;
}
return storage;
}
#endif // INDEXER_H
+1
View File
@@ -15,6 +15,7 @@ class IndexerBase
public:
IndexerBase();
virtual ~IndexerBase() = default;
virtual IndexerCommandType getSupportedIndexerCommandType() const = 0;
virtual std::shared_ptr<IntermediateStorage> index(std::shared_ptr<IndexerCommand> indexerCommand) = 0;
virtual void interrupt() = 0;
+13 -3
View File
@@ -14,11 +14,16 @@ namespace
}
std::wstring NameHierarchy::serialize(const NameHierarchy& nameHierarchy)
{
return serializeRange(nameHierarchy, 0, nameHierarchy.size());
}
std::wstring NameHierarchy::serializeRange(const NameHierarchy& nameHierarchy, size_t first, size_t last)
{
std::wstringstream ss;
ss << nameDelimiterTypeToString(nameHierarchy.getDelimiter());
ss << META_DELIMITER;
for (size_t i = 0; i < nameHierarchy.size(); i++)
for (size_t i = first; i < last && i < nameHierarchy.size(); i++)
{
if (i > 0)
{
@@ -102,10 +107,10 @@ NameHierarchy::NameHierarchy(const NameDelimiterType delimiter)
{
}
NameHierarchy::NameHierarchy(const std::wstring& name, const NameDelimiterType delimiter)
NameHierarchy::NameHierarchy(std::wstring name, const NameDelimiterType delimiter)
: m_delimiter(delimiter)
{
push(std::make_shared<NameElement>(name));
push(std::make_shared<NameElement>(std::move(name)));
}
NameHierarchy::NameHierarchy(const std::vector<std::wstring>& names, const NameDelimiterType delimiter)
@@ -138,6 +143,11 @@ void NameHierarchy::push(std::shared_ptr<NameElement> element)
m_elements.push_back(element);
}
void NameHierarchy::push(std::wstring name)
{
m_elements.push_back(std::make_shared<NameElement>(std::move(name)));
}
void NameHierarchy::pop()
{
m_elements.pop_back();
+4 -1
View File
@@ -12,11 +12,13 @@ class NameHierarchy
{
public:
static std::wstring serialize(const NameHierarchy& nameHierarchy);
static std::wstring serializeRange(const NameHierarchy& nameHierarchy, size_t first, size_t last);
static NameHierarchy deserialize(const std::wstring& serializedName);
NameHierarchy(const NameDelimiterType delimiter = NAME_DELIMITER_UNKNOWN);
NameHierarchy(const std::wstring& name, const NameDelimiterType delimiter);
NameHierarchy(const std::vector<std::wstring>& names, const NameDelimiterType delimiter);
NameHierarchy(std::wstring name, const NameDelimiterType delimiter);
NameHierarchy(const NameHierarchy& other);
NameHierarchy(NameHierarchy&& other);
~NameHierarchy();
@@ -25,6 +27,7 @@ public:
void setDelimiter(const NameDelimiterType delimiter);
void push(std::shared_ptr<NameElement> element);
void push(std::wstring name);
void pop();
std::shared_ptr<NameElement> back() const;
+54 -136
View File
@@ -2,32 +2,28 @@
#include "Edge.h"
#include "Node.h"
#include "LocationType.h"
#include "ParseLocation.h"
#include "logging.h"
ParserClientImpl::ParserClientImpl()
ParserClientImpl::ParserClientImpl(IntermediateStorage* const storage)
: m_storage(storage)
{
}
void ParserClientImpl::setStorage(std::shared_ptr<IntermediateStorage> storage)
{
m_storage = storage;
}
void ParserClientImpl::resetStorage()
{
m_storage.reset();
}
Id ParserClientImpl::recordSymbol(
const NameHierarchy& symbolName, SymbolKind symbolKind,
AccessKind access, DefinitionKind definitionKind
)
{
Id nodeId = addNodeHierarchy(symbolName, symbolKindToNodeType(symbolKind));
addSymbol(nodeId, definitionKind);
addAccess(nodeId, access);
if (definitionKind != DEFINITION_NONE)
{
m_storage->addSymbol(StorageSymbol(nodeId, definitionKindToInt(definitionKind)));
}
if (access != ACCESS_NONE)
{
m_storage->addComponentAccess(StorageComponentAccess(nodeId, accessKindToInt(access)));
}
return nodeId;
}
@@ -38,7 +34,7 @@ Id ParserClientImpl::recordSymbolWithLocation(
)
{
Id nodeId = recordSymbol(symbolName, symbolKind, access, definitionKind);
addSourceLocation(nodeId, location, locationTypeToInt(LOCATION_TOKEN));
addSourceLocation(nodeId, location, LOCATION_TOKEN);
return nodeId;
}
@@ -49,7 +45,7 @@ Id ParserClientImpl::recordSymbolWithLocationAndScope(
)
{
Id nodeId = recordSymbolWithLocation(symbolName, symbolKind, location, access, definitionKind);
addSourceLocation(nodeId, scopeLocation, locationTypeToInt(LOCATION_SCOPE));
addSourceLocation(nodeId, scopeLocation, LOCATION_SCOPE);
return nodeId;
}
@@ -59,7 +55,7 @@ Id ParserClientImpl::recordSymbolWithLocationAndScopeAndSignature(
AccessKind access, DefinitionKind definitionKind)
{
Id nodeId = recordSymbolWithLocationAndScope(symbolName, symbolKind, location, scopeLocation, access, definitionKind);
addSourceLocation(nodeId, signatureLocation, locationTypeToInt(LOCATION_SIGNATURE));
addSourceLocation(nodeId, signatureLocation, LOCATION_SIGNATURE);
return nodeId;
}
@@ -70,30 +66,36 @@ void ParserClientImpl::recordReference(
Id contextNodeId = addNodeHierarchy(contextName);
Id referencedNodeId = addNodeHierarchy(referencedName);
Id edgeId = addEdge(referenceKindToEdgeType(referenceKind), contextNodeId, referencedNodeId);
addSourceLocation(edgeId, location, locationTypeToInt(LOCATION_TOKEN));
addSourceLocation(edgeId, location, LOCATION_TOKEN);
}
void ParserClientImpl::recordQualifierLocation(const NameHierarchy& qualifierName, const ParseLocation& location)
{
Id nodeId = addNodeHierarchy(qualifierName, NodeType::NODE_SYMBOL);
addSourceLocation(nodeId, location, locationTypeToInt(LOCATION_QUALIFIER));
addSourceLocation(nodeId, location, LOCATION_QUALIFIER);
}
void ParserClientImpl::recordLocalSymbol(const std::wstring& name, const ParseLocation& location)
{
const Id localSymbolId = addLocalSymbol(name);
addSourceLocation(localSymbolId, location, locationTypeToInt(LOCATION_LOCAL_SYMBOL));
const Id localSymbolId = m_storage->addLocalSymbol(name);
addSourceLocation(localSymbolId, location, LOCATION_LOCAL_SYMBOL);
}
void ParserClientImpl::recordFile(const FilePath& filePath, bool indexed)
{
const Id nodeId = addNodeHierarchy(NameHierarchy(filePath.wstr(), NAME_DELIMITER_FILE), NodeType::NODE_FILE);
addFile(nodeId, filePath, indexed);
const Id fileId = addFileName(filePath);
m_storage->addFile(StorageFile(fileId, filePath.wstr(), indexed, true));
}
void ParserClientImpl::recordComment(const ParseLocation& location)
{
addCommentLocation(location);
m_storage->addCommentLocation(StorageCommentLocationData(
addFileName(location.filePath),
location.startLineNumber,
location.startColumnNumber,
location.endLineNumber,
location.endColumnNumber
));
}
void ParserClientImpl::doRecordError(
@@ -101,7 +103,15 @@ void ParserClientImpl::doRecordError(
{
if (location.isValid())
{
addError(message, fatal, indexed, location, translationUnit);
m_storage->addError(StorageErrorData(
message,
location.filePath.wstr(),
location.startLineNumber,
location.startColumnNumber,
translationUnit.wstr(),
fatal,
indexed
));
}
}
@@ -189,31 +199,15 @@ Edge::EdgeType ParserClientImpl::referenceKindToEdgeType(ReferenceKind reference
return Edge::EDGE_UNDEFINED;
}
void ParserClientImpl::addAccess(Id nodeId, AccessKind access)
{
if (access != ACCESS_NONE)
{
addComponentAccess(nodeId, accessKindToInt(access));
}
}
Id ParserClientImpl::addNodeHierarchy(const NameHierarchy& nameHierarchy, NodeType nodeType)
{
if (nameHierarchy.size() == 0)
{
return 0;
}
Id parentNodeId = 0;
NameHierarchy currentNameHierarchy(nameHierarchy.getDelimiter());
for (size_t i = 0; i < nameHierarchy.size(); i++)
for (size_t i = 1; i <= nameHierarchy.size(); i++)
{
currentNameHierarchy.push(nameHierarchy[i]);
const bool currentIsLastElement = (i == nameHierarchy.size() - 1);
const NodeType currentType = (currentIsLastElement ? nodeType : NodeType::NODE_SYMBOL); // TODO: rename to unknown!
const NodeType currentType = (i == nameHierarchy.size() ? nodeType : NodeType::NODE_SYMBOL); // TODO: rename to unknown!
Id nodeId = addNode(currentType, currentNameHierarchy);
Id nodeId = m_storage->addNode(StorageNodeData(
NodeType::typeToInt(currentType.getType()), NameHierarchy::serializeRange(nameHierarchy, 0, i)));
if (parentNodeId != 0)
{
@@ -225,69 +219,32 @@ Id ParserClientImpl::addNodeHierarchy(const NameHierarchy& nameHierarchy, NodeTy
return parentNodeId;
}
Id ParserClientImpl::addNode(NodeType nodeType, const NameHierarchy& nameHierarchy)
Id ParserClientImpl::addFileName(const FilePath& filePath)
{
if (!m_storage)
const std::wstring file = filePath.wstr();
auto it = m_fileIdMap.find(file);
if (it != m_fileIdMap.end())
{
return 0;
return it->second;
}
return m_storage->addNode(StorageNodeData(NodeType::typeToInt(nodeType.getType()), NameHierarchy::serialize(nameHierarchy)));
}
void ParserClientImpl::addFile(Id id, const FilePath& filePath, bool indexed)
{
if (m_storage)
{
m_storage->addFile(StorageFile(id, filePath.wstr(), indexed, true));
}
}
void ParserClientImpl::addSymbol(Id id, DefinitionKind definitionKind)
{
if (!m_storage)
{
return;
}
if (definitionKind != DEFINITION_NONE)
{
m_storage->addSymbol(StorageSymbol(id, definitionKindToInt(definitionKind)));
}
const Id fileId = addNodeHierarchy(NameHierarchy(file, NAME_DELIMITER_FILE), NodeType::NODE_FILE);
m_fileIdMap.emplace(file, fileId);
return fileId;
}
Id ParserClientImpl::addEdge(int type, Id sourceId, Id targetId)
{
if (!m_storage)
if (sourceId && targetId)
{
return 0;
return m_storage->addEdge(StorageEdgeData(type, sourceId, targetId));
}
if (!sourceId || !targetId)
{
return 0;
}
return m_storage->addEdge(StorageEdgeData(type, sourceId, targetId));
return 0;
}
Id ParserClientImpl::addLocalSymbol(const std::wstring& name)
void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& location, LocationType type)
{
if (!m_storage)
{
return 0;
}
return m_storage->addLocalSymbol(name);
}
void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& location, int type)
{
if (!m_storage)
{
return;
}
if (!location.isValid())
{
return;
@@ -300,12 +257,12 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca
}
Id sourceLocationId = m_storage->addSourceLocation(StorageSourceLocationData(
addNodeHierarchy(NameHierarchy(location.filePath.wstr(), NAME_DELIMITER_FILE), NodeType::NODE_FILE),
addFileName(location.filePath),
location.startLineNumber,
location.startColumnNumber,
location.endLineNumber,
location.endColumnNumber,
type
locationTypeToInt(type)
));
m_storage->addOccurrence(StorageOccurrence(
@@ -313,42 +270,3 @@ void ParserClientImpl::addSourceLocation(Id elementId, const ParseLocation& loca
sourceLocationId
));
}
void ParserClientImpl::addComponentAccess(Id nodeId , int type)
{
if (!m_storage)
{
return;
}
m_storage->addComponentAccess(StorageComponentAccess(nodeId, type));
}
void ParserClientImpl::addCommentLocation(const ParseLocation& location)
{
if (!m_storage)
{
return;
}
m_storage->addCommentLocation(StorageCommentLocationData(
addNodeHierarchy(NameHierarchy(location.filePath.wstr(), NAME_DELIMITER_FILE), NodeType::NODE_FILE),
location.startLineNumber,
location.startColumnNumber,
location.endLineNumber,
location.endColumnNumber
));
}
void ParserClientImpl::addError(
const std::wstring& message, bool fatal, bool indexed, const ParseLocation& location, const FilePath& translationUnit)
{
if (!m_storage)
{
return;
}
m_storage->addError(StorageErrorData(
message, location.filePath.wstr(), location.startLineNumber, location.startColumnNumber, translationUnit.wstr(), fatal, indexed
));
}
+8 -13
View File
@@ -6,16 +6,14 @@
#include "DefinitionKind.h"
#include "Node.h"
#include "IntermediateStorage.h"
#include "LocationType.h"
#include "ParserClient.h"
class ParserClientImpl
: public ParserClient
{
public:
ParserClientImpl();
void setStorage(std::shared_ptr<IntermediateStorage> storage);
void resetStorage();
ParserClientImpl(IntermediateStorage* const storage);
Id recordSymbol(
const NameHierarchy& symbolName, SymbolKind symbolKind,
@@ -54,20 +52,17 @@ private:
NodeType symbolKindToNodeType(SymbolKind symbolType) const;
Edge::EdgeType referenceKindToEdgeType(ReferenceKind referenceKind) const;
void addAccess(Id nodeId, AccessKind access);
Id addNodeHierarchy(const NameHierarchy& nameHierarchy, NodeType nodeType = NodeType::NODE_SYMBOL);
Id addNode(NodeType nodeType, const NameHierarchy& nameHierarchy);
void addFile(Id id, const FilePath& filePath, bool indexed);
void addSymbol(Id id, DefinitionKind definitionKind);
Id addNodeHierarchy(const NameHierarchy& nameHierarchy, NodeType nodeType = NodeType::NODE_SYMBOL);
Id addFileName(const FilePath& filePath);
Id addEdge(int type, Id sourceId, Id targetId);
Id addLocalSymbol(const std::wstring& name);
void addSourceLocation(Id elementId, const ParseLocation& location, int type);
void addComponentAccess(Id nodeId , int type);
void addCommentLocation(const ParseLocation& location);
void addSourceLocation(Id elementId, const ParseLocation& location, LocationType type);
void addError(const std::wstring& message, bool fatal, bool indexed,
const ParseLocation& location, const FilePath& sourceFilePath);
std::shared_ptr<IntermediateStorage> m_storage;
IntermediateStorage* const m_storage;
std::map<std::wstring, Id> m_fileIdMap;
};
#endif // PARSER_CLIENT_IMPL_H
+7 -40
View File
@@ -2,25 +2,13 @@
#include "CxxParser.h"
#include "FileRegister.h"
#include "IndexerStateInfo.h"
#include "ParserClientImpl.h"
IndexerCxx::IndexerCxx()
: m_indexerStateInfo(std::make_shared<IndexerStateInfo>())
{
m_indexerStateInfo->indexingInterrupted = false;
}
void IndexerCxx::interrupt()
{
m_indexerStateInfo->indexingInterrupted = true;
}
std::shared_ptr<IntermediateStorage> IndexerCxx::doIndex(std::shared_ptr<IndexerCommandCxx> indexerCommand)
{
std::shared_ptr<ParserClientImpl> parserClient = std::make_shared<ParserClientImpl>();
std::shared_ptr<CxxParser> parser = std::make_shared<CxxParser>(
void IndexerCxx::doIndex(
std::shared_ptr<IndexerCommandCxx> indexerCommand,
std::shared_ptr<ParserClientImpl> parserClient,
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo
){
CxxParser parser(
parserClient,
std::make_shared<FileRegister>(
indexerCommand->getSourceFilePath(), indexerCommand->getIndexedPaths(), indexerCommand->getExcludeFilters()
@@ -28,26 +16,5 @@ std::shared_ptr<IntermediateStorage> IndexerCxx::doIndex(std::shared_ptr<Indexer
m_indexerStateInfo
);
std::shared_ptr<IntermediateStorage> storage = std::make_shared<IntermediateStorage>();
parserClient->setStorage(storage);
parser->buildIndex(indexerCommand);
parserClient->resetStorage();
if (parserClient->hasFatalErrors())
{
storage->setAllFilesIncomplete();
}
else
{
storage->setFilesWithErrorsIncomplete();
}
if (m_indexerStateInfo->indexingInterrupted)
{
return std::shared_ptr<IntermediateStorage>();
}
return storage;
parser.buildIndex(indexerCommand);
}
+6 -13
View File
@@ -1,24 +1,17 @@
#ifndef INDEXER_CXX_H
#define INDEXER_CXX_H
#include <memory>
#include <mutex>
#include "Indexer.h"
#include "IndexerCommandCxx.h"
struct IndexerStateInfo;
class IndexerCxx: public Indexer<IndexerCommandCxx>
class IndexerCxx
: public Indexer<IndexerCommandCxx>
{
public:
IndexerCxx();
void interrupt() override;
private:
std::shared_ptr<IntermediateStorage> doIndex(std::shared_ptr<IndexerCommandCxx> indexerCommand) override;
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo;
void doIndex(
std::shared_ptr<IndexerCommandCxx> indexerCommand,
std::shared_ptr<ParserClientImpl> parserClient,
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo) override;
};
#endif // INDEXER_CXX_H
+6 -42
View File
@@ -1,52 +1,16 @@
#include "IndexerJava.h"
#include "IndexerCommandJava.h"
#include "IndexerStateInfo.h"
#include "JavaParser.h"
#include "ParserClientImpl.h"
IndexerJava::IndexerJava()
: m_indexerStateInfo(std::make_shared<IndexerStateInfo>())
{
m_indexerStateInfo->indexingInterrupted = false;
}
IndexerJava::~IndexerJava()
{
JavaParser::clearCaches();
}
void IndexerJava::interrupt()
{
m_indexerStateInfo->indexingInterrupted = true;
}
std::shared_ptr<IntermediateStorage> IndexerJava::doIndex(std::shared_ptr<IndexerCommandJava> indexerCommand)
{
std::shared_ptr<ParserClientImpl> parserClient = std::make_shared<ParserClientImpl>();
std::shared_ptr<JavaParser> parser = std::make_shared<JavaParser>(parserClient, m_indexerStateInfo);
std::shared_ptr<IntermediateStorage> storage = std::make_shared<IntermediateStorage>();
parserClient->setStorage(storage);
parser->buildIndex(indexerCommand);
parserClient->resetStorage();
if (parserClient->hasFatalErrors())
{
storage->setAllFilesIncomplete();
}
else
{
storage->setFilesWithErrorsIncomplete();
}
if (m_indexerStateInfo->indexingInterrupted)
{
return std::shared_ptr<IntermediateStorage>();
}
return storage;
void IndexerJava::doIndex(
std::shared_ptr<IndexerCommandJava> indexerCommand,
std::shared_ptr<ParserClientImpl> parserClient,
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo
){
JavaParser(parserClient, m_indexerStateInfo).buildIndex(indexerCommand);
}
+6 -8
View File
@@ -1,24 +1,22 @@
#ifndef INDEXER_JAVA_H
#define INDEXER_JAVA_H
#include <memory>
#include "Indexer.h"
#include "IndexerCommandJava.h"
struct IndexerStateInfo;
class IndexerJava: public Indexer<IndexerCommandJava>
class IndexerJava
: public Indexer<IndexerCommandJava>
{
public:
IndexerJava();
virtual ~IndexerJava();
void interrupt() override;
private:
std::shared_ptr<IntermediateStorage> doIndex(std::shared_ptr<IndexerCommandJava> indexerCommand) override;
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo;
void doIndex(
std::shared_ptr<IndexerCommandJava> indexerCommand,
std::shared_ptr<ParserClientImpl> parserClient,
std::shared_ptr<IndexerStateInfo> m_indexerStateInfo) override;
};
#endif // INDEXER_JAVA_H