data: parser performance optimization for large projects

* moved inserting everything into the database to the end of parsing a file. this reduces many duplicate queries.
* removed ParserClient interface from Storage.
* errors are updated after parsing a file because we only if there are any duplicate errors after merging the new errors into the database.
* renamed field in db
* added fatal column to error db
This commit is contained in:
malte_langkabel
2016-03-10 18:24:28 +01:00
parent e7c7178057
commit 9a7b818dba
17 changed files with 1254 additions and 1055 deletions
+4 -466
View File
@@ -175,30 +175,13 @@ void Storage::finishParsing()
buildHierarchyCache();
}
void Storage::startParsingFile(const FilePath& filePath)
void Storage::injectData(std::shared_ptr<IntermediateStorage> injectedStorage)
{
m_sqliteStorage.beginTransaction();
}
int totalErrorCount = getErrorCount().total;
void Storage::finishParsingFile(const FilePath& filePath)
{
m_sqliteStorage.commitTransaction();
}
injectedStorage->transferToStorage(m_sqliteStorage);
void Storage::onError(const ParseLocation& location, const std::string& message, bool fatal)
{
log(std::string(fatal ? "FATAL " : "") + "ERROR", message, location);
if (!location.isValid())
{
return;
}
size_t totalErrorCount = getErrorCount().total;
m_sqliteStorage.addError(message, fatal, location.filePath.str(), location.startLineNumber, location.startColumnNumber);
if ((int)totalErrorCount != getErrorCount().total)
if (totalErrorCount != getErrorCount().total)
{
MessageShowErrors msg(getErrorCount());
msg.setSendAsTask(false);
@@ -206,394 +189,6 @@ void Storage::onError(const ParseLocation& location, const std::string& message,
}
}
Id Storage::onTypedefParsed(
const ParseLocation& location, const NameHierarchy& typedefName, AccessType access
){
log("typedef", typedefName.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_TYPEDEF, typedefName, true);
addSourceLocation(nodeId, location);
addAccess(nodeId, access);
return nodeId;
}
Id Storage::onClassParsed(
const ParseLocation& location, const NameHierarchy& nameHierarchy, AccessType access,
const ParseLocation& scopeLocation
){
log("class", nameHierarchy.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_CLASS, nameHierarchy, scopeLocation.isValid());
addSourceLocation(nodeId, location);
addSourceLocation(nodeId, scopeLocation, true);
addAccess(nodeId, access);
return nodeId;
}
Id Storage::onStructParsed(
const ParseLocation& location, const NameHierarchy& nameHierarchy, AccessType access,
const ParseLocation& scopeLocation
){
log("struct", nameHierarchy.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_STRUCT, nameHierarchy, scopeLocation.isValid());
addSourceLocation(nodeId, location);
addSourceLocation(nodeId, scopeLocation, true);
addAccess(nodeId, access);
return nodeId;
}
Id Storage::onGlobalVariableParsed(const ParseLocation& location, const NameHierarchy& variable)
{
log("global", variable.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_GLOBAL_VARIABLE, variable, true);
addSourceLocation(nodeId, location);
return nodeId;
}
Id Storage::onFieldParsed(const ParseLocation& location, const NameHierarchy& field, AccessType access)
{
log("field", field.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_FIELD, field, true);
addSourceLocation(nodeId, location);
addAccess(nodeId, access);
return nodeId;
}
Id Storage::onFunctionParsed(
const ParseLocation& location, const NameHierarchy& function, const ParseLocation& scopeLocation
){
log("function", function.getQualifiedNameWithSignature(), location);
Id nodeId = addNodeHierarchy(Node::NODE_FUNCTION, function, true);
addSourceLocation(nodeId, location);
addSourceLocation(nodeId, scopeLocation, true);
return nodeId;
}
Id Storage::onMethodParsed(
const ParseLocation& location, const NameHierarchy& method, AccessType access, AbstractionType abstraction,
const ParseLocation& scopeLocation
){
log("method", method.getQualifiedNameWithSignature(), location);
Id nodeId = addNodeHierarchy(Node::NODE_METHOD, method, location.isValid() && scopeLocation.isValid());
addSourceLocation(nodeId, location);
addSourceLocation(nodeId, scopeLocation, true);
addAccess(nodeId, access);
return nodeId;
}
Id Storage::onNamespaceParsed(
const ParseLocation& location, const NameHierarchy& nameHierarchy, const ParseLocation& scopeLocation
){
log("namespace", nameHierarchy.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_NAMESPACE, nameHierarchy, true);
addSourceLocation(nodeId, location);
addSourceLocation(nodeId, scopeLocation, true);
return nodeId;
}
Id Storage::onEnumParsed(
const ParseLocation& location, const NameHierarchy& nameHierarchy, AccessType access,
const ParseLocation& scopeLocation
){
log("enum", nameHierarchy.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_ENUM, nameHierarchy, true);
addSourceLocation(nodeId, location);
addSourceLocation(nodeId, scopeLocation, true);
addAccess(nodeId, access);
return nodeId;
}
Id Storage::onEnumConstantParsed(const ParseLocation& location, const NameHierarchy& nameHierarchy)
{
log("enum constant", nameHierarchy.getQualifiedName(), location);
Id nodeId = addNodeHierarchy(Node::NODE_ENUM_CONSTANT, nameHierarchy, true);
addSourceLocation(nodeId, location);
return nodeId;
}
Id Storage::onInheritanceParsed(
const ParseLocation& location, const NameHierarchy& childNameHierarchy,
const NameHierarchy& parentNameHierarchy, AccessType access
){
log("inheritance", childNameHierarchy.getQualifiedName() + " : " + parentNameHierarchy.getQualifiedName(), location);
Id childNodeId = addNodeHierarchy(Node::NODE_TYPE, childNameHierarchy, false);
Id parentNodeId = addNodeHierarchy(Node::NODE_TYPE, parentNameHierarchy, false);
Id edgeId = addEdge(childNodeId, parentNodeId, Edge::EDGE_INHERITANCE, location);
return edgeId;
}
Id Storage::onMethodOverrideParsed(
const ParseLocation& location, const NameHierarchy& overridden, const NameHierarchy& overrider)
{
log("override", overridden.getQualifiedNameWithSignature() + " -> " + overrider.getQualifiedNameWithSignature(), location);
Id baseNodeId = addNodeHierarchy(Node::NODE_FUNCTION, overridden, false);
Id overriderNodeId = addNodeHierarchy(Node::NODE_FUNCTION, overrider, false);
Id edgeId = addEdge(overriderNodeId, baseNodeId, Edge::EDGE_OVERRIDE, location);
return edgeId;
}
Id Storage::onCallParsed(const ParseLocation& location, const NameHierarchy& caller, const NameHierarchy& callee)
{
log("call", caller.getQualifiedNameWithSignature() + " -> " + callee.getQualifiedNameWithSignature(), location);
Id callerNodeId = addNodeHierarchy(Node::NODE_FUNCTION, caller, false);
Id calleeNodeId = addNodeHierarchy(Node::NODE_FUNCTION, callee, false);
Id edgeId = addEdge(callerNodeId, calleeNodeId, Edge::EDGE_CALL, location);
return edgeId;
}
Id Storage::onFieldUsageParsed(
const ParseLocation& location, const NameHierarchy& userNameHierarchy, const NameHierarchy& usedNameHierarchy
){
log("field usage", userNameHierarchy.getQualifiedNameWithSignature() + " -> " + usedNameHierarchy.getQualifiedName(), location);
Id userNodeId = addNodeHierarchy(Node::NODE_FUNCTION, userNameHierarchy, false);
Id usedNodeId = addNodeHierarchy(Node::NODE_FIELD, usedNameHierarchy, false);
Id edgeId = addEdge(userNodeId, usedNodeId, Edge::EDGE_USAGE, location);
return edgeId;
}
Id Storage::onGlobalVariableUsageParsed( // or static variable used
const ParseLocation& location, const NameHierarchy& userNameHierarchy, const NameHierarchy& usedNameHierarchy
){
log("global usage", userNameHierarchy.getQualifiedNameWithSignature() + " -> " + usedNameHierarchy.getQualifiedNameWithSignature(), location);
Id userNodeId = addNodeHierarchy(Node::NODE_FUNCTION, userNameHierarchy, false);
Id usedNodeId = addNodeHierarchy(Node::NODE_GLOBAL_VARIABLE, usedNameHierarchy, false);
Id edgeId = addEdge(userNodeId, usedNodeId, Edge::EDGE_USAGE, location);
return edgeId;
}
Id Storage::onEnumConstantUsageParsed(
const ParseLocation& location, const NameHierarchy& userNameHierarchy, const NameHierarchy& usedNameHierarchy
){
log("enum constant usage", userNameHierarchy.getQualifiedNameWithSignature() + " -> " + usedNameHierarchy.getQualifiedNameWithSignature(), location);
Id userNodeId = addNodeHierarchy(Node::NODE_UNDEFINED, userNameHierarchy, false);
Id usedNodeId = addNodeHierarchy(Node::NODE_ENUM_CONSTANT, usedNameHierarchy, false);
Id edgeId = addEdge(userNodeId, usedNodeId, Edge::EDGE_USAGE, location);
return edgeId;
}
Id Storage::onTypeUsageParsed(const ParseLocation& location, const NameHierarchy& user, const NameHierarchy& used)
{
log("type usage", user.getQualifiedNameWithSignature() + " -> " + used.getQualifiedName(), location);
if (!location.isValid())
{
return 0;
}
Id functionNodeId = addNodeHierarchy(Node::NODE_UNDEFINED, user, false);
Id typeNodeId = addNodeHierarchy(Node::NODE_TYPE, used, false);
Id edgeId = addEdge(functionNodeId, typeNodeId, Edge::EDGE_TYPE_USAGE, location);
return edgeId;
}
Id Storage::onTemplateArgumentTypeParsed(
const ParseLocation& location, const NameHierarchy& argumentTypeNameHierarchy,
const NameHierarchy& templateNameHierarchy)
{
log(
"template argument type",
argumentTypeNameHierarchy.getQualifiedName() + " -> " + templateNameHierarchy.getQualifiedName(),
location
);
Id argumentNodeId = addNodeHierarchy(Node::NODE_TYPE, argumentTypeNameHierarchy, false);
Id templateNodeId = addNodeHierarchy(Node::NODE_UNDEFINED, templateNameHierarchy, false);
addEdge(templateNodeId, argumentNodeId, Edge::EDGE_TEMPLATE_ARGUMENT, location);
return argumentNodeId;
}
Id Storage::onTemplateDefaultArgumentTypeParsed(
const ParseLocation& location, const NameHierarchy& defaultArgumentTypeNameHierarchy,
const NameHierarchy& templateParameterNameHierarchy
){
log(
"template default argument",
defaultArgumentTypeNameHierarchy.getQualifiedNameWithSignature() + " -> " + templateParameterNameHierarchy.getQualifiedName(),
location
);
Id defaultArgumentNodeId = addNodeHierarchy(Node::NODE_TYPE, defaultArgumentTypeNameHierarchy, false);
Id parameterNodeId = addNodeHierarchy(Node::NODE_TYPE, templateParameterNameHierarchy, false);
addEdge(parameterNodeId, defaultArgumentNodeId, Edge::EDGE_TEMPLATE_DEFAULT_ARGUMENT, location);
return defaultArgumentNodeId;
}
Id Storage::onTemplateParameterTypeParsed(
const ParseLocation& location, const NameHierarchy& templateParameterTypeNameHierarchy
){
log("template parameter type", templateParameterTypeNameHierarchy.getQualifiedName(), location);
Id parameterNodeId = addNodeHierarchy(Node::NODE_TEMPLATE_PARAMETER_TYPE, templateParameterTypeNameHierarchy, true);
addSourceLocation(parameterNodeId, location, false);
addAccess(parameterNodeId, TokenComponentAccess::ACCESS_TEMPLATE);
return parameterNodeId;
}
Id Storage::onTemplateSpecializationParsed(
const ParseLocation& location, const NameHierarchy& specializedNameHierarchy,
const NameHierarchy& specializedFromNameHierarchy
){
log(
"template record specialization",
specializedNameHierarchy.getQualifiedName() + " -> " + specializedFromNameHierarchy.getQualifiedName(),
location
);
Id specializedId = addNodeHierarchy(Node::NODE_TYPE, specializedNameHierarchy, false);
Id recordNodeId = addNodeHierarchy(Node::NODE_TYPE, specializedFromNameHierarchy, false);
Id edgeId = addEdge(specializedId, recordNodeId, Edge::EDGE_TEMPLATE_SPECIALIZATION_OF, location);
return edgeId;
}
Id Storage::onTemplateMemberFunctionSpecializationParsed(
const ParseLocation& location, const NameHierarchy& instantiatedFunction, const NameHierarchy& specializedFunction
){
log(
"template member function specialization",
instantiatedFunction.getQualifiedNameWithSignature() + " -> " + specializedFunction.getQualifiedNameWithSignature(),
location
);
Id instantiatedFunctionNodeId = addNodeHierarchy(Node::NODE_FUNCTION, instantiatedFunction, false);
Id specializedFunctionNodeId = addNodeHierarchy(Node::NODE_FUNCTION, specializedFunction, false);
Id edgeId = addEdge(instantiatedFunctionNodeId, specializedFunctionNodeId, Edge::EDGE_TEMPLATE_MEMBER_SPECIALIZATION_OF, location);
return edgeId;
}
Id Storage::onTemplateFunctionSpecializationParsed(
const ParseLocation& location, const NameHierarchy specializedFunction, const NameHierarchy templateFunction
){
log("function template specialization", specializedFunction.getQualifiedNameWithSignature(), location);
Id specializedNodeId = addNodeHierarchy(Node::NODE_TYPE, specializedFunction, true);
Id functionNodeId = addNodeHierarchy(Node::NODE_TYPE, templateFunction, false);
Id edgeId = addEdge(specializedNodeId, functionNodeId, Edge::EDGE_TEMPLATE_SPECIALIZATION_OF, location);
return edgeId;
}
Id Storage::onFileParsed(const FileInfo& fileInfo)
{
log("file", fileInfo.path.str(), ParseLocation());
Id fileNodeId = getFileNodeId(fileInfo.path);
if (fileNodeId == 0)
{
NameHierarchy nameHierarchy;
nameHierarchy.push(std::make_shared<NameElement>(fileInfo.path.fileName()));
fileNodeId = m_sqliteStorage.addFile(
NameHierarchy::serialize(nameHierarchy),
fileInfo.path.str(),
utility::timeToString(fileInfo.lastWriteTime)
);
}
return fileNodeId;
}
Id Storage::onFileIncludeParsed(const ParseLocation& location, const FileInfo& fileInfo, const FileInfo& includedFileInfo)
{
log("include", includedFileInfo.path.str(), location);
const Id fileNodeId = onFileParsed(fileInfo);
const Id includedFileNodeId = onFileParsed(includedFileInfo);
addEdge(fileNodeId, includedFileNodeId, Edge::EDGE_INCLUDE, location);
return fileNodeId;
}
Id Storage::onMacroDefineParsed(
const ParseLocation& location, const NameHierarchy& macroNameHierarchy, const ParseLocation& scopeLocation
){
log("macro", macroNameHierarchy.getQualifiedName(), location);
Id macroId = addNodeHierarchy(Node::NODE_MACRO, macroNameHierarchy, true);
addSourceLocation(macroId, location);
addSourceLocation(macroId, scopeLocation, true);
Id fileNodeId = getFileNodeId(location.filePath);
addEdge(fileNodeId, macroId, Edge::EDGE_MACRO_USAGE, location);
return macroId;
}
Id Storage::onMacroExpandParsed(const ParseLocation &location, const NameHierarchy& macroNameHierarchy)
{
log("macro use", macroNameHierarchy.getQualifiedName(), location);
Id macroExpandId = addNodeHierarchy(Node::NODE_MACRO, macroNameHierarchy, false);
Id fileNodeId = getFileNodeId(location.filePath);
Id edgeId = addEdge(fileNodeId, macroExpandId, Edge::EDGE_MACRO_USAGE, location);
return edgeId;
}
Id Storage::onCommentParsed(const ParseLocation& location)
{
log("comment", "no name", location);
Id fileNodeId = m_sqliteStorage.getFileByPath(location.filePath.str()).id;
Id commentId = m_sqliteStorage.addCommentLocation(
fileNodeId,
location.startLineNumber,
location.startColumnNumber,
location.endLineNumber,
location.endColumnNumber
);
return commentId;
}
Id Storage::getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const
{
return m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(nameHierarchy)).id;
@@ -1449,54 +1044,6 @@ void Storage::addEdgesToGraph(const std::vector<Id> edgeIds, Graph* graph) const
}
}
TokenComponentAccess::AccessType Storage::convertAccessType(ParserClient::AccessType access) const
{
switch (access)
{
case ACCESS_PUBLIC:
return TokenComponentAccess::ACCESS_PUBLIC;
case ACCESS_PROTECTED:
return TokenComponentAccess::ACCESS_PROTECTED;
case ACCESS_PRIVATE:
return TokenComponentAccess::ACCESS_PRIVATE;
case ACCESS_NONE:
return TokenComponentAccess::ACCESS_NONE;
}
}
void Storage::addAccess(Id nodeId, TokenComponentAccess::AccessType access)
{
if (access == TokenComponentAccess::ACCESS_NONE)
{
return;
}
std::vector<StorageEdge> memberEdges = m_sqliteStorage.getEdgesByTargetType(nodeId, Edge::EDGE_MEMBER);
if (memberEdges.size() != 1)
{
LOG_ERROR_STREAM(<< "Cannot assign access" << access << " to node id " << nodeId << " because it's no child.");
return;
}
m_sqliteStorage.addComponentAccess(memberEdges[0].id, access);
}
void Storage::addAccess(Id nodeId, ParserClient::AccessType access)
{
addAccess(nodeId, convertAccessType(access));
}
TokenComponentAccess::AccessType Storage::getAccess(Id nodeId) const
{
std::vector<StorageEdge> memberEdges = m_sqliteStorage.getEdgesByTargetType(nodeId, Edge::EDGE_MEMBER);
if (memberEdges.size() == 1)
{
return TokenComponentAccess::intToType(m_sqliteStorage.getComponentAccessByMemberEdgeId(memberEdges[0].id).type);
}
return TokenComponentAccess::ACCESS_NONE;
}
void Storage::addComponentAccessToGraph(Graph* graph) const
{
std::vector<Id> memberEdgeIds;
@@ -1542,12 +1089,3 @@ void Storage::buildHierarchyCache()
m_hierarchyCache.createConnection(edge.id, edge.sourceNodeId, edge.targetNodeId, isVisible);
}
}
void Storage::log(std::string type, std::string str, const ParseLocation& location) const
{
LOG_INFO_STREAM_BARE(
<< type << ": " << str << " <" << location.filePath.str() << " "
<< location.startLineNumber << ":" << location.startColumnNumber << " "
<< location.endLineNumber << ":" << location.endColumnNumber << ">"
);
}