ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

src: added optional indexer AST logging

Browse Source 
* removed old AstVisitor
* added new implementation called CxxAstVisitor
* added CxxVerboseAstVisitor that may wrap the CxxAstVisitor to log the AST while visiting
* added the same structure for the Java visitors
* removed all the logs that were fired when recording stuff
* Added UI setting for verbose indexer logging.
* implemented AST name obfuscation for Java and Cxx
* unified test visitors for Java and Cxx
* implemented recording using directive decl
* removed most of the old onXxxParsed methods
This commit is contained in:
malte_langkabel
2016-11-01 10:37:30 +01:00
parent a4240def56
commit 9e4a02a33a
36 changed files with 2145 additions and 3208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java_indexer/src/io/coati/JavaAstVisitor.java
+16 -34
View File
@@ -46,21 +46,14 @@ import me.tomassetti.symbolsolver.model.typesystem.*;
public class JavaAstVisitor extends JavaAstVisitorAdapter
{
private int m_callbackId = -1;
protected int m_callbackId = -1;
private String m_filePath;
private FileContent m_fileContent;
private TypeSolver m_typeSolver;
private List<DeclContext> m_context = new ArrayList<DeclContext>();
private boolean m_verbose = false;
static int errorCount = 0;
public JavaAstVisitor(int callbackId, String filePath, FileContent fileContent, TypeSolver typeSolver)
{
if (m_verbose)
{
System.out.println("indexing file: " + filePath);
}
{
m_callbackId = callbackId;
m_filePath = filePath;
m_fileContent = fileContent;
@@ -80,7 +73,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
String packageName = JavaparserDeclNameResolver.getQualifiedName(nameExpr).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithScope(
JavaIndexer.recordSymbolWithLocationAndScope(
m_callbackId, packageName, SymbolType.PACKAGE,
nameExpr.getBegin().line, nameExpr.getBegin().column, nameExpr.getEnd().line, nameExpr.getEnd().column,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
@@ -96,7 +89,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
String qualifiedName = JavaparserDeclNameResolver.getQualifiedDeclName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithScope(
JavaIndexer.recordSymbolWithLocationAndScope(
m_callbackId, qualifiedName, (n.isInterface() ? SymbolType.INTERFACE : SymbolType.CLASS),
nameExpr.getBegin().line, nameExpr.getBegin().column, nameExpr.getEnd().line, nameExpr.getEnd().column,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
@@ -128,7 +121,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
qualifiedName += "\tn";
qualifiedName += n.getName() + "\ts\tp";
JavaIndexer.recordSymbol(
JavaIndexer.recordSymbolWithLocation(
m_callbackId, qualifiedName, SymbolType.TYPE_PARAMETER,
n.getBegin().line, n.getBegin().column, n.getBegin().line, n.getBegin().column + n.getName().length() - 1,
AccessKind.TYPE_PARAMETER, false
@@ -147,7 +140,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
String qualifiedName = JavaparserDeclNameResolver.getQualifiedDeclName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithScope(
JavaIndexer.recordSymbolWithLocationAndScope(
m_callbackId, qualifiedName, SymbolType.ENUM,
nameExpr.getBegin().line, nameExpr.getBegin().column, nameExpr.getEnd().line, nameExpr.getEnd().column,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
@@ -168,7 +161,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
{
String qualifiedName = JavaparserDeclNameResolver.getQualifiedDeclName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbol(
JavaIndexer.recordSymbolWithLocation(
m_callbackId, qualifiedName, SymbolType.ENUM_CONSTANT,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
AccessKind.NONE, false
@@ -187,7 +180,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
String qualifiedName = JavaparserDeclNameResolver.getQualifiedDeclName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithScope(
JavaIndexer.recordSymbolWithLocationAndScope(
m_callbackId, qualifiedName, SymbolType.METHOD,
nameExpr.getBegin().line, nameExpr.getBegin().column, nameExpr.getEnd().line, nameExpr.getEnd().column,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
@@ -207,7 +200,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
String qualifiedName = JavaparserDeclNameResolver.getQualifiedDeclName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithScope(
JavaIndexer.recordSymbolWithLocationAndScope(
m_callbackId, qualifiedName, SymbolType.METHOD,
nameExpr.getBegin().line, nameExpr.getBegin().column, nameExpr.getEnd().line, nameExpr.getEnd().column,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
@@ -290,7 +283,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
String qualifiedName = JavaparserDeclNameResolver.getQualifiedDeclName(varDecl, m_typeSolver).toSerializedNameHierarchy();
VariableDeclaratorId varDeclId = varDecl.getId();
JavaIndexer.recordSymbol(
JavaIndexer.recordSymbolWithLocation(
m_callbackId, qualifiedName, SymbolType.FIELD,
varDeclId.getBegin().line, varDeclId.getBegin().column, varDeclId.getEnd().line, varDeclId.getEnd().column,
AccessKind.fromAccessSpecifier(Modifier.getAccessSpecifier(n.getModifiers())), false
@@ -470,7 +463,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
{
String referencedName = JavaparserTypeNameResolver.getQualifiedTypeName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithoutLocation(
JavaIndexer.recordSymbol(
m_callbackId, referencedName, SymbolType.BUILTIN_TYPE,
AccessKind.NONE, true
);
@@ -497,7 +490,7 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
{
String referencedName = JavaparserTypeNameResolver.getQualifiedTypeName(n, m_typeSolver).toSerializedNameHierarchy();
JavaIndexer.recordSymbolWithoutLocation(
JavaIndexer.recordSymbol(
m_callbackId, referencedName, SymbolType.BUILTIN_TYPE,
AccessKind.NONE, true
);
@@ -615,7 +608,6 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
MethodUsage solvedMethod = JavaParserFacade.get(m_typeSolver).solveMethodAsUsage(n);
qualifiedName = getQualifiedName(solvedMethod);
}
catch (UnsupportedOperationException e)
{
recordException(e, n);
@@ -632,7 +624,6 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
{
recordException(e, n);
}
}
if (!qualifiedName.isEmpty())
@@ -653,31 +644,22 @@ public class JavaAstVisitor extends JavaAstVisitorAdapter
private void recordException(Exception e, Node n)
{
JavaIndexer.recordSymbol(
JavaIndexer.logError(m_callbackId, e.getClass() + " at " + m_filePath + "<"+ n.getBegin().line + ", " + n.getBegin().column + ">");
JavaIndexer.recordSymbolWithLocation(
m_callbackId, "unsolved-symbol\ts\tp", SymbolType.TYPE_MAX,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
AccessKind.DEFAULT, false
);
errorCount++;
if (m_verbose)
{
System.out.println(e + " at location " + n.getBegin().line + ", " + n.getBegin().column + " [errors: " + errorCount + "]");
}
}
private void recordError(Error e, Node n)
{
JavaIndexer.recordSymbol(
JavaIndexer.logError(m_callbackId, e.getClass() + " at " + m_filePath + "<"+ n.getBegin().line + ", " + n.getBegin().column + ">");
JavaIndexer.recordSymbolWithLocation(
m_callbackId, "unsolved-symbol\ts\tp", SymbolType.TYPE_MAX,
n.getBegin().line, n.getBegin().column, n.getEnd().line, n.getEnd().column,
AccessKind.DEFAULT, false
);
errorCount++;
if (m_verbose)
{
System.out.println(e + " at location " + n.getBegin().line + ", " + n.getBegin().column + " [errors: " + errorCount + "]");
}
}
java_indexer/src/io/coati/JavaIndexer.java
+40 -28
View File
@@ -20,9 +20,9 @@ import me.tomassetti.symbolsolver.resolution.typesolvers.JreTypeSolver;
public class JavaIndexer
{
public static void processFile(int address, String filePath, String fileContent, String classPath)
public static void processFile(int address, String filePath, String fileContent, String classPath, int verbose)
{
// System.out.println("indexing file: " + filePath);
logInfo(address, "indexing source file: " + filePath);
try
{
@@ -51,8 +51,12 @@ public class JavaIndexer
}
CompilationUnit cu = JavaParser.parse(new StringReader(fileContent), true);
JavaAstVisitor astVisitor = new JavaAstVisitor(address, filePath, new FileContent(fileContent), typeSolver);
// JavaAstVisitor astVisitor = new ASTDumper(address, filePath, new FileContent(fileContent), typeSolver);
JavaAstVisitor astVisitor = (
verbose == 1 ?
new JavaVerboseAstVisitor(address, filePath, new FileContent(fileContent), typeSolver) :
new JavaAstVisitor(address, filePath, new FileContent(fileContent), typeSolver)
);
cu.accept(astVisitor, null);
}
catch (ParseProblemException e)
@@ -61,8 +65,6 @@ public class JavaIndexer
{
if (problem.message.startsWith("Encountered unexpected token"))
{
int startLine = Integer.parseInt(problem.message.substring(
problem.message.indexOf("line ") + ("line ").length(),
problem.message.indexOf(",")
@@ -117,38 +119,40 @@ public class JavaIndexer
return packageName;
}
// helpers
static public void recordSymbol(
int address, String symbolName, SymbolType symbolType,
AccessKind access, boolean isImplicit
)
{
recordSymbol(
address, symbolName, symbolType.getValue(),
access.getValue(), (isImplicit ? 1 : 0)
);
}
static public void recordSymbolWithLocation(
int address, String symbolName, SymbolType symbolType,
int beginLine, int beginColumn, int endLine, int endColumn,
AccessKind access, boolean isImplicit
)
{
recordSymbol(
recordSymbolWithLocation(
address, symbolName, symbolType.getValue(),
beginLine, beginColumn, endLine, endColumn,
access.getValue(), (isImplicit ? 1 : 0)
);
}
static public void recordSymbolWithoutLocation(
int address, String symbolName, SymbolType symbolType,
AccessKind access, boolean isImplicit
)
{
recordSymbolWithoutLocation(
address, symbolName, symbolType.getValue(),
access.getValue(), (isImplicit ? 1 : 0)
);
}
static public void recordSymbolWithScope(
static public void recordSymbolWithLocationAndScope(
int address, String symbolName, SymbolType symbolType,
int beginLine, int beginColumn, int endLine, int endColumn,
int scopeBeginLine, int scopeBeginColumn, int scopeEndLine, int scopeEndColumn,
AccessKind access, boolean isImplicit
)
{
recordSymbolWithScope(
recordSymbolWithLocationAndScope(
address, symbolName, symbolType.getValue(),
beginLine, beginColumn, endLine, endColumn,
scopeBeginLine, scopeBeginColumn, scopeEndLine, scopeEndColumn,
@@ -176,19 +180,27 @@ public class JavaIndexer
beginLine, beginColumn, endLine, endColumn
);
}
// the following methods are defines in the native c++ code
static public native void logInfo(int address, String info);
static public native void logWarning(int address, String warning);
static public native void logError(int address, String error);
static private native void recordSymbol(
int address, String symbolName, int symbolType,
int access, int isImplicit
);
static private native void recordSymbolWithLocation(
int address, String symbolName, int symbolType,
int beginLine, int beginColumn, int endLine, int endColumn,
int access, int isImplicit
);
static private native void recordSymbolWithoutLocation(
int address, String symbolName, int symbolType,
int access, int isImplicit
);
static private native void recordSymbolWithScope(
static private native void recordSymbolWithLocationAndScope(
int address, String symbolName, int symbolType,
int beginLine, int beginColumn, int endLine, int endColumn,
int scopeBeginLine, int scopeBeginColumn, int scopeEndLine, int scopeEndColumn,
@@ -199,11 +211,11 @@ public class JavaIndexer
int address, int referenceKind, String referencedName, String contextName, int beginLine, int beginColumn, int endLine, int endColumn
);
static native void recordLocalSymbol(
static public native void recordLocalSymbol(
int address, String symbolName, int beginLine, int beginColumn, int endLine, int endColumn
);
static native void recordComment(
static public native void recordComment(
int address, int beginLine, int beginColumn, int endLine, int endColumn
);
java_indexer/src/io/coati/ASTDumper.java → java_indexer/src/io/coati/JavaVerboseAstVisitor.java
+19 -13
View File
@@ -4,20 +4,34 @@ import com.github.javaparser.ast.*;
import com.github.javaparser.ast.body.*;
import com.github.javaparser.ast.comments.*;
import com.github.javaparser.ast.expr.*;
import com.github.javaparser.ast.nodeTypes.NodeWithName;
import com.github.javaparser.ast.stmt.*;
import com.github.javaparser.ast.type.*;
import me.tomassetti.symbolsolver.model.resolution.TypeSolver;
public class ASTDumper extends JavaAstVisitor{
public class JavaVerboseAstVisitor extends JavaAstVisitor{
public ASTDumper(int callbackId, String filePath, FileContent fileContent, TypeSolver typeSolver) {
public JavaVerboseAstVisitor(int callbackId, String filePath, FileContent fileContent, TypeSolver typeSolver) {
super(callbackId, filePath, fileContent, typeSolver);
}
int indent = 0;
String indentSymbol = "| ";
private String obfuscate(final String s)
{
if (s.isEmpty())
{
return "";
}
else if (s.length() <= 2)
{
return s;
}
return s.substring(0, 1) + ".." + s.substring(s.length() - 1, s.length());
}
private void dump(Node n)
{
String line = "";
@@ -28,21 +42,13 @@ public class ASTDumper extends JavaAstVisitor{
}
line += n.getClass().getName();
if (n instanceof NameExpr)
if (n instanceof NodeWithName<?>)
{
line += " [" + ((NameExpr)n).getName() + "]";
}
else if (n instanceof ClassOrInterfaceType)
{
line += " [" + ((ClassOrInterfaceType)n).getName() + "]";
}
else if (n instanceof MethodCallExpr)
{
line += " [" + ((MethodCallExpr)n).getName() + "]";
line += " [" + obfuscate(((NodeWithName<?>)n).getName()) + "]";
}
line += " line: " + n.getBegin().line;
System.out.println(line);
JavaIndexer.logInfo(m_callbackId, line);
}
//- Compilation Unit ----------------------------------