ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: removed storing commandline for errors

Browse Source 
* reason: commandline string may be large (7GB for 20k errors in the UnrealEngine)
This commit is contained in:
mlangkabel
2017-12-12 10:04:44 +01:00
parent 4da2b1f8f4
commit 928aa45bfc
17 changed files with 43 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/TaskBuildIndex.cpp
+1 -1
View File
@@ -144,7 +144,7 @@ void TaskBuildIndex::doExit(std::shared_ptr<Blackboard> blackboard)
is->addError(StorageErrorData(
"The translation unit threw an exception during indexing. Please check if the source file "
"conforms to the specified language standard and all necessary options are defined within your project "
"setup.", "", path, 1, 1, true, true
"setup.", path, 1, 1, true, true
));
LOG_INFO_STREAM(<< "crashed translation unit: " << path.str());
}
src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h
+2 -5
View File
@@ -117,7 +117,6 @@ struct SharedStorageErrorData
{
SharedStorageErrorData(
const std::string& message,
const std::string& commandline,
const std::string& filePath,
uint lineNumber,
uint columnNumber,
@@ -126,7 +125,6 @@ struct SharedStorageErrorData
SharedMemory::Allocator* allocator
)
: message(message.c_str(), allocator)
, commandline(commandline.c_str(), allocator)
, filePath(filePath.c_str(), allocator)
, lineNumber(lineNumber)
, columnNumber(columnNumber)
@@ -135,7 +133,6 @@ struct SharedStorageErrorData
{}
SharedMemory::String message;
SharedMemory::String commandline;
SharedMemory::String filePath;
uint lineNumber;
@@ -148,14 +145,14 @@ struct SharedStorageErrorData
inline SharedStorageErrorData toShared(const StorageErrorData& error, SharedMemory::Allocator* allocator)
{
return SharedStorageErrorData(
error.message, error.commandline, error.filePath.str(),
error.message, error.filePath.str(),
error.lineNumber, error.columnNumber, error.fatal, error.indexed, allocator);
}
inline StorageErrorData fromShared(const SharedStorageErrorData& error)
{
return StorageErrorData(
error.message.c_str(), error.commandline.c_str(), FilePath(error.filePath.c_str()),
error.message.c_str(), FilePath(error.filePath.c_str()),
error.lineNumber, error.columnNumber, error.fatal, error.indexed);
}
src/lib/data/parser/ParserClient.cpp
+2 -2
View File
@@ -80,9 +80,9 @@ ParserClient::~ParserClient()
}
void ParserClient::recordError(
const ParseLocation& location, const std::string& message, const std::string& commandline, bool fatal, bool indexed)
const ParseLocation& location, const std::string& message, bool fatal, bool indexed)
{
doRecordError(location, message, commandline, fatal, indexed);
doRecordError(location, message, fatal, indexed);
if (fatal)
{
src/lib/data/parser/ParserClient.h
+4 -4
View File
@@ -49,8 +49,8 @@ public:
virtual void recordQualifierLocation(
const NameHierarchy& qualifierName, const ParseLocation& location) = 0;
void recordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed);
void recordError(
const ParseLocation& location, const std::string& message, bool fatal, bool indexed);
virtual void recordLocalSymbol(const std::string& name, const ParseLocation& location) = 0;
virtual void recordFile(const FileInfo& fileInfo) = 0;
@@ -59,8 +59,8 @@ public:
bool hasFatalErrors() const;
protected:
virtual void doRecordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed) = 0;
virtual void doRecordError(
const ParseLocation& location, const std::string& message, bool fatal, bool indexed) = 0;
bool m_hasFatalErrors;
};
src/lib/data/parser/ParserClientImpl.cpp
+4 -4
View File
@@ -91,11 +91,11 @@ void ParserClientImpl::recordComment(const ParseLocation& location)
}
void ParserClientImpl::doRecordError(
const ParseLocation& location, const std::string& message, const std::string& commandline, bool fatal, bool indexed)
const ParseLocation& location, const std::string& message, bool fatal, bool indexed)
{
if (location.isValid())
{
addError(message, commandline, fatal, indexed, location);
addError(message, fatal, indexed, location);
}
}
@@ -333,7 +333,7 @@ void ParserClientImpl::addCommentLocation(const ParseLocation& location)
}
void ParserClientImpl::addError(
const std::string& message, const std::string& commandline, bool fatal, bool indexed, const ParseLocation& location)
const std::string& message, bool fatal, bool indexed, const ParseLocation& location)
{
if (!m_storage)
{
@@ -341,6 +341,6 @@ void ParserClientImpl::addError(
}
m_storage->addError(StorageErrorData(
message, commandline, location.filePath, location.startLineNumber, location.startColumnNumber, fatal, indexed
message, location.filePath, location.startLineNumber, location.startColumnNumber, fatal, indexed
));
}
src/lib/data/parser/ParserClientImpl.h
+3 -3
View File
@@ -44,8 +44,8 @@ public:
virtual void recordComment(const ParseLocation& location) override;
private:
virtual void doRecordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed) override;
virtual void doRecordError(
const ParseLocation& location, const std::string& message, bool fatal, bool indexed) override;
NodeType symbolKindToNodeType(SymbolKind symbolType) const;
Edge::EdgeType referenceKindToEdgeType(ReferenceKind referenceKind) const;
@@ -60,7 +60,7 @@ private:
void addSourceLocation(Id elementId, const ParseLocation& location, int type);
void addComponentAccess(Id nodeId , int type);
void addCommentLocation(const ParseLocation& location);
void addError(const std::string& message, const std::string& commandline, bool fatal, bool indexed,
void addError(const std::string& message, bool fatal, bool indexed,
const ParseLocation& location);
std::shared_ptr<IntermediateStorage> m_storage;
src/lib/data/storage/IntermediateStorage.cpp
-1
View File
@@ -44,7 +44,6 @@ size_t IntermediateStorage::getByteSize(size_t stringSize) const
byteSize += sizeof(StorageErrorData);
byteSize += stringSize + storageError.filePath.str().size();
byteSize += stringSize + storageError.message.size();
byteSize += stringSize + storageError.commandline.size();
}
for (const StorageNode& storageNode: getStorageNodes())
src/lib/data/storage/sqlite/SqliteIndexStorage.cpp
+14 -18
View File
@@ -269,15 +269,12 @@ StorageError SqliteIndexStorage::addError(const StorageErrorData& data)
if (id == 0)
{
const std::string sanitizedCommandline = utility::replace(data.commandline, "'", "''");
m_insertErrorStmt.bind(1, sanitizedMessage.c_str());
m_insertErrorStmt.bind(2, sanitizedCommandline.c_str());
m_insertErrorStmt.bind(3, data.fatal);
m_insertErrorStmt.bind(4, data.indexed);
m_insertErrorStmt.bind(5, data.filePath.str().c_str());
m_insertErrorStmt.bind(6, int(data.lineNumber));
m_insertErrorStmt.bind(7, int(data.columnNumber));
m_insertErrorStmt.bind(2, data.fatal);
m_insertErrorStmt.bind(3, data.indexed);
m_insertErrorStmt.bind(4, data.filePath.str().c_str());
m_insertErrorStmt.bind(5, int(data.lineNumber));
m_insertErrorStmt.bind(6, int(data.columnNumber));
const bool success = executeStatement(m_insertErrorStmt);
if (success)
@@ -954,7 +951,6 @@ void SqliteIndexStorage::setupTables()
"CREATE TABLE IF NOT EXISTS error("
"id INTEGER NOT NULL, "
"message TEXT, "
"commandline TEXT, "
"fatal INTEGER NOT NULL, "
"indexed INTEGER NOT NULL, "
"file_path TEXT, "
@@ -1042,7 +1038,7 @@ void SqliteIndexStorage::setupPrecompiledStatements()
"LIMIT 1;"
);
m_insertErrorStmt = m_database.compileStatement(
"INSERT INTO error(message, commandline, fatal, indexed, file_path, line_number, column_number) VALUES(?, ?, ?, ?, ?, ?, ?);"
"INSERT INTO error(message, fatal, indexed, file_path, line_number, column_number) VALUES(?, ?, ?, ?, ?, ?);"
);
}
catch (CppSQLite3Exception& e)
@@ -1288,7 +1284,7 @@ template <>
std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::string& query) const
{
CppSQLite3Query q = executeQuery(
"SELECT message, commandline, fatal, indexed, file_path, line_number, column_number FROM error " + query + ";"
"SELECT message, fatal, indexed, file_path, line_number, column_number FROM error " + query + ";"
);
std::vector<StorageError> errors;
@@ -1296,17 +1292,17 @@ std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::
while (!q.eof())
{
const std::string message = q.getStringField(0, "");
const std::string commandline = q.getStringField(1, "");
const bool fatal = q.getIntField(2, 0);
const bool indexed = q.getIntField(3, 0);
const std::string filePath = q.getStringField(4, "");
const int lineNumber = q.getIntField(5, -1);
const int columnNumber = q.getIntField(6, -1);
const bool fatal = q.getIntField(1, 0);
const bool indexed = q.getIntField(2, 0);
const std::string filePath = q.getStringField(3, "");
const int lineNumber = q.getIntField(4, -1);
const int columnNumber = q.getIntField(5, -1);
if (lineNumber != -1 && columnNumber != -1)
{
errors.push_back(StorageError(
id, message, commandline, FilePath(filePath), lineNumber, columnNumber, fatal, indexed));
id, message, FilePath(filePath), lineNumber, columnNumber, fatal, indexed)
);
id++;
}
src/lib/data/storage/type/StorageError.h
-5
View File
@@ -18,7 +18,6 @@ struct StorageErrorData
StorageErrorData(
const std::string& message,
const std::string& commandline,
const FilePath& filePath,
uint lineNumber,
uint columnNumber,
@@ -26,7 +25,6 @@ struct StorageErrorData
bool indexed
)
: message(message)
, commandline(commandline)
, filePath(filePath)
, lineNumber(lineNumber)
, columnNumber(columnNumber)
@@ -35,7 +33,6 @@ struct StorageErrorData
{}
std::string message;
std::string commandline;
FilePath filePath;
uint lineNumber;
@@ -60,7 +57,6 @@ struct StorageError: public StorageErrorData
StorageError(
Id id,
const std::string& message,
const std::string& commandline,
const FilePath& filePath,
uint lineNumber,
uint columnNumber,
@@ -69,7 +65,6 @@ struct StorageError: public StorageErrorData
)
: StorageErrorData(
message,
commandline,
filePath,
lineNumber,
columnNumber,
src/lib_cxx/data/parser/cxx/CxxDiagnosticConsumer.cpp
-6
View File
@@ -94,14 +94,8 @@ void CxxDiagnosticConsumer::HandleDiagnostic(clang::DiagnosticsEngine::Level lev
m_client->recordError(
location,
message,
m_commandline,
level == clang::DiagnosticsEngine::Fatal,
m_register->hasFilePath(location.filePath)
);
}
}
void CxxDiagnosticConsumer::setCommandLine(const std::string& commandline)
{
m_commandline = commandline;
}
src/lib_cxx/data/parser/cxx/CxxDiagnosticConsumer.h
-4
View File
@@ -25,15 +25,11 @@ public:
void HandleDiagnostic(clang::DiagnosticsEngine::Level level, const clang::Diagnostic& info);
void setCommandLine(const std::string& commandline);
private:
std::shared_ptr<ParserClient> m_client;
std::shared_ptr<FileRegister> m_register;
std::shared_ptr<CanonicalFilePathCache> m_canonicalFilePathCache;
std::string m_commandline;
bool m_isParsingFile;
bool m_useLogging;
};
src/lib_cxx/data/parser/cxx/CxxParser.cpp
-11
View File
@@ -116,17 +116,6 @@ void CxxParser::runTool(clang::tooling::CompilationDatabase* compilationDatabase
std::shared_ptr<CxxDiagnosticConsumer> diagnostics = getDiagnostics(canonicalFilePathCache, true);
std::vector<clang::tooling::CompileCommand> compileCommands = compilationDatabase->getCompileCommands(sourceFilePath.str());
if (compileCommands.size() == 1)
{
std::string commandline;
for (const std::string& str : compileCommands[0].CommandLine)
{
commandline += str + " ";
}
diagnostics->setCommandLine(commandline);
}
tool.setDiagnosticConsumer(diagnostics.get());
ASTActionFactory actionFactory(m_client, m_fileRegister, canonicalFilePathCache);
src/lib_gui/qt/view/QtErrorView.cpp
+2 -4
View File
@@ -76,16 +76,15 @@ void QtErrorView::initView()
m_table->setItemDelegate(new SelectableDelegate());
// Setup Table Headers
m_model->setColumnCount(7);
m_model->setColumnCount(COLUMN_MAX + 1);
m_table->setColumnWidth(COLUMN::TYPE, 80);
m_table->setColumnWidth(COLUMN::MESSAGE, 450);
m_table->setColumnWidth(COLUMN::FILE, 300);
m_table->setColumnWidth(COLUMN::LINE, 50);
m_table->setColumnWidth(COLUMN::COMMANDLINE, 120);
m_table->setColumnHidden(COLUMN::ID, true);
QStringList headers;
headers << "Type" << "Message" << "File" << "Line" << "Command Line" << "Indexed";
headers << "Type" << "Message" << "File" << "Line" << "Indexed";
m_model->setHorizontalHeaderLabels(headers);
connect(m_table->selectionModel(), &QItemSelectionModel::currentRowChanged,
@@ -331,7 +330,6 @@ void QtErrorView::addErrorToTable(const ErrorInfo& error)
m_model->item(rowNumber, COLUMN::FILE)->setToolTip(error.filePath.str().c_str());
m_model->setItem(rowNumber, COLUMN::LINE, new QStandardItem(QString::number(error.lineNumber)));
m_model->setItem(rowNumber, COLUMN::COMMANDLINE, new QStandardItem(error.commandline.c_str()));
m_model->setItem(rowNumber, COLUMN::INDEXED, new QStandardItem(error.indexed ? "yes" : "no"));
src/lib_gui/qt/view/QtErrorView.h
+3 -3
View File
@@ -50,9 +50,9 @@ private:
MESSAGE = 1,
FILE = 2,
LINE = 3,
COMMANDLINE = 4,
INDEXED = 5,
ID = 6
INDEXED = 4,
ID = 5,
COLUMN_MAX = ID
};
void setStyleSheet() const;
src/lib_java/data/parser/java/JavaParser.cpp
+2 -3
View File
@@ -264,8 +264,7 @@ void JavaParser::doRecordError(
m_client->recordError(
ParseLocation(m_currentFilePath, beginLine, beginColumn, endLine, endColumn),
m_javaEnvironment->toStdString(jMessage),
"",
fatal, indexed
fatal,
indexed
);
}
src/test/helper/DumpParserClient.h
+1 -1
View File
@@ -82,7 +82,7 @@ public:
std::string m_lines;
private:
virtual void doRecordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
virtual void doRecordError(const ParseLocation& location, const std::string& message,
bool fatal, bool indexed) override
{
recordLine("ERROR: " + addLocationSuffix(message + " [" + location.filePath.fileName(), location) + "]\n");
src/test/helper/TestParserClient.h
+5 -2
View File
@@ -162,8 +162,11 @@ public:
std::vector<std::string> imports;
private:
virtual void doRecordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed) override
virtual void doRecordError(
const ParseLocation& location,
const std::string& message,
bool fatal,
bool indexed) override
{
errors.push_back(addLocationSuffix(message, location));
}