ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: removed storing commandline for errors

Browse Source 
* reason: commandline string may be large (7GB for 20k errors in the UnrealEngine)
This commit is contained in:
mlangkabel
2017-12-12 10:04:44 +01:00
parent 4da2b1f8f4
commit 928aa45bfc
17 changed files with 43 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/indexer/TaskBuildIndex.cpp
+1 -1
View File
@@ -144,7 +144,7 @@ void TaskBuildIndex::doExit(std::shared_ptr<Blackboard> blackboard)
is->addError(StorageErrorData(
"The translation unit threw an exception during indexing. Please check if the source file "
"conforms to the specified language standard and all necessary options are defined within your project "
"setup.", "", path, 1, 1, true, true
"setup.", path, 1, 1, true, true
));
LOG_INFO_STREAM(<< "crashed translation unit: " << path.str());
}
src/lib/data/indexer/interprocess/shared_types/SharedStorageTypes.h
+2 -5
View File
@@ -117,7 +117,6 @@ struct SharedStorageErrorData
{
SharedStorageErrorData(
const std::string& message,
const std::string& commandline,
const std::string& filePath,
uint lineNumber,
uint columnNumber,
@@ -126,7 +125,6 @@ struct SharedStorageErrorData
SharedMemory::Allocator* allocator
)
: message(message.c_str(), allocator)
, commandline(commandline.c_str(), allocator)
, filePath(filePath.c_str(), allocator)
, lineNumber(lineNumber)
, columnNumber(columnNumber)
@@ -135,7 +133,6 @@ struct SharedStorageErrorData
{}
SharedMemory::String message;
SharedMemory::String commandline;
SharedMemory::String filePath;
uint lineNumber;
@@ -148,14 +145,14 @@ struct SharedStorageErrorData
inline SharedStorageErrorData toShared(const StorageErrorData& error, SharedMemory::Allocator* allocator)
{
return SharedStorageErrorData(
error.message, error.commandline, error.filePath.str(),
error.message, error.filePath.str(),
error.lineNumber, error.columnNumber, error.fatal, error.indexed, allocator);
}
inline StorageErrorData fromShared(const SharedStorageErrorData& error)
{
return StorageErrorData(
error.message.c_str(), error.commandline.c_str(), FilePath(error.filePath.c_str()),
error.message.c_str(), FilePath(error.filePath.c_str()),
error.lineNumber, error.columnNumber, error.fatal, error.indexed);
}
src/lib/data/parser/ParserClient.cpp
+2 -2
View File
@@ -80,9 +80,9 @@ ParserClient::~ParserClient()
}
void ParserClient::recordError(
const ParseLocation& location, const std::string& message, const std::string& commandline, bool fatal, bool indexed)
const ParseLocation& location, const std::string& message, bool fatal, bool indexed)
{
doRecordError(location, message, commandline, fatal, indexed);
doRecordError(location, message, fatal, indexed);
if (fatal)
{
src/lib/data/parser/ParserClient.h
+4 -4
View File
@@ -49,8 +49,8 @@ public:
virtual void recordQualifierLocation(
const NameHierarchy& qualifierName, const ParseLocation& location) = 0;
void recordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed);
void recordError(
const ParseLocation& location, const std::string& message, bool fatal, bool indexed);
virtual void recordLocalSymbol(const std::string& name, const ParseLocation& location) = 0;
virtual void recordFile(const FileInfo& fileInfo) = 0;
@@ -59,8 +59,8 @@ public:
bool hasFatalErrors() const;
protected:
virtual void doRecordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed) = 0;
virtual void doRecordError(
const ParseLocation& location, const std::string& message, bool fatal, bool indexed) = 0;
bool m_hasFatalErrors;
};
src/lib/data/parser/ParserClientImpl.cpp
+4 -4
View File
@@ -91,11 +91,11 @@ void ParserClientImpl::recordComment(const ParseLocation& location)
}
void ParserClientImpl::doRecordError(
const ParseLocation& location, const std::string& message, const std::string& commandline, bool fatal, bool indexed)
const ParseLocation& location, const std::string& message, bool fatal, bool indexed)
{
if (location.isValid())
{
addError(message, commandline, fatal, indexed, location);
addError(message, fatal, indexed, location);
}
}
@@ -333,7 +333,7 @@ void ParserClientImpl::addCommentLocation(const ParseLocation& location)
}
void ParserClientImpl::addError(
const std::string& message, const std::string& commandline, bool fatal, bool indexed, const ParseLocation& location)
const std::string& message, bool fatal, bool indexed, const ParseLocation& location)
{
if (!m_storage)
{
@@ -341,6 +341,6 @@ void ParserClientImpl::addError(
}
m_storage->addError(StorageErrorData(
message, commandline, location.filePath, location.startLineNumber, location.startColumnNumber, fatal, indexed
message, location.filePath, location.startLineNumber, location.startColumnNumber, fatal, indexed
));
}
src/lib/data/parser/ParserClientImpl.h
+3 -3
View File
@@ -44,8 +44,8 @@ public:
virtual void recordComment(const ParseLocation& location) override;
private:
virtual void doRecordError(const ParseLocation& location, const std::string& message, const std::string& commandline,
bool fatal, bool indexed) override;
virtual void doRecordError(
const ParseLocation& location, const std::string& message, bool fatal, bool indexed) override;
NodeType symbolKindToNodeType(SymbolKind symbolType) const;
Edge::EdgeType referenceKindToEdgeType(ReferenceKind referenceKind) const;
@@ -60,7 +60,7 @@ private:
void addSourceLocation(Id elementId, const ParseLocation& location, int type);
void addComponentAccess(Id nodeId , int type);
void addCommentLocation(const ParseLocation& location);
void addError(const std::string& message, const std::string& commandline, bool fatal, bool indexed,
void addError(const std::string& message, bool fatal, bool indexed,
const ParseLocation& location);
std::shared_ptr<IntermediateStorage> m_storage;
src/lib/data/storage/IntermediateStorage.cpp
-1
View File
@@ -44,7 +44,6 @@ size_t IntermediateStorage::getByteSize(size_t stringSize) const
byteSize += sizeof(StorageErrorData);
byteSize += stringSize + storageError.filePath.str().size();
byteSize += stringSize + storageError.message.size();
byteSize += stringSize + storageError.commandline.size();
}
for (const StorageNode& storageNode: getStorageNodes())
src/lib/data/storage/sqlite/SqliteIndexStorage.cpp
+14 -18
View File
@@ -269,15 +269,12 @@ StorageError SqliteIndexStorage::addError(const StorageErrorData& data)
if (id == 0)
{
const std::string sanitizedCommandline = utility::replace(data.commandline, "'", "''");
m_insertErrorStmt.bind(1, sanitizedMessage.c_str());
m_insertErrorStmt.bind(2, sanitizedCommandline.c_str());
m_insertErrorStmt.bind(3, data.fatal);
m_insertErrorStmt.bind(4, data.indexed);
m_insertErrorStmt.bind(5, data.filePath.str().c_str());
m_insertErrorStmt.bind(6, int(data.lineNumber));
m_insertErrorStmt.bind(7, int(data.columnNumber));
m_insertErrorStmt.bind(2, data.fatal);
m_insertErrorStmt.bind(3, data.indexed);
m_insertErrorStmt.bind(4, data.filePath.str().c_str());
m_insertErrorStmt.bind(5, int(data.lineNumber));
m_insertErrorStmt.bind(6, int(data.columnNumber));
const bool success = executeStatement(m_insertErrorStmt);
if (success)
@@ -954,7 +951,6 @@ void SqliteIndexStorage::setupTables()
"CREATE TABLE IF NOT EXISTS error("
"id INTEGER NOT NULL, "
"message TEXT, "
"commandline TEXT, "
"fatal INTEGER NOT NULL, "
"indexed INTEGER NOT NULL, "
"file_path TEXT, "
@@ -1042,7 +1038,7 @@ void SqliteIndexStorage::setupPrecompiledStatements()
"LIMIT 1;"
);
m_insertErrorStmt = m_database.compileStatement(
"INSERT INTO error(message, commandline, fatal, indexed, file_path, line_number, column_number) VALUES(?, ?, ?, ?, ?, ?, ?);"
"INSERT INTO error(message, fatal, indexed, file_path, line_number, column_number) VALUES(?, ?, ?, ?, ?, ?);"
);
}
catch (CppSQLite3Exception& e)
@@ -1288,7 +1284,7 @@ template <>
std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::string& query) const
{
CppSQLite3Query q = executeQuery(
"SELECT message, commandline, fatal, indexed, file_path, line_number, column_number FROM error " + query + ";"
"SELECT message, fatal, indexed, file_path, line_number, column_number FROM error " + query + ";"
);
std::vector<StorageError> errors;
@@ -1296,17 +1292,17 @@ std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::
while (!q.eof())
{
const std::string message = q.getStringField(0, "");
const std::string commandline = q.getStringField(1, "");
const bool fatal = q.getIntField(2, 0);
const bool indexed = q.getIntField(3, 0);
const std::string filePath = q.getStringField(4, "");
const int lineNumber = q.getIntField(5, -1);
const int columnNumber = q.getIntField(6, -1);
const bool fatal = q.getIntField(1, 0);
const bool indexed = q.getIntField(2, 0);
const std::string filePath = q.getStringField(3, "");
const int lineNumber = q.getIntField(4, -1);
const int columnNumber = q.getIntField(5, -1);
if (lineNumber != -1 && columnNumber != -1)
{
errors.push_back(StorageError(
id, message, commandline, FilePath(filePath), lineNumber, columnNumber, fatal, indexed));
id, message, FilePath(filePath), lineNumber, columnNumber, fatal, indexed)
);
id++;
}
src/lib/data/storage/type/StorageError.h
-5
View File
@@ -18,7 +18,6 @@ struct StorageErrorData
StorageErrorData(
const std::string& message,
const std::string& commandline,
const FilePath& filePath,
uint lineNumber,
uint columnNumber,
@@ -26,7 +25,6 @@ struct StorageErrorData
bool indexed
)
: message(message)
, commandline(commandline)
, filePath(filePath)
, lineNumber(lineNumber)
, columnNumber(columnNumber)
@@ -35,7 +33,6 @@ struct StorageErrorData
{}
std::string message;
std::string commandline;
FilePath filePath;
uint lineNumber;
@@ -60,7 +57,6 @@ struct StorageError: public StorageErrorData
StorageError(
Id id,
const std::string& message,
const std::string& commandline,
const FilePath& filePath,
uint lineNumber,
uint columnNumber,
@@ -69,7 +65,6 @@ struct StorageError: public StorageErrorData
)
: StorageErrorData(
message,
commandline,
filePath,
lineNumber,
columnNumber,