data, ui: comment handling

* CodeView displays comments as atomic regions. They are either displayed as a whole or not at all. Do nothing halfway!
* added CommentHandler that handles comments detected by clang.
* stored CommentLocations in database.

src/lib/CMakeLists.txt

@@ -22,6 +22,8 @@ add_files(
 	data/parser/cxx/ASTConsumer.h
 	data/parser/cxx/ASTVisitor.cpp
 	data/parser/cxx/ASTVisitor.h
+	data/parser/cxx/CommentHandler.cpp
+	data/parser/cxx/CommentHandler.h
 	data/parser/cxx/CxxDiagnosticConsumer.cpp
 	data/parser/cxx/CxxDiagnosticConsumer.h
 	data/parser/cxx/CxxParser.cpp

src/lib/component/controller/CodeController.cpp

@@ -257,7 +257,19 @@ std::vector<CodeView::CodeSnippetParams> CodeController::getSnippetsForFile(std:
 			}
 		);
 
-		ranges = fileScopedMerger.merge();
+		std::vector<SnippetMerger::Range> atomicRanges;
+		m_storageAccess->getCommentLocationsInFile(file->getFilePath())->forEachStartTokenLocation(
+			[&](TokenLocation* location)
+			{
+				atomicRanges.push_back(SnippetMerger::Range(
+					SnippetMerger::Border(location->getLineNumber(), false),
+					SnippetMerger::Border(location->getOtherTokenLocation()->getLineNumber(), false)
+				));
+			}
+		);
+		atomicRanges = SnippetMerger::Range::mergeAdjacent(atomicRanges);
+		
+		ranges = fileScopedMerger.merge(atomicRanges);
 	}
 
 	const int snippetExpandRange = ApplicationSettings::getInstance()->getCodeSnippetExpandRange();

src/lib/component/controller/helper/SnippetMerger.cpp

@@ -14,8 +14,9 @@ void SnippetMerger::addChild(std::shared_ptr<SnippetMerger> child)
 	m_children.push_back(child);
 }
 
-std::deque<SnippetMerger::Range> SnippetMerger::merge() const
+std::deque<SnippetMerger::Range> SnippetMerger::merge(std::vector<SnippetMerger::Range> atomicRanges) const
 {
+	const int snippetExpandRange = ApplicationSettings::getInstance()->getCodeSnippetExpandRange();
 	std::deque<Range> merged;
 	if (m_children.size() == 0)
 	{
@@ -25,10 +26,10 @@ std::deque<SnippetMerger::Range> SnippetMerger::merge() const
 	{
 		for (size_t i = 0; i < m_children.size(); i++)
 		{
-			std::deque<Range> mergedFromChild = m_children[i]->merge();
+			std::deque<Range> mergedFromChild = m_children[i]->merge(atomicRanges);
 			for (size_t j = 0; j < mergedFromChild.size(); j++)
 			{
-				merged.push_back(mergedFromChild[j]);
+				merged.push_back(getExpandedRegardingAtomicRanges(mergedFromChild[j], snippetExpandRange, atomicRanges));
 			}
 		}
 		std::sort(merged.begin(), merged.end(),
@@ -39,36 +40,51 @@ std::deque<SnippetMerger::Range> SnippetMerger::merge() const
 		);
 
 		// merge children
-		const int snippetExpandRange = ApplicationSettings::getInstance()->getCodeSnippetExpandRange();
 		const int snippetMergeRange = 2 * snippetExpandRange + 1;	// +1 since snippets that end/start with consequtive 
 																	// lines should be merged as well.
-		for (size_t i = 0; i < merged.size() - 1; i++)
-		{
-			const Range first = merged[i];
-			const Range second = merged[i + 1];
-			if (first.end.row + snippetMergeRange >= second.start.row)
-			{
-				merged.erase(merged.begin() + i, merged.begin() + i + 2);
-				merged.insert(merged.begin() + i, Range(
-					first.start.row < second.start.row ? first.start : second.start,
-					first.end.row > second.end.row ? first.end : second.end
-				));
-				i--;
-			}
-		}
+		merged = Range::mergeAdjacent(merged, snippetMergeRange);
 
 		// snap to own borders
 		const int snippetSnapRange = ApplicationSettings::getInstance()->getCodeSnippetSnapRange();
-		if (m_start + snippetSnapRange >= merged.front().start.row)
+		if ((m_start < merged.front().start.row) && 
+			(merged.front().start.row <= m_start + snippetSnapRange))
 		{
 			merged.front().start.row = m_start;
-			merged.front().start.strong = true;
+			merged.front().start.strong = false;
 		}
-		if (m_end - snippetSnapRange <= merged.back().end.row)
+		if ((m_end - snippetSnapRange <= merged.back().end.row) && 
+			(merged.back().end.row < m_end))
 		{
 			merged.back().end.row = m_end;
-			merged.back().end.strong = true;
+			merged.back().end.strong = false;
 		}
 	}
 	return merged;
 }
+
+SnippetMerger::Range SnippetMerger::getExpandedRegardingAtomicRanges(
+	Range range, const int snippetExpandRange, const std::vector<Range>& atomicRanges
+) const
+{
+	const int rangeStartThreshold = range.start.row - snippetExpandRange;
+	const int rangeEndThreshold = range.end.row + snippetExpandRange;
+	for (size_t i = 0; i < atomicRanges.size(); i++)
+	{
+		if ((!range.start.strong) &&
+			(atomicRanges[i].end.row >= rangeStartThreshold) &&
+			(atomicRanges[i].start.row < rangeStartThreshold))
+		{
+			range.start.row = std::min(range.start.row, atomicRanges[i].start.row);
+			range.start.strong = true;
+		}
+		if ((!range.end.strong) &&
+			(atomicRanges[i].start.row <= rangeEndThreshold) &&
+			(atomicRanges[i].end.row > rangeEndThreshold))
+		{
+			range.end.row = std::max(range.end.row, atomicRanges[i].end.row);
+			range.end.strong = true;
+		}
+	}
+	return range;
+}
+

src/lib/component/controller/helper/SnippetMerger.h

@@ -16,6 +16,26 @@ public:
 	};
 	struct Range
 	{
+		template <template<class, class> class ContainerType>
+		static ContainerType<Range, std::allocator<Range>> mergeAdjacent(ContainerType<Range, std::allocator<Range>> ranges, int rowDifference = 1)
+		{
+			for (size_t i = 0; i < ranges.size() - 1; i++)
+			{
+				const Range first = ranges[i];
+				const Range second = ranges[i + 1];
+				if (first.end.row + rowDifference >= second.start.row)
+				{
+					ranges.erase(ranges.begin() + i, ranges.begin() + i + 2);
+					ranges.insert(ranges.begin() + i, Range(
+						first.start.row < second.start.row ? first.start : second.start,
+						first.end.row > second.end.row ? first.end : second.end
+						));
+					i--;
+				}
+			}
+			return ranges;
+		}
+
 		Range (Border start, Border end): start(start), end(end) {}
 		Border start;
 		Border end;
@@ -23,9 +43,11 @@ public:
 
 	SnippetMerger(int startRow, int endRow);
 	void addChild(std::shared_ptr<SnippetMerger> child);
-	std::deque<Range> merge() const;
+	std::deque<Range> merge(std::vector<SnippetMerger::Range> atomicRanges) const;
 
 private:
+	Range getExpandedRegardingAtomicRanges(Range range, const int snippetExpandRange, const std::vector<Range>& atomicRanges) const;
+
 	const int m_start;
 	const int m_end;
 	std::vector<std::shared_ptr<SnippetMerger>> m_children;

src/lib/data/SqliteStorage.cpp

@@ -164,6 +164,18 @@ Id SqliteStorage::addSignature(Id nodeId, const std::string& signature)
 	return m_database.lastRowId();
 }
 
+Id SqliteStorage::addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol)
+{
+	m_database.execDML((
+		"INSERT INTO comment_location(id, file_node_id, start_line, start_column, end_line, end_column) "
+		"VALUES(NULL, "  + std::to_string(fileNodeId) + ", "
+		+ std::to_string(startLine) + ", " + std::to_string(startCol) + ", "
+		+ std::to_string(endLine) + ", " + std::to_string(endCol) + ");"
+	).c_str());
+
+	return m_database.lastRowId();
+}
+
 Id SqliteStorage::addError(const std::string& message, const std::string& filePath, uint lineNumber, uint columnNumber)
 {
 	std::string sanitizedMessage = utility::replace(message, "'", "''");
@@ -727,6 +739,36 @@ Id SqliteStorage::getNodeIdBySignature(const std::string& signature) const
 	return 0;
 }
 
+std::vector<StorageCommentLocation> SqliteStorage::getCommentLocationsInFile(const FilePath& filePath) const
+{
+	Id fileNodeId = getFileByPath(filePath.str()).id;
+	CppSQLite3Query q = m_database.execQuery((
+		"SELECT id, file_node_id, start_line, start_column, end_line, end_column FROM comment_location "
+		"WHERE file_node_id == " + std::to_string(fileNodeId) + ";"
+	).c_str());
+
+	std::vector<StorageCommentLocation> commentLocations;
+	while (!q.eof())
+	{
+		const Id id = q.getIntField(0, 0);
+		const Id fileNodeId = q.getIntField(1, 0);
+		const int startLineNumber = q.getIntField(2, -1);
+		const int startColNumber = q.getIntField(3, -1);
+		const int endLineNumber = q.getIntField(4, -1);
+		const int endColNumber = q.getIntField(5, -1);
+
+		if (id != 0 && fileNodeId != 0 && startLineNumber != -1 && startColNumber != -1 && endLineNumber != -1 && endColNumber != -1)
+		{
+			commentLocations.push_back(StorageCommentLocation(
+				id, fileNodeId, startLineNumber, startColNumber, endLineNumber, endColNumber
+			));
+		}
+		q.nextRow();
+	}
+
+	return commentLocations;
+}
+
 std::vector<StorageError> SqliteStorage::getAllErrors() const
 {
 	CppSQLite3Query q = m_database.execQuery(
@@ -777,6 +819,7 @@ int SqliteStorage::getSourceLocationCount() const
 void SqliteStorage::clearTables()
 {
 	m_database.execDML("DROP TABLE IF EXISTS main.error;");
+	m_database.execDML("DROP TABLE IF EXISTS main.comment_location;");
 	m_database.execDML("DROP TABLE IF EXISTS main.function_signature;");
 	m_database.execDML("DROP TABLE IF EXISTS main.component_access;");
 	m_database.execDML("DROP TABLE IF EXISTS main.source_location;");
@@ -878,6 +921,18 @@ void SqliteStorage::setupTables()
 			"FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);"
 	);
 
+	m_database.execDML(
+		"CREATE TABLE IF NOT EXISTS comment_location("
+			"id INTEGER NOT NULL, "
+			"file_node_id INTEGER, "
+			"start_line INTEGER, "
+			"start_column INTEGER, "
+			"end_line INTEGER, "
+			"end_column INTEGER, "
+			"PRIMARY KEY(id), "
+			"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
+	);
+
 	m_database.execDML(
 		"CREATE TABLE IF NOT EXISTS error("
 			"id INTEGER NOT NULL, "

src/lib/data/SqliteStorage.h

@@ -42,6 +42,7 @@ public:
 	Id addComponentAccess(Id memberEdgeId, int type);
 	Id addSignature(Id nodeId, const std::string& signature);
 
+	Id addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol);
 	Id addError(const std::string& message, const std::string& filePath, uint lineNumber, uint columnNumber);
 
 	void removeElement(Id id);
@@ -51,6 +52,7 @@ public:
 	void removeFiles(const std::vector<Id>& fileIds);
 	void removeUnusedNameHierarchyElements();
 
+	void removeCommentLocationsInFiles(const std::vector<FilePath>& filePaths);
 	void removeErrorsInFiles(const std::vector<FilePath>& filePaths);
 
 	StorageNode getFirstNode() const;
@@ -105,6 +107,7 @@ public:
 	std::vector<StorageComponentAccess> getComponentAccessByMemberEdgeIds(const std::vector<Id>& memberEdgeIds) const;
 	Id getNodeIdBySignature(const std::string& signature) const;
 
+	std::vector<StorageCommentLocation> getCommentLocationsInFile(const FilePath& filePath) const;
 	std::vector<StorageError> getAllErrors() const;
 
 	int getNodeCount() const;

src/lib/data/Storage.cpp

@@ -713,6 +713,22 @@ Id Storage::onMacroExpandParsed(const ParseLocation &location, const NameHierarc
 	return edgeId;
 }
 
+Id Storage::onCommentParsed(const ParseLocation& location)
+{
+	log("comment", "no name", location);
+
+	Id fileNodeId = m_sqliteStorage.getFileByPath(location.filePath.str()).id;
+	Id commentId = m_sqliteStorage.addCommentLocation(
+		fileNodeId,
+		location.startLineNumber,
+		location.startColumnNumber,
+		location.endLineNumber,
+		location.endColumnNumber
+	);
+
+	return commentId;
+}
+
 Id Storage::getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const
 {
 	Id currentId = 0;
@@ -1167,7 +1183,7 @@ std::shared_ptr<TokenLocationFile> Storage::getTokenLocationOfParentScope(const
 	const TokenLocation* parent = child;
 	const FilePath filePath = child->getFilePath();
 
-	std::shared_ptr<TokenLocationFile> locationFile = m_sqliteStorage.getTokenLocationsForFile(filePath);
+	std::shared_ptr<TokenLocationFile> locationFile = m_sqliteStorage.getTokenLocationsForFile(filePath); // TODO: sqlite should not know TokenLocationFile!
 	locationFile->forEachStartTokenLocation(
 		[&](TokenLocation* tokenLocation) -> void
 		{
@@ -1197,6 +1213,26 @@ std::shared_ptr<TokenLocationFile> Storage::getTokenLocationOfParentScope(const
 	return file;
 }
 
+std::shared_ptr<TokenLocationFile> Storage::getCommentLocationsInFile(const FilePath& filePath) const
+{
+	std::shared_ptr<TokenLocationFile> file = std::make_shared<TokenLocationFile>(filePath);
+
+	std::vector<StorageCommentLocation> storageLocations = m_sqliteStorage.getCommentLocationsInFile(filePath);
+	for (size_t i = 0; i < storageLocations.size(); i++)
+	{
+		TokenLocation* loc = file->addTokenLocation(
+			storageLocations[i].id, 
+			0, // comment token location has no element.
+			storageLocations[i].startLine, 
+			storageLocations[i].startCol, 
+			storageLocations[i].endLine, 
+			storageLocations[i].endCol
+		);
+	}
+
+	return file;
+}
+
 std::shared_ptr<TextAccess> Storage::getFileContent(const FilePath& filePath) const
 {
 	return m_sqliteStorage.getFileContentByPath(filePath.str());

src/lib/data/Storage.h

@@ -127,6 +127,8 @@ public:
 		const ParseLocation& location, const NameHierarchy& macroNameHierarchy, const ParseLocation& scopeLocation);
 	virtual Id onMacroExpandParsed(const ParseLocation& location, const NameHierarchy& macroNameHierarchy);
 
+	virtual Id onCommentParsed(const ParseLocation& location);
+
 	// StorageAccess implementation
 	virtual Id getIdForNodeWithNameHierarchy(const NameHierarchy& nameHierarchy) const;
 	virtual Id getIdForEdge(
@@ -160,6 +162,7 @@ public:
 
 	virtual TokenLocationCollection getErrorTokenLocations(std::vector<std::string>* errorMessages) const;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationOfParentScope(const TokenLocation* child) const;
+	virtual std::shared_ptr<TokenLocationFile> getCommentLocationsInFile(const FilePath& filePath) const;
 
 	virtual std::shared_ptr<TextAccess> getFileContent(const FilePath& filePath) const;
 	virtual TimePoint getFileModificationTime(const FilePath& filePath) const;

src/lib/data/StorageTypes.h

@@ -8,7 +8,10 @@
 struct StorageEdge
 {
 	StorageEdge(Id id, int type, Id sourceNodeId, Id targetNodeId)
-		: id(id), type(type), sourceNodeId(sourceNodeId), targetNodeId(targetNodeId)
+		: id(id)
+		, type(type)
+		, sourceNodeId(sourceNodeId)
+		, targetNodeId(targetNodeId)
 	{}
 
 	Id id;
@@ -20,7 +23,10 @@ struct StorageEdge
 struct StorageNode
 {
 	StorageNode(Id id, int type, Id nameId, bool defined)
-		: id(id), type(type), nameId(nameId), defined(defined)
+		: id(id)
+		, type(type)
+		, nameId(nameId)
+		, defined(defined)
 	{}
 
 	Id id;
@@ -32,7 +38,10 @@ struct StorageNode
 struct StorageFile
 {
 	StorageFile(Id id, Id nameId, const std::string& filePath, const std::string& modificationTime)
-		: id(id), nameId(nameId), filePath(filePath), modificationTime(modificationTime)
+		: id(id)
+		, nameId(nameId)
+		, filePath(filePath)
+		, modificationTime(modificationTime)
 	{}
 
 	Id id;
@@ -55,8 +64,14 @@ struct StorageNameHierarchyElement
 struct StorageSourceLocation
 {
 	StorageSourceLocation(Id id, Id elementId, Id fileNodeId, int startLine, int startCol, int endLine, int endCol, bool isScope)
-		: id(id), elementId(elementId), fileNodeId(fileNodeId)
-		, startLine(startLine), startCol(startCol), endLine(endLine), endCol(endCol), isScope(isScope)
+		: id(id)
+		, elementId(elementId)
+		, fileNodeId(fileNodeId)
+		, startLine(startLine)
+		, startCol(startCol)
+		, endLine(endLine)
+		, endCol(endCol)
+		, isScope(isScope)
 	{}
 
 	Id id;
@@ -80,6 +95,25 @@ struct StorageComponentAccess
 	int type;
 };
 
+struct StorageCommentLocation
+{
+	StorageCommentLocation(Id id, Id fileNodeId, int startLine, int startCol, int endLine, int endCol)
+		: id(id)
+		, fileNodeId(fileNodeId)
+		, startLine(startLine)
+		, startCol(startCol)
+		, endLine(endLine)
+		, endCol(endCol)
+	{}
+
+	Id id;
+	Id fileNodeId;
+	int startLine;
+	int startCol;
+	int endLine;
+	int endCol;
+};
+
 struct StorageError
 {
 	StorageError(const std::string& message, const std::string& filePath, uint lineNumber, uint columnNumber)

src/lib/data/access/StorageAccess.h

@@ -55,6 +55,7 @@ public:
 
 	virtual TokenLocationCollection getErrorTokenLocations(std::vector<std::string>* errorMessages) const = 0;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationOfParentScope(const TokenLocation* child) const = 0;
+	virtual std::shared_ptr<TokenLocationFile> getCommentLocationsInFile(const FilePath& filePath) const = 0;
 
 	virtual std::shared_ptr<TextAccess> getFileContent(const FilePath& filePath) const = 0;
 	virtual TimePoint getFileModificationTime(const FilePath& filePath) const = 0;

src/lib/data/access/StorageAccessProxy.cpp

@@ -235,6 +235,16 @@ std::shared_ptr<TokenLocationFile> StorageAccessProxy::getTokenLocationOfParentS
 	return std::make_shared<TokenLocationFile>("");
 }
 
+std::shared_ptr<TokenLocationFile> StorageAccessProxy::getCommentLocationsInFile(const FilePath& filePath) const
+{
+	if (hasSubject())
+	{
+		return m_subject->getCommentLocationsInFile(filePath);
+	}
+
+	return std::make_shared<TokenLocationFile>("");
+}
+
 std::shared_ptr<TextAccess> StorageAccessProxy::getFileContent(const FilePath& filePath) const
 {
 	if (hasSubject())

src/lib/data/access/StorageAccessProxy.h

@@ -45,6 +45,7 @@ public:
 
 	virtual TokenLocationCollection getErrorTokenLocations(std::vector<std::string>* errorMessages) const;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationOfParentScope(const TokenLocation* child) const;
+	virtual std::shared_ptr<TokenLocationFile> getCommentLocationsInFile(const FilePath& filePath) const;
 
 	virtual std::shared_ptr<TextAccess> getFileContent(const FilePath& filePath) const;
 	virtual TimePoint getFileModificationTime(const FilePath& filePath) const;

src/lib/data/parser/ParserClient.h

@@ -145,6 +145,8 @@ public:
 		const ParseLocation& location, const NameHierarchy& macroNameHierarchy, const ParseLocation& scopeLocation) = 0;
 	virtual Id onMacroExpandParsed(
 		const ParseLocation& location, const NameHierarchy& macroNameHierarchy) = 0;
+
+	virtual Id onCommentParsed(const ParseLocation& location) = 0;
 };
 
 #endif // PARSER_CLIENT_H

src/lib/data/parser/cxx/ASTAction.cpp

@@ -2,11 +2,13 @@
 
 #include "clang/Lex/Preprocessor.h"
 
+#include "data/parser/cxx/CommentHandler.h"
 #include "data/parser/cxx/PreprocessorCallbacks.h"
 
 ASTAction::ASTAction(ParserClient* client, FileRegister* fileRegister)
 	: m_client(client)
 	, m_fileRegister(fileRegister)
+	, m_commentHandler(client)
 {
 }
 
@@ -24,7 +26,7 @@ bool ASTAction::BeginSourceFileAction(clang::CompilerInstance& compiler, llvm::S
 	clang::Preprocessor& preprocessor = compiler.getPreprocessor();
 	preprocessor.addPPCallbacks(
 		llvm::make_unique<PreprocessorCallbacks>(compiler.getSourceManager(), m_client, m_fileRegister));
-
+	preprocessor.addCommentHandler(&m_commentHandler);
 	return true;
 }
 

src/lib/data/parser/cxx/ASTAction.h

@@ -5,6 +5,7 @@
 #include "clang/Frontend/FrontendAction.h"
 
 #include "data/parser/cxx/ASTConsumer.h"
+#include "data/parser/cxx/CommentHandler.h"
 #include "utility/file/FileRegister.h"
 
 class ASTAction : public clang::ASTFrontendAction
@@ -22,6 +23,8 @@ protected:
 private:
 	ParserClient* m_client;
 	FileRegister* m_fileRegister;
+	CommentHandler m_commentHandler;
+
 };
 
 #endif // AST_ACTION_H

src/lib/data/parser/cxx/CommentHandler.cpp

@@ -0,0 +1,28 @@
+#include "data/parser/cxx/CommentHandler.h"
+
+#include "data/parser/ParserClient.h"
+CommentHandler::CommentHandler(ParserClient* client)
+	: m_client(client)
+{
+}
+
+CommentHandler::~CommentHandler()
+{
+}
+
+bool CommentHandler::HandleComment(clang::Preprocessor& preprocessor, clang::SourceRange sourceRange)
+{
+	clang::SourceManager& sourceManager = preprocessor.getSourceManager();
+	const clang::PresumedLoc& presumedBegin = sourceManager.getPresumedLoc(sourceRange.getBegin(), false);
+	const clang::PresumedLoc& presumedEnd = sourceManager.getPresumedLoc(sourceRange.getEnd(), false);
+	
+	m_client->onCommentParsed(ParseLocation(
+		presumedBegin.getFilename(),
+		presumedBegin.getLine(),
+		presumedBegin.getColumn(),
+		presumedEnd.getLine(),
+		presumedEnd.getColumn()
+	));
+
+	return false;
+}

src/lib/data/parser/cxx/CommentHandler.h

@@ -0,0 +1,21 @@
+#ifndef COMMENT_HANDLER_H
+#define COMMENT_HANDLER_H
+
+#include "clang/Lex/Preprocessor.h"
+
+class ParserClient;
+
+class CommentHandler
+	: public clang::CommentHandler
+{
+public:
+	CommentHandler(ParserClient* client);
+	virtual ~CommentHandler();
+
+	virtual bool HandleComment(clang::Preprocessor& preprocessor, clang::SourceRange sourceRange);
+
+private:
+	ParserClient* m_client;
+};
+
+#endif // COMMENT_HANDLER_H