logic: Fulltextsearch

* fulltextsearch enabled in sqlite
* fulltextsearch start search with @
* codeview performance improved
* strip sRGB profile from png files to get rid of libpng warning at runtime

src/lib/data/SqliteStorage.cpp

@@ -3,6 +3,7 @@
 #include "data/graph/Node.h"
 #include "data/location/TokenLocation.h"
 #include "data/DefinitionType.h"
+#include "data/parser/ParseLocation.h"
 #include "data/SqliteIndex.h"
 #include "utility/logging/logging.h"
 #include "utility/text/TextAccess.h"
@@ -207,7 +208,8 @@ Id SqliteStorage::addError(const std::string& message, bool fatal, const std::st
 
 	m_database.execDML((
 		"INSERT INTO error(message, fatal, file_path, line_number, column_number) "
-		"VALUES ('" + sanitizedMessage + "', " + std::to_string(fatal) + ", '" + filePath + "', " + std::to_string(lineNumber) + ", " + std::to_string(columnNumber) + ");"
+		"VALUES ('" + sanitizedMessage + "', " + std::to_string(fatal) + ", '" + filePath +
+		"', " + std::to_string(lineNumber) + ", " + std::to_string(columnNumber) + ");"
 	).c_str());
 
 	return m_database.lastRowId();
@@ -416,6 +418,92 @@ std::vector<StorageEdge> SqliteStorage::getEdgesByTargetType(Id targetId, int ty
 	return edges;
 }
 
+void SqliteStorage::optimizeFTSTable() const
+{
+	try
+	{
+		CppSQLite3Query q = m_database.execQuery("INSERT INTO file(file) VALUES('optimize');");
+	}
+	catch(CppSQLite3Exception e)
+	{
+		LOG_ERROR(e.errorMessage());
+	}
+}
+
+std::vector<ParseLocation> SqliteStorage::getFullTextSearch(const std::string& searchTerm) const
+{
+	std::vector<ParseLocation> matches;
+	try
+	{
+		CppSQLite3Query q = m_database.execQuery((
+					"SELECT id,offsets(file) FROM file WHERE content MATCH '\"*" + searchTerm + "*\"'"
+		).c_str());
+
+		while(!q.eof())
+		{
+			const Id fileId = q.getIntField(0,0);
+
+			// convert the string "0 2 0 2" to int vector
+			std::stringstream temp_results(q.getStringField(1,0));
+			std::vector<int> results((std::istream_iterator<int>(temp_results)),std::istream_iterator<int>());
+
+			ParseLocation location;
+			location.filePath = getFileById(fileId).filePath;
+			std::shared_ptr<TextAccess> file = getFileContentByPath(location.filePath.str());
+
+			int charsInPreviousLines = 0;
+			int lineNumber = 1;
+			std::string line;
+			// results
+			// i   ... col
+			// i+1 ... term
+			// i+2 ... offset
+			// i+3 ... length
+			line = file->getLine(lineNumber);
+			for (size_t i = 0; i < results.size() ; i+=4)
+			{
+				while( ((charsInPreviousLines + (int)line.length()) < results[i+2]) && results[i+1] == 0 )
+				{
+					lineNumber++;
+					charsInPreviousLines += line.length();
+					line = file->getLine(lineNumber);
+				}
+
+				//only set start if its the first term of the match
+				if ( results[i+1] == 0 )
+				{
+					location.startLineNumber = lineNumber;
+					// +1 to be consistent with the rest of the codebase
+					location.startColumnNumber = results[i+2] - charsInPreviousLines + 1;
+				}
+
+				while( (charsInPreviousLines + (int)line.length()) < (results[i+2] + results[i+3]) )
+				{
+					lineNumber++;
+					charsInPreviousLines += line.length();
+					line = file->getLine(lineNumber);
+				}
+
+				location.endLineNumber = lineNumber;
+				location.endColumnNumber = results[i+2] + results[i+3] - charsInPreviousLines;
+
+				// add match if the next term is the first term of a match or its the last term in the file
+				if ( (i+4 < results.size() && results[i+5] == 0) || i+4 >= results.size() )
+				{
+					matches.push_back(location);
+				}
+			}
+
+			q.nextRow();
+		}
+	}
+	catch(CppSQLite3Exception e)
+	{
+		LOG_ERROR(e.errorMessage());
+	}
+	return matches;
+}
+
 StorageNode SqliteStorage::getNodeById(Id id) const
 {
 	if (id != 0)
@@ -475,6 +563,23 @@ std::vector<StorageFile> SqliteStorage::getAllFiles() const
 	return getAllFiles("");
 }
 
+std::vector<Id> SqliteStorage::getAllFileIds() const
+{
+	std::vector<Id> ids;
+
+	CppSQLite3Query q = m_database.execQuery(
+		"SELECT id FROM file;"
+	);
+
+	while (!q.eof())
+	{
+		ids.push_back(q.getIntField(0,0));
+		q.nextRow();
+	}
+
+	return ids;
+}
+
 std::shared_ptr<TextAccess> SqliteStorage::getFileContentByPath(const std::string& filePath) const
 {
 	CppSQLite3Query q = m_database.execQuery((
@@ -786,16 +891,24 @@ void SqliteStorage::setupTables()
 		"CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);"
 	);
 
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS file("
-			"id INTEGER NOT NULL, "
-			"path TEXT, "
-			"modification_time TEXT, "
-			"content TEXT, "
-			"loc INTEGER, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);"
-	);
+	try
+	{
+		m_database.execDML(
+			"CREATE VIRTUAL TABLE IF NOT EXISTS file USING fts4("
+			//"CREATE TABLE IF NOT EXISTS file ("
+				"id INTEGER NOT NULL, "
+				"path TEXT, "
+				"modification_time TEXT, "
+				"content TEXT, "
+				"loc INTEGER, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);"
+		);
+	}
+	catch (CppSQLite3Exception& e)
+	{
+		std::cerr << e.errorCode() << ":" << e.errorMessage() << std::endl;
+	}
 
 	m_database.execDML(
 		"CREATE TABLE IF NOT EXISTS local_symbol("

src/lib/data/SqliteStorage.h

@@ -9,13 +9,14 @@
 
 #include "utility/file/FilePath.h"
 #include "utility/types.h"
-#include "data/name/NameHierarchy.h"
 #include "data/location/TokenLocationFile.h"
 #include "data/location/TokenLocationCollection.h"
+#include "data/name/NameHierarchy.h"
 #include "data/StorageTypes.h"
 
 class TextAccess;
 class Version;
+struct ParseLocation;
 
 class SqliteStorage
 {
@@ -83,6 +84,7 @@ public:
 	StorageFile getFileById(const Id id) const;
 	StorageFile getFileByPath(const FilePath& filePath) const;
 
+	std::vector<Id> getAllFileIds() const;
 	std::vector<StorageFile> getFilesByPaths(const std::vector<FilePath>& filePaths) const;
 	std::vector<StorageFile> getAllFiles() const;
 	std::shared_ptr<TextAccess> getFileContentByPath(const std::string& filePath) const;
@@ -100,6 +102,9 @@ public:
 	StorageComponentAccess getComponentAccessByMemberEdgeId(Id memberEdgeId) const;
 	std::vector<StorageComponentAccess> getComponentAccessByMemberEdgeIds(const std::vector<Id>& memberEdgeIds) const;
 
+	std::vector<ParseLocation> getFullTextSearch(const std::string& searchTerm) const;
+	void optimizeFTSTable() const;
+
 	std::vector<StorageCommentLocation> getCommentLocationsInFile(const FilePath& filePath) const;
 	std::vector<StorageError> getAllErrors() const;
 	std::vector<StorageError> getFatalErrors() const;

src/lib/data/Storage.cpp

@@ -172,6 +172,7 @@ void Storage::finishParsing()
 {
 	buildSearchIndex();
 	buildHierarchyCache();
+	optimizeFTSTable();
 }
 
 void Storage::injectData(std::shared_ptr<IntermediateStorage> injectedStorage)
@@ -217,6 +218,29 @@ Node::NodeType Storage::getNodeTypeForNodeWithId(Id nodeId) const
 	return Node::intToType(m_sqliteStorage.getNodeById(nodeId).type);
 }
 
+std::shared_ptr<TokenLocationCollection> Storage::getFullTextSearchLocations(const std::string& searchTerm) const
+{
+	std::shared_ptr<TokenLocationCollection> collection = std::make_shared<TokenLocationCollection>();
+
+	std::vector<ParseLocation> parseLocations = m_sqliteStorage.getFullTextSearch(searchTerm);
+	size_t i = 0;
+	for(ParseLocation location : parseLocations)
+	{
+		collection->addTokenLocation(
+				i,
+				0,
+				location.filePath,
+				location.startLineNumber,
+				location.startColumnNumber,
+				location.endLineNumber,
+				location.endColumnNumber
+		)->setType(LOCATION_FULLTEXTSEARCH_MATCH);
+		i++;
+	}
+
+	return collection;
+}
+
 std::vector<SearchMatch> Storage::getAutocompletionMatches(const std::string& query) const
 {
 	std::vector<SearchResult> commandResults = m_commandIndex.search(query, 0);
@@ -316,7 +340,8 @@ std::vector<SearchMatch> Storage::getAutocompletionMatches(const std::string& qu
 			match.nodeType = Node::intToType(firstNode->type);
 			match.typeName = Node::getTypeString(match.nodeType);
 
-			if (intToDefinitionType(firstNode->definitionType) == DEFINITION_NONE && match.nodeType != Node::NODE_UNDEFINED)
+			if (intToDefinitionType(firstNode->definitionType) == DEFINITION_NONE
+					&& match.nodeType != Node::NODE_UNDEFINED)
 			{
 				match.typeName = "undefined " + match.typeName;
 			}
@@ -401,6 +426,7 @@ std::shared_ptr<Graph> Storage::getGraphForActiveTokenIds(const std::vector<Id>&
 	std::vector<Id> edgeIds;
 	bool addAggregations = false;
 
+	//m_sqliteStorage.getFullTextSearch("const int");
 	if (tokenIds.size() == 1)
 	{
 		const Id elementId = tokenIds[0];
@@ -575,7 +601,9 @@ std::vector<Id> Storage::getTokenIdsForMatches(const std::vector<SearchMatch>& m
 	{
 		for (size_t i = 0; i < match.nameHierarchies.size(); i++)
 		{
-			idSet.insert(m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(match.nameHierarchies[i])).id);
+			idSet.insert(
+				m_sqliteStorage.getNodeBySerializedName(NameHierarchy::serialize(match.nameHierarchies[i])).id
+			);
 		}
 	}
 
@@ -638,9 +666,10 @@ std::shared_ptr<TokenLocationCollection> Storage::getTokenLocationsForTokenIds(c
 
 	std::vector<Id> fileIds;
 	std::vector<Id> nonFileIds;
+	std::vector<Id> allFileIds = m_sqliteStorage.getAllFileIds();
 	for (size_t i = 0; i < tokenIds.size(); i++)
 	{
-		if (m_sqliteStorage.isFile(tokenIds[i]))
+		if (std::find(allFileIds.begin(), allFileIds.end(),tokenIds[i]) != allFileIds.end())
 		{
 			fileIds.push_back(tokenIds[i]);
 		}
@@ -653,7 +682,9 @@ std::shared_ptr<TokenLocationCollection> Storage::getTokenLocationsForTokenIds(c
 	for (Id fileId: fileIds)
 	{
 		StorageFile storageFile = m_sqliteStorage.getFileById(fileId);
-		collection->addTokenLocationFileAsPlainCopy(m_sqliteStorage.getTokenLocationsForFile(storageFile.filePath).get());
+		collection->addTokenLocationFileAsPlainCopy(
+				m_sqliteStorage.getTokenLocationsForFile(storageFile.filePath).get()
+		);
 	}
 
 	Cache<Id, std::string> filePathCache(
@@ -688,7 +719,9 @@ std::shared_ptr<TokenLocationCollection> Storage::getTokenLocationsForTokenIds(c
 	return collection;
 }
 
-std::shared_ptr<TokenLocationCollection> Storage::getTokenLocationsForLocationIds(const std::vector<Id>& locationIds) const
+std::shared_ptr<TokenLocationCollection> Storage::getTokenLocationsForLocationIds(
+		const std::vector<Id>& locationIds
+) const
 {
 	std::shared_ptr<TokenLocationCollection> collection = std::make_shared<TokenLocationCollection>();
 
@@ -925,7 +958,11 @@ void Storage::addEdgesToGraph(const std::vector<Id>& edgeIds, Graph* graph) cons
 	}
 }
 
-void Storage::addNodesWithChildrenAndEdgesToGraph(const std::vector<Id>& nodeIds, const std::vector<Id>& edgeIds, Graph* graph) const
+void Storage::addNodesWithChildrenAndEdgesToGraph(
+	const std::vector<Id>& nodeIds, 
+	const std::vector<Id>& edgeIds, 
+	Graph* graph
+) const
 {
 	std::set<Id> parentNodeIds;
 
@@ -1099,3 +1136,8 @@ void Storage::buildHierarchyCache()
 		m_hierarchyCache.createConnection(edge.id, edge.sourceNodeId, edge.targetNodeId, isVisible);
 	}
 }
+
+void Storage::optimizeFTSTable()
+{
+	m_sqliteStorage.optimizeFTSTable();
+}

src/lib/data/Storage.h

@@ -10,6 +10,7 @@
 #include "data/graph/token_component/TokenComponentAccess.h"
 #include "data/location/TokenLocationCollection.h"
 #include "data/parser/ParserClient.h"
+#include "data/parser/ParseLocation.h"
 #include "data/search/SearchIndex.h"
 #include "data/HierarchyCache.h"
 #include "data/SqliteStorage.h"
@@ -54,6 +55,7 @@ public:
 	virtual NameHierarchy getNameHierarchyForNodeWithId(Id nodeId) const;
 	virtual Node::NodeType getNodeTypeForNodeWithId(Id nodeId) const;
 
+	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(const std::string& searchTerm) const;
 	virtual std::vector<SearchMatch> getAutocompletionMatches(const std::string& query) const;
 	virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& elementIds) const;
 
@@ -69,8 +71,12 @@ public:
 	virtual Id getTokenIdForFileNode(const FilePath& filePath) const;
 	virtual std::vector<Id> getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const;
 
-	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForTokenIds(const std::vector<Id>& tokenIds) const;
-	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForLocationIds(const std::vector<Id>& locationIds) const;
+	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForTokenIds(
+			const std::vector<Id>& tokenIds
+	) const;
+	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForLocationIds(
+			const std::vector<Id>& locationIds
+	) const;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationsForFile(const std::string& filePath) const;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationsForLinesInFile(
 		const std::string& filePath, uint firstLineNumber, uint lastLineNumber
@@ -96,13 +102,17 @@ private:
 
 	void addNodesToGraph(const std::vector<Id>& nodeIds, Graph* graph) const;
 	void addEdgesToGraph(const std::vector<Id>& edgeIds, Graph* graph) const;
-	void addNodesWithChildrenAndEdgesToGraph(const std::vector<Id>& nodeIds, const std::vector<Id>& edgeIds, Graph* graph) const;
+	void addNodesWithChildrenAndEdgesToGraph(
+			const std::vector<Id>& nodeIds,
+			const std::vector<Id>& edgeIds, Graph* graph
+	) const;
 
 	void addAggregationEdgesToGraph(const Id nodeId, Graph* graph) const;
 	void addComponentAccessToGraph(Graph* graph) const;
 
 	void buildSearchIndex();
 	void buildHierarchyCache();
+	void optimizeFTSTable();
 
 	void log(std::string type, std::string str, const ParseLocation& location) const;
 

src/lib/data/access/StorageAccess.h

@@ -9,6 +9,7 @@
 #include "utility/file/FileInfo.h"
 #include "utility/file/FilePath.h"
 
+#include "data/parser/ParseLocation.h"
 #include "data/graph/Node.h"
 #include "data/search/SearchMatch.h"
 #include "data/ErrorCountInfo.h"
@@ -36,6 +37,8 @@ public:
 	virtual NameHierarchy getNameHierarchyForNodeWithId(Id id) const = 0;
 	virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const = 0;
 
+	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(
+			const std::string& searchTerm) const = 0;
 	virtual std::vector<SearchMatch> getAutocompletionMatches(const std::string& query) const = 0;
 	virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const = 0;
 
@@ -51,8 +54,10 @@ public:
 	virtual Id getTokenIdForFileNode(const FilePath& filePath) const = 0;
 	virtual std::vector<Id> getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const = 0;
 
-	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForTokenIds(const std::vector<Id>& tokenIds) const = 0;
-	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForLocationIds(const std::vector<Id>& locationIds) const = 0;
+	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForTokenIds(
+			const std::vector<Id>& tokenIds) const = 0;
+	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForLocationIds(
+			const std::vector<Id>& locationIds) const = 0;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationsForFile(const std::string& filePath) const = 0;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationsForLinesInFile(
 		const std::string& filePath, uint firstLineNumber, uint lastLineNumber) const = 0;

src/lib/data/access/StorageAccessProxy.cpp

@@ -93,6 +93,17 @@ std::vector<SearchMatch> StorageAccessProxy::getAutocompletionMatches(const std:
 	return std::vector<SearchMatch>();
 }
 
+std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getFullTextSearchLocations(
+		const std::string &searchTerm) const
+{
+	if (hasSubject())
+	{
+		return m_subject->getFullTextSearchLocations(searchTerm);
+	}
+
+	return std::make_shared<TokenLocationCollection>();
+}
+
 std::vector<SearchMatch> StorageAccessProxy::getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const
 {
 	if (hasSubject())
@@ -183,7 +194,8 @@ std::vector<Id> StorageAccessProxy::getTokenIdsForAggregationEdge(Id sourceId, I
 	return std::vector<Id>();
 }
 
-std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getTokenLocationsForTokenIds(const std::vector<Id>& tokenIds) const
+std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getTokenLocationsForTokenIds(
+		const std::vector<Id>& tokenIds) const
 {
 	if (hasSubject())
 	{
@@ -193,7 +205,8 @@ std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getTokenLocationsFo
 	return std::make_shared<TokenLocationCollection>();
 }
 
-std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getTokenLocationsForLocationIds(const std::vector<Id>& locationIds) const
+std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getTokenLocationsForLocationIds(
+		const std::vector<Id>& locationIds) const
 {
 	if (hasSubject())
 	{

src/lib/data/access/StorageAccessProxy.h

@@ -22,6 +22,7 @@ public:
 	virtual NameHierarchy getNameHierarchyForNodeWithId(Id id) const;
 	virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const;
 
+	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(const std::string& searchTerm) const;
 	virtual std::vector<SearchMatch> getAutocompletionMatches(const std::string& query) const;
 	virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const;
 
@@ -37,8 +38,12 @@ public:
 	virtual Id getTokenIdForFileNode(const FilePath& filePath) const;
 	virtual std::vector<Id> getTokenIdsForAggregationEdge(Id sourceId, Id targetId) const;
 
-	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForTokenIds(const std::vector<Id>& tokenIds) const;
-	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForLocationIds(const std::vector<Id>& locationIds) const;
+	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForTokenIds(
+			const std::vector<Id>& tokenIds
+	) const;
+	virtual std::shared_ptr<TokenLocationCollection> getTokenLocationsForLocationIds(
+			const std::vector<Id>& locationIds
+	) const;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationsForFile(const std::string& filePath) const;
 	virtual std::shared_ptr<TokenLocationFile> getTokenLocationsForLinesInFile(
 		const std::string& filePath, uint firstLineNumber, uint lastLineNumber

src/lib/data/location/LocationType.cpp

@@ -10,6 +10,8 @@ int locationTypeToInt(LocationType type)
 		return 1;
 	case LOCATION_LOCAL_SYMBOL:
 		return 2;
+	case LOCATION_FULLTEXTSEARCH_MATCH:
+		return 3;
 	}
 }
 
@@ -23,6 +25,8 @@ LocationType intToLocationType(int value)
 		return LOCATION_SCOPE;
 	case 2:
 		return LOCATION_LOCAL_SYMBOL;
+	case 3:
+		return LOCATION_FULLTEXTSEARCH_MATCH;
 	}
 	return LOCATION_TOKEN;
 }

src/lib/data/location/LocationType.h

@@ -5,7 +5,8 @@ enum LocationType
 {
 	LOCATION_TOKEN,
 	LOCATION_SCOPE,
-	LOCATION_LOCAL_SYMBOL
+	LOCATION_LOCAL_SYMBOL,
+	LOCATION_FULLTEXTSEARCH_MATCH
 };
 
 int locationTypeToInt(LocationType type);

src/lib/data/location/TokenLocation.cpp

@@ -182,6 +182,11 @@ bool TokenLocation::isScopeTokenLocation() const
 	return m_type == LOCATION_SCOPE;
 }
 
+bool TokenLocation::isFullTextSearchMatch() const
+{
+	return m_type == LOCATION_FULLTEXTSEARCH_MATCH;
+}
+
 std::ostream& operator<<(std::ostream& ostream, const TokenLocation& location)
 {
 	if ((&location)->isStartTokenLocation())

src/lib/data/location/TokenLocation.h

@@ -50,6 +50,7 @@ public:
 	bool isEndTokenLocation() const;
 
 	bool isScopeTokenLocation() const;
+	bool isFullTextSearchMatch() const;
 
 private:
 	const Id m_id;			// own id