logic: fulltextsearch with suffixarray

* delete test.sqlite in storagetestsuite in desstructor
* remove new and delete from suffixarray
* case sensitive fts
* fixed fulltext results annotated like errors
* lazy suffix array building
* fulltextsearch folder added
* go back too branch without _publish
* suffixarray working
* logic: Improved code view performance

CMakeLists.txt

@@ -1,9 +1,4 @@
-cmake_minimum_required(VERSION 2.8.9)
-
-# to get CMP0043 warnings, upgrade to version 3 and change the line above to VERSION 3.0 and remove the line below
-if(POLICY CMP0043)
-	cmake_policy(SET CMP0043 OLD)
-endif()
+cmake_minimum_required(VERSION 3.1)
 
 include(cmake/add_files.cmake)
 include(cmake/create_source_groups.cmake)
@@ -11,8 +6,6 @@ include(cmake/version.cmake)
 include(cmake/version_setup.cmake)
 include(cmake/licenses.cmake)
 
-# set Standerd build type to Release
-set(CMAKE_BUILD_TYPE_INIT "Release")
 
 # Variables --------------------------------------------------------------------
 
@@ -40,6 +33,9 @@ endif ()
 
 project(${PROJECT_NAME})
 
+# set Standerd build type to Release
+set(CMAKE_BUILD_TYPE_INIT "Release")
+
 #RPATH
 if(UNIX AND NOT APPLE)
 	set(CMAKE_SKIP_BUILD_RPATH FALSE)
@@ -50,17 +46,19 @@ if(UNIX AND NOT APPLE)
 endif()
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_C_STANDARD 99)
 
 # Settings ---------------------------------------------------------------------
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option -fcolor-diagnostics")
+endif()
 
 if (UNIX)
-	add_definitions(-std=c++11 -Wno-unknown-warning-option -fcolor-diagnostics -DQT_COMPILING_QSTRING_COMPAT_CPP )
+	#add_definitions(-std=c++11 -Wno-unknown-warning-option -fcolor-diagnostics -DQT_COMPILING_QSTRING_COMPAT_CPP )
+	add_definitions(-fno-omit-frame-pointer  -DQT_COMPILING_QSTRING_COMPAT_CPP )
 endif ()
 
-# enable fts4 module for sqlite
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS")
-
 # Clang ------------------------------------------------------------------------
 
 if (UNIX AND APPLE)

cmake/external.cmake

@@ -2,12 +2,16 @@
 find_package(PythonInterp REQUIRED)
 
 execute_process(
-    COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/script/getSystemString.py"
-    OUTPUT_VARIABLE SYSSTRING
-    OUTPUT_STRIP_TRAILING_WHITESPACE
+	COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/script/getSystemString.py"
+	OUTPUT_VARIABLE SYSSTRING
+	OUTPUT_STRIP_TRAILING_WHITESPACE
 )
 
 set(EXTERNAL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../Coati_External")
 set(EXTERNAL_SRC "${EXTERNAL_ROOT}/src")
-set(EXTERNAL_BUILD "${EXTERNAL_ROOT}/${SYSSTRING}/${CMAKE_BUILD_TYPE}")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+	set(EXTERNAL_BUILD "${EXTERNAL_ROOT}/${SYSSTRING}/Debug")
+else()
+	set(EXTERNAL_BUILD "${EXTERNAL_ROOT}/${SYSSTRING}/Release")
+endif()
 

setup/git/git_pre_commit_hook.sh

@@ -20,6 +20,13 @@ elif [ "$(expr substr $(uname -s) 1 10)" == "MINGW32_NT" ]; then
 	PLATFORM='Windows'
 fi
 
+function fallback {
+	FALLBACK_BRANCH=${BRANCH_NAME:3}
+	FALLBACK_BRANCH="${FALLBACK_BRANCH%?}"
+	git checkout ${FALLBACK_BRANCH}
+	git branch -D ${BRANCH_NAME}
+}
+
 function build {
 	echo -e $INFO Building $1 \($2\)
 	cmake --build build/$2 --target $1 > /dev/null
@@ -30,6 +37,7 @@ function build {
 		echo -e $PASS Building $1 \($2\) passed
 	else
 		echo -e $FAIL Building $1 \($2\) failed
+		fallback
 		exit 1
 	fi
 }
@@ -43,6 +51,7 @@ function build_type {
 		if [ $? -ne 0 ]
 		then
 			echo -e $FAIL At least one build or test failed, no push to $branch
+			fallback
 			exit 1
 		fi
 	else
@@ -65,6 +74,7 @@ function run_tests {
 		echo -e $PASS $1 Tests passed
 	else
 		echo -e $FAIL $1 Tests failed
+		fallback
 		exit 1
 	fi
 	cd $ROOTDIR

src/lib/CMakeLists.txt

@@ -79,6 +79,11 @@ add_files(
 	data/access/StorageAccessProxy.cpp
 	data/access/StorageAccessProxy.h
 
+	data/fulltextsearch/FullTextSearchIndex.cpp
+	data/fulltextsearch/FullTextSearchIndex.h
+	data/fulltextsearch/SuffixArray.cpp
+	data/fulltextsearch/SuffixArray.h
+
 	data/graph/token_component/TokenComponent.cpp
 	data/graph/token_component/TokenComponent.h
 	data/graph/token_component/TokenComponentAbstraction.cpp

src/lib/component/controller/CodeController.cpp

@@ -241,7 +241,9 @@ void CodeController::handleMessage(MessageShowErrors* message)
 void CodeController::handleMessage(MessageSearchFullText* message)
 {
 	CodeView* view = getView();
-	std::vector<CodeSnippetParams> snippets = getSnippetsForFullTextSearch(message->searchTerm);
+	view->setErrorInfos(std::vector<ErrorInfo>());
+
+	std::vector<CodeSnippetParams> snippets = getSnippetsForFullTextSearch(message->searchTerm, message->caseSensitive);
 	view->showCodeSnippets(snippets, std::vector<Id>());
 
 	showContents(message);
@@ -603,23 +605,32 @@ std::shared_ptr<TokenLocationFile> CodeController::getTokenLocationOfParentScope
 	return file;
 }
 
-
 std::vector<CodeSnippetParams> CodeController::getSnippetsForFullTextSearch(
-		const std::string& searchTerm) const
+		const std::string& searchTerm, bool caseSensitive) const
 {
 	std::shared_ptr<TokenLocationCollection> collection =
-		m_storageAccess->getFullTextSearchLocations(searchTerm);
+		m_storageAccess->getFullTextSearchLocations(searchTerm, caseSensitive);
 
 	std::vector<CodeSnippetParams> snippets;
+	snippets.reserve(collection->getTokenLocationFileCount());
 
 	collection->forEachTokenLocationFile(
 		[&](std::shared_ptr<TokenLocationFile> file) -> void
 		{
+			//CodeSnippetParams params;
+			//params.locationFile = file;
+			//params.startLineNumber = 1;
+
+			//std::shared_ptr<TextAccess> textAccess = m_storageAccess->getFileContent(file->getFilePath());
+			//params.code = textAccess->getText();
+
+			//snippets.push_back(params);
+
 			//if (snippets.size() < 10)
-			{
+			//{
 				std::vector<CodeSnippetParams> fileSnippets = getSnippetsForFile(file);
 				snippets.insert(snippets.end(), fileSnippets.begin(), fileSnippets.end());
-			}
+			//}
 			//else
 			//{
 				//CodeSnippetParams params;

src/lib/component/controller/CodeController.h

@@ -69,7 +69,7 @@ private:
 	std::vector<CodeSnippetParams> getSnippetsForFile(
 		std::shared_ptr<TokenLocationFile> activeTokenLocations, std::shared_ptr<TokenLocationFile> fileLocations) const;
 	std::vector<CodeSnippetParams> getSnippetsForFile(std::shared_ptr<TokenLocationFile> file) const;
-	std::vector<CodeSnippetParams> getSnippetsForFullTextSearch(const std::string& searchTerm) const;
+	std::vector<CodeSnippetParams> getSnippetsForFullTextSearch(const std::string& searchTerm, bool caseSensitive) const;
 	std::shared_ptr<SnippetMerger> buildMergerHierarchy(
 		TokenLocation* location, std::shared_ptr<TokenLocationFile> context, SnippetMerger& fileScopedMerger, std::map<int, std::shared_ptr<SnippetMerger>>& mergers) const;
 	std::shared_ptr<TokenLocationFile> getTokenLocationOfParentScope(const TokenLocation* location, std::shared_ptr<TokenLocationFile> context) const;

src/lib/component/controller/SearchController.cpp

@@ -59,7 +59,12 @@ void SearchController::handleMessage(MessageSearchAutocomplete* message)
 void SearchController::handleMessage(MessageSearchFullText* message)
 {
 	LOG_INFO("fulltext string: \"" + message->searchTerm + "\"");
-	SearchMatch match("@" + message->searchTerm);
+	std::string prefix = "@";
+	if (message->caseSensitive)
+	{
+		prefix += "@";
+	}
+	SearchMatch match(prefix + message->searchTerm);
 	getView()->setMatches(std::vector<SearchMatch>(1, match));
 }
 

src/lib/data/PersistentStorage.cpp

@@ -7,6 +7,7 @@
 #include "utility/logging/logging.h"
 #include "utility/messaging/type/MessageClearErrorCount.h"
 #include "utility/messaging/type/MessageShowErrors.h"
+#include "utility/text/TextAccess.h"
 #include "utility/TimePoint.h"
 #include "utility/utility.h"
 #include "utility/utilityString.h"
@@ -254,6 +255,7 @@ void PersistentStorage::clearCaches()
 	m_fileNodeIds.clear();
 	m_fileNodePaths.clear();
 	m_hierarchyCache.clear();
+	m_fullTextSearchIndex.clear();
 }
 
 std::set<FilePath> PersistentStorage::getDependingFilePaths(const std::set<FilePath>& filePaths)
@@ -357,7 +359,6 @@ void PersistentStorage::finishParsing()
 	buildSearchIndex();
 	buildFilePathMaps();
 	buildHierarchyCache();
-	optimizeFTSTable();
 }
 
 void PersistentStorage::optimizeMemory()
@@ -389,24 +390,76 @@ Node::NodeType PersistentStorage::getNodeTypeForNodeWithId(Id nodeId) const
 	return Node::intToType(m_sqliteStorage.getNodeById(nodeId).type);
 }
 
-std::shared_ptr<TokenLocationCollection> PersistentStorage::getFullTextSearchLocations(const std::string& searchTerm) const
+std::shared_ptr<TokenLocationCollection> PersistentStorage::getFullTextSearchLocations(
+		const std::string& searchTerm, bool caseSensitive
+) const
 {
+	if (m_fullTextSearchIndex.fileCount() == 0)
+	{
+		buildFullTextSearchIndex();
+	}
+
 	std::shared_ptr<TokenLocationCollection> collection = std::make_shared<TokenLocationCollection>();
 
-	std::vector<ParseLocation> parseLocations = m_sqliteStorage.getFullTextSearch(searchTerm);
-	size_t i = 0;
-	for(ParseLocation location : parseLocations)
+	std::vector<FullTextSearchResult> hits = m_fullTextSearchIndex.searchForTerm(searchTerm);
+
+	int termLength = searchTerm.length();
+	FilePath filepath;
+	std::shared_ptr<TextAccess> file;
+	ParseLocation location;
+	for (size_t i = 0; i < hits.size(); i++)
 	{
-		collection->addTokenLocation(
-			i,
-			0,
-			location.filePath,
-			location.startLineNumber,
-			location.startColumnNumber,
-			location.endLineNumber,
-			location.endColumnNumber
-		)->setType(LOCATION_FULLTEXTSEARCH_MATCH);
-		i++;
+		filepath = getFileNodePath(hits[i].fileId);
+		file = getFileContent(filepath);
+
+		int charsInPreviousLines = 0;
+		int lineNumber = 1;
+		std::string line;
+		line = file->getLine(lineNumber);
+
+		for (int pos : hits[i].positions)
+		{
+			bool addHit = true;
+			while( (charsInPreviousLines + (int)line.length()) < pos)
+			{
+				lineNumber++;
+				charsInPreviousLines += line.length();
+				line = file->getLine(lineNumber);
+			}
+			location.startLineNumber = lineNumber;
+			location.startColumnNumber = pos - charsInPreviousLines + 1;
+
+			if ( caseSensitive )
+			{
+				if( line.substr(location.startColumnNumber-1, termLength) != searchTerm )
+				{
+					addHit = false;
+				}
+			}
+
+			while( (charsInPreviousLines + (int)line.length()) < pos + termLength)
+			{
+				lineNumber++;
+				charsInPreviousLines += line.length();
+				line = file->getLine(lineNumber);
+			}
+
+			location.endLineNumber = lineNumber;
+			location.endColumnNumber = pos + termLength - charsInPreviousLines;
+
+			if ( addHit )
+			{
+				collection->addTokenLocation(
+					i,
+					0,
+					filepath,
+					location.startLineNumber,
+					location.startColumnNumber,
+					location.endLineNumber,
+					location.endColumnNumber
+				)->setType(LOCATION_FULLTEXT);
+			}
+		}
 	}
 
 	return collection;
@@ -597,7 +650,6 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForActiveTokenIds(const std::v
 	std::vector<Id> edgeIds;
 	bool addAggregations = false;
 
-	//m_sqliteStorage.getFullTextSearch("const int");
 	if (tokenIds.size() == 1)
 	{
 		const Id elementId = tokenIds[0];
@@ -1252,6 +1304,14 @@ void PersistentStorage::buildFilePathMaps()
 	}
 }
 
+void PersistentStorage::buildFullTextSearchIndex() const
+{
+	for (StorageFile file : m_sqliteStorage.getAllFiles())
+	{
+		m_fullTextSearchIndex.addFile(file.id, m_sqliteStorage.getFileContentById(file.id)->getText());
+	}
+}
+
 void PersistentStorage::buildHierarchyCache()
 {
 	std::vector<StorageEdge> memberEdges = m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER));
@@ -1267,7 +1327,3 @@ void PersistentStorage::buildHierarchyCache()
 	}
 }
 
-void PersistentStorage::optimizeFTSTable()
-{
-	m_sqliteStorage.optimizeFTSTable();
-}

src/lib/data/PersistentStorage.h

@@ -7,6 +7,7 @@
 #include "utility/file/FilePath.h"
 
 #include "data/access/StorageAccess.h"
+#include "data/fulltextsearch/FullTextSearchIndex.h"
 #include "data/graph/token_component/TokenComponentAccess.h"
 #include "data/location/TokenLocationCollection.h"
 #include "data/parser/ParserClient.h"
@@ -47,9 +48,6 @@ public:
 	virtual void startInjection();
 	virtual void finishInjection();
 
-
-
-
 	FilePath getDbFilePath() const;
 	Version getVersion() const;
 
@@ -79,7 +77,8 @@ public:
 	virtual NameHierarchy getNameHierarchyForNodeWithId(Id nodeId) const;
 	virtual Node::NodeType getNodeTypeForNodeWithId(Id nodeId) const;
 
-	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(const std::string& searchTerm) const;
+	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(
+			const std::string& searchTerm, bool caseSensitive) const;
 	virtual std::vector<SearchMatch> getAutocompletionMatches(const std::string& query) const;
 	virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& elementIds) const;
 
@@ -135,8 +134,8 @@ private:
 
 	void buildSearchIndex();
 	void buildFilePathMaps();
+	void buildFullTextSearchIndex() const;
 	void buildHierarchyCache();
-	void optimizeFTSTable();
 
 	void log(std::string type, std::string str, const ParseLocation& location) const;
 
@@ -145,6 +144,8 @@ private:
 	SearchIndex m_commandIndex;
 	SearchIndex m_elementIndex;
 
+	mutable FullTextSearchIndex m_fullTextSearchIndex;
+
 	SqliteStorage m_sqliteStorage;
 
 	mutable std::map <FilePath, Id> m_fileNodeIds;

src/lib/data/SqliteStorage.cpp

@@ -373,92 +373,6 @@ void SqliteStorage::optimizeMemory() const
 	}
 }
 
-void SqliteStorage::optimizeFTSTable() const
-{
-	try
-	{
-		CppSQLite3Query q = m_database.execQuery("INSERT INTO file(file) VALUES('optimize');");
-	}
-	catch(CppSQLite3Exception e)
-	{
-		LOG_ERROR(e.errorMessage());
-	}
-}
-
-std::vector<ParseLocation> SqliteStorage::getFullTextSearch(const std::string& searchTerm) const
-{
-	std::vector<ParseLocation> matches;
-	try
-	{
-		CppSQLite3Query q = m_database.execQuery((
-					"SELECT id,offsets(file) FROM file WHERE content MATCH '\"*" + searchTerm + "*\"'"
-		).c_str());
-
-		while(!q.eof())
-		{
-			const Id fileId = q.getIntField(0,0);
-
-			// convert the string "0 2 0 2" to int vector
-			std::stringstream temp_results(q.getStringField(1,0));
-			std::vector<int> results((std::istream_iterator<int>(temp_results)),std::istream_iterator<int>());
-
-			ParseLocation location;
-			location.filePath = getFileById(fileId).filePath;
-			std::shared_ptr<TextAccess> file = getFileContentByPath(location.filePath.str());
-
-			int charsInPreviousLines = 0;
-			int lineNumber = 1;
-			std::string line;
-			// results
-			// i   ... col
-			// i+1 ... term
-			// i+2 ... offset
-			// i+3 ... length
-			line = file->getLine(lineNumber);
-			for (size_t i = 0; i < results.size() ; i+=4)
-			{
-				while( ((charsInPreviousLines + (int)line.length()) < results[i+2]) && results[i+1] == 0 )
-				{
-					lineNumber++;
-					charsInPreviousLines += line.length();
-					line = file->getLine(lineNumber);
-				}
-
-				//only set start if its the first term of the match
-				if ( results[i+1] == 0 )
-				{
-					location.startLineNumber = lineNumber;
-					// +1 to be consistent with the rest of the codebase
-					location.startColumnNumber = results[i+2] - charsInPreviousLines + 1;
-				}
-
-				while( (charsInPreviousLines + (int)line.length()) < (results[i+2] + results[i+3]) )
-				{
-					lineNumber++;
-					charsInPreviousLines += line.length();
-					line = file->getLine(lineNumber);
-				}
-
-				location.endLineNumber = lineNumber;
-				location.endColumnNumber = results[i+2] + results[i+3] - charsInPreviousLines;
-
-				// add match if the next term is the first term of a match or its the last term in the file
-				if ( (i+4 < results.size() && results[i+5] == 0) || i+4 >= results.size() )
-				{
-					matches.push_back(location);
-				}
-			}
-
-			q.nextRow();
-		}
-	}
-	catch(CppSQLite3Exception e)
-	{
-		LOG_ERROR(e.errorMessage());
-	}
-	return matches;
-}
-
 StorageNode SqliteStorage::getNodeById(Id id) const
 {
 	if (id != 0)
@@ -518,6 +432,20 @@ std::vector<StorageFile> SqliteStorage::getFilesByPaths(const std::vector<FilePa
 	return getAll<StorageFile>("WHERE file.path IN ('" + utility::join(utility::toStrings(filePaths), "', '") + "')");
 }
 
+std::shared_ptr<TextAccess> SqliteStorage::getFileContentById(Id fileId) const
+{
+	CppSQLite3Query q = m_database.execQuery((
+		"SELECT content FROM file WHERE id = '" + std::to_string(fileId) + "';"
+	).c_str());
+
+	if (!q.eof())
+	{
+		return TextAccess::createFromString(q.getStringField(0, ""));
+	}
+
+	return TextAccess::createFromString("");
+}
+
 std::shared_ptr<TextAccess> SqliteStorage::getFileContentByPath(const std::string& filePath) const
 {
 	CppSQLite3Query q = m_database.execQuery((
@@ -704,54 +632,55 @@ void SqliteStorage::clearTables()
 
 void SqliteStorage::setupTables()
 {
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS meta("
-			"id INTEGER, "
-			"key TEXT, "
-			"value TEXT, "
-			"PRIMARY KEY(id));"
-	);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS element("
-			"id INTEGER, "
-			"PRIMARY KEY(id));"
-	);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS edge("
-			"id INTEGER NOT NULL, "
-			"type INTEGER NOT NULL, "
-			"source_node_id INTEGER NOT NULL, "
-			"target_node_id INTEGER NOT NULL, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE, "
-			"FOREIGN KEY(source_node_id) REFERENCES node(id) ON DELETE CASCADE, "
-			"FOREIGN KEY(target_node_id) REFERENCES node(id) ON DELETE CASCADE);"
-	);
-
-	m_database.execDML( // used for checking for duplicates during code analysis // TODO: move to createIndexesForAnalysis() or prepareForAnalysis
-		"CREATE INDEX IF NOT EXISTS edge_multipart_index ON edge(type, source_node_id, target_node_id);"
-	);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS node("
-			"id INTEGER NOT NULL, "
-			"type INTEGER NOT NULL, "
-			"serialized_name TEXT, "
-			"definition_type INTEGER NOT NULL, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
-	);
-
-	m_database.execDML(
-		"CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);"
-	);
-
 	try
 	{
 		m_database.execDML(
-			"CREATE VIRTUAL TABLE IF NOT EXISTS file USING fts4("
+			"CREATE TABLE IF NOT EXISTS meta("
+				"id INTEGER, "
+				"key TEXT, "
+				"value TEXT, "
+				"PRIMARY KEY(id));"
+		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS element("
+				"id INTEGER, "
+				"PRIMARY KEY(id));"
+		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS edge("
+				"id INTEGER NOT NULL, "
+				"type INTEGER NOT NULL, "
+				"source_node_id INTEGER NOT NULL, "
+				"target_node_id INTEGER NOT NULL, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE, "
+				"FOREIGN KEY(source_node_id) REFERENCES node(id) ON DELETE CASCADE, "
+				"FOREIGN KEY(target_node_id) REFERENCES node(id) ON DELETE CASCADE);"
+		);
+
+		// TODO: move to createIndexesForAnalysis() or prepareForAnalysis
+		m_database.execDML( // used for checking for duplicates during code analysis 
+			"CREATE INDEX IF NOT EXISTS edge_multipart_index ON edge(type, source_node_id, target_node_id);"
+		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS node("
+				"id INTEGER NOT NULL, "
+				"type INTEGER NOT NULL, "
+				"serialized_name TEXT, "
+				"definition_type INTEGER NOT NULL, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
+		);
+
+		m_database.execDML(
+			"CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);"
+		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS file("
 				"id INTEGER NOT NULL, "
 				"path TEXT, "
 				"modification_time TEXT, "
@@ -760,75 +689,75 @@ void SqliteStorage::setupTables()
 				"PRIMARY KEY(id), "
 				"FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);"
 		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS local_symbol("
+				"id INTEGER NOT NULL, "
+				"name TEXT, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
+		);
+
+		m_database.execDML(
+			"CREATE INDEX IF NOT EXISTS local_symbol_name_index ON local_symbol(name);"
+		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS source_location("
+				"id INTEGER NOT NULL, "
+				"element_id INTEGER, "
+				"file_node_id INTEGER, "
+				"start_line INTEGER, "
+				"start_column INTEGER, "
+				"end_line INTEGER, "
+				"end_column INTEGER, "
+				"type INTEGER, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, "
+				"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
+		);
+
+		SqliteIndex("source_location_element_id_index", "source_location(element_id)").createOnDatabase(m_database);
+		SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS component_access("
+				"id INTEGER NOT NULL, "
+				"edge_id INTEGER, "
+				"type INTEGER NOT NULL, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(edge_id) REFERENCES edge(id) ON DELETE CASCADE);"
+		);
+
+		SqliteIndex("component_access_edge_id_index", "component_access(edge_id)").createOnDatabase(m_database);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS comment_location("
+				"id INTEGER NOT NULL, "
+				"file_node_id INTEGER, "
+				"start_line INTEGER, "
+				"start_column INTEGER, "
+				"end_line INTEGER, "
+				"end_column INTEGER, "
+				"PRIMARY KEY(id), "
+				"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
+		);
+
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS error("
+				"id INTEGER NOT NULL, "
+				"message TEXT, "
+				"fatal INTEGER NOT NULL, "
+				"file_path TEXT, "
+				"line_number INTEGER, "
+				"column_number INTEGER, "
+				"PRIMARY KEY(id));"
+		);
 	}
 	catch (CppSQLite3Exception& e)
 	{
 		LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage());
 	}
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS local_symbol("
-			"id INTEGER NOT NULL, "
-			"name TEXT, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);"
-	);
-
-	m_database.execDML(
-		"CREATE INDEX IF NOT EXISTS local_symbol_name_index ON local_symbol(name);"
-	);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS source_location("
-			"id INTEGER NOT NULL, "
-			"element_id INTEGER, "
-			"file_node_id INTEGER, "
-			"start_line INTEGER, "
-			"start_column INTEGER, "
-			"end_line INTEGER, "
-			"end_column INTEGER, "
-			"type INTEGER, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, "
-			"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
-	);
-
-	SqliteIndex("source_location_element_id_index", "source_location(element_id)").createOnDatabase(m_database);
-	SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS component_access("
-			"id INTEGER NOT NULL, "
-			"edge_id INTEGER, "
-			"type INTEGER NOT NULL, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(edge_id) REFERENCES edge(id) ON DELETE CASCADE);"
-	);
-
-	SqliteIndex("component_access_edge_id_index", "component_access(edge_id)").createOnDatabase(m_database);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS comment_location("
-			"id INTEGER NOT NULL, "
-			"file_node_id INTEGER, "
-			"start_line INTEGER, "
-			"start_column INTEGER, "
-			"end_line INTEGER, "
-			"end_column INTEGER, "
-			"PRIMARY KEY(id), "
-			"FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);"
-	);
-
-	m_database.execDML(
-		"CREATE TABLE IF NOT EXISTS error("
-			"id INTEGER NOT NULL, "
-			"message TEXT, "
-			"fatal INTEGER NOT NULL, "
-			"file_path TEXT, "
-			"line_number INTEGER, "
-			"column_number INTEGER, "
-			"PRIMARY KEY(id));"
-	);
 }
 
 bool SqliteStorage::hasTable(const std::string& tableName) const

src/lib/data/SqliteStorage.h

@@ -81,6 +81,7 @@ public:
 
 	std::vector<StorageFile> getFilesByPaths(const std::vector<FilePath>& filePaths) const;
 	std::shared_ptr<TextAccess> getFileContentByPath(const std::string& filePath) const;
+	std::shared_ptr<TextAccess> getFileContentById(Id fileId) const;
 
 	void setNodeType(int type, Id nodeId);
 	void setNodeDefinitionType(int definitionType, Id nodeId);
@@ -98,7 +99,6 @@ public:
 	void optimizeMemory() const;
 
 	std::vector<ParseLocation> getFullTextSearch(const std::string& searchTerm) const;
-	void optimizeFTSTable() const;
 
 	std::vector<StorageCommentLocation> getCommentLocationsInFile(const FilePath& filePath) const;
 	std::vector<StorageError> getFatalErrors() const;

src/lib/data/access/StorageAccess.h

@@ -36,7 +36,7 @@ public:
 	virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const = 0;
 
 	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(
-			const std::string& searchTerm) const = 0;
+			const std::string& searchTerm, bool caseSensitive) const = 0;
 	virtual std::vector<SearchMatch> getAutocompletionMatches(const std::string& query) const = 0;
 	virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const = 0;
 

src/lib/data/access/StorageAccessProxy.cpp

@@ -84,11 +84,11 @@ std::vector<SearchMatch> StorageAccessProxy::getAutocompletionMatches(const std:
 }
 
 std::shared_ptr<TokenLocationCollection> StorageAccessProxy::getFullTextSearchLocations(
-		const std::string &searchTerm) const
+		const std::string &searchTerm, bool caseSensitive) const
 {
 	if (hasSubject())
 	{
-		return m_subject->getFullTextSearchLocations(searchTerm);
+		return m_subject->getFullTextSearchLocations(searchTerm, caseSensitive);
 	}
 
 	return std::make_shared<TokenLocationCollection>();

src/lib/data/access/StorageAccessProxy.h

@@ -20,7 +20,8 @@ public:
 	virtual NameHierarchy getNameHierarchyForNodeWithId(Id id) const;
 	virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const;
 
-	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(const std::string& searchTerm) const;
+	virtual std::shared_ptr<TokenLocationCollection> getFullTextSearchLocations(
+			const std::string& searchTerm, bool caseSensitive) const;
 	virtual std::vector<SearchMatch> getAutocompletionMatches(const std::string& query) const;
 	virtual std::vector<SearchMatch> getSearchMatchesForTokenIds(const std::vector<Id>& tokenIds) const;
 

src/lib/data/fulltextsearch/FullTextSearchIndex.cpp

@@ -0,0 +1,45 @@
+#include "data/fulltextsearch/FullTextSearchIndex.h"
+#include <limits>
+
+#include "utility/utility.h"
+#include "utility/logging/logging.h"
+
+void FullTextSearchIndex::addFile(Id fileId, const std::string& file)
+{
+	if( file.empty() )
+	{
+		LOG_ERROR("empty file not added to fulltextsearch index");
+	}
+
+	if ( file.size() >= std::numeric_limits<int>::max() )
+	{
+		LOG_ERROR("file too big not added to fulltextsearch index");
+	}
+
+	FullTextSearchFile fts_file(fileId, SuffixArray(file));
+	m_files.push_back(fts_file);
+}
+
+std::vector<FullTextSearchResult> FullTextSearchIndex::searchForTerm(const std::string& term) const
+{
+	std::vector<FullTextSearchResult> ret;
+	FullTextSearchResult hit;
+	for (auto f : m_files)
+	{
+		hit.fileId = f.fileId;
+		hit.positions = f.array.searchForTerm(term);
+		ret.push_back(hit);
+	}
+	return ret;
+}
+
+size_t FullTextSearchIndex::fileCount() const
+{
+	return m_files.size();
+}
+
+void FullTextSearchIndex::clear()
+{
+	m_files.clear();
+}
+

src/lib/data/fulltextsearch/FullTextSearchIndex.h

@@ -0,0 +1,44 @@
+#ifndef FULLTEXTSEARCH_INDEX_H
+#define FULLTEXTSEARCH_INDEX_H
+
+#include <vector>
+#include <unordered_map>
+
+#include "data/fulltextsearch/SuffixArray.h"
+#include "utility/types.h"
+
+class StorageAccess;
+
+// contains all fulltextsearch results of one file
+struct FullTextSearchResult
+{
+	Id fileId;
+	std::vector<int> positions;
+};
+
+struct FullTextSearchFile
+{
+	FullTextSearchFile(Id fileId, SuffixArray array)
+		: fileId(fileId)
+		, array(array)
+	{
+	};
+	Id fileId;
+	SuffixArray array;
+};
+
+class FullTextSearchIndex
+{
+public:
+	void addFile(Id fileId, const std::string& file);
+	std::vector<FullTextSearchResult> searchForTerm(const std::string& term) const;
+
+	size_t fileCount() const;
+
+	void clear();
+
+private:
+	std::vector<FullTextSearchFile> m_files;
+};
+
+#endif // FULLTEXTSEARCH_INDEX_H

src/lib/data/fulltextsearch/SuffixArray.cpp

@@ -0,0 +1,182 @@
+#include "SuffixArray.h"
+
+#include <iostream>
+#include <algorithm>
+
+struct suffix
+{
+	int index;
+	int rank[2];
+};
+
+int SuffixArray::cmp(struct suffix a, struct suffix b)
+{
+	return (a.rank[0] == b.rank[0])? (a.rank[1] < b.rank[1] ?1: 0):
+		(a.rank[0] < b.rank[0] ?1: 0);
+}
+
+SuffixArray::SuffixArray(const std::string& text)
+	: m_text(text)
+{
+	std::transform(m_text.begin(), m_text.end(), m_text.begin(), ::tolower);
+	m_array = buildSuffixArray();
+	m_lcp = buildLCP();
+}
+
+void SuffixArray::printArray() const
+{
+	std::cout << "Suffix Array : \n";
+	printArr(m_array);
+}
+
+void SuffixArray::printLCP() const
+{
+	std::cout << "\nLCP Array : \n";
+	printArr(m_lcp);
+}
+
+std::vector<int> SuffixArray::buildLCP()
+{
+	int n = m_array.size();
+
+	std::vector<int> lcp(n, 0);
+	std::vector<int> invSuff(n, 0);
+
+	for (int i=0; i < n; i++)
+	{
+		invSuff[m_array[i]] = i;
+	}
+
+	int k = 0;
+
+	for (int i=0; i<n; i++)
+	{
+		if (invSuff[i] == n-1)
+		{
+			k = 0;
+			continue;
+		}
+
+		int j = m_array[invSuff[i]+1];
+
+		while (i+k<n && j+k<n && m_text[i+k]==m_text[j+k])
+		{
+			k++;
+		}
+
+		lcp[invSuff[i]] = k;
+
+		if (k>0)
+		{
+			k--;
+		}
+	}
+
+	return lcp;
+}
+
+std::vector<int> SuffixArray::searchForTerm(const std::string& searchTerm) const
+{
+	std::string term = searchTerm;
+	std::transform(term.begin(), term.end(), term.begin(), ::tolower);
+
+	int termLength = term.length();
+	int l = 0;
+	int r = m_text.length()-1;
+	int m;
+
+	std::vector<int> matches;
+	int compareResult;
+	while (l+1 < r)
+	{
+		m = (l+r+1)/2;
+		compareResult = term.compare(m_text.substr(m_array[m], termLength));
+		if( compareResult < 0)
+		{
+			r = m;
+		}
+		else if (compareResult > 0)
+		{
+			l = m;
+		}
+		else
+		{
+			matches.push_back(m_array[m]);
+			for (int lower = m-1; m_lcp[lower] >= termLength; lower--)
+			{
+				matches.push_back(m_array[lower]);
+			}
+			for (int higher = m+1; m_lcp[higher-1] >= termLength; higher++)
+			{
+				matches.push_back(m_array[higher]);
+			}
+			break;
+		}
+	}
+
+	std::sort(matches.begin(), matches.end());
+
+	return matches;
+}
+
+std::vector<int> SuffixArray::buildSuffixArray()
+{
+	int n = m_text.length();
+	std::vector<suffix> suffixes;
+	suffixes.reserve(n);
+
+	suffix s;
+	for (int i = 0; i < n; i++)
+	{
+		s.index = i;
+		s.rank[0] = m_text[i] - 'a';
+		s.rank[1] = ((i+1) < n)? (m_text[i + 1] - 'a'): -1;
+		suffixes.push_back(s);
+	}
+
+	std::sort(suffixes.begin(), suffixes.end(), SuffixArray::cmp);
+
+	std::vector<int> ind (n,0);
+	for (int k = 4; k < 2*n; k = k*2)
+	{
+		int rank = 0;
+		int prev_rank = suffixes[0].rank[0];
+		suffixes[0].rank[0] = rank;
+		ind[suffixes[0].index] = 0;
+
+		for (int i = 1; i < n; i++)
+		{
+			if (suffixes[i].rank[0] == prev_rank &&
+					suffixes[i].rank[1] == suffixes[i-1].rank[1])
+			{
+				prev_rank = suffixes[i].rank[0];
+				suffixes[i].rank[0] = rank;
+			}
+			else
+			{
+				prev_rank = suffixes[i].rank[0];
+				suffixes[i].rank[0] = ++rank;
+			}
+			ind[suffixes[i].index] = i;
+		}
+
+		for (int i = 0; i < n; i++)
+		{
+			int nextindex = suffixes[i].index + k/2;
+			suffixes[i].rank[1] = (nextindex < n)?
+				suffixes[ind[nextindex]].rank[0]: -1;
+		}
+
+		std::sort(suffixes.begin(), suffixes.end(), cmp);
+	}
+
+	std::vector<int>suffixArr;
+	for (int i = 0; i < n; i++)
+	{
+		suffixArr.push_back(suffixes[i].index);
+	}
+
+	return  suffixArr;
+}
+
+

src/lib/data/fulltextsearch/SuffixArray.h

@@ -0,0 +1,36 @@
+#ifndef SUFFIX_ARRAY_H
+#define SUFFIX_ARRAY_H
+
+#include <vector>
+#include <string>
+#include <iostream>
+
+class SuffixArray
+{
+public:
+	SuffixArray(const std::string& text);
+	std::vector<int> searchForTerm(const std::string& searchTerm) const;
+	static int cmp(struct suffix a, struct suffix b);
+
+	void printArray() const;
+	void printLCP() const;
+
+private:
+	template <typename T>
+	void printArr(std::vector<T>arr) const
+	{
+		for (size_t i = 0; i < arr.size(); i++)
+		{
+			std::cout << arr[i] << " ";
+		}
+		std::cout << std::endl;
+	}
+
+	std::vector<int> buildLCP();
+	std::vector<int> buildSuffixArray();
+	std::vector<int> m_array;
+	std::vector<int> m_lcp;
+	std::string m_text;
+};
+
+#endif // SUFFIX_ARRAY_H

src/lib/data/location/LocationType.cpp

@@ -10,7 +10,7 @@ int locationTypeToInt(LocationType type)
 		return 1;
 	case LOCATION_LOCAL_SYMBOL:
 		return 2;
-	case LOCATION_FULLTEXTSEARCH_MATCH:
+	case LOCATION_FULLTEXT:
 		return 3;
 	}
 }
@@ -26,7 +26,7 @@ LocationType intToLocationType(int value)
 	case 2:
 		return LOCATION_LOCAL_SYMBOL;
 	case 3:
-		return LOCATION_FULLTEXTSEARCH_MATCH;
+		return LOCATION_FULLTEXT;
 	}
 	return LOCATION_TOKEN;
 }

src/lib/data/location/LocationType.h

@@ -6,7 +6,7 @@ enum LocationType
 	LOCATION_TOKEN,
 	LOCATION_SCOPE,
 	LOCATION_LOCAL_SYMBOL,
-	LOCATION_FULLTEXTSEARCH_MATCH
+	LOCATION_FULLTEXT
 };
 
 int locationTypeToInt(LocationType type);

src/lib/data/location/TokenLocation.cpp

@@ -184,7 +184,7 @@ bool TokenLocation::isScopeTokenLocation() const
 
 bool TokenLocation::isFullTextSearchMatch() const
 {
-	return m_type == LOCATION_FULLTEXTSEARCH_MATCH;
+	return m_type == LOCATION_FULLTEXT;
 }
 
 std::ostream& operator<<(std::ostream& ostream, const TokenLocation& location)

src/lib/utility/messaging/type/MessageSearchFullText.h

@@ -7,8 +7,9 @@
 class MessageSearchFullText: public Message<MessageSearchFullText>
 {
 public:
-	MessageSearchFullText(const std::string& searchTerm)
+	MessageSearchFullText(const std::string& searchTerm, bool caseSensitive = false)
 		: searchTerm(searchTerm)
+		, caseSensitive(caseSensitive)
 	{
 	}
 
@@ -23,6 +24,7 @@ public:
 	}
 
 	const std::string searchTerm;
+	bool caseSensitive;
 };
 
 #endif // MESSAGE_SEARCH_FULLTEXT_H

src/lib_gui/qt/element/QtCodeArea.cpp

@@ -938,7 +938,7 @@ const QtCodeArea::AnnotationColor& QtCodeArea::getAnnotationColorForAnnotation(c
 	{
 		i = 9;
 	}
-	else if (annotation.locationType == LOCATION_FULLTEXTSEARCH_MATCH)
+	else if (annotation.locationType == LOCATION_FULLTEXT)
 	{
 		i = 12;
 	}

src/lib_gui/qt/element/QtSmartSearchBox.cpp

@@ -33,8 +33,18 @@ void QtSearchElement::onChecked(bool)
 
 void QtSmartSearchBox::fullTextSearch()
 {
-	LOG_INFO_STREAM(<< "FullTextsearch: " << text().toStdString().substr(1));
-	MessageSearchFullText(text().toStdString().substr(1)).dispatch();
+	std::string term = text().toStdString().substr(1);
+	if(term.at(0) == '@')
+	{
+		term = term.substr(1);
+		LOG_INFO_STREAM(<< "FullTextsearch(case sensitive): " << term);
+		MessageSearchFullText(term, true).dispatch();
+	}
+	else
+	{
+		LOG_INFO_STREAM(<< "FullTextsearch: " << term);
+		MessageSearchFullText(term).dispatch();
+	}
 }
 
 void QtSmartSearchBox::search()

src/test/StorageTestSuite.h

@@ -233,6 +233,11 @@ private:
 			clear();
 		}
 
+		~TestStorage()
+		{
+			boost::filesystem::remove("data/test.sqlite");
+		}
+
 		std::shared_ptr<TokenLocationCollection> getLocationCollectionForTokenId(Id id) const
 		{
 			std::vector<Id> tokenIds;