From 6edcd77eb239360b17b96029ccc552f3ae018a25 Mon Sep 17 00:00:00 2001 From: Andreas Stallinger Date: Tue, 7 Jun 2016 08:49:34 +0200 Subject: [PATCH] logic: fulltextsearch with suffixarray * delete test.sqlite in storagetestsuite in desstructor * remove new and delete from suffixarray * case sensitive fts * fixed fulltext results annotated like errors * lazy suffix array building * fulltextsearch folder added * go back too branch without _publish * suffixarray working * logic: Improved code view performance --- CMakeLists.txt | 24 +- cmake/external.cmake | 12 +- setup/git/git_pre_commit_hook.sh | 10 + src/lib/CMakeLists.txt | 5 + .../component/controller/CodeController.cpp | 23 +- src/lib/component/controller/CodeController.h | 2 +- .../component/controller/SearchController.cpp | 7 +- src/lib/data/PersistentStorage.cpp | 96 ++++-- src/lib/data/PersistentStorage.h | 11 +- src/lib/data/SqliteStorage.cpp | 319 +++++++----------- src/lib/data/SqliteStorage.h | 2 +- src/lib/data/access/StorageAccess.h | 2 +- src/lib/data/access/StorageAccessProxy.cpp | 4 +- src/lib/data/access/StorageAccessProxy.h | 3 +- .../fulltextsearch/FullTextSearchIndex.cpp | 45 +++ .../data/fulltextsearch/FullTextSearchIndex.h | 44 +++ src/lib/data/fulltextsearch/SuffixArray.cpp | 182 ++++++++++ src/lib/data/fulltextsearch/SuffixArray.h | 36 ++ src/lib/data/location/LocationType.cpp | 4 +- src/lib/data/location/LocationType.h | 2 +- src/lib/data/location/TokenLocation.cpp | 2 +- .../messaging/type/MessageSearchFullText.h | 4 +- src/lib_gui/qt/element/QtCodeArea.cpp | 2 +- src/lib_gui/qt/element/QtSmartSearchBox.cpp | 14 +- src/test/StorageTestSuite.h | 5 + 25 files changed, 602 insertions(+), 258 deletions(-) create mode 100644 src/lib/data/fulltextsearch/FullTextSearchIndex.cpp create mode 100644 src/lib/data/fulltextsearch/FullTextSearchIndex.h create mode 100644 src/lib/data/fulltextsearch/SuffixArray.cpp create mode 100644 src/lib/data/fulltextsearch/SuffixArray.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fcab942d..ba7d5013 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,4 @@ -cmake_minimum_required(VERSION 2.8.9) - -# to get CMP0043 warnings, upgrade to version 3 and change the line above to VERSION 3.0 and remove the line below -if(POLICY CMP0043) - cmake_policy(SET CMP0043 OLD) -endif() +cmake_minimum_required(VERSION 3.1) include(cmake/add_files.cmake) include(cmake/create_source_groups.cmake) @@ -11,8 +6,6 @@ include(cmake/version.cmake) include(cmake/version_setup.cmake) include(cmake/licenses.cmake) -# set Standerd build type to Release -set(CMAKE_BUILD_TYPE_INIT "Release") # Variables -------------------------------------------------------------------- @@ -40,6 +33,9 @@ endif () project(${PROJECT_NAME}) +# set Standerd build type to Release +set(CMAKE_BUILD_TYPE_INIT "Release") + #RPATH if(UNIX AND NOT APPLE) set(CMAKE_SKIP_BUILD_RPATH FALSE) @@ -50,17 +46,19 @@ if(UNIX AND NOT APPLE) endif() set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_C_STANDARD 99) # Settings --------------------------------------------------------------------- +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option -fcolor-diagnostics") +endif() if (UNIX) - add_definitions(-std=c++11 -Wno-unknown-warning-option -fcolor-diagnostics -DQT_COMPILING_QSTRING_COMPAT_CPP ) + #add_definitions(-std=c++11 -Wno-unknown-warning-option -fcolor-diagnostics -DQT_COMPILING_QSTRING_COMPAT_CPP ) + add_definitions(-fno-omit-frame-pointer -DQT_COMPILING_QSTRING_COMPAT_CPP ) endif () -# enable fts4 module for sqlite -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS") - # Clang ------------------------------------------------------------------------ if (UNIX AND APPLE) diff --git a/cmake/external.cmake b/cmake/external.cmake index 3fcabafd..6649c18e 100644 --- a/cmake/external.cmake +++ b/cmake/external.cmake @@ -2,12 +2,16 @@ find_package(PythonInterp REQUIRED) execute_process( - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/script/getSystemString.py" - OUTPUT_VARIABLE SYSSTRING - OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/script/getSystemString.py" + OUTPUT_VARIABLE SYSSTRING + OUTPUT_STRIP_TRAILING_WHITESPACE ) set(EXTERNAL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../Coati_External") set(EXTERNAL_SRC "${EXTERNAL_ROOT}/src") -set(EXTERNAL_BUILD "${EXTERNAL_ROOT}/${SYSSTRING}/${CMAKE_BUILD_TYPE}") +if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(EXTERNAL_BUILD "${EXTERNAL_ROOT}/${SYSSTRING}/Debug") +else() + set(EXTERNAL_BUILD "${EXTERNAL_ROOT}/${SYSSTRING}/Release") +endif() diff --git a/setup/git/git_pre_commit_hook.sh b/setup/git/git_pre_commit_hook.sh index 3222ae63..33b00b4d 100755 --- a/setup/git/git_pre_commit_hook.sh +++ b/setup/git/git_pre_commit_hook.sh @@ -20,6 +20,13 @@ elif [ "$(expr substr $(uname -s) 1 10)" == "MINGW32_NT" ]; then PLATFORM='Windows' fi +function fallback { + FALLBACK_BRANCH=${BRANCH_NAME:3} + FALLBACK_BRANCH="${FALLBACK_BRANCH%?}" + git checkout ${FALLBACK_BRANCH} + git branch -D ${BRANCH_NAME} +} + function build { echo -e $INFO Building $1 \($2\) cmake --build build/$2 --target $1 > /dev/null @@ -30,6 +37,7 @@ function build { echo -e $PASS Building $1 \($2\) passed else echo -e $FAIL Building $1 \($2\) failed + fallback exit 1 fi } @@ -43,6 +51,7 @@ function build_type { if [ $? -ne 0 ] then echo -e $FAIL At least one build or test failed, no push to $branch + fallback exit 1 fi else @@ -65,6 +74,7 @@ function run_tests { echo -e $PASS $1 Tests passed else echo -e $FAIL $1 Tests failed + fallback exit 1 fi cd $ROOTDIR diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 822c565d..9e762725 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -79,6 +79,11 @@ add_files( data/access/StorageAccessProxy.cpp data/access/StorageAccessProxy.h + data/fulltextsearch/FullTextSearchIndex.cpp + data/fulltextsearch/FullTextSearchIndex.h + data/fulltextsearch/SuffixArray.cpp + data/fulltextsearch/SuffixArray.h + data/graph/token_component/TokenComponent.cpp data/graph/token_component/TokenComponent.h data/graph/token_component/TokenComponentAbstraction.cpp diff --git a/src/lib/component/controller/CodeController.cpp b/src/lib/component/controller/CodeController.cpp index d92aa9c4..76556e1a 100644 --- a/src/lib/component/controller/CodeController.cpp +++ b/src/lib/component/controller/CodeController.cpp @@ -241,7 +241,9 @@ void CodeController::handleMessage(MessageShowErrors* message) void CodeController::handleMessage(MessageSearchFullText* message) { CodeView* view = getView(); - std::vector snippets = getSnippetsForFullTextSearch(message->searchTerm); + view->setErrorInfos(std::vector()); + + std::vector snippets = getSnippetsForFullTextSearch(message->searchTerm, message->caseSensitive); view->showCodeSnippets(snippets, std::vector()); showContents(message); @@ -603,23 +605,32 @@ std::shared_ptr CodeController::getTokenLocationOfParentScope return file; } - std::vector CodeController::getSnippetsForFullTextSearch( - const std::string& searchTerm) const + const std::string& searchTerm, bool caseSensitive) const { std::shared_ptr collection = - m_storageAccess->getFullTextSearchLocations(searchTerm); + m_storageAccess->getFullTextSearchLocations(searchTerm, caseSensitive); std::vector snippets; + snippets.reserve(collection->getTokenLocationFileCount()); collection->forEachTokenLocationFile( [&](std::shared_ptr file) -> void { + //CodeSnippetParams params; + //params.locationFile = file; + //params.startLineNumber = 1; + + //std::shared_ptr textAccess = m_storageAccess->getFileContent(file->getFilePath()); + //params.code = textAccess->getText(); + + //snippets.push_back(params); + //if (snippets.size() < 10) - { + //{ std::vector fileSnippets = getSnippetsForFile(file); snippets.insert(snippets.end(), fileSnippets.begin(), fileSnippets.end()); - } + //} //else //{ //CodeSnippetParams params; diff --git a/src/lib/component/controller/CodeController.h b/src/lib/component/controller/CodeController.h index 9d99d6b3..be61a6ac 100644 --- a/src/lib/component/controller/CodeController.h +++ b/src/lib/component/controller/CodeController.h @@ -69,7 +69,7 @@ private: std::vector getSnippetsForFile( std::shared_ptr activeTokenLocations, std::shared_ptr fileLocations) const; std::vector getSnippetsForFile(std::shared_ptr file) const; - std::vector getSnippetsForFullTextSearch(const std::string& searchTerm) const; + std::vector getSnippetsForFullTextSearch(const std::string& searchTerm, bool caseSensitive) const; std::shared_ptr buildMergerHierarchy( TokenLocation* location, std::shared_ptr context, SnippetMerger& fileScopedMerger, std::map>& mergers) const; std::shared_ptr getTokenLocationOfParentScope(const TokenLocation* location, std::shared_ptr context) const; diff --git a/src/lib/component/controller/SearchController.cpp b/src/lib/component/controller/SearchController.cpp index 29e1c56d..9b72e604 100644 --- a/src/lib/component/controller/SearchController.cpp +++ b/src/lib/component/controller/SearchController.cpp @@ -59,7 +59,12 @@ void SearchController::handleMessage(MessageSearchAutocomplete* message) void SearchController::handleMessage(MessageSearchFullText* message) { LOG_INFO("fulltext string: \"" + message->searchTerm + "\""); - SearchMatch match("@" + message->searchTerm); + std::string prefix = "@"; + if (message->caseSensitive) + { + prefix += "@"; + } + SearchMatch match(prefix + message->searchTerm); getView()->setMatches(std::vector(1, match)); } diff --git a/src/lib/data/PersistentStorage.cpp b/src/lib/data/PersistentStorage.cpp index fd7c0875..1c60f59c 100644 --- a/src/lib/data/PersistentStorage.cpp +++ b/src/lib/data/PersistentStorage.cpp @@ -7,6 +7,7 @@ #include "utility/logging/logging.h" #include "utility/messaging/type/MessageClearErrorCount.h" #include "utility/messaging/type/MessageShowErrors.h" +#include "utility/text/TextAccess.h" #include "utility/TimePoint.h" #include "utility/utility.h" #include "utility/utilityString.h" @@ -254,6 +255,7 @@ void PersistentStorage::clearCaches() m_fileNodeIds.clear(); m_fileNodePaths.clear(); m_hierarchyCache.clear(); + m_fullTextSearchIndex.clear(); } std::set PersistentStorage::getDependingFilePaths(const std::set& filePaths) @@ -357,7 +359,6 @@ void PersistentStorage::finishParsing() buildSearchIndex(); buildFilePathMaps(); buildHierarchyCache(); - optimizeFTSTable(); } void PersistentStorage::optimizeMemory() @@ -389,24 +390,76 @@ Node::NodeType PersistentStorage::getNodeTypeForNodeWithId(Id nodeId) const return Node::intToType(m_sqliteStorage.getNodeById(nodeId).type); } -std::shared_ptr PersistentStorage::getFullTextSearchLocations(const std::string& searchTerm) const +std::shared_ptr PersistentStorage::getFullTextSearchLocations( + const std::string& searchTerm, bool caseSensitive +) const { + if (m_fullTextSearchIndex.fileCount() == 0) + { + buildFullTextSearchIndex(); + } + std::shared_ptr collection = std::make_shared(); - std::vector parseLocations = m_sqliteStorage.getFullTextSearch(searchTerm); - size_t i = 0; - for(ParseLocation location : parseLocations) + std::vector hits = m_fullTextSearchIndex.searchForTerm(searchTerm); + + int termLength = searchTerm.length(); + FilePath filepath; + std::shared_ptr file; + ParseLocation location; + for (size_t i = 0; i < hits.size(); i++) { - collection->addTokenLocation( - i, - 0, - location.filePath, - location.startLineNumber, - location.startColumnNumber, - location.endLineNumber, - location.endColumnNumber - )->setType(LOCATION_FULLTEXTSEARCH_MATCH); - i++; + filepath = getFileNodePath(hits[i].fileId); + file = getFileContent(filepath); + + int charsInPreviousLines = 0; + int lineNumber = 1; + std::string line; + line = file->getLine(lineNumber); + + for (int pos : hits[i].positions) + { + bool addHit = true; + while( (charsInPreviousLines + (int)line.length()) < pos) + { + lineNumber++; + charsInPreviousLines += line.length(); + line = file->getLine(lineNumber); + } + location.startLineNumber = lineNumber; + location.startColumnNumber = pos - charsInPreviousLines + 1; + + if ( caseSensitive ) + { + if( line.substr(location.startColumnNumber-1, termLength) != searchTerm ) + { + addHit = false; + } + } + + while( (charsInPreviousLines + (int)line.length()) < pos + termLength) + { + lineNumber++; + charsInPreviousLines += line.length(); + line = file->getLine(lineNumber); + } + + location.endLineNumber = lineNumber; + location.endColumnNumber = pos + termLength - charsInPreviousLines; + + if ( addHit ) + { + collection->addTokenLocation( + i, + 0, + filepath, + location.startLineNumber, + location.startColumnNumber, + location.endLineNumber, + location.endColumnNumber + )->setType(LOCATION_FULLTEXT); + } + } } return collection; @@ -597,7 +650,6 @@ std::shared_ptr PersistentStorage::getGraphForActiveTokenIds(const std::v std::vector edgeIds; bool addAggregations = false; - //m_sqliteStorage.getFullTextSearch("const int"); if (tokenIds.size() == 1) { const Id elementId = tokenIds[0]; @@ -1252,6 +1304,14 @@ void PersistentStorage::buildFilePathMaps() } } +void PersistentStorage::buildFullTextSearchIndex() const +{ + for (StorageFile file : m_sqliteStorage.getAllFiles()) + { + m_fullTextSearchIndex.addFile(file.id, m_sqliteStorage.getFileContentById(file.id)->getText()); + } +} + void PersistentStorage::buildHierarchyCache() { std::vector memberEdges = m_sqliteStorage.getEdgesByType(Edge::typeToInt(Edge::EDGE_MEMBER)); @@ -1267,7 +1327,3 @@ void PersistentStorage::buildHierarchyCache() } } -void PersistentStorage::optimizeFTSTable() -{ - m_sqliteStorage.optimizeFTSTable(); -} diff --git a/src/lib/data/PersistentStorage.h b/src/lib/data/PersistentStorage.h index 451dfefb..180033fe 100644 --- a/src/lib/data/PersistentStorage.h +++ b/src/lib/data/PersistentStorage.h @@ -7,6 +7,7 @@ #include "utility/file/FilePath.h" #include "data/access/StorageAccess.h" +#include "data/fulltextsearch/FullTextSearchIndex.h" #include "data/graph/token_component/TokenComponentAccess.h" #include "data/location/TokenLocationCollection.h" #include "data/parser/ParserClient.h" @@ -47,9 +48,6 @@ public: virtual void startInjection(); virtual void finishInjection(); - - - FilePath getDbFilePath() const; Version getVersion() const; @@ -79,7 +77,8 @@ public: virtual NameHierarchy getNameHierarchyForNodeWithId(Id nodeId) const; virtual Node::NodeType getNodeTypeForNodeWithId(Id nodeId) const; - virtual std::shared_ptr getFullTextSearchLocations(const std::string& searchTerm) const; + virtual std::shared_ptr getFullTextSearchLocations( + const std::string& searchTerm, bool caseSensitive) const; virtual std::vector getAutocompletionMatches(const std::string& query) const; virtual std::vector getSearchMatchesForTokenIds(const std::vector& elementIds) const; @@ -135,8 +134,8 @@ private: void buildSearchIndex(); void buildFilePathMaps(); + void buildFullTextSearchIndex() const; void buildHierarchyCache(); - void optimizeFTSTable(); void log(std::string type, std::string str, const ParseLocation& location) const; @@ -145,6 +144,8 @@ private: SearchIndex m_commandIndex; SearchIndex m_elementIndex; + mutable FullTextSearchIndex m_fullTextSearchIndex; + SqliteStorage m_sqliteStorage; mutable std::map m_fileNodeIds; diff --git a/src/lib/data/SqliteStorage.cpp b/src/lib/data/SqliteStorage.cpp index aa52b735..21b49b9f 100644 --- a/src/lib/data/SqliteStorage.cpp +++ b/src/lib/data/SqliteStorage.cpp @@ -373,92 +373,6 @@ void SqliteStorage::optimizeMemory() const } } -void SqliteStorage::optimizeFTSTable() const -{ - try - { - CppSQLite3Query q = m_database.execQuery("INSERT INTO file(file) VALUES('optimize');"); - } - catch(CppSQLite3Exception e) - { - LOG_ERROR(e.errorMessage()); - } -} - -std::vector SqliteStorage::getFullTextSearch(const std::string& searchTerm) const -{ - std::vector matches; - try - { - CppSQLite3Query q = m_database.execQuery(( - "SELECT id,offsets(file) FROM file WHERE content MATCH '\"*" + searchTerm + "*\"'" - ).c_str()); - - while(!q.eof()) - { - const Id fileId = q.getIntField(0,0); - - // convert the string "0 2 0 2" to int vector - std::stringstream temp_results(q.getStringField(1,0)); - std::vector results((std::istream_iterator(temp_results)),std::istream_iterator()); - - ParseLocation location; - location.filePath = getFileById(fileId).filePath; - std::shared_ptr file = getFileContentByPath(location.filePath.str()); - - int charsInPreviousLines = 0; - int lineNumber = 1; - std::string line; - // results - // i ... col - // i+1 ... term - // i+2 ... offset - // i+3 ... length - line = file->getLine(lineNumber); - for (size_t i = 0; i < results.size() ; i+=4) - { - while( ((charsInPreviousLines + (int)line.length()) < results[i+2]) && results[i+1] == 0 ) - { - lineNumber++; - charsInPreviousLines += line.length(); - line = file->getLine(lineNumber); - } - - //only set start if its the first term of the match - if ( results[i+1] == 0 ) - { - location.startLineNumber = lineNumber; - // +1 to be consistent with the rest of the codebase - location.startColumnNumber = results[i+2] - charsInPreviousLines + 1; - } - - while( (charsInPreviousLines + (int)line.length()) < (results[i+2] + results[i+3]) ) - { - lineNumber++; - charsInPreviousLines += line.length(); - line = file->getLine(lineNumber); - } - - location.endLineNumber = lineNumber; - location.endColumnNumber = results[i+2] + results[i+3] - charsInPreviousLines; - - // add match if the next term is the first term of a match or its the last term in the file - if ( (i+4 < results.size() && results[i+5] == 0) || i+4 >= results.size() ) - { - matches.push_back(location); - } - } - - q.nextRow(); - } - } - catch(CppSQLite3Exception e) - { - LOG_ERROR(e.errorMessage()); - } - return matches; -} - StorageNode SqliteStorage::getNodeById(Id id) const { if (id != 0) @@ -518,6 +432,20 @@ std::vector SqliteStorage::getFilesByPaths(const std::vector("WHERE file.path IN ('" + utility::join(utility::toStrings(filePaths), "', '") + "')"); } +std::shared_ptr SqliteStorage::getFileContentById(Id fileId) const +{ + CppSQLite3Query q = m_database.execQuery(( + "SELECT content FROM file WHERE id = '" + std::to_string(fileId) + "';" + ).c_str()); + + if (!q.eof()) + { + return TextAccess::createFromString(q.getStringField(0, "")); + } + + return TextAccess::createFromString(""); +} + std::shared_ptr SqliteStorage::getFileContentByPath(const std::string& filePath) const { CppSQLite3Query q = m_database.execQuery(( @@ -704,54 +632,55 @@ void SqliteStorage::clearTables() void SqliteStorage::setupTables() { - m_database.execDML( - "CREATE TABLE IF NOT EXISTS meta(" - "id INTEGER, " - "key TEXT, " - "value TEXT, " - "PRIMARY KEY(id));" - ); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS element(" - "id INTEGER, " - "PRIMARY KEY(id));" - ); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS edge(" - "id INTEGER NOT NULL, " - "type INTEGER NOT NULL, " - "source_node_id INTEGER NOT NULL, " - "target_node_id INTEGER NOT NULL, " - "PRIMARY KEY(id), " - "FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE, " - "FOREIGN KEY(source_node_id) REFERENCES node(id) ON DELETE CASCADE, " - "FOREIGN KEY(target_node_id) REFERENCES node(id) ON DELETE CASCADE);" - ); - - m_database.execDML( // used for checking for duplicates during code analysis // TODO: move to createIndexesForAnalysis() or prepareForAnalysis - "CREATE INDEX IF NOT EXISTS edge_multipart_index ON edge(type, source_node_id, target_node_id);" - ); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS node(" - "id INTEGER NOT NULL, " - "type INTEGER NOT NULL, " - "serialized_name TEXT, " - "definition_type INTEGER NOT NULL, " - "PRIMARY KEY(id), " - "FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);" - ); - - m_database.execDML( - "CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);" - ); - try { m_database.execDML( - "CREATE VIRTUAL TABLE IF NOT EXISTS file USING fts4(" + "CREATE TABLE IF NOT EXISTS meta(" + "id INTEGER, " + "key TEXT, " + "value TEXT, " + "PRIMARY KEY(id));" + ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS element(" + "id INTEGER, " + "PRIMARY KEY(id));" + ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS edge(" + "id INTEGER NOT NULL, " + "type INTEGER NOT NULL, " + "source_node_id INTEGER NOT NULL, " + "target_node_id INTEGER NOT NULL, " + "PRIMARY KEY(id), " + "FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE, " + "FOREIGN KEY(source_node_id) REFERENCES node(id) ON DELETE CASCADE, " + "FOREIGN KEY(target_node_id) REFERENCES node(id) ON DELETE CASCADE);" + ); + + // TODO: move to createIndexesForAnalysis() or prepareForAnalysis + m_database.execDML( // used for checking for duplicates during code analysis + "CREATE INDEX IF NOT EXISTS edge_multipart_index ON edge(type, source_node_id, target_node_id);" + ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS node(" + "id INTEGER NOT NULL, " + "type INTEGER NOT NULL, " + "serialized_name TEXT, " + "definition_type INTEGER NOT NULL, " + "PRIMARY KEY(id), " + "FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);" + ); + + m_database.execDML( + "CREATE INDEX IF NOT EXISTS node_serialized_name_index ON node(serialized_name);" + ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS file(" "id INTEGER NOT NULL, " "path TEXT, " "modification_time TEXT, " @@ -760,75 +689,75 @@ void SqliteStorage::setupTables() "PRIMARY KEY(id), " "FOREIGN KEY(id) REFERENCES node(id) ON DELETE CASCADE);" ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS local_symbol(" + "id INTEGER NOT NULL, " + "name TEXT, " + "PRIMARY KEY(id), " + "FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);" + ); + + m_database.execDML( + "CREATE INDEX IF NOT EXISTS local_symbol_name_index ON local_symbol(name);" + ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS source_location(" + "id INTEGER NOT NULL, " + "element_id INTEGER, " + "file_node_id INTEGER, " + "start_line INTEGER, " + "start_column INTEGER, " + "end_line INTEGER, " + "end_column INTEGER, " + "type INTEGER, " + "PRIMARY KEY(id), " + "FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, " + "FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);" + ); + + SqliteIndex("source_location_element_id_index", "source_location(element_id)").createOnDatabase(m_database); + SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS component_access(" + "id INTEGER NOT NULL, " + "edge_id INTEGER, " + "type INTEGER NOT NULL, " + "PRIMARY KEY(id), " + "FOREIGN KEY(edge_id) REFERENCES edge(id) ON DELETE CASCADE);" + ); + + SqliteIndex("component_access_edge_id_index", "component_access(edge_id)").createOnDatabase(m_database); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS comment_location(" + "id INTEGER NOT NULL, " + "file_node_id INTEGER, " + "start_line INTEGER, " + "start_column INTEGER, " + "end_line INTEGER, " + "end_column INTEGER, " + "PRIMARY KEY(id), " + "FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);" + ); + + m_database.execDML( + "CREATE TABLE IF NOT EXISTS error(" + "id INTEGER NOT NULL, " + "message TEXT, " + "fatal INTEGER NOT NULL, " + "file_path TEXT, " + "line_number INTEGER, " + "column_number INTEGER, " + "PRIMARY KEY(id));" + ); } catch (CppSQLite3Exception& e) { LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage()); } - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS local_symbol(" - "id INTEGER NOT NULL, " - "name TEXT, " - "PRIMARY KEY(id), " - "FOREIGN KEY(id) REFERENCES element(id) ON DELETE CASCADE);" - ); - - m_database.execDML( - "CREATE INDEX IF NOT EXISTS local_symbol_name_index ON local_symbol(name);" - ); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS source_location(" - "id INTEGER NOT NULL, " - "element_id INTEGER, " - "file_node_id INTEGER, " - "start_line INTEGER, " - "start_column INTEGER, " - "end_line INTEGER, " - "end_column INTEGER, " - "type INTEGER, " - "PRIMARY KEY(id), " - "FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE, " - "FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);" - ); - - SqliteIndex("source_location_element_id_index", "source_location(element_id)").createOnDatabase(m_database); - SqliteIndex("source_location_file_node_id_index", "source_location(file_node_id)").createOnDatabase(m_database); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS component_access(" - "id INTEGER NOT NULL, " - "edge_id INTEGER, " - "type INTEGER NOT NULL, " - "PRIMARY KEY(id), " - "FOREIGN KEY(edge_id) REFERENCES edge(id) ON DELETE CASCADE);" - ); - - SqliteIndex("component_access_edge_id_index", "component_access(edge_id)").createOnDatabase(m_database); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS comment_location(" - "id INTEGER NOT NULL, " - "file_node_id INTEGER, " - "start_line INTEGER, " - "start_column INTEGER, " - "end_line INTEGER, " - "end_column INTEGER, " - "PRIMARY KEY(id), " - "FOREIGN KEY(file_node_id) REFERENCES node(id) ON DELETE CASCADE);" - ); - - m_database.execDML( - "CREATE TABLE IF NOT EXISTS error(" - "id INTEGER NOT NULL, " - "message TEXT, " - "fatal INTEGER NOT NULL, " - "file_path TEXT, " - "line_number INTEGER, " - "column_number INTEGER, " - "PRIMARY KEY(id));" - ); } bool SqliteStorage::hasTable(const std::string& tableName) const diff --git a/src/lib/data/SqliteStorage.h b/src/lib/data/SqliteStorage.h index 9cd191bd..0a847bc6 100644 --- a/src/lib/data/SqliteStorage.h +++ b/src/lib/data/SqliteStorage.h @@ -81,6 +81,7 @@ public: std::vector getFilesByPaths(const std::vector& filePaths) const; std::shared_ptr getFileContentByPath(const std::string& filePath) const; + std::shared_ptr getFileContentById(Id fileId) const; void setNodeType(int type, Id nodeId); void setNodeDefinitionType(int definitionType, Id nodeId); @@ -98,7 +99,6 @@ public: void optimizeMemory() const; std::vector getFullTextSearch(const std::string& searchTerm) const; - void optimizeFTSTable() const; std::vector getCommentLocationsInFile(const FilePath& filePath) const; std::vector getFatalErrors() const; diff --git a/src/lib/data/access/StorageAccess.h b/src/lib/data/access/StorageAccess.h index d17fcdab..56d023e0 100644 --- a/src/lib/data/access/StorageAccess.h +++ b/src/lib/data/access/StorageAccess.h @@ -36,7 +36,7 @@ public: virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const = 0; virtual std::shared_ptr getFullTextSearchLocations( - const std::string& searchTerm) const = 0; + const std::string& searchTerm, bool caseSensitive) const = 0; virtual std::vector getAutocompletionMatches(const std::string& query) const = 0; virtual std::vector getSearchMatchesForTokenIds(const std::vector& tokenIds) const = 0; diff --git a/src/lib/data/access/StorageAccessProxy.cpp b/src/lib/data/access/StorageAccessProxy.cpp index 8251c656..2ceb965b 100644 --- a/src/lib/data/access/StorageAccessProxy.cpp +++ b/src/lib/data/access/StorageAccessProxy.cpp @@ -84,11 +84,11 @@ std::vector StorageAccessProxy::getAutocompletionMatches(const std: } std::shared_ptr StorageAccessProxy::getFullTextSearchLocations( - const std::string &searchTerm) const + const std::string &searchTerm, bool caseSensitive) const { if (hasSubject()) { - return m_subject->getFullTextSearchLocations(searchTerm); + return m_subject->getFullTextSearchLocations(searchTerm, caseSensitive); } return std::make_shared(); diff --git a/src/lib/data/access/StorageAccessProxy.h b/src/lib/data/access/StorageAccessProxy.h index d7c3a924..241d987c 100644 --- a/src/lib/data/access/StorageAccessProxy.h +++ b/src/lib/data/access/StorageAccessProxy.h @@ -20,7 +20,8 @@ public: virtual NameHierarchy getNameHierarchyForNodeWithId(Id id) const; virtual Node::NodeType getNodeTypeForNodeWithId(Id id) const; - virtual std::shared_ptr getFullTextSearchLocations(const std::string& searchTerm) const; + virtual std::shared_ptr getFullTextSearchLocations( + const std::string& searchTerm, bool caseSensitive) const; virtual std::vector getAutocompletionMatches(const std::string& query) const; virtual std::vector getSearchMatchesForTokenIds(const std::vector& tokenIds) const; diff --git a/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp b/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp new file mode 100644 index 00000000..0eba07d9 --- /dev/null +++ b/src/lib/data/fulltextsearch/FullTextSearchIndex.cpp @@ -0,0 +1,45 @@ +#include "data/fulltextsearch/FullTextSearchIndex.h" +#include + +#include "utility/utility.h" +#include "utility/logging/logging.h" + +void FullTextSearchIndex::addFile(Id fileId, const std::string& file) +{ + if( file.empty() ) + { + LOG_ERROR("empty file not added to fulltextsearch index"); + } + + if ( file.size() >= std::numeric_limits::max() ) + { + LOG_ERROR("file too big not added to fulltextsearch index"); + } + + FullTextSearchFile fts_file(fileId, SuffixArray(file)); + m_files.push_back(fts_file); +} + +std::vector FullTextSearchIndex::searchForTerm(const std::string& term) const +{ + std::vector ret; + FullTextSearchResult hit; + for (auto f : m_files) + { + hit.fileId = f.fileId; + hit.positions = f.array.searchForTerm(term); + ret.push_back(hit); + } + return ret; +} + +size_t FullTextSearchIndex::fileCount() const +{ + return m_files.size(); +} + +void FullTextSearchIndex::clear() +{ + m_files.clear(); +} + diff --git a/src/lib/data/fulltextsearch/FullTextSearchIndex.h b/src/lib/data/fulltextsearch/FullTextSearchIndex.h new file mode 100644 index 00000000..cb90f1bb --- /dev/null +++ b/src/lib/data/fulltextsearch/FullTextSearchIndex.h @@ -0,0 +1,44 @@ +#ifndef FULLTEXTSEARCH_INDEX_H +#define FULLTEXTSEARCH_INDEX_H + +#include +#include + +#include "data/fulltextsearch/SuffixArray.h" +#include "utility/types.h" + +class StorageAccess; + +// contains all fulltextsearch results of one file +struct FullTextSearchResult +{ + Id fileId; + std::vector positions; +}; + +struct FullTextSearchFile +{ + FullTextSearchFile(Id fileId, SuffixArray array) + : fileId(fileId) + , array(array) + { + }; + Id fileId; + SuffixArray array; +}; + +class FullTextSearchIndex +{ +public: + void addFile(Id fileId, const std::string& file); + std::vector searchForTerm(const std::string& term) const; + + size_t fileCount() const; + + void clear(); + +private: + std::vector m_files; +}; + +#endif // FULLTEXTSEARCH_INDEX_H diff --git a/src/lib/data/fulltextsearch/SuffixArray.cpp b/src/lib/data/fulltextsearch/SuffixArray.cpp new file mode 100644 index 00000000..eee8b602 --- /dev/null +++ b/src/lib/data/fulltextsearch/SuffixArray.cpp @@ -0,0 +1,182 @@ +#include "SuffixArray.h" + +#include +#include + +struct suffix +{ + int index; + int rank[2]; +}; + +int SuffixArray::cmp(struct suffix a, struct suffix b) +{ + return (a.rank[0] == b.rank[0])? (a.rank[1] < b.rank[1] ?1: 0): + (a.rank[0] < b.rank[0] ?1: 0); +} + +SuffixArray::SuffixArray(const std::string& text) + : m_text(text) +{ + std::transform(m_text.begin(), m_text.end(), m_text.begin(), ::tolower); + m_array = buildSuffixArray(); + m_lcp = buildLCP(); +} + +void SuffixArray::printArray() const +{ + std::cout << "Suffix Array : \n"; + printArr(m_array); +} + +void SuffixArray::printLCP() const +{ + std::cout << "\nLCP Array : \n"; + printArr(m_lcp); +} + +std::vector SuffixArray::buildLCP() +{ + int n = m_array.size(); + + std::vector lcp(n, 0); + std::vector invSuff(n, 0); + + for (int i=0; i < n; i++) + { + invSuff[m_array[i]] = i; + } + + int k = 0; + + for (int i=0; i0) + { + k--; + } + } + + return lcp; +} + +std::vector SuffixArray::searchForTerm(const std::string& searchTerm) const +{ + std::string term = searchTerm; + std::transform(term.begin(), term.end(), term.begin(), ::tolower); + + int termLength = term.length(); + int l = 0; + int r = m_text.length()-1; + int m; + + std::vector matches; + int compareResult; + while (l+1 < r) + { + m = (l+r+1)/2; + compareResult = term.compare(m_text.substr(m_array[m], termLength)); + if( compareResult < 0) + { + r = m; + } + else if (compareResult > 0) + { + l = m; + } + else + { + matches.push_back(m_array[m]); + for (int lower = m-1; m_lcp[lower] >= termLength; lower--) + { + matches.push_back(m_array[lower]); + } + for (int higher = m+1; m_lcp[higher-1] >= termLength; higher++) + { + matches.push_back(m_array[higher]); + } + break; + } + } + + std::sort(matches.begin(), matches.end()); + + return matches; +} + +std::vector SuffixArray::buildSuffixArray() +{ + int n = m_text.length(); + std::vector suffixes; + suffixes.reserve(n); + + suffix s; + for (int i = 0; i < n; i++) + { + s.index = i; + s.rank[0] = m_text[i] - 'a'; + s.rank[1] = ((i+1) < n)? (m_text[i + 1] - 'a'): -1; + suffixes.push_back(s); + } + + std::sort(suffixes.begin(), suffixes.end(), SuffixArray::cmp); + + std::vector ind (n,0); + for (int k = 4; k < 2*n; k = k*2) + { + int rank = 0; + int prev_rank = suffixes[0].rank[0]; + suffixes[0].rank[0] = rank; + ind[suffixes[0].index] = 0; + + for (int i = 1; i < n; i++) + { + if (suffixes[i].rank[0] == prev_rank && + suffixes[i].rank[1] == suffixes[i-1].rank[1]) + { + prev_rank = suffixes[i].rank[0]; + suffixes[i].rank[0] = rank; + } + else + { + prev_rank = suffixes[i].rank[0]; + suffixes[i].rank[0] = ++rank; + } + ind[suffixes[i].index] = i; + } + + for (int i = 0; i < n; i++) + { + int nextindex = suffixes[i].index + k/2; + suffixes[i].rank[1] = (nextindex < n)? + suffixes[ind[nextindex]].rank[0]: -1; + } + + std::sort(suffixes.begin(), suffixes.end(), cmp); + } + + std::vectorsuffixArr; + for (int i = 0; i < n; i++) + { + suffixArr.push_back(suffixes[i].index); + } + + return suffixArr; +} + + diff --git a/src/lib/data/fulltextsearch/SuffixArray.h b/src/lib/data/fulltextsearch/SuffixArray.h new file mode 100644 index 00000000..39a59177 --- /dev/null +++ b/src/lib/data/fulltextsearch/SuffixArray.h @@ -0,0 +1,36 @@ +#ifndef SUFFIX_ARRAY_H +#define SUFFIX_ARRAY_H + +#include +#include +#include + +class SuffixArray +{ +public: + SuffixArray(const std::string& text); + std::vector searchForTerm(const std::string& searchTerm) const; + static int cmp(struct suffix a, struct suffix b); + + void printArray() const; + void printLCP() const; + +private: + template + void printArr(std::vectorarr) const + { + for (size_t i = 0; i < arr.size(); i++) + { + std::cout << arr[i] << " "; + } + std::cout << std::endl; + } + + std::vector buildLCP(); + std::vector buildSuffixArray(); + std::vector m_array; + std::vector m_lcp; + std::string m_text; +}; + +#endif // SUFFIX_ARRAY_H diff --git a/src/lib/data/location/LocationType.cpp b/src/lib/data/location/LocationType.cpp index 64c983ac..2cbbff42 100644 --- a/src/lib/data/location/LocationType.cpp +++ b/src/lib/data/location/LocationType.cpp @@ -10,7 +10,7 @@ int locationTypeToInt(LocationType type) return 1; case LOCATION_LOCAL_SYMBOL: return 2; - case LOCATION_FULLTEXTSEARCH_MATCH: + case LOCATION_FULLTEXT: return 3; } } @@ -26,7 +26,7 @@ LocationType intToLocationType(int value) case 2: return LOCATION_LOCAL_SYMBOL; case 3: - return LOCATION_FULLTEXTSEARCH_MATCH; + return LOCATION_FULLTEXT; } return LOCATION_TOKEN; } diff --git a/src/lib/data/location/LocationType.h b/src/lib/data/location/LocationType.h index 09b8fdcf..fea30ebb 100644 --- a/src/lib/data/location/LocationType.h +++ b/src/lib/data/location/LocationType.h @@ -6,7 +6,7 @@ enum LocationType LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_LOCAL_SYMBOL, - LOCATION_FULLTEXTSEARCH_MATCH + LOCATION_FULLTEXT }; int locationTypeToInt(LocationType type); diff --git a/src/lib/data/location/TokenLocation.cpp b/src/lib/data/location/TokenLocation.cpp index 26b7499a..0236c5b6 100644 --- a/src/lib/data/location/TokenLocation.cpp +++ b/src/lib/data/location/TokenLocation.cpp @@ -184,7 +184,7 @@ bool TokenLocation::isScopeTokenLocation() const bool TokenLocation::isFullTextSearchMatch() const { - return m_type == LOCATION_FULLTEXTSEARCH_MATCH; + return m_type == LOCATION_FULLTEXT; } std::ostream& operator<<(std::ostream& ostream, const TokenLocation& location) diff --git a/src/lib/utility/messaging/type/MessageSearchFullText.h b/src/lib/utility/messaging/type/MessageSearchFullText.h index 695472ce..c2bbe481 100644 --- a/src/lib/utility/messaging/type/MessageSearchFullText.h +++ b/src/lib/utility/messaging/type/MessageSearchFullText.h @@ -7,8 +7,9 @@ class MessageSearchFullText: public Message { public: - MessageSearchFullText(const std::string& searchTerm) + MessageSearchFullText(const std::string& searchTerm, bool caseSensitive = false) : searchTerm(searchTerm) + , caseSensitive(caseSensitive) { } @@ -23,6 +24,7 @@ public: } const std::string searchTerm; + bool caseSensitive; }; #endif // MESSAGE_SEARCH_FULLTEXT_H diff --git a/src/lib_gui/qt/element/QtCodeArea.cpp b/src/lib_gui/qt/element/QtCodeArea.cpp index 134c1dc5..754a3e63 100644 --- a/src/lib_gui/qt/element/QtCodeArea.cpp +++ b/src/lib_gui/qt/element/QtCodeArea.cpp @@ -938,7 +938,7 @@ const QtCodeArea::AnnotationColor& QtCodeArea::getAnnotationColorForAnnotation(c { i = 9; } - else if (annotation.locationType == LOCATION_FULLTEXTSEARCH_MATCH) + else if (annotation.locationType == LOCATION_FULLTEXT) { i = 12; } diff --git a/src/lib_gui/qt/element/QtSmartSearchBox.cpp b/src/lib_gui/qt/element/QtSmartSearchBox.cpp index 3aad0af9..5ee27a52 100644 --- a/src/lib_gui/qt/element/QtSmartSearchBox.cpp +++ b/src/lib_gui/qt/element/QtSmartSearchBox.cpp @@ -33,8 +33,18 @@ void QtSearchElement::onChecked(bool) void QtSmartSearchBox::fullTextSearch() { - LOG_INFO_STREAM(<< "FullTextsearch: " << text().toStdString().substr(1)); - MessageSearchFullText(text().toStdString().substr(1)).dispatch(); + std::string term = text().toStdString().substr(1); + if(term.at(0) == '@') + { + term = term.substr(1); + LOG_INFO_STREAM(<< "FullTextsearch(case sensitive): " << term); + MessageSearchFullText(term, true).dispatch(); + } + else + { + LOG_INFO_STREAM(<< "FullTextsearch: " << term); + MessageSearchFullText(term).dispatch(); + } } void QtSmartSearchBox::search() diff --git a/src/test/StorageTestSuite.h b/src/test/StorageTestSuite.h index 1f8b911a..c7f203ce 100644 --- a/src/test/StorageTestSuite.h +++ b/src/test/StorageTestSuite.h @@ -233,6 +233,11 @@ private: clear(); } + ~TestStorage() + { + boost::filesystem::remove("data/test.sqlite"); + } + std::shared_ptr getLocationCollectionForTokenId(Id id) const { std::vector tokenIds;