diff --git a/script/download_python_indexer.sh b/script/download_python_indexer.sh index 79921693..6693b6c4 100755 --- a/script/download_python_indexer.sh +++ b/script/download_python_indexer.sh @@ -1,6 +1,6 @@ #!/bin/bash -SOURCETRAIL_PYTHON_INDEXER_VERSION="v0_db23_p4" +SOURCETRAIL_PYTHON_INDEXER_VERSION="v0_db24_p0" # Determine current platform PLATFORM='unknown' @@ -14,7 +14,7 @@ fi PACKAGE_NAME="SourcetrailPythonIndexer_${SOURCETRAIL_PYTHON_INDEXER_VERSION}-${PLATFORM}" PACKAGE_FILE_NAME="${PACKAGE_NAME}.zip" -PACKAGE_URL="https://github.com/CoatiSoftware/SourcetrailPythonIndexer/releases/download/${SOURCETRAIL_PYTHON_INDEXER_VERSION}/${PACKAGE_FILE_NAME}" +PACKAGE_URL="https://github.com/CoatiSoftware/SourcetrailPythonIndexer/releases/download/${SOURCETRAIL_PYTHON_INDEXER_VERSION}_beta/${PACKAGE_FILE_NAME}" TEMP_PATH="build/temp" TARGET_PATH="bin/app/data/python" diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index e91597b4..4028b8ec 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -138,11 +138,14 @@ add_files( data/graph/token_component/TokenComponentFilePath.cpp data/graph/token_component/TokenComponentFilePath.h data/graph/token_component/TokenComponentInheritanceChain.h + data/graph/token_component/TokenComponentIsAmbiguous.h data/graph/token_component/TokenComponentStatic.cpp data/graph/token_component/TokenComponentStatic.h data/graph/Edge.cpp data/graph/Edge.h + data/graph/ElementComponentKind.cpp + data/graph/ElementComponentKind.h data/graph/Graph.cpp data/graph/Graph.h data/graph/Node.cpp @@ -250,6 +253,7 @@ add_files( data/storage/type/StorageBookmarkedNode.h data/storage/type/StorageComponentAccess.h data/storage/type/StorageEdge.h + data/storage/type/StorageElementComponent.h data/storage/type/StorageError.h data/storage/type/StorageFile.h data/storage/type/StorageLocalSymbol.h diff --git a/src/lib/component/controller/ActivationController.cpp b/src/lib/component/controller/ActivationController.cpp index f1eac641..aea80566 100644 --- a/src/lib/component/controller/ActivationController.cpp +++ b/src/lib/component/controller/ActivationController.cpp @@ -12,6 +12,7 @@ #include "MessageRefreshUI.h" #include "MessageStatus.h" #include "MessageScrollToLine.h" +#include "MessageTooltipShow.h" #include "utility.h" ActivationController::ActivationController(StorageAccess* storageAccess) @@ -109,6 +110,16 @@ void ActivationController::handleMessage(MessageActivateSourceLocations* message { msg.addNode(nodeId); } + + if (message->containsUnsolvedLocations && msg.nodes.size() == 1 && + m_storageAccess->getNameHierarchyForNodeId(msg.nodes[0].nodeId).getQualifiedName() == L"unsolved symbol") + { + MessageTooltipShow m(message->locationIds, {}, TOOLTIP_ORIGIN_CODE); + m.force = true; + m.dispatch(); + return; + } + msg.setSchedulerId(message->getSchedulerId()); msg.dispatchImmediately(); } diff --git a/src/lib/component/controller/IDECommunicationController.cpp b/src/lib/component/controller/IDECommunicationController.cpp index 56643664..3f8d648b 100644 --- a/src/lib/component/controller/IDECommunicationController.cpp +++ b/src/lib/component/controller/IDECommunicationController.cpp @@ -100,7 +100,8 @@ void IDECommunicationController::handleSetActiveTokenMessage( { const SourceLocation* endLocation = startLocation->getEndLocation(); - if ((startLocation->getType() == LOCATION_TOKEN || startLocation->getType() == LOCATION_QUALIFIER) + if ((startLocation->getType() == LOCATION_TOKEN || startLocation->getType() == LOCATION_QUALIFIER + || startLocation->getType() == LOCATION_UNSOLVED) && startLocation->getLineNumber() == endLocation->getLineNumber() && startLocation->getColumnNumber() <= cursorColumn && endLocation->getColumnNumber() + 1 >= cursorColumn) diff --git a/src/lib/component/controller/TooltipController.cpp b/src/lib/component/controller/TooltipController.cpp index 9f56f8b8..fac58578 100644 --- a/src/lib/component/controller/TooltipController.cpp +++ b/src/lib/component/controller/TooltipController.cpp @@ -91,9 +91,9 @@ void TooltipController::handleMessage(MessageTooltipShow* message) // If a tooltip list would only display one token, then just activate it instead. // This can happen when edges pointing to the token use the same source location e.g. override edges - if (info.snippets.size() == 1) + if (!message->force && info.snippets.size() == 1) { - MessageActivateSourceLocations(message->sourceLocationIds).dispatch(); + MessageActivateSourceLocations(message->sourceLocationIds, false).dispatch(); } else if (info.snippets.size()) { diff --git a/src/lib/component/view/GraphViewStyle.cpp b/src/lib/component/view/GraphViewStyle.cpp index 41b67d36..845d0546 100644 --- a/src/lib/component/view/GraphViewStyle.cpp +++ b/src/lib/component/view/GraphViewStyle.cpp @@ -609,7 +609,7 @@ GraphViewStyle::NodeStyle GraphViewStyle::getStyleOfGroupNode(GroupType type, bo } GraphViewStyle::EdgeStyle GraphViewStyle::getStyleForEdgeType( - Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge) + Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge, bool isAmbiguous) { EdgeStyle style; @@ -706,6 +706,11 @@ GraphViewStyle::EdgeStyle GraphViewStyle::getStyleForEdgeType( break; } + if (isAmbiguous) + { + style.dashed = true; + } + return style; } diff --git a/src/lib/component/view/GraphViewStyle.h b/src/lib/component/view/GraphViewStyle.h index 734dac49..b5862cc3 100644 --- a/src/lib/component/view/GraphViewStyle.h +++ b/src/lib/component/view/GraphViewStyle.h @@ -130,7 +130,8 @@ public: static NodeStyle getStyleOfTextNode(int fontSizeDiff); static NodeStyle getStyleOfGroupNode(GroupType type, bool isFocused); - static EdgeStyle getStyleForEdgeType(Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge); + static EdgeStyle getStyleForEdgeType( + Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge, bool isAmbiguous); static int toGridOffset(int x); static int toGridSize(int x); diff --git a/src/lib/data/graph/ElementComponentKind.cpp b/src/lib/data/graph/ElementComponentKind.cpp new file mode 100644 index 00000000..27feed02 --- /dev/null +++ b/src/lib/data/graph/ElementComponentKind.cpp @@ -0,0 +1,23 @@ +#include "ElementComponentKind.h" + +int elementComponentKindToInt(ElementComponentKind kind) +{ + return static_cast(kind); +} + +ElementComponentKind intToElementComponentKind(int value) +{ + const ElementComponentKind kinds[] = { + ElementComponentKind::IS_AMBIGUOUS + }; + + for (ElementComponentKind kind : kinds) + { + if (value == elementComponentKindToInt(kind)) + { + return kind; + } + } + + return ElementComponentKind::NONE; +} diff --git a/src/lib/data/graph/ElementComponentKind.h b/src/lib/data/graph/ElementComponentKind.h new file mode 100644 index 00000000..378f9b30 --- /dev/null +++ b/src/lib/data/graph/ElementComponentKind.h @@ -0,0 +1,13 @@ +#ifndef ELEMENT_COMPONENT_KIND_H +#define ELEMENT_COMPONENT_KIND_H + +enum class ElementComponentKind +{ + NONE = 0, + IS_AMBIGUOUS = 1 +}; + +int elementComponentKindToInt(ElementComponentKind type); +ElementComponentKind intToElementComponentKind(int value); + +#endif // ELEMENT_COMPONENT_KIND_H diff --git a/src/lib/data/graph/token_component/TokenComponentIsAmbiguous.h b/src/lib/data/graph/token_component/TokenComponentIsAmbiguous.h new file mode 100644 index 00000000..237eb9ea --- /dev/null +++ b/src/lib/data/graph/token_component/TokenComponentIsAmbiguous.h @@ -0,0 +1,16 @@ +#ifndef TOKEN_COMPONENT_IS_AMBIGUOUS_H +#define TOKEN_COMPONENT_IS_AMBIGUOUS_H + +#include "TokenComponent.h" + +class TokenComponentIsAmbiguous + : public TokenComponent +{ +public: + inline virtual std::shared_ptr copy() const + { + return std::make_shared(*this); + } +}; + +#endif // TOKEN_COMPONENT_IS_AMBIGUOUS_H diff --git a/src/lib/data/indexer/IndexerCommandCustom.cpp b/src/lib/data/indexer/IndexerCommandCustom.cpp index 18efd635..6397c695 100644 --- a/src/lib/data/indexer/IndexerCommandCustom.cpp +++ b/src/lib/data/indexer/IndexerCommandCustom.cpp @@ -19,6 +19,26 @@ IndexerCommandCustom::IndexerCommandCustom( bool runInParallel ) : IndexerCommand(sourceFilePath) + , m_type(getStaticIndexerCommandType()) + , m_customCommand(customCommand) + , m_projectFilePath(projectFilePath) + , m_databaseFilePath(databaseFilePath) + , m_databaseVersion(databaseVersion) + , m_runInParallel(runInParallel) +{ +} + +IndexerCommandCustom::IndexerCommandCustom( + IndexerCommandType type, + const std::wstring& customCommand, + const FilePath& projectFilePath, + const FilePath& databaseFilePath, + const std::wstring& databaseVersion, + const FilePath& sourceFilePath, + bool runInParallel +) + : IndexerCommand(sourceFilePath) + , m_type(type) , m_customCommand(customCommand) , m_projectFilePath(projectFilePath) , m_databaseFilePath(databaseFilePath) @@ -29,7 +49,7 @@ IndexerCommandCustom::IndexerCommandCustom( IndexerCommandType IndexerCommandCustom::getIndexerCommandType() const { - return getStaticIndexerCommandType(); + return m_type; } size_t IndexerCommandCustom::getByteSize(size_t stringSize) const diff --git a/src/lib/data/indexer/IndexerCommandCustom.h b/src/lib/data/indexer/IndexerCommandCustom.h index e217056f..679af1d9 100644 --- a/src/lib/data/indexer/IndexerCommandCustom.h +++ b/src/lib/data/indexer/IndexerCommandCustom.h @@ -13,7 +13,16 @@ public: static IndexerCommandType getStaticIndexerCommandType(); IndexerCommandCustom( - const std::wstring& customCommand, + const std::wstring& customCommand, + const FilePath& projectFilePath, + const FilePath& databaseFilePath, + const std::wstring& databaseVersion, + const FilePath& sourceFilePath, + bool runInParallel); + + IndexerCommandCustom( + IndexerCommandType type, + const std::wstring& customCommand, const FilePath& projectFilePath, const FilePath& databaseFilePath, const std::wstring& databaseVersion, @@ -33,6 +42,7 @@ protected: QJsonObject doSerialize() const override; private: + IndexerCommandType m_type; std::wstring m_customCommand; FilePath m_projectFilePath; FilePath m_databaseFilePath; diff --git a/src/lib/data/indexer/IndexerCommandType.cpp b/src/lib/data/indexer/IndexerCommandType.cpp index dfe774f8..d0272d10 100644 --- a/src/lib/data/indexer/IndexerCommandType.cpp +++ b/src/lib/data/indexer/IndexerCommandType.cpp @@ -22,7 +22,6 @@ IndexerCommandType stringToIndexerCommandType(const std::string& s) { if (s == indexerCommandTypeToString(INDEXER_COMMAND_CXX)) return INDEXER_COMMAND_CXX; if (s == indexerCommandTypeToString(INDEXER_COMMAND_JAVA)) return INDEXER_COMMAND_JAVA; - if (s == indexerCommandTypeToString(INDEXER_COMMAND_PYTHON)) return INDEXER_COMMAND_PYTHON; if (s == indexerCommandTypeToString(INDEXER_COMMAND_CUSTOM)) return INDEXER_COMMAND_CUSTOM; return INDEXER_COMMAND_UNKNOWN; } diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp index 6c7f4492..ba450f4a 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.cpp +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.cpp @@ -1,7 +1,9 @@ #include "TaskExecuteCustomCommands.h" +#include "ApplicationSettings.h" #include "Blackboard.h" #include "DialogView.h" +#include "ElementComponentKind.h" #include "FileSystem.h" #include "IndexerCommandCustom.h" #include "IndexerCommandProvider.h" @@ -9,6 +11,9 @@ #include "MessageShowStatus.h" #include "MessageStatus.h" #include "PersistentStorage.h" +#include "SourceLocationCollection.h" +#include "SourceLocationFile.h" +#include "TextAccess.h" #include "utility.h" #include "utilityApp.h" #include "utilityString.h" @@ -26,6 +31,7 @@ TaskExecuteCustomCommands::TaskExecuteCustomCommands( , m_indexerThreadCount(indexerThreadCount) , m_projectDirectory(projectDirectory) , m_indexerCommandCount(m_indexerCommandProvider->size()) + , m_hasPythonCommands(false) { } @@ -46,6 +52,11 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr blackboard) m_targetDatabaseFilePath = indexerCommand->getDatabaseFilePath(); } + if (indexerCommand->getIndexerCommandType() == INDEXER_COMMAND_PYTHON) + { + m_hasPythonCommands = true; + } + if (indexerCommand->getRunInParallel()) { m_parallelCommands.push_back(indexerCommand); @@ -66,7 +77,7 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr return STATE_SUCCESS; } - m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), { }); + m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), {}); std::vector> indexerThreads; @@ -105,6 +116,11 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr } FileSystem::remove(sourceDatabaseFilePath); } + + if (m_hasPythonCommands && ApplicationSettings::getInstance()->getPythonPostProcessingEnabled()) + { + runPythonPostProcessing(targetStorage); + } } return STATE_SUCCESS; @@ -143,7 +159,6 @@ void TaskExecuteCustomCommands::executeParallelIndexerCommands(int threadId, std } indexerCommand = m_parallelCommands.back(); m_parallelCommands.pop_back(); - } if (threadId != 0) @@ -235,4 +250,115 @@ void TaskExecuteCustomCommands::runIndexerCommand(std::shared_ptrupdate("indexed_source_file_count", [=](int count) { return count + 1; }); } -} \ No newline at end of file +} + +void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& storage) +{ + LOG_INFO("Starting Python post processing."); + + std::vector unsolvedLocationIds; + for (const StorageSourceLocation location : storage.getStorageSourceLocations()) + { + if (intToLocationType(location.type) == LOCATION_UNSOLVED) + { + unsolvedLocationIds.push_back(location.id); + } + } + + std::shared_ptr locationCollection = storage.getSourceLocationsForLocationIds(unsolvedLocationIds); + + std::map> nodeNameToStorageNodes; + if (locationCollection->getSourceLocationCount() > 0) + { + for (const StorageNode& node : storage.getStorageNodes()) + { + nodeNameToStorageNodes[NameHierarchy::deserialize(node.serializedName).back().getName()].push_back(node); + } + } + + struct DataToInsert + { + StorageEdgeData edgeData; + Id sourceLocationId; + }; + + storage.setMode(SqliteIndexStorage::STORAGE_MODE_READ); + + std::vector dataToInsert; + std::set elementsToDelete; + locationCollection->forEachSourceLocationFile([&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr locationFile) + { + std::shared_ptr textAccess = TextAccess::createFromFile(locationFile->getFilePath()); + + if (textAccess) + { + locationFile->forEachStartSourceLocation([textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc) + { + if (!startLoc) + { + return; + } + const SourceLocation* endLoc = startLoc->getOtherLocation(); + if (!endLoc) + { + return; + } + + const std::wstring token = utility::decodeFromUtf8(textAccess->getLine(startLoc->getLineNumber()).substr(startLoc->getColumnNumber() - 1, endLoc->getColumnNumber() - startLoc->getColumnNumber() + 1)); + + for (const Id tokenId : startLoc->getTokenIds()) + { + const StorageEdge edge = storage.getEdgeById(tokenId); + if (edge.id != 0) + { + for (const StorageNode& targetNode : nodeNameToStorageNodes[token]) + { + if (Edge::intToType(edge.type) == Edge::EDGE_CALL && + ( + NodeType::intToType(targetNode.type) != NodeType::NODE_FUNCTION || + NodeType::intToType(targetNode.type) != NodeType::NODE_METHOD + ) + ){ + continue; + } + dataToInsert.push_back({ StorageEdgeData(edge.type, edge.sourceNodeId, targetNode.id) , startLoc->getLocationId() }); + elementsToDelete.insert(edge.id); + } + } + } + } + ); + } + } + ); + + storage.setMode(SqliteIndexStorage::STORAGE_MODE_WRITE); + + storage.startInjection(); + + std::vector edgesToInsert; + for (const DataToInsert& data : dataToInsert) + { + edgesToInsert.push_back(StorageEdge(0, data.edgeData)); + } + const std::vector ambiguousEdgeIds = storage.addEdges(edgesToInsert); + + if (ambiguousEdgeIds.size() == dataToInsert.size()) + { + for (size_t i = 0; i < ambiguousEdgeIds.size(); i++) + { + storage.addElementComponent(StorageElementComponentData(ambiguousEdgeIds[i], elementComponentKindToInt(ElementComponentKind::IS_AMBIGUOUS), L"")); + storage.addOccurrence(StorageOccurrence(ambiguousEdgeIds[i], dataToInsert[i].sourceLocationId)); + } + + storage.removeElements(utility::toVector(elementsToDelete)); + storage.finishInjection(); + } + else + { + LOG_ERROR("Error occurred while running Python post processing. Rolling back all changes."); + storage.rollbackInjection(); + } + + LOG_INFO("Finished Python post processing."); +} diff --git a/src/lib/data/indexer/TaskExecuteCustomCommands.h b/src/lib/data/indexer/TaskExecuteCustomCommands.h index 3c42d4ab..edeac9b7 100644 --- a/src/lib/data/indexer/TaskExecuteCustomCommands.h +++ b/src/lib/data/indexer/TaskExecuteCustomCommands.h @@ -37,7 +37,9 @@ private: void executeParallelIndexerCommands(int threadId, std::shared_ptr blackboard); void runIndexerCommand(std::shared_ptr indexerCommand, std::shared_ptr blackboard); - +public: + static void runPythonPostProcessing(PersistentStorage& storage); +private: std::unique_ptr m_indexerCommandProvider; std::shared_ptr m_storage; std::shared_ptr m_dialogView; @@ -51,6 +53,7 @@ private: std::vector> m_parallelCommands; std::mutex m_parallelCommandsMutex; FilePath m_targetDatabaseFilePath; + bool m_hasPythonCommands; std::set m_sourceDatabaseFilePaths; std::mutex m_sourceDatabaseFilePathsMutex; }; diff --git a/src/lib/data/location/LocationType.cpp b/src/lib/data/location/LocationType.cpp index 0701fefb..216088a4 100644 --- a/src/lib/data/location/LocationType.cpp +++ b/src/lib/data/location/LocationType.cpp @@ -27,6 +27,8 @@ LocationType intToLocationType(int value) return LOCATION_FULLTEXT_SEARCH; case LOCATION_SCREEN_SEARCH: return LOCATION_SCREEN_SEARCH; + case LOCATION_UNSOLVED: + return LOCATION_UNSOLVED; } return LOCATION_TOKEN; } diff --git a/src/lib/data/location/LocationType.h b/src/lib/data/location/LocationType.h index cf879805..b966c3d7 100644 --- a/src/lib/data/location/LocationType.h +++ b/src/lib/data/location/LocationType.h @@ -11,7 +11,8 @@ enum LocationType LOCATION_COMMENT = 5, LOCATION_ERROR = 6, LOCATION_FULLTEXT_SEARCH = 7, - LOCATION_SCREEN_SEARCH = 8 + LOCATION_SCREEN_SEARCH = 8, + LOCATION_UNSOLVED = 9 }; int locationTypeToInt(LocationType type); diff --git a/src/lib/data/storage/PersistentStorage.cpp b/src/lib/data/storage/PersistentStorage.cpp index 26e1ed2b..e5f5b1dc 100644 --- a/src/lib/data/storage/PersistentStorage.cpp +++ b/src/lib/data/storage/PersistentStorage.cpp @@ -7,10 +7,12 @@ #include "TokenComponentAggregation.h" #include "TokenComponentFilePath.h" #include "TokenComponentInheritanceChain.h" +#include "TokenComponentIsAmbiguous.h" #include "Graph.h" #include "SourceLocationCollection.h" #include "SourceLocationFile.h" #include "AccessKind.h" +#include "ElementComponentKind.h" #include "ParseLocation.h" #include "NodeTypeSet.h" #include "ApplicationSettings.h" @@ -138,11 +140,26 @@ void PersistentStorage::addComponentAccesses(const std::vector& ids) +{ + m_sqliteIndexStorage.removeElements(ids); +} + const std::vector& PersistentStorage::getStorageNodes() const { return m_storageData.nodes = m_sqliteIndexStorage.getAll(); @@ -207,6 +224,13 @@ void PersistentStorage::finishInjection() afterErrorRecording(); } +void PersistentStorage::rollbackInjection() +{ + m_sqliteIndexStorage.rollbackTransaction(); + + afterErrorRecording(); +} + void PersistentStorage::beforeErrorRecording() { m_preInjectionErrorCount = m_sqliteIndexStorage.getErrorCount(); @@ -1170,6 +1194,7 @@ std::shared_ptr PersistentStorage::getGraphForActiveTokenIds( } addComponentAccessToGraph(graph); + addComponentIsAmbiguousToGraph(graph); if (isActiveNamespace) { @@ -1258,6 +1283,7 @@ std::shared_ptr PersistentStorage::getGraphForTrail( addNodesWithParentsAndEdgesToGraph(utility::toVector(nodeIds), utility::toVector(edgeIds), graph.get(), false); addComponentAccessToGraph(graph.get()); + addComponentIsAmbiguousToGraph(graph.get()); return graph; } @@ -1382,7 +1408,7 @@ std::shared_ptr PersistentStorage::getSourceLocationsF for (const StorageSourceLocation& sourceLocation: m_sqliteIndexStorage.getAllByIds(locationIds)) { const LocationType type = intToLocationType(sourceLocation.type); - if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL) + if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL && type != LOCATION_UNSOLVED) { continue; } @@ -1446,7 +1472,7 @@ std::shared_ptr PersistentStorage::getSourceLocationsF } const LocationType type = intToLocationType(location.type); - if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL) + if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL && type != LOCATION_UNSOLVED) { continue; } @@ -1473,7 +1499,7 @@ std::shared_ptr PersistentStorage::getSourceLocationsForFile TRACE(); return m_sqliteIndexStorage.getSourceLocationsForFile(filePath)->getFilteredByTypes({ - LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL + LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL, LOCATION_UNSOLVED }); } @@ -1485,7 +1511,7 @@ std::shared_ptr PersistentStorage::getSourceLocationsForLine return m_sqliteIndexStorage.getSourceLocationsForLinesInFile( filePath, startLine, endLine)->getFilteredByLines(startLine, endLine)->getFilteredByTypes({ - LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL + LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL, LOCATION_UNSOLVED }); } @@ -2819,6 +2845,28 @@ void PersistentStorage::addComponentAccessToGraph(Graph* graph) const } } +void PersistentStorage::addComponentIsAmbiguousToGraph(Graph* graph) const +{ + TRACE(); + + std::vector edgeIds; + graph->forEachEdge( + [&edgeIds](Edge* edge) + { + edgeIds.push_back(edge->getId()); + } + ); + + int componentKind = elementComponentKindToInt(ElementComponentKind::IS_AMBIGUOUS); + for (const StorageElementComponent& component : m_sqliteIndexStorage.getElementComponentsByElementIds(edgeIds)) + { + if (component.type == componentKind) + { + graph->getEdgeById(component.elementId)->addComponent(std::make_shared()); + } + } +} + void PersistentStorage::addCompleteFlagsToSourceLocationCollection(SourceLocationCollection* collection) const { TRACE(); diff --git a/src/lib/data/storage/PersistentStorage.h b/src/lib/data/storage/PersistentStorage.h index 4776ca64..dc6e344d 100644 --- a/src/lib/data/storage/PersistentStorage.h +++ b/src/lib/data/storage/PersistentStorage.h @@ -34,8 +34,12 @@ public: void addOccurrences(const std::vector& occurrences) override; void addComponentAccess(const StorageComponentAccess& componentAccess) override; void addComponentAccesses(const std::vector& componentAccesses) override; + void addElementComponent(const StorageElementComponentData& data) /*override*/; Id addError(const StorageErrorData& data) override; + void removeElement(const Id id); + void removeElements(const std::vector& ids); + const std::vector& getStorageNodes() const override; const std::vector& getStorageFiles() const override; const std::vector& getStorageSymbols() const override; @@ -48,6 +52,7 @@ public: void startInjection() override; void finishInjection() override; + void rollbackInjection(); void beforeErrorRecording(); void afterErrorRecording(); @@ -198,6 +203,7 @@ private: void addAggregationEdgesToGraph(Id nodeId, const std::vector& edgesToAggregate, Graph* graph) const; void addFileContentsToGraph(Id fileId, Graph* graph) const; void addComponentAccessToGraph(Graph* graph) const; + void addComponentIsAmbiguousToGraph(Graph* graph) const; void addCompleteFlagsToSourceLocationCollection(SourceLocationCollection* collection) const; void addInheritanceChainsToGraph(const std::vector& nodeIds, Graph* graph) const; diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp index 36362834..a68fdd2d 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.cpp @@ -11,7 +11,7 @@ #include "SourceLocationFile.h" #include "utilityString.h" -const size_t SqliteIndexStorage::s_storageVersion = 23; +const size_t SqliteIndexStorage::s_storageVersion = 24; namespace { @@ -424,6 +424,17 @@ bool SqliteIndexStorage::addComponentAccesses(const std::vector SqliteIndexStorage::getComponentAccessesByNo return doGetAll("WHERE node_id IN (" + utility::join(utility::toStrings(nodeIds), ',') + ")"); } +std::vector SqliteIndexStorage::getElementComponentsByElementIds(const std::vector& elementIds) const +{ + return doGetAll("WHERE element_id IN (" + utility::join(utility::toStrings(elementIds), ',') + ")"); +} + std::vector SqliteIndexStorage::getAllErrorInfos() const { std::vector errorInfos; @@ -1107,6 +1123,7 @@ void SqliteIndexStorage::clearTables() m_database.execDML("DROP TABLE IF EXISTS main.symbol;"); m_database.execDML("DROP TABLE IF EXISTS main.node;"); m_database.execDML("DROP TABLE IF EXISTS main.edge;"); + m_database.execDML("DROP TABLE IF EXISTS main.element_component;"); m_database.execDML("DROP TABLE IF EXISTS main.element;"); m_database.execDML("DROP TABLE IF EXISTS main.meta;"); } @@ -1126,6 +1143,17 @@ void SqliteIndexStorage::setupTables() "PRIMARY KEY(id));" ); + m_database.execDML( + "CREATE TABLE IF NOT EXISTS element_component(" + " id INTEGER, " + " element_id INTEGER, " + " type INTEGER, " + " data TEXT, " + " PRIMARY KEY(id), " + " FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE" + ");" + ); + m_database.execDML( "CREATE TABLE IF NOT EXISTS edge(" "id INTEGER NOT NULL, " @@ -1319,6 +1347,9 @@ void SqliteIndexStorage::setupPrecompiledStatements() m_insertElementStmt = m_database.compileStatement( "INSERT INTO element(id) VALUES(NULL);" ); + m_insertElementComponentStmt = m_database.compileStatement( + "INSERT INTO element_component(id, element_id, type, data) VALUES(NULL, ?, ?, ?);" + ); m_insertFileStmt = m_database.compileStatement( "INSERT INTO file(id, path, language, modification_time, indexed, complete, line_count) VALUES(?, ?, ?, ?, ?, ?, ?);" ); @@ -1526,6 +1557,31 @@ void SqliteIndexStorage::forEach(const std::string& quer } } +template <> +void SqliteIndexStorage::forEach(const std::string& query, std::function func) const +{ + CppSQLite3Query q = executeQuery( + "SELECT id, element_id, type, data FROM element_component " + query + ";" + ); + + while (!q.eof()) + { + const Id id = q.getIntField(0, 0); + const Id elementId = q.getIntField(1, 0); + const int type = q.getIntField(2, -1); + const std::string data = q.getStringField(3, ""); + + if (id != 0 && elementId != 0 && type != -1) + { + func(StorageElementComponent( + id, elementId, type, utility::decodeFromUtf8(data) + )); + } + + q.nextRow(); + } +} + template <> void SqliteIndexStorage::forEach(const std::string& query, std::function func) const { diff --git a/src/lib/data/storage/sqlite/SqliteIndexStorage.h b/src/lib/data/storage/sqlite/SqliteIndexStorage.h index 127013df..2c5c13ba 100644 --- a/src/lib/data/storage/sqlite/SqliteIndexStorage.h +++ b/src/lib/data/storage/sqlite/SqliteIndexStorage.h @@ -13,6 +13,7 @@ #include "StorageComponentAccess.h" #include "StorageEdge.h" #include "StorageError.h" +#include "StorageElementComponent.h" #include "StorageFile.h" #include "StorageLocalSymbol.h" #include "StorageNode.h" @@ -65,6 +66,7 @@ public: bool addOccurrences(const std::vector& occurrences); bool addComponentAccess(const StorageComponentAccess& componentAccess); bool addComponentAccesses(const std::vector& componentAccesses); + int addElementComponent(const StorageElementComponentData& storageElementComponentData); StorageError addError(const StorageErrorData& data); void removeElement(Id id); @@ -118,8 +120,10 @@ public: std::vector getOccurrencesForLocationIds(const std::vector& locationIds) const; std::vector getOccurrencesForElementIds(const std::vector& elementIds) const; - StorageComponentAccess getComponentAccessByNodeId(Id memberEdgeId) const; - std::vector getComponentAccessesByNodeIds(const std::vector& memberEdgeIds) const; + StorageComponentAccess getComponentAccessByNodeId(Id nodeId) const; + std::vector getComponentAccessesByNodeIds(const std::vector& nodeIds) const; + + std::vector getElementComponentsByElementIds(const std::vector& elementIds) const; std::vector getAllErrorInfos() const; @@ -351,6 +355,7 @@ private: InsertBatchStatement m_insertComponentAccessBatchStatement; CppSQLite3Statement m_insertElementStmt; + CppSQLite3Statement m_insertElementComponentStmt; CppSQLite3Statement m_insertFileStmt; CppSQLite3Statement m_insertFileContentStmt; CppSQLite3Statement m_checkErrorExistsStmt; @@ -374,6 +379,8 @@ void SqliteIndexStorage::forEach(const std::string& query, st template <> void SqliteIndexStorage::forEach(const std::string& query, std::function func) const; template <> +void SqliteIndexStorage::forEach(const std::string& query, std::function func) const; +template <> void SqliteIndexStorage::forEach(const std::string& query, std::function func) const; #endif // SQLITE_INDEX_STORAGE_H diff --git a/src/lib/data/storage/type/StorageElementComponent.h b/src/lib/data/storage/type/StorageElementComponent.h new file mode 100644 index 00000000..aed6bfff --- /dev/null +++ b/src/lib/data/storage/type/StorageElementComponent.h @@ -0,0 +1,47 @@ +#ifndef STORAGE_ELEMENT_COMPONENT_H +#define STORAGE_ELEMENT_COMPONENT_H + +#include + +#include "types.h" + +struct StorageElementComponentData +{ + StorageElementComponentData() + : elementId(0) + , type(0) + , data(L"") + {} + + StorageElementComponentData(Id elementId, int type, std::wstring data) + : elementId(elementId) + , type(type) + , data(std::move(data)) + {} + + Id elementId; + int type; + std::wstring data; +}; + +struct StorageElementComponent : public StorageElementComponentData +{ + StorageElementComponent() + : StorageElementComponentData() + , id(0) + {} + + StorageElementComponent(Id id, const StorageElementComponentData& data) + : StorageElementComponentData(data) + , id(id) + {} + + StorageElementComponent(Id id, Id elementId, int type, std::wstring data) + : StorageElementComponentData(elementId, type, data) + , id(id) + {} + + Id id; +}; + +#endif // STORAGE_ELEMENT_COMPONENT_H diff --git a/src/lib/settings/ApplicationSettings.cpp b/src/lib/settings/ApplicationSettings.cpp index 6962f53a..6067d436 100644 --- a/src/lib/settings/ApplicationSettings.cpp +++ b/src/lib/settings/ApplicationSettings.cpp @@ -400,6 +400,16 @@ void ApplicationSettings::setHasPrefilledMavenPath(bool v) setValue("indexing/java/has_prefilled_maven_path", v); } +bool ApplicationSettings::getPythonPostProcessingEnabled() const +{ + return getValue("indexing/python/post_processing", true); +} + +void ApplicationSettings::setPythonPostProcessingEnabled(bool enabled) +{ + setValue("indexing/python/post_processing", enabled); +} + std::vector ApplicationSettings::getHeaderSearchPaths() const { return getPathValues("indexing/cxx/header_search_paths/header_search_path"); diff --git a/src/lib/settings/ApplicationSettings.h b/src/lib/settings/ApplicationSettings.h index 73032288..ad8279d3 100644 --- a/src/lib/settings/ApplicationSettings.h +++ b/src/lib/settings/ApplicationSettings.h @@ -115,6 +115,9 @@ public: bool getHasPrefilledMavenPath() const; void setHasPrefilledMavenPath(bool v); + bool getPythonPostProcessingEnabled() const; + void setPythonPostProcessingEnabled(bool enabled); + std::vector getHeaderSearchPaths() const; std::vector getHeaderSearchPathsExpanded() const; bool setHeaderSearchPaths(const std::vector& headerSearchPaths); diff --git a/src/lib/utility/messaging/type/MessageActivateSourceLocations.h b/src/lib/utility/messaging/type/MessageActivateSourceLocations.h index 4dcdadf1..c08be4a3 100644 --- a/src/lib/utility/messaging/type/MessageActivateSourceLocations.h +++ b/src/lib/utility/messaging/type/MessageActivateSourceLocations.h @@ -9,8 +9,9 @@ class MessageActivateSourceLocations : public Message { public: - MessageActivateSourceLocations(const std::vector& locationIds) + MessageActivateSourceLocations(const std::vector& locationIds, bool containsUnsolvedLocations) : locationIds(locationIds) + , containsUnsolvedLocations(containsUnsolvedLocations) { setSchedulerId(TabId::currentTab()); } @@ -29,6 +30,7 @@ public: } const std::vector locationIds; + const bool containsUnsolvedLocations; }; #endif // MESSAGE_ACTIVATE_SOURCE_LOCATIONS_H diff --git a/src/lib/utility/messaging/type/MessageTooltipShow.h b/src/lib/utility/messaging/type/MessageTooltipShow.h index 44a39d88..1a237bcf 100644 --- a/src/lib/utility/messaging/type/MessageTooltipShow.h +++ b/src/lib/utility/messaging/type/MessageTooltipShow.h @@ -38,6 +38,8 @@ public: const std::vector localSymbolIds; const TooltipOrigin origin; + + bool force = false; }; #endif // MESSAGE_TOOLTIP_SHOW_H diff --git a/src/lib_gui/qt/element/QtCodeField.cpp b/src/lib_gui/qt/element/QtCodeField.cpp index 3c298a9b..5476c114 100644 --- a/src/lib_gui/qt/element/QtCodeField.cpp +++ b/src/lib_gui/qt/element/QtCodeField.cpp @@ -217,7 +217,12 @@ void QtCodeField::paintEvent(QPaintEvent* event) continue; } - painter.setPen(QPen(color.border.c_str())); + QPen pen(color.border.c_str()); + if (annotation.locationType == LOCATION_UNSOLVED) + { + pen.setStyle(Qt::DashLine); + } + painter.setPen(pen); painter.setBrush(QBrush(color.fill.c_str())); if (annotation.locationType == LOCATION_SCOPE) @@ -451,9 +456,12 @@ void QtCodeField::activateAnnotations(const std::vector& anno std::set tokenIds; std::set localSymbolIds; + bool containsUnsolved = false; + for (const Annotation* annotation : annotations) { - if (annotation->locationType == LOCATION_TOKEN || annotation->locationType == LOCATION_QUALIFIER) + if (annotation->locationType == LOCATION_TOKEN || annotation->locationType == LOCATION_QUALIFIER || + annotation->locationType == LOCATION_UNSOLVED) { if (annotation->locationId > 0) { @@ -464,6 +472,11 @@ void QtCodeField::activateAnnotations(const std::vector& anno { tokenIds.insert(annotation->tokenIds.begin(), annotation->tokenIds.end()); } + + if (annotation->locationType == LOCATION_UNSOLVED) + { + containsUnsolved = true; + } } else if (annotation->locationType == LOCATION_LOCAL_SYMBOL) { @@ -480,7 +493,7 @@ void QtCodeField::activateAnnotations(const std::vector& anno } else if (locationIds.size()) { - MessageActivateSourceLocations(locationIds).dispatch(); + MessageActivateSourceLocations(locationIds, containsUnsolved).dispatch(); } else if (tokenIds.size()) // fallback for links in project description { @@ -681,7 +694,8 @@ std::vector QtCodeField::getInteractiveAnnotatio for (const Annotation& annotation : m_annotations) { const LocationType& type = annotation.locationType; - if ((type == LOCATION_TOKEN || type == LOCATION_QUALIFIER || type == LOCATION_LOCAL_SYMBOL || type == LOCATION_ERROR) + if ((type == LOCATION_TOKEN || type == LOCATION_QUALIFIER || type == LOCATION_LOCAL_SYMBOL + || type == LOCATION_UNSOLVED || type == LOCATION_ERROR) && pos >= annotation.start && pos <= annotation.end) { annotations.push_back(&annotation); @@ -697,7 +711,7 @@ void QtCodeField::checkOpenInTabActionEnabled(QPoint position) for (const Annotation* annotation : getInteractiveAnnotationsForPosition(position)) { const LocationType& type = annotation->locationType; - if (type == LOCATION_TOKEN || type == LOCATION_QUALIFIER) + if (type == LOCATION_TOKEN || type == LOCATION_QUALIFIER || type == LOCATION_UNSOLVED) { locationIds.emplace_back(annotation->locationId); } diff --git a/src/lib_gui/qt/view/graphElements/QtGraphEdge.cpp b/src/lib_gui/qt/view/graphElements/QtGraphEdge.cpp index 7647d96c..292a85e1 100644 --- a/src/lib_gui/qt/view/graphElements/QtGraphEdge.cpp +++ b/src/lib_gui/qt/view/graphElements/QtGraphEdge.cpp @@ -7,6 +7,7 @@ #include "Edge.h" #include "TokenComponentAggregation.h" #include "TokenComponentInheritanceChain.h" +#include "TokenComponentIsAmbiguous.h" #include "QtLineItemAngled.h" #include "QtLineItemBezier.h" #include "QtLineItemStraight.h" @@ -95,7 +96,8 @@ void QtGraphEdge::updateLine() const QtGraphNode* target = m_target; Edge::EdgeType type = (getData() ? getData()->getType() : Edge::EDGE_AGGREGATION); - GraphViewStyle::EdgeStyle style = GraphViewStyle::getStyleForEdgeType(type, m_isActive | m_isFocused, false, m_isTrailEdge); + GraphViewStyle::EdgeStyle style = + GraphViewStyle::getStyleForEdgeType(type, m_isActive | m_isFocused, false, m_isTrailEdge, isAmbiguous()); Vec4i ownerRect = owner->getBoundingRect(); Vec4i targetRect = target->getBoundingRect(); @@ -363,6 +365,11 @@ void QtGraphEdge::focusIn() TooltipInfo info; info.title = Edge::getReadableTypeString(type); + if (isAmbiguous()) + { + info.title = L"ambiguous " + info.title; + } + if (type == Edge::EDGE_AGGREGATION && m_direction == TokenComponentAggregation::DIRECTION_NONE) { info.title = L"bidirectional " + info.title; @@ -506,3 +513,8 @@ void QtGraphEdge::clearPath() { m_path.clear(); } + +bool QtGraphEdge::isAmbiguous() const +{ + return m_data && m_data->getComponent(); +} diff --git a/src/lib_gui/qt/view/graphElements/QtGraphEdge.h b/src/lib_gui/qt/view/graphElements/QtGraphEdge.h index 010f7da1..10309dd8 100644 --- a/src/lib_gui/qt/view/graphElements/QtGraphEdge.h +++ b/src/lib_gui/qt/view/graphElements/QtGraphEdge.h @@ -60,6 +60,8 @@ public: void setUseBezier(bool useBezier); void clearPath(); + bool isAmbiguous() const; + protected: virtual void mousePressEvent(QGraphicsSceneMouseEvent* event); virtual void mouseMoveEvent(QGraphicsSceneMouseEvent* event); diff --git a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp index 27e1de65..ee25bc13 100644 --- a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp +++ b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp @@ -373,7 +373,20 @@ void QtProjectWizzardContentPreferences::populate(QGridLayout* layout, int& row) addGap(layout, row); - // C/C++ + + addTitle("Python", layout, row); + + m_pythonPostProcessing = addCheckBox("Post Processing", + "Add ambiguous edges for unsolved references", + "

Enable a post processing step to solve unsolved references after the indexing is done.

" + "

These references will be marked \"ambiguous\" to indicate that some of these edges may never " + "be encountered during runtime of the indexed code because the post processing only relies on " + "symbol names and types.

", + layout, row); + + addGap(layout, row); + + addTitle("C/C++", layout, row); } @@ -447,6 +460,8 @@ void QtProjectWizzardContentPreferences::load() { m_mavenPath->setText(QString::fromStdWString(appSettings->getMavenPath().wstr())); } + + m_pythonPostProcessing->setChecked(appSettings->getPythonPostProcessingEnabled()); } void QtProjectWizzardContentPreferences::save() @@ -510,6 +525,8 @@ void QtProjectWizzardContentPreferences::save() appSettings->setMavenPath(FilePath(m_mavenPath->getText().toStdWString())); } + appSettings->setPythonPostProcessingEnabled(m_pythonPostProcessing->isChecked()); + appSettings->save(); } diff --git a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h index 9810e0fc..998e6654 100644 --- a/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h +++ b/src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h @@ -125,6 +125,8 @@ private: QtPathListBox* m_jreSystemLibraryPaths; QLineEdit* m_jvmMaximumMemory; QtLocationPicker* m_mavenPath; + + QCheckBox* m_pythonPostProcessing; }; #endif // QT_PROJECT_WIZZARD_CONTENT_PREFERENCES_H diff --git a/src/lib_python/project/SourceGroupPythonEmpty.cpp b/src/lib_python/project/SourceGroupPythonEmpty.cpp index 5f5661f3..7b5b370d 100644 --- a/src/lib_python/project/SourceGroupPythonEmpty.cpp +++ b/src/lib_python/project/SourceGroupPythonEmpty.cpp @@ -57,6 +57,7 @@ std::vector> SourceGroupPythonEmpty::getIndexerC if (filesToIndex.find(sourceFilePath) != filesToIndex.end()) { indexerCommands.push_back(std::make_shared( + INDEXER_COMMAND_PYTHON, L"\"" + ResourcePaths::getPythonPath().wstr() + L"SourcetrailPythonIndexer\"" + args, m_settings->getProjectSettings()->getProjectFilePath(), m_settings->getProjectSettings()->getTempDBFilePath(),