logic: implemented Python post processing to add ambiguous edges for unsolved symbols

script/download_python_indexer.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-SOURCETRAIL_PYTHON_INDEXER_VERSION="v0_db23_p4"
+SOURCETRAIL_PYTHON_INDEXER_VERSION="v0_db24_p0"
 
 # Determine current platform
 PLATFORM='unknown'
@@ -14,7 +14,7 @@ fi
 
 PACKAGE_NAME="SourcetrailPythonIndexer_${SOURCETRAIL_PYTHON_INDEXER_VERSION}-${PLATFORM}"
 PACKAGE_FILE_NAME="${PACKAGE_NAME}.zip"
-PACKAGE_URL="https://github.com/CoatiSoftware/SourcetrailPythonIndexer/releases/download/${SOURCETRAIL_PYTHON_INDEXER_VERSION}/${PACKAGE_FILE_NAME}"
+PACKAGE_URL="https://github.com/CoatiSoftware/SourcetrailPythonIndexer/releases/download/${SOURCETRAIL_PYTHON_INDEXER_VERSION}_beta/${PACKAGE_FILE_NAME}"
 TEMP_PATH="build/temp"
 TARGET_PATH="bin/app/data/python"
 

src/lib/CMakeLists.txt

@@ -138,11 +138,14 @@ add_files(
 	data/graph/token_component/TokenComponentFilePath.cpp
 	data/graph/token_component/TokenComponentFilePath.h
 	data/graph/token_component/TokenComponentInheritanceChain.h
+	data/graph/token_component/TokenComponentIsAmbiguous.h
 	data/graph/token_component/TokenComponentStatic.cpp
 	data/graph/token_component/TokenComponentStatic.h
 
 	data/graph/Edge.cpp
 	data/graph/Edge.h
+	data/graph/ElementComponentKind.cpp
+	data/graph/ElementComponentKind.h
 	data/graph/Graph.cpp
 	data/graph/Graph.h
 	data/graph/Node.cpp
@@ -250,6 +253,7 @@ add_files(
 	data/storage/type/StorageBookmarkedNode.h
 	data/storage/type/StorageComponentAccess.h
 	data/storage/type/StorageEdge.h
+	data/storage/type/StorageElementComponent.h
 	data/storage/type/StorageError.h
 	data/storage/type/StorageFile.h
 	data/storage/type/StorageLocalSymbol.h

src/lib/component/controller/ActivationController.cpp

@@ -12,6 +12,7 @@
 #include "MessageRefreshUI.h"
 #include "MessageStatus.h"
 #include "MessageScrollToLine.h"
+#include "MessageTooltipShow.h"
 #include "utility.h"
 
 ActivationController::ActivationController(StorageAccess* storageAccess)
@@ -109,6 +110,16 @@ void ActivationController::handleMessage(MessageActivateSourceLocations* message
 	{
 		msg.addNode(nodeId);
 	}
+
+	if (message->containsUnsolvedLocations && msg.nodes.size() == 1 &&
+		m_storageAccess->getNameHierarchyForNodeId(msg.nodes[0].nodeId).getQualifiedName() == L"unsolved symbol")
+	{
+		MessageTooltipShow m(message->locationIds, {}, TOOLTIP_ORIGIN_CODE);
+		m.force = true;
+		m.dispatch();
+		return;
+	}
+
 	msg.setSchedulerId(message->getSchedulerId());
 	msg.dispatchImmediately();
 }

src/lib/component/controller/IDECommunicationController.cpp

@@ -100,7 +100,8 @@ void IDECommunicationController::handleSetActiveTokenMessage(
 				{
 					const SourceLocation* endLocation = startLocation->getEndLocation();
 
-					if ((startLocation->getType() == LOCATION_TOKEN || startLocation->getType() == LOCATION_QUALIFIER)
+					if ((startLocation->getType() == LOCATION_TOKEN || startLocation->getType() == LOCATION_QUALIFIER
+						|| startLocation->getType() == LOCATION_UNSOLVED)
 						&& startLocation->getLineNumber() == endLocation->getLineNumber()
 						&& startLocation->getColumnNumber() <= cursorColumn
 						&& endLocation->getColumnNumber() + 1 >= cursorColumn)

src/lib/component/controller/TooltipController.cpp

@@ -91,9 +91,9 @@ void TooltipController::handleMessage(MessageTooltipShow* message)
 
 		// If a tooltip list would only display one token, then just activate it instead.
 		// This can happen when edges pointing to the token use the same source location e.g. override edges
-		if (info.snippets.size() == 1)
+		if (!message->force && info.snippets.size() == 1)
 		{
-			MessageActivateSourceLocations(message->sourceLocationIds).dispatch();
+			MessageActivateSourceLocations(message->sourceLocationIds, false).dispatch();
 		}
 		else if (info.snippets.size())
 		{

src/lib/component/view/GraphViewStyle.cpp

@@ -609,7 +609,7 @@ GraphViewStyle::NodeStyle GraphViewStyle::getStyleOfGroupNode(GroupType type, bo
 }
 
 GraphViewStyle::EdgeStyle GraphViewStyle::getStyleForEdgeType(
-	Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge)
+	Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge, bool isAmbiguous)
 {
 	EdgeStyle style;
 
@@ -706,6 +706,11 @@ GraphViewStyle::EdgeStyle GraphViewStyle::getStyleForEdgeType(
 		break;
 	}
 
+	if (isAmbiguous)
+	{
+		style.dashed = true;
+	}
+
 	return style;
 }
 

src/lib/component/view/GraphViewStyle.h

@@ -130,7 +130,8 @@ public:
 	static NodeStyle getStyleOfTextNode(int fontSizeDiff);
 	static NodeStyle getStyleOfGroupNode(GroupType type, bool isFocused);
 
-	static EdgeStyle getStyleForEdgeType(Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge);
+	static EdgeStyle getStyleForEdgeType(
+		Edge::EdgeType type, bool isActive, bool isFocused, bool isTrailEdge, bool isAmbiguous);
 
 	static int toGridOffset(int x);
 	static int toGridSize(int x);

src/lib/data/graph/ElementComponentKind.cpp

@@ -0,0 +1,23 @@
+#include "ElementComponentKind.h"
+
+int elementComponentKindToInt(ElementComponentKind kind)
+{
+	return static_cast<int>(kind);
+}
+
+ElementComponentKind intToElementComponentKind(int value)
+{
+	const ElementComponentKind kinds[] = {
+		ElementComponentKind::IS_AMBIGUOUS
+	};
+
+	for (ElementComponentKind kind : kinds)
+	{
+		if (value == elementComponentKindToInt(kind))
+		{
+			return kind;
+		}
+	}
+
+	return ElementComponentKind::NONE;
+}

src/lib/data/graph/ElementComponentKind.h

@@ -0,0 +1,13 @@
+#ifndef ELEMENT_COMPONENT_KIND_H
+#define ELEMENT_COMPONENT_KIND_H
+
+enum class ElementComponentKind
+{
+	NONE = 0,
+	IS_AMBIGUOUS = 1
+};
+
+int elementComponentKindToInt(ElementComponentKind type);
+ElementComponentKind intToElementComponentKind(int value);
+
+#endif // ELEMENT_COMPONENT_KIND_H

src/lib/data/graph/token_component/TokenComponentIsAmbiguous.h

@@ -0,0 +1,16 @@
+#ifndef TOKEN_COMPONENT_IS_AMBIGUOUS_H
+#define TOKEN_COMPONENT_IS_AMBIGUOUS_H
+
+#include "TokenComponent.h"
+
+class TokenComponentIsAmbiguous
+	: public TokenComponent
+{
+public:
+	inline virtual std::shared_ptr<TokenComponent> copy() const
+	{
+		return std::make_shared<TokenComponentIsAmbiguous>(*this);
+	}
+};
+
+#endif // TOKEN_COMPONENT_IS_AMBIGUOUS_H

src/lib/data/indexer/IndexerCommandCustom.cpp

@@ -19,6 +19,26 @@ IndexerCommandCustom::IndexerCommandCustom(
 	bool runInParallel
 )
 	: IndexerCommand(sourceFilePath)
+	, m_type(getStaticIndexerCommandType())
+	, m_customCommand(customCommand)
+	, m_projectFilePath(projectFilePath)
+	, m_databaseFilePath(databaseFilePath)
+	, m_databaseVersion(databaseVersion)
+	, m_runInParallel(runInParallel)
+{
+}
+
+IndexerCommandCustom::IndexerCommandCustom(
+	IndexerCommandType type,
+	const std::wstring& customCommand,
+	const FilePath& projectFilePath,
+	const FilePath& databaseFilePath,
+	const std::wstring& databaseVersion,
+	const FilePath& sourceFilePath,
+	bool runInParallel
+)
+	: IndexerCommand(sourceFilePath)
+	, m_type(type)
 	, m_customCommand(customCommand)
 	, m_projectFilePath(projectFilePath)
 	, m_databaseFilePath(databaseFilePath)
@@ -29,7 +49,7 @@ IndexerCommandCustom::IndexerCommandCustom(
 
 IndexerCommandType IndexerCommandCustom::getIndexerCommandType() const
 {
-	return getStaticIndexerCommandType();
+	return m_type;
 }
 
 size_t IndexerCommandCustom::getByteSize(size_t stringSize) const

src/lib/data/indexer/IndexerCommandCustom.h

@@ -13,7 +13,16 @@ public:
 	static IndexerCommandType getStaticIndexerCommandType();
 
 	IndexerCommandCustom(
-		const std::wstring& customCommand, 
+		const std::wstring& customCommand,
+		const FilePath& projectFilePath,
+		const FilePath& databaseFilePath,
+		const std::wstring& databaseVersion,
+		const FilePath& sourceFilePath,
+		bool runInParallel);
+
+	IndexerCommandCustom(
+		IndexerCommandType type,
+		const std::wstring& customCommand,
 		const FilePath& projectFilePath,
 		const FilePath& databaseFilePath,
 		const std::wstring& databaseVersion,
@@ -33,6 +42,7 @@ protected:
 	QJsonObject doSerialize() const override;
 
 private:
+	IndexerCommandType m_type;
 	std::wstring m_customCommand;
 	FilePath m_projectFilePath;
 	FilePath m_databaseFilePath;

src/lib/data/indexer/IndexerCommandType.cpp

@@ -22,7 +22,6 @@ IndexerCommandType stringToIndexerCommandType(const std::string& s)
 {
 	if (s == indexerCommandTypeToString(INDEXER_COMMAND_CXX)) return INDEXER_COMMAND_CXX;
 	if (s == indexerCommandTypeToString(INDEXER_COMMAND_JAVA)) return INDEXER_COMMAND_JAVA;
-	if (s == indexerCommandTypeToString(INDEXER_COMMAND_PYTHON)) return INDEXER_COMMAND_PYTHON;
 	if (s == indexerCommandTypeToString(INDEXER_COMMAND_CUSTOM)) return INDEXER_COMMAND_CUSTOM;
 	return INDEXER_COMMAND_UNKNOWN;
 }

src/lib/data/indexer/TaskExecuteCustomCommands.cpp

@@ -1,7 +1,9 @@
 #include "TaskExecuteCustomCommands.h"
 
+#include "ApplicationSettings.h"
 #include "Blackboard.h"
 #include "DialogView.h"
+#include "ElementComponentKind.h"
 #include "FileSystem.h"
 #include "IndexerCommandCustom.h"
 #include "IndexerCommandProvider.h"
@@ -9,6 +11,9 @@
 #include "MessageShowStatus.h"
 #include "MessageStatus.h"
 #include "PersistentStorage.h"
+#include "SourceLocationCollection.h"
+#include "SourceLocationFile.h"
+#include "TextAccess.h"
 #include "utility.h"
 #include "utilityApp.h"
 #include "utilityString.h"
@@ -26,6 +31,7 @@ TaskExecuteCustomCommands::TaskExecuteCustomCommands(
 	, m_indexerThreadCount(indexerThreadCount)
 	, m_projectDirectory(projectDirectory)
 	, m_indexerCommandCount(m_indexerCommandProvider->size())
+	, m_hasPythonCommands(false)
 {
 }
 
@@ -46,6 +52,11 @@ void TaskExecuteCustomCommands::doEnter(std::shared_ptr<Blackboard> blackboard)
 					m_targetDatabaseFilePath = indexerCommand->getDatabaseFilePath();
 				}
 
+				if (indexerCommand->getIndexerCommandType() == INDEXER_COMMAND_PYTHON)
+				{
+					m_hasPythonCommands = true;
+				}
+
 				if (indexerCommand->getRunInParallel())
 				{
 					m_parallelCommands.push_back(indexerCommand);
@@ -66,7 +77,7 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard>
 		return STATE_SUCCESS;
 	}
 
-	m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), { });
+	m_dialogView->updateCustomIndexingDialog(0, 0, m_indexerCommandProvider->size(), {});
 
 
 	std::vector<std::shared_ptr<std::thread>> indexerThreads;
@@ -105,6 +116,11 @@ Task::TaskState TaskExecuteCustomCommands::doUpdate(std::shared_ptr<Blackboard>
 			}
 			FileSystem::remove(sourceDatabaseFilePath);
 		}
+
+		if (m_hasPythonCommands && ApplicationSettings::getInstance()->getPythonPostProcessingEnabled())
+		{
+			runPythonPostProcessing(targetStorage);
+		}
 	}
 
 	return STATE_SUCCESS;
@@ -143,7 +159,6 @@ void TaskExecuteCustomCommands::executeParallelIndexerCommands(int threadId, std
 			}
 			indexerCommand = m_parallelCommands.back();
 			m_parallelCommands.pop_back();
-
 		}
 
 		if (threadId != 0)
@@ -235,4 +250,115 @@ void TaskExecuteCustomCommands::runIndexerCommand(std::shared_ptr<IndexerCommand
 		indexedSourceFileCount++;
 		blackboard->update<int>("indexed_source_file_count", [=](int count) { return count + 1; });
 	}
-}
+}
+
+void TaskExecuteCustomCommands::runPythonPostProcessing(PersistentStorage& storage)
+{
+	LOG_INFO("Starting Python post processing.");
+
+	std::vector<Id> unsolvedLocationIds;
+	for (const StorageSourceLocation location : storage.getStorageSourceLocations())
+	{
+		if (intToLocationType(location.type) == LOCATION_UNSOLVED)
+		{
+			unsolvedLocationIds.push_back(location.id);
+		}
+	}
+
+	std::shared_ptr<SourceLocationCollection> locationCollection = storage.getSourceLocationsForLocationIds(unsolvedLocationIds);
+
+	std::map<std::wstring, std::vector<StorageNode>> nodeNameToStorageNodes;
+	if (locationCollection->getSourceLocationCount() > 0)
+	{
+		for (const StorageNode& node : storage.getStorageNodes())
+		{
+			nodeNameToStorageNodes[NameHierarchy::deserialize(node.serializedName).back().getName()].push_back(node);
+		}
+	}
+
+	struct DataToInsert
+	{
+		StorageEdgeData edgeData;
+		Id sourceLocationId;
+	};
+
+	storage.setMode(SqliteIndexStorage::STORAGE_MODE_READ);
+
+	std::vector<DataToInsert> dataToInsert;
+	std::set<Id> elementsToDelete;
+	locationCollection->forEachSourceLocationFile([&nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](std::shared_ptr<SourceLocationFile> locationFile)
+		{
+			std::shared_ptr<TextAccess> textAccess = TextAccess::createFromFile(locationFile->getFilePath());
+
+			if (textAccess)
+			{
+				locationFile->forEachStartSourceLocation([textAccess, &nodeNameToStorageNodes, &storage, &dataToInsert, &elementsToDelete](const SourceLocation* startLoc)
+					{
+						if (!startLoc)
+						{
+							return;
+						}
+						const SourceLocation* endLoc = startLoc->getOtherLocation();
+						if (!endLoc)
+						{
+							return;
+						}
+
+						const std::wstring token = utility::decodeFromUtf8(textAccess->getLine(startLoc->getLineNumber()).substr(startLoc->getColumnNumber() - 1, endLoc->getColumnNumber() - startLoc->getColumnNumber() + 1));
+				
+						for (const Id tokenId : startLoc->getTokenIds())
+						{
+							const StorageEdge edge = storage.getEdgeById(tokenId);
+							if (edge.id != 0)
+							{
+								for (const StorageNode& targetNode : nodeNameToStorageNodes[token])
+								{
+									if (Edge::intToType(edge.type) == Edge::EDGE_CALL && 
+										(
+											NodeType::intToType(targetNode.type) != NodeType::NODE_FUNCTION ||
+											NodeType::intToType(targetNode.type) != NodeType::NODE_METHOD
+										)
+									){
+										continue;
+									}
+									dataToInsert.push_back({ StorageEdgeData(edge.type, edge.sourceNodeId, targetNode.id) , startLoc->getLocationId() });
+									elementsToDelete.insert(edge.id);
+								}
+							}
+						}
+					}
+				);
+			}
+		}
+	);
+
+	storage.setMode(SqliteIndexStorage::STORAGE_MODE_WRITE);
+
+	storage.startInjection();
+
+	std::vector<StorageEdge> edgesToInsert;
+	for (const DataToInsert& data : dataToInsert)
+	{
+		edgesToInsert.push_back(StorageEdge(0, data.edgeData));
+	}
+	const std::vector<Id> ambiguousEdgeIds = storage.addEdges(edgesToInsert);
+
+	if (ambiguousEdgeIds.size() == dataToInsert.size())
+	{
+		for (size_t i = 0; i < ambiguousEdgeIds.size(); i++)
+		{
+			storage.addElementComponent(StorageElementComponentData(ambiguousEdgeIds[i], elementComponentKindToInt(ElementComponentKind::IS_AMBIGUOUS), L""));
+			storage.addOccurrence(StorageOccurrence(ambiguousEdgeIds[i], dataToInsert[i].sourceLocationId));
+		}
+
+		storage.removeElements(utility::toVector(elementsToDelete));
+		storage.finishInjection();
+	}
+	else
+	{
+		LOG_ERROR("Error occurred while running Python post processing. Rolling back all changes.");
+		storage.rollbackInjection();
+	}
+
+	LOG_INFO("Finished Python post processing.");
+}

src/lib/data/indexer/TaskExecuteCustomCommands.h

@@ -37,7 +37,9 @@ private:
 
 	void executeParallelIndexerCommands(int threadId, std::shared_ptr<Blackboard> blackboard);
 	void runIndexerCommand(std::shared_ptr<IndexerCommandCustom> indexerCommand, std::shared_ptr<Blackboard> blackboard);
-
+public:
+	static void runPythonPostProcessing(PersistentStorage& storage);
+private:
 	std::unique_ptr<IndexerCommandProvider> m_indexerCommandProvider;
 	std::shared_ptr<PersistentStorage> m_storage;
 	std::shared_ptr<DialogView> m_dialogView;
@@ -51,6 +53,7 @@ private:
 	std::vector<std::shared_ptr<IndexerCommandCustom>> m_parallelCommands;
 	std::mutex m_parallelCommandsMutex;
 	FilePath m_targetDatabaseFilePath;
+	bool m_hasPythonCommands;
 	std::set<FilePath> m_sourceDatabaseFilePaths;
 	std::mutex m_sourceDatabaseFilePathsMutex;
 };

src/lib/data/location/LocationType.cpp

@@ -27,6 +27,8 @@ LocationType intToLocationType(int value)
 		return LOCATION_FULLTEXT_SEARCH;
 	case LOCATION_SCREEN_SEARCH:
 		return LOCATION_SCREEN_SEARCH;
+	case LOCATION_UNSOLVED:
+		return LOCATION_UNSOLVED;
 	}
 	return LOCATION_TOKEN;
 }

src/lib/data/location/LocationType.h

@@ -11,7 +11,8 @@ enum LocationType
 	LOCATION_COMMENT = 5,
 	LOCATION_ERROR = 6,
 	LOCATION_FULLTEXT_SEARCH = 7,
-	LOCATION_SCREEN_SEARCH = 8
+	LOCATION_SCREEN_SEARCH = 8,
+	LOCATION_UNSOLVED = 9
 };
 
 int locationTypeToInt(LocationType type);

src/lib/data/storage/PersistentStorage.cpp

@@ -7,10 +7,12 @@
 #include "TokenComponentAggregation.h"
 #include "TokenComponentFilePath.h"
 #include "TokenComponentInheritanceChain.h"
+#include "TokenComponentIsAmbiguous.h"
 #include "Graph.h"
 #include "SourceLocationCollection.h"
 #include "SourceLocationFile.h"
 #include "AccessKind.h"
+#include "ElementComponentKind.h"
 #include "ParseLocation.h"
 #include "NodeTypeSet.h"
 #include "ApplicationSettings.h"
@@ -138,11 +140,26 @@ void PersistentStorage::addComponentAccesses(const std::vector<StorageComponentA
 	m_sqliteIndexStorage.addComponentAccesses(componentAccesses);
 }
 
+void PersistentStorage::addElementComponent(const StorageElementComponentData& data)
+{
+	m_sqliteIndexStorage.addElementComponent(data);
+}
+
 Id PersistentStorage::addError(const StorageErrorData& data)
 {
 	return m_sqliteIndexStorage.addError(data).id;
 }
 
+void PersistentStorage::removeElement(const Id id)
+{
+	m_sqliteIndexStorage.removeElement(id);
+}
+
+void PersistentStorage::removeElements(const std::vector<Id>& ids)
+{
+	m_sqliteIndexStorage.removeElements(ids);
+}
+
 const std::vector<StorageNode>& PersistentStorage::getStorageNodes() const
 {
 	return m_storageData.nodes = m_sqliteIndexStorage.getAll<StorageNode>();
@@ -207,6 +224,13 @@ void PersistentStorage::finishInjection()
 	afterErrorRecording();
 }
 
+void PersistentStorage::rollbackInjection()
+{
+	m_sqliteIndexStorage.rollbackTransaction();
+
+	afterErrorRecording();
+}
+
 void PersistentStorage::beforeErrorRecording()
 {
 	m_preInjectionErrorCount = m_sqliteIndexStorage.getErrorCount();
@@ -1170,6 +1194,7 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForActiveTokenIds(
 	}
 
 	addComponentAccessToGraph(graph);
+	addComponentIsAmbiguousToGraph(graph);
 
 	if (isActiveNamespace)
 	{
@@ -1258,6 +1283,7 @@ std::shared_ptr<Graph> PersistentStorage::getGraphForTrail(
 
 	addNodesWithParentsAndEdgesToGraph(utility::toVector(nodeIds), utility::toVector(edgeIds), graph.get(), false);
 	addComponentAccessToGraph(graph.get());
+	addComponentIsAmbiguousToGraph(graph.get());
 
 	return graph;
 }
@@ -1382,7 +1408,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getSourceLocationsF
 		for (const StorageSourceLocation& sourceLocation: m_sqliteIndexStorage.getAllByIds<StorageSourceLocation>(locationIds))
 		{
 			const LocationType type = intToLocationType(sourceLocation.type);
-			if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL)
+			if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL && type != LOCATION_UNSOLVED)
 			{
 				continue;
 			}
@@ -1446,7 +1472,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getSourceLocationsF
 		}
 
 		const LocationType type = intToLocationType(location.type);
-		if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL)
+		if (type != LOCATION_TOKEN && type != LOCATION_SCOPE && type != LOCATION_LOCAL_SYMBOL && type != LOCATION_UNSOLVED)
 		{
 			continue;
 		}
@@ -1473,7 +1499,7 @@ std::shared_ptr<SourceLocationFile> PersistentStorage::getSourceLocationsForFile
 	TRACE();
 
 	return m_sqliteIndexStorage.getSourceLocationsForFile(filePath)->getFilteredByTypes({
-		LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL
+		LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL, LOCATION_UNSOLVED
 	});
 }
 
@@ -1485,7 +1511,7 @@ std::shared_ptr<SourceLocationFile> PersistentStorage::getSourceLocationsForLine
 
 	return m_sqliteIndexStorage.getSourceLocationsForLinesInFile(
 		filePath, startLine, endLine)->getFilteredByLines(startLine, endLine)->getFilteredByTypes({
-			LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL
+			LOCATION_TOKEN, LOCATION_SCOPE, LOCATION_QUALIFIER, LOCATION_LOCAL_SYMBOL, LOCATION_UNSOLVED
 		});
 }
 
@@ -2819,6 +2845,28 @@ void PersistentStorage::addComponentAccessToGraph(Graph* graph) const
 	}
 }
 
+void PersistentStorage::addComponentIsAmbiguousToGraph(Graph* graph) const
+{
+	TRACE();
+
+	std::vector<Id> edgeIds;
+	graph->forEachEdge(
+		[&edgeIds](Edge* edge)
+		{
+			edgeIds.push_back(edge->getId());
+		}
+	);
+
+	int componentKind = elementComponentKindToInt(ElementComponentKind::IS_AMBIGUOUS);
+	for (const StorageElementComponent& component : m_sqliteIndexStorage.getElementComponentsByElementIds(edgeIds))
+	{
+		if (component.type == componentKind)
+		{
+			graph->getEdgeById(component.elementId)->addComponent(std::make_shared<TokenComponentIsAmbiguous>());
+		}
+	}
+}
+
 void PersistentStorage::addCompleteFlagsToSourceLocationCollection(SourceLocationCollection* collection) const
 {
 	TRACE();

src/lib/data/storage/PersistentStorage.h

@@ -34,8 +34,12 @@ public:
 	void addOccurrences(const std::vector<StorageOccurrence>& occurrences) override;
 	void addComponentAccess(const StorageComponentAccess& componentAccess) override;
 	void addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses) override;
+	void addElementComponent(const StorageElementComponentData& data) /*override*/;
 	Id addError(const StorageErrorData& data) override;
 
+	void removeElement(const Id id);
+	void removeElements(const std::vector<Id>& ids);
+
 	const std::vector<StorageNode>& getStorageNodes() const override;
 	const std::vector<StorageFile>& getStorageFiles() const override;
 	const std::vector<StorageSymbol>& getStorageSymbols() const override;
@@ -48,6 +52,7 @@ public:
 
 	void startInjection() override;
 	void finishInjection() override;
+	void rollbackInjection();
 
 	void beforeErrorRecording();
 	void afterErrorRecording();
@@ -198,6 +203,7 @@ private:
 	void addAggregationEdgesToGraph(Id nodeId, const std::vector<StorageEdge>& edgesToAggregate, Graph* graph) const;
 	void addFileContentsToGraph(Id fileId, Graph* graph) const;
 	void addComponentAccessToGraph(Graph* graph) const;
+	void addComponentIsAmbiguousToGraph(Graph* graph) const;
 
 	void addCompleteFlagsToSourceLocationCollection(SourceLocationCollection* collection) const;
 	void addInheritanceChainsToGraph(const std::vector<Id>& nodeIds, Graph* graph) const;

src/lib/data/storage/sqlite/SqliteIndexStorage.cpp

@@ -11,7 +11,7 @@
 #include "SourceLocationFile.h"
 #include "utilityString.h"
 
-const size_t SqliteIndexStorage::s_storageVersion = 23;
+const size_t SqliteIndexStorage::s_storageVersion = 24;
 
 namespace
 {
@@ -424,6 +424,17 @@ bool SqliteIndexStorage::addComponentAccesses(const std::vector<StorageComponent
 	return m_insertComponentAccessBatchStatement.execute(componentAccesses, this);
 }
 
+int SqliteIndexStorage::addElementComponent(const StorageElementComponentData& storageElementComponentData)
+{
+	m_insertElementComponentStmt.bind(1, int(storageElementComponentData.elementId));
+	m_insertElementComponentStmt.bind(2, storageElementComponentData.type);
+	m_insertElementComponentStmt.bind(3, utility::encodeToUtf8(storageElementComponentData.data).c_str());
+	executeStatement(m_insertElementComponentStmt);
+	int id = m_database.lastRowId();
+	m_insertElementComponentStmt.reset();
+	return id;
+}
+
 StorageError SqliteIndexStorage::addError(const StorageErrorData& data)
 {
 	const std::wstring sanitizedMessage = utility::replace(data.message, L"'", L"''");
@@ -968,6 +979,11 @@ std::vector<StorageComponentAccess> SqliteIndexStorage::getComponentAccessesByNo
 	return doGetAll<StorageComponentAccess>("WHERE node_id IN (" + utility::join(utility::toStrings(nodeIds), ',') + ")");
 }
 
+std::vector<StorageElementComponent> SqliteIndexStorage::getElementComponentsByElementIds(const std::vector<Id>& elementIds) const
+{
+	return doGetAll<StorageElementComponent>("WHERE element_id IN (" + utility::join(utility::toStrings(elementIds), ',') + ")");
+}
+
 std::vector<ErrorInfo> SqliteIndexStorage::getAllErrorInfos() const
 {
 	std::vector<ErrorInfo> errorInfos;
@@ -1107,6 +1123,7 @@ void SqliteIndexStorage::clearTables()
 		m_database.execDML("DROP TABLE IF EXISTS main.symbol;");
 		m_database.execDML("DROP TABLE IF EXISTS main.node;");
 		m_database.execDML("DROP TABLE IF EXISTS main.edge;");
+		m_database.execDML("DROP TABLE IF EXISTS main.element_component;");
 		m_database.execDML("DROP TABLE IF EXISTS main.element;");
 		m_database.execDML("DROP TABLE IF EXISTS main.meta;");
 	}
@@ -1126,6 +1143,17 @@ void SqliteIndexStorage::setupTables()
 				"PRIMARY KEY(id));"
 		);
 
+		m_database.execDML(
+			"CREATE TABLE IF NOT EXISTS element_component("
+			"	id INTEGER, "
+			"	element_id INTEGER, "
+			"	type INTEGER, "
+			"	data TEXT, "
+			"	PRIMARY KEY(id), "
+			"	FOREIGN KEY(element_id) REFERENCES element(id) ON DELETE CASCADE"
+			");"
+		);
+
 		m_database.execDML(
 			"CREATE TABLE IF NOT EXISTS edge("
 				"id INTEGER NOT NULL, "
@@ -1319,6 +1347,9 @@ void SqliteIndexStorage::setupPrecompiledStatements()
 		m_insertElementStmt = m_database.compileStatement(
 			"INSERT INTO element(id) VALUES(NULL);"
 		);
+		m_insertElementComponentStmt = m_database.compileStatement(
+			"INSERT INTO element_component(id, element_id, type, data) VALUES(NULL, ?, ?, ?);"
+		);
 		m_insertFileStmt = m_database.compileStatement(
 			"INSERT INTO file(id, path, language, modification_time, indexed, complete, line_count) VALUES(?, ?, ?, ?, ?, ?, ?);"
 		);
@@ -1526,6 +1557,31 @@ void SqliteIndexStorage::forEach<StorageComponentAccess>(const std::string& quer
 	}
 }
 
+template <>
+void SqliteIndexStorage::forEach<StorageElementComponent>(const std::string& query, std::function<void(StorageElementComponent&&)> func) const
+{
+	CppSQLite3Query q = executeQuery(
+		"SELECT id, element_id, type, data FROM element_component " + query + ";"
+	);
+
+	while (!q.eof())
+	{
+		const Id id = q.getIntField(0, 0);
+		const Id elementId = q.getIntField(1, 0);
+		const int type = q.getIntField(2, -1);
+		const std::string data = q.getStringField(3, "");
+
+		if (id != 0 && elementId != 0 && type != -1)
+		{
+			func(StorageElementComponent(
+				id, elementId, type, utility::decodeFromUtf8(data)
+			));
+		}
+
+		q.nextRow();
+	}
+}
+
 template <>
 void SqliteIndexStorage::forEach<StorageError>(const std::string& query, std::function<void(StorageError&&)> func) const
 {

src/lib/data/storage/sqlite/SqliteIndexStorage.h

@@ -13,6 +13,7 @@
 #include "StorageComponentAccess.h"
 #include "StorageEdge.h"
 #include "StorageError.h"
+#include "StorageElementComponent.h"
 #include "StorageFile.h"
 #include "StorageLocalSymbol.h"
 #include "StorageNode.h"
@@ -65,6 +66,7 @@ public:
 	bool addOccurrences(const std::vector<StorageOccurrence>& occurrences);
 	bool addComponentAccess(const StorageComponentAccess& componentAccess);
 	bool addComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses);
+	int addElementComponent(const StorageElementComponentData& storageElementComponentData);
 	StorageError addError(const StorageErrorData& data);
 
 	void removeElement(Id id);
@@ -118,8 +120,10 @@ public:
 	std::vector<StorageOccurrence> getOccurrencesForLocationIds(const std::vector<Id>& locationIds) const;
 	std::vector<StorageOccurrence> getOccurrencesForElementIds(const std::vector<Id>& elementIds) const;
 
-	StorageComponentAccess getComponentAccessByNodeId(Id memberEdgeId) const;
-	std::vector<StorageComponentAccess> getComponentAccessesByNodeIds(const std::vector<Id>& memberEdgeIds) const;
+	StorageComponentAccess getComponentAccessByNodeId(Id nodeId) const;
+	std::vector<StorageComponentAccess> getComponentAccessesByNodeIds(const std::vector<Id>& nodeIds) const;
+
+	std::vector<StorageElementComponent> getElementComponentsByElementIds(const std::vector<Id>& elementIds) const;
 
 	std::vector<ErrorInfo> getAllErrorInfos() const;
 
@@ -351,6 +355,7 @@ private:
 	InsertBatchStatement<StorageComponentAccess> m_insertComponentAccessBatchStatement;
 
 	CppSQLite3Statement m_insertElementStmt;
+	CppSQLite3Statement m_insertElementComponentStmt;
 	CppSQLite3Statement m_insertFileStmt;
 	CppSQLite3Statement m_insertFileContentStmt;
 	CppSQLite3Statement m_checkErrorExistsStmt;
@@ -374,6 +379,8 @@ void SqliteIndexStorage::forEach<StorageOccurrence>(const std::string& query, st
 template <>
 void SqliteIndexStorage::forEach<StorageComponentAccess>(const std::string& query, std::function<void(StorageComponentAccess&&)> func) const;
 template <>
+void SqliteIndexStorage::forEach<StorageElementComponent>(const std::string& query, std::function<void(StorageElementComponent&&)> func) const;
+template <>
 void SqliteIndexStorage::forEach<StorageError>(const std::string& query, std::function<void(StorageError&&)> func) const;
 
 #endif // SQLITE_INDEX_STORAGE_H

src/lib/data/storage/type/StorageElementComponent.h

@@ -0,0 +1,47 @@
+#ifndef STORAGE_ELEMENT_COMPONENT_H
+#define STORAGE_ELEMENT_COMPONENT_H
+
+#include <string>
+
+#include "types.h"
+
+struct StorageElementComponentData
+{
+	StorageElementComponentData()
+		: elementId(0)
+		, type(0)
+		, data(L"")
+	{}
+
+	StorageElementComponentData(Id elementId, int type, std::wstring data)
+		: elementId(elementId)
+		, type(type)
+		, data(std::move(data))
+	{}
+
+	Id elementId;
+	int type;
+	std::wstring data;
+};
+
+struct StorageElementComponent : public StorageElementComponentData
+{
+	StorageElementComponent()
+		: StorageElementComponentData()
+		, id(0)
+	{}
+
+	StorageElementComponent(Id id, const StorageElementComponentData& data)
+		: StorageElementComponentData(data)
+		, id(id)
+	{}
+
+	StorageElementComponent(Id id, Id elementId, int type, std::wstring data)
+		: StorageElementComponentData(elementId, type, data)
+		, id(id)
+	{}
+
+	Id id;
+};
+
+#endif // STORAGE_ELEMENT_COMPONENT_H

src/lib/settings/ApplicationSettings.cpp

@@ -400,6 +400,16 @@ void ApplicationSettings::setHasPrefilledMavenPath(bool v)
 	setValue<bool>("indexing/java/has_prefilled_maven_path", v);
 }
 
+bool ApplicationSettings::getPythonPostProcessingEnabled() const
+{
+	return getValue<bool>("indexing/python/post_processing", true);
+}
+
+void ApplicationSettings::setPythonPostProcessingEnabled(bool enabled)
+{
+	setValue<bool>("indexing/python/post_processing", enabled);
+}
+
 std::vector<FilePath> ApplicationSettings::getHeaderSearchPaths() const
 {
 	return getPathValues("indexing/cxx/header_search_paths/header_search_path");

src/lib/settings/ApplicationSettings.h

@@ -115,6 +115,9 @@ public:
 	bool getHasPrefilledMavenPath() const;
 	void setHasPrefilledMavenPath(bool v);
 
+	bool getPythonPostProcessingEnabled() const;
+	void setPythonPostProcessingEnabled(bool enabled);
+
 	std::vector<FilePath> getHeaderSearchPaths() const;
 	std::vector<FilePath> getHeaderSearchPathsExpanded() const;
 	bool setHeaderSearchPaths(const std::vector<FilePath>& headerSearchPaths);

src/lib/utility/messaging/type/MessageActivateSourceLocations.h

@@ -9,8 +9,9 @@ class MessageActivateSourceLocations
 	: public Message<MessageActivateSourceLocations>
 {
 public:
-	MessageActivateSourceLocations(const std::vector<Id>& locationIds)
+	MessageActivateSourceLocations(const std::vector<Id>& locationIds, bool containsUnsolvedLocations)
 		: locationIds(locationIds)
+		, containsUnsolvedLocations(containsUnsolvedLocations)
 	{
 		setSchedulerId(TabId::currentTab());
 	}
@@ -29,6 +30,7 @@ public:
 	}
 
 	const std::vector<Id> locationIds;
+	const bool containsUnsolvedLocations;
 };
 
 #endif // MESSAGE_ACTIVATE_SOURCE_LOCATIONS_H

src/lib/utility/messaging/type/MessageTooltipShow.h

@@ -38,6 +38,8 @@ public:
 	const std::vector<Id> localSymbolIds;
 
 	const TooltipOrigin origin;
+
+	bool force = false;
 };
 
 #endif // MESSAGE_TOOLTIP_SHOW_H

src/lib_gui/qt/element/QtCodeField.cpp

@@ -217,7 +217,12 @@ void QtCodeField::paintEvent(QPaintEvent* event)
 			continue;
 		}
 
-		painter.setPen(QPen(color.border.c_str()));
+		QPen pen(color.border.c_str());
+		if (annotation.locationType == LOCATION_UNSOLVED)
+		{
+			pen.setStyle(Qt::DashLine);
+		}
+		painter.setPen(pen);
 		painter.setBrush(QBrush(color.fill.c_str()));
 
 		if (annotation.locationType == LOCATION_SCOPE)
@@ -451,9 +456,12 @@ void QtCodeField::activateAnnotations(const std::vector<const Annotation*>& anno
 	std::set<Id> tokenIds;
 	std::set<Id> localSymbolIds;
 
+	bool containsUnsolved = false;
+
 	for (const Annotation* annotation : annotations)
 	{
-		if (annotation->locationType == LOCATION_TOKEN || annotation->locationType == LOCATION_QUALIFIER)
+		if (annotation->locationType == LOCATION_TOKEN || annotation->locationType == LOCATION_QUALIFIER ||
+			annotation->locationType == LOCATION_UNSOLVED)
 		{
 			if (annotation->locationId > 0)
 			{
@@ -464,6 +472,11 @@ void QtCodeField::activateAnnotations(const std::vector<const Annotation*>& anno
 			{
 				tokenIds.insert(annotation->tokenIds.begin(), annotation->tokenIds.end());
 			}
+
+			if (annotation->locationType == LOCATION_UNSOLVED)
+			{
+				containsUnsolved = true;
+			}
 		}
 		else if (annotation->locationType == LOCATION_LOCAL_SYMBOL)
 		{
@@ -480,7 +493,7 @@ void QtCodeField::activateAnnotations(const std::vector<const Annotation*>& anno
 	}
 	else if (locationIds.size())
 	{
-		MessageActivateSourceLocations(locationIds).dispatch();
+		MessageActivateSourceLocations(locationIds, containsUnsolved).dispatch();
 	}
 	else if (tokenIds.size()) // fallback for links in project description
 	{
@@ -681,7 +694,8 @@ std::vector<const QtCodeField::Annotation*> QtCodeField::getInteractiveAnnotatio
 	for (const Annotation& annotation : m_annotations)
 	{
 		const LocationType& type = annotation.locationType;
-		if ((type == LOCATION_TOKEN || type == LOCATION_QUALIFIER || type == LOCATION_LOCAL_SYMBOL || type == LOCATION_ERROR)
+		if ((type == LOCATION_TOKEN || type == LOCATION_QUALIFIER || type == LOCATION_LOCAL_SYMBOL
+			|| type == LOCATION_UNSOLVED || type == LOCATION_ERROR)
 			&& pos >= annotation.start && pos <= annotation.end)
 		{
 			annotations.push_back(&annotation);
@@ -697,7 +711,7 @@ void QtCodeField::checkOpenInTabActionEnabled(QPoint position)
 	for (const Annotation* annotation : getInteractiveAnnotationsForPosition(position))
 	{
 		const LocationType& type = annotation->locationType;
-		if (type == LOCATION_TOKEN || type == LOCATION_QUALIFIER)
+		if (type == LOCATION_TOKEN || type == LOCATION_QUALIFIER || type == LOCATION_UNSOLVED)
 		{
 			locationIds.emplace_back(annotation->locationId);
 		}

src/lib_gui/qt/view/graphElements/QtGraphEdge.cpp

@@ -7,6 +7,7 @@
 #include "Edge.h"
 #include "TokenComponentAggregation.h"
 #include "TokenComponentInheritanceChain.h"
+#include "TokenComponentIsAmbiguous.h"
 #include "QtLineItemAngled.h"
 #include "QtLineItemBezier.h"
 #include "QtLineItemStraight.h"
@@ -95,7 +96,8 @@ void QtGraphEdge::updateLine()
 	const QtGraphNode* target = m_target;
 
 	Edge::EdgeType type = (getData() ? getData()->getType() : Edge::EDGE_AGGREGATION);
-	GraphViewStyle::EdgeStyle style = GraphViewStyle::getStyleForEdgeType(type, m_isActive | m_isFocused, false, m_isTrailEdge);
+	GraphViewStyle::EdgeStyle style =
+		GraphViewStyle::getStyleForEdgeType(type, m_isActive | m_isFocused, false, m_isTrailEdge, isAmbiguous());
 
 	Vec4i ownerRect = owner->getBoundingRect();
 	Vec4i targetRect = target->getBoundingRect();
@@ -363,6 +365,11 @@ void QtGraphEdge::focusIn()
 			TooltipInfo info;
 			info.title = Edge::getReadableTypeString(type);
 
+			if (isAmbiguous())
+			{
+				info.title = L"ambiguous " + info.title;
+			}
+
 			if (type == Edge::EDGE_AGGREGATION && m_direction == TokenComponentAggregation::DIRECTION_NONE)
 			{
 				info.title = L"bidirectional " + info.title;
@@ -506,3 +513,8 @@ void QtGraphEdge::clearPath()
 {
 	m_path.clear();
 }
+
+bool QtGraphEdge::isAmbiguous() const
+{
+	return m_data && m_data->getComponent<TokenComponentIsAmbiguous>();
+}

src/lib_gui/qt/view/graphElements/QtGraphEdge.h

@@ -60,6 +60,8 @@ public:
 	void setUseBezier(bool useBezier);
 	void clearPath();
 
+	bool isAmbiguous() const;
+
 protected:
 	virtual void mousePressEvent(QGraphicsSceneMouseEvent* event);
 	virtual void mouseMoveEvent(QGraphicsSceneMouseEvent* event);

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.cpp

@@ -373,7 +373,20 @@ void QtProjectWizzardContentPreferences::populate(QGridLayout* layout, int& row)
 
 	addGap(layout, row);
 
-	// C/C++
+
+	addTitle("Python", layout, row);
+
+	m_pythonPostProcessing = addCheckBox("Post Processing",
+		"Add ambiguous edges for unsolved references",
+		"<p>Enable a post processing step to solve unsolved references after the indexing is done. </p>"
+		"<p>These references will be marked \"ambiguous\" to indicate that some of these edges may never "
+		"be encountered during runtime of the indexed code because the post processing only relies on "
+		"symbol names and types.</p>",
+		layout, row);
+
+	addGap(layout, row);
+
+
 	addTitle("C/C++", layout, row);
 }
 
@@ -447,6 +460,8 @@ void QtProjectWizzardContentPreferences::load()
 	{
 		m_mavenPath->setText(QString::fromStdWString(appSettings->getMavenPath().wstr()));
 	}
+
+	m_pythonPostProcessing->setChecked(appSettings->getPythonPostProcessingEnabled());
 }
 
 void QtProjectWizzardContentPreferences::save()
@@ -510,6 +525,8 @@ void QtProjectWizzardContentPreferences::save()
 		appSettings->setMavenPath(FilePath(m_mavenPath->getText().toStdWString()));
 	}
 
+	appSettings->setPythonPostProcessingEnabled(m_pythonPostProcessing->isChecked());
+
 	appSettings->save();
 }
 

src/lib_gui/qt/window/project_wizzard/QtProjectWizzardContentPreferences.h

@@ -125,6 +125,8 @@ private:
 	QtPathListBox* m_jreSystemLibraryPaths;
 	QLineEdit* m_jvmMaximumMemory;
 	QtLocationPicker* m_mavenPath;
+
+	QCheckBox* m_pythonPostProcessing;
 };
 
 #endif // QT_PROJECT_WIZZARD_CONTENT_PREFERENCES_H

src/lib_python/project/SourceGroupPythonEmpty.cpp

@@ -57,6 +57,7 @@ std::vector<std::shared_ptr<IndexerCommand>> SourceGroupPythonEmpty::getIndexerC
 		if (filesToIndex.find(sourceFilePath) != filesToIndex.end())
 		{
 			indexerCommands.push_back(std::make_shared<IndexerCommandCustom>(
+				INDEXER_COMMAND_PYTHON,
 				L"\"" + ResourcePaths::getPythonPath().wstr() + L"SourcetrailPythonIndexer\"" + args,
 				m_settings->getProjectSettings()->getProjectFilePath(),
 				m_settings->getProjectSettings()->getTempDBFilePath(),