diff --git a/src/lib_gui/qt/element/QtCodeArea.cpp b/src/lib_gui/qt/element/QtCodeArea.cpp
index 37da504c..a545da10 100644
--- a/src/lib_gui/qt/element/QtCodeArea.cpp
+++ b/src/lib_gui/qt/element/QtCodeArea.cpp
@@ -855,16 +855,30 @@ void QtCodeArea::annotateText()
 		}
 
 		const AnnotationColor& newColor = getAnnotationColorForAnnotation(annotation);
-		if ((newColor.text != oldColor.text || !m_wasAnnotated))
+		if (newColor.text != oldColor.text || (!m_wasAnnotated && newColor.text != "transparent"))
 		{
 			if (newColor.text.size() > 0 && newColor.text != "transparent")
 			{
-				if (!annotation.oldTextColor.isValid())
+				bool isDuplicateAnnotation = false;
+				for (Annotation* a : m_colorChangedAnnotations)
 				{
-					annotation.oldTextColor = m_highlighter->getFormat(annotation.start, annotation.end).foreground().color();
+					if (a->start == annotation.start && a->end == annotation.end && a->tokenId != annotation.tokenId)
+					{
+						isDuplicateAnnotation = true;
+						break;
+					}
 				}
 
-				setTextColorForAnnotation(annotation, QColor(newColor.text.c_str()));
+				if (!isDuplicateAnnotation)
+				{
+					if (!annotation.oldTextColor.isValid())
+					{
+						annotation.oldTextColor = m_highlighter->getFormat(annotation.start, annotation.end).foreground().color();
+					}
+
+					setTextColorForAnnotation(annotation, QColor(newColor.text.c_str()));
+					m_colorChangedAnnotations.push_back(&annotation);
+				}
 			}
 			else if (annotation.oldTextColor.isValid())
 			{
diff --git a/src/lib_gui/qt/element/QtCodeArea.h b/src/lib_gui/qt/element/QtCodeArea.h
index 5b9a36bc..e88f732e 100644
--- a/src/lib_gui/qt/element/QtCodeArea.h
+++ b/src/lib_gui/qt/element/QtCodeArea.h
@@ -185,6 +185,9 @@ private:
 	std::vector<Annotation> m_annotations;
 	std::vector<const Annotation*> m_hoveredAnnotations;
 
+	// Remove when annotations become unique regarding start and end, and store multiple tokenIds
+	std::vector<Annotation*> m_colorChangedAnnotations;
+
 	int m_digits;
 
 	bool m_isSelecting;