enhance(emoji): update to Unicode 17, unify and lazy-load emoji data (#39363)

Generate emoji data from Unicode 17's `emoji-test.txt`, keeping existing
aliases. `public/assets/emoji.json` is now the single emoji data file,
also loaded by the backend. Rendered emoji drop their `aria-label`, the
dark theme inverts key on a new `data-alias` attribute instead.

Skin tone variants and their Gitea-only aliases are removed, GitHub has
none either.

Emoji autocompletion is now lazy-loaded with the markdown editor,
shrinking the index JS chunk from 653KB to 563KB.

---------

Signed-off-by: silverwind <me@silverwind.io>
Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: bircni <bircni@icloud.com>

modules/emoji/emoji.go

@@ -8,24 +8,23 @@ import (
 	"sort"
 	"strings"
 	"sync/atomic"
+	"unicode/utf8"
 
+	"gitea.dev/modules/json"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/public"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/util"
 )
 
-// Gemoji is a set of emoji data.
-type Gemoji []Emoji
-
 // Emoji represents a single emoji and associated data.
 type Emoji struct {
-	Emoji          string
-	Description    string
-	Aliases        []string
-	UnicodeVersion string
-	SkinTones      bool
+	Emoji   string   `json:"emoji"`
+	Aliases []string `json:"aliases"`
 }
 
 type globalVarsStruct struct {
+	emojis         []Emoji
 	codeMap        map[string]int    // emoji Unicode code to its emoji data.
 	aliasMap       map[string]int    // the alias to its emoji data.
 	trie           *util.TrieNode    // trie for finding emoji positions.
@@ -43,8 +42,13 @@ func globalVars() *globalVarsStruct {
 	}
 	// although there can be concurrent calls, the result should be the same, and there is no performance problem
 	vars = &globalVarsStruct{}
-	vars.codeMap = make(map[string]int, len(GemojiData))
-	vars.aliasMap = make(map[string]int, len(GemojiData))
+	if data, err := public.AssetFS().ReadFile("assets/emoji.json"); err != nil {
+		log.Error("Unable to read assets/emoji.json: %v", err)
+	} else if err = json.Unmarshal(data, &vars.emojis); err != nil {
+		log.Error("Unable to parse assets/emoji.json: %v", err)
+	}
+	vars.codeMap = make(map[string]int, len(vars.emojis))
+	vars.aliasMap = make(map[string]int, len(vars.emojis))
 	vars.trie = &util.TrieNode{}
 
 	// process emoji codes and aliases
@@ -52,11 +56,11 @@ func globalVars() *globalVarsStruct {
 	aliasPairs := make([]string, 0)
 
 	// sort from largest to small so we match combined emoji first
-	sort.Slice(GemojiData, func(i, j int) bool {
-		return len(GemojiData[i].Emoji) > len(GemojiData[j].Emoji)
+	sort.Slice(vars.emojis, func(i, j int) bool {
+		return len(vars.emojis[i].Emoji) > len(vars.emojis[j].Emoji)
 	})
 
-	for idx, emoji := range GemojiData {
+	for idx, emoji := range vars.emojis {
 		if emoji.Emoji == "" || len(emoji.Aliases) == 0 {
 			continue
 		}
@@ -95,29 +99,34 @@ func globalVars() *globalVarsStruct {
 }
 
 // FromCode retrieves the emoji data based on the provided Unicode code
-// e.g.: "\u2618" will return the Gemoji data for "shamrock".
+// e.g.: "\u2618" will return the emoji data for "shamrock".
 func FromCode(code string) *Emoji {
-	i, ok := globalVars().codeMap[code]
+	vars := globalVars()
+	i, ok := vars.codeMap[code]
+	if !ok {
+		i, ok = vars.codeMap[removeSkinTones(code)]
+	}
 	if !ok {
 		return nil
 	}
 
-	return &GemojiData[i]
+	return &vars.emojis[i]
 }
 
 // FromAlias retrieves the emoji data based on the provided alias in the form "alias" or ":alias:"
-// e.g.: "shamrock" or ":shamrock:" will return the Gemoji data for "shamrock".
+// e.g.: "shamrock" or ":shamrock:" will return the emoji data for "shamrock".
 func FromAlias(alias string) *Emoji {
 	if strings.HasPrefix(alias, ":") && strings.HasSuffix(alias, ":") {
 		alias = alias[1 : len(alias)-1]
 	}
 
-	i, ok := globalVars().aliasMap[alias]
+	vars := globalVars()
+	i, ok := vars.aliasMap[alias]
 	if !ok {
 		return nil
 	}
 
-	return &GemojiData[i]
+	return &vars.emojis[i]
 }
 
 // ReplaceCodes replaces all emoji codes with the first corresponding emoji alias in the form of ":alias:"
@@ -139,8 +148,36 @@ func FindEmojiSubmatchIndex(s string) []int {
 			continue
 		}
 		if matchLen := vars.trie.Match(s, i); matchLen > 0 {
-			return []int{i, i + matchLen}
+			return []int{i, i + skinTonedLen(vars, s[i:], matchLen)}
 		}
 	}
 	return nil
 }
+
+func isSkinTone(r rune) bool {
+	return r >= 0x1f3fb && r <= 0x1f3ff
+}
+
+func removeSkinTones(s string) string {
+	return strings.Map(func(r rune) rune { return util.Iif(isSkinTone(r), -1, r) }, s)
+}
+
+// skinTonedLen extends a match at the start of s over skin tones, which the emoji data omits
+func skinTonedLen(vars *globalVarsStruct, s string, matchLen int) int {
+	if r, _ := utf8.DecodeRuneInString(s[matchLen:]); !isSkinTone(r) {
+		return matchLen
+	}
+	tonelessLen := vars.trie.Match(removeSkinTones(s[:min(len(s), 2*len(vars.emojis[0].Emoji))]), 0)
+	end := 0
+	for end < len(s) {
+		r, size := utf8.DecodeRuneInString(s[end:])
+		if !isSkinTone(r) {
+			if tonelessLen == 0 {
+				break
+			}
+			tonelessLen -= size
+		}
+		end += size
+	}
+	return end
+}

modules/emoji/emoji_test.go

@@ -61,6 +61,11 @@ func TestReplacers(t *testing.T) {
 	}
 }
 
+func TestMain(m *testing.M) {
+	setting.SetupGiteaTestEnv()
+	m.Run()
+}
+
 const (
 	testInputWithEmojis = "This is a test string containing some emojis like \U0001f44d and \U0001f37a and some text in between."
 	testInputNoEmojis   = "This is a test string containing no emojis at all, just plain old ASCII text, which should ideally be scanned very quickly by our trie implementation."
@@ -89,6 +94,14 @@ func TestFindEmojiSubmatchIndex(t *testing.T) {
 			"\u0001\U0001f44d",
 			[]int{1, 1 + len("\U0001f44d")},
 		},
+		{
+			"👩🏿‍❤️‍👩🏿",
+			[]int{0, len("👩🏿‍❤️‍👩🏿")},
+		},
+		{
+			"🏽👍",
+			[]int{len("🏽"), len("🏽👍")},
+		},
 		{
 			// This package can handle keycap emoji if it is registered in the emoji data.
 			// However, many other places (e.g.: markup rendering) also might not handle such cases correctly.