enhance(emoji): update to Unicode 17, unify and lazy-load emoji data (#39363)

Generate emoji data from Unicode 17's `emoji-test.txt`, keeping existing
aliases. `public/assets/emoji.json` is now the single emoji data file,
also loaded by the backend. Rendered emoji drop their `aria-label`, the
dark theme inverts key on a new `data-alias` attribute instead.

Skin tone variants and their Gitea-only aliases are removed, GitHub has
none either.

Emoji autocompletion is now lazy-loaded with the markdown editor,
shrinking the index JS chunk from 653KB to 563KB.

---------

Signed-off-by: silverwind <me@silverwind.io>
Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: bircni <bircni@icloud.com>

.gitattributes

@@ -2,6 +2,7 @@
 *.tmpl linguist-language=Handlebars
 *.pb.go linguist-generated
 /assets/*.json linguist-generated
+/public/assets/emoji.json linguist-generated
 /public/assets/img/svg/*.svg linguist-generated
 /templates/swagger/*.generated.json linguist-generated
 /options/fileicon/** linguist-generated

.github/workflows/files-changed.yml

@@ -57,7 +57,7 @@ jobs:
             backend:
               - "**/*.go"
               - "templates/**/*.tmpl"
-              - "assets/emoji.json"
+              - "public/assets/emoji.json"
               - "go.mod"
               - "go.sum"
               - "Makefile"
@@ -77,7 +77,7 @@ jobs:
               - "tools/**/*.json"
               - "tools/playwright.sh"
               - "tsconfig.json"
-              - "assets/emoji.json"
+              - "public/assets/emoji.json"
               - "package.json"
               - "pnpm-lock.yaml"
               - "pnpm-workspace.yaml"

Makefile

@@ -637,6 +637,10 @@ lockfile-check:
 generate-gitignore: ## update gitignore files
 	$(GO) run build/generate-gitignores.go
 
+.PHONY: generate-emoji
+generate-emoji: ## update emoji data from Unicode
+	$(GO) run build/generate-emoji.go
+
 .PHONY: generate-images
 generate-images: | node_modules ## generate images
 	cd tools && node generate-images.ts $(TAGS)

build/generate-emoji.go

@@ -7,213 +7,126 @@
 package main
 
 import (
-	"flag"
+	"bufio"
+	"errors"
 	"fmt"
-	"go/format"
-	"io"
 	"log"
 	"net/http"
 	"os"
 	"regexp"
-	"sort"
-	"strconv"
+	"slices"
 	"strings"
-	"unicode/utf8"
 
 	"gitea.dev/modules/json"
 )
 
 const (
-	gemojiURL         = "https://raw.githubusercontent.com/rhysd/gemoji/537ff2d7e0496e9964824f7f73ec7ece88c9765a/db/emoji.json"
-	maxUnicodeVersion = 16
+	emojiTestURL = "https://www.unicode.org/Public/17.0.0/emoji/emoji-test.txt"
+	jsonFile     = "public/assets/emoji.json"
 )
 
-var flagOut = flag.String("o", "modules/emoji/emoji_data.go", "out")
-
-// Gemoji is a set of emoji data.
-type Gemoji []Emoji
-
-// Emoji represents a single emoji and associated data.
-type Emoji struct {
-	Emoji          string   `json:"emoji"`
-	Description    string   `json:"description,omitempty"`
-	Aliases        []string `json:"aliases"`
-	UnicodeVersion string   `json:"unicode_version,omitempty"`
-	SkinTones      bool     `json:"skin_tones,omitempty"`
+type emoji struct {
+	Emoji       string   `json:"emoji"`
+	Aliases     []string `json:"aliases"`
+	description string
 }
 
-// Don't include some fields in JSON
-func (e Emoji) MarshalJSON() ([]byte, error) {
-	type emoji Emoji
-	x := emoji(e)
-	x.UnicodeVersion = ""
-	x.Description = ""
-	x.SkinTones = false
-	return json.Marshal(x)
-}
+var slugRe = regexp.MustCompile(`[^a-z0-9]+`)
 
 func main() {
-	flag.Parse()
-
-	// generate data
-	buf, err := generate()
-	if err != nil {
-		log.Fatalf("generate err: %v", err)
-	}
-
-	// write
-	err = os.WriteFile(*flagOut, buf, 0o644)
-	if err != nil {
-		log.Fatalf("WriteFile err: %v", err)
+	if err := generate(); err != nil {
+		log.Fatal(err)
 	}
 }
 
-var replacer = strings.NewReplacer(
-	"main.Gemoji", "Gemoji",
-	"main.Emoji", "\n",
-	"}}", "},\n}",
-	", Description:", ", ",
-	", Aliases:", ", ",
-	", UnicodeVersion:", ", ",
-	", SkinTones:", ", ",
-)
+func generate() error {
+	// the existing file is also the alias source, so existing aliases stay stable
+	existing, err := os.ReadFile(jsonFile)
+	if err != nil {
+		return err
+	}
+	var existingEmojis []*emoji
+	if err := json.Unmarshal(existing, &existingEmojis); err != nil {
+		return err
+	}
+	existingAliases := make(map[string][]string, len(existingEmojis))
+	for _, e := range existingEmojis {
+		existingAliases[e.Emoji] = e.Aliases
+	}
 
-var emojiRE = regexp.MustCompile(`\{Emoji:"([^"]*)"`)
+	emojis, err := fetchEmojis(existingAliases)
+	if err != nil {
+		return err
+	}
 
-func generate() ([]byte, error) {
-	// load gemoji data
-	res, err := http.Get(gemojiURL)
+	lines := make([]string, len(emojis))
+	for i, e := range emojis {
+		line, err := json.Marshal(e)
+		if err != nil {
+			return err
+		}
+		lines[i] = string(line)
+	}
+	return os.WriteFile(jsonFile, []byte("[\n"+strings.Join(lines, ",\n")+"\n]\n"), 0o644)
+}
+
+func isSkinTone(r rune) bool {
+	return r >= 0x1f3fb && r <= 0x1f3ff
+}
+
+func fetchEmojis(existingAliases map[string][]string) ([]*emoji, error) {
+	res, err := http.Get(emojiTestURL)
 	if err != nil {
 		return nil, err
 	}
 	defer res.Body.Close()
-
-	// read all
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetching %s: %s", emojiTestURL, res.Status)
 	}
 
-	// unmarshal
-	var data Gemoji
-	err = json.Unmarshal(body, &data)
-	if err != nil {
-		return nil, err
-	}
-
-	skinTones := make(map[string]string)
-
-	skinTones["\U0001f3fb"] = "Light Skin Tone"
-	skinTones["\U0001f3fc"] = "Medium-Light Skin Tone"
-	skinTones["\U0001f3fd"] = "Medium Skin Tone"
-	skinTones["\U0001f3fe"] = "Medium-Dark Skin Tone"
-	skinTones["\U0001f3ff"] = "Dark Skin Tone"
-
-	var tmp Gemoji
-
-	// filter out emoji that require greater than max unicode version
-	for i := range data {
-		val, _ := strconv.ParseFloat(data[i].UnicodeVersion, 64)
-		if int(val) <= maxUnicodeVersion {
-			tmp = append(tmp, data[i])
-		}
-	}
-	data = tmp
-
-	sort.Slice(data, func(i, j int) bool {
-		return data[i].Aliases[0] < data[j].Aliases[0]
-	})
-
-	aliasMap := make(map[string]int, len(data))
-
-	for i, e := range data {
-		if e.Emoji == "" || len(e.Aliases) == 0 {
+	var emojis []*emoji
+	scanner := bufio.NewScanner(res.Body)
+	for scanner.Scan() {
+		// e.g. "1F44D ; fully-qualified # 👍 E0.6 thumbs up"
+		_, rest, _ := strings.Cut(scanner.Text(), ";")
+		status, comment, _ := strings.Cut(rest, "#")
+		if strings.TrimSpace(status) != "fully-qualified" {
 			continue
 		}
-		for _, a := range e.Aliases {
-			if a == "" {
-				continue
+		fields := strings.SplitN(strings.TrimSpace(comment), " ", 3)
+		if strings.ContainsFunc(fields[0], isSkinTone) {
+			continue
+		}
+		emojis = append(emojis, &emoji{Emoji: fields[0], description: fields[2]})
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	var errs []error
+	aliasOwners := map[string]string{}
+	for _, e := range emojis {
+		e.Aliases = existingAliases[e.Emoji]
+		if e.Aliases == nil {
+			e.Aliases = []string{strings.Trim(slugRe.ReplaceAllString(strings.ReplaceAll(strings.ToLower(e.description), "’", ""), "_"), "_")}
+		}
+		delete(existingAliases, e.Emoji)
+		for _, alias := range e.Aliases {
+			if owner, ok := aliasOwners[alias]; ok {
+				errs = append(errs, fmt.Errorf("alias %q used by both %q and %q", alias, owner, e.Emoji))
 			}
-			aliasMap[a] = i
+			aliasOwners[alias] = e.Emoji
 		}
 	}
-
-	// gitea customizations
-	i, ok := aliasMap["tada"]
-	if ok {
-		data[i].Aliases = append(data[i].Aliases, "hooray")
+	for code, aliases := range existingAliases {
+		errs = append(errs, fmt.Errorf("emoji %q with aliases %v is missing from Unicode data", code, aliases))
 	}
-	i, ok = aliasMap["laughing"]
-	if ok {
-		data[i].Aliases = append(data[i].Aliases, "laugh")
+	if len(errs) > 0 {
+		return nil, errors.Join(errs...)
 	}
 
-	// write a JSON file to use with tribute (write before adding skin tones since we can't support them there yet)
-	file, _ := json.MarshalIndent(data, "", "  ")
-	_ = os.WriteFile("assets/emoji.json", append(file, '\n'), 0o644)
-
-	// Add skin tones to emoji that support it
-	var (
-		s              []string
-		newEmoji       string
-		newDescription string
-		newData        Emoji
-	)
-
-	for i := range data {
-		if data[i].SkinTones {
-			for k, v := range skinTones {
-				s = strings.Split(data[i].Emoji, "")
-
-				if utf8.RuneCountInString(data[i].Emoji) == 1 {
-					s = append(s, k)
-				} else {
-					// insert into slice after first element because all emoji that support skin tones
-					// have that modifier placed at this spot
-					s = append(s, "")
-					copy(s[2:], s[1:])
-					s[1] = k
-				}
-
-				newEmoji = strings.Join(s, "")
-				newDescription = data[i].Description + ": " + v
-				newAlias := data[i].Aliases[0] + "_" + strings.ReplaceAll(v, " ", "_")
-
-				newData = Emoji{newEmoji, newDescription, []string{newAlias}, "12.0", false}
-				data = append(data, newData)
-			}
-		}
-	}
-
-	sort.Slice(data, func(i, j int) bool {
-		return data[i].Aliases[0] < data[j].Aliases[0]
+	slices.SortFunc(emojis, func(a, b *emoji) int {
+		return strings.Compare(a.Aliases[0], b.Aliases[0])
 	})
-
-	// add header
-	str := replacer.Replace(fmt.Sprintf(hdr, gemojiURL, data))
-
-	// change the format of the unicode string
-	str = emojiRE.ReplaceAllStringFunc(str, func(s string) string {
-		var err error
-		s, err = strconv.Unquote(s[len("{Emoji:"):])
-		if err != nil {
-			panic(err)
-		}
-		return "{" + strconv.QuoteToASCII(s)
-	})
-
-	// format
-	return format.Source([]byte(str))
+	return emojis, nil
 }
-
-const hdr = `
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-
-package emoji
-
-// Code generated by build/generate-emoji.go. DO NOT EDIT.
-// Sourced from %s
-var GemojiData = %#v
-`

modules/emoji/emoji.go

@@ -8,24 +8,23 @@ import (
 	"sort"
 	"strings"
 	"sync/atomic"
+	"unicode/utf8"
 
+	"gitea.dev/modules/json"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/public"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/util"
 )
 
-// Gemoji is a set of emoji data.
-type Gemoji []Emoji
-
 // Emoji represents a single emoji and associated data.
 type Emoji struct {
-	Emoji          string
-	Description    string
-	Aliases        []string
-	UnicodeVersion string
-	SkinTones      bool
+	Emoji   string   `json:"emoji"`
+	Aliases []string `json:"aliases"`
 }
 
 type globalVarsStruct struct {
+	emojis         []Emoji
 	codeMap        map[string]int    // emoji Unicode code to its emoji data.
 	aliasMap       map[string]int    // the alias to its emoji data.
 	trie           *util.TrieNode    // trie for finding emoji positions.
@@ -43,8 +42,13 @@ func globalVars() *globalVarsStruct {
 	}
 	// although there can be concurrent calls, the result should be the same, and there is no performance problem
 	vars = &globalVarsStruct{}
-	vars.codeMap = make(map[string]int, len(GemojiData))
-	vars.aliasMap = make(map[string]int, len(GemojiData))
+	if data, err := public.AssetFS().ReadFile("assets/emoji.json"); err != nil {
+		log.Error("Unable to read assets/emoji.json: %v", err)
+	} else if err = json.Unmarshal(data, &vars.emojis); err != nil {
+		log.Error("Unable to parse assets/emoji.json: %v", err)
+	}
+	vars.codeMap = make(map[string]int, len(vars.emojis))
+	vars.aliasMap = make(map[string]int, len(vars.emojis))
 	vars.trie = &util.TrieNode{}
 
 	// process emoji codes and aliases
@@ -52,11 +56,11 @@ func globalVars() *globalVarsStruct {
 	aliasPairs := make([]string, 0)
 
 	// sort from largest to small so we match combined emoji first
-	sort.Slice(GemojiData, func(i, j int) bool {
-		return len(GemojiData[i].Emoji) > len(GemojiData[j].Emoji)
+	sort.Slice(vars.emojis, func(i, j int) bool {
+		return len(vars.emojis[i].Emoji) > len(vars.emojis[j].Emoji)
 	})
 
-	for idx, emoji := range GemojiData {
+	for idx, emoji := range vars.emojis {
 		if emoji.Emoji == "" || len(emoji.Aliases) == 0 {
 			continue
 		}
@@ -95,29 +99,34 @@ func globalVars() *globalVarsStruct {
 }
 
 // FromCode retrieves the emoji data based on the provided Unicode code
-// e.g.: "\u2618" will return the Gemoji data for "shamrock".
+// e.g.: "\u2618" will return the emoji data for "shamrock".
 func FromCode(code string) *Emoji {
-	i, ok := globalVars().codeMap[code]
+	vars := globalVars()
+	i, ok := vars.codeMap[code]
+	if !ok {
+		i, ok = vars.codeMap[removeSkinTones(code)]
+	}
 	if !ok {
 		return nil
 	}
 
-	return &GemojiData[i]
+	return &vars.emojis[i]
 }
 
 // FromAlias retrieves the emoji data based on the provided alias in the form "alias" or ":alias:"
-// e.g.: "shamrock" or ":shamrock:" will return the Gemoji data for "shamrock".
+// e.g.: "shamrock" or ":shamrock:" will return the emoji data for "shamrock".
 func FromAlias(alias string) *Emoji {
 	if strings.HasPrefix(alias, ":") && strings.HasSuffix(alias, ":") {
 		alias = alias[1 : len(alias)-1]
 	}
 
-	i, ok := globalVars().aliasMap[alias]
+	vars := globalVars()
+	i, ok := vars.aliasMap[alias]
 	if !ok {
 		return nil
 	}
 
-	return &GemojiData[i]
+	return &vars.emojis[i]
 }
 
 // ReplaceCodes replaces all emoji codes with the first corresponding emoji alias in the form of ":alias:"
@@ -139,8 +148,36 @@ func FindEmojiSubmatchIndex(s string) []int {
 			continue
 		}
 		if matchLen := vars.trie.Match(s, i); matchLen > 0 {
-			return []int{i, i + matchLen}
+			return []int{i, i + skinTonedLen(vars, s[i:], matchLen)}
 		}
 	}
 	return nil
 }
+
+func isSkinTone(r rune) bool {
+	return r >= 0x1f3fb && r <= 0x1f3ff
+}
+
+func removeSkinTones(s string) string {
+	return strings.Map(func(r rune) rune { return util.Iif(isSkinTone(r), -1, r) }, s)
+}
+
+// skinTonedLen extends a match at the start of s over skin tones, which the emoji data omits
+func skinTonedLen(vars *globalVarsStruct, s string, matchLen int) int {
+	if r, _ := utf8.DecodeRuneInString(s[matchLen:]); !isSkinTone(r) {
+		return matchLen
+	}
+	tonelessLen := vars.trie.Match(removeSkinTones(s[:min(len(s), 2*len(vars.emojis[0].Emoji))]), 0)
+	end := 0
+	for end < len(s) {
+		r, size := utf8.DecodeRuneInString(s[end:])
+		if !isSkinTone(r) {
+			if tonelessLen == 0 {
+				break
+			}
+			tonelessLen -= size
+		}
+		end += size
+	}
+	return end
+}

modules/emoji/emoji_test.go

@@ -61,6 +61,11 @@ func TestReplacers(t *testing.T) {
 	}
 }
 
+func TestMain(m *testing.M) {
+	setting.SetupGiteaTestEnv()
+	m.Run()
+}
+
 const (
 	testInputWithEmojis = "This is a test string containing some emojis like \U0001f44d and \U0001f37a and some text in between."
 	testInputNoEmojis   = "This is a test string containing no emojis at all, just plain old ASCII text, which should ideally be scanned very quickly by our trie implementation."
@@ -89,6 +94,14 @@ func TestFindEmojiSubmatchIndex(t *testing.T) {
 			"\u0001\U0001f44d",
 			[]int{1, 1 + len("\U0001f44d")},
 		},
+		{
+			"👩🏿‍❤️‍👩🏿",
+			[]int{0, len("👩🏿‍❤️‍👩🏿")},
+		},
+		{
+			"🏽👍",
+			[]int{len("🏽"), len("🏽👍")},
+		},
 		{
 			// This package can handle keycap emoji if it is registered in the emoji data.
 			// However, many other places (e.g.: markup rendering) also might not handle such cases correctly.

modules/markup/html_emoji.go

@@ -14,16 +14,14 @@ import (
 	"golang.org/x/net/html/atom"
 )
 
-func createEmoji(ctx *RenderContext, content, name string) *html.Node {
+func createEmoji(ctx *RenderContext, content, alias string) *html.Node {
 	span := &html.Node{
 		Type: html.ElementNode,
 		Data: atom.Span.String(),
 		Attr: []html.Attribute{},
 	}
 	span.Attr = append(span.Attr, ctx.RenderInternal.NodeSafeAttr("class", "emoji"))
-	if name != "" {
-		span.Attr = append(span.Attr, html.Attribute{Key: "aria-label", Val: name})
-	}
+	span.Attr = append(span.Attr, html.Attribute{Key: "data-alias", Val: alias})
 
 	text := &html.Node{
 		Type: html.TextNode,
@@ -41,7 +39,6 @@ func createCustomEmoji(ctx *RenderContext, alias string) *html.Node {
 		Attr: []html.Attribute{},
 	}
 	span.Attr = append(span.Attr, ctx.RenderInternal.NodeSafeAttr("class", "emoji"))
-	span.Attr = append(span.Attr, html.Attribute{Key: "aria-label", Val: alias})
 
 	img := &html.Node{
 		Type:     html.ElementNode,
@@ -88,7 +85,7 @@ func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) {
 		converted := emoji.FromAlias(alias)
 		if converted != nil {
 			// standard emoji
-			replaceContent(node, m[0], m[1], createEmoji(ctx, converted.Emoji, converted.Description))
+			replaceContent(node, m[0], m[1], createEmoji(ctx, converted.Emoji, converted.Aliases[0]))
 			node = node.NextSibling.NextSibling
 			start = 0 // restart searching start since node has changed
 		} else if _, exist := setting.UI.CustomEmojisMap[alias]; exist {
@@ -116,7 +113,7 @@ func emojiProcessor(ctx *RenderContext, node *html.Node) {
 		start = m[1]
 		val := emoji.FromCode(codepoint)
 		if val != nil {
-			replaceContent(node, m[0], m[1], createEmoji(ctx, codepoint, val.Description))
+			replaceContent(node, m[0], m[1], createEmoji(ctx, codepoint, val.Aliases[0]))
 			node = node.NextSibling.NextSibling
 			start = 0
 		}

modules/markup/html_test.go

@@ -9,14 +9,17 @@ import (
 	"testing"
 
 	"gitea.dev/modules/emoji"
+	"gitea.dev/modules/json"
 	"gitea.dev/modules/markup"
 	"gitea.dev/modules/markup/common"
 	"gitea.dev/modules/markup/markdown"
+	"gitea.dev/modules/public"
 	"gitea.dev/modules/setting"
 	testModule "gitea.dev/modules/test"
 	"gitea.dev/modules/util"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var (
@@ -334,45 +337,47 @@ func TestRender_emoji(t *testing.T) {
 	}
 
 	// Make sure we can successfully match every emoji in our dataset with regex
-	for i := range emoji.GemojiData {
-		test(
-			emoji.GemojiData[i].Emoji,
-			`<p><span class="emoji" aria-label="`+emoji.GemojiData[i].Description+`">`+emoji.GemojiData[i].Emoji+`</span></p>`)
-	}
-	for i := range emoji.GemojiData {
-		test(
-			":"+emoji.GemojiData[i].Aliases[0]+":",
-			`<p><span class="emoji" aria-label="`+emoji.GemojiData[i].Description+`">`+emoji.GemojiData[i].Emoji+`</span></p>`)
+	data, err := public.AssetFS().ReadFile("assets", "emoji.json")
+	require.NoError(t, err)
+	var emojis []emoji.Emoji
+	require.NoError(t, json.Unmarshal(data, &emojis))
+	for _, e := range emojis {
+		expected := `<p><span class="emoji" data-alias="` + e.Aliases[0] + `">` + e.Emoji + `</span></p>`
+		test(e.Emoji, expected)
+		test(":"+e.Aliases[0]+":", expected)
 	}
 
 	// Text that should be turned into or recognized as emoji
 	test(
 		":gitea:",
-		`<p><span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span></p>`)
+		`<p><span class="emoji"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span></p>`)
 	test(
 		":custom-emoji:",
 		`<p>:custom-emoji:</p>`)
 	setting.UI.CustomEmojisMap["custom-emoji"] = ":custom-emoji:"
 	test(
 		":custom-emoji:",
-		`<p><span class="emoji" aria-label="custom-emoji"><img alt=":custom-emoji:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/custom-emoji.png"/></span></p>`)
+		`<p><span class="emoji"><img alt=":custom-emoji:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/custom-emoji.png"/></span></p>`)
 	test(
 		"这是字符:1::+1: some🐊 \U0001f44d:custom-emoji: :gitea:",
-		`<p>这是字符:1:<span class="emoji" aria-label="thumbs up">👍</span> some<span class="emoji" aria-label="crocodile">🐊</span> `+
-			`<span class="emoji" aria-label="thumbs up">👍</span><span class="emoji" aria-label="custom-emoji"><img alt=":custom-emoji:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/custom-emoji.png"/></span> `+
-			`<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span></p>`)
+		`<p>这是字符:1:<span class="emoji" data-alias="+1">👍</span> some<span class="emoji" data-alias="crocodile">🐊</span> `+
+			`<span class="emoji" data-alias="+1">👍</span><span class="emoji"><img alt=":custom-emoji:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/custom-emoji.png"/></span> `+
+			`<span class="emoji"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span></p>`)
 	test(
 		"Some text with 😄 in the middle",
-		`<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle</p>`)
+		`<p>Some text with <span class="emoji" data-alias="smile">😄</span> in the middle</p>`)
 	test(
 		"Some text with :smile: in the middle",
-		`<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle</p>`)
+		`<p>Some text with <span class="emoji" data-alias="smile">😄</span> in the middle</p>`)
 	test(
 		"Some text with 😄😄 2 emoji next to each other",
-		`<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span><span class="emoji" aria-label="grinning face with smiling eyes">😄</span> 2 emoji next to each other</p>`)
+		`<p>Some text with <span class="emoji" data-alias="smile">😄</span><span class="emoji" data-alias="smile">😄</span> 2 emoji next to each other</p>`)
 	test(
 		"😎🤪🔐🤑❓",
-		`<p><span class="emoji" aria-label="smiling face with sunglasses">😎</span><span class="emoji" aria-label="zany face">🤪</span><span class="emoji" aria-label="locked with key">🔐</span><span class="emoji" aria-label="money-mouth face">🤑</span><span class="emoji" aria-label="red question mark">❓</span></p>`)
+		`<p><span class="emoji" data-alias="sunglasses">😎</span><span class="emoji" data-alias="zany_face">🤪</span><span class="emoji" data-alias="closed_lock_with_key">🔐</span><span class="emoji" data-alias="money_mouth_face">🤑</span><span class="emoji" data-alias="question">❓</span></p>`)
+	test(
+		"👍🏽🧑🏽‍💻👩🏿‍❤️‍👩🏿",
+		`<p><span class="emoji" data-alias="+1">👍🏽</span><span class="emoji" data-alias="technologist">🧑🏽‍💻</span><span class="emoji" data-alias="couple_with_heart_woman_woman">👩🏿‍❤️‍👩🏿</span></p>`)
 
 	// should match nothing
 	test(":100:200", `<p>:100:200</p>`)
@@ -380,7 +385,7 @@ func TestRender_emoji(t *testing.T) {
 	test(":not exist:", `<p>:not exist:</p>`)
 	test("foo `:smile:", "<p>foo `:smile:</p>")
 	test("foo `:smile:`", `<p>foo <code>:smile:</code></p>`)
-	test("foo ` :smile:", "<p>foo ` <span class=\"emoji\" aria-label=\"grinning face with smiling eyes\">😄</span></p>")
+	test("foo ` :smile:", "<p>foo ` <span class=\"emoji\" data-alias=\"smile\">😄</span></p>")
 }
 
 func TestRender_ShortLinks(t *testing.T) {
@@ -532,10 +537,10 @@ func TestPostProcess(t *testing.T) {
 	// Test that other post-processing still works.
 	test(
 		":gitea:",
-		`<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`)
+		`<span class="emoji"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`)
 	test(
 		"Some text with 😄 in the middle",
-		`Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle`)
+		`Some text with <span class="emoji" data-alias="smile">😄</span> in the middle`)
 	test("http://localhost:3000/person/repo/issues/4#issuecomment-1234",
 		`<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`)
 

modules/markup/main_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	setting.IsInTesting = true
+	setting.SetupGiteaTestEnv()
 	markup.RenderBehaviorForTesting.DisableAdditionalAttributes = true
 	markup.RefreshFileNamePatterns()
 	os.Exit(m.Run())

modules/markup/markdown/main_test.go

@@ -4,7 +4,6 @@
 package markdown
 
 import (
-	"os"
 	"testing"
 
 	"gitea.dev/modules/markup"
@@ -12,7 +11,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	setting.IsInTesting = true
+	setting.SetupGiteaTestEnv()
 	markup.RenderBehaviorForTesting.DisableAdditionalAttributes = true
-	os.Exit(m.Run())
+	m.Run()
 }

modules/markup/markdown/markdown_test.go

@@ -323,8 +323,8 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) {
 }
 
 func TestRenderEmojiInLinks_Issue12331(t *testing.T) {
-	testcase := `[Link with emoji :moon: in text](https://gitea.io)`
-	expected := `<p><a href="https://gitea.io" rel="nofollow">Link with emoji <span class="emoji" aria-label="waxing gibbous moon">🌔</span> in text</a></p>
+	testcase := `[Link with emoji :moon: in text](https://gitea.com)`
+	expected := `<p><a href="https://gitea.com" rel="nofollow">Link with emoji <span class="emoji" data-alias="moon">🌔</span> in text</a></p>
 `
 	res, err := markdown.RenderString(markup.NewTestRenderContext(), testcase)
 	assert.NoError(t, err)
@@ -542,7 +542,7 @@ mail@domain.com
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare
 <a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a>
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit
-<span class="emoji" aria-label="thumbs up">👍</span>
+<span class="emoji" data-alias="+1">👍</span>
 <a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a>
 @mention-user test
 #123

modules/markup/sanitizer_default.go

@@ -34,6 +34,9 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy {
 	// Line numbers on codepreview
 	policy.AllowAttrs("data-line-number").OnElements("span")
 
+	// emoji aliases for dark theme inversion
+	policy.AllowAttrs("data-alias").OnElements("span")
+
 	// HINT: CUSTOM-URL-SCHEMES-ALLOW: setting custom means also allow them besides http/https, no custom means "allow all"
 	if len(setting.Markdown.CustomURLSchemes) > 0 {
 		policy.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)

modules/public/vitedev.go

@@ -183,7 +183,7 @@ func IsViteDevRequest(req *http.Request) bool {
 	//   - "{RepoRoot}/assets/*.json" just happens to live under the dir name "assets"; it is not related to frontend assets
 	//   - BAD DESIGN: indeed it is a "conflicted and polluted name" sample
 	switch path {
-	case "/assets/emoji.json", "/assets/codemirror-languages.json":
+	case "/assets/codemirror-languages.json":
 		return true
 	}
 	return false

modules/templates/util_render_test.go

@@ -38,9 +38,9 @@ https://example.com/file.bin
 ![remote image](https://example.com/image.jpg)
 [[local image|image.jpg]]
 [[remote link|https://example.com/image.jpg]]
-https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash
+http://localhost:3000/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare
-https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb
+http://localhost:3000/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit
 :+1:
 mail@domain.com
@@ -52,6 +52,7 @@ mail@domain.com
 }
 
 func TestMain(m *testing.M) {
+	setting.SetupGiteaTestEnv()
 	setting.Markdown.RenderOptionsComment.ShortIssuePattern = true
 	markup.Init(&markup.RenderHelperFuncs{
 		IsUsernameMentionable: func(ctx context.Context, username string) bool {
@@ -123,11 +124,11 @@ func TestRenderRepoComment(t *testing.T) {
 ![remote image](<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>)
 [[local image|image.jpg]]
 [[remote link|<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>]]
-<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a>
+<a href="http://localhost:3000/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a>
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare
-<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code>88fc37a3c0</code></a>
+<a href="http://localhost:3000/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code>88fc37a3c0</code></a>
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit
-<span class="emoji" aria-label="thumbs up">👍</span>
+<span class="emoji" data-alias="+1">👍</span>
 <a href="mailto:mail@domain.com">mail@domain.com</a>
 <a href="/mention-user">@mention-user</a> test
 <a href="/user13/repo11/issues/123" class="ref-issue">#123</a>
@@ -170,11 +171,11 @@ https://example.com/file.bin
 ![remote image](https://example.com/image.jpg)
 [[local image|image.jpg]]
 [[remote link|https://example.com/image.jpg]]
-https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash
+http://localhost:3000/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare
-https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb
+http://localhost:3000/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit
-<span class="emoji" aria-label="thumbs up">👍</span>
+<span class="emoji" data-alias="+1">👍</span>
 mail@domain.com
 @mention-user test
 <a href="/user13/repo11/issues/123" class="ref-issue">#123</a>
@@ -203,10 +204,10 @@ func TestRenderIssueTitleCodeSpan(t *testing.T) {
 		{"`#123`", `<code class="inline-code-block">#123</code>`, false},
 		{"`88fc37a3c0a4dda553bdcfc80c178a58247f42fb`", `<code class="inline-code-block">88fc37a3c0a4dda553bdcfc80c178a58247f42fb</code>`, false},
 		{"foo `:100:", "foo `:100:", true},
-		{"foo ` :100:", `foo ` + "`" + ` <span class="emoji" aria-label="hundred points">💯</span>`, true},
-		{":100:", `<span class="emoji" aria-label="hundred points">💯</span>`, true},
+		{"foo ` :100:", `foo ` + "`" + ` <span class="emoji" data-alias="100">💯</span>`, true},
+		{":100:", `<span class="emoji" data-alias="100">💯</span>`, true},
 		{"#123", `<a href="/user13/repo11/issues/123" class="ref-issue">#123</a>`, false},
-		{"`x`:100:", `<code class="inline-code-block">x</code><span class="emoji" aria-label="hundred points">💯</span>`, true},
+		{"`x`:100:", `<code class="inline-code-block">x</code><span class="emoji" data-alias="100">💯</span>`, true},
 		{"a `:100:` b `:+1:` c", `a <code class="inline-code-block">:100:</code> b <code class="inline-code-block">:+1:</code> c`, true},
 	}
 	for _, c := range cases {
@@ -230,11 +231,11 @@ func TestRenderMarkdownToHtml(t *testing.T) {
 <a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a>
 <a href="/image.jpg" rel="nofollow"><img src="/image.jpg" title="local image" alt="local image"/></a>
 <a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a>
-<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a>
+<a href="http://localhost:3000/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a>
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare
-<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow"><code>88fc37a3c0</code></a>
+<a href="http://localhost:3000/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow"><code>88fc37a3c0</code></a>
 com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit
-<span class="emoji" aria-label="thumbs up">👍</span>
+<span class="emoji" data-alias="+1">👍</span>
 <a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a>
 <a href="/mention-user" rel="nofollow">@mention-user</a> test
 #123

tests/e2e/issue-comment.test.ts

@@ -25,6 +25,18 @@ test('comment on and close an issue', async ({page, request}) => {
   await expect(page.getByRole('button', {name: 'Reopen Issue'})).toBeVisible();
 });
 
+test('emoji autocompletion in issue description', async ({page, request}) => {
+  const repoName = `e2e-emoji-${randomString(8)}`;
+  await Promise.all([apiCreateRepo(request, {name: repoName, autoInit: false}), login(page)]);
+  await page.goto(`/${env.GITEA_TEST_E2E_USER}/${repoName}/issues/new`);
+  const textarea = page.getByPlaceholder('Leave a comment');
+  await textarea.focus();
+  await textarea.pressSequentially(':tada');
+  await expect(page.getByRole('option', {name: '🎉 tada'})).toBeVisible();
+  await textarea.press('Tab');
+  await expect(textarea).toHaveValue('🎉');
+});
+
 test('unsaved issue description prompts before leaving', async ({page, request}) => {
   const repoName = `e2e-are-you-sure-${randomString(8)}`;
   await Promise.all([apiCreateRepo(request, {name: repoName, autoInit: false}), login(page)]);

tests/fuzz/fuzz_test.go

@@ -13,6 +13,11 @@ import (
 	"gitea.dev/modules/setting"
 )
 
+func TestMain(m *testing.M) {
+	setting.SetupGiteaTestEnv()
+	m.Run()
+}
+
 func newFuzzRenderContext() *markup.RenderContext {
 	return markup.NewTestRenderContext("https://example.com/go-gitea/gitea", map[string]string{"user": "go-gitea", "repo": "gitea"})
 }

web_src/css/themes/theme-gitea-dark.css

@@ -313,25 +313,25 @@ gitea-theme-meta-info {
 }
 
 /* invert emojis that are hard to read otherwise */
-.emoji[aria-label="check mark"],
-.emoji[aria-label="currency exchange"],
-.emoji[aria-label="TOP arrow"],
-.emoji[aria-label="END arrow"],
-.emoji[aria-label="ON! arrow"],
-.emoji[aria-label="SOON arrow"],
-.emoji[aria-label="heavy dollar sign"],
-.emoji[aria-label="copyright"],
-.emoji[aria-label="registered"],
-.emoji[aria-label="trade mark"],
-.emoji[aria-label="multiply"],
-.emoji[aria-label="plus"],
-.emoji[aria-label="minus"],
-.emoji[aria-label="divide"],
-.emoji[aria-label="curly loop"],
-.emoji[aria-label="double curly loop"],
-.emoji[aria-label="wavy dash"],
-.emoji[aria-label="paw prints"],
-.emoji[aria-label="musical note"],
-.emoji[aria-label="musical notes"] {
+.emoji[data-alias="heavy_check_mark"],
+.emoji[data-alias="currency_exchange"],
+.emoji[data-alias="top"],
+.emoji[data-alias="end"],
+.emoji[data-alias="on"],
+.emoji[data-alias="soon"],
+.emoji[data-alias="heavy_dollar_sign"],
+.emoji[data-alias="copyright"],
+.emoji[data-alias="registered"],
+.emoji[data-alias="tm"],
+.emoji[data-alias="heavy_multiplication_x"],
+.emoji[data-alias="heavy_plus_sign"],
+.emoji[data-alias="heavy_minus_sign"],
+.emoji[data-alias="heavy_division_sign"],
+.emoji[data-alias="curly_loop"],
+.emoji[data-alias="loop"],
+.emoji[data-alias="wavy_dash"],
+.emoji[data-alias="feet"],
+.emoji[data-alias="musical_note"],
+.emoji[data-alias="notes"] {
   filter: invert(100%) hue-rotate(180deg);
 }

web_src/js/features/comp/ComboMarkdownEditor.ts

@@ -1,6 +1,5 @@
 import '@github/markdown-toolbar-element';
 import '@github/text-expander-element';
-import {attachTribute} from '../tribute.ts';
 import {hideElem, showElem, autosize, isElemVisible, generateElemId} from '../../utils/dom.ts';
 import {
   EventUploadStateChanged,
@@ -12,7 +11,6 @@ import {handleGlobalEnterQuickSubmit} from './QuickSubmit.ts';
 import {renderPreviewPanelContent} from '../repo-editor.ts';
 import {toggleTasklistCheckbox} from '../../markup/tasklist.ts';
 import {easyMDEToolbarActions, type EasyMdeToolbarAction} from './EasyMDEToolbarActions.ts';
-import {initTextExpander} from './TextExpander.ts';
 import {showErrorToast} from '../../modules/toast.ts';
 import {POST} from '../../modules/fetch.ts';
 import {
@@ -105,7 +103,10 @@ export class ComboMarkdownEditor {
     this.prepareEasyMDEToolbarActions();
     this.setupContainer();
     this.setupTab();
-    await this.setupDropzone(); // textarea depends on dropzone
+    await Promise.all([
+      this.setupDropzone(), // textarea depends on dropzone
+      this.setupTextExpander(),
+    ]);
     this.setupTextarea();
 
     await this.switchToUserPreference();
@@ -129,6 +130,10 @@ export class ComboMarkdownEditor {
     this.previewUrl = this.container.getAttribute('data-preview-url')!;
     this.previewContext = this.container.getAttribute('data-preview-context')!;
     this.updateEditorContainerTabPage('writer');
+  }
+
+  async setupTextExpander() {
+    const {initTextExpander} = await import('./TextExpander.ts');
     initTextExpander(this.container.querySelector('text-expander')!);
   }
 
@@ -343,8 +348,9 @@ export class ComboMarkdownEditor {
 
   async switchToEasyMDE() {
     if (this.easyMDE) return;
-    const [{default: EasyMDE}] = await Promise.all([
+    const [{default: EasyMDE}, {attachTribute}] = await Promise.all([
       import('easymde'),
+      import('../tribute.ts'),
       import('../../../css/easymde.css'),
     ]);
     const easyMDEOpt: EasyMDE.Options = {
@@ -386,7 +392,7 @@ export class ComboMarkdownEditor {
       },
     });
     this.applyEditorHeights(this.container.querySelector('.CodeMirror-scroll')!, this.options.editorHeights);
-    await attachTribute(this.easyMDE.codemirror.getInputField());
+    attachTribute(this.easyMDE.codemirror.getInputField());
     if (this.dropzone) {
       initEasyMDEPaste(this.easyMDE, this.dropzone);
     }

web_src/js/features/emoji.ts

@@ -1,4 +1,4 @@
-import emojis from '../../../assets/emoji.json' with {type: 'json'};
+import emojis from '../../../public/assets/emoji.json' with {type: 'json'};
 import {html} from '../utils/html.ts';
 
 const {assetUrlPrefix, customEmojis} = window.config;

web_src/js/features/tribute.ts

@@ -1,11 +1,10 @@
 import {emojiKeys, emojiHTML, emojiString} from './emoji.ts';
 import {html, htmlRaw} from '../utils/html.ts';
 import {fetchMentions} from '../utils/match.ts';
-import type {TributeCollection} from 'tributejs';
+import Tribute, {type TributeCollection} from 'tributejs';
 import type {Mention} from '../types.ts';
 
-export async function attachTribute(element: HTMLElement) {
-  const {default: Tribute} = await import('tributejs');
+export function attachTribute(element: HTMLElement) {
   const mentionsUrl = element.closest('[data-mentions-url]')?.getAttribute('data-mentions-url');
 
   const emojiCollection: TributeCollection<string> = { // emojis

web_src/js/utils/match.ts

@@ -1,4 +1,4 @@
-import emojis from '../../../assets/emoji.json' with {type: 'json'};
+import emojis from '../../../public/assets/emoji.json' with {type: 'json'};
 import {GET} from '../modules/fetch.ts';
 import {showErrorToast} from '../modules/toast.ts';
 import {parseIssuePageInfo} from '../utils.ts';