fix: charset lookup (#39187) (#39197)

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

custom/conf/app.example.ini

@@ -1027,7 +1027,7 @@ LEVEL = Info
 ;; If the charsets have equal confidence, tie-breaking will be done by order in this list
 ;; with charsets earlier in the list chosen in preference to those later.
 ;; Adding "defaults" will place the unused charsets at that position.
-;DETECTED_CHARSETS_ORDER = UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE, ISO-8859, windows-1252, ISO-8859, windows-1250, ISO-8859, ISO-8859, ISO-8859, windows-1253, ISO-8859, windows-1255, ISO-8859, windows-1251, windows-1256, KOI8-R, ISO-8859, windows-1254, Shift_JIS, GB18030, EUC-JP, EUC-KR, Big5, ISO-2022, ISO-2022, ISO-2022, IBM424_rtl, IBM424_ltr, IBM420_rtl, IBM420_ltr
+;DETECTED_CHARSETS_ORDER = UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE, ISO-8859, windows-1252, ISO-8859, windows-1250, ISO-8859, ISO-8859, ISO-8859, windows-1253, ISO-8859, windows-1255, ISO-8859, windows-1251, windows-1256, KOI8-R, ISO-8859, windows-1254, Shift_JIS, GB18030, EUC-JP, EUC-KR, Big5, ISO-2022, ISO-2022, ISO-2022
 ;;
 ;; Default ANSI charset to override non-UTF-8 charsets to
 ;ANSI_CHARSET =

modules/charset/charset.go

@@ -17,6 +17,8 @@ import (
 
 	"github.com/gogs/chardet"
 	"golang.org/x/net/html/charset"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/unicode/utf32"
 	"golang.org/x/text/transform"
 )
 
@@ -28,12 +30,26 @@ var globalVars = sync.OnceValue(func() (ret struct {
 	invisibleRangeTable *unicode.RangeTable
 },
 ) {
-	ret.utf8Bom = []byte{'\xef', '\xbb', '\xbf'}
+	ret.utf8Bom = []byte("\xef\xbb\xbf")
 	ret.ambiguousTableMap = newAmbiguousTableMap()
 	ret.invisibleRangeTable = newInvisibleRangeTable()
 	return ret
 })
 
+func Lookup(label string) (e encoding.Encoding, name string) {
+	e, name = charset.Lookup(label)
+	if e != nil {
+		return e, name
+	}
+	switch {
+	case strings.EqualFold(label, "UTF-32BE"):
+		return utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM), "UTF-32BE"
+	case strings.EqualFold(label, "UTF-32LE"):
+		return utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM), "UTF-32LE"
+	}
+	return nil, ""
+}
+
 type ConvertOpts struct {
 	KeepBOM           bool
 	ErrorReplacement  []byte
@@ -57,7 +73,7 @@ func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
 		return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd)
 	}
 
-	encoding, _ := charset.Lookup(charsetLabel)
+	encoding, _ := Lookup(charsetLabel)
 	if encoding == nil {
 		// unknown charset, don't do any processing
 		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
@@ -86,7 +102,7 @@ func ToUTF8(content []byte, opts ConvertOpts) []byte {
 		return maybeRemoveBOM(content, opts)
 	}
 
-	encoding, _ := charset.Lookup(charsetLabel)
+	encoding, _ := Lookup(charsetLabel)
 	if encoding == nil {
 		setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel)
 		if opts.ErrorReturnOrigin {

modules/charset/charset_test.go

@@ -245,3 +245,10 @@ func TestToUTF8WithFallbackReader(t *testing.T) {
 		}
 	}
 }
+
+func TestDefaultDetectedCharsetsOrder(t *testing.T) {
+	for _, charsetName := range setting.DefaultDetectedCharsetsOrder() {
+		e, _ := Lookup(charsetName)
+		assert.NotNil(t, e, "charset %s is not registered", charsetName)
+	}
+}

modules/setting/repository.go

@@ -126,41 +126,7 @@ var (
 			TrustedSSHKeys    []string `ini:"TRUSTED_SSH_KEYS"`
 		} `ini:"repository.signing"`
 	}{
-		DetectedCharsetsOrder: []string{
+		DetectedCharsetsOrder:                   DefaultDetectedCharsetsOrder(),
-			"UTF-8",
-			"UTF-16BE",
-			"UTF-16LE",
-			"UTF-32BE",
-			"UTF-32LE",
-			"ISO-8859-1",
-			"windows-1252",
-			"ISO-8859-2",
-			"windows-1250",
-			"ISO-8859-5",
-			"ISO-8859-6",
-			"ISO-8859-7",
-			"windows-1253",
-			"ISO-8859-8-I",
-			"windows-1255",
-			"ISO-8859-8",
-			"windows-1251",
-			"windows-1256",
-			"KOI8-R",
-			"ISO-8859-9",
-			"windows-1254",
-			"Shift_JIS",
-			"GB18030",
-			"EUC-JP",
-			"EUC-KR",
-			"Big5",
-			"ISO-2022-JP",
-			"ISO-2022-KR",
-			"ISO-2022-CN",
-			"IBM424_rtl",
-			"IBM424_ltr",
-			"IBM420_rtl",
-			"IBM420_ltr",
-		},
 		DetectedCharsetScore:                    map[string]int{},
 		AnsiCharset:                             "",
 		ForcePrivate:                            false,
@@ -294,6 +260,40 @@ var (
 	ScriptType   = "bash"
 )
 
+func DefaultDetectedCharsetsOrder() []string {
+	return []string{
+		"UTF-8",
+		"UTF-16BE",
+		"UTF-16LE",
+		"UTF-32BE",
+		"UTF-32LE",
+		"ISO-8859-1",
+		"windows-1252",
+		"ISO-8859-2",
+		"windows-1250",
+		"ISO-8859-5",
+		"ISO-8859-6",
+		"ISO-8859-7",
+		"windows-1253",
+		"ISO-8859-8-I",
+		"windows-1255",
+		"ISO-8859-8",
+		"windows-1251",
+		"windows-1256",
+		"KOI8-R",
+		"ISO-8859-9",
+		"windows-1254",
+		"Shift_JIS",
+		"GB18030",
+		"EUC-JP",
+		"EUC-KR",
+		"Big5",
+		"ISO-2022-JP",
+		"ISO-2022-KR",
+		"ISO-2022-CN",
+	}
+}
+
 func loadRepositoryFrom(rootCfg ConfigProvider) {
 	var err error
 	// Determine and create root git repository path.

modules/setting/setting.go

@@ -256,5 +256,5 @@ func PanicInDevOrTesting(msg string, a ...any) {
 	if !IsProd || IsInTesting {
 		panic(fmt.Sprintf(msg, a...))
 	}
-	log.Error(msg, a...)
+	log.ErrorWithSkip(1, msg, a...)
 }

services/gitdiff/gitdiff.go

@@ -42,7 +42,6 @@ import (
 
 	"github.com/alecthomas/chroma/v2"
 	"github.com/sergi/go-diff/diffmatchpatch"
-	stdcharset "golang.org/x/net/html/charset"
 	"golang.org/x/text/encoding"
 	"golang.org/x/text/transform"
 )
@@ -889,7 +888,7 @@ parsingLoop:
 			}
 			charsetLabel, _ := charset.DetectEncoding(buffer.Bytes())
 			if charsetLabel != "UTF-8" {
-				charsetEncoding, _ := stdcharset.Lookup(charsetLabel)
+				charsetEncoding, _ := charset.Lookup(charsetLabel)
 				if charsetEncoding != nil {
 					diffLineTypeDecoders[lineType] = charsetEncoding.NewDecoder()
 				}