ynkmr/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-09-02 11:03:23 +09:00
Code Issues Projects Releases Wiki Activity

fix: charset lookup (#39187)

Browse Source
This commit is contained in:
wxiaoguang
2026-08-31 11:48:52 +00:00
committed by GitHub
parent 7c36ae7668
commit be3a6d1647
6 changed files with 64 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/charset/charset.go
+19 -3
View File
@@ -17,6 +17,8 @@ import (
"github.com/gogs/chardet"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode/utf32"
"golang.org/x/text/transform"
)
@@ -28,12 +30,26 @@ var globalVars = sync.OnceValue(func() (ret struct {
invisibleRangeTable *unicode.RangeTable
},
) {
ret.utf8Bom = []byte{'\xef', '\xbb', '\xbf'}
ret.utf8Bom = []byte("\xef\xbb\xbf")
ret.ambiguousTableMap = newAmbiguousTableMap()
ret.invisibleRangeTable = newInvisibleRangeTable()
return ret
})
func Lookup(label string) (e encoding.Encoding, name string) {
e, name = charset.Lookup(label)
if e != nil {
return e, name
}
switch {
case strings.EqualFold(label, "UTF-32BE"):
return utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM), "UTF-32BE"
case strings.EqualFold(label, "UTF-32LE"):
return utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM), "UTF-32LE"
}
return nil, ""
}
type ConvertOpts struct {
KeepBOM bool
ErrorReplacement []byte
@@ -57,7 +73,7 @@ func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd)
}
encoding, _ := charset.Lookup(charsetLabel)
encoding, _ := Lookup(charsetLabel)
if encoding == nil {
// unknown charset, don't do any processing
return io.MultiReader(bytes.NewReader(buf[:n]), rd)
@@ -86,7 +102,7 @@ func ToUTF8(content []byte, opts ConvertOpts) []byte {
return maybeRemoveBOM(content, opts)
}
encoding, _ := charset.Lookup(charsetLabel)
encoding, _ := Lookup(charsetLabel)
if encoding == nil {
setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel)
if opts.ErrorReturnOrigin {
modules/charset/charset_test.go
+7
View File
@@ -245,3 +245,10 @@ func TestToUTF8WithFallbackReader(t *testing.T) {
}
}
}
func TestDefaultDetectedCharsetsOrder(t *testing.T) {
for _, charsetName := range setting.DefaultDetectedCharsetsOrder() {
e, _ := Lookup(charsetName)
assert.NotNil(t, e, "charset %s is not registered", charsetName)
}
}
modules/setting/repository.go
+35 -35
View File
@@ -128,41 +128,7 @@ var (
TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"`
} `ini:"repository.signing"`
}{
DetectedCharsetsOrder: []string{
"UTF-8",
"UTF-16BE",
"UTF-16LE",
"UTF-32BE",
"UTF-32LE",
"ISO-8859-1",
"windows-1252",
"ISO-8859-2",
"windows-1250",
"ISO-8859-5",
"ISO-8859-6",
"ISO-8859-7",
"windows-1253",
"ISO-8859-8-I",
"windows-1255",
"ISO-8859-8",
"windows-1251",
"windows-1256",
"KOI8-R",
"ISO-8859-9",
"windows-1254",
"Shift_JIS",
"GB18030",
"EUC-JP",
"EUC-KR",
"Big5",
"ISO-2022-JP",
"ISO-2022-KR",
"ISO-2022-CN",
"IBM424_rtl",
"IBM424_ltr",
"IBM420_rtl",
"IBM420_ltr",
},
DetectedCharsetsOrder: DefaultDetectedCharsetsOrder(),
DetectedCharsetScore: map[string]int{},
AnsiCharset: "",
ForcePrivate: false,
@@ -297,6 +263,40 @@ var (
ScriptType = "bash"
)
func DefaultDetectedCharsetsOrder() []string {
return []string{
"UTF-8",
"UTF-16BE",
"UTF-16LE",
"UTF-32BE",
"UTF-32LE",
"ISO-8859-1",
"windows-1252",
"ISO-8859-2",
"windows-1250",
"ISO-8859-5",
"ISO-8859-6",
"ISO-8859-7",
"windows-1253",
"ISO-8859-8-I",
"windows-1255",
"ISO-8859-8",
"windows-1251",
"windows-1256",
"KOI8-R",
"ISO-8859-9",
"windows-1254",
"Shift_JIS",
"GB18030",
"EUC-JP",
"EUC-KR",
"Big5",
"ISO-2022-JP",
"ISO-2022-KR",
"ISO-2022-CN",
}
}
func loadRepositoryFrom(rootCfg ConfigProvider) {
var err error
// Determine and create root git repository path.
modules/setting/setting.go
+1 -1
View File
@@ -257,5 +257,5 @@ func PanicInDevOrTesting(msg string, a ...any) {
if !IsProd || IsInTesting {
panic(fmt.Sprintf(msg, a...))
}
log.Error(msg, a...)
log.ErrorWithSkip(1, msg, a...)
}