ynkmr/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-09-02 02:53:23 +09:00
Code Issues Projects Releases Wiki Activity

fix: charset lookup (#39187) (#39197)

Browse Source 
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Giteabot
2026-09-01 08:35:50 +02:00
committed by GitHub
co-authored by wxiaoguang
parent 146cc3eec5
commit f0b4a4142b
6 changed files with 64 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
custom/conf/app.example.ini
+1 -1
View File
@@ -1027,7 +1027,7 @@ LEVEL = Info
;; If the charsets have equal confidence, tie-breaking will be done by order in this list
;; with charsets earlier in the list chosen in preference to those later.
;; Adding "defaults" will place the unused charsets at that position.
;DETECTED_CHARSETS_ORDER = UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE, ISO-8859, windows-1252, ISO-8859, windows-1250, ISO-8859, ISO-8859, ISO-8859, windows-1253, ISO-8859, windows-1255, ISO-8859, windows-1251, windows-1256, KOI8-R, ISO-8859, windows-1254, Shift_JIS, GB18030, EUC-JP, EUC-KR, Big5, ISO-2022, ISO-2022, ISO-2022, IBM424_rtl, IBM424_ltr, IBM420_rtl, IBM420_ltr
;DETECTED_CHARSETS_ORDER = UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE, ISO-8859, windows-1252, ISO-8859, windows-1250, ISO-8859, ISO-8859, ISO-8859, windows-1253, ISO-8859, windows-1255, ISO-8859, windows-1251, windows-1256, KOI8-R, ISO-8859, windows-1254, Shift_JIS, GB18030, EUC-JP, EUC-KR, Big5, ISO-2022, ISO-2022, ISO-2022
;;
;; Default ANSI charset to override non-UTF-8 charsets to
;ANSI_CHARSET =
modules/charset/charset.go
+19 -3
View File
@@ -17,6 +17,8 @@ import (
"github.com/gogs/chardet"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode/utf32"
"golang.org/x/text/transform"
)
@@ -28,12 +30,26 @@ var globalVars = sync.OnceValue(func() (ret struct {
invisibleRangeTable *unicode.RangeTable
},
) {
ret.utf8Bom = []byte{'\xef', '\xbb', '\xbf'}
ret.utf8Bom = []byte("\xef\xbb\xbf")
ret.ambiguousTableMap = newAmbiguousTableMap()
ret.invisibleRangeTable = newInvisibleRangeTable()
return ret
})
func Lookup(label string) (e encoding.Encoding, name string) {
e, name = charset.Lookup(label)
if e != nil {
return e, name
}
switch {
case strings.EqualFold(label, "UTF-32BE"):
return utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM), "UTF-32BE"
case strings.EqualFold(label, "UTF-32LE"):
return utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM), "UTF-32LE"
}
return nil, ""
}
type ConvertOpts struct {
KeepBOM bool
ErrorReplacement []byte
@@ -57,7 +73,7 @@ func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd)
}
encoding, _ := charset.Lookup(charsetLabel)
encoding, _ := Lookup(charsetLabel)
if encoding == nil {
// unknown charset, don't do any processing
return io.MultiReader(bytes.NewReader(buf[:n]), rd)
@@ -86,7 +102,7 @@ func ToUTF8(content []byte, opts ConvertOpts) []byte {
return maybeRemoveBOM(content, opts)
}
encoding, _ := charset.Lookup(charsetLabel)
encoding, _ := Lookup(charsetLabel)
if encoding == nil {
setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel)
if opts.ErrorReturnOrigin {
modules/charset/charset_test.go
+7
View File
@@ -245,3 +245,10 @@ func TestToUTF8WithFallbackReader(t *testing.T) {
}
}
}
func TestDefaultDetectedCharsetsOrder(t *testing.T) {
for _, charsetName := range setting.DefaultDetectedCharsetsOrder() {
e, _ := Lookup(charsetName)
assert.NotNil(t, e, "charset %s is not registered", charsetName)
}
}
modules/setting/repository.go
+35 -35
View File
@@ -126,41 +126,7 @@ var (
TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"`
} `ini:"repository.signing"`
}{
DetectedCharsetsOrder: []string{
"UTF-8",
"UTF-16BE",
"UTF-16LE",
"UTF-32BE",
"UTF-32LE",
"ISO-8859-1",
"windows-1252",
"ISO-8859-2",
"windows-1250",
"ISO-8859-5",
"ISO-8859-6",
"ISO-8859-7",
"windows-1253",
"ISO-8859-8-I",
"windows-1255",
"ISO-8859-8",
"windows-1251",
"windows-1256",
"KOI8-R",
"ISO-8859-9",
"windows-1254",
"Shift_JIS",
"GB18030",
"EUC-JP",
"EUC-KR",
"Big5",
"ISO-2022-JP",
"ISO-2022-KR",
"ISO-2022-CN",
"IBM424_rtl",
"IBM424_ltr",
"IBM420_rtl",
"IBM420_ltr",
},
DetectedCharsetsOrder: DefaultDetectedCharsetsOrder(),
DetectedCharsetScore: map[string]int{},
AnsiCharset: "",
ForcePrivate: false,
@@ -294,6 +260,40 @@ var (
ScriptType = "bash"
)
func DefaultDetectedCharsetsOrder() []string {
return []string{
"UTF-8",
"UTF-16BE",
"UTF-16LE",
"UTF-32BE",
"UTF-32LE",
"ISO-8859-1",
"windows-1252",
"ISO-8859-2",
"windows-1250",
"ISO-8859-5",
"ISO-8859-6",
"ISO-8859-7",
"windows-1253",
"ISO-8859-8-I",
"windows-1255",
"ISO-8859-8",
"windows-1251",
"windows-1256",
"KOI8-R",
"ISO-8859-9",
"windows-1254",
"Shift_JIS",
"GB18030",
"EUC-JP",
"EUC-KR",
"Big5",
"ISO-2022-JP",
"ISO-2022-KR",
"ISO-2022-CN",
}
}
func loadRepositoryFrom(rootCfg ConfigProvider) {
var err error
// Determine and create root git repository path.
modules/setting/setting.go
+1 -1
View File
@@ -256,5 +256,5 @@ func PanicInDevOrTesting(msg string, a ...any) {
if !IsProd || IsInTesting {
panic(fmt.Sprintf(msg, a...))
}
log.Error(msg, a...)
log.ErrorWithSkip(1, msg, a...)
}
services/gitdiff/gitdiff.go
+1 -2
View File
@@ -42,7 +42,6 @@ import (
"github.com/alecthomas/chroma/v2"
"github.com/sergi/go-diff/diffmatchpatch"
stdcharset "golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
)
@@ -889,7 +888,7 @@ parsingLoop:
}
charsetLabel, _ := charset.DetectEncoding(buffer.Bytes())
if charsetLabel != "UTF-8" {
charsetEncoding, _ := stdcharset.Lookup(charsetLabel)
charsetEncoding, _ := charset.Lookup(charsetLabel)
if charsetEncoding != nil {
diffLineTypeDecoders[lineType] = charsetEncoding.NewDecoder()
}