From 7b12543e08b8b7440552b005938c12cec6615713 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 14 Nov 2024 17:44:51 +0800 Subject: [PATCH] fix --- custom/conf/app.example.ini | 2 +- modules/httplib/serve.go | 4 +-- modules/setting/attachment.go | 26 ++++++++--------- modules/typesniffer/typesniffer.go | 39 ++++++++++++++++++++----- modules/typesniffer/typesniffer_test.go | 30 +++++++++++++++++++ web_src/js/utils.test.ts | 2 +- web_src/js/utils.ts | 2 +- 7 files changed, 80 insertions(+), 25 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 7d5b3961bc8..dcd396ab58e 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1912,7 +1912,7 @@ LEVEL = Info ;ENABLED = true ;; ;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. -;ALLOWED_TYPES = .csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip +;ALLOWED_TYPES = .avif,.csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.webp,.xls,.xlsx,.zip ;; ;; Max size of each file. Defaults to 2048MB ;MAX_SIZE = 2048 diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index 2e3e6a7c423..8fb667876e2 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -46,7 +46,7 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { w.Header().Add(gzhttp.HeaderNoCompression, "1") } - contentType := typesniffer.ApplicationOctetStream + contentType := typesniffer.MimeTypeApplicationOctetStream if opts.ContentType != "" { if opts.ContentTypeCharset != "" { contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset) @@ -107,7 +107,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri } else if isPlain { opts.ContentType = "text/plain" } else { - opts.ContentType = typesniffer.ApplicationOctetStream + opts.ContentType = typesniffer.MimeTypeApplicationOctetStream } } diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go index 0fdabb50320..c11b0c478ae 100644 --- a/modules/setting/attachment.go +++ b/modules/setting/attachment.go @@ -3,33 +3,33 @@ package setting -// Attachment settings -var Attachment = struct { +type AttachmentSettingType struct { Storage *Storage AllowedTypes string MaxSize int64 MaxFiles int Enabled bool -}{ - Storage: &Storage{}, - AllowedTypes: ".cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip", - MaxSize: 2048, - MaxFiles: 5, - Enabled: true, } +var Attachment AttachmentSettingType + func loadAttachmentFrom(rootCfg ConfigProvider) (err error) { + Attachment = AttachmentSettingType{ + AllowedTypes: ".avif,.cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.webp,.xls,.xlsx,.zip", + MaxSize: 2048, + MaxFiles: 5, + Enabled: true, + } sec, _ := rootCfg.GetSection("attachment") if sec == nil { Attachment.Storage, err = getStorage(rootCfg, "attachments", "", nil) return err } - Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip") - Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(2048) - Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5) - Attachment.Enabled = sec.Key("ENABLED").MustBool(true) - + Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(Attachment.AllowedTypes) + Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(Attachment.MaxSize) + Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(Attachment.MaxFiles) + Attachment.Enabled = sec.Key("ENABLED").MustBool(Attachment.Enabled) Attachment.Storage, err = getStorage(rootCfg, "attachments", "", sec) return err } diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 6aec5c285e2..cc0f97dcd5e 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -5,10 +5,12 @@ package typesniffer import ( "bytes" + "encoding/binary" "fmt" "io" "net/http" "regexp" + "slices" "strings" "code.gitea.io/gitea/modules/util" @@ -18,10 +20,10 @@ import ( const sniffLen = 1024 const ( - // SvgMimeType MIME type of SVG images. - SvgMimeType = "image/svg+xml" - // ApplicationOctetStream MIME type of binary files. - ApplicationOctetStream = "application/octet-stream" + MimeTypeImageSvg = "image/svg+xml" + MimeTypeImageAvif = "image/avif" + + MimeTypeApplicationOctetStream = "application/octet-stream" ) var ( @@ -47,7 +49,7 @@ func (ct SniffedType) IsImage() bool { // IsSvgImage detects if data is an SVG image format func (ct SniffedType) IsSvgImage() bool { - return strings.Contains(ct.contentType, SvgMimeType) + return strings.Contains(ct.contentType, MimeTypeImageSvg) } // IsPDF detects if data is a PDF format @@ -81,6 +83,25 @@ func (ct SniffedType) GetMimeType() string { return strings.SplitN(ct.contentType, ";", 2)[0] } +func detectFileTypeBox(data []byte) (brands []string, found bool) { + if len(data) < 12 { + return nil, false + } + boxSize := int(binary.BigEndian.Uint32(data[:4])) + if boxSize < 12 || boxSize > len(data) { + return nil, false + } + tag := string(data[4:8]) + if tag != "ftyp" { + return nil, false + } + brands = append(brands, string(data[8:12])) + for i := 16; i+4 <= boxSize; i += 4 { + brands = append(brands, string(data[i:i+4])) + } + return brands, true +} + // DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty. func DetectContentType(data []byte) SniffedType { if len(data) == 0 { @@ -94,7 +115,6 @@ func DetectContentType(data []byte) SniffedType { } // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888 - detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html") detectByXML := strings.Contains(ct, "text/xml") if detectByHTML || detectByXML { @@ -102,7 +122,7 @@ func DetectContentType(data []byte) SniffedType { dataProcessed = bytes.TrimSpace(dataProcessed) if detectByHTML && svgTagRegex.Match(dataProcessed) || detectByXML && svgTagInXMLRegex.Match(dataProcessed) { - ct = SvgMimeType + ct = MimeTypeImageSvg } } @@ -116,6 +136,11 @@ func DetectContentType(data []byte) SniffedType { } } + fileTypeBrands, found := detectFileTypeBox(data) + if found && slices.Contains(fileTypeBrands, "avif") { + ct = MimeTypeImageAvif + } + if ct == "application/ogg" { dataHead := data if len(dataHead) > 256 { diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go index 731fac11e71..3e5db3308b5 100644 --- a/modules/typesniffer/typesniffer_test.go +++ b/modules/typesniffer/typesniffer_test.go @@ -134,3 +134,33 @@ func TestDetectContentTypeOgg(t *testing.T) { assert.NoError(t, err) assert.True(t, st.IsVideo()) } + +func TestDetectFileTypeBox(t *testing.T) { + _, found := detectFileTypeBox([]byte("\x00\x00\xff\xffftypAAAA....")) + assert.False(t, found) + + brands, found := detectFileTypeBox([]byte("\x00\x00\x00\x0cftypAAAA")) + assert.True(t, found) + assert.Equal(t, []string{"AAAA"}, brands) + + brands, found = detectFileTypeBox([]byte("\x00\x00\x00\x10ftypAAAA....BBBB")) + assert.True(t, found) + assert.Equal(t, []string{"AAAA"}, brands) + + brands, found = detectFileTypeBox([]byte("\x00\x00\x00\x14ftypAAAA....BBBB")) + assert.True(t, found) + assert.Equal(t, []string{"AAAA", "BBBB"}, brands) + + _, found = detectFileTypeBox([]byte("\x00\x00\x00\x14ftypAAAA....BBB")) + assert.False(t, found) + + brands, found = detectFileTypeBox([]byte("\x00\x00\x00\x13ftypAAAA....BBB")) + assert.True(t, found) + assert.Equal(t, []string{"AAAA"}, brands) +} + +func TestDetectContentTypeAvif(t *testing.T) { + buf := []byte("\x00\x00\x00\x20ftypavif.......................") + st := DetectContentType(buf) + assert.Equal(t, MimeTypeImageAvif, st.contentType) +} diff --git a/web_src/js/utils.test.ts b/web_src/js/utils.test.ts index 647676bf205..ac9d4fab91b 100644 --- a/web_src/js/utils.test.ts +++ b/web_src/js/utils.test.ts @@ -118,7 +118,7 @@ test('encodeURLEncodedBase64, decodeURLEncodedBase64', () => { }); test('file detection', () => { - for (const name of ['a.jpg', '/a.jpeg', '.file.png', '.webp', 'file.svg']) { + for (const name of ['a.avif', 'a.jpg', '/a.jpeg', '.file.png', '.webp', 'file.svg']) { expect(isImageFile({name})).toBeTruthy(); } for (const name of ['', 'a.jpg.x', '/path.png/x', 'webp']) { diff --git a/web_src/js/utils.ts b/web_src/js/utils.ts index 4fed74e20f0..bd872f094ca 100644 --- a/web_src/js/utils.ts +++ b/web_src/js/utils.ts @@ -165,7 +165,7 @@ export function sleep(ms: number): Promise { } export function isImageFile({name, type}: {name: string, type?: string}): boolean { - return /\.(jpe?g|png|gif|webp|svg|heic)$/i.test(name || '') || type?.startsWith('image/'); + return /\.(avif|jpe?g|png|gif|webp|svg|heic)$/i.test(name || '') || type?.startsWith('image/'); } export function isVideoFile({name, type}: {name: string, type?: string}): boolean {