From 21ed4fd8da4c8992518dcfb01aa7306f7406f735 Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 7 Jan 2022 01:18:52 +0000 Subject: [PATCH] Add warning for BIDI characters in page renders and in diffs (#17562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #17514 Given the comments I've adjusted this somewhat. The numbers of characters detected are increased and include things like the use of U+300 to make à instead of à and non-breaking spaces. There is a button which can be used to escape the content to show it. Signed-off-by: Andrew Thornton Co-authored-by: Gwyneth Morgan Co-authored-by: silverwind Co-authored-by: wxiaoguang --- modules/charset/escape.go | 230 ++++++++++++++++++ modules/charset/escape_test.go | 202 +++++++++++++++ options/locale/locale_en-US.ini | 11 + routers/web/repo/blame.go | 6 + routers/web/repo/lfs.go | 5 +- routers/web/repo/view.go | 37 ++- routers/web/repo/wiki.go | 8 +- services/gitdiff/gitdiff.go | 42 +++- services/gitdiff/gitdiff_test.go | 20 +- templates/repo/blame.tmpl | 7 +- templates/repo/diff/blob_excerpt.tmpl | 18 +- templates/repo/diff/box.tmpl | 6 +- templates/repo/diff/section_split.tmpl | 68 +++++- templates/repo/diff/section_unified.tmpl | 19 +- templates/repo/editor/diff_preview.tmpl | 2 +- .../repo/issue/view_content/comments.tmpl | 2 +- templates/repo/settings/lfs_file.tmpl | 5 + templates/repo/unicode_escape_prompt.tmpl | 17 ++ templates/repo/view_file.tmpl | 57 +++-- templates/repo/wiki/view.tmpl | 25 +- web_src/js/features/common-global.js | 14 +- web_src/js/features/repo-legacy.js | 3 + web_src/js/features/repo-unicode-escape.js | 28 +++ web_src/less/_base.less | 10 + web_src/less/_repository.less | 43 ++++ web_src/less/_review.less | 11 + 26 files changed, 809 insertions(+), 87 deletions(-) create mode 100644 modules/charset/escape.go create mode 100644 modules/charset/escape_test.go create mode 100644 templates/repo/unicode_escape_prompt.tmpl create mode 100644 web_src/js/features/repo-unicode-escape.js diff --git a/modules/charset/escape.go b/modules/charset/escape.go new file mode 100644 index 00000000000..abe813b465a --- /dev/null +++ b/modules/charset/escape.go @@ -0,0 +1,230 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package charset + +import ( + "bytes" + "fmt" + "io" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/text/unicode/bidi" +) + +// EscapeStatus represents the findings of the unicode escaper +type EscapeStatus struct { + Escaped bool + HasError bool + HasBadRunes bool + HasControls bool + HasSpaces bool + HasMarks bool + HasBIDI bool + BadBIDI bool + HasRTLScript bool + HasLTRScript bool +} + +// Or combines two EscapeStatus structs into one representing the conjunction of the two +func (status EscapeStatus) Or(other EscapeStatus) EscapeStatus { + st := status + st.Escaped = st.Escaped || other.Escaped + st.HasError = st.HasError || other.HasError + st.HasBadRunes = st.HasBadRunes || other.HasBadRunes + st.HasControls = st.HasControls || other.HasControls + st.HasSpaces = st.HasSpaces || other.HasSpaces + st.HasMarks = st.HasMarks || other.HasMarks + st.HasBIDI = st.HasBIDI || other.HasBIDI + st.BadBIDI = st.BadBIDI || other.BadBIDI + st.HasRTLScript = st.HasRTLScript || other.HasRTLScript + st.HasLTRScript = st.HasLTRScript || other.HasLTRScript + return st +} + +// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string +func EscapeControlString(text string) (EscapeStatus, string) { + sb := &strings.Builder{} + escaped, _ := EscapeControlReader(strings.NewReader(text), sb) + return escaped, sb.String() +} + +// EscapeControlBytes escapes the unicode control sequences a provided []byte and returns the findings as an EscapeStatus and the escaped []byte +func EscapeControlBytes(text []byte) (EscapeStatus, []byte) { + buf := &bytes.Buffer{} + escaped, _ := EscapeControlReader(bytes.NewReader(text), buf) + return escaped, buf.Bytes() +} + +// EscapeControlReader escapes the unicode control sequences a provided Reader writing the escaped output to the output and returns the findings as an EscapeStatus and an error +func EscapeControlReader(text io.Reader, output io.Writer) (escaped EscapeStatus, err error) { + buf := make([]byte, 4096) + readStart := 0 + var n int + var writePos int + + lineHasBIDI := false + lineHasRTLScript := false + lineHasLTRScript := false + +readingloop: + for err == nil { + n, err = text.Read(buf[readStart:]) + bs := buf[:n+readStart] + i := 0 + + for i < len(bs) { + r, size := utf8.DecodeRune(bs[i:]) + // Now handle the codepoints + switch { + case r == utf8.RuneError: + if writePos < i { + if _, err = output.Write(bs[writePos:i]); err != nil { + escaped.HasError = true + return + } + writePos = i + } + // runes can be at most 4 bytes - so... + if len(bs)-i <= 3 { + // if not request more data + copy(buf, bs[i:]) + readStart = n - i + writePos = 0 + continue readingloop + } + // this is a real broken rune + escaped.HasBadRunes = true + escaped.Escaped = true + if err = writeBroken(output, bs[i:i+size]); err != nil { + escaped.HasError = true + return + } + writePos += size + case r == '\n': + if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript { + escaped.BadBIDI = true + } + lineHasBIDI = false + lineHasRTLScript = false + lineHasLTRScript = false + + case r == '\r' || r == '\t' || r == ' ': + // These are acceptable control characters and space characters + case unicode.IsSpace(r): + escaped.HasSpaces = true + escaped.Escaped = true + if writePos < i { + if _, err = output.Write(bs[writePos:i]); err != nil { + escaped.HasError = true + return + } + } + if err = writeEscaped(output, r); err != nil { + escaped.HasError = true + return + } + writePos = i + size + case unicode.Is(unicode.Bidi_Control, r): + escaped.Escaped = true + escaped.HasBIDI = true + if writePos < i { + if _, err = output.Write(bs[writePos:i]); err != nil { + escaped.HasError = true + return + } + } + lineHasBIDI = true + if err = writeEscaped(output, r); err != nil { + escaped.HasError = true + return + } + writePos = i + size + case unicode.Is(unicode.C, r): + escaped.Escaped = true + escaped.HasControls = true + if writePos < i { + if _, err = output.Write(bs[writePos:i]); err != nil { + escaped.HasError = true + return + } + } + if err = writeEscaped(output, r); err != nil { + escaped.HasError = true + return + } + writePos = i + size + case unicode.Is(unicode.M, r): + escaped.Escaped = true + escaped.HasMarks = true + if writePos < i { + if _, err = output.Write(bs[writePos:i]); err != nil { + escaped.HasError = true + return + } + } + if err = writeEscaped(output, r); err != nil { + escaped.HasError = true + return + } + writePos = i + size + default: + p, _ := bidi.Lookup(bs[i : i+size]) + c := p.Class() + if c == bidi.R || c == bidi.AL { + lineHasRTLScript = true + escaped.HasRTLScript = true + } else if c == bidi.L { + lineHasLTRScript = true + escaped.HasLTRScript = true + } + } + i += size + } + if n > 0 { + // we read something... + // write everything unwritten + if writePos < i { + if _, err = output.Write(bs[writePos:i]); err != nil { + escaped.HasError = true + return + } + } + + // reset the starting positions for the next read + readStart = 0 + writePos = 0 + } + } + if readStart > 0 { + // this means that there is an incomplete or broken rune at 0-readStart and we read nothing on the last go round + escaped.Escaped = true + escaped.HasBadRunes = true + if err = writeBroken(output, buf[:readStart]); err != nil { + escaped.HasError = true + return + } + } + if err == io.EOF { + if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript { + escaped.BadBIDI = true + } + err = nil + return + } + escaped.HasError = true + return +} + +func writeBroken(output io.Writer, bs []byte) (err error) { + _, err = fmt.Fprintf(output, `<%X>`, bs) + return +} + +func writeEscaped(output io.Writer, r rune) (err error) { + _, err = fmt.Fprintf(output, `%c`, r, r) + return +} diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go new file mode 100644 index 00000000000..dec92b49929 --- /dev/null +++ b/modules/charset/escape_test.go @@ -0,0 +1,202 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package charset + +import ( + "reflect" + "strings" + "testing" +) + +type escapeControlTest struct { + name string + text string + status EscapeStatus + result string +} + +var escapeControlTests = []escapeControlTest{ + { + name: "", + }, + { + name: "single line western", + text: "single line western", + result: "single line western", + status: EscapeStatus{HasLTRScript: true}, + }, + { + name: "multi line western", + text: "single line western\nmulti line western\n", + result: "single line western\nmulti line western\n", + status: EscapeStatus{HasLTRScript: true}, + }, + { + name: "multi line western non-breaking space", + text: "single line western\nmulti line western\n", + result: `single line western` + "\n" + `multi line western` + "\n", + status: EscapeStatus{Escaped: true, HasLTRScript: true, HasSpaces: true}, + }, + { + name: "mixed scripts: western + japanese", + text: "日属秘ぞしちゅ。Then some western.", + result: "日属秘ぞしちゅ。Then some western.", + status: EscapeStatus{HasLTRScript: true}, + }, + { + name: "japanese", + text: "日属秘ぞしちゅ。", + result: "日属秘ぞしちゅ。", + status: EscapeStatus{HasLTRScript: true}, + }, + { + name: "hebrew", + text: "עד תקופת יוון העתיקה היה העיסוק במתמטיקה תכליתי בלבד: היא שימשה כאוסף של נוסחאות לחישוב קרקע, אוכלוסין וכו'. פריצת הדרך של היוונים, פרט לתרומותיהם הגדולות לידע המתמטי, הייתה בלימוד המתמטיקה כשלעצמה, מתוקף ערכה הרוחני. יחסם של חלק מהיוונים הקדמונים למתמטיקה היה דתי - למשל, הכת שאסף סביבו פיתגורס האמינה כי המתמטיקה היא הבסיס לכל הדברים. היוונים נחשבים ליוצרי מושג ההוכחה המתמטית, וכן לראשונים שעסקו במתמטיקה לשם עצמה, כלומר כתחום מחקרי עיוני ומופשט ולא רק כעזר שימושי. עם זאת, לצדה", + result: "עד תקופת יוון העתיקה היה העיסוק במתמטיקה תכליתי בלבד: היא שימשה כאוסף של נוסחאות לחישוב קרקע, אוכלוסין וכו'. פריצת הדרך של היוונים, פרט לתרומותיהם הגדולות לידע המתמטי, הייתה בלימוד המתמטיקה כשלעצמה, מתוקף ערכה הרוחני. יחסם של חלק מהיוונים הקדמונים למתמטיקה היה דתי - למשל, הכת שאסף סביבו פיתגורס האמינה כי המתמטיקה היא הבסיס לכל הדברים. היוונים נחשבים ליוצרי מושג ההוכחה המתמטית, וכן לראשונים שעסקו במתמטיקה לשם עצמה, כלומר כתחום מחקרי עיוני ומופשט ולא רק כעזר שימושי. עם זאת, לצדה", + status: EscapeStatus{HasRTLScript: true}, + }, + { + name: "more hebrew", + text: `בתקופה מאוחרת יותר, השתמשו היוונים בשיטת סימון מתקדמת יותר, שבה הוצגו המספרים לפי 22 אותיות האלפבית היווני. לסימון המספרים בין 1 ל-9 נקבעו תשע האותיות הראשונות, בתוספת גרש ( ' ) בצד ימין של האות, למעלה; תשע האותיות הבאות ייצגו את העשרות מ-10 עד 90, והבאות את המאות. לסימון הספרות בין 1000 ל-900,000, השתמשו היוונים באותן אותיות, אך הוסיפו לאותיות את הגרש דווקא מצד שמאל של האותיות, למטה. ממיליון ומעלה, כנראה השתמשו היוונים בשני תגים במקום אחד. + + המתמטיקאי הבולט הראשון ביוון העתיקה, ויש האומרים בתולדות האנושות, הוא תאלס (624 לפנה"ס - 546 לפנה"ס בקירוב).[1] לא יהיה זה משולל יסוד להניח שהוא האדם הראשון שהוכיח משפט מתמטי, ולא רק גילה אותו. תאלס הוכיח שישרים מקבילים חותכים מצד אחד של שוקי זווית קטעים בעלי יחסים שווים (משפט תאלס הראשון), שהזווית המונחת על קוטר במעגל היא זווית ישרה (משפט תאלס השני), שהקוטר מחלק את המעגל לשני חלקים שווים, ושזוויות הבסיס במשולש שווה-שוקיים שוות זו לזו. מיוחסות לו גם שיטות למדידת גובהן של הפירמידות בעזרת מדידת צילן ולקביעת מיקומה של ספינה הנראית מן החוף. + + בשנים 582 לפנה"ס עד 496 לפנה"ס, בקירוב, חי מתמטיקאי חשוב במיוחד - פיתגורס. המקורות הראשוניים עליו מועטים, וההיסטוריונים מתקשים להפריד את העובדות משכבת המסתורין והאגדות שנקשרו בו. ידוע שסביבו התקבצה האסכולה הפיתגוראית מעין כת פסבדו-מתמטית שהאמינה ש"הכל מספר", או ליתר דיוק הכל ניתן לכימות, וייחסה למספרים משמעויות מיסטיות. ככל הנראה הפיתגוראים ידעו לבנות את הגופים האפלטוניים, הכירו את הממוצע האריתמטי, הממוצע הגאומטרי והממוצע ההרמוני והגיעו להישגים חשובים נוספים. ניתן לומר שהפיתגוראים גילו את היותו של השורש הריבועי של 2, שהוא גם האלכסון בריבוע שאורך צלעותיו 1, אי רציונלי, אך תגליתם הייתה למעשה רק שהקטעים "חסרי מידה משותפת", ומושג המספר האי רציונלי מאוחר יותר.[2] אזכור ראשון לקיומם של קטעים חסרי מידה משותפת מופיע בדיאלוג "תאיטיטוס" של אפלטון, אך רעיון זה היה מוכר עוד קודם לכן, במאה החמישית לפנה"ס להיפאסוס, בן האסכולה הפיתגוראית, ואולי לפיתגורס עצמו.[3]`, + result: `בתקופה מאוחרת יותר, השתמשו היוונים בשיטת סימון מתקדמת יותר, שבה הוצגו המספרים לפי 22 אותיות האלפבית היווני. לסימון המספרים בין 1 ל-9 נקבעו תשע האותיות הראשונות, בתוספת גרש ( ' ) בצד ימין של האות, למעלה; תשע האותיות הבאות ייצגו את העשרות מ-10 עד 90, והבאות את המאות. לסימון הספרות בין 1000 ל-900,000, השתמשו היוונים באותן אותיות, אך הוסיפו לאותיות את הגרש דווקא מצד שמאל של האותיות, למטה. ממיליון ומעלה, כנראה השתמשו היוונים בשני תגים במקום אחד. + + המתמטיקאי הבולט הראשון ביוון העתיקה, ויש האומרים בתולדות האנושות, הוא תאלס (624 לפנה"ס - 546 לפנה"ס בקירוב).[1] לא יהיה זה משולל יסוד להניח שהוא האדם הראשון שהוכיח משפט מתמטי, ולא רק גילה אותו. תאלס הוכיח שישרים מקבילים חותכים מצד אחד של שוקי זווית קטעים בעלי יחסים שווים (משפט תאלס הראשון), שהזווית המונחת על קוטר במעגל היא זווית ישרה (משפט תאלס השני), שהקוטר מחלק את המעגל לשני חלקים שווים, ושזוויות הבסיס במשולש שווה-שוקיים שוות זו לזו. מיוחסות לו גם שיטות למדידת גובהן של הפירמידות בעזרת מדידת צילן ולקביעת מיקומה של ספינה הנראית מן החוף. + + בשנים 582 לפנה"ס עד 496 לפנה"ס, בקירוב, חי מתמטיקאי חשוב במיוחד - פיתגורס. המקורות הראשוניים עליו מועטים, וההיסטוריונים מתקשים להפריד את העובדות משכבת המסתורין והאגדות שנקשרו בו. ידוע שסביבו התקבצה האסכולה הפיתגוראית מעין כת פסבדו-מתמטית שהאמינה ש"הכל מספר", או ליתר דיוק הכל ניתן לכימות, וייחסה למספרים משמעויות מיסטיות. ככל הנראה הפיתגוראים ידעו לבנות את הגופים האפלטוניים, הכירו את הממוצע האריתמטי, הממוצע הגאומטרי והממוצע ההרמוני והגיעו להישגים חשובים נוספים. ניתן לומר שהפיתגוראים גילו את היותו של השורש הריבועי של 2, שהוא גם האלכסון בריבוע שאורך צלעותיו 1, אי רציונלי, אך תגליתם הייתה למעשה רק שהקטעים "חסרי מידה משותפת", ומושג המספר האי רציונלי מאוחר יותר.[2] אזכור ראשון לקיומם של קטעים חסרי מידה משותפת מופיע בדיאלוג "תאיטיטוס" של אפלטון, אך רעיון זה היה מוכר עוד קודם לכן, במאה החמישית לפנה"ס להיפאסוס, בן האסכולה הפיתגוראית, ואולי לפיתגורס עצמו.[3]`, + status: EscapeStatus{HasRTLScript: true}, + }, + { + name: "Mixed RTL+LTR", + text: `Many computer programs fail to display bidirectional text correctly. +For example, the Hebrew name Sarah (שרה) is spelled: sin (ש) (which appears rightmost), +then resh (ר), and finally heh (ה) (which should appear leftmost).`, + result: `Many computer programs fail to display bidirectional text correctly. +For example, the Hebrew name Sarah (שרה) is spelled: sin (ש) (which appears rightmost), +then resh (ר), and finally heh (ה) (which should appear leftmost).`, + status: EscapeStatus{ + HasRTLScript: true, + HasLTRScript: true, + }, + }, + { + name: "Mixed RTL+LTR+BIDI", + text: `Many computer programs fail to display bidirectional text correctly. + For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" + + `sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).`, + result: `Many computer programs fail to display bidirectional text correctly. + For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066" + `` + "\n" + + `sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).`, + status: EscapeStatus{ + Escaped: true, + HasBIDI: true, + HasRTLScript: true, + HasLTRScript: true, + }, + }, + { + name: "Accented characters", + text: string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}), + result: string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}), + status: EscapeStatus{HasLTRScript: true}, + }, + { + name: "Program", + text: "string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})", + result: "string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})", + status: EscapeStatus{HasLTRScript: true}, + }, + { + name: "CVE testcase", + text: "if access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {", + result: `if access_level != "user` + "\u202e" + ` ` + "\u2066" + `// Check if admin` + "\u2069" + ` ` + "\u2066" + `" {`, + status: EscapeStatus{Escaped: true, HasBIDI: true, BadBIDI: true, HasLTRScript: true}, + }, + { + name: "Mixed testcase with fail", + text: `Many computer programs fail to display bidirectional text correctly. + For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" + + `sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).` + + "\nif access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {\n", + result: `Many computer programs fail to display bidirectional text correctly. + For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066" + `` + "\n" + + `sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).` + + "\n" + `if access_level != "user` + "\u202e" + ` ` + "\u2066" + `// Check if admin` + "\u2069" + ` ` + "\u2066" + `" {` + "\n", + status: EscapeStatus{Escaped: true, HasBIDI: true, BadBIDI: true, HasLTRScript: true, HasRTLScript: true}, + }, +} + +func TestEscapeControlString(t *testing.T) { + for _, tt := range escapeControlTests { + t.Run(tt.name, func(t *testing.T) { + status, result := EscapeControlString(tt.text) + if !reflect.DeepEqual(status, tt.status) { + t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status) + } + if result != tt.result { + t.Errorf("EscapeControlString()\nresult= %v,\nwanted= %v", result, tt.result) + } + }) + } +} + +func TestEscapeControlBytes(t *testing.T) { + for _, tt := range escapeControlTests { + t.Run(tt.name, func(t *testing.T) { + status, result := EscapeControlBytes([]byte(tt.text)) + if !reflect.DeepEqual(status, tt.status) { + t.Errorf("EscapeControlBytes() status = %v, wanted= %v", status, tt.status) + } + if string(result) != tt.result { + t.Errorf("EscapeControlBytes()\nresult= %v,\nwanted= %v", result, tt.result) + } + }) + } +} + +func TestEscapeControlReader(t *testing.T) { + // lets add some control characters to the tests + tests := make([]escapeControlTest, 0, len(escapeControlTests)*3) + copy(tests, escapeControlTests) + for _, test := range escapeControlTests { + test.name += " (+Control)" + test.text = "\u001E" + test.text + test.result = `` + "\u001e" + `` + test.result + test.status.Escaped = true + test.status.HasControls = true + tests = append(tests, test) + } + + for _, test := range escapeControlTests { + test.name += " (+Mark)" + test.text = "\u0300" + test.text + test.result = `` + "\u0300" + `` + test.result + test.status.Escaped = true + test.status.HasMarks = true + tests = append(tests, test) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := strings.NewReader(tt.text) + output := &strings.Builder{} + status, err := EscapeControlReader(input, output) + result := output.String() + if err != nil { + t.Errorf("EscapeControlReader(): err = %v", err) + } + + if !reflect.DeepEqual(status, tt.status) { + t.Errorf("EscapeControlReader() status = %v, wanted= %v", status, tt.status) + } + if result != tt.result { + t.Errorf("EscapeControlReader()\nresult= %v,\nwanted= %v", result, tt.result) + } + }) + } +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3d60df5d682..eacd74e1a00 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1005,6 +1005,16 @@ file_view_rendered = View Rendered file_view_raw = View Raw file_permalink = Permalink file_too_large = The file is too large to be shown. +bidi_bad_header = `This file contains unexpected Bidirectional Unicode characters!` +bidi_bad_description = `This file contains unexpected Bidirectional Unicode characters that may be processed differently from what appears below. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to reveal hidden characters.` +bidi_bad_description_escaped = `This file contains unexpected Bidirectional Unicode characters. Hidden unicode characters are escaped below. Use the Unescape button to show how they render.` +unicode_header = `This file contains hidden Unicode characters!` +unicode_description = `This file contains hidden Unicode characters that may be processed differently from what appears below. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to reveal hidden characters.` +unicode_description_escaped = `This file contains hidden Unicode characters. Hidden unicode characters are escaped below. Use the Unescape button to show how they render.` +line_unicode = `This line has hidden unicode characters` + +escape_control_characters = Escape +unescape_control_characters = Unescape file_copy_permalink = Copy Permalink video_not_supported_in_browser = Your browser does not support the HTML5 'video' tag. audio_not_supported_in_browser = Your browser does not support the HTML5 'audio' tag. @@ -2101,6 +2111,7 @@ diff.protected = Protected diff.image.side_by_side = Side by Side diff.image.swipe = Swipe diff.image.overlay = Overlay +diff.has_escaped = This line has hidden Unicode characters releases.desc = Track project versions and downloads. release.releases = Releases diff --git a/routers/web/repo/blame.go b/routers/web/repo/blame.go index 75246c3acb6..bff6a039e88 100644 --- a/routers/web/repo/blame.go +++ b/routers/web/repo/blame.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/highlight" @@ -39,6 +40,7 @@ type blameRow struct { CommitMessage string CommitSince gotemplate.HTML Code gotemplate.HTML + EscapeStatus charset.EscapeStatus } // RefBlame render blame page @@ -233,6 +235,7 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m } var lines = make([]string, 0) rows := make([]*blameRow, 0) + escapeStatus := charset.EscapeStatus{} var i = 0 var commitCnt = 0 @@ -277,11 +280,14 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m fileName := fmt.Sprintf("%v", ctx.Data["FileName"]) line = highlight.Code(fileName, language, line) + br.EscapeStatus, line = charset.EscapeControlString(line) br.Code = gotemplate.HTML(line) rows = append(rows, br) + escapeStatus = escapeStatus.Or(br.EscapeStatus) } } + ctx.Data["EscapeStatus"] = escapeStatus ctx.Data["BlameRows"] = rows ctx.Data["CommitCnt"] = commitCnt } diff --git a/routers/web/repo/lfs.go b/routers/web/repo/lfs.go index 6cc05430dde..8943641381f 100644 --- a/routers/web/repo/lfs.go +++ b/routers/web/repo/lfs.go @@ -300,10 +300,11 @@ func LFSFileGet(ctx *context.Context) { rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) // Building code view blocks with line number on server side. - fileContent, _ := io.ReadAll(rd) + escapedContent := &bytes.Buffer{} + ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, escapedContent) var output bytes.Buffer - lines := strings.Split(string(fileContent), "\n") + lines := strings.Split(escapedContent.String(), "\n") //Remove blank line at the end of file if len(lines) > 0 && lines[len(lines)-1] == "" { lines = lines[:len(lines)-1] diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 384681caf6d..e8c02b64b87 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -339,21 +339,24 @@ func renderDirectory(ctx *context.Context, treeLink string) { }, rd, &result) if err != nil { log.Error("Render failed: %v then fallback", err) - bs, _ := io.ReadAll(rd) + buf := &bytes.Buffer{} + ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf) ctx.Data["FileContent"] = strings.ReplaceAll( - gotemplate.HTMLEscapeString(string(bs)), "\n", `
`, + gotemplate.HTMLEscapeString(buf.String()), "\n", `
`, ) } else { - ctx.Data["FileContent"] = result.String() + ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlString(result.String()) } } else { ctx.Data["IsRenderedHTML"] = true - buf, err = io.ReadAll(rd) + buf := &bytes.Buffer{} + ctx.Data["EscapeStatus"], err = charset.EscapeControlReader(rd, buf) if err != nil { - log.Error("ReadAll failed: %v", err) + log.Error("Read failed: %v", err) } + ctx.Data["FileContent"] = strings.ReplaceAll( - gotemplate.HTMLEscapeString(string(buf)), "\n", `
`, + gotemplate.HTMLEscapeString(buf.String()), "\n", `
`, ) } } @@ -502,12 +505,15 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.ServerError("Render", err) return } - ctx.Data["FileContent"] = result.String() + ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlString(result.String()) } else if readmeExist { - buf, _ := io.ReadAll(rd) + buf := &bytes.Buffer{} ctx.Data["IsRenderedHTML"] = true + + ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf) + ctx.Data["FileContent"] = strings.ReplaceAll( - gotemplate.HTMLEscapeString(string(buf)), "\n", `
`, + gotemplate.HTMLEscapeString(buf.String()), "\n", `
`, ) } else { buf, _ := io.ReadAll(rd) @@ -540,7 +546,15 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st language = "" } } - ctx.Data["FileContent"] = highlight.File(lineNums, blob.Name(), language, buf) + fileContent := highlight.File(lineNums, blob.Name(), language, buf) + status, _ := charset.EscapeControlReader(bytes.NewReader(buf), io.Discard) + ctx.Data["EscapeStatus"] = status + statuses := make([]charset.EscapeStatus, len(fileContent)) + for i, line := range fileContent { + statuses[i], fileContent[i] = charset.EscapeControlString(line) + } + ctx.Data["FileContent"] = fileContent + ctx.Data["LineEscapeStatus"] = statuses } if !isLFSFile { if ctx.Repo.CanEnableEditor() { @@ -588,7 +602,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.ServerError("Render", err) return } - ctx.Data["FileContent"] = result.String() + + ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlString(result.String()) } } diff --git a/routers/web/repo/wiki.go b/routers/web/repo/wiki.go index d449800b844..d8666c7a294 100644 --- a/routers/web/repo/wiki.go +++ b/routers/web/repo/wiki.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -232,7 +233,8 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { ctx.ServerError("Render", err) return nil, nil } - ctx.Data["content"] = buf.String() + + ctx.Data["EscapeStatus"], ctx.Data["content"] = charset.EscapeControlString(buf.String()) buf.Reset() if err := markdown.Render(rctx, bytes.NewReader(sidebarContent), &buf); err != nil { @@ -243,7 +245,7 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { return nil, nil } ctx.Data["sidebarPresent"] = sidebarContent != nil - ctx.Data["sidebarContent"] = buf.String() + ctx.Data["sidebarEscapeStatus"], ctx.Data["sidebarContent"] = charset.EscapeControlString(buf.String()) buf.Reset() if err := markdown.Render(rctx, bytes.NewReader(footerContent), &buf); err != nil { @@ -254,7 +256,7 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { return nil, nil } ctx.Data["footerPresent"] = footerContent != nil - ctx.Data["footerContent"] = buf.String() + ctx.Data["footerEscapeStatus"], ctx.Data["footerContent"] = charset.EscapeControlString(buf.String()) // get commit count - wiki revisions commitsCount, _ := wikiRepo.FileCommitsCount("master", pageFilename) diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 166660b87e1..292c270b7e3 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -169,11 +169,11 @@ func getDiffLineSectionInfo(treePath, line string, lastLeftIdx, lastRightIdx int } // escape a line's content or return
needed for copy/paste purposes -func getLineContent(content string) string { +func getLineContent(content string) DiffInline { if len(content) > 0 { - return html.EscapeString(content) + return DiffInlineWithUnicodeEscape(template.HTML(html.EscapeString(content))) } - return "
" + return DiffInline{Content: "
"} } // DiffSection represents a section of a DiffFile. @@ -411,7 +411,7 @@ func fixupBrokenSpans(diffs []diffmatchpatch.Diff) []diffmatchpatch.Diff { return fixedup } -func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineType) template.HTML { +func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineType) DiffInline { buf := bytes.NewBuffer(nil) match := "" @@ -483,7 +483,7 @@ func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineT buf.Write(codeTagSuffix) } } - return template.HTML(buf.Bytes()) + return DiffInlineWithUnicodeEscape(template.HTML(buf.String())) } // GetLine gets a specific line by type (add or del) and file line number @@ -535,10 +535,28 @@ func init() { diffMatchPatch.DiffEditCost = 100 } +// DiffInline is a struct that has a content and escape status +type DiffInline struct { + EscapeStatus charset.EscapeStatus + Content template.HTML +} + +// DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped +func DiffInlineWithUnicodeEscape(s template.HTML) DiffInline { + status, content := charset.EscapeControlString(string(s)) + return DiffInline{EscapeStatus: status, Content: template.HTML(content)} +} + +// DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped +func DiffInlineWithHighlightCode(fileName, language, code string) DiffInline { + status, content := charset.EscapeControlString(highlight.Code(fileName, language, code)) + return DiffInline{EscapeStatus: status, Content: template.HTML(content)} +} + // GetComputedInlineDiffFor computes inline diff for the given line. -func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) template.HTML { +func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) DiffInline { if setting.Git.DisableDiffHighlight { - return template.HTML(getLineContent(diffLine.Content[1:])) + return getLineContent(diffLine.Content[1:]) } var ( @@ -555,26 +573,26 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) tem // try to find equivalent diff line. ignore, otherwise switch diffLine.Type { case DiffLineSection: - return template.HTML(getLineContent(diffLine.Content[1:])) + return getLineContent(diffLine.Content[1:]) case DiffLineAdd: compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx) if compareDiffLine == nil { - return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content[1:])) + return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:]) } diff1 = compareDiffLine.Content diff2 = diffLine.Content case DiffLineDel: compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx) if compareDiffLine == nil { - return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content[1:])) + return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:]) } diff1 = diffLine.Content diff2 = compareDiffLine.Content default: if strings.IndexByte(" +-", diffLine.Content[0]) > -1 { - return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content[1:])) + return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:]) } - return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content)) + return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content) } diffRecord := diffMatchPatch.DiffMain(highlight.Code(diffSection.FileName, language, diff1[1:]), highlight.Code(diffSection.FileName, language, diff2[1:]), true) diff --git a/services/gitdiff/gitdiff_test.go b/services/gitdiff/gitdiff_test.go index 21afdb4cac4..b64ac092aad 100644 --- a/services/gitdiff/gitdiff_test.go +++ b/services/gitdiff/gitdiff_test.go @@ -38,14 +38,14 @@ func TestDiffToHTML(t *testing.T) { {Type: dmp.DiffInsert, Text: "bar"}, {Type: dmp.DiffDelete, Text: " baz"}, {Type: dmp.DiffEqual, Text: " biz"}, - }, DiffLineAdd)) + }, DiffLineAdd).Content) assertEqual(t, "foo bar biz", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: "foo "}, {Type: dmp.DiffDelete, Text: "bar"}, {Type: dmp.DiffInsert, Text: " baz"}, {Type: dmp.DiffEqual, Text: " biz"}, - }, DiffLineDel)) + }, DiffLineDel).Content) assertEqual(t, "if !nohl && (lexer != nil || r.GuessLanguage) {", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: "if !nohl && lexer != nil"}, {Type: dmp.DiffInsert, Text: " || r.GuessLanguage)"}, {Type: dmp.DiffEqual, Text: " {"}, - }, DiffLineAdd)) + }, DiffLineAdd).Content) assertEqual(t, "tagURL := fmt.Sprintf("## [%s](%s/%s/%s/%s?q=&type=all&state=closed&milestone=%d) - %s", ge.Milestone\", ge.BaseURL, ge.Owner, ge.Repo, from, milestoneID, time.Now().Format("2006-01-02"))", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: "tagURL := , milestoneID, time.Now().Format("2006-01-02")"}, {Type: dmp.DiffInsert, Text: "ge.Milestone, from, milestoneID"}, {Type: dmp.DiffEqual, Text: ")"}, - }, DiffLineDel)) + }, DiffLineDel).Content) assertEqual(t, "r.WrapperRenderer(w, language, true, attrs, false)", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: "r.WrapperRenderer(w, "}, @@ -71,14 +71,14 @@ func TestDiffToHTML(t *testing.T) { {Type: dmp.DiffEqual, Text: "c"}, {Type: dmp.DiffDelete, Text: "lass=\"p\">, true, attrs"}, {Type: dmp.DiffEqual, Text: ", false)"}, - }, DiffLineDel)) + }, DiffLineDel).Content) assertEqual(t, "language, true, attrs, false)", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffInsert, Text: "language, true, attrs"}, {Type: dmp.DiffEqual, Text: ", false)"}, - }, DiffLineAdd)) + }, DiffLineAdd).Content) assertEqual(t, "print("// ", sys.argv)", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: "print"}, @@ -87,14 +87,14 @@ func TestDiffToHTML(t *testing.T) { {Type: dmp.DiffInsert, Text: "class=\"p\">("}, {Type: dmp.DiffEqual, Text: ""// ", sys.argv"}, {Type: dmp.DiffInsert, Text: ")"}, - }, DiffLineAdd)) + }, DiffLineAdd).Content) assertEqual(t, "sh 'useradd -u $(stat -c "%u" .gitignore) jenkins'", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: "sh "}, {Type: dmp.DiffDelete, Text: "4;useradd -u 111 jenkins""}, {Type: dmp.DiffInsert, Text: "9;useradd -u $(stat -c "%u" .gitignore) jenkins'"}, {Type: dmp.DiffEqual, Text: ";"}, - }, DiffLineAdd)) + }, DiffLineAdd).Content) assertEqual(t, " <h4 class="release-list-title df ac">", diffToHTML("", []dmp.Diff{ {Type: dmp.DiffEqual, Text: " <h"}, @@ -102,7 +102,7 @@ func TestDiffToHTML(t *testing.T) { {Type: dmp.DiffEqual, Text: "3"}, {Type: dmp.DiffInsert, Text: "4;release-list-title df ac""}, {Type: dmp.DiffEqual, Text: ">"}, - }, DiffLineAdd)) + }, DiffLineAdd).Content) } func TestParsePatch_skipTo(t *testing.T) { @@ -718,7 +718,7 @@ func TestDiffToHTML_14231(t *testing.T) { expected := ` run(db)` output := diffToHTML("main.v", diffRecord, DiffLineAdd) - assertEqual(t, expected, output) + assertEqual(t, expected, output.Content) } func TestNoCrashes(t *testing.T) { diff --git a/templates/repo/blame.tmpl b/templates/repo/blame.tmpl index cdd31c0eba7..3dc3522275b 100644 --- a/templates/repo/blame.tmpl +++ b/templates/repo/blame.tmpl @@ -16,11 +16,13 @@ {{end}} {{.i18n.Tr "repo.normal_view"}} {{.i18n.Tr "repo.file_history"}} + {{.i18n.Tr "repo.unescape_control_characters"}} +
-
+
{{range $row := .BlameRows}} @@ -52,6 +54,9 @@ + {{if $.EscapeStatus.Escaped}} + + {{end}} diff --git a/templates/repo/diff/blob_excerpt.tmpl b/templates/repo/diff/blob_excerpt.tmpl index 792c539ac59..e529ed3bcd0 100644 --- a/templates/repo/diff/blob_excerpt.tmpl +++ b/templates/repo/diff/blob_excerpt.tmpl @@ -19,14 +19,26 @@ {{end}} - + {{else}} - + - + {{end}} {{end}} diff --git a/templates/repo/diff/box.tmpl b/templates/repo/diff/box.tmpl index 3ab7a11bbd1..f115a5f4994 100644 --- a/templates/repo/diff/box.tmpl +++ b/templates/repo/diff/box.tmpl @@ -94,6 +94,10 @@ {{if $file.IsProtected}} {{$.i18n.Tr "repo.diff.protected"}} {{end}} + {{if not (or $file.IsIncomplete $file.IsBin $file.IsSubmodule)}} + {{$.i18n.Tr "repo.unescape_control_characters"}} + + {{end}} {{if and (not $file.IsSubmodule) (not $.PageIsWiki)}} {{if $file.IsDeleted}} {{$.i18n.Tr "repo.diff.view_file"}} @@ -104,7 +108,7 @@
-
+
{{if or $file.IsIncomplete $file.IsBin}}
{{if $file.IsIncomplete}} diff --git a/templates/repo/diff/section_split.tmpl b/templates/repo/diff/section_split.tmpl index 81223642db7..754f7cec10e 100644 --- a/templates/repo/diff/section_split.tmpl +++ b/templates/repo/diff/section_split.tmpl @@ -21,23 +21,75 @@ {{svg "octicon-fold"}} {{end}} - -
+ {{$inlineDiff := $section.GetComputedInlineDiffFor $line}} + + {{else if and (eq .GetType 3) $hasmatch}}{{/* DEL */}} {{$match := index $section.Lines $line.Match}} + {{- $leftDiff := ""}}{{if $line.LeftIdx}}{{$leftDiff = $section.GetComputedInlineDiffFor $line}}{{end}} + {{- $rightDiff := ""}}{{if $match.RightIdx}}{{$rightDiff = $section.GetComputedInlineDiffFor $match}}{{end}} + - + + - + {{else}} + {{$inlineDiff := $section.GetComputedInlineDiffFor $line}} + - + + - + {{end}} {{if and (eq .GetType 3) $hasmatch}} @@ -45,6 +97,7 @@ {{if or (gt (len $line.Comments) 0) (gt (len $match.Comments) 0)}} + + + + {{end}} + {{$inlineDiff := $section.GetComputedInlineDiffFor $line -}} + {{if eq .GetType 4}} - + {{else}} - + {{end}} {{if gt (len $line.Comments) 0}} - + diff --git a/templates/repo/editor/diff_preview.tmpl b/templates/repo/editor/diff_preview.tmpl index 0ed330c57bd..e6956648ca8 100644 --- a/templates/repo/editor/diff_preview.tmpl +++ b/templates/repo/editor/diff_preview.tmpl @@ -1,6 +1,6 @@
-
+
{{if $row.EscapeStatus.Escaped}}{{end}} {{$row.Code}} {{$.section.GetComputedInlineDiffFor $line}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}{{$inlineDiff}}{{$inlineDiff.Content}} {{if $line.LeftIdx}}{{end}}{{if $line.LeftIdx}}{{$.section.GetComputedInlineDiffFor $line}}{{end}}{{/* + */}}{{if $line.LeftIdx}}{{/* + */}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}{{$inlineDiff.Content}}{{/* + */}}{{else}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}} {{if $line.RightIdx}}{{end}}{{if $line.RightIdx}}{{$.section.GetComputedInlineDiffFor $line}}{{end}}{{/* + */}}{{if $line.RightIdx}}{{/* + */}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}{{$inlineDiff.Content}}{{/* + */}}{{else}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}
{{$section.GetComputedInlineDiffFor $line}}{{if $inlineDiff.EscapeStatus.Escaped}}{{end}}{{$inlineDiff.Content}}{{if $line.LeftIdx}}{{if $leftDiff.EscapeStatus.Escaped}}{{end}}{{end}} {{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{svg "octicon-plus"}}{{end}}{{if $line.LeftIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}{{/* + */}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/* + */}}{{/* + */}}{{svg "octicon-plus"}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}{{if $line.LeftIdx}}{{/* + */}}{{$leftDiff.Content}}{{/* + */}}{{else}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}} {{if $match.RightIdx}}{{if $rightDiff.EscapeStatus.Escaped}}{{end}}{{end}} {{if $match.RightIdx}}{{end}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{svg "octicon-plus"}}{{end}}{{if $match.RightIdx}}{{$section.GetComputedInlineDiffFor $match}}{{end}}{{/* + */}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/* + */}}{{/* + */}}{{svg "octicon-plus"}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}{{if $match.RightIdx}}{{/* + */}}{{$rightDiff.Content}}{{/* + */}}{{else}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}{{if $line.LeftIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}{{end}}{{end}} {{if $line.LeftIdx}}{{end}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2))}}{{svg "octicon-plus"}}{{end}}{{if $line.LeftIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}{{/* + */}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2))}}{{/* + */}}{{/* + */}}{{svg "octicon-plus"}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}{{if $line.LeftIdx}}{{/* + */}}{{$inlineDiff.Content}}{{/* + */}}{{else}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}} {{if $line.RightIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}{{end}}{{end}} {{if $line.RightIdx}}{{end}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3))}}{{svg "octicon-plus"}}{{end}}{{if $line.RightIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}{{/* + */}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3))}}{{/* + */}}{{/* + */}}{{svg "octicon-plus"}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}{{if $line.RightIdx}}{{/* + */}}{{$inlineDiff.Content}}{{/* + */}}{{else}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}
{{if gt (len $line.Comments) 0}} @@ -59,6 +112,7 @@ {{end}} {{if eq $line.GetCommentSide "proposed"}} @@ -75,6 +129,7 @@ {{else if gt (len $line.Comments) 0}}
{{if gt (len $line.Comments) 0}} @@ -84,6 +139,7 @@ {{end}} {{if eq $line.GetCommentSide "proposed"}} diff --git a/templates/repo/diff/section_unified.tmpl b/templates/repo/diff/section_unified.tmpl index 74634a760f5..93f9af52b47 100644 --- a/templates/repo/diff/section_unified.tmpl +++ b/templates/repo/diff/section_unified.tmpl @@ -25,16 +25,29 @@ {{if $inlineDiff.EscapeStatus.Escaped}}{{end}} {{$section.GetComputedInlineDiffFor $line}}{{/* + */}}{{$inlineDiff.Content}}{{/* + */}} + {{$line.Content}} + {{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{svg "octicon-plus"}}{{end}}{{$section.GetComputedInlineDiffFor $line}}{{/* + */}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/* + */}}{{/* + */}}{{svg "octicon-plus"}}{{/* + */}}{{/* + */}}{{end}}{{/* + */}}{{$inlineDiff.Content}}{{/* + */}}
{{template "repo/diff/conversation" mergeinto $.root "comments" $line.Comments}}
{{template "repo/diff/section_unified" dict "file" .File "root" $}} diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 026e1de0fd1..3242a5b3e56 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -512,7 +512,7 @@ {{$file := (index $diff.Files 0)}}
-
+
{{template "repo/diff/section_unified" dict "file" $file "root" $}} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index f6510f17db3..a4d6b21f1c9 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -8,10 +8,15 @@

{{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}}
+ {{if .EscapeStatus.Escaped}} + + {{.i18n.Tr "repo.escape_control_characters"}} + {{end}} {{$.i18n.Tr "repo.settings.lfs_findcommits"}}

+ {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
{{if .IsMarkup}} {{if .FileContent}}{{.FileContent | Safe}}{{end}} diff --git a/templates/repo/unicode_escape_prompt.tmpl b/templates/repo/unicode_escape_prompt.tmpl new file mode 100644 index 00000000000..d45df012e15 --- /dev/null +++ b/templates/repo/unicode_escape_prompt.tmpl @@ -0,0 +1,17 @@ +{{if .EscapeStatus.BadBIDI}} +
+ {{svg "octicon-x" 16 "close inside"}} +
+ {{$.root.i18n.Tr "repo.bidi_bad_header"}} +
+

{{$.root.i18n.Tr "repo.bidi_bad_description" | Str2html}}

+
+{{else if .EscapeStatus.Escaped}} +
+ {{svg "octicon-x" 16 "close inside"}} +
+ {{$.root.i18n.Tr "repo.unicode_header"}} +
+

{{$.root.i18n.Tr "repo.unicode_description" | Str2html}}

+
+{{end}} diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 6bd54cc8e53..d5308c154b4 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -30,7 +30,6 @@
{{end}}
- {{if not .ReadmeInList}}
{{if .HasSourceRenderedToggle}}
@@ -38,33 +37,42 @@ {{svg "octicon-file" 15}}
{{end}} -
- {{.i18n.Tr "repo.file_raw"}} - {{if not .IsViewCommit}} - {{.i18n.Tr "repo.file_permalink"}} - {{end}} - {{if .IsRepresentableAsText}} - {{.i18n.Tr "repo.blame"}} - {{end}} - {{.i18n.Tr "repo.file_history"}} -
- {{svg "octicon-download"}} - {{if .Repository.CanEnableEditor}} - {{if .CanEditFile}} - {{svg "octicon-pencil"}} - {{else}} - {{svg "octicon-pencil"}} - {{end}} - {{if .CanDeleteFile}} - {{svg "octicon-trash"}} - {{else}} - {{svg "octicon-trash"}} + {{if not .ReadmeInList}} +
+ {{.i18n.Tr "repo.file_raw"}} + {{if not .IsViewCommit}} + {{.i18n.Tr "repo.file_permalink"}} + {{end}} + {{if .IsRepresentableAsText}} + {{.i18n.Tr "repo.blame"}} + {{end}} + {{.i18n.Tr "repo.file_history"}} + {{if .EscapeStatus.Escaped}} + + {{.i18n.Tr "repo.escape_control_characters"}} + {{end}} +
+ {{svg "octicon-download"}} + {{if .Repository.CanEnableEditor}} + {{if .CanEditFile}} + {{svg "octicon-pencil"}} + {{else}} + {{svg "octicon-pencil"}} + {{end}} + {{if .CanDeleteFile}} + {{svg "octicon-trash"}} + {{else}} + {{svg "octicon-trash"}} + {{end}} {{end}} + {{else if .EscapeStatus.Escaped}} + + {{.i18n.Tr "repo.escape_control_characters"}} {{end}}
- {{end}}
+ {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
{{if .IsMarkup}} {{if .FileContent}}{{.FileContent | Safe}}{{end}} @@ -104,6 +112,9 @@ {{$line := Add $idx 1}}
+ {{if $.EscapeStatus.Escaped}} + + {{end}} {{end}} diff --git a/templates/repo/wiki/view.tmpl b/templates/repo/wiki/view.tmpl index b71c950e174..db0ce148785 100644 --- a/templates/repo/wiki/view.tmpl +++ b/templates/repo/wiki/view.tmpl @@ -45,6 +45,10 @@
+ {{if .EscapeStatus.Escaped}} + + {{.i18n.Tr "repo.escape_control_characters"}} + {{end}} {{if and .CanWriteWiki (not .Repository.IsMirror)}}
{{.i18n.Tr "repo.wiki.edit_page_button"}} @@ -61,26 +65,29 @@
{{end}}
-
- {{.content | Str2html}} +
+ {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}} + {{.content | Safe}}
{{if .sidebarPresent}}
-
+
{{if and .CanWriteWiki (not .Repository.IsMirror)}} {{svg "octicon-pencil"}} {{end}} - {{.sidebarContent | Str2html}} + {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .sidebarEscapeStatus "root" $}} + {{.sidebarContent | Safe}}
{{end}}
{{if .footerPresent}} -
- {{if and .CanWriteWiki (not .Repository.IsMirror)}} - {{svg "octicon-pencil"}} - {{end}} - {{.footerContent | Str2html}} +
+ {{if and .CanWriteWiki (not .Repository.IsMirror)}} + {{svg "octicon-pencil"}} + {{end}} + {{template "repo/unicode_escape_prompt" dict "footerEscapeStatus" .sidebarEscapeStatus "root" $}} + {{.footerContent | Safe}}
{{end}}
diff --git a/web_src/js/features/common-global.js b/web_src/js/features/common-global.js index 92c9fb81550..bf9d21ac49b 100644 --- a/web_src/js/features/common-global.js +++ b/web_src/js/features/common-global.js @@ -297,8 +297,20 @@ export function initGlobalButtons() { }); $('.hide-panel.button').on('click', function (event) { - $($(this).data('panel')).hide(); + // a `.hide-panel.button` can hide a panel, by `data-panel="selector"` or `data-panel-closest="selector"` event.preventDefault(); + let sel = $(this).attr('data-panel'); + if (sel) { + $(sel).hide(); + return; + } + sel = $(this).attr('data-panel-closest'); + if (sel) { + $(this).closest(sel).hide(); + return; + } + // should never happen, otherwise there is a bug in code + alert('Nothing to hide'); }); $('.show-modal.button').on('click', function () { diff --git a/web_src/js/features/repo-legacy.js b/web_src/js/features/repo-legacy.js index fccec8ccac9..c364beada9a 100644 --- a/web_src/js/features/repo-legacy.js +++ b/web_src/js/features/repo-legacy.js @@ -10,6 +10,7 @@ import { initRepoIssueWipToggle, initRepoPullRequestMerge, initRepoPullRequestUpdate, updateIssuesMeta, } from './repo-issue.js'; +import {initUnicodeEscapeButton} from './repo-unicode-escape.js'; import {svg} from '../svg.js'; import {htmlEscape} from 'escape-goat'; import {initRepoBranchTagDropdown} from '../components/RepoBranchTagDropdown.js'; @@ -533,6 +534,8 @@ export function initRepository() { easyMDE.codemirror.refresh(); }); } + + initUnicodeEscapeButton(); } function initRepoIssueCommentEdit() { diff --git a/web_src/js/features/repo-unicode-escape.js b/web_src/js/features/repo-unicode-escape.js new file mode 100644 index 00000000000..5791c23155a --- /dev/null +++ b/web_src/js/features/repo-unicode-escape.js @@ -0,0 +1,28 @@ +export function initUnicodeEscapeButton() { + $(document).on('click', 'a.escape-button', (e) => { + e.preventDefault(); + $(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').addClass('unicode-escaped'); + $(e.target).hide(); + $(e.target).siblings('a.unescape-button').show(); + }); + $(document).on('click', 'a.unescape-button', (e) => { + e.preventDefault(); + $(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').removeClass('unicode-escaped'); + $(e.target).hide(); + $(e.target).siblings('a.escape-button').show(); + }); + $(document).on('click', 'a.toggle-escape-button', (e) => { + e.preventDefault(); + const fileContent = $(e.target).parents('.file-content, .non-diff-file-content'); + const fileView = fileContent.find('.file-code, .file-view'); + if (fileView.hasClass('unicode-escaped')) { + fileView.removeClass('unicode-escaped'); + fileContent.find('a.unescape-button').hide(); + fileContent.find('a.escape-button').show(); + } else { + fileView.addClass('unicode-escaped'); + fileContent.find('a.unescape-button').show(); + fileContent.find('a.escape-button').hide(); + } + }); +} diff --git a/web_src/less/_base.less b/web_src/less/_base.less index 741efeadca7..c19030cccf4 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -668,6 +668,12 @@ a.ui.card:hover, color: var(--color-text-dark); } +.ui.error.message .header, +.ui.warning.message .header { + color: inherit; + filter: saturate(2); +} + .dont-break-out { overflow-wrap: break-word; word-wrap: break-word; @@ -1569,6 +1575,10 @@ a.ui.label:hover { } } +.lines-escape { + width: 0; +} + .lines-code { background-color: var(--color-code-bg); padding-left: 5px; diff --git a/web_src/less/_repository.less b/web_src/less/_repository.less index 7320f3e3025..4894a0a2c92 100644 --- a/web_src/less/_repository.less +++ b/web_src/less/_repository.less @@ -76,6 +76,24 @@ } } + .unicode-escaped .escaped-code-point { + &[data-escaped]::before { + visibility: visible; + content: attr(data-escaped); + font-family: var(--fonts-monospace); + color: var(--color-red); + } + + .char { + display: none; + } + } + + .broken-code-point { + font-family: var(--fonts-monospace); + color: blue; + } + .metas { .menu { overflow-x: auto; @@ -3020,6 +3038,26 @@ td.blob-excerpt { padding-left: 8px; } +.ui.message.unicode-escape-prompt { + margin-bottom: 0; + border-radius: 0; + display: flex; + flex-direction: column; +} + +.wiki-content-sidebar .ui.message.unicode-escape-prompt, +.wiki-content-footer .ui.message.unicode-escape-prompt { + p { + display: none; + } +} + +/* fomantic's last-child selector does not work with hidden last child */ +.ui.buttons .unescape-button { + border-top-right-radius: .28571429rem; + border-bottom-right-radius: .28571429rem; +} + .webhook-info { padding: 7px 12px; margin: 10px 0; @@ -3110,6 +3148,7 @@ td.blob-excerpt { .code-diff-unified .del-code, .code-diff-unified .del-code td, .code-diff-split .del-code .lines-num-old, +.code-diff-split .del-code .lines-escape-old, .code-diff-split .del-code .lines-type-marker-old, .code-diff-split .del-code .lines-code-old { background: var(--color-diff-removed-row-bg); @@ -3120,9 +3159,11 @@ td.blob-excerpt { .code-diff-unified .add-code td, .code-diff-split .add-code .lines-num-new, .code-diff-split .add-code .lines-type-marker-new, +.code-diff-split .add-code .lines-escape-new, .code-diff-split .add-code .lines-code-new, .code-diff-split .del-code .add-code.lines-num-new, .code-diff-split .del-code .add-code.lines-type-marker-new, +.code-diff-split .del-code .add-code.lines-escape-new, .code-diff-split .del-code .add-code.lines-code-new { background: var(--color-diff-added-row-bg); border-color: var(--color-diff-added-row-border); @@ -3131,7 +3172,9 @@ td.blob-excerpt { .code-diff-split .del-code .lines-num-new, .code-diff-split .del-code .lines-type-marker-new, .code-diff-split .del-code .lines-code-new, +.code-diff-split .del-code .lines-escape-new, .code-diff-split .add-code .lines-num-old, +.code-diff-split .add-code .lines-escape-old, .code-diff-split .add-code .lines-type-marker-old, .code-diff-split .add-code .lines-code-old { background: var(--color-diff-inactive); diff --git a/web_src/less/_review.less b/web_src/less/_review.less index 12bd6a608a8..1070ad7dded 100644 --- a/web_src/less/_review.less +++ b/web_src/less/_review.less @@ -16,6 +16,17 @@ } } +.lines-escape a.toggle-escape-button::before { + visibility: visible; + content: '⚠️'; + font-family: var(--fonts-emoji); + color: var(--color-red); +} + +.repository .diff-file-box .code-diff td.lines-escape { + padding-left: 0 !important; +} + .diff-file-box .lines-code:hover .ui.button.add-code-comment { opacity: 1; }
{{if (index $.LineEscapeStatus $idx).Escaped}}{{end}}{{$code | Safe}}