Fix commenting on non-utf8 encoded files (#11916) (#11950)

* Add comment on non-unicode line to force fail

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Just quote/unquote patch

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>

integrations/api_pull_review_test.go

@@ -86,6 +86,11 @@ func TestAPIPullReview(t *testing.T) {
 			Body:       "first old line",
 			OldLineNum: 1,
 			NewLineNum: 0,
+		}, {
+			Path:       "iso-8859-1.txt",
+			Body:       "this line contains a non-utf-8 character",
+			OldLineNum: 0,
+			NewLineNum: 1,
 		},
 		},
 	})
@@ -93,7 +98,7 @@ func TestAPIPullReview(t *testing.T) {
 	DecodeJSON(t, resp, &review)
 	assert.EqualValues(t, 6, review.ID)
 	assert.EqualValues(t, "PENDING", review.State)
-	assert.EqualValues(t, 2, review.CodeCommentsCount)
+	assert.EqualValues(t, 3, review.CodeCommentsCount)
 
 	// test SubmitPullReview
 	req = NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/reviews/%d?token=%s", repo.OwnerName, repo.Name, pullIssue.Index, review.ID, token), &api.SubmitPullReviewOptions{
@@ -104,7 +109,7 @@ func TestAPIPullReview(t *testing.T) {
 	DecodeJSON(t, resp, &review)
 	assert.EqualValues(t, 6, review.ID)
 	assert.EqualValues(t, "APPROVED", review.State)
-	assert.EqualValues(t, 2, review.CodeCommentsCount)
+	assert.EqualValues(t, 3, review.CodeCommentsCount)
 
 	// test DeletePullReview
 	req = NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/reviews?token=%s", repo.OwnerName, repo.Name, pullIssue.Index, token), &api.CreatePullReviewOptions{

integrations/gitea-repositories-meta/user2/repo1.git/objects/90/dcd07da077d1e7cd6032b52d1f79ae2b5f19b2

@@ -0,0 +1,2 @@
+x
e<EFBFBD>±NÄ0D©#åæŽ4 
+JÄA
Å5”Ž³—,—xÑzsVþ<EFBFBD>‚5„DåÑØ»ž7ý,=®®o.áEå<EFBFBD>¢áq5J=éˆý<EFBFBD>È	rÄ=>4§ú
O!óŠý<EFBFBD>ã´ðÐ6ms˜8ƒ¾&\Ea¾tÍT„´I¢z”‰Ô…!	¢dso@a›Ú&ÌK5üB)›r4–”Q¦`YèLÚ
¯²b	›<EFBFBD>¾o`Ûaä3¹@(<EFBFBD>ÒeýÔ­5
ô<EFBFBD>ÂH—\sÔHÿ9Ÿ9Rª3)Îë@ŽSùã_"§‘4sE0”Rºñ§¤.‘U|/€m¦Û¿]U÷ÌzÀ

integrations/gitea-repositories-meta/user2/repo1.git/refs/pull/3/head

@@ -1 +1 @@
-4a357436d925b5c974181ff12a994538ddc5a269
+5f22f7d0d95d614d25a5b68592adb345a4b5c7fd

models/issue_comment.go

@@ -9,7 +9,9 @@ package models
 import (
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
@@ -139,7 +141,8 @@ type Comment struct {
 	RenderedContent string `xorm:"-"`
 
 	// Path represents the 4 lines of code cemented by this comment
-	Patch string `xorm:"TEXT"`
+	Patch       string `xorm:"-"`
+	PatchQuoted string `xorm:"TEXT patch"`
 
 	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
@@ -183,6 +186,33 @@ func (c *Comment) loadIssue(e Engine) (err error) {
 	return
 }
 
+// BeforeInsert will be invoked by XORM before inserting a record
+func (c *Comment) BeforeInsert() {
+	c.PatchQuoted = c.Patch
+	if !utf8.ValidString(c.Patch) {
+		c.PatchQuoted = strconv.Quote(c.Patch)
+	}
+}
+
+// BeforeUpdate will be invoked by XORM before updating a record
+func (c *Comment) BeforeUpdate() {
+	c.PatchQuoted = c.Patch
+	if !utf8.ValidString(c.Patch) {
+		c.PatchQuoted = strconv.Quote(c.Patch)
+	}
+}
+
+// AfterLoad is invoked from XORM after setting the values of all fields of this object.
+func (c *Comment) AfterLoad(session *xorm.Session) {
+	c.Patch = c.PatchQuoted
+	if len(c.PatchQuoted) > 0 && c.PatchQuoted[0] == '"' {
+		unquoted, err := strconv.Unquote(c.PatchQuoted)
+		if err == nil {
+			c.Patch = unquoted
+		}
+	}
+}
+
 func (c *Comment) loadPoster(e Engine) (err error) {
 	if c.PosterID <= 0 || c.Poster != nil {
 		return nil