From 0ddca40529c5df6dfa629b7dfa7790c8467e557e Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 22 Jun 2021 16:06:04 -0400 Subject: [PATCH 01/10] Don't render title as list item This fixes an issue where "12. April" would get rendered as "1. April" because it looks like a Markdown list item to our renderer. Now, we parse titles as titles, instead of standalone text, which causes the renderer to give us the results we want. This also adds some basic tests for the applyBasicMarkdown() func. Closes #470 --- postrender.go | 8 +++++++- postrender_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 postrender_test.go diff --git a/postrender.go b/postrender.go index 8e71109..0992d30 100644 --- a/postrender.go +++ b/postrender.go @@ -11,6 +11,7 @@ package writefreely import ( + "bytes" "encoding/json" "fmt" "html" @@ -191,7 +192,12 @@ func applyBasicMarkdown(data []byte) string { blackfriday.HTML_SMARTYPANTS_DASHES // Generate Markdown - md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions) + // This passes the supplied title into blackfriday.Markdown() as an H1 header, so we only render HTML that + // belongs in an H1. + md := blackfriday.Markdown(append([]byte("# "), data...), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions) + // Remove H1 markup + md = bytes.TrimSpace(md) // blackfriday.Markdown adds a newline at the end of the

+ md = md[len("

") : len(md)-len("

")] // Strip out bad HTML policy := bluemonday.UGCPolicy() policy.AllowAttrs("class", "id").Globally() diff --git a/postrender_test.go b/postrender_test.go new file mode 100644 index 0000000..240e755 --- /dev/null +++ b/postrender_test.go @@ -0,0 +1,36 @@ +/* + * Copyright © 2021 A Bunch Tell LLC. + * + * This file is part of WriteFreely. + * + * WriteFreely is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, included + * in the LICENSE file in this source code package. + */ + +package writefreely + +import "testing" + +func TestApplyBasicMarkdown(t *testing.T) { + tests := []struct { + name string + in string + result string + }{ + {"plain", "Hello, World!", "Hello, World!"}, + {"multibyte", "こんにちは", `こんにちは`}, + {"bold", "**안녕하세요**", `안녕하세요`}, + {"link", "[WriteFreely](https://writefreely.org)", `WriteFreely`}, + {"date", "12. April", `12. April`}, + {"table", "| Hi | There |", `| Hi | There |`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res := applyBasicMarkdown([]byte(test.in)) + if res != test.result { + t.Errorf("%s: wanted %s, got %s", test.name, test.result, res) + } + }) + } +} From 3008668a7d436cf945dff81408e64aef83dbb9f8 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 23 Jun 2021 12:43:05 -0400 Subject: [PATCH 02/10] Error when generic OAuth provider doesn't return a user ID ...on the OAuth access token inspection call. This returns an error and privately (via logs) prompts the admin to add a `map_user_id` config value. Fixes #469 --- oauth_generic.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/oauth_generic.go b/oauth_generic.go index ba8b97e..ccf6a0f 100644 --- a/oauth_generic.go +++ b/oauth_generic.go @@ -1,8 +1,20 @@ +/* + * Copyright © 2020-2021 A Bunch Tell LLC and respective authors. + * + * This file is part of WriteFreely. + * + * WriteFreely is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, included + * in the LICENSE file in this source code package. + */ + package writefreely import ( "context" "errors" + "fmt" + "github.com/writeas/web-core/log" "net/http" "net/url" "strings" @@ -118,6 +130,10 @@ func (c genericOauthClient) inspectOauthAccessToken(ctx context.Context, accessT // map each relevant field in inspectResponse to the mapped field from the config var inspectResponse InspectResponse inspectResponse.UserID, _ = genericInterface[c.MapUserID].(string) + if inspectResponse.UserID == "" { + log.Error("[CONFIGURATION ERROR] Generic OAuth provider returned empty UserID value (`%s`).\n Do you need to configure a different `map_user_id` value for this provider?", c.MapUserID) + return nil, fmt.Errorf("no UserID (`%s`) value returned", c.MapUserID) + } inspectResponse.Username, _ = genericInterface[c.MapUsername].(string) inspectResponse.DisplayName, _ = genericInterface[c.MapDisplayName].(string) inspectResponse.Email, _ = genericInterface[c.MapEmail].(string) From e91ffe2dcb7a5f2df897b2f4a731c4540d26c03d Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 23 Jun 2021 16:50:23 -0400 Subject: [PATCH 03/10] Remove additional slash in /feed/ redirect --- collections.go | 2 ++ posts.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/collections.go b/collections.go index f79cc2d..beadf19 100644 --- a/collections.go +++ b/collections.go @@ -242,6 +242,8 @@ func (c *Collection) DisplayCanonicalURL() string { return d + p } +// RedirectingCanonicalURL returns the fully-qualified canonical URL for the Collection, with a trailing slash. The +// hostName field needs to be populated for this to work correctly. func (c *Collection) RedirectingCanonicalURL(isRedir bool) string { if c.hostName == "" { // If this is true, the human programmers screwed up. So ask for a bug report and fail, fail, fail diff --git a/posts.go b/posts.go index 4d8d019..d8754c4 100644 --- a/posts.go +++ b/posts.go @@ -1461,7 +1461,7 @@ func viewCollectionPost(app *App, w http.ResponseWriter, r *http.Request) error if slug == "feed" { // User tried to access blog feed without a trailing slash, and // there's no post with a slug "feed" - return impart.HTTPError{http.StatusFound, c.CanonicalURL() + "/feed/"} + return impart.HTTPError{http.StatusFound, c.CanonicalURL() + "feed/"} } po := &Post{ From f933b3617086a7995e81855be14576cfc51d8f43 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 23 Jun 2021 17:38:22 -0400 Subject: [PATCH 04/10] Prevent out of bounds error when post has no title --- postrender.go | 4 ++++ postrender_test.go | 1 + 2 files changed, 5 insertions(+) diff --git a/postrender.go b/postrender.go index 0992d30..1cb4f0e 100644 --- a/postrender.go +++ b/postrender.go @@ -182,6 +182,10 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c } func applyBasicMarkdown(data []byte) string { + if len(data) == 0 { + return "" + } + mdExtensions := 0 | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_SPACE_HEADERS | diff --git a/postrender_test.go b/postrender_test.go index 240e755..262d550 100644 --- a/postrender_test.go +++ b/postrender_test.go @@ -18,6 +18,7 @@ func TestApplyBasicMarkdown(t *testing.T) { in string result string }{ + {"empty", "", ""}, {"plain", "Hello, World!", "Hello, World!"}, {"multibyte", "こんにちは", `こんにちは`}, {"bold", "**안녕하세요**", `안녕하세요`}, From 720a8c19759d02f845090d59e9ce50205b89f46f Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 25 Jun 2021 11:16:03 -0400 Subject: [PATCH 05/10] Don't include img alt text in post Summary() Fixes #306 --- app.go | 2 +- feed.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- postrender.go | 2 +- posts.go | 4 ++-- read.go | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app.go b/app.go index 40eb858..c6000f6 100644 --- a/app.go +++ b/app.go @@ -30,7 +30,7 @@ import ( "github.com/gorilla/schema" "github.com/gorilla/sessions" "github.com/manifoldco/promptui" - stripmd "github.com/writeas/go-strip-markdown" + stripmd "github.com/writeas/go-strip-markdown/v2" "github.com/writeas/impart" "github.com/writeas/web-core/auth" "github.com/writeas/web-core/converter" diff --git a/feed.go b/feed.go index 32c6591..d85baf8 100644 --- a/feed.go +++ b/feed.go @@ -17,7 +17,7 @@ import ( . "github.com/gorilla/feeds" "github.com/gorilla/mux" - stripmd "github.com/writeas/go-strip-markdown" + stripmd "github.com/writeas/go-strip-markdown/v2" "github.com/writeas/web-core/log" ) diff --git a/go.mod b/go.mod index 8344d4c..cb8bd79 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/urfave/cli/v2 v2.3.0 github.com/writeas/activity v0.1.2 github.com/writeas/activityserve v0.0.0-20200409150223-d7ab3eaa4481 - github.com/writeas/go-strip-markdown v2.0.1+incompatible + github.com/writeas/go-strip-markdown/v2 v2.1.1 github.com/writeas/go-webfinger v1.1.0 github.com/writeas/httpsig v1.0.0 github.com/writeas/impart v1.1.1 diff --git a/go.sum b/go.sum index 354a1dd..5f2eb13 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,8 @@ github.com/writeas/activity v0.1.2 h1:Y12B5lIrabfqKE7e7HFCWiXrlfXljr9tlkFm2mp7Dg github.com/writeas/activity v0.1.2/go.mod h1:mYYgiewmEM+8tlifirK/vl6tmB2EbjYaxwb+ndUw5T0= github.com/writeas/activityserve v0.0.0-20200409150223-d7ab3eaa4481 h1:BiSivIxLQFcKoUorpNN3rNwwFG5bITPnqUSyIccfdh0= github.com/writeas/activityserve v0.0.0-20200409150223-d7ab3eaa4481/go.mod h1:4akDJSl+sSp+QhrQKMqzAqdV1gJ1pPx6XPI77zgMM8o= -github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= -github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= +github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28= +github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA= github.com/writeas/go-webfinger v1.1.0 h1:MzNyt0ry/GMsRmJGftn2o9mPwqK1Q5MLdh4VuJCfb1Q= github.com/writeas/go-webfinger v1.1.0/go.mod h1:w2VxyRO/J5vfNjJHYVubsjUGHd3RLDoVciz0DE3ApOc= github.com/writeas/go-writeas v1.1.0 h1:WHGm6wriBkxYAOGbvriXH8DlMUGOi6jhSZLUZKQ+4mQ= diff --git a/postrender.go b/postrender.go index 8e71109..9c10063 100644 --- a/postrender.go +++ b/postrender.go @@ -23,7 +23,7 @@ import ( "unicode/utf8" "github.com/microcosm-cc/bluemonday" - stripmd "github.com/writeas/go-strip-markdown" + stripmd "github.com/writeas/go-strip-markdown/v2" "github.com/writeas/impart" blackfriday "github.com/writeas/saturday" "github.com/writeas/web-core/log" diff --git a/posts.go b/posts.go index 4d8d019..6ab912a 100644 --- a/posts.go +++ b/posts.go @@ -26,7 +26,7 @@ import ( "github.com/guregu/null/zero" "github.com/kylemcc/twitter-text-go/extract" "github.com/microcosm-cc/bluemonday" - stripmd "github.com/writeas/go-strip-markdown" + stripmd "github.com/writeas/go-strip-markdown/v2" "github.com/writeas/impart" "github.com/writeas/monday" "github.com/writeas/slug" @@ -234,7 +234,7 @@ func (p Post) Summary() string { } p.Content = stripHTMLWithoutEscaping(p.Content) // and Markdown - p.Content = stripmd.Strip(p.Content) + p.Content = stripmd.StripOptions(p.Content, stripmd.Options{SkipImages: true}) title := p.Title.String var desc string diff --git a/read.go b/read.go index 35a671f..f2706a9 100644 --- a/read.go +++ b/read.go @@ -21,7 +21,7 @@ import ( . "github.com/gorilla/feeds" "github.com/gorilla/mux" - stripmd "github.com/writeas/go-strip-markdown" + stripmd "github.com/writeas/go-strip-markdown/v2" "github.com/writeas/impart" "github.com/writeas/web-core/log" "github.com/writeas/web-core/memo" From ed771380fbe4283fb53e2905156b5c615270a2f9 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 25 Jun 2021 12:10:19 -0400 Subject: [PATCH 06/10] Fix SQLite database lock on Reader Fetching posts for the Reader involves an additional query on each row, which previously ran into our connection limit to the database and caused it to lock up. This increases the connection limit from 1 to 2, to allow this. This is meant to be a quick, safe fix, but there could always be a better solution. Fixes #467 --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index 40eb858..d188493 100644 --- a/app.go +++ b/app.go @@ -797,7 +797,7 @@ func connectToDatabase(app *App) { os.Exit(1) } db, err = sql.Open("sqlite3_with_regex", app.cfg.Database.FileName+"?parseTime=true&cached=shared") - db.SetMaxOpenConns(1) + db.SetMaxOpenConns(2) } else { log.Error("Invalid database type '%s'. Only 'mysql' and 'sqlite3' are supported right now.", app.cfg.Database.Type) os.Exit(1) From 1bdcf7096abde9b326793089af19209a7c7578f7 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 25 Jun 2021 12:39:59 -0400 Subject: [PATCH 07/10] Fix "Collection.hostName is empty!" log on Stats page Fixes #468 --- account.go | 3 ++- database.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/account.go b/account.go index 72d12ee..6ab73bd 100644 --- a/account.go +++ b/account.go @@ -1021,9 +1021,10 @@ func viewStats(app *App, u *User, w http.ResponseWriter, r *http.Request) error if c.OwnerID != u.ID { return ErrCollectionNotFound } + c.hostName = app.cfg.App.Host } - topPosts, err := app.db.GetTopPosts(u, alias) + topPosts, err := app.db.GetTopPosts(u, alias, c.hostName) if err != nil { log.Error("Unable to get top posts: %v", err) return err diff --git a/database.go b/database.go index fefc3c1..f474ae9 100644 --- a/database.go +++ b/database.go @@ -76,7 +76,7 @@ type writestore interface { GetMeStats(u *User) userMeStats GetTotalCollections() (int64, error) GetTotalPosts() (int64, error) - GetTopPosts(u *User, alias string) (*[]PublicPost, error) + GetTopPosts(u *User, alias string, hostName string) (*[]PublicPost, error) GetAnonymousPosts(u *User, page int) (*[]PublicPost, error) GetUserPosts(u *User) (*[]PublicPost, error) @@ -1767,7 +1767,7 @@ func (db *datastore) GetTotalPosts() (postCount int64, err error) { return } -func (db *datastore) GetTopPosts(u *User, alias string) (*[]PublicPost, error) { +func (db *datastore) GetTopPosts(u *User, alias string, hostName string) (*[]PublicPost, error) { params := []interface{}{u.ID} where := "" if alias != "" { @@ -1802,6 +1802,7 @@ func (db *datastore) GetTopPosts(u *User, alias string) (*[]PublicPost, error) { c.Title = title.String c.Description = description.String c.Views = views.Int64 + c.hostName = hostName pubPost.Collection = &CollectionObj{Collection: c} } From d37ab544e814b663bede266fa9ae83adf5e67d81 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 25 Jun 2021 17:07:42 -0400 Subject: [PATCH 08/10] Prevent out of bounds error on title with only whitespace --- postrender.go | 2 +- postrender_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/postrender.go b/postrender.go index 1cb4f0e..369ce51 100644 --- a/postrender.go +++ b/postrender.go @@ -182,7 +182,7 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c } func applyBasicMarkdown(data []byte) string { - if len(data) == 0 { + if len(bytes.TrimSpace(data)) == 0 { return "" } diff --git a/postrender_test.go b/postrender_test.go index 262d550..531c0f1 100644 --- a/postrender_test.go +++ b/postrender_test.go @@ -19,6 +19,12 @@ func TestApplyBasicMarkdown(t *testing.T) { result string }{ {"empty", "", ""}, + {"empty spaces", " ", ""}, + {"empty tabs", "\t", ""}, + {"empty newline", "\n", ""}, + {"nums", "123", "123"}, + {"dot", ".", "."}, + {"dash", "-", "-"}, {"plain", "Hello, World!", "Hello, World!"}, {"multibyte", "こんにちは", `こんにちは`}, {"bold", "**안녕하세요**", `안녕하세요`}, From de5e91cb718c6120a3ed7e65ff9f1ede548f46ea Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 29 Jun 2021 13:07:10 -0400 Subject: [PATCH 09/10] Fix trailing backslash on lists in Classic editor Previously, when editing a post with an unordered list in it via the Classic editor, backslashes (\) would get added to the end of each list item. This fixes that. Closes #480 --- prose/prose.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/prose/prose.js b/prose/prose.js index 3096c9e..ae8b3ce 100644 --- a/prose/prose.js +++ b/prose/prose.js @@ -40,13 +40,7 @@ class ProseMirrorView { $title.value = title; } - const doc = writeFreelyMarkdownParser.parse( - // Replace all "solo" \n's with \\\n for correct markdown parsing - // Can't use lookahead or lookbehind because it's not supported on Safari - content.replace(/([^]{0,1})(\n)([^]{0,1})/g, (match, p1, p2, p3) => { - return p1 !== "\n" && p3 !== "\n" ? p1 + "\\\n" + p3 : match; - }) - ); + const doc = writeFreelyMarkdownParser.parse(content) this.view = new EditorView(target, { state: EditorState.create({ From 940d220bf392792f17a877c1314f5fbc03106447 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 30 Jun 2021 10:33:08 -0400 Subject: [PATCH 10/10] Bump version to 0.13.1 --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index 163774b..4c4aeb9 100644 --- a/app.go +++ b/app.go @@ -56,7 +56,7 @@ var ( debugging bool // Software version can be set from git env using -ldflags - softwareVer = "0.13.0" + softwareVer = "0.13.1" // DEPRECATED VARS isSingleUser bool