From ee4fe2f4adf635284080f91118d87675ad843b42 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Fri, 16 Aug 2019 14:27:24 -0700 Subject: [PATCH 01/89] add basic text file imports this adds basic support for importing files as blog posts. .txt and .md are supported at this time and the collection is selectable, defaulting to draft. if a collection is specified the post is federated. --- account.go | 13 +-- account_import.go | 134 +++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 12 +++ routes.go | 9 +- templates/user/import.tmpl | 33 +++++++ templates/user/include/header.tmpl | 2 + 7 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 account_import.go create mode 100644 templates/user/import.tmpl diff --git a/account.go b/account.go index 1cf259b..9db9b1f 100644 --- a/account.go +++ b/account.go @@ -13,6 +13,13 @@ package writefreely import ( "encoding/json" "fmt" + "html/template" + "net/http" + "regexp" + "strings" + "sync" + "time" + "github.com/gorilla/mux" "github.com/gorilla/sessions" "github.com/guregu/null/zero" @@ -22,12 +29,6 @@ import ( "github.com/writeas/web-core/log" "github.com/writeas/writefreely/author" "github.com/writeas/writefreely/page" - "html/template" - "net/http" - "regexp" - "strings" - "sync" - "time" ) type ( diff --git a/account_import.go b/account_import.go new file mode 100644 index 0000000..c8fe8a2 --- /dev/null +++ b/account_import.go @@ -0,0 +1,134 @@ +package writefreely + +import ( + "fmt" + "html/template" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/hashicorp/go-multierror" + "github.com/writeas/impart" + wfimport "github.com/writeas/import" + "github.com/writeas/web-core/log" +) + +func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { + // Fetch extra user data + p := NewUserPage(app, r, u, "Import", nil) + + c, err := app.db.GetCollections(u) + if err != nil { + return impart.HTTPError{http.StatusInternalServerError, fmt.Sprintf("unable to fetch collections: %v", err)} + } + + d := struct { + *UserPage + Collections *[]Collection + Flashes []template.HTML + }{ + UserPage: p, + Collections: c, + Flashes: []template.HTML{}, + } + + flashes, _ := getSessionFlashes(app, w, r, nil) + for _, flash := range flashes { + d.Flashes = append(d.Flashes, template.HTML(flash)) + } + + showUserPage(w, "import", d) + return nil +} + +func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { + // limit 10MB per submission + r.ParseMultipartForm(10 << 20) + files := r.MultipartForm.File["files"] + var fileErrs []error + for _, formFile := range files { + // TODO: count uploaded files that succeed and report back with message + file, err := formFile.Open() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) + log.Error("import textfile: open from form: %v", err) + continue + } + defer file.Close() + + tempFile, err := ioutil.TempFile("", "post-upload-*.txt") + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) + log.Error("import textfile: create temp file: %v", err) + continue + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, file) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) + log.Error("import textfile: copy to temp: %v", err) + continue + } + + info, err := tempFile.Stat() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) + log.Error("import textfile: stat temp file: %v", err) + continue + } + post, err := wfimport.FromFile(filepath.Join(os.TempDir(), info.Name())) + if err == wfimport.ErrEmptyFile { + // not a real error so don't log + _ = addSessionFlash(app, w, r, fmt.Sprintf("%s was empty, import skipped", formFile.Filename), nil) + continue + } else if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to read copy of %s", formFile.Filename)) + log.Error("import textfile: file to post: %v", err) + continue + } + + post.Collection = r.PostFormValue("collection") + coll, _ := app.db.GetCollection(post.Collection) + if coll == nil { + coll = &Collection{ + ID: 0, + } + } + + submittedPost := SubmittedPost{ + Title: &post.Title, + Content: &post.Content, + Font: "norm", + } + if coll.ID != 0 && app.cfg.App.Federation { + token, err := app.db.GetAccessToken(u.ID) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to authenticate uploading: %s", formFile.Filename)) + log.Error("import textfile: get accesstoken: %+v", err) + continue + } + ownedPost, err := app.db.CreateOwnedPost(&submittedPost, token, coll.Alias, app.cfg.App.Host) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create owned post for %s", formFile.Filename)) + log.Error("import textfile: create owned post: %v", err) + continue + } + go federatePost(app, ownedPost, coll.ID, false) + } else { + _, err = app.db.CreatePost(u.ID, coll.ID, &submittedPost) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create post from %s", formFile.Filename)) + log.Error("import textfile: create db post: %v", err) + continue + } + } + } + if len(fileErrs) != 0 { + _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) + } + + return impart.HTTPError{http.StatusFound, "/me/import"} +} diff --git a/go.mod b/go.mod index cc5fc57..8c62e9e 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/gorilla/schema v1.0.2 github.com/gorilla/sessions v1.1.3 github.com/guregu/null v3.4.0+incompatible + github.com/hashicorp/go-multierror v1.0.0 github.com/ikeikeikeike/go-sitemap-generator v1.0.1 github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 github.com/imdario/mergo v0.3.7 // indirect @@ -56,6 +57,7 @@ require ( github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 github.com/writeas/httpsig v1.0.0 github.com/writeas/impart v1.1.0 + github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 github.com/writeas/nerds v1.0.0 github.com/writeas/openssl-go v1.0.0 // indirect @@ -63,7 +65,7 @@ require ( github.com/writeas/slug v1.2.0 github.com/writeas/web-core v1.0.0 github.com/writefreely/go-nodeinfo v1.2.0 - golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f // indirect + golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 // indirect golang.org/x/sys v0.0.0-20190209173611-3b5209105503 // indirect diff --git a/go.sum b/go.sum index 8898bec..30e4a9d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +code.as/core/socks v1.0.0 h1:SPQXNp4SbEwjOAP9VzUahLHak8SDqy5n+9cm9tpjZOs= +code.as/core/socks v1.0.0/go.mod h1:BAXBy5O9s2gmw6UxLqNJcVbWY7C/UPs+801CcSsfWOY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f h1:m1tYqjD/N0vF/S8s/ZKz/eccUr8RAAcrOK2MhXeTegA= @@ -80,6 +82,10 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM= github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/ikeikeikeike/go-sitemap-generator v1.0.1 h1:49Fn8gro/B12vCY8pf5/+/Jpr3kwB9TvP0MSymo69SY= github.com/ikeikeikeike/go-sitemap-generator v1.0.1/go.mod h1:QI+zWsz6yQyxkG9LWNcnu0f7aiAE5tPdsZOsICgmd1c= github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM= @@ -151,10 +157,16 @@ github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6Fk github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2/go.mod h1:w2VxyRO/J5vfNjJHYVubsjUGHd3RLDoVciz0DE3ApOc= +github.com/writeas/go-writeas v1.1.0 h1:WHGm6wriBkxYAOGbvriXH8DlMUGOi6jhSZLUZKQ+4mQ= +github.com/writeas/go-writeas v1.1.0/go.mod h1:oh9U1rWaiE0p3kzdKwwvOpNXgp0P0IELI7OLOwV4fkA= github.com/writeas/httpsig v1.0.0 h1:peIAoIA3DmlP8IG8tMNZqI4YD1uEnWBmkcC9OFPjt3A= github.com/writeas/httpsig v1.0.0/go.mod h1:7ClMGSrSVXJbmiLa17bZ1LrG1oibGZmUMlh3402flPY= github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= +github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvVuhO9UiQ9f+U1aR/x5B4MP7YQHaU= +github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= +github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs= +github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 h1:baEp0631C8sT2r/hqwypIw2snCFZa6h7U6TojoLHu/c= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219/go.mod h1:NyM35ayknT7lzO6O/1JpfgGyv+0W9Z9q7aE0J8bXxfQ= github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo= diff --git a/routes.go b/routes.go index 724c532..022a7ea 100644 --- a/routes.go +++ b/routes.go @@ -11,13 +11,14 @@ package writefreely import ( + "net/http" + "path/filepath" + "strings" + "github.com/gorilla/mux" "github.com/writeas/go-webfinger" "github.com/writeas/web-core/log" "github.com/writefreely/go-nodeinfo" - "net/http" - "path/filepath" - "strings" ) // InitStaticRoutes adds routes for serving static files. @@ -93,6 +94,8 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { me.HandleFunc("/posts/export.json", handler.Download(viewExportPosts, UserLevelUser)).Methods("GET") me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET") me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET") + me.HandleFunc("/import", handler.User(viewImport)).Methods("GET") + me.HandleFunc("/import", handler.User(handleImport)).Methods("POST") me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET") me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET") me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET") diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl new file mode 100644 index 0000000..c258783 --- /dev/null +++ b/templates/user/import.tmpl @@ -0,0 +1,33 @@ +{{define "import"}} +{{template "header" .}} + +
+

Import

+

Upload text or markdown files to import as posts.

+
+
+ + +
+ + +
+ +
+
+ {{if .Flashes}} + + {{end}} +
+ +{{template "footer" .}} +{{end}} \ No newline at end of file diff --git a/templates/user/include/header.tmpl b/templates/user/include/header.tmpl index 312d0b8..e8fd908 100644 --- a/templates/user/include/header.tmpl +++ b/templates/user/include/header.tmpl @@ -27,6 +27,7 @@ {{if .IsAdmin}}
  • Admin
    • {{end}}
  • Settings
  • Export
    • +
  • Import
  • Log out
    • @@ -45,6 +46,7 @@ {{if .IsAdmin}}
  • Admin dashboard
    • {{end}}
  • Account settings
  • Export
    • +
  • Import
    • {{if .CanInvite}}
  • Invite people
    • {{end}}
  • Log out
    • From 0ca198c715a81161d79ba65c23046424a5fbe601 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Sat, 17 Aug 2019 16:18:40 -0700 Subject: [PATCH 02/89] include nice alert message on success different template action for partial or complete import success --- account_import.go | 25 +++++++++++++++++++++++-- templates/user/import.tmpl | 5 +++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/account_import.go b/account_import.go index c8fe8a2..3fd434e 100644 --- a/account_import.go +++ b/account_import.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "github.com/hashicorp/go-multierror" "github.com/writeas/impart" @@ -28,6 +29,8 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error *UserPage Collections *[]Collection Flashes []template.HTML + Message string + InfoMsg bool }{ UserPage: p, Collections: c, @@ -36,7 +39,14 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error flashes, _ := getSessionFlashes(app, w, r, nil) for _, flash := range flashes { - d.Flashes = append(d.Flashes, template.HTML(flash)) + if strings.HasPrefix(flash, "SUCCESS: ") { + d.Message = strings.TrimPrefix(flash, "SUCCESS: ") + } else if strings.HasPrefix(flash, "INFO: ") { + d.Message = strings.TrimPrefix(flash, "INFO: ") + d.InfoMsg = true + } else { + d.Flashes = append(d.Flashes, template.HTML(flash)) + } } showUserPage(w, "import", d) @@ -48,8 +58,9 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err r.ParseMultipartForm(10 << 20) files := r.MultipartForm.File["files"] var fileErrs []error + filesSubmitted := len(files) + var filesImported int for _, formFile := range files { - // TODO: count uploaded files that succeed and report back with message file, err := formFile.Open() if err != nil { fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) @@ -125,10 +136,20 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err continue } } + filesImported++ } if len(fileErrs) != 0 { _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) } + if filesImported == filesSubmitted { + verb := "posts" + if filesSubmitted == 1 { + verb = "post" + } + _ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil) + } else if filesImported > 0 { + _ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil) + } return impart.HTTPError{http.StatusFound, "/me/import"} } diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index c258783..3833095 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -2,6 +2,11 @@ {{template "header" .}}
    + {{if .Message}} +
    +

    {{.Message}}

    +
    + {{end}}

    Import

    Upload text or markdown files to import as posts.

    From 6c5d89ac86d3c51746a0501a2135e92e4ba16631 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Mon, 19 Aug 2019 09:05:52 -0700 Subject: [PATCH 03/89] move import post handler under /api handler for post request to import is now under /api/me/import form target updated also allow all plaintext files in form --- routes.go | 2 +- templates/user/import.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/routes.go b/routes.go index 022a7ea..7dcdc65 100644 --- a/routes.go +++ b/routes.go @@ -95,7 +95,6 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET") me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET") me.HandleFunc("/import", handler.User(viewImport)).Methods("GET") - me.HandleFunc("/import", handler.User(handleImport)).Methods("POST") me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET") me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET") me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET") @@ -108,6 +107,7 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { apiMe.HandleFunc("/password", handler.All(updatePassphrase)).Methods("POST") apiMe.HandleFunc("/self", handler.All(updateSettings)).Methods("POST") apiMe.HandleFunc("/invites", handler.User(handleCreateUserInvite)).Methods("POST") + apiMe.HandleFunc("/import", handler.User(handleImport)).Methods("POST") // Sign up validation write.HandleFunc("/api/alias", handler.All(handleUsernameCheck)).Methods("POST") diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 3833095..002cf14 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -10,9 +10,9 @@

    Import

    Upload text or markdown files to import as posts.

    -
    + - +
    +
    - {{template "footer" .}} {{end}} From 98ca449b664061e028f7526817973db1f512e6d2 Mon Sep 17 00:00:00 2001 From: Matti R Date: Tue, 14 Jan 2020 12:02:43 -0500 Subject: [PATCH 40/89] add arm-6 --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile b/Makefile index 7fd7b49..85f02d3 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,12 @@ build-darwin: deps fi xgo --targets=darwin/amd64, -dest build/ $(LDFLAGS) -tags='sqlite' -out writefreely ./cmd/writefreely +build-arm6: deps + @hash xgo > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + $(GOGET) -u src.techknowlogick.com/xgo; \ + fi + xgo --targets=linux/arm-6, -dest build/ $(LDFLAGS) -tags='sqlite' -out writefreely ./cmd/writefreely + build-arm7: deps @hash xgo > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ $(GOGET) -u src.techknowlogick.com/xgo; \ @@ -85,6 +91,10 @@ release : clean ui assets mv build/$(BINARY_NAME)-linux-amd64 $(BUILDPATH)/$(BINARY_NAME) tar -cvzf $(BINARY_NAME)_$(GITREV)_linux_amd64.tar.gz -C build $(BINARY_NAME) rm $(BUILDPATH)/$(BINARY_NAME) + $(MAKE) build-arm6 + mv build/$(BINARY_NAME)-linux-arm-6 $(BUILDPATH)/$(BINARY_NAME) + tar -cvzf $(BINARY_NAME)_$(GITREV)_linux_arm6.tar.gz -C build $(BINARY_NAME) + rm $(BUILDPATH)/$(BINARY_NAME) $(MAKE) build-arm7 mv build/$(BINARY_NAME)-linux-arm-7 $(BUILDPATH)/$(BINARY_NAME) tar -cvzf $(BINARY_NAME)_$(GITREV)_linux_arm7.tar.gz -C build $(BINARY_NAME) From 2b066997d131e298607bc6864bc17718871994c2 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 14 Jan 2020 12:21:11 -0500 Subject: [PATCH 41/89] Fix unix timestamp in file upload File API gives timestamp in milliseconds, not seconds, so this converts it correctly. Ref T609 --- account_import.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/account_import.go b/account_import.go index b34f3a7..a7b5f25 100644 --- a/account_import.go +++ b/account_import.go @@ -146,7 +146,8 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err if collAlias != "" { post.Collection = collAlias } - dateTime := time.Unix(fileDates[formFile.Filename], 0) + ts := fileDates[formFile.Filename] / 1000 // Get timestamp in seconds, not milliseconds + dateTime := time.Unix(ts, 0) post.Created = &dateTime created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ From 65e2e5126bb11ff32ba5d4cdf21c22e229a7a7ad Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 14 Jan 2020 12:24:57 -0500 Subject: [PATCH 42/89] Revert "Fix unix timestamp in file upload" This reverts commit 2b066997d131e298607bc6864bc17718871994c2. --- account_import.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/account_import.go b/account_import.go index a7b5f25..b34f3a7 100644 --- a/account_import.go +++ b/account_import.go @@ -146,8 +146,7 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err if collAlias != "" { post.Collection = collAlias } - ts := fileDates[formFile.Filename] / 1000 // Get timestamp in seconds, not milliseconds - dateTime := time.Unix(ts, 0) + dateTime := time.Unix(fileDates[formFile.Filename], 0) post.Created = &dateTime created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ From 3e97625cca86484a2836befe418cfc7af3fc70ab Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 14 Jan 2020 12:26:02 -0500 Subject: [PATCH 43/89] Fix Unix timestamps on client during import File API gives timestamp in milliseconds, not seconds, so this converts it on the client-side and sends it the correct time to the server. Ref T609 --- templates/user/import.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 7642023..3400e2f 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -48,7 +48,7 @@ const files = e.target.files; let dateMap = {}; for (let file of files) { - dateMap[file.name] = file.lastModified; + dateMap[file.name] = file.lastModified / 1000; } fileDates.value = JSON.stringify(dateMap); }) From 0766e6cb366a464327c8e20f2da987e71d5bcebc Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Tue, 14 Jan 2020 10:44:56 -0800 Subject: [PATCH 44/89] fixes imported post times changes the client side to round the unix time to avoid floats alters the time to match the client time zone on the server side --- account_import.go | 8 ++++++++ templates/user/import.tmpl | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/account_import.go b/account_import.go index b34f3a7..1098798 100644 --- a/account_import.go +++ b/account_import.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "strconv" "strings" "time" @@ -85,6 +86,7 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err log.Error("invalid form data for file dates: %v", err) return impart.HTTPError{http.StatusBadRequest, "form data for file dates was invalid"} } + fileTZ := r.FormValue("tz") files := r.MultipartForm.File["files"] var fileErrs []error filesSubmitted := len(files) @@ -147,6 +149,12 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err post.Collection = collAlias } dateTime := time.Unix(fileDates[formFile.Filename], 0) + offset, err := strconv.Atoi(fileTZ) + if err != nil { + log.Error("form time zone offset not a valid integer: %v", err) + continue + } + dateTime = dateTime.Add(time.Minute * time.Duration(offset)) post.Created = &dateTime created := post.Created.Format("2006-01-02T15:04:05Z") submittedPost := SubmittedPost{ diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 3400e2f..06ff8b7 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -32,6 +32,7 @@ + {{end}} From 4d5c89e7efec6fad7b3fae7f36f92f0c58a9b30a Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 16 Jan 2020 13:37:44 -0500 Subject: [PATCH 49/89] Fix false login state on OAuth signup page Having a `Username` field populated in the page data tells the base template to display navigation that only a logged in user should see. So this renames the field to `LoginUsername`, similar to our login.tmpl page. Ref T712 --- oauth_signup.go | 12 ++++++------ pages/signup-oauth.tmpl | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/oauth_signup.go b/oauth_signup.go index cf90af6..96b9936 100644 --- a/oauth_signup.go +++ b/oauth_signup.go @@ -29,9 +29,9 @@ type viewOauthSignupVars struct { ClientID string TokenHash string - Username string - Alias string - Email string + LoginUsername string + Alias string + Email string } const ( @@ -184,9 +184,9 @@ func (h oauthHandler) showOauthSignupPage(app *App, w http.ResponseWriter, r *ht ClientID: tp.ClientID, TokenHash: tp.TokenHash, - Username: username, - Alias: alias, - Email: email, + LoginUsername: username, + Alias: collTitle, + Email: email, } // Display any error messages diff --git a/pages/signup-oauth.tmpl b/pages/signup-oauth.tmpl index 3fd4255..40cc2e6 100644 --- a/pages/signup-oauth.tmpl +++ b/pages/signup-oauth.tmpl @@ -85,7 +85,7 @@ form dd { From 6842ab2e3be7ffae9ce46a0613714a95cc1dd135 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 16 Jan 2020 13:50:37 -0500 Subject: [PATCH 50/89] Rename collTitle from alias "alias" is the name of a different collection field, so this renames the variable internally to make things clearer. --- oauth_signup.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/oauth_signup.go b/oauth_signup.go index 96b9936..58071c6 100644 --- a/oauth_signup.go +++ b/oauth_signup.go @@ -22,7 +22,7 @@ type viewOauthSignupVars struct { AccessToken string TokenUsername string - TokenAlias string + TokenAlias string // TODO: rename this to match the data it represents: the collection title TokenEmail string TokenRemoteUser string Provider string @@ -30,7 +30,7 @@ type viewOauthSignupVars struct { TokenHash string LoginUsername string - Alias string + Alias string // TODO: rename this to match the data it represents: the collection title Email string } @@ -52,7 +52,7 @@ const ( type oauthSignupPageParams struct { AccessToken string TokenUsername string - TokenAlias string + TokenAlias string // TODO: rename this to match the data it represents: the collection title TokenEmail string TokenRemoteUser string ClientID string @@ -131,8 +131,8 @@ func (h oauthHandler) validateOauthSignup(r *http.Request) error { if len(username) > 100 { return impart.HTTPError{Status: http.StatusBadRequest, Message: "Username is too long."} } - alias := r.FormValue(oauthParamAlias) - if len(alias) == 0 { + collTitle := r.FormValue(oauthParamAlias) + if len(collTitle) == 0 { return impart.HTTPError{Status: http.StatusBadRequest, Message: "Alias is too short."} } password := r.FormValue("password") @@ -151,7 +151,7 @@ func (h oauthHandler) validateOauthSignup(r *http.Request) error { func (h oauthHandler) showOauthSignupPage(app *App, w http.ResponseWriter, r *http.Request, tp *oauthSignupPageParams, errMsg error) error { username := tp.TokenUsername - alias := tp.TokenAlias + collTitle := tp.TokenAlias email := tp.TokenEmail session, err := app.sessionStore.Get(r, cookieName) @@ -164,7 +164,7 @@ func (h oauthHandler) showOauthSignupPage(app *App, w http.ResponseWriter, r *ht username = tmpValue } if tmpValue := r.FormValue(oauthParamAlias); len(tmpValue) > 0 { - alias = tmpValue + collTitle = tmpValue } if tmpValue := r.FormValue(oauthParamEmail); len(tmpValue) > 0 { email = tmpValue From 130c9eb7475d93ab7d3a4a016c26d09ce3a8464c Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 16 Jan 2020 13:58:14 -0500 Subject: [PATCH 51/89] Change Blog Title to Display Name in OAuth signup Ref T712 --- oauth_signup.go | 2 +- pages/signup-oauth.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oauth_signup.go b/oauth_signup.go index 58071c6..5b960ca 100644 --- a/oauth_signup.go +++ b/oauth_signup.go @@ -133,7 +133,7 @@ func (h oauthHandler) validateOauthSignup(r *http.Request) error { } collTitle := r.FormValue(oauthParamAlias) if len(collTitle) == 0 { - return impart.HTTPError{Status: http.StatusBadRequest, Message: "Alias is too short."} + return impart.HTTPError{Status: http.StatusBadRequest, Message: "Display name is too short."} } password := r.FormValue("password") if len(password) == 0 { diff --git a/pages/signup-oauth.tmpl b/pages/signup-oauth.tmpl index 40cc2e6..21e8ed5 100644 --- a/pages/signup-oauth.tmpl +++ b/pages/signup-oauth.tmpl @@ -77,9 +77,9 @@ form dd {
    -
    From dcdd4dd1ef19170e22f47ebef4be30a698fcd7eb Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 16 Jan 2020 14:39:18 -0500 Subject: [PATCH 55/89] Add and update copyright notices --- author/author.go | 2 +- oauth_signup.go | 10 ++++++++++ oauth_slack.go | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/author/author.go b/author/author.go index e2e9508..0114905 100644 --- a/author/author.go +++ b/author/author.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * diff --git a/oauth_signup.go b/oauth_signup.go index 10d2306..220afbd 100644 --- a/oauth_signup.go +++ b/oauth_signup.go @@ -1,3 +1,13 @@ +/* + * Copyright © 2020 A Bunch Tell LLC. + * + * This file is part of WriteFreely. + * + * WriteFreely is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, included + * in the LICENSE file in this source code package. + */ + package writefreely import ( diff --git a/oauth_slack.go b/oauth_slack.go index f700c2c..1db3613 100644 --- a/oauth_slack.go +++ b/oauth_slack.go @@ -1,3 +1,13 @@ +/* + * Copyright © 2020 A Bunch Tell LLC. + * + * This file is part of WriteFreely. + * + * WriteFreely is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, included + * in the LICENSE file in this source code package. + */ + package writefreely import ( From c1ec6b26051ee76f63d12f9581218fe479bf8f64 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 16 Jan 2020 14:43:32 -0500 Subject: [PATCH 56/89] Fix copyright years in oauth_slack.go --- oauth_slack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oauth_slack.go b/oauth_slack.go index 1db3613..35db156 100644 --- a/oauth_slack.go +++ b/oauth_slack.go @@ -1,5 +1,5 @@ /* - * Copyright © 2020 A Bunch Tell LLC. + * Copyright © 2019-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * From 8e09e72979128d1d16d28df0df17e544037da9f0 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 16 Jan 2020 14:47:23 -0500 Subject: [PATCH 57/89] Require authenticated user for editor access Previously, anyone could access the editor even if they weren't logged in. They couldn't do much in that case (publishing would fail), but it could potentially cause some confusion. Now, users will be sent to the login page, and then redirected back to the editor once successfully logged in. --- routes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes.go b/routes.go index 7784d71..ba531fb 100644 --- a/routes.go +++ b/routes.go @@ -169,9 +169,9 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { draftEditPrefix := "" if apper.App().cfg.App.SingleUser { draftEditPrefix = "/d" - write.HandleFunc("/me/new", handler.Web(handleViewPad, UserLevelOptional)).Methods("GET") + write.HandleFunc("/me/new", handler.Web(handleViewPad, UserLevelUser)).Methods("GET") } else { - write.HandleFunc("/new", handler.Web(handleViewPad, UserLevelOptional)).Methods("GET") + write.HandleFunc("/new", handler.Web(handleViewPad, UserLevelUser)).Methods("GET") } // All the existing stuff From 2c075c0347a4d123bc260eb935e1becd67edb2bc Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Sun, 19 Jan 2020 15:57:58 -0800 Subject: [PATCH 58/89] update upgrade script for recent changes changes accounted for - the tar directory structure had changed to use a subdirectory - there are now multiple linux targets released bugs - the service must be stopped before replacing the binary - migrations were not being run during an upgrade --- scripts/upgrade-server.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/upgrade-server.sh b/scripts/upgrade-server.sh index c8e004a..ef10452 100755 --- a/scripts/upgrade-server.sh +++ b/scripts/upgrade-server.sh @@ -31,7 +31,7 @@ fi # go ahead and check for the latest release on linux echo "Checking for updates..." -url=`curl -s https://api.github.com/repos/writeas/writefreely/releases/latest | grep 'browser_' | grep linux | cut -d\" -f4` +url=`curl -s https://api.github.com/repos/writeas/writefreely/releases/latest | grep 'browser_' | grep 'linux' | grep 'amd64' | cut -d\" -f4` # check current version @@ -82,13 +82,25 @@ filename=${parts[-1]} echo "Extracting files..." tar -zxf $tempdir/$filename -C $tempdir +# stop service +echo "Stopping writefreely systemd service..." +if `systemctl start writefreely`; then + echo "Success, service stopped." +else + echo "Upgrade failed to stop the systemd service, exiting early." + exit 1 +fi + # copy files echo "Copying files..." -cp -r $tempdir/{pages,static,templates,writefreely} . +cp -r $tempdir/writefreely/{pages,static,templates,writefreely} . + +# migrate db +./writefreely -migrate # restart service echo "Restarting writefreely systemd service..." -if `systemctl restart writefreely`; then +if `systemctl start writefreely`; then echo "Success, version has been upgraded to $latest." else echo "Upgrade complete, but failed to restart service." From b336e95e1244007426a4c3fb66d7691424c9d3ed Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 20 Jan 2020 15:20:45 -0500 Subject: [PATCH 59/89] Render HTML entities in Drafts list Previously, we'd show the raw HTML entities in the summaries of Draft posts, instead of rendering them. This fixes that. --- posts.go | 4 ++++ templates/user/articles.tmpl | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/posts.go b/posts.go index d2fbcca..608f2fc 100644 --- a/posts.go +++ b/posts.go @@ -229,6 +229,10 @@ func (p Post) Summary() string { return shortPostDescription(p.Content) } +func (p Post) SummaryHTML() template.HTML { + return template.HTML(p.Summary()) +} + // Excerpt shows any text that comes before a (more) tag. // TODO: use HTMLExcerpt in templates instead of this method func (p *Post) Excerpt() template.HTML { diff --git a/templates/user/articles.tmpl b/templates/user/articles.tmpl index 3edb89c..5dbe47b 100644 --- a/templates/user/articles.tmpl +++ b/templates/user/articles.tmpl @@ -34,7 +34,7 @@ {{end}} {{ end }} - {{if .Summary}}

    {{.Summary}}

    {{end}} + {{if .Summary}}

    {{.SummaryHTML}}

    {{end}}
    {{end}} {{ else }}

    You haven't saved any drafts yet.

    They'll show up here once you do. {{if not .SingleUser}}Find your blog posts from the Blogs page.{{end}}

    From 30032e74a0199c247aea8f0efdeb0a6f28c63db3 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 20 Jan 2020 15:25:37 -0500 Subject: [PATCH 60/89] Add helpful text on Drafts page --- templates/user/articles.tmpl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/templates/user/articles.tmpl b/templates/user/articles.tmpl index 5dbe47b..16fb4e3 100644 --- a/templates/user/articles.tmpl +++ b/templates/user/articles.tmpl @@ -12,7 +12,10 @@

    drafts

    -{{ if .AnonymousPosts }}
    +{{ if .AnonymousPosts }} +

    These are your draft posts. You can share them individually (without a blog) or move them to your blog when you're ready.

    + +
    {{ range $el := .AnonymousPosts }}

    @@ -36,8 +39,9 @@

    {{if .Summary}}

    {{.SummaryHTML}}

    {{end}}
    {{end}} -
    {{ else }}

    You haven't saved any drafts yet.

    -

    They'll show up here once you do. {{if not .SingleUser}}Find your blog posts from the Blogs page.{{end}}

    +
    {{ else }}
    +

    Your anonymous and draft posts will show up here once you've published some. You'll be able to share them individually (without a blog) or move them to a blog when you're ready.

    + {{if not .SingleUser}}

    Alternatively, see your blogs and their posts on your Blogs page.

    {{end}}

    Start writing

    {{ end }}
    From bc9843dfa37dd0714c0c20c8f5f21234815535c3 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 23 Jan 2020 11:47:35 -0500 Subject: [PATCH 61/89] Add timeout on ActivityPub requests --- activitypub.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/activitypub.go b/activitypub.go index a18a636..62294cd 100644 --- a/activitypub.go +++ b/activitypub.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018-2019 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * @@ -62,6 +62,12 @@ func (ru *RemoteUser) AsPerson() *activitystreams.Person { } } +func activityPubClient() http.Client { + return http.Client{ + Timeout: 15 * time.Second, + } +} + func handleFetchCollectionActivities(app *App, w http.ResponseWriter, r *http.Request) error { w.Header().Set("Server", serverSoftware) @@ -502,7 +508,7 @@ func makeActivityPost(hostName string, p *activitystreams.Person, url string, m } } - resp, err := http.DefaultClient.Do(r) + resp, err := activityPubClient().Do(r) if err != nil { return err } @@ -538,7 +544,7 @@ func resolveIRI(hostName, url string) ([]byte, error) { } } - resp, err := http.DefaultClient.Do(r) + resp, err := activityPubClient().Do(r) if err != nil { return nil, err } From 8d3e755c8f5b0afe3b5bb2950dd3806a4af08f45 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 23 Jan 2020 12:03:23 -0500 Subject: [PATCH 62/89] Return pointer to http.Client in activityPubClient() --- activitypub.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitypub.go b/activitypub.go index 62294cd..b133f9d 100644 --- a/activitypub.go +++ b/activitypub.go @@ -62,8 +62,8 @@ func (ru *RemoteUser) AsPerson() *activitystreams.Person { } } -func activityPubClient() http.Client { - return http.Client{ +func activityPubClient() *http.Client { + return &http.Client{ Timeout: 15 * time.Second, } } From bf8dcff01eb26e0dcb9941fdf151d5c3f4af8317 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 27 Jan 2020 09:19:12 -0500 Subject: [PATCH 63/89] Quit AP goroutine early when there's no "to" Previously, we'd sleep for 2 seconds and then return for no reason. This fixes that. --- activitypub.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/activitypub.go b/activitypub.go index b133f9d..ba2bb27 100644 --- a/activitypub.go +++ b/activitypub.go @@ -388,6 +388,11 @@ func handleFetchCollectionInbox(app *App, w http.ResponseWriter, r *http.Request } go func() { + if to == nil { + log.Error("No to! %v", err) + return + } + time.Sleep(2 * time.Second) am, err := a.Serialize() if err != nil { @@ -396,10 +401,6 @@ func handleFetchCollectionInbox(app *App, w http.ResponseWriter, r *http.Request } am["@context"] = []string{activitystreams.Namespace} - if to == nil { - log.Error("No to! %v", err) - return - } err = makeActivityPost(app.cfg.App.Host, p, fullActor.Inbox, am) if err != nil { log.Error("Unable to make activity POST: %v", err) From ae1a892be00d4b38486658b658a12bd14420194d Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 04:56:23 -0500 Subject: [PATCH 64/89] Upgrade gorilla/sessions to v1.2.0 This gets rid of the gorilla/context dependency, which might have been causing a memory leak. We noticed some serious memory leakage on Write.as that seemed to point to this library. One heap snapshot: flat flat% sum% cum cum% 259.13MB 30.41% 30.41% 268.13MB 31.46% net/textproto.(*Reader).ReadMIMEHeader 105.71MB 12.40% 42.81% 105.71MB 12.40% github.com/gorilla/context.Set 78.53MB 9.21% 52.03% 125.53MB 14.73% github.com/gorilla/sessions.(*Registry).Get 55.51MB 6.51% 58.54% 82.52MB 9.68% net/http.(*Request).WithContext 38.01MB 4.46% 63.00% 38.01MB 4.46% github.com/gorilla/mux.extractVars 35MB 4.11% 67.11% 53MB 6.22% context.WithCancel 34.50MB 4.05% 71.16% 34.50MB 4.05% context.WithValue 27MB 3.17% 74.32% 27MB 3.17% net/http.cloneURL 26MB 3.05% 77.38% 26MB 3.05% github.com/gorilla/sessions.NewSession 18MB 2.11% 79.49% 18MB 2.11% context.(*cancelCtx).Done 16.50MB 1.94% 81.42% 16.50MB 1.94% syscall.anyToSockaddr 14MB 1.64% 83.07% 47MB 5.52% github.com/gorilla/sessions.(*CookieStore).New 13.50MB 1.58% 84.65% 51.51MB 6.04% github.com/gorilla/mux.(*Route).Match 11.67MB 1.37% 86.02% 13.21MB 1.55% regexp.(*Regexp).replaceAll 9.72MB 1.14% 87.16% 22.94MB 2.69% regexp.(*Regexp).ReplaceAllString 9.50MB 1.11% 88.28% 115.21MB 13.52% github.com/gorilla/sessions.GetRegistry With the help of these articles, we tracked it down to this dependency, and upgraded the library, which seems to have completely fixed the issue so far: https://rover.rocks/golang-memory-leak/ https://medium.com/@walterwu_22843/golang-memory-leak-while-handling-huge-amount-of-http-request-35cc970cb75e This should fix #133 --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f6aa8b7..adfcccb 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/gorilla/feeds v1.1.0 github.com/gorilla/mux v1.7.0 github.com/gorilla/schema v1.0.2 - github.com/gorilla/sessions v1.1.3 + github.com/gorilla/sessions v1.2.0 github.com/guregu/null v3.4.0+incompatible github.com/hashicorp/go-multierror v1.0.0 github.com/ikeikeikeike/go-sitemap-generator v1.0.1 diff --git a/go.sum b/go.sum index 5b8b88a..f958de2 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc h1:cJlkeAx1QYgO5N80aF5xRGstVsRQwgLR7uA2FnP1ZjY= github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= -github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk= github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= @@ -56,8 +54,8 @@ github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU= -github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= +github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM= github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= From 8fce34b70b039b5ff9528ad8494a6f4a7921ae6c Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 05:24:22 -0500 Subject: [PATCH 65/89] Tidy up Go mod files --- go.mod | 6 ++---- go.sum | 10 ---------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index adfcccb..519a6ea 100644 --- a/go.mod +++ b/go.mod @@ -18,10 +18,8 @@ require ( github.com/gorilla/sessions v1.2.0 github.com/guregu/null v3.4.0+incompatible github.com/hashicorp/go-multierror v1.0.0 - github.com/ikeikeikeike/go-sitemap-generator v1.0.1 github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 github.com/jtolds/gls v4.2.1+incompatible // indirect - github.com/kr/pretty v0.1.0 github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec github.com/lunixbochs/vtclean v1.0.0 // indirect github.com/manifoldco/promptui v0.3.2 @@ -32,7 +30,7 @@ require ( github.com/nicksnyder/go-i18n v1.10.0 // indirect github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d github.com/pelletier/go-toml v1.2.0 // indirect - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.8.1 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect @@ -53,7 +51,7 @@ require ( golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 // indirect golang.org/x/sys v0.0.0-20190209173611-3b5209105503 // indirect - golang.org/x/tools v0.0.0-20190208222737-3744606dbb67 + golang.org/x/tools v0.0.0-20190208222737-3744606dbb67 // indirect google.golang.org/appengine v1.4.0 // indirect gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c // indirect gopkg.in/ini.v1 v1.41.0 diff --git a/go.sum b/go.sum index f958de2..4cb46da 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,6 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/ikeikeikeike/go-sitemap-generator v1.0.1 h1:49Fn8gro/B12vCY8pf5/+/Jpr3kwB9TvP0MSymo69SY= -github.com/ikeikeikeike/go-sitemap-generator v1.0.1/go.mod h1:QI+zWsz6yQyxkG9LWNcnu0f7aiAE5tPdsZOsICgmd1c= github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM= github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2/go.mod h1:WtaVKD9TeruTED9ydiaOJU08qGoEPP/LyzTKiD3jEsw= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= @@ -135,12 +133,6 @@ github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= github.com/writeas/impart v1.1.1-0.20191230230525-d3c45ced010d h1:PK7DOj3JE6MGf647esPrKzXEHFjGWX2hl22uX79ixaE= github.com/writeas/impart v1.1.1-0.20191230230525-d3c45ced010d/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= -github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvVuhO9UiQ9f+U1aR/x5B4MP7YQHaU= -github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= -github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs= -github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= -github.com/writeas/import v0.1.1 h1:SbYltT+nxrJBUe0xQWJqeKMHaupbxV0a6K3RtwcE4yY= -github.com/writeas/import v0.1.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM= github.com/writeas/import v0.2.0 h1:Ov23JW9Rnjxk06rki1Spar45bNX647HhwhAZj3flJiY= github.com/writeas/import v0.2.0/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 h1:baEp0631C8sT2r/hqwypIw2snCFZa6h7U6TojoLHu/c= @@ -154,8 +146,6 @@ github.com/writeas/saturday v1.7.1 h1:lYo1EH6CYyrFObQoA9RNWHVlpZA5iYL5Opxo7PYAnZ github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= github.com/writeas/slug v1.2.0 h1:EMQ+cwLiOcA6EtFwUgyw3Ge18x9uflUnOnR6bp/J+/g= github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQFubfQ= -github.com/writeas/web-core v1.0.0 h1:5VKkCakQgdKZcbfVKJXtRpc5VHrkflusCl/KRCPzpQ0= -github.com/writeas/web-core v1.0.0/go.mod h1:Si3chV7VWgY8CsV+3gRolMXSO2Vx1ZFAQ/mkrpvmyEE= github.com/writeas/web-core v1.2.0 h1:CYqvBd+byi1cK4mCr1NZ6CjILuMOFmiFecv+OACcmG0= github.com/writeas/web-core v1.2.0/go.mod h1:vTYajviuNBAxjctPp2NUYdgjofywVkxUGpeaERF3SfI= github.com/writefreely/go-nodeinfo v1.2.0 h1:La+YbTCvmpTwFhBSlebWDDL81N88Qf/SCAvRLR7F8ss= From be0885698e5afdaf689d23f54e1798cf9ce56160 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 05:46:59 -0500 Subject: [PATCH 66/89] Change "restarting" to "starting" in upgrade script --- scripts/upgrade-server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade-server.sh b/scripts/upgrade-server.sh index ef10452..36281bf 100755 --- a/scripts/upgrade-server.sh +++ b/scripts/upgrade-server.sh @@ -99,7 +99,7 @@ cp -r $tempdir/writefreely/{pages,static,templates,writefreely} . ./writefreely -migrate # restart service -echo "Restarting writefreely systemd service..." +echo "Starting writefreely systemd service..." if `systemctl start writefreely`; then echo "Success, version has been upgraded to $latest." else From b25cec8381006780f39a0433fef4ddaa03ca6545 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 05:49:12 -0500 Subject: [PATCH 67/89] Update copyright in upgrade script --- scripts/upgrade-server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade-server.sh b/scripts/upgrade-server.sh index 36281bf..581085d 100755 --- a/scripts/upgrade-server.sh +++ b/scripts/upgrade-server.sh @@ -11,7 +11,7 @@ ## have not installed the binary `writefreely` in another location. ## ############################################################################### # -# Copyright © 2019 A Bunch Tell LLC. +# Copyright © 2019-2020 A Bunch Tell LLC. # # This file is part of WriteFreely. # From 4d5f58a7e65733f1fe6e4c8819708abdfcc295cd Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 06:42:32 -0500 Subject: [PATCH 68/89] Fix date-based post header links Posts without an explicit title render the date as the post header in lists of posts (like on the blog index and tag pages). This updates localdate.js to properly adjust those dates, too. --- static/js/localdate.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/static/js/localdate.js b/static/js/localdate.js index 19fa502..879ebe4 100644 --- a/static/js/localdate.js +++ b/static/js/localdate.js @@ -1,9 +1,16 @@ -function toLocalDate(el) { - var d = new Date(el.getAttribute("datetime")); - el.textContent = d.toLocaleDateString(navigator.language || "en-US", { year: 'numeric', month: 'long', day: 'numeric' }); +function toLocalDate(dateEl, displayEl) { + var d = new Date(dateEl.getAttribute("datetime")); + displayEl.textContent = d.toLocaleDateString(navigator.language || "en-US", { year: 'numeric', month: 'long', day: 'numeric' }); } -var $dates = document.querySelectorAll("time"); +// Adjust dates on individual post pages, and on posts in a list *with* an explicit title +var $dates = document.querySelectorAll("article > time"); for (var i=0; i < $dates.length; i++) { - toLocalDate($dates[i]); + toLocalDate($dates[i], $dates[i]); } + +// Adjust dates on posts in a list without an explicit title, where they act as the header +$dates = document.querySelectorAll("h2.post-title > time"); +for (i=0; i < $dates.length; i++) { + toLocalDate($dates[i], $dates[i].querySelector('a')); +} \ No newline at end of file From d6b7a5925f20ba7ca3d9f210188a3f1ceba07cf1 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 09:11:02 -0500 Subject: [PATCH 69/89] Restrict /invite/{code} route to valid chars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, loading something like /invite/fFdblk😄 would return a 500, due to a mix of collations in MySQL while SELECTing for an invite with an ID of 'fFdblk😄'. This restricts the route to [a-zA-Z0-9] chars, to prevent this. --- routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes.go b/routes.go index ba531fb..023aca2 100644 --- a/routes.go +++ b/routes.go @@ -161,7 +161,7 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { // Handle special pages first write.HandleFunc("/login", handler.Web(viewLogin, UserLevelNoneRequired)) write.HandleFunc("/signup", handler.Web(handleViewLanding, UserLevelNoneRequired)) - write.HandleFunc("/invite/{code}", handler.Web(handleViewInvite, UserLevelOptional)).Methods("GET") + write.HandleFunc("/invite/{code:[a-zA-Z0-9]+}", handler.Web(handleViewInvite, UserLevelOptional)).Methods("GET") // TODO: show a reader-specific 404 page if the function is disabled write.HandleFunc("/read", handler.Web(viewLocalTimeline, UserLevelReader)) RouteRead(handler, UserLevelReader, write.PathPrefix("/read").Subrouter()) From 50901d2446ca9dc473a3820778aaff5aae250607 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Jan 2020 13:01:21 -0500 Subject: [PATCH 70/89] Fix date format in `datetime` attribute Previously, the date format in this attribute for posts was invalid. This caused local date rendering to fail in Firefox. This fixes that. Closes #253 --- templates/chorus-collection-post.tmpl | 2 +- templates/collection-post.tmpl | 2 +- templates/include/posts.tmpl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/chorus-collection-post.tmpl b/templates/chorus-collection-post.tmpl index c231b64..f88e334 100644 --- a/templates/chorus-collection-post.tmpl +++ b/templates/chorus-collection-post.tmpl @@ -58,7 +58,7 @@ body#post header { {{if .Suspended}} {{template "user-suspended"}} {{end}} -
    {{if .IsScheduled}}

    Scheduled

    {{end}}{{if .Title.String}}

    {{.FormattedDisplayTitle}}

    {{end}}{{if $.Collection.Format.ShowDates}}{{end}}
    {{.HTMLContent}}
    +
    {{if .IsScheduled}}

    Scheduled

    {{end}}{{if .Title.String}}

    {{.FormattedDisplayTitle}}

    {{end}}{{if $.Collection.Format.ShowDates}}{{end}}
    {{.HTMLContent}}
    {{ if .Collection.ShowFooterBranding }}