From ee4fe2f4adf635284080f91118d87675ad843b42 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Fri, 16 Aug 2019 14:27:24 -0700 Subject: [PATCH 001/166] add basic text file imports this adds basic support for importing files as blog posts. .txt and .md are supported at this time and the collection is selectable, defaulting to draft. if a collection is specified the post is federated. --- account.go | 13 +-- account_import.go | 134 +++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 12 +++ routes.go | 9 +- templates/user/import.tmpl | 33 +++++++ templates/user/include/header.tmpl | 2 + 7 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 account_import.go create mode 100644 templates/user/import.tmpl diff --git a/account.go b/account.go index 1cf259b..9db9b1f 100644 --- a/account.go +++ b/account.go @@ -13,6 +13,13 @@ package writefreely import ( "encoding/json" "fmt" + "html/template" + "net/http" + "regexp" + "strings" + "sync" + "time" + "github.com/gorilla/mux" "github.com/gorilla/sessions" "github.com/guregu/null/zero" @@ -22,12 +29,6 @@ import ( "github.com/writeas/web-core/log" "github.com/writeas/writefreely/author" "github.com/writeas/writefreely/page" - "html/template" - "net/http" - "regexp" - "strings" - "sync" - "time" ) type ( diff --git a/account_import.go b/account_import.go new file mode 100644 index 0000000..c8fe8a2 --- /dev/null +++ b/account_import.go @@ -0,0 +1,134 @@ +package writefreely + +import ( + "fmt" + "html/template" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/hashicorp/go-multierror" + "github.com/writeas/impart" + wfimport "github.com/writeas/import" + "github.com/writeas/web-core/log" +) + +func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { + // Fetch extra user data + p := NewUserPage(app, r, u, "Import", nil) + + c, err := app.db.GetCollections(u) + if err != nil { + return impart.HTTPError{http.StatusInternalServerError, fmt.Sprintf("unable to fetch collections: %v", err)} + } + + d := struct { + *UserPage + Collections *[]Collection + Flashes []template.HTML + }{ + UserPage: p, + Collections: c, + Flashes: []template.HTML{}, + } + + flashes, _ := getSessionFlashes(app, w, r, nil) + for _, flash := range flashes { + d.Flashes = append(d.Flashes, template.HTML(flash)) + } + + showUserPage(w, "import", d) + return nil +} + +func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { + // limit 10MB per submission + r.ParseMultipartForm(10 << 20) + files := r.MultipartForm.File["files"] + var fileErrs []error + for _, formFile := range files { + // TODO: count uploaded files that succeed and report back with message + file, err := formFile.Open() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) + log.Error("import textfile: open from form: %v", err) + continue + } + defer file.Close() + + tempFile, err := ioutil.TempFile("", "post-upload-*.txt") + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) + log.Error("import textfile: create temp file: %v", err) + continue + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, file) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) + log.Error("import textfile: copy to temp: %v", err) + continue + } + + info, err := tempFile.Stat() + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) + log.Error("import textfile: stat temp file: %v", err) + continue + } + post, err := wfimport.FromFile(filepath.Join(os.TempDir(), info.Name())) + if err == wfimport.ErrEmptyFile { + // not a real error so don't log + _ = addSessionFlash(app, w, r, fmt.Sprintf("%s was empty, import skipped", formFile.Filename), nil) + continue + } else if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to read copy of %s", formFile.Filename)) + log.Error("import textfile: file to post: %v", err) + continue + } + + post.Collection = r.PostFormValue("collection") + coll, _ := app.db.GetCollection(post.Collection) + if coll == nil { + coll = &Collection{ + ID: 0, + } + } + + submittedPost := SubmittedPost{ + Title: &post.Title, + Content: &post.Content, + Font: "norm", + } + if coll.ID != 0 && app.cfg.App.Federation { + token, err := app.db.GetAccessToken(u.ID) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to authenticate uploading: %s", formFile.Filename)) + log.Error("import textfile: get accesstoken: %+v", err) + continue + } + ownedPost, err := app.db.CreateOwnedPost(&submittedPost, token, coll.Alias, app.cfg.App.Host) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create owned post for %s", formFile.Filename)) + log.Error("import textfile: create owned post: %v", err) + continue + } + go federatePost(app, ownedPost, coll.ID, false) + } else { + _, err = app.db.CreatePost(u.ID, coll.ID, &submittedPost) + if err != nil { + fileErrs = append(fileErrs, fmt.Errorf("failed to create post from %s", formFile.Filename)) + log.Error("import textfile: create db post: %v", err) + continue + } + } + } + if len(fileErrs) != 0 { + _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) + } + + return impart.HTTPError{http.StatusFound, "/me/import"} +} diff --git a/go.mod b/go.mod index cc5fc57..8c62e9e 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/gorilla/schema v1.0.2 github.com/gorilla/sessions v1.1.3 github.com/guregu/null v3.4.0+incompatible + github.com/hashicorp/go-multierror v1.0.0 github.com/ikeikeikeike/go-sitemap-generator v1.0.1 github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 github.com/imdario/mergo v0.3.7 // indirect @@ -56,6 +57,7 @@ require ( github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 github.com/writeas/httpsig v1.0.0 github.com/writeas/impart v1.1.0 + github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 github.com/writeas/nerds v1.0.0 github.com/writeas/openssl-go v1.0.0 // indirect @@ -63,7 +65,7 @@ require ( github.com/writeas/slug v1.2.0 github.com/writeas/web-core v1.0.0 github.com/writefreely/go-nodeinfo v1.2.0 - golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f // indirect + golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 // indirect golang.org/x/sys v0.0.0-20190209173611-3b5209105503 // indirect diff --git a/go.sum b/go.sum index 8898bec..30e4a9d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +code.as/core/socks v1.0.0 h1:SPQXNp4SbEwjOAP9VzUahLHak8SDqy5n+9cm9tpjZOs= +code.as/core/socks v1.0.0/go.mod h1:BAXBy5O9s2gmw6UxLqNJcVbWY7C/UPs+801CcSsfWOY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Unknwon/com v0.0.0-20181010210213-41959bdd855f h1:m1tYqjD/N0vF/S8s/ZKz/eccUr8RAAcrOK2MhXeTegA= @@ -80,6 +82,10 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM= github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/ikeikeikeike/go-sitemap-generator v1.0.1 h1:49Fn8gro/B12vCY8pf5/+/Jpr3kwB9TvP0MSymo69SY= github.com/ikeikeikeike/go-sitemap-generator v1.0.1/go.mod h1:QI+zWsz6yQyxkG9LWNcnu0f7aiAE5tPdsZOsICgmd1c= github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM= @@ -151,10 +157,16 @@ github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6Fk github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2/go.mod h1:w2VxyRO/J5vfNjJHYVubsjUGHd3RLDoVciz0DE3ApOc= +github.com/writeas/go-writeas v1.1.0 h1:WHGm6wriBkxYAOGbvriXH8DlMUGOi6jhSZLUZKQ+4mQ= +github.com/writeas/go-writeas v1.1.0/go.mod h1:oh9U1rWaiE0p3kzdKwwvOpNXgp0P0IELI7OLOwV4fkA= github.com/writeas/httpsig v1.0.0 h1:peIAoIA3DmlP8IG8tMNZqI4YD1uEnWBmkcC9OFPjt3A= github.com/writeas/httpsig v1.0.0/go.mod h1:7ClMGSrSVXJbmiLa17bZ1LrG1oibGZmUMlh3402flPY= github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= +github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvVuhO9UiQ9f+U1aR/x5B4MP7YQHaU= +github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= +github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs= +github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 h1:baEp0631C8sT2r/hqwypIw2snCFZa6h7U6TojoLHu/c= github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219/go.mod h1:NyM35ayknT7lzO6O/1JpfgGyv+0W9Z9q7aE0J8bXxfQ= github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo= diff --git a/routes.go b/routes.go index 724c532..022a7ea 100644 --- a/routes.go +++ b/routes.go @@ -11,13 +11,14 @@ package writefreely import ( + "net/http" + "path/filepath" + "strings" + "github.com/gorilla/mux" "github.com/writeas/go-webfinger" "github.com/writeas/web-core/log" "github.com/writefreely/go-nodeinfo" - "net/http" - "path/filepath" - "strings" ) // InitStaticRoutes adds routes for serving static files. @@ -93,6 +94,8 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { me.HandleFunc("/posts/export.json", handler.Download(viewExportPosts, UserLevelUser)).Methods("GET") me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET") me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET") + me.HandleFunc("/import", handler.User(viewImport)).Methods("GET") + me.HandleFunc("/import", handler.User(handleImport)).Methods("POST") me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET") me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET") me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET") diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl new file mode 100644 index 0000000..c258783 --- /dev/null +++ b/templates/user/import.tmpl @@ -0,0 +1,33 @@ +{{define "import"}} +{{template "header" .}} + +
+

Import

+

Upload text or markdown files to import as posts.

+
+
+ + +
+ + +
+ +
+
+ {{if .Flashes}} + + {{end}} +
+ +{{template "footer" .}} +{{end}} \ No newline at end of file diff --git a/templates/user/include/header.tmpl b/templates/user/include/header.tmpl index 312d0b8..e8fd908 100644 --- a/templates/user/include/header.tmpl +++ b/templates/user/include/header.tmpl @@ -27,6 +27,7 @@ {{if .IsAdmin}}
  • Admin
    • {{end}}
  • Settings
  • Export
    • +
  • Import
  • Log out
    • @@ -45,6 +46,7 @@ {{if .IsAdmin}}
  • Admin dashboard
    • {{end}}
  • Account settings
  • Export
    • +
  • Import
    • {{if .CanInvite}}
  • Invite people
    • {{end}}
  • Log out
    • From 0ca198c715a81161d79ba65c23046424a5fbe601 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Sat, 17 Aug 2019 16:18:40 -0700 Subject: [PATCH 002/166] include nice alert message on success different template action for partial or complete import success --- account_import.go | 25 +++++++++++++++++++++++-- templates/user/import.tmpl | 5 +++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/account_import.go b/account_import.go index c8fe8a2..3fd434e 100644 --- a/account_import.go +++ b/account_import.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "github.com/hashicorp/go-multierror" "github.com/writeas/impart" @@ -28,6 +29,8 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error *UserPage Collections *[]Collection Flashes []template.HTML + Message string + InfoMsg bool }{ UserPage: p, Collections: c, @@ -36,7 +39,14 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error flashes, _ := getSessionFlashes(app, w, r, nil) for _, flash := range flashes { - d.Flashes = append(d.Flashes, template.HTML(flash)) + if strings.HasPrefix(flash, "SUCCESS: ") { + d.Message = strings.TrimPrefix(flash, "SUCCESS: ") + } else if strings.HasPrefix(flash, "INFO: ") { + d.Message = strings.TrimPrefix(flash, "INFO: ") + d.InfoMsg = true + } else { + d.Flashes = append(d.Flashes, template.HTML(flash)) + } } showUserPage(w, "import", d) @@ -48,8 +58,9 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err r.ParseMultipartForm(10 << 20) files := r.MultipartForm.File["files"] var fileErrs []error + filesSubmitted := len(files) + var filesImported int for _, formFile := range files { - // TODO: count uploaded files that succeed and report back with message file, err := formFile.Open() if err != nil { fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) @@ -125,10 +136,20 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err continue } } + filesImported++ } if len(fileErrs) != 0 { _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) } + if filesImported == filesSubmitted { + verb := "posts" + if filesSubmitted == 1 { + verb = "post" + } + _ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil) + } else if filesImported > 0 { + _ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil) + } return impart.HTTPError{http.StatusFound, "/me/import"} } diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index c258783..3833095 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -2,6 +2,11 @@ {{template "header" .}}
    + {{if .Message}} +
    +

    {{.Message}}

    +
    + {{end}}

    Import

    Upload text or markdown files to import as posts.

    From 6c5d89ac86d3c51746a0501a2135e92e4ba16631 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Mon, 19 Aug 2019 09:05:52 -0700 Subject: [PATCH 003/166] move import post handler under /api handler for post request to import is now under /api/me/import form target updated also allow all plaintext files in form --- routes.go | 2 +- templates/user/import.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/routes.go b/routes.go index 022a7ea..7dcdc65 100644 --- a/routes.go +++ b/routes.go @@ -95,7 +95,6 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { me.HandleFunc("/export", handler.User(viewExportOptions)).Methods("GET") me.HandleFunc("/export.json", handler.Download(viewExportFull, UserLevelUser)).Methods("GET") me.HandleFunc("/import", handler.User(viewImport)).Methods("GET") - me.HandleFunc("/import", handler.User(handleImport)).Methods("POST") me.HandleFunc("/settings", handler.User(viewSettings)).Methods("GET") me.HandleFunc("/invites", handler.User(handleViewUserInvites)).Methods("GET") me.HandleFunc("/logout", handler.Web(viewLogout, UserLevelNone)).Methods("GET") @@ -108,6 +107,7 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { apiMe.HandleFunc("/password", handler.All(updatePassphrase)).Methods("POST") apiMe.HandleFunc("/self", handler.All(updateSettings)).Methods("POST") apiMe.HandleFunc("/invites", handler.User(handleCreateUserInvite)).Methods("POST") + apiMe.HandleFunc("/import", handler.User(handleImport)).Methods("POST") // Sign up validation write.HandleFunc("/api/alias", handler.All(handleUsernameCheck)).Methods("POST") diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 3833095..002cf14 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -10,9 +10,9 @@

    Import

    Upload text or markdown files to import as posts.

    -
    + - +
    +