From 232d6b56c9f1d0e9c9f78e2c1a6fd393429ffcc1 Mon Sep 17 00:00:00 2001 From: Rob Loranger Date: Wed, 21 Aug 2019 15:43:44 -0700 Subject: [PATCH] allow importing from zip this allows zip files in the import upload form plain files are still uploaded to the selected collection but zip contents are considered drafts with the exception of subdirectories, these are added to the corresponding collections. if no collection is found then it will be created. --- account_import.go | 164 +++++++++++++++++++++++++++++-------- go.sum | 3 + templates/user/import.tmpl | 2 +- 3 files changed, 135 insertions(+), 34 deletions(-) diff --git a/account_import.go b/account_import.go index 88a9a91..579770b 100644 --- a/account_import.go +++ b/account_import.go @@ -5,6 +5,7 @@ import ( "html/template" "io" "io/ioutil" + "mime/multipart" "net/http" "os" "path/filepath" @@ -55,36 +56,58 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error { // limit 10MB per submission + // TODO: increase? r.ParseMultipartForm(10 << 20) files := r.MultipartForm.File["files"] - var fileErrs []error filesSubmitted := len(files) - var filesImported int - for _, formFile := range files { - file, err := formFile.Open() - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename)) - log.Error("import textfile: open from form: %v", err) - continue - } - defer file.Close() + var filesImported, collsImported int + var errs []error + // TODO: support multiple zip uploads at once + if filesSubmitted == 1 && files[0].Header.Get("Content-Type") == "application/zip" { + filesSubmitted, filesImported, collsImported, errs = importZipPosts(app, w, r, files[0], u) + } else { + filesImported, errs = importFilePosts(app, w, r, files, u) + } - tempFile, err := ioutil.TempFile("", "post-upload-*.txt") - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)) - log.Error("import textfile: create temp file: %v", err) - continue + if len(errs) != 0 { + _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(errs), nil) + } + if filesImported == filesSubmitted { + postAdj := "posts" + if filesSubmitted == 1 { + postAdj = "post" } - defer tempFile.Close() + if collsImported != 0 { + collAdj := "collections" + if collsImported == 1 { + collAdj = "collection" + } + _ = addSessionFlash(app, w, r, fmt.Sprintf( + "SUCCESS: Import complete, %d %s imported across %d %s.", + filesImported, + postAdj, + collsImported, + collAdj, + ), nil) + } else { + _ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, postAdj), nil) + } + } else if filesImported > 0 { + _ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil) + } + return impart.HTTPError{http.StatusFound, "/me/import"} +} - _, err = io.Copy(tempFile, file) - if err != nil { - fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)) - log.Error("import textfile: copy to temp: %v", err) +func importFilePosts(app *App, w http.ResponseWriter, r *http.Request, files []*multipart.FileHeader, u *User) (int, []error) { + var fileErrs []error + var count int + for _, formFile := range files { + if filepath.Ext(formFile.Filename) == ".zip" { + fileErrs = append(fileErrs, fmt.Errorf("zips are supported as a single upload only: %s", formFile.Filename)) + log.Info("zip included in bulk files, skipping") continue } - - info, err := tempFile.Stat() + info, err := formFileToTemp(formFile) if err != nil { fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename)) log.Error("import textfile: stat temp file: %v", err) @@ -142,20 +165,95 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err false, ) } - filesImported++ + count++ + } + return count, fileErrs +} + +func importZipPosts(app *App, w http.ResponseWriter, r *http.Request, file *multipart.FileHeader, u *User) (filesSubmitted, importedPosts, importedColls int, errs []error) { + info, err := formFileToTemp(file) + if err != nil { + errs = append(errs, fmt.Errorf("upload temp file: %v", err)) + return } - if len(fileErrs) != 0 { - _ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil) + + postMap, err := wfimport.FromZipDirs(filepath.Join(os.TempDir(), info.Name())) + if err != nil { + errs = append(errs, fmt.Errorf("parse posts and collections from zip: %v", err)) + return } - if filesImported == filesSubmitted { - verb := "posts" - if filesSubmitted == 1 { - verb = "post" + for collKey, posts := range postMap { + // TODO: will posts ever be 0? should skip if so + collObj := CollectionObj{} + importedColls++ + if collKey != wfimport.DraftsKey { + coll, err := app.db.GetCollection(collKey) + if err == ErrCollectionNotFound { + coll, err = app.db.CreateCollection(app.cfg, collKey, collKey, u.ID) + if err != nil { + errs = append(errs, fmt.Errorf("create non existent collection: %v", err)) + continue + } + coll.hostName = app.cfg.App.Host + collObj.Collection = *coll + } else if err != nil { + errs = append(errs, fmt.Errorf("get collection: %v", err)) + continue + } + collObj.Collection = *coll + } + + for _, post := range posts { + if post != nil { + filesSubmitted++ + created := post.Created.Format("2006-01-02T15:04:05Z") + submittedPost := SubmittedPost{ + Title: &post.Title, + Content: &post.Content, + Font: "norm", + Created: &created, + } + rp, err := app.db.CreatePost(u.ID, collObj.Collection.ID, &submittedPost) + if err != nil { + errs = append(errs, fmt.Errorf("create post: %v", err)) + } + + if collObj.Collection.ID != 0 && app.cfg.App.Federation { + go federatePost( + app, + &PublicPost{ + Post: rp, + Collection: &collObj, + }, + collObj.Collection.ID, + false, + ) + } + importedPosts++ + } } - _ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil) - } else if filesImported > 0 { - _ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil) } - return impart.HTTPError{http.StatusFound, "/me/import"} + return +} + +func formFileToTemp(formFile *multipart.FileHeader) (os.FileInfo, error) { + file, err := formFile.Open() + if err != nil { + return nil, fmt.Errorf("failed to open form file: %s", formFile.Filename) + } + defer file.Close() + + tempFile, err := ioutil.TempFile("", fmt.Sprintf("upload-*%s", filepath.Ext(formFile.Filename))) + if err != nil { + return nil, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename) + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, file) + if err != nil { + return nil, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename) + } + + return tempFile.Stat() } diff --git a/go.sum b/go.sum index de39ce4..64400b9 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,7 @@ github.com/gogs/minwinsvc v0.0.0-20170301035411-95be6356811a/go.mod h1:TUIZ+29jo github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 h1:6DVPu65tee05kY0/rciBQ47ue+AnuY8KTayV6VHikIo= github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= @@ -169,6 +170,8 @@ github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvV github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs= github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= +github.com/writeas/import v0.1.0 h1:ZbAOb6QL24tgZOEAmEJ/hk59fF4UGOX8sQLaYE+yNiA= +github.com/writeas/import v0.1.0/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE= github.com/writeas/import v0.1.1 h1:SbYltT+nxrJBUe0xQWJqeKMHaupbxV0a6K3RtwcE4yY= github.com/writeas/import v0.1.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM= github.com/writeas/import v0.2.0 h1:Ov23JW9Rnjxk06rki1Spar45bNX647HhwhAZj3flJiY= diff --git a/templates/user/import.tmpl b/templates/user/import.tmpl index 0fd8a93..0c04f06 100644 --- a/templates/user/import.tmpl +++ b/templates/user/import.tmpl @@ -12,7 +12,7 @@
- +