allow importing from zip

this allows zip files in the import upload form plain files are still uploaded to the selected collection but zip contents are considered drafts with the exception of subdirectories, these are added to the corresponding collections. if no collection is found then it will be created.
6 years ago · 232d6b56c9
parent 2fa2086654
commit 232d6b56c9
3 changed files with 135 additions and 34 deletions
--- a/account_import.go
+++ b/account_import.go
@ -5,6 +5,7 @@ import (
 	"html/template"
 	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"os"
 	"path/filepath"
@ -55,36 +56,58 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error

 func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
 	// limit 10MB per submission
+	// TODO: increase?
 	r.ParseMultipartForm(10 << 20)
 	files := r.MultipartForm.File["files"]
-	var fileErrs []error
 	filesSubmitted := len(files)
-	var filesImported int
-	for _, formFile := range files {
-		file, err := formFile.Open()
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename))
-			log.Error("import textfile: open from form: %v", err)
-			continue
-		}
-		defer file.Close()
+	var filesImported, collsImported int
+	var errs []error
+	// TODO: support multiple zip uploads at once
+	if filesSubmitted == 1 && files[0].Header.Get("Content-Type") == "application/zip" {
+		filesSubmitted, filesImported, collsImported, errs = importZipPosts(app, w, r, files[0], u)
+	} else {
+		filesImported, errs = importFilePosts(app, w, r, files, u)
+	}

-		tempFile, err := ioutil.TempFile("", "post-upload-*.txt")
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename))
-			log.Error("import textfile: create temp file: %v", err)
-			continue
+	if len(errs) != 0 {
+		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(errs), nil)
+	}
+	if filesImported == filesSubmitted {
+		postAdj := "posts"
+		if filesSubmitted == 1 {
+			postAdj = "post"
 		}
-		defer tempFile.Close()
+		if collsImported != 0 {
+			collAdj := "collections"
+			if collsImported == 1 {
+				collAdj = "collection"
+			}
+			_ = addSessionFlash(app, w, r, fmt.Sprintf(
+				"SUCCESS: Import complete, %d %s imported across %d %s.",
+				filesImported,
+				postAdj,
+				collsImported,
+				collAdj,
+			), nil)
+		} else {
+			_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, postAdj), nil)
+		}
+	} else if filesImported > 0 {
+		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
+	}
+	return impart.HTTPError{http.StatusFound, "/me/import"}
+}

-		_, err = io.Copy(tempFile, file)
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename))
-			log.Error("import textfile: copy to temp: %v", err)
+func importFilePosts(app *App, w http.ResponseWriter, r *http.Request, files []*multipart.FileHeader, u *User) (int, []error) {
+	var fileErrs []error
+	var count int
+	for _, formFile := range files {
+		if filepath.Ext(formFile.Filename) == ".zip" {
+			fileErrs = append(fileErrs, fmt.Errorf("zips are supported as a single upload only: %s", formFile.Filename))
+			log.Info("zip included in bulk files, skipping")
 			continue
 		}
-
-		info, err := tempFile.Stat()
+		info, err := formFileToTemp(formFile)
 		if err != nil {
 			fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename))
 			log.Error("import textfile: stat temp file: %v", err)
@ -142,20 +165,95 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err
 				false,
 			)
 		}
-		filesImported++
+		count++
+	}
+	return count, fileErrs
+}
+
+func importZipPosts(app *App, w http.ResponseWriter, r *http.Request, file *multipart.FileHeader, u *User) (filesSubmitted, importedPosts, importedColls int, errs []error) {
+	info, err := formFileToTemp(file)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("upload temp file: %v", err))
+		return
 	}
-	if len(fileErrs) != 0 {
-		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil)
+
+	postMap, err := wfimport.FromZipDirs(filepath.Join(os.TempDir(), info.Name()))
+	if err != nil {
+		errs = append(errs, fmt.Errorf("parse posts and collections from zip: %v", err))
+		return
 	}

-	if filesImported == filesSubmitted {
-		verb := "posts"
-		if filesSubmitted == 1 {
-			verb = "post"
+	for collKey, posts := range postMap {
+		// TODO: will posts ever be 0? should skip if so
+		collObj := CollectionObj{}
+		importedColls++
+		if collKey != wfimport.DraftsKey {
+			coll, err := app.db.GetCollection(collKey)
+			if err == ErrCollectionNotFound {
+				coll, err = app.db.CreateCollection(app.cfg, collKey, collKey, u.ID)
+				if err != nil {
+					errs = append(errs, fmt.Errorf("create non existent collection: %v", err))
+					continue
+				}
+				coll.hostName = app.cfg.App.Host
+				collObj.Collection = *coll
+			} else if err != nil {
+				errs = append(errs, fmt.Errorf("get collection: %v", err))
+				continue
+			}
+			collObj.Collection = *coll
+		}
+
+		for _, post := range posts {
+			if post != nil {
+				filesSubmitted++
+				created := post.Created.Format("2006-01-02T15:04:05Z")
+				submittedPost := SubmittedPost{
+					Title:   &post.Title,
+					Content: &post.Content,
+					Font:    "norm",
+					Created: &created,
+				}
+				rp, err := app.db.CreatePost(u.ID, collObj.Collection.ID, &submittedPost)
+				if err != nil {
+					errs = append(errs, fmt.Errorf("create post: %v", err))
+				}
+
+				if collObj.Collection.ID != 0 && app.cfg.App.Federation {
+					go federatePost(
+						app,
+						&PublicPost{
+							Post:       rp,
+							Collection: &collObj,
+						},
+						collObj.Collection.ID,
+						false,
+					)
+				}
+				importedPosts++
+			}
 		}
-		_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil)
-	} else if filesImported > 0 {
-		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
 	}
-	return impart.HTTPError{http.StatusFound, "/me/import"}
+	return
+}
+
+func formFileToTemp(formFile *multipart.FileHeader) (os.FileInfo, error) {
+	file, err := formFile.Open()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open form file: %s", formFile.Filename)
+	}
+	defer file.Close()
+
+	tempFile, err := ioutil.TempFile("", fmt.Sprintf("upload-*%s", filepath.Ext(formFile.Filename)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)
+	}
+	defer tempFile.Close()
+
+	_, err = io.Copy(tempFile, file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)
+	}
+
+	return tempFile.Stat()
 }
--- a/go.sum
+++ b/go.sum
@ -61,6 +61,7 @@ github.com/gogs/minwinsvc v0.0.0-20170301035411-95be6356811a/go.mod h1:TUIZ+29jo
 github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 h1:6DVPu65tee05kY0/rciBQ47ue+AnuY8KTayV6VHikIo=
 github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg=
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
@ -169,6 +170,8 @@ github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvV
 github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
 github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs=
 github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
+github.com/writeas/import v0.1.0 h1:ZbAOb6QL24tgZOEAmEJ/hk59fF4UGOX8sQLaYE+yNiA=
+github.com/writeas/import v0.1.0/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
 github.com/writeas/import v0.1.1 h1:SbYltT+nxrJBUe0xQWJqeKMHaupbxV0a6K3RtwcE4yY=
 github.com/writeas/import v0.1.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM=
 github.com/writeas/import v0.2.0 h1:Ov23JW9Rnjxk06rki1Spar45bNX647HhwhAZj3flJiY=
--- a/templates/user/import.tmpl
+++ b/templates/user/import.tmpl
@ -12,7 +12,7 @@
 	<div class="formContainer">
 		<form id="importPosts" class="import" enctype="multipart/form-data" action="/api/me/import" method="POST">
 			<label for="file" hidden>Browse files to upload</label>
-			<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain"/>
+			<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain, application/zip"/>
 			<br />
 			<label for="collection">Select a blog to import the posts under.</label>
 			<select name="collection">