allow importing from zip

this allows zip files in the import upload form

plain files are still uploaded to the selected collection but zip
contents are considered drafts with the exception of subdirectories,
these are added to the corresponding collections. if no collection is
found then it will be created.

account_import.go

@@ -5,6 +5,7 @@ import (
 	"html/template"
 	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -55,36 +56,58 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error
 
 func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
 	// limit 10MB per submission
+	// TODO: increase?
 	r.ParseMultipartForm(10 << 20)
 	files := r.MultipartForm.File["files"]
-	var fileErrs []error
 	filesSubmitted := len(files)
-	var filesImported int
+	var filesImported, collsImported int
+	var errs []error
+	// TODO: support multiple zip uploads at once
+	if filesSubmitted == 1 && files[0].Header.Get("Content-Type") == "application/zip" {
+		filesSubmitted, filesImported, collsImported, errs = importZipPosts(app, w, r, files[0], u)
+	} else {
+		filesImported, errs = importFilePosts(app, w, r, files, u)
+	}
+
+	if len(errs) != 0 {
+		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(errs), nil)
+	}
+	if filesImported == filesSubmitted {
+		postAdj := "posts"
+		if filesSubmitted == 1 {
+			postAdj = "post"
+		}
+		if collsImported != 0 {
+			collAdj := "collections"
+			if collsImported == 1 {
+				collAdj = "collection"
+			}
+			_ = addSessionFlash(app, w, r, fmt.Sprintf(
+				"SUCCESS: Import complete, %d %s imported across %d %s.",
+				filesImported,
+				postAdj,
+				collsImported,
+				collAdj,
+			), nil)
+		} else {
+			_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, postAdj), nil)
+		}
+	} else if filesImported > 0 {
+		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
+	}
+	return impart.HTTPError{http.StatusFound, "/me/import"}
+}
+
+func importFilePosts(app *App, w http.ResponseWriter, r *http.Request, files []*multipart.FileHeader, u *User) (int, []error) {
+	var fileErrs []error
+	var count int
 	for _, formFile := range files {
-		file, err := formFile.Open()
+		if filepath.Ext(formFile.Filename) == ".zip" {
-		if err != nil {
+			fileErrs = append(fileErrs, fmt.Errorf("zips are supported as a single upload only: %s", formFile.Filename))
-			fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename))
+			log.Info("zip included in bulk files, skipping")
-			log.Error("import textfile: open from form: %v", err)
 			continue
 		}
-		defer file.Close()
+		info, err := formFileToTemp(formFile)
-
-		tempFile, err := ioutil.TempFile("", "post-upload-*.txt")
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename))
-			log.Error("import textfile: create temp file: %v", err)
-			continue
-		}
-		defer tempFile.Close()
-
-		_, err = io.Copy(tempFile, file)
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename))
-			log.Error("import textfile: copy to temp: %v", err)
-			continue
-		}
-
-		info, err := tempFile.Stat()
 		if err != nil {
 			fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename))
 			log.Error("import textfile: stat temp file: %v", err)
@@ -142,20 +165,95 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err
 				false,
 			)
 		}
-		filesImported++
+		count++
 	}
-	if len(fileErrs) != 0 {
+	return count, fileErrs
-		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil)
+}
+
+func importZipPosts(app *App, w http.ResponseWriter, r *http.Request, file *multipart.FileHeader, u *User) (filesSubmitted, importedPosts, importedColls int, errs []error) {
+	info, err := formFileToTemp(file)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("upload temp file: %v", err))
+		return
 	}
 
-	if filesImported == filesSubmitted {
+	postMap, err := wfimport.FromZipDirs(filepath.Join(os.TempDir(), info.Name()))
-		verb := "posts"
+	if err != nil {
-		if filesSubmitted == 1 {
+		errs = append(errs, fmt.Errorf("parse posts and collections from zip: %v", err))
-			verb = "post"
+		return
 	}
-		_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil)
+
-	} else if filesImported > 0 {
+	for collKey, posts := range postMap {
-		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
+		// TODO: will posts ever be 0? should skip if so
+		collObj := CollectionObj{}
+		importedColls++
+		if collKey != wfimport.DraftsKey {
+			coll, err := app.db.GetCollection(collKey)
+			if err == ErrCollectionNotFound {
+				coll, err = app.db.CreateCollection(app.cfg, collKey, collKey, u.ID)
+				if err != nil {
+					errs = append(errs, fmt.Errorf("create non existent collection: %v", err))
+					continue
 				}
-	return impart.HTTPError{http.StatusFound, "/me/import"}
+				coll.hostName = app.cfg.App.Host
+				collObj.Collection = *coll
+			} else if err != nil {
+				errs = append(errs, fmt.Errorf("get collection: %v", err))
+				continue
+			}
+			collObj.Collection = *coll
+		}
+
+		for _, post := range posts {
+			if post != nil {
+				filesSubmitted++
+				created := post.Created.Format("2006-01-02T15:04:05Z")
+				submittedPost := SubmittedPost{
+					Title:   &post.Title,
+					Content: &post.Content,
+					Font:    "norm",
+					Created: &created,
+				}
+				rp, err := app.db.CreatePost(u.ID, collObj.Collection.ID, &submittedPost)
+				if err != nil {
+					errs = append(errs, fmt.Errorf("create post: %v", err))
+				}
+
+				if collObj.Collection.ID != 0 && app.cfg.App.Federation {
+					go federatePost(
+						app,
+						&PublicPost{
+							Post:       rp,
+							Collection: &collObj,
+						},
+						collObj.Collection.ID,
+						false,
+					)
+				}
+				importedPosts++
+			}
+		}
+	}
+	return
+}
+
+func formFileToTemp(formFile *multipart.FileHeader) (os.FileInfo, error) {
+	file, err := formFile.Open()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open form file: %s", formFile.Filename)
+	}
+	defer file.Close()
+
+	tempFile, err := ioutil.TempFile("", fmt.Sprintf("upload-*%s", filepath.Ext(formFile.Filename)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)
+	}
+	defer tempFile.Close()
+
+	_, err = io.Copy(tempFile, file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)
+	}
+
+	return tempFile.Stat()
 }

go.sum

@@ -61,6 +61,7 @@ github.com/gogs/minwinsvc v0.0.0-20170301035411-95be6356811a/go.mod h1:TUIZ+29jo
 github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 h1:6DVPu65tee05kY0/rciBQ47ue+AnuY8KTayV6VHikIo=
 github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg=
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
@@ -169,6 +170,8 @@ github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvV
 github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
 github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs=
 github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
+github.com/writeas/import v0.1.0 h1:ZbAOb6QL24tgZOEAmEJ/hk59fF4UGOX8sQLaYE+yNiA=
+github.com/writeas/import v0.1.0/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
 github.com/writeas/import v0.1.1 h1:SbYltT+nxrJBUe0xQWJqeKMHaupbxV0a6K3RtwcE4yY=
 github.com/writeas/import v0.1.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM=
 github.com/writeas/import v0.2.0 h1:Ov23JW9Rnjxk06rki1Spar45bNX647HhwhAZj3flJiY=

templates/user/import.tmpl

@@ -12,7 +12,7 @@
 	<div class="formContainer">
 		<form id="importPosts" class="import" enctype="multipart/form-data" action="/api/me/import" method="POST">
 			<label for="file" hidden>Browse files to upload</label>
-			<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain"/>
+			<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain, application/zip"/>
 			<br />
 			<label for="collection">Select a blog to import the posts under.</label>
 			<select name="collection">