hide elements before they are displayed

user feedback logic was updated to report if zero posts were found in a
zip and form submissions disable the submit button until the form input
for files changes again, preventing possible duplicate submissions on
large zip uploads.

updated to v0.2.1 wfimport to prevent early error returns when an
invalid file is present in a zip.

account_import.go

@@ -5,6 +5,7 @@ import (
 	"html/template"
 	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -55,36 +56,60 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error
 
 func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
 	// limit 10MB per submission
+	// TODO: increase?
 	r.ParseMultipartForm(10 << 20)
 	files := r.MultipartForm.File["files"]
-	var fileErrs []error
 	filesSubmitted := len(files)
-	var filesImported int
+	var filesImported, collsImported int
+	var errs []error
+	// TODO: support multiple zip uploads at once
+	if filesSubmitted == 1 && files[0].Header.Get("Content-Type") == "application/zip" {
+		filesSubmitted, filesImported, collsImported, errs = importZipPosts(app, w, r, files[0], u)
+	} else {
+		filesImported, errs = importFilePosts(app, w, r, files, u)
+	}
+
+	if len(errs) != 0 {
+		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(errs), nil)
+	}
+	if filesImported == filesSubmitted && filesSubmitted != 0 {
+		postAdj := "posts"
+		if filesSubmitted == 1 {
+			postAdj = "post"
+		}
+		if collsImported != 0 {
+			collAdj := "collections"
+			if collsImported == 1 {
+				collAdj = "collection"
+			}
+			_ = addSessionFlash(app, w, r, fmt.Sprintf(
+				"SUCCESS: Import complete, %d %s imported across %d %s.",
+				filesImported,
+				postAdj,
+				collsImported,
+				collAdj,
+			), nil)
+		} else {
+			_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, postAdj), nil)
+		}
+	} else if filesImported == 0 && filesSubmitted == 0 {
+		_ = addSessionFlash(app, w, r, "INFO: 0 valid posts found", nil)
+	} else if filesImported > 0 {
+		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
+	}
+	return impart.HTTPError{http.StatusFound, "/me/import"}
+}
+
+func importFilePosts(app *App, w http.ResponseWriter, r *http.Request, files []*multipart.FileHeader, u *User) (int, []error) {
+	var fileErrs []error
+	var count int
 	for _, formFile := range files {
-		file, err := formFile.Open()
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename))
-			log.Error("import textfile: open from form: %v", err)
+		if filepath.Ext(formFile.Filename) == ".zip" {
+			fileErrs = append(fileErrs, fmt.Errorf("zips are supported as a single upload only: %s", formFile.Filename))
+			log.Info("zip included in bulk files, skipping")
 			continue
 		}
-		defer file.Close()
-
-		tempFile, err := ioutil.TempFile("", "post-upload-*.txt")
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename))
-			log.Error("import textfile: create temp file: %v", err)
-			continue
-		}
-		defer tempFile.Close()
-
-		_, err = io.Copy(tempFile, file)
-		if err != nil {
-			fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename))
-			log.Error("import textfile: copy to temp: %v", err)
-			continue
-		}
-
-		info, err := tempFile.Stat()
+		info, err := formFileToTemp(formFile)
 		if err != nil {
 			fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename))
 			log.Error("import textfile: stat temp file: %v", err)
@@ -142,20 +167,98 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err
 				false,
 			)
 		}
-		filesImported++
+		count++
 	}
-	if len(fileErrs) != 0 {
-		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil)
+	return count, fileErrs
 }
 
-	if filesImported == filesSubmitted {
-		verb := "posts"
-		if filesSubmitted == 1 {
-			verb = "post"
+func importZipPosts(app *App, w http.ResponseWriter, r *http.Request, file *multipart.FileHeader, u *User) (filesSubmitted, importedPosts, importedColls int, errs []error) {
+	info, err := formFileToTemp(file)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("upload temp file: %v", err))
+		return
 	}
-		_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil)
-	} else if filesImported > 0 {
-		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
+
+	postMap, err := wfimport.FromZipDirs(filepath.Join(os.TempDir(), info.Name()))
+	if err != nil {
+		errs = append(errs, fmt.Errorf("parse posts and collections from zip: %v", err))
+		return
 	}
-	return impart.HTTPError{http.StatusFound, "/me/import"}
+
+	for collKey, posts := range postMap {
+		if len(posts) == 0 {
+			continue
+		}
+		collObj := CollectionObj{}
+		if collKey != wfimport.DraftsKey {
+			coll, err := app.db.GetCollection(collKey)
+			if err == ErrCollectionNotFound {
+				coll, err = app.db.CreateCollection(app.cfg, collKey, collKey, u.ID)
+				if err != nil {
+					errs = append(errs, fmt.Errorf("create non existent collection: %v", err))
+					continue
+				}
+				coll.hostName = app.cfg.App.Host
+				collObj.Collection = *coll
+			} else if err != nil {
+				errs = append(errs, fmt.Errorf("get collection: %v", err))
+				continue
+			}
+			collObj.Collection = *coll
+			importedColls++
+		}
+
+		for _, post := range posts {
+			if post != nil {
+				filesSubmitted++
+				created := post.Created.Format("2006-01-02T15:04:05Z")
+				submittedPost := SubmittedPost{
+					Title:   &post.Title,
+					Content: &post.Content,
+					Font:    "norm",
+					Created: &created,
+				}
+				rp, err := app.db.CreatePost(u.ID, collObj.Collection.ID, &submittedPost)
+				if err != nil {
+					errs = append(errs, fmt.Errorf("create post: %v", err))
+					continue
+				}
+
+				if collObj.Collection.ID != 0 && app.cfg.App.Federation {
+					go federatePost(
+						app,
+						&PublicPost{
+							Post:       rp,
+							Collection: &collObj,
+						},
+						collObj.Collection.ID,
+						false,
+					)
+				}
+				importedPosts++
+			}
+		}
+	}
+	return
+}
+
+func formFileToTemp(formFile *multipart.FileHeader) (os.FileInfo, error) {
+	file, err := formFile.Open()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open form file: %s", formFile.Filename)
+	}
+	defer file.Close()
+
+	tempFile, err := ioutil.TempFile("", fmt.Sprintf("upload-*%s", filepath.Ext(formFile.Filename)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)
+	}
+	defer tempFile.Close()
+
+	_, err = io.Copy(tempFile, file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)
+	}
+
+	return tempFile.Stat()
 }

go.mod

@@ -57,7 +57,7 @@ require (
 	github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2
 	github.com/writeas/httpsig v1.0.0
 	github.com/writeas/impart v1.1.0
-	github.com/writeas/import v0.2.0
+	github.com/writeas/import v0.2.1
 	github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219
 	github.com/writeas/nerds v1.0.0
 	github.com/writeas/openssl-go v1.0.0 // indirect

go.sum

@@ -61,6 +61,7 @@ github.com/gogs/minwinsvc v0.0.0-20170301035411-95be6356811a/go.mod h1:TUIZ+29jo
 github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 h1:6DVPu65tee05kY0/rciBQ47ue+AnuY8KTayV6VHikIo=
 github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg=
 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
@@ -169,10 +170,14 @@ github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvV
 github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
 github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs=
 github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
+github.com/writeas/import v0.1.0 h1:ZbAOb6QL24tgZOEAmEJ/hk59fF4UGOX8sQLaYE+yNiA=
+github.com/writeas/import v0.1.0/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
 github.com/writeas/import v0.1.1 h1:SbYltT+nxrJBUe0xQWJqeKMHaupbxV0a6K3RtwcE4yY=
 github.com/writeas/import v0.1.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM=
 github.com/writeas/import v0.2.0 h1:Ov23JW9Rnjxk06rki1Spar45bNX647HhwhAZj3flJiY=
 github.com/writeas/import v0.2.0/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM=
+github.com/writeas/import v0.2.1 h1:3k+bDNCyqaWdZinyUZtEO4je3mR6fr/nE4ozTh9/9Wg=
+github.com/writeas/import v0.2.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM=
 github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 h1:baEp0631C8sT2r/hqwypIw2snCFZa6h7U6TojoLHu/c=
 github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219/go.mod h1:NyM35ayknT7lzO6O/1JpfgGyv+0W9Z9q7aE0J8bXxfQ=
 github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo=

less/app.less

@@ -6,5 +6,6 @@
 @import "effects";
 @import "admin";
 @import "pages/error";
+@import "pages/import";
 @import "lib/elements";
 @import "lib/material";

less/pages/import.less

@@ -0,0 +1,18 @@
+form.import-form {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  &span.row {
+    justify-content: space-around;
+  }
+
+  input[type=file] {
+    width: 100%;
+    align-self: center;
+  }
+
+  input[type=submit] {
+    width: 100%;
+    margin-top: .5rem;
+  }
+}

static/js/import.js

@@ -0,0 +1,55 @@
+const fileForm = document.querySelector('form.import-form');
+const selectElem = document.querySelector('select[name="collection"]');
+const submitElem = document.querySelector('input[type="submit"]');
+const zipInfo = document.querySelector('span.zip > ul.info');
+const zipWarning = document.querySelector('span.zip > p.error');
+const fileInput = document.querySelector('input[type="file"]')
+
+document.onreadystatechange = () => {
+  if ( document.readyState === "interactive") {
+    selectElem.disabled = true;
+    submitElem.disabled = true;
+    zipInfo.hidden = true;
+    zipWarning.hidden = true;
+  }
+}
+
+fileInput.onchange = function() {
+  if ( this.files.length === 1 ) {
+    if ( this.files[0].type === 'application/zip' ) {
+      selectElem.disabled = true;
+      submitElem.disabled = false;
+      zipInfo.hidden = false;
+      zipWarning.hidden = true;
+    } else {
+      selectElem.disabled = false;
+      submitElem.disabled = false;
+      zipInfo.hidden = true;
+      zipWarning.hidden = true;
+    }
+  }
+
+  if ( this.files.length > 1 ) {
+    selectElem.disabled = false;
+    submitElem.disabled = false;
+    var zips = 0;
+    Array.from(this.files).forEach(file => {
+        if ( file.name.endsWith(".zip") ) {
+          zips++;
+        }
+    })
+    if ( zips > 0 ) {
+      zipInfo.hidden = true;
+      zipWarning.hidden = false;
+    } else {
+      zipInfo.hidden = true;
+      zipWarning.hidden = true;
+    }
+  }
+}
+
+submitElem.addEventListener("click", (e) => {
+  e.preventDefault();
+  submitElem.disabled = true;
+  fileForm.submit();
+});

templates/user/import.tmpl

@@ -8,12 +8,13 @@
 	</div>
 	{{end}}
 	<h2 id="posts-header">Import</h2>
-	<p>Upload text or markdown files to import as posts.</p>
-	<div class="formContainer">
-		<form id="importPosts" class="import" enctype="multipart/form-data" action="/api/me/import" method="POST">
+	<p>This form allows you to import posts from files on your computer.</p>
+	<p>Any number text or markdown files are supported, as well as zip archives of posts.</p>
+	<div>
+		<form class="import-form" enctype="multipart/form-data" action="/api/me/import" method="POST">
 			<label for="file" hidden>Browse files to upload</label>
-			<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain"/>
-			<br />
+			<input name="files" type="file" multiple accept="text/*, application/zip"/>
+			<span class="row">
 				<label for="collection">Select a blog to import the posts under.</label>
 				<select name="collection">
 					{{range $i, $el := .Collections}}
@@ -23,7 +24,17 @@
 					{{end}}
 					<option value="" selected>drafts</option>
 				</select>
-			<br />
+			</span>
+			<span class="row zip">
+				<p class="error">
+					WARNING: zip files must be uploaded separately, selected zips will be <strong>skipped</strong>.
+				</p>
+				<ul class="info">
+					<li>Root level zip files are imported as drafts</li>
+					<li>ZIP sub-directories are imported as blog collections.<br/>
+					If no blog exists matching the sub-directory name, one will be created if possible.</li>
+				</ul>
+			</span>
 			<input type="submit" value="Import" />
 		</form>
 	</div>
@@ -35,4 +46,5 @@
 </div>
 
 {{template "footer" .}}
+<script src="/js/import.js"></script>
 {{end}}