allow importing from zip
this allows zip files in the import upload form plain files are still uploaded to the selected collection but zip contents are considered drafts with the exception of subdirectories, these are added to the corresponding collections. if no collection is found then it will be created.
This commit is contained in:
Rob Loranger
parent
2fa2086654
commit
232d6b56c9
3 changed files with 138 additions and 37 deletions
|
|
@ -5,6 +5,7 @@ import (
|
|||
"html/template"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -55,36 +56,58 @@ func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error
|
|||
|
||||
func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
|
||||
// limit 10MB per submission
|
||||
// TODO: increase?
|
||||
r.ParseMultipartForm(10 << 20)
|
||||
files := r.MultipartForm.File["files"]
|
||||
var fileErrs []error
|
||||
filesSubmitted := len(files)
|
||||
var filesImported int
|
||||
var filesImported, collsImported int
|
||||
var errs []error
|
||||
// TODO: support multiple zip uploads at once
|
||||
if filesSubmitted == 1 && files[0].Header.Get("Content-Type") == "application/zip" {
|
||||
filesSubmitted, filesImported, collsImported, errs = importZipPosts(app, w, r, files[0], u)
|
||||
} else {
|
||||
filesImported, errs = importFilePosts(app, w, r, files, u)
|
||||
}
|
||||
|
||||
if len(errs) != 0 {
|
||||
_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(errs), nil)
|
||||
}
|
||||
if filesImported == filesSubmitted {
|
||||
postAdj := "posts"
|
||||
if filesSubmitted == 1 {
|
||||
postAdj = "post"
|
||||
}
|
||||
if collsImported != 0 {
|
||||
collAdj := "collections"
|
||||
if collsImported == 1 {
|
||||
collAdj = "collection"
|
||||
}
|
||||
_ = addSessionFlash(app, w, r, fmt.Sprintf(
|
||||
"SUCCESS: Import complete, %d %s imported across %d %s.",
|
||||
filesImported,
|
||||
postAdj,
|
||||
collsImported,
|
||||
collAdj,
|
||||
), nil)
|
||||
} else {
|
||||
_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, postAdj), nil)
|
||||
}
|
||||
} else if filesImported > 0 {
|
||||
_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
|
||||
}
|
||||
return impart.HTTPError{http.StatusFound, "/me/import"}
|
||||
}
|
||||
|
||||
func importFilePosts(app *App, w http.ResponseWriter, r *http.Request, files []*multipart.FileHeader, u *User) (int, []error) {
|
||||
var fileErrs []error
|
||||
var count int
|
||||
for _, formFile := range files {
|
||||
file, err := formFile.Open()
|
||||
if err != nil {
|
||||
fileErrs = append(fileErrs, fmt.Errorf("failed to open form file: %s", formFile.Filename))
|
||||
log.Error("import textfile: open from form: %v", err)
|
||||
if filepath.Ext(formFile.Filename) == ".zip" {
|
||||
fileErrs = append(fileErrs, fmt.Errorf("zips are supported as a single upload only: %s", formFile.Filename))
|
||||
log.Info("zip included in bulk files, skipping")
|
||||
continue
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
tempFile, err := ioutil.TempFile("", "post-upload-*.txt")
|
||||
if err != nil {
|
||||
fileErrs = append(fileErrs, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename))
|
||||
log.Error("import textfile: create temp file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer tempFile.Close()
|
||||
|
||||
_, err = io.Copy(tempFile, file)
|
||||
if err != nil {
|
||||
fileErrs = append(fileErrs, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename))
|
||||
log.Error("import textfile: copy to temp: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := tempFile.Stat()
|
||||
info, err := formFileToTemp(formFile)
|
||||
if err != nil {
|
||||
fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename))
|
||||
log.Error("import textfile: stat temp file: %v", err)
|
||||
|
|
@ -142,20 +165,95 @@ func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) err
|
|||
false,
|
||||
)
|
||||
}
|
||||
filesImported++
|
||||
count++
|
||||
}
|
||||
if len(fileErrs) != 0 {
|
||||
_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(fileErrs), nil)
|
||||
return count, fileErrs
|
||||
}
|
||||
|
||||
func importZipPosts(app *App, w http.ResponseWriter, r *http.Request, file *multipart.FileHeader, u *User) (filesSubmitted, importedPosts, importedColls int, errs []error) {
|
||||
info, err := formFileToTemp(file)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("upload temp file: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
if filesImported == filesSubmitted {
|
||||
verb := "posts"
|
||||
if filesSubmitted == 1 {
|
||||
verb = "post"
|
||||
postMap, err := wfimport.FromZipDirs(filepath.Join(os.TempDir(), info.Name()))
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("parse posts and collections from zip: %v", err))
|
||||
return
|
||||
}
|
||||
_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, verb), nil)
|
||||
} else if filesImported > 0 {
|
||||
_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
|
||||
|
||||
for collKey, posts := range postMap {
|
||||
// TODO: will posts ever be 0? should skip if so
|
||||
collObj := CollectionObj{}
|
||||
importedColls++
|
||||
if collKey != wfimport.DraftsKey {
|
||||
coll, err := app.db.GetCollection(collKey)
|
||||
if err == ErrCollectionNotFound {
|
||||
coll, err = app.db.CreateCollection(app.cfg, collKey, collKey, u.ID)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("create non existent collection: %v", err))
|
||||
continue
|
||||
}
|
||||
return impart.HTTPError{http.StatusFound, "/me/import"}
|
||||
coll.hostName = app.cfg.App.Host
|
||||
collObj.Collection = *coll
|
||||
} else if err != nil {
|
||||
errs = append(errs, fmt.Errorf("get collection: %v", err))
|
||||
continue
|
||||
}
|
||||
collObj.Collection = *coll
|
||||
}
|
||||
|
||||
for _, post := range posts {
|
||||
if post != nil {
|
||||
filesSubmitted++
|
||||
created := post.Created.Format("2006-01-02T15:04:05Z")
|
||||
submittedPost := SubmittedPost{
|
||||
Title: &post.Title,
|
||||
Content: &post.Content,
|
||||
Font: "norm",
|
||||
Created: &created,
|
||||
}
|
||||
rp, err := app.db.CreatePost(u.ID, collObj.Collection.ID, &submittedPost)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("create post: %v", err))
|
||||
}
|
||||
|
||||
if collObj.Collection.ID != 0 && app.cfg.App.Federation {
|
||||
go federatePost(
|
||||
app,
|
||||
&PublicPost{
|
||||
Post: rp,
|
||||
Collection: &collObj,
|
||||
},
|
||||
collObj.Collection.ID,
|
||||
false,
|
||||
)
|
||||
}
|
||||
importedPosts++
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func formFileToTemp(formFile *multipart.FileHeader) (os.FileInfo, error) {
|
||||
file, err := formFile.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open form file: %s", formFile.Filename)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
tempFile, err := ioutil.TempFile("", fmt.Sprintf("upload-*%s", filepath.Ext(formFile.Filename)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)
|
||||
}
|
||||
defer tempFile.Close()
|
||||
|
||||
_, err = io.Copy(tempFile, file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)
|
||||
}
|
||||
|
||||
return tempFile.Stat()
|
||||
}
|
||||
|
|
|
|||
3
go.sum
3
go.sum
|
|
@ -61,6 +61,7 @@ github.com/gogs/minwinsvc v0.0.0-20170301035411-95be6356811a/go.mod h1:TUIZ+29jo
|
|||
github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 h1:6DVPu65tee05kY0/rciBQ47ue+AnuY8KTayV6VHikIo=
|
||||
github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg=
|
||||
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
|
||||
|
|
@ -169,6 +170,8 @@ github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06 h1:S6oKKP8GhSoyZUvV
|
|||
github.com/writeas/import v0.0.0-20190815214647-baae8acd8d06/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
|
||||
github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e h1:31PkvDTWkjzC1nGzWw9uAE92ZfcVyFX/K9L9ejQjnEs=
|
||||
github.com/writeas/import v0.0.0-20190815235139-628d10daaa9e/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
|
||||
github.com/writeas/import v0.1.0 h1:ZbAOb6QL24tgZOEAmEJ/hk59fF4UGOX8sQLaYE+yNiA=
|
||||
github.com/writeas/import v0.1.0/go.mod h1:f3K8z7YnJwKnPIT4h7980n9C6cQb4DIB2QcxVCTB7lE=
|
||||
github.com/writeas/import v0.1.1 h1:SbYltT+nxrJBUe0xQWJqeKMHaupbxV0a6K3RtwcE4yY=
|
||||
github.com/writeas/import v0.1.1/go.mod h1:gFe0Pl7ZWYiXbI0TJxeMMyylPGZmhVvCfQxhMEc8CxM=
|
||||
github.com/writeas/import v0.2.0 h1:Ov23JW9Rnjxk06rki1Spar45bNX647HhwhAZj3flJiY=
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
<div class="formContainer">
|
||||
<form id="importPosts" class="import" enctype="multipart/form-data" action="/api/me/import" method="POST">
|
||||
<label for="file" hidden>Browse files to upload</label>
|
||||
<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain"/>
|
||||
<input class="fileInput" name="files" type="file" multiple accept="text/markdown, text/plain, application/zip"/>
|
||||
<br />
|
||||
<label for="collection">Select a blog to import the posts under.</label>
|
||||
<select name="collection">
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue