writefreely/account_import.go

package writefreely

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/go-multierror"
	"github.com/writeas/impart"
	wfimport "github.com/writeas/import"
	"github.com/writeas/web-core/log"
)

func viewImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
	// Fetch extra user data
	p := NewUserPage(app, r, u, "Import", nil)

	c, err := app.db.GetCollections(u)
	if err != nil {
		return impart.HTTPError{http.StatusInternalServerError, fmt.Sprintf("unable to fetch collections: %v", err)}
	}

	d := struct {
		*UserPage
		Collections *[]Collection
		Flashes     []template.HTML
		Message     string
		InfoMsg     bool
	}{
		UserPage:    p,
		Collections: c,
		Flashes:     []template.HTML{},
	}

	flashes, _ := getSessionFlashes(app, w, r, nil)
	for _, flash := range flashes {
		if strings.HasPrefix(flash, "SUCCESS: ") {
			d.Message = strings.TrimPrefix(flash, "SUCCESS: ")
		} else if strings.HasPrefix(flash, "INFO: ") {
			d.Message = strings.TrimPrefix(flash, "INFO: ")
			d.InfoMsg = true
		} else {
			d.Flashes = append(d.Flashes, template.HTML(flash))
		}
	}

	showUserPage(w, "import", d)
	return nil
}

func handleImport(app *App, u *User, w http.ResponseWriter, r *http.Request) error {
	// limit 10MB per submission
	// TODO: increase?
	r.ParseMultipartForm(10 << 20)
	files := r.MultipartForm.File["files"]
	filesSubmitted := len(files)
	var filesImported, collsImported int
	var errs []error
	// TODO: support multiple zip uploads at once
	if filesSubmitted == 1 && files[0].Header.Get("Content-Type") == "application/zip" {
		filesSubmitted, filesImported, collsImported, errs = importZipPosts(app, w, r, files[0], u)
	} else {
		filesImported, errs = importFilePosts(app, w, r, files, u)
	}

	if len(errs) != 0 {
		_ = addSessionFlash(app, w, r, multierror.ListFormatFunc(errs), nil)
	}
	if filesImported == filesSubmitted && filesSubmitted != 0 {
		postAdj := "posts"
		if filesSubmitted == 1 {
			postAdj = "post"
		}
		if collsImported != 0 {
			collAdj := "collections"
			if collsImported == 1 {
				collAdj = "collection"
			}
			_ = addSessionFlash(app, w, r, fmt.Sprintf(
				"SUCCESS: Import complete, %d %s imported across %d %s.",
				filesImported,
				postAdj,
				collsImported,
				collAdj,
			), nil)
		} else {
			_ = addSessionFlash(app, w, r, fmt.Sprintf("SUCCESS: Import complete, %d %s imported.", filesImported, postAdj), nil)
		}
	} else if filesImported == 0 && filesSubmitted == 0 {
		_ = addSessionFlash(app, w, r, "INFO: 0 valid posts found", nil)
	} else if filesImported > 0 {
		_ = addSessionFlash(app, w, r, fmt.Sprintf("INFO: %d of %d posts imported, see details below.", filesImported, filesSubmitted), nil)
	}
	return impart.HTTPError{http.StatusFound, "/me/import"}
}

func importFilePosts(app *App, w http.ResponseWriter, r *http.Request, files []*multipart.FileHeader, u *User) (int, []error) {
	var fileErrs []error
	var count int
	for _, formFile := range files {
		if filepath.Ext(formFile.Filename) == ".zip" {
			fileErrs = append(fileErrs, fmt.Errorf("zips are supported as a single upload only: %s", formFile.Filename))
			log.Info("zip included in bulk files, skipping")
			continue
		}
		info, err := formFileToTemp(formFile)
		if err != nil {
			fileErrs = append(fileErrs, fmt.Errorf("failed to get file info of: %s", formFile.Filename))
			log.Error("import textfile: stat temp file: %v", err)
			continue
		}
		post, err := wfimport.FromFile(filepath.Join(os.TempDir(), info.Name()))
		if err == wfimport.ErrEmptyFile {
			// not a real error so don't log
			_ = addSessionFlash(app, w, r, fmt.Sprintf("%s was empty, import skipped", formFile.Filename), nil)
			continue
		} else if err == wfimport.ErrInvalidContentType {
			// same as above
			_ = addSessionFlash(app, w, r, fmt.Sprintf("%s is not a supported post file", formFile.Filename), nil)
			continue
		} else if err != nil {
			fileErrs = append(fileErrs, fmt.Errorf("failed to read copy of %s", formFile.Filename))
			log.Error("import textfile: file to post: %v", err)
			continue
		}

		post.Collection = r.PostFormValue("collection")
		coll, _ := app.db.GetCollection(post.Collection)
		if coll == nil {
			coll = &Collection{
				ID: 0,
			}
		}
		coll.hostName = app.cfg.App.Host
		created := post.Created.Format("2006-01-02T15:04:05Z")
		submittedPost := SubmittedPost{
			Title:   &post.Title,
			Content: &post.Content,
			Font:    "norm",
			Created: &created,
		}
		rp, err := app.db.CreatePost(u.ID, coll.ID, &submittedPost)
		if err != nil {
			fileErrs = append(fileErrs, fmt.Errorf("failed to create post from %s", formFile.Filename))
			log.Error("import textfile: create db post: %v", err)
			continue
		}

		// create public post

		if coll.ID != 0 && app.cfg.App.Federation {
			go federatePost(
				app,
				&PublicPost{
					Post: rp,
					Collection: &CollectionObj{
						Collection: *coll,
					},
				},
				coll.ID,
				false,
			)
		}
		count++
	}
	return count, fileErrs
}

func importZipPosts(app *App, w http.ResponseWriter, r *http.Request, file *multipart.FileHeader, u *User) (filesSubmitted, importedPosts, importedColls int, errs []error) {
	info, err := formFileToTemp(file)
	if err != nil {
		errs = append(errs, fmt.Errorf("upload temp file: %v", err))
		return
	}

	postMap, err := wfimport.FromZipDirs(filepath.Join(os.TempDir(), info.Name()))
	if err != nil {
		errs = append(errs, fmt.Errorf("parse posts and collections from zip: %v", err))
		return
	}

	for collKey, posts := range postMap {
		if len(posts) == 0 {
			continue
		}
		collObj := CollectionObj{}
		if collKey != wfimport.DraftsKey {
			coll, err := app.db.GetCollection(collKey)
			if err == ErrCollectionNotFound {
				coll, err = app.db.CreateCollection(app.cfg, collKey, collKey, u.ID)
				if err != nil {
					errs = append(errs, fmt.Errorf("create non existent collection: %v", err))
					continue
				}
				coll.hostName = app.cfg.App.Host
				collObj.Collection = *coll
			} else if err != nil {
				errs = append(errs, fmt.Errorf("get collection: %v", err))
				continue
			}
			collObj.Collection = *coll
			importedColls++
		}

		for _, post := range posts {
			if post != nil {
				filesSubmitted++
				created := post.Created.Format("2006-01-02T15:04:05Z")
				submittedPost := SubmittedPost{
					Title:   &post.Title,
					Content: &post.Content,
					Font:    "norm",
					Created: &created,
				}
				rp, err := app.db.CreatePost(u.ID, collObj.Collection.ID, &submittedPost)
				if err != nil {
					errs = append(errs, fmt.Errorf("create post: %v", err))
					continue
				}

				if collObj.Collection.ID != 0 && app.cfg.App.Federation {
					go federatePost(
						app,
						&PublicPost{
							Post:       rp,
							Collection: &collObj,
						},
						collObj.Collection.ID,
						false,
					)
				}
				importedPosts++
			}
		}
	}
	return
}

func formFileToTemp(formFile *multipart.FileHeader) (os.FileInfo, error) {
	file, err := formFile.Open()
	if err != nil {
		return nil, fmt.Errorf("failed to open form file: %s", formFile.Filename)
	}
	defer file.Close()

	tempFile, err := ioutil.TempFile("", fmt.Sprintf("upload-*%s", filepath.Ext(formFile.Filename)))
	if err != nil {
		return nil, fmt.Errorf("failed to create temporary file for: %s", formFile.Filename)
	}
	defer tempFile.Close()

	_, err = io.Copy(tempFile, file)
	if err != nil {
		return nil, fmt.Errorf("failed to copy file into temporary location: %s", formFile.Filename)
	}

	return tempFile.Stat()
}