add funtionality to import zip/tgz with maildirs/mboxes to account page

so users can easily take their email out of somewhere else, and import it into mox.

this goes a little way to give feedback as the import progresses: upload
progress is shown (surprisingly, browsers aren't doing this...), imported
mailboxes/messages are counted (batched) and import issues/warnings are
displayed, all sent over an SSE connection. an import token is stored in
sessionstorage. if you reload the page (e.g. after a connection error), the
browser will reconnect to the running import and show its progress again. and
you can just abort the import before it is finished and committed, and nothing
will have changed.

this also imports flags/keywords from mbox files.

README.md

@@ -140,12 +140,13 @@ documentation.
 
 ## How do I import/export email?
 
-Use the "mox import maildir" or "mox import mbox" subcommands. You could also
-use your IMAP email client, add your mox account, and copy or move messages
-from one account to the other.
+Use the import functionality on the accounts web page to import a zip/tgz with
+maildirs/mbox files, or use the "mox import maildir" or "mox import mbox"
+subcommands. You could also use your IMAP email client, add your mox account,
+and copy or move messages from one account to the other.
 
-Similarly, see the "mox export maildir" and "mox export mbox" subcommands to
-export email.
+Similarly, see the export functionality on the accounts web page and the "mox
+export maildir" and "mox export mbox" subcommands to export email.
 
 ## How can I help?
 
@@ -168,8 +169,8 @@ work.
 ## How do I change my password?
 
 Regular users (doing IMAP/SMTP with authentication) can change their password
-at the account page, e.g. http://127.0.0.1/account/. Or you can set a password
-with "mox setaccountpassword".
+at the account page, e.g. http://127.0.0.1/. Or you can set a password with "mox
+setaccountpassword".
 
 The admin password can be changed with "mox setadminpassword".
 

doc.go

@@ -206,56 +206,44 @@ The message is printed to stdout and is in standard internet mail format.
 
 Import a maildir into an account.
 
-By default, messages will train the junk filter based on their flags and
-mailbox naming. If the destination mailbox name starts with "junk" or "spam"
-(case insensitive), messages are imported and trained as junk regardless of
-pre-existing flags. Use the -train=false flag to prevent training the filter.
+By default, messages will train the junk filter based on their flags and, if
+"automatic junk flags" configuration is set, based on mailbox naming.
 
 If the destination mailbox is "Sent", the recipients of the messages are added
 to the message metadata, causing later incoming messages from these recipients
 to be accepted, unless other reputation signals prevent that.
 
-The message "read"/"seen" flag can be overridden during import with the
--markread flag.
+Users can also import mailboxes/messages through the account web page by
+uploading a zip or tgz file with mbox and/or maildirs.
 
-Mailbox flags, like "seen", "answered", "forwarded", will be imported. An
-attempt is made to parse dovecot keyword files.
+Mailbox flags, like "seen", "answered", will be imported. An optional
+dovecot-keywords file can specify additional flags, like Forwarded/Junk/NotJunk.
 
 The maildir files/directories are read by the mox process, so make sure it has
 access to the maildir directories/files.
 
 	usage: mox import maildir accountname mailboxname maildir
-	  -markread
-	    	mark all imported messages as read
-	  -train
-	    	train junkfilter with messages (default true)
 
 # mox import mbox
 
 Import an mbox into an account.
 
-Using mbox is not recommended, maildir is a better format.
+Using mbox is not recommended, maildir is a better defined format.
 
-By default, messages will train the junk filter based on their flags and
-mailbox naming. If the destination mailbox name starts with "junk" or "spam"
-(case insensitive), messages are imported and trained as junk regardless of
-pre-existing flags. Use the -train=false flag to prevent training the filter.
+By default, messages will train the junk filter based on their flags and, if
+"automatic junk flags" configuration is set, based on mailbox naming.
 
 If the destination mailbox is "Sent", the recipients of the messages are added
 to the message metadata, causing later incoming messages from these recipients
 to be accepted, unless other reputation signals prevent that.
 
-The message "read"/"seen" flag can be overridden during import with the
--markread flag.
+Users can also import mailboxes/messages through the account web page by
+uploading a zip or tgz file with mbox and/or maildirs.
 
 The mailbox is read by the mox process, so make sure it has access to the
 maildir directories/files.
 
 	usage: mox import mbox accountname mailboxname mbox
-	  -markread
-	    	mark all imported messages as read
-	  -train
-	    	train junkfilter with messages (default true)
 
 # mox export maildir
 

http/account.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"io"
 	"net"
@@ -106,6 +107,63 @@ func accountHandle(w http.ResponseWriter, r *http.Request) {
 	ctx := context.WithValue(r.Context(), mlog.CidKey, mox.Cid())
 	log := xlog.WithContext(ctx).Fields(mlog.Field("userauth", ""))
 
+	// Without authentication. The token is unguessable.
+	if r.URL.Path == "/importprogress" {
+		if r.Method != "GET" {
+			http.Error(w, "405 - method not allowed - get required", http.StatusMethodNotAllowed)
+			return
+		}
+
+		q := r.URL.Query()
+		token := q.Get("token")
+		if token == "" {
+			http.Error(w, "400 - bad request - missing token", http.StatusBadRequest)
+			return
+		}
+
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			log.Error("internal error: ResponseWriter not a http.Flusher")
+			http.Error(w, "500 - internal error - cannot sync to http connection", 500)
+			return
+		}
+
+		l := importListener{token, make(chan importEvent, 100), make(chan bool, 1)}
+		importers.Register <- &l
+		ok = <-l.Register
+		if !ok {
+			http.Error(w, "400 - bad request - unknown token, import may have finished more than a minute ago", http.StatusBadRequest)
+			return
+		}
+		defer func() {
+			importers.Unregister <- &l
+		}()
+
+		h := w.Header()
+		h.Set("Content-Type", "text/event-stream")
+		h.Set("Cache-Control", "no-cache")
+		_, err := w.Write([]byte(": keepalive\n\n"))
+		if err != nil {
+			return
+		}
+		flusher.Flush()
+
+		ctx := r.Context()
+		for {
+			select {
+			case e := <-l.Events:
+				_, err := w.Write(e.SSEMsg)
+				flusher.Flush()
+				if err != nil {
+					return
+				}
+
+			case <-ctx.Done():
+				return
+			}
+		}
+	}
+
 	accName := checkAccountAuth(ctx, log, w, r)
 	if accName == "" {
 		// Response already sent.
@@ -165,6 +223,54 @@ func accountHandle(w http.ResponseWriter, r *http.Request) {
 			log.Errorx("exporting mail", err)
 		}
 
+	case "/import":
+		if r.Method != "POST" {
+			http.Error(w, "405 - method not allowed - post required", http.StatusMethodNotAllowed)
+			return
+		}
+
+		f, _, err := r.FormFile("file")
+		if err != nil {
+			if errors.Is(err, http.ErrMissingFile) {
+				http.Error(w, "400 - bad request - missing file", http.StatusBadRequest)
+			} else {
+				http.Error(w, "500 - internal server error - "+err.Error(), http.StatusInternalServerError)
+			}
+			return
+		}
+		defer f.Close()
+		skipMailboxPrefix := r.FormValue("skipMailboxPrefix")
+		tmpf, err := os.CreateTemp("", "mox-import")
+		if err != nil {
+			http.Error(w, "500 - internal server error - "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+		defer func() {
+			if tmpf != nil {
+				tmpf.Close()
+			}
+		}()
+		if err := os.Remove(tmpf.Name()); err != nil {
+			log.Errorx("removing temporary file", err)
+			http.Error(w, "500 - internal server error - "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if _, err := io.Copy(tmpf, f); err != nil {
+			log.Errorx("copying import to temporary file", err)
+			http.Error(w, "500 - internal server error - "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+		token, err := importStart(log, accName, tmpf, skipMailboxPrefix)
+		if err != nil {
+			log.Errorx("starting import", err)
+			http.Error(w, "500 - internal server error - "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+		tmpf = nil // importStart is now responsible for closing.
+
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(map[string]string{"ImportToken": token})
+
 	default:
 		if strings.HasPrefix(r.URL.Path, "/api/") {
 			accountSherpaHandler.ServeHTTP(w, r.WithContext(context.WithValue(ctx, authCtxKey, accName)))
@@ -230,3 +336,11 @@ func (Account) DestinationSave(ctx context.Context, destName string, oldDest, ne
 	err := mox.DestinationSave(ctx, accountName, destName, newDest)
 	xcheckf(ctx, err, "saving destination")
 }
+
+// ImportAbort aborts an import that is in progress. If the import exists and isn't
+// finished, no changes will have been made by the import.
+func (Account) ImportAbort(ctx context.Context, importToken string) error {
+	req := importAbortRequest{importToken, make(chan error)}
+	importers.Abort <- req
+	return <-req.Response
+}