implement storing non-system/well-known flags (keywords) for messages…

… and mailboxes, with imap

the mailbox select/examine responses now return all flags used in a mailbox in
the FLAGS response. and indicate in the PERMANENTFLAGS response that clients
can set new keywords. we store these values on the new Message.Keywords field.
system/well-known flags are still in Message.Flags, so we're recognizing those
and handling them separately.

the imap store command handles the new flags. as does the append command, and
the search command.

we store keywords in a mailbox when a message in that mailbox gets the keyword.
we don't automatically remove the keywords from a mailbox. there is currently
no way at all to remove a keyword from a mailbox.

the import commands now handle non-system/well-known keywords too, when
importing from mbox/maildir.

jmap requires keyword support, so best to get it out of the way now.

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	go.etcd.io/bbolt v1.3.7
 	golang.org/x/crypto v0.8.0
+	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 	golang.org/x/net v0.9.0
 	golang.org/x/text v0.9.0
 )

go.sum

@@ -236,6 +236,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

http/account_test.go

@@ -14,14 +14,19 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 
+	"github.com/mjl-/bstore"
+
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/store"
 )
 
+var ctxbg = context.Background()
+
 func tcheck(t *testing.T, err error, msg string) {
 	t.Helper()
 	if err != nil {
@@ -50,7 +55,7 @@ func TestAccount(t *testing.T) {
 		if authHdr != "" {
 			r.Header.Add("Authorization", authHdr)
 		}
-		ok := checkAccountAuth(context.Background(), log, w, r)
+		ok := checkAccountAuth(ctxbg, log, w, r)
 		if ok != expect {
 			t.Fatalf("got %v, expected %v", ok, expect)
 		}
@@ -59,7 +64,7 @@ func TestAccount(t *testing.T) {
 	const authOK = "Basic bWpsQG1veC5leGFtcGxlOnRlc3QxMjM0"      // [email protected]:test1234
 	const authBad = "Basic bWpsQG1veC5leGFtcGxlOmJhZHBhc3N3b3Jk" // [email protected]:badpassword
 
-	authCtx := context.WithValue(context.Background(), authCtxKey, "mjl")
+	authCtx := context.WithValue(ctxbg, authCtxKey, "mjl")
 
 	test(authOK, "") // No password set yet.
 	Account{}.SetPassword(authCtx, "test1234")
@@ -132,6 +137,39 @@ func TestAccount(t *testing.T) {
 	testImport("../testdata/importtest.mbox.zip", 2)
 	testImport("../testdata/importtest.maildir.tgz", 2)
 
+	// Check there are messages, with the right flags.
+	acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
+		_, err = bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get()
+		tcheck(t, err, `fetching message with keywords "other" and "test"`)
+
+		mb, err := acc.MailboxFind(tx, "importtest")
+		tcheck(t, err, "looking up mailbox importtest")
+		if mb == nil {
+			t.Fatalf("missing mailbox importtest")
+		}
+		sort.Strings(mb.Keywords)
+		if strings.Join(mb.Keywords, " ") != "other test" {
+			t.Fatalf(`expected mailbox keywords "other" and "test", got %v`, mb.Keywords)
+		}
+
+		n, err := bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "custom").Count()
+		tcheck(t, err, `fetching message with keyword "custom"`)
+		if n != 2 {
+			t.Fatalf(`got %d messages with keyword "custom", expected 2`, n)
+		}
+
+		mb, err = acc.MailboxFind(tx, "maildir")
+		tcheck(t, err, "looking up mailbox maildir")
+		if mb == nil {
+			t.Fatalf("missing mailbox maildir")
+		}
+		if strings.Join(mb.Keywords, " ") != "custom" {
+			t.Fatalf(`expected mailbox keywords "custom", got %v`, mb.Keywords)
+		}
+
+		return nil
+	})
+
 	testExport := func(httppath string, iszip bool, expectFiles int) {
 		t.Helper()
 

http/admin_test.go

@@ -1,7 +1,6 @@
 package http
 
 import (
-	"context"
 	"crypto/ed25519"
 	"net"
 	"net/http/httptest"
@@ -29,7 +28,7 @@ func TestAdminAuth(t *testing.T) {
 		if authHdr != "" {
 			r.Header.Add("Authorization", authHdr)
 		}
-		ok := checkAdminAuth(context.Background(), passwordfile, w, r)
+		ok := checkAdminAuth(ctxbg, passwordfile, w, r)
 		if ok != expect {
 			t.Fatalf("got %v, expected %v", ok, expect)
 		}
@@ -125,9 +124,9 @@ func TestCheckDomain(t *testing.T) {
 	close(done)
 	dialer := &net.Dialer{Deadline: time.Now().Add(-time.Second), Cancel: done}
 
-	checkDomain(context.Background(), resolver, dialer, "mox.example")
+	checkDomain(ctxbg, resolver, dialer, "mox.example")
 	// todo: check returned data
 
-	Admin{}.Domains(context.Background())        // todo: check results
-	dnsblsStatus(context.Background(), resolver) // todo: check results
+	Admin{}.Domains(ctxbg)        // todo: check results
+	dnsblsStatus(ctxbg, resolver) // todo: check results
 }

http/import.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/exp/maps"
 	"golang.org/x/text/unicode/norm"
 
 	"github.com/mjl-/bstore"
@@ -361,10 +362,16 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 	mailboxes := map[string]store.Mailbox{}
 	messages := map[string]int{}
 
-	// For maildirs, we are likely to get a possible dovecot-keywords file after having imported the messages. Once we see the keywords, we use them. But before that time we remember which messages miss a keywords. Once the keywords become available, we'll fix up the flags for the unknown messages
+	// For maildirs, we are likely to get a possible dovecot-keywords file after having
+	// imported the messages. Once we see the keywords, we use them. But before that
+	// time we remember which messages miss a keywords. Once the keywords become
+	// available, we'll fix up the flags for the unknown messages
 	mailboxKeywords := map[string]map[rune]string{}                // Mailbox to 'a'-'z' to flag name.
 	mailboxMissingKeywordMessages := map[string]map[int64]string{} // Mailbox to message id to string consisting of the unrecognized flags.
 
+	// We keep the mailboxes we deliver to up to date with their keywords (non-system flags).
+	destMailboxKeywords := map[int64]map[string]bool{}
+
 	// Previous mailbox an event was sent for. We send an event for new mailboxes, when
 	// another 100 messages were added, when adding a message to another mailbox, and
 	// finally at the end as a closing statement.
@@ -471,6 +478,15 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		m.MailboxID = mb.ID
 		m.MailboxOrigID = mb.ID
 
+		if len(m.Keywords) > 0 {
+			if destMailboxKeywords[mb.ID] == nil {
+				destMailboxKeywords[mb.ID] = map[string]bool{}
+			}
+			for _, k := range m.Keywords {
+				destMailboxKeywords[mb.ID][k] = true
+			}
+		}
+
 		// Parse message and store parsed information for later fast retrieval.
 		p, err := message.EnsurePart(f, m.Size)
 		if err != nil {
@@ -503,7 +519,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 			return
 		}
 		deliveredIDs = append(deliveredIDs, m.ID)
-		changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags})
+		changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords})
 		messages[mb.Name]++
 		if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name {
 			prevMailbox = mb.Name
@@ -583,7 +599,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 
 		// Parse flags. See https://cr.yp.to/proto/maildir.html.
 		var keepFlags string
-		flags := store.Flags{}
+		var flags store.Flags
+		keywords := map[string]bool{}
 		t = strings.SplitN(path.Base(filename), ":2,", 2)
 		if len(t) == 2 {
 			for _, c := range t[1] {
@@ -602,12 +619,12 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 					flags.Flagged = true
 				default:
 					if c >= 'a' && c <= 'z' {
-						keywords, ok := mailboxKeywords[mailbox]
+						dovecotKeywords, ok := mailboxKeywords[mailbox]
 						if !ok {
 							// No keywords file seen yet, we'll try later if it comes in.
 							keepFlags += string(c)
-						} else if kw, ok := keywords[c]; ok {
-							flagSet(&flags, strings.ToLower(kw))
+						} else if kw, ok := dovecotKeywords[c]; ok {
+							flagSet(&flags, keywords, strings.ToLower(kw))
 						}
 					}
 				}
@@ -617,6 +634,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		m := store.Message{
 			Received: received,
 			Flags:    flags,
+			Keywords: maps.Keys(keywords),
 			Size:     size,
 		}
 		xdeliver(mb, &m, f, filename)
@@ -663,38 +681,52 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		default:
 			if path.Base(name) == "dovecot-keywords" {
 				mailbox := path.Dir(name)
-				keywords := map[rune]string{}
+				dovecotKeywords := map[rune]string{}
 				words, err := store.ParseDovecotKeywords(r, log)
 				log.Check(err, "parsing dovecot keywords for mailbox", mlog.Field("mailbox", mailbox))
 				for i, kw := range words {
-					keywords['a'+rune(i)] = kw
+					dovecotKeywords['a'+rune(i)] = kw
 				}
-				mailboxKeywords[mailbox] = keywords
+				mailboxKeywords[mailbox] = dovecotKeywords
 
 				for id, chars := range mailboxMissingKeywordMessages[mailbox] {
 					var flags, zeroflags store.Flags
+					keywords := map[string]bool{}
 					for _, c := range chars {
-						kw, ok := keywords[c]
+						kw, ok := dovecotKeywords[c]
 						if !ok {
-							problemf("unspecified message flag %c for message id %d (continuing)", c, id)
+							problemf("unspecified dovecot message flag %c for message id %d (continuing)", c, id)
 							continue
 						}
-						flagSet(&flags, strings.ToLower(kw))
+						flagSet(&flags, keywords, strings.ToLower(kw))
 					}
-					if flags == zeroflags {
+					if flags == zeroflags && len(keywords) == 0 {
 						continue
 					}
+
 					m := store.Message{ID: id}
 					err := tx.Get(&m)
 					ximportcheckf(err, "get imported message for flag update")
+
 					m.Flags = m.Flags.Set(flags, flags)
+					m.Keywords = maps.Keys(keywords)
+
+					if len(m.Keywords) > 0 {
+						if destMailboxKeywords[m.MailboxID] == nil {
+							destMailboxKeywords[m.MailboxID] = map[string]bool{}
+						}
+						for _, k := range m.Keywords {
+							destMailboxKeywords[m.MailboxID][k] = true
+						}
+					}
+
 					// We train before updating, training may set m.TrainedJunk.
 					if jf != nil && m.NeedsTraining() {
 						openTrainMessage(&m)
 					}
 					err = tx.Update(&m)
 					ximportcheckf(err, "updating message after flag update")
-					changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags})
+					changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags, Keywords: m.Keywords})
 				}
 				delete(mailboxMissingKeywordMessages, mailbox)
 			} else {
@@ -744,6 +776,19 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		sendEvent("count", importCount{prevMailbox, messages[prevMailbox]})
 	}
 
+	// Update mailboxes with keywords.
+	for mbID, keywords := range destMailboxKeywords {
+		mb := store.Mailbox{ID: mbID}
+		err := tx.Get(&mb)
+		ximportcheckf(err, "loading mailbox for updating keywords")
+		var changed bool
+		mb.Keywords, changed = store.MergeKeywords(mb.Keywords, maps.Keys(keywords))
+		if changed {
+			err = tx.Update(&mb)
+			ximportcheckf(err, "updating mailbox with keywords")
+		}
+	}
+
 	err = tx.Commit()
 	tx = nil
 	ximportcheckf(err, "commit")
@@ -768,9 +813,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 	sendEvent("done", importDone{})
 }
 
-func flagSet(flags *store.Flags, word string) {
-	// todo: custom labels, e.g. $label1, JunkRecorded?
-
+func flagSet(flags *store.Flags, keywords map[string]bool, word string) {
 	switch word {
 	case "forwarded", "$forwarded":
 		flags.Forwarded = true
@@ -782,5 +825,9 @@ func flagSet(flags *store.Flags, word string) {
 		flags.Phishing = true
 	case "mdnsent", "$mdnsent":
 		flags.MDNSent = true
+	default:
+		if store.ValidLowercaseKeyword(word) {
+			keywords[word] = true
+		}
 	}
 }

imapclient/parse.go

@@ -178,9 +178,9 @@ func (c *Conn) xrespCode() (string, CodeArg) {
 		l := []string{} // Must be non-nil.
 		if c.take(' ') {
 			c.xtake("(")
-			l = []string{c.xflag()}
+			l = []string{c.xflagPerm()}
 			for c.take(' ') {
-				l = append(l, c.xflag())
+				l = append(l, c.xflagPerm())
 			}
 			c.xtake(")")
 		}
@@ -694,17 +694,28 @@ func (c *Conn) xliteral() []byte {
 
 // ../rfc/9051:6565
 // todo: stricter
-func (c *Conn) xflag() string {
+func (c *Conn) xflag0(allowPerm bool) string {
 	s := ""
 	if c.take('\\') {
-		s = "\\"
+		s = `\`
+		if allowPerm && c.take('*') {
+			return `\*`
+		}
 	} else if c.take('$') {
 		s = "$"
 	}
 	s += c.xatom()
 	return s
 }
 
+func (c *Conn) xflag() string {
+	return c.xflag0(false)
+}
+
+func (c *Conn) xflagPerm() string {
+	return c.xflag0(true)
+}
+
 func (c *Conn) xsection() string {
 	c.xtake("[")
 	s := c.xtakeuntil(']')

imapserver/append_test.go

@@ -44,14 +44,14 @@ func TestAppend(t *testing.T) {
 	tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}")
 	tc2.xcode("TRYCREATE")
 
-	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc2.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
 	tc2.xuntagged(imapclient.UntaggedExists(1))
 	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 1})
 
 	tc.transactf("ok", "noop")
 	uid1 := imapclient.FetchUID(1)
-	flagsSeen := imapclient.FetchFlags{`\Seen`}
-	tc.xuntagged(imapclient.UntaggedExists(1), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, flagsSeen}})
+	flags := imapclient.FetchFlags{`\Seen`, "label1", "$label2"}
+	tc.xuntagged(imapclient.UntaggedExists(1), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, flags}})
 	tc3.transactf("ok", "noop")
 	tc3.xuntagged() // Inbox is not selected, nothing to report.
 

imapserver/fetch.go

@@ -189,12 +189,12 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 		err := cmd.tx.Update(m)
 		xcheckf(err, "marking message as seen")
 
-		cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags})
+		cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
 	}
 
 	if cmd.needFlags {
 		m := cmd.xensureMessage()
-		data = append(data, bare("FLAGS"), flaglist(m.Flags))
+		data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
 	}
 
 	// Write errors are turned into panics because we write through c.