webmail: when moving a single message out of/to the inbox, ask if use…

…r wants to create a rule to automatically do that server-side for future deliveries

if the message has a list-id header, we assume this is a (mailing) list
message, and we require a dkim/spf-verified domain (we prefer the shortest that
is a suffix of the list-id value). the rule we would add will mark such
messages as from a mailing list, changing filtering rules on incoming messages
(not enforcing dmarc policies). messages will be matched on list-id header and
will only match if they have the same dkim/spf-verified domain.

if the message doesn't have a list-id header, we'll ask to match based on
"message from" address.

we don't ask the user in several cases:
- if the destination/source mailbox is a special-use mailbox (e.g.
  trash,archive,sent,junk; inbox isn't included)
- if the rule already exist (no point in adding it again).
- if the user said "no, not for this list-id/from-address" in the past.
- if the user said "no, not for messages moved to this mailbox" in the past.

we'll add the rule if the message was moved out of the inbox.
if the message was moved to the inbox, we check if there is a matching rule
that we can remove.

we now remember the "no" answers (for list-id, msg-from-addr and mailbox) in
the account database.

to implement the msgfrom rules, this adds support to rulesets for matching on
message "from" address. before, we could match on smtp from address (and other
fields). rulesets now also have a field for comments. webmail adds a note that
it created the rule, with the date.

manual editing of the rulesets is still in the webaccount page. this webmail
functionality is just a convenient way to add/remove common rules.

config/config.go

@@ -444,26 +444,29 @@ func (d Destination) Equal(o Destination) bool {
 
 type Ruleset struct {
 	SMTPMailFromRegexp string            `sconf:"optional" sconf-doc:"Matches if this regular expression matches (a substring of) the SMTP MAIL FROM address (not the message From-header). E.g. '^user@example\\.org$'."`
+	MsgFromRegexp      string            `sconf:"optional" sconf-doc:"Matches if this regular expression matches (a substring of) the single address in the message From header."`
 	VerifiedDomain     string            `sconf:"optional" sconf-doc:"Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain."`
 	HeadersRegexp      map[string]string `sconf:"optional" sconf-doc:"Matches if these header field/value regular expressions all match (substrings of) the message headers. Header fields and valuees are converted to lower case before matching. Whitespace is trimmed from the value before matching. A header field can occur multiple times in a message, only one instance has to match. For mailing lists, you could match on ^list-id$ with the value typically the mailing list address in angled brackets with @ replaced with a dot, e.g. <name\\.lists\\.example\\.org>."`
-	// todo: add a SMTPRcptTo check, and MessageFrom that works on a properly parsed From header.
+	// todo: add a SMTPRcptTo check
 
 	// todo: once we implement ARC, we can use dkim domains that we cannot verify but that the arc-verified forwarding mail server was able to verify.
 	IsForward              bool   `sconf:"optional" sconf-doc:"Influences spam filtering only, this option does not change whether a message matches this ruleset. Can only be used together with SMTPMailFromRegexp and VerifiedDomain. SMTPMailFromRegexp must be set to the address used to deliver the forwarded message, e.g. '^user(|\\+.*)@forward\\.example$'. Changes to junk analysis: 1. Messages are not rejected for failing a DMARC policy, because a legitimate forwarded message without valid/intact/aligned DKIM signature would be rejected because any verified SPF domain will be 'unaligned', of the forwarding mail server. 2. The sending mail server IP address, and sending EHLO and MAIL FROM domains and matching DKIM domain aren't used in future reputation-based spam classifications (but other verified DKIM domains are) because the forwarding server is not a useful spam signal for future messages."`
 	ListAllowDomain        string `sconf:"optional" sconf-doc:"Influences spam filtering only, this option does not change whether a message matches this ruleset. If this domain matches an SPF- and/or DKIM-verified (sub)domain, the message is accepted without further spam checks, such as a junk filter or DMARC reject evaluation. DMARC rejects should not apply for mailing lists that are not configured to rewrite the From-header of messages that don't have a passing DKIM signature of the From-domain. Otherwise, by rejecting messages, you may be automatically unsubscribed from the mailing list. The assumption is that mailing lists do their own spam filtering/moderation."`
 	AcceptRejectsToMailbox string `sconf:"optional" sconf-doc:"Influences spam filtering only, this option does not change whether a message matches this ruleset. If a message is classified as spam, it isn't rejected during the SMTP transaction (the normal behaviour), but accepted during the SMTP transaction and delivered to the specified mailbox. The specified mailbox is not automatically cleaned up like the account global Rejects mailbox, unless set to that Rejects mailbox."`
 
 	Mailbox string `sconf-doc:"Mailbox to deliver to if this ruleset matches."`
+	Comment string `sconf:"optional" sconf-doc:"Free-form comments."`
 
 	SMTPMailFromRegexpCompiled *regexp.Regexp      `sconf:"-" json:"-"`
+	MsgFromRegexpCompiled      *regexp.Regexp      `sconf:"-" json:"-"`
 	VerifiedDNSDomain          dns.Domain          `sconf:"-"`
 	HeadersRegexpCompiled      [][2]*regexp.Regexp `sconf:"-" json:"-"`
 	ListAllowDNSDomain         dns.Domain          `sconf:"-"`
 }
 
 // Equal returns whether r and o are equal, only looking at their user-changeable fields.
 func (r Ruleset) Equal(o Ruleset) bool {
-	if r.SMTPMailFromRegexp != o.SMTPMailFromRegexp || r.VerifiedDomain != o.VerifiedDomain || r.IsForward != o.IsForward || r.ListAllowDomain != o.ListAllowDomain || r.AcceptRejectsToMailbox != o.AcceptRejectsToMailbox || r.Mailbox != o.Mailbox {
+	if r.SMTPMailFromRegexp != o.SMTPMailFromRegexp || r.MsgFromRegexp != o.MsgFromRegexp || r.VerifiedDomain != o.VerifiedDomain || r.IsForward != o.IsForward || r.ListAllowDomain != o.ListAllowDomain || r.AcceptRejectsToMailbox != o.AcceptRejectsToMailbox || r.Mailbox != o.Mailbox || r.Comment != o.Comment {
 		return false
 	}
 	if !reflect.DeepEqual(r.HeadersRegexp, o.HeadersRegexp) {

config/doc.go

@@ -998,6 +998,10 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 							# address (not the message From-header). E.g. '^user@example\.org$'. (optional)
 							SMTPMailFromRegexp:
 
+							# Matches if this regular expression matches (a substring of) the single address
+							# in the message From header. (optional)
+							MsgFromRegexp:
+
 							# Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain.
 							# (optional)
 							VerifiedDomain:
@@ -1048,6 +1052,9 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 							# Mailbox to deliver to if this ruleset matches.
 							Mailbox:
 
+							# Free-form comments. (optional)
+							Comment:
+
 					# Full name to use in message From header when composing messages coming from this
 					# address with webmail. (optional)
 					FullName:

mox-/config.go

@@ -1335,6 +1335,14 @@ func prepareDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string,
 					}
 					c.Accounts[accName].Destinations[addrName].Rulesets[i].SMTPMailFromRegexpCompiled = r
 				}
+				if rs.MsgFromRegexp != "" {
+					n++
+					r, err := regexp.Compile(rs.MsgFromRegexp)
+					if err != nil {
+						addErrorf("invalid MsgFrom regular expression: %v", err)
+					}
+					c.Accounts[accName].Destinations[addrName].Rulesets[i].MsgFromRegexpCompiled = r
+				}
 				if rs.VerifiedDomain != "" {
 					n++
 					d, err := dns.ParseDomain(rs.VerifiedDomain)

store/account.go

@@ -757,6 +757,37 @@ type FromAddressSettings struct {
 	ViewMode    ViewMode
 }
 
+// RulesetNoListID records a user "no" response to the question of
+// creating/removing a ruleset after moving a message with list-id header from/to
+// the inbox.
+type RulesetNoListID struct {
+	ID            int64
+	RcptToAddress string `bstore:"nonzero"`
+	ListID        string `bstore:"nonzero"`
+	ToInbox       bool   // Otherwise from Inbox to other mailbox.
+}
+
+// RulesetNoMsgFrom records a user "no" response to the question of
+// creating/moveing a ruleset after moving a mesage with message "from" address
+// from/to the inbox.
+type RulesetNoMsgFrom struct {
+	ID             int64
+	RcptToAddress  string `bstore:"nonzero"`
+	MsgFromAddress string `bstore:"nonzero"` // Unicode.
+	ToInbox        bool   // Otherwise from Inbox to other mailbox.
+}
+
+// RulesetNoMailbox represents a "never from/to this mailbox" response to the
+// question of adding/removing a ruleset after moving a message.
+type RulesetNoMailbox struct {
+	ID int64
+
+	// The mailbox from/to which the move has happened.
+	// Not a references, if mailbox is deleted, an entry becomes ineffective.
+	MailboxID int64 `bstore:"nonzero"`
+	ToMailbox bool  // Whether MailboxID is the destination of the move (instead of source).
+}
+
 // Types stored in DB.
 var DBTypes = []any{
 	NextUIDValidity{},
@@ -774,6 +805,9 @@ var DBTypes = []any{
 	LoginSession{},
 	Settings{},
 	FromAddressSettings{},
+	RulesetNoListID{},
+	RulesetNoMsgFrom{},
+	RulesetNoMailbox{},
 }
 
 // Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
@@ -1758,7 +1792,7 @@ func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, erro
 	return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil
 }
 
-// MessageRuleset returns the first ruleset (if any) that message the message
+// MessageRuleset returns the first ruleset (if any) that matches the message
 // represented by msgPrefix and msgFile, with smtp and validation fields from m.
 func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {
 	if len(dest.Rulesets) == 0 {
@@ -1786,6 +1820,11 @@ ruleset:
 				continue ruleset
 			}
 		}
+		if rs.MsgFromRegexpCompiled != nil {
+			if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {
+				continue ruleset
+			}
+		}
 
 		if !rs.VerifiedDNSDomain.IsZero() {
 			d := rs.VerifiedDNSDomain.Name()

testdata/webmail/domains.conf

@@ -6,24 +6,26 @@ Domains:
 					PrivateKeyFile: testsel.rsakey.pkcs8.pem
 			Sign:
 				- testsel
+	other.example: nil
 Accounts:
-	other:
-		Domain: mox.example
-		Destinations:
-			[email protected]: nil
 	mjl:
-		MaxOutgoingMessagesPerDay: 30
-		MaxFirstTimeRecipientsPerDay: 10
 		Domain: mox.example
 		Destinations:
 			[email protected]: nil
-			møx@mox.example: nil
+			mox@other.example: nil
 			mó[email protected]: nil
+			mø[email protected]: nil
 		RejectsMailbox: Rejects
 		JunkFilter:
-			Threshold: 0.95
+			Threshold: 0.950000
 			Params:
 				Twograms: true
-				MaxPower: 0.1
+				MaxPower: 0.100000
 				TopWords: 10
-				IgnoreWords: 0.1
+				IgnoreWords: 0.100000
+		MaxOutgoingMessagesPerDay: 30
+		MaxFirstTimeRecipientsPerDay: 10
+	other:
+		Domain: mox.example
+		Destinations:
+			[email protected]: nil