add per-account quota for total message size disk usage

so a single user cannot fill up the disk.
by default, there is (still) no limit. a default can be set in the config file
for all accounts, and a per-account max size can be set that would override any
global setting.

this does not take into account disk usage of the index database. and also not
of any file system overhead.

config/config.go

@@ -68,9 +68,10 @@ type Static struct {
 	DefaultMailboxes []string             `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
 	Transports       map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
 	// Awkward naming of fields to get intended default behaviour for zero values.
-	NoOutgoingDMARCReports          bool `sconf:"optional" sconf-doc:"Do not send DMARC reports (aggregate only). By default, aggregate reports on DMARC evaluations are sent to domains if their DMARC policy requests them. Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 hours, rounded up so a whole number of intervals cover 24 hours, aligned at whole days in UTC. Reports are sent from the postmaster@<mailhostname> address."`
-	NoOutgoingTLSReports            bool `sconf:"optional" sconf-doc:"Do not send TLS reports. By default, reports about failed SMTP STARTTLS connections and related MTA-STS/DANE policies are sent to domains if their TLSRPT DNS record requests them. Reports covering a 24 hour UTC interval are sent daily. Reports are sent from the postmaster address of the configured domain the mailhostname is in. If there is no such domain, or it does not have DKIM configured, no reports are sent."`
-	OutgoingTLSReportsForAllSuccess bool `sconf:"optional" sconf-doc:"Also send TLS reports if there were no SMTP STARTTLS connection failures. By default, reports are only sent when at least one failure occurred. If a report is sent, it does always include the successful connection counts as well."`
+	NoOutgoingDMARCReports          bool  `sconf:"optional" sconf-doc:"Do not send DMARC reports (aggregate only). By default, aggregate reports on DMARC evaluations are sent to domains if their DMARC policy requests them. Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 hours, rounded up so a whole number of intervals cover 24 hours, aligned at whole days in UTC. Reports are sent from the postmaster@<mailhostname> address."`
+	NoOutgoingTLSReports            bool  `sconf:"optional" sconf-doc:"Do not send TLS reports. By default, reports about failed SMTP STARTTLS connections and related MTA-STS/DANE policies are sent to domains if their TLSRPT DNS record requests them. Reports covering a 24 hour UTC interval are sent daily. Reports are sent from the postmaster address of the configured domain the mailhostname is in. If there is no such domain, or it does not have DKIM configured, no reports are sent."`
+	OutgoingTLSReportsForAllSuccess bool  `sconf:"optional" sconf-doc:"Also send TLS reports if there were no SMTP STARTTLS connection failures. By default, reports are only sent when at least one failure occurred. If a report is sent, it does always include the successful connection counts as well."`
+	QuotaMessageSize                int64 `sconf:"optional" sconf-doc:"Default maximum total message size for accounts, only applicable if greater than zero. Can be overridden per account. Attempting to add new messages beyond the maximum size will result in an error. Useful to prevent a single account from filling storage. The quota only applies to the email message files, not to any file system overhead and also not the message index database file (account for approximately 15% overhead)."`
 
 	// All IPs that were explicitly listen on for external SMTP. Only set when there
 	// are no unspecified external SMTP listeners and there is at most one for IPv4 and
@@ -354,6 +355,7 @@ type Account struct {
 	SubjectPass  struct {
 		Period time.Duration `sconf-doc:"How long unique values are accepted after generating, e.g. 12h."` // todo: have a reasonable default for this?
 	} `sconf:"optional" sconf-doc:"If configured, messages classified as weakly spam are rejected with instructions to retry delivery, but this time with a signed token added to the subject. During the next delivery attempt, the signed token will bypass the spam filter. Messages with a clear spam signal, such as a known bad reputation, are rejected/delayed without a signed token."`
+	QuotaMessageSize   int64  `sconf:"optional" sconf-doc:"Default maximum total message size for the account, overriding any globally configured maximum size if non-zero. A negative value can be used to have no limit in case there is a limit by default. Attempting to add new messages beyond the maximum size will result in an error. Useful to prevent a single account from filling storage."`
 	RejectsMailbox     string `sconf:"optional" sconf-doc:"Mail that looks like spam will be rejected, but a copy can be stored temporarily in a mailbox, e.g. Rejects. If mail isn't coming in when you expect, you can look there. The mail still isn't accepted, so the remote mail server may retry (hopefully, if legitimate), or give up (hopefully, if indeed a spammer). Messages are automatically removed from this mailbox, so do not set it to a mailbox that has messages you want to keep."`
 	KeepRejects        bool   `sconf:"optional" sconf-doc:"Don't automatically delete mail in the RejectsMailbox listed above. This can be useful, e.g. for future spam training."`
 	AutomaticJunkFlags struct {

config/doc.go

@@ -588,6 +588,14 @@ describe-static" and "mox config describe-domains":
 	# (optional)
 	OutgoingTLSReportsForAllSuccess: false
 
+	# Default maximum total message size for accounts, only applicable if greater than
+	# zero. Can be overridden per account. Attempting to add new messages beyond the
+	# maximum size will result in an error. Useful to prevent a single account from
+	# filling storage. The quota only applies to the email message files, not to any
+	# file system overhead and also not the message index database file (account for
+	# approximately 15% overhead). (optional)
+	QuotaMessageSize: 0
+
 # domains.conf
 
 	# NOTE: This config file is in 'sconf' format. Indent with tabs. Comments must be
@@ -849,6 +857,13 @@ describe-static" and "mox config describe-domains":
 				# How long unique values are accepted after generating, e.g. 12h.
 				Period: 0s
 
+			# Default maximum total message size for the account, overriding any globally
+			# configured maximum size if non-zero. A negative value can be used to have no
+			# limit in case there is a limit by default. Attempting to add new messages beyond
+			# the maximum size will result in an error. Useful to prevent a single account
+			# from filling storage. (optional)
+			QuotaMessageSize: 0
+
 			# Mail that looks like spam will be rejected, but a copy can be stored temporarily
 			# in a mailbox, e.g. Rejects. If mail isn't coming in when you expect, you can
 			# look there. The mail still isn't accepted, so the remote mail server may retry

ctl.go

@@ -685,11 +685,13 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
 		acc.WithWLock(func() {
 			var changes []store.Change
 			err = acc.DB.Write(ctx, func(tx *bstore.Tx) error {
-				return bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
+				var totalSize int64
+				err := bstore.QueryTx[store.Mailbox](tx).ForEach(func(mb store.Mailbox) error {
 					mc, err := mb.CalculateCounts(tx)
 					if err != nil {
 						return fmt.Errorf("calculating counts for mailbox %q: %w", mb.Name, err)
 					}
+					totalSize += mc.Size
 
 					if !mb.HaveCounts || mc != mb.MailboxCounts {
 						_, err := fmt.Fprintf(w, "for %s setting new counts %s (was %s)\n", mb.Name, mc, mb.MailboxCounts)
@@ -703,6 +705,23 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
 					}
 					return nil
 				})
+				if err != nil {
+					return err
+				}
+
+				du := store.DiskUsage{ID: 1}
+				if err := tx.Get(&du); err != nil {
+					return fmt.Errorf("get disk usage: %v", err)
+				}
+				if du.MessageSize != totalSize {
+					_, err := fmt.Fprintf(w, "setting new total message size %d (was %d)\n", totalSize, du.MessageSize)
+					ctl.xcheck(err, "write")
+					du.MessageSize = totalSize
+					if err := tx.Update(&du); err != nil {
+						return fmt.Errorf("update disk usage: %v", err)
+					}
+				}
+				return nil
 			})
 			ctl.xcheck(err, "write transaction for mailbox counts")
 

doc.go

@@ -977,12 +977,13 @@ Ensure messages in the database have a pre-parsed MIME form in the database.
 
 # mox recalculatemailboxcounts
 
-Recalculate message counts for all mailboxes in the account.
+Recalculate message counts for all mailboxes in the account, and total message size for quota.
 
 When a message is added to/removed from a mailbox, or when message flags change,
-the total, unread, unseen and deleted messages are accounted, and the total size
-of the mailbox. In case of a bug in this accounting, the numbers could become
-incorrect. This command will find, fix and print them.
+the total, unread, unseen and deleted messages are accounted, the total size of
+the mailbox, and the total message size for the account. In case of a bug in
+this accounting, the numbers could become incorrect. This command will find, fix
+and print them.
 
 	usage: mox recalculatemailboxcounts account
 

gentestdata.go

@@ -284,7 +284,7 @@ Accounts:
 		xcheckf(err, "creating temp file for delivery")
 		_, err = fmt.Fprint(mf, msg)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false)
+		err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false, true)
 
 		mfname := mf.Name()
 		xcheckf(err, "add message to account test1")
@@ -344,7 +344,7 @@ Accounts:
 		xcheckf(err, "creating temp file for delivery")
 		_, err = fmt.Fprint(mf0, msg0)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false)
+		err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false, true)
 		xcheckf(err, "add message to account test2")
 
 		mf0name := mf0.Name()
@@ -375,7 +375,7 @@ Accounts:
 		xcheckf(err, "creating temp file for delivery")
 		_, err = fmt.Fprint(mf1, msg1)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false)
+		err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false, true)
 		xcheckf(err, "add message to account test2")
 
 		mf1name := mf1.Name()

imapclient/parse.go

@@ -116,7 +116,7 @@ func (c *Conn) xrespText() RespText {
 
 var knownCodes = stringMap(
 	// Without parameters.
-	"ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE",
+	"ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE", "OVERQUOTA",
 	// With parameters.
 	"BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID",
 	"HIGHESTMODSEQ", "MODIFIED",

imapserver/append_test.go

@@ -78,4 +78,15 @@ func TestAppend(t *testing.T) {
 		},
 	}
 	tc2.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, xbs}})
+
+	tclimit := startArgs(t, false, false, true, true, "limit")
+	defer tclimit.close()
+	tclimit.client.Login("[email protected]", "testtest")
+	tclimit.client.Select("inbox")
+	// First message of 1 byte is within limits.
+	tclimit.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tclimit.xuntagged(imapclient.UntaggedExists(1))
+	// Second message would take account past limit.
+	tclimit.transactf("no", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tclimit.xcode("OVERQUOTA")
 }

imapserver/copy_test.go

@@ -58,4 +58,15 @@ func TestCopy(t *testing.T) {
 		imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(3), imapclient.FetchFlags(nil)}},
 		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), imapclient.FetchFlags(nil)}},
 	)
+
+	tclimit := startArgs(t, false, false, true, true, "limit")
+	defer tclimit.close()
+	tclimit.client.Login("[email protected]", "testtest")
+	tclimit.client.Select("inbox")
+	// First message of 1 byte is within limits.
+	tclimit.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tclimit.xuntagged(imapclient.UntaggedExists(1))
+	// Second message would take account past limit.
+	tclimit.transactf("no", "copy 1:* Trash")
+	tclimit.xcode("OVERQUOTA")
 }

imapserver/fetch.go

@@ -243,6 +243,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 			err := tx.Update(&mb)
 			xcheckf(err, "updating mailbox counts")
 			cmd.changes = append(cmd.changes, mb.ChangeCounts())
+			// No need to update account total message size.
 		}
 	})
 
@@ -349,6 +350,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 		m.ModSeq = cmd.xmodseq()
 		err := cmd.tx.Update(m)
 		xcheckf(err, "marking message as seen")
+		// No need to update account total message size.
 
 		cmd.changes = append(cmd.changes, m.ChangeFlags(origFlags))
 	}

imapserver/server.go

@@ -2750,13 +2750,20 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
 				Size:          mw.Size,
 			}
 
+			ok, maxSize, err := c.account.CanAddMessageSize(tx, m.Size)
+			xcheckf(err, "checking quota")
+			if !ok {
+				// ../rfc/9051:5155
+				xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
+			}
+
 			mb.Add(m.MailboxCounts())
 
 			// Update mailbox before delivering, which updates uidnext which we mustn't overwrite.
 			err = tx.Update(&mb)
 			xcheckf(err, "updating mailbox counts")
 
-			err := c.account.DeliverMessage(c.log, tx, &m, msgFile, true, false, false)
+			err = c.account.DeliverMessage(c.log, tx, &m, msgFile, true, false, false, true)
 			xcheckf(err, "delivering message")
 		})
 
@@ -2923,10 +2930,12 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (remove []store.M
 
 			removeIDs := make([]int64, len(remove))
 			anyIDs := make([]any, len(remove))
+			var totalSize int64
 			for i, m := range remove {
 				removeIDs[i] = m.ID
 				anyIDs[i] = m.ID
 				mb.Sub(m.MailboxCounts())
+				totalSize += m.Size
 				// Update "remove", because RetrainMessage below will save the message.
 				remove[i].Expunged = true
 				remove[i].ModSeq = modseq
@@ -2947,6 +2956,9 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (remove []store.M
 			err = tx.Update(&mb)
 			xcheckf(err, "updating mailbox counts")
 
+			err = c.account.AddMessageSize(c.log, tx, -totalSize)
+			xcheckf(err, "updating disk usage")
+
 			// Mark expunged messages as not needing training, then retrain them, so if they
 			// were trained, they get untrained.
 			for i := range remove {
@@ -3208,6 +3220,20 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
 				xserverErrorf("uid and message mismatch")
 			}
 
+			// See if quota allows copy.
+			var totalSize int64
+			for _, m := range xmsgs {
+				totalSize += m.Size
+			}
+			if ok, maxSize, err := c.account.CanAddMessageSize(tx, totalSize); err != nil {
+				xcheckf(err, "checking quota")
+			} else if !ok {
+				// ../rfc/9051:5155
+				xusercodeErrorf("OVERQUOTA", "account over maximum total message size %d", maxSize)
+			}
+			err = c.account.AddMessageSize(c.log, tx, totalSize)
+			xcheckf(err, "updating disk usage")
+
 			msgs := map[store.UID]store.Message{}
 			for _, m := range xmsgs {
 				msgs[m.UID] = m

imapserver/server_test.go

@@ -327,14 +327,14 @@ func xparseNumSet(s string) imapclient.NumSet {
 var connCounter int64
 
 func start(t *testing.T) *testconn {
-	return startArgs(t, true, false, true)
+	return startArgs(t, true, false, true, true, "mjl")
 }
 
 func startNoSwitchboard(t *testing.T) *testconn {
-	return startArgs(t, false, false, true)
+	return startArgs(t, false, false, true, false, "mjl")
 }
 
-func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn {
+func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS, setPassword bool, accname string) *testconn {
 	limitersInit() // Reset rate limiters.
 
 	if first {
@@ -343,9 +343,9 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
 	mox.Context = ctxbg
 	mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf")
 	mox.MustLoadConfig(true, false)
-	acc, err := store.OpenAccount(pkglog, "mjl")
+	acc, err := store.OpenAccount(pkglog, accname)
 	tcheck(t, err, "open account")
-	if first {
+	if setPassword {
 		err = acc.SetPassword(pkglog, "testtest")
 		tcheck(t, err, "set password")
 	}

imapserver/starttls_test.go

@@ -13,11 +13,11 @@ func TestStarttls(t *testing.T) {
 	tc.client.Login("[email protected]", "testtest")
 	tc.close()
 
-	tc = startArgs(t, true, true, false)
+	tc = startArgs(t, true, true, false, true, "mjl")
 	tc.transactf("bad", "starttls") // TLS already active.
 	tc.close()
 
-	tc = startArgs(t, true, false, false)
+	tc = startArgs(t, true, false, false, true, "mjl")
 	tc.transactf("no", `login "[email protected]" "testtest"`)
 	tc.xcode("PRIVACYREQUIRED")
 	tc.transactf("no", "authenticate PLAIN %s", base64.StdEncoding.EncodeToString([]byte("\u0000[email protected]\u0000testtest")))

import.go

@@ -282,7 +282,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		const sync = false
 		const notrain = true
 		const nothreads = true
-		err := a.DeliverMessage(ctl.log, tx, m, mf, sync, notrain, nothreads)
+		const updateDiskUsage = false
+		err := a.DeliverMessage(ctl.log, tx, m, mf, sync, notrain, nothreads, updateDiskUsage)
 		ctl.xcheck(err, "delivering message")
 		deliveredIDs = append(deliveredIDs, m.ID)
 		ctl.log.Debug("delivered message", slog.Int64("id", m.ID))
@@ -313,9 +314,20 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 
 		conf, _ := a.Conf()
 
+		maxSize := a.QuotaMessageSize()
+		var addSize int64
+		du := store.DiskUsage{ID: 1}
+		err = tx.Get(&du)
+		ctl.xcheck(err, "get disk usage")
+
 		process := func(m *store.Message, msgf *os.File, origPath string) {
 			defer store.CloseRemoveTempFile(ctl.log, msgf, "message to import")
 
+			addSize += m.Size
+			if maxSize > 0 && du.MessageSize+addSize > maxSize {
+				ctl.xcheck(fmt.Errorf("account over maximum total message size %d", maxSize), "checking quota")
+			}
+
 			for _, kw := range m.Keywords {
 				mailboxKeywords[kw] = true
 			}
@@ -407,6 +419,9 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		ctl.xcheck(err, "updating message counts and keywords in mailbox")
 		changes = append(changes, mb.ChangeCounts())
 
+		err = a.AddMessageSize(ctl.log, tx, addSize)
+		xcheckf(err, "updating total message size")
+
 		err = tx.Commit()
 		ctl.xcheck(err, "commit")
 		tx = nil