implement outgoing dmarc aggregate reporting

in smtpserver, we store dmarc evaluations (under the right conditions).
in dmarcdb, we periodically (hourly) send dmarc reports if there are
evaluations. for failed deliveries, we deliver the dsn quietly to a submailbox
of the postmaster mailbox.

this is on by default, but can be disabled in mox.conf.

.gitignore

@@ -5,6 +5,7 @@
 /testdata/check/
 /testdata/ctl/data/
 /testdata/ctl/dkim/
+/testdata/dmarcdb/data/
 /testdata/empty/
 /testdata/exportmaildir/
 /testdata/exportmbox/

README.md

@@ -8,7 +8,8 @@ See Quickstart below to get started.
 - SMTP (with extensions) for receiving, submitting and delivering email.
 - IMAP4 (with extensions) for giving email clients access to email.
 - Webmail for reading/sending email from the browser.
-- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC reports.
+- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC aggregate
+  reports.
 - Reputation tracking, learning (per user) host-, domain- and
   sender address-based reputation from (Non-)Junk email classification.
 - Bayesian spam filtering that learns (per user) from (Non-)Junk email.
@@ -113,7 +114,7 @@ https://nlnet.nl/project/Mox/.
 
 ## Roadmap
 
-- Sending DMARC and TLS reports (currently only receiving)
+- Sending TLS reports (currently only receiving)
 - Authentication other than HTTP-basic for webmail/webadmin/webaccount
 - Per-domain webmail and IMAP/SMTP host name (and TLS cert) and client settings
 - Make mox Go packages more easily reusable, each pulling in fewer (internal)

backup.go

@@ -279,7 +279,8 @@ func backupctl(ctx context.Context, ctl *ctl) {
 	if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
 		xerrx("writing moxversion", err)
 	}
-	backupDB(dmarcdb.DB, "dmarcrpt.db")
+	backupDB(dmarcdb.ReportsDB, "dmarcrpt.db")
+	backupDB(dmarcdb.EvalDB, "dmarceval.db")
 	backupDB(mtastsdb.DB, "mtasts.db")
 	backupDB(tlsrptdb.DB, "tlsrpt.db")
 	backupFile("receivedid.key")
@@ -529,7 +530,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
 		}
 
 		switch p {
-		case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
+		case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
 			// Already handled.
 			return nil
 		case "lastknownversion": // Optional file, not yet handled.

config/config.go

@@ -57,9 +57,10 @@ type Static struct {
 		Account string
 		Mailbox string `sconf-doc:"E.g. Postmaster or Inbox."`
 	} `sconf-doc:"Destination for emails delivered to postmaster addresses: a plain 'postmaster' without domain, 'postmaster@<hostname>' (also for each listener with SMTP enabled), and as fallback for each domain without explicitly configured postmaster destination."`
-	InitialMailboxes InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
-	DefaultMailboxes []string             `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
-	Transports       map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
+	InitialMailboxes       InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
+	DefaultMailboxes       []string             `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
+	Transports             map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
+	NoOutgoingDMARCReports bool                 `sconf:"optional" sconf-doc:"Do not send DMARC reports (aggregate only). By default, aggregate reports on DMARC evaluations are sent to domains if their DMARC policy requests them. Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 hours, rounded up so a whole number of intervals cover 24 hours, aligned at whole days in UTC."`
 
 	// All IPs that were explicitly listen on for external SMTP. Only set when there
 	// are no unspecified external SMTP listeners and there is at most one for IPv4 and

config/doc.go

@@ -549,6 +549,13 @@ describe-static" and "mox config describe-domains":
 				# typically the hostname of the host in the Address field.
 				RemoteHostname:
 
+	# Do not send DMARC reports (aggregate only). By default, aggregate reports on
+	# DMARC evaluations are sent to domains if their DMARC policy requests them.
+	# Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24
+	# hours, rounded up so a whole number of intervals cover 24 hours, aligned at
+	# whole days in UTC. (optional)
+	NoOutgoingDMARCReports: false
+
 # domains.conf
 
 	# NOTE: This config file is in 'sconf' format. Indent with tabs. Comments must be

dkim/dkim.go

@@ -382,14 +382,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
 
 		h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig)
 		if err != nil {
-			results = append(results, Result{StatusPermerror, nil, nil, false, err})
+			results = append(results, Result{StatusPermerror, sig, nil, false, err})
 			continue
 		}
 
 		// ../rfc/6376:2560
 		if err := policy(sig); err != nil {
 			err := fmt.Errorf("%w: %s", ErrPolicy, err)
-			results = append(results, Result{StatusPolicy, nil, nil, false, err})
+			results = append(results, Result{StatusPolicy, sig, nil, false, err})
 			continue
 		}
 

dmarc/dmarc.go

@@ -75,7 +75,9 @@ type Result struct {
 	Reject bool
 	// Result of DMARC validation. A message can fail validation, but still
 	// not be rejected, e.g. if the policy is "none".
-	Status Status
+	Status          Status
+	AlignedSPFPass  bool
+	AlignedDKIMPass bool
 	// Domain with the DMARC DNS record. May be the organizational domain instead of
 	// the domain in the From-header.
 	Domain dns.Domain
@@ -142,7 +144,7 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain)
 			return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
 		}
 		if record != nil {
-			// ../ ../rfc/7489:1388
+			// ../rfc/7489:1388
 			return StatusNone, nil, "", result.Authentic, ErrMultipleRecords
 		}
 		text = txt
@@ -152,14 +154,15 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain)
 	return StatusNone, record, text, result.Authentic, rerr
 }
 
-func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, *Record, string, bool, error) {
+func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, []*Record, []string, bool, error) {
+	// ../rfc/7489:1566
 	name := dmarcDomain.ASCII + "._report._dmarc." + extDestDomain.ASCII + "."
 	txts, result, err := dns.WithPackage(resolver, "dmarc").LookupTXT(ctx, name)
 	if err != nil && !dns.IsNotFound(err) {
-		return StatusTemperror, nil, "", result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err)
+		return StatusTemperror, nil, nil, result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err)
 	}
-	var record *Record
-	var text string
+	var records []*Record
+	var texts []string
 	var rerr error = ErrNoRecord
 	for _, txt := range txts {
 		r, isdmarc, err := ParseRecordNoRequired(txt)
@@ -171,44 +174,44 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain
 			r, isdmarc, err = &xr, true, nil
 		}
 		if !isdmarc {
-			// ../rfc/7489:1374
+			// ../rfc/7489:1586
 			continue
-		} else if err != nil {
-			return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
 		}
-		if record != nil {
-			// ../ ../rfc/7489:1388
-			return StatusNone, nil, "", result.Authentic, ErrMultipleRecords
+		texts = append(texts, txt)
+		records = append(records, r)
+		if err != nil {
+			return StatusPermerror, records, texts, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
 		}
-		text = txt
-		record = r
+		// Multiple records are allowed for the _report record, unlike for policies. ../rfc/7489:1593
 		rerr = nil
 	}
-	return StatusNone, record, text, result.Authentic, rerr
+	return StatusNone, records, texts, result.Authentic, rerr
 }
 
 // LookupExternalReportsAccepted returns whether the extDestDomain has opted in
 // to receiving dmarc reports for dmarcDomain (where the dmarc record was found),
 // through a "._report._dmarc." DNS TXT DMARC record.
 //
-// Callers should look at status for interpretation, not err, because err will
-// be set to ErrNoRecord when the DNS TXT record isn't present, which means the
-// extDestDomain does not opt in (not a failure condition).
+// accepts is true if the external domain has opted in.
+// If a temporary error occurred, the returned status is StatusTemperror, and a
+// later retry may give an authoritative result.
+// The returned error is ErrNoRecord if no opt-in DNS record exists, which is
+// not a failure condition.
 //
 // The normally invalid "v=DMARC1" record is accepted since it is used as
 // example in RFC 7489.
 //
 // authentic indicates if the DNS results were DNSSEC-verified.
-func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, record *Record, txt string, authentic bool, rerr error) {
+func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) {
 	log := xlog.WithContext(ctx)
 	start := time.Now()
 	defer func() {
-		log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("record", record), mlog.Field("duration", time.Since(start)))
+		log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start)))
 	}()
 
-	status, record, txt, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
+	status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
 	accepts = rerr == nil
-	return accepts, status, record, txt, authentic, rerr
+	return accepts, status, records, txts, authentic, rerr
 }
 
 // Verify evaluates the DMARC policy for the domain in the From-header of a
@@ -241,7 +244,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 
 	status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from)
 	if record == nil {
-		return false, Result{false, status, recordDomain, record, authentic, err}
+		return false, Result{false, status, false, false, recordDomain, record, authentic, err}
 	}
 	result.Domain = recordDomain
 	result.Record = record
@@ -251,8 +254,8 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 	// See ../rfc/7489:1432
 	useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand.Intn(100) < record.Percentage
 
-	// We reject treat "quarantine" and "reject" the same. Thus, we also don't
-	// "downgrade" from reject to quarantine if this message was sampled out.
+	// We treat "quarantine" and "reject" the same. Thus, we also don't "downgrade"
+	// from reject to quarantine if this message was sampled out.
 	// ../rfc/7489:1446 ../rfc/7489:1024
 	if recordDomain != from && record.SubdomainPolicy != PolicyEmpty {
 		result.Reject = record.SubdomainPolicy != PolicyNone
@@ -282,9 +285,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 	// ../rfc/7489:1319
 	// ../rfc/7489:544
 	if spfResult == spf.StatusPass && spfIdentity != nil && (*spfIdentity == from || result.Record.ASPF == "r" && pubsuffix(from) == pubsuffix(*spfIdentity)) {
-		result.Reject = false
-		result.Status = StatusPass
-		return
+		result.AlignedSPFPass = true
 	}
 
 	for _, dkimResult := range dkimResults {
@@ -296,10 +297,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 		// ../rfc/7489:511
 		if dkimResult.Status == dkim.StatusPass && dkimResult.Sig != nil && (dkimResult.Sig.Domain == from || result.Record.ADKIM == "r" && pubsuffix(from) == pubsuffix(dkimResult.Sig.Domain)) {
 			// ../rfc/7489:535
-			result.Reject = false
-			result.Status = StatusPass
-			return
+			result.AlignedDKIMPass = true
+			break
 		}
 	}
+
+	if result.AlignedSPFPass || result.AlignedDKIMPass {
+		result.Reject = false
+		result.Status = StatusPass
+	}
 	return
 }

dmarc/dmarc_test.go

@@ -84,7 +84,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) {
 	test("example.com", "simple2.example", StatusNone, true, nil)
 	test("example.com", "one.example", StatusNone, true, nil)
 	test("example.com", "absent.example", StatusNone, false, ErrNoRecord)
-	test("example.com", "multiple.example", StatusNone, false, ErrMultipleRecords)
+	test("example.com", "multiple.example", StatusNone, true, nil)
 	test("example.com", "malformed.example", StatusPermerror, false, ErrSyntax)
 	test("example.com", "temperror.example", StatusTemperror, false, ErrDNS)
 }
@@ -137,15 +137,15 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		true, Result{true, StatusFail, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Accept with spf pass.
 	test("reject.example",
 		[]dkim.Result{},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "sub.reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Accept with dkim pass.
@@ -161,7 +161,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusFail,
 		&dns.Domain{ASCII: "reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Reject due to spf and dkim "strict".
@@ -181,23 +181,23 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "sub.strict.example"},
-		true, Result{true, StatusFail, dns.Domain{ASCII: "strict.example"}, &strict, false, nil},
+		true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "strict.example"}, &strict, false, nil},
 	)
 
 	// No dmarc policy, nothing to say.
 	test("absent.example",
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
+		false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
 	)
 
 	// No dmarc policy, spf pass does nothing.
 	test("absent.example",
 		[]dkim.Result{},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "absent.example"},
-		false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
+		false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
 	)
 
 	none := DefaultRecord
@@ -207,7 +207,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "none.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "none.example"}, &none, false, nil},
+		true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "none.example"}, &none, false, nil},
 	)
 
 	// No actual reject due to pct=0.
@@ -218,7 +218,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		false, Result{true, StatusFail, dns.Domain{ASCII: "test.example"}, &testr, false, nil},
+		false, Result{true, StatusFail, false, false, dns.Domain{ASCII: "test.example"}, &testr, false, nil},
 	)
 
 	// No reject if subdomain has "none" policy.
@@ -229,15 +229,15 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusFail,
 		&dns.Domain{ASCII: "sub.subnone.example"},
-		true, Result{false, StatusFail, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil},
+		true, Result{false, StatusFail, false, false, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil},
 	)
 
 	// No reject if spf temperror and no other pass.
 	test("reject.example",
 		[]dkim.Result{},
 		spf.StatusTemperror,
 		&dns.Domain{ASCII: "mail.reject.example"},
-		true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// No reject if dkim temperror and no other pass.
@@ -253,7 +253,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusNone,
 		nil,
-		true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// No reject if spf temperror but still dkim pass.
@@ -269,7 +269,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusTemperror,
 		&dns.Domain{ASCII: "mail.reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// No reject if dkim temperror but still spf pass.
@@ -285,15 +285,15 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "mail.reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Bad DMARC record results in permerror without reject.
 	test("malformed.example",
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		false, Result{false, StatusPermerror, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax},
+		false, Result{false, StatusPermerror, false, false, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax},
 	)
 
 	// DKIM domain that is higher-level than organizational can not result in a pass. ../rfc/7489:525
@@ -309,6 +309,6 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusNone,
 		nil,
-		true, Result{true, StatusFail, dns.Domain{ASCII: "example.com"}, &reject, false, nil},
+		true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "example.com"}, &reject, false, nil},
 	)
 }