mod_smtp_greylisting: Add ability to greylist messages in SMTP.

This adds the ability to greylist incoming email, a common technique
used to defer potentially spammy messages which results in "less
spam by deferring", with the idea that many spammers do not retry
temporarily failures.

The implementation here, while following RFC 6647 to some extent,
takes a different approach from conventional approaches to greylisting,
with the primary aim of reducing the chances that legitimate (ham,
or non-spam) mail is delayed for any reason. In particular, greylisting
is contingent about meeting two conditions:

* A minimum fail count. Many spammers deviate from the SMTP standards
  in way that commit an outright protocol violation or are, at the very
  least, suspicious. We already track this and tarpit senders that
  increase the fail count, but this is also a good sign that the message
  could be spam, and greylisting might make sense.
* A minimum spam score, as reported by SpamAssassin. This is less
  conventional, since typically greylisting is performed in order to
  deter potential spam to reduce system load, as spam filtering is a
  resource-intensive process. However, as our focus is less on
  high-throughput and more on effectiveness and convenience, it can
  make sense to analyze the message for spam, and greylist the message
  if it has a higher spam score.

Collectively, conditioning greylisting on these two criteria allows
us to avoid greylisting at all for the majority of legitimate mail.
This avoids delays that are commonly a source of frustration with
greylisting.

README.rst

@@ -556,6 +556,15 @@ Configuration
    # Heavily penalize mail from domains with no SPF record
    score SPF_NONE 3.0
 
+   # No valid author signature and from-domain does not exist
+   score DKIM_ADSP_NXDOMAIN 5.0
+
+   # No valid author signature, domain signs all mail and suggests discarding the rest (DISCARD)
+   score DKIM_ADSP_DISCARD 5.0
+
+   # No valid author signature, domain signs all mail (ALL)
+   score DKIM_ADSP_ALL 5.0
+
    # Email is not in English
    score UNWANTED_LANGUAGE_BODY 3.5
 

bbs/module.c

@@ -1326,10 +1326,12 @@ static int bbs_module_exists(const char *name)
 	return bbs_file_exists(fn);
 }
 
-/*! \brief Whether a module is currently running or not */
-static int bbs_module_running(const char *name)
+int bbs_module_running(const char *name)
 {
-	struct bbs_module *mod = find_resource(name);
+	struct bbs_module *mod;
+	RWLIST_RDLOCK(&modules);
+	mod = find_resource(name);
+	RWLIST_UNLOCK(&modules);
 	return mod ? 1 : 0;
 }
 

configs/mod_mail.conf

@@ -6,7 +6,7 @@ maildir=/home/bbs/maildir ; Where users' email is stored.
                 ; WARNING: You could open the catchall mailbox up to receiving a lot of spam by enabling this!
 			    ; The specified catch all mailbox must belong to a user directly (it cannot be an alias).
 			    ; Default is none (disabled unless specified).
-				; The catch all address applies to ALL domains.
+				; The catch all address applies to ALL domains. To add a catch-all for a single domain, use an alias instead.
 quota=10000000  ; Default maximum mail quota (in bytes), allowed per mailbox. Default is 10 MB.
                 ; A per-mailbox quota override can be imposed by specifying the quota in bytes in a .quota file in a mailbox's root maildir.
 trashdays=7     ; Number of days messages can stay in Trash before being automatically permanently deleted.
@@ -93,5 +93,6 @@ trashdays=7     ; Number of days messages can stay in Trash before being automat
 ;abuse = sysop
 ;root = sysop
 
+;* = sysop ; Catch-all for all domains. Equivalent to enabling the catchall setting in [general]. If present, MUST be first. Order matters!
+;*@bbs.example.net = sysop ; Define catch-all for entire domain. Be sure to define these BEFORE any other aliases for the domain. Order matters!
 ;[email protected] = sysop
-;*@bbs.example.net = sysop

configs/mod_smtp_greylisting.conf

@@ -0,0 +1,12 @@
+; mod_smtp_greylisting - SMTP message greylisting
+; This module is active as long as this config file is present.
+; Greylisting should only be done on the mail server that receives mail directly from the Internet.
+; Spam filtering, if being done, should be done on the same server so that the spam score is available for greylisting checks.
+[general]
+; Define conditions required to evaluate messages for greylisting.
+; Messages meeting this criteria will be greylisted. You can fine tune these to control what messages get greylisted.
+; On one extreme, set both to 0 to greylist every incoming message. On the other, to greylist only the most obviously spammy messages, increase min_spamscore.
+; Since greylisting may incur delays in receiving legitimate mail, it is recommended that min_spamscore be set to at least 1, to avoid unnecessarily delaying ALL messages.
+; It is recommended that you change min_failcount only in response to observation of real traffic.
+min_spamscore = 2 ; Minimum rounded X-Spam-Score value required to consider greylisting a message. The header value is a float (e.g. 7.3) but is rounded down for comparison. Default is 2.
+min_failcount = 1 ; Minimum SMTP failure count (~protocol violations or suspicious activity) to consider greylisting a message. Default is 1.

configs/modules.conf

@@ -72,14 +72,15 @@ load = mod_sieve.so
 load = mod_smtp_client.so
 load = mod_smtp_delivery_external.so
 load = mod_smtp_delivery_local.so
+load = mod_smtp_mailing_lists.so
 load = mod_smtp_fetchmail.so
 load = mod_smtp_filter.so
 load = mod_smtp_filter_arc.so
 load = mod_smtp_filter_dkim.so
 load = mod_smtp_filter_dmarc.so
-load = mod_spamassassin.so
-load = mod_smtp_mailing_lists.so
+load = mod_smtp_greylisting.so
 load = mod_smtp_recipient_monitor.so
+load = mod_spamassassin.so
 load = mod_webmail.so
 load = net_imap.so
 load = net_nntp.so

include/module.h

@@ -131,6 +131,13 @@ int bbs_module_unload(const char *name);
  */
 int bbs_module_reload(const char *name, int try_delayed);
 
+/*!
+ * \brief Whether a specified module is running
+ * \param name Module name. File extension (.so suffix) is optional.
+ * \retval 1 if running, 0 if not running
+ */
+int bbs_module_running(const char *name);
+
 /*! \brief Autoload all modules */
 int load_modules(void);
 

include/net_smtp.h

@@ -164,6 +164,12 @@ const char *smtp_sender_ip(struct smtp_session *smtp);
 /*! \brief Get SMTP protocol used */
 const char *smtp_protname(struct smtp_session *smtp);
 
+/*!
+ * \brief Get SMTP recipient list from RCPT TO
+ * \note Do not use this from user mail filters (e.g. Sieve/MailScript)
+ */
+struct stringlist *smtp_recipients(struct smtp_session *smtp);
+
 /*! \brief Get the MAIL FROM address */
 const char *smtp_from(struct smtp_session *smtp);
 
@@ -206,9 +212,20 @@ size_t smtp_message_estimated_size(struct smtp_session *smtp);
  */
 const char *smtp_message_content_type(struct smtp_session *smtp);
 
+/*!
+ * \brief Get the Message-ID of a message, if available
+ * \param smtp
+ * \return Message-ID value
+ * \return NULL if unavailable
+ */
+const char *smtp_messageid(struct smtp_session *smtp);
+
 /*! \brief Time that message was received */
 time_t smtp_received_time(struct smtp_session *smtp);
 
+/*! \brief Score of protocol violation or spaminess severity */
+unsigned int smtp_failure_count(struct smtp_session *smtp);
+
 /*! \brief Get RFC822 message as string */
 const char *smtp_message_body(struct smtp_filter_data *f);
 
@@ -252,6 +269,7 @@ struct smtp_msg_process {
 	unsigned int userid;		/*!< User ID (outgoing only) */
 	enum smtp_direction dir;	/*!< Full direction (IN, OUT, or SUBMIT) */
 	unsigned int direction:1;	/*!< 0 = incoming, 1 = outgoing */
+	enum smtp_filter_scope scope; /*!< COMBINED (for outside a delivery handler) or INDIVIDUAL (for within a delivery handler) */
 	enum msg_process_iteration iteration; /*!< Which processing pass this is */
 	/* Outputs */
 	unsigned int bounce:1;		/*!< Whether to send a bounce. This on its own does not also implicitly drop the message, that bit must be explicitly set. */
@@ -268,7 +286,7 @@ void smtp_mproc_init(struct smtp_session *smtp, struct smtp_msg_process *mproc);
 
 /*!
  * \brief Register an SMTP processor callback to run on each message received or sent. Callback will be called 3x, once for each msg_process_order.
- * \param cb Callback that should return nonzero to stop processing further callbacks
+ * \param cb Callback that should return 1 to stop processing further callbacks and -1 to additionally terminate the SMTP transaction (after having already responded)
  */
 #define smtp_register_processor(cb) __smtp_register_processor(cb, BBS_MODULE_SELF)
 
@@ -280,9 +298,9 @@ int smtp_unregister_processor(int (*cb)(struct smtp_msg_process *mproc));
 /*!
  * \brief Run SMTP callbacks for a message (only called by net_smtp)
  * \param mproc
- * \retval 0 to continue, -1 to abort transaction immediately
+ * \retval 0 to continue (some or all callbacks were executed and none returned -1), -1 to abort transaction immediately (because a callback returned -1)
  */
-int smtp_run_callbacks(struct smtp_msg_process *mproc);
+int smtp_run_callbacks(struct smtp_msg_process *mproc, enum smtp_filter_scope scope);
 
 struct smtp_response {
 	/* Response */
@@ -298,12 +316,14 @@ struct smtp_response {
  * \param mbox Mailbox or NULL
  * \param resp
  * \param dir
+ * \param scope
  * \param recipient Recipient, with <>
  * \param datalen Message size
  * \param freedata
- * \retval 0 to continue, nonzero to return the return value
+ * \retval 0 to continue, -1 if message should be aborted and a failure response generated, 1 if message is being silently dropped
  */
-int smtp_run_delivery_callbacks(struct smtp_session *smtp, struct smtp_msg_process *mproc, struct mailbox *mbox, struct smtp_response **restrict resp, enum smtp_direction dir, const char *recipient, size_t datalen, void **freedata);
+int smtp_run_delivery_callbacks(struct smtp_session *smtp, struct smtp_msg_process *mproc, struct mailbox *mbox, struct smtp_response **restrict resp,
+	enum smtp_direction dir, enum smtp_filter_scope scope, const char *recipient, size_t datalen, void **freedata);
 
 #define smtp_abort(r, c, sub, msg) \
 	r->code = c; \

modules/mod_mail.c

@@ -458,6 +458,14 @@ static const char *resolve_alias(const char *user, const char *domain)
 	const char *retval = NULL;
 	struct alias *alias;
 
+	/* Note that we do not look for the most explicit match,
+	 * just for a match. For this reason, *@example.com
+	 * should always be defined in the config BEFORE
+	 * anything else at the domain (and * should be first, if present),
+	 * since we add using head insert, so the first things will be
+	 * last and thus if a more specific match exists, we would have
+	 * encountered it first. */
+
 	RWLIST_RDLOCK(&aliases);
 	RWLIST_TRAVERSE(&aliases, alias, entry) {
 		int user_match;

modules/mod_mailscript.c

@@ -654,6 +654,15 @@ static int mailscript(struct smtp_msg_process *mproc)
 	char filepath[256];
 	const char *mboxmaildir;
 
+	if (mproc->scope != SMTP_SCOPE_INDIVIDUAL) {
+		/* Filters are only run for individual delivery.
+		 * Even global rules should use SMTP_SCOPE_INDIVIDUAL,
+		 * since they could manipulate the mailbox in some way,
+		 * and we don't have a single mailbox if processing
+		 * a message that will get delivered to multiple recipients. */
+		return 0;
+	}
+
 	/* Calculate maildir path, if we have a mailbox */
 	if (mproc->userid) {
 		snprintf(filepath, sizeof(filepath), "%s/%d", mailbox_maildir(NULL), mproc->userid);

modules/mod_menu_handlers.c

@@ -141,6 +141,7 @@ static int __exec_handler(struct bbs_node *node, char *args, int isolated)
 		 */
 		return -1;
 	}
+
 	/* Who knows what this external program did. Prompt the user for confirmation before returning to menu, if the program exited nonzero,
 	 * or if returned almost immediately (since some executions will just print something to STDOUT),
 	 * and since the screen will be cleared after returning, this would erase that output otherwise).

modules/mod_sieve.c

@@ -581,6 +581,10 @@ static int sieve(struct smtp_msg_process *mproc)
 	char filepath[256];
 	const char *mboxmaildir;
 
+	if (mproc->scope != SMTP_SCOPE_INDIVIDUAL) {
+		return 0; /* Filters are only run for individual delivery */
+	}
+
 	if (mproc->direction != SMTP_MSG_DIRECTION_IN) {
 		return 0; /* Currently, Sieve can only be used for filtering inbound mail. If support for Sieve extension for outbound mail is added, this could change. */
 	}

modules/mod_smtp_delivery_external.c

@@ -1682,12 +1682,12 @@ static int external_delivery(struct smtp_session *smtp, struct smtp_response *re
 	 * There is no mailbox corresponding to this filter execution,
 	 * so this is purely for global before/after rules
 	 * that may want to target non-mailbox mail. */
-	res = smtp_run_delivery_callbacks(smtp, &mproc, NULL, &resp, SMTP_DIRECTION_OUT, recipient, datalen, freedata);
+	res = smtp_run_delivery_callbacks(smtp, &mproc, NULL, &resp, SMTP_DIRECTION_OUT, SMTP_SCOPE_INDIVIDUAL, recipient, datalen, freedata);
 	if (res) {
 		return res;
 	}
 	if (!resp) {
-		resp = &tmpresp; /* We already set the error, don't allow appendmsg to override it if we're not going to drop immediately */
+		resp = &tmpresp;
 	}
 	res = -1; /* Reset to -1 before continuing */
 
@@ -1996,6 +1996,7 @@ static int unload_module(void)
 	bbs_cli_unregister_multiple(cli_commands_mailq);
 	bbs_pthread_cancel_kill(queue_thread);
 	bbs_pthread_join(queue_thread, NULL);
+	/*! \todo BUGBUG Possible for queue_lock to get destroyed while still being used, need to properly wait */
 	bbs_rwlock_destroy(&queue_lock);
 	RWLIST_WRLOCK_REMOVE_ALL(&static_relays, entry, free_static_relay);
 	return res;

modules/mod_smtp_delivery_local.c

@@ -245,7 +245,7 @@ static int do_local_delivery(struct smtp_session *smtp, struct smtp_response *re
 		return -1;
 	}
 
-	res = smtp_run_delivery_callbacks(smtp, &mproc, mbox, &resp, SMTP_DIRECTION_IN, recipient, datalen, freedata);
+	res = smtp_run_delivery_callbacks(smtp, &mproc, mbox, &resp, SMTP_DIRECTION_IN, SMTP_SCOPE_INDIVIDUAL, recipient, datalen, freedata);
 	if (res) {
 		return res;
 	}