feat: add background mail polling option for all workers

Check for new mail (recent, unseen, exists counts) with an external
command, or for imap with the STATUS command, at start or on
reconnection and every X time duration

IMAP:
The selected folder is skipped, per specification. Additional config
options are included for including/excluding folders explicitly.

Maildir/Notmuch:
An external command will be run in the background to check for new mail.
An optional timeout can be used with maildir/notmuch. Default is 10s

New account options:
check-mail
check-mail-cmd (maildir/notmuch only)
check-mail-timeout (maildir/notmuch only), default 10s
check-mail-include (IMAP only)
check-mail-exclude (IMAP only)

If unset, or set less than or equal to 0, check-mail will be ignored

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
Tested-by: Moritz Poldrack <moritz@poldrack.dev>
Acked-by: Robin Jarry <robin@jarry.cc>

14 files changed, 267 insertions, 42 deletions

diff --git a/config/config.go b/config/config.go
index 80f7f751..f87649c3 100644
--- a/config/config.go
+++ b/config/config.go
@@ -105,6 +105,13 @@ type AccountConfig struct {
 	EnableFoldersSort bool     `ini:"enable-folders-sort"`
 	FoldersSort       []string `ini:"folders-sort" delim:","`
 
+	// CheckMail
+	CheckMail        time.Duration `ini:"check-mail"`
+	CheckMailCmd     string        `ini:"check-mail-cmd"`
+	CheckMailTimeout time.Duration `ini:"check-mail-timeout"`
+	CheckMailInclude []string      `ini:"check-mail-include"`
+	CheckMailExclude []string      `ini:"check-mail-exclude"`
+
 	// PGP Config
 	PgpKeyId                string `ini:"pgp-key-id"`
 	PgpAutoSign             bool   `ini:"pgp-auto-sign"`
@@ -224,6 +231,7 @@ func loadAccountConfig(path string) ([]AccountConfig, error) {
 			Name:              _sec,
 			Params:            make(map[string]string),
 			EnableFoldersSort: true,
+			CheckMailTimeout:  10 * time.Second,
 		}
 		if err = sec.MapTo(&account); err != nil {
 			return nil, err
diff --git a/doc/aerc-config.5.scd b/doc/aerc-config.5.scd
index 79025508..b7fba82d 100644
--- a/doc/aerc-config.5.scd
+++ b/doc/aerc-config.5.scd
@@ -525,6 +525,20 @@ Note that many of these configuration options are written for you, such as
 
 	Default: Archive
 
+*check-mail*
+	Specifies an interval to check for new mail. Mail will be checked at
+	startup, and every interval. IMAP accounts will check for mail in all
+	unselected folders, and the selected folder will continue to receive PUSH
+	mail notifications. Maildir/Notmuch folders must use *check-mail-cmd* in
+	conjunction with this option. See *aerc-maildir* and *aerc-notmuch* for
+	more information.
+
+	Setting this option to 0 will disable check-mail
+
+	Example: 5m
+
+	Default: 0
+
 *copy-to*
 	Specifies a folder to copy sent mails to, usually "Sent".
 
diff --git a/doc/aerc-imap.5.scd b/doc/aerc-imap.5.scd
index 5a20749a..99640b64 100644
--- a/doc/aerc-imap.5.scd
+++ b/doc/aerc-imap.5.scd
@@ -96,6 +96,21 @@ available:
 	This option is only supported on linux. On other platforms, it will be
 	ignored.
 
+*check-mail-include*
+	Specifies the comma separated list of folders to include when checking for
+	new mail with *check-mail*. Names prefixed with ~ are interpreted as regular
+	expressions.
+
+	Default: all folders
+
+*check-mail-exclude*
+	Specifies the comma separated list of folders to exclude when checking for
+	new mail with *check-mail*. Names prefixed with ~ are interpreted as regular
+	expressions.
+	Note that this overrides anything from *check-mail-include*.
+
+	Default: no folders
+
 # SEE ALSO
 
 *aerc*(1) *aerc-config*(5)
diff --git a/doc/aerc-maildir.5.scd b/doc/aerc-maildir.5.scd
index 7f6e7ccd..80bc0931 100644
--- a/doc/aerc-maildir.5.scd
+++ b/doc/aerc-maildir.5.scd
@@ -15,6 +15,21 @@ must be added manually to the *aerc-config*(5) file.
 
 The following maildir-specific options are available:
 
+*check-mail-cmd*
+	Command to run in conjunction with *check-mail* option.
+
+	Example:
+		mbsync -a
+
+	Default: none
+
+*check-mail-timeout*
+	Timeout for the *check-mail-cmd*. The command will be stopped if it does
+	not complete in this interval and an error will be displayed. Increase from
+	the default if repeated errors occur
+
+	Default: 10s
+
 *source*
 	maildir://path
 
diff --git a/doc/aerc-notmuch.5.scd b/doc/aerc-notmuch.5.scd
index a411c51f..44b6b9d2 100644
--- a/doc/aerc-notmuch.5.scd
+++ b/doc/aerc-notmuch.5.scd
@@ -20,6 +20,21 @@ must be added manually.
 In accounts.conf (see *aerc-config*(5)), the following notmuch-specific
 options are available:
 
+*check-mail-cmd*
+	Command to run in conjunction with *check-mail* option.
+
+	Example:
+		mbsync -a
+
+	Default: none
+
+*check-mail-timeout*
+	Timeout for the *check-mail-cmd*. The command will be stopped if it does
+	not complete in this interval and an error will be displayed. Increase from
+	the default if repeated errors occur
+
+	Default: 10s
+
 *source*
 	notmuch://path
 
diff --git a/lib/msgstore.go b/lib/msgstore.go
index dc18137f..6774f59f 100644
--- a/lib/msgstore.go
+++ b/lib/msgstore.go
@@ -193,7 +193,9 @@ func (store *MessageStore) Update(msg types.WorkerMessage) {
 	switch msg := msg.(type) {
 	case *types.DirectoryInfo:
 		store.DirInfo = *msg.Info
-		store.Sort(store.sortCriteria, nil)
+		if !msg.SkipSort {
+			store.Sort(store.sortCriteria, nil)
+		}
 		update = true
 	case *types.DirectoryContents:
 		newMap := make(map[uint32]*models.MessageInfo)
diff --git a/lib/statusline/state.go b/lib/statusline/state.go
index 3fecd0fe..54746fbd 100644
--- a/lib/statusline/state.go
+++ b/lib/statusline/state.go
@@ -64,9 +64,13 @@ func (s *State) SetWidth(w int) bool {
 	return changeState
 }
 
+func (s *State) Connected() bool {
+	return s.acct.Connected
+}
+
 type SetStateFunc func(s *State, folder string)
 
-func Connected(state bool) SetStateFunc {
+func SetConnected(state bool) SetStateFunc {
 	return func(s *State, folder string) {
 		s.acct.ConnActivity = ""
 		s.acct.Connected = state
diff --git a/widgets/account.go b/widgets/account.go
index b34396bd..e913cb7e 100644
--- a/widgets/account.go
+++ b/widgets/account.go
@@ -33,6 +33,7 @@ type AccountView struct {
 	msglist *MessageList
 	worker  *types.Worker
 	state   *statusline.State
+	newConn bool // True if this is a first run after a new connection/reconnection
 }
 
 func (acct *AccountView) UiConfig() config.UIConfig {
@@ -100,6 +101,9 @@ func NewAccountView(aerc *Aerc, conf *config.AercConfig, acct *config.AccountCon
 	worker.PostAction(&types.Configure{Config: acct}, nil)
 	worker.PostAction(&types.Connect{}, nil)
 	view.SetStatus(statusline.ConnectionActivity("Connecting..."))
+	if acct.CheckMail.Minutes() > 0 {
+		view.CheckMailTimer(acct.CheckMail)
+	}
 
 	return view, nil
 }
@@ -258,13 +262,14 @@ func (acct *AccountView) onMessage(msg types.WorkerMessage) {
 				}
 				acct.msglist.SetInitDone()
 				acct.logger.Println("Connected.")
-				acct.SetStatus(statusline.Connected(true))
+				acct.SetStatus(statusline.SetConnected(true))
+				acct.newConn = true
 			})
 		case *types.Disconnect:
 			acct.dirlist.UpdateList(nil)
 			acct.msglist.SetStore(nil)
 			acct.logger.Println("Disconnected.")
-			acct.SetStatus(statusline.Connected(false))
+			acct.SetStatus(statusline.SetConnected(false))
 		case *types.OpenDirectory:
 			if store, ok := acct.dirlist.SelectedMsgStore(); ok {
 				// If we've opened this dir before, we can re-render it from
@@ -279,6 +284,11 @@ func (acct *AccountView) onMessage(msg types.WorkerMessage) {
 			acct.dirlist.UpdateList(nil)
 		case *types.RemoveDirectory:
 			acct.dirlist.UpdateList(nil)
+		case *types.FetchMessageHeaders:
+			if acct.newConn && acct.AccountConfig().CheckMail.Minutes() > 0 {
+				acct.newConn = false
+				acct.CheckMail()
+			}
 		}
 	case *types.DirectoryInfo:
 		if store, ok := acct.dirlist.MsgStore(msg.Info.Name); ok {
@@ -327,7 +337,7 @@ func (acct *AccountView) onMessage(msg types.WorkerMessage) {
 		acct.labels = msg.Labels
 	case *types.ConnError:
 		acct.logger.Printf("connection error: [%s] %v", acct.acct.Name, msg.Error)
-		acct.SetStatus(statusline.Connected(false))
+		acct.SetStatus(statusline.SetConnected(false))
 		acct.PushError(msg.Error)
 		acct.msglist.SetStore(nil)
 		acct.worker.PostAction(&types.Reconnect{}, nil)
@@ -349,3 +359,33 @@ func (acct *AccountView) GetSortCriteria() []*types.SortCriterion {
 	}
 	return criteria
 }
+
+func (acct *AccountView) CheckMail() {
+	// Exclude selected mailbox, per IMAP specification
+	exclude := append(acct.AccountConfig().CheckMailExclude, acct.dirlist.Selected())
+	dirs := acct.dirlist.List()
+	dirs = acct.dirlist.FilterDirs(dirs, acct.AccountConfig().CheckMailInclude, false)
+	dirs = acct.dirlist.FilterDirs(dirs, exclude, true)
+	acct.logger.Printf("Checking for new mail on account %s", acct.Name())
+	acct.SetStatus(statusline.ConnectionActivity("Checking for new mail..."))
+	msg := &types.CheckMail{
+		Directories: dirs,
+		Command:     acct.acct.CheckMailCmd,
+		Timeout:     acct.acct.CheckMailTimeout,
+	}
+	acct.worker.PostAction(msg, func(_ types.WorkerMessage) {
+		acct.SetStatus(statusline.ConnectionActivity(""))
+	})
+}
+
+func (acct *AccountView) CheckMailTimer(d time.Duration) {
+	ticker := time.NewTicker(d)
+	go func() {
+		for range ticker.C {
+			if !acct.state.Connected() {
+				continue
+			}
+			acct.CheckMail()
+		}
+	}()
+}
diff --git a/widgets/dirlist.go b/widgets/dirlist.go
index 412ed06c..ca0f6c19 100644
--- a/widgets/dirlist.go
+++ b/widgets/dirlist.go
@@ -40,6 +40,8 @@ type DirectoryLister interface {
 	SelectedMsgStore() (*lib.MessageStore, bool)
 	MsgStore(string) (*lib.MessageStore, bool)
 	SetMsgStore(string, *lib.MessageStore)
+
+	FilterDirs([]string, []string, bool) []string
 }
 
 type DirectoryList struct {
@@ -441,38 +443,41 @@ func (dirlist *DirectoryList) sortDirsByFoldersSortConfig() {
 // dirstore, based on AccountConfig.Folders (inclusion) and
 // AccountConfig.FoldersExclude (exclusion), in that order.
 func (dirlist *DirectoryList) filterDirsByFoldersConfig() {
-	filterDirs := func(orig, filters []string, exclude bool) []string {
-		if len(filters) == 0 {
-			return orig
-		}
-		var dest []string
-		for _, folder := range orig {
-			// When excluding, include things by default, and vice-versa
-			include := exclude
-			for _, f := range filters {
-				if folderMatches(folder, f) {
-					// If matched an exclusion, don't include
-					// If matched an inclusion, do include
-					include = !exclude
-					break
-				}
-			}
-			if include {
-				dest = append(dest, folder)
-			}
-		}
-		return dest
-	}
-
 	dirlist.dirs = dirlist.store.List()
 
 	// 'folders' (if available) is used to make the initial list and
 	// 'folders-exclude' removes from that list.
 	configFolders := dirlist.acctConf.Folders
-	dirlist.dirs = filterDirs(dirlist.dirs, configFolders, false)
+	dirlist.dirs = dirlist.FilterDirs(dirlist.dirs, configFolders, false)
 
 	configFoldersExclude := dirlist.acctConf.FoldersExclude
-	dirlist.dirs = filterDirs(dirlist.dirs, configFoldersExclude, true)
+	dirlist.dirs = dirlist.FilterDirs(dirlist.dirs, configFoldersExclude, true)
+}
+
+// FilterDirs filters directories by the supplied filter. If exclude is false,
+// the filter will only include directories from orig which exist in filters.
+// If exclude is true, the directories in filters are removed from orig
+func (dirlist *DirectoryList) FilterDirs(orig, filters []string, exclude bool) []string {
+	if len(filters) == 0 {
+		return orig
+	}
+	var dest []string
+	for _, folder := range orig {
+		// When excluding, include things by default, and vice-versa
+		include := exclude
+		for _, f := range filters {
+			if folderMatches(folder, f) {
+				// If matched an exclusion, don't include
+				// If matched an inclusion, do include
+				include = !exclude
+				break
+			}
+		}
+		if include {
+			dest = append(dest, folder)
+		}
+	}
+	return dest
 }
 
 func (dirlist *DirectoryList) SelectedMsgStore() (*lib.MessageStore, bool) {
diff --git a/worker/imap/checkmail.go b/worker/imap/checkmail.go
new file mode 100644
index 00000000..d9dcfd33
--- /dev/null
+++ b/worker/imap/checkmail.go
@@ -0,0 +1,40 @@
+package imap
+
+import (
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-imap"
+)
+
+func (w *IMAPWorker) handleCheckMailMessage(msg *types.CheckMail) {
+	items := []imap.StatusItem{
+		imap.StatusMessages,
+		imap.StatusRecent,
+		imap.StatusUnseen,
+	}
+	for _, dir := range msg.Directories {
+		w.worker.Logger.Printf("Getting status of directory %s", dir)
+		status, err := w.client.Status(dir, items)
+		if err != nil {
+			w.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+		} else {
+			w.worker.PostMessage(&types.DirectoryInfo{
+				Info: &models.DirectoryInfo{
+					Flags:          status.Flags,
+					Name:           status.Name,
+					ReadOnly:       status.ReadOnly,
+					AccurateCounts: true,
+
+					Exists: int(status.Messages),
+					Recent: int(status.Recent),
+					Unseen: int(status.Unseen),
+				},
+				SkipSort: true,
+			}, nil)
+		}
+	}
+	w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+}
diff --git a/worker/imap/worker.go b/worker/imap/worker.go
index eabaae02..da0716e1 100644
--- a/worker/imap/worker.go
+++ b/worker/imap/worker.go
@@ -190,6 +190,8 @@ func (w *IMAPWorker) handleMessage(msg types.WorkerMessage) error {
 		w.handleAppendMessage(msg)
 	case *types.SearchDirectory:
 		w.handleSearchDirectory(msg)
+	case *types.CheckMail:
+		w.handleCheckMailMessage(msg)
 	default:
 		reterr = errUnsupported
 	}
diff --git a/worker/maildir/worker.go b/worker/maildir/worker.go
index 4a2a19b8..d3de8447 100644
--- a/worker/maildir/worker.go
+++ b/worker/maildir/worker.go
@@ -2,12 +2,14 @@ package maildir
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -60,17 +62,25 @@ func (w *Worker) Run() {
 
 func (w *Worker) handleAction(action types.WorkerMessage) {
 	msg := w.worker.ProcessAction(action)
-	if err := w.handleMessage(msg); err == errUnsupported {
-		w.worker.PostMessage(&types.Unsupported{
-			Message: types.RespondTo(msg),
-		}, nil)
-	} else if err != nil {
-		w.worker.PostMessage(&types.Error{
-			Message: types.RespondTo(msg),
-			Error:   err,
-		}, nil)
-	} else {
-		w.done(msg)
+	switch msg := msg.(type) {
+	// Explicitly handle all asynchronous actions. Async actions are
+	// responsible for posting their own Done message
+	case *types.CheckMail:
+		go w.handleCheckMail(msg)
+	default:
+		// Default handling, will be performed synchronously
+		if err := w.handleMessage(msg); err == errUnsupported {
+			w.worker.PostMessage(&types.Unsupported{
+				Message: types.RespondTo(msg),
+			}, nil)
+		} else if err != nil {
+			w.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+		} else {
+			w.done(msg)
+		}
 	}
 }
 
@@ -672,3 +682,24 @@ func (w *Worker) msgInfoFromUid(uid uint32) (*models.MessageInfo, error) {
 	}
 	return info, nil
 }
+
+func (w *Worker) handleCheckMail(msg *types.CheckMail) {
+	ctx, cancel := context.WithTimeout(context.Background(), msg.Timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "sh", "-c", msg.Command)
+	ch := make(chan error)
+	go func() {
+		err := cmd.Run()
+		ch <- err
+	}()
+	select {
+	case <-ctx.Done():
+		w.err(msg, fmt.Errorf("checkmail: timed out"))
+	case err := <-ch:
+		if err != nil {
+			w.err(msg, fmt.Errorf("checkmail: error running command: %v", err))
+		} else {
+			w.done(msg)
+		}
+	}
+}
diff --git a/worker/notmuch/worker.go b/worker/notmuch/worker.go
index 4091ea43..c1426f28 100644
--- a/worker/notmuch/worker.go
+++ b/worker/notmuch/worker.go
@@ -6,10 +6,12 @@ package notmuch
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -128,6 +130,9 @@ func (w *worker) handleMessage(msg types.WorkerMessage) error {
 		return w.handleSearchDirectory(msg)
 	case *types.ModifyLabels:
 		return w.handleModifyLabels(msg)
+	case *types.CheckMail:
+		go w.handleCheckMail(msg)
+		return nil
 
 		// not implemented, they are generally not used
 		// in a notmuch based workflow
@@ -616,3 +621,24 @@ func (w *worker) sort(uids []uint32,
 	}
 	return sortedUids, nil
 }
+
+func (w *worker) handleCheckMail(msg *types.CheckMail) {
+	ctx, cancel := context.WithTimeout(context.Background(), msg.Timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "sh", "-c", msg.Command)
+	ch := make(chan error)
+	go func() {
+		err := cmd.Run()
+		ch <- err
+	}()
+	select {
+	case <-ctx.Done():
+		w.err(msg, fmt.Errorf("checkmail: timed out"))
+	case err := <-ch:
+		if err != nil {
+			w.err(msg, fmt.Errorf("checkmail: error running command: %v", err))
+		} else {
+			w.done(msg)
+		}
+	}
+}
diff --git a/worker/types/messages.go b/worker/types/messages.go
index d8f1f568..5cd3768c 100644
--- a/worker/types/messages.go
+++ b/worker/types/messages.go
@@ -167,6 +167,13 @@ type AppendMessage struct {
 	Length      int
 }
 
+type CheckMail struct {
+	Message
+	Directories []string
+	Command     string
+	Timeout     time.Duration
+}
+
 // Messages
 
 type Directory struct {
@@ -176,7 +183,8 @@ type Directory struct {
 
 type DirectoryInfo struct {
 	Message
-	Info *models.DirectoryInfo
+	Info     *models.DirectoryInfo
+	SkipSort bool
 }
 
 type DirectoryContents struct {