imap: add option to cache headers

Add option to cache headers for imap accounts. Cache db is located at
$XDG_CACHE_DIR/aerc/{account name}. The cache is cleaned of stale
entries when aerc is first opened.

Two new account level configuration options are introduced:
* cache-headers (Default: false)
* cache-max-age (Default: 30 days (720 hours))

The change in worker/imap/open.go is to set the selected directory. This
is required to access the UIDVALIDITY field, which is used in
combination with the message ID to form the key for use in the cache db.
The key structure is: "header.{UIDVALIDITY}.{UID}"

Where reasonable, cache does not stop aerc from running. In general, if
there is an error in the cache, aerc should continue working as usual.
Errors are either displayed to the user or logged.

All messages are stored without flags, and when retrieved have the flags
set to SEEN. This is to prevent UI flashes. A new method to
FetchMessageFlags is introduced to update flags of cached headers. This
is done asynchronously, and the user will see their messages appear and
then any flags updated. The message will initially show as SEEN, but
will update to unread. I considered updating the cache with the
last-known flag state, however it seems prudent to spare the R/W cycle
and assume that - eventually - all messages will end up read, and if it
isn't the update will occur rather quickly.

Note that leveldb puts a lock on the database, preventing multiple
instances of aerc from accessing the cache at the same time.

Much of this work is based on previous efforts by Vladimír Magyar.

Implements: https://todo.sr.ht/~rjarry/aerc/2
Thanks: Vladimír Magyar <vladimir@mgyar.me>
Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
Tested-by: inwit <inwit@sindominio.net>
Reviewed-by: Koni Marti <koni.marti@gmail.com>
Acked-by: Robin Jarry <robin@jarry.cc>

6 files changed, 248 insertions, 12 deletions

diff --git a/worker/imap/cache.go b/worker/imap/cache.go
new file mode 100644
index 00000000..ecbedd89
--- /dev/null
+++ b/worker/imap/cache.go
@@ -0,0 +1,174 @@
+package imap
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-message"
+	"github.com/emersion/go-message/mail"
+	"github.com/emersion/go-message/textproto"
+	"github.com/mitchellh/go-homedir"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type CachedHeader struct {
+	BodyStructure models.BodyStructure
+	Envelope      models.Envelope
+	InternalDate  time.Time
+	Uid           uint32
+	Header        []byte
+	Created       time.Time
+}
+
+// initCacheDb opens (or creates) the database for the cache. One database is
+// created per account
+func (w *IMAPWorker) initCacheDb(acct string) {
+	cd, err := cacheDir()
+	if err != nil {
+		w.cache = nil
+		w.worker.Logger.Panicf("cache: unable to find cache directory: %v", err)
+		return
+	}
+	p := path.Join(cd, acct)
+	db, err := leveldb.OpenFile(p, nil)
+	if err != nil {
+		w.cache = nil
+		w.worker.Logger.Printf("cache: error opening cache db: %v", err)
+		return
+	}
+	w.cache = db
+	w.worker.Logger.Printf("cache: cache db opened: %s", p)
+	if w.config.cacheMaxAge.Hours() > 0 {
+		go w.cleanCache()
+	}
+}
+
+func (w *IMAPWorker) cacheHeader(mi *models.MessageInfo) {
+	uv := fmt.Sprintf("%d", w.selected.UidValidity)
+	uid := fmt.Sprintf("%d", mi.Uid)
+	w.worker.Logger.Printf("cache: caching header for message %s.%s", uv, uid)
+	hdr := bytes.NewBuffer(nil)
+	err := textproto.WriteHeader(hdr, mi.RFC822Headers.Header.Header)
+	if err != nil {
+		w.worker.Logger.Printf("cache: error writing header %s.%s: %v", uv, uid, err)
+		return
+	}
+	h := &CachedHeader{
+		BodyStructure: *mi.BodyStructure,
+		Envelope:      *mi.Envelope,
+		InternalDate:  mi.InternalDate,
+		Uid:           mi.Uid,
+		Header:        hdr.Bytes(),
+		Created:       time.Now(),
+	}
+	data := bytes.NewBuffer(nil)
+	enc := gob.NewEncoder(data)
+	err = enc.Encode(h)
+	if err != nil {
+		w.worker.Logger.Printf("cache: error encoding message %s.%s: %v", uv, uid, err)
+		return
+	}
+	err = w.cache.Put([]byte("header."+uv+"."+uid), data.Bytes(), nil)
+	if err != nil {
+		w.worker.Logger.Printf("cache: error writing header to database for message %s.%s: %v", uv, uid, err)
+		return
+	}
+}
+
+func (w *IMAPWorker) getCachedHeaders(msg *types.FetchMessageHeaders) []uint32 {
+	w.worker.Logger.Println("Retrieving headers from cache")
+	var need, found []uint32
+	uv := fmt.Sprintf("%d", w.selected.UidValidity)
+	for _, uid := range msg.Uids {
+		u := fmt.Sprintf("%d", uid)
+		data, err := w.cache.Get([]byte("header."+uv+"."+u), nil)
+		if err != nil {
+			need = append(need, uid)
+			continue
+		}
+		ch := &CachedHeader{}
+		dec := gob.NewDecoder(bytes.NewReader(data))
+		err = dec.Decode(ch)
+		if err != nil {
+			w.worker.Logger.Printf("cache: error decoding cached header %s.%s: %v", uv, u, err)
+			need = append(need, uid)
+			continue
+		}
+		hr := bytes.NewReader(ch.Header)
+		textprotoHeader, err := textproto.ReadHeader(bufio.NewReader(hr))
+		if err != nil {
+			w.worker.Logger.Printf("cache: error reading cached header %s.%s: %v", uv, u, err)
+			need = append(need, uid)
+			continue
+		}
+
+		hdr := &mail.Header{Header: message.Header{Header: textprotoHeader}}
+		mi := &models.MessageInfo{
+			BodyStructure: &ch.BodyStructure,
+			Envelope:      &ch.Envelope,
+			Flags:         []models.Flag{models.SeenFlag}, // Always return a SEEN flag
+			Uid:           ch.Uid,
+			RFC822Headers: hdr,
+		}
+		found = append(found, uid)
+		w.worker.Logger.Printf("cache: located cached header %s.%s", uv, u)
+		w.worker.PostMessage(&types.MessageInfo{
+			Message: types.RespondTo(msg),
+			Info:    mi,
+		}, nil)
+	}
+	if len(found) > 0 {
+		w.worker.PostAction(&types.FetchMessageFlags{
+			Uids: found,
+		}, nil)
+	}
+	return need
+}
+
+func cacheDir() (string, error) {
+	dir, err := os.UserCacheDir()
+	if err != nil {
+		dir, err = homedir.Expand("~/.cache")
+		if err != nil {
+			return "", err
+		}
+	}
+	return path.Join(dir, "aerc"), nil
+}
+
+// cleanCache removes stale entries from the selected mailbox cachedb
+func (w *IMAPWorker) cleanCache() {
+	start := time.Now()
+	var scanned, removed int
+	iter := w.cache.NewIterator(nil, nil)
+	for iter.Next() {
+		data := iter.Value()
+		ch := &CachedHeader{}
+		dec := gob.NewDecoder(bytes.NewReader(data))
+		err := dec.Decode(ch)
+		if err != nil {
+			w.worker.Logger.Printf("cache: error cleaning database %d: %v", w.selected.UidValidity, err)
+			continue
+		}
+		exp := ch.Created.Add(w.config.cacheMaxAge)
+		if exp.Before(time.Now()) {
+			err = w.cache.Delete(iter.Key(), nil)
+			if err != nil {
+				w.worker.Logger.Printf("cache: error cleaning database %d: %v", w.selected.UidValidity, err)
+				continue
+			}
+			removed = removed + 1
+		}
+		scanned = scanned + 1
+	}
+	iter.Release()
+	elapsed := time.Since(start)
+	w.worker.Logger.Printf("cache: cleaned cache, removed %d of %d entries in %f seconds", removed, scanned, elapsed.Seconds())
+}
diff --git a/worker/imap/configure.go b/worker/imap/configure.go
index ddaaf93f..f5151880 100644
--- a/worker/imap/configure.go
+++ b/worker/imap/configure.go
@@ -56,6 +56,9 @@ func (w *IMAPWorker) handleConfigure(msg *types.Configure) error {
 
 	w.config.reconnect_maxwait = 30 * time.Second
 
+	w.config.cacheEnabled = false
+	w.config.cacheMaxAge = 30 * 24 * time.Hour // 30 days
+
 	for key, value := range msg.Config.Params {
 		switch key {
 		case "idle-timeout":
@@ -114,9 +117,26 @@ func (w *IMAPWorker) handleConfigure(msg *types.Configure) error {
 					value, err)
 			}
 			w.config.keepalive_interval = int(val.Seconds())
+		case "cache-headers":
+			cache, err := strconv.ParseBool(value)
+			if err != nil {
+				// Return an error here because the user tried to set header
+				// caching, and we want them to know they didn't set it right -
+				// one way or the other
+				return fmt.Errorf("invalid cache-headers value %v: %v", value, err)
+			}
+			w.config.cacheEnabled = cache
+		case "cache-max-age":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf("invalid cache-max-age value %v: %v", value, err)
+			}
+			w.config.cacheMaxAge = val
 		}
 	}
-
+	if w.config.cacheEnabled {
+		w.initCacheDb(msg.Config.Name)
+	}
 	w.idler = newIdler(w.config, w.worker)
 	w.observer = newObserver(w.config, w.worker)
 
diff --git a/worker/imap/fetch.go b/worker/imap/fetch.go
index 765039bf..e8a8251b 100644
--- a/worker/imap/fetch.go
+++ b/worker/imap/fetch.go
@@ -17,7 +17,15 @@ import (
 
 func (imapw *IMAPWorker) handleFetchMessageHeaders(
 	msg *types.FetchMessageHeaders) {
-
+	toFetch := msg.Uids
+	if imapw.config.cacheEnabled && imapw.cache != nil {
+		toFetch = imapw.getCachedHeaders(msg)
+	}
+	if len(toFetch) == 0 {
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)},
+			nil)
+		return
+	}
 	imapw.worker.Logger.Printf("Fetching message headers")
 	section := &imap.BodySectionName{
 		BodyPartName: imap.BodyPartName{
@@ -34,7 +42,7 @@ func (imapw *IMAPWorker) handleFetchMessageHeaders(
 		imap.FetchUid,
 		section.FetchItem(),
 	}
-	imapw.handleFetchMessages(msg, msg.Uids, items,
+	imapw.handleFetchMessages(msg, toFetch, items,
 		func(_msg *imap.Message) error {
 			reader := _msg.GetBody(section)
 			textprotoHeader, err := textproto.ReadHeader(bufio.NewReader(reader))
@@ -48,17 +56,21 @@ func (imapw *IMAPWorker) handleFetchMessageHeaders(
 				return nil
 			}
 			header := &mail.Header{Header: message.Header{Header: textprotoHeader}}
+			info := &models.MessageInfo{
+				BodyStructure: translateBodyStructure(_msg.BodyStructure),
+				Envelope:      translateEnvelope(_msg.Envelope),
+				Flags:         translateImapFlags(_msg.Flags),
+				InternalDate:  _msg.InternalDate,
+				RFC822Headers: header,
+				Uid:           _msg.Uid,
+			}
 			imapw.worker.PostMessage(&types.MessageInfo{
 				Message: types.RespondTo(msg),
-				Info: &models.MessageInfo{
-					BodyStructure: translateBodyStructure(_msg.BodyStructure),
-					Envelope:      translateEnvelope(_msg.Envelope),
-					Flags:         translateImapFlags(_msg.Flags),
-					InternalDate:  _msg.InternalDate,
-					RFC822Headers: header,
-					Uid:           _msg.Uid,
-				},
+				Info:    info,
 			}, nil)
+			if imapw.config.cacheEnabled && imapw.cache != nil {
+				imapw.cacheHeader(info)
+			}
 			return nil
 		})
 }
@@ -169,6 +181,24 @@ func (imapw *IMAPWorker) handleFetchFullMessages(
 		})
 }
 
+func (imapw *IMAPWorker) handleFetchMessageFlags(msg *types.FetchMessageFlags) {
+	items := []imap.FetchItem{
+		imap.FetchFlags,
+		imap.FetchUid,
+	}
+	imapw.handleFetchMessages(msg, msg.Uids, items,
+		func(_msg *imap.Message) error {
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info: &models.MessageInfo{
+					Flags: translateImapFlags(_msg.Flags),
+					Uid:   _msg.Uid,
+				},
+			}, nil)
+			return nil
+		})
+}
+
 func (imapw *IMAPWorker) handleFetchMessages(
 	msg types.WorkerMessage, uids []uint32, items []imap.FetchItem,
 	procFunc func(*imap.Message) error) {
diff --git a/worker/imap/open.go b/worker/imap/open.go
index 238f1e25..65060fe5 100644
--- a/worker/imap/open.go
+++ b/worker/imap/open.go
@@ -12,13 +12,14 @@ import (
 func (imapw *IMAPWorker) handleOpenDirectory(msg *types.OpenDirectory) {
 	imapw.worker.Logger.Printf("Opening %s", msg.Directory)
 
-	_, err := imapw.client.Select(msg.Directory, false)
+	sel, err := imapw.client.Select(msg.Directory, false)
 	if err != nil {
 		imapw.worker.PostMessage(&types.Error{
 			Message: types.RespondTo(msg),
 			Error:   err,
 		}, nil)
 	} else {
+		imapw.selected = sel
 		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
 	}
 }
diff --git a/worker/imap/worker.go b/worker/imap/worker.go
index da0716e1..e890bb8d 100644
--- a/worker/imap/worker.go
+++ b/worker/imap/worker.go
@@ -9,6 +9,7 @@ import (
 	sortthread "github.com/emersion/go-imap-sortthread"
 	"github.com/emersion/go-imap/client"
 	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb"
 
 	"git.sr.ht/~rjarry/aerc/lib"
 	"git.sr.ht/~rjarry/aerc/models"
@@ -49,6 +50,8 @@ type imapConfig struct {
 	keepalive_period   time.Duration
 	keepalive_probes   int
 	keepalive_interval int
+	cacheEnabled       bool
+	cacheMaxAge        time.Duration
 }
 
 type IMAPWorker struct {
@@ -63,6 +66,7 @@ type IMAPWorker struct {
 
 	idler    *idler
 	observer *observer
+	cache    *leveldb.DB
 }
 
 func NewIMAPWorker(worker *types.Worker) (types.Backend, error) {
@@ -178,6 +182,8 @@ func (w *IMAPWorker) handleMessage(msg types.WorkerMessage) error {
 		w.handleFetchMessageBodyPart(msg)
 	case *types.FetchFullMessages:
 		w.handleFetchFullMessages(msg)
+	case *types.FetchMessageFlags:
+		w.handleFetchMessageFlags(msg)
 	case *types.DeleteMessages:
 		w.handleDeleteMessages(msg)
 	case *types.FlagMessages:
diff --git a/worker/types/messages.go b/worker/types/messages.go
index d2d98fdf..e303ade4 100644
--- a/worker/types/messages.go
+++ b/worker/types/messages.go
@@ -133,6 +133,11 @@ type FetchMessageBodyPart struct {
 	Part []int
 }
 
+type FetchMessageFlags struct {
+	Message
+	Uids []uint32
+}
+
 type DeleteMessages struct {
 	Message
 	Uids []uint32