mbox: implement an mbox backend worker

Implement an mbox backend worker. Worker can be used for testing and
development by mocking a backend for the message store. Worker does not
modify the actual mbox file on disk; all operations are performed in
memory.

To use the mbox backend, create an mbox account in the accounts.conf
where the source uses the "mbox://" scheme, such as

source = mbox://~/mbox/

or

source = mbox://~/mbox/file.mbox

If the mbox source points to a directory, all files in this directory
with the .mbox suffix will be opened as folders.

If an outgoing smtp server is defined for the mbox account, replies can
be sent to emails that are stored in the mbox file.

Signed-off-by: Koni Marti <koni.marti@gmail.com>
Acked-by: Robin Jarry <robin@jarry.cc>

4 files changed, 692 insertions, 0 deletions

diff --git a/worker/mbox/create.go b/worker/mbox/create.go
new file mode 100644
index 00000000..7c4d9f7b
--- /dev/null
+++ b/worker/mbox/create.go
@@ -0,0 +1,60 @@
+package mboxer
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func createMailboxContainer(path string) (*mailboxContainer, error) {
+
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	mbdata := &mailboxContainer{mailboxes: make(map[string]*container)}
+
+	openMboxFile := func(path string, r io.Reader) error {
+		// read mbox file
+		messages, err := Read(r)
+		if err != nil {
+			return err
+		}
+		_, name := filepath.Split(path)
+		name = strings.TrimSuffix(name, ".mbox")
+		mbdata.mailboxes[name] = &container{filename: path, messages: messages}
+		return nil
+	}
+
+	if fileInfo.IsDir() {
+		files, err := filepath.Glob(filepath.Join(path, "*.mbox"))
+		if err != nil {
+			return nil, err
+		}
+		for _, file := range files {
+			f, err := os.Open(file)
+			if err != nil {
+				continue
+			}
+			if err := openMboxFile(file, f); err != nil {
+				return nil, err
+			}
+			f.Close()
+		}
+	} else {
+		if err := openMboxFile(path, file); err != nil {
+			return nil, err
+		}
+	}
+
+	return mbdata, nil
+}
diff --git a/worker/mbox/io.go b/worker/mbox/io.go
new file mode 100644
index 00000000..38469160
--- /dev/null
+++ b/worker/mbox/io.go
@@ -0,0 +1,50 @@
+package mboxer
+
+import (
+	"io"
+	"io/ioutil"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/lib"
+	"github.com/emersion/go-mbox"
+)
+
+func Read(r io.Reader) ([]lib.RawMessage, error) {
+	mbr := mbox.NewReader(r)
+	uid := uint32(0)
+	messages := make([]lib.RawMessage, 0)
+	for {
+		msg, err := mbr.NextMessage()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		content, err := ioutil.ReadAll(msg)
+		if err != nil {
+			return nil, err
+		}
+
+		messages = append(messages, &message{
+			uid: uid, flags: []models.Flag{models.SeenFlag}, content: content,
+		})
+
+		uid++
+	}
+	return messages, nil
+}
+
+func Write(w io.Writer, reader io.Reader, from string, date time.Time) error {
+	wc := mbox.NewWriter(w)
+	mw, err := wc.CreateMessage(from, time.Now())
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(mw, reader)
+	if err != nil {
+		return err
+	}
+	return wc.Close()
+}
diff --git a/worker/mbox/models.go b/worker/mbox/models.go
new file mode 100644
index 00000000..f97530ed
--- /dev/null
+++ b/worker/mbox/models.go
@@ -0,0 +1,203 @@
+package mboxer
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/lib"
+)
+
+type mailboxContainer struct {
+	mailboxes map[string]*container
+}
+
+func (md *mailboxContainer) Names() []string {
+	files := make([]string, 0)
+	for file := range md.mailboxes {
+		files = append(files, file)
+	}
+	return files
+}
+
+func (md *mailboxContainer) Mailbox(f string) (*container, bool) {
+	mb, ok := md.mailboxes[f]
+	return mb, ok
+}
+
+func (md *mailboxContainer) Create(file string) *container {
+	md.mailboxes[file] = &container{filename: file}
+	return md.mailboxes[file]
+}
+
+func (md *mailboxContainer) Remove(file string) error {
+	delete(md.mailboxes, file)
+	return nil
+}
+
+func (md *mailboxContainer) DirectoryInfo(file string) *models.DirectoryInfo {
+	var exists int
+	if md, ok := md.Mailbox(file); ok {
+		exists = len(md.Uids())
+	}
+	return &models.DirectoryInfo{
+		Name:           file,
+		Flags:          []string{},
+		ReadOnly:       false,
+		Exists:         exists,
+		Recent:         0,
+		Unseen:         0,
+		AccurateCounts: false,
+		Caps: &models.Capabilities{
+			Sort:   true,
+			Thread: false,
+		},
+	}
+}
+
+func (md *mailboxContainer) Copy(dest, src string, uids []uint32) error {
+	srcmbox, ok := md.Mailbox(src)
+	if !ok {
+		return fmt.Errorf("source %s not found", src)
+	}
+	destmbox, ok := md.Mailbox(dest)
+	if !ok {
+		return fmt.Errorf("destination %s not found", dest)
+	}
+	for _, uidSrc := range srcmbox.Uids() {
+		found := false
+		for _, uid := range uids {
+			if uid == uidSrc {
+				found = true
+				break
+			}
+		}
+		if found {
+			msg, err := srcmbox.Message(uidSrc)
+			if err != nil {
+				return fmt.Errorf("could not get message with uid %d from folder %s", uidSrc, src)
+			}
+			r, err := msg.NewReader()
+			if err != nil {
+				return fmt.Errorf("could not get reader for message with uid %d", uidSrc)
+			}
+			flags, err := msg.ModelFlags()
+			if err != nil {
+				return fmt.Errorf("could not get flags for message with uid %d", uidSrc)
+			}
+			destmbox.Append(r, flags)
+		}
+	}
+	md.mailboxes[dest] = destmbox
+	return nil
+}
+
+type container struct {
+	filename string
+	messages []lib.RawMessage
+}
+
+func (f *container) Uids() []uint32 {
+	uids := make([]uint32, len(f.messages))
+	for i, m := range f.messages {
+		uids[i] = m.UID()
+	}
+	return uids
+}
+
+func (f *container) Message(uid uint32) (lib.RawMessage, error) {
+	for _, m := range f.messages {
+		if uid == m.UID() {
+			return m, nil
+		}
+	}
+	return &message{}, fmt.Errorf("uid [%d] not found", uid)
+}
+
+func (f *container) Delete(uids []uint32) (deleted []uint32) {
+	newMessages := make([]lib.RawMessage, 0)
+	for _, m := range f.messages {
+		del := false
+		for _, uid := range uids {
+			if m.UID() == uid {
+				del = true
+				break
+			}
+		}
+		if del {
+			deleted = append(deleted, m.UID())
+		} else {
+			newMessages = append(newMessages, m)
+		}
+	}
+	f.messages = newMessages
+	return
+}
+
+func (f *container) newUid() (next uint32) {
+	for _, m := range f.messages {
+		if uid := m.UID(); uid > next {
+			next = uid
+		}
+	}
+	next++
+	return
+}
+
+func (f *container) Append(r io.Reader, flags []models.Flag) error {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	f.messages = append(f.messages, &message{
+		uid:     f.newUid(),
+		flags:   flags,
+		content: data,
+	})
+	return nil
+}
+
+// message implements the lib.RawMessage interface
+type message struct {
+	uid     uint32
+	flags   []models.Flag
+	content []byte
+}
+
+func (m *message) NewReader() (io.ReadCloser, error) {
+	return ioutil.NopCloser(bytes.NewReader(m.content)), nil
+}
+
+func (m *message) ModelFlags() ([]models.Flag, error) {
+	return m.flags, nil
+}
+
+func (m *message) Labels() ([]string, error) {
+	return nil, nil
+}
+
+func (m *message) UID() uint32 {
+	return m.uid
+}
+
+func (m *message) SetFlag(flag models.Flag, state bool) error {
+	flagSet := make(map[models.Flag]bool)
+	flags, err := m.ModelFlags()
+	if err != nil {
+		return err
+	}
+	for _, f := range flags {
+		flagSet[f] = true
+	}
+	flagSet[flag] = state
+	newFlags := make([]models.Flag, 0)
+	for flag, isSet := range flagSet {
+		if isSet {
+			newFlags = append(newFlags, flag)
+		}
+	}
+	m.flags = newFlags
+	return nil
+}
diff --git a/worker/mbox/worker.go b/worker/mbox/worker.go
new file mode 100644
index 00000000..c7f105b5
--- /dev/null
+++ b/worker/mbox/worker.go
@@ -0,0 +1,379 @@
+package mboxer
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/handlers"
+	"git.sr.ht/~rjarry/aerc/worker/lib"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	gomessage "github.com/emersion/go-message"
+)
+
+func init() {
+	handlers.RegisterWorkerFactory("mbox", NewWorker)
+}
+
+var errUnsupported = fmt.Errorf("unsupported command")
+
+type mboxWorker struct {
+	data   *mailboxContainer
+	name   string
+	folder *container
+	worker *types.Worker
+}
+
+func NewWorker(worker *types.Worker) (types.Backend, error) {
+	return &mboxWorker{
+		worker: worker,
+	}, nil
+}
+
+func (w *mboxWorker) handleMessage(msg types.WorkerMessage) error {
+	var reterr error // will be returned at the end, needed to support idle
+
+	switch msg := msg.(type) {
+
+	case *types.Unsupported:
+		// No-op
+
+	case *types.Configure:
+		u, err := url.Parse(msg.Config.Source)
+		if err != nil {
+			reterr = err
+			break
+		}
+		dir := u.Path
+		if u.Host == "~" {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				reterr = err
+				break
+			}
+			dir = filepath.Join(home, u.Path)
+		} else {
+			dir = filepath.Join(u.Host, u.Path)
+		}
+		w.data, err = createMailboxContainer(dir)
+		if err != nil || w.data == nil {
+			w.data = &mailboxContainer{
+				mailboxes: make(map[string]*container),
+			}
+			reterr = err
+			break
+		} else {
+			w.worker.Logger.Printf("mbox: configured with mbox file %s", dir)
+		}
+
+	case *types.Connect, *types.Reconnect, *types.Disconnect:
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.ListDirectories:
+		dirs := w.data.Names()
+		sort.Strings(dirs)
+		for _, name := range dirs {
+			w.worker.PostMessage(&types.Directory{
+				Message: types.RespondTo(msg),
+				Dir: &models.Directory{
+					Name:       name,
+					Attributes: nil,
+				},
+			}, nil)
+			w.worker.PostMessage(&types.DirectoryInfo{
+				Info: w.data.DirectoryInfo(name),
+			}, nil)
+		}
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.OpenDirectory:
+		w.name = msg.Directory
+		var ok bool
+		w.folder, ok = w.data.Mailbox(w.name)
+		if !ok {
+			w.folder = w.data.Create(w.name)
+			w.worker.PostMessage(&types.Done{
+				Message: types.RespondTo(&types.CreateDirectory{})}, nil)
+		}
+		w.worker.PostMessage(&types.DirectoryInfo{
+			Info: w.data.DirectoryInfo(msg.Directory),
+		}, nil)
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+		w.worker.Logger.Printf("mbox: %s opened\n", msg.Directory)
+
+	case *types.FetchDirectoryContents:
+		var infos []*models.MessageInfo
+		for _, uid := range w.folder.Uids() {
+			m, err := w.folder.Message(uid)
+			if err != nil {
+				w.worker.Logger.Println("mbox: could not get message", err)
+				continue
+			}
+			info, err := lib.MessageInfo(m)
+			if err != nil {
+				w.worker.Logger.Println("mbox: could not get message info", err)
+				continue
+			}
+			infos = append(infos, info)
+		}
+		uids, err := lib.Sort(infos, msg.SortCriteria)
+		if err != nil {
+			reterr = err
+			break
+		}
+		if len(uids) == 0 {
+			reterr = fmt.Errorf("mbox: no uids in directory")
+			break
+		}
+		w.worker.PostMessage(&types.DirectoryContents{
+			Message: types.RespondTo(msg),
+			Uids:    uids,
+		}, nil)
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.FetchDirectoryThreaded:
+		reterr = errUnsupported
+
+	case *types.CreateDirectory:
+		w.data.Create(msg.Directory)
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.RemoveDirectory:
+		if err := w.data.Remove(msg.Directory); err != nil {
+			reterr = err
+			break
+		}
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.FetchMessageHeaders:
+		for _, uid := range msg.Uids {
+			m, err := w.folder.Message(uid)
+			if err != nil {
+				reterr = err
+				break
+			}
+			msgInfo, err := lib.MessageInfo(m)
+			if err != nil {
+				reterr = err
+				break
+			} else {
+				w.worker.PostMessage(&types.MessageInfo{
+					Message: types.RespondTo(msg),
+					Info:    msgInfo,
+				}, nil)
+			}
+		}
+		w.worker.PostMessage(
+			&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.FetchMessageBodyPart:
+		m, err := w.folder.Message(msg.Uid)
+		if err != nil {
+			w.worker.Logger.Printf("could not get message %d: %v", msg.Uid, err)
+			reterr = err
+			break
+		}
+
+		contentReader, err := m.NewReader()
+		if err != nil {
+			reterr = fmt.Errorf("could not get message reader: %v", err)
+			break
+		}
+
+		fullMsg, err := gomessage.Read(contentReader)
+		if err != nil {
+			reterr = fmt.Errorf("could not read message: %v", err)
+			break
+		}
+
+		r, err := lib.FetchEntityPartReader(fullMsg, msg.Part)
+		if err != nil {
+			w.worker.Logger.Printf(
+				"could not get body part reader for message=%d, parts=%#v: %v",
+				msg.Uid, msg.Part, err)
+			reterr = err
+			break
+		}
+
+		w.worker.PostMessage(&types.MessageBodyPart{
+			Message: types.RespondTo(msg),
+			Part: &models.MessageBodyPart{
+				Reader: r,
+				Uid:    msg.Uid,
+			},
+		}, nil)
+
+	case *types.FetchFullMessages:
+		for _, uid := range msg.Uids {
+			m, err := w.folder.Message(uid)
+			if err != nil {
+				w.worker.Logger.Printf("could not get message for uid %d: %v", uid, err)
+				continue
+			}
+			r, err := m.NewReader()
+			if err != nil {
+				w.worker.Logger.Printf("could not get message reader: %v", err)
+				continue
+			}
+			defer r.Close()
+			b, err := ioutil.ReadAll(r)
+			if err != nil {
+				w.worker.Logger.Printf("could not get message reader: %v", err)
+				continue
+			}
+			w.worker.PostMessage(&types.FullMessage{
+				Message: types.RespondTo(msg),
+				Content: &models.FullMessage{
+					Uid:    uid,
+					Reader: bytes.NewReader(b),
+				},
+			}, nil)
+		}
+		w.worker.PostMessage(&types.Done{
+			Message: types.RespondTo(msg),
+		}, nil)
+
+	case *types.DeleteMessages:
+		deleted := w.folder.Delete(msg.Uids)
+		if len(deleted) > 0 {
+			w.worker.PostMessage(&types.MessagesDeleted{
+				Message: types.RespondTo(msg),
+				Uids:    deleted,
+			}, nil)
+		}
+
+		w.worker.PostMessage(&types.DirectoryInfo{
+			Info: w.data.DirectoryInfo(w.name),
+		}, nil)
+
+		w.worker.PostMessage(
+			&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.FlagMessages:
+		for _, uid := range msg.Uids {
+			m, err := w.folder.Message(uid)
+			if err != nil {
+				w.worker.Logger.Printf("could not get message: %v", err)
+				continue
+			}
+			if err := m.(*message).SetFlag(msg.Flag, msg.Enable); err != nil {
+				w.worker.Logger.Printf("could change flag %v to %v on message: %v", msg.Flag, msg.Enable, err)
+				continue
+			}
+			info, err := lib.MessageInfo(m)
+			if err != nil {
+				w.worker.Logger.Printf("could not get message info: %v", err)
+				continue
+			}
+
+			w.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info:    info,
+			}, nil)
+		}
+
+		w.worker.PostMessage(&types.DirectoryInfo{
+			Info: w.data.DirectoryInfo(w.name),
+		}, nil)
+
+		w.worker.PostMessage(
+			&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.CopyMessages:
+		err := w.data.Copy(msg.Destination, w.name, msg.Uids)
+		if err != nil {
+			reterr = err
+			break
+		}
+
+		w.worker.PostMessage(&types.DirectoryInfo{
+			Info: w.data.DirectoryInfo(w.name),
+		}, nil)
+
+		w.worker.PostMessage(&types.DirectoryInfo{
+			Info: w.data.DirectoryInfo(msg.Destination),
+		}, nil)
+
+		w.worker.PostMessage(
+			&types.Done{Message: types.RespondTo(msg)}, nil)
+
+	case *types.SearchDirectory:
+		criteria, err := lib.GetSearchCriteria(msg.Argv)
+		if err != nil {
+			reterr = err
+			break
+		}
+		w.worker.Logger.Printf("Searching with parsed criteria: %#v", criteria)
+		m := make([]lib.RawMessage, 0, len(w.folder.Uids()))
+		for _, uid := range w.folder.Uids() {
+			msg, err := w.folder.Message(uid)
+			if err != nil {
+				w.worker.Logger.Println("faild to get message for uid:", uid)
+				continue
+			}
+			m = append(m, msg)
+		}
+		uids, err := lib.Search(m, criteria)
+		if err != nil {
+			reterr = err
+			break
+		}
+		w.worker.PostMessage(&types.SearchResults{
+			Message: types.RespondTo(msg),
+			Uids:    uids,
+		}, nil)
+
+	case *types.AppendMessage:
+		if msg.Destination == "" {
+			reterr = fmt.Errorf("AppendMessage with empty destination directory")
+			break
+		}
+		folder, ok := w.data.Mailbox(msg.Destination)
+		if !ok {
+			folder = w.data.Create(msg.Destination)
+			w.worker.PostMessage(&types.Done{
+				Message: types.RespondTo(&types.CreateDirectory{})}, nil)
+		}
+
+		if err := folder.Append(msg.Reader, msg.Flags); err != nil {
+			reterr = err
+			break
+		} else {
+			w.worker.PostMessage(&types.DirectoryInfo{
+				Info: w.data.DirectoryInfo(msg.Destination),
+			}, nil)
+			w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+		}
+
+	case *types.AnsweredMessages:
+		reterr = errUnsupported
+	default:
+		reterr = errUnsupported
+	}
+
+	return reterr
+}
+
+func (w *mboxWorker) Run() {
+	for {
+		select {
+		case msg := <-w.worker.Actions:
+			msg = w.worker.ProcessAction(msg)
+			if err := w.handleMessage(msg); err == errUnsupported {
+				w.worker.PostMessage(&types.Unsupported{
+					Message: types.RespondTo(msg),
+				}, nil)
+			} else if err != nil {
+				w.worker.PostMessage(&types.Error{
+					Message: types.RespondTo(msg),
+					Error:   err,
+				}, nil)
+			}
+		}
+	}
+}