maildir: keep less data in memory for sorting

Sorting opens and reads portions of every file within a directory in
order to gather the data needed. Specifically, RFC822Headers and
BodyStructure are not needed. The RFC822Headers field stores a lot of
information, and the BodyStructure field requires parsing the body of
the email.

Don't set these two values when parsing.

Note: in my testing, this dropped sorting a 52k archive from 2.2gb of
ram usage, to < 500mb

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
Acked-by: Robin Jarry <robin@jarry.cc>

3 files changed, 67 insertions, 1 deletions

diff --git a/worker/lib/parse.go b/worker/lib/parse.go
index 616784c6..f57a56ac 100644
--- a/worker/lib/parse.go
+++ b/worker/lib/parse.go
@@ -302,6 +302,54 @@ func MessageInfo(raw RawMessage) (*models.MessageInfo, error) {
 	}, nil
 }
 
+// MessageHeaders populates a models.MessageInfo struct for the message.
+// based on the reader returned by NewReader. Minimal information is included.
+// There is no body structure or RFC822Headers set
+func MessageHeaders(raw RawMessage) (*models.MessageInfo, error) {
+	var parseErr error
+	r, err := raw.NewReader()
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	msg, err := ReadMessage(r)
+	if err != nil {
+		return nil, fmt.Errorf("could not read message: %w", err)
+	}
+	h := &mail.Header{Header: msg.Header}
+	env, err := parseEnvelope(h)
+	if err != nil && !errors.Is(err, DateParseError) {
+		return nil, fmt.Errorf("could not parse envelope: %w", err)
+		// if only the date parsing failed we still get the rest of the
+		// envelop structure in a valid state.
+		// Date parsing errors are fairly common and it's better to be
+		// slightly off than to not be able to read the mails at all
+		// hence we continue here
+	}
+	recDate, _ := parseReceivedHeader(h)
+	if recDate.IsZero() {
+		// better than nothing, if incorrect
+		recDate = env.Date
+	}
+	flags, err := raw.ModelFlags()
+	if err != nil {
+		return nil, err
+	}
+	labels, err := raw.Labels()
+	if err != nil {
+		return nil, err
+	}
+	return &models.MessageInfo{
+		Envelope:     env,
+		Flags:        flags,
+		Labels:       labels,
+		InternalDate: recDate,
+		Size:         0,
+		Uid:          raw.UID(),
+		Error:        parseErr,
+	}, nil
+}
+
 // NewCRLFReader returns a reader with CRLF line endings
 func NewCRLFReader(r io.Reader) io.Reader {
 	var buf bytes.Buffer
diff --git a/worker/maildir/message.go b/worker/maildir/message.go
index 3c8ce9ef..d9433823 100644
--- a/worker/maildir/message.go
+++ b/worker/maildir/message.go
@@ -76,6 +76,12 @@ func (m Message) MessageInfo() (*models.MessageInfo, error) {
 	return lib.MessageInfo(m)
 }
 
+// MessageHeaders populates a models.MessageInfo struct for the message with
+// minimal information, used for sorting and threading.
+func (m Message) MessageHeaders() (*models.MessageInfo, error) {
+	return lib.MessageHeaders(m)
+}
+
 // NewBodyPartReader creates a new io.Reader for the requested body part(s) of
 // the message.
 func (m Message) NewBodyPartReader(requestedParts []int) (io.Reader, error) {
diff --git a/worker/maildir/worker.go b/worker/maildir/worker.go
index 30faddac..de7e8e89 100644
--- a/worker/maildir/worker.go
+++ b/worker/maildir/worker.go
@@ -448,7 +448,7 @@ func (w *Worker) sort(uids []uint32, criteria []*types.SortCriterion) ([]uint32,
 		wg.Add(1)
 		go func(uid uint32) {
 			defer wg.Done()
-			info, err := w.msgInfoFromUid(uid)
+			info, err := w.msgHeadersFromUid(uid)
 			if err != nil {
 				logging.Errorf("could not get message info: %v", err)
 				return
@@ -729,6 +729,18 @@ func (w *Worker) msgInfoFromUid(uid uint32) (*models.MessageInfo, error) {
 	return info, nil
 }
 
+func (w *Worker) msgHeadersFromUid(uid uint32) (*models.MessageInfo, error) {
+	m, err := w.c.Message(*w.selected, uid)
+	if err != nil {
+		return nil, err
+	}
+	info, err := m.MessageHeaders()
+	if err != nil {
+		return nil, err
+	}
+	return info, nil
+}
+
 func (w *Worker) handleCheckMail(msg *types.CheckMail) {
 	if msg.Command == "" {
 		w.err(msg, fmt.Errorf("checkmail: no command specified"))