commands: add OptionsProvider and OptionCompleter

Improve command completion by supporting option flags and option
arguments completion. Option completion is activated when the command
implements the OptionsProvider interface. Implementing the
OptionCompleter allows the completion of individual option arguments.

The completion interfaces harmonizes the completion behavior in aerc,
makes the completion code clearer and simplifies the completion
functionality.

With this patch, the Complete method on commands will only have to deal
with the actual completion, i.e. paths, folders, etc and not worry about
options. To remove all options and its mandatory arguments from args,
use commands.Operands().

Signed-off-by: Koni Marti <koni.marti@gmail.com>
Acked-by: Robin Jarry <robin@jarry.cc>

8 files changed, 632 insertions, 38 deletions

diff --git a/commands/commands.go b/commands/commands.go
index 0a7050e9..f50a1e8a 100644
--- a/commands/commands.go
+++ b/commands/commands.go
@@ -23,6 +23,16 @@ type Command interface {
 	Complete(*widgets.Aerc, []string) []string
 }
 
+type OptionsProvider interface {
+	Command
+	Options() string
+}
+
+type OptionCompleter interface {
+	OptionsProvider
+	CompleteOption(*widgets.Aerc, rune, string) []string
+}
+
 type Commands map[string]Command
 
 func NewCommands() *Commands {
@@ -131,49 +141,98 @@ func (cmds *Commands) ExecuteCommand(
 	return NoSuchCommand(args[0])
 }
 
-func (cmds *Commands) GetCompletions(aerc *widgets.Aerc, cmd string) []string {
-	args, err := shlex.Split(cmd)
+// GetCompletions returns the completion options and the command prefix
+func (cmds *Commands) GetCompletions(
+	aerc *widgets.Aerc, cmd string,
+) (options []string, prefix string) {
+	log.Tracef("completing command: %s", cmd)
+
+	// start completion
+	args, err := splitCmd(cmd)
 	if err != nil {
-		return nil
+		return
 	}
 
 	// nothing entered, list all commands
 	if len(args) == 0 {
-		names := cmds.Names()
-		sort.Strings(names)
-		return names
+		options = cmds.Names()
+		sort.Strings(options)
+		return
+	}
+
+	// complete command name
+	spaceTerminated := cmd[len(cmd)-1] == ' '
+	if len(args) == 1 && !spaceTerminated {
+		for _, n := range cmds.Names() {
+			options = append(options, n+" ")
+		}
+		options = CompletionFromList(aerc, options, args)
+
+		return
+	}
+
+	// look for command in dictionary
+	c, ok := cmds.dict()[args[0]]
+	if !ok {
+		return
 	}
 
 	// complete options
-	if len(args) > 1 || cmd[len(cmd)-1] == ' ' {
-		if cmd, ok := cmds.dict()[args[0]]; ok {
-			var completions []string
-			if len(args) > 1 {
-				completions = cmd.Complete(aerc, args[1:])
-			} else {
-				completions = cmd.Complete(aerc, []string{})
+	var spec string
+	if provider, ok := c.(OptionsProvider); ok {
+		spec = provider.Options()
+	}
+
+	parser, err := newParser(cmd, spec, spaceTerminated)
+	if err != nil {
+		log.Debugf("completion parser failed: %v", err)
+		return
+	}
+
+	switch parser.kind {
+	case SHORT_OPTION:
+		for _, r := range strings.ReplaceAll(spec, ":", "") {
+			if strings.ContainsRune(parser.flag, r) {
+				continue
 			}
-			if completions != nil && len(completions) == 0 {
-				return nil
+			option := string(r)
+			if strings.Contains(spec, option+":") {
+				option += " "
 			}
-
-			options := make([]string, 0)
-			for _, option := range completions {
-				options = append(options, args[0]+" "+option)
+			options = append(options, option)
+		}
+		prefix = cmd
+	case OPTION_ARGUMENT:
+		cmpl, ok := c.(OptionCompleter)
+		if !ok {
+			return
+		}
+		stem := cmd
+		if parser.arg != "" {
+			stem = strings.TrimSuffix(cmd, parser.arg)
+		}
+		pad := ""
+		if !strings.HasSuffix(stem, " ") {
+			pad += " "
+		}
+		s := parser.flag
+		r := rune(s[len(s)-1])
+		for _, option := range cmpl.CompleteOption(aerc, r, parser.arg) {
+			options = append(options, pad+escape(option)+" ")
+		}
+		prefix = stem
+	case OPERAND:
+		stem := strings.Join(args[:parser.optind], " ")
+		for _, option := range c.Complete(aerc, args[1:]) {
+			if strings.Contains(option, "  ") {
+				option = escape(option)
 			}
-			return options
+			options = append(options, " "+option)
 		}
-		return nil
+		prefix = stem
 	}
 
-	// complete available commands
-	names := cmds.Names()
-	options := FilterList(names, args[0], "", aerc.SelectedAccountUiConfig().FuzzyComplete)
-
-	if len(options) > 0 {
-		return options
-	}
-	return nil
+	return
 }
 
 func GetFolders(aerc *widgets.Aerc, args []string) []string {
@@ -246,3 +305,19 @@ func hasUpper(s string) bool {
 	}
 	return false
 }
+
+// splitCmd splits the command into arguments
+func splitCmd(cmd string) ([]string, error) {
+	args, err := shlex.Split(cmd)
+	if err != nil {
+		return nil, err
+	}
+	return args, nil
+}
+
+func escape(s string) string {
+	if strings.Contains(s, " ") {
+		return strings.ReplaceAll(s, " ", "\\ ")
+	}
+	return s
+}
diff --git a/commands/completion_helpers.go b/commands/completion_helpers.go
new file mode 100644
index 00000000..96a423ee
--- /dev/null
+++ b/commands/completion_helpers.go
@@ -0,0 +1,78 @@
+package commands
+
+import (
+	"fmt"
+	"net/mail"
+	"strings"
+
+	"git.sr.ht/~rjarry/aerc/completer"
+	"git.sr.ht/~rjarry/aerc/config"
+	"git.sr.ht/~rjarry/aerc/log"
+	"git.sr.ht/~rjarry/aerc/widgets"
+)
+
+// GetAddress uses the address-book-cmd for address completion
+func GetAddress(aerc *widgets.Aerc, search string) []string {
+	var options []string
+
+	cmd := aerc.SelectedAccount().AccountConfig().AddressBookCmd
+	if cmd == "" {
+		cmd = config.Compose.AddressBookCmd
+		if cmd == "" {
+			return nil
+		}
+	}
+
+	cmpl := completer.New(cmd, func(err error) {
+		aerc.PushError(
+			fmt.Sprintf("could not complete header: %v", err))
+		log.Warnf("could not complete header: %v", err)
+	})
+
+	if len(search) > config.Ui.CompletionMinChars && cmpl != nil {
+		addrList, _ := cmpl.ForHeader("to")(search)
+		for _, full := range addrList {
+			addr, err := mail.ParseAddress(full)
+			if err != nil {
+				continue
+			}
+			options = append(options, addr.Address)
+		}
+	}
+
+	return options
+}
+
+// GetFlagList returns a list of available flags for completion
+func GetFlagList() []string {
+	return []string{"Seen", "Answered", "Flagged"}
+}
+
+// GetDateList returns a list of date terms for completion
+func GetDateList() []string {
+	return []string{
+		"today", "yesterday", "this_week", "this_month",
+		"this_year", "last_week", "last_month", "last_year",
+		"Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
+		"Saturday", "Sunday",
+	}
+}
+
+// Operands returns a slice without any option flags or mandatory option
+// arguments
+func Operands(args []string, spec string) []string {
+	var result []string
+	for i := 0; i < len(args); i++ {
+		if s := args[i]; s == "--" {
+			return args[i+1:]
+		} else if strings.HasPrefix(s, "-") && len(spec) > 0 {
+			r := string(s[len(s)-1]) + ":"
+			if strings.Contains(spec, r) {
+				i++
+			}
+			continue
+		}
+		result = append(result, args[i])
+	}
+	return result
+}
diff --git a/commands/completion_helpers_test.go b/commands/completion_helpers_test.go
new file mode 100644
index 00000000..876dc26d
--- /dev/null
+++ b/commands/completion_helpers_test.go
@@ -0,0 +1,44 @@
+package commands_test
+
+import (
+	"strings"
+	"testing"
+
+	"git.sr.ht/~rjarry/aerc/commands"
+)
+
+func TestCommands_Operand(t *testing.T) {
+	tests := []struct {
+		args []string
+		spec string
+		want string
+	}{
+		{
+			args: []string{"cmd", "-a", "-b", "arg1", "-c", "bla"},
+			spec: "ab:c",
+			want: "cmdbla",
+		},
+		{
+			args: []string{"cmd", "-a", "-b", "arg1", "-c", "--", "bla"},
+			spec: "ab:c",
+			want: "bla",
+		},
+		{
+			args: []string{"cmd", "-a", "-b", "arg1", "-c", "bla"},
+			spec: "ab:c:",
+			want: "cmd",
+		},
+		{
+			args: nil,
+			spec: "ab:c:",
+			want: "",
+		},
+	}
+	for i, test := range tests {
+		arg := strings.Join(commands.Operands(test.args, test.spec), "")
+		if arg != test.want {
+			t.Errorf("failed test %d: want '%s', got '%s'", i,
+				test.want, arg)
+		}
+	}
+}
diff --git a/commands/parser.go b/commands/parser.go
new file mode 100644
index 00000000..e8146506
--- /dev/null
+++ b/commands/parser.go
@@ -0,0 +1,134 @@
+package commands
+
+import (
+	"strings"
+)
+
+type completionType int
+
+const (
+	NONE completionType = iota
+	COMMAND
+	OPERAND
+	SHORT_OPTION
+	OPTION_ARGUMENT
+)
+
+type parser struct {
+	tokens []string
+	optind int
+	spec   string
+	space  bool
+	kind   completionType
+	flag   string
+	arg    string
+	err    error
+}
+
+func newParser(cmd, spec string, spaceTerminated bool) (*parser, error) {
+	args, err := splitCmd(cmd)
+	if err != nil {
+		return nil, err
+	}
+
+	p := &parser{
+		tokens: args,
+		optind: 0,
+		spec:   spec,
+		space:  spaceTerminated,
+		kind:   NONE,
+		flag:   "",
+		arg:    "",
+		err:    nil,
+	}
+
+	state := command
+	for state != nil {
+		state = state(p)
+	}
+
+	return p, p.err
+}
+
+func (p *parser) empty() bool {
+	return len(p.tokens) == 0
+}
+
+func (p *parser) peek() string {
+	return p.tokens[0]
+}
+
+func (p *parser) advance() string {
+	if p.empty() {
+		return ""
+	}
+	tok := p.tokens[0]
+	p.tokens = p.tokens[1:]
+	p.optind++
+	return tok
+}
+
+func (p *parser) set(t completionType) {
+	p.kind = t
+}
+
+func (p *parser) hasArgument() bool {
+	n := len(p.flag)
+	if n > 0 {
+		s := string(p.flag[n-1]) + ":"
+		return strings.Contains(p.spec, s)
+	}
+	return false
+}
+
+type stateFn func(*parser) stateFn
+
+func command(p *parser) stateFn {
+	p.set(COMMAND)
+	p.advance()
+	return peek(p)
+}
+
+func peek(p *parser) stateFn {
+	if p.empty() {
+		if p.space {
+			return operand
+		}
+		return nil
+	}
+	if p.spec == "" {
+		return operand
+	}
+	s := p.peek()
+	switch {
+	case s == "--":
+		p.advance()
+	case strings.HasPrefix(s, "-"):
+		return short_option
+	}
+	return operand
+}
+
+func short_option(p *parser) stateFn {
+	p.set(SHORT_OPTION)
+	tok := p.advance()
+	p.flag = tok[1:]
+	if p.hasArgument() {
+		return option_argument
+	}
+	return peek(p)
+}
+
+func option_argument(p *parser) stateFn {
+	p.set(OPTION_ARGUMENT)
+	p.arg = p.advance()
+	if p.empty() && len(p.arg) == 0 {
+		return nil
+	}
+	return peek(p)
+}
+
+func operand(p *parser) stateFn {
+	p.set(OPERAND)
+	return nil
+}
diff --git a/commands/parser_test.go b/commands/parser_test.go
new file mode 100644
index 00000000..d6ccd385
--- /dev/null
+++ b/commands/parser_test.go
@@ -0,0 +1,258 @@
+package commands
+
+import (
+	"testing"
+)
+
+var parserTests = []struct {
+	name       string
+	cmd        string
+	wantType   completionType
+	wantFlag   string
+	wantArg    string
+	wantOptind int
+}{
+	{
+		name:       "empty command",
+		cmd:        "",
+		wantType:   COMMAND,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 0,
+	},
+	{
+		name:       "command only",
+		cmd:        "cmd",
+		wantType:   COMMAND,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 1,
+	},
+	{
+		name:       "with space",
+		cmd:        "cmd ",
+		wantType:   OPERAND,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 1,
+	},
+	{
+		name:       "with two spaces",
+		cmd:        "cmd  ",
+		wantType:   OPERAND,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 1,
+	},
+	{
+		name:       "with single option flag",
+		cmd:        "cmd -",
+		wantType:   SHORT_OPTION,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with single option flag two spaces",
+		cmd:        "cmd  -",
+		wantType:   SHORT_OPTION,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with single option flag completed",
+		cmd:        "cmd -a",
+		wantType:   SHORT_OPTION,
+		wantFlag:   "a",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with single option flag completed and space",
+		cmd:        "cmd -a ",
+		wantType:   OPERAND,
+		wantFlag:   "a",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with single option flag completed and two spaces",
+		cmd:        "cmd -a ",
+		wantType:   OPERAND,
+		wantFlag:   "a",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with two single option flag completed",
+		cmd:        "cmd -b -a",
+		wantType:   SHORT_OPTION,
+		wantFlag:   "a",
+		wantArg:    "",
+		wantOptind: 3,
+	},
+	{
+		name:       "with two single option flag combined",
+		cmd:        "cmd -ab",
+		wantType:   SHORT_OPTION,
+		wantFlag:   "ab",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with two single option flag and space",
+		cmd:        "cmd -ab ",
+		wantType:   OPERAND,
+		wantFlag:   "ab",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with mandatory option flag",
+		cmd:        "cmd -f",
+		wantType:   OPTION_ARGUMENT,
+		wantFlag:   "f",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with mandatory option flag and space",
+		cmd:        "cmd -f ",
+		wantType:   OPTION_ARGUMENT,
+		wantFlag:   "f",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with mandatory option flag and two spaces",
+		cmd:        "cmd -f  ",
+		wantType:   OPTION_ARGUMENT,
+		wantFlag:   "f",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with mandatory option flag and completed",
+		cmd:        "cmd -f a",
+		wantType:   OPTION_ARGUMENT,
+		wantFlag:   "f",
+		wantArg:    "a",
+		wantOptind: 3,
+	},
+	{
+		name:       "with mandatory option flag and completed quote",
+		cmd:        "cmd -f 'a b'",
+		wantType:   OPTION_ARGUMENT,
+		wantFlag:   "f",
+		wantArg:    "a b",
+		wantOptind: 3,
+	},
+	{
+		name:       "with mandatory option flag and operand",
+		cmd:        "cmd -f 'a b' hello",
+		wantType:   OPERAND,
+		wantFlag:   "f",
+		wantArg:    "a b",
+		wantOptind: 3,
+	},
+	{
+		name:       "with mandatory option flag and two spaces between",
+		cmd:        "cmd -f  a",
+		wantType:   OPTION_ARGUMENT,
+		wantFlag:   "f",
+		wantArg:    "a",
+		wantOptind: 3,
+	},
+	{
+		name:       "with mandatory option flag and more spaces",
+		cmd:        "cmd -f  a ",
+		wantType:   OPERAND,
+		wantFlag:   "f",
+		wantArg:    "a",
+		wantOptind: 3,
+	},
+	{
+		name:       "with template data",
+		cmd:        "cmd -a {{if .Size}}  hello {{else}} {{end}}",
+		wantType:   OPERAND,
+		wantFlag:   "a",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with operand",
+		cmd:        "cmd -ab /tmp/aerc-",
+		wantType:   OPERAND,
+		wantFlag:   "ab",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+	{
+		name:       "with operand indicator",
+		cmd:        "cmd -ab -- /tmp/aerc-",
+		wantType:   OPERAND,
+		wantFlag:   "ab",
+		wantArg:    "",
+		wantOptind: 3,
+	},
+	{
+		name:       "hyphen connected command",
+		cmd:        "cmd-dmc",
+		wantType:   COMMAND,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 1,
+	},
+	{
+		name:       "incomplete hyphen connected command",
+		cmd:        "cmd-",
+		wantType:   COMMAND,
+		wantFlag:   "",
+		wantArg:    "",
+		wantOptind: 1,
+	},
+	{
+		name:       "hyphen connected command with option",
+		cmd:        "cmd-dmc -a",
+		wantType:   SHORT_OPTION,
+		wantFlag:   "a",
+		wantArg:    "",
+		wantOptind: 2,
+	},
+}
+
+func TestCommands_Parser(t *testing.T) {
+	for i, test := range parserTests {
+		n := len(test.cmd)
+		spaceTerminated := n > 0 && test.cmd[n-1] == ' '
+		parser, err := newParser(test.cmd, "abf:", spaceTerminated)
+		if err != nil {
+			t.Errorf("parser error: %v", err)
+		}
+
+		if test.wantType != parser.kind {
+			t.Errorf("test %d '%s': completion type does not match: "+
+				"want %d, but got %d", i, test.cmd, test.wantType,
+				parser.kind)
+		}
+
+		if test.wantFlag != parser.flag {
+			t.Errorf("test %d '%s': flag does not match: "+
+				"want %s, but got %s", i, test.cmd, test.wantFlag,
+				parser.flag)
+		}
+
+		if test.wantArg != parser.arg {
+			t.Errorf("test %d '%s': arg does not match: "+
+				"want %s, but got %s", i, test.cmd, test.wantArg,
+				parser.arg)
+		}
+
+		if test.wantOptind != parser.optind {
+			t.Errorf("test %d '%s': optind does not match: "+
+				"want %d, but got %d", i, test.cmd, test.wantOptind,
+				parser.optind)
+		}
+	}
+}
diff --git a/commands/prompt.go b/commands/prompt.go
index 8746bcfe..a93d19a9 100644
--- a/commands/prompt.go
+++ b/commands/prompt.go
@@ -36,7 +36,7 @@ func (Prompt) Complete(aerc *widgets.Aerc, args []string) []string {
 		if hascommand {
 			return nil
 		}
-		cs = GlobalCommands.GetCompletions(aerc, args[1])
+		cs, _ = GlobalCommands.GetCompletions(aerc, args[1])
 	}
 	if cs == nil {
 		return nil
diff --git a/main.go b/main.go
index 99557d2b..9dd166fe 100644
--- a/main.go
+++ b/main.go
@@ -88,13 +88,18 @@ func execCommand(
 	return nil
 }
 
-func getCompletions(aerc *widgets.Aerc, cmd string) []string {
+func getCompletions(aerc *widgets.Aerc, cmd string) ([]string, string) {
 	var completions []string
+	var prefix string
 	for _, set := range getCommands(aerc.SelectedTabContent()) {
-		completions = append(completions, set.GetCompletions(aerc, cmd)...)
+		options, s := set.GetCompletions(aerc, cmd)
+		if s != "" {
+			prefix = s
+		}
+		completions = append(completions, options...)
 	}
 	sort.Strings(completions)
-	return completions
+	return completions, prefix
 }
 
 // set at build time
@@ -198,7 +203,7 @@ func main() {
 		msg *models.MessageInfo,
 	) error {
 		return execCommand(aerc, ui, cmd, acct, msg)
-	}, func(cmd string) []string {
+	}, func(cmd string) ([]string, string) {
 		return getCompletions(aerc, cmd)
 	}, &commands.CmdHistory, deferLoop)
 
diff --git a/widgets/aerc.go b/widgets/aerc.go
index f4812dd4..9f88ecb6 100644
--- a/widgets/aerc.go
+++ b/widgets/aerc.go
@@ -28,7 +28,7 @@ type Aerc struct {
 	accounts    map[string]*AccountView
 	cmd         func([]string, *config.AccountConfig, *models.MessageInfo) error
 	cmdHistory  lib.History
-	complete    func(cmd string) []string
+	complete    func(cmd string) ([]string, string)
 	focused     ui.Interactive
 	grid        *ui.Grid
 	simulating  int
@@ -54,7 +54,7 @@ type Choice struct {
 func NewAerc(
 	crypto crypto.Provider,
 	cmd func([]string, *config.AccountConfig, *models.MessageInfo) error,
-	complete func(cmd string) []string, cmdHistory lib.History,
+	complete func(cmd string) ([]string, string), cmdHistory lib.History,
 	deferLoop chan struct{},
 ) *Aerc {
 	tabs := ui.NewTabs(config.Ui)
@@ -290,7 +290,7 @@ func (aerc *Aerc) simulate(strokes []config.KeyStroke) {
 	// If we are still focused on the exline, turn on tab complete
 	if exline, ok := aerc.focused.(*ExLine); ok {
 		exline.TabComplete(func(cmd string) ([]string, string) {
-			return aerc.complete(cmd), ""
+			return aerc.complete(cmd)
 		})
 		// send tab to text input to trigger completion
 		exline.Event(tcell.NewEventKey(tcell.KeyTab, 0, tcell.ModNone))
@@ -594,7 +594,7 @@ func (aerc *Aerc) BeginExCommand(cmd string) {
 		tabComplete = nil
 	} else {
 		tabComplete = func(cmd string) ([]string, string) {
-			return aerc.complete(cmd), ""
+			return aerc.complete(cmd)
 		}
 	}
 	exline := NewExLine(cmd, func(cmd string) {