1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
package query
import (
"fmt"
"strings"
"unicode"
)
type token struct {
qualifier string
value string
}
// tokenize parse and break a input into tokens ready to be
// interpreted later by a parser to get the semantic.
func tokenize(query string) ([]token, error) {
fields, err := splitQuery(query)
if err != nil {
return nil, err
}
var tokens []token
for _, field := range fields {
split := strings.Split(field, ":")
if len(split) != 2 {
return nil, fmt.Errorf("can't tokenize \"%s\"", field)
}
if len(split[0]) == 0 {
return nil, fmt.Errorf("can't tokenize \"%s\": empty qualifier", field)
}
if len(split[1]) == 0 {
return nil, fmt.Errorf("empty value for qualifier \"%s\"", split[0])
}
tokens = append(tokens, token{
qualifier: split[0],
value: removeQuote(split[1]),
})
}
return tokens, nil
}
func splitQuery(query string) ([]string, error) {
lastQuote := rune(0)
inQuote := false
isToken := func(r rune) bool {
switch {
case !inQuote && isQuote(r):
lastQuote = r
inQuote = true
return true
case inQuote && r == lastQuote:
lastQuote = rune(0)
inQuote = false
return true
case inQuote:
return true
default:
return !unicode.IsSpace(r)
}
}
var result []string
var token strings.Builder
for _, r := range query {
if isToken(r) {
token.WriteRune(r)
} else {
if token.Len() > 0 {
result = append(result, token.String())
token.Reset()
}
}
}
if inQuote {
return nil, fmt.Errorf("unmatched quote")
}
if token.Len() > 0 {
result = append(result, token.String())
}
return result, nil
}
func isQuote(r rune) bool {
return r == '"' || r == '\''
}
func removeQuote(field string) string {
runes := []rune(field)
if len(runes) >= 2 {
r1 := runes[0]
r2 := runes[len(runes)-1]
if r1 == r2 && isQuote(r1) {
return string(runes[1 : len(runes)-1])
}
}
return field
}
|
