1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
package query_test
import (
"testing"
"github.com/jpappel/atlas/pkg/query"
)
type Token = query.Token
const (
TOK_UNKNOWN = query.TOK_UNKNOWN
TOK_CLAUSE_OR = query.TOK_CLAUSE_OR
TOK_CLAUSE_AND = query.TOK_CLAUSE_AND
TOK_CLAUSE_START = query.TOK_CLAUSE_START
TOK_CLAUSE_END = query.TOK_CLAUSE_END
TOK_OP_NEG = query.TOK_OP_NEG
TOK_OP_EQ = query.TOK_OP_EQ
TOK_OP_AP = query.TOK_OP_AP
TOK_OP_NE = query.TOK_OP_NE
TOK_OP_LT = query.TOK_OP_LT
TOK_OP_LE = query.TOK_OP_LE
TOK_OP_GE = query.TOK_OP_GE
TOK_OP_GT = query.TOK_OP_GT
TOK_OP_RE = query.TOK_OP_RE
TOK_CAT_TITLE = query.TOK_CAT_TITLE
TOK_CAT_AUTHOR = query.TOK_CAT_AUTHOR
TOK_CAT_DATE = query.TOK_CAT_DATE
TOK_CAT_FILETIME = query.TOK_CAT_FILETIME
TOK_CAT_TAGS = query.TOK_CAT_TAGS
TOK_CAT_HEADINGS = query.TOK_CAT_HEADINGS
TOK_CAT_LINKS = query.TOK_CAT_LINKS
TOK_CAT_META = query.TOK_CAT_META
TOK_VAL_STR = query.TOK_VAL_STR
TOK_VAL_DATETIME = query.TOK_VAL_DATETIME
)
func TestLex(t *testing.T) {
tests := []struct {
name string
query string
want []query.Token
}{
{"empty query", "", []Token{{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"}, {Type: TOK_CLAUSE_END}}},
{"quoted statement", `a:"ken thompson"`, []Token{
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "ken thompson"},
{Type: TOK_CLAUSE_END},
}},
{"invalid token", `foo:bar`, []Token{
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{TOK_UNKNOWN, "foo:bar"},
{Type: TOK_CLAUSE_END},
}},
{"simple query", "a:a t:b d:01010001", []Token{
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "a"},
{TOK_CAT_TAGS, "t"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "b"},
{TOK_CAT_DATE, "d"}, {TOK_OP_AP, ":"}, {TOK_VAL_DATETIME, "01010001"},
{Type: TOK_CLAUSE_END},
}},
{"leading subclause", "(or a:a a:b)", []Token{
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_OR, "or"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "a"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "b"},
{Type: TOK_CLAUSE_END},
{Type: TOK_CLAUSE_END},
}},
{"clause after clause", "(or a:a a:b) (or a:c a:d)", []Token{
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_OR, "or"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "a"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "b"},
{Type: TOK_CLAUSE_END},
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_OR, "or"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "c"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "d"},
{Type: TOK_CLAUSE_END},
{Type: TOK_CLAUSE_END},
}},
{"nested clauses", "a:a (or t:b t!=c) or d<=01010001 and -T~foo t/bar", []Token{
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "a"},
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_OR, "or"},
{TOK_CAT_TAGS, "t"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "b"},
{TOK_CAT_TAGS, "t"}, {TOK_OP_NE, "!="}, {TOK_VAL_STR, "c"},
{Type: TOK_CLAUSE_END},
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_OR, "or"},
{TOK_CAT_DATE, "d"}, {TOK_OP_LE, "<="}, {TOK_VAL_DATETIME, "01010001"},
{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
{TOK_OP_NEG, "-"}, {TOK_CAT_TITLE, "T"}, {TOK_OP_AP, "~"}, {TOK_VAL_STR, "foo"},
{TOK_CAT_TAGS, "t"}, {TOK_OP_RE, "/"}, {TOK_VAL_STR, "bar"},
{Type: TOK_CLAUSE_END},
{Type: TOK_CLAUSE_END},
{Type: TOK_CLAUSE_END},
}},
// FIXME: change parser so this test passes
// {"consecutive clause starts", "a:a (or (and a:b a:c) a:d)", []Token{
// {Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
// {TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "a"},
// {Type: TOK_CLAUSE_START}, {TOK_CLAUSE_OR, "or"},
// {Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"},
// {TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "b"},
// {TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "c"},
// {Type: TOK_CLAUSE_END},
// {TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "d"},
// {Type: TOK_CLAUSE_END},
// }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := query.Lex(tt.query)
gl, wl := len(got), len(tt.want)
if gl != wl {
t.Errorf("Got %d tokens wanted %d\n", gl, wl)
}
for i := range min(gl, wl) {
gt, wt := got[i], tt.want[i]
if !gt.Equal(wt) {
t.Errorf("Got different token than wanted at %d\n", i)
t.Logf("(%v) != (%v)\n", gt.String(), wt.String())
break
}
}
if t.Failed() {
t.Log("Got\n", query.TokensStringify(got))
t.Log("Want\n", query.TokensStringify(tt.want))
}
})
}
}
|
