From bb84196aaf227904e8cb9afc846a829a1e6e0365 Mon Sep 17 00:00:00 2001 From: JP Appel Date: Sun, 15 Jun 2025 13:26:17 -0400 Subject: Update tests and fix minor bugs --- pkg/query/lexer_test.go | 41 ++++++- pkg/query/outputs.go | 4 +- pkg/query/outputs_test.go | 26 ++++- pkg/query/parser.go | 4 +- pkg/query/parser_test.go | 282 +++++++++++++++------------------------------- 5 files changed, 152 insertions(+), 205 deletions(-) (limited to 'pkg/query') diff --git a/pkg/query/lexer_test.go b/pkg/query/lexer_test.go index 0cfc1de..d931b96 100644 --- a/pkg/query/lexer_test.go +++ b/pkg/query/lexer_test.go @@ -1,14 +1,45 @@ -package query +package query_test import ( "testing" + + "github.com/jpappel/atlas/pkg/query" +) + +type Token = query.Token + +const ( + TOK_UNKNOWN = query.TOK_UNKNOWN + TOK_CLAUSE_OR = query.TOK_CLAUSE_OR + TOK_CLAUSE_AND = query.TOK_CLAUSE_AND + TOK_CLAUSE_START = query.TOK_CLAUSE_START + TOK_CLAUSE_END = query.TOK_CLAUSE_END + TOK_OP_NEG = query.TOK_OP_NEG + TOK_OP_EQ = query.TOK_OP_EQ + TOK_OP_AP = query.TOK_OP_AP + TOK_OP_NE = query.TOK_OP_NE + TOK_OP_LT = query.TOK_OP_LT + TOK_OP_LE = query.TOK_OP_LE + TOK_OP_GE = query.TOK_OP_GE + TOK_OP_GT = query.TOK_OP_GT + TOK_OP_PIPE = query.TOK_OP_PIPE + TOK_OP_ARG = query.TOK_OP_ARG + TOK_CAT_TITLE = query.TOK_CAT_TITLE + TOK_CAT_AUTHOR = query.TOK_CAT_AUTHOR + TOK_CAT_DATE = query.TOK_CAT_DATE + TOK_CAT_FILETIME = query.TOK_CAT_FILETIME + TOK_CAT_TAGS = query.TOK_CAT_TAGS + TOK_CAT_LINKS = query.TOK_CAT_LINKS + TOK_CAT_META = query.TOK_CAT_META + TOK_VAL_STR = query.TOK_VAL_STR + TOK_VAL_DATETIME = query.TOK_VAL_DATETIME ) func TestLex(t *testing.T) { tests := []struct { name string query string - want []Token + want []query.Token }{ {"empty query", "", []Token{{Type: TOK_CLAUSE_START}, {TOK_CLAUSE_AND, "and"}, {Type: TOK_CLAUSE_END}}}, {"quoted statement", `a:"ken thompson"`, []Token{ @@ -66,7 +97,7 @@ func TestLex(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := Lex(tt.query) + got := query.Lex(tt.query) gl, wl := len(got), len(tt.want) if gl != wl { @@ -83,8 +114,8 @@ func TestLex(t *testing.T) { } if t.Failed() { - t.Log("Got\n", TokensStringify(got)) - t.Log("Want\n", TokensStringify(tt.want)) + t.Log("Got\n", query.TokensStringify(got)) + t.Log("Want\n", query.TokensStringify(tt.want)) } }) } diff --git a/pkg/query/outputs.go b/pkg/query/outputs.go index 00b9ddf..7dac42e 100644 --- a/pkg/query/outputs.go +++ b/pkg/query/outputs.go @@ -102,7 +102,7 @@ func (o JsonOutput) Output(docs []*index.Document) (string, error) { return string(b), nil } -func parseOutputFormat(formatStr string) ([]OutputToken, []string, error) { +func ParseOutputFormat(formatStr string) ([]OutputToken, []string, error) { toks := make([]OutputToken, 0, 16) curTok := make([]rune, 0, 16) strToks := make([]string, 0, 8) @@ -157,7 +157,7 @@ func parseOutputFormat(formatStr string) ([]OutputToken, []string, error) { } func NewCustomOutput(formatStr string, datetimeFormat string) (CustomOutput, error) { - outToks, strToks, err := parseOutputFormat(formatStr) + outToks, strToks, err := ParseOutputFormat(formatStr) if err != nil { return CustomOutput{}, err } diff --git a/pkg/query/outputs_test.go b/pkg/query/outputs_test.go index b5fdba7..8a1bb29 100644 --- a/pkg/query/outputs_test.go +++ b/pkg/query/outputs_test.go @@ -1,44 +1,58 @@ -package query +package query_test import ( "errors" "slices" "testing" + + "github.com/jpappel/atlas/pkg/query" +) + +const ( + OUT_TOK_STR = query.OUT_TOK_STR + OUT_TOK_PATH = query.OUT_TOK_PATH + OUT_TOK_TITLE = query.OUT_TOK_TITLE + OUT_TOK_DATE = query.OUT_TOK_DATE + OUT_TOK_FILETIME = query.OUT_TOK_FILETIME + OUT_TOK_AUTHORS = query.OUT_TOK_AUTHORS + OUT_TOK_TAGS = query.OUT_TOK_TAGS + OUT_TOK_LINKS = query.OUT_TOK_LINKS + OUT_TOK_META = query.OUT_TOK_META ) func Test_parseOutputFormat(t *testing.T) { tests := []struct { name string formatStr string - wantToks []OutputToken + wantToks []query.OutputToken wantStrToks []string wantErr error }{ { "one big string", "here is a string with no placeholders", - []OutputToken{OUT_TOK_STR}, + []query.OutputToken{OUT_TOK_STR}, []string{"here is a string with no placeholders"}, nil, }, { "default format", "%p %T %d authors:%a tags:%t", - []OutputToken{OUT_TOK_PATH, OUT_TOK_STR, OUT_TOK_TITLE, OUT_TOK_STR, OUT_TOK_DATE, OUT_TOK_STR, OUT_TOK_AUTHORS, OUT_TOK_STR, OUT_TOK_TAGS}, + []query.OutputToken{OUT_TOK_PATH, OUT_TOK_STR, OUT_TOK_TITLE, OUT_TOK_STR, OUT_TOK_DATE, OUT_TOK_STR, OUT_TOK_AUTHORS, OUT_TOK_STR, OUT_TOK_TAGS}, []string{" ", " ", " authors:", " tags:"}, nil, }, { "literal percents", "%%%p%%%T%%", - []OutputToken{OUT_TOK_STR, OUT_TOK_PATH, OUT_TOK_STR, OUT_TOK_TITLE, OUT_TOK_STR}, + []query.OutputToken{OUT_TOK_STR, OUT_TOK_PATH, OUT_TOK_STR, OUT_TOK_TITLE, OUT_TOK_STR}, []string{"%", "%", "%"}, nil, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotToks, gotStrToks, gotErr := parseOutputFormat(tt.formatStr) + gotToks, gotStrToks, gotErr := query.ParseOutputFormat(tt.formatStr) if !errors.Is(gotErr, tt.wantErr) { t.Errorf("Recieved unexpected error: got %v want %v", gotErr, tt.wantErr) diff --git a/pkg/query/parser.go b/pkg/query/parser.go index 7ac9918..14cc227 100644 --- a/pkg/query/parser.go +++ b/pkg/query/parser.go @@ -290,17 +290,17 @@ func (root Clause) Order() int { func (root *Clause) DFS() iter.Seq[*Clause] { return func(yield func(*Clause) bool) { - stack := make([]*Clause, 0, len(root.Clauses)) + stack := make([]*Clause, 0, len(root.Clauses)+1) stack = append(stack, root) for len(stack) != 0 { node := stack[len(stack)-1] + stack = stack[:len(stack)-1] if !yield(node) { return } - stack := stack[:len(stack)-1] stack = append(stack, node.Clauses...) } } diff --git a/pkg/query/parser_test.go b/pkg/query/parser_test.go index 6ea5c10..e3ab971 100644 --- a/pkg/query/parser_test.go +++ b/pkg/query/parser_test.go @@ -1,222 +1,124 @@ package query_test import ( + "errors" "slices" "testing" "github.com/jpappel/atlas/pkg/query" ) -func TestClause_Flatten(t *testing.T) { +const ( + CAT_UNKNOWN = query.CAT_UNKNOWN + CAT_TITLE = query.CAT_TITLE + CAT_AUTHOR = query.CAT_AUTHOR + CAT_DATE = query.CAT_DATE + CAT_FILETIME = query.CAT_FILETIME + CAT_TAGS = query.CAT_TAGS + CAT_LINKS = query.CAT_LINKS + CAT_META = query.CAT_META + + OP_UNKNOWN = query.OP_UNKNOWN + OP_EQ = query.OP_EQ + OP_AP = query.OP_AP + OP_NE = query.OP_NE + OP_LT = query.OP_LT + OP_LE = query.OP_LE + OP_GE = query.OP_GE + OP_GT = query.OP_GT + OP_PIPE = query.OP_PIPE + OP_ARG = query.OP_ARG +) + +func TestParse(t *testing.T) { tests := []struct { - name string - root *query.Clause - expected query.Clause - }{ - { - "empty", - &query.Clause{}, - query.Clause{}, + name string + tokens []query.Token + want *query.Clause + wantErr error + }{{ + "empty clause", + []query.Token{ + {Type: TOK_CLAUSE_START}, {Type: TOK_CLAUSE_AND}, {Type: TOK_CLAUSE_END}, }, - { - "empty with child", - &query.Clause{ - Operator: query.COP_OR, - Clauses: []*query.Clause{ - { - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - }, - }, - }, - }, - query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - }, - }, + &query.Clause{Operator: query.COP_AND}, + nil, + }, { + "simple clause", + []query.Token{ + {Type: TOK_CLAUSE_START}, {Type: TOK_CLAUSE_AND}, + {TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "ken thompson"}, + {Type: TOK_CLAUSE_END}, }, - { - "already flat", - &query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - }, - }, - query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - }, + &query.Clause{ + Operator: query.COP_AND, + Statements: []query.Statement{ + {Category: CAT_AUTHOR, Operator: OP_AP, Value: query.StringValue{"ken thompson"}}, }, }, - { - "flatten 1 layer, multiple clauses", - &query.Clause{ - Operator: query.COP_OR, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - }, - Clauses: []*query.Clause{ - {Operator: query.COP_OR, Statements: []query.Statement{{Category: query.CAT_AUTHOR, Operator: query.OP_NE, Value: query.StringValue{"pj"}}}}, - {Operator: query.COP_OR, Statements: []query.Statement{{Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"barfoo"}}}}, - }, - }, - query.Clause{ - Operator: query.COP_OR, - Statements: []query.Statement{ - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_NE, Value: query.StringValue{"pj"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"barfoo"}}, - }, - }, + nil, + }, { + "nested clause", + []query.Token{ + {Type: TOK_CLAUSE_START}, {Type: TOK_CLAUSE_AND}, + {TOK_CAT_AUTHOR, "a"}, {TOK_OP_AP, ":"}, {TOK_VAL_STR, "Alonzo Church"}, + {Type: TOK_CLAUSE_START}, {Type: TOK_CLAUSE_OR}, + {TOK_CAT_AUTHOR, "a"}, {TOK_OP_EQ, "="}, {TOK_VAL_STR, "Alan Turing"}, + {Type: TOK_CLAUSE_END}, + {Type: TOK_CLAUSE_END}, }, - { - "flatten 2 layers", - &query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - }, - Clauses: []*query.Clause{ - { - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_NE, Value: query.StringValue{"pj"}}, - }, - Clauses: []*query.Clause{ - { - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"barfoo"}}, - }, - }, - }, - }, - }, - }, - query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_NE, Value: query.StringValue{"pj"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"barfoo"}}, - }, - }, - }, - { - "flatten 1 child keep 1 child", - &query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - }, - Clauses: []*query.Clause{ - { - Operator: query.COP_OR, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_NE, Value: query.StringValue{"pj"}}, - }, - }, - { - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"barfoo"}}, - }, - }, - }, + &query.Clause{ + Operator: query.COP_AND, + Statements: []query.Statement{ + {Category: CAT_AUTHOR, Operator: OP_AP, Value: query.StringValue{"Alonzo Church"}}, }, - query.Clause{ - Operator: query.COP_AND, - Statements: []query.Statement{ - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"foobar"}}, - {Category: query.CAT_TITLE, Operator: query.OP_AP, Value: query.StringValue{"a very interesting title"}}, - {Category: query.CAT_TAGS, Operator: query.OP_EQ, Value: query.StringValue{"barfoo"}}, - }, - Clauses: []*query.Clause{ - { - Operator: query.COP_OR, - Statements: []query.Statement{ - {Category: query.CAT_AUTHOR, Operator: query.OP_AP, Value: query.StringValue{"jp"}}, - {Category: query.CAT_AUTHOR, Operator: query.OP_NE, Value: query.StringValue{"pj"}}, - }, + Clauses: []*query.Clause{ + { + Operator: query.COP_OR, + Statements: []query.Statement{ + {Category: CAT_AUTHOR, Operator: OP_EQ, Value: query.StringValue{"Alan Turing"}}, }, }, }, }, - } + nil, + }} for _, tt := range tests { - o := query.Optimizer{} t.Run(tt.name, func(t *testing.T) { - o.Flatten(tt.root) - - slices.SortFunc(tt.root.Statements, query.StatementCmp) - slices.SortFunc(tt.expected.Statements, query.StatementCmp) - - stmtsEq := slices.EqualFunc(tt.root.Statements, tt.expected.Statements, - func(a, b query.Statement) bool { - return a.Category == b.Category && a.Operator == b.Operator && a.Negated == b.Negated && a.Value.Compare(b.Value) == 0 - }, - ) - - if !stmtsEq { - t.Error("Statments not equal") - if gL, wL := len(tt.root.Statements), len(tt.expected.Statements); gL != wL { - t.Logf("Different number of statements: got %d want %d\n", gL, wL) - } + gotC, gotErr := query.Parse(tt.tokens) + if !errors.Is(gotErr, tt.wantErr) { + t.Fatalf("Different parse error than expected: got %v, want %v", gotErr, tt.wantErr) + } else if gotErr != nil { + return } - gotL, wantL := len(tt.root.Clauses), len(tt.expected.Clauses) + got := slices.Collect(gotC.DFS()) + want := slices.Collect(tt.want.DFS()) + gotL, wantL := len(got), len(want) if gotL != wantL { - t.Errorf("Incorrect number of children clauses: got %d want %d\n", gotL, wantL) + t.Errorf("Different number of clauses than expected: got %d, want %d", gotL, wantL) } - }) - } -} -func TestParse(t *testing.T) { - tests := []struct { - name string // description of this test case - // Named input parameters for target function. - tokens []query.Token - want *query.Clause - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, gotErr := query.Parse(tt.tokens) - if gotErr != nil { - if !tt.wantErr { - t.Errorf("Parse() failed: %v", gotErr) + for i := range min(gotL, wantL) { + gotC, wantC := got[i], want[i] + + if gotC.Operator != wantC.Operator { + t.Error("Different clause operator than expected") + } else if !slices.EqualFunc(gotC.Statements, wantC.Statements, + func(s1, s2 query.Statement) bool { + return s1.Negated == s2.Negated && s1.Category == s2.Category && s1.Operator == s2.Operator && s1.Value.Compare(s2.Value) == 0 + }) { + t.Error("Different statements than expected") + } else if len(gotC.Clauses) != len(wantC.Clauses) { + t.Error("Different number of child clauses than expected") + } + + if t.Failed() { + t.Log("Got\n", gotC) + t.Log("Want\n", wantC) + break } - return - } - if tt.wantErr { - t.Fatal("Parse() succeeded unexpectedly") - } - // TODO: update the condition below to compare got with tt.want. - if true { - t.Errorf("Parse() = %v, want %v", got, tt.want) } }) } -- cgit v1.2.3