From a4d86693394ba9b181b5928c1c6e8c31c9bb2b64 Mon Sep 17 00:00:00 2001 From: JP Appel Date: Mon, 30 Jun 2025 17:18:45 -0400 Subject: Implement compiled query execution --- cmd/atlas.go | 133 +++++++---------------------------------------------------- cmd/index.go | 75 +++++++++++++++++++++++++++++++++ cmd/query.go | 84 +++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 118 deletions(-) create mode 100644 cmd/index.go create mode 100644 cmd/query.go (limited to 'cmd') diff --git a/cmd/atlas.go b/cmd/atlas.go index 08e1a00..aa01bac 100644 --- a/cmd/atlas.go +++ b/cmd/atlas.go @@ -1,7 +1,6 @@ package main import ( - "context" "errors" "flag" "fmt" @@ -15,11 +14,11 @@ import ( "github.com/adrg/xdg" "github.com/jpappel/atlas/pkg/data" - "github.com/jpappel/atlas/pkg/index" "github.com/jpappel/atlas/pkg/query" "github.com/jpappel/atlas/pkg/shell" ) +const VERSION = "0.0.1" const ExitCommand = 2 // exit because of a command parsing error const dateFormat = time.RFC3339 // TODO: make a flag @@ -81,15 +80,8 @@ func main() { flag.Parse() args := flag.Args() - queryFlags := struct { - Output query.Outputer - CustomFormat string - OptimizationLevel int - }{} - indexFlags := struct { - Filters []index.DocFilter - index.ParseOpts - }{} + queryFlags := QueryFlags{Outputer: query.DefaultOutput{}} + indexFlags := IndexFlags{} if len(args) < 1 { fmt.Fprintln(os.Stderr, "No Command provided") @@ -101,54 +93,10 @@ func main() { command := args[0] switch command { - case "query": - // NOTE: providing `-outFormat` before `-outCustomFormat` might ignore user specified format - queryFs.Func("outFormat", "output `format` for queries (default, json, custom)", - func(arg string) error { - switch arg { - case "default": - queryFlags.Output = query.DefaultOutput{} - return nil - case "json": - queryFlags.Output = query.JsonOutput{} - return nil - case "custom": - var err error - queryFlags.Output, err = query.NewCustomOutput(queryFlags.CustomFormat, dateFormat) - return err - } - return fmt.Errorf("Unrecognized output format: %s", arg) - }) - queryFs.StringVar(&queryFlags.CustomFormat, "outCustomFormat", query.DefaultOutputFormat, "format string for --outFormat custom, see EXAMPLES for more details") - queryFs.IntVar(&queryFlags.OptimizationLevel, "optLevel", 0, "optimization `level` for queries, 0 is automatic, <0 to disable") - - queryFs.Parse(args[1:]) + case "query", "q": + setupQueryFlags(args, queryFs, &queryFlags) case "index": - indexFs.BoolVar(&indexFlags.IgnoreDateError, "ignoreBadDates", false, "ignore malformed dates while indexing") - indexFs.BoolVar(&indexFlags.IgnoreMetaError, "ignoreMetaError", false, "ignore errors while parsing general YAML header info") - indexFs.BoolVar(&indexFlags.ParseMeta, "parseMeta", true, "parse YAML header values other title, authors, date, tags") - - customFilters := false - indexFlags.Filters = index.DefaultFilters() - indexFs.Func("filter", - "accept or reject files from indexing, applied in supplied order"+ - "\n(default Ext_.md, MaxSize_204800, YAMLHeader, ExcludeParent_templates)\n"+ - index.FilterHelp, - func(s string) error { - if !customFilters { - indexFlags.Filters = indexFlags.Filters[:0] - } - - filter, err := index.ParseFilter(s) - if err != nil { - return err - } - indexFlags.Filters = append(indexFlags.Filters, filter) - - return nil - }) - - indexFs.Parse(args[1:]) + setupIndexFlags(args, indexFs, &indexFlags) case "help": printHelp() flag.PrintDefaults() @@ -186,68 +134,15 @@ func main() { slog.SetDefault(logger) querier := data.NewQuery(globalFlags.DBPath) - defer querier.Close() - - go func() { - if r := recover(); r != nil { - os.Exit(1) - } - }() // command specific + var exitCode int switch command { - case "query": + case "query", "q": searchQuery := strings.Join(queryFs.Args(), " ") - tokens := query.Lex(searchQuery) - clause, err := query.Parse(tokens) - if err != nil { - fmt.Fprintln(os.Stderr, "Failed to parse query: ", err) - panic(err) - } - - if queryFlags.OptimizationLevel >= 0 { - o := query.NewOptimizer(clause, globalFlags.NumWorkers) - o.Optimize(queryFlags.OptimizationLevel) - } - - artifact, err := clause.Compile() - if err != nil { - panic(err) - } - fmt.Println("query\n", artifact.Query) - fmt.Println("args\n", strings.Join(artifact.Args, ", ")) - // TODO: evaluate query - // s, err := queryFlags.Output.Output(nil) - // if err != nil { - // slog.Error("Error while outputing query results", slog.String("err", err.Error())) - // return - // } - // fmt.Print(s) + exitCode = int(runQuery(globalFlags, queryFlags, querier, searchQuery)) case "index": - idx := index.Index{Root: globalFlags.IndexRoot, Filters: indexFlags.Filters} - if logger.Enabled(context.Background(), slog.LevelDebug) { - filterNames := make([]string, 0, len(indexFlags.Filters)) - for _, filter := range indexFlags.Filters { - filterNames = append(filterNames, filter.Name) - } - logger.Debug("index", - slog.String("indexRoot", globalFlags.IndexRoot), - slog.String("filters", strings.Join(filterNames, ", ")), - ) - } - - traversedFiles := idx.Traverse(globalFlags.NumWorkers) - fmt.Print("Crawled ", len(traversedFiles)) - - filteredFiles := idx.Filter(traversedFiles, globalFlags.NumWorkers) - fmt.Print(", Filtered ", len(filteredFiles)) - - idx.Documents = index.ParseDocs(filteredFiles, globalFlags.NumWorkers, indexFlags.ParseOpts) - fmt.Print(", Parsed ", len(idx.Documents), "\n") - - if err := querier.Put(idx); err != nil { - panic(err) - } + exitCode = int(runIndex(globalFlags, indexFlags, querier)) case "shell": state := make(shell.State) env := make(map[string]string) @@ -255,13 +150,15 @@ func main() { env["workers"] = fmt.Sprint(globalFlags.NumWorkers) env["db_path"] = globalFlags.DBPath env["index_root"] = globalFlags.IndexRoot - env["version"] = "0.0.1" + env["version"] = VERSION - interpreter := shell.NewInterpreter(state, env, globalFlags.NumWorkers) + interpreter := shell.NewInterpreter(state, env, globalFlags.NumWorkers, querier) if err := interpreter.Run(); err != nil && err != io.EOF { slog.Error("Fatal error occured", slog.String("err", err.Error())) - panic(err) + exitCode = 1 } } + querier.Close() + os.Exit(exitCode) } diff --git a/cmd/index.go b/cmd/index.go new file mode 100644 index 0000000..2f45f78 --- /dev/null +++ b/cmd/index.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log/slog" + "os" + "strings" + + "github.com/jpappel/atlas/pkg/data" + "github.com/jpappel/atlas/pkg/index" +) + +type IndexFlags struct { + Filters []index.DocFilter + index.ParseOpts +} + +func setupIndexFlags(args []string, fs *flag.FlagSet, flags *IndexFlags) { + fs.BoolVar(&flags.IgnoreDateError, "ignoreBadDates", false, "ignore malformed dates while indexing") + fs.BoolVar(&flags.IgnoreMetaError, "ignoreMetaError", false, "ignore errors while parsing general YAML header info") + fs.BoolVar(&flags.ParseMeta, "parseMeta", true, "parse YAML header values other title, authors, date, tags") + + customFilters := false + flags.Filters = index.DefaultFilters() + fs.Func("filter", + "accept or reject files from indexing, applied in supplied order"+ + "\n(default Ext_.md, MaxSize_204800, YAMLHeader, ExcludeParent_templates)\n"+ + index.FilterHelp, + func(s string) error { + if !customFilters { + flags.Filters = flags.Filters[:0] + } + + filter, err := index.ParseFilter(s) + if err != nil { + return err + } + flags.Filters = append(flags.Filters, filter) + + return nil + }) + + fs.Parse(args[1:]) +} + +func runIndex(gFlags GlobalFlags, iFlags IndexFlags, db *data.Query) byte { + idx := index.Index{Root: gFlags.IndexRoot, Filters: iFlags.Filters} + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + filterNames := make([]string, 0, len(iFlags.Filters)) + for _, filter := range iFlags.Filters { + filterNames = append(filterNames, filter.Name) + } + slog.Default().Debug("index", + slog.String("indexRoot", gFlags.IndexRoot), + slog.String("filters", strings.Join(filterNames, ", ")), + ) + } + + traversedFiles := idx.Traverse(gFlags.NumWorkers) + fmt.Print("Crawled ", len(traversedFiles)) + + filteredFiles := idx.Filter(traversedFiles, gFlags.NumWorkers) + fmt.Print(", Filtered ", len(filteredFiles)) + + idx.Documents = index.ParseDocs(filteredFiles, gFlags.NumWorkers, iFlags.ParseOpts) + fmt.Print(", Parsed ", len(idx.Documents), "\n") + + if err := db.Put(idx); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + return 0 +} diff --git a/cmd/query.go b/cmd/query.go new file mode 100644 index 0000000..40c9e58 --- /dev/null +++ b/cmd/query.go @@ -0,0 +1,84 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/jpappel/atlas/pkg/data" + "github.com/jpappel/atlas/pkg/index" + "github.com/jpappel/atlas/pkg/query" +) + +type QueryFlags struct { + Outputer query.Outputer + CustomFormat string + OptimizationLevel int +} + +func setupQueryFlags(args []string, fs *flag.FlagSet, flags *QueryFlags) { + // NOTE: providing `-outFormat` before `-outCustomFormat` might ignore user specified format + fs.Func("outFormat", "output `format` for queries (default, json, custom)", + func(arg string) error { + switch arg { + case "default": + flags.Outputer = query.DefaultOutput{} + return nil + case "json": + flags.Outputer = query.JsonOutput{} + return nil + case "custom": + var err error + flags.Outputer, err = query.NewCustomOutput(flags.CustomFormat, dateFormat) + return err + } + return fmt.Errorf("Unrecognized output format: %s", arg) + }) + fs.StringVar(&flags.CustomFormat, "outCustomFormat", query.DefaultOutputFormat, "format string for --outFormat custom, see EXAMPLES for more details") + fs.IntVar(&flags.OptimizationLevel, "optLevel", 0, "optimization `level` for queries, 0 is automatic, <0 to disable") + + fs.Parse(args[1:]) +} + +func runQuery(gFlags GlobalFlags, qFlags QueryFlags, db *data.Query, searchQuery string) byte { + tokens := query.Lex(searchQuery) + clause, err := query.Parse(tokens) + if err != nil { + fmt.Fprintln(os.Stderr, "Failed to parse query: ", err) + return 1 + } + + o := query.NewOptimizer(clause, gFlags.NumWorkers) + o.Optimize(qFlags.OptimizationLevel) + + artifact, err := clause.Compile() + if err != nil { + fmt.Fprintln(os.Stderr, "Failed to compile query: ", err) + return 1 + } + + results, err := db.Execute(artifact) + if err != nil { + fmt.Fprintln(os.Stderr, "Failed to execute query: ", err) + return 1 + } + + if len(results) == 0 { + fmt.Println("No results.") + return 0 + } + + outputableResults := make([]*index.Document, 0, len(results)) + for _, v := range results { + outputableResults = append(outputableResults, v) + } + + s, err := qFlags.Outputer.Output(outputableResults) + if err != nil { + fmt.Fprintln(os.Stderr, "Failed to output results: ", err) + return 1 + } + + fmt.Println(s) + return 0 +} -- cgit v1.2.3