diff options
| -rw-r--r-- | README.md | 46 | ||||
| -rw-r--r-- | cmd/help.go | 155 | ||||
| -rw-r--r-- | cmd/index.go | 22 | ||||
| -rw-r--r-- | cmd/query.go | 77 | ||||
| -rw-r--r-- | cmd/server.go | 2 | ||||
| -rw-r--r-- | cmd/shell.go | 28 | ||||
| -rw-r--r-- | main.go | 42 | ||||
| -rw-r--r-- | pkg/index/index.go | 35 | ||||
| -rw-r--r-- | pkg/server/server.go | 32 | ||||
| -rw-r--r-- | pkg/shell/interpreter.go | 4 |
10 files changed, 299 insertions, 144 deletions
@@ -1,3 +1,49 @@ # Atlas A tool for querying markdown files with YAML metadata. + +## Build + +```bash +make +``` + +### Install + +Default installation path is `$HOME/.local/bin` + +```bash +make install +``` + +## Usage + +``` +atlas is a note indexing and querying tool + +Usage: + atlas [global-flags] <command> + +Commands: + index <subcommand> - build, update, or modify an index + query <subcommand> - search against an index + shell - start a debug shell + server - start an http query server (EXPERIMENTAL) + help <help-topic> - print help info + +Global Flags: + -dateFormat format + format for dates (see https://pkg.go.dev/time#Layout for more details) (default "2006-01-02T15:04:05Z07:00") + -db path + path to document database (default "/home/goose/.local/share/atlas/default.db") + -logFile file + file to log errors to, use '-' for stdout and empty for stderr + -logJson + log to json + -logLevel level + set log level (debug, info, warn, error) (default "error") + -numWorkers uint + number of worker threads to use (defaults to core count) (default 4) + -root directory + root directory for indexing (default "/home/goose/doc") +``` diff --git a/cmd/help.go b/cmd/help.go index b3844ac..e57ce2f 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -1,29 +1,146 @@ package cmd import ( + "flag" "fmt" + "io" "os" + + "github.com/jpappel/atlas/pkg/shell" + "github.com/jpappel/atlas/pkg/util" ) -var CommandHelp map[string]string - -func PrintHelp() { - fmt.Println("atlas is a note indexing and querying tool") - fmt.Printf("\nUsage:\n %s [global-flags] <command>\n\n", os.Args[0]) - fmt.Println("Commands:") - fmt.Println(" index - build, update, or modify an index") - fmt.Println(" query - search against an index") - fmt.Println(" shell - start a debug shell") - fmt.Println(" server - start an http query server (EXPERIMENTAL)") - fmt.Println(" help - print this help then exit") +var helpTopics = []string{ + "index", "i", + "index build", "i build", + "index update", "i update", + "index tidy", "i tidy", + "query", "q", + "shell", + "server", +} + +func PrintHelp(w io.Writer) { + fmt.Fprintln(w, "atlas is a note indexing and querying tool") + fmt.Fprintf(w, "\nUsage:\n %s [global-flags] <command>\n\n", os.Args[0]) + fmt.Fprintln(w, "Commands:") + fmt.Fprintln(w, " index <subcommand> - build, update, or modify an index") + fmt.Fprintln(w, " query <subcommand> - search against an index") + fmt.Fprintln(w, " shell - start a debug shell") + fmt.Fprintln(w, " server - start an http query server (EXPERIMENTAL)") + fmt.Fprintln(w, " help <help-topic> - print help info") } -func init() { - CommandHelp = make(map[string]string) - CommandHelp["query"] = "" - CommandHelp["index"] = "" - CommandHelp["server"] = "" - CommandHelp["completions"] = "" - CommandHelp["shell"] = "" - CommandHelp["help"] = "" +func PrintGlobalFlags(w io.Writer) { + fmt.Fprintln(w, "\nGlobal Flags:") + PrintFlagSet(w, flag.CommandLine) +} + +func PrintFlagSet(w io.Writer, fs *flag.FlagSet) { + w_ := fs.Output() + fs.SetOutput(w) + fs.PrintDefaults() + fs.SetOutput(w_) +} + +func Help(topic string, w io.Writer) { + fs := flag.NewFlagSet(topic, flag.ExitOnError) + switch topic { + case "index", "i": + SetupIndexFlags(nil, fs, &IndexFlags{}) + fmt.Fprintf(w, "%s [global-flags] index [index-flags] <subcommand>\n\n", os.Args[0]) + fmt.Fprintln(w, "Subcommands:") + fmt.Fprintln(w, " build - create a new index") + fmt.Fprintln(w, " update - update an existing index") + fmt.Fprintln(w, " tidy - cleanup an index") + fmt.Fprintf(w, "\nSee %s help index <subcommand> for subcommand help\n\n", os.Args[0]) + fmt.Fprintln(w, "Index Flags:") + PrintFlagSet(w, fs) + case "i build", "index build": + fmt.Fprintf(w, "%s [global-flags] index [index-flags] build\n\n", os.Args[0]) + fmt.Fprintln(w, "Crawl files starting at `-root` to build an index stored in `-db`") + fmt.Fprintln(w, "Use this subcommand to generate the initial index, then update it with `atlas index update`") + case "i update", "index update": + fmt.Fprintf(w, "%s [global-flags] index [index-flags] update\n\n", os.Args[0]) + fmt.Fprintln(w, "Crawl files starting at `-root` to update an index stored in `-db`") + fmt.Fprintln(w, "Use this subcommand to update an existing index.") + fmt.Fprintln(w, "Deleted documents are removed from the index. To remove unused authors and tags run `atlas index tidy`") + case "i tidy", "index tidy": + fmt.Fprintf(w, "%s [global-flags] index tidy\n\n", os.Args[0]) + fmt.Fprintln(w, "Remove unused authors or tags and optimize the database") + case "query", "q": + SetupQueryFlags(nil, fs, &QueryFlags{}, "") + fmt.Fprintf(w, "%s [global-flags] query [query-flags] <query>...\n\n", os.Args[0]) + fmt.Fprintln(w, "Execute a query against the connected database") + fmt.Fprintln(w, "Query Flags:") + PrintFlagSet(w, fs) + fmt.Fprintln(w, "\nOutput Format:") + help := `The output format of query results can be customized by setting -outCustomFormat. + + The output of each document has the value of -docSeparator appended to it. + Dates are formated using -dateFormat + Lists use -listSeparator to delimit elements + + Placeholder - Type - Value + %p - Str - path + %T - Str - title + %d - Date - date + %f - Date - filetime + %a - List - authors + %t - List - tags + %l - List - links + %m - Str - meta + + Examples: + "%p %T %d tags:%t" -> '/a/path/to/document A Title 2006-01-02T15:04:05Z07:00 tags:tag1, tag2\n' + "<h1><a href="%p">%T</a></h1>" -> '<h1><a href="/a/path/to/document">A Title</a></h1>\n' + +` + fmt.Fprint(w, help) + case "shell": + fmt.Fprintf(w, "%s [global-flags] shell\n", os.Args[0]) + fmt.Fprintln(w, "Simple shell for debugging queries") + fmt.Fprintln(w, "\nShell Help:") + shell.PrintHelp(w) + case "server": + SetupServerFlags(nil, fs, &ServerFlags{}) + fmt.Fprintf(w, "%s [global-flags] server [server-flags]", os.Args[0]) + fmt.Fprintln(w, "Run a server to execute queries over HTTP or a unix domain socket") + fmt.Fprintln(w, "HTTP Server:") + fmt.Fprintln(w, " To execute a query POST it in the request body to /search") + fmt.Fprintln(w, " ex. curl -d 'T:notes d>=\"January 1, 2025\"' 127.0.0.1:8080/search") + fmt.Fprintln(w, " To have the backend use the query params `sortBy` and `sortOrder`") + fmt.Fprintln(w, " sortBy: path, title, date, filetime, meta") + fmt.Fprintln(w, " sortOrder: desc, descending") + fmt.Fprintln(w, "Server Flags:") + PrintFlagSet(w, fs) + case "help", "": + PrintHelp(w) + fmt.Fprintln(w, "\nHelp Topics:") + curLineLen := 2 + fmt.Fprint(w, " ") + for i, topic := range helpTopics { + if curLineLen+len(topic) < 80 { + curLineLen += len(topic) + fmt.Fprint(w, topic) + } else { + fmt.Fprintln(w, topic) + fmt.Fprint(w, " ") + curLineLen = 2 + } + if i == len(helpTopics)-1 { + fmt.Fprintln(w) + } else if curLineLen != 2 { + fmt.Fprint(w, ", ") + curLineLen += 3 + } + } + PrintGlobalFlags(w) + default: + fmt.Fprintln(os.Stderr, "Unrecognized topic: ", topic) + if suggestion, ok := util.Nearest(topic, helpTopics, util.LevensteinDistance, 3); ok { + fmt.Fprintf(w, "Did you mean %s?\n", suggestion) + } + fmt.Fprintln(w, "See `atlas help`") + } } diff --git a/cmd/index.go b/cmd/index.go index 5454a8a..358182c 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -23,7 +23,7 @@ func SetupIndexFlags(args []string, fs *flag.FlagSet, flags *IndexFlags) { flags.ParseMeta = true fs.BoolVar(&flags.IgnoreDateError, "ignoreBadDates", false, "ignore malformed dates while indexing") fs.BoolVar(&flags.IgnoreMetaError, "ignoreMetaError", false, "ignore errors while parsing general YAML header info") - fs.BoolFunc("ignoreMeta", "don't parse YAML header values other title, authors, date, tags", func(s string) error { + fs.BoolFunc("ignoreMeta", "only parse title, authors, date, tags from YAML headers", func(s string) error { flags.ParseMeta = false return nil }) @@ -33,20 +33,6 @@ func SetupIndexFlags(args []string, fs *flag.FlagSet, flags *IndexFlags) { }) fs.BoolVar(&flags.IgnoreHidden, "ignoreHidden", false, "ignore hidden files while crawling") - fs.Usage = func() { - f := fs.Output() - fmt.Fprintf(f, "Usage of %s %s\n", os.Args[0], fs.Name()) - fmt.Fprintf(f, " %s [global-flags] %s [index-flags] <subcommand>\n\n", os.Args[0], fs.Name()) - fmt.Fprintln(f, "Subcommands:") - fmt.Fprintln(f, "build - create a new index") - fmt.Fprintln(f, "update - update an existing index") - fmt.Fprintln(f, "tidy - cleanup an index") - fmt.Fprintln(f, "\nIndex Flags:") - fs.PrintDefaults() - fmt.Fprintln(f, "\nGlobal Flags:") - flag.PrintDefaults() - } - customFilters := false flags.Filters = index.DefaultFilters() fs.Func("filter", @@ -67,6 +53,12 @@ func SetupIndexFlags(args []string, fs *flag.FlagSet, flags *IndexFlags) { return nil }) + fs.Usage = func() { + f := fs.Output() + Help("index", f) + PrintGlobalFlags(f) + } + fs.Parse(args) remainingArgs := fs.Args() diff --git a/cmd/query.go b/cmd/query.go index d00a792..4b28e09 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -3,10 +3,8 @@ package cmd import ( "flag" "fmt" - "log/slog" "os" "slices" - "strings" "github.com/jpappel/atlas/pkg/data" "github.com/jpappel/atlas/pkg/index" @@ -47,43 +45,17 @@ func SetupQueryFlags(args []string, fs *flag.FlagSet, flags *QueryFlags, dateFor fs.StringVar(&flags.SortBy, "sortBy", "", "category to sort by (path,title,date,filetime,meta)") fs.BoolVar(&flags.SortDesc, "sortDesc", false, "sort in descending order") - fs.StringVar(&flags.CustomFormat, "outCustomFormat", query.DefaultOutputFormat, "format string for --outFormat custom, see Output Format for more details") + fs.StringVar(&flags.CustomFormat, "outCustomFormat", query.DefaultOutputFormat, "`format` string for --outFormat custom, see `atlas help query` for more details") fs.IntVar(&flags.OptimizationLevel, "optLevel", 0, "optimization `level` for queries, 0 is automatic, <0 to disable") fs.StringVar(&flags.DocumentSeparator, "docSeparator", "\n", "separator for custom output format") fs.StringVar(&flags.ListSeparator, "listSeparator", ", ", "separator for list fields") fs.Usage = func() { - f := fs.Output() - fmt.Fprintf(f, "Usage of %s %s\n", os.Args[0], fs.Name()) - fmt.Fprintf(f, " %s [global-flags] %s [query-flags]\n\n", - os.Args[0], fs.Name()) - fmt.Fprintln(f, "Query Flags:") - fs.PrintDefaults() - fmt.Fprintln(f, "\nOutput Format:") - help := `The output format of query results can be customized by setting -outCustomFormat. - - The output of each document has the value of -docSeparator appended to it. - Dates are formated using -dateFormat - Lists use -listSeparator to delimit elements - - Placeholder - Type - Value - %p - Str - path - %T - Str - title - %d - Date - date - %f - Date - filetime - %a - List - authors - %t - List - tags - %l - List - links - %m - Str - meta - - Examples: - "%p %T %d tags:%t" -> '/a/path/to/document A Title 2006-01-02T15:04:05Z07:00 tags:tag1, tag2\n' - "<h1><a href="%p">%T</a></h1>" -> '<h1><a href="/a/path/to/document">A Title</a></h1>\n' - -` - fmt.Fprint(f, help) - fmt.Fprintln(f, "Global Flags:") - flag.PrintDefaults() + w := fs.Output() + fmt.Fprintf(w, "%s [global-flags] query [query-flags] <query>...\n\n", os.Args[0]) + fmt.Fprintln(w, "Query Flags:") + PrintFlagSet(w, fs) + PrintGlobalFlags(w) } fs.Parse(args) @@ -122,40 +94,11 @@ func RunQuery(gFlags GlobalFlags, qFlags QueryFlags, db *data.Query, searchQuery outputableResults = append(outputableResults, v) } - var docCmp func(a, b *index.Document) int - descMod := 1 - if qFlags.SortDesc { - descMod = -1 - } - switch qFlags.SortBy { - case "": - case "path": - docCmp = func(a, b *index.Document) int { - return descMod * strings.Compare(a.Path, b.Path) - } - case "title": - docCmp = func(a, b *index.Document) int { - return descMod * strings.Compare(a.Title, b.Title) - } - case "date": - docCmp = func(a, b *index.Document) int { - return descMod * a.Date.Compare(b.Date) - } - case "filetime": - docCmp = func(a, b *index.Document) int { - return descMod * a.FileTime.Compare(b.FileTime) - } - case "meta": - docCmp = func(a, b *index.Document) int { - return descMod * strings.Compare(a.OtherMeta, b.OtherMeta) - } - default: - slog.Error("Unrecognized category to sort by, leaving documents unsorted") - qFlags.SortBy = "" - } - if qFlags.SortBy != "" { - slices.SortFunc(outputableResults, docCmp) + docCmp, ok := index.NewDocCmp(qFlags.SortBy, qFlags.SortDesc) + if ok { + slices.SortFunc(outputableResults, docCmp) + } } _, err = qFlags.Outputer.OutputTo(os.Stdout, outputableResults) diff --git a/cmd/server.go b/cmd/server.go index 6182221..45fe197 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -22,7 +22,7 @@ type ServerFlags struct { } func SetupServerFlags(args []string, fs *flag.FlagSet, flags *ServerFlags) { - fs.StringVar(&flags.Address, "address", "", "the address to listen on, prefix with 'unix:' to create a unixsocket") + fs.StringVar(&flags.Address, "address", "127.0.0.1", "the address to listen on, prefix with 'unix:' to create a unixsocket") fs.IntVar(&flags.Port, "port", 8080, "the port to bind to") fs.Parse(args) diff --git a/cmd/shell.go b/cmd/shell.go new file mode 100644 index 0000000..218ecd0 --- /dev/null +++ b/cmd/shell.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "fmt" + "io" + "log/slog" + + "github.com/jpappel/atlas/pkg/data" + "github.com/jpappel/atlas/pkg/shell" +) + +func RunShell(gFlags GlobalFlags, db *data.Query, version string) byte { + state := make(shell.State) + env := make(map[string]string) + + env["workers"] = fmt.Sprint(gFlags.NumWorkers) + env["db_path"] = gFlags.DBPath + env["index_root"] = gFlags.IndexRoot + env["version"] = version + + interpreter := shell.NewInterpreter(state, env, gFlags.NumWorkers, db) + if err := interpreter.Run(); err != nil && err != io.EOF { + slog.Error("Fatal error occured", slog.String("err", err.Error())) + return 1 + } + + return 0 +} @@ -3,18 +3,13 @@ package main import ( "flag" "fmt" - "io" "log/slog" - "maps" "os" - "slices" "strings" "github.com/jpappel/atlas/cmd" "github.com/jpappel/atlas/pkg/data" "github.com/jpappel/atlas/pkg/query" - "github.com/jpappel/atlas/pkg/shell" - "github.com/jpappel/atlas/pkg/util" ) const VERSION = "0.4.1" @@ -53,9 +48,8 @@ func main() { if len(args) < 1 { fmt.Fprintln(os.Stderr, "No Command provided") - cmd.PrintHelp() - fmt.Fprintln(flag.CommandLine.Output(), "\nGlobal Flags:") - flag.PrintDefaults() + cmd.PrintHelp(os.Stderr) + cmd.PrintGlobalFlags(os.Stderr) os.Exit(ExitCommand) } command := args[0] @@ -70,22 +64,16 @@ func main() { case "completions": completionsFs.Parse(args[1:]) case "help": - cmd.PrintHelp() - flag.PrintDefaults() + if len(args) > 1 { + cmd.Help(strings.Join(args[1:], " "), os.Stdout) + } else { + cmd.Help("", os.Stdout) + } return case "shell": shellFs.Parse(args[1:]) default: - fmt.Fprintln(os.Stderr, "Unrecognized command: ", command) - suggestedCommand, ok := util.Nearest( - command, - slices.Collect(maps.Keys(cmd.CommandHelp)), - util.LevensteinDistance, 3, - ) - if ok { - fmt.Fprintf(os.Stderr, "Did you mean %s?\n\n", suggestedCommand) - } - cmd.PrintHelp() + cmd.Help(command, os.Stderr) os.Exit(ExitCommand) } @@ -162,19 +150,7 @@ func main() { exitCode = 2 } case "shell": - state := make(shell.State) - env := make(map[string]string) - - env["workers"] = fmt.Sprint(globalFlags.NumWorkers) - env["db_path"] = globalFlags.DBPath - env["index_root"] = globalFlags.IndexRoot - env["version"] = VERSION - - interpreter := shell.NewInterpreter(state, env, globalFlags.NumWorkers, querier) - if err := interpreter.Run(); err != nil && err != io.EOF { - slog.Error("Fatal error occured", slog.String("err", err.Error())) - exitCode = 1 - } + exitCode = int(cmd.RunShell(globalFlags, querier, VERSION)) } querier.Close() diff --git a/pkg/index/index.go b/pkg/index/index.go index 50f642e..42f5051 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -336,6 +336,39 @@ func (idx Index) Filter(paths []string, numWorkers uint) []string { return fPaths } +// Create a comparison function for documents by field. +// Allowed fields: path,title,date,filetime,meta +func NewDocCmp(field string, reverse bool) (func(*Document, *Document) int, bool) { + descMod := 1 + if reverse { + descMod = -1 + } + switch field { + case "path": + return func(a, b *Document) int { + return descMod * strings.Compare(a.Path, b.Path) + }, true + case "title": + return func(a, b *Document) int { + return descMod * strings.Compare(a.Title, b.Title) + }, true + case "date": + return func(a, b *Document) int { + return descMod * a.Date.Compare(b.Date) + }, true + case "filetime": + return func(a, b *Document) int { + return descMod * a.FileTime.Compare(b.FileTime) + }, true + case "meta": + return func(a, b *Document) int { + return descMod * strings.Compare(a.OtherMeta, b.OtherMeta) + }, true + } + + return nil, false +} + func ParseDoc(path string, opts ParseOpts) (*Document, error) { doc := &Document{Path: path, parseOpts: opts} @@ -430,5 +463,5 @@ func ParseDocs(paths []string, numWorkers uint, opts ParseOpts) (map[string]*Doc } func init() { - linkRegex = regexp.MustCompile(`\[.*\]\(\s*(\S+)\s*\)`) + linkRegex = regexp.MustCompile(`\[.*\]\(\s*([^\)]+)\s*\)`) } diff --git a/pkg/server/server.go b/pkg/server/server.go index a7a5395..68750a2 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,6 +6,7 @@ import ( "io" "log/slog" "net/http" + "slices" "strings" "sync" "time" @@ -21,11 +22,20 @@ type Server interface { } func info(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(` - <h1>Atlas Server</h1> - <p>This is the experimental atlas server! - Try POSTing a query to <pre>/search</pre></p> - `)) + w.Write([]byte(`<h1>Atlas Server</h1> +<p>This is the experimental atlas server! +Try POSTing a query to <pre>/search</pre></p> +<hr> +<p>You can sort the results using the query param <pre>sortBy</pre> +<ul> +<li>path</li> +<li>title</li> +<li>date</li> +<li>filetime</li> +<li>meta</li> +</ul> +You can change the order using <pre>sortOrder</pre> with asc or desc +</p>`)) } func NewMux(db *data.Query) *http.ServeMux { @@ -37,7 +47,7 @@ func NewMux(db *data.Query) *http.ServeMux { } mux.HandleFunc("/", info) - mux.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("POST /search", func(w http.ResponseWriter, r *http.Request) { b := &strings.Builder{} if _, err := io.Copy(b, r.Body); err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -70,6 +80,16 @@ func NewMux(db *data.Query) *http.ServeMux { } } + queryParams := r.URL.Query() + if queryParams.Has("sortBy") { + sortBy := queryParams.Get("sortBy") + sortOrder := queryParams.Get("sortOrder") + docCmp, ok := index.NewDocCmp(sortBy, sortOrder == "desc" || sortOrder == "descending") + if ok { + slices.SortFunc(docs, docCmp) + } + } + if !maxFileTime.IsZero() { w.Header().Add("Last-Modified", maxFileTime.UTC().Format(http.TimeFormat)) } diff --git a/pkg/shell/interpreter.go b/pkg/shell/interpreter.go index c222de0..bb0a441 100644 --- a/pkg/shell/interpreter.go +++ b/pkg/shell/interpreter.go @@ -183,7 +183,7 @@ out: } switch t.Type { case ITOK_CMD_HELP: - printHelp(w) + PrintHelp(w) break out case ITOK_CMD_EXIT: return true, nil @@ -755,7 +755,7 @@ func (inter Interpreter) Tokenize(line string) []IToken { return tokens } -func printHelp(w io.Writer) { +func PrintHelp(w io.Writer) { fmt.Fprintln(w, "Shitty debug shell for atlas") fmt.Fprintln(w, "help - print this help") fmt.Fprintln(w, "exit - exit interactive mode") |