1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
package main
import (
"context"
"flag"
"fmt"
"log/slog"
"os"
"strings"
"github.com/jpappel/atlas/pkg/data"
"github.com/jpappel/atlas/pkg/index"
)
type IndexFlags struct {
Filters []index.DocFilter
index.ParseOpts
}
func setupIndexFlags(args []string, fs *flag.FlagSet, flags *IndexFlags) {
fs.BoolVar(&flags.IgnoreDateError, "ignoreBadDates", false, "ignore malformed dates while indexing")
fs.BoolVar(&flags.IgnoreMetaError, "ignoreMetaError", false, "ignore errors while parsing general YAML header info")
fs.BoolVar(&flags.ParseMeta, "parseMeta", true, "parse YAML header values other title, authors, date, tags")
fs.BoolVar(&flags.ParseLinks, "parseLinks", true, "parse file contents for links")
customFilters := false
flags.Filters = index.DefaultFilters()
fs.Func("filter",
"accept or reject files from indexing, applied in supplied order"+
"\n(default Ext_.md, MaxSize_204800, YAMLHeader, ExcludeParent_templates)\n"+
index.FilterHelp,
func(s string) error {
if !customFilters {
flags.Filters = flags.Filters[:0]
}
filter, err := index.ParseFilter(s)
if err != nil {
return err
}
flags.Filters = append(flags.Filters, filter)
return nil
})
fs.Parse(args[1:])
}
func runIndex(gFlags GlobalFlags, iFlags IndexFlags, db *data.Query) byte {
idx := index.Index{Root: gFlags.IndexRoot, Filters: iFlags.Filters}
if slog.Default().Enabled(context.Background(), slog.LevelDebug) {
filterNames := make([]string, 0, len(iFlags.Filters))
for _, filter := range iFlags.Filters {
filterNames = append(filterNames, filter.Name)
}
slog.Default().Debug("index",
slog.String("indexRoot", gFlags.IndexRoot),
slog.String("filters", strings.Join(filterNames, ", ")),
)
}
traversedFiles := idx.Traverse(gFlags.NumWorkers)
fmt.Print("Crawled ", len(traversedFiles))
filteredFiles := idx.Filter(traversedFiles, gFlags.NumWorkers)
fmt.Print(", Filtered ", len(filteredFiles))
idx.Documents = index.ParseDocs(filteredFiles, gFlags.NumWorkers, iFlags.ParseOpts)
fmt.Print(", Parsed ", len(idx.Documents), "\n")
if err := db.Put(idx); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
return 0
}
|
