diff options
| -rw-r--r-- | cmd/atlas.go | 10 | ||||
| -rw-r--r-- | pkg/index/index.go | 37 | ||||
| -rw-r--r-- | pkg/index/index_test.go | 3 |
3 files changed, 31 insertions, 19 deletions
diff --git a/cmd/atlas.go b/cmd/atlas.go index f411a2b..3eabfbb 100644 --- a/cmd/atlas.go +++ b/cmd/atlas.go @@ -72,8 +72,8 @@ func main() { CustomFormat string }{} indexFlags := struct { - ignoreBadDates bool - Filters []index.DocFilter + Filters []index.DocFilter + index.ParseOpts }{} if len(args) < 1 { @@ -107,7 +107,9 @@ func main() { queryFs.Parse(args[1:]) case "index": - indexFs.BoolVar(&indexFlags.ignoreBadDates, "ignoreBadDates", false, "ignore malformed dates while indexing") + indexFs.BoolVar(&indexFlags.IgnoreDateError, "ignoreBadDates", false, "ignore malformed dates while indexing") + indexFs.BoolVar(&indexFlags.IgnoreMetaError, "ignoreMetaError", false, "ignore errors while parsing general YAML header info") + indexFs.BoolVar(&indexFlags.ParseMeta, "parseMeta", true, "parse YAML header values other title, authors, date, tags") customFilters := false indexFlags.Filters = index.DefaultFilters() @@ -198,7 +200,7 @@ func main() { filteredFiles := idx.Filter(traversedFiles, globalFlags.NumWorkers) fmt.Print(", Filtered ", len(filteredFiles)) - idx.Documents = index.ParseDocs(filteredFiles, globalFlags.NumWorkers, indexFlags.ignoreBadDates) + idx.Documents = index.ParseDocs(filteredFiles, globalFlags.NumWorkers, indexFlags.ParseOpts) fmt.Print(", Parsed ", len(idx.Documents), "\n") if err := querier.Put(idx); err != nil { diff --git a/pkg/index/index.go b/pkg/index/index.go index a96c227..419607c 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -27,6 +27,13 @@ type Document struct { Tags []string `yaml:"tags,omitempty" json:"tags"` Links []string `yaml:"-" json:"links"` OtherMeta string `yaml:"-" json:"meta"` + parseOpts ParseOpts +} + +type ParseOpts struct { + ParseMeta bool + IgnoreDateError bool + IgnoreMetaError bool } type infoPath struct { @@ -87,17 +94,21 @@ func (doc *Document) UnmarshalYAML(node ast.Node) error { } if keyPath == "$.date" { - if err := doc.parseDateNode(v); err != nil { + if err := doc.parseDateNode(v); err != nil && !doc.parseOpts.IgnoreDateError { return err } } else if keyPath == "$.author" { if err := doc.parseAuthor(v); err != nil { return err } - } else { + } else if doc.parseOpts.ParseMeta { field, err := kv.MarshalYAML() if err != nil { - return err + if doc.parseOpts.IgnoreMetaError { + continue + } else { + return err + } } buf.Write(field) buf.WriteByte('\n') @@ -314,9 +325,8 @@ func (idx Index) Filter(paths []string, numWorkers uint) []string { return fPaths } -func ParseDoc(path string) (*Document, error) { - doc := &Document{} - doc.Path = path +func ParseDoc(path string, opts ParseOpts) (*Document, error) { + doc := &Document{Path: path, parseOpts: opts} f, err := os.Open(path) if err != nil { @@ -336,7 +346,6 @@ func ParseDoc(path string) (*Document, error) { return nil, fmt.Errorf("Can't find YAML header in %s", path) } - // FIXME: decoder reads past yaml header into document if err := yaml.NewDecoder(io.LimitReader(f, pos)).Decode(doc); err != nil { return nil, errors.Join(ErrHeaderParse, err) } @@ -345,17 +354,17 @@ func ParseDoc(path string) (*Document, error) { return doc, nil } -func ParseDocs(paths []string, numWorkers uint) map[string]*Document { +func ParseDocs(paths []string, numWorkers uint, opts ParseOpts) map[string]*Document { jobs := make(chan string, numWorkers) - results := make(chan Document, numWorkers) + results := make(chan *Document, numWorkers) docs := make(map[string]*Document, len(paths)) wg := &sync.WaitGroup{} wg.Add(int(numWorkers)) for range numWorkers { - go func(jobs <-chan string, results chan<- Document, wg *sync.WaitGroup) { + go func(jobs <-chan string, results chan<- *Document, wg *sync.WaitGroup) { for path := range jobs { - doc, err := ParseDoc(path) + doc, err := ParseDoc(path, opts) if err != nil { // TODO: propagate error slog.Error("Error occured while parsing file", @@ -364,7 +373,7 @@ func ParseDocs(paths []string, numWorkers uint) map[string]*Document { continue } - results <- *doc + results <- doc } wg.Done() }(jobs, results, wg) @@ -377,13 +386,13 @@ func ParseDocs(paths []string, numWorkers uint) map[string]*Document { close(jobs) }(jobs, paths) - go func(results chan Document, wg *sync.WaitGroup) { + go func(results chan *Document, wg *sync.WaitGroup) { wg.Wait() close(results) }(results, wg) for doc := range results { - docs[doc.Path] = &doc + docs[doc.Path] = doc } return docs diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go index 4e55325..3f9b900 100644 --- a/pkg/index/index_test.go +++ b/pkg/index/index_test.go @@ -262,7 +262,8 @@ func TestIndex_ParseOne(t *testing.T) { path := tt.pathMaker(t) tt.want.Path = path - got, gotErr := ParseDoc(path) + // TODO: add ParseOpts as test param + got, gotErr := ParseDoc(path, ParseOpts{ParseMeta: true}) if !errors.Is(gotErr, tt.wantErr) { t.Errorf("Recieved unexpected error: want %v got %v", tt.wantErr, gotErr) |