From 10d5e4db7ad9fe9855a19977f7bc037b058bfead Mon Sep 17 00:00:00 2001 From: JP Appel Date: Tue, 5 Aug 2025 12:40:15 -0400 Subject: Add headings to db serializer and deserializer --- pkg/data/get.go | 23 ++++++++++++++++------- pkg/data/get_test.go | 11 +++++++---- pkg/data/put.go | 29 ++++++++++++----------------- pkg/data/put_test.go | 1 + pkg/data/update.go | 20 ++++++++++++++------ pkg/data/update_test.go | 7 +++++++ 6 files changed, 57 insertions(+), 34 deletions(-) diff --git a/pkg/data/get.go b/pkg/data/get.go index 396f132..7025c3a 100644 --- a/pkg/data/get.go +++ b/pkg/data/get.go @@ -66,14 +66,15 @@ func (f *Fill) document(ctx context.Context) error { var title sql.NullString var dateEpoch sql.NullInt64 var fileTimeEpoch sql.NullInt64 + var headings sql.NullString var meta sql.NullString row := f.Db.QueryRowContext(ctx, ` - SELECT id, title, date, fileTime, meta + SELECT id, title, date, fileTime, headings, meta FROM Documents WHERE path = ? `, f.Path) - if err := row.Scan(&f.id, &title, &dateEpoch, &fileTimeEpoch, &meta); err != nil { + if err := row.Scan(&f.id, &title, &dateEpoch, &fileTimeEpoch, &headings, &meta); err != nil { return err } @@ -86,6 +87,9 @@ func (f *Fill) document(ctx context.Context) error { if fileTimeEpoch.Valid { f.doc.FileTime = time.Unix(fileTimeEpoch.Int64, 0) } + if headings.Valid { + f.doc.Headings = headings.String + } if meta.Valid { f.doc.OtherMeta = meta.String } @@ -98,7 +102,7 @@ func (f *FillMany) documents(ctx context.Context, rows *sql.Rows) error { if rows == nil { var err error rows, err = f.Db.QueryContext(ctx, ` - SELECT id, path, title, date, fileTime, meta + SELECT id, path, title, date, fileTime, headings, meta FROM Documents `) if err != nil { @@ -107,7 +111,7 @@ func (f *FillMany) documents(ctx context.Context, rows *sql.Rows) error { defer rows.Close() } else if cols, err := rows.ColumnTypes(); err != nil { return err - } else if len(cols) != 6 { + } else if len(cols) != 7 { return fmt.Errorf("Not enough columns to fill documents with") } else if t := cols[0].DatabaseTypeName(); t != "INTEGER" { return fmt.Errorf("Expected integer for id column fill, got %s", t) @@ -119,17 +123,19 @@ func (f *FillMany) documents(ctx context.Context, rows *sql.Rows) error { return fmt.Errorf("Expected integer for date column fill, got %s", t) } else if t := cols[4].DatabaseTypeName(); t != "INT" { return fmt.Errorf("Expected integer for filetime column fill, got %s", t) - } else if t := cols[5].DatabaseTypeName(); t != "BLOB" { + } else if t := cols[5].DatabaseTypeName(); t != "TEXT" { + return fmt.Errorf("Expected text for headings column fill, got %s", t) + } else if t := cols[6].DatabaseTypeName(); t != "BLOB" { return fmt.Errorf("Expected text for meta column fill, got %s", t) } var id int var docPath string - var title, meta sql.NullString + var title, headings, meta sql.NullString var dateEpoch, filetimeEpoch sql.NullInt64 for rows.Next() { - if err := rows.Scan(&id, &docPath, &title, &dateEpoch, &filetimeEpoch, &meta); err != nil { + if err := rows.Scan(&id, &docPath, &title, &dateEpoch, &filetimeEpoch, &headings, &meta); err != nil { return err } @@ -146,6 +152,9 @@ func (f *FillMany) documents(ctx context.Context, rows *sql.Rows) error { if filetimeEpoch.Valid { doc.FileTime = time.Unix(filetimeEpoch.Int64, 0) } + if headings.Valid { + doc.Headings = headings.String + } if meta.Valid { doc.OtherMeta = meta.String } diff --git a/pkg/data/get_test.go b/pkg/data/get_test.go index 9ee8a2a..4cadc76 100644 --- a/pkg/data/get_test.go +++ b/pkg/data/get_test.go @@ -15,8 +15,8 @@ func singleDoc(t *testing.T) *sql.DB { db := data.NewMemDB("test") if _, err := db.Exec(` - INSERT INTO Documents (path, title, date, fileTime) - VALUES ("/file", "A file", 1, 2) + INSERT INTO Documents (path, title, date, fileTime, headings) + VALUES ("/file", "A file", 1, 2, '# A Heading' || CHAR(10)) `); err != nil { t.Fatal("err inserting doc:", err) } @@ -71,8 +71,9 @@ func multiDoc(t *testing.T) *sql.DB { db := data.NewMemDB("test") if _, err := db.Exec(` - INSERT INTO Documents (path, title, date, fileTime) - VALUES ("/notes/anote.md", "A note", 1, 2), ("README.md", "read this file!", 3, 4) + INSERT INTO Documents (path, title, date, fileTime, headings) + VALUES ("/notes/anote.md", "A note", 1, 2, '#Top Level' || CHAR(10) || '## Second Level' || CHAR(10)), + ("README.md", "read this file!", 3, 4, NULL) `); err != nil { t.Fatal("err inserting doc:", err) } @@ -143,6 +144,7 @@ func TestFill_Get(t *testing.T) { Authors: []string{"jp"}, Tags: []string{"foo", "bar", "oof", "baz"}, Links: []string{"link1", "link2"}, + Headings: "# A Heading\n", }, nil, }, @@ -186,6 +188,7 @@ func TestFillMany_Get(t *testing.T) { FileTime: time.Unix(2, 0), Authors: []string{"jp"}, Tags: []string{"foo", "baz"}, + Headings: "#Top Level\n## Second Level\n", Links: []string{"/home"}, }, "README.md": { diff --git a/pkg/data/put.go b/pkg/data/put.go index 6ceccc3..4a9a039 100644 --- a/pkg/data/put.go +++ b/pkg/data/put.go @@ -103,17 +103,15 @@ func (p PutMany) Insert() error { func (p *Put) document() error { title := sql.NullString{String: p.Doc.Title, Valid: p.Doc.Title != ""} - date := sql.NullInt64{Int64: p.Doc.Date.Unix(), Valid: !p.Doc.Date.IsZero()} - filetime := sql.NullInt64{Int64: p.Doc.FileTime.Unix(), Valid: !p.Doc.FileTime.IsZero()} - + headings := sql.NullString{String: p.Doc.Headings, Valid: p.Doc.Headings != ""} meta := sql.NullString{String: p.Doc.OtherMeta, Valid: p.Doc.OtherMeta != ""} result, err := p.tx.Exec(` - INSERT INTO Documents(path, title, date, fileTime, meta) - VALUES (?,?,?,?,?) - `, p.Doc.Path, title, date, filetime, meta) + INSERT INTO Documents(path, title, date, fileTime, headings, meta) + VALUES (?,?,?,?,?,?) + `, p.Doc.Path, title, date, filetime, headings, meta) if err != nil { return err } @@ -127,33 +125,30 @@ func (p *Put) document() error { } func (p *PutMany) documents(ctx context.Context) error { - stmt, err := p.db.PrepareContext(ctx, ` - INSERT INTO Documents(path, title, date, fileTime, meta) - VALUES (?,?,?,?,?) - `) + tx, err := p.db.BeginTx(ctx, nil) if err != nil { return err } - defer stmt.Close() - tx, err := p.db.BeginTx(ctx, nil) + stmt, err := tx.PrepareContext(ctx, ` + INSERT INTO Documents(path, title, date, fileTime, headings, meta) + VALUES (?,?,?,?,?,?) + `) if err != nil { return err } - - txStmt := tx.StmtContext(ctx, stmt) + defer stmt.Close() // PERF: profile this, grabbing the docId here might save time by simpliyfying // future inserts for _, doc := range p.pathDocs { title := sql.NullString{String: doc.Title, Valid: doc.Title != ""} date := sql.NullInt64{Int64: doc.Date.Unix(), Valid: !doc.Date.IsZero()} - filetime := sql.NullInt64{Int64: doc.FileTime.Unix(), Valid: !doc.FileTime.IsZero()} - + headings := sql.NullString{String: doc.Headings, Valid: doc.Headings != ""} meta := sql.NullString{String: doc.OtherMeta, Valid: doc.OtherMeta != ""} - res, err := txStmt.ExecContext(ctx, doc.Path, title, date, filetime, meta) + res, err := stmt.ExecContext(ctx, doc.Path, title, date, filetime, headings, meta) if err != nil { tx.Rollback() return err diff --git a/pkg/data/put_test.go b/pkg/data/put_test.go index 12c9e24..a450196 100644 --- a/pkg/data/put_test.go +++ b/pkg/data/put_test.go @@ -30,6 +30,7 @@ func TestPut_Insert(t *testing.T) { FileTime: time.Unix(2, 0), Authors: []string{"jp"}, Tags: []string{"foo", "bar", "oof", "baz"}, + Headings: "#A Heading\n## Another Heading", Links: []string{"link_1", "link_2", "link_3"}, }, nil, diff --git a/pkg/data/update.go b/pkg/data/update.go index 1a50563..6aecb41 100644 --- a/pkg/data/update.go +++ b/pkg/data/update.go @@ -129,18 +129,20 @@ func (u *Update) document() (bool, error) { title := sql.NullString{String: u.Doc.Title, Valid: u.Doc.Title != ""} date := sql.NullInt64{Int64: u.Doc.Date.Unix(), Valid: !u.Doc.Date.IsZero()} + headings := sql.NullString{String: u.Doc.Headings, Valid: u.Doc.Headings != ""} meta := sql.NullString{String: u.Doc.OtherMeta, Valid: u.Doc.OtherMeta != ""} _, err := u.tx.Exec(` - INSERT INTO Documents(path, title, date, fileTime, meta) - VALUES (?,?,?,?,?) + INSERT INTO Documents(path, title, date, fileTime, headings, meta) + VALUES (?,?,?,?,?,?) ON CONFLICT(path) DO UPDATE SET title=excluded.title, date=excluded.date, fileTime=excluded.fileTime, + headings=excluded.headings, meta=excluded.meta - `, u.Doc.Path, title, date, filetime, meta) + `, u.Doc.Path, title, date, filetime, headings, meta) if err != nil { return true, err } @@ -160,6 +162,7 @@ func (u *UpdateMany) documents() (bool, error) { title TEXT, date INT, fileTime INT, + headings TEXT, meta BLOB )`) if err != nil { @@ -167,7 +170,7 @@ func (u *UpdateMany) documents() (bool, error) { } defer u.tx.Exec("DROP TABLE temp.updateDocs") - tempInsertStmt, err := u.tx.Prepare("INSERT INTO temp.updateDocs VALUES (?,?,?,?,?)") + tempInsertStmt, err := u.tx.Prepare("INSERT INTO temp.updateDocs VALUES (?,?,?,?,?,?)") if err != nil { return false, err } @@ -186,11 +189,15 @@ func (u *UpdateMany) documents() (bool, error) { Int64: doc.Date.Unix(), Valid: !doc.Date.IsZero(), } + headings := sql.NullString{ + String: doc.Headings, + Valid: doc.Headings != "", + } meta := sql.NullString{ String: doc.OtherMeta, Valid: doc.OtherMeta != "", } - if _, err := tempInsertStmt.Exec(path, title, date, filetime, meta); err != nil { + if _, err := tempInsertStmt.Exec(path, title, date, filetime, headings, meta); err != nil { return false, err } } @@ -206,12 +213,13 @@ func (u *UpdateMany) documents() (bool, error) { } _, err = u.tx.Exec(` - INSERT INTO Documents (path, title, date, fileTime, meta) + INSERT INTO Documents (path, title, date, fileTime, headings, meta) SELECT * FROM updateDocs WHERE TRUE ON CONFLICT(path) DO UPDATE SET title=excluded.title, date=excluded.date, fileTime=excluded.fileTime, + headings=excluded.headings, meta=excluded.meta WHERE excluded.fileTime > Documents.fileTime `) diff --git a/pkg/data/update_test.go b/pkg/data/update_test.go index 95b425f..dc2365b 100644 --- a/pkg/data/update_test.go +++ b/pkg/data/update_test.go @@ -32,6 +32,7 @@ func TestUpdate_Update(t *testing.T) { FileTime: time.Unix(2, 0), Authors: []string{"jp"}, Tags: []string{"foo", "bar", "oof", "baz"}, + Headings: "#A Heading\n", Links: []string{"link_1", "link_2", "link_3"}, }, nil, @@ -48,6 +49,7 @@ func TestUpdate_Update(t *testing.T) { FileTime: time.Unix(2, 0), Authors: []string{"jp"}, Tags: []string{"foo", "bar", "oof", "baz"}, + Headings: "#Old Heading\n", Links: []string{"link_1", "link_2", "link_3"}, }) @@ -64,6 +66,7 @@ func TestUpdate_Update(t *testing.T) { FileTime: time.Unix(3, 0), Authors: []string{"jp", "pj"}, Tags: []string{"foo", "bar", "oof"}, + Headings: "#New Heading\n", Links: []string{"link_4"}, }, nil, @@ -115,6 +118,7 @@ func TestUpdateMany_Update(t *testing.T) { FileTime: time.Unix(2, 0), Authors: []string{"jp"}, Tags: []string{"foo", "bar", "oof", "baz"}, + Headings: "# Some Heading\n", Links: []string{"link_1", "link_2", "link_3"}, }, "/bfile": { @@ -190,6 +194,7 @@ func TestUpdateMany_Update(t *testing.T) { FileTime: time.Unix(2, 0), Authors: []string{"jp"}, Tags: []string{"foo", "bar", "oof", "baz"}, + Headings: "# A Original\n", Links: []string{"link_1", "link_2", "link_3"}, }, "/bfile": { @@ -199,6 +204,7 @@ func TestUpdateMany_Update(t *testing.T) { FileTime: time.Unix(4, 0), Authors: []string{"pj"}, Tags: []string{"foo", "gar"}, + Headings: "# B Original\n", Links: []string{"link_4"}, }, } @@ -220,6 +226,7 @@ func TestUpdateMany_Update(t *testing.T) { FileTime: time.Unix(10, 0), Authors: []string{"jp"}, Tags: []string{"foo", "bar", "bing", "baz"}, + Headings: "# A New\n", Links: []string{"link_1", "link_3"}, }, "/bfile": { -- cgit v1.2.3