From fbfdc6877d99f9485edc2357a1d8abff68d99132 Mon Sep 17 00:00:00 2001 From: Jean-Michel Tremblay Date: Fri, 19 Jun 2026 07:12:50 -0400 Subject: [PATCH] Add scheduled metadata-only DB backups (spec item 11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - internal/backup: weekly (configurable) snapshots via the existing VACUUM INTO + blob-strip path; restart-safe (only backs up if newest is older than the interval), retains BACKUP_KEEP most recent, prunes the rest. Runs in a background goroutine; failures are logged, never fatal. - Config: BACKUP_DIR (./data/backups), BACKUP_INTERVAL (168h; 0 disables), BACKUP_KEEP (8). - Fix: blob-strip now clears attachments.image_data too, not just receipts. - Fix: STORAGE_DIR is the storage ROOT (default ./data) — receipts/ and attachments/ live under it; corrects the doubled-nesting from item 10. Co-Authored-By: Claude Opus 4.8 --- .env.example | 10 ++- cmd/hsa/main.go | 8 ++ internal/backup/backup.go | 124 ++++++++++++++++++++++++++++++ internal/backup/backup_test.go | 100 ++++++++++++++++++++++++ internal/config/config.go | 25 +++++- internal/storage/snapshot.go | 7 +- internal/storage/snapshot_test.go | 24 +++++- spec.md | 25 +++++- 8 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 internal/backup/backup.go create mode 100644 internal/backup/backup_test.go diff --git a/.env.example b/.env.example index 9ed93b0..4f73c18 100644 --- a/.env.example +++ b/.env.example @@ -25,12 +25,20 @@ LISTEN_ADDR=:8080 # Storage (defaults shown; relative paths are resolved from the app's working dir). # For the LXC deployment use absolute paths under /var/lib/hsa. +# STORAGE_DIR is the storage ROOT — receipt files go under /receipts/ +# / and attachments under /attachments//. DB_PATH=./data/hsa.db -STORAGE_DIR=./data/receipts +STORAGE_DIR=./data # Max upload size in megabytes (reject larger). Camera photos can be ~10MB. MAX_UPLOAD_MB=32 +# Scheduled metadata-only DB backups (no image blobs). On by default, weekly. +# BACKUP_INTERVAL accepts a Go duration (e.g. 168h, 24h); set 0 to disable. +BACKUP_DIR=./data/backups +BACKUP_INTERVAL=168h +BACKUP_KEEP=8 + # People + categories catalog (seeded into the DB on startup). CONFIG_PATH=./config.json diff --git a/cmd/hsa/main.go b/cmd/hsa/main.go index 17cc846..b841296 100644 --- a/cmd/hsa/main.go +++ b/cmd/hsa/main.go @@ -6,7 +6,9 @@ import ( "net/http" "os" "path/filepath" + "time" + "maisym.com/hsa/internal/backup" "maisym.com/hsa/internal/classify" "maisym.com/hsa/internal/config" "maisym.com/hsa/internal/storage" @@ -63,6 +65,12 @@ func main() { log.Fatalf("server init: %v", err) } + // Scheduled metadata-only DB backups (best-effort, runs for the process life). + if cfg.BackupEvery > 0 { + go backup.Schedule(context.Background(), store, cfg.BackupDir, cfg.BackupEvery, cfg.BackupKeep, time.Now) + log.Printf("DB backups: every %s into %s (keep %d, metadata-only)", cfg.BackupEvery, cfg.BackupDir, cfg.BackupKeep) + } + log.Printf("HSA listening on %s (issuer %s)", cfg.ListenAddr, cfg.IssuerURL) if err := http.ListenAndServe(cfg.ListenAddr, srv.Routes()); err != nil { log.Fatalf("listen: %v", err) diff --git a/internal/backup/backup.go b/internal/backup/backup.go new file mode 100644 index 0000000..129fe02 --- /dev/null +++ b/internal/backup/backup.go @@ -0,0 +1,124 @@ +// Package backup writes periodic metadata-only snapshots of the database to a +// local directory, on a schedule, so a fresh copy of the receipt metadata always +// survives even if the live DB is lost. Blobs (receipt + attachment images) are +// excluded to keep the backups small — the on-disk files and the full-blob export +// remain the source for the images themselves. +package backup + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// Snapshotter writes a point-in-time DB copy. *storage.Store satisfies this. +type Snapshotter interface { + SnapshotTo(destPath string, includeBlobs bool) error +} + +const ( + prefix = "hsa-backup-" + suffix = ".db" + stamp = "2006-01-02-150405" +) + +// Schedule runs metadata-only backups into dir every `every`, keeping the most +// recent `keep`. It blocks until ctx is cancelled, so run it in a goroutine. +// +// On start it backs up immediately unless a backup younger than `every` already +// exists — so frequent restarts don't spam the directory, and a gap longer than +// the interval is covered right away. `now` is injected for testability (pass +// time.Now). +func Schedule(ctx context.Context, s Snapshotter, dir string, every time.Duration, keep int, now func() time.Time) { + if err := os.MkdirAll(dir, 0o700); err != nil { + log.Printf("backup: cannot create %s: %v (backups disabled)", dir, err) + return + } + for { + timer := time.NewTimer(untilNext(dir, every, now())) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + if err := writeBackup(s, dir, now()); err != nil { + log.Printf("backup: %v", err) + continue + } + if err := prune(dir, keep); err != nil { + log.Printf("backup prune: %v", err) + } + } +} + +// untilNext is how long to wait before the next backup: zero if none exists or the +// newest is already older than `every`, otherwise the remaining time. +func untilNext(dir string, every time.Duration, now time.Time) time.Duration { + newest := newestBackup(dir) + if newest.IsZero() { + return 0 + } + if elapsed := now.Sub(newest); elapsed < every { + return every - elapsed + } + return 0 +} + +func writeBackup(s Snapshotter, dir string, now time.Time) error { + name := prefix + now.UTC().Format(stamp) + suffix + if err := s.SnapshotTo(filepath.Join(dir, name), false); err != nil { + return fmt.Errorf("write %s: %w", name, err) + } + log.Printf("backup: wrote %s", name) + return nil +} + +// backupFiles returns the backup filenames in chronological order (the timestamped +// names sort lexically the same as by time). +func backupFiles(dir string) []string { + ents, err := os.ReadDir(dir) + if err != nil { + return nil + } + var names []string + for _, e := range ents { + if !e.IsDir() && strings.HasPrefix(e.Name(), prefix) && strings.HasSuffix(e.Name(), suffix) { + names = append(names, e.Name()) + } + } + sort.Strings(names) + return names +} + +func newestBackup(dir string) time.Time { + names := backupFiles(dir) + if len(names) == 0 { + return time.Time{} + } + last := strings.TrimSuffix(strings.TrimPrefix(names[len(names)-1], prefix), suffix) + t, err := time.Parse(stamp, last) // layout has no zone → parsed as UTC + if err != nil { + return time.Time{} + } + return t +} + +// prune deletes the oldest backups beyond the most recent `keep`. +func prune(dir string, keep int) error { + names := backupFiles(dir) + if keep <= 0 || len(names) <= keep { + return nil + } + for _, name := range names[:len(names)-keep] { + if err := os.Remove(filepath.Join(dir, name)); err != nil { + return err + } + } + return nil +} diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go new file mode 100644 index 0000000..65ab711 --- /dev/null +++ b/internal/backup/backup_test.go @@ -0,0 +1,100 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// fakeSnap writes a tiny file so backup files exist on disk for the tests, and +// records how many times it ran. +type fakeSnap struct{ calls int } + +func (f *fakeSnap) SnapshotTo(dest string, includeBlobs bool) error { + f.calls++ + return os.WriteFile(dest, []byte("snap"), 0o600) +} + +func TestWriteBackupAndNewest(t *testing.T) { + dir := t.TempDir() + s := &fakeSnap{} + now := time.Date(2026, 6, 10, 9, 0, 0, 0, time.UTC) + + if err := writeBackup(s, dir, now); err != nil { + t.Fatal(err) + } + files := backupFiles(dir) + if len(files) != 1 || files[0] != "hsa-backup-2026-06-10-090000.db" { + t.Fatalf("files = %v", files) + } + if got := newestBackup(dir); !got.Equal(now) { + t.Errorf("newestBackup = %v, want %v", got, now) + } +} + +func TestUntilNext(t *testing.T) { + dir := t.TempDir() + every := 168 * time.Hour + now := time.Date(2026, 6, 10, 9, 0, 0, 0, time.UTC) + + // No backups yet → back up immediately. + if d := untilNext(dir, every, now); d != 0 { + t.Errorf("empty dir: untilNext = %v, want 0", d) + } + + // A fresh backup → wait nearly the full interval. + _ = writeBackup(&fakeSnap{}, dir, now.Add(-1*time.Hour)) + if d := untilNext(dir, every, now); d <= 0 || d > every { + t.Errorf("fresh backup: untilNext = %v, want ~167h", d) + } + + // An old backup → back up now. + dir2 := t.TempDir() + _ = writeBackup(&fakeSnap{}, dir2, now.Add(-200*time.Hour)) + if d := untilNext(dir2, every, now); d != 0 { + t.Errorf("stale backup: untilNext = %v, want 0", d) + } +} + +func TestPruneKeepsMostRecent(t *testing.T) { + dir := t.TempDir() + s := &fakeSnap{} + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i := 0; i < 5; i++ { + _ = writeBackup(s, dir, base.AddDate(0, 0, i)) + } + if err := prune(dir, 2); err != nil { + t.Fatal(err) + } + files := backupFiles(dir) + if len(files) != 2 { + t.Fatalf("after prune: %v", files) + } + // The two newest (Jan 04, Jan 05) survive. + if !strings.Contains(files[0], "2026-01-04") || !strings.Contains(files[1], "2026-01-05") { + t.Errorf("wrong files kept: %v", files) + } +} + +func TestScheduleRunsOnceThenStops(t *testing.T) { + dir := t.TempDir() + s := &fakeSnap{} + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + // A real store would be passed in production; here the fake is enough. + go Schedule(ctx, s, dir, time.Hour, 4, func() time.Time { + return time.Date(2026, 6, 10, 9, 0, 0, 0, time.UTC) + }) + // Wait for the immediate first backup to land. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(filepath.Join(dir, "hsa-backup-2026-06-10-090000.db")); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("Schedule did not write an initial backup") +} diff --git a/internal/config/config.go b/internal/config/config.go index 05eb8bb..f68862f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,6 +7,7 @@ import ( "os" "strconv" "strings" + "time" ) // DefaultClassifyModel is the model used for receipt classification unless @@ -32,6 +33,10 @@ type Config struct { ConfigPath string // path to config.json (people + categories) ClassifyAPIKey string // Anthropic API key; empty disables auto-classification ClassifyModel string // model id for classification + + BackupDir string // directory for scheduled metadata-only DB backups + BackupEvery time.Duration // backup interval; <= 0 disables scheduled backups + BackupKeep int // number of most-recent backups to retain } // Load reads configuration from the environment, validating required values. @@ -69,7 +74,9 @@ func Load() (Config, error) { c.SecureCookies = strings.HasPrefix(c.RedirectURL, "https://") c.DBPath = envOr("DB_PATH", "./data/hsa.db") - c.StorageDir = envOr("STORAGE_DIR", "./data/receipts") + // STORAGE_DIR is the storage ROOT; receipts and attachments are stored under + // /receipts// and /attachments//. + c.StorageDir = envOr("STORAGE_DIR", "./data") maxMB := 32 if v := strings.TrimSpace(os.Getenv("MAX_UPLOAD_MB")); v != "" { @@ -84,6 +91,22 @@ func Load() (Config, error) { c.ClassifyAPIKey = strings.TrimSpace(os.Getenv("CLAUDE_API_KEY")) c.ClassifyModel = envOr("CLASSIFY_MODEL", DefaultClassifyModel) + // Scheduled metadata-only DB backups (on by default, weekly). Set + // BACKUP_INTERVAL=0 to disable. + c.BackupDir = envOr("BACKUP_DIR", "./data/backups") + c.BackupEvery = 7 * 24 * time.Hour + if v := strings.TrimSpace(os.Getenv("BACKUP_INTERVAL")); v != "" { + if d, err := time.ParseDuration(v); err == nil { + c.BackupEvery = d + } + } + c.BackupKeep = 8 + if v := strings.TrimSpace(os.Getenv("BACKUP_KEEP")); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + c.BackupKeep = n + } + } + return c, nil } diff --git a/internal/storage/snapshot.go b/internal/storage/snapshot.go index d6b4d0c..5b098bd 100644 --- a/internal/storage/snapshot.go +++ b/internal/storage/snapshot.go @@ -21,13 +21,16 @@ func (s *Store) SnapshotTo(destPath string, includeBlobs bool) error { } // Strip blobs from the copy only. image_data is NOT NULL, so use an empty blob. + // Both receipts and their attachments carry blobs. snap, err := sql.Open("sqlite", destPath) if err != nil { return fmt.Errorf("open snapshot: %w", err) } defer snap.Close() - if _, err := snap.Exec(`UPDATE receipts SET image_data = X''`); err != nil { - return fmt.Errorf("strip blobs: %w", err) + for _, table := range []string{"receipts", "attachments"} { + if _, err := snap.Exec(`UPDATE ` + table + ` SET image_data = X''`); err != nil { + return fmt.Errorf("strip %s blobs: %w", table, err) + } } if _, err := snap.Exec(`VACUUM`); err != nil { return fmt.Errorf("vacuum stripped snapshot: %w", err) diff --git a/internal/storage/snapshot_test.go b/internal/storage/snapshot_test.go index fb3f55c..02caa36 100644 --- a/internal/storage/snapshot_test.go +++ b/internal/storage/snapshot_test.go @@ -3,6 +3,8 @@ package storage import ( "path/filepath" "testing" + + "maisym.com/hsa/internal/receipt" ) func TestSnapshotWithBlobs(t *testing.T) { @@ -38,6 +40,15 @@ func TestSnapshotWithoutBlobs(t *testing.T) { if err := s.Insert(r); err != nil { t.Fatal(err) } + // An attachment blob must be stripped too. + att := receipt.Attachment{ + ID: "att1", ReceiptID: r.ID, UploadedBy: "jm", UploadedAt: r.UploadedAt, + FilePath: "attachments/2026/x.png", ImageData: []byte("blobbytes"), + FileSizeBytes: 9, OriginalFilename: "x.png", MimeType: "image/png", + } + if err := s.InsertAttachment(att); err != nil { + t.Fatal(err) + } dest := filepath.Join(t.TempDir(), "snap-noblob.db") if err := s.SnapshotTo(dest, false); err != nil { @@ -63,9 +74,20 @@ func TestSnapshotWithoutBlobs(t *testing.T) { t.Fatal(err) } if len(got.ImageData) != 0 { - t.Errorf("blob not stripped: got %d bytes", len(got.ImageData)) + t.Errorf("receipt blob not stripped: got %d bytes", len(got.ImageData)) } if got.AmountCents != r.AmountCents { t.Errorf("metadata lost: amount = %d, want %d", got.AmountCents, r.AmountCents) } + // Attachment metadata survives, blob stripped. + gotAtt, err := snap.GetAttachment("att1") + if err != nil { + t.Fatal(err) + } + if len(gotAtt.ImageData) != 0 { + t.Errorf("attachment blob not stripped: got %d bytes", len(gotAtt.ImageData)) + } + if gotAtt.OriginalFilename != "x.png" { + t.Errorf("attachment metadata lost: %+v", gotAtt) + } } diff --git a/spec.md b/spec.md index 5543e82..0324c9c 100644 --- a/spec.md +++ b/spec.md @@ -306,4 +306,27 @@ Lifecycle / interactions: Out of scope (for now): - Adding attachments to an already-saved receipt (would need an edit/detail page). - No AI parsing of attachments; no per-attachment metadata beyond the file. - - No reordering UI beyond upload order. \ No newline at end of file + - No reordering UI beyond upload order. + +11. Scheduled metadata-only DB backups + +The app writes a periodic, metadata-only snapshot of the database to a local +directory so the receipt metadata always has a recent recoverable copy, without an +external cron job. + - Built on the existing snapshot primitive (VACUUM INTO, then blob-strip): the + backup excludes image blobs from BOTH receipts and attachments, so it is tiny. + The on-disk files and the full-blob export (/export/db) remain the source for + the images themselves. + - On by default, weekly. Config: BACKUP_DIR (default ./data/backups), + BACKUP_INTERVAL (Go duration, default 168h; set 0 to disable), BACKUP_KEEP + (most-recent copies to retain, default 8). + - Files are named hsa-backup-.db so they sort chronologically; + older copies beyond BACKUP_KEEP are pruned. + - Restart-safe: on startup it backs up immediately only if the newest existing + backup is older than the interval, so frequent restarts don't spam the dir and + a long gap is covered right away. Runs in a background goroutine for the life + of the process; failures are logged, never fatal. + +Storage-root note: STORAGE_DIR is the storage ROOT (not the receipts subdir). +Receipts live under /receipts// and attachments under +/attachments// (default STORAGE_DIR=./data). \ No newline at end of file