From 5acef6869ac01352cf0e105151c5ad146924048e Mon Sep 17 00:00:00 2001 From: Jean-Michel Tremblay Date: Mon, 22 Jun 2026 22:48:28 -0400 Subject: [PATCH] Add receiptscan pilot (Go + Claude cloud API) Pilot tool that sends a receipt photo to Claude, classifies the budget category, and for groceries locates + categorizes each line item, rendering an annotated PNG with per-category subtotals. Uses the Claude cloud API (claude-opus-4-8) for pixel-accurate bounding boxes; reads CLAUDE_API_KEY from ../.env. Complementary to the local-only OCR/ tools (ollama, no cloud). Co-Authored-By: Claude Opus 4.8 --- receiptscan/README.md | 80 ++++ receiptscan/go.mod | 5 + receiptscan/go.sum | 2 + receiptscan/main.go | 853 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 940 insertions(+) create mode 100644 receiptscan/README.md create mode 100644 receiptscan/go.mod create mode 100644 receiptscan/go.sum create mode 100644 receiptscan/main.go diff --git a/receiptscan/README.md b/receiptscan/README.md new file mode 100644 index 0000000..4ced7ce --- /dev/null +++ b/receiptscan/README.md @@ -0,0 +1,80 @@ +# receiptscan + +Sends a receipt photo to Claude, classifies the receipt into a budget category, +and (for groceries) locates + categorizes every line item, then renders an +annotated PNG: each line item is lightly shaded by food category with a category +tag in the margin, plus a per-category subtotal sidebar. + +## Run + +```bash +/home/jm/.local/go/bin/go -C /home/jm/programming/HSAmanager/receiptscan run . +``` + +- Image path is hard-coded to `/home/jm/aldi.jpg` (or pass one as an argument). +- Reads `CLAUDE_API_KEY` from `../.env` (or `ANTHROPIC_API_KEY` in the env). +- Output: `_annotated.png` next to the input. +- Go is installed locally at `/home/jm/.local/go` (no root). `GOTOOLCHAIN=local` + avoids auto-downloading a toolchain. + +### Useful env flags +- `RECEIPT_CACHE=read` — re-render from the saved `.cache.json` without + calling the API (fast iteration on rendering). +- `RECEIPT_DEBUG=1` — print the raw API response, the row-fit, and per-item row + assignments to stderr. +- `RECEIPT_ANALYZE=1` — dump detected text rows vs. model boxes and exit. + +## Model query + +- Model: `claude-opus-4-8` (returns true 1:1 image-pixel coordinates; Haiku does + not localize accurately and produced vertically-stretched boxes). +- Adaptive thinking + `output_config.effort: high`. Without thinking the model + often returned a placeholder/empty list. +- Structured output via `output_config.format` (`json_schema`, see schema in + `main.go` → `analyze`): `receipt_type`, `currency`, `total_amount`, and + `line_items[]` of `{name, price, category, box{x,y,width,height}}`. +- The model intermittently returns an empty `line_items` for a grocery receipt, + so the call is retried up to 4× until items come back. + +### Prompt + +``` +You are reading a photographed grocery store receipt. The image is exactly + pixels wide and pixels tall. All coordinates you return must be in +pixels in that space, with (0,0) at the top-left. + +1. Set receipt_type to "groceries" and read the grand total (total_amount) + with its currency. + +2. This receipt has many purchased line items (dozens). Return EVERY one of + them - do not stop early and do not return an empty list. For each line item: + - name and price exactly as printed, + - a tight pixel rectangle (box) enclosing that whole printed line, from the + item name on the left through its price on the right, + - a food category, exactly one of: dairy, meat, bakery_bread, berries, + non_berry_fruits, vegetables, desserts, processed_food, pet_stuff, + other_groceries. + Use other_groceries only when nothing more specific fits. Do NOT include + subtotal, tax, payment, or total rows as line items. + +Return only the structured object. +``` + +Budget categories for `receipt_type`: house_maintenance, activities, restaurant, +groceries, hobbies, alcohol, medical, education, other. + +## How box alignment works + +The model's returned y-coordinates are vertically stretched (~1.35× the true +line pitch) and only roughly placed, so boxes are **not** drawn at the model +coordinates. Instead the image itself is used to place them: + +1. Detect horizontal text-row bands (dark-pixel projection in the item column). +2. Keep only **item rows**: bands with ink in the price column (right) AND the + item-code column (far left), at or below the model's first item. This + excludes centered headers, weight/quantity sub-lines, paper-edge shadows, and + the payment/total block. +3. The first N price-bearing rows (N = item count from the model) are the N + items, in order — map item *i* → row *i*. + +This keeps alignment correct regardless of the model's coordinate drift. diff --git a/receiptscan/go.mod b/receiptscan/go.mod new file mode 100644 index 0000000..5ffffba --- /dev/null +++ b/receiptscan/go.mod @@ -0,0 +1,5 @@ +module receiptscan + +go 1.26 + +require golang.org/x/image v0.21.0 diff --git a/receiptscan/go.sum b/receiptscan/go.sum new file mode 100644 index 0000000..93f7530 --- /dev/null +++ b/receiptscan/go.sum @@ -0,0 +1,2 @@ +golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s= +golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78= diff --git a/receiptscan/main.go b/receiptscan/main.go new file mode 100644 index 0000000..105173c --- /dev/null +++ b/receiptscan/main.go @@ -0,0 +1,853 @@ +// Command receiptscan sends a receipt image to Claude, asks it to classify the +// receipt into a budget category and (for groceries) to locate and categorize +// every line item, then renders an annotated image: each line item is lightly +// shaded by its food category, labeled, and summarized with per-category +// subtotals in a sidebar. +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "image" + "image/color" + "image/draw" + _ "image/jpeg" + "image/png" + "io" + "math" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "golang.org/x/image/font" + "golang.org/x/image/font/basicfont" + "golang.org/x/image/math/fixed" +) + +const ( + model = "claude-opus-4-8" + endpoint = "https://api.anthropic.com/v1/messages" +) + +// ---------- structured response from Claude ---------- + +type Box struct { + X int `json:"x"` + Y int `json:"y"` + Width int `json:"width"` + Height int `json:"height"` +} + +type LineItem struct { + Name string `json:"name"` + Price float64 `json:"price"` + Category string `json:"category"` + Box Box `json:"box"` +} + +type Receipt struct { + ReceiptType string `json:"receipt_type"` + Currency string `json:"currency"` + TotalAmount float64 `json:"total_amount"` + LineItems []LineItem `json:"line_items"` +} + +// Food categories and their pale highlight colors (used for groceries). +var categoryColor = map[string]color.RGBA{ + "dairy": {165, 205, 255, 255}, // light blue + "meat": {255, 150, 150, 255}, // red + "bakery_bread": {235, 205, 150, 255}, // tan + "berries": {220, 175, 235, 255}, // light purple + "non_berry_fruits": {255, 245, 150, 255}, // pale yellow + "vegetables": {180, 235, 175, 255}, // light green + "desserts": {255, 190, 220, 255}, // pink + "processed_food": {210, 200, 235, 255}, // lavender + "pet_stuff": {215, 200, 185, 255}, // light brown + "other_groceries": {220, 220, 220, 255}, // light grey +} + +var categoryOrder = []string{ + "dairy", "meat", "bakery_bread", "berries", "non_berry_fruits", + "vegetables", "desserts", "processed_food", "pet_stuff", "other_groceries", +} + +func prettyCat(c string) string { + switch c { + case "bakery_bread": + return "Bakery / Bread" + case "non_berry_fruits": + return "Non-berry fruits" + case "processed_food": + return "Processed food" + case "pet_stuff": + return "Pet stuff" + case "other_groceries": + return "Other groceries" + } + return strings.Title(c) +} + +func main() { + imgPath := "/home/jm/aldi.jpg" + if len(os.Args) > 1 { + imgPath = os.Args[1] + } + + apiKey, err := loadAPIKey() + if err != nil { + fatal(err) + } + + raw, err := os.ReadFile(imgPath) + if err != nil { + fatal(fmt.Errorf("read image: %w", err)) + } + + // Decode once to learn the true pixel dimensions; Claude (Opus 4.7+) returns + // coordinates 1:1 with actual image pixels for images up to 2576px on the + // long edge, so we hand it the real dimensions to ground the boxes. + srcImg, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + fatal(fmt.Errorf("decode image: %w", err)) + } + w, h := srcImg.Bounds().Dx(), srcImg.Bounds().Dy() + fmt.Printf("Image %s is %dx%d px. Asking %s...\n", filepath.Base(imgPath), w, h, model) + + // Cache the model response so rendering can be iterated without re-calling. + cacheFile := strings.TrimSuffix(imgPath, filepath.Ext(imgPath)) + ".cache.json" + var receipt *Receipt + if os.Getenv("RECEIPT_CACHE") == "read" { + receipt, err = loadReceipt(cacheFile) + if err != nil { + fatal(fmt.Errorf("load cache: %w", err)) + } + fmt.Printf("Loaded cached analysis from %s\n", cacheFile) + } else { + // The model occasionally returns an empty line_items list for a grocery + // receipt; retry until it enumerates them (or attempts are exhausted). + for attempt := 1; attempt <= 4; attempt++ { + receipt, err = analyze(apiKey, raw, w, h) + if err != nil { + fatal(err) + } + if receipt.ReceiptType != "groceries" || len(receipt.LineItems) > 0 { + break + } + fmt.Printf(" (attempt %d returned no line items, retrying...)\n", attempt) + } + if b, e := json.MarshalIndent(receipt, "", " "); e == nil { + _ = os.WriteFile(cacheFile, b, 0644) + } + } + + // Detected text rows from the image; used to snap model boxes onto real rows. + rows := detectTextRows(srcImg, receipt) + + if os.Getenv("RECEIPT_ANALYZE") != "" { + analyzeAlignment(receipt, rows) + return + } + + alignBoxes(srcImg, receipt, rows) + printSummary(receipt) + + outPath := strings.TrimSuffix(imgPath, filepath.Ext(imgPath)) + "_annotated.png" + if err := render(srcImg, receipt, outPath); err != nil { + fatal(err) + } + fmt.Printf("\nAnnotated image written to %s\n", outPath) +} + +// ---------- API call ---------- + +func analyze(apiKey string, img []byte, w, h int) (*Receipt, error) { + schema := map[string]any{ + "type": "object", + "additionalProperties": false, + "required": []string{"receipt_type", "currency", "total_amount", "line_items"}, + "properties": map[string]any{ + "receipt_type": map[string]any{ + "type": "string", + "enum": []string{"house_maintenance", "activities", "restaurant", + "groceries", "hobbies", "alcohol", "medical", "education", "other"}, + "description": "Best-fitting budget category for the whole receipt.", + }, + "currency": map[string]any{"type": "string", "description": "ISO-ish currency symbol or code, e.g. $, USD, EUR."}, + "total_amount": map[string]any{"type": "number", "description": "Grand total paid."}, + "line_items": map[string]any{ + "type": "array", + "description": "One entry per purchased line item. Empty unless the receipt is groceries.", + "items": map[string]any{ + "type": "object", + "additionalProperties": false, + "required": []string{"name", "price", "category", "box"}, + "properties": map[string]any{ + "name": map[string]any{"type": "string", "description": "Item name as printed."}, + "price": map[string]any{"type": "number", "description": "Line item price."}, + "category": map[string]any{ + "type": "string", + "enum": categoryOrder, + }, + "box": map[string]any{ + "type": "object", + "additionalProperties": false, + "required": []string{"x", "y", "width", "height"}, + "description": "Pixel rectangle tightly enclosing the whole line (name + price).", + "properties": map[string]any{ + "x": map[string]any{"type": "integer", "description": "Left edge, pixels from left."}, + "y": map[string]any{"type": "integer", "description": "Top edge, pixels from top."}, + "width": map[string]any{"type": "integer"}, + "height": map[string]any{"type": "integer"}, + }, + }, + }, + }, + }, + }, + } + + prompt := fmt.Sprintf(`You are reading a photographed grocery store receipt. The image is exactly %d pixels wide and %d pixels tall. All coordinates you return must be in pixels in that space, with (0,0) at the top-left. + +1. Set receipt_type to "groceries" and read the grand total (total_amount) with its currency. + +2. This receipt has many purchased line items (dozens). Return EVERY one of them - do not stop early and do not return an empty list. For each line item: + - name and price exactly as printed, + - a tight pixel rectangle (box) enclosing that whole printed line, from the item name on the left through its price on the right, + - a food category, exactly one of: dairy, meat, bakery_bread, berries, non_berry_fruits, vegetables, desserts, processed_food, pet_stuff, other_groceries. + Use other_groceries only when nothing more specific fits. Do NOT include subtotal, tax, payment, or total rows as line items. + +Return only the structured object.`, w, h) + + reqBody := map[string]any{ + "model": model, + "max_tokens": 24000, + "output_config": map[string]any{ + "format": map[string]any{"type": "json_schema", "schema": schema}, + }, + } + // Per-model thinking config: Haiku 4.5 (a 4.x model) uses budgeted extended + // thinking and rejects the effort param; 4.6+ models use adaptive thinking. + if strings.Contains(model, "haiku") { + reqBody["thinking"] = map[string]any{"type": "enabled", "budget_tokens": 8000} + } else { + reqBody["thinking"] = map[string]any{"type": "adaptive"} + reqBody["output_config"].(map[string]any)["effort"] = "high" + } + reqBody["messages"] = []any{ + map[string]any{ + "role": "user", + "content": []any{ + map[string]any{ + "type": "image", + "source": map[string]any{ + "type": "base64", + "media_type": "image/jpeg", + "data": base64.StdEncoding.EncodeToString(img), + }, + }, + map[string]any{"type": "text", "text": prompt}, + }, + }, + } + + buf, _ := json.Marshal(reqBody) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(buf)) + req.Header.Set("x-api-key", apiKey) + req.Header.Set("anthropic-version", "2023-06-01") + req.Header.Set("content-type", "application/json") + + resp, err := (&http.Client{Timeout: 6 * time.Minute}).Do(req) + if err != nil { + return nil, fmt.Errorf("request: %w", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if os.Getenv("RECEIPT_DEBUG") != "" { + fmt.Fprintf(os.Stderr, "[debug] http=%d body=%s\n", resp.StatusCode, string(body)) + } + + var parsed struct { + StopReason string `json:"stop_reason"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + Error *struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + if parsed.Error != nil { + return nil, fmt.Errorf("api error (%s): %s", parsed.Error.Type, parsed.Error.Message) + } + if parsed.StopReason == "refusal" { + return nil, fmt.Errorf("model refused the request") + } + + var jsonText string + for _, b := range parsed.Content { + if b.Type == "text" { + jsonText = b.Text + break + } + } + if os.Getenv("RECEIPT_DEBUG") != "" { + fmt.Fprintf(os.Stderr, "[debug] stop_reason=%s json_len=%d\n%s\n", parsed.StopReason, len(jsonText), jsonText) + } + if jsonText == "" { + return nil, fmt.Errorf("no text block in response") + } + + var r Receipt + if err := json.Unmarshal([]byte(jsonText), &r); err != nil { + return nil, fmt.Errorf("parse structured output: %w\n%s", err, jsonText) + } + return &r, nil +} + +// ---------- row detection & snapping ---------- + +type rowBand struct{ top, bottom, center int } + +// detectTextRows finds horizontal bands of printed text within the column that +// the model's line-item boxes occupy, by projecting dark pixels per scanline. +func detectTextRows(img image.Image, r *Receipt) []rowBand { + b := img.Bounds() + w, h := b.Dx(), b.Dy() + + minX, maxX := w, 0 + for _, it := range r.LineItems { + if it.Box.X < minX { + minX = it.Box.X + } + if it.Box.X+it.Box.Width > maxX { + maxX = it.Box.X + it.Box.Width + } + } + if minX < 0 { + minX = 0 + } + if maxX > w || maxX <= minX { + maxX = w + } + span := maxX - minX + if span <= 0 { + return nil + } + + dark := make([]float64, h) + for y := 0; y < h; y++ { + cnt := 0 + for x := minX; x < maxX; x++ { + rr, gg, bb, _ := img.At(b.Min.X+x, b.Min.Y+y).RGBA() + lum := (299*rr + 587*gg + 114*bb) / 1000 >> 8 // 0..255 + if lum < 150 { + cnt++ + } + } + dark[y] = float64(cnt) / float64(span) + } + + const thr = 0.03 + var bands []rowBand + inBand, start := false, 0 + for y := 0; y < h; y++ { + if dark[y] > thr { + if !inBand { + inBand, start = true, y + } + } else if inBand { + inBand = false + if y-start >= 8 { + bands = append(bands, rowBand{start, y, (start + y) / 2}) + } + } + } + if inBand && h-start >= 8 { + bands = append(bands, rowBand{start, h, (start + h) / 2}) + } + return bands +} + +func analyzeAlignment(r *Receipt, rows []rowBand) { + fmt.Printf("\n%d detected text bands:\n", len(rows)) + for i, rw := range rows { + fmt.Printf(" band %2d: y %4d..%4d center %4d (h%2d)\n", i, rw.top, rw.bottom, rw.center, rw.bottom-rw.top) + } + fmt.Printf("\n%d model items:\n", len(r.LineItems)) + for i, it := range r.LineItems { + fmt.Printf(" item %2d: top %4d center %4d %s\n", i, it.Box.Y, it.Box.Y+it.Box.Height/2, it.Name) + } +} + +// alignBoxes places each line item onto its true printed row. Item rows are the +// rows that carry a price in the right-hand column; headers, weight/quantity +// sub-lines, and totals don't, so the first N price-bearing rows (top to bottom) +// are exactly the N items, in order. This avoids fitting unreliable model +// coordinates. If price detection doesn't yield enough rows, fall back to the +// coordinate-fit snap. +func alignBoxes(img image.Image, r *Receipt, rows []rowBand) { + n := len(r.LineItems) + if n == 0 { + return + } + itemRows := pricedRows(img, rows, r) + if os.Getenv("RECEIPT_DEBUG") != "" { + fmt.Fprintf(os.Stderr, "[align] items=%d pricedRows=%d\n", n, len(itemRows)) + } + if len(itemRows) >= n { + for i := 0; i < n; i++ { + b := itemRows[i] + if os.Getenv("RECEIPT_DEBUG") != "" { + fmt.Fprintf(os.Stderr, " item %2d -> row center %4d %s\n", i, b.center, r.LineItems[i].Name) + } + r.LineItems[i].Box.Y = b.top - 1 + r.LineItems[i].Box.Height = b.bottom - b.top + 2 + } + return + } + snapBoxes(r, rows) // fallback +} + +// pricedRows returns the bands that look like item lines: ink in the price +// column (right end of the boxes) AND ink in the item-code column (just left of +// the item name). Requiring both excludes centered header lines (which can reach +// the price column but have nothing at the far left). Top-to-bottom order. +func pricedRows(img image.Image, all []rowBand, r *Receipt) []rowBand { + b := img.Bounds() + right, nameLeft, minY := 0, b.Dx(), b.Dy() + for _, it := range r.LineItems { + if x := it.Box.X + it.Box.Width; x > right { + right = x + } + if it.Box.X < nameLeft { + nameLeft = it.Box.X + } + if it.Box.Y < minY { + minY = it.Box.Y + } + } + if right <= 0 || right > b.Dx() { + right = b.Dx() + } + priceL := clamp(right-110, 0, b.Dx()) + codeL := clamp(nameLeft-95, 0, b.Dx()) + codeR := clamp(nameLeft-15, 0, b.Dx()) + // The model is accurate near the top, so rows above its first item are header + // or paper-edge artifacts, not items. + floorY := minY - 30 + + var out []rowBand + for _, rw := range all { + if rw.center < floorY { + continue + } + if inkFrac(img, codeL, codeR, rw.top, rw.bottom) > 0.03 && + inkFrac(img, priceL, right, rw.top, rw.bottom) > 0.04 { + out = append(out, rw) + } + } + return out +} + +func inkFrac(img image.Image, x0, x1, y0, y1 int) float64 { + b := img.Bounds() + cnt, tot := 0, 0 + for y := y0; y < y1; y++ { + for x := x0; x < x1; x++ { + rr, gg, bb, _ := img.At(b.Min.X+x, b.Min.Y+y).RGBA() + if (299*rr+587*gg+114*bb)/1000>>8 < 150 { + cnt++ + } + tot++ + } + } + if tot == 0 { + return 0 + } + return float64(cnt) / float64(tot) +} + +func clamp(v, lo, hi int) int { + if v < lo { + return lo + } + if v > hi { + return hi + } + return v +} + +// snapBoxes corrects the model's boxes onto the real text rows. The model +// returns vertically-stretched coordinates (line pitch ~1.35x the true pitch), +// so we first recover the best linear map a*y+b that lands the item centers on +// detected text bands (a 1-D iterative-closest-point fit), then snap each item +// to its band with a monotonic (strictly increasing) constraint so sub-lines, +// headers and the payment block are skipped automatically. +func snapBoxes(r *Receipt, rows []rowBand) { + n := len(r.LineItems) + if n == 0 || len(rows) < n { + return + } + M := make([]float64, n) + for i, it := range r.LineItems { + M[i] = float64(it.Box.Y) + float64(it.Box.Height)/2 + } + bandC := make([]float64, len(rows)) + for i, b := range rows { + bandC[i] = float64(b.center) + } + + // The scale is pinned to the ratio of true line pitch to the model's line + // pitch (robust to a few double-gaps from sub-lines). Searching scale freely + // lets the fit march the tail items onto the evenly-spaced payment rows. + a0 := medianDiff(bandC) / medianDiff(M) + if a0 <= 0 || math.IsNaN(a0) { + a0 = 0.75 + } + + bestA, bestB, bestCost := a0, 0.0, math.Inf(1) + for a := a0 * 0.9; a <= a0*1.1; a += a0 * 0.004 { + for b := -350.0; b <= 250.0; b += 2 { + if c, ok := assignCost(a, b, M, rows); ok && c < bestCost { + bestCost, bestA, bestB = c, a, b + } + } + } + if os.Getenv("RECEIPT_DEBUG") != "" { + fmt.Fprintf(os.Stderr, "[snap] a0=%.3f -> a=%.3f b=%.1f avgResidual=%.1fpx\n", + a0, bestA, bestB, bestCost/float64(n)) + } + + idxs, _ := assignBands(bestA, bestB, M, rows) + for i := range r.LineItems { + band := rows[idxs[i]] + r.LineItems[i].Box.Y = band.top - 1 + r.LineItems[i].Box.Height = band.bottom - band.top + 2 + } +} + +// assignBands maps each item (in order) to a strictly-increasing band index via +// greedy nearest-after-previous. Returns the indices and total residual. +func assignBands(a, b float64, M []float64, rows []rowBand) ([]int, float64) { + idxs := make([]int, len(M)) + prev, total := -1, 0.0 + for i := range M { + y := a*M[i] + b + idx, bd := -1, math.Inf(1) + for j := prev + 1; j < len(rows); j++ { + if d := math.Abs(float64(rows[j].center) - y); d < bd { + bd, idx = d, j + } else if float64(rows[j].center) > y { + break // distances only grow past y + } + } + if idx < 0 { + idx = len(rows) - 1 + } + idxs[i], prev, total = idx, idx, total+bd + } + return idxs, total +} + +// assignCost returns the residual of a monotonic assignment; ok is false if the +// transform can't fit all items into strictly-increasing bands. +func assignCost(a, b float64, M []float64, rows []rowBand) (float64, bool) { + if len(rows) < len(M) { + return 0, false + } + prev := -1 + for i := range M { + y := a*M[i] + b + idx, bd := -1, math.Inf(1) + for j := prev + 1; j < len(rows); j++ { + if d := math.Abs(float64(rows[j].center) - y); d < bd { + bd, idx = d, j + } else if float64(rows[j].center) > y { + break + } + } + if idx < 0 || len(rows)-idx < len(M)-i { + return 0, false // ran out of bands for the remaining items + } + prev = idx + _ = bd + } + _, total := assignBands(a, b, M, rows) + return total, true +} + +func medianDiff(xs []float64) float64 { + if len(xs) < 2 { + return 0 + } + d := make([]float64, 0, len(xs)-1) + for i := 1; i < len(xs); i++ { + if xs[i]-xs[i-1] > 0 { + d = append(d, xs[i]-xs[i-1]) + } + } + if len(d) == 0 { + return 0 + } + sort.Float64s(d) + return d[len(d)/2] +} + +func loadReceipt(path string) (*Receipt, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var r Receipt + if err := json.Unmarshal(b, &r); err != nil { + return nil, err + } + return &r, nil +} + +// ---------- rendering ---------- + +const sidebarW = 420 + +func render(src image.Image, r *Receipt, outPath string) error { + b := src.Bounds() + w, h := b.Dx(), b.Dy() + + canvas := image.NewRGBA(image.Rect(0, 0, w+sidebarW, h)) + // white background + draw.Draw(canvas, canvas.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) + // the receipt photo on the left + draw.Draw(canvas, image.Rect(0, 0, w, h), src, b.Min, draw.Src) + + subtotals := map[string]float64{} + + for _, it := range r.LineItems { + col, ok := categoryColor[it.Category] + if !ok { + col = categoryColor["other_groceries"] + } + subtotals[it.Category] += it.Price + + rect := clampRect(it.Box, w, h) + // pale category highlight over the whole printed line + fill := color.RGBA{col.R, col.G, col.B, 110} + draw.Draw(canvas, rect, image.NewUniform(fill), image.Point{}, draw.Over) + drawBorder(canvas, rect, col, 1) + + // category tag in the margin to the right of the line (no overlap with text) + yMid := (rect.Min.Y + rect.Max.Y) / 2 + drawTag(canvas, rect.Max.X+8, yMid, prettyCat(it.Category), col) + } + + drawSidebar(canvas, w, h, r, subtotals) + + f, err := os.Create(outPath) + if err != nil { + return err + } + defer f.Close() + return png.Encode(f, canvas) +} + +func drawSidebar(canvas *image.RGBA, w, h int, r *Receipt, subtotals map[string]float64) { + x0 := w + 1 + // faint divider + draw.Draw(canvas, image.Rect(w, 0, w+1, h), image.NewUniform(color.RGBA{200, 200, 200, 255}), image.Point{}, draw.Src) + + black := color.RGBA{20, 20, 20, 255} + y := 28 + drawText(canvas, x0+16, y, "RECEIPT SUMMARY", black) + y += 26 + drawText(canvas, x0+16, y, fmt.Sprintf("Type: %s", prettyCat(r.ReceiptType)), black) + y += 20 + drawText(canvas, x0+16, y, fmt.Sprintf("Total: %s%.2f", sym(r.Currency), r.TotalAmount), black) + y += 30 + drawText(canvas, x0+16, y, "Category subtotals", black) + y += 8 + draw.Draw(canvas, image.Rect(x0+16, y, w+sidebarW-16, y+1), image.NewUniform(color.RGBA{200, 200, 200, 255}), image.Point{}, draw.Src) + y += 20 + + var catSum float64 + for _, c := range categoryOrder { + v, ok := subtotals[c] + if !ok { + continue + } + catSum += v + col := categoryColor[c] + // swatch + draw.Draw(canvas, image.Rect(x0+16, y-11, x0+34, y+3), image.NewUniform(col), image.Point{}, draw.Src) + drawBorder(canvas, image.Rect(x0+16, y-11, x0+34, y+3), black, 1) + drawText(canvas, x0+42, y, fmt.Sprintf("%-16s %s%6.2f", prettyCat(c), sym(r.Currency), v), black) + y += 22 + } + + y += 6 + draw.Draw(canvas, image.Rect(x0+16, y, w+sidebarW-16, y+1), image.NewUniform(color.RGBA{200, 200, 200, 255}), image.Point{}, draw.Src) + y += 20 + drawText(canvas, x0+16, y, fmt.Sprintf("Items subtotal: %s%.2f", sym(r.Currency), catSum), black) + y += 20 + drawText(canvas, x0+16, y, fmt.Sprintf("(taxes/fees not itemized)"), color.RGBA{120, 120, 120, 255}) +} + +// drawTag draws a solid colored chip with a category name, vertically centered +// on y, with its left edge at x. Placed in the margin beside each line. +func drawTag(dst *image.RGBA, x, y int, s string, col color.RGBA) { + wpx := len(s)*7 + 10 + r := image.Rect(x, y-9, x+wpx, y+8) + draw.Draw(dst, r, image.NewUniform(color.RGBA{col.R, col.G, col.B, 255}), image.Point{}, draw.Over) + drawBorder(dst, r, darken(col), 1) + drawText(dst, x+5, y+4, s, color.RGBA{30, 30, 30, 255}) +} + +func darken(c color.RGBA) color.RGBA { + return color.RGBA{c.R * 7 / 10, c.G * 7 / 10, c.B * 7 / 10, 255} +} + +func drawText(dst *image.RGBA, x, y int, s string, c color.Color) { + d := &font.Drawer{ + Dst: dst, + Src: image.NewUniform(c), + Face: basicfont.Face7x13, + Dot: fixed.P(x, y), + } + d.DrawString(s) +} + +func drawBorder(dst *image.RGBA, r image.Rectangle, c color.Color, thick int) { + u := image.NewUniform(c) + draw.Draw(dst, image.Rect(r.Min.X, r.Min.Y, r.Max.X, r.Min.Y+thick), u, image.Point{}, draw.Over) + draw.Draw(dst, image.Rect(r.Min.X, r.Max.Y-thick, r.Max.X, r.Max.Y), u, image.Point{}, draw.Over) + draw.Draw(dst, image.Rect(r.Min.X, r.Min.Y, r.Min.X+thick, r.Max.Y), u, image.Point{}, draw.Over) + draw.Draw(dst, image.Rect(r.Max.X-thick, r.Min.Y, r.Max.X, r.Max.Y), u, image.Point{}, draw.Over) +} + +func clampRect(box Box, w, h int) image.Rectangle { + x0, y0 := box.X, box.Y + x1, y1 := box.X+box.Width, box.Y+box.Height + if x0 < 0 { + x0 = 0 + } + if y0 < 0 { + y0 = 0 + } + if x1 > w { + x1 = w + } + if y1 > h { + y1 = h + } + if x1 <= x0 { + x1 = x0 + 1 + } + if y1 <= y0 { + y1 = y0 + 1 + } + return image.Rect(x0, y0, x1, y1) +} + +// ---------- helpers ---------- + +func printSummary(r *Receipt) { + fmt.Printf("\nReceipt type: %s\nTotal: %s%.2f\n", prettyCat(r.ReceiptType), sym(r.Currency), r.TotalAmount) + if len(r.LineItems) == 0 { + return + } + fmt.Printf("\n%d line items:\n", len(r.LineItems)) + subtotals := map[string]float64{} + for _, it := range r.LineItems { + subtotals[it.Category] += it.Price + fmt.Printf(" %-28s %s%6.2f %s\n", trunc(it.Name, 28), sym(r.Currency), it.Price, prettyCat(it.Category)) + } + fmt.Println("\nCategory subtotals:") + for _, c := range categoryOrder { + if v, ok := subtotals[c]; ok { + fmt.Printf(" %-18s %s%6.2f\n", prettyCat(c), sym(r.Currency), v) + } + } +} + +func trunc(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n-1] + "…" +} + +func sym(currency string) string { + switch strings.ToUpper(strings.TrimSpace(currency)) { + case "USD", "$", "": + return "$" + case "EUR": + return "€" + case "GBP": + return "£" + case "CAD": + return "C$" + } + return currency + " " +} + +// loadAPIKey reads CLAUDE_API_KEY (or ANTHROPIC_API_KEY) from the environment, +// falling back to a .env file in the working directory or this program's folder. +func loadAPIKey() (string, error) { + for _, k := range []string{"CLAUDE_API_KEY", "ANTHROPIC_API_KEY"} { + if v := os.Getenv(k); v != "" { + return v, nil + } + } + candidates := []string{".env"} + if exe, err := os.Executable(); err == nil { + candidates = append(candidates, filepath.Join(filepath.Dir(exe), ".env")) + } + candidates = append(candidates, + "/home/jm/programming/HSAmanager/.env", + "/home/jm/programming/HSAmanager/receiptscan/.env") + for _, p := range candidates { + if v := keyFromEnvFile(p); v != "" { + return v, nil + } + } + return "", fmt.Errorf("no CLAUDE_API_KEY / ANTHROPIC_API_KEY in env or .env") +} + +func keyFromEnvFile(path string) string { + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + line = strings.TrimPrefix(line, "export ") + for _, k := range []string{"CLAUDE_API_KEY", "ANTHROPIC_API_KEY"} { + if strings.HasPrefix(line, k+"=") { + v := strings.TrimSpace(line[len(k)+1:]) + v = strings.Trim(v, `"'`) + if v != "" { + return v + } + } + } + } + return "" +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) +}