diff --git a/README.md b/README.md index e3aa2fd..97ff49c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ local model through Ollama. Your data never leaves your computer unless you conn - **Kaspi Gold PDF statements** — parsed by text coordinates and reconciled with the statement header totals to the tiyn; re-importing or overlapping statements never duplicates operations. + Pick or drop several at once: they are collected with a preview (card, period, operations, + totals, whose, already imported) and imported, oldest first, when you press Import. - **Batched questions** — the local model guesses the category of each new merchant, questions pile up in batches, and a whole batch is answered at once; answered merchants are categorized automatically. - **Home dashboard** — this month so far against the same days of the last one, the month's pace, diff --git a/internal/web/gate.go b/internal/web/gate.go index fc93080..3ee8639 100644 --- a/internal/web/gate.go +++ b/internal/web/gate.go @@ -548,8 +548,15 @@ func (g *Gate) drop(id string) { // there and signs this browser in to it. func (g *Gate) importNewAccount(w http.ResponseWriter, r *http.Request, v viewer, list []accounts.Account) { ctx := r.Context() - upload := r.PostFormValue("upload") - b, err := g.held(upload) + // a held upload, or a statement from the list of ones waiting + upload, staged := r.PostFormValue("upload"), r.PostFormValue("staged") + var b []byte + var err error + if staged != "" { + b, err = g.stagedBytes(v.Account.ID, staged) + } else { + b, err = g.held(upload) + } if err != nil { http.Redirect(w, r, "/ui?msg="+urlq("The uploaded statement is gone (they are kept for an hour). Upload it again."), http.StatusSeeOther) return @@ -593,7 +600,11 @@ func (g *Gate) importNewAccount(w http.ResponseWriter, r *http.Request, v viewer return } } - g.drop(upload) + if staged != "" { + g.unstage(v.Account.ID, staged) + } else { + g.drop(upload) + } if err := g.startSession(w, r, na.ID); err != nil { g.fail(w, err) return diff --git a/internal/web/staging.go b/internal/web/staging.go new file mode 100644 index 0000000..c7dd1dd --- /dev/null +++ b/internal/web/staging.go @@ -0,0 +1,363 @@ +package web + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "spendbot/internal/accounts" + "spendbot/internal/importer" + "spendbot/internal/statement" +) + +// Statements are collected first and imported on a button press: several files at once are +// parsed for a preview — card, period, operations, totals, whose — and wait on disk in the +// account's staging folder (a week at most) until the person imports or removes them. + +const ( + maxStaged = 40 + maxBatchBytes = 200 << 20 + stagedTTL = 7 * 24 * time.Hour +) + +// stagedFile is a statement waiting to be imported. What is parsed at upload is kept; whether +// it is already imported or another person's is worked out when the list is shown. +type stagedFile struct { + ID string `json:"id"` + Name string `json:"name"` + Size int `json:"size"` + Added time.Time `json:"added"` + Account string `json:"account"` + Holder string `json:"holder"` + From time.Time `json:"from"` + To time.Time `json:"to"` + Ops int `json:"ops"` + Verify string `json:"verify"` // the totals do not match: why + ParseErr string `json:"parse_error"` + + Known bool `json:"-"` // a statement for this card and period is already imported + Other bool `json:"-"` // another person's statement +} + +func (g *Gate) stageDir(accountID int64) string { + return filepath.Join(g.uploads, "staged", strconv.FormatInt(accountID, 10)) +} + +func validID(id string) bool { + return len(id) == 32 && strings.Trim(id, "0123456789abcdef") == "" +} + +// stage keeps a file with its preview. +func (g *Gate) stage(accountID int64, f stagedFile, b []byte) error { + dir := g.stageDir(accountID) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + meta, err := json.Marshal(f) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, f.ID+".pdf"), b, 0o600); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, f.ID+".json"), meta, 0o600) +} + +// staged lists the files waiting, the oldest period first; ones older than a week are removed. +func (g *Gate) staged(accountID int64) ([]stagedFile, error) { + dir := g.stageDir(accountID) + metas, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err != nil { + return nil, err + } + var out []stagedFile + for _, m := range metas { + b, err := os.ReadFile(m) + if err != nil { + continue + } + var f stagedFile + if json.Unmarshal(b, &f) != nil || !validID(f.ID) { + continue + } + if g.now().Sub(f.Added) > stagedTTL { + g.unstage(accountID, f.ID) + continue + } + out = append(out, f) + } + sort.SliceStable(out, func(i, j int) bool { + if !out[i].From.Equal(out[j].From) { + return out[i].From.Before(out[j].From) + } + return out[i].Added.Before(out[j].Added) + }) + return out, nil +} + +func (g *Gate) stagedBytes(accountID int64, id string) ([]byte, error) { + if !validID(id) { + return nil, errors.New("no such file") + } + return os.ReadFile(filepath.Join(g.stageDir(accountID), id+".pdf")) +} + +func (g *Gate) unstage(accountID int64, id string) { + if !validID(id) { + return + } + for _, ext := range []string{".pdf", ".json"} { + if err := os.Remove(filepath.Join(g.stageDir(accountID), id+ext)); err != nil && !errors.Is(err, os.ErrNotExist) { + g.log.Warn("web: remove a staged file", "err", err) + } + } +} + +func newID() (string, error) { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return "", err + } + return hex.EncodeToString(raw), nil +} + +// preview parses a statement for the list without importing it. +func (s *Server) preview(name string, b []byte, now time.Time) stagedFile { + f := stagedFile{Name: name, Size: len(b), Added: now} + st, err := statement.ParseKaspiBytes(b, s.loc) + if err != nil { + f.ParseErr = err.Error() + return f + } + f.Account, f.Holder, f.From, f.To, f.Ops = st.Account, st.Holder, st.From, st.To, len(st.Ops) + if err := st.Verify(); err != nil { + f.Verify = err.Error() + } + return f +} + +// stageUpload takes one or more PDF files and adds them to the list. +func (s *Server) stageUpload(w http.ResponseWriter, r *http.Request) { + v := viewerFrom(r.Context()) + if s.gate == nil || v.Account.ID == 0 { + http.NotFound(w, r) + return + } + back := func(msg string) { http.Redirect(w, r, "/ui/import/staged?msg="+urlq(msg), http.StatusSeeOther) } + r.Body = http.MaxBytesReader(w, r.Body, maxBatchBytes) + if err := r.ParseMultipartForm(32 << 20); err != nil { + back("The files are too large together (up to 200 MB at once); add them in parts.") + return + } + defer func() { _ = r.MultipartForm.RemoveAll() }() + files := r.MultipartForm.File["pdf"] + if len(files) == 0 { + back("Choose one or more PDF statements.") + return + } + have, err := s.gate.staged(v.Account.ID) + if err != nil { + s.fail(w, err) + return + } + added, skipped := 0, 0 + for _, fh := range files { + if len(have)+added >= maxStaged { + skipped++ + continue + } + if fh.Size > maxUpload { + skipped++ + continue + } + fr, err := fh.Open() + if err != nil { + skipped++ + continue + } + b, err := io.ReadAll(fr) + fr.Close() + if err != nil { + skipped++ + continue + } + f := s.preview(trimTo(filepath.Base(fh.Filename), 120), b, s.now()) + if f.ID, err = newID(); err != nil { + s.fail(w, err) + return + } + if err := s.gate.stage(v.Account.ID, f, b); err != nil { + s.fail(w, err) + return + } + added++ + } + msg := fmt.Sprintf("%d %s added — check them and press Import.", added, plural(added, "statement", "statements")) + if skipped > 0 { + msg += fmt.Sprintf(" %d skipped (over 20 MB, unreadable, or over %d waiting).", skipped, maxStaged) + } + back(msg) +} + +type stagedData struct { + Page + Flash string + Files []stagedFile + Ready int // can be imported + Mine string + CanCreate bool + Results []stagedResult + NewFor *stagedFile // the file a new account is being made for +} + +type stagedResult struct { + Name string + From time.Time + To time.Time + Result importer.Result + Err string +} + +// stagedPage lists the files waiting with their previews. +func (s *Server) stagedPage(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + v := viewerFrom(ctx) + if s.gate == nil || v.Account.ID == 0 { + http.NotFound(w, r) + return + } + d := &stagedData{Flash: r.URL.Query().Get("msg"), Mine: v.Account.Holder, CanCreate: v.Account.Hash != "" || !v.Open} + var err error + if d.Files, err = s.gate.staged(v.Account.ID); err != nil { + s.fail(w, err) + return + } + known, err := s.st.Statements(ctx, s.loc) + if err != nil { + s.fail(w, err) + return + } + for i := range d.Files { + f := &d.Files[i] + for _, k := range known { + if k.Account == f.Account && k.From.Equal(f.From) && k.To.Equal(f.To) && k.Ops >= f.Ops { + f.Known = true + } + } + f.Other = f.Holder != "" && v.Account.Holder != "" && !accounts.SameHolder(f.Holder, v.Account.Holder) + if f.ParseErr == "" { + d.Ready++ + } + if f.ID == r.URL.Query().Get("new") && d.CanCreate { + d.NewFor = f + } + } + s.show(w, r, "staged.html", "Statements to import", "home", d) +} + +// stagedAction imports the chosen files, oldest period first, or removes one or all. +func (s *Server) stagedAction(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + v := viewerFrom(ctx) + if s.gate == nil || v.Account.ID == 0 { + http.NotFound(w, r) + return + } + back := func(msg string) { http.Redirect(w, r, "/ui/import/staged?msg="+urlq(msg), http.StatusSeeOther) } + if err := r.ParseForm(); err != nil { + http.Error(w, "invalid form", http.StatusBadRequest) + return + } + files, err := s.gate.staged(v.Account.ID) + if err != nil { + s.fail(w, err) + return + } + switch r.PostFormValue("action") { + case "remove": + s.gate.unstage(v.Account.ID, r.PostFormValue("id")) + back("Removed from the list.") + case "clear": + for _, f := range files { + s.gate.unstage(v.Account.ID, f.ID) + } + back("The list is empty.") + case "import": + chosen := map[string]bool{} + for _, id := range r.PostForm["id"] { + chosen[id] = true + } + d := &stagedData{Mine: v.Account.Holder} + holder := v.Account.Holder + for _, f := range files { // already the oldest period first + if !chosen[f.ID] || f.ParseErr != "" { + continue + } + res := stagedResult{Name: f.Name, From: f.From, To: f.To} + b, err := s.gate.stagedBytes(v.Account.ID, f.ID) + if err != nil { + res.Err = "the file is gone; add it again" + d.Results = append(d.Results, res) + continue + } + st, err := statement.ParseKaspiBytes(b, s.loc) + if err != nil { + res.Err = err.Error() + d.Results = append(d.Results, res) + continue + } + if res.Result, err = importer.Import(ctx, s.st, st, s.now()); err != nil { + s.fail(w, err) + return + } + // the first statement of an account tells whose it is + if holder == "" && st.Holder != "" { + holder = st.Holder + if err := s.gate.SetHolder(ctx, v.Account.ID, holder); err != nil { + s.fail(w, err) + return + } + } + s.gate.unstage(v.Account.ID, f.ID) + d.Results = append(d.Results, res) + } + if len(d.Results) == 0 { + back("Tick at least one statement to import.") + return + } + s.log.Info("web: statements imported", "files", len(d.Results)) + if s.kick != nil { + s.kick.Kick() + } + if d.Files, err = s.gate.staged(v.Account.ID); err != nil { + s.fail(w, err) + return + } + s.show(w, r, "staged.html", "Statements imported", "home", d) + default: + http.Error(w, "unknown action", http.StatusBadRequest) + } +} + +// stagedCount is how many statements wait, for the home page. +func (s *Server) stagedCount(r *http.Request) int { + v := viewerFrom(r.Context()) + if s.gate == nil || v.Account.ID == 0 { + return 0 + } + fs, err := s.gate.staged(v.Account.ID) + if err != nil { + return 0 + } + return len(fs) +} diff --git a/internal/web/staging_test.go b/internal/web/staging_test.go new file mode 100644 index 0000000..2ebb6a9 --- /dev/null +++ b/internal/web/staging_test.go @@ -0,0 +1,119 @@ +package web + +import ( + "bytes" + "context" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "spendbot/internal/store" +) + +// stageFiles posts files to the uploader like a browser picking several at once. +func stageFiles(t *testing.T, h http.Handler, files map[string][]byte) *httptest.ResponseRecorder { + t.Helper() + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for name, b := range files { + fw, _ := mw.CreateFormFile("pdf", name) + fw.Write(b) + } + mw.Close() + r := httptest.NewRequest("POST", "/ui/import/stage", &body) + r.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w +} + +func stagingServer(t *testing.T) (http.Handler, *store.Store, *Gate) { + t.Helper() + st, err := store.Open(context.Background(), t.TempDir()+"/t.db") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + mux := http.NewServeMux() + g := gateFor(t, mux, New(st, nil, 3, time.Minute, almaty, slogDiscard()), "") + return mux, st, g +} + +func TestStagingBrokenFile(t *testing.T) { + h, st, g := stagingServer(t) + w := stageFiles(t, h, map[string][]byte{"notes.pdf": []byte("not a pdf")}) + if !strings.Contains(w.Header().Get("Location"), "1+statement+added") { + t.Fatalf("stage: %s", w.Header().Get("Location")) + } + b := (&browser{h: h}).req("GET", "/ui/import/staged", nil).Body.String() + if !strings.Contains(b, "not a readable Kaspi Gold statement") || !strings.Contains(b, "disabled") { + t.Fatalf("list:\n%s", tail(b)) + } + if home := (&browser{h: h}).req("GET", "/ui", nil).Body.String(); !strings.Contains(home, "1 statement waits") { + t.Error("home must say a statement waits") + } + files, _ := g.staged(1) + // a broken file cannot be imported, only removed + if loc := (&browser{h: h}).req("POST", "/ui/import/staged", url.Values{"action": {"import"}, "id": {files[0].ID}}).Header().Get("Location"); !strings.Contains(loc, "Tick+at+least") { + t.Errorf("import a broken file: %s", loc) + } + if rows, _ := st.Ledger(context.Background(), almaty); len(rows) != 0 { + t.Error("nothing is imported") + } + (&browser{h: h}).req("POST", "/ui/import/staged", url.Values{"action": {"remove"}, "id": {files[0].ID}}) + if files, _ = g.staged(1); len(files) != 0 { + t.Error("remove") + } + if _, err := os.Stat(filepath.Join(g.stageDir(1), "x.pdf")); err == nil { + t.Error("nothing is left on disk") + } +} + +// Real statements in statements/: collect two, import them oldest first, add one again. +func TestStagingRealStatements(t *testing.T) { + names, _ := filepath.Glob("../../statements/*.pdf") + if len(names) < 2 { + t.Skip("needs two statements in statements/") + } + h, st, g := stagingServer(t) + files := map[string][]byte{} + for _, n := range names[:2] { + b, _ := os.ReadFile(n) + files[filepath.Base(n)] = b + } + stageFiles(t, h, files) + if rows, _ := st.Ledger(context.Background(), almaty); len(rows) != 0 { + t.Fatal("collecting imports nothing") + } + staged, _ := g.staged(1) + if len(staged) != 2 || staged[0].Ops == 0 || staged[0].From.After(staged[1].From) { + t.Fatalf("staged, oldest first: %+v", staged) + } + page := (&browser{h: h}).req("GET", "/ui/import/staged", nil).Body.String() + if strings.Count(page, `type="checkbox" name="id"`) != 2 || !strings.Contains(page, "totals match") { + t.Fatalf("list: ids %d, match %v, differ %v", strings.Count(page, `type="checkbox" name="id"`), strings.Contains(page, "totals match"), strings.Contains(page, "totals differ")) + } + w := (&browser{h: h}).req("POST", "/ui/import/staged", url.Values{"action": {"import"}, "id": {staged[0].ID, staged[1].ID}}) + if b := w.Body.String(); w.Code != 200 || !strings.Contains(b, "Statements imported") || strings.Count(b, "operations") != 2 { + t.Fatalf("results %d:\n%s", w.Code, tail(b)) + } + if left, _ := g.staged(1); len(left) != 0 { + t.Error("imported files leave the list") + } + if a, _ := g.reg.Get(context.Background(), 1); a.Holder == "" { + t.Error("the first statement tells whose the account is") + } + // the same statement again is marked as imported and not ticked + b, _ := os.ReadFile(names[0]) + stageFiles(t, h, map[string][]byte{"again.pdf": b}) + page = (&browser{h: h}).req("GET", "/ui/import/staged", nil).Body.String() + if !strings.Contains(page, "already imported") || strings.Contains(page, `checked aria-label="Import again.pdf"`) { + t.Errorf("again:\n%s", tail(page)) + } +} diff --git a/internal/web/templates/index.html b/internal/web/templates/index.html index f23dd51..449d921 100644 --- a/internal/web/templates/index.html +++ b/internal/web/templates/index.html @@ -102,35 +102,6 @@

{{if $s}}{{$s.Month.Title}}{{else}}Welcome{{end}}

{{end}}

Upload a statement

-
- {{template "icon" "upload"}} -
- Kaspi Gold PDF statement -
Kaspi.kz → My Bank → Kaspi Gold → Statement, up to one year. Uploading the same period again duplicates nothing.
-
The upload starts as soon as you pick a file — or drop it here.
-
- - - -
- +{{if .Staged}}{{template "icon" "upload"}}
{{.Staged}} {{plural .Staged "statement waits" "statements wait"}} to be imported. Check and import them →
{{end}} +{{template "uploader" .}} {{template "foot" .}} diff --git a/internal/web/templates/layout.html b/internal/web/templates/layout.html index c581b6a..7b0cc4a 100644 --- a/internal/web/templates/layout.html +++ b/internal/web/templates/layout.html @@ -367,6 +367,8 @@ .stackbar { display: flex; height: 18px; border-radius: 6px; overflow: hidden; gap: 2px; background: var(--surface); } .stackbar > * { display: block; height: 100%; min-width: 2px; } .stackbar > a:hover, .stackbar > a:focus-visible { filter: brightness(1.12); outline: 2px solid var(--ink); outline-offset: -2px; } +.staged-bar { position: sticky; bottom: 0; display: flex; gap: 14px; align-items: center; flex-wrap: wrap; padding: 14px 0; background: linear-gradient(transparent, var(--bg) 35%); margin-top: 6px; } +.staged-row input[type=checkbox] { width: 18px; height: 18px; } .long-view { display: flex; gap: 10px 32px; flex-wrap: wrap; padding: 14px 20px; margin-top: 14px; } .long-view a { display: flex; flex-direction: column; text-decoration: none; } .long-view a b { font-size: 18px; } diff --git a/internal/web/templates/parts.html b/internal/web/templates/parts.html index 88e26ef..5319700 100644 --- a/internal/web/templates/parts.html +++ b/internal/web/templates/parts.html @@ -115,5 +115,40 @@ {{end}}{{end}} +{{/* The statement uploader: one or many PDFs, picked or dropped; they are collected, not imported. */}} +{{define "uploader"}} +
+ {{template "icon" "upload"}} +
+ Kaspi Gold PDF statements +
Kaspi.kz → My Bank → Kaspi Gold → Statement, up to a year each. Pick several at once: they are collected and checked first, and imported when you press Import.
+
Pick or drop one or more files here.
+
+ + + +
+ +{{end}} + {{/* passnote: cash that only passed through the card and is left out; dict Sum N Period */}} {{define "passnote"}}{{if .Sum}}

{{template "icon" "wallet"}} {{compact .Sum}} in {{.N}} cash {{plural .N "withdrawal" "withdrawals"}} only passed through the card — money put on it from another bank, at an ATM or by people and taken out as cash within a week — and {{plural .N "is" "are"}} not counted as spending. Review

{{end}}{{end}} diff --git a/internal/web/templates/staged.html b/internal/web/templates/staged.html new file mode 100644 index 0000000..5fbce6d --- /dev/null +++ b/internal/web/templates/staged.html @@ -0,0 +1,98 @@ +{{template "head" .Page}} +
+

{{if .Results}}Statements imported{{else}}Statements to import{{end}}

+
{{if .Results}}{{len .Results}} {{plural (len .Results) "statement" "statements"}} imported, oldest first{{else}}Collected here until you press Import · nothing is imported yet{{end}}
+
{{template "icon" "left"}}Home
+
+{{template "flash" .Flash}} + +{{if .Results}} +{{$sum := 0}} +
+ {{range .Results}} +
+ +
{{.Name}} +
{{if .Err}}{{.Err}}{{else}}{{date .From}} – {{date .To}} + · {{.Result.Total}} operations· new: {{add .Result.Auto .Result.Review}} purchases, {{.Result.Info}} other + · already there: {{add .Result.Duplicates .Result.Matched}}{{if .Result.Verify}}totals differ{{end}}{{end}}
+ {{if not .Err}}{{if .Result.Review}}{{.Result.Review}} need a category{{end}}{{end}} +
+ {{end}} +
+

The analysis runs in the background: questions about new merchants and insights appear on the home page.

+
HomeSee the operations
+{{end}} + +{{with .NewFor}} +
+

A new account for {{.Holder}}

cancel
+

{{.Name}} · {{date .From}} – {{date .To}} goes there, and this browser signs in to the new account. Sign out to come back.

+ +
+
+
+
+
+
+
+{{end}} +{{if .Files}} +{{if .Results}}

Still waiting

{{end}} +
+ +
+ {{range .Files}} + + {{end}} +
+
+ + Oldest first; statements already imported add only what is new. Unticked ones stay in the list. +
+
+{{range .Files}} + +{{end}} +
+{{else if not .Results}} +
{{template "icon" "upload"}}
No statements waiting.
+{{end}} + +

Add statements

+{{template "uploader" .}} + + +{{template "foot" .}} diff --git a/internal/web/web.go b/internal/web/web.go index 37cdfb9..b481e02 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -238,6 +238,9 @@ func (s *Server) Handler() http.Handler { ui.HandleFunc("GET /ui/batch/{id}", s.batch) ui.HandleFunc("POST /ui/batch/{id}", s.answer) ui.HandleFunc("POST /ui/import", s.importPDF) + ui.HandleFunc("POST /ui/import/stage", s.stageUpload) + ui.HandleFunc("GET /ui/import/staged", s.stagedPage) + ui.HandleFunc("POST /ui/import/staged", s.stagedAction) ui.HandleFunc("POST /ui/insight/{id}/dismiss", s.dismiss) s.handler = ui }) @@ -260,6 +263,7 @@ func (s *Server) fail(w http.ResponseWriter, err error) { type indexData struct { Page Update *update.Status // a newer release, shown on the first account's home page + Staged int // statements waiting to be imported Summary *homeSummary // nil — no data yet ModelHint string // the model is not ready — hint to open the settings Open []store.Batch @@ -294,6 +298,7 @@ func (s *Server) index(w http.ResponseWriter, r *http.Request) { d.Waiting, d.BatchAt = n, oldest.Add(s.batchMaxAge) d.Flash = r.URL.Query().Get("msg") d.ModelHint = s.modelHint(ctx) + d.Staged = s.stagedCount(r) if st := s.updateStatus(); st != nil && st.Newer && s.canUpdate() { d.Update = st }