Leave cash that only passed through the card out of spending

internal/analysis/ledger.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"spendbot/internal/analytics"
 	"spendbot/internal/clickhouse"
 	"spendbot/internal/store"
 )
@@ -40,3 +41,22 @@ func LedgerFromClickHouse(ctx context.Context, ch *clickhouse.Client, loc *time.
 	}
 	return out, nil
 }
+
+// MarkPassThrough leaves out of spending the cash that only passed through the card, unless
+// the person turned that off; every reader of the ledger calls it, so pages and insights
+// count the same.
+func MarkPassThrough(ctx context.Context, st *store.Store, rows []store.LedgerRow) ([]store.LedgerRow, error) {
+	on, err := st.PassThroughCash(ctx)
+	if err != nil || !on {
+		return rows, err
+	}
+	names, err := st.IncomeSources(ctx)
+	if err != nil {
+		return nil, err
+	}
+	sources := map[string]bool{}
+	for _, n := range names {
+		sources[n] = true
+	}
+	return analytics.MarkPassThrough(rows, sources), nil
+}

internal/analysis/runner.go

@@ -54,6 +54,14 @@ func New(st *store.Store, ch *clickhouse.Client, cls *classify.Classifier, provi
 // ledger returns operations for insights: from ClickHouse when configured and reachable,
 // otherwise from SQLite. The computation is the same, the source does not change the numbers.
 func (a *Analyzer) ledger(ctx context.Context) ([]store.LedgerRow, error) {
+	rows, err := a.rawLedger(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return MarkPassThrough(ctx, a.st, rows)
+}
+
+func (a *Analyzer) rawLedger(ctx context.Context) ([]store.LedgerRow, error) {
 	if a.ch != nil {
 		rows, err := LedgerFromClickHouse(ctx, a.ch, a.opts.Location)
 		if err == nil {

internal/analytics/analytics.go

@@ -29,11 +29,12 @@ const (
 )
 
 // IsSpend reports spending: a purchase (refunds negative) or a cash withdrawal in tenge,
-// unless the person skipped it. Transfers to people, top-ups and incoming money are not
-// spending — except transfers the person gave a category (rent and such): those are done.
-// Wallet payments and manual entries have no statement kind.
+// unless the person skipped it or the cash only passed through the card (MarkPassThrough).
+// Transfers to people, top-ups and incoming money are not spending — except transfers the
+// person gave a category (rent and such): those are done. Wallet payments and manual entries
+// have no statement kind.
 func IsSpend(r store.LedgerRow) bool {
-	if r.Currency != "KZT" || r.Status == store.StatusIgnored {
+	if r.Currency != "KZT" || r.Status == store.StatusIgnored || r.PassThrough > 0 && r.Kind == cash {
 		return false
 	}
 	switch r.Kind {

internal/analytics/explore.go

@@ -196,6 +196,7 @@ type Op struct {
 	Full        int64 // the whole operation
 	Categories  []string
 	Spend       bool
+	PassThrough bool // cash that only passed through the card
 }
 
 // Editable reports whether the category of the operation can be changed: purchases, cash
@@ -278,7 +279,7 @@ func Operations(all []store.LedgerRow, f Filter) Ops {
 		}
 		o := Op{TxID: r.TxID, At: r.At, Merchant: merchantName(r), MerchantKey: merchantKey(r), Kind: r.Kind,
 			Status: r.Status, Source: r.Source, Amount: amount, Full: r.Amount, Categories: r.Categories,
-			Spend: spend || IsSpend(r)}
+			Spend: spend || IsSpend(r), PassThrough: r.PassThrough > 0 && r.Kind == cash}
 		if len(terms) > 0 && !matches(o, terms) {
 			continue
 		}

internal/analytics/income.go

@@ -110,6 +110,8 @@ type Flow struct {
 	Deficit   int     // months that ended in the red
 	Basis     string
 	People    bool
+	Passed    int64 // cash that only passed through the card, left out of Out
+	PassedN   int   // withdrawals of it
 }
 
 // DeclaredFor sums the stated salaries covering a month ('YYYY-MM'); an open end runs on.
@@ -138,6 +140,7 @@ func CurrentSalary(ps []store.SalaryPeriod, month string) (amount int64, current
 // CashFlow adds up income and spending by month.
 func CashFlow(all []store.LedgerRow, f FlowFilter) Flow {
 	fl := Flow{Basis: f.Basis, People: f.People}
+	fl.Passed, fl.PassedN = PassedThrough(all, f.Period)
 	if fl.Basis != BasisDeclared && fl.Basis != BasisCard {
 		fl.Basis = BasisAuto
 	}
@@ -173,7 +176,7 @@ func CashFlow(all []store.LedgerRow, f FlowFilter) Flow {
 		case r.Kind == kaspi.TopUp && r.Amount < 0 && f.Sources[r.Merchant]:
 			m.Other -= r.Amount
 		case r.Kind == kaspi.TopUp && r.Amount < 0 && IsPerson(r.Merchant):
-			m.PeopleIn -= r.Amount
+			m.PeopleIn -= r.Amount + r.PassThrough // what left as cash is neither spending nor theirs to repay
 		case r.Kind == kaspi.Transfer && r.Status != store.StatusDone && IsPerson(r.Merchant):
 			m.PeopleOut += r.Amount
 		}

internal/analytics/passthrough.go

@@ -0,0 +1,118 @@
+package analytics
+
+import (
+	"sort"
+	"time"
+
+	"spendbot/internal/kaspi"
+	"spendbot/internal/store"
+)
+
+// PassWindow is how long money put on the card may stay there and still count as passing
+// through when it leaves as cash.
+const PassWindow = 7 * 24 * time.Hour
+
+// MarkPassThrough marks cash that only passed through the card: money put on it that is not
+// income — the person's own money from another bank, a cash deposit, money from a person —
+// and taken out as cash within PassWindow. Such a withdrawal is not spending: whoever cashes
+// out millions for someone else or moves money between banks through an ATM did not spend it,
+// and counting it would inflate every number built on spending. The top-ups that paid for it
+// carry the part of them that left, so it is not counted as money from people either.
+//
+// A withdrawal counts only when the money put on the card before it covers it whole; one with
+// a category the person gave stays spending. sources are top-up senders counted as income:
+// their money is not passing through. Rows are marked in place and returned.
+func MarkPassThrough(rows []store.LedgerRow, sources map[string]bool) []store.LedgerRow {
+	order := make([]int, 0, len(rows))
+	for i, r := range rows {
+		rows[i].PassThrough = 0
+		if r.Currency == "KZT" && r.Status != store.StatusIgnored && (isFunding(r, sources) || isPassCash(r)) {
+			order = append(order, i)
+		}
+	}
+	sort.SliceStable(order, func(a, b int) bool {
+		ra, rb := rows[order[a]], rows[order[b]]
+		if !ra.At.Equal(rb.At) {
+			return ra.At.Before(rb.At)
+		}
+		// statements give dates without times: money in on a day comes before cash out of it
+		if (ra.Amount < 0) != (rb.Amount < 0) {
+			return ra.Amount < 0
+		}
+		return ra.TxID < rb.TxID
+	})
+	type fund struct {
+		i    int
+		left int64
+	}
+	var pool []fund // oldest first
+	for _, i := range order {
+		r := rows[i]
+		if r.Amount < 0 {
+			pool = append(pool, fund{i: i, left: -r.Amount})
+			continue
+		}
+		since := r.At.Add(-PassWindow)
+		for len(pool) > 0 && rows[pool[0].i].At.Before(since) {
+			pool = pool[1:]
+		}
+		var have int64
+		for _, f := range pool {
+			have += f.left
+		}
+		if have < r.Amount {
+			continue
+		}
+		rows[i].PassThrough = r.Amount
+		need := r.Amount
+		for need > 0 {
+			f := &pool[0]
+			take := min(f.left, need)
+			f.left -= take
+			rows[f.i].PassThrough += take
+			need -= take
+			if f.left == 0 {
+				pool = pool[1:]
+			}
+		}
+	}
+	return rows
+}
+
+// isFunding reports money put on the card that is not income and may leave again as cash.
+func isFunding(r store.LedgerRow, sources map[string]bool) bool {
+	return r.Kind == kaspi.TopUp && r.Amount < 0 && !IsSalary(r) && !sources[r.Merchant] &&
+		(IsOwnMoney(r.Merchant) || IsPerson(r.Merchant))
+}
+
+// isPassCash reports a cash withdrawal that may have only passed through.
+func isPassCash(r store.LedgerRow) bool {
+	return r.Kind == cash && r.Amount > 0 && len(r.Categories) == 0
+}
+
+// PassedThrough sums the cash withdrawals in p that passed through the card.
+func PassedThrough(rows []store.LedgerRow, p Period) (sum int64, n int) {
+	for _, r := range rows {
+		if r.PassThrough > 0 && r.Kind == cash && p.contains(r.At) {
+			sum += r.Amount
+			n++
+		}
+	}
+	return sum, n
+}
+
+// Withdrawals counts the cash withdrawals in p without a category: those that count and
+// those the person skipped — what store.SkipWithdrawals and CountWithdrawals would change.
+func Withdrawals(rows []store.LedgerRow, p Period) (counted, skipped int) {
+	for _, r := range rows {
+		if r.Kind != cash || r.Amount <= 0 || len(r.Categories) > 0 || !p.contains(r.At) {
+			continue
+		}
+		if r.Status == store.StatusIgnored {
+			skipped++
+		} else {
+			counted++
+		}
+	}
+	return counted, skipped
+}

internal/analytics/passthrough_test.go

@@ -0,0 +1,85 @@
+package analytics_test
+
+import (
+	"testing"
+	"time"
+
+	. "spendbot/internal/analytics"
+	"spendbot/internal/kaspi"
+	"spendbot/internal/store"
+)
+
+func passRow(id int64, at time.Time, kind, details string, amount int64) store.LedgerRow {
+	return store.LedgerRow{TxID: id, At: at, Amount: amount * tg, Currency: "KZT", Merchant: details,
+		Kind: kind, Status: store.StatusInfo, Source: store.SourceImport}
+}
+
+func passCash(id int64, at time.Time, amount int64) store.LedgerRow {
+	r := passRow(id, at, kaspi.Withdrawal, "", amount)
+	r.Status = store.StatusReview
+	return r
+}
+
+func TestMarkPassThrough(t *testing.T) {
+	day0 := time.Date(2026, 3, 2, 0, 0, 0, 0, almaty) // statements give dates only
+	rows := []store.LedgerRow{
+		// covered whole by own money from another bank the day before
+		passRow(1, day0, kaspi.TopUp, "С карты другого банка", -1_000_000),
+		passCash(2, day0.AddDate(0, 0, 1), 1_000_000),
+		// a cash deposit and its withdrawal on one day; the withdrawal is listed first
+		passCash(3, day0.AddDate(0, 0, 3), 200_000),
+		passRow(4, day0.AddDate(0, 0, 3), kaspi.TopUp, "В Kaspi Банкомате", -200_000),
+		// salary is income: cash from it is spending
+		passRow(5, day0.AddDate(0, 0, 10), kaspi.TopUp, kaspi.Salary, -500_000),
+		passCash(6, day0.AddDate(0, 0, 10), 100_000),
+		// money from a person, partly cashed out
+		passRow(7, day0.AddDate(0, 0, 20), kaspi.TopUp, "Aigerim A.", -300_000),
+		passCash(8, day0.AddDate(0, 0, 21), 120_000),
+		// not covered whole: stays spending, and does not use up the money before it
+		passCash(9, day0.AddDate(0, 0, 22), 400_000),
+		// money put on the card too long ago does not cover it
+		passRow(10, day0.AddDate(0, 1, 0), kaspi.TopUp, "С карты другого банка", -50_000),
+		passCash(11, day0.AddDate(0, 1, 8), 50_000),
+		// a withdrawal the person gave a category stays spending
+		passRow(12, day0.AddDate(0, 2, 0), kaspi.TopUp, "С карты другого банка", -70_000),
+		passCash(13, day0.AddDate(0, 2, 0), 70_000),
+	}
+	rows[len(rows)-1].Categories, rows[len(rows)-1].Splits = []string{"Rent"}, []int64{70_000 * tg}
+	MarkPassThrough(rows, nil)
+
+	want := map[int64]int64{1: 1_000_000, 2: 1_000_000, 3: 200_000, 4: 200_000, 7: 120_000, 8: 120_000}
+	for _, r := range rows {
+		if r.PassThrough != want[r.TxID]*tg {
+			t.Errorf("tx %d: passed %d, want %d", r.TxID, r.PassThrough/tg, want[r.TxID])
+		}
+		if r.Kind == kaspi.Withdrawal && IsSpend(r) == (want[r.TxID] > 0) {
+			t.Errorf("tx %d: spending %v", r.TxID, IsSpend(r))
+		}
+	}
+	all := Period{From: day0, To: day0.AddDate(1, 0, 0)}
+	if sum, n := PassedThrough(rows, all); sum != 1_320_000*tg || n != 3 {
+		t.Errorf("passed through %d in %d", sum/tg, n)
+	}
+	// spending and its drill-downs leave the passed cash out
+	d := Build(rows, all, nil)
+	if d.Totals.Cash != (100_000+400_000+50_000+70_000)*tg {
+		t.Errorf("cash %d", d.Totals.Cash/tg)
+	}
+	if ops := Operations(rows, Filter{Period: all, Merchant: CashKey}); ops.Total != d.Totals.Cash {
+		t.Errorf("cash operations %d", ops.Total/tg)
+	}
+	// money from a person that left as cash is not money from people
+	fl := CashFlow(rows, FlowFilter{Period: all, People: true})
+	if fl.PeopleIn != 180_000*tg || fl.Passed != 1_320_000*tg || fl.PassedN != 3 {
+		t.Errorf("people in %d, passed %d in %d", fl.PeopleIn/tg, fl.Passed/tg, fl.PassedN)
+	}
+	if counted, skipped := Withdrawals(rows, all); counted != 6 || skipped != 0 {
+		t.Errorf("withdrawals %d counted, %d skipped", counted, skipped)
+	}
+
+	// an income source is not passing through, and marking again starts over
+	MarkPassThrough(rows, map[string]bool{"С карты другого банка": true})
+	if rows[0].PassThrough != 0 || rows[1].PassThrough != 0 || !IsSpend(rows[1]) {
+		t.Errorf("income source: %+v %+v", rows[0], rows[1])
+	}
+}

internal/store/edit.go

@@ -162,3 +162,55 @@ func (s *Store) MergeCategory(ctx context.Context, from, into int64) error {
 		return nil
 	})
 }
+
+const keyPassThrough = "cash_pass_through"
+
+// PassThroughCash reports whether cash that only passed through the card is left out of
+// spending (analytics.MarkPassThrough). On unless the person turned it off.
+func (s *Store) PassThroughCash(ctx context.Context) (bool, error) {
+	v, err := s.GetKV(ctx, keyPassThrough)
+	return v != "off", err
+}
+
+func (s *Store) SetPassThroughCash(ctx context.Context, on bool) error {
+	v := "on"
+	if !on {
+		v = "off"
+	}
+	return s.SetKV(ctx, keyPassThrough, v)
+}
+
+// SkipWithdrawals marks the cash withdrawals in [from, to) skipped — not spending — and returns
+// how many it changed. Withdrawals with a category the person gave are left alone; questions
+// about the skipped ones are dropped, as for one skipped operation.
+func (s *Store) SkipWithdrawals(ctx context.Context, from, to time.Time) (int, error) {
+	var n int64
+	err := s.inTx(ctx, func(tx *sql.Tx) error {
+		const which = `kind = ? AND status != ? AND amount_minor > 0 AND occurred_at >= ? AND occurred_at < ?
+			AND NOT EXISTS (SELECT 1 FROM splits WHERE tx_id = transactions.id)`
+		args := []any{kaspi.Withdrawal, StatusIgnored, fmtTime(from), fmtTime(to)}
+		if _, err := tx.ExecContext(ctx, "DELETE FROM questions WHERE tx_id IN (SELECT id FROM transactions WHERE "+which+")", args...); err != nil {
+			return err
+		}
+		res, err := tx.ExecContext(ctx, "UPDATE transactions SET status = ? WHERE "+which, append([]any{StatusIgnored}, args...)...)
+		if err != nil {
+			return err
+		}
+		n, err = res.RowsAffected()
+		return err
+	})
+	return int(n), err
+}
+
+// CountWithdrawals undoes SkipWithdrawals: skipped cash withdrawals in [from, to) are
+// spending again, waiting for a category as when they were imported.
+func (s *Store) CountWithdrawals(ctx context.Context, from, to time.Time) (int, error) {
+	res, err := s.db.ExecContext(ctx, `UPDATE transactions SET status = ?
+		WHERE kind = ? AND status = ? AND amount_minor > 0 AND occurred_at >= ? AND occurred_at < ?`,
+		StatusReview, kaspi.Withdrawal, StatusIgnored, fmtTime(from), fmtTime(to))
+	if err != nil {
+		return 0, err
+	}
+	n, err := res.RowsAffected()
+	return int(n), err
+}