diff --git a/CLAUDE.md b/CLAUDE.md index c3687eb..715e7ba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,6 +44,12 @@ GET /api/analysis/status rather than blocking. Report the decision line and wher report lives. Always present the result as **third-party generated research, not a recommendation**, and never restate it as your own investment advice. +**"refresh navs" / "update nav"** — `python3 app/navs.py`. Fetches AMFI's daily NAV file +(public, no key, nothing personal sent) and marks every mutual fund position. Zerodha +funds match on ISIN; Paytm funds carry none, so the scheme name must reduce to exactly one +Direct/Growth scheme or it is skipped and reported. Report which funds moved and by how +much. `--dry-run` writes nothing. + **"backup"** — `python3 app/backup.py snapshot manual` (or POST /api/backup). Only the database file is copied; code lives in git, not Drive. Snapshots go to `/trans`, auto-detected on macOS and Linux. Report the filename and diff --git a/README.md b/README.md index 12f50f4..1a940d9 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,7 @@ In Claude Code, in this folder: | `accounts` | shows your broker account numbers | | `classify tickers` | maps unclassified holdings to sectors | | `bootstrap` | rebuilds the database from scratch | +| `refresh navs` | marks mutual funds at today's NAV from AMFI | To import files instead, drop them in `sync/inbox/` and run `python3 app/ingest.py inbox`. Re-importing is always safe — every importer deduplicates, so overlapping exports and diff --git a/app/analytics.py b/app/analytics.py index 6dc1849..6fc22b0 100644 --- a/app/analytics.py +++ b/app/analytics.py @@ -36,10 +36,63 @@ def _txns(c, market=None): return out def _positions(c, market=None): + """Positions marked at the best price available. + + The broker's price is whatever it reported at the last sync. `quotes` can hold + something newer — AMFI publishes fund NAVs daily, and a fund held at Paytm has no + broker price at all, only the NAV of its last transaction. A quote is used only when + it is dated later than the position snapshot, so a stale quote can never override a + fresh broker price. + """ br = _brokers(market) - q = "SELECT ticker,quantity,price FROM positions" + _bw(br) - return {r["ticker"]: Position(r["ticker"], r["quantity"], r["price"]) - for r in c.execute(q, br)} + latest = {} + for r in c.execute( + "SELECT q.ticker, q.date, q.price FROM quotes q" + " JOIN (SELECT ticker, MAX(date) d FROM quotes GROUP BY ticker) m" + " ON m.ticker = q.ticker AND m.d = q.date"): + if r["price"]: + latest[r["ticker"]] = (r["date"], r["price"]) + + out = {} + for r in c.execute("SELECT ticker,quantity,price,asof FROM positions" + _bw(br), br): + price = r["price"] + q = latest.get(r["ticker"]) + if q and r["asof"] and q[0] > r["asof"]: + price = q[1] + out[r["ticker"]] = Position(r["ticker"], r["quantity"], price) + return out + + +def price_asof(c, market=None): + """How old the marks are, per asset class, worst first. + + Reported as the OLDEST effective price date in each group, not the newest: one fund + priced today does not make the fund holdings current, and averaging would hide the + month-old NAV that is the only figure worth warning about. + + A position's effective date is its broker snapshot, or a later quote if one exists — + matching how _positions() actually marks it. + """ + br = _brokers(market) + latest = {r["ticker"]: r["d"] for r in + c.execute("SELECT ticker, MAX(date) d FROM quotes GROUP BY ticker")} + groups = collections.defaultdict(list) + for r in c.execute("SELECT ticker,asset,asof FROM positions" + _bw(br), br): + eff = r["asof"] + q = latest.get(r["ticker"]) + if q and eff and q > eff: + eff = q + if eff: + groups[r["asset"] or "equity"].append(eff) + + today = dt.date.today() + out = [] + for asset, dates in groups.items(): + oldest, newest = min(dates), max(dates) + out.append({"asset": asset, "n": len(dates), "oldest": oldest, "newest": newest, + "days": (today - dt.date.fromisoformat(oldest)).days}) + out.sort(key=lambda x: -x["days"]) + return out def cost_basis(c, market=None): """{ticker: cost of the shares still held}, plus the tickers where it is a guess. @@ -167,6 +220,7 @@ def results(c, as_of=None, market=None): # Fall back to the broker's own average cost so value and P/L are still real; XIRR # genuinely cannot be computed without dated flows, and says so rather than showing 0. seen = {r["ticker"] for r in rows} + marks = _positions(c, market) # one source of truth for what a share is worth br = _brokers(market) # Once a tradebook is loaded, "import a tradebook" stops being the right explanation. # What is left are holdings the equity tradebook structurally cannot contain: bonds @@ -180,7 +234,12 @@ def results(c, as_of=None, market=None): continue inv = 0.0 if M.demerged_from(p["ticker"], CFG.demergers()) \ else (p["avg_cost"] or 0) * p["quantity"] - val = (p["price"] or 0) * p["quantity"] + # Mark through the same path as everything else. This branch used to read + # positions.price directly, so a holding with no transactions — which is every + # mutual fund — silently ignored a fresher quote while the rest of the portfolio + # used it. Two pricing paths is one too many. + marked = marks.get(p["ticker"]) + val = (marked.price if marked else (p["price"] or 0)) * p["quantity"] rows.append({ "ticker": p["ticker"], "xirr": None, "note": _no_history_note(p["asset"], has_txns), @@ -623,7 +682,10 @@ def health(c, market=None): maxd = c.execute("SELECT MAX(date) d FROM transactions" + _bw(br), br).fetchone()["d"] stale = (dt.date.today() - dt.date.fromisoformat(maxd)).days if maxd else None + pa = price_asof(c, market) + worst = max((x["days"] for x in pa if x["days"] is not None), default=None) return {"last_transaction_date": maxd, "days_stale": stale, "runs": runs, + "prices": pa, "price_days_stale": worst, "n_transactions": c.execute("SELECT COUNT(*) n FROM transactions" + _bw(br), br).fetchone()["n"], "n_positions": c.execute("SELECT COUNT(*) n FROM positions" diff --git a/app/navs.py b/app/navs.py new file mode 100644 index 0000000..aea53fc --- /dev/null +++ b/app/navs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Refresh mutual fund NAVs from AMFI. + +Fund prices are the weakest thing in the database. Zerodha reports a live NAV only for +funds you still hold there; Paytm reports none at all, so those positions are marked at +the NAV of their last transaction and drift further out of date every week. + +AMFI publishes every Indian scheme's NAV daily as one public text file — no key, no +account, nothing personal sent. This fetches it and marks the funds we hold. + + python3 app/navs.py fetch and update + python3 app/navs.py --dry-run show what would change, write nothing + +Matching is deliberately strict. A NAV attached to the wrong fund is worse than a stale +one, because a stale price is visibly stale and a wrong one is not. Where a fund carries +an ISIN it is matched on that alone. Where it does not — Paytm statements have no ISIN — +the scheme name must reduce to exactly one Direct/Growth scheme after dropping filler +words, and anything ambiguous is skipped and reported rather than guessed at. A resolved +ISIN is stored so the guess is made once and can be audited afterwards. + +Equities are not covered: AMFI is funds only. Stock prices still come from the broker at +sync time. +""" +import os, re, sys, urllib.request, datetime as dt + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import db as D + +URL = "https://www.amfiindia.com/spages/NAVAll.txt" +# Words that appear in almost every scheme name and so carry no identifying signal. +_FILLER = {"FUND", "PLAN", "OPTION", "SCHEME", "DIRECT", "GROWTH", "THE", "OF", "AND"} + + +def _tokens(s): + return {t for t in re.findall(r"[A-Z0-9&]+", (s or "").upper()) if t not in _FILLER} + + +def fetch(url=URL, timeout=30): + req = urllib.request.Request(url, headers={"User-Agent": "trans (github.com/winnertechserv/trans)"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read().decode("utf-8", "replace") + + +def parse(raw): + """-> list of {isin, name, plan, opt, nav, date, tok}. Rows without an ISIN are + dropped: they cannot be matched safely and are not worth guessing about.""" + out = [] + for line in raw.split("\n"): + p = line.split(";") + if len(p) < 8 or p[0].strip() == "Scheme Code": + continue + _, isin1, isin2, name, plan, opt, nav, date = [x.strip() for x in p[:8]] + isin = isin1 if isin1 and isin1 != "-" else (isin2 if isin2 and isin2 != "-" else None) + if not isin: + continue + try: + nav_f = float(nav) + except ValueError: + continue + out.append({"isin": isin, "name": name, "plan": plan, "opt": opt, + "nav": nav_f, "date": _date(date), "tok": _tokens(name)}) + return out + + +def _date(s): + """'03-Sep-2026' -> '2026-09-03'. Returns None rather than a wrong date.""" + try: + return dt.datetime.strptime(s, "%d-%b-%Y").date().isoformat() + except (ValueError, TypeError): + return None + + +def by_isin(schemes): + return {s["isin"]: s for s in schemes} + + +def resolve_name(schemes, name): + """-> (scheme, reason). Only a single unambiguous Direct/Growth match is accepted.""" + want = _tokens(name) + if not want: + return None, "no usable words in the name" + cands = [s for s in schemes + if s["tok"] == want + and "DIRECT" in s["plan"].upper() + and "GROWTH" in s["opt"].upper() + and "IDCW" not in s["opt"].upper()] + if len(cands) == 1: + return cands[0], "matched on scheme name" + if not cands: + return None, "no Direct/Growth scheme matches that name" + return None, f"ambiguous — {len(cands)} schemes match" + + +def update(c, schemes, dry_run=False): + """Mark every mutual fund position we can identify. -> list of report rows.""" + isins = by_isin(schemes) + names = {r["ticker"]: r["text_value"] for r in c.execute( + "SELECT ticker,text_value FROM fundamentals WHERE metric='name'" + " AND text_value IS NOT NULL")} + # An ISIN resolved on a previous run, so a name is only ever matched once. + known = {r["ticker"]: r["text_value"] for r in c.execute( + "SELECT ticker,text_value FROM fundamentals WHERE metric='isin'" + " AND text_value IS NOT NULL")} + + report = [] + for r in c.execute("SELECT ticker,quantity,price,asof,broker FROM positions" + " WHERE asset='mf' ORDER BY ticker"): + tick = r["ticker"] + label = names.get(tick, tick) + isin = tick if re.fullmatch(r"INF[0-9A-Z]{9}", tick) else known.get(tick) + why = "isin on the holding" if isin == tick else ( + "isin resolved earlier" if isin else None) + + s = isins.get(isin) if isin else None + if s is None: + s, why = resolve_name(schemes, label) + if s is not None: + isin = s["isin"] + + if s is None: + report.append({"ticker": tick, "name": label, "status": "skipped", + "detail": why, "old": r["price"], "new": None}) + continue + + drift = (s["nav"] / r["price"] - 1) * 100 if r["price"] else None + report.append({"ticker": tick, "name": label, "status": "updated", + "detail": why, "old": r["price"], "new": s["nav"], + "date": s["date"], "drift": drift, "isin": isin, + "matched": f"{s['name']} {s['plan']} {s['opt']}".strip()}) + if dry_run: + continue + c.execute("INSERT INTO quotes(ticker,date,price,prev_close) VALUES(?,?,?,NULL)" + " ON CONFLICT(ticker,date) DO UPDATE SET price=excluded.price", + (tick, s["date"], s["nav"])) + if isin and tick != isin: + c.execute("INSERT INTO fundamentals(ticker,asof,metric,value,text_value)" + " VALUES(?,?,?,NULL,?) ON CONFLICT(ticker,asof,metric)" + " DO UPDATE SET text_value=excluded.text_value", + (tick, dt.date.today().isoformat(), "isin", isin)) + if not dry_run: + c.commit() + return report + + +def main(): + dry = "--dry-run" in sys.argv + print(f"fetching {URL}") + try: + raw = fetch() + except Exception as e: + print(f" could not reach AMFI: {type(e).__name__}: {e}") + print(" fund prices are unchanged; nothing was written.") + return 1 + schemes = parse(raw) + print(f" {len(schemes)} schemes\n") + + c = D.connect() + rows = update(c, schemes, dry_run=dry) + if not rows: + print("no mutual fund positions to price.") + return 0 + for r in rows: + if r["status"] == "updated": + d = f"{r['drift']:+.1f}%" if r["drift"] is not None else " n/a" + print(f" {r['name'][:38]:<40} {r['old']:>9,.2f} -> {r['new']:>9,.2f}" + f" {d:>7} as of {r['date']}") + if r["detail"] == "matched on scheme name": + print(f" matched by name to: {r['matched'][:66]}") + else: + print(f" {r['name'][:38]:<40} skipped — {r['detail']}") + n = sum(1 for r in rows if r["status"] == "updated") + print(f"\n{'would update' if dry else 'updated'} {n} of {len(rows)} fund(s)") + if dry: + print("dry run — nothing written") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/static/index.html b/app/static/index.html index 2def105..7e33326 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -16,6 +16,7 @@ .kpi b .pos{color:var(--pos)} .kpi b .neg{color:var(--neg)} .kpi b .na{color:var(--mut)} .kpi.date b{font-size:13px;font-weight:500;color:var(--mut);padding-top:5px} +.pxage{display:block;font-size:11px;color:var(--warn);font-weight:500} .kpi span{color:var(--mut);font-size:11px;text-transform:uppercase;letter-spacing:.06em} nav{display:flex;gap:2px;padding:0 14px;border-bottom:1px solid var(--line);overflow-x:auto} nav button{background:none;border:0;padding:11px 14px;color:var(--mut);cursor:pointer; @@ -304,9 +305,20 @@ +`still held. Counted that way rather than by every flow landing in the window, ` +`because a sale whose purchase predates it would bring proceeds with none of the ` +`cost and read as a spectacular year.`], - ['Data through',h.last_transaction_date - ? h.last_transaction_date+(h.days_stale>1?` (${h.days_stale}d old)`:'') - : 'holdings only', null, 'date'] + // Two different ages, and conflating them hid a month-old fund NAV behind a + // one-day-old trade. Transactions lead; prices are named underneath when they lag. + ['Data through', + (h.last_transaction_date + ? h.last_transaction_date+(h.days_stale>1?` (${h.days_stale}d old)`:'') + : 'holdings only') + + (h.price_days_stale>1 ? `prices ${h.price_days_stale}d old` : ''), + (h.prices||[]).length + ? 'Marks were last refreshed: ' + + h.prices.map(p=>`${p.asset} ${p.days}d ago`).join(', ') + + '. Transactions and prices age separately — a recent trade does not mean ' + + 'current prices.' + : null, + 'date'] ].map(([s,v,t,cls])=>`
` +`${s}${v}
`).join(''); @@ -339,6 +351,13 @@ $('#main').innerHTML=` ${h.days_stale>2?``:''} + ${h.price_days_stale>7?``:''} ${(o.unreconciled||[]).length?`