From e46e4b7268e9554e5f8019497a8fdaab30f35aef Mon Sep 17 00:00:00 2001 From: Abhinav Srivastava Date: Thu, 3 Sep 2026 15:58:20 -0500 Subject: [PATCH] Refresh fund NAVs from AMFI, and say how old the prices are MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prices were never updated except by a broker sync, and nothing said so. `app/navs.py` fetches AMFI's daily NAV file — public, no key, nothing personal sent — and marks every mutual fund position. This matters most for Paytm, which reports no NAV at all: those funds were marked at the NAV of their last transaction, and one was 24 days stale. Refreshing corrected India from 97,31,673 to 97,20,028. Matching refuses to guess. A NAV on the wrong fund is worse than a stale one, because a stale price is visibly stale and a wrong one is not. Funds with an ISIN match on it. Paytm funds have none, so the scheme name must reduce to exactly one Direct/Growth scheme after filler words are dropped — which correctly matched "HDFC Index Fund - BSE Sensex Plan" to AMFI's "HDFC BSE Sensex Index Fund" despite the word order, and refuses anything ambiguous. Regular plans and IDCW options are excluded, since an IDCW NAV would permanently understate a growth holding. Resolved ISINs are stored so a name is matched once and can be audited. "Data through" measured the wrong thing. It reported the last transaction date, so a trade this morning made the header look healthy while fund NAVs were a month old. Price age is now reported per asset class as the OLDEST mark in each group, not the newest — one fund priced today does not make the fund holdings current — shown beside the transaction date and bannered past a week. The quotes table was dead: written by upsert_quotes, read by nothing, 0 rows. It now backs pricing. A quote is used only when dated later than the broker snapshot, so a stale quote can never override a fresh sync. Writing the tests found a real bug. results() had a second pricing path for holdings with no transactions — which is every mutual fund — that read positions.price directly and ignored quotes entirely. So the funds this feature exists to reprice were the one group it would not have reached. Both paths now go through _positions(). 22 new tests: feed parsing, the matcher's refusals, and that a stale quote loses to a fresh broker price. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 6 ++ README.md | 1 + app/analytics.py | 70 +++++++++++++++- app/navs.py | 180 ++++++++++++++++++++++++++++++++++++++++++ app/static/index.html | 25 +++++- docs/IMPORTING.md | 29 ++++++- tests/test_navs.py | 176 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 app/navs.py create mode 100644 tests/test_navs.py diff --git a/CLAUDE.md b/CLAUDE.md index c3687eb..715e7ba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,6 +44,12 @@ GET /api/analysis/status rather than blocking. Report the decision line and wher report lives. Always present the result as **third-party generated research, not a recommendation**, and never restate it as your own investment advice. +**"refresh navs" / "update nav"** — `python3 app/navs.py`. Fetches AMFI's daily NAV file +(public, no key, nothing personal sent) and marks every mutual fund position. Zerodha +funds match on ISIN; Paytm funds carry none, so the scheme name must reduce to exactly one +Direct/Growth scheme or it is skipped and reported. Report which funds moved and by how +much. `--dry-run` writes nothing. + **"backup"** — `python3 app/backup.py snapshot manual` (or POST /api/backup). Only the database file is copied; code lives in git, not Drive. Snapshots go to `/trans`, auto-detected on macOS and Linux. Report the filename and diff --git a/README.md b/README.md index 12f50f4..1a940d9 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,7 @@ In Claude Code, in this folder: | `accounts` | shows your broker account numbers | | `classify tickers` | maps unclassified holdings to sectors | | `bootstrap` | rebuilds the database from scratch | +| `refresh navs` | marks mutual funds at today's NAV from AMFI | To import files instead, drop them in `sync/inbox/` and run `python3 app/ingest.py inbox`. Re-importing is always safe — every importer deduplicates, so overlapping exports and diff --git a/app/analytics.py b/app/analytics.py index 6dc1849..6fc22b0 100644 --- a/app/analytics.py +++ b/app/analytics.py @@ -36,10 +36,63 @@ def _txns(c, market=None): return out def _positions(c, market=None): + """Positions marked at the best price available. + + The broker's price is whatever it reported at the last sync. `quotes` can hold + something newer — AMFI publishes fund NAVs daily, and a fund held at Paytm has no + broker price at all, only the NAV of its last transaction. A quote is used only when + it is dated later than the position snapshot, so a stale quote can never override a + fresh broker price. + """ br = _brokers(market) - q = "SELECT ticker,quantity,price FROM positions" + _bw(br) - return {r["ticker"]: Position(r["ticker"], r["quantity"], r["price"]) - for r in c.execute(q, br)} + latest = {} + for r in c.execute( + "SELECT q.ticker, q.date, q.price FROM quotes q" + " JOIN (SELECT ticker, MAX(date) d FROM quotes GROUP BY ticker) m" + " ON m.ticker = q.ticker AND m.d = q.date"): + if r["price"]: + latest[r["ticker"]] = (r["date"], r["price"]) + + out = {} + for r in c.execute("SELECT ticker,quantity,price,asof FROM positions" + _bw(br), br): + price = r["price"] + q = latest.get(r["ticker"]) + if q and r["asof"] and q[0] > r["asof"]: + price = q[1] + out[r["ticker"]] = Position(r["ticker"], r["quantity"], price) + return out + + +def price_asof(c, market=None): + """How old the marks are, per asset class, worst first. + + Reported as the OLDEST effective price date in each group, not the newest: one fund + priced today does not make the fund holdings current, and averaging would hide the + month-old NAV that is the only figure worth warning about. + + A position's effective date is its broker snapshot, or a later quote if one exists — + matching how _positions() actually marks it. + """ + br = _brokers(market) + latest = {r["ticker"]: r["d"] for r in + c.execute("SELECT ticker, MAX(date) d FROM quotes GROUP BY ticker")} + groups = collections.defaultdict(list) + for r in c.execute("SELECT ticker,asset,asof FROM positions" + _bw(br), br): + eff = r["asof"] + q = latest.get(r["ticker"]) + if q and eff and q > eff: + eff = q + if eff: + groups[r["asset"] or "equity"].append(eff) + + today = dt.date.today() + out = [] + for asset, dates in groups.items(): + oldest, newest = min(dates), max(dates) + out.append({"asset": asset, "n": len(dates), "oldest": oldest, "newest": newest, + "days": (today - dt.date.fromisoformat(oldest)).days}) + out.sort(key=lambda x: -x["days"]) + return out def cost_basis(c, market=None): """{ticker: cost of the shares still held}, plus the tickers where it is a guess. @@ -167,6 +220,7 @@ def results(c, as_of=None, market=None): # Fall back to the broker's own average cost so value and P/L are still real; XIRR # genuinely cannot be computed without dated flows, and says so rather than showing 0. seen = {r["ticker"] for r in rows} + marks = _positions(c, market) # one source of truth for what a share is worth br = _brokers(market) # Once a tradebook is loaded, "import a tradebook" stops being the right explanation. # What is left are holdings the equity tradebook structurally cannot contain: bonds @@ -180,7 +234,12 @@ def results(c, as_of=None, market=None): continue inv = 0.0 if M.demerged_from(p["ticker"], CFG.demergers()) \ else (p["avg_cost"] or 0) * p["quantity"] - val = (p["price"] or 0) * p["quantity"] + # Mark through the same path as everything else. This branch used to read + # positions.price directly, so a holding with no transactions — which is every + # mutual fund — silently ignored a fresher quote while the rest of the portfolio + # used it. Two pricing paths is one too many. + marked = marks.get(p["ticker"]) + val = (marked.price if marked else (p["price"] or 0)) * p["quantity"] rows.append({ "ticker": p["ticker"], "xirr": None, "note": _no_history_note(p["asset"], has_txns), @@ -623,7 +682,10 @@ def health(c, market=None): maxd = c.execute("SELECT MAX(date) d FROM transactions" + _bw(br), br).fetchone()["d"] stale = (dt.date.today() - dt.date.fromisoformat(maxd)).days if maxd else None + pa = price_asof(c, market) + worst = max((x["days"] for x in pa if x["days"] is not None), default=None) return {"last_transaction_date": maxd, "days_stale": stale, "runs": runs, + "prices": pa, "price_days_stale": worst, "n_transactions": c.execute("SELECT COUNT(*) n FROM transactions" + _bw(br), br).fetchone()["n"], "n_positions": c.execute("SELECT COUNT(*) n FROM positions" diff --git a/app/navs.py b/app/navs.py new file mode 100644 index 0000000..aea53fc --- /dev/null +++ b/app/navs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Refresh mutual fund NAVs from AMFI. + +Fund prices are the weakest thing in the database. Zerodha reports a live NAV only for +funds you still hold there; Paytm reports none at all, so those positions are marked at +the NAV of their last transaction and drift further out of date every week. + +AMFI publishes every Indian scheme's NAV daily as one public text file — no key, no +account, nothing personal sent. This fetches it and marks the funds we hold. + + python3 app/navs.py fetch and update + python3 app/navs.py --dry-run show what would change, write nothing + +Matching is deliberately strict. A NAV attached to the wrong fund is worse than a stale +one, because a stale price is visibly stale and a wrong one is not. Where a fund carries +an ISIN it is matched on that alone. Where it does not — Paytm statements have no ISIN — +the scheme name must reduce to exactly one Direct/Growth scheme after dropping filler +words, and anything ambiguous is skipped and reported rather than guessed at. A resolved +ISIN is stored so the guess is made once and can be audited afterwards. + +Equities are not covered: AMFI is funds only. Stock prices still come from the broker at +sync time. +""" +import os, re, sys, urllib.request, datetime as dt + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import db as D + +URL = "https://www.amfiindia.com/spages/NAVAll.txt" +# Words that appear in almost every scheme name and so carry no identifying signal. +_FILLER = {"FUND", "PLAN", "OPTION", "SCHEME", "DIRECT", "GROWTH", "THE", "OF", "AND"} + + +def _tokens(s): + return {t for t in re.findall(r"[A-Z0-9&]+", (s or "").upper()) if t not in _FILLER} + + +def fetch(url=URL, timeout=30): + req = urllib.request.Request(url, headers={"User-Agent": "trans (github.com/winnertechserv/trans)"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read().decode("utf-8", "replace") + + +def parse(raw): + """-> list of {isin, name, plan, opt, nav, date, tok}. Rows without an ISIN are + dropped: they cannot be matched safely and are not worth guessing about.""" + out = [] + for line in raw.split("\n"): + p = line.split(";") + if len(p) < 8 or p[0].strip() == "Scheme Code": + continue + _, isin1, isin2, name, plan, opt, nav, date = [x.strip() for x in p[:8]] + isin = isin1 if isin1 and isin1 != "-" else (isin2 if isin2 and isin2 != "-" else None) + if not isin: + continue + try: + nav_f = float(nav) + except ValueError: + continue + out.append({"isin": isin, "name": name, "plan": plan, "opt": opt, + "nav": nav_f, "date": _date(date), "tok": _tokens(name)}) + return out + + +def _date(s): + """'03-Sep-2026' -> '2026-09-03'. Returns None rather than a wrong date.""" + try: + return dt.datetime.strptime(s, "%d-%b-%Y").date().isoformat() + except (ValueError, TypeError): + return None + + +def by_isin(schemes): + return {s["isin"]: s for s in schemes} + + +def resolve_name(schemes, name): + """-> (scheme, reason). Only a single unambiguous Direct/Growth match is accepted.""" + want = _tokens(name) + if not want: + return None, "no usable words in the name" + cands = [s for s in schemes + if s["tok"] == want + and "DIRECT" in s["plan"].upper() + and "GROWTH" in s["opt"].upper() + and "IDCW" not in s["opt"].upper()] + if len(cands) == 1: + return cands[0], "matched on scheme name" + if not cands: + return None, "no Direct/Growth scheme matches that name" + return None, f"ambiguous — {len(cands)} schemes match" + + +def update(c, schemes, dry_run=False): + """Mark every mutual fund position we can identify. -> list of report rows.""" + isins = by_isin(schemes) + names = {r["ticker"]: r["text_value"] for r in c.execute( + "SELECT ticker,text_value FROM fundamentals WHERE metric='name'" + " AND text_value IS NOT NULL")} + # An ISIN resolved on a previous run, so a name is only ever matched once. + known = {r["ticker"]: r["text_value"] for r in c.execute( + "SELECT ticker,text_value FROM fundamentals WHERE metric='isin'" + " AND text_value IS NOT NULL")} + + report = [] + for r in c.execute("SELECT ticker,quantity,price,asof,broker FROM positions" + " WHERE asset='mf' ORDER BY ticker"): + tick = r["ticker"] + label = names.get(tick, tick) + isin = tick if re.fullmatch(r"INF[0-9A-Z]{9}", tick) else known.get(tick) + why = "isin on the holding" if isin == tick else ( + "isin resolved earlier" if isin else None) + + s = isins.get(isin) if isin else None + if s is None: + s, why = resolve_name(schemes, label) + if s is not None: + isin = s["isin"] + + if s is None: + report.append({"ticker": tick, "name": label, "status": "skipped", + "detail": why, "old": r["price"], "new": None}) + continue + + drift = (s["nav"] / r["price"] - 1) * 100 if r["price"] else None + report.append({"ticker": tick, "name": label, "status": "updated", + "detail": why, "old": r["price"], "new": s["nav"], + "date": s["date"], "drift": drift, "isin": isin, + "matched": f"{s['name']} {s['plan']} {s['opt']}".strip()}) + if dry_run: + continue + c.execute("INSERT INTO quotes(ticker,date,price,prev_close) VALUES(?,?,?,NULL)" + " ON CONFLICT(ticker,date) DO UPDATE SET price=excluded.price", + (tick, s["date"], s["nav"])) + if isin and tick != isin: + c.execute("INSERT INTO fundamentals(ticker,asof,metric,value,text_value)" + " VALUES(?,?,?,NULL,?) ON CONFLICT(ticker,asof,metric)" + " DO UPDATE SET text_value=excluded.text_value", + (tick, dt.date.today().isoformat(), "isin", isin)) + if not dry_run: + c.commit() + return report + + +def main(): + dry = "--dry-run" in sys.argv + print(f"fetching {URL}") + try: + raw = fetch() + except Exception as e: + print(f" could not reach AMFI: {type(e).__name__}: {e}") + print(" fund prices are unchanged; nothing was written.") + return 1 + schemes = parse(raw) + print(f" {len(schemes)} schemes\n") + + c = D.connect() + rows = update(c, schemes, dry_run=dry) + if not rows: + print("no mutual fund positions to price.") + return 0 + for r in rows: + if r["status"] == "updated": + d = f"{r['drift']:+.1f}%" if r["drift"] is not None else " n/a" + print(f" {r['name'][:38]:<40} {r['old']:>9,.2f} -> {r['new']:>9,.2f}" + f" {d:>7} as of {r['date']}") + if r["detail"] == "matched on scheme name": + print(f" matched by name to: {r['matched'][:66]}") + else: + print(f" {r['name'][:38]:<40} skipped — {r['detail']}") + n = sum(1 for r in rows if r["status"] == "updated") + print(f"\n{'would update' if dry else 'updated'} {n} of {len(rows)} fund(s)") + if dry: + print("dry run — nothing written") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/static/index.html b/app/static/index.html index 2def105..7e33326 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -16,6 +16,7 @@ .kpi b .pos{color:var(--pos)} .kpi b .neg{color:var(--neg)} .kpi b .na{color:var(--mut)} .kpi.date b{font-size:13px;font-weight:500;color:var(--mut);padding-top:5px} +.pxage{display:block;font-size:11px;color:var(--warn);font-weight:500} .kpi span{color:var(--mut);font-size:11px;text-transform:uppercase;letter-spacing:.06em} nav{display:flex;gap:2px;padding:0 14px;border-bottom:1px solid var(--line);overflow-x:auto} nav button{background:none;border:0;padding:11px 14px;color:var(--mut);cursor:pointer; @@ -304,9 +305,20 @@ +`still held. Counted that way rather than by every flow landing in the window, ` +`because a sale whose purchase predates it would bring proceeds with none of the ` +`cost and read as a spectacular year.`], - ['Data through',h.last_transaction_date - ? h.last_transaction_date+(h.days_stale>1?` (${h.days_stale}d old)`:'') - : 'holdings only', null, 'date'] + // Two different ages, and conflating them hid a month-old fund NAV behind a + // one-day-old trade. Transactions lead; prices are named underneath when they lag. + ['Data through', + (h.last_transaction_date + ? h.last_transaction_date+(h.days_stale>1?` (${h.days_stale}d old)`:'') + : 'holdings only') + + (h.price_days_stale>1 ? `prices ${h.price_days_stale}d old` : ''), + (h.prices||[]).length + ? 'Marks were last refreshed: ' + + h.prices.map(p=>`${p.asset} ${p.days}d ago`).join(', ') + + '. Transactions and prices age separately — a recent trade does not mean ' + + 'current prices.' + : null, + 'date'] ].map(([s,v,t,cls])=>`
` +`${s}${v}
`).join(''); @@ -339,6 +351,13 @@ $('#main').innerHTML=` ${h.days_stale>2?``:''} + ${h.price_days_stale>7?``:''} ${(o.unreconciled||[]).length?`