From 209a7c15bb9561a4caad45535bcf86177a900df5 Mon Sep 17 00:00:00 2001 From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:10:05 -0700 Subject: [PATCH 1/2] feat(fred): get_observations, aligned and downsampled (#33) The tool the server exists for, and where the token argument is cashed. Three things it does that a wrapper over /series/observations does not: Columnar output. FRED sends {realtime_start, realtime_end, date, value} per observation, and the two realtime fields hold the same value on every row. Measured on 20 years of DFF: 694,388 chars of raw payload become 2,856. Several series share one date index, so a comparison is one call and arrives aligned, with nulls (not forward-fill) where a quarterly series has no monthly print. Summary before downsampling. latest, prior, change, min, max, mean and count are computed over every observation; only the point list is thinned to max_points. The same DFF call returns 120 of 7,305 points and still reports the true 20-year min of 0.04 and max of 5.41. The fixture puts both extremes in the interior of the series precisely so this test can fail if the order is ever reversed. Argument correction. units="yoy" becomes pc1, start="5y" and "2020" and "18 months" become real dates, frequency="monthly" becomes m. None of these are guessable and all of them are unambiguous, so they are translated rather than returned as an HTTP 400 about a variable name. A bad series ID among good ones costs only its own column. --- plugins/fred/src/dates.py | 87 +++++++++ plugins/fred/src/schemas.py | 110 +++++++++++ plugins/fred/src/shaping.py | 103 +++++++++++ plugins/fred/src/tools.py | 127 ++++++++++++- plugins/fred/tests/fixtures/fred_api.py | 71 ++++++++ .../tests/integration/test_observations.py | 147 +++++++++++++++ plugins/fred/tests/unit/test_dates.py | 90 +++++++++ .../tests/unit/test_observations_shaping.py | 171 ++++++++++++++++++ plugins/fred/tests/unit/test_schemas.py | 93 +++++++++- plugins/fred/tests/unit/test_server.py | 2 +- 10 files changed, 992 insertions(+), 9 deletions(-) create mode 100644 plugins/fred/src/dates.py create mode 100644 plugins/fred/tests/integration/test_observations.py create mode 100644 plugins/fred/tests/unit/test_dates.py create mode 100644 plugins/fred/tests/unit/test_observations_shaping.py diff --git a/plugins/fred/src/dates.py b/plugins/fred/src/dates.py new file mode 100644 index 0000000..dd2bec5 --- /dev/null +++ b/plugins/fred/src/dates.py @@ -0,0 +1,87 @@ +"""Date parsing for arguments FRED will only accept as YYYY-MM-DD. + +A model asked for "the last five years" writes ``start="5y"`` or ``start="2020"`` or +``start="last 5 years"``. FRED answers all three with HTTP 400 and a message about a +variable name. Since none of these are ambiguous, they are translated rather than +rejected. +""" + +import re +from datetime import date + +# "5y", "5 years", "last 5 years", "past 18 months", "10yr". +_RELATIVE = re.compile( + r"^(?:last|past)?\s*(\d+)\s*(d|w|m|q|y|yr|day|days|week|weeks|month|months|quarter|quarters|year|years)$" +) + +_UNIT_MONTHS = {"m": 1, "mo": 1, "month": 1, "months": 1, "q": 3, "quarter": 3, "quarters": 3} +_UNIT_YEARS = {"y": 1, "yr": 1, "year": 1, "years": 1} +_UNIT_DAYS = {"d": 1, "day": 1, "days": 1, "w": 7, "week": 7, "weeks": 7} + + +def today() -> date: + """Indirection so tests can pin the clock without patching the stdlib.""" + return date.today() + + +def parse(value: str, *, field: str) -> str: + """Return a YYYY-MM-DD string, or "" for an unset value. + + Accepted, beyond a full date: a bare year ("2020"), a year-month ("2020-01"), + "today"/"now", "ytd", and a relative span ("5y", "18 months", "last 10 years") + measured back from today. + """ + text = " ".join(str(value).strip().lower().split()) + if not text: + return "" + + if text in ("today", "now"): + return today().isoformat() + if text in ("ytd", "year to date", "this year"): + return today().replace(month=1, day=1).isoformat() + + if re.fullmatch(r"\d{4}", text): + return f"{text}-01-01" + if re.fullmatch(r"\d{4}-\d{2}", text): + return f"{text}-01" + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", text): + # Round-tripped through date() so 2020-13-01 is caught here rather than by FRED. + try: + return date.fromisoformat(text).isoformat() + except ValueError as exc: + raise ValueError(f"{field}={value!r} is not a real date: {exc}") from exc + + match = _RELATIVE.match(text) + if match: + amount, unit = int(match.group(1)), match.group(2) + return _ago(amount, unit).isoformat() + + raise ValueError( + f"{field}={value!r} could not be read as a date. Use YYYY-MM-DD, a year " + '("2020"), a year-month ("2020-01"), a span back from today ("5y", ' + '"18 months"), "ytd", or "today".' + ) + + +def _ago(amount: int, unit: str) -> date: + """Today, minus ``amount`` of ``unit``. Calendar arithmetic, no dependency.""" + now = today() + if unit in _UNIT_DAYS: + return date.fromordinal(max(1, now.toordinal() - amount * _UNIT_DAYS[unit])) + if unit in _UNIT_YEARS: + months = amount * 12 + else: + months = amount * _UNIT_MONTHS[unit] + + total = now.year * 12 + (now.month - 1) - months + year, month = divmod(total, 12) + month += 1 + # Clamp the day: three months before the 31st is not the 31st of a 30-day month. + day = min(now.day, _days_in_month(year, month)) + return date(year, month, day) + + +def _days_in_month(year: int, month: int) -> int: + if month == 12: + return 31 + return (date(year + (month == 12), month % 12 + 1, 1).toordinal()) - date(year, month, 1).toordinal() diff --git a/plugins/fred/src/schemas.py b/plugins/fred/src/schemas.py index 356cdcd..e65062b 100644 --- a/plugins/fred/src/schemas.py +++ b/plugins/fred/src/schemas.py @@ -14,6 +14,8 @@ from pydantic import BaseModel, Field, field_validator, model_validator +from . import dates + MAX_SERIES_PER_CALL = 20 # FRED's frequency codes, and the words a model writes instead. @@ -75,6 +77,45 @@ SERIES_INCLUDES = ("metadata", "notes", "release", "categories", "tags") +# What each FRED units code actually does, echoed in the response so the numbers are +# not left to be guessed at. +UNITS_MEANING: dict[str, str] = { + "lin": "levels, as published", + "chg": "change from the previous period", + "ch1": "change from a year ago", + "pch": "percent change from the previous period", + "pc1": "percent change from a year ago", + "pca": "compounded annual rate of change", + "cch": "continuously compounded rate of change", + "cca": "continuously compounded annual rate of change", + "log": "natural log", +} + +# The single highest-value table in this file. "Year over year percent change" is the +# most-asked transformation in all of economics and FRED spells it "pc1". +UNITS_ALIASES: dict[str, str] = { + "": "lin", "lin": "lin", "level": "lin", "levels": "lin", "none": "lin", + "raw": "lin", "as published": "lin", "nominal": "lin", + "pc1": "pc1", "yoy": "pc1", "y/y": "pc1", "yoy%": "pc1", "year over year": "pc1", + "year-over-year": "pc1", "annual percent change": "pc1", "percent change from year ago": "pc1", + "percent change from a year ago": "pc1", "inflation": "pc1", + "pch": "pch", "mom": "pch", "m/m": "pch", "month over month": "pch", + "percent change": "pch", "pct change": "pch", "pct_change": "pch", + "percent change from previous period": "pch", + "chg": "chg", "change": "chg", "diff": "chg", "difference": "chg", + "change from previous period": "chg", + "ch1": "ch1", "change from year ago": "ch1", "change from a year ago": "ch1", + "pca": "pca", "annualized": "pca", "saar": "pca", "annual rate": "pca", + "compounded annual rate of change": "pca", + "cch": "cch", "cca": "cca", "log": "log", "natural log": "log", "ln": "log", +} # fmt: skip + +AGGREGATION_ALIASES: dict[str, str] = { + "avg": "avg", "average": "avg", "mean": "avg", + "sum": "sum", "total": "sum", + "eop": "eop", "end": "eop", "end of period": "eop", "last": "eop", "close": "eop", +} # fmt: skip + def normalize_series_ids(value: Any) -> Any: """Accept the several ways a model writes a list of series IDs. @@ -231,3 +272,72 @@ def _known_includes(cls, value: list[str]) -> list[str]: def wants(self, part: str) -> bool: return part in self.include + + +class ObservationArgs(BaseModel): + """Arguments for get_observations.""" + + series_ids: list[str] = Field(min_length=1, max_length=MAX_SERIES_PER_CALL) + start: str = "" + end: str = "" + units: str = "lin" + frequency: str = "" + aggregation_method: str = "avg" + # Floor of 2 because downsampling always keeps the first and last point; a cap of + # one cannot express a series. + max_points: int = Field(default=120, ge=2, le=2000) + + @model_validator(mode="before") + @classmethod + def _correct(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + data = dict(data) + if "series_ids" in data: + data["series_ids"] = normalize_series_ids(data["series_ids"]) + for field in ("start", "end"): + if data.get(field) is not None: + data[field] = dates.parse(data[field], field=field) + if data.get("units") is not None: + raw = str(data["units"]).strip().lower() + data["units"] = UNITS_ALIASES.get(raw, raw) + if "frequency" in data: + data["frequency"] = normalize_frequency(data["frequency"]) + if data.get("aggregation_method") is not None: + raw = str(data["aggregation_method"]).strip().lower() + data["aggregation_method"] = AGGREGATION_ALIASES.get(raw, raw) + return data + + @field_validator("units") + @classmethod + def _known_units(cls, value: str) -> str: + if value not in UNITS_MEANING: + raise ValueError( + f"unknown units {value!r}; use a FRED code ({', '.join(UNITS_MEANING)}) " + "or a phrase such as 'yoy', 'percent change', 'level'" + ) + return value + + @field_validator("frequency") + @classmethod + def _known_frequency(cls, value: str) -> str: + if value and value not in FREQUENCY_TAGS: + raise ValueError(f"unknown frequency {value!r}; use one of {', '.join(FREQUENCY_TAGS)}") + return value + + @field_validator("aggregation_method") + @classmethod + def _known_aggregation(cls, value: str) -> str: + if value not in ("avg", "sum", "eop"): + raise ValueError(f"unknown aggregation_method {value!r}; use avg, sum, or eop") + return value + + @model_validator(mode="after") + def _range_is_the_right_way_round(self) -> "ObservationArgs": + if self.start and self.end and self.start > self.end: + raise ValueError(f"start ({self.start}) is after end ({self.end}); swap them") + return self + + @property + def units_meaning(self) -> str: + return UNITS_MEANING[self.units] diff --git a/plugins/fred/src/shaping.py b/plugins/fred/src/shaping.py index 257a84b..e525db3 100644 --- a/plugins/fred/src/shaping.py +++ b/plugins/fred/src/shaping.py @@ -66,3 +66,106 @@ def series_list(payload: dict, *, notes: bool = False) -> list[dict]: def first_or_empty(payload: dict, key: str) -> dict[str, Any]: items = payload.get(key) or [] return items[0] if items else {} + + +# --- observations ------------------------------------------------------------------ +# +# FRED returns one object per observation: +# +# {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", +# "date": "2025-01-01", "value": "2.99098"} +# +# On a normal (non-vintage) request the two realtime fields hold the same value on +# every single row, and the key names repeat once per observation. Twenty years of a +# daily series is around 5,000 of those. Columnar output drops all of it. + +Number = float | None + + +def parse_value(raw: object) -> Number: + """FRED writes values as strings and missing ones as ".", not null.""" + if raw is None: + return None + text = str(raw).strip() + if text in ("", "."): + return None + try: + return float(text) + except ValueError: + return None + + +def observation_pairs(payload: dict) -> list[tuple[str, Number]]: + """(date, value) for each observation, values parsed.""" + return [(obs["date"], parse_value(obs.get("value"))) for obs in payload.get("observations", []) if obs.get("date")] + + +def align(per_series: dict[str, list[tuple[str, Number]]]) -> tuple[list[str], dict[str, list[Number]]]: + """Put several series on one shared, sorted date index. + + This is what makes comparison a single tool call. A monthly and a quarterly series + will not line up, so the index is the union of their dates and the gaps are null + rather than silently dropped or forward-filled, which would invent data. + """ + dates = sorted({d for pairs in per_series.values() for d, _ in pairs}) + index = {date: position for position, date in enumerate(dates)} + + columns: dict[str, list[Number]] = {} + for series_id, pairs in per_series.items(): + column: list[Number] = [None] * len(dates) + for date, value in pairs: + column[index[date]] = value + columns[series_id] = column + return dates, columns + + +def summarize(pairs: list[tuple[str, Number]]) -> dict: + """The figures worth having, computed over every observation. + + Deliberately computed *before* downsampling. A 5,000-point daily series thinned to + 120 still reports its true min, max, and latest value; a summary computed after + would quietly report the extremes of the sample instead of the series. + """ + observed = [(date, value) for date, value in pairs if value is not None] + if not observed: + return {"observations": len(pairs), "count": 0} + + values = [value for _, value in observed] + latest_date, latest = observed[-1] + summary: dict[str, Any] = { + "latest": latest, + "latest_date": latest_date, + "min": min(values), + "max": max(values), + "mean": round(sum(values) / len(values), 6), + "start_date": observed[0][0], + "end_date": latest_date, + "count": len(observed), + "observations": len(pairs), + } + if len(observed) > 1: + prior = observed[-2][1] + summary["prior"] = prior + summary["change"] = round(latest - prior, 6) + if prior: + summary["pct_change"] = round((latest - prior) / abs(prior) * 100, 4) + return summary + + +def downsample(dates: list[str], columns: dict[str, list[Number]], max_points: int) -> tuple[list, dict, int]: + """Thin to evenly spaced points, always keeping the first and the last. + + Returns the kept dates, the kept columns, and how many points were dropped, so a + truncated series is never mistaken for a complete one. + """ + total = len(dates) + if total <= max_points: + return dates, columns, 0 + + step = (total - 1) / (max_points - 1) + keep = sorted({round(i * step) for i in range(max_points)} | {0, total - 1}) + return ( + [dates[i] for i in keep], + {series_id: [column[i] for i in keep] for series_id, column in columns.items()}, + total - len(keep), + ) diff --git a/plugins/fred/src/tools.py b/plugins/fred/src/tools.py index 8338026..6d7ded7 100644 --- a/plugins/fred/src/tools.py +++ b/plugins/fred/src/tools.py @@ -14,7 +14,7 @@ from . import shaping from .client import FredClient from .errors import guarded_tool -from .schemas import GetSeriesArgs, SearchArgs +from .schemas import GetSeriesArgs, ObservationArgs, SearchArgs _client: FredClient | None = None @@ -42,6 +42,15 @@ def fmt(data: object) -> str: return json.dumps(data, indent=2, default=str) +def _http_detail(exc: httpx.HTTPStatusError) -> str: + """FRED's own explanation, which the status code alone does not carry.""" + try: + detail = str(exc.response.json().get("error_message", "")).strip() + except ValueError: + detail = "" + return " ".join(detail.split()) or f"HTTP {exc.response.status_code}" + + async def _search_series(args: SearchArgs) -> dict: """Run one of the three discovery paths and return the shared output shape.""" client = get_client() @@ -109,14 +118,9 @@ async def optional(part: str, path: str) -> dict | None: optional("tags", "/series/tags"), ) except httpx.HTTPStatusError as exc: - detail = "" - try: - detail = str(exc.response.json().get("error_message", "")).strip() - except ValueError: - pass return { "id": series_id, - "error": " ".join(detail.split()) or f"HTTP {exc.response.status_code}", + "error": _http_detail(exc), "suggestion": "Check this ID with search_series; the others in this call were returned.", } @@ -131,6 +135,67 @@ async def optional(part: str, path: str) -> dict | None: return out +Pairs = list[tuple[str, shaping.Number]] + + +async def _observations_for(args: ObservationArgs, series_id: str) -> tuple[str, Pairs | str]: + """One series' observations, or its own error message. A bad ID among good ones + should cost only its own column, not the whole comparison.""" + try: + payload = await get_client().get( + "/series/observations", + series_id=series_id, + observation_start=args.start, + observation_end=args.end, + units=args.units, + frequency=args.frequency, + # FRED ignores this unless frequency is set, so sending it always is safe. + aggregation_method=args.aggregation_method, + sort_order="asc", + ) + except httpx.HTTPStatusError as exc: + return series_id, _http_detail(exc) + return series_id, shaping.observation_pairs(payload) + + +async def _get_observations(args: ObservationArgs) -> dict: + fetched = await asyncio.gather(*(_observations_for(args, sid) for sid in args.series_ids)) + + per_series: dict[str, Pairs] = {} + errors: dict[str, str] = {} + for series_id, outcome in fetched: + if isinstance(outcome, str): + errors[series_id] = outcome + else: + per_series[series_id] = outcome + + # Summaries first, over every observation. Downsampling after, so a thinned series + # still reports its true latest value and true extremes. + summary = {series_id: shaping.summarize(pairs) for series_id, pairs in per_series.items()} + dates, columns = shaping.align(per_series) + total = len(dates) + dates, columns, dropped = shaping.downsample(dates, columns, args.max_points) + + points: dict[str, object] = {"returned": len(dates), "total": total, "dropped": dropped} + if dropped: + points["note"] = "evenly spaced sample; the summary covers every observation" + + result: dict[str, object] = { + "units": args.units, + "units_meaning": args.units_meaning, + "dates": dates, + "values": columns, + "summary": summary, + "points": points, + } + if args.frequency: + result["frequency"] = args.frequency + result["aggregation_method"] = args.aggregation_method + if errors: + result["errors"] = errors + return result + + def register_all(mcp: MCPServer) -> None: """Register every tool on the given MCP server.""" @@ -207,3 +272,51 @@ async def get_series(series_ids: list[str] | str, include: list[str] | str | Non args = GetSeriesArgs.model_validate({"series_ids": series_ids, "include": include or ["metadata"]}) results = await asyncio.gather(*(_fetch_one_series(args, sid) for sid in args.series_ids)) return fmt({"series": list(results)}) + + @mcp.tool() + @guarded_tool + async def get_observations( + series_ids: list[str] | str, + start: str = "", + end: str = "", + units: str = "lin", + frequency: str = "", + aggregation_method: str = "avg", + max_points: int = 120, + ) -> str: + """Get the actual numbers. Pass several series at once to compare them. + + Series come back on one shared date index, so a comparison is a single call: + {"dates": [...], "values": {"UNRATE": [...], "CPIAUCSL": [...]}}. Where a + series has no observation for a date (a quarterly series against a monthly + one) the value is null rather than filled in. + + start and end accept YYYY-MM-DD, a bare year ("2020"), a year-month + ("2020-01"), a span back from today ("5y", "18 months", "last 10 years"), + "ytd", or "today". Omit both for the full history. + + units transforms the series server-side, so do not do this arithmetic + yourself. Say "yoy" for year-over-year percent change (FRED calls it pc1), + "percent change" for period-over-period, "change" for a difference, + "annualized", or "level" for the published numbers. + + frequency aggregates to a coarser interval only ("monthly", "quarterly", + "annual"), with aggregation_method of avg, sum, or eop. + + Every series gets a summary (latest, prior, change, min, max, mean, count) + computed over ALL its observations. Only the returned point list is thinned + to max_points, so a 20-year daily series still reports its true extremes + without spending 5,000 points to do it. + """ + args = ObservationArgs.model_validate( + { + "series_ids": series_ids, + "start": start, + "end": end, + "units": units, + "frequency": frequency, + "aggregation_method": aggregation_method, + "max_points": max_points, + } + ) + return fmt(await _get_observations(args)) diff --git a/plugins/fred/tests/fixtures/fred_api.py b/plugins/fred/tests/fixtures/fred_api.py index 6898aa3..59fd21f 100644 --- a/plugins/fred/tests/fixtures/fred_api.py +++ b/plugins/fred/tests/fixtures/fred_api.py @@ -10,6 +10,8 @@ the filter_variable pairing, the popularity ordering). """ +from datetime import date + import httpx UNRATE = { @@ -71,6 +73,46 @@ SERIES = {s["id"]: s for s in (UNRATE, CPIAUCSL, UNRATENSA, GDPC1)} +# Monthly, with a "." in the middle: FRED's missing-value marker, not a null. +UNRATE_OBS = [ + ("2025-01-01", "4.0"), + ("2025-02-01", "4.1"), + ("2025-03-01", "."), + ("2025-04-01", "4.2"), + ("2025-05-01", "4.4"), + ("2025-06-01", "4.3"), +] + +# Quarterly, so it lines up with UNRATE on only two of its dates. This is the pair the +# alignment tests use: a union index with nulls where a series has no observation. +GDPC1_OBS = [ + ("2025-01-01", "23000.0"), + ("2025-04-01", "23150.5"), +] + +# A long daily series for the downsampling tests. The knots put the peak and the trough +# in the *interior*, away from the first and last points that downsampling always keeps. +# That is the whole point: a summary computed after thinning would report the extremes +# of the sample, and with extremes at the endpoints the test could not tell the +# difference. +_DAILY_KNOTS = [(0, 200.0), (80, 300.0), (200, 50.0), (365, 180.0)] + + +def _daily_series() -> list[tuple[str, str]]: + values: list[float] = [] + for (i0, v0), (i1, v1) in zip(_DAILY_KNOTS, _DAILY_KNOTS[1:]): + values.extend(v0 + (v1 - v0) * (i - i0) / (i1 - i0) for i in range(i0, i1)) + values.append(_DAILY_KNOTS[-1][1]) + + first = date(2020, 1, 1).toordinal() + return [(date.fromordinal(first + i).isoformat(), f"{v:.4f}") for i, v in enumerate(values)] + + +DAILY_OBS = _daily_series() +DAILY_MIN, DAILY_MAX = 50.0, 300.0 + +OBSERVATIONS = {"UNRATE": UNRATE_OBS, "GDPC1": GDPC1_OBS, "CPIAUCSL": UNRATE_OBS, "DGS10": DAILY_OBS} + class MockFred: """Routes FRED paths to captured payloads and records every request.""" @@ -97,6 +139,8 @@ def _handle(self, request: httpx.Request) -> httpx.Response: # "/series" and would otherwise be swallowed by this branch. if path.endswith("/fred/series"): return self._one_series(params.get("series_id", "")) + if path.endswith("/series/observations"): + return self._observations(params) if path.endswith("/series/search"): return self._series_list(params, [UNRATE, UNRATENSA, CPIAUCSL, GDPC1]) if path.endswith("/release/series") or path.endswith("/category/series"): @@ -141,6 +185,33 @@ def _handle(self, request: httpx.Request) -> httpx.Response: ) return _error(404, f"Not Found. No handler for {path}.") + def _observations(self, params: dict[str, str]) -> httpx.Response: + series_id = params.get("series_id", "") + if series_id not in OBSERVATIONS: + return _error(400, "Bad Request. The series does not exist.") + + rows = OBSERVATIONS[series_id] + start, end = params.get("observation_start"), params.get("observation_end") + if start: + rows = [r for r in rows if r[0] >= start] + if end: + rows = [r for r in rows if r[0] <= end] + + # Real-time fields carry the same value on every row, which is the redundancy + # the columnar shaping exists to remove; the fixture reproduces it faithfully. + return _ok( + { + "realtime_start": "2026-08-05", + "realtime_end": "2026-08-05", + "units": params.get("units", "lin"), + "count": len(rows), + "observations": [ + {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "date": d, "value": v} + for d, v in rows + ], + } + ) + def _one_series(self, series_id: str) -> httpx.Response: if series_id not in SERIES: return _error(400, "Bad Request. The series does not exist.") diff --git a/plugins/fred/tests/integration/test_observations.py b/plugins/fred/tests/integration/test_observations.py new file mode 100644 index 0000000..857dd98 --- /dev/null +++ b/plugins/fred/tests/integration/test_observations.py @@ -0,0 +1,147 @@ +"""get_observations, driven through the registered MCP server.""" + +import pytest + +from ..fixtures.fred_api import DAILY_MAX, DAILY_MIN, DAILY_OBS + +pytestmark = pytest.mark.integration + + +class TestSingleSeries: + async def test_columnar_output(self, call): + out = await call("get_observations", series_ids="UNRATE") + assert out["dates"][:2] == ["2025-01-01", "2025-02-01"] + assert out["values"]["UNRATE"][:2] == [4.0, 4.1] + + async def test_freds_missing_marker_becomes_null(self, call): + out = await call("get_observations", series_ids="UNRATE") + assert out["values"]["UNRATE"][2] is None # "." in the fixture + + async def test_the_summary_skips_the_gap(self, call): + summary = (await call("get_observations", series_ids="UNRATE"))["summary"]["UNRATE"] + assert summary["latest"] == 4.3 + assert summary["latest_date"] == "2025-06-01" + assert summary["count"] == 5 + assert summary["observations"] == 6 + assert (summary["min"], summary["max"]) == (4.0, 4.4) + + async def test_units_are_echoed_with_their_meaning(self, call): + out = await call("get_observations", series_ids="UNRATE", units="yoy") + assert out["units"] == "pc1" + assert out["units_meaning"] == "percent change from a year ago" + + +class TestParametersSent: + async def test_yoy_becomes_pc1_on_the_wire(self, call, fred): + await call("get_observations", series_ids="UNRATE", units="year over year") + assert fred.query("/series/observations")["units"] == "pc1" + + async def test_dates_are_normalized_before_the_request(self, call, fred): + await call("get_observations", series_ids="UNRATE", start="2025", end="2025-06") + params = fred.query("/series/observations") + assert params["observation_start"] == "2025-01-01" + assert params["observation_end"] == "2025-06-01" + + async def test_the_range_is_applied(self, call): + out = await call("get_observations", series_ids="UNRATE", start="2025-04-01") + assert out["dates"] == ["2025-04-01", "2025-05-01", "2025-06-01"] + + async def test_observations_come_back_oldest_first(self, call, fred): + await call("get_observations", series_ids="UNRATE") + assert fred.query("/series/observations")["sort_order"] == "asc" + + async def test_frequency_and_aggregation_are_echoed_only_when_set(self, call): + plain = await call("get_observations", series_ids="UNRATE") + assert "frequency" not in plain + + aggregated = await call( + "get_observations", series_ids="UNRATE", frequency="quarterly", aggregation_method="total" + ) + assert aggregated["frequency"] == "q" + assert aggregated["aggregation_method"] == "sum" + + +class TestComparison: + async def test_two_series_share_one_date_index(self, call): + # The claim the tool exists for: comparison is one call, already aligned. + out = await call("get_observations", series_ids="UNRATE,GDPC1") + assert out["dates"] == [ + "2025-01-01", + "2025-02-01", + "2025-03-01", + "2025-04-01", + "2025-05-01", + "2025-06-01", + ] + assert out["values"]["UNRATE"][0] == 4.0 + assert out["values"]["GDPC1"] == [23000.0, None, None, 23150.5, None, None] + + async def test_each_series_gets_its_own_summary(self, call): + out = await call("get_observations", series_ids=["UNRATE", "GDPC1"]) + assert set(out["summary"]) == {"UNRATE", "GDPC1"} + assert out["summary"]["GDPC1"]["latest"] == 23150.5 + + async def test_one_series_is_fetched_per_id(self, call, fred): + await call("get_observations", series_ids=["UNRATE", "GDPC1"]) + observation_calls = [p for p, _ in fred.requests if p.endswith("/series/observations")] + assert len(observation_calls) == 2 + + async def test_a_bad_id_does_not_take_down_the_good_one(self, call): + out = await call("get_observations", series_ids=["UNRATE", "NOPE"]) + assert out["values"]["UNRATE"][0] == 4.0 + assert "does not exist" in out["errors"]["NOPE"] + assert "NOPE" not in out["values"] + + +class TestDownsampling: + async def test_a_long_series_is_thinned(self, call): + out = await call("get_observations", series_ids="DGS10") + assert out["points"]["total"] == len(DAILY_OBS) + assert out["points"]["returned"] == 120 + assert out["points"]["dropped"] == len(DAILY_OBS) - 120 + assert len(out["dates"]) == 120 + + async def test_the_summary_still_reports_the_true_extremes(self, call): + # THE test for this tool. Both extremes sit in the interior of the fixture, so + # a summary computed after thinning would miss them and quietly report the + # sample's range as the series' range. + summary = (await call("get_observations", series_ids="DGS10"))["summary"]["DGS10"] + assert summary["min"] == DAILY_MIN + assert summary["max"] == DAILY_MAX + assert summary["count"] == len(DAILY_OBS) + assert summary["observations"] == len(DAILY_OBS) + + async def test_the_first_and_last_dates_survive(self, call): + out = await call("get_observations", series_ids="DGS10") + assert out["dates"][0] == DAILY_OBS[0][0] + assert out["dates"][-1] == DAILY_OBS[-1][0] + + async def test_thinning_is_stated_not_silent(self, call): + out = await call("get_observations", series_ids="DGS10") + assert "note" in out["points"] + + async def test_a_short_series_is_not_marked_as_thinned(self, call): + out = await call("get_observations", series_ids="UNRATE") + assert out["points"]["dropped"] == 0 + assert "note" not in out["points"] + + async def test_max_points_is_respected(self, call): + out = await call("get_observations", series_ids="DGS10", max_points=10) + assert out["points"]["returned"] == 10 + + +class TestGuidedFailures: + async def test_an_invented_unit_never_reaches_the_api(self, call, fred): + out = await call("get_observations", series_ids="UNRATE", units="cagr") + assert any("yoy" in s for s in out["suggestions"]) + assert not fred.requests + + async def test_an_unreadable_date_never_reaches_the_api(self, call, fred): + out = await call("get_observations", series_ids="UNRATE", start="whenever") + assert any("YYYY-MM-DD" in s for s in out["suggestions"]) + assert not fred.requests + + async def test_a_backwards_range_is_caught_locally(self, call, fred): + out = await call("get_observations", series_ids="UNRATE", start="2025-01-01", end="2020-01-01") + assert any("swap them" in s for s in out["suggestions"]) + assert not fred.requests diff --git a/plugins/fred/tests/unit/test_dates.py b/plugins/fred/tests/unit/test_dates.py new file mode 100644 index 0000000..7141d76 --- /dev/null +++ b/plugins/fred/tests/unit/test_dates.py @@ -0,0 +1,90 @@ +"""Relative and partial dates become the YYYY-MM-DD that FRED will accept.""" + +from datetime import date + +import pytest + +from src import dates + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def _pinned_clock(monkeypatch): + # Mid-month and mid-year, so the month and year arithmetic is actually exercised + # rather than passing by luck on the 1st of January. + monkeypatch.setattr(dates, "today", lambda: date(2026, 8, 5)) + + +class TestExplicitDates: + @pytest.mark.parametrize( + "written,expected", + [ + ("2020-01-15", "2020-01-15"), + ("2020", "2020-01-01"), + ("2020-03", "2020-03-01"), + (" 2020-01-15 ", "2020-01-15"), + ("", ""), + (" ", ""), + ], + ) + def test_forms(self, written, expected): + assert dates.parse(written, field="start") == expected + + def test_an_impossible_date_is_caught_here_not_by_fred(self): + with pytest.raises(ValueError, match="not a real date"): + dates.parse("2020-13-01", field="start") + + def test_february_30_is_rejected(self): + with pytest.raises(ValueError, match="not a real date"): + dates.parse("2021-02-30", field="start") + + +class TestRelativeDates: + @pytest.mark.parametrize( + "written,expected", + [ + ("today", "2026-08-05"), + ("now", "2026-08-05"), + ("ytd", "2026-01-01"), + ("year to date", "2026-01-01"), + ("5y", "2021-08-05"), + ("5 years", "2021-08-05"), + ("last 5 years", "2021-08-05"), + ("past 5 years", "2021-08-05"), + ("10yr", "2016-08-05"), + ("6m", "2026-02-05"), + ("18 months", "2025-02-05"), + ("2 quarters", "2026-02-05"), + ("30d", "2026-07-06"), + ("2 weeks", "2026-07-22"), + ], + ) + def test_spans_back_from_today(self, written, expected): + assert dates.parse(written, field="start") == expected + + def test_case_and_spacing_are_forgiven(self): + assert dates.parse("Last 5 Years", field="start") == "2021-08-05" + + def test_a_month_shift_onto_a_shorter_month_clamps(self, monkeypatch): + # 3 months before the 31st of May is the 28th of February, not the 31st. + monkeypatch.setattr(dates, "today", lambda: date(2026, 5, 31)) + assert dates.parse("3m", field="start") == "2026-02-28" + + def test_a_year_shift_across_a_leap_day_clamps(self, monkeypatch): + monkeypatch.setattr(dates, "today", lambda: date(2024, 2, 29)) + assert dates.parse("1y", field="start") == "2023-02-28" + + def test_a_span_crossing_the_year_boundary(self, monkeypatch): + monkeypatch.setattr(dates, "today", lambda: date(2026, 2, 10)) + assert dates.parse("6m", field="start") == "2025-08-10" + + +class TestFailure: + def test_nonsense_names_the_field_and_lists_the_forms(self): + with pytest.raises(ValueError) as exc: + dates.parse("whenever", field="end") + message = str(exc.value) + assert "end='whenever'" in message + assert "YYYY-MM-DD" in message + assert "ytd" in message diff --git a/plugins/fred/tests/unit/test_observations_shaping.py b/plugins/fred/tests/unit/test_observations_shaping.py new file mode 100644 index 0000000..31cc0bc --- /dev/null +++ b/plugins/fred/tests/unit/test_observations_shaping.py @@ -0,0 +1,171 @@ +"""Columnar alignment, summaries, and downsampling. + +The assertion this file exists for: the summary is computed over every observation and +the point list is thinned afterwards, so a long series still reports its true extremes. +""" + +import pytest + +from src import shaping + +pytestmark = pytest.mark.unit + + +class TestParseValue: + @pytest.mark.parametrize("raw,expected", [("4.2", 4.2), ("0", 0.0), ("-1.5", -1.5), (" 4.2 ", 4.2)]) + def test_strings_become_floats(self, raw, expected): + assert shaping.parse_value(raw) == expected + + @pytest.mark.parametrize("raw", [".", "", None, "n/a"]) + def test_freds_missing_marker_becomes_none(self, raw): + # FRED writes a missing observation as "." and never as null. + assert shaping.parse_value(raw) is None + + def test_zero_is_not_mistaken_for_missing(self): + assert shaping.parse_value("0.0") == 0.0 + + +class TestObservationPairs: + def test_drops_the_realtime_padding(self): + payload = { + "observations": [ + {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "date": "2025-01-01", "value": "4.0"}, + {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "date": "2025-02-01", "value": "."}, + ] + } + assert shaping.observation_pairs(payload) == [("2025-01-01", 4.0), ("2025-02-01", None)] + + def test_an_empty_payload(self): + assert shaping.observation_pairs({}) == [] + + +class TestAlign: + def test_one_series_is_its_own_index(self): + dates, columns = shaping.align({"A": [("2025-01-01", 1.0), ("2025-02-01", 2.0)]}) + assert dates == ["2025-01-01", "2025-02-01"] + assert columns == {"A": [1.0, 2.0]} + + def test_two_frequencies_union_with_nulls_in_the_gaps(self): + # Monthly against quarterly. The gaps are null, not forward-filled: filling + # them would invent observations that FRED never published. + monthly = [("2025-01-01", 4.0), ("2025-02-01", 4.1), ("2025-03-01", 4.2)] + quarterly = [("2025-01-01", 23000.0)] + dates, columns = shaping.align({"M": monthly, "Q": quarterly}) + assert dates == ["2025-01-01", "2025-02-01", "2025-03-01"] + assert columns["M"] == [4.0, 4.1, 4.2] + assert columns["Q"] == [23000.0, None, None] + + def test_dates_present_only_in_the_second_series_still_appear(self): + dates, columns = shaping.align({"A": [("2025-02-01", 1.0)], "B": [("2025-01-01", 2.0)]}) + assert dates == ["2025-01-01", "2025-02-01"] + assert columns["A"] == [None, 1.0] + assert columns["B"] == [2.0, None] + + def test_no_series(self): + assert shaping.align({}) == ([], {}) + + def test_every_column_is_the_length_of_the_index(self): + dates, columns = shaping.align( + {"A": [("2025-01-01", 1.0)], "B": [("2025-02-01", 2.0)], "C": [("2025-03-01", 3.0)]} + ) + assert all(len(column) == len(dates) for column in columns.values()) + + +class TestSummarize: + def test_the_figures_a_question_actually_asks_for(self): + pairs = [("2025-01-01", 4.0), ("2025-02-01", 4.1), ("2025-03-01", 4.4)] + s = shaping.summarize(pairs) + assert s["latest"] == 4.4 + assert s["latest_date"] == "2025-03-01" + assert s["prior"] == 4.1 + assert s["change"] == pytest.approx(0.3) + assert s["pct_change"] == pytest.approx(7.3171, abs=1e-3) + assert (s["min"], s["max"]) == (4.0, 4.4) + assert s["count"] == 3 + + def test_missing_values_are_skipped_but_still_counted(self): + pairs = [("2025-01-01", 4.0), ("2025-02-01", None), ("2025-03-01", 4.4)] + s = shaping.summarize(pairs) + assert s["count"] == 2 # observations with a value + assert s["observations"] == 3 # rows FRED returned + assert s["latest"] == 4.4 + assert s["prior"] == 4.0 # the null is skipped, not treated as the prior + + def test_latest_is_the_last_real_value_not_a_trailing_null(self): + pairs = [("2025-01-01", 4.0), ("2025-02-01", 4.4), ("2025-03-01", None)] + s = shaping.summarize(pairs) + assert s["latest"] == 4.4 + assert s["latest_date"] == "2025-02-01" + + def test_a_single_observation_has_no_change(self): + s = shaping.summarize([("2025-01-01", 4.0)]) + assert s["latest"] == 4.0 + assert "change" not in s and "prior" not in s + + def test_all_missing(self): + assert shaping.summarize([("2025-01-01", None)]) == {"observations": 1, "count": 0} + + def test_empty(self): + assert shaping.summarize([]) == {"observations": 0, "count": 0} + + def test_a_zero_prior_does_not_divide_by_zero(self): + s = shaping.summarize([("2025-01-01", 0.0), ("2025-02-01", 2.0)]) + assert s["change"] == 2.0 + assert "pct_change" not in s + + def test_pct_change_is_signed_correctly_from_a_negative_prior(self): + # Falling further below zero is a decrease, and dividing by a raw negative + # would report it as an increase. + s = shaping.summarize([("2025-01-01", -2.0), ("2025-02-01", -3.0)]) + assert s["change"] == -1.0 + assert s["pct_change"] == pytest.approx(-50.0) + + +class TestDownsample: + def _series(self, n): + dates = [f"2020-{i:04d}" for i in range(n)] + return dates, {"A": [float(i) for i in range(n)]} + + def test_a_short_series_is_untouched(self): + dates, columns = self._series(50) + out_dates, out_columns, dropped = shaping.downsample(dates, columns, 120) + assert (out_dates, out_columns, dropped) == (dates, columns, 0) + + def test_exactly_at_the_cap_is_untouched(self): + dates, columns = self._series(120) + assert shaping.downsample(dates, columns, 120)[2] == 0 + + def test_thins_to_the_cap(self): + dates, columns = self._series(5000) + out_dates, out_columns, dropped = shaping.downsample(dates, columns, 120) + assert len(out_dates) == 120 + assert len(out_columns["A"]) == 120 + assert dropped == 4880 + + def test_the_first_and_last_points_always_survive(self): + # Otherwise the series appears to start and end somewhere it does not. + dates, columns = self._series(5000) + out_dates, out_columns, _ = shaping.downsample(dates, columns, 120) + assert out_dates[0] == dates[0] + assert out_dates[-1] == dates[-1] + assert out_columns["A"][0] == 0.0 + assert out_columns["A"][-1] == 4999.0 + + def test_the_sample_is_evenly_spaced(self): + dates, columns = self._series(1000) + _, out_columns, _ = shaping.downsample(dates, columns, 100) + gaps = {b - a for a, b in zip(out_columns["A"], out_columns["A"][1:])} + assert max(gaps) - min(gaps) <= 1 # even to within rounding + + def test_every_column_stays_aligned_with_the_dates(self): + dates, columns = self._series(1000) + columns["B"] = [float(i) * 2 for i in range(1000)] + out_dates, out_columns, _ = shaping.downsample(dates, columns, 50) + assert all(len(column) == len(out_dates) for column in out_columns.values()) + assert all(b == a * 2 for a, b in zip(out_columns["A"], out_columns["B"])) + + def test_the_minimum_cap_still_produces_the_endpoints(self): + dates, columns = self._series(1000) + out_dates, _, dropped = shaping.downsample(dates, columns, 2) + assert out_dates == [dates[0], dates[-1]] + assert dropped == 998 diff --git a/plugins/fred/tests/unit/test_schemas.py b/plugins/fred/tests/unit/test_schemas.py index 476d4a7..9c60995 100644 --- a/plugins/fred/tests/unit/test_schemas.py +++ b/plugins/fred/tests/unit/test_schemas.py @@ -8,7 +8,7 @@ import pytest from pydantic import ValidationError -from src.schemas import GetSeriesArgs, SearchArgs, normalize_series_ids +from src.schemas import GetSeriesArgs, ObservationArgs, SearchArgs, normalize_series_ids pytestmark = pytest.mark.unit @@ -172,3 +172,94 @@ def test_order_by_is_corrected_and_checked(self): def test_limit_is_bounded(self, limit): with pytest.raises(ValidationError): SearchArgs(query="x", limit=limit) + + +class TestUnitsCorrection: + """The single most valuable table in the correction layer: nobody remembers pc1.""" + + @pytest.mark.parametrize( + "written,expected", + [ + ("yoy", "pc1"), + ("YoY", "pc1"), + ("year over year", "pc1"), + ("year-over-year", "pc1"), + ("percent change from a year ago", "pc1"), + ("inflation", "pc1"), + ("pc1", "pc1"), + ("percent change", "pch"), + ("pct_change", "pch"), + ("mom", "pch"), + ("month over month", "pch"), + ("change", "chg"), + ("diff", "chg"), + ("change from a year ago", "ch1"), + ("annualized", "pca"), + ("saar", "pca"), + ("level", "lin"), + ("levels", "lin"), + ("raw", "lin"), + ("none", "lin"), + ("", "lin"), + ("natural log", "log"), + ("ln", "log"), + ], + ) + def test_phrases_become_fred_codes(self, written, expected): + assert ObservationArgs(series_ids="UNRATE", units=written).units == expected + + def test_every_code_carries_its_meaning(self): + args = ObservationArgs(series_ids="UNRATE", units="yoy") + assert args.units_meaning == "percent change from a year ago" + + def test_an_invented_transformation_is_rejected_with_the_real_options(self): + with pytest.raises(ValidationError) as exc: + ObservationArgs(series_ids="UNRATE", units="cagr") + assert "yoy" in str(exc.value) + + +class TestAggregationCorrection: + @pytest.mark.parametrize( + "written,expected", + [("average", "avg"), ("mean", "avg"), ("total", "sum"), ("end of period", "eop"), ("last", "eop")], + ) + def test_words_become_codes(self, written, expected): + assert ObservationArgs(series_ids="X", aggregation_method=written).aggregation_method == expected + + def test_unknown_is_rejected(self): + with pytest.raises(ValidationError, match="unknown aggregation_method"): + ObservationArgs(series_ids="X", aggregation_method="median") + + +class TestObservationRange: + def test_relative_dates_are_resolved(self, monkeypatch): + from datetime import date + + from src import dates as dates_module + + monkeypatch.setattr(dates_module, "today", lambda: date(2026, 8, 5)) + args = ObservationArgs(series_ids="UNRATE", start="5y", end="today") + assert (args.start, args.end) == ("2021-08-05", "2026-08-05") + + def test_a_backwards_range_is_caught_before_the_api_sees_it(self): + with pytest.raises(ValidationError, match="swap them"): + ObservationArgs(series_ids="UNRATE", start="2025-01-01", end="2020-01-01") + + def test_an_equal_range_is_fine(self): + # One specific observation date is a legitimate request. + args = ObservationArgs(series_ids="UNRATE", start="2025-01-01", end="2025-01-01") + assert args.start == args.end + + def test_an_unparseable_date_is_guidance_not_a_400(self): + with pytest.raises(ValidationError, match="could not be read as a date"): + ObservationArgs(series_ids="UNRATE", start="whenever") + + @pytest.mark.parametrize("cap", [1, 2001]) + def test_max_points_is_bounded(self, cap): + with pytest.raises(ValidationError): + ObservationArgs(series_ids="UNRATE", max_points=cap) + + def test_a_finer_frequency_is_still_expressible_here(self): + # Only FRED knows a series' native frequency, so "finer than the series" is + # its call; this layer only rejects frequencies FRED has no code for. + assert ObservationArgs(series_ids="UNRATE", frequency="daily").frequency == "d" diff --git a/plugins/fred/tests/unit/test_server.py b/plugins/fred/tests/unit/test_server.py index 5eb8a9b..97ee13b 100644 --- a/plugins/fred/tests/unit/test_server.py +++ b/plugins/fred/tests/unit/test_server.py @@ -20,7 +20,7 @@ async def test_server_builds_and_exposes_its_tools(): # Grows with the stack in #30. Asserting the exact set is what catches a tool that was # written but never registered, which is otherwise invisible until a user asks for it. -EXPECTED_TOOLS: set[str] = {"search_series", "get_series"} +EXPECTED_TOOLS: set[str] = {"search_series", "get_series", "get_observations"} def test_instructions_name_the_entry_point(): From 1030ca27e02214f8430b052beb1121356c30b18d Mon Sep 17 00:00:00 2001 From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:19:08 -0700 Subject: [PATCH 2/2] chore(fred): bump to 0.3.0 for get_observations (#33) --- plugins/fred/.claude-plugin/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/fred/.claude-plugin/plugin.json b/plugins/fred/.claude-plugin/plugin.json index 2496455..54d8847 100644 --- a/plugins/fred/.claude-plugin/plugin.json +++ b/plugins/fred/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fred", - "version": "0.2.0", + "version": "0.3.0", "description": "MCP server for the FRED API: economic time series from the St. Louis Fed, with search, aligned multi-series observations, revision history, and the release calendar.", "author": { "name": "Walker Hughes"