diff --git a/plugins/fred/.claude-plugin/plugin.json b/plugins/fred/.claude-plugin/plugin.json index 2496455..54d8847 100644 --- a/plugins/fred/.claude-plugin/plugin.json +++ b/plugins/fred/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fred", - "version": "0.2.0", + "version": "0.3.0", "description": "MCP server for the FRED API: economic time series from the St. Louis Fed, with search, aligned multi-series observations, revision history, and the release calendar.", "author": { "name": "Walker Hughes" diff --git a/plugins/fred/src/dates.py b/plugins/fred/src/dates.py new file mode 100644 index 0000000..dd2bec5 --- /dev/null +++ b/plugins/fred/src/dates.py @@ -0,0 +1,87 @@ +"""Date parsing for arguments FRED will only accept as YYYY-MM-DD. + +A model asked for "the last five years" writes ``start="5y"`` or ``start="2020"`` or +``start="last 5 years"``. FRED answers all three with HTTP 400 and a message about a +variable name. Since none of these are ambiguous, they are translated rather than +rejected. +""" + +import re +from datetime import date + +# "5y", "5 years", "last 5 years", "past 18 months", "10yr". +_RELATIVE = re.compile( + r"^(?:last|past)?\s*(\d+)\s*(d|w|m|q|y|yr|day|days|week|weeks|month|months|quarter|quarters|year|years)$" +) + +_UNIT_MONTHS = {"m": 1, "mo": 1, "month": 1, "months": 1, "q": 3, "quarter": 3, "quarters": 3} +_UNIT_YEARS = {"y": 1, "yr": 1, "year": 1, "years": 1} +_UNIT_DAYS = {"d": 1, "day": 1, "days": 1, "w": 7, "week": 7, "weeks": 7} + + +def today() -> date: + """Indirection so tests can pin the clock without patching the stdlib.""" + return date.today() + + +def parse(value: str, *, field: str) -> str: + """Return a YYYY-MM-DD string, or "" for an unset value. + + Accepted, beyond a full date: a bare year ("2020"), a year-month ("2020-01"), + "today"/"now", "ytd", and a relative span ("5y", "18 months", "last 10 years") + measured back from today. + """ + text = " ".join(str(value).strip().lower().split()) + if not text: + return "" + + if text in ("today", "now"): + return today().isoformat() + if text in ("ytd", "year to date", "this year"): + return today().replace(month=1, day=1).isoformat() + + if re.fullmatch(r"\d{4}", text): + return f"{text}-01-01" + if re.fullmatch(r"\d{4}-\d{2}", text): + return f"{text}-01" + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", text): + # Round-tripped through date() so 2020-13-01 is caught here rather than by FRED. + try: + return date.fromisoformat(text).isoformat() + except ValueError as exc: + raise ValueError(f"{field}={value!r} is not a real date: {exc}") from exc + + match = _RELATIVE.match(text) + if match: + amount, unit = int(match.group(1)), match.group(2) + return _ago(amount, unit).isoformat() + + raise ValueError( + f"{field}={value!r} could not be read as a date. Use YYYY-MM-DD, a year " + '("2020"), a year-month ("2020-01"), a span back from today ("5y", ' + '"18 months"), "ytd", or "today".' + ) + + +def _ago(amount: int, unit: str) -> date: + """Today, minus ``amount`` of ``unit``. Calendar arithmetic, no dependency.""" + now = today() + if unit in _UNIT_DAYS: + return date.fromordinal(max(1, now.toordinal() - amount * _UNIT_DAYS[unit])) + if unit in _UNIT_YEARS: + months = amount * 12 + else: + months = amount * _UNIT_MONTHS[unit] + + total = now.year * 12 + (now.month - 1) - months + year, month = divmod(total, 12) + month += 1 + # Clamp the day: three months before the 31st is not the 31st of a 30-day month. + day = min(now.day, _days_in_month(year, month)) + return date(year, month, day) + + +def _days_in_month(year: int, month: int) -> int: + if month == 12: + return 31 + return (date(year + (month == 12), month % 12 + 1, 1).toordinal()) - date(year, month, 1).toordinal() diff --git a/plugins/fred/src/schemas.py b/plugins/fred/src/schemas.py index 356cdcd..e65062b 100644 --- a/plugins/fred/src/schemas.py +++ b/plugins/fred/src/schemas.py @@ -14,6 +14,8 @@ from pydantic import BaseModel, Field, field_validator, model_validator +from . import dates + MAX_SERIES_PER_CALL = 20 # FRED's frequency codes, and the words a model writes instead. @@ -75,6 +77,45 @@ SERIES_INCLUDES = ("metadata", "notes", "release", "categories", "tags") +# What each FRED units code actually does, echoed in the response so the numbers are +# not left to be guessed at. +UNITS_MEANING: dict[str, str] = { + "lin": "levels, as published", + "chg": "change from the previous period", + "ch1": "change from a year ago", + "pch": "percent change from the previous period", + "pc1": "percent change from a year ago", + "pca": "compounded annual rate of change", + "cch": "continuously compounded rate of change", + "cca": "continuously compounded annual rate of change", + "log": "natural log", +} + +# The single highest-value table in this file. "Year over year percent change" is the +# most-asked transformation in all of economics and FRED spells it "pc1". +UNITS_ALIASES: dict[str, str] = { + "": "lin", "lin": "lin", "level": "lin", "levels": "lin", "none": "lin", + "raw": "lin", "as published": "lin", "nominal": "lin", + "pc1": "pc1", "yoy": "pc1", "y/y": "pc1", "yoy%": "pc1", "year over year": "pc1", + "year-over-year": "pc1", "annual percent change": "pc1", "percent change from year ago": "pc1", + "percent change from a year ago": "pc1", "inflation": "pc1", + "pch": "pch", "mom": "pch", "m/m": "pch", "month over month": "pch", + "percent change": "pch", "pct change": "pch", "pct_change": "pch", + "percent change from previous period": "pch", + "chg": "chg", "change": "chg", "diff": "chg", "difference": "chg", + "change from previous period": "chg", + "ch1": "ch1", "change from year ago": "ch1", "change from a year ago": "ch1", + "pca": "pca", "annualized": "pca", "saar": "pca", "annual rate": "pca", + "compounded annual rate of change": "pca", + "cch": "cch", "cca": "cca", "log": "log", "natural log": "log", "ln": "log", +} # fmt: skip + +AGGREGATION_ALIASES: dict[str, str] = { + "avg": "avg", "average": "avg", "mean": "avg", + "sum": "sum", "total": "sum", + "eop": "eop", "end": "eop", "end of period": "eop", "last": "eop", "close": "eop", +} # fmt: skip + def normalize_series_ids(value: Any) -> Any: """Accept the several ways a model writes a list of series IDs. @@ -231,3 +272,72 @@ def _known_includes(cls, value: list[str]) -> list[str]: def wants(self, part: str) -> bool: return part in self.include + + +class ObservationArgs(BaseModel): + """Arguments for get_observations.""" + + series_ids: list[str] = Field(min_length=1, max_length=MAX_SERIES_PER_CALL) + start: str = "" + end: str = "" + units: str = "lin" + frequency: str = "" + aggregation_method: str = "avg" + # Floor of 2 because downsampling always keeps the first and last point; a cap of + # one cannot express a series. + max_points: int = Field(default=120, ge=2, le=2000) + + @model_validator(mode="before") + @classmethod + def _correct(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + data = dict(data) + if "series_ids" in data: + data["series_ids"] = normalize_series_ids(data["series_ids"]) + for field in ("start", "end"): + if data.get(field) is not None: + data[field] = dates.parse(data[field], field=field) + if data.get("units") is not None: + raw = str(data["units"]).strip().lower() + data["units"] = UNITS_ALIASES.get(raw, raw) + if "frequency" in data: + data["frequency"] = normalize_frequency(data["frequency"]) + if data.get("aggregation_method") is not None: + raw = str(data["aggregation_method"]).strip().lower() + data["aggregation_method"] = AGGREGATION_ALIASES.get(raw, raw) + return data + + @field_validator("units") + @classmethod + def _known_units(cls, value: str) -> str: + if value not in UNITS_MEANING: + raise ValueError( + f"unknown units {value!r}; use a FRED code ({', '.join(UNITS_MEANING)}) " + "or a phrase such as 'yoy', 'percent change', 'level'" + ) + return value + + @field_validator("frequency") + @classmethod + def _known_frequency(cls, value: str) -> str: + if value and value not in FREQUENCY_TAGS: + raise ValueError(f"unknown frequency {value!r}; use one of {', '.join(FREQUENCY_TAGS)}") + return value + + @field_validator("aggregation_method") + @classmethod + def _known_aggregation(cls, value: str) -> str: + if value not in ("avg", "sum", "eop"): + raise ValueError(f"unknown aggregation_method {value!r}; use avg, sum, or eop") + return value + + @model_validator(mode="after") + def _range_is_the_right_way_round(self) -> "ObservationArgs": + if self.start and self.end and self.start > self.end: + raise ValueError(f"start ({self.start}) is after end ({self.end}); swap them") + return self + + @property + def units_meaning(self) -> str: + return UNITS_MEANING[self.units] diff --git a/plugins/fred/src/shaping.py b/plugins/fred/src/shaping.py index 257a84b..e525db3 100644 --- a/plugins/fred/src/shaping.py +++ b/plugins/fred/src/shaping.py @@ -66,3 +66,106 @@ def series_list(payload: dict, *, notes: bool = False) -> list[dict]: def first_or_empty(payload: dict, key: str) -> dict[str, Any]: items = payload.get(key) or [] return items[0] if items else {} + + +# --- observations ------------------------------------------------------------------ +# +# FRED returns one object per observation: +# +# {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", +# "date": "2025-01-01", "value": "2.99098"} +# +# On a normal (non-vintage) request the two realtime fields hold the same value on +# every single row, and the key names repeat once per observation. Twenty years of a +# daily series is around 5,000 of those. Columnar output drops all of it. + +Number = float | None + + +def parse_value(raw: object) -> Number: + """FRED writes values as strings and missing ones as ".", not null.""" + if raw is None: + return None + text = str(raw).strip() + if text in ("", "."): + return None + try: + return float(text) + except ValueError: + return None + + +def observation_pairs(payload: dict) -> list[tuple[str, Number]]: + """(date, value) for each observation, values parsed.""" + return [(obs["date"], parse_value(obs.get("value"))) for obs in payload.get("observations", []) if obs.get("date")] + + +def align(per_series: dict[str, list[tuple[str, Number]]]) -> tuple[list[str], dict[str, list[Number]]]: + """Put several series on one shared, sorted date index. + + This is what makes comparison a single tool call. A monthly and a quarterly series + will not line up, so the index is the union of their dates and the gaps are null + rather than silently dropped or forward-filled, which would invent data. + """ + dates = sorted({d for pairs in per_series.values() for d, _ in pairs}) + index = {date: position for position, date in enumerate(dates)} + + columns: dict[str, list[Number]] = {} + for series_id, pairs in per_series.items(): + column: list[Number] = [None] * len(dates) + for date, value in pairs: + column[index[date]] = value + columns[series_id] = column + return dates, columns + + +def summarize(pairs: list[tuple[str, Number]]) -> dict: + """The figures worth having, computed over every observation. + + Deliberately computed *before* downsampling. A 5,000-point daily series thinned to + 120 still reports its true min, max, and latest value; a summary computed after + would quietly report the extremes of the sample instead of the series. + """ + observed = [(date, value) for date, value in pairs if value is not None] + if not observed: + return {"observations": len(pairs), "count": 0} + + values = [value for _, value in observed] + latest_date, latest = observed[-1] + summary: dict[str, Any] = { + "latest": latest, + "latest_date": latest_date, + "min": min(values), + "max": max(values), + "mean": round(sum(values) / len(values), 6), + "start_date": observed[0][0], + "end_date": latest_date, + "count": len(observed), + "observations": len(pairs), + } + if len(observed) > 1: + prior = observed[-2][1] + summary["prior"] = prior + summary["change"] = round(latest - prior, 6) + if prior: + summary["pct_change"] = round((latest - prior) / abs(prior) * 100, 4) + return summary + + +def downsample(dates: list[str], columns: dict[str, list[Number]], max_points: int) -> tuple[list, dict, int]: + """Thin to evenly spaced points, always keeping the first and the last. + + Returns the kept dates, the kept columns, and how many points were dropped, so a + truncated series is never mistaken for a complete one. + """ + total = len(dates) + if total <= max_points: + return dates, columns, 0 + + step = (total - 1) / (max_points - 1) + keep = sorted({round(i * step) for i in range(max_points)} | {0, total - 1}) + return ( + [dates[i] for i in keep], + {series_id: [column[i] for i in keep] for series_id, column in columns.items()}, + total - len(keep), + ) diff --git a/plugins/fred/src/tools.py b/plugins/fred/src/tools.py index 8338026..6d7ded7 100644 --- a/plugins/fred/src/tools.py +++ b/plugins/fred/src/tools.py @@ -14,7 +14,7 @@ from . import shaping from .client import FredClient from .errors import guarded_tool -from .schemas import GetSeriesArgs, SearchArgs +from .schemas import GetSeriesArgs, ObservationArgs, SearchArgs _client: FredClient | None = None @@ -42,6 +42,15 @@ def fmt(data: object) -> str: return json.dumps(data, indent=2, default=str) +def _http_detail(exc: httpx.HTTPStatusError) -> str: + """FRED's own explanation, which the status code alone does not carry.""" + try: + detail = str(exc.response.json().get("error_message", "")).strip() + except ValueError: + detail = "" + return " ".join(detail.split()) or f"HTTP {exc.response.status_code}" + + async def _search_series(args: SearchArgs) -> dict: """Run one of the three discovery paths and return the shared output shape.""" client = get_client() @@ -109,14 +118,9 @@ async def optional(part: str, path: str) -> dict | None: optional("tags", "/series/tags"), ) except httpx.HTTPStatusError as exc: - detail = "" - try: - detail = str(exc.response.json().get("error_message", "")).strip() - except ValueError: - pass return { "id": series_id, - "error": " ".join(detail.split()) or f"HTTP {exc.response.status_code}", + "error": _http_detail(exc), "suggestion": "Check this ID with search_series; the others in this call were returned.", } @@ -131,6 +135,67 @@ async def optional(part: str, path: str) -> dict | None: return out +Pairs = list[tuple[str, shaping.Number]] + + +async def _observations_for(args: ObservationArgs, series_id: str) -> tuple[str, Pairs | str]: + """One series' observations, or its own error message. A bad ID among good ones + should cost only its own column, not the whole comparison.""" + try: + payload = await get_client().get( + "/series/observations", + series_id=series_id, + observation_start=args.start, + observation_end=args.end, + units=args.units, + frequency=args.frequency, + # FRED ignores this unless frequency is set, so sending it always is safe. + aggregation_method=args.aggregation_method, + sort_order="asc", + ) + except httpx.HTTPStatusError as exc: + return series_id, _http_detail(exc) + return series_id, shaping.observation_pairs(payload) + + +async def _get_observations(args: ObservationArgs) -> dict: + fetched = await asyncio.gather(*(_observations_for(args, sid) for sid in args.series_ids)) + + per_series: dict[str, Pairs] = {} + errors: dict[str, str] = {} + for series_id, outcome in fetched: + if isinstance(outcome, str): + errors[series_id] = outcome + else: + per_series[series_id] = outcome + + # Summaries first, over every observation. Downsampling after, so a thinned series + # still reports its true latest value and true extremes. + summary = {series_id: shaping.summarize(pairs) for series_id, pairs in per_series.items()} + dates, columns = shaping.align(per_series) + total = len(dates) + dates, columns, dropped = shaping.downsample(dates, columns, args.max_points) + + points: dict[str, object] = {"returned": len(dates), "total": total, "dropped": dropped} + if dropped: + points["note"] = "evenly spaced sample; the summary covers every observation" + + result: dict[str, object] = { + "units": args.units, + "units_meaning": args.units_meaning, + "dates": dates, + "values": columns, + "summary": summary, + "points": points, + } + if args.frequency: + result["frequency"] = args.frequency + result["aggregation_method"] = args.aggregation_method + if errors: + result["errors"] = errors + return result + + def register_all(mcp: MCPServer) -> None: """Register every tool on the given MCP server.""" @@ -207,3 +272,51 @@ async def get_series(series_ids: list[str] | str, include: list[str] | str | Non args = GetSeriesArgs.model_validate({"series_ids": series_ids, "include": include or ["metadata"]}) results = await asyncio.gather(*(_fetch_one_series(args, sid) for sid in args.series_ids)) return fmt({"series": list(results)}) + + @mcp.tool() + @guarded_tool + async def get_observations( + series_ids: list[str] | str, + start: str = "", + end: str = "", + units: str = "lin", + frequency: str = "", + aggregation_method: str = "avg", + max_points: int = 120, + ) -> str: + """Get the actual numbers. Pass several series at once to compare them. + + Series come back on one shared date index, so a comparison is a single call: + {"dates": [...], "values": {"UNRATE": [...], "CPIAUCSL": [...]}}. Where a + series has no observation for a date (a quarterly series against a monthly + one) the value is null rather than filled in. + + start and end accept YYYY-MM-DD, a bare year ("2020"), a year-month + ("2020-01"), a span back from today ("5y", "18 months", "last 10 years"), + "ytd", or "today". Omit both for the full history. + + units transforms the series server-side, so do not do this arithmetic + yourself. Say "yoy" for year-over-year percent change (FRED calls it pc1), + "percent change" for period-over-period, "change" for a difference, + "annualized", or "level" for the published numbers. + + frequency aggregates to a coarser interval only ("monthly", "quarterly", + "annual"), with aggregation_method of avg, sum, or eop. + + Every series gets a summary (latest, prior, change, min, max, mean, count) + computed over ALL its observations. Only the returned point list is thinned + to max_points, so a 20-year daily series still reports its true extremes + without spending 5,000 points to do it. + """ + args = ObservationArgs.model_validate( + { + "series_ids": series_ids, + "start": start, + "end": end, + "units": units, + "frequency": frequency, + "aggregation_method": aggregation_method, + "max_points": max_points, + } + ) + return fmt(await _get_observations(args)) diff --git a/plugins/fred/tests/fixtures/fred_api.py b/plugins/fred/tests/fixtures/fred_api.py index 6898aa3..59fd21f 100644 --- a/plugins/fred/tests/fixtures/fred_api.py +++ b/plugins/fred/tests/fixtures/fred_api.py @@ -10,6 +10,8 @@ the filter_variable pairing, the popularity ordering). """ +from datetime import date + import httpx UNRATE = { @@ -71,6 +73,46 @@ SERIES = {s["id"]: s for s in (UNRATE, CPIAUCSL, UNRATENSA, GDPC1)} +# Monthly, with a "." in the middle: FRED's missing-value marker, not a null. +UNRATE_OBS = [ + ("2025-01-01", "4.0"), + ("2025-02-01", "4.1"), + ("2025-03-01", "."), + ("2025-04-01", "4.2"), + ("2025-05-01", "4.4"), + ("2025-06-01", "4.3"), +] + +# Quarterly, so it lines up with UNRATE on only two of its dates. This is the pair the +# alignment tests use: a union index with nulls where a series has no observation. +GDPC1_OBS = [ + ("2025-01-01", "23000.0"), + ("2025-04-01", "23150.5"), +] + +# A long daily series for the downsampling tests. The knots put the peak and the trough +# in the *interior*, away from the first and last points that downsampling always keeps. +# That is the whole point: a summary computed after thinning would report the extremes +# of the sample, and with extremes at the endpoints the test could not tell the +# difference. +_DAILY_KNOTS = [(0, 200.0), (80, 300.0), (200, 50.0), (365, 180.0)] + + +def _daily_series() -> list[tuple[str, str]]: + values: list[float] = [] + for (i0, v0), (i1, v1) in zip(_DAILY_KNOTS, _DAILY_KNOTS[1:]): + values.extend(v0 + (v1 - v0) * (i - i0) / (i1 - i0) for i in range(i0, i1)) + values.append(_DAILY_KNOTS[-1][1]) + + first = date(2020, 1, 1).toordinal() + return [(date.fromordinal(first + i).isoformat(), f"{v:.4f}") for i, v in enumerate(values)] + + +DAILY_OBS = _daily_series() +DAILY_MIN, DAILY_MAX = 50.0, 300.0 + +OBSERVATIONS = {"UNRATE": UNRATE_OBS, "GDPC1": GDPC1_OBS, "CPIAUCSL": UNRATE_OBS, "DGS10": DAILY_OBS} + class MockFred: """Routes FRED paths to captured payloads and records every request.""" @@ -97,6 +139,8 @@ def _handle(self, request: httpx.Request) -> httpx.Response: # "/series" and would otherwise be swallowed by this branch. if path.endswith("/fred/series"): return self._one_series(params.get("series_id", "")) + if path.endswith("/series/observations"): + return self._observations(params) if path.endswith("/series/search"): return self._series_list(params, [UNRATE, UNRATENSA, CPIAUCSL, GDPC1]) if path.endswith("/release/series") or path.endswith("/category/series"): @@ -141,6 +185,33 @@ def _handle(self, request: httpx.Request) -> httpx.Response: ) return _error(404, f"Not Found. No handler for {path}.") + def _observations(self, params: dict[str, str]) -> httpx.Response: + series_id = params.get("series_id", "") + if series_id not in OBSERVATIONS: + return _error(400, "Bad Request. The series does not exist.") + + rows = OBSERVATIONS[series_id] + start, end = params.get("observation_start"), params.get("observation_end") + if start: + rows = [r for r in rows if r[0] >= start] + if end: + rows = [r for r in rows if r[0] <= end] + + # Real-time fields carry the same value on every row, which is the redundancy + # the columnar shaping exists to remove; the fixture reproduces it faithfully. + return _ok( + { + "realtime_start": "2026-08-05", + "realtime_end": "2026-08-05", + "units": params.get("units", "lin"), + "count": len(rows), + "observations": [ + {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "date": d, "value": v} + for d, v in rows + ], + } + ) + def _one_series(self, series_id: str) -> httpx.Response: if series_id not in SERIES: return _error(400, "Bad Request. The series does not exist.") diff --git a/plugins/fred/tests/integration/test_observations.py b/plugins/fred/tests/integration/test_observations.py new file mode 100644 index 0000000..857dd98 --- /dev/null +++ b/plugins/fred/tests/integration/test_observations.py @@ -0,0 +1,147 @@ +"""get_observations, driven through the registered MCP server.""" + +import pytest + +from ..fixtures.fred_api import DAILY_MAX, DAILY_MIN, DAILY_OBS + +pytestmark = pytest.mark.integration + + +class TestSingleSeries: + async def test_columnar_output(self, call): + out = await call("get_observations", series_ids="UNRATE") + assert out["dates"][:2] == ["2025-01-01", "2025-02-01"] + assert out["values"]["UNRATE"][:2] == [4.0, 4.1] + + async def test_freds_missing_marker_becomes_null(self, call): + out = await call("get_observations", series_ids="UNRATE") + assert out["values"]["UNRATE"][2] is None # "." in the fixture + + async def test_the_summary_skips_the_gap(self, call): + summary = (await call("get_observations", series_ids="UNRATE"))["summary"]["UNRATE"] + assert summary["latest"] == 4.3 + assert summary["latest_date"] == "2025-06-01" + assert summary["count"] == 5 + assert summary["observations"] == 6 + assert (summary["min"], summary["max"]) == (4.0, 4.4) + + async def test_units_are_echoed_with_their_meaning(self, call): + out = await call("get_observations", series_ids="UNRATE", units="yoy") + assert out["units"] == "pc1" + assert out["units_meaning"] == "percent change from a year ago" + + +class TestParametersSent: + async def test_yoy_becomes_pc1_on_the_wire(self, call, fred): + await call("get_observations", series_ids="UNRATE", units="year over year") + assert fred.query("/series/observations")["units"] == "pc1" + + async def test_dates_are_normalized_before_the_request(self, call, fred): + await call("get_observations", series_ids="UNRATE", start="2025", end="2025-06") + params = fred.query("/series/observations") + assert params["observation_start"] == "2025-01-01" + assert params["observation_end"] == "2025-06-01" + + async def test_the_range_is_applied(self, call): + out = await call("get_observations", series_ids="UNRATE", start="2025-04-01") + assert out["dates"] == ["2025-04-01", "2025-05-01", "2025-06-01"] + + async def test_observations_come_back_oldest_first(self, call, fred): + await call("get_observations", series_ids="UNRATE") + assert fred.query("/series/observations")["sort_order"] == "asc" + + async def test_frequency_and_aggregation_are_echoed_only_when_set(self, call): + plain = await call("get_observations", series_ids="UNRATE") + assert "frequency" not in plain + + aggregated = await call( + "get_observations", series_ids="UNRATE", frequency="quarterly", aggregation_method="total" + ) + assert aggregated["frequency"] == "q" + assert aggregated["aggregation_method"] == "sum" + + +class TestComparison: + async def test_two_series_share_one_date_index(self, call): + # The claim the tool exists for: comparison is one call, already aligned. + out = await call("get_observations", series_ids="UNRATE,GDPC1") + assert out["dates"] == [ + "2025-01-01", + "2025-02-01", + "2025-03-01", + "2025-04-01", + "2025-05-01", + "2025-06-01", + ] + assert out["values"]["UNRATE"][0] == 4.0 + assert out["values"]["GDPC1"] == [23000.0, None, None, 23150.5, None, None] + + async def test_each_series_gets_its_own_summary(self, call): + out = await call("get_observations", series_ids=["UNRATE", "GDPC1"]) + assert set(out["summary"]) == {"UNRATE", "GDPC1"} + assert out["summary"]["GDPC1"]["latest"] == 23150.5 + + async def test_one_series_is_fetched_per_id(self, call, fred): + await call("get_observations", series_ids=["UNRATE", "GDPC1"]) + observation_calls = [p for p, _ in fred.requests if p.endswith("/series/observations")] + assert len(observation_calls) == 2 + + async def test_a_bad_id_does_not_take_down_the_good_one(self, call): + out = await call("get_observations", series_ids=["UNRATE", "NOPE"]) + assert out["values"]["UNRATE"][0] == 4.0 + assert "does not exist" in out["errors"]["NOPE"] + assert "NOPE" not in out["values"] + + +class TestDownsampling: + async def test_a_long_series_is_thinned(self, call): + out = await call("get_observations", series_ids="DGS10") + assert out["points"]["total"] == len(DAILY_OBS) + assert out["points"]["returned"] == 120 + assert out["points"]["dropped"] == len(DAILY_OBS) - 120 + assert len(out["dates"]) == 120 + + async def test_the_summary_still_reports_the_true_extremes(self, call): + # THE test for this tool. Both extremes sit in the interior of the fixture, so + # a summary computed after thinning would miss them and quietly report the + # sample's range as the series' range. + summary = (await call("get_observations", series_ids="DGS10"))["summary"]["DGS10"] + assert summary["min"] == DAILY_MIN + assert summary["max"] == DAILY_MAX + assert summary["count"] == len(DAILY_OBS) + assert summary["observations"] == len(DAILY_OBS) + + async def test_the_first_and_last_dates_survive(self, call): + out = await call("get_observations", series_ids="DGS10") + assert out["dates"][0] == DAILY_OBS[0][0] + assert out["dates"][-1] == DAILY_OBS[-1][0] + + async def test_thinning_is_stated_not_silent(self, call): + out = await call("get_observations", series_ids="DGS10") + assert "note" in out["points"] + + async def test_a_short_series_is_not_marked_as_thinned(self, call): + out = await call("get_observations", series_ids="UNRATE") + assert out["points"]["dropped"] == 0 + assert "note" not in out["points"] + + async def test_max_points_is_respected(self, call): + out = await call("get_observations", series_ids="DGS10", max_points=10) + assert out["points"]["returned"] == 10 + + +class TestGuidedFailures: + async def test_an_invented_unit_never_reaches_the_api(self, call, fred): + out = await call("get_observations", series_ids="UNRATE", units="cagr") + assert any("yoy" in s for s in out["suggestions"]) + assert not fred.requests + + async def test_an_unreadable_date_never_reaches_the_api(self, call, fred): + out = await call("get_observations", series_ids="UNRATE", start="whenever") + assert any("YYYY-MM-DD" in s for s in out["suggestions"]) + assert not fred.requests + + async def test_a_backwards_range_is_caught_locally(self, call, fred): + out = await call("get_observations", series_ids="UNRATE", start="2025-01-01", end="2020-01-01") + assert any("swap them" in s for s in out["suggestions"]) + assert not fred.requests diff --git a/plugins/fred/tests/unit/test_dates.py b/plugins/fred/tests/unit/test_dates.py new file mode 100644 index 0000000..7141d76 --- /dev/null +++ b/plugins/fred/tests/unit/test_dates.py @@ -0,0 +1,90 @@ +"""Relative and partial dates become the YYYY-MM-DD that FRED will accept.""" + +from datetime import date + +import pytest + +from src import dates + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def _pinned_clock(monkeypatch): + # Mid-month and mid-year, so the month and year arithmetic is actually exercised + # rather than passing by luck on the 1st of January. + monkeypatch.setattr(dates, "today", lambda: date(2026, 8, 5)) + + +class TestExplicitDates: + @pytest.mark.parametrize( + "written,expected", + [ + ("2020-01-15", "2020-01-15"), + ("2020", "2020-01-01"), + ("2020-03", "2020-03-01"), + (" 2020-01-15 ", "2020-01-15"), + ("", ""), + (" ", ""), + ], + ) + def test_forms(self, written, expected): + assert dates.parse(written, field="start") == expected + + def test_an_impossible_date_is_caught_here_not_by_fred(self): + with pytest.raises(ValueError, match="not a real date"): + dates.parse("2020-13-01", field="start") + + def test_february_30_is_rejected(self): + with pytest.raises(ValueError, match="not a real date"): + dates.parse("2021-02-30", field="start") + + +class TestRelativeDates: + @pytest.mark.parametrize( + "written,expected", + [ + ("today", "2026-08-05"), + ("now", "2026-08-05"), + ("ytd", "2026-01-01"), + ("year to date", "2026-01-01"), + ("5y", "2021-08-05"), + ("5 years", "2021-08-05"), + ("last 5 years", "2021-08-05"), + ("past 5 years", "2021-08-05"), + ("10yr", "2016-08-05"), + ("6m", "2026-02-05"), + ("18 months", "2025-02-05"), + ("2 quarters", "2026-02-05"), + ("30d", "2026-07-06"), + ("2 weeks", "2026-07-22"), + ], + ) + def test_spans_back_from_today(self, written, expected): + assert dates.parse(written, field="start") == expected + + def test_case_and_spacing_are_forgiven(self): + assert dates.parse("Last 5 Years", field="start") == "2021-08-05" + + def test_a_month_shift_onto_a_shorter_month_clamps(self, monkeypatch): + # 3 months before the 31st of May is the 28th of February, not the 31st. + monkeypatch.setattr(dates, "today", lambda: date(2026, 5, 31)) + assert dates.parse("3m", field="start") == "2026-02-28" + + def test_a_year_shift_across_a_leap_day_clamps(self, monkeypatch): + monkeypatch.setattr(dates, "today", lambda: date(2024, 2, 29)) + assert dates.parse("1y", field="start") == "2023-02-28" + + def test_a_span_crossing_the_year_boundary(self, monkeypatch): + monkeypatch.setattr(dates, "today", lambda: date(2026, 2, 10)) + assert dates.parse("6m", field="start") == "2025-08-10" + + +class TestFailure: + def test_nonsense_names_the_field_and_lists_the_forms(self): + with pytest.raises(ValueError) as exc: + dates.parse("whenever", field="end") + message = str(exc.value) + assert "end='whenever'" in message + assert "YYYY-MM-DD" in message + assert "ytd" in message diff --git a/plugins/fred/tests/unit/test_observations_shaping.py b/plugins/fred/tests/unit/test_observations_shaping.py new file mode 100644 index 0000000..31cc0bc --- /dev/null +++ b/plugins/fred/tests/unit/test_observations_shaping.py @@ -0,0 +1,171 @@ +"""Columnar alignment, summaries, and downsampling. + +The assertion this file exists for: the summary is computed over every observation and +the point list is thinned afterwards, so a long series still reports its true extremes. +""" + +import pytest + +from src import shaping + +pytestmark = pytest.mark.unit + + +class TestParseValue: + @pytest.mark.parametrize("raw,expected", [("4.2", 4.2), ("0", 0.0), ("-1.5", -1.5), (" 4.2 ", 4.2)]) + def test_strings_become_floats(self, raw, expected): + assert shaping.parse_value(raw) == expected + + @pytest.mark.parametrize("raw", [".", "", None, "n/a"]) + def test_freds_missing_marker_becomes_none(self, raw): + # FRED writes a missing observation as "." and never as null. + assert shaping.parse_value(raw) is None + + def test_zero_is_not_mistaken_for_missing(self): + assert shaping.parse_value("0.0") == 0.0 + + +class TestObservationPairs: + def test_drops_the_realtime_padding(self): + payload = { + "observations": [ + {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "date": "2025-01-01", "value": "4.0"}, + {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "date": "2025-02-01", "value": "."}, + ] + } + assert shaping.observation_pairs(payload) == [("2025-01-01", 4.0), ("2025-02-01", None)] + + def test_an_empty_payload(self): + assert shaping.observation_pairs({}) == [] + + +class TestAlign: + def test_one_series_is_its_own_index(self): + dates, columns = shaping.align({"A": [("2025-01-01", 1.0), ("2025-02-01", 2.0)]}) + assert dates == ["2025-01-01", "2025-02-01"] + assert columns == {"A": [1.0, 2.0]} + + def test_two_frequencies_union_with_nulls_in_the_gaps(self): + # Monthly against quarterly. The gaps are null, not forward-filled: filling + # them would invent observations that FRED never published. + monthly = [("2025-01-01", 4.0), ("2025-02-01", 4.1), ("2025-03-01", 4.2)] + quarterly = [("2025-01-01", 23000.0)] + dates, columns = shaping.align({"M": monthly, "Q": quarterly}) + assert dates == ["2025-01-01", "2025-02-01", "2025-03-01"] + assert columns["M"] == [4.0, 4.1, 4.2] + assert columns["Q"] == [23000.0, None, None] + + def test_dates_present_only_in_the_second_series_still_appear(self): + dates, columns = shaping.align({"A": [("2025-02-01", 1.0)], "B": [("2025-01-01", 2.0)]}) + assert dates == ["2025-01-01", "2025-02-01"] + assert columns["A"] == [None, 1.0] + assert columns["B"] == [2.0, None] + + def test_no_series(self): + assert shaping.align({}) == ([], {}) + + def test_every_column_is_the_length_of_the_index(self): + dates, columns = shaping.align( + {"A": [("2025-01-01", 1.0)], "B": [("2025-02-01", 2.0)], "C": [("2025-03-01", 3.0)]} + ) + assert all(len(column) == len(dates) for column in columns.values()) + + +class TestSummarize: + def test_the_figures_a_question_actually_asks_for(self): + pairs = [("2025-01-01", 4.0), ("2025-02-01", 4.1), ("2025-03-01", 4.4)] + s = shaping.summarize(pairs) + assert s["latest"] == 4.4 + assert s["latest_date"] == "2025-03-01" + assert s["prior"] == 4.1 + assert s["change"] == pytest.approx(0.3) + assert s["pct_change"] == pytest.approx(7.3171, abs=1e-3) + assert (s["min"], s["max"]) == (4.0, 4.4) + assert s["count"] == 3 + + def test_missing_values_are_skipped_but_still_counted(self): + pairs = [("2025-01-01", 4.0), ("2025-02-01", None), ("2025-03-01", 4.4)] + s = shaping.summarize(pairs) + assert s["count"] == 2 # observations with a value + assert s["observations"] == 3 # rows FRED returned + assert s["latest"] == 4.4 + assert s["prior"] == 4.0 # the null is skipped, not treated as the prior + + def test_latest_is_the_last_real_value_not_a_trailing_null(self): + pairs = [("2025-01-01", 4.0), ("2025-02-01", 4.4), ("2025-03-01", None)] + s = shaping.summarize(pairs) + assert s["latest"] == 4.4 + assert s["latest_date"] == "2025-02-01" + + def test_a_single_observation_has_no_change(self): + s = shaping.summarize([("2025-01-01", 4.0)]) + assert s["latest"] == 4.0 + assert "change" not in s and "prior" not in s + + def test_all_missing(self): + assert shaping.summarize([("2025-01-01", None)]) == {"observations": 1, "count": 0} + + def test_empty(self): + assert shaping.summarize([]) == {"observations": 0, "count": 0} + + def test_a_zero_prior_does_not_divide_by_zero(self): + s = shaping.summarize([("2025-01-01", 0.0), ("2025-02-01", 2.0)]) + assert s["change"] == 2.0 + assert "pct_change" not in s + + def test_pct_change_is_signed_correctly_from_a_negative_prior(self): + # Falling further below zero is a decrease, and dividing by a raw negative + # would report it as an increase. + s = shaping.summarize([("2025-01-01", -2.0), ("2025-02-01", -3.0)]) + assert s["change"] == -1.0 + assert s["pct_change"] == pytest.approx(-50.0) + + +class TestDownsample: + def _series(self, n): + dates = [f"2020-{i:04d}" for i in range(n)] + return dates, {"A": [float(i) for i in range(n)]} + + def test_a_short_series_is_untouched(self): + dates, columns = self._series(50) + out_dates, out_columns, dropped = shaping.downsample(dates, columns, 120) + assert (out_dates, out_columns, dropped) == (dates, columns, 0) + + def test_exactly_at_the_cap_is_untouched(self): + dates, columns = self._series(120) + assert shaping.downsample(dates, columns, 120)[2] == 0 + + def test_thins_to_the_cap(self): + dates, columns = self._series(5000) + out_dates, out_columns, dropped = shaping.downsample(dates, columns, 120) + assert len(out_dates) == 120 + assert len(out_columns["A"]) == 120 + assert dropped == 4880 + + def test_the_first_and_last_points_always_survive(self): + # Otherwise the series appears to start and end somewhere it does not. + dates, columns = self._series(5000) + out_dates, out_columns, _ = shaping.downsample(dates, columns, 120) + assert out_dates[0] == dates[0] + assert out_dates[-1] == dates[-1] + assert out_columns["A"][0] == 0.0 + assert out_columns["A"][-1] == 4999.0 + + def test_the_sample_is_evenly_spaced(self): + dates, columns = self._series(1000) + _, out_columns, _ = shaping.downsample(dates, columns, 100) + gaps = {b - a for a, b in zip(out_columns["A"], out_columns["A"][1:])} + assert max(gaps) - min(gaps) <= 1 # even to within rounding + + def test_every_column_stays_aligned_with_the_dates(self): + dates, columns = self._series(1000) + columns["B"] = [float(i) * 2 for i in range(1000)] + out_dates, out_columns, _ = shaping.downsample(dates, columns, 50) + assert all(len(column) == len(out_dates) for column in out_columns.values()) + assert all(b == a * 2 for a, b in zip(out_columns["A"], out_columns["B"])) + + def test_the_minimum_cap_still_produces_the_endpoints(self): + dates, columns = self._series(1000) + out_dates, _, dropped = shaping.downsample(dates, columns, 2) + assert out_dates == [dates[0], dates[-1]] + assert dropped == 998 diff --git a/plugins/fred/tests/unit/test_schemas.py b/plugins/fred/tests/unit/test_schemas.py index 476d4a7..9c60995 100644 --- a/plugins/fred/tests/unit/test_schemas.py +++ b/plugins/fred/tests/unit/test_schemas.py @@ -8,7 +8,7 @@ import pytest from pydantic import ValidationError -from src.schemas import GetSeriesArgs, SearchArgs, normalize_series_ids +from src.schemas import GetSeriesArgs, ObservationArgs, SearchArgs, normalize_series_ids pytestmark = pytest.mark.unit @@ -172,3 +172,94 @@ def test_order_by_is_corrected_and_checked(self): def test_limit_is_bounded(self, limit): with pytest.raises(ValidationError): SearchArgs(query="x", limit=limit) + + +class TestUnitsCorrection: + """The single most valuable table in the correction layer: nobody remembers pc1.""" + + @pytest.mark.parametrize( + "written,expected", + [ + ("yoy", "pc1"), + ("YoY", "pc1"), + ("year over year", "pc1"), + ("year-over-year", "pc1"), + ("percent change from a year ago", "pc1"), + ("inflation", "pc1"), + ("pc1", "pc1"), + ("percent change", "pch"), + ("pct_change", "pch"), + ("mom", "pch"), + ("month over month", "pch"), + ("change", "chg"), + ("diff", "chg"), + ("change from a year ago", "ch1"), + ("annualized", "pca"), + ("saar", "pca"), + ("level", "lin"), + ("levels", "lin"), + ("raw", "lin"), + ("none", "lin"), + ("", "lin"), + ("natural log", "log"), + ("ln", "log"), + ], + ) + def test_phrases_become_fred_codes(self, written, expected): + assert ObservationArgs(series_ids="UNRATE", units=written).units == expected + + def test_every_code_carries_its_meaning(self): + args = ObservationArgs(series_ids="UNRATE", units="yoy") + assert args.units_meaning == "percent change from a year ago" + + def test_an_invented_transformation_is_rejected_with_the_real_options(self): + with pytest.raises(ValidationError) as exc: + ObservationArgs(series_ids="UNRATE", units="cagr") + assert "yoy" in str(exc.value) + + +class TestAggregationCorrection: + @pytest.mark.parametrize( + "written,expected", + [("average", "avg"), ("mean", "avg"), ("total", "sum"), ("end of period", "eop"), ("last", "eop")], + ) + def test_words_become_codes(self, written, expected): + assert ObservationArgs(series_ids="X", aggregation_method=written).aggregation_method == expected + + def test_unknown_is_rejected(self): + with pytest.raises(ValidationError, match="unknown aggregation_method"): + ObservationArgs(series_ids="X", aggregation_method="median") + + +class TestObservationRange: + def test_relative_dates_are_resolved(self, monkeypatch): + from datetime import date + + from src import dates as dates_module + + monkeypatch.setattr(dates_module, "today", lambda: date(2026, 8, 5)) + args = ObservationArgs(series_ids="UNRATE", start="5y", end="today") + assert (args.start, args.end) == ("2021-08-05", "2026-08-05") + + def test_a_backwards_range_is_caught_before_the_api_sees_it(self): + with pytest.raises(ValidationError, match="swap them"): + ObservationArgs(series_ids="UNRATE", start="2025-01-01", end="2020-01-01") + + def test_an_equal_range_is_fine(self): + # One specific observation date is a legitimate request. + args = ObservationArgs(series_ids="UNRATE", start="2025-01-01", end="2025-01-01") + assert args.start == args.end + + def test_an_unparseable_date_is_guidance_not_a_400(self): + with pytest.raises(ValidationError, match="could not be read as a date"): + ObservationArgs(series_ids="UNRATE", start="whenever") + + @pytest.mark.parametrize("cap", [1, 2001]) + def test_max_points_is_bounded(self, cap): + with pytest.raises(ValidationError): + ObservationArgs(series_ids="UNRATE", max_points=cap) + + def test_a_finer_frequency_is_still_expressible_here(self): + # Only FRED knows a series' native frequency, so "finer than the series" is + # its call; this layer only rejects frequencies FRED has no code for. + assert ObservationArgs(series_ids="UNRATE", frequency="daily").frequency == "d" diff --git a/plugins/fred/tests/unit/test_server.py b/plugins/fred/tests/unit/test_server.py index 5eb8a9b..97ee13b 100644 --- a/plugins/fred/tests/unit/test_server.py +++ b/plugins/fred/tests/unit/test_server.py @@ -20,7 +20,7 @@ async def test_server_builds_and_exposes_its_tools(): # Grows with the stack in #30. Asserting the exact set is what catches a tool that was # written but never registered, which is otherwise invisible until a user asks for it. -EXPECTED_TOOLS: set[str] = {"search_series", "get_series"} +EXPECTED_TOOLS: set[str] = {"search_series", "get_series", "get_observations"} def test_instructions_name_the_entry_point():