diff --git a/CHANGELOG.md b/CHANGELOG.md index 30ad497..a0ac993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.15.4] - 2026-06-30 + +### Fixed + +- **Auto-pagination no longer repeats the first page** (`*_all` iterators, e.g. `client.twitter.tweets.search_all`). The shared `paginate()` helper now stops when the backend returns the same cursor it was given, instead of re-fetching the page it just yielded — previously a non-advancing cursor from the API could cause a repeat-page loop. (SCR-52) + ## [0.15.3] - 2026-06-30 ### Added diff --git a/pyproject.toml b/pyproject.toml index 691dfeb..d655975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scrapebadger" -version = "0.15.3" +version = "0.15.4" description = "Official Python SDK for ScrapeBadger - Async web scraping APIs for Twitter and more" readme = "README.md" license = { text = "MIT" } diff --git a/src/scrapebadger/__init__.py b/src/scrapebadger/__init__.py index bc2ca33..1403620 100644 --- a/src/scrapebadger/__init__.py +++ b/src/scrapebadger/__init__.py @@ -420,7 +420,7 @@ async def main(): Video as YoutubeVideo, ) -__version__ = "0.15.3" +__version__ = "0.15.4" __all__ = [ # TikTok core models diff --git a/src/scrapebadger/_internal/pagination.py b/src/scrapebadger/_internal/pagination.py index 622708c..08b5cf0 100644 --- a/src/scrapebadger/_internal/pagination.py +++ b/src/scrapebadger/_internal/pagination.py @@ -179,13 +179,16 @@ async def paginate( yield item_parser(item) items_yielded += 1 - # Update cursor for next page - cursor = response.get("next_cursor") + # Advance the cursor for the next page. + next_cursor = response.get("next_cursor") pages_fetched += 1 - # Check if we've reached the end before potentially sleeping. - if not cursor: + # Stop at the last page, or if the backend echoes the same cursor it was + # given — advancing on a non-advancing cursor would re-fetch the page we + # just yielded (the "repeats first page" bug). (SCR-52) + if not next_cursor or next_cursor == cursor: break + cursor = next_cursor # Rate-limit-aware throttling between pages. _maybe_throttle_between_pages(headers) diff --git a/tests/test_pagination.py b/tests/test_pagination.py index 820e9a8..23ca015 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -110,6 +110,24 @@ async def test_empty_data_field_stops_iteration(self, base_client: BaseClient) - assert results == [] + async def test_echoed_cursor_stops_iteration(self, base_client: BaseClient) -> None: + """Stops when the backend echoes the same cursor it was given (SCR-52). + + A non-advancing cursor would otherwise re-fetch the page just yielded, + causing the 'repeats first page' loop. Only two pages are mocked, so a + third call would raise StopIteration and fail this test. + """ + page1, h1 = _make_page([{"id": "1"}], next_cursor="c") + page2, h2 = _make_page([{"id": "2"}], next_cursor="c") # echoes "c" + base_client.get_with_headers = AsyncMock( # type: ignore[method-assign] + side_effect=[(page1, h1), (page2, h2)] + ) + + results = [item async for item in paginate(base_client, "/v1/test", {}, lambda x: x)] + + assert results == [{"id": "1"}, {"id": "2"}] + assert base_client.get_with_headers.call_count == 2 + async def test_item_parser_is_applied(self, base_client: BaseClient) -> None: """Applies item_parser to each raw dict.""" page, headers = _make_page([{"value": 1}, {"value": 2}])