fix(pagination): stop *_all iterators repeating the first page (SCR-52)

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.15.4] - 2026-06-30
+
+### Fixed
+
+- **Auto-pagination no longer repeats the first page** (`*_all` iterators, e.g. `client.twitter.tweets.search_all`). The shared `paginate()` helper now stops when the backend returns the same cursor it was given, instead of re-fetching the page it just yielded — previously a non-advancing cursor from the API could cause a repeat-page loop. (SCR-52)
+
 ## [0.15.3] - 2026-06-30
 
 ### Added

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapebadger"
-version = "0.15.3"
+version = "0.15.4"
 description = "Official Python SDK for ScrapeBadger - Async web scraping APIs for Twitter and more"
 readme = "README.md"
 license = { text = "MIT" }

src/scrapebadger/__init__.py

@@ -420,7 +420,7 @@ async def main():
     Video as YoutubeVideo,
 )
 
-__version__ = "0.15.3"
+__version__ = "0.15.4"
 
 __all__ = [
     # TikTok core models

src/scrapebadger/_internal/pagination.py

@@ -179,13 +179,16 @@ async def paginate(
             yield item_parser(item)
             items_yielded += 1
 
-        # Update cursor for next page
-        cursor = response.get("next_cursor")
+        # Advance the cursor for the next page.
+        next_cursor = response.get("next_cursor")
         pages_fetched += 1
 
-        # Check if we've reached the end before potentially sleeping.
-        if not cursor:
+        # Stop at the last page, or if the backend echoes the same cursor it was
+        # given — advancing on a non-advancing cursor would re-fetch the page we
+        # just yielded (the "repeats first page" bug). (SCR-52)
+        if not next_cursor or next_cursor == cursor:
             break
+        cursor = next_cursor
 
         # Rate-limit-aware throttling between pages.
         _maybe_throttle_between_pages(headers)

tests/test_pagination.py

@@ -110,6 +110,24 @@ async def test_empty_data_field_stops_iteration(self, base_client: BaseClient) -
 
         assert results == []
 
+    async def test_echoed_cursor_stops_iteration(self, base_client: BaseClient) -> None:
+        """Stops when the backend echoes the same cursor it was given (SCR-52).
+
+        A non-advancing cursor would otherwise re-fetch the page just yielded,
+        causing the 'repeats first page' loop. Only two pages are mocked, so a
+        third call would raise StopIteration and fail this test.
+        """
+        page1, h1 = _make_page([{"id": "1"}], next_cursor="c")
+        page2, h2 = _make_page([{"id": "2"}], next_cursor="c")  # echoes "c"
+        base_client.get_with_headers = AsyncMock(  # type: ignore[method-assign]
+            side_effect=[(page1, h1), (page2, h2)]
+        )
+
+        results = [item async for item in paginate(base_client, "/v1/test", {}, lambda x: x)]
+
+        assert results == [{"id": "1"}, {"id": "2"}]
+        assert base_client.get_with_headers.call_count == 2
+
     async def test_item_parser_is_applied(self, base_client: BaseClient) -> None:
         """Applies item_parser to each raw dict."""
         page, headers = _make_page([{"value": 1}, {"value": 2}])