feat: Article rich content parsing + bookmarks cursor pagination

tests/test_article_parsing.py

@@ -0,0 +1,264 @@
+"""Tests for Twitter Article (long-form) Draft.js parsing enhancements.
+
+Covers:
+- _render_article_text_block: inline styles (Bold/Italic/Code/Strikethrough),
+  entity links, and mixed style+link on the same block.
+- _extract_atomic_content: DIVIDER, TWEET, MARKDOWN entity types.
+- _parse_article: end-to-end with synthetic article data.
+"""
+
+from __future__ import annotations
+
+
+from twitter_cli.parser import (
+    _extract_atomic_content,
+    _normalize_article_entity_map,
+    _parse_article,
+    _render_article_text_block,
+)
+
+
+# ── _render_article_text_block ──────────────────────────────────────────
+
+
+class TestRenderArticleTextBlock:
+    def test_plain_text_unchanged(self):
+        block = {"text": "hello world", "entityRanges": [], "inlineStyleRanges": []}
+        assert _render_article_text_block(block, {}) == "hello world"
+
+    def test_empty_text(self):
+        assert _render_article_text_block({"text": "", "entityRanges": [], "inlineStyleRanges": []}, {}) == ""
+
+    def test_bold(self):
+        block = {
+            "text": "LEVEL 1 — ONE-SHOT PROMPTS",
+            "entityRanges": [],
+            "inlineStyleRanges": [{"offset": 0, "length": 26, "style": "Bold"}],
+        }
+        result = _render_article_text_block(block, {})
+        assert result == "**LEVEL 1 — ONE-SHOT PROMPTS**"
+
+    def test_italic(self):
+        block = {
+            "text": "emphasis here",
+            "entityRanges": [],
+            "inlineStyleRanges": [{"offset": 9, "length": 4, "style": "Italic"}],
+        }
+        result = _render_article_text_block(block, {})
+        assert result == "emphasis *here*"
+
+    def test_code(self):
+        block = {
+            "text": "run npm install",
+            "entityRanges": [],
+            "inlineStyleRanges": [{"offset": 4, "length": 11, "style": "Code"}],
+        }
+        result = _render_article_text_block(block, {})
+        assert result == "run `npm install`"
+
+    def test_strikethrough(self):
+        block = {
+            "text": "old text new",
+            "entityRanges": [],
+            "inlineStyleRanges": [{"offset": 0, "length": 7, "style": "Strikethrough"}],
+        }
+        result = _render_article_text_block(block, {})
+        assert result == "~~old tex~~t new"  # length 7 from offset 0
+
+    def test_style_case_insensitive(self):
+        """Twitter API returns 'Bold' (Title case), not 'BOLD'."""
+        block = {
+            "text": "hello",
+            "entityRanges": [],
+            "inlineStyleRanges": [{"offset": 0, "length": 5, "style": "bold"}],
+        }
+        assert "**hello**" == _render_article_text_block(block, {})
+
+    def test_out_of_bounds_style_ignored(self):
+        block = {
+            "text": "short",
+            "entityRanges": [],
+            "inlineStyleRanges": [{"offset": 0, "length": 100, "style": "Bold"}],
+        }
+        assert _render_article_text_block(block, {}) == "short"
+
+    def test_link(self):
+        block = {
+            "text": "Click here for more",
+            "entityRanges": [{"key": 0, "offset": 6, "length": 4}],
+            "inlineStyleRanges": [],
+        }
+        entity_map = {"0": {"type": "LINK", "data": {"url": "https://example.com"}}}
+        result = _render_article_text_block(block, entity_map)
+        assert result == "Click [here](https://example.com) for more"
+
+    def test_link_with_paren_in_url(self):
+        block = {
+            "text": "see Wikipedia",
+            "entityRanges": [{"key": 0, "offset": 4, "length": 9}],
+            "inlineStyleRanges": [],
+        }
+        entity_map = {"0": {"type": "LINK", "data": {"url": "https://en.wikipedia.org/wiki/Test_(page)"}}}
+        result = _render_article_text_block(block, entity_map)
+        assert "%29" in result  # ) should be encoded
+
+    def test_bold_and_link_mixed(self):
+        """The key regression: bold and link on the same block must not corrupt each other's offsets."""
+        block = {
+            "text": "Click here for more",
+            "entityRanges": [{"key": 0, "offset": 6, "length": 4}],
+            "inlineStyleRanges": [{"offset": 0, "length": 5, "style": "Bold"}],
+        }
+        entity_map = {"0": {"type": "LINK", "data": {"url": "https://example.com"}}}
+        result = _render_article_text_block(block, entity_map)
+        assert "**Click**" in result
+        assert "[here](https://example.com)" in result
+
+    def test_multiple_non_overlapping_styles(self):
+        block = {
+            "text": "bold text and italic text",
+            "entityRanges": [],
+            "inlineStyleRanges": [
+                {"offset": 0, "length": 4, "style": "Bold"},
+                {"offset": 14, "length": 6, "style": "Italic"},
+            ],
+        }
+        result = _render_article_text_block(block, {})
+        assert result == "**bold** text and *italic* text"
+
+
+# ── _extract_atomic_content ─────────────────────────────────────────────
+
+
+class TestExtractAtomicContent:
+    def test_divider(self):
+        block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]}
+        entity_map = {"0": {"type": "DIVIDER", "data": {}}}
+        assert _extract_atomic_content(block, entity_map) == ["---"]
+
+    def test_embedded_tweet(self):
+        block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]}
+        entity_map = {"0": {"type": "TWEET", "data": {"tweetId": "123456"}}}
+        result = _extract_atomic_content(block, entity_map)
+        assert len(result) == 1
+        assert "Embedded Tweet" in result[0]
+        assert "123456" in result[0]
+        assert "https://x.com/i/status/123456" in result[0]
+
+    def test_markdown_block(self):
+        block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]}
+        entity_map = {"0": {"type": "MARKDOWN", "data": {"markdown": "# Heading"}}}
+        assert _extract_atomic_content(block, entity_map) == ["# Heading"]
+
+    def test_unknown_type_ignored(self):
+        block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]}
+        entity_map = {"0": {"type": "UNKNOWN", "data": {}}}
+        assert _extract_atomic_content(block, entity_map) == []
+
+    def test_empty_entity_ranges(self):
+        assert _extract_atomic_content({"entityRanges": []}, {}) == []
+
+    def test_multiple_entities(self):
+        block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}, {"key": 1, "length": 1, "offset": 2}]}
+        entity_map = {
+            "0": {"type": "DIVIDER", "data": {}},
+            "1": {"type": "TWEET", "data": {"tweetId": "999"}},
+        }
+        result = _extract_atomic_content(block, entity_map)
+        assert len(result) == 2
+        assert result[0] == "---"
+        assert "999" in result[1]
+
+
+# ── _normalize_article_entity_map ───────────────────────────────────────
+
+
+class TestNormalizeEntityMap:
+    def test_list_format(self):
+        """Twitter API returns entityMap as [{key, value}, ...]."""
+        raw = [{"key": "0", "value": {"type": "LINK", "data": {}}}]
+        result = _normalize_article_entity_map(raw)
+        assert result == {"0": {"type": "LINK", "data": {}}}
+
+    def test_dict_format(self):
+        raw = {"0": {"type": "LINK", "data": {}}}
+        result = _normalize_article_entity_map(raw)
+        assert result == {"0": {"type": "LINK", "data": {}}}
+
+    def test_empty(self):
+        assert _normalize_article_entity_map([]) == {}
+        assert _normalize_article_entity_map({}) == {}
+
+
+# ── _parse_article (end-to-end with synthetic data) ─────────────────────
+
+
+class TestParseArticle:
+    def _make_article(self, blocks, entity_map=None, media_entities=None):
+        return {
+            "article": {
+                "article_results": {
+                    "result": {
+                        "title": "Test Article",
+                        "content_state": {
+                            "blocks": blocks,
+                            "entityMap": entity_map or [],
+                        },
+                        "media_entities": media_entities or [],
+                        "cover_media": {},
+                    }
+                }
+            }
+        }
+
+    def test_basic_blocks(self):
+        blocks = [
+            {"type": "header-two", "text": "Section", "entityRanges": [], "inlineStyleRanges": []},
+            {"type": "unstyled", "text": "Body text", "entityRanges": [], "inlineStyleRanges": []},
+            {"type": "blockquote", "text": "A quote", "entityRanges": [], "inlineStyleRanges": []},
+        ]
+        result = _parse_article(self._make_article(blocks))
+        text = result["article_text"]
+        assert "## Section" in text
+        assert "Body text" in text
+        assert "> A quote" in text
+
+    def test_divider_between_sections(self):
+        blocks = [
+            {"type": "unstyled", "text": "Before", "entityRanges": [], "inlineStyleRanges": []},
+            {"type": "atomic", "text": " ", "entityRanges": [{"key": 0, "length": 1, "offset": 0}], "inlineStyleRanges": []},
+            {"type": "unstyled", "text": "After", "entityRanges": [], "inlineStyleRanges": []},
+        ]
+        entity_map = [{"key": "0", "value": {"type": "DIVIDER", "data": {}}}]
+        result = _parse_article(self._make_article(blocks, entity_map))
+        text = result["article_text"]
+        assert "Before" in text
+        assert "---" in text
+        assert "After" in text
+
+    def test_image_with_media_entities(self):
+        blocks = [
+            {"type": "atomic", "text": " ", "entityRanges": [{"key": 0, "length": 1, "offset": 0}], "inlineStyleRanges": []},
+        ]
+        entity_map = [{
+            "key": "0",
+            "value": {
+                "type": "MEDIA",
+                "data": {
+                    "caption": "A screenshot",
+                    "mediaItems": [{"mediaId": "111"}],
+                },
+            },
+        }]
+        media_entities = [{
+            "media_id": "111",
+            "media_info": {"original_img_url": "https://pbs.twimg.com/media/test.jpg"},
+        }]
+        result = _parse_article(self._make_article(blocks, entity_map, media_entities))
+        text = result["article_text"]
+        assert "![A screenshot](https://pbs.twimg.com/media/test.jpg)" in text
+
+    def test_no_article_returns_none(self):
+        result = _parse_article({"legacy": {}})
+        assert result["article_title"] is None
+        assert result["article_text"] is None

tests/test_client.py

@@ -25,7 +25,7 @@
 )
 from twitter_cli.parser import (
     _deep_get,
-    _extract_atomic_markdown,
+    _extract_atomic_content,
     _extract_cursor,
     _extract_media,
     _normalize_article_entity_map,
@@ -531,7 +531,7 @@ def test_extracts_markdown_entity(self):
             "4": {"type": "MARKDOWN", "data": {"markdown": "```markdown\nconst answer = 42;\n```"}}
         }
 
-        assert _extract_atomic_markdown(block, entity_map) == ["```markdown\nconst answer = 42;\n```"]
+        assert _extract_atomic_content(block, entity_map) == ["```markdown\nconst answer = 42;\n```"]
 
     def test_ignores_non_markdown_entities(self):
         block = {"entityRanges": [{"key": 0}, {"key": 1}]}
@@ -540,13 +540,13 @@ def test_ignores_non_markdown_entities(self):
             "1": {"type": "LINK", "data": {"url": "https://example.com"}},
         }
 
-        assert _extract_atomic_markdown(block, entity_map) == []
+        assert _extract_atomic_content(block, entity_map) == []
 
     def test_ignores_blank_markdown(self):
         block = {"entityRanges": [{"key": 4}]}
         entity_map = {"4": {"type": "MARKDOWN", "data": {"markdown": "   \n"}}}
 
-        assert _extract_atomic_markdown(block, entity_map) == []
+        assert _extract_atomic_content(block, entity_map) == []
 
 
 class TestRenderArticleTextBlock:

twitter_cli/cli.py

@@ -358,25 +358,54 @@ def _emit_timeline_structured(tweets, next_cursor, *, as_json, as_yaml):
     return emit_structured(payload, as_json=as_json, as_yaml=as_yaml)
 
 
-def _run_bookmarks_command(max_count, as_json, as_yaml, output_file, do_filter, compact=False, full_text=False):
-    # type: (Optional[int], bool, bool, Optional[str], bool, bool, bool) -> None
+def _run_bookmarks_command(max_count, as_json, as_yaml, output_file, do_filter, compact=False, full_text=False, cursor=None):
+    # type: (Optional[int], bool, bool, Optional[str], bool, bool, bool, Optional[str]) -> None
+    """Fetch bookmarks with optional cursor-based pagination.
+
+    When ``cursor`` is supplied, continues a previous fetch from that cursor.
+    The JSON/YAML envelope emits ``pagination.nextCursor`` so callers can page
+    through bookmarks in small, resumable batches.
+    """
     config = load_config()
+    rich_output = use_rich_output(as_json=as_json, as_yaml=as_yaml, compact=compact)
 
     def _run():
-        client = _get_client(config)
-        _fetch_and_display(
-            lambda count: client.fetch_bookmarks(count),
-            "bookmarks",
-            "🔖",
-            max_count,
-            as_json,
-            as_yaml,
-            output_file,
-            do_filter,
-            config,
-            compact=compact,
+        client = _get_client(config, quiet=not rich_output)
+        fetch_count = _resolve_configured_count(config, max_count)
+        if rich_output:
+            console.print("🔖 Fetching bookmarks (%d tweets)...\n" % fetch_count)
+        start = time.time()
+        tweets, next_cursor = client.fetch_bookmarks(
+            fetch_count, cursor=cursor, return_cursor=True,
+        )
+        elapsed = time.time() - start
+        if rich_output:
+            console.print("✅ Fetched %d bookmarks in %.1fs\n" % (len(tweets), elapsed))
+
+        filtered = _apply_filter(tweets, do_filter, config, rich_output=rich_output)
+
+        if output_file:
+            Path(output_file).write_text(tweets_to_json(filtered), encoding="utf-8")
+            if rich_output:
+                console.print("💾 Saved to %s\n" % output_file)
+
+        if compact:
+            click.echo(tweets_to_compact_json(filtered))
+            return
+
+        save_tweet_cache(filtered)
+
+        if _emit_timeline_structured(filtered, next_cursor, as_json=as_json, as_yaml=as_yaml):
+            return
+
+        print_tweet_table(
+            filtered,
+            console,
+            title="🔖 Bookmarks — %d tweets" % len(filtered),
             full_text=full_text,
         )
+        _print_show_hint()
+        console.print()
 
     _run_guarded(_run)
 
@@ -509,13 +538,14 @@ def favorites(ctx, max_count, as_json, as_yaml, output_file, do_filter, full_tex
 
 @cli.group(name="bookmarks", invoke_without_command=True)
 @click.option("--max", "-n", "max_count", type=int, default=None, help="Max number of tweets to fetch.")
+@click.option("--cursor", type=str, default=None, help="Pagination cursor to continue a previous bookmarks fetch (enables resumable batched fetching).")
 @structured_output_options
 @click.option("--output", "-o", "output_file", type=str, default=None, help="Save tweets to JSON file.")
 @click.option("--filter", "do_filter", is_flag=True, help="Enable score-based filtering.")
 @click.option("--full-text", is_flag=True, help="Show full tweet text in table output.")
 @click.pass_context
-def bookmarks(ctx, max_count, as_json, as_yaml, output_file, do_filter, full_text):
-    # type: (Any, Optional[int], bool, bool, Optional[str], bool, bool) -> None
+def bookmarks(ctx, max_count, cursor, as_json, as_yaml, output_file, do_filter, full_text):
+    # type: (Any, Optional[int], Optional[str], bool, bool, Optional[str], bool, bool) -> None
     """Fetch bookmarked tweets, or manage bookmark folders."""
     if ctx.invoked_subcommand is None:
         _run_bookmarks_command(
@@ -526,6 +556,7 @@ def bookmarks(ctx, max_count, as_json, as_yaml, output_file, do_filter, full_tex
             do_filter,
             compact=ctx.obj.get("compact", False),
             full_text=full_text,
+            cursor=cursor,
         )