diff --git a/tests/test_article_parsing.py b/tests/test_article_parsing.py new file mode 100644 index 0000000..315db07 --- /dev/null +++ b/tests/test_article_parsing.py @@ -0,0 +1,264 @@ +"""Tests for Twitter Article (long-form) Draft.js parsing enhancements. + +Covers: +- _render_article_text_block: inline styles (Bold/Italic/Code/Strikethrough), + entity links, and mixed style+link on the same block. +- _extract_atomic_content: DIVIDER, TWEET, MARKDOWN entity types. +- _parse_article: end-to-end with synthetic article data. +""" + +from __future__ import annotations + + +from twitter_cli.parser import ( + _extract_atomic_content, + _normalize_article_entity_map, + _parse_article, + _render_article_text_block, +) + + +# ── _render_article_text_block ────────────────────────────────────────── + + +class TestRenderArticleTextBlock: + def test_plain_text_unchanged(self): + block = {"text": "hello world", "entityRanges": [], "inlineStyleRanges": []} + assert _render_article_text_block(block, {}) == "hello world" + + def test_empty_text(self): + assert _render_article_text_block({"text": "", "entityRanges": [], "inlineStyleRanges": []}, {}) == "" + + def test_bold(self): + block = { + "text": "LEVEL 1 — ONE-SHOT PROMPTS", + "entityRanges": [], + "inlineStyleRanges": [{"offset": 0, "length": 26, "style": "Bold"}], + } + result = _render_article_text_block(block, {}) + assert result == "**LEVEL 1 — ONE-SHOT PROMPTS**" + + def test_italic(self): + block = { + "text": "emphasis here", + "entityRanges": [], + "inlineStyleRanges": [{"offset": 9, "length": 4, "style": "Italic"}], + } + result = _render_article_text_block(block, {}) + assert result == "emphasis *here*" + + def test_code(self): + block = { + "text": "run npm install", + "entityRanges": [], + "inlineStyleRanges": [{"offset": 4, "length": 11, "style": "Code"}], + } + result = _render_article_text_block(block, {}) + assert result == "run `npm install`" + + def test_strikethrough(self): + block = { + "text": "old text new", + "entityRanges": [], + "inlineStyleRanges": [{"offset": 0, "length": 7, "style": "Strikethrough"}], + } + result = _render_article_text_block(block, {}) + assert result == "~~old tex~~t new" # length 7 from offset 0 + + def test_style_case_insensitive(self): + """Twitter API returns 'Bold' (Title case), not 'BOLD'.""" + block = { + "text": "hello", + "entityRanges": [], + "inlineStyleRanges": [{"offset": 0, "length": 5, "style": "bold"}], + } + assert "**hello**" == _render_article_text_block(block, {}) + + def test_out_of_bounds_style_ignored(self): + block = { + "text": "short", + "entityRanges": [], + "inlineStyleRanges": [{"offset": 0, "length": 100, "style": "Bold"}], + } + assert _render_article_text_block(block, {}) == "short" + + def test_link(self): + block = { + "text": "Click here for more", + "entityRanges": [{"key": 0, "offset": 6, "length": 4}], + "inlineStyleRanges": [], + } + entity_map = {"0": {"type": "LINK", "data": {"url": "https://example.com"}}} + result = _render_article_text_block(block, entity_map) + assert result == "Click [here](https://example.com) for more" + + def test_link_with_paren_in_url(self): + block = { + "text": "see Wikipedia", + "entityRanges": [{"key": 0, "offset": 4, "length": 9}], + "inlineStyleRanges": [], + } + entity_map = {"0": {"type": "LINK", "data": {"url": "https://en.wikipedia.org/wiki/Test_(page)"}}} + result = _render_article_text_block(block, entity_map) + assert "%29" in result # ) should be encoded + + def test_bold_and_link_mixed(self): + """The key regression: bold and link on the same block must not corrupt each other's offsets.""" + block = { + "text": "Click here for more", + "entityRanges": [{"key": 0, "offset": 6, "length": 4}], + "inlineStyleRanges": [{"offset": 0, "length": 5, "style": "Bold"}], + } + entity_map = {"0": {"type": "LINK", "data": {"url": "https://example.com"}}} + result = _render_article_text_block(block, entity_map) + assert "**Click**" in result + assert "[here](https://example.com)" in result + + def test_multiple_non_overlapping_styles(self): + block = { + "text": "bold text and italic text", + "entityRanges": [], + "inlineStyleRanges": [ + {"offset": 0, "length": 4, "style": "Bold"}, + {"offset": 14, "length": 6, "style": "Italic"}, + ], + } + result = _render_article_text_block(block, {}) + assert result == "**bold** text and *italic* text" + + +# ── _extract_atomic_content ───────────────────────────────────────────── + + +class TestExtractAtomicContent: + def test_divider(self): + block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]} + entity_map = {"0": {"type": "DIVIDER", "data": {}}} + assert _extract_atomic_content(block, entity_map) == ["---"] + + def test_embedded_tweet(self): + block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]} + entity_map = {"0": {"type": "TWEET", "data": {"tweetId": "123456"}}} + result = _extract_atomic_content(block, entity_map) + assert len(result) == 1 + assert "Embedded Tweet" in result[0] + assert "123456" in result[0] + assert "https://x.com/i/status/123456" in result[0] + + def test_markdown_block(self): + block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]} + entity_map = {"0": {"type": "MARKDOWN", "data": {"markdown": "# Heading"}}} + assert _extract_atomic_content(block, entity_map) == ["# Heading"] + + def test_unknown_type_ignored(self): + block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}]} + entity_map = {"0": {"type": "UNKNOWN", "data": {}}} + assert _extract_atomic_content(block, entity_map) == [] + + def test_empty_entity_ranges(self): + assert _extract_atomic_content({"entityRanges": []}, {}) == [] + + def test_multiple_entities(self): + block = {"entityRanges": [{"key": 0, "length": 1, "offset": 0}, {"key": 1, "length": 1, "offset": 2}]} + entity_map = { + "0": {"type": "DIVIDER", "data": {}}, + "1": {"type": "TWEET", "data": {"tweetId": "999"}}, + } + result = _extract_atomic_content(block, entity_map) + assert len(result) == 2 + assert result[0] == "---" + assert "999" in result[1] + + +# ── _normalize_article_entity_map ─────────────────────────────────────── + + +class TestNormalizeEntityMap: + def test_list_format(self): + """Twitter API returns entityMap as [{key, value}, ...].""" + raw = [{"key": "0", "value": {"type": "LINK", "data": {}}}] + result = _normalize_article_entity_map(raw) + assert result == {"0": {"type": "LINK", "data": {}}} + + def test_dict_format(self): + raw = {"0": {"type": "LINK", "data": {}}} + result = _normalize_article_entity_map(raw) + assert result == {"0": {"type": "LINK", "data": {}}} + + def test_empty(self): + assert _normalize_article_entity_map([]) == {} + assert _normalize_article_entity_map({}) == {} + + +# ── _parse_article (end-to-end with synthetic data) ───────────────────── + + +class TestParseArticle: + def _make_article(self, blocks, entity_map=None, media_entities=None): + return { + "article": { + "article_results": { + "result": { + "title": "Test Article", + "content_state": { + "blocks": blocks, + "entityMap": entity_map or [], + }, + "media_entities": media_entities or [], + "cover_media": {}, + } + } + } + } + + def test_basic_blocks(self): + blocks = [ + {"type": "header-two", "text": "Section", "entityRanges": [], "inlineStyleRanges": []}, + {"type": "unstyled", "text": "Body text", "entityRanges": [], "inlineStyleRanges": []}, + {"type": "blockquote", "text": "A quote", "entityRanges": [], "inlineStyleRanges": []}, + ] + result = _parse_article(self._make_article(blocks)) + text = result["article_text"] + assert "## Section" in text + assert "Body text" in text + assert "> A quote" in text + + def test_divider_between_sections(self): + blocks = [ + {"type": "unstyled", "text": "Before", "entityRanges": [], "inlineStyleRanges": []}, + {"type": "atomic", "text": " ", "entityRanges": [{"key": 0, "length": 1, "offset": 0}], "inlineStyleRanges": []}, + {"type": "unstyled", "text": "After", "entityRanges": [], "inlineStyleRanges": []}, + ] + entity_map = [{"key": "0", "value": {"type": "DIVIDER", "data": {}}}] + result = _parse_article(self._make_article(blocks, entity_map)) + text = result["article_text"] + assert "Before" in text + assert "---" in text + assert "After" in text + + def test_image_with_media_entities(self): + blocks = [ + {"type": "atomic", "text": " ", "entityRanges": [{"key": 0, "length": 1, "offset": 0}], "inlineStyleRanges": []}, + ] + entity_map = [{ + "key": "0", + "value": { + "type": "MEDIA", + "data": { + "caption": "A screenshot", + "mediaItems": [{"mediaId": "111"}], + }, + }, + }] + media_entities = [{ + "media_id": "111", + "media_info": {"original_img_url": "https://pbs.twimg.com/media/test.jpg"}, + }] + result = _parse_article(self._make_article(blocks, entity_map, media_entities)) + text = result["article_text"] + assert "![A screenshot](https://pbs.twimg.com/media/test.jpg)" in text + + def test_no_article_returns_none(self): + result = _parse_article({"legacy": {}}) + assert result["article_title"] is None + assert result["article_text"] is None diff --git a/tests/test_client.py b/tests/test_client.py index c1393d3..be71a5a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -25,7 +25,7 @@ ) from twitter_cli.parser import ( _deep_get, - _extract_atomic_markdown, + _extract_atomic_content, _extract_cursor, _extract_media, _normalize_article_entity_map, @@ -531,7 +531,7 @@ def test_extracts_markdown_entity(self): "4": {"type": "MARKDOWN", "data": {"markdown": "```markdown\nconst answer = 42;\n```"}} } - assert _extract_atomic_markdown(block, entity_map) == ["```markdown\nconst answer = 42;\n```"] + assert _extract_atomic_content(block, entity_map) == ["```markdown\nconst answer = 42;\n```"] def test_ignores_non_markdown_entities(self): block = {"entityRanges": [{"key": 0}, {"key": 1}]} @@ -540,13 +540,13 @@ def test_ignores_non_markdown_entities(self): "1": {"type": "LINK", "data": {"url": "https://example.com"}}, } - assert _extract_atomic_markdown(block, entity_map) == [] + assert _extract_atomic_content(block, entity_map) == [] def test_ignores_blank_markdown(self): block = {"entityRanges": [{"key": 4}]} entity_map = {"4": {"type": "MARKDOWN", "data": {"markdown": " \n"}}} - assert _extract_atomic_markdown(block, entity_map) == [] + assert _extract_atomic_content(block, entity_map) == [] class TestRenderArticleTextBlock: diff --git a/twitter_cli/cli.py b/twitter_cli/cli.py index d2dc523..c54dd6d 100644 --- a/twitter_cli/cli.py +++ b/twitter_cli/cli.py @@ -358,25 +358,54 @@ def _emit_timeline_structured(tweets, next_cursor, *, as_json, as_yaml): return emit_structured(payload, as_json=as_json, as_yaml=as_yaml) -def _run_bookmarks_command(max_count, as_json, as_yaml, output_file, do_filter, compact=False, full_text=False): - # type: (Optional[int], bool, bool, Optional[str], bool, bool, bool) -> None +def _run_bookmarks_command(max_count, as_json, as_yaml, output_file, do_filter, compact=False, full_text=False, cursor=None): + # type: (Optional[int], bool, bool, Optional[str], bool, bool, bool, Optional[str]) -> None + """Fetch bookmarks with optional cursor-based pagination. + + When ``cursor`` is supplied, continues a previous fetch from that cursor. + The JSON/YAML envelope emits ``pagination.nextCursor`` so callers can page + through bookmarks in small, resumable batches. + """ config = load_config() + rich_output = use_rich_output(as_json=as_json, as_yaml=as_yaml, compact=compact) def _run(): - client = _get_client(config) - _fetch_and_display( - lambda count: client.fetch_bookmarks(count), - "bookmarks", - "🔖", - max_count, - as_json, - as_yaml, - output_file, - do_filter, - config, - compact=compact, + client = _get_client(config, quiet=not rich_output) + fetch_count = _resolve_configured_count(config, max_count) + if rich_output: + console.print("🔖 Fetching bookmarks (%d tweets)...\n" % fetch_count) + start = time.time() + tweets, next_cursor = client.fetch_bookmarks( + fetch_count, cursor=cursor, return_cursor=True, + ) + elapsed = time.time() - start + if rich_output: + console.print("✅ Fetched %d bookmarks in %.1fs\n" % (len(tweets), elapsed)) + + filtered = _apply_filter(tweets, do_filter, config, rich_output=rich_output) + + if output_file: + Path(output_file).write_text(tweets_to_json(filtered), encoding="utf-8") + if rich_output: + console.print("💾 Saved to %s\n" % output_file) + + if compact: + click.echo(tweets_to_compact_json(filtered)) + return + + save_tweet_cache(filtered) + + if _emit_timeline_structured(filtered, next_cursor, as_json=as_json, as_yaml=as_yaml): + return + + print_tweet_table( + filtered, + console, + title="🔖 Bookmarks — %d tweets" % len(filtered), full_text=full_text, ) + _print_show_hint() + console.print() _run_guarded(_run) @@ -509,13 +538,14 @@ def favorites(ctx, max_count, as_json, as_yaml, output_file, do_filter, full_tex @cli.group(name="bookmarks", invoke_without_command=True) @click.option("--max", "-n", "max_count", type=int, default=None, help="Max number of tweets to fetch.") +@click.option("--cursor", type=str, default=None, help="Pagination cursor to continue a previous bookmarks fetch (enables resumable batched fetching).") @structured_output_options @click.option("--output", "-o", "output_file", type=str, default=None, help="Save tweets to JSON file.") @click.option("--filter", "do_filter", is_flag=True, help="Enable score-based filtering.") @click.option("--full-text", is_flag=True, help="Show full tweet text in table output.") @click.pass_context -def bookmarks(ctx, max_count, as_json, as_yaml, output_file, do_filter, full_text): - # type: (Any, Optional[int], bool, bool, Optional[str], bool, bool) -> None +def bookmarks(ctx, max_count, cursor, as_json, as_yaml, output_file, do_filter, full_text): + # type: (Any, Optional[int], Optional[str], bool, bool, Optional[str], bool, bool) -> None """Fetch bookmarked tweets, or manage bookmark folders.""" if ctx.invoked_subcommand is None: _run_bookmarks_command( @@ -526,6 +556,7 @@ def bookmarks(ctx, max_count, as_json, as_yaml, output_file, do_filter, full_tex do_filter, compact=ctx.obj.get("compact", False), full_text=full_text, + cursor=cursor, ) diff --git a/twitter_cli/client.py b/twitter_cli/client.py index 0436c8e..4ffa1d2 100644 --- a/twitter_cli/client.py +++ b/twitter_cli/client.py @@ -177,9 +177,13 @@ def fetch_following_feed(self, count=20, include_promoted=False, cursor=None, re return_cursor=return_cursor, ) - def fetch_bookmarks(self, count=50): - # type: (int) -> List[Tweet] - """Fetch bookmarked tweets.""" + def fetch_bookmarks(self, count=50, cursor=None, return_cursor=False): + # type: (int, Optional[str], bool) -> Any + """Fetch bookmarked tweets. + + When ``return_cursor`` is True, returns a ``(tweets, next_cursor)`` + tuple so callers can paginate by passing ``cursor`` on the next call. + """ def get_instructions(data): # type: (Any) -> Any instructions = _deep_get(data, "data", "bookmark_timeline", "timeline", "instructions") @@ -187,7 +191,13 @@ def get_instructions(data): instructions = _deep_get(data, "data", "bookmark_timeline_v2", "timeline", "instructions") return instructions - return self._fetch_timeline("Bookmarks", count, get_instructions) + return self._fetch_timeline( + "Bookmarks", + count, + get_instructions, + start_cursor=cursor, + return_cursor=return_cursor, + ) def fetch_bookmark_folders(self): # type: () -> List[BookmarkFolder] diff --git a/twitter_cli/parser.py b/twitter_cli/parser.py index 58378be..311e525 100644 --- a/twitter_cli/parser.py +++ b/twitter_cli/parser.py @@ -196,9 +196,15 @@ def _extract_article_media_url_map(article_results): return media_url_map -def _extract_atomic_markdown(block, entity_map): +def _extract_atomic_content(block, entity_map): # type: (Dict[str, Any], Dict[str, Any]) -> List[str] - """Extract embedded markdown/code payloads from atomic Draft.js entities.""" + """Extract content from atomic Draft.js entities. + + Handles: + - MARKDOWN: raw markdown/code blocks + - DIVIDER: horizontal rules (---) + - TWEET: embedded tweet links + """ parts = [] # type: List[str] for entity_range in block.get("entityRanges", []) or []: if not isinstance(entity_range, dict): @@ -207,28 +213,58 @@ def _extract_atomic_markdown(block, entity_map): entity = entity_map.get(str(entity_key)) if entity_key is not None else None if not isinstance(entity, dict): continue - if str(entity.get("type") or "").upper() != "MARKDOWN": - continue - markdown = _deep_get(entity, "data", "markdown") - if isinstance(markdown, str) and markdown.strip(): - parts.append(markdown.strip()) + entity_type = str(entity.get("type") or "").upper() + if entity_type == "MARKDOWN": + markdown = _deep_get(entity, "data", "markdown") + if isinstance(markdown, str) and markdown.strip(): + parts.append(markdown.strip()) + elif entity_type == "DIVIDER": + parts.append("---") + elif entity_type == "TWEET": + tweet_id = _deep_get(entity, "data", "tweetId") + if isinstance(tweet_id, str) and tweet_id: + parts.append("> [Embedded Tweet](https://x.com/i/status/%s)" % tweet_id) return parts def _render_article_text_block(block, entity_map): # type: (Dict[str, Any], Dict[str, Any]) -> str - """Render a Draft.js text block, converting inline hyperlinks to Markdown.""" + """Render a Draft.js text block, converting inline hyperlinks and styles to Markdown. + + Both inlineStyleRanges and entityRanges (links) reference the *original* + text offsets. We collect all replacement operations as (start, end, text) + tuples, sort by offset descending, and apply right-to-left so that earlier + insertions don't shift later offsets. + """ text = block.get("text", "") if not isinstance(text, str) or not text: return "" - entity_ranges = block.get("entityRanges", []) or [] - if not entity_ranges: - return text + # Collect operations as (start, end, replacement) tuples + ops = [] # type: List[Tuple[int, int, str]] - rendered = text - ranges = [] - for entity_range in entity_ranges: + # Inline styles (Bold, Italic, Code, Strikethrough) + style_markers = { + "BOLD": ("**", "**"), "ITALIC": ("*", "*"), + "CODE": ("`", "`"), "STRIKETHROUGH": ("~~", "~~"), + } # type: Dict[str, Tuple[str, str]] + for sr in block.get("inlineStyleRanges", []) or []: + if not isinstance(sr, dict): + continue + style = str(sr.get("style", "")).upper() + offset = sr.get("offset") + length = sr.get("length") + if not isinstance(offset, int) or not isinstance(length, int) or length <= 0: + continue + if offset < 0 or offset + length > len(text): + continue + if style in style_markers: + open_m, close_m = style_markers[style] + segment = text[offset:offset + length] + ops.append((offset, offset + length, "%s%s%s" % (open_m, segment, close_m))) + + # Entity links + for entity_range in block.get("entityRanges", []) or []: if not isinstance(entity_range, dict): continue entity_key = entity_range.get("key") @@ -241,26 +277,31 @@ def _render_article_text_block(block, entity_map): length = entity_range.get("length") if not isinstance(offset, int) or not isinstance(length, int) or length <= 0: continue + if offset < 0 or offset + length > len(text): + continue url = _deep_get(entity, "data", "url") if not isinstance(url, str) or not url.strip(): continue - ranges.append((offset, length, url.strip())) - - for offset, length, url in sorted(ranges, reverse=True): - if offset < 0 or offset + length > len(rendered): - continue - label = rendered[offset:offset + length] + label = text[offset:offset + length] if not label: continue - # Escape markdown special chars: ] in labels and ) in URLs safe_label = label.replace("[", "\\[").replace("]", "\\]") - safe_url = url.replace(")", "%29") - rendered = "%s[%s](%s)%s" % ( - rendered[:offset], - safe_label, - safe_url, - rendered[offset + length:], - ) + safe_url = url.strip().replace(")", "%29") + ops.append((offset, offset + length, "[%s](%s)" % (safe_label, safe_url))) + + if not ops: + return text + + # Sort by start offset descending so right-to-left application preserves offsets. + # For overlapping ranges at the same start, process longer ranges first (larger end). + ops.sort(key=lambda x: (x[0], x[1]), reverse=True) + + rendered = text + for start, end, replacement in ops: + if start < 0 or end > len(rendered): + continue + rendered = "%s%s%s" % (rendered[:start], replacement, rendered[end:]) + return rendered @@ -334,7 +375,7 @@ def _parse_article(tweet_data): for block in blocks: block_type = block.get("type", "unstyled") # type: str if block_type == "atomic": - parts.extend(_extract_atomic_markdown(block, entity_map)) + parts.extend(_extract_atomic_content(block, entity_map)) parts.extend(_extract_article_images(block, entity_map, media_url_map)) ordered_counter = 0 continue