Bump requests from 2.28.2 to 2.31.0

.github/workflows/publish_lambda.yml

@@ -32,10 +32,10 @@ jobs:
     - name: Package (daily, weekly, 5 minutes & dump)
       if: success()
       run: |
-        zip -ur lys_daily.zip lys_daily.py common.py twitter_utils.py
-        zip -ur lys_weekly.zip lys_weekly.py common.py twitter_utils.py
-        zip -ur lys_5minutes.zip lys_5minutes.py common.py twitter_utils.py
-        zip -ur lys_dump.zip lys_dump.py common.py twitter_utils.py
+        zip -ur lys_daily.zip lys_daily.py common.py twitter_utils.py bluesky_utils.py
+        zip -ur lys_weekly.zip lys_weekly.py common.py twitter_utils.py bluesky_utils.py
+        zip -ur lys_5minutes.zip lys_5minutes.py common.py twitter_utils.py bluesky_utils.py
+        zip -ur lys_dump.zip lys_dump.py common.py twitter_utils.py bluesky_utils.py
         cd lib
         zip -ur ../lys_daily.zip *
         zip -ur ../lys_weekly.zip *

bluesky_utils.py

@@ -0,0 +1,152 @@
+import requests
+import os
+import datetime
+
+from bs4 import BeautifulSoup
+from common import get_short_url
+
+
+def get_session():
+    BLUESKY_ACCOUNT_HANDLE=os.environ['BLUESKY_ACCOUNT_HANDLE']
+    BLUESKY_ACCOUNT_APP_PASSWORD=os.environ['BLUESKY_ACCOUNT_APP_PASSWORD']
+
+    # json with accessJwt and refreshJwt
+    response = requests.post(
+        "https://bsky.social/xrpc/com.atproto.server.createSession",
+        json={"identifier": BLUESKY_ACCOUNT_HANDLE, "password": BLUESKY_ACCOUNT_APP_PASSWORD},
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def get_timestamp():
+    return datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def parse_url_links(post_body, link_descriptions):
+    spans = []
+    last_appearance_of = {}
+    for ld in link_descriptions:
+        previous_appearance = (last_appearance_of.get(ld['text']) + 1) if ld['text'] in last_appearance_of else 0
+        start = post_body.find(ld['text'], previous_appearance)
+        # keep track of the last appearance of a link text to handle posts where a short link appears multiple times
+        last_appearance_of[ld['text']] = start
+        # we have to "count in utf-8" to make sure emojis are counted for the right amount of characters
+        start = len(post_body[:start].encode("utf-8"))
+        end = start + len(ld['text'].encode("utf-8"))
+        spans.append({
+            "start": start,
+            "end": end,
+            "url": ld['url'],
+        })
+    return spans
+
+
+# stolen from https://atproto.com/blog/create-post
+def get_facets(link_spans, menion_spans=[]):
+    facets = []
+    for m in menion_spans:
+        resp = requests.get(
+            "https://bsky.social/xrpc/com.atproto.identity.resolveHandle",
+            params={"handle": m["handle"]},
+        )
+        # If the handle can't be resolved, just skip it!
+        # It will be rendered as text in the post instead of a link
+        if resp.status_code == 400:
+            continue
+        did = resp.json()["did"]
+        facets.append({
+            "index": {
+                "byteStart": m["start"],
+                "byteEnd": m["end"],
+            },
+            "features": [{"$type": "app.bsky.richtext.facet#mention", "did": did}],
+        })
+    for u in link_spans:
+        facets.append({
+            "index": {
+                "byteStart": u["start"],
+                "byteEnd": u["end"],
+            },
+            "features": [
+                {
+                    "$type": "app.bsky.richtext.facet#link",
+                    "uri": u["url"],
+                }
+            ],
+        })
+    return facets
+
+
+def get_facets_for_event_links_in_string(events, string):
+    link_descriptions = []
+
+    for event in events:
+        # extract links to insert in the post alongside their associated string
+        for link in event['watchLinks']:
+            link_string = get_short_url(link['link'])
+            link_descriptions.append({'text': link_string, 'url': link['link']})
+            if 'accountRequired' in link and link['accountRequired']:
+                account_help_link = "https://lyseurovision.github.io/help.html#account-" + event['country']
+                link_descriptions.append({'text': get_short_url(account_help_link), 'url': account_help_link})
+
+    spans = parse_url_links(string, link_descriptions)
+    facets = get_facets(spans)
+    return facets
+
+
+def generate_url_card(url):
+    # the required fields for every embed card
+    card = {
+        "uri": url,
+        "title": "",
+        "description": "",
+    }
+
+    # fetch the HTML
+    resp = requests.get(url)
+    resp.raise_for_status()
+    soup = BeautifulSoup(resp.text, "html.parser")
+
+    # parse out the "og:title" and "og:description" HTML meta tags
+    title_tag = soup.find("meta", property="og:title")
+    if title_tag:
+        card["title"] = title_tag["content"]
+    description_tag = soup.find("meta", property="og:description")
+    if description_tag:
+        card["description"] = description_tag["content"]
+
+    return {
+        "$type": "app.bsky.embed.external",
+        "external": card,
+    }
+
+
+def generate_post(post_string, facets=[], include_card=False, url_for_card=None):
+    post = {
+        "$type": "app.bsky.feed.post",
+        "text": post_string,
+        "facets": facets,
+        "createdAt": get_timestamp(),
+        "langs": ["en-US"]
+    }
+
+    if include_card:
+        # generate social card for the recommended link
+        post["embed"] = generate_url_card(url_for_card)
+
+    return post
+
+
+def publish_post(session, post):
+    resp = requests.post(
+        "https://bsky.social/xrpc/com.atproto.repo.createRecord",
+        headers={"Authorization": "Bearer " + session["accessJwt"]},
+        json={
+            "repo": session["did"],
+            "collection": "app.bsky.feed.post",
+            "record": post
+        }
+    )
+    resp.raise_for_status()
+    return resp.json()

common.py

@@ -69,6 +69,7 @@
 CLOCK_EMOJI = "\U0001F553"
 TV_EMOJI = "\U0001F4FA"
 ALERT_EMOJI = "\U0001F6A8"
+DOWN_ARROW_EMOJI = "\U00002B07\U0000FE0F"
 
 BLUESKY="bluesky"
 TWITTER="twitter"
@@ -109,6 +110,15 @@ def get_watch_link_string(watch_link, country, shorten_urls=False):
     return watch_link_string
 
 
+def get_first_watch_link(event, live=True):
+    links = event['watchLinks']
+    if live:
+        links = list(filter(lambda l: l['live'], links))
+    if len(links) == 0:
+        return None
+    return links[0]['link']
+
+
 def get_current_season_range_for_date(date):
     if date.month > 8:
         season_start = datetime.datetime(date.year, 9, 1, 0, 0, 0)

lys_5minutes.py

@@ -7,42 +7,44 @@
 except ImportError:
     pass
 
-from common import DATETIME_CET_FORMAT, flag_emojis, ALERT_EMOJI, BLUESKY, TWITTER, get_watch_link_string
+from common import DATETIME_CET_FORMAT, flag_emojis, ALERT_EMOJI, DOWN_ARROW_EMOJI, BLUESKY, TWITTER, get_watch_link_string, get_short_url
 from twitter_utils import create_tweepy_client, send_tweet
+from bluesky_utils import get_session, get_facets_for_event_links_in_string, generate_post, publish_post
+
+
+def generate_event_string(event, shorten_urls=False):
+    flag = (flag_emojis[event['country']] + " ") if event['country'] in flag_emojis else ""
+    watch_link_string = ""
+    try:
+        watch_links = event['watchLinks']
+        # tweeting only links that can be watched live
+        for watch_link in list(filter(lambda wl: 'live' in wl and wl['live'], watch_links)):
+            if watch_link_string != "":
+                watch_link_string += " OR "
+            if "link" in watch_link:
+                watch_link_string += get_watch_link_string(watch_link, event['country'], shorten_urls)
+    except KeyError:
+        pass
+    if watch_link_string == "":    
+        watch_link_string = "(no watch link found)"
+    else:
+        watch_link_string = "(" + watch_link_string + ")"
+    event_string = "\n{}{} - {} {}".format(flag, event['name'], event['stage'], watch_link_string)
+    return event_string
 
 
 def generate_twitter_event_strings(events):
     event_strings = []
     for event in events:
-        flag = (flag_emojis[event['country']] + " ") if event['country'] in flag_emojis else ""
-        watch_link_string = ""
-        try:
-            watch_links = event['watchLinks']
-            # tweeting only links that can be watched live
-            for watch_link in list(filter(lambda wl: 'live' in wl and wl['live'], watch_links)):
-                if watch_link_string != "":
-                    watch_link_string += " OR "
-                if "link" in watch_link:
-                    watch_link_string += watch_link['link'] + ((" (" + watch_link['comment'] + ")") if "comment" in watch_link and watch_link['comment'] != "" and watch_link['comment'] != "Recommended link" else "")
-                additional_comments = []
-                if "geoblocked" in watch_link and watch_link['geoblocked']:
-                    additional_comments.append("geoblocked")
-                if "accountRequired" in watch_link and watch_link['accountRequired']:
-                    additional_comments.append("account required: https://lyseurovision.github.io/help.html#account-" + event['country'])
-                if len(additional_comments) > 0:
-                    watch_link_string += " (" + ", ".join(additional_comments) + ")"
-        except KeyError:
-            pass
-        if watch_link_string == "":    
-            watch_link_string = "(no watch link found)"
-        else:
-            watch_link_string = "(" + watch_link_string + ")"
-        event_string = "\n{}{} - {} {}".format(flag, event['name'], event['stage'], watch_link_string)
-        event_strings.append(event_string)
+        event_strings.append(generate_event_string(event))
     return event_strings
 
 
-def build_tweets(event_strings):
+def generate_bluesky_event_string(event):
+    return generate_event_string(event, shorten_urls=True)
+
+
+def build_twitter_posts(event_strings):
     tweets = []
     tweet = ALERT_EMOJI + " 5 MINUTES REMINDER!"
     for string in event_strings:
@@ -56,17 +58,68 @@ def build_tweets(event_strings):
     return tweets
 
 
+def build_bluesky_posts(events):
+    post_header = ALERT_EMOJI + " 5 MINUTES REMINDER!"
+    posts = []
+    posts_events = []
+    is_thread=False
+    tmp_post = ""
+    post_events = []
+    for idx, event in enumerate(events):
+        event_string = generate_bluesky_event_string(event)
+        # bluesky character limit = 300; leaving room for the header
+        if len(tmp_post+event_string) < 260:
+            # add the event string to the current post
+            tmp_post += "\n---------" + event_string
+            # flag the index of the event in the list as part of the current post
+            post_events.append(event)
+        else:
+            # we're ready to save the first post
+            # add the header
+            post = post_header
+            # if we're here, we're about to create/continue a thread, because the next event doesn't fit in the current post
+            if not is_thread:
+                # if we haven't started a thread yet, this means this is the first post of the thread
+                post += " (thread " + DOWN_ARROW_EMOJI + ")"
+            else:
+                # otherwise, we're just adding another post to the thread
+                post += " (cont.)"
+            is_thread = True
+            # the post is complete, we save it and save the indices of the events that are part of it
+            post += tmp_post
+            posts.append(post)
+            posts_events.append(post_events)
+            # we reset the tmp post, and open it with the next event (the one we couldn't add to the previous post)
+            tmp_post = "\n---------" + event_string
+            post_events = [event]
+    # if we're out of events but still have event strings we haven't saved to a post, we do it now
+    if len(tmp_post) > 0:
+        post = post_header
+        if is_thread:
+            post += " (cont.)"
+        post += tmp_post
+        posts.append(post)
+        posts_events.append(post_events)
+
+    # finally, we return the complete list of posts, alongside the indices of the events that compose it
+    return (posts, posts_events)
+
+
 def generate_twitter_thread(events):
     event_strings = generate_twitter_event_strings(events)
-    return build_tweets(event_strings)
+    return build_twitter_posts(event_strings)
 
 
 def generate_bluesky_thread(events):
-    # TODO identify which post of the thread includes which event (to add right facets in right post)
-    # TODO return a list of list, that includes for each post a list of the indexes of the links that are present in the post?
-    # ex: posts = ["Post0=link 0, link 1", "Post1=link 2"], links_idx = [[0, 1], [2]]
-    # => we can then iterate on each list of links_idx and generate the facets for each post individually
-    return []
+    (post_bodies, post_events) = build_bluesky_posts(events)
+    posts = []
+    for idx, body in enumerate(post_bodies):
+        # get the events included in the post
+        events = post_events[idx]
+        # extract link facets from post
+        facets = get_facets_for_event_links_in_string(events, body)
+        posts.append(generate_post(body, facets, include_card=False))
+    return posts
 
 
 def generate_thread(events, target):
@@ -105,6 +158,22 @@ def post_to_twitter(tweets, is_test=True):
 
 def post_to_bluesky(posts, is_test=True):
     output = []
+    session = get_session()
+
+    # publish the first post of the thread
+    if not is_test:
+        parent = root = publish_post(session, posts[0])
+    output.append(posts[0]['text'])
+
+    for post in posts[1:]:
+        if not is_test:
+            post['reply'] = {
+                "root": root,
+                "parent": parent
+            }
+            parent = publish_post(session, post)
+        output.append(post['text'])
+
     return output