Refactor, update module versions

.gitignore

@@ -159,3 +159,5 @@ dmypy.json
 *.code-workspace
 
 # End of https://www.toptal.com/developers/gitignore/api/python,vscode
+
+.DS_Store

main.py

@@ -3,6 +3,7 @@
 import re
 import sys
 import datetime
+import pytz
 import urllib.parse as urlparse
 from datetime import datetime as dt
 from typing import Final, List, Dict, Tuple, Union
@@ -18,6 +19,8 @@
 CREDENTIAL_INFO: Dict[str, str] = {}
 CALENDAR_TYPE: Final[str] = 'ABC' if os.environ.get('CALENDAR_TYPE') == 'ABC' else 'ALL'
 
+ABC_PATTERN = re.compile(r"\/abc\d{3}$")
+
 if os.environ.get('ENV') == 'local':
     # ローカルでテスト
     # `ENV=local python3 main.py` みたいに使う
@@ -59,7 +62,7 @@
 
 @dataclass
 class TimeWithStrTimeZone:
-    time: datetime.datetime
+    time: datetime.datetime # Timezone aware
     time_zone: str = 'Japan'
 
     def get_as_obj(self):
@@ -81,10 +84,10 @@ def __eq__(self, other: "TimeWithStrTimeZone") -> bool:
 @dataclass
 class CalendarEvent:
     summary: str
-    created_at: datetime.datetime
-    updated_at: datetime.datetime
-    start_at: InitVar[datetime.datetime]
-    end_at: InitVar[datetime.datetime]
+    created_at: datetime.datetime # UTC
+    updated_at: datetime.datetime # UTC
+    start_at: InitVar[datetime.datetime] # Timezone aware
+    end_at: InitVar[datetime.datetime] # Timezone aware
     url: str
     start_at_with_time_zone: TimeWithStrTimeZone = field(init=False)
     end_at_with_timw_zone: TimeWithStrTimeZone = field(init=False)
@@ -99,14 +102,13 @@ def get_description(self) -> str:
         """
         :returns: Description formatted for Google Calendar
         """
-        created_at_jst_str: Final[str] = utc_to_jst_str(self.created_at)
-        updated_at_jst_str: Final[str] = utc_to_jst_str(self.updated_at)
+        created_at_jst_str: Final[str] = get_jst_str(self.created_at)
+        updated_at_jst_str: Final[str] = get_jst_str(self.updated_at)
         return f"created at: {created_at_jst_str}\nlast modified at: {updated_at_jst_str}"
 
     def get_as_obj(self) -> dict:
         """
-        :returns: Dictionary formatted for Google Calendar API
-        以下のような形で返す
+        :returns: Dictionary formatted for Google Calendar API like below
         {
             'summary': 'ABC001',
             'location': 'https://atcoder.jp/contests/abc001',
@@ -130,12 +132,19 @@ def get_as_obj(self) -> dict:
             'end': self.end_at_with_timw_zone.get_as_obj()
         }
 
-    def is_abc(self) -> bool:
-        m = re.search(r"\/abc\d{3}$", self.url)
-        if m is None:
-            return False
+    def starts_before(self, time: datetime.datetime) -> bool:
+        """Check if the event starts before the time
 
-        return True
+        Args:
+            time (datetime.datetime): Timezone aware time
+
+        Returns:
+            bool: Whether or not the event starts before the time
+        """
+        return self.start_at_with_time_zone.time < time
+
+    def is_abc(self) -> bool:
+        return ABC_PATTERN.search(self.url) is not None
 
     @classmethod
     def parse_event(cls, event_item_obj: dict) -> "CalendarEvent":
@@ -155,28 +164,29 @@ def parse_event(cls, event_item_obj: dict) -> "CalendarEvent":
 
     @classmethod
     def parse_text_obj_to_calendarevent(
-            cls, name_obj, start_datetime_obj, duration_obj, now: datetime.datetime
+            cls, name_obj, start_datetime_obj, duration_obj, utc_now: datetime.datetime
         ) -> "CalendarEvent":
         """
         :param name_obj: Object, which contains name and url of the contest, obtained from html
         :param start_datetime_obj: Object, which contains start time, obtained from html
         :param duration_obj:
             Object, which contains duration time (1:30 as one hour and 30 minutes),
             obtained from html
-        :param now: The time this program started
+        :param utc_now: The time this program started
         :returns: Parsed Calendar Event
         """
         contest_title: str = name_obj.text
         contest_url: str = urlparse.urljoin(ATCODER_BASE_URL, name_obj.attrs['href'])
-        start_at: datetime.datetime = dt.strptime(start_datetime_obj.text, '%Y-%m-%d %H:%M:%S+0900')
+        # Timezone aware datetime object (%z: timezone offset)
+        start_at: datetime.datetime = dt.strptime(start_datetime_obj.text, '%Y-%m-%d %H:%M:%S%z')
         contest_hours, contest_minutes = map(int, duration_obj.text.split(':'))
         contest_duration: datetime.timedelta = datetime.timedelta(hours=contest_hours,
                                                                   minutes=contest_minutes)
         end_at: datetime.datetime = start_at + contest_duration
         return CalendarEvent(
             summary=contest_title,
-            created_at=now,
-            updated_at=now,
+            created_at=utc_now,
+            updated_at=utc_now,
             start_at=start_at,
             end_at=end_at,
             url=contest_url
@@ -206,14 +216,19 @@ def update_for_diff(cls,
         return True
 
 
-def utc_to_jst_str(time: datetime.datetime) -> str:
-    """
-    :returns: Time converted to JST as string
+def get_jst_str(time: datetime.datetime) -> str:
+    """Get JST formatted string from timezone aware datetime object
+
+    Args:
+        time: Timezone aware datetime object
+
+    Returns:
+        str: JST formatted string
     """
-    return f"{(time + datetime.timedelta(hours=9)).strftime('%Y/%m/%d %H:%M:%S')} JST"
+    return time.astimezone(pytz.timezone("Asia/Tokyo")).strftime('%Y/%m/%d %H:%M:%S JST')
 
 
-def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]:
+def get_atcoder_schedule(utc_now: datetime.datetime) -> List[CalendarEvent]:
     res = requests.get(urlparse.urljoin(ATCODER_BASE_URL, "contests/?lang=ja"), timeout=10)
     res.raise_for_status()
     soup = bs4.BeautifulSoup(res.content, 'html.parser')
@@ -229,14 +244,22 @@ def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]:
         sys.exit(1)
 
     event_list = [
-        CalendarEvent.parse_text_obj_to_calendarevent(nobj, sobj, dobj, now)
+        CalendarEvent.parse_text_obj_to_calendarevent(nobj, sobj, dobj, utc_now)
         for nobj, sobj, dobj in zip(name_objs, start_datetime_objs, duration_objs)
     ]
 
     return event_list
 
 # pylint: disable=invalid-name
 def parse_datetime(t: str) -> datetime.datetime:
+    """Get timezone aware datetime object from string
+
+    Args:
+        t (str): Timestamp in string (with timezone offset %z)
+
+    Returns:
+        datetime.datetime: Timezone aware datetime object
+    """
     try:
         return datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S%z")
     except ValueError:
@@ -252,8 +275,8 @@ def get_registered_events(
         # pylint: disable=no-member
         events = API_SERVICE.events().list(
             calendarId=CALENDAR_ID,
-            timeMin=f"{time_from.isoformat()}Z",
-            timeMax=f"{time_to.isoformat()}Z",
+            timeMin=f"{time_from.isoformat(timespec='seconds')}",
+            timeMax=f"{time_to.isoformat(timespec='seconds')}",
             pageToken=page_token
         ).execute()
         registered_events += [
@@ -266,39 +289,20 @@ def get_registered_events(
     return registered_events
 
 def get_registered_events_dict(
-        time_from: datetime.datetime, time_to: datetime.datetime
-    ) -> Tuple[Dict[str, CalendarEvent], Dict[str, CalendarEvent]]:
-    """Returns registered event dictionary
+        registered_events: List[CalendarEvent]
+    ) -> Tuple[Dict[str, int], Dict[str, int]]:
+    """Returns dictionary that maps from event url/summary to event index
     Args:
-        time_from (datetime.datetime): This function looks for events after this
-        time_to (datetime.datetime): This function looks for events after this
+        registered_events (List[CalendarEvent]): List of registered events
 
     Returns:
-        Tuple[Dict[str, CalendarEvent], Dict[str, CalendarEvent]]:
-            first Dict -> key: event summary, value: event,
-            second Dict -> key: event url, value: event
+        Tuple[Dict[str, int], Dict[str, int]]:
+            first Dict -> key: event summary, value: index of registered event,
+            second Dict -> key: event url, value: index of registered event
     """
 
-    summary_to_registered: Dict[str, CalendarEvent] = {}
-    url_to_registered: Dict[str, CalendarEvent] = {}
-
-    page_token = None
-    while True:
-        # pylint: disable=no-member
-        events = API_SERVICE.events().list(
-            calendarId=CALENDAR_ID,
-            timeMin=f"{time_from.isoformat()}Z",
-            timeMax=f"{time_to.isoformat()}Z",
-            pageToken=page_token
-        ).execute()
-        for event_item_obj in events["items"]:
-            event = CalendarEvent.parse_event(event_item_obj)
-            summary_to_registered[event.summary] = event
-            url_to_registered[event.url] = event
-
-        page_token = events.get('nextPageToken')
-        if not page_token:
-            break
+    summary_to_registered: Dict[str, int] = {event.summary: i for i, event in enumerate(registered_events)}
+    url_to_registered: Dict[str, int] = {event.url: i for i, event in enumerate(registered_events)}
 
     return summary_to_registered, url_to_registered
 
@@ -347,40 +351,77 @@ def add_event(event: CalendarEvent, batch: Union[BatchHttpRequest, None] = None)
     # pylint: disable=no-member
     API_SERVICE.events().insert(calendarId=CALENDAR_ID, body=event.get_as_obj()).execute()
 
-# These are needed in Cloud Functions
+# TODO(k1832): WIP. Refactor this.
+def delete_all_within_three_years():
+    utc_now = datetime.datetime.now(datetime.UTC)
+    three_years_later = utc_now + datetime.timedelta(days=365*3)
+    batch = API_SERVICE.new_batch_http_request()
+
+    offset = 0
+    events_to_delete = get_registered_events(utc_now, three_years_later)
+
+    ONE_BATCH_LIMIT = 995
+
+    while offset < len(events_to_delete):
+        end = min(len(events_to_delete), offset + ONE_BATCH_LIMIT)
+        for event in events_to_delete[offset:end]:
+            batch.add(
+                API_SERVICE.events().delete(
+                    calendarId=CALENDAR_ID,
+                    eventId=event.id
+                )
+            )
+        batch.execute()
+        offset = end
+
+# `data` and `context` are passed by Cloud Functions and not used.
 # pylint: disable=unused-argument
 def main(data, context):
-    now = datetime.datetime.utcnow()
-    upcoming_contests = get_atcoder_schedule(now)
+    utc_now = datetime.datetime.now(datetime.UTC)
+    upcoming_contests: List[CalendarEvent] = get_atcoder_schedule(utc_now)
     print(f"{len(upcoming_contests)} contests have been retrieved.")
-    eight_week_later = now + datetime.timedelta(weeks=8)
 
     if not upcoming_contests:
         print("There is no upcoming contests.")
         sys.exit()
 
-    updated_count = 0
-    inserted_count = 0
+    # Google Calendar events: From: now, To: 2 years later
+    # AtCoder events: From: now, To: Unlimited (as long as it's on the first page?)
+
+    # Therefore, we need to ignore AtCoder events that are older than
+    # certain time (365 days for now) to avoid the duplicate events
+    atcoder_event_ignore_threashold = utc_now + datetime.timedelta(days=365)
+    two_year_later = utc_now + datetime.timedelta(days=365*2) # 2 years
+
+
+    # Get registered events within 2 years from Google Calendar
+    registered_events: List[CalendarEvent] = get_registered_events(utc_now, two_year_later)
+    summary_to_registered, url_to_registered = get_registered_events_dict(registered_events)
 
-    summary_to_registered, url_to_registered = get_registered_events_dict(now, eight_week_later)
 
     # pylint: disable=no-member
     batch = API_SERVICE.new_batch_http_request()
 
+    updated_count = 0
+    inserted_count = 0
     for upcoming in upcoming_contests:
+        if not upcoming.starts_before(atcoder_event_ignore_threashold):
+            continue
         if CALENDAR_TYPE == 'ABC' and not upcoming.is_abc():
             continue
 
         # 2 contests are the same contest if they have either same summary (title) or url
         if upcoming.summary in summary_to_registered:
-            registered = summary_to_registered[upcoming.summary]
+            index = summary_to_registered[upcoming.summary]
+            registered = registered_events[index]
             if CalendarEvent.update_for_diff(registered, upcoming, batch):
                 updated_count += 1
 
             continue
 
         if upcoming.url in url_to_registered:
-            registered = url_to_registered[upcoming.url]
+            index = url_to_registered[upcoming.url]
+            registered = registered_events[index]
             if CalendarEvent.update_for_diff(registered, upcoming, batch):
                 updated_count += 1
 

requirements.txt

@@ -1,5 +1,20 @@
-beautifulsoup4==4.11.1
-google_api_python_client==2.64.0
-protobuf==4.21.7
-requests==2.28.1
-pylint==2.15.4
+beautifulsoup4==4.12.2
+cachetools==5.3.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+google-api-core==2.14.0
+google-api-python-client==2.109.0
+google-auth==2.24.0
+google-auth-httplib2==0.1.1
+googleapis-common-protos==1.61.0
+httplib2==0.22.0
+idna==3.6
+protobuf==4.25.1
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pyparsing==3.1.1
+requests==2.31.0
+rsa==4.9
+soupsieve==2.5
+uritemplate==4.1.1
+urllib3==2.1.0