From b1d0a10d577e567eb5a3bf30225acfce166ca412 Mon Sep 17 00:00:00 2001 From: Keita Date: Sun, 11 Jun 2023 16:00:18 +0800 Subject: [PATCH 1/9] Update Python modules --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8a5f6a..bbee869 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -beautifulsoup4==4.11.1 -google_api_python_client==2.64.0 -protobuf==4.21.7 -requests==2.28.1 -pylint==2.15.4 +beautifulsoup4==4.12.2 +google_api_python_client==2.88.0 +protobuf==4.23.2 +requests==2.31.0 +pylint==2.17.4 From 8ebab175f8982be855a55a8b2e7dbd60ff0d8856 Mon Sep 17 00:00:00 2001 From: Keita Date: Sun, 11 Jun 2023 16:01:52 +0800 Subject: [PATCH 2/9] Change data structure to save memory --- main.py | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/main.py b/main.py index 8db89dc..4d2ba4d 100644 --- a/main.py +++ b/main.py @@ -266,39 +266,20 @@ def get_registered_events( return registered_events def get_registered_events_dict( - time_from: datetime.datetime, time_to: datetime.datetime - ) -> Tuple[Dict[str, CalendarEvent], Dict[str, CalendarEvent]]: - """Returns registered event dictionary + registered_events: List[CalendarEvent] + ) -> Tuple[Dict[str, int], Dict[str, int]]: + """Returns dictionary that maps from event url/summary to event index Args: - time_from (datetime.datetime): This function looks for events after this - time_to (datetime.datetime): This function looks for events after this + registered_events (List[CalendarEvent]): List of registered events Returns: - Tuple[Dict[str, CalendarEvent], Dict[str, CalendarEvent]]: - first Dict -> key: event summary, value: event, - second Dict -> key: event url, value: event + Tuple[Dict[str, int], Dict[str, int]]: + first Dict -> key: event summary, value: index of registered event, + second Dict -> key: event url, value: index of registered event """ - summary_to_registered: Dict[str, CalendarEvent] = {} - url_to_registered: Dict[str, CalendarEvent] = {} - - page_token = None - while True: - # pylint: disable=no-member - events = API_SERVICE.events().list( - calendarId=CALENDAR_ID, - timeMin=f"{time_from.isoformat()}Z", - timeMax=f"{time_to.isoformat()}Z", - pageToken=page_token - ).execute() - for event_item_obj in events["items"]: - event = CalendarEvent.parse_event(event_item_obj) - summary_to_registered[event.summary] = event - url_to_registered[event.url] = event - - page_token = events.get('nextPageToken') - if not page_token: - break + summary_to_registered: Dict[str, int] = {event.summary: i for i, event in enumerate(registered_events)} + url_to_registered: Dict[str, int] = {event.url: i for i, event in enumerate(registered_events)} return summary_to_registered, url_to_registered @@ -351,7 +332,7 @@ def add_event(event: CalendarEvent, batch: Union[BatchHttpRequest, None] = None) # pylint: disable=unused-argument def main(data, context): now = datetime.datetime.utcnow() - upcoming_contests = get_atcoder_schedule(now) + upcoming_contests: List[CalendarEvent] = get_atcoder_schedule(now) print(f"{len(upcoming_contests)} contests have been retrieved.") eight_week_later = now + datetime.timedelta(weeks=8) @@ -362,7 +343,8 @@ def main(data, context): updated_count = 0 inserted_count = 0 - summary_to_registered, url_to_registered = get_registered_events_dict(now, eight_week_later) + registered_events: List[CalendarEvent] = get_registered_events(now, eight_week_later) + summary_to_registered, url_to_registered = get_registered_events_dict(registered_events) # pylint: disable=no-member batch = API_SERVICE.new_batch_http_request() @@ -373,14 +355,16 @@ def main(data, context): # 2 contests are the same contest if they have either same summary (title) or url if upcoming.summary in summary_to_registered: - registered = summary_to_registered[upcoming.summary] + index = summary_to_registered[upcoming.summary] + registered = registered_events[index] if CalendarEvent.update_for_diff(registered, upcoming, batch): updated_count += 1 continue if upcoming.url in url_to_registered: - registered = url_to_registered[upcoming.url] + index = url_to_registered[upcoming.url] + registered = registered_events[index] if CalendarEvent.update_for_diff(registered, upcoming, batch): updated_count += 1 From 4492a37547ee178c2ddada4506ba7eb7887de67c Mon Sep 17 00:00:00 2001 From: Keita Date: Sun, 11 Jun 2023 16:03:07 +0800 Subject: [PATCH 3/9] Compile regex for efficiency --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 4d2ba4d..cfc927d 100644 --- a/main.py +++ b/main.py @@ -18,6 +18,8 @@ CREDENTIAL_INFO: Dict[str, str] = {} CALENDAR_TYPE: Final[str] = 'ABC' if os.environ.get('CALENDAR_TYPE') == 'ABC' else 'ALL' +ABC_PATTERN = re.compile(r"\/abc\d{3}$") + if os.environ.get('ENV') == 'local': # ローカルでテスト # `ENV=local python3 main.py` みたいに使う @@ -131,7 +133,7 @@ def get_as_obj(self) -> dict: } def is_abc(self) -> bool: - m = re.search(r"\/abc\d{3}$", self.url) + m = ABC_PATTERN.search(self.url) if m is None: return False From d1ce0a97f1036ba40b4c853656645703a2770e28 Mon Sep 17 00:00:00 2001 From: Keita Date: Sat, 15 Jul 2023 23:41:54 +0800 Subject: [PATCH 4/9] [WIP] Add "delete all" --- main.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/main.py b/main.py index cfc927d..b696fa1 100644 --- a/main.py +++ b/main.py @@ -330,6 +330,29 @@ def add_event(event: CalendarEvent, batch: Union[BatchHttpRequest, None] = None) # pylint: disable=no-member API_SERVICE.events().insert(calendarId=CALENDAR_ID, body=event.get_as_obj()).execute() +# WIP +def delete_all_events(): + now = datetime.datetime.utcnow() + eight_week_later = now + datetime.timedelta(weeks=8) + batch = API_SERVICE.new_batch_http_request() + + offset = 0 + events_to_delete = get_registered_events(now, eight_week_later) + + ONE_BATCH_LIMIT = 995 + + while offset < len(events_to_delete): + end = min(len(events_to_delete), offset + ONE_BATCH_LIMIT) + for event in events_to_delete[offset:end]: + batch.add( + API_SERVICE.events().delete( + calendarId=CALENDAR_ID, + eventId=event.id + ) + ) + batch.execute() + offset = end + # These are needed in Cloud Functions # pylint: disable=unused-argument def main(data, context): From 45301edeb149d64fbcb3f91e09fdf7c850db868c Mon Sep 17 00:00:00 2001 From: Keita Date: Sat, 30 Sep 2023 13:32:51 +0800 Subject: [PATCH 5/9] Refactor --- main.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/main.py b/main.py index b696fa1..c36ec05 100644 --- a/main.py +++ b/main.py @@ -133,11 +133,7 @@ def get_as_obj(self) -> dict: } def is_abc(self) -> bool: - m = ABC_PATTERN.search(self.url) - if m is None: - return False - - return True + return ABC_PATTERN.search(self.url) is not None @classmethod def parse_event(cls, event_item_obj: dict) -> "CalendarEvent": From 9d207d9f11585e597d97f454529d264dc875ae6c Mon Sep 17 00:00:00 2001 From: Keita Date: Sat, 30 Sep 2023 13:34:48 +0800 Subject: [PATCH 6/9] Add TODO comment to WIP function --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index c36ec05..026730e 100644 --- a/main.py +++ b/main.py @@ -326,7 +326,7 @@ def add_event(event: CalendarEvent, batch: Union[BatchHttpRequest, None] = None) # pylint: disable=no-member API_SERVICE.events().insert(calendarId=CALENDAR_ID, body=event.get_as_obj()).execute() -# WIP +# TODO(k1832): WIP. Refactor this. def delete_all_events(): now = datetime.datetime.utcnow() eight_week_later = now + datetime.timedelta(weeks=8) From 02bb4d7e182774bfd21f6f253fd99b2877f45769 Mon Sep 17 00:00:00 2001 From: Keita Date: Sun, 3 Mar 2024 16:23:25 +0800 Subject: [PATCH 7/9] Update dependencies and gitignore --- .gitignore | 2 ++ requirements.txt | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5918636..11c182f 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,5 @@ dmypy.json *.code-workspace # End of https://www.toptal.com/developers/gitignore/api/python,vscode + +.DS_Store diff --git a/requirements.txt b/requirements.txt index bbee869..ee1306e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,20 @@ beautifulsoup4==4.12.2 -google_api_python_client==2.88.0 -protobuf==4.23.2 +cachetools==5.3.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +google-api-core==2.14.0 +google-api-python-client==2.109.0 +google-auth==2.24.0 +google-auth-httplib2==0.1.1 +googleapis-common-protos==1.61.0 +httplib2==0.22.0 +idna==3.6 +protobuf==4.25.1 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pyparsing==3.1.1 requests==2.31.0 -pylint==2.17.4 +rsa==4.9 +soupsieve==2.5 +uritemplate==4.1.1 +urllib3==2.1.0 From 7ea1da586da9f9ff19f38804313109257446aa1a Mon Sep 17 00:00:00 2001 From: Keita Date: Sun, 14 Apr 2024 17:15:36 +0800 Subject: [PATCH 8/9] Fix event duplication bug Previously, event duplication occurred for contests starting after 8 weeks due to limited calendar event retrieval. This fix extends the retrieval range to 2 years and ignores AtCoder events older than 1 year. --- main.py | 81 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/main.py b/main.py index 026730e..478dde9 100644 --- a/main.py +++ b/main.py @@ -61,7 +61,7 @@ @dataclass class TimeWithStrTimeZone: - time: datetime.datetime + time: datetime.datetime # Timezone aware time_zone: str = 'Japan' def get_as_obj(self): @@ -83,10 +83,10 @@ def __eq__(self, other: "TimeWithStrTimeZone") -> bool: @dataclass class CalendarEvent: summary: str - created_at: datetime.datetime - updated_at: datetime.datetime - start_at: InitVar[datetime.datetime] - end_at: InitVar[datetime.datetime] + created_at: datetime.datetime # UTC + updated_at: datetime.datetime # UTC + start_at: InitVar[datetime.datetime] # Timezone aware + end_at: InitVar[datetime.datetime] # Timezone aware url: str start_at_with_time_zone: TimeWithStrTimeZone = field(init=False) end_at_with_timw_zone: TimeWithStrTimeZone = field(init=False) @@ -132,6 +132,17 @@ def get_as_obj(self) -> dict: 'end': self.end_at_with_timw_zone.get_as_obj() } + def starts_before(self, time: datetime.datetime) -> bool: + """Check if the event starts before the time + + Args: + time (datetime.datetime): Timezone aware time + + Returns: + bool: Whether or not the event starts before the time + """ + return self.start_at_with_time_zone.time < time + def is_abc(self) -> bool: return ABC_PATTERN.search(self.url) is not None @@ -153,7 +164,7 @@ def parse_event(cls, event_item_obj: dict) -> "CalendarEvent": @classmethod def parse_text_obj_to_calendarevent( - cls, name_obj, start_datetime_obj, duration_obj, now: datetime.datetime + cls, name_obj, start_datetime_obj, duration_obj, utc_now: datetime.datetime ) -> "CalendarEvent": """ :param name_obj: Object, which contains name and url of the contest, obtained from html @@ -161,20 +172,21 @@ def parse_text_obj_to_calendarevent( :param duration_obj: Object, which contains duration time (1:30 as one hour and 30 minutes), obtained from html - :param now: The time this program started + :param utc_now: The time this program started :returns: Parsed Calendar Event """ contest_title: str = name_obj.text contest_url: str = urlparse.urljoin(ATCODER_BASE_URL, name_obj.attrs['href']) - start_at: datetime.datetime = dt.strptime(start_datetime_obj.text, '%Y-%m-%d %H:%M:%S+0900') + # Timezone aware datetime object (%z: timezone offset) + start_at: datetime.datetime = dt.strptime(start_datetime_obj.text, '%Y-%m-%d %H:%M:%S%z') contest_hours, contest_minutes = map(int, duration_obj.text.split(':')) contest_duration: datetime.timedelta = datetime.timedelta(hours=contest_hours, minutes=contest_minutes) end_at: datetime.datetime = start_at + contest_duration return CalendarEvent( summary=contest_title, - created_at=now, - updated_at=now, + created_at=utc_now, + updated_at=utc_now, start_at=start_at, end_at=end_at, url=contest_url @@ -211,7 +223,7 @@ def utc_to_jst_str(time: datetime.datetime) -> str: return f"{(time + datetime.timedelta(hours=9)).strftime('%Y/%m/%d %H:%M:%S')} JST" -def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]: +def get_atcoder_schedule(utc_now: datetime.datetime) -> List[CalendarEvent]: res = requests.get(urlparse.urljoin(ATCODER_BASE_URL, "contests/?lang=ja"), timeout=10) res.raise_for_status() soup = bs4.BeautifulSoup(res.content, 'html.parser') @@ -227,7 +239,7 @@ def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]: sys.exit(1) event_list = [ - CalendarEvent.parse_text_obj_to_calendarevent(nobj, sobj, dobj, now) + CalendarEvent.parse_text_obj_to_calendarevent(nobj, sobj, dobj, utc_now) for nobj, sobj, dobj in zip(name_objs, start_datetime_objs, duration_objs) ] @@ -235,6 +247,14 @@ def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]: # pylint: disable=invalid-name def parse_datetime(t: str) -> datetime.datetime: + """Get timezone aware datetime object from string + + Args: + t (str): Timestamp in string (with timezone offset %z) + + Returns: + datetime.datetime: Timezone aware datetime object + """ try: return datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S%z") except ValueError: @@ -250,8 +270,8 @@ def get_registered_events( # pylint: disable=no-member events = API_SERVICE.events().list( calendarId=CALENDAR_ID, - timeMin=f"{time_from.isoformat()}Z", - timeMax=f"{time_to.isoformat()}Z", + timeMin=f"{time_from.isoformat(timespec='seconds')}", + timeMax=f"{time_to.isoformat(timespec='seconds')}", pageToken=page_token ).execute() registered_events += [ @@ -327,13 +347,13 @@ def add_event(event: CalendarEvent, batch: Union[BatchHttpRequest, None] = None) API_SERVICE.events().insert(calendarId=CALENDAR_ID, body=event.get_as_obj()).execute() # TODO(k1832): WIP. Refactor this. -def delete_all_events(): - now = datetime.datetime.utcnow() - eight_week_later = now + datetime.timedelta(weeks=8) +def delete_all_within_three_years(): + utc_now = datetime.datetime.now(datetime.UTC) + three_years_later = utc_now + datetime.timedelta(days=365*3) batch = API_SERVICE.new_batch_http_request() offset = 0 - events_to_delete = get_registered_events(now, eight_week_later) + events_to_delete = get_registered_events(utc_now, three_years_later) ONE_BATCH_LIMIT = 995 @@ -349,28 +369,39 @@ def delete_all_events(): batch.execute() offset = end -# These are needed in Cloud Functions +# `data` and `context` are passed by Cloud Functions and not used. # pylint: disable=unused-argument def main(data, context): - now = datetime.datetime.utcnow() - upcoming_contests: List[CalendarEvent] = get_atcoder_schedule(now) + utc_now = datetime.datetime.now(datetime.UTC) + upcoming_contests: List[CalendarEvent] = get_atcoder_schedule(utc_now) print(f"{len(upcoming_contests)} contests have been retrieved.") - eight_week_later = now + datetime.timedelta(weeks=8) if not upcoming_contests: print("There is no upcoming contests.") sys.exit() - updated_count = 0 - inserted_count = 0 + # Google Calendar events: From: now, To: 2 years later + # AtCoder events: From: now, To: Unlimited (as long as it's on the first page?) + + # Therefore, we need to ignore AtCoder events that are older than + # certain time (365 days for now) to avoid the duplicate events + atcoder_event_ignore_threashold = utc_now + datetime.timedelta(days=365) + two_year_later = utc_now + datetime.timedelta(days=365*2) # 2 years + - registered_events: List[CalendarEvent] = get_registered_events(now, eight_week_later) + # Get registered events within 2 years from Google Calendar + registered_events: List[CalendarEvent] = get_registered_events(utc_now, two_year_later) summary_to_registered, url_to_registered = get_registered_events_dict(registered_events) + # pylint: disable=no-member batch = API_SERVICE.new_batch_http_request() + updated_count = 0 + inserted_count = 0 for upcoming in upcoming_contests: + if not upcoming.starts_before(atcoder_event_ignore_threashold): + continue if CALENDAR_TYPE == 'ABC' and not upcoming.is_abc(): continue From ce3257a690088e6baa04cd55e9e055eba56bcf04 Mon Sep 17 00:00:00 2001 From: Keita Date: Tue, 16 Apr 2024 16:49:21 +0800 Subject: [PATCH 9/9] Change way to get JST timestamp --- main.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 478dde9..6ca480e 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ import re import sys import datetime +import pytz import urllib.parse as urlparse from datetime import datetime as dt from typing import Final, List, Dict, Tuple, Union @@ -101,14 +102,13 @@ def get_description(self) -> str: """ :returns: Description formatted for Google Calendar """ - created_at_jst_str: Final[str] = utc_to_jst_str(self.created_at) - updated_at_jst_str: Final[str] = utc_to_jst_str(self.updated_at) + created_at_jst_str: Final[str] = get_jst_str(self.created_at) + updated_at_jst_str: Final[str] = get_jst_str(self.updated_at) return f"created at: {created_at_jst_str}\nlast modified at: {updated_at_jst_str}" def get_as_obj(self) -> dict: """ - :returns: Dictionary formatted for Google Calendar API - 以下のような形で返す + :returns: Dictionary formatted for Google Calendar API like below { 'summary': 'ABC001', 'location': 'https://atcoder.jp/contests/abc001', @@ -216,11 +216,16 @@ def update_for_diff(cls, return True -def utc_to_jst_str(time: datetime.datetime) -> str: - """ - :returns: Time converted to JST as string +def get_jst_str(time: datetime.datetime) -> str: + """Get JST formatted string from timezone aware datetime object + + Args: + time: Timezone aware datetime object + + Returns: + str: JST formatted string """ - return f"{(time + datetime.timedelta(hours=9)).strftime('%Y/%m/%d %H:%M:%S')} JST" + return time.astimezone(pytz.timezone("Asia/Tokyo")).strftime('%Y/%m/%d %H:%M:%S JST') def get_atcoder_schedule(utc_now: datetime.datetime) -> List[CalendarEvent]: