diff --git a/.gitignore b/.gitignore index 5918636..11c182f 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,5 @@ dmypy.json *.code-workspace # End of https://www.toptal.com/developers/gitignore/api/python,vscode + +.DS_Store diff --git a/main.py b/main.py index 8db89dc..6ca480e 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ import re import sys import datetime +import pytz import urllib.parse as urlparse from datetime import datetime as dt from typing import Final, List, Dict, Tuple, Union @@ -18,6 +19,8 @@ CREDENTIAL_INFO: Dict[str, str] = {} CALENDAR_TYPE: Final[str] = 'ABC' if os.environ.get('CALENDAR_TYPE') == 'ABC' else 'ALL' +ABC_PATTERN = re.compile(r"\/abc\d{3}$") + if os.environ.get('ENV') == 'local': # ローカルでテスト # `ENV=local python3 main.py` みたいに使う @@ -59,7 +62,7 @@ @dataclass class TimeWithStrTimeZone: - time: datetime.datetime + time: datetime.datetime # Timezone aware time_zone: str = 'Japan' def get_as_obj(self): @@ -81,10 +84,10 @@ def __eq__(self, other: "TimeWithStrTimeZone") -> bool: @dataclass class CalendarEvent: summary: str - created_at: datetime.datetime - updated_at: datetime.datetime - start_at: InitVar[datetime.datetime] - end_at: InitVar[datetime.datetime] + created_at: datetime.datetime # UTC + updated_at: datetime.datetime # UTC + start_at: InitVar[datetime.datetime] # Timezone aware + end_at: InitVar[datetime.datetime] # Timezone aware url: str start_at_with_time_zone: TimeWithStrTimeZone = field(init=False) end_at_with_timw_zone: TimeWithStrTimeZone = field(init=False) @@ -99,14 +102,13 @@ def get_description(self) -> str: """ :returns: Description formatted for Google Calendar """ - created_at_jst_str: Final[str] = utc_to_jst_str(self.created_at) - updated_at_jst_str: Final[str] = utc_to_jst_str(self.updated_at) + created_at_jst_str: Final[str] = get_jst_str(self.created_at) + updated_at_jst_str: Final[str] = get_jst_str(self.updated_at) return f"created at: {created_at_jst_str}\nlast modified at: {updated_at_jst_str}" def get_as_obj(self) -> dict: """ - :returns: Dictionary formatted for Google Calendar API - 以下のような形で返す + :returns: Dictionary formatted for Google Calendar API like below { 'summary': 'ABC001', 'location': 'https://atcoder.jp/contests/abc001', @@ -130,12 +132,19 @@ def get_as_obj(self) -> dict: 'end': self.end_at_with_timw_zone.get_as_obj() } - def is_abc(self) -> bool: - m = re.search(r"\/abc\d{3}$", self.url) - if m is None: - return False + def starts_before(self, time: datetime.datetime) -> bool: + """Check if the event starts before the time - return True + Args: + time (datetime.datetime): Timezone aware time + + Returns: + bool: Whether or not the event starts before the time + """ + return self.start_at_with_time_zone.time < time + + def is_abc(self) -> bool: + return ABC_PATTERN.search(self.url) is not None @classmethod def parse_event(cls, event_item_obj: dict) -> "CalendarEvent": @@ -155,7 +164,7 @@ def parse_event(cls, event_item_obj: dict) -> "CalendarEvent": @classmethod def parse_text_obj_to_calendarevent( - cls, name_obj, start_datetime_obj, duration_obj, now: datetime.datetime + cls, name_obj, start_datetime_obj, duration_obj, utc_now: datetime.datetime ) -> "CalendarEvent": """ :param name_obj: Object, which contains name and url of the contest, obtained from html @@ -163,20 +172,21 @@ def parse_text_obj_to_calendarevent( :param duration_obj: Object, which contains duration time (1:30 as one hour and 30 minutes), obtained from html - :param now: The time this program started + :param utc_now: The time this program started :returns: Parsed Calendar Event """ contest_title: str = name_obj.text contest_url: str = urlparse.urljoin(ATCODER_BASE_URL, name_obj.attrs['href']) - start_at: datetime.datetime = dt.strptime(start_datetime_obj.text, '%Y-%m-%d %H:%M:%S+0900') + # Timezone aware datetime object (%z: timezone offset) + start_at: datetime.datetime = dt.strptime(start_datetime_obj.text, '%Y-%m-%d %H:%M:%S%z') contest_hours, contest_minutes = map(int, duration_obj.text.split(':')) contest_duration: datetime.timedelta = datetime.timedelta(hours=contest_hours, minutes=contest_minutes) end_at: datetime.datetime = start_at + contest_duration return CalendarEvent( summary=contest_title, - created_at=now, - updated_at=now, + created_at=utc_now, + updated_at=utc_now, start_at=start_at, end_at=end_at, url=contest_url @@ -206,14 +216,19 @@ def update_for_diff(cls, return True -def utc_to_jst_str(time: datetime.datetime) -> str: - """ - :returns: Time converted to JST as string +def get_jst_str(time: datetime.datetime) -> str: + """Get JST formatted string from timezone aware datetime object + + Args: + time: Timezone aware datetime object + + Returns: + str: JST formatted string """ - return f"{(time + datetime.timedelta(hours=9)).strftime('%Y/%m/%d %H:%M:%S')} JST" + return time.astimezone(pytz.timezone("Asia/Tokyo")).strftime('%Y/%m/%d %H:%M:%S JST') -def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]: +def get_atcoder_schedule(utc_now: datetime.datetime) -> List[CalendarEvent]: res = requests.get(urlparse.urljoin(ATCODER_BASE_URL, "contests/?lang=ja"), timeout=10) res.raise_for_status() soup = bs4.BeautifulSoup(res.content, 'html.parser') @@ -229,7 +244,7 @@ def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]: sys.exit(1) event_list = [ - CalendarEvent.parse_text_obj_to_calendarevent(nobj, sobj, dobj, now) + CalendarEvent.parse_text_obj_to_calendarevent(nobj, sobj, dobj, utc_now) for nobj, sobj, dobj in zip(name_objs, start_datetime_objs, duration_objs) ] @@ -237,6 +252,14 @@ def get_atcoder_schedule(now: datetime.datetime) -> List[CalendarEvent]: # pylint: disable=invalid-name def parse_datetime(t: str) -> datetime.datetime: + """Get timezone aware datetime object from string + + Args: + t (str): Timestamp in string (with timezone offset %z) + + Returns: + datetime.datetime: Timezone aware datetime object + """ try: return datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S%z") except ValueError: @@ -252,8 +275,8 @@ def get_registered_events( # pylint: disable=no-member events = API_SERVICE.events().list( calendarId=CALENDAR_ID, - timeMin=f"{time_from.isoformat()}Z", - timeMax=f"{time_to.isoformat()}Z", + timeMin=f"{time_from.isoformat(timespec='seconds')}", + timeMax=f"{time_to.isoformat(timespec='seconds')}", pageToken=page_token ).execute() registered_events += [ @@ -266,39 +289,20 @@ def get_registered_events( return registered_events def get_registered_events_dict( - time_from: datetime.datetime, time_to: datetime.datetime - ) -> Tuple[Dict[str, CalendarEvent], Dict[str, CalendarEvent]]: - """Returns registered event dictionary + registered_events: List[CalendarEvent] + ) -> Tuple[Dict[str, int], Dict[str, int]]: + """Returns dictionary that maps from event url/summary to event index Args: - time_from (datetime.datetime): This function looks for events after this - time_to (datetime.datetime): This function looks for events after this + registered_events (List[CalendarEvent]): List of registered events Returns: - Tuple[Dict[str, CalendarEvent], Dict[str, CalendarEvent]]: - first Dict -> key: event summary, value: event, - second Dict -> key: event url, value: event + Tuple[Dict[str, int], Dict[str, int]]: + first Dict -> key: event summary, value: index of registered event, + second Dict -> key: event url, value: index of registered event """ - summary_to_registered: Dict[str, CalendarEvent] = {} - url_to_registered: Dict[str, CalendarEvent] = {} - - page_token = None - while True: - # pylint: disable=no-member - events = API_SERVICE.events().list( - calendarId=CALENDAR_ID, - timeMin=f"{time_from.isoformat()}Z", - timeMax=f"{time_to.isoformat()}Z", - pageToken=page_token - ).execute() - for event_item_obj in events["items"]: - event = CalendarEvent.parse_event(event_item_obj) - summary_to_registered[event.summary] = event - url_to_registered[event.url] = event - - page_token = events.get('nextPageToken') - if not page_token: - break + summary_to_registered: Dict[str, int] = {event.summary: i for i, event in enumerate(registered_events)} + url_to_registered: Dict[str, int] = {event.url: i for i, event in enumerate(registered_events)} return summary_to_registered, url_to_registered @@ -347,40 +351,77 @@ def add_event(event: CalendarEvent, batch: Union[BatchHttpRequest, None] = None) # pylint: disable=no-member API_SERVICE.events().insert(calendarId=CALENDAR_ID, body=event.get_as_obj()).execute() -# These are needed in Cloud Functions +# TODO(k1832): WIP. Refactor this. +def delete_all_within_three_years(): + utc_now = datetime.datetime.now(datetime.UTC) + three_years_later = utc_now + datetime.timedelta(days=365*3) + batch = API_SERVICE.new_batch_http_request() + + offset = 0 + events_to_delete = get_registered_events(utc_now, three_years_later) + + ONE_BATCH_LIMIT = 995 + + while offset < len(events_to_delete): + end = min(len(events_to_delete), offset + ONE_BATCH_LIMIT) + for event in events_to_delete[offset:end]: + batch.add( + API_SERVICE.events().delete( + calendarId=CALENDAR_ID, + eventId=event.id + ) + ) + batch.execute() + offset = end + +# `data` and `context` are passed by Cloud Functions and not used. # pylint: disable=unused-argument def main(data, context): - now = datetime.datetime.utcnow() - upcoming_contests = get_atcoder_schedule(now) + utc_now = datetime.datetime.now(datetime.UTC) + upcoming_contests: List[CalendarEvent] = get_atcoder_schedule(utc_now) print(f"{len(upcoming_contests)} contests have been retrieved.") - eight_week_later = now + datetime.timedelta(weeks=8) if not upcoming_contests: print("There is no upcoming contests.") sys.exit() - updated_count = 0 - inserted_count = 0 + # Google Calendar events: From: now, To: 2 years later + # AtCoder events: From: now, To: Unlimited (as long as it's on the first page?) + + # Therefore, we need to ignore AtCoder events that are older than + # certain time (365 days for now) to avoid the duplicate events + atcoder_event_ignore_threashold = utc_now + datetime.timedelta(days=365) + two_year_later = utc_now + datetime.timedelta(days=365*2) # 2 years + + + # Get registered events within 2 years from Google Calendar + registered_events: List[CalendarEvent] = get_registered_events(utc_now, two_year_later) + summary_to_registered, url_to_registered = get_registered_events_dict(registered_events) - summary_to_registered, url_to_registered = get_registered_events_dict(now, eight_week_later) # pylint: disable=no-member batch = API_SERVICE.new_batch_http_request() + updated_count = 0 + inserted_count = 0 for upcoming in upcoming_contests: + if not upcoming.starts_before(atcoder_event_ignore_threashold): + continue if CALENDAR_TYPE == 'ABC' and not upcoming.is_abc(): continue # 2 contests are the same contest if they have either same summary (title) or url if upcoming.summary in summary_to_registered: - registered = summary_to_registered[upcoming.summary] + index = summary_to_registered[upcoming.summary] + registered = registered_events[index] if CalendarEvent.update_for_diff(registered, upcoming, batch): updated_count += 1 continue if upcoming.url in url_to_registered: - registered = url_to_registered[upcoming.url] + index = url_to_registered[upcoming.url] + registered = registered_events[index] if CalendarEvent.update_for_diff(registered, upcoming, batch): updated_count += 1 diff --git a/requirements.txt b/requirements.txt index b8a5f6a..ee1306e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,20 @@ -beautifulsoup4==4.11.1 -google_api_python_client==2.64.0 -protobuf==4.21.7 -requests==2.28.1 -pylint==2.15.4 +beautifulsoup4==4.12.2 +cachetools==5.3.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +google-api-core==2.14.0 +google-api-python-client==2.109.0 +google-auth==2.24.0 +google-auth-httplib2==0.1.1 +googleapis-common-protos==1.61.0 +httplib2==0.22.0 +idna==3.6 +protobuf==4.25.1 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pyparsing==3.1.1 +requests==2.31.0 +rsa==4.9 +soupsieve==2.5 +uritemplate==4.1.1 +urllib3==2.1.0