diff --git a/lolstaticdata/champions/__main__.py b/lolstaticdata/champions/__main__.py index 054770cb..05a3a60f 100644 --- a/lolstaticdata/champions/__main__.py +++ b/lolstaticdata/champions/__main__.py @@ -2,6 +2,7 @@ import json import time import argparse +import requests from bs4 import BeautifulSoup from ..common import utils @@ -10,7 +11,11 @@ def get_ability_filenames(url): - soup = utils.download_soup(url, use_cache=False) + try: + soup = utils.download_soup(url, use_cache=False) + except requests.exceptions.RequestException as e: + utils.print_warning(f"Could not load ability icon listing ({url}): {e}") + return [] soup = BeautifulSoup(soup, "lxml") filenames = [] @@ -55,10 +60,20 @@ def print_runtime(): process_skins=skins or process_all, ) - latest_version = utils.get_latest_patch_version() - ddragon_champions = utils.download_json( - f"http://ddragon.leagueoflegends.com/cdn/{latest_version}/data/en_US/championFull.json" - )["data"] + try: + latest_version = utils.get_latest_patch_version() + except requests.exceptions.RequestException as e: + utils.print_error(f"Unable to determine latest patch version from Data Dragon: {e}") + return + + print(f"Fetching Champion data for patch version: {latest_version}") + try: + ddragon_champions = utils.download_json( + f"http://ddragon.leagueoflegends.com/cdn/{latest_version}/data/en_US/championFull.json" + )["data"] + except requests.exceptions.RequestException as e: + utils.print_error(f"Unable to load champion metadata for patch {latest_version}: {e}") + return factions = {} if lore or process_all: @@ -71,7 +86,7 @@ def print_runtime(): for universe_entry in universe_stats: factions[universe_entry["slug"]] = universe_entry["associated-faction-slug"] except Exception: - print("ERROR: Unable to load/parse universe file, location may have changed") + utils.print_error("Unable to load/parse universe file, location may have changed") ability_key_to_identifier = { "P": "passive", @@ -88,7 +103,7 @@ def print_runtime(): champion_key = champion.key if champion_key not in ddragon_champions: - print(f"WARNING: Skipping champion '{champion_key}' (not present in Data Dragon metadata).") + utils.print_warning(f"Skipping champion '{champion_key}' (not present in Data Dragon metadata).") continue ddragon_champion = ddragon_champions[champion_key] @@ -140,7 +155,7 @@ def print_runtime(): if champion and not champions: end_time = time.time() print_runtime() - print(f"WARNING: Champion '{champion}' was not found. Try the exact in-game name, e.g. \"Jarvan IV\".") + utils.print_warning(f"Champion '{champion}' was not found. Try the exact in-game name, e.g. \"Jarvan IV\".") return jsonfn = os.path.join(directory, "champions.json") diff --git a/lolstaticdata/champions/pull_champions_wiki.py b/lolstaticdata/champions/pull_champions_wiki.py index f02904fc..98d98a45 100644 --- a/lolstaticdata/champions/pull_champions_wiki.py +++ b/lolstaticdata/champions/pull_champions_wiki.py @@ -33,6 +33,8 @@ to_enum_like, download_json, strip_lua_comments, + print_error, + print_warning, ) from .modelchampion import ( Champion, @@ -339,7 +341,7 @@ def _render_champion_data(self, name: str, data: Dict) -> Champion: flat=data["stats"]["range"], per_level=data["stats"].get("range_lvl", 0), ), - critical_strike_damage=Stat(flat=data["stats"].get("crit_base", 175)), + critical_strike_damage=Stat(flat=data["stats"].get("crit_base", 200)), critical_strike_damage_modifier=Stat(flat=data["stats"].get("crit_mod", 1.0)), movespeed=Movespeed(flat=data["stats"]["ms"]), acquisition_radius=Stat(flat=data["stats"].get("acquisition_radius", 800)), @@ -492,7 +494,10 @@ def _pull_champion_ability(self, champion_name, ability_name) -> HTMLAbilityWrap try: return HTMLAbilityWrapper(soup) except ValueError: - print(f"WARNING: Ability data could not be found for {ability_name}. The Wiki page may be empty: https://wiki.leagueoflegends.com/en-us/Template:Data_{champion_name}/{ability_name}") + print_warning( + f"Ability data could not be found for {ability_name}. " + f"The Wiki page may be empty: https://wiki.leagueoflegends.com/en-us/Template:Data_{champion_name}/{ability_name}" + ) def _get_ability_effects(self, name, skill_dict): effects = [] @@ -721,8 +726,8 @@ def _render_modifiers(self, mods: str, nvalues: int) -> List[Modifier]: try: parsed_modifiers = ParsingAndRegex.split_modifiers(mods) except Exception as error: - print("ERROR: FAILURE TO SPLIT MODIFIER") - print("ERROR:", error) + print_error("FAILURE TO SPLIT MODIFIER") + print_error(str(error)) return modifiers for lvling in parsed_modifiers: @@ -730,8 +735,8 @@ def _render_modifiers(self, mods: str, nvalues: int) -> List[Modifier]: modifier = self._render_modifier(lvling, nvalues) modifiers.append(modifier) except Exception as error: - print(f"ERROR: FAILURE TO PARSE MODIFIER: {lvling}") - print("ERROR:", error) + print_error(f"FAILURE TO PARSE MODIFIER: {lvling}") + print_error(str(error)) modifier = Modifier( values=[0 for _ in range(nvalues)], units=[lvling for _ in range(nvalues)], @@ -1015,7 +1020,7 @@ def regex_slash_separated(string: str, nvalues: int) -> Tuple[List[str], List[Un if nvalues == 3 and len(values) == 5: values = [values[0], values[2], values[4]] if nvalues is not None and len(values) != nvalues: - print(f"WARNING: Unexpected number of modifier values: {values} (expected {nvalues})") + print_warning(f"Unexpected number of modifier values: {values} (expected {nvalues})") return not_parsed, values raise ValueError(f"Could not parse slash-separated string: {string}") diff --git a/lolstaticdata/common/utils.py b/lolstaticdata/common/utils.py index ed7c8a90..b403e809 100644 --- a/lolstaticdata/common/utils.py +++ b/lolstaticdata/common/utils.py @@ -14,6 +14,26 @@ Json = Union[dict, list, str, int, float, bool, None] +ANSI_RED = "\033[31m" +ANSI_ORANGE = "\033[38;5;214m" +ANSI_RESET = "\033[0m" + + +def _print_colored(prefix: str, message: str, color: str, code: Union[int, str, None] = None): + if code is None: + print(f"{color}{prefix}: {message}{ANSI_RESET}") + else: + print(f"{color}{prefix} [{code}]: {message}{ANSI_RESET}") + + +def print_error(message: str, code: Union[int, str, None] = None): + _print_colored("ERROR", message, ANSI_RED, code=code) + + +def print_warning(message: str, code: Union[int, str, None] = None): + _print_colored("WARNING", message, ANSI_ORANGE, code=code) + + def to_enum_like(string: str) -> str: return string.upper().replace(" ", "_") @@ -131,6 +151,9 @@ def download_json(url: str, use_cache: bool = True) -> Json: else: headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"} page = requests.get(url, headers=headers) + if page.status_code >= 400: + print_error(f"HTTP {page.status_code} while fetching JSON: {url}", code=page.status_code) + raise requests.exceptions.HTTPError(f"HTTP {page.status_code}: {url}", response=page) j = page.json() if use_cache: with open(fn, "w") as f: @@ -153,6 +176,9 @@ def download_soup(url: str, use_cache: bool = True, dir: str = f"__cache__"): html = f.read() else: page = requests.get(url) + if page.status_code >= 400: + print_error(f"HTTP {page.status_code} while fetching page: {url}", code=page.status_code) + raise requests.exceptions.HTTPError(f"HTTP {page.status_code}: {url}", response=page) # html = page.content.decode(page.encoding) html = page.text if use_cache: diff --git a/lolstaticdata/items/__main__.py b/lolstaticdata/items/__main__.py index 60d24f01..4b8eb347 100644 --- a/lolstaticdata/items/__main__.py +++ b/lolstaticdata/items/__main__.py @@ -1,13 +1,21 @@ import os import shutil import json +import requests from .pull_items_wiki import WikiItem, get_item_urls from .pull_items_dragon import DragonItem from collections import OrderedDict +from ..common.utils import print_error def main(): directory = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../..")) + try: + latest_version = DragonItem.get_latest_version() + except requests.exceptions.RequestException as e: + print_error(f"Unable to determine latest patch version from Data Dragon: {e}") + return False + print(f"Fetching Item data for patch version: {latest_version}") if not os.path.exists(os.path.join(directory, "items")): os.mkdir(os.path.join(directory, "items")) @@ -16,7 +24,11 @@ def main(): if not os.path.exists(os.path.join(directory, "__wiki__")): os.mkdir(os.path.join(directory, "__wiki__")) - cdragon = DragonItem.get_cdragon() + try: + cdragon = DragonItem.get_cdragon() + except requests.exceptions.RequestException as e: + print_error(f"Unable to load CommunityDragon items list: {e}") + return False wikiItems = get_item_urls(False) jsons = {} @@ -61,8 +73,9 @@ def main(): with open(jsonfn, "w", encoding="utf8") as f: json.dump(jsons, f, indent=2, ensure_ascii=False) del jsons + return True if __name__ == "__main__": - main() - print("Hello! What a surprise, it worked!") + if main(): + print("Hello! What a surprise, it worked!") diff --git a/lolstaticdata/items/modelitem.py b/lolstaticdata/items/modelitem.py index 3f3c1e05..22d2c736 100644 --- a/lolstaticdata/items/modelitem.py +++ b/lolstaticdata/items/modelitem.py @@ -70,6 +70,14 @@ class ItemRanks(OrderedEnum): SPECIAL = "SPECIAL" +class ItemModes(OrderedEnum): + CLASSIC = "classic" + ARAM = "aram" + NEXUS_BLITZ = "nexusBlitz" + ARENA = "arena" + MAYHEM = "mayhem" + + @dataclasses_json.dataclass_json(letter_case=dataclasses_json.LetterCase.CAMEL) @dataclass class Stats(object): @@ -157,6 +165,7 @@ class Item(object): stats: Stats shop: Shop iconOverlay: str + modes: List[str] def __json__(self, *args, **kwargs): # Use dataclasses_json to get the dict diff --git a/lolstaticdata/items/pull_items_dragon.py b/lolstaticdata/items/pull_items_dragon.py index 93b57301..fc509cd1 100644 --- a/lolstaticdata/items/pull_items_dragon.py +++ b/lolstaticdata/items/pull_items_dragon.py @@ -1,7 +1,8 @@ import json import os +import requests -from ..common.utils import download_json +from ..common.utils import download_json, print_warning from .pull_items_wiki import WikiItem from .modelitem import Item, Shop @@ -13,9 +14,22 @@ def get_latest_version(): class DragonItem: - latest_version = get_latest_version() - version = latest_version.split(".") - version = str(version[0]) + "." + str(version[1]) + _latest_version = None + _version = None + + @classmethod + def get_latest_version(cls): + if cls._latest_version is None: + cls._latest_version = get_latest_version() + return cls._latest_version + + @classmethod + def get_version(cls): + if cls._version is None: + latest_version = cls.get_latest_version() + version_parts = latest_version.split(".") + cls._version = str(version_parts[0]) + "." + str(version_parts[1]) + return cls._version @staticmethod def get_cdragon(): # cdragon to list @@ -26,13 +40,25 @@ def get_cdragon(): # cdragon to list cdragon = [i for i in j if str(i["id"])] return cdragon - @staticmethod - def get_item_plaintext(item): - url = f"https://raw.communitydragon.org/{DragonItem.version}/game/en_us/data/menu/en_us/lol.stringtable.json" - j = download_json(url, use_cache=True) + _stringtable_cache = None + _stringtable_loaded = False + + @classmethod + def get_item_plaintext(cls, item): + if not cls._stringtable_loaded: + url = f"https://raw.communitydragon.org/latest/game/en_us/data/menu/en_us/lol.stringtable.json" + try: + cls._stringtable_cache = download_json(url, use_cache=True) + except requests.exceptions.RequestException as e: + print_warning(f"Could not fetch CommunityDragon stringtable ({url}): {e}") + cls._stringtable_cache = None + cls._stringtable_loaded = True + + if cls._stringtable_cache is None: + return None try: - return j['entries']["game_item_plaintext_" + str(item)] - except: + return cls._stringtable_cache['entries']["game_item_plaintext_" + str(item)] + except (KeyError, TypeError): return None @classmethod @@ -75,6 +101,7 @@ def get_item_cdragon(cls, cdrag): rank="", special_recipe=special_recipe, iconOverlay=None, + modes=[], ) return item @@ -88,7 +115,7 @@ def _get_skin_path(cls, path): path = path.lower() path = ( "https://raw.communitydragon.org/{}/plugins/rcp-be-lol-game-data/global/default/assets".format( - DragonItem.version + cls.get_version() ) + path ) @@ -101,14 +128,14 @@ def get_json_ddragon( cls, ): # Main Function, gets items from ddragon, compares them with cdragon and then gets the items from the wiki # I didn't want make a request to cdragon for every item - url = "http://ddragon.leagueoflegends.com/cdn/{}/data/en_US/item.json".format(get_latest_version()) + url = "http://ddragon.leagueoflegends.com/cdn/{}/data/en_US/item.json".format(cls.get_latest_version()) p = download_json(url, use_cache=True) return p["data"] @classmethod def get_ddragon(cls, ddragon: int, p: dict): # print(ddragon) - baseurl = "http://ddragon.leagueoflegends.com/cdn/{}/img/item/".format(get_latest_version()) # icon base url + baseurl = "http://ddragon.leagueoflegends.com/cdn/{}/img/item/".format(cls.get_latest_version()) # icon base url icon = baseurl + p[ddragon]["image"]["full"] plaintext = p[ddragon]["plaintext"] # simple description purchasable = p[ddragon]["gold"]["purchasable"] # is this purchasable or is it upgraded (seraph's embrace) @@ -140,5 +167,6 @@ def get_ddragon(cls, ddragon: int, p: dict): stats=[], shop=shop, rank="", + modes=[], ) return item diff --git a/lolstaticdata/items/pull_items_wiki.py b/lolstaticdata/items/pull_items_wiki.py index 4113d94e..32f49bd7 100644 --- a/lolstaticdata/items/pull_items_wiki.py +++ b/lolstaticdata/items/pull_items_wiki.py @@ -13,10 +13,12 @@ Active, ItemAttributes, ItemRanks, + ItemModes, ) from ..common.utils import ( download_soup, strip_lua_comments, + print_warning, ) from ..common.modelcommon import ( ArmorPenetration, @@ -47,6 +49,30 @@ class WikiItem: + WIKI_MODE_MAP = { + "classic sr 5v5": ItemModes.CLASSIC, + "aram": ItemModes.ARAM, + "nb": ItemModes.NEXUS_BLITZ, + "ar": ItemModes.ARENA, + "mayhem": ItemModes.MAYHEM, + } + + @classmethod + def _parse_modes(cls, item_data: dict) -> List[str]: + modes = [] + wiki_modes = item_data.get("modes") + if not isinstance(wiki_modes, dict): + return modes + for wiki_key, enabled in wiki_modes.items(): + if not enabled: + continue + mode = cls.WIKI_MODE_MAP.get(wiki_key) + if mode is None: + print_warning(f"WARNING: Unknown item mode '{wiki_key}'") + continue + modes.append(mode) + return modes + @classmethod def _parse_passives(cls, item_data: dict) -> List[Passive]: effects = [] @@ -630,7 +656,7 @@ def _parse_item_data(cls, item_data: dict, item_name:str,wiki_data:dict) -> Item item_data[x] = wiki_data[parent_item][x] except KeyError as e: clear_keys.append(x) - print(f"WARNING: Couldn't find inherited value of '{x}' for {item_name} - inherited from {parent_item}") + print_warning(f"Couldn't find inherited value of '{x}' for {item_name} - inherited from {parent_item}") break if x in "effects": for l in item_data[x]: @@ -767,6 +793,7 @@ def _parse_item_data(cls, item_data: dict, item_name:str,wiki_data:dict) -> Item rank=rank, special_recipe=0, iconOverlay=ornn, + modes=cls._parse_modes(item_data), ) return item