diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1975ed1..442976b 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,10 +11,6 @@ on: description: 'Build and release Windows executables' type: boolean default: false - release_gui: - description: 'Include Windows GUI executable (requires GooeyEx)' - type: boolean - default: false permissions: contents: read @@ -85,14 +81,11 @@ jobs: python-version: '3.11' - name: Install dependencies run: python -m pip install pyinstaller ".[il2cpp]" - - name: Install GUI dependencies - if: inputs.release_gui - run: python -m pip install ".[gui]" + # NOTE: The GUI executable is not built here: its dependency GooeyEx is not + # published on public PyPI. Build it locally with `pyinstaller sssekai-gui.spec` + # from an environment where GooeyEx is installed. - name: Build Windows executables run: pyinstaller sssekai.spec - - name: Build Windows GUI executable - if: inputs.release_gui - run: pyinstaller sssekai-gui.spec - name: Create GitHub release shell: pwsh env: diff --git a/.vscode/launch.json b/.vscode/launch.json index 8e58e2c..38db2ac 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -276,30 +276,6 @@ "console": "integratedTerminal", "justMyCode": false }, - { - "name": "Python: Convert Sekai Character Mesh", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/scripts/blender_mesh_importer.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "C:\\Users\\mos9527\\.sssekai\\abcache\\live_pv\\model\\character\\body\\21\\0001\\ladies_s" - ], - "justMyCode": false - }, - { - "name": "Python: Dump Live2D Model Parameter names", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/scripts/dump_moc3_assets.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "C:\\Users\\mos9527\\.sssekai\\abcache\\live2d\\model" - ], - "justMyCode": false - }, { "name": "Python: apphash", "type": "python", @@ -353,17 +329,6 @@ ], "justMyCode": true }, - { - "name": "Python: live2dextract Generic", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/scripts/live2dextract_generic.py", - "args": [ - "C:\\Users\\Huang\\Desktop\\Data", - "C:\\Users\\Huang\\Desktop\\Data_result" - ], - "justMyCode": false - }, { "name": "Python: rla2json", "type": "python", diff --git a/README.md b/README.md index bac07f8..7e6727a 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,14 @@ To use the latest development version, you can install directly from GitHub: pip install -U git+https://github.com/MetaSekaiLab/sssekai ``` ## Using the GUI version -Additionally, you can use the GUI version of `sssekai` by installing the `[gui]` extra, which in turn installs [GooeyEx](https://github.com/mos9527/GooeyEx/). +The GUI depends on [GooeyEx](https://github.com/mos9527/GooeyEx/) **>= 0.1.1**, which is not published on PyPI. Install it manually from upstream (a `git` executable and network access are required), then run the GUI: ```bash -pip install -U "sssekai[gui]" -``` -Then you can run the GUI version by: -```bash +pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx" sssekai-gui ``` + +`GooeyEx` 0.1.0 (the newest version on PyPI) crashes on import; verify you got 0.1.1 or newer with `python -c "import GooeyEx; print(GooeyEx.__version__)"`. If git is unavailable, use the CLI executable from Releases instead. # 问题反馈 使用问题和缺陷请提交至本仓库的 [Issues](https://github.com/MetaSekaiLab/sssekai/issues)。 diff --git a/setup.py b/setup.py index 835a96c..37320f7 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,12 @@ "coloredlogs", "fsspec", ], - extras_require={"gui": ["GooeyEx>=0.1.1"], "il2cpp": ["lief"], "criware": ["PyCriCodecsEx"]}, + # NOTE: No `gui` extra on purpose. The GUI requires GooeyEx >= 0.1.1, which + # is not published on PyPI (0.1.0 crashes at import without a wx.App), and + # PyPI rejects direct-URL requirements. See README for manual installation. + # The unused `criware` extra was removed with its dependency for the same + # reason: PyCriCodecsEx is not installable and nothing imports it. + extras_require={"il2cpp": ["lief"]}, entry_points={ "console_scripts": [ "sssekai = sssekai.__main__:__main__", diff --git a/sssekai/__gui__.py b/sssekai/__gui__.py index 87e3aca..293fd1e 100644 --- a/sssekai/__gui__.py +++ b/sssekai/__gui__.py @@ -9,9 +9,13 @@ from GooeyEx import Gooey, GooeyParser except ImportError as e: print( - "Please install sssekai[gui] through your Python package manager to use the GUI" + "The GUI requires GooeyEx >= 0.1.1, which is not published on PyPI.\n" + "Install it manually, then retry:\n" + ' pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx"\n' + "Otherwise use the CLI executable from the Releases page.", + file=sys.stderr, ) - raise e + raise SystemExit(1) from e from sssekai.unity import sssekai_set_unity_version diff --git a/sssekai/__main__.py b/sssekai/__main__.py index cd60571..d643dcf 100644 --- a/sssekai/__main__.py +++ b/sssekai/__main__.py @@ -178,7 +178,7 @@ def gooey_only(**kwargs): ) group = abcache_parser.add_argument_group( "game version options", - "NOTE: these options are used to *update* the cache database. Use --no-update to skip updating. Also, you may want to read the Wiki to know how to get these values.", + "NOTE: these options are used to *update* the cache database. Use --no-update to skip updating. You can obtain these values with the `apphash` subcommand.", ) group.add_argument( "--app-region", @@ -329,8 +329,8 @@ def gooey_only(**kwargs): abserve_parser.add_argument( "--host", type=str, - help="""address of the interface to listen on (default: %(default)s)""", - default="0.0.0.0", + help="""address of the interface to listen on (default: %(default)s). Use 0.0.0.0 to expose the server to the network.""", + default="127.0.0.1", ) abserve_parser.add_argument( "--port", diff --git a/sssekai/abcache/__init__.py b/sssekai/abcache/__init__.py index 3df59b2..6173510 100644 --- a/sssekai/abcache/__init__.py +++ b/sssekai/abcache/__init__.py @@ -14,6 +14,24 @@ REGION_OPTIONS = REGION_JP_EN | REGION_ROW +class RedactedStr(str): + """A credential that behaves like a str but hides its value in repr. + + `dataclasses.asdict` deep-copies values, so `field(repr=False)` alone does + not keep tokens out of `repr(asdict(...))`. Pickles as a plain str so cache + files stay compatible with older versions. + """ + + def __repr__(self): + return "" + + def __deepcopy__(self, memo): + return self + + def __reduce_ex__(self, protocol): + return str, (str(self),) + + def fromdict(klass: type, d: Union[Mapping, List], warn_missing_fields=True): """ Recursively convert a dictionary to a dataclass instance, if applicable. @@ -74,7 +92,9 @@ class AbCacheConfig: app_platform: str app_hash: str - auth_credential: str = None # JWT token for JP/EN, Base64 encoded JWT token for ROW + auth_credential: str = field( + default=None, repr=False + ) # JWT token for JP/EN, Base64 encoded JWT token for ROW row_ab_version: str = None # Override AB version in url for ROW @@ -84,6 +104,13 @@ class AbCacheConfig: version: Tuple[int, int, int] = (0, 0, 0) # Internal versioning + def __setattr__(self, name, value): + # Credentials are also assigned after construction (CLI, auth.py). + if name == "auth_credential" and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + @property def auth_jwt(self) -> str: if self.app_region in REGION_JP_EN: @@ -209,10 +236,16 @@ class SekaiUserRegistrationData: class SekaiUserData: userRegistration: SekaiUserRegistrationData # JP/EN Only - credential: Optional[str] = None + credential: Optional[str] = field(default=None, repr=False) updatedResources: Optional[dict] = None # ROW - sessionToken: Optional[str] = None + sessionToken: Optional[str] = field(default=None, repr=False) + + def __setattr__(self, name, value): + if name in ("credential", "sessionToken") and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) @property def user_credentials(self): @@ -221,7 +254,7 @@ def user_credentials(self): @dataclass class SekaiUserAuthData: - sessionToken: str + sessionToken: str = field(repr=False) appVersion: str multiPlayVersion: str dataVersion: str @@ -244,11 +277,34 @@ class SekaiUserAuthData: cdnVersion: Optional[int] = None configs: Optional[List[dict]] = None + def __setattr__(self, name, value): + if name == "sessionToken" and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + +class RedactedHeaders(dict): + """Request headers kept for cache reuse. Hidden from repr/asdict so tokens + (e.g. X-Session-Token) never reach logs. Pickles as a plain dict so cache + files stay loadable by both old and new versions.""" + + def __repr__(self): + return "" % len(self) + + def __deepcopy__(self, memo): + # dict's default deepcopy returns a plain dict, which would leak through + # dataclasses.asdict(). Header values are str, so a shallow copy suffices. + return RedactedHeaders(self) + + def __reduce_ex__(self, protocol): + return dict, (dict(self),) + @dataclass class SSSekaiDatabase: config: AbCacheConfig = None - cached_headers: dict = field(default_factory=dict) + cached_headers: dict = field(default_factory=RedactedHeaders) sekai_user_data: SekaiUserData = None sekai_user_auth_data: SekaiUserAuthData = None @@ -256,6 +312,20 @@ class SSSekaiDatabase: sekai_system_data: SekaiSystemData = None sekai_gameversion_data: SekaiGameVersionData = None + def __repr__(self) -> str: + # Two layers guard this: RedactedHeaders hides its own values, and this + # repr also covers a plain dict (databases built by hand, or caches not + # yet normalized by load()) that would otherwise print in full. + values = ", ".join( + "%s=%r" % (f.name, getattr(self, f.name, None)) + for f in fields(self) + if f.name != "cached_headers" + ) + return "SSSekaiDatabase(%s, cached_headers=<%d values>)" % ( + values, + len(self.cached_headers or ()), + ) + def reset(self): self.sekai_user_data = None self.sekai_user_auth_data = None @@ -263,6 +333,38 @@ def reset(self): self.sekai_system_data = None self.sekai_gameversion_data = None + def redact_secrets(self): + """Re-wrap secrets loaded from disk as RedactedStr. + + RedactedStr pickles as a plain str (so caches stay readable by older + versions), so values read back from a cache file must be wrapped again + before they can reach a repr/asdict. Assignment goes through each + dataclass's __setattr__, which does the wrapping. + """ + if not isinstance(self.cached_headers, RedactedHeaders): + self.cached_headers = RedactedHeaders(self.cached_headers or {}) + # sssekai <= 0.8.2 stored the raw API response here, so old caches may + # hold a plain dict (which also breaks attribute access such as + # SEKAI_CREDENTIAL). Normalize it to the dataclass we expect. + if isinstance(self.sekai_user_data, dict): + self.sekai_user_data = fromdict( + SekaiUserData, self.sekai_user_data, False + ) + secrets = [] + if self.config is not None: + secrets.append((self.config, ("auth_credential",))) + secrets += [ + (self.sekai_user_data, ("credential", "sessionToken")), + (self.sekai_user_auth_data, ("sessionToken",)), + ] + for owner, names in secrets: + if owner is None: + continue + for name in names: + value = getattr(owner, name, None) + if value is not None: + setattr(owner, name, value) + class AbCacheBundleNotFoundError(Exception): bundleName: str @@ -716,7 +818,7 @@ def update(self): def save(self, f: BinaryIO): self._update_request_headers() - self.database.cached_headers = self.headers + self.database.cached_headers = RedactedHeaders(self.headers) self.database.config.version = __version_tuple__ logger.info("Saving cache: %s" % self) dump(self.database, f) @@ -724,13 +826,15 @@ def save(self, f: BinaryIO): def load(self, f: BinaryIO): logger.info("Loading cache") self.database = load(f) + # Values from disk are plain str/dict; re-wrap so they stay out of logs. + self.database.redact_secrets() + logger.info("Loaded cache: %s" % self) if not self.database.config.is_up_to_date: logger.warning( "Cache is outdated (cache: %s, current: %s). It's HIGHLY recommended to update the cache to avoid issues." % (self.database.config.cache_version_string, __version__) ) self.headers.update(self.database.cached_headers) - logger.info("Cache loaded: %s" % self) @staticmethod def from_file(f: BinaryIO): diff --git a/sssekai/abcache/auth.py b/sssekai/abcache/auth.py index ba96e1d..37dbc7e 100644 --- a/sssekai/abcache/auth.py +++ b/sssekai/abcache/auth.py @@ -26,5 +26,5 @@ def register_as_anonymous_user_sega(cache: AbCache) -> SekaiUserData: data = cache.response_to_dict(resp) cache.config.auth_credential = data["credential"] logger.info("Success. User ID=%s" % cache.config.auth_userID) - cache.database.sekai_user_data = data - return fromdict(SekaiUserData, data) + cache.database.sekai_user_data = fromdict(SekaiUserData, data, False) + return cache.database.sekai_user_data diff --git a/sssekai/entrypoint/apphash.py b/sssekai/entrypoint/apphash.py index da2573f..95423b3 100644 --- a/sssekai/entrypoint/apphash.py +++ b/sssekai/entrypoint/apphash.py @@ -261,7 +261,7 @@ def main_apphash(args): AbCacheConfig( app_region="{region}", app_version="{app_version}", - ab_version="{ab_version}", + row_ab_version="{ab_version}", app_hash="{app_hash}", app_platform="{platform}" ) diff --git a/sssekai/entrypoint/rla2json.py b/sssekai/entrypoint/rla2json.py index 12b275e..78f7fe3 100644 --- a/sssekai/entrypoint/rla2json.py +++ b/sssekai/entrypoint/rla2json.py @@ -57,6 +57,7 @@ def __dump_from_frame_gen(frame_gen): ) header = datas.get("sekai.rlh", None) assert header, "RLH Header file not found!" + header = json.loads(header.decode("utf-8")) version = tuple(map(int, header["version"].split("."))) logger.info("Version: %d.%d" % version) logger.info("Count: %d" % len(header["splitFileIds"])) diff --git a/sssekai/fmt/rla.py b/sssekai/fmt/rla.py index d6c7ced..1ebd52d 100644 --- a/sssekai/fmt/rla.py +++ b/sssekai/fmt/rla.py @@ -1,11 +1,15 @@ from base64 import b64decode, b64encode from struct import unpack as s_unpack +import struct, zlib from io import BytesIO from collections import defaultdict from typing import Generator, Tuple, TypeVar, List +from logging import getLogger import math, gzip import msgpack +logger = getLogger(__name__) + RLA_VERSIONS = [(1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (1, 7)] read_int = lambda stream, nbytes, signed=False: int.from_bytes( @@ -32,6 +36,12 @@ def __init__(self, missing): super().__init__(f"Missing %d split packet(s): %s" % (len(missing), missing)) +class StreamingDataTruncatedException(Exception): + """Raised when a payload is incomplete or corrupt and cannot be decoded + in full. In non-strict mode callers drop the whole frame rather than + returning a silently shortened one.""" + + # Sekai_Streaming_StreamingCommon__CheckHeader def decode_buffer_base64(buffer: bytes) -> tuple[int, bytes]: """Decodes the 'RTVL' SSE Message payload into a header signature and data. @@ -124,10 +134,11 @@ def decode_streaming_data( version (tuple): RLA version decoder_signature (int): Decoder signature buffer (bytes): Decoded payload data - strict (bool, optional): If False, incomplete packets will be returned as is. Defaults to True. + strict (bool, optional): If False, truncated payloads raise StreamingDataTruncatedException instead of propagating the underlying error. Defaults to True. Raises: Exception: If the packet is incomplete and strict is True. + StreamingDataTruncatedException: If the packet is incomplete and strict is False. Returns: dict: Parsed streaming data payload @@ -227,16 +238,20 @@ def get_next_mask(): ) # ReadString # HACK: Workaround for incomplete packets. + # In non-strict mode a truncated array aborts the whole frame instead of + # being silently returned as a shorter one: callers must be able to tell a + # complete frame from a truncated one (see StreamingDataTruncatedException). def gen_exception_handler(generator): try: for item in generator: yield item + except StreamingDataTruncatedException: + raise except Exception as e: # XXX: This shouldn't happen unless the packet is corrupted. if strict: raise e - else: - return + raise StreamingDataTruncatedException(str(e)) from e gen_get_next_array = lambda reader: ( reader() for _ in range(get_next_int()) @@ -433,7 +448,7 @@ def read_rla_frames( Args: reader (Generator[Tuple[T, bytes], None, None]): A generator that yields a tuple of timestamp and buffer data version (tuple, optional): RLA version, found in respective RLH (JSON) header files. range: see RLA_VERSIONS. Defaults to (1,0). - strict (bool, optional): If False, incomplete packets will be returned as is. Defaults to True. + strict (bool, optional): If False, malformed or incomplete packets are skipped instead of raising. Defaults to True. Yields: Generator[Tuple[T, dict], None, None]: A generator that yields a tuple of timestamp and parsed frame data. Timestamp format is provided by the reader. @@ -453,13 +468,42 @@ def decode_buffer(header_signature, data): payload = decode_streaming_data(version, decoder_signature, data, strict) return payload + # Errors that indicate a bad packet. Anything else (TypeError, AttributeError, + # ...) is a bug in this decoder and must not be silently swallowed. + DECODE_ERRORS = ( + AssertionError, + IndexError, + KeyError, + ValueError, + UnicodeError, + EOFError, + struct.error, + gzip.BadGzipFile, + zlib.error, + StreamingDataTruncatedException, + ) + + def decode_or_skip(header_signature, data): + """Decode a packet, or return None to skip it when not in strict mode.""" + try: + return decode_buffer(header_signature, data) + except DECODE_ERRORS as e: + if strict: + raise + logger.debug("Skipping undecodable RLA packet: %s", e) + return None + for timestamp, buffer in reader: try: split_info, header_signature, data = decode_buffer_base64(buffer) except Exception as e: if strict: raise e - # Otherwise fail silently + # NOTE: A packet that fails to parse has no trustworthy splitId, so + # it cannot be matched against pending __buffers entries. Pending + # groups are left as is and expire at end of stream. + logger.debug("Skipping malformed RLA packet: %s", e) + continue if split_info: splitId, splitIndex, splitNum, dataLength, totalDataLength, base64 = ( split_info @@ -472,14 +516,36 @@ def decode_buffer(header_signature, data): __buffers[splitId], key=lambda x: x[1][1] ) # splitIndex ) + del __buffers[splitId] if strict: assert len(buffer) == totalDataLength, "incomplete split packet" - del __buffers[splitId] + elif len(buffer) != totalDataLength: + logger.debug( + "Skipping incomplete split packet %s (%d/%d bytes)", + splitId, + len(buffer), + totalDataLength, + ) + continue if base64: - buffer = b64decode(buffer) - yield timestamp, decode_buffer(header_signature, buffer) + try: + buffer = b64decode(buffer) + except Exception as e: + if strict: + raise e + logger.debug( + "Skipping split packet %s with malformed base64: %s", + splitId, + e, + ) + continue + payload = decode_or_skip(header_signature, buffer) + if payload is not None: + yield timestamp, payload else: - yield timestamp, decode_buffer(header_signature, data) + payload = decode_or_skip(header_signature, data) + if payload is not None: + yield timestamp, payload if __buffers and strict: missing = [(payload[0][0], splitId) for splitId, payload in __buffers.items()] @@ -496,7 +562,7 @@ def read_archive_rla_frames( Args: src (BytesIO): Source RLA file stream version (tuple, optional): RLA version, found in respective RLH (JSON) header files. range: see RLA_VERSIONS. Defaults to (1,0). - strict (bool, optional): If False, incomplete packets will be returned as is. Defaults to True. + strict (bool, optional): If False, malformed or incomplete packets are skipped instead of raising. Defaults to True. Yields: Generator[Tuple[int, dict], None, None]: A generator that yields a tuple of timestamp and parsed frame data diff --git a/tests/test_rla2json.py b/tests/test_rla2json.py new file mode 100644 index 0000000..de4f434 --- /dev/null +++ b/tests/test_rla2json.py @@ -0,0 +1,64 @@ +from . import * + +import json +from types import SimpleNamespace + + +class _FakeTextAsset: + def __init__(self, name, script: bytes): + self.m_Name = name + self.m_Script = script.decode("utf-8", "surrogateescape") + + +class _FakeObject: + def __init__(self, name, script: bytes): + from UnityPy.enums import ClassIDType + + self.type = ClassIDType.TextAsset + self._data = _FakeTextAsset(name, script) + + def read(self): + return self._data + + +class _FakeEnv: + def __init__(self, objects): + self.objects = objects + + +def test_rla2json_archive_header_is_parsed(monkeypatch): + """0.7.18 regression: the RLH header is bytes but was indexed as a dict.""" + from sssekai.entrypoint import rla2json + + header = json.dumps( + {"version": "1.0", "splitSeconds": 0, "splitFileIds": []} + ).encode("utf-8") + env = _FakeEnv([_FakeObject("sekai.rlh", header)]) + monkeypatch.setattr(rla2json, "load_assetbundle", lambda f: env) + + args = SimpleNamespace( + input=sample_file_path("rla", "streaming_live_vbs_1-0_0.bin"), + outdir=TEMP_DIR, + strict=False, + ) + # Should not raise; no split files means nothing is dumped. + rla2json.main_rla2json(args) + + +def test_rla2json_missing_header(monkeypatch): + from sssekai.entrypoint import rla2json + + env = _FakeEnv([]) + monkeypatch.setattr(rla2json, "load_assetbundle", lambda f: env) + + args = SimpleNamespace( + input=sample_file_path("rla", "streaming_live_vbs_1-0_0.bin"), + outdir=TEMP_DIR, + strict=False, + ) + try: + rla2json.main_rla2json(args) + except AssertionError: + pass + else: + raise AssertionError("missing RLH header should assert") diff --git a/tests/test_rla_error_handling.py b/tests/test_rla_error_handling.py new file mode 100644 index 0000000..22fed6f --- /dev/null +++ b/tests/test_rla_error_handling.py @@ -0,0 +1,186 @@ +from . import * + +import binascii + +# --- Packet builders ------------------------------------------------------- +# RTVL wire format: "RTVL" + 6 hex total length + base64 flag + split flag + +# 3 digit signature + body. Split bodies start with five zero-padded decimal +# fields: splitId(5) splitIndex(5) splitNum(5) dataLength(10) totalDataLength(10). +# A non-split body is signature + 5 byte payload header + streaming data; the +# 5 byte header's first byte after the signature selects raw vs gzip decoding. + + +def _split_packet(sid, idx, num, declared_total, payload, b64=True, sig=2): + info = b"%05d%05d%05d%010d%010d" % ( + sid, + idx, + num, + len(payload), + declared_total, + ) + body = info + payload + total = len(body) + 15 + flag = b"T" if b64 else b"F" + return b"RTVL" + b"%06X" % total + flag + b"T" + b"%03d" % sig + body + + +def _payload_body(streaming_payload, sig=0): + """Body of a non-split packet: decoder signature + streaming payload.""" + return bytes([sig]) + b"\x01\x00\x00\x00" + streaming_payload + + +def _payload_packet(streaming_payload, sig=0): + body = _payload_body(streaming_payload, sig) + total = len(body) + 15 + return b"RTVL" + b"%06X" % total + b"F" + b"F" + b"%03d" % sig + body + + +VALID_PAYLOAD = b"\x08\x00\x00\x00" + b"\x00\x00\x00\x00" + b"\x02\x00" + b"\x00\x00" +# pose count = 0x7fffffff with no pose data -> array reader hits EOF +TRUNCATED_PAYLOAD = ( + b"\x08\x00\x00\x00" + b"\x7f\xff\xff\xff" + b"\x02\x00" + b"\x00\x0c" +) + + +# --- Non-split packets ----------------------------------------------------- + + +def test_rla_garbage_is_skipped_when_not_strict(): + from sssekai.fmt.rla import read_rla_frames + + frames = list(read_rla_frames([(1, b"GARBAGE")], (1, 6), strict=False)) + assert frames == [] + + +def test_rla_garbage_raises_when_strict(): + from sssekai.fmt.rla import read_rla_frames + + try: + list(read_rla_frames([(1, b"GARBAGE")], (1, 6), strict=True)) + except AssertionError: + pass + else: + raise AssertionError("strict mode should raise on malformed packets") + + +def test_rla_payload_failure_is_skipped_when_not_strict(monkeypatch): + """A valid RTVL header with an undecodable payload must not abort the stream.""" + from sssekai.fmt import rla + + packet = _payload_packet(VALID_PAYLOAD) + calls = {"n": 0} + + def fake_decode_payload(buffer): + calls["n"] += 1 + raise ValueError("cannot decode payload") + + monkeypatch.setattr(rla, "decode_buffer_payload", fake_decode_payload) + frames = list(rla.read_rla_frames([(1, packet)], (1, 6), strict=False)) + assert frames == [] + assert calls["n"] == 1 + + +def test_rla_payload_failure_raises_when_strict(monkeypatch): + from sssekai.fmt import rla + + def fake_decode_payload(buffer): + raise ValueError("cannot decode payload") + + monkeypatch.setattr(rla, "decode_buffer_payload", fake_decode_payload) + packet = _payload_packet(VALID_PAYLOAD) + try: + list(rla.read_rla_frames([(1, packet)], (1, 6), strict=True)) + except ValueError: + pass + else: + raise AssertionError("strict mode should propagate payload errors") + + +def test_rla_truncated_payload_raises_truncated_error(): + """Truncation is reported distinctly so callers can tell it from corruption.""" + from sssekai.fmt.rla import StreamingDataTruncatedException, decode_streaming_data + + try: + decode_streaming_data((1, 6), 0, TRUNCATED_PAYLOAD, strict=False) + except StreamingDataTruncatedException: + pass + else: + raise AssertionError( + "truncated payload should raise StreamingDataTruncatedException" + ) + + +def test_rla_truncated_payload_drops_frame_but_keeps_stream(): + from sssekai.fmt.rla import read_rla_frames + + packets = [ + (1, _payload_packet(VALID_PAYLOAD)), + (2, _payload_packet(TRUNCATED_PAYLOAD)), + ] + frames = list(read_rla_frames(packets, (1, 6), strict=False)) + assert [(tick, f["type"]) for tick, f in frames] == [(1, "MotionData")] + + +# --- Split packets --------------------------------------------------------- + + +def test_rla_split_bad_base64_skipped_when_not_strict(): + from sssekai.fmt.rla import read_rla_frames + + # "abcde" is not valid base64 (length 5 mod 4 == 1) + packet = _split_packet(1, 0, 1, 5, b"abcde", b64=True) + assert list(read_rla_frames([(1, packet)], (1, 6), strict=False)) == [] + + +def test_rla_split_bad_base64_raises_when_strict(): + from sssekai.fmt.rla import read_rla_frames + + packet = _split_packet(1, 0, 1, 5, b"abcde", b64=True) + try: + list(read_rla_frames([(1, packet)], (1, 6), strict=True)) + except binascii.Error: + pass + else: + raise AssertionError("strict mode should raise on malformed split base64") + + +def test_rla_split_length_mismatch_skipped_when_not_strict(): + from sssekai.fmt.rla import read_rla_frames + + packet = _split_packet(1, 0, 1, 999, b"abcdef", b64=False) + assert list(read_rla_frames([(1, packet)], (1, 6), strict=False)) == [] + + +def test_rla_split_length_mismatch_raises_when_strict(): + from sssekai.fmt.rla import read_rla_frames + + packet = _split_packet(1, 0, 1, 999, b"abcdef", b64=False) + try: + list(read_rla_frames([(1, packet)], (1, 6), strict=True)) + except AssertionError as e: + assert "incomplete split packet" in str(e) + else: + raise AssertionError("strict mode should assert on split length mismatch") + + +def test_rla_split_group_is_cleared_after_skip(): + """A discarded group must not bleed into a later packet with the same id.""" + from sssekai.fmt.rla import read_rla_frames + + bad = _split_packet(7, 0, 1, 999, b"abcdef", b64=False) + good_body = _payload_body(VALID_PAYLOAD) + good = _split_packet(7, 0, 1, len(good_body), good_body, b64=False, sig=0) + frames = list(read_rla_frames([(1, bad), (2, good)], (1, 6), strict=False)) + assert [(tick, f["type"]) for tick, f in frames] == [(2, "MotionData")] + + +def test_rla_split_reassembles_in_index_order(): + from sssekai.fmt.rla import read_rla_frames + + body = _payload_body(VALID_PAYLOAD) + half = len(body) // 2 + p0 = _split_packet(3, 0, 2, len(body), body[:half], b64=False, sig=0) + p1 = _split_packet(3, 1, 2, len(body), body[half:], b64=False, sig=0) + # Delivered out of order; reassembly sorts by splitIndex. + frames = list(read_rla_frames([(2, p1), (1, p0)], (1, 6), strict=False)) + assert [f["type"] for _, f in frames] == ["MotionData"] diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..9e927e8 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,181 @@ +from . import * + +import base64 +import dataclasses +import json + +SECRET = "SUPER_SECRET_JWT_TOKEN_1234567890" + + +def _fake_jwt(user_id=1, signature="sig"): + payload = base64.b64encode(json.dumps({"userId": user_id}).encode()).decode() + return "e30." + payload.rstrip("=") + "." + signature + + +def _make_cache(credential=SECRET): + from sssekai.abcache import AbCache, AbCacheConfig + + return AbCache( + AbCacheConfig("jp", "1.0.0", "android", "hash", auth_credential=credential) + ) + + +def _make_secret_cache(): + """Credential + cached headers both carry the secret.""" + cache = _make_cache(_fake_jwt()) + cache.config.auth_credential = SECRET + cache.database.cached_headers["X-Session-Token"] = SECRET + cache.database.cached_headers["Cookie"] = SECRET + return cache + + +def test_config_repr_hides_credential(): + cache = _make_cache() + assert SECRET not in repr(cache.config) + assert SECRET not in repr(cache) + + +def test_user_data_repr_hides_tokens(): + from sssekai.abcache import ( + SekaiUserAuthData, + SekaiUserData, + SekaiUserRegistrationData, + ) + + user = SekaiUserData( + userRegistration=SekaiUserRegistrationData(userId=1), + credential=SECRET, + sessionToken=SECRET, + ) + auth = SekaiUserAuthData( + sessionToken=SECRET, + appVersion="1.0.0", + multiPlayVersion="1.0.0", + dataVersion="1.0.0", + assetVersion="1.0.0", + ) + assert SECRET not in repr(user) + assert SECRET not in repr(auth) + + +def test_database_repr_and_asdict_hide_cached_headers(): + cache = _make_secret_cache() + assert SECRET not in repr(cache.database) + asdict = dataclasses.asdict(cache.database) + assert SECRET not in repr(asdict) + # asdict's secret-free view must still cover the token-bearing field. + assert "cached_headers" in asdict + + +def test_legacy_plain_dict_headers_are_normalized_on_load(caplog): + """Caches written by older versions store a plain dict in cached_headers, + and save() replaces that field with the live session headers. So the token + must be put on the session headers to actually reach the cache file.""" + import io + import pickle + + caplog.set_level(DEBUG) + cache = _make_secret_cache() + cache.headers["X-Session-Token"] = SECRET + cache.headers["Cookie"] = SECRET + buf = io.BytesIO() + cache.save(buf) + buf.seek(0) + # Round-trip the raw pickle with the header field demoted to a plain dict, + # exactly as an older sssekai wrote it. + database = pickle.load(buf) + database.cached_headers = dict(database.cached_headers) + assert SECRET in repr(database.cached_headers) + buf = io.BytesIO(pickle.dumps(database)) + from sssekai.abcache import AbCache, RedactedHeaders + + loaded = AbCache.from_file(buf) + assert isinstance(loaded.database.cached_headers, RedactedHeaders) + assert dict(loaded.database.cached_headers)["X-Session-Token"] == SECRET + assert SECRET not in repr(loaded.database) + assert SECRET not in repr(dataclasses.asdict(loaded.database)) + assert SECRET not in caplog.text + + +def test_save_and_load_do_not_log_credential(caplog): + import io + + from sssekai.abcache import AbCache + + cache = _make_secret_cache() + caplog.clear() + caplog.set_level(DEBUG) + buf = io.BytesIO() + cache.save(buf) + buf.seek(0) + AbCache.from_file(buf) + assert SECRET not in caplog.text + + +def test_registered_user_data_is_dataclass_and_not_leaked(caplog, monkeypatch): + from sssekai.abcache import SekaiUserData + from sssekai.abcache.auth import register_as_anonymous_user_sega + + cache = _make_cache(credential=None) + cache.headers["X-Platform"] = "Android" + cache.headers["X-DeviceModel"] = "test" + cache.headers["X-OperatingSystem"] = "Android" + + monkeypatch.setattr(cache, "_update_request_headers", lambda: None) + monkeypatch.setattr(cache, "update_signatures", lambda: None) + monkeypatch.setattr(cache, "request_packed", lambda *a, **kw: object()) + monkeypatch.setattr( + cache, + "response_to_dict", + lambda resp: { + "credential": _fake_jwt(signature=SECRET), + "userRegistration": {"userId": 1}, + }, + ) + + caplog.clear() + caplog.set_level(DEBUG) + register_as_anonymous_user_sega(cache) + assert isinstance(cache.database.sekai_user_data, SekaiUserData) + # The credential must really be present, otherwise the assertions below + # would pass vacuously. + assert SECRET in cache.database.sekai_user_data.credential + asdict = dataclasses.asdict(cache.database) + assert SECRET not in repr(cache.database) + assert SECRET not in repr(asdict) + assert SECRET not in caplog.text + + +def test_abserve_host_defaults_to_loopback(): + from sssekai.__main__ import create_parser + + parser = create_parser() + args = parser.parse_args(["abserve"]) + assert args.host == "127.0.0.1" + + +def test_legacy_dict_user_data_is_normalized_and_redacted(caplog): + """sssekai <= 0.8.2 stored the raw API response dict in sekai_user_data.""" + import io + import pickle + + from sssekai.abcache import AbCache, AbCacheConfig, SekaiUserData + + SECRET_LOCAL = "LEGACY_CREDENTIAL_XYZ" + cache = AbCache(AbCacheConfig("jp", "1", "android", "h")) + cache.database.sekai_user_data = { + "credential": SECRET_LOCAL, + "userRegistration": {"userId": 1}, + } + buf = io.BytesIO() + pickle.dump(cache.database, buf) + buf.seek(0) + + caplog.clear() + caplog.set_level(DEBUG) + loaded = AbCache.from_file(buf) + assert isinstance(loaded.database.sekai_user_data, SekaiUserData) + assert loaded.SEKAI_CREDENTIAL == SECRET_LOCAL + assert SECRET_LOCAL not in repr(loaded.database) + assert SECRET_LOCAL not in repr(dataclasses.asdict(loaded.database)) + assert SECRET_LOCAL not in caplog.text