From a71b564fec94302c7bef0a7590595c05f303d85f Mon Sep 17 00:00:00 2001 From: ht5161143 Date: Thu, 27 Aug 2026 10:26:50 +0000 Subject: [PATCH 1/2] feat(ytmusic): add YouTube Music account import Adds a YouTube Music source alongside the existing Spotify/Tidal/Qobuz/ Deezer playlist sources: browse the signed-in account's own library playlists (plus a virtual "Liked Music" entry, pinned first) and feed them through the existing discovery -> sync -> download pipeline. Auth reuses the existing Settings -> YouTube "Paste cookies.txt" flow; no new credential UI or settings surface. Adds __Secure-1PSIDTS / __Secure-3PSIDTS to the essential-cookie allowlist, which YouTube Music's privacy-sensitive endpoints (Liked Music) require even when SAPISIDHASH auth otherwise verifies. Backend: core/ytmusic_library.py (account listing), the YTMusicPlaylistSource adapter, and the /api/ytmusic/* route family in api/source_playlists.py, which reuses the YouTube discovery worker verbatim (identical track shape) rather than duplicating it. Also adds a bounded retry around ytmusicapi's get_playlist pagination, which can silently truncate on a bad continuation page (temporary, pending a fix upstream in ytmusicapi). Frontend: a routed 'ytmusic' vertical in the Sync page, following the same table-driven pattern as Tidal/Qobuz. --- api/source_playlists.py | 396 ++++++++++++++++++ core/playlists/sources/base.py | 2 + core/playlists/sources/bootstrap.py | 9 + core/playlists/sources/ytmusic.py | 127 ++++++ core/youtube_cookies.py | 4 + core/youtube_music_meta.py | 74 +++- core/ytmusic_library.py | 199 +++++++++ tests/test_playlist_sources_adapters.py | 93 ++++ tests/test_youtube_cookies.py | 15 + tests/test_youtube_music_meta.py | 100 +++++ tests/test_ytmusic_library.py | 196 +++++++++ web_server.py | 20 +- webui/src/routes/sync/-sync.api.test.ts | 3 + webui/src/routes/sync/-sync.api.ts | 30 +- webui/src/routes/sync/-sync.modal-core.ts | 3 + webui/src/routes/sync/-sync.shell.test.ts | 15 +- webui/src/routes/sync/-sync.shell.ts | 6 +- webui/src/routes/sync/-sync.sources.test.ts | 6 +- webui/src/routes/sync/-sync.sources.ts | 32 ++ webui/src/routes/sync/-sync.use-page.test.ts | 4 +- webui/src/routes/sync/-sync.verticals.test.ts | 4 +- webui/src/routes/sync/-sync.verticals.ts | 24 +- .../src/routes/sync/-ui/account-tab.test.tsx | 109 ++++- webui/src/routes/sync/-ui/account-tab.tsx | 71 +++- webui/src/routes/sync/-ui/sync-page.tsx | 8 +- webui/src/routes/sync/-ui/sync-shell.test.tsx | 20 + webui/src/routes/sync/-ui/sync-shell.tsx | 2 +- webui/static/style.css | 4 + 28 files changed, 1522 insertions(+), 54 deletions(-) create mode 100644 core/playlists/sources/ytmusic.py create mode 100644 core/ytmusic_library.py create mode 100644 tests/test_ytmusic_library.py diff --git a/api/source_playlists.py b/api/source_playlists.py index 4c8be0c9b..d5393ad31 100644 --- a/api/source_playlists.py +++ b/api/source_playlists.py @@ -86,6 +86,7 @@ _tidal_enrichment_worker = None _dev_mode_enabled = None _get_automation_deps = None +_ytmusic_auth_headers = None def configure(**deps): @@ -4636,6 +4637,401 @@ def convert_youtube_results_to_spotify_tracks(discovery_results): return convert_results_to_spotify_tracks(discovery_results, "YouTube") +# =================================================================== +# YOUTUBE MUSIC (ACCOUNT) API ENDPOINTS +# =================================================================== +# +# Distinct from the /api/youtube/* family above: that one parses a PASTED +# playlist URL. This one lists the SIGNED-IN account's own library playlists +# (core.ytmusic_library) and fetches individual playlists by id +# (core.youtube_music_meta.fetch_ytmusic_playlist) + +# Global state for the YouTube Music management (persistent across page reloads) +ytmusic_discovery_states = {} # Key: YT Music playlist id ('LM' for Liked Music), Value: persistent playlist state +ytmusic_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="ytmusic_discovery") + + +@bp.route('/api/ytmusic/playlists', methods=['GET']) +def get_ytmusic_playlists(): + """List the signed-in YouTube Music account's own library playlists, Liked Music first.""" + from core.ytmusic_library import fetch_library_playlists, fetch_liked_music_row, library_playlists_to_rows + auth = _ytmusic_auth_headers() + if not auth: + return jsonify({"error": "YouTube Music not authenticated. Settings → Downloads → Download Source: YouTube Only → Paste cookies.txt → Save."}), 401 + try: + rows = library_playlists_to_rows(fetch_library_playlists(auth)) + liked_row = fetch_liked_music_row(auth) + if liked_row: + rows.insert(0, liked_row) + + playlist_data = [{ + "id": row["id"], + "name": row["name"], + "owner": row.get("owner") or "Unknown", + "track_count": row.get("track_count", 0), + "image_url": row.get("image_url"), + "description": row.get("description") or "", + "tracks": [], + } for row in rows] + + logger.info(f"Loaded {len(playlist_data)} YouTube Music playlists") + return jsonify(playlist_data) + except Exception as e: + logger.error(f"Error getting YouTube Music playlists: {e}") + return jsonify({"error": str(e)}), 500 + + +@bp.route('/api/ytmusic/playlist/', methods=['GET']) +def get_ytmusic_playlist_tracks(playlist_id): + """Fetch full track details for a specific YouTube Music playlist.""" + from core.youtube_music_meta import fetch_ytmusic_playlist + from core.ytmusic_library import ytmusic_playlist_url + auth = _ytmusic_auth_headers() + if not auth: + return jsonify({"error": "YouTube Music not authenticated."}), 401 + try: + data = fetch_ytmusic_playlist(ytmusic_playlist_url(playlist_id), auth) + if not data: + return jsonify({"error": "Playlist not found or unable to access. This may be due to privacy settings or account restrictions."}), 404 + if not data.get('tracks'): + return jsonify({"error": "This playlist appears to have no tracks or they cannot be accessed"}), 403 + + playlist_dict = { + 'id': data.get('id', playlist_id), + 'name': data.get('name', 'YouTube Music Playlist'), + 'description': '', + 'owner': 'You', + 'track_count': data.get('track_count', len(data['tracks'])), + 'image_url': data.get('image_url'), + 'tracks': data['tracks'], + } + logger.info(f"Loaded {len(data['tracks'])} tracks from YouTube Music playlist: {playlist_dict['name']}") + return jsonify(playlist_dict) + except Exception as e: + logger.error(f"Error getting YouTube Music playlist tracks: {e}") + return jsonify({"error": str(e)}), 500 + + +@bp.route('/api/ytmusic/discovery/start/', methods=['POST']) +def start_ytmusic_discovery(playlist_id): + """Start Spotify discovery process for a YouTube Music playlist""" + from core.youtube_music_meta import fetch_ytmusic_playlist + from core.ytmusic_library import ytmusic_playlist_url + try: + auth = _ytmusic_auth_headers() + if not auth: + return jsonify({"error": "YouTube Music not authenticated."}), 401 + + # Fetch this single playlist fresh — no need to re-fetch the whole library. + target_playlist = fetch_ytmusic_playlist(ytmusic_playlist_url(playlist_id), auth) + + if not target_playlist: + return jsonify({"error": "YouTube Music playlist not found"}), 404 + + if not target_playlist.get('tracks'): + return jsonify({"error": "Playlist has no tracks"}), 400 + + if playlist_id in ytmusic_discovery_states: + state = ytmusic_discovery_states[playlist_id] + if state['phase'] == 'discovering': + return jsonify({"error": "Discovery already in progress"}), 400 + state['playlist'] = target_playlist + state['phase'] = 'discovering' + state['status'] = 'discovering' + state['discovery_progress'] = 0 + state['spotify_matches'] = 0 + state['spotify_total'] = len(target_playlist['tracks']) + state['discovery_results'] = [] + state['last_accessed'] = time.time() + else: + state = { + 'playlist': target_playlist, + 'phase': 'discovering', # discovering -> discovered -> syncing -> sync_complete -> downloading -> download_complete + 'status': 'discovering', + 'discovery_progress': 0, + 'spotify_matches': 0, + 'spotify_total': len(target_playlist['tracks']), + 'discovery_results': [], + 'sync_playlist_id': None, + 'converted_spotify_playlist_id': None, + 'download_process_id': None, + 'created_at': time.time(), + 'last_accessed': time.time(), + 'discovery_future': None, + 'sync_progress': {} + } + ytmusic_discovery_states[playlist_id] = state + + add_activity_item("", "YouTube Music Discovery Started", f"'{target_playlist['name']}' - {len(target_playlist['tracks'])} tracks", "Now") + + future = ytmusic_discovery_executor.submit(_run_ytmusic_discovery_worker, playlist_id) + state['discovery_future'] = future + + logger.info(f"Started Spotify discovery for YouTube Music playlist: {target_playlist['name']}") + return jsonify({"success": True, "message": "Discovery started"}) + + except Exception as e: + logger.error(f"Error starting YouTube Music discovery: {e}") + return jsonify({"error": str(e)}), 500 + + +@bp.route('/api/ytmusic/discovery/status/', methods=['GET']) +def get_ytmusic_discovery_status(playlist_id): + """Get real-time discovery status for a YouTube Music playlist""" + return _get_source_discovery_status(ytmusic_discovery_states, playlist_id, "YouTube Music discovery not found", "YouTube Music") + + +@bp.route('/api/ytmusic/discovery/update_match', methods=['POST']) +def update_ytmusic_discovery_match(): + """Update a YouTube Music discovery result with manually selected Spotify track.""" + try: + data = request.get_json() + identifier = data.get('identifier') # playlist_id (bare or mirrored_) + track_index = data.get('track_index') + spotify_track = data.get('spotify_track') + + if not identifier or track_index is None or not spotify_track: + return jsonify({'error': 'Missing required fields'}), 400 + + state = ytmusic_discovery_states.get(identifier) + + if not state: + return jsonify({'error': 'Discovery state not found'}), 404 + + if track_index >= len(state['discovery_results']): + return jsonify({'error': 'Invalid track index'}), 400 + + result = state['discovery_results'][track_index] + old_status = result.get('status') + + result['status'] = 'Found' + result['status_class'] = 'found' + result['spotify_track'] = spotify_track['name'] + result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) + result['spotify_album'] = spotify_track['album'] + result['spotify_id'] = spotify_track['id'] + + duration_ms = spotify_track.get('duration_ms', 0) + if duration_ms: + minutes = duration_ms // 60000 + seconds = (duration_ms % 60000) // 1000 + result['duration'] = f"{minutes}:{seconds:02d}" + else: + result['duration'] = '0:00' + + result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) + result['wing_it_fallback'] = False + result['manual_match'] = True + + if old_status != 'found' and old_status != 'Found': + state['spotify_matches'] = state.get('spotify_matches', 0) + 1 + + logger.info(f"Manual match updated: ytmusic - {identifier} - track {track_index}") + logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") + + from core.discovery.manual_match import derive_manual_match_provider + match_source = derive_manual_match_provider( + spotify_track, _get_active_discovery_source() + ) + matched_data = None + + try: + original_track = result.get('youtube_track', {}) + original_name = original_track.get('name', spotify_track['name']) + original_artists = original_track.get('artists', []) + if original_artists: + original_artist = original_artists[0] if isinstance(original_artists[0], str) else original_artists[0].get('name', '') + else: + original_artist = '' + + cache_key = _get_discovery_cache_key(original_name, original_artist) + artists_list = spotify_track['artists'] + if isinstance(artists_list, list): + artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] + image_url = spotify_track.get('image_url') or '' + album_raw = spotify_track.get('album', '') + if isinstance(album_raw, dict): + album_obj = dict(album_raw) + if image_url and not album_obj.get('image_url'): + album_obj['image_url'] = image_url + if image_url and not album_obj.get('images'): + album_obj['images'] = [{'url': image_url}] + else: + album_obj = {'name': album_raw or ''} + if image_url: + album_obj['image_url'] = image_url + album_obj['images'] = [{'url': image_url}] + + matched_data = { + 'id': spotify_track['id'], + 'name': spotify_track['name'], + 'artists': artists_list, + 'album': album_obj, + 'duration_ms': spotify_track.get('duration_ms', 0), + 'image_url': image_url, + 'source': match_source, + } + cache_db = get_database() + cache_db.save_discovery_cache_match( + cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, + original_name, original_artist + ) + logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") + except Exception as cache_err: + logger.error(f"Error saving manual fix to discovery cache: {cache_err}") + + if matched_data is not None and identifier.startswith('mirrored_'): + try: + tracks = state['playlist']['tracks'] + if track_index < len(tracks): + db_track_id = tracks[track_index].get('db_track_id') + if db_track_id: + db = get_database() + extra_data = { + 'discovered': True, + 'provider': match_source, + 'confidence': 1.0, + 'matched_data': matched_data, + 'manual_match': True, + 'wing_it_fallback': False, + 'unmatched_by_user': False, + } + db.update_mirrored_track_extra_data(db_track_id, extra_data) + result['matched_data'] = matched_data + logger.info(f"Persisted manual fix to DB for track {db_track_id}") + except Exception as wb_err: + logger.error(f"Error persisting manual fix to DB: {wb_err}") + + return jsonify({'success': True, 'result': result}) + + except Exception as e: + logger.error(f"Error updating YouTube Music discovery match: {e}") + return jsonify({'error': str(e)}), 500 + + +@bp.route('/api/ytmusic/playlists/states', methods=['GET']) +def get_ytmusic_playlist_states(): + """Get all stored YouTube Music discovery states for frontend hydration""" + return _get_source_playlist_states(ytmusic_discovery_states, "YouTube Music", "YouTube Music") + + +@bp.route('/api/ytmusic/state/', methods=['GET']) +def get_ytmusic_playlist_state(playlist_id): + """Get specific YouTube Music playlist state (detailed version of the status endpoint)""" + try: + if playlist_id not in ytmusic_discovery_states: + return jsonify({"error": "YouTube Music playlist not found"}), 404 + + state = ytmusic_discovery_states[playlist_id] + state['last_accessed'] = time.time() + + response = { + 'playlist_id': playlist_id, + 'playlist': state['playlist'], + 'phase': state['phase'], + 'status': state['status'], + 'discovery_progress': state['discovery_progress'], + 'spotify_matches': state['spotify_matches'], + 'spotify_total': state['spotify_total'], + 'discovery_results': state['discovery_results'], + 'sync_playlist_id': state.get('sync_playlist_id'), + 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), + 'download_process_id': state.get('download_process_id'), + 'sync_progress': state.get('sync_progress', {}), + 'created_at': state['created_at'], + 'last_accessed': state['last_accessed'] + } + + return jsonify(response) + + except Exception as e: + logger.error(f"Error getting YouTube Music playlist state: {e}") + return jsonify({"error": str(e)}), 500 + + +@bp.route('/api/ytmusic/reset/', methods=['POST']) +def reset_ytmusic_playlist(playlist_id): + """Reset YouTube Music playlist to fresh phase (clear discovery/sync data)""" + return _reset_source_playlist(ytmusic_discovery_states, playlist_id, "YouTube Music", "YouTube Music playlist not found") + + +@bp.route('/api/ytmusic/delete/', methods=['POST']) +def delete_ytmusic_playlist(playlist_id): + """Delete YouTube Music playlist state completely""" + return _delete_source_playlist(ytmusic_discovery_states, playlist_id, "YouTube Music", "YouTube Music playlist not found") + + +@bp.route('/api/ytmusic/update_phase/', methods=['POST']) +def update_ytmusic_playlist_phase(playlist_id): + """Update YouTube Music playlist phase (used when modal closes to reset from download_complete to discovered)""" + return _update_source_playlist_phase(ytmusic_discovery_states, playlist_id, "YouTube Music playlist not found", "YouTube Music", _PHASE_LIST, False) + + +def _build_ytmusic_discovery_deps(): + """Build the YoutubeDiscoveryDeps bundle pointed at ytmusic_discovery_states.""" + return _discovery_youtube.YoutubeDiscoveryDeps( + youtube_playlist_states=ytmusic_discovery_states, + spotify_client=_spotify_client(), + matching_engine=_matching_engine(), + pause_enrichment_workers=_pause_enrichment_workers, + resume_enrichment_workers=_resume_enrichment_workers, + get_active_discovery_source=_get_active_discovery_source, + get_metadata_fallback_client=_get_metadata_fallback_client, + get_discovery_cache_key=_get_discovery_cache_key, + validate_discovery_cache_artist=_validate_discovery_cache_artist, + extract_artist_name=_extract_artist_name, + spotify_rate_limited=_spotify_rate_limited, + discovery_score_candidates=_discovery_score_candidates, + get_metadata_cache=get_metadata_cache, + build_discovery_wing_it_stub=_build_discovery_wing_it_stub, + get_database=get_database, + add_activity_item=add_activity_item, + # No per-video artist recovery here: fetch_ytmusic_playlist already + # gets a real artist from the catalog API, unlike yt-dlp's flat + # extraction (which is what recover_youtube_artist works around). + recover_youtube_artist=None, + ) + + +def _run_ytmusic_discovery_worker(playlist_id): + return _discovery_youtube.run_youtube_discovery_worker(playlist_id, _build_ytmusic_discovery_deps()) + + +def convert_ytmusic_results_to_spotify_tracks(discovery_results): + """Convert YouTube Music discovery results to Spotify tracks format for sync""" + return convert_results_to_spotify_tracks(discovery_results, "YouTube Music") + + +@bp.route('/api/ytmusic/sync/start/', methods=['POST']) +def start_ytmusic_sync(playlist_id): + """Start sync process for a YouTube Music playlist using discovered Spotify tracks""" + # Unlike start_youtube_sync, mirrored playlists never reach this endpoint: + # the frontend's 'mirrored' vertical routes every mirrored sync through + # /api/youtube/* regardless of the original source, and + # api/mirrored_playlists.py only ever registers into + # youtube_playlist_states — so no mirrored_ pipeline-collision guard + # is needed here. + return _start_source_sync( + ytmusic_discovery_states, playlist_id, sync_id_prefix="ytmusic", + not_found_message="YouTube Music playlist not found", + not_ready_message="YouTube Music playlist not ready for sync", + convert_fn=convert_ytmusic_results_to_spotify_tracks, + name_getter=_pl_name_strict, image_getter=_pl_image_dict, + activity_label="YouTube Music", error_label="YouTube Music") + + +@bp.route('/api/ytmusic/sync/status/', methods=['GET']) +def get_ytmusic_sync_status(playlist_id): + """Get sync status for a YouTube Music playlist""" + return _get_source_sync_status(ytmusic_discovery_states, playlist_id, "YouTube Music playlist not found", "YouTube Music", "YouTube Music playlist", _pl_name_safe) + + +@bp.route('/api/ytmusic/sync/cancel/', methods=['POST']) +def cancel_ytmusic_sync(playlist_id): + """Cancel sync for a YouTube Music playlist""" + return _cancel_source_sync(ytmusic_discovery_states, playlist_id, "YouTube Music", "YouTube Music playlist not found") + + # Add these new endpoints to the end of web_server.py # Sync background worker logic lives in core/discovery/sync.py. diff --git a/core/playlists/sources/base.py b/core/playlists/sources/base.py index dda634165..4dc8447f9 100644 --- a/core/playlists/sources/base.py +++ b/core/playlists/sources/base.py @@ -38,6 +38,7 @@ SOURCE_TIDAL = "tidal" SOURCE_QOBUZ = "qobuz" SOURCE_YOUTUBE = "youtube" +SOURCE_YTMUSIC = "ytmusic" SOURCE_ITUNES_LINK = "itunes_link" SOURCE_LISTENBRAINZ = "listenbrainz" SOURCE_LASTFM = "lastfm" @@ -50,6 +51,7 @@ SOURCE_TIDAL, SOURCE_QOBUZ, SOURCE_YOUTUBE, + SOURCE_YTMUSIC, SOURCE_ITUNES_LINK, SOURCE_LISTENBRAINZ, SOURCE_LASTFM, diff --git a/core/playlists/sources/bootstrap.py b/core/playlists/sources/bootstrap.py index 1d2aa77b3..0141e96cd 100644 --- a/core/playlists/sources/bootstrap.py +++ b/core/playlists/sources/bootstrap.py @@ -23,6 +23,7 @@ SOURCE_SPOTIFY_PUBLIC, SOURCE_TIDAL, SOURCE_YOUTUBE, + SOURCE_YTMUSIC, ) from core.playlists.sources.deezer import DeezerPlaylistSource from core.playlists.sources.itunes_link import ITunesLinkPlaylistSource @@ -37,6 +38,7 @@ from core.playlists.sources.spotify_public import SpotifyPublicPlaylistSource from core.playlists.sources.tidal import TidalPlaylistSource from core.playlists.sources.youtube import YouTubePlaylistSource +from core.playlists.sources.ytmusic import YTMusicPlaylistSource def build_playlist_source_registry( @@ -47,6 +49,7 @@ def build_playlist_source_registry( deezer_client_getter: Callable[[], Any], itunes_link_parser: Optional[Callable[[str], Optional[dict]]] = None, youtube_parser: Optional[Callable[[str], Optional[dict]]] = None, + ytmusic_auth_getter: Optional[Callable[[], Optional[dict]]] = None, listenbrainz_manager_getter: Optional[Callable[[], Any]] = None, lastfm_manager_getter: Optional[Callable[[], Any]] = None, personalized_manager_getter: Optional[Callable[[], Any]] = None, @@ -78,6 +81,12 @@ def build_playlist_source_registry( lambda: ITunesLinkPlaylistSource(itunes_link_parser or _no_url_parser), ) + _no_auth = lambda: None + reg.register( + SOURCE_YTMUSIC, + lambda: YTMusicPlaylistSource(ytmusic_auth_getter or _no_auth), + ) + _no_manager = lambda: None reg.register( SOURCE_LISTENBRAINZ, diff --git a/core/playlists/sources/ytmusic.py b/core/playlists/sources/ytmusic.py new file mode 100644 index 000000000..2b728a986 --- /dev/null +++ b/core/playlists/sources/ytmusic.py @@ -0,0 +1,127 @@ +"""YouTube Music (account) playlist source adapter. + +This adapter is the signed-in account vertical: it returns the account's +own library playlists plus a virtual "Liked Music" entry. Auth reuses the +existing Settings -> YouTube cookies. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Optional + +from core.playlists.sources.base import ( + NormalizedTrack, + PlaylistDetail, + PlaylistMeta, + PlaylistSource, + SOURCE_YTMUSIC, +) +from core.youtube_music_meta import fetch_ytmusic_playlist +from core.ytmusic_library import ( + fetch_liked_music_row, + fetch_library_playlists, + library_playlists_to_rows, + ytmusic_playlist_url, +) + + +class YTMusicPlaylistSource(PlaylistSource): + name = SOURCE_YTMUSIC + supports_listing = True + supports_refresh = True + requires_auth = True + + def __init__(self, auth_getter: Callable[[], Optional[Dict[str, str]]]): + """``auth_getter`` matches ``web_server._ytmusic_auth_headers`` — + zero-arg, returns a ytmusicapi browser-auth header dict or ``None`` + when Settings -> YouTube has no cookies configured. Injected (not + called eagerly) for the same late-binding reason every other + adapter here takes a getter.""" + self._auth_getter = auth_getter + + def _auth(self) -> Optional[Dict[str, str]]: + try: + return self._auth_getter() + except Exception: + return None + + def is_authenticated(self) -> bool: + return bool(self._auth()) + + def list_playlists(self) -> List[PlaylistMeta]: + auth = self._auth() + if not auth: + return [] + + raw = fetch_library_playlists(auth) + rows = library_playlists_to_rows(raw) + metas = [self._meta_from_row(row) for row in rows] + + # Virtual "Liked Music" playlist, pinned FIRST, count-only — matches + # YouTube Music's own UI, where it's the prominent/first library + # entry (deliberately unlike Spotify's "Liked Songs" / Tidal's + # "Favorite Tracks" in this app, which are appended at the end — + # a per-source call, not a shared convention). Omitted entirely + # when there's nothing liked yet. + liked_row = fetch_liked_music_row(auth) + if liked_row: + metas.insert(0, self._meta_from_row(liked_row)) + + return metas + + def get_playlist(self, playlist_id: str) -> Optional[PlaylistDetail]: + auth = self._auth() + if not auth: + return None + data = fetch_ytmusic_playlist(ytmusic_playlist_url(playlist_id), auth) + if not data: + return None + + tracks_raw = data.get("tracks") or [] + meta = PlaylistMeta( + source=self.name, + source_playlist_id=playlist_id, + name=data.get("name", "YouTube Music Playlist"), + track_count=int(data.get("track_count", len(tracks_raw))), + image_url=data.get("image_url") or None, + source_url=data.get("url") or ytmusic_playlist_url(playlist_id), + ) + tracks = [self._track_from_yt(t, idx) for idx, t in enumerate(tracks_raw) if t] + return PlaylistDetail(meta=meta, tracks=tracks) + + def refresh_playlist(self, playlist_id: str) -> Optional[PlaylistDetail]: + return self.get_playlist(playlist_id) + + # ---- projection helpers ------------------------------------------------ + + def _meta_from_row(self, row: Dict[str, Any]) -> PlaylistMeta: + playlist_id = str(row["id"]) + return PlaylistMeta( + source=self.name, + source_playlist_id=playlist_id, + name=row["name"], + owner=row.get("owner"), + description=row.get("description"), + image_url=row.get("image_url"), + track_count=int(row.get("track_count") or 0), + source_url=ytmusic_playlist_url(playlist_id), + ) + + def _track_from_yt(self, track: dict, position: int) -> NormalizedTrack: + artists = track.get("artists") or [] + artist_name = artists[0] if artists else "Unknown Artist" + return NormalizedTrack( + position=position, + track_name=track.get("name", "Unknown Track"), + artist_name=artist_name, + album_name=(track.get("album") or "").strip() or None, + duration_ms=int(track.get("duration_ms", 0) or 0), + source_track_id=str(track.get("id", "")), + needs_discovery=False, + extra={ + "url": track.get("url"), + "raw_title": track.get("raw_title"), + "raw_artist": track.get("raw_artist"), + "video_type": track.get("video_type"), + }, + ) diff --git a/core/youtube_cookies.py b/core/youtube_cookies.py index 495373f0c..4ce88c788 100644 --- a/core/youtube_cookies.py +++ b/core/youtube_cookies.py @@ -37,10 +37,14 @@ # A browser export can carry 90 KB+ of cookies across every Google property, # and YouTube rejects a request whose headers are that large (HTTP 413). Only # these actually matter for auth. +# +# __Secure-1PSIDTS / __Secure-3PSIDTS are rotating session-refresh tokens +# Google now binds the SID/SAPISID family to. _ESSENTIAL_COOKIES = frozenset({ "APISID", "HSID", "SSID", "SID", "SAPISID", "__Secure-1PAPISID", "__Secure-3PAPISID", "__Secure-1PSID", "__Secure-3PSID", + "__Secure-1PSIDTS", "__Secure-3PSIDTS", "LOGIN_INFO", "PREF", "SOCS", "VISITOR_INFO1_LIVE", "YSC", }) diff --git a/core/youtube_music_meta.py b/core/youtube_music_meta.py index e578d1f77..52262cd41 100644 --- a/core/youtube_music_meta.py +++ b/core/youtube_music_meta.py @@ -272,6 +272,76 @@ def ytmusic_playlist_to_payload( } +# TEMPORARY — remove once ytmusicapi ships get_playlist(validate_responses=...). +# +# ytmusicapi's plain continuation loop (ytmusicapi/continuations.py, +# get_continuations()) silently stops paginating on the FIRST malformed or +# empty continuation page — no retry, no exception, just `break`. A transient +# hiccup on any one page of a large playlist truncates the whole result, and +# the caller has no way to tell "genuinely done" apart from "gave up early". +# +# ytmusicapi already has the real fix for this shape of bug — +# get_validated_continuations() in the same file, which retries a short page +# up to 3 times before accepting it — and get_library_songs() already takes +# a validate_responses flag that uses it. get_playlist() doesn't expose that +# flag yet: sigma67/ytmusicapi#778 (bug report) / #953 (fix PR) are open but +# unmerged as of writing. Once #953 ships, delete _get_playlist_paginated +# below and go back to calling client.get_playlist(...) directly with +# validate_responses=True. +# +# CALIBRATION — trackCount counts entries this endpoint can never return a +# row for (deleted / region-blocked videos; ytmusic_playlist_to_payload +# already drops those placeholder rows on purpose), so a SMALL gap is +# normal, not truncation, and retrying never closes it — a retry against a +# stable small gap just re-fetches the same result at the cost of a slow, +# blocking request. A genuine truncation looks nothing like that: a severe +# shortfall that a single retry recovers from. The threshold below is set to +# catch the second shape and leave the first alone. +_YTMUSIC_PAGINATION_COMPLETE_RATIO = 0.9 +_YTMUSIC_PAGINATION_RETRY_ATTEMPTS = 2 + + +def _get_playlist_paginated(client: Any, playlist_id: str) -> Dict[str, Any]: + """``client.get_playlist(playlist_id, limit=None)``, retried when the + result looks TRUNCATED (not just short). See the TEMPORARY note above. + + ``trackCount`` comes from the playlist HEADER (fetched before pagination + starts); ``len(tracks)`` is what pagination actually produced. A small + gap between them is normal (see CALIBRATION above) and is accepted + as-is; only a gap below ``_YTMUSIC_PAGINATION_COMPLETE_RATIO`` is treated + as a truncation bug worth retrying. This is a mitigation, not a fix: it + cannot guarantee completeness, only make a severely-short result less + likely, and it gives up after a couple of attempts rather than retrying + forever against a page ytmusicapi genuinely can't get past. + """ + best: Optional[Dict[str, Any]] = None + best_count = -1 + for attempt in range(1, _YTMUSIC_PAGINATION_RETRY_ATTEMPTS + 1): + try: + raw = client.get_playlist(playlist_id, limit=None) + except Exception: + # A later attempt failing shouldn't throw away a good earlier + # one; the FIRST attempt failing is a real failure and should + # propagate so the caller falls back to yt-dlp, same as before + # this wrapper existed. + if best is not None: + break + raise + + tracks = raw.get("tracks") or [] + got = len(tracks) + declared = raw.get("trackCount") + if got > best_count: + best, best_count = raw, got + if not isinstance(declared, int) or declared <= 0 or got >= declared * _YTMUSIC_PAGINATION_COMPLETE_RATIO: + break # complete, a normal small gap, or no ground truth to compare against + logger.info( + "YouTube Music pagination looked truncated (%d/%s tracks) for %s — retrying (%d/%d)", + got, declared, playlist_id, attempt, _YTMUSIC_PAGINATION_RETRY_ATTEMPTS, + ) + return best + + def fetch_ytmusic_playlist( url: str, auth: Optional[Dict[str, str]] = None ) -> Optional[Dict[str, Any]]: @@ -298,7 +368,9 @@ def fetch_ytmusic_playlist( try: client = YTMusic(auth) if auth else YTMusic() # limit=None pages the whole playlist; the default stops at 100. - raw = client.get_playlist(playlist_id, limit=None) + # _get_playlist_paginated retries a short result — see its TEMPORARY + # note above; delete once ytmusicapi#953 ships. + raw = _get_playlist_paginated(client, playlist_id) except Exception as e: # noqa: BLE001 - see docstring: all failures fall back logger.info( "YouTube Music lookup failed for %s (%s: %s) — falling back to yt-dlp", diff --git a/core/ytmusic_library.py b/core/ytmusic_library.py new file mode 100644 index 000000000..0d9476d77 --- /dev/null +++ b/core/ytmusic_library.py @@ -0,0 +1,199 @@ +"""Read the signed-in YouTube Music account's own library via ytmusicapi. + +Companion to ``core.youtube_music_meta``, which reads a SINGLE playlist by +URL (anonymous or authenticated). This module answers a different +question — "what playlists does THIS account have?" — which is what +``YouTubePlaylistSource.supports_listing = False`` cannot do. See +``core.playlists.sources.ytmusic.YTMusicPlaylistSource``, the adapter that +consumes it. + +Same house rules as ``youtube_music_meta``: ``ytmusicapi`` is optional at +runtime, every function is best-effort and returns ``None`` (never raises) +on any failure, and ``auth`` is a ytmusicapi browser-auth header dict — see +``core.youtube_cookies.ytmusic_auth_headers``. + +Two calls, not one, for Liked Music: ``get_library_playlists()`` is +unreliable for it — some accounts get an "Auto playlist" row for it in the +regular library grid (``playlistId: 'LM'``) with no usable ``count``, others +get nothing at all (verified against ytmusicapi 1.12's ``parse_playlist`` — +that grid isn't the same shelf ``get_liked_songs()`` reads). Either way it +needs its own lookup by the reserved id ``"LM"`` (``LIKED_MUSIC_ID``, +re-exported from ``youtube_music_meta``), the same one +``ytmusicapi.YTMusic.get_liked_songs()`` uses internally. ``get_playlist(id, +limit=0)`` fetches the header (title, track count, thumbnail) without +paging any tracks — the same cheap-count trick +``TidalClient.get_collection_tracks_count()`` uses for Favorite Tracks. +``library_playlist_to_row`` drops any ``LIKED_MUSIC_ID`` row it sees in the +regular grid, so the two calls can never produce two "Liked Music" entries. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Mapping, Optional + +from core.youtube_music_meta import LIKED_MUSIC_ID +from utils.logging_config import get_logger + +logger = get_logger("ytmusic_library") + +LIKED_MUSIC_NAME = "Liked Music" + + +def _thumbnail_url(entry: Mapping[str, Any]) -> str: + thumbnails = entry.get("thumbnails") or [] + if thumbnails and isinstance(thumbnails[-1], Mapping): + return str(thumbnails[-1].get("url") or "") + return "" + + +def _owner_name(author: Any) -> Optional[str]: + """``author`` shape varies: absent, a plain string, a single dict, or a + list of ``{name, id}`` dicts (ytmusicapi's ``parse_artists_runs``).""" + if isinstance(author, list) and author: + first = author[0] + name = first.get("name") if isinstance(first, Mapping) else str(first) + return (name or "").strip() or None + if isinstance(author, Mapping): + return (str(author.get("name") or "")).strip() or None + if isinstance(author, str) and author.strip(): + return author.strip() + return None + + +def library_playlist_to_row(entry: Any) -> Optional[Dict[str, Any]]: + """Project one ``get_library_playlists()`` row, or ``None``. Pure. + + Requires a ``playlistId`` and a ``title`` — a playlist that can't be + identified or shown is not a usable row. Also drops a ``LIKED_MUSIC_ID`` + row outright: some accounts surface an "Auto playlist" entry for Liked + Music in this same grid with no usable ``count``, and + ``fetch_liked_music_row`` already builds the correct row for it from its + own dedicated header lookup. Letting both through gives two "Liked Music" + rows sharing one id, and whichever the frontend renders second silently + wins.""" + if not isinstance(entry, Mapping): + return None + playlist_id = str(entry.get("playlistId") or "").strip() + title = str(entry.get("title") or "").strip() + if not playlist_id or not title: + return None + if playlist_id == LIKED_MUSIC_ID: + return None + + count_raw = entry.get("count") + try: + track_count = int(count_raw) if count_raw not in (None, "") else 0 + except (TypeError, ValueError): + track_count = 0 + + return { + "id": playlist_id, + "name": title, + "track_count": track_count, + "image_url": _thumbnail_url(entry) or None, + "description": (str(entry.get("description") or "").strip() or None), + "owner": _owner_name(entry.get("author")), + } + + +def library_playlists_to_rows(raw: Optional[List[Any]]) -> List[Dict[str, Any]]: + """Project every ``get_library_playlists()`` row. Pure. ``raw=None`` -> ``[]``.""" + if not raw: + return [] + rows: List[Dict[str, Any]] = [] + for entry in raw: + row = library_playlist_to_row(entry) + if row: + rows.append(row) + return rows + + +def fetch_library_playlists(auth: Optional[Dict[str, str]]) -> Optional[List[Any]]: + """Fetch the signed-in account's own library playlists, or ``None`` on any failure. + + ``auth`` is required — the library has no anonymous view, so a missing + ``auth`` short-circuits before the (guaranteed to fail) network call. + + Never raises: missing ytmusicapi, a network blip, an expired cookie jar + and an upstream response-shape change all mean the same thing to the + caller — treat the account as unreachable right now. + """ + if not auth: + return None + try: + from ytmusicapi import YTMusic + except ImportError: + logger.debug("ytmusicapi not installed — cannot list library playlists") + return None + + try: + client = YTMusic(auth) + return client.get_library_playlists(limit=None) + except Exception as e: # noqa: BLE001 - see docstring: all failures return None + logger.info( + "YouTube Music library playlists fetch failed (%s: %s)", + type(e).__name__, e, + ) + return None + + +def fetch_liked_music_row(auth: Optional[Dict[str, str]]) -> Optional[Dict[str, Any]]: + """Fetch a listing row for the account's Liked Music, or ``None``. + + ``None`` covers both "the call failed" and "there is nothing liked yet" + (``track_count <= 0``) — callers that want a virtual playlist card treat + both the same way Tidal's Favorite Tracks entry does: don't show an + empty ghost row. + """ + if not auth: + return None + try: + from ytmusicapi import YTMusic + except ImportError: + logger.debug("ytmusicapi not installed — cannot fetch Liked Music") + return None + + try: + client = YTMusic(auth) + header = client.get_playlist(LIKED_MUSIC_ID, limit=0) + except Exception as e: # noqa: BLE001 - see docstring: all failures return None + logger.info( + "YouTube Music Liked Music header fetch failed (%s: %s)", + type(e).__name__, e, + ) + return None + + if not isinstance(header, Mapping): + return None + track_count_raw = header.get("trackCount") + try: + track_count = int(track_count_raw) if track_count_raw is not None else 0 + except (TypeError, ValueError): + track_count = 0 + if track_count <= 0: + return None + + return { + "id": LIKED_MUSIC_ID, + "name": str(header.get("title") or LIKED_MUSIC_NAME), + "track_count": track_count, + "image_url": _thumbnail_url(header) or None, + "description": None, + "owner": "You", + } + + +def ytmusic_playlist_url(playlist_id: str) -> str: + """The ``music.youtube.com`` URL form ``fetch_ytmusic_playlist`` / + ``parse_youtube_playlist`` expect.""" + return f"https://music.youtube.com/playlist?list={playlist_id}" + + +__all__ = [ + "LIKED_MUSIC_NAME", + "library_playlist_to_row", + "library_playlists_to_rows", + "fetch_library_playlists", + "fetch_liked_music_row", + "ytmusic_playlist_url", +] diff --git a/tests/test_playlist_sources_adapters.py b/tests/test_playlist_sources_adapters.py index 89b9de989..d0b2195a4 100644 --- a/tests/test_playlist_sources_adapters.py +++ b/tests/test_playlist_sources_adapters.py @@ -33,6 +33,8 @@ from core.playlists.sources.spotify_public import SpotifyPublicPlaylistSource from core.playlists.sources.tidal import TidalPlaylistSource from core.playlists.sources.youtube import YouTubePlaylistSource +import core.playlists.sources.ytmusic as _ytmusic_module +from core.playlists.sources.ytmusic import YTMusicPlaylistSource # ─── Spotify ──────────────────────────────────────────────────────────── @@ -325,6 +327,97 @@ def test_youtube_adapter_failed_parse(): assert src.get_playlist("https://bad") is None +def test_ytmusic_adapter_unauthenticated_short_circuits(): + src = YTMusicPlaylistSource(lambda: None) + assert src.is_authenticated() is False + assert src.list_playlists() == [] + assert src.get_playlist("PL123") is None + + +def test_ytmusic_adapter_auth_getter_raising_is_treated_as_unauthenticated(): + def boom(): + raise RuntimeError("cookie parse blew up") + + src = YTMusicPlaylistSource(boom) + assert src.is_authenticated() is False + assert src.list_playlists() == [] + + +def test_ytmusic_adapter_lists_library_plus_liked_music(monkeypatch): + monkeypatch.setattr(_ytmusic_module, "fetch_library_playlists", lambda auth: ["raw-row"]) + monkeypatch.setattr( + _ytmusic_module, "library_playlists_to_rows", + lambda raw: [{ + "id": "pl1", "name": "Road Trip", "track_count": 10, + "image_url": "thumb1", "description": None, "owner": None, + }], + ) + monkeypatch.setattr( + _ytmusic_module, "fetch_liked_music_row", + lambda auth: { + "id": "LM", "name": "Liked Music", "track_count": 5, + "image_url": "thumb2", "description": None, "owner": "You", + }, + ) + + src = YTMusicPlaylistSource(lambda: {"Cookie": "x"}) + assert src.is_authenticated() is True + metas = src.list_playlists() + # Liked Music is pinned FIRST — matches YouTube Music's own UI, unlike + # the Spotify "Liked Songs" / Tidal "Favorite Tracks" virtual-playlist + # convention elsewhere in this app, which appends at the end. + assert [m.source_playlist_id for m in metas] == ["LM", "pl1"] + assert metas[0].name == "Liked Music" + assert metas[0].owner == "You" + assert metas[1].source == "ytmusic" + assert metas[1].name == "Road Trip" + assert metas[1].track_count == 10 + + +def test_ytmusic_adapter_omits_liked_music_when_none_liked(monkeypatch): + monkeypatch.setattr(_ytmusic_module, "fetch_library_playlists", lambda auth: []) + monkeypatch.setattr(_ytmusic_module, "library_playlists_to_rows", lambda raw: []) + monkeypatch.setattr(_ytmusic_module, "fetch_liked_music_row", lambda auth: None) + + src = YTMusicPlaylistSource(lambda: {"Cookie": "x"}) + assert src.list_playlists() == [] + + +def test_ytmusic_adapter_get_playlist_projection(monkeypatch): + def fake_fetch(url, auth): + assert url == "https://music.youtube.com/playlist?list=pl1" + assert auth == {"Cookie": "x"} + return { + "id": "pl1", "name": "Road Trip", "track_count": 1, + "url": url, "image_url": "thumb", + "tracks": [{ + "id": "vid1", "name": "Track", "artists": ["Artist"], + "album": "Album", "duration_ms": 200_000, + "url": "https://youtu.be/vid1", + }], + } + + monkeypatch.setattr(_ytmusic_module, "fetch_ytmusic_playlist", fake_fetch) + + src = YTMusicPlaylistSource(lambda: {"Cookie": "x"}) + detail = src.get_playlist("pl1") + assert detail is not None + assert detail.meta.source == "ytmusic" + assert detail.meta.source_playlist_id == "pl1" + assert detail.meta.name == "Road Trip" + assert detail.tracks[0].source_track_id == "vid1" + assert detail.tracks[0].album_name == "Album" + + # refresh_playlist is the identical call — the point of being ID-backed. + assert src.refresh_playlist("pl1") is not None + + +def test_ytmusic_adapter_get_playlist_failed_fetch(monkeypatch): + monkeypatch.setattr(_ytmusic_module, "fetch_ytmusic_playlist", lambda url, auth: None) + src = YTMusicPlaylistSource(lambda: {"Cookie": "x"}) + assert src.get_playlist("pl1") is None + + # ─── iTunes Link ──────────────────────────────────────────────────────── diff --git a/tests/test_youtube_cookies.py b/tests/test_youtube_cookies.py index 525eeb684..020c53184 100644 --- a/tests/test_youtube_cookies.py +++ b/tests/test_youtube_cookies.py @@ -191,6 +191,21 @@ def test_auth_headers_drop_non_essential_cookies(): assert "bulky-unrelated-value" not in cookie +def test_auth_headers_keep_sidts_rotating_tokens(): + # Regression for the "Sign in to listen to your liked tracks" bug: + # SAPISIDHASH alone verifies fine and generic library calls work, but + # Liked Music (list=LM) serves the signed-out view without these two + # (sigma67/ytmusicapi#962). + jar = ( + _JAR + + ".google.de\tTRUE\t/\tTRUE\t1799999999\t__Secure-1PSIDTS\tsidts-1p\n" + + ".google.de\tTRUE\t/\tTRUE\t1799999999\t__Secure-3PSIDTS\tsidts-3p\n" + ) + cookie = ytmusic_auth_headers(parse_netscape_cookies(jar))["Cookie"] + assert "__Secure-1PSIDTS=sidts-1p" in cookie + assert "__Secure-3PSIDTS=sidts-3p" in cookie + + def test_sapisid_aliases_are_accepted_in_priority_order(): for name in ("__Secure-3PAPISID", "__Secure-1PAPISID", "SAPISID"): headers = ytmusic_auth_headers({name: "v"}, timestamp=1) diff --git a/tests/test_youtube_music_meta.py b/tests/test_youtube_music_meta.py index e9dcf5bed..b1bcf8d32 100644 --- a/tests/test_youtube_music_meta.py +++ b/tests/test_youtube_music_meta.py @@ -16,6 +16,7 @@ import sys from core.youtube_music_meta import ( + _get_playlist_paginated, playlist_id_from_url, search_ytmusic_songs, ytmusic_playlist_to_payload, @@ -296,3 +297,102 @@ def test_search_ytmusic_songs_sorts_atv_first(monkeypatch): ]) hits = search_ytmusic_songs("Example Artist - Example Track") assert [h["id"] for h in hits] == ["atv", "omv", "ugc"] + + +# TEMPORARY, delete alongside the function once +# sigma67/ytmusicapi#953 ships and get_playlist grows its own validate_responses. + + +class _CountingClient: + """A fake ytmusicapi client whose get_playlist() replays one canned + response per call, in order, and records how many times it was called.""" + + def __init__(self, responses): + self._responses = list(responses) + self.calls = 0 + + def get_playlist(self, playlist_id, limit=None): + self.calls += 1 + # Repeat the last response if asked for more calls than scripted — + # keeps a test's intent ("it should have stopped by now") legible + # instead of an IndexError burying the real assertion. + idx = min(self.calls - 1, len(self._responses) - 1) + return self._responses[idx] + + +def test_paginated_no_retry_when_first_attempt_is_already_complete(): + client = _CountingClient([{"tracks": [1, 2, 3], "trackCount": 3}]) + result = _get_playlist_paginated(client, "PL1") + assert result["tracks"] == [1, 2, 3] + assert client.calls == 1 + + +def test_paginated_does_not_retry_a_normal_small_gap(): + # 143/144 (99.3%) — a real, reproducible shape from production: one + # dead/region-blocked video that legitimately produces no row. Must be + # accepted on the first attempt, not chased with a full refetch. + client = _CountingClient([{"tracks": list(range(143)), "trackCount": 144}]) + result = _get_playlist_paginated(client, "PL1") + assert len(result["tracks"]) == 143 + assert client.calls == 1 + + +def test_paginated_retries_a_severely_truncated_result(): + # 2000/3132 (63.9%) — the real bug shape: catastrophically short, not + # "a couple of dead entries". Recovers to the normal ceiling on retry. + client = _CountingClient([ + {"tracks": list(range(2000)), "trackCount": 3132}, + {"tracks": list(range(3032)), "trackCount": 3132}, + ]) + result = _get_playlist_paginated(client, "PL1") + assert len(result["tracks"]) == 3032 + assert client.calls == 2 + + +def test_paginated_gives_up_after_max_attempts_and_keeps_the_best_seen(): + # Never crosses the completeness threshold — bounded, not infinite. + client = _CountingClient([ + {"tracks": [1], "trackCount": 100}, + {"tracks": [1, 2], "trackCount": 100}, # the best of the two + ]) + result = _get_playlist_paginated(client, "PL1") + assert result["tracks"] == [1, 2] + assert client.calls == 2 + + +def test_paginated_does_not_retry_without_a_trackcount_to_validate_against(): + client = _CountingClient([{"tracks": [1], "trackCount": None}]) + result = _get_playlist_paginated(client, "PL1") + assert result["tracks"] == [1] + assert client.calls == 1 + + +def test_paginated_first_attempt_failure_propagates(): + # Unchanged from before this wrapper existed: fetch_ytmusic_playlist's + # own try/except catches this and falls back to yt-dlp. + class _Boom: + def get_playlist(self, playlist_id, limit=None): + raise RuntimeError("network blip") + + try: + _get_playlist_paginated(_Boom(), "PL1") + assert False, "expected the exception to propagate" + except RuntimeError: + pass + + +def test_paginated_later_attempt_failure_keeps_the_partial_result(): + class _FailsOnSecondCall: + def __init__(self): + self.calls = 0 + + def get_playlist(self, playlist_id, limit=None): + self.calls += 1 + if self.calls == 1: + return {"tracks": [1, 2], "trackCount": 3} + raise RuntimeError("network blip") + + client = _FailsOnSecondCall() + result = _get_playlist_paginated(client, "PL1") + assert result["tracks"] == [1, 2] + assert client.calls == 2 diff --git a/tests/test_ytmusic_library.py b/tests/test_ytmusic_library.py new file mode 100644 index 000000000..3ab96de78 --- /dev/null +++ b/tests/test_ytmusic_library.py @@ -0,0 +1,196 @@ +"""Seam tests for the YouTube Music ACCOUNT-library reads. + +``core.ytmusic_library`` answers "what playlists does the signed-in +account have?" — a different question from ``core.youtube_music_meta`` +(which reads a single playlist by id/URL, anonymous or authenticated). +These pin the projection of ``get_library_playlists()`` rows and the +Liked Music virtual-row lookup, and — as with every other best-effort +ytmusicapi seam in this codebase — that every failure mode returns +``None``/``[]`` rather than raising. + +Fixtures are trimmed copies of real ytmusicapi 1.12 responses (verified +against its ``mixins/library.py`` and ``parsers/browsing.py`` source). +""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace + +from core.youtube_music_meta import LIKED_MUSIC_ID +from core.ytmusic_library import ( + fetch_liked_music_row, + fetch_library_playlists, + library_playlist_to_row, + library_playlists_to_rows, + ytmusic_playlist_url, +) + +LIBRARY_ROW = { + "playlistId": "PLQwVIlKxHM6rz0fDJVv_0UlXGEWf-bFys", + "title": "Road Trip", + "thumbnails": [{"url": "https://example.com/small.jpg"}, {"url": "https://example.com/large.jpg"}], + "count": "42", + "owned": True, +} + + +# ── URL helper ─────────────────────────────────────────────────────────── + + +def test_ytmusic_playlist_url_form(): + assert ytmusic_playlist_url("PL123") == "https://music.youtube.com/playlist?list=PL123" + assert ytmusic_playlist_url("LM") == "https://music.youtube.com/playlist?list=LM" + + +# ── library_playlist_to_row / library_playlists_to_rows ───────────────── + + +def test_library_row_projects_all_fields(): + row = library_playlist_to_row(LIBRARY_ROW) + assert row["id"] == "PLQwVIlKxHM6rz0fDJVv_0UlXGEWf-bFys" + assert row["name"] == "Road Trip" + assert row["track_count"] == 42 + assert row["image_url"] == "https://example.com/large.jpg" + assert row["description"] is None + assert row["owner"] is None + + +def test_library_row_requires_id_and_title(): + assert library_playlist_to_row({**LIBRARY_ROW, "playlistId": ""}) is None + assert library_playlist_to_row({**LIBRARY_ROW, "title": ""}) is None + assert library_playlist_to_row("junk") is None + + +def test_library_row_missing_count_defaults_to_zero(): + row = library_playlist_to_row({**LIBRARY_ROW, "count": None}) + assert row["track_count"] == 0 + row = library_playlist_to_row({k: v for k, v in LIBRARY_ROW.items() if k != "count"}) + assert row["track_count"] == 0 + + +def test_library_row_author_shapes(): + # list of {name, id} dicts — ytmusicapi's parse_artists_runs shape + row = library_playlist_to_row({**LIBRARY_ROW, "author": [{"name": "Some User", "id": "UC123"}]}) + assert row["owner"] == "Some User" + # bare string + row = library_playlist_to_row({**LIBRARY_ROW, "author": "Some User"}) + assert row["owner"] == "Some User" + # absent + row = library_playlist_to_row({**LIBRARY_ROW}) + assert row["owner"] is None + + +def test_library_playlists_to_rows_filters_unusable_and_handles_none(): + rows = library_playlists_to_rows([LIBRARY_ROW, {"playlistId": "", "title": "no id"}, "junk"]) + assert len(rows) == 1 + assert rows[0]["id"] == LIBRARY_ROW["playlistId"] + assert library_playlists_to_rows(None) == [] + + +def test_library_row_drops_the_liked_music_auto_playlist_entry(): + # Some accounts surface Liked Music as an "Auto playlist" row in the + # regular library grid, with no usable count. fetch_liked_music_row + # already builds the correct row for this id from its own header lookup; + # letting this one through too produces two "Liked Music" cards sharing + # one id. + auto_playlist_row = { + "playlistId": LIKED_MUSIC_ID, + "title": "Liked Music", + "description": "Auto playlist", + "owned": True, + # deliberately no "count" — this is the observed real shape + } + assert library_playlist_to_row(auto_playlist_row) is None + assert library_playlists_to_rows([LIBRARY_ROW, auto_playlist_row]) == [ + library_playlist_to_row(LIBRARY_ROW) + ] + assert library_playlists_to_rows([]) == [] + + +# ── fetch_library_playlists ────────────────────────────────────────────── + + +def _install_fake_ytmusic(monkeypatch, *, library_impl=None, playlist_impl=None): + class _FakeYTMusic: + def __init__(self, *_a, **_k): + pass + + def get_library_playlists(self, limit=None): + return library_impl(limit) if library_impl else [] + + def get_playlist(self, playlist_id, limit=100): + return playlist_impl(playlist_id, limit) if playlist_impl else {} + + monkeypatch.setitem(sys.modules, "ytmusicapi", SimpleNamespace(YTMusic=_FakeYTMusic)) + + +def test_fetch_library_playlists_requires_auth(): + assert fetch_library_playlists(None) is None + assert fetch_library_playlists({}) is None + + +def test_fetch_library_playlists_missing_dependency_returns_none(monkeypatch): + monkeypatch.setitem(sys.modules, "ytmusicapi", None) + assert fetch_library_playlists({"Cookie": "x"}) is None + + +def test_fetch_library_playlists_exception_returns_none(monkeypatch): + class _Boom: + def __init__(self, *_a, **_k): + raise RuntimeError("inner-tube down") + + monkeypatch.setitem(sys.modules, "ytmusicapi", SimpleNamespace(YTMusic=_Boom)) + assert fetch_library_playlists({"Cookie": "x"}) is None + + +def test_fetch_library_playlists_returns_raw_rows(monkeypatch): + _install_fake_ytmusic(monkeypatch, library_impl=lambda limit: [LIBRARY_ROW]) + raw = fetch_library_playlists({"Cookie": "x"}) + assert raw == [LIBRARY_ROW] + + +# ── fetch_liked_music_row ──────────────────────────────────────────────── + + +def test_fetch_liked_music_row_requires_auth(): + assert fetch_liked_music_row(None) is None + assert fetch_liked_music_row({}) is None + + +def test_fetch_liked_music_row_missing_dependency_returns_none(monkeypatch): + monkeypatch.setitem(sys.modules, "ytmusicapi", None) + assert fetch_liked_music_row({"Cookie": "x"}) is None + + +def test_fetch_liked_music_row_exception_returns_none(monkeypatch): + _install_fake_ytmusic(monkeypatch, playlist_impl=lambda pid, limit: (_ for _ in ()).throw(RuntimeError("boom"))) + assert fetch_liked_music_row({"Cookie": "x"}) is None + + +def test_fetch_liked_music_row_empty_is_none(monkeypatch): + # Nothing liked yet — don't show a ghost card. + _install_fake_ytmusic(monkeypatch, playlist_impl=lambda pid, limit: {"title": "Liked Music", "trackCount": 0}) + assert fetch_liked_music_row({"Cookie": "x"}) is None + + +def test_fetch_liked_music_row_projects_header(monkeypatch): + def _playlist(pid, limit): + assert pid == "LM" + assert limit == 0 # header-only fetch, no track paging + return { + "title": "Liked Music", + "trackCount": 128, + "thumbnails": [{"url": "https://example.com/liked.jpg"}], + } + + _install_fake_ytmusic(monkeypatch, playlist_impl=_playlist) + row = fetch_liked_music_row({"Cookie": "x"}) + assert row == { + "id": "LM", + "name": "Liked Music", + "track_count": 128, + "image_url": "https://example.com/liked.jpg", + "description": None, + "owner": "You", + } diff --git a/web_server.py b/web_server.py index 6fc10ddd4..7adcdd727 100644 --- a/web_server.py +++ b/web_server.py @@ -1280,6 +1280,12 @@ def _download_pool_size(): # --- Automation Progress Tracking --- _scan_library_automation_id = None _automation_deps = None +# Module global so routes outside _register_automation_handlers (e.g. the +# YouTube Music account listing route) can reach the registry without going +# through AutomationDeps. Mirrors _automation_deps: stays None until +# _register_automation_handlers() runs (which itself no-ops without a live +# automation_engine) — callers must treat None as "not ready yet". +_playlist_source_registry = None # Playlist-native manual pipeline runs share the automation dependency # bundle, but keep their own small progress state for the playlist UI. @@ -1301,7 +1307,7 @@ def _register_automation_handlers(): closures still live below until subsequent commits in the same branch finish the lift. """ - global _automation_deps + global _automation_deps, _playlist_source_registry if not automation_engine: return @@ -1367,6 +1373,7 @@ def _discover_callable_for_registry(tracks): qobuz_client_getter=_get_qobuz_client_for_sync, deezer_client_getter=_get_deezer_client, youtube_parser=parse_youtube_playlist, + ytmusic_auth_getter=_ytmusic_auth_headers, itunes_link_parser=_itunes_link_parser_for_registry, listenbrainz_manager_getter=_lb_manager_for_registry, lastfm_manager_getter=_lb_manager_for_registry, @@ -3193,6 +3200,10 @@ def _build_system_stats(): for _url_hash, state in youtube_playlist_states.items(): if state.get('phase') == 'syncing': active_syncs += 1 + # Count YouTube Music playlist syncs + for _playlist_id, state in ytmusic_discovery_states.items(): + if state.get('phase') == 'syncing': + active_syncs += 1 # Count Tidal playlist syncs for _playlist_id, state in tidal_discovery_states.items(): if state.get('phase') == 'syncing': @@ -17782,6 +17793,7 @@ def get_spotify_track(track_id): _run_playlist_organize_download, _run_sync_task, _run_youtube_discovery_worker, + _run_ytmusic_discovery_worker, _save_source_bubble_snapshot, _sync_discovery_results_to_mirrored, _validate_discovery_cache_artist, @@ -17800,6 +17812,8 @@ def get_spotify_track(track_id): tidal_discovery_states, youtube_discovery_executor, youtube_playlist_states, + ytmusic_discovery_executor, + ytmusic_discovery_states, ) # --- Discover Download Snapshot System --- @@ -20479,6 +20493,7 @@ def _has_active_discovery(): try: for states in (tidal_discovery_states, qobuz_discovery_states, deezer_discovery_states, youtube_playlist_states, + ytmusic_discovery_states, beatport_chart_states, listenbrainz_playlist_states, spotify_public_discovery_states, itunes_link_discovery_states): for state in list(states.values()): @@ -21029,6 +21044,7 @@ def _set_dev_mode_impl(value): _tidal_enrichment_worker=lambda: tidal_enrichment_worker, _dev_mode_enabled=lambda: dev_mode_enabled, _get_automation_deps=lambda: _automation_deps, + _ytmusic_auth_headers=_ytmusic_auth_headers, ) app.register_blueprint(_bp_sp()) @@ -21486,6 +21502,7 @@ def _reconcile_discovery_sync_phases(): (spotify_public_discovery_states, "Spotify Link playlist", _pl_name_strict), (itunes_link_discovery_states, "iTunes Link", _pl_name_strict), (youtube_playlist_states, "YouTube playlist", _pl_name_safe), + (ytmusic_discovery_states, "YouTube Music playlist", _pl_name_safe), (listenbrainz_playlist_states, "ListenBrainz playlist", _pl_name_safe), ) for states_dict, activity_subject, name_getter in targets: @@ -21548,6 +21565,7 @@ def _emit_discovery_progress_loop(): 'qobuz': lambda: qobuz_discovery_states, 'deezer': lambda: deezer_discovery_states, 'youtube': lambda: youtube_playlist_states, + 'ytmusic': lambda: ytmusic_discovery_states, 'beatport': lambda: beatport_chart_states, 'listenbrainz': lambda: listenbrainz_playlist_states, 'spotify_public': lambda: spotify_public_discovery_states, diff --git a/webui/src/routes/sync/-sync.api.test.ts b/webui/src/routes/sync/-sync.api.test.ts index 95c323e3c..3f1b9a0d3 100644 --- a/webui/src/routes/sync/-sync.api.test.ts +++ b/webui/src/routes/sync/-sync.api.test.ts @@ -198,6 +198,9 @@ describe('page-level endpoints', () => { stubFetch({ playlists: [{ id: 'c' }] }); expect(await fetchSourcePlaylists('tidal')).toEqual([{ id: 'c' }]); expect(calls[0].url).toBe('/api/tidal/playlists'); + stubFetch({ playlists: [{ id: 'd' }] }); + expect(await fetchSourcePlaylists('ytmusic')).toEqual([{ id: 'd' }]); + expect(calls[0].url).toBe('/api/ytmusic/playlists'); }); it('account sync status + arl status hit the account endpoints', async () => { diff --git a/webui/src/routes/sync/-sync.api.ts b/webui/src/routes/sync/-sync.api.ts index c0cdb5b68..5e0a8b054 100644 --- a/webui/src/routes/sync/-sync.api.ts +++ b/webui/src/routes/sync/-sync.api.ts @@ -400,28 +400,36 @@ export async function fetchYouTubePlaylists(): Promise[] /* ── Source playlist lists ────────────────────────────────────────────────── */ +/** Account-vertical base path -> display noun, for the fetch-failure toast. */ +const ACCOUNT_SOURCE_NOUN: Record<'tidal' | 'qobuz' | 'ytmusic', string> = { + tidal: 'Tidal', + qobuz: 'Qobuz', + ytmusic: 'YouTube Music', +}; + /** - * GET /api/tidal/playlists | /api/qobuz/playlists (vertical heads). Throws - * the backend error on !ok (the vanilla's sync-services.js 14-17 throw, - * which lands in the tab's ❌ placeholder + toast). + * GET /api/tidal/playlists | /api/qobuz/playlists | /api/ytmusic/playlists + * (vertical heads). Throws the backend error on !ok (the vanilla's + * sync-services.js 14-17 throw, which lands in the tab's ❌ placeholder + + * toast). */ -export async function fetchSourcePlaylists(base: 'tidal' | 'qobuz'): Promise { +export async function fetchSourcePlaylists( + base: 'tidal' | 'qobuz' | 'ytmusic', +): Promise { const response = await fetch(`/api/${base}/playlists`); if (!response.ok) { const error = await readJson<{ error?: string }>(response); - throw new Error( - error.error || `Failed to fetch ${base === 'tidal' ? 'Tidal' : 'Qobuz'} playlists`, - ); + throw new Error(error.error || `Failed to fetch ${ACCOUNT_SOURCE_NOUN[base]} playlists`); } const data = await readJson(response); return Array.isArray(data) ? data : ((data as { playlists?: unknown[] })?.playlists ?? []); } -/** GET /api/tidal/playlist/ | /api/qobuz/playlist/ — the on-demand - * track fetch the account verticals run per playlist (sync-services.js 39, - * 1550, 1653). */ +/** GET /api/tidal/playlist/ | /api/qobuz/playlist/ | + * /api/ytmusic/playlist/ — the on-demand track fetch the account + * verticals run per playlist (sync-services.js 39, 1550, 1653). */ export async function fetchAccountPlaylist( - base: 'tidal' | 'qobuz', + base: 'tidal' | 'qobuz' | 'ytmusic', id: string, ): Promise> { return readJson(await fetch(`/api/${base}/playlist/${id}`)); diff --git a/webui/src/routes/sync/-sync.modal-core.ts b/webui/src/routes/sync/-sync.modal-core.ts index d538b83f3..4aa53567e 100644 --- a/webui/src/routes/sync/-sync.modal-core.ts +++ b/webui/src/routes/sync/-sync.modal-core.ts @@ -51,6 +51,7 @@ export function modalTitle(source: SyncSourceId, fakeHash: string): string { beatport: '🎵 Beatport Chart Discovery', listenbrainz: '🎵 ListenBrainz Playlist Discovery', youtube: '🎵 YouTube Playlist Discovery', + ytmusic: '🎵 YouTube Music Playlist Discovery', mirrored: '🎵 Mirrored Playlist Discovery', }; return titles[source]; @@ -80,6 +81,7 @@ export function modalSourceLabel( beatport: 'Beatport', listenbrainz: 'LB', youtube: 'YT', + ytmusic: 'YT Music', mirrored: 'Source', }; return labels[source]; @@ -98,6 +100,7 @@ export function descriptionSourceWord(source: SyncSourceId, fakeHash: string): s beatport: 'Beatport', listenbrainz: 'ListenBrainz', youtube: 'YouTube', + ytmusic: 'YouTube Music', mirrored: 'mirrored', }; return words[source]; diff --git a/webui/src/routes/sync/-sync.shell.test.ts b/webui/src/routes/sync/-sync.shell.test.ts index 771972add..d8355bf33 100644 --- a/webui/src/routes/sync/-sync.shell.test.ts +++ b/webui/src/routes/sync/-sync.shell.test.ts @@ -63,16 +63,21 @@ describe('the tab table matches the markup it was transcribed from', () => { */ const SHELL = readFileSync(resolve(__dirname, '__fixtures__/-vanilla-sync-markup.html'), 'utf8'); + /** + * drop items not in the vanilla JS code + */ + const TRANSCRIBED_TABS = SYNC_TABS.filter((t) => t.id !== 'ytmusic'); + it('has the same fifteen ids, in the same order', () => { const inMarkup = [ ...SHELL.matchAll(/class="sync-tab-button[^"]*"[^>]*data-tab="([^"]+)"/g), ].map((m) => m[1]); expect(inMarkup).toHaveLength(15); - expect(SYNC_TABS.map((t) => t.id)).toEqual(inMarkup); + expect(TRANSCRIBED_TABS.map((t) => t.id)).toEqual(inMarkup); }); it('has the same label for each, and uses it as the title too', () => { - for (const t of SYNC_TABS) { + for (const t of TRANSCRIBED_TABS) { const row = new RegExp( `data-tab="${t.id}"[^>]*title="([^"]+)"[\\s\\S]{0,160}?sync-tab-label">([^<]+)<`, ).exec(SHELL); @@ -83,7 +88,7 @@ describe('the tab table matches the markup it was transcribed from', () => { }); it('has the same sprite for each', () => { - for (const t of SYNC_TABS) { + for (const t of TRANSCRIBED_TABS) { const row = new RegExp(`data-tab="${t.id}"[^>]*>\\s* { it('marks the same tabs as link tabs', () => { const inMarkup = [...SHELL.matchAll(/data-tab="([^"]+)" data-link="true"/g)].map((m) => m[1]); - expect(SYNC_TABS.filter((t) => t.link).map((t) => t.id)).toEqual(inMarkup); + expect(TRANSCRIBED_TABS.filter((t) => t.link).map((t) => t.id)).toEqual(inMarkup); }); it('DIVERGES from the markup on which tab opens, deliberately', () => { @@ -176,7 +181,7 @@ describe('the strip (syncStripTabs)', () => { it('every tab is still REACHABLE even when it is not in the strip', () => { // The panels all still render; only the chips were reduced. const stripped = SYNC_TABS.filter((t) => !SYNC_PRIMARY_TAB_IDS.includes(t.id)); - expect(stripped.length).toBe(12); + expect(stripped.length).toBe(13); for (const t of stripped) { expect(syncStripTabs(t.id).map((x) => x.id)).toContain(t.id); } diff --git a/webui/src/routes/sync/-sync.shell.ts b/webui/src/routes/sync/-sync.shell.ts index bb39dfe15..37247d374 100644 --- a/webui/src/routes/sync/-sync.shell.ts +++ b/webui/src/routes/sync/-sync.shell.ts @@ -14,6 +14,7 @@ export type SyncTabId = | 'deezer' | 'deezer-link' | 'youtube' + | 'ytmusic' | 'beatport' | 'listenbrainz-sync' | 'lastfm-sync' @@ -41,9 +42,9 @@ export interface SyncTab { } /** - * All fifteen, in the order the strip renders them. `server` is first and is + * All sixteen, in the order the strip renders them. `server` is first and is * the default; a divider follows it (index.html 2253), separating "your media - * server" from the fifteen sources. + * server" from the sources. */ export const SYNC_TABS: readonly SyncTab[] = [ { id: 'server', label: 'Server Playlists', icon: 'server-icon' }, @@ -55,6 +56,7 @@ export const SYNC_TABS: readonly SyncTab[] = [ { id: 'deezer', label: 'Deezer', icon: 'deezer-icon' }, { id: 'deezer-link', label: 'Deezer Link', icon: 'deezer-icon', link: true }, { id: 'youtube', label: 'YouTube', icon: 'youtube-icon' }, + { id: 'ytmusic', label: 'YouTube Music', icon: 'ytmusic-icon' }, { id: 'beatport', label: 'Beatport', icon: 'beatport-icon' }, // 3760-3763: the id is `listenbrainz-sync`, NOT `listenbrainz`, because the // vanilla resolves panels by `${tabId}-tab-content` and the DISCOVER page diff --git a/webui/src/routes/sync/-sync.sources.test.ts b/webui/src/routes/sync/-sync.sources.test.ts index 62cefde25..1b09a5356 100644 --- a/webui/src/routes/sync/-sync.sources.test.ts +++ b/webui/src/routes/sync/-sync.sources.test.ts @@ -203,7 +203,10 @@ describe('the endpoint table is anchored to the live vanilla', () => { // B's mutation run showed most endpoint strings could be renamed with the // suite green. These assert the real paths appear in the real sources. it('every configured endpoint path exists in webui/static', () => { - const ids = Object.keys(SYNC_SOURCES) as (keyof typeof SYNC_SOURCES)[]; + // remove items not in the vanilla JS code + const ids = (Object.keys(SYNC_SOURCES) as (keyof typeof SYNC_SOURCES)[]).filter( + (id) => id !== 'ytmusic', + ); for (const id of ids) { const api = SYNC_SOURCES[id].api; const paths = [ @@ -248,6 +251,7 @@ describe('discovery-completion toasts (the per-source drift, 9204/11076)', () => // _discoveryCompleteToast, shared by youtube and the mirrored rows that // ride its poller (9204). youtube: 'Discovery complete!', + ytmusic: 'Discovery complete!', mirrored: 'Discovery complete!', // ListenBrainz words its own (11076, 11171). listenbrainz: 'ListenBrainz discovery complete!', diff --git a/webui/src/routes/sync/-sync.sources.ts b/webui/src/routes/sync/-sync.sources.ts index a3bdf03eb..55c360560 100644 --- a/webui/src/routes/sync/-sync.sources.ts +++ b/webui/src/routes/sync/-sync.sources.ts @@ -22,6 +22,7 @@ export type SyncSourceId = | 'qobuz' | 'deezer' | 'youtube' + | 'ytmusic' | 'beatport' | 'spotify_public' | 'itunes_link' @@ -306,6 +307,37 @@ export const SYNC_SOURCES: Record = { }, }, + ytmusic: { + id: 'ytmusic', + heroLabel: 'YouTube Music', + api: { + ...std('/api/ytmusic'), + updatePhase: (id) => `/api/ytmusic/update_phase/${id}`, + state: (id) => `/api/ytmusic/state/${id}`, + playlistsStates: '/api/ytmusic/playlists/states', + reset: (id) => `/api/ytmusic/reset/${id}`, + resetBody: 'none', + }, + ids: { fakeHashPrefix: 'ytmusic_', vpidPrefix: 'ytmusic_', stateFlag: 'is_ytmusic_playlist' }, + discovery: { + pollMs: 1000, + pollPolicy: 'always', + wingItInSocket: false, + wingItInPoll: false, + startBody: 'none', + }, + sync: { pollMs: 1000, percentFormula: 'processed' }, + ux: { + openModalImmediately: false, + cardProgressFormat: 'slash-text', + foundVariant: 'lenient', + // Shares youtube's download modal (M3U export + quality-profile chrome), + downloadEntry: 'youtube', + discoveryCompleteToast: 'Discovery complete!', + resetErrorNoun: 'playlist', + }, + }, + beatport: { id: 'beatport', heroLabel: 'Beatport', diff --git a/webui/src/routes/sync/-sync.use-page.test.ts b/webui/src/routes/sync/-sync.use-page.test.ts index e8f818936..e0ddc15d6 100644 --- a/webui/src/routes/sync/-sync.use-page.test.ts +++ b/webui/src/routes/sync/-sync.use-page.test.ts @@ -158,9 +158,9 @@ describe('sidebar visibility', () => { }); describe('what it assembles', () => { - it('builds nine verticals and ONE pipeline controller', () => { + it('builds ten verticals and ONE pipeline controller', () => { const { result } = renderHook(() => useSyncPage()); - expect(Object.keys(result.current.verticals)).toHaveLength(9); + expect(Object.keys(result.current.verticals)).toHaveLength(10); expect(typeof result.current.pipeline.run).toBe('function'); expect(typeof result.current.pipeline.resume).toBe('function'); }); diff --git a/webui/src/routes/sync/-sync.verticals.test.ts b/webui/src/routes/sync/-sync.verticals.test.ts index 340482e78..0007b8746 100644 --- a/webui/src/routes/sync/-sync.verticals.test.ts +++ b/webui/src/routes/sync/-sync.verticals.test.ts @@ -21,10 +21,10 @@ describe('the id list cannot drift from the table', () => { expect([...SYNC_VERTICAL_IDS].sort()).toEqual(Object.keys(SYNC_SOURCES).sort()); }); - it('is NINE, not fifteen — tabs and verticals are different counts', () => { + it('is TEN, not sixteen — tabs and verticals are different counts', () => { // Last.fm rides ListenBrainz's vertical, Deezer-link rides Deezer's. If // this number ever equals the tab count, someone has conflated the two. - expect(SYNC_VERTICAL_IDS).toHaveLength(9); + expect(SYNC_VERTICAL_IDS).toHaveLength(10); }); it('has no duplicates, which would break hook order', () => { diff --git a/webui/src/routes/sync/-sync.verticals.ts b/webui/src/routes/sync/-sync.verticals.ts index 427c57be8..722ac8a96 100644 --- a/webui/src/routes/sync/-sync.verticals.ts +++ b/webui/src/routes/sync/-sync.verticals.ts @@ -1,20 +1,23 @@ /** - * The nine source verticals, built from ONE table rather than nine hand-written + * The ten source verticals, built from ONE table rather than ten hand-written * hook calls. * * kettui's fourth observation is that similar things get reimplemented per - * feature instead of sharing an abstraction, and nine near-identical + * feature instead of sharing an abstraction, and ten near-identical * `useSourceVertical(SYNC_SOURCES.x, …)` calls at a page's top level is exactly - * that shape: nine places to forget an option, nine places to update when the - * vertical contract changes, and no way to tell that a tenth source was added - * and missed. The registry below is the abstraction — the table is the source - * of truth and the page reads a map out of it. + * that shape: ten places to forget an option, ten places to update when the + * vertical contract changes, and no way to tell that an eleventh source was + * added and missed. The registry below is the abstraction — the table is the + * source of truth and the page reads a map out of it. * - * NINE, not fifteen: several tabs share a vertical. Last.fm Radio rides + * TEN, not sixteen: several tabs share a vertical. Last.fm Radio rides * ListenBrainz's machinery (same MB-track shape), Deezer-link rides Deezer's, - * and the three URL-import tabs each have their own. The tab count and the - * vertical count are different numbers and conflating them is how a tab ends - * up with a vertical that belongs to something else. + * and the three URL-import tabs each have their own. `ytmusic` is the one + * exception in the other direction — it's its OWN vertical (a real account + * with real discovery/sync state) even though it shares youtube's worker and + * result shape; see `-sync.sources.ts`. The tab count and the vertical count + * are different numbers and conflating them is how a tab ends up with a + * vertical that belongs to something else. * * HOOK ORDER IS SAFE because `SYNC_VERTICAL_IDS` is a frozen module constant: * the same ids in the same order on every render, which is all the rules of @@ -38,6 +41,7 @@ export const SYNC_VERTICAL_IDS = [ 'qobuz', 'deezer', 'youtube', + 'ytmusic', 'beatport', 'spotify_public', 'itunes_link', diff --git a/webui/src/routes/sync/-ui/account-tab.test.tsx b/webui/src/routes/sync/-ui/account-tab.test.tsx index 46bb64301..aff33846d 100644 --- a/webui/src/routes/sync/-ui/account-tab.test.tsx +++ b/webui/src/routes/sync/-ui/account-tab.test.tsx @@ -12,7 +12,7 @@ import { forgetAccountPlaylists } from '../-sync.account-cache'; import { fetchAccountPlaylist } from '../-sync.api'; import { SYNC_SOURCES } from '../-sync.sources'; import { useSourceVertical } from '../-sync.use-vertical'; -import { QobuzTab, TidalTab } from './account-tab'; +import { QobuzTab, TidalTab, YTMusicTab } from './account-tab'; import { hydrateStatesForLoaded, resumeIfInFlight } from './url-import-tab'; interface Call { @@ -453,6 +453,113 @@ describe('QobuzTab', () => { }); }); +function YTMusicHarness({ onOpen }: { onOpen?: (id: string) => void }) { + const vertical = useSourceVertical(SYNC_SOURCES.ytmusic); + const [openId, setOpenId] = useState(null); + return ( +
+ + {openId ?? 'none'} + + {JSON.stringify(vertical.states[openId ?? 'ytm1']?.playlist?.tracks ?? null)} + +
+ ); +} + +describe('YTMusicTab', () => { + it('starts on the click-Refresh placeholder and loads nothing', () => { + stubFetch(); + render(); + expect( + screen.getByText("Click 'Refresh' to load your YouTube Music playlists."), + ).toBeInTheDocument(); + expect(calls).toEqual([]); + }); + + it('a fresh click fetches tracks from the ytmusic endpoint (not qobuz), then opens', async () => { + stubFetch(); + const overlay = vi.fn(); + const hideOverlay = vi.fn(); + window.showLoadingOverlay = overlay as typeof window.showLoadingOverlay; + window.hideLoadingOverlay = hideOverlay as typeof window.hideLoadingOverlay; + let clickFetches = 0; + responder = (url) => { + if (url === '/api/ytmusic/playlists') { + return [{ id: 'ytm1', name: 'YTM Mix', track_count: 0 }]; + } + if (url === '/api/ytmusic/playlist/ytm1') { + clickFetches += 1; + return clickFetches === 1 + ? { tracks: [] } + : { tracks: [{ id: 'v1', name: 'Vid', artists: ['Ch'], duration_ms: 5 }] }; + } + if (url === '/api/ytmusic/playlists/states') return { states: [] }; + return { success: true }; + }; + render(); + fireEvent.click(screen.getByText('🔄 Refresh')); + await waitFor(() => expect(screen.getByText('YTM Mix')).toBeInTheDocument()); + await waitFor(() => + expect(calls.some((c) => c.url === '/api/ytmusic/playlists/states')).toBe(true), + ); + + fireEvent.click(screen.getByText('YTM Mix')); + await waitFor(() => expect(screen.getByTestId('open-id')).toHaveTextContent('ytm1')); + expect(overlay).toHaveBeenCalledWith('Loading YTM Mix...'); + expect(hideOverlay).toHaveBeenCalled(); + // Every track fetch — background crawl AND the click — hit ytmusic's own + // endpoint. The bug this guards against sent the click's fetch to qobuz's. + expect(calls.some((c) => c.url === '/api/qobuz/playlist/ytm1')).toBe(false); + expect(clickFetches).toBe(2); + expect(JSON.parse(screen.getByTestId('seeded').textContent!)).toEqual([ + { id: 'v1', name: 'Vid', artists: ['Ch'], album: '', duration_ms: 5, track_number: 0 }, + ]); + }); + + it("no tracks → 'Could not load tracks for this playlist', no open", async () => { + stubFetch(); + const toast = vi.fn(); + window.showToast = toast as typeof window.showToast; + responder = (url) => { + if (url === '/api/ytmusic/playlists') { + return [{ id: 'ytm2', name: 'Empty Mix', track_count: 0 }]; + } + if (url === '/api/ytmusic/playlists/states') return { states: [] }; + return { tracks: [] }; + }; + render(); + fireEvent.click(screen.getByText('🔄 Refresh')); + await waitFor(() => expect(screen.getByText('Empty Mix')).toBeInTheDocument()); + fireEvent.click(screen.getByText('Empty Mix')); + await waitFor(() => + expect(toast).toHaveBeenCalledWith('Could not load tracks for this playlist', 'error'), + ); + expect(screen.getByTestId('open-id')).toHaveTextContent('none'); + }); + + it('unauthenticated account (401) surfaces the backend error, not a blank list', async () => { + // The list route 401s when Settings → YouTube has no cookies configured; + // fetchSourcePlaylists throws on !ok, same as every other account vertical, + // and AccountVerticalTab.load() renders that into the error placeholder. + calls = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + calls.push({ url, method: 'GET', body: undefined }); + return new Response(JSON.stringify({ error: 'YouTube Music not authenticated.' }), { + status: 401, + }); + }), + ); + render(); + fireEvent.click(screen.getByText('🔄 Refresh')); + await waitFor(() => + expect(screen.getByText('❌ Error: YouTube Music not authenticated.')).toBeInTheDocument(), + ); + }); +}); + describe('shared helpers', () => { it('fetchAccountPlaylist hits the per-source playlist endpoint', async () => { stubFetch(); diff --git a/webui/src/routes/sync/-ui/account-tab.tsx b/webui/src/routes/sync/-ui/account-tab.tsx index f9d4dadd8..4e0087c5b 100644 --- a/webui/src/routes/sync/-ui/account-tab.tsx +++ b/webui/src/routes/sync/-ui/account-tab.tsx @@ -1,13 +1,19 @@ /** - * The Tidal and Qobuz account-vertical tabs (sync-services.js 4-227 and - * 1516-1720): click Refresh to load the account's playlists, cards render - * instantly from metadata, tracks fetch per-playlist in the background and - * auto-mirror, then the saved discovery states hydrate (with the P5b - * resume-on-in-flight fix). The two differ only in their FRESH card click: - * Tidal opens the shared modal immediately with whatever tracks are cached - * (#867 — the backend discovery fetch is the source of truth, 152-166); - * Qobuz fetches the track list behind a loading overlay first and refuses to - * open without tracks (1648-1680). + * The Tidal, Qobuz and YouTube Music account-vertical tabs (sync-services.js + * 4-227 and 1516-1720 for the first two; YTMusicTab has no vanilla line — + * it's the new YouTube Music ACCOUNT vertical, added alongside the existing + * youtube URL-paste one). Click Refresh to load the account's playlists, + * cards render instantly from metadata, tracks fetch per-playlist in the + * background and auto-mirror, then the saved discovery states hydrate (with + * the P5b resume-on-in-flight fix). They differ only in their FRESH card + * click: Tidal opens the shared modal immediately with whatever tracks are + * cached (#867 — the backend discovery fetch is the source of truth, + * 152-166); Qobuz and YouTube Music fetch the track list behind a loading + * overlay first and refuse to open without tracks (1648-1680) — YouTube + * Music follows Qobuz's shape rather than Tidal's #867 treatment because + * that treatment was a deliberate, not-yet-extended choice for the existing + * verticals (see the `openModalImmediately` note in -sync.sources.ts); a + * brand-new vertical has no standing to claim it unilaterally. * * Declared divergences (the P5a/P5b pattern): card clicks open the React * DiscoveryModal in every phase (the vanilla's downloading branches reopened @@ -54,8 +60,8 @@ import { hydrateStatesForLoaded } from './url-import-tab'; */ const TRACK_CRAWL_CONCURRENCY = 3; -/** The qobuz fresh-click track projection (sync-services.js 1657-1661). */ -function qobuzFreshTracks(tracks: unknown[]): Record[] { +/** The fresh-click track projection (sync-services.js 1657-1661). */ +function normalizeFreshTracks(tracks: unknown[]): Record[] { return (tracks as Record[]).map((t) => ({ id: t.id, name: t.name, @@ -67,14 +73,14 @@ function qobuzFreshTracks(tracks: unknown[]): Record[] { } interface AccountTabChrome { - base: 'tidal' | 'qobuz'; + base: 'tidal' | 'qobuz' | 'ytmusic'; title: string; refreshBtnId: string; refreshBtnClass: string; containerId: string; initialPlaceholder: string; loadingPlaceholder: string; - /** 'Tidal' | 'Qobuz' — the toast/placeholder noun. */ + /** 'Tidal' | 'Qobuz' | 'YouTube Music' — the toast/placeholder noun. */ noun: string; cardIdPrefix: string; cardClassName: string; @@ -253,15 +259,15 @@ function AccountVerticalTab({ onOpen(sourceId); return; } - // Qobuz: fetch the track list behind the overlay first (1649-1680). + // Qobuz (1649-1680) / YouTube Music: fetch the track list behind the overlay first. let tracks = Array.isArray(playlist.tracks) ? (playlist.tracks as unknown[]) : []; if (tracks.length === 0) { window.showLoadingOverlay?.(`Loading ${asString(playlist.name)}...`); try { - const fullData = await fetchAccountPlaylist('qobuz', sourceId); + const fullData = await fetchAccountPlaylist(chrome.base, sourceId); const fetched = Array.isArray(fullData.tracks) ? (fullData.tracks as unknown[]) : []; if (fetched.length > 0) { - tracks = qobuzFreshTracks(fetched); + tracks = normalizeFreshTracks(fetched); setPlaylistTracks(sourceId, tracks); } } catch { @@ -382,3 +388,36 @@ export function QobuzTab({ /> ); } + +/** + * The YouTube Music ACCOUNT vertical (not present in the vanilla JS code). + * Browses the signed-in account's own library playlists (+ Liked Music), + * the auth-based counterpart to the `youtube` tab's URL-paste flow. + */ +export function YTMusicTab({ + vertical, + onOpen, +}: { + vertical: SourceVertical; + onOpen: (sourceId: string) => void; +}) { + return ( + + ); +} diff --git a/webui/src/routes/sync/-ui/sync-page.tsx b/webui/src/routes/sync/-ui/sync-page.tsx index c55427649..5333a22ea 100644 --- a/webui/src/routes/sync/-ui/sync-page.tsx +++ b/webui/src/routes/sync/-ui/sync-page.tsx @@ -29,7 +29,7 @@ import { normalizeSyncTab } from '../-sync.shell'; import { useAutoSync } from '../-sync.use-autosync'; import { useSyncHistory } from '../-sync.use-history'; import { useSyncPage } from '../-sync.use-page'; -import { QobuzTab, TidalTab } from './account-tab'; +import { QobuzTab, TidalTab, YTMusicTab } from './account-tab'; import { DeezerArlTab, SpotifyTab } from './account-tabs'; import { ActivityModal, type ActivityTab } from './activity-modal'; import { AddPlaylistSheet } from './add-playlist-sheet'; @@ -286,6 +286,12 @@ export function SyncPage() { onPendingConsumed={clearPending} /> ), + ytmusic: ( + openSourceModal('ytmusic', sourceId)} + /> + ), 'listenbrainz-sync': ( ), diff --git a/webui/src/routes/sync/-ui/sync-shell.test.tsx b/webui/src/routes/sync/-ui/sync-shell.test.tsx index 433d11006..727436052 100644 --- a/webui/src/routes/sync/-ui/sync-shell.test.tsx +++ b/webui/src/routes/sync/-ui/sync-shell.test.tsx @@ -136,6 +136,26 @@ describe('the tab strip', () => { expect(btns.map((b) => b.getAttribute('data-tab'))).toEqual(['mirrored', 'server', 'beatport']); }); + it('opens YouTube Music as a routed tab, same as the other sources', () => { + // ytmusic has no permanent chip (like spotify-public, tidal, etc.) — it + // is reached through Add playlist / the account-listing flow, which + // opens it by id exactly like the other routed sources. + let open!: (tab: string) => void; + const { container } = renderShell({ + registerOpenTab: (fn) => { + open = fn as (tab: string) => void; + }, + }); + act(() => { + open('ytmusic'); + }); + const withRouted = Array.from(container.querySelectorAll('.sync-tab-button')).map((b) => + b.getAttribute('data-tab'), + ); + expect(withRouted).toEqual(['mirrored', 'server', 'beatport', 'ytmusic']); + expect(container.querySelector('[data-tab="ytmusic"]')?.className).toContain('active'); + }); + it('opens on Mirrored — the library, not a source directory', () => { const { container } = renderShell(); expect(container.querySelector('[data-tab="mirrored"]')?.className).toContain('active'); diff --git a/webui/src/routes/sync/-ui/sync-shell.tsx b/webui/src/routes/sync/-ui/sync-shell.tsx index 2c563d4a9..1d39bc279 100644 --- a/webui/src/routes/sync/-ui/sync-shell.tsx +++ b/webui/src/routes/sync/-ui/sync-shell.tsx @@ -1,5 +1,5 @@ /** - * The sync page's chrome — header, the fifteen-tab strip, and the panel + * The sync page's chrome — header, the sixteen-tab strip, and the panel * switch. index.html 2226-2295 plus the tab handler at sync-services.js * 3694-3803. * diff --git a/webui/static/style.css b/webui/static/style.css index 8ca9fb2c8..4504cda1b 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -15265,6 +15265,10 @@ body:has(.download-missing-modal[style*="display: flex"]) #helper-float-btn { background-image: url('data:image/svg+xml;charset=utf-8,'); } +.ytmusic-icon { + background-image: url('data:image/svg+xml;charset=utf-8,'); +} + .qobuz-icon { background-image: url('data:image/svg+xml;charset=utf-8,'); } From 7ad1e340f165afcbf069b08c8ecb20b4d6d9fdb3 Mon Sep 17 00:00:00 2001 From: ht5161143 Date: Thu, 27 Aug 2026 13:22:12 +0000 Subject: [PATCH 2/2] fix(youtube): parse #HttpOnly_-prefixed cookies.txt rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Netscape cookies.txt format marks an HttpOnly cookie by prefixing its domain field with "#HttpOnly_" rather than leaving the line plain. The parser treated any line starting with "#" as a comment, silently dropping exactly the session-identity cookies (SID, __Secure-1PSID/3PSID, HSID, SSID, the SIDTS tokens) that make the difference between an authenticated request and a signed-out one — SAPISIDHASH auth still "succeeds" (those cookies aren't HttpOnly) while the account reads as logged out. Affects any cookies.txt export that uses this convention, which most modern export tools do — not specific to the YouTube Music vertical. --- core/youtube_cookies.py | 32 +++++++++++++++++++++++--------- tests/test_youtube_cookies.py | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/core/youtube_cookies.py b/core/youtube_cookies.py index 4ce88c788..d38e5a0b0 100644 --- a/core/youtube_cookies.py +++ b/core/youtube_cookies.py @@ -27,6 +27,25 @@ # browser name. Anything else non-empty is treated as a browser for cookiesfrombrowser. PASTE_MODE = "custom" +# Netscape cookies.txt convention: an HttpOnly cookie's domain field is prefixed +# with this marker instead of being left plain. These are exactly the +# session-identity cookies (SID, __Secure-1PSID, HSID, SSID, the SIDTS tokens) +# that actually authenticate a request — treating the whole line as a comment +# silently drops the cookies needed to look signed in. +_HTTPONLY_PREFIX = "#HttpOnly_" + + +def _cookie_line_fields(raw: str) -> Optional[list]: + """Split one cookies.txt line into its tab-separated fields, or ``None`` + for a blank line or a genuine comment. Strips ``_HTTPONLY_PREFIX`` first.""" + line = raw.rstrip("\n") + stripped = line.lstrip() + if stripped.startswith(_HTTPONLY_PREFIX): + line = stripped[len(_HTTPONLY_PREFIX):] + elif not line or stripped.startswith("#"): + return None + return line.split("\t") + # ytmusicapi speaks to the same YouTube backend but wants HEADERS, not a cookie # file, so the pasted cookies.txt has to be projected into them (below). @@ -92,10 +111,8 @@ def looks_like_cookiefile(content: Any) -> bool: if not content or not isinstance(content, str): return False for raw in content.splitlines(): - line = raw.rstrip("\n") - if not line or line.lstrip().startswith("#"): - continue - if len(line.split("\t")) >= 6: + fields = _cookie_line_fields(raw) + if fields is not None and len(fields) >= 6: return True return False @@ -138,11 +155,8 @@ def parse_netscape_cookies(content: Any) -> Dict[str, str]: if not content or not isinstance(content, str): return cookies for raw in content.splitlines(): - line = raw.rstrip("\n") - if not line or line.lstrip().startswith("#"): - continue - fields = line.split("\t") - if len(fields) < 7: + fields = _cookie_line_fields(raw) + if fields is None or len(fields) < 7: continue name, value = fields[5].strip(), fields[6].strip() if name: diff --git a/tests/test_youtube_cookies.py b/tests/test_youtube_cookies.py index 020c53184..d8b32744f 100644 --- a/tests/test_youtube_cookies.py +++ b/tests/test_youtube_cookies.py @@ -169,6 +169,29 @@ def test_parse_skips_comments_and_short_rows(): assert parse_netscape_cookies(12345) == {} +def test_parse_reads_httponly_prefixed_rows(): + # Netscape format marks an HttpOnly cookie by prefixing its domain field + # with "#HttpOnly_" instead of leaving the line plain. Treating that as an + # ordinary comment silently drops exactly the session-identity cookies + # (SID, __Secure-1PSID, HSID, ...) that authenticate the request — the + # export still "looks" complete but the account reads as signed out. + jar = ( + "# Netscape HTTP Cookie File\n" + "#HttpOnly_.google.de\tTRUE\t/\tTRUE\t1799999999\t__Secure-1PSID\thttponly-psid\n" + "#HttpOnly_.google.de\tTRUE\t/\tTRUE\t1799999999\tHSID\thttponly-hsid\n" + ".google.de\tTRUE\t/\tTRUE\t1799999999\t__Secure-3PAPISID\tsecret-sapisid\n" + ) + cookies = parse_netscape_cookies(jar) + assert cookies["__Secure-1PSID"] == "httponly-psid" + assert cookies["HSID"] == "httponly-hsid" + assert cookies["__Secure-3PAPISID"] == "secret-sapisid" + + +def test_looks_like_cookiefile_accepts_httponly_only_export(): + jar = "#HttpOnly_.google.de\tTRUE\t/\tTRUE\t1799999999\t__Secure-1PSID\thttponly-psid\n" + assert looks_like_cookiefile(jar) is True + + def test_later_duplicate_row_wins(): jar = _JAR + ".youtube.com\tTRUE\t/\tTRUE\t1799999999\tSID\tnewer-sid\n" assert parse_netscape_cookies(jar)["SID"] == "newer-sid"