diff --git a/changelog/v0.1.0-beta3.txt b/changelog/v0.1.0-beta3.txt index 75cdd808..5d018eeb 100644 --- a/changelog/v0.1.0-beta3.txt +++ b/changelog/v0.1.0-beta3.txt @@ -216,6 +216,8 @@ PLAYLISTS - Playlists whose id contains "VL" open correctly, playlist headers show their owner again, and changing the 4K, HD or subtitles search filter now runs the search again. - Long YouTube playlists load past their first 100 videos as you scroll, show their real title and video count, and download or save in full. Sorting a playlist by date or popularity reads the whole playlist first. - Searching inside a channel keeps returning results past the first page, and long comment threads keep their Show more replies button. +- Playlist tracks that came from sync or an import without a length, view count or upload date get them filled in when the playlist is open, a few a second, and keep them. Sorting by date uses each video's exact upload time, and a drag made meanwhile is kept. +- Videos that can't be filled in, such as removed ones, are skipped for a week instead of being looked up on every visit. DOWNLOADS - Videos now save as MP4 with AAC audio in every quality, VP9 and AV1 included, so they open in more players. A video with no AAC track still saves as MKV. diff --git a/src-tauri/src/api/innertube/extractors/category.rs b/src-tauri/src/api/innertube/extractors/category.rs deleted file mode 100644 index 6b347d78..00000000 --- a/src-tauri/src/api/innertube/extractors/category.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! A video's category ("Gaming", "Education"), which the recommendation engine -//! clusters by. The clients that serve streams return no microformat and the -//! watch answer has no category, so it costs one small `MWEB` player request, -//! asked without a signature timestamp: the answer is unplayable and carries no -//! streams, about 8 KB (Flow for Android's `videoCategory`). - -use std::collections::HashMap; -use std::sync::{Mutex, OnceLock}; - -use crate::api::innertube::InnertubeClient; -use crate::api::innertube::core::clients; -use crate::api::innertube::core::player_requests::PlayerFlights; -use crate::api::innertube::transport::request::Host; -use crate::errors::AppResult; - -// A category never changes, so every one asked this session is kept. -const CAPACITY: usize = 1024; - -fn cache() -> &'static Mutex>> { - static CACHE: OnceLock>>> = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(HashMap::new())) -} - -impl InnertubeClient { - pub async fn get_video_category(&self, video_id: &str) -> AppResult> { - static FLIGHTS: OnceLock>> = OnceLock::new(); - if let Some(category) = cache() - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get(video_id) - { - return Ok(category.clone()); - } - let flight = FLIGHTS - .get_or_init(PlayerFlights::default) - .for_video(video_id); - let category = flight - .get_or_init(|| async { - let mut payload = serde_json::json!({ - "videoId": video_id, - "contentCheckOk": true, - "racyCheckOk": true, - }); - // A failed request is not remembered, so a later watch asks again. - let response = self - .post_via( - &self.client, - Host::Www, - "player", - &clients::MWEB, - &mut payload, - None, - None, - ) - .await - .ok()?; - let category = response["microformat"]["playerMicroformatRenderer"]["category"] - .as_str() - .filter(|category| !category.is_empty()) - .map(ToOwned::to_owned); - let mut cache = cache() - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if cache.len() >= CAPACITY { - cache.clear(); - } - cache.insert(video_id.to_string(), category.clone()); - category - }) - .await - .clone(); - Ok(category) - } -} diff --git a/src-tauri/src/api/innertube/extractors/mod.rs b/src-tauri/src/api/innertube/extractors/mod.rs index 169e8aa1..10b9f722 100644 --- a/src-tauri/src/api/innertube/extractors/mod.rs +++ b/src-tauri/src/api/innertube/extractors/mod.rs @@ -1,5 +1,4 @@ pub mod audio_preference; -pub mod category; pub mod channel; pub mod comments; pub mod explore; @@ -12,4 +11,5 @@ pub mod search; pub mod shorts; pub mod storyboard; pub mod subscription; +pub mod video_metadata; pub mod watch_next; diff --git a/src-tauri/src/api/innertube/extractors/video_metadata.rs b/src-tauri/src/api/innertube/extractors/video_metadata.rs new file mode 100644 index 00000000..67f39b32 --- /dev/null +++ b/src-tauri/src/api/innertube/extractors/video_metadata.rs @@ -0,0 +1,173 @@ +//! What a listed video is: its length, views, upload date and category. The +//! clients that serve streams return no microformat, so this costs one small +//! `MWEB` player request, asked without a signature timestamp: the answer is +//! unplayable and carries no streams, 5 to 10 KB (Flow for Android's +//! `videoCategory`). It stays out of the playback caches, so filling in a list +//! never pushes out what is playing. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +use serde_json::Value; + +use crate::api::innertube::InnertubeClient; +use crate::api::innertube::core::clients; +use crate::api::innertube::core::player_requests::PlayerFlights; +use crate::api::innertube::transport::request::Host; +use crate::errors::{AppError, AppResult}; +use crate::models::video::VideoMetadata; + +// Every answer this session is kept; a failure is not, so a later ask tries again. +const CAPACITY: usize = 1024; + +fn cache() -> &'static Mutex> { + static CACHE: OnceLock>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// The upload time as epoch milliseconds, from a full timestamp or a bare date. +fn published_at_ms(text: &str) -> Option { + chrono::DateTime::parse_from_rfc3339(text) + .map(|date| date.timestamp_millis()) + .ok() + .or_else(|| { + chrono::NaiveDate::parse_from_str(text, "%Y-%m-%d") + .ok()? + .and_hms_opt(0, 0, 0) + .map(|date| date.and_utc().timestamp_millis()) + }) +} + +/// The metadata in an `MWEB` player answer, or none when it describes no video +/// (a removed or private one answers `ERROR` without `videoDetails`). +#[must_use] +pub fn parse_video_metadata(response: &Value) -> Option { + let details = response.get("videoDetails")?; + let microformat = &response["microformat"]["playerMicroformatRenderer"]; + let text = |value: &Value| { + value + .as_str() + .filter(|text| !text.is_empty()) + .map(ToOwned::to_owned) + }; + let is_live = microformat["liveBroadcastDetails"]["isLiveNow"] + .as_bool() + .unwrap_or(false); + Some(VideoMetadata { + title: text(&details["title"]).unwrap_or_default(), + channel_name: text(&details["author"]).unwrap_or_default(), + channel_id: text(&details["channelId"]), + // A stream that is live reports how long it has run, not a length. + duration_seconds: details["lengthSeconds"] + .as_str() + .and_then(|seconds| seconds.parse().ok()) + .filter(|seconds| *seconds > 0 && !is_live), + view_count: details["viewCount"] + .as_str() + .and_then(|views| views.parse().ok()), + published_at_ms: microformat["publishDate"] + .as_str() + .or_else(|| microformat["uploadDate"].as_str()) + .and_then(published_at_ms), + is_live, + category: text(µformat["category"]), + }) +} + +impl InnertubeClient { + pub async fn get_video_metadata(&self, video_id: &str) -> AppResult { + static FLIGHTS: OnceLock>> = OnceLock::new(); + if let Some(metadata) = cache() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(video_id) + { + return Ok(metadata.clone()); + } + let flight = FLIGHTS + .get_or_init(PlayerFlights::default) + .for_video(video_id); + let metadata = flight + .get_or_init(|| async { + let mut payload = serde_json::json!({ + "videoId": video_id, + "contentCheckOk": true, + "racyCheckOk": true, + }); + let response = self + .post_via( + &self.client, + Host::Www, + "player", + &clients::MWEB, + &mut payload, + None, + None, + ) + .await + .ok()?; + let metadata = parse_video_metadata(&response)?; + let mut cache = cache() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if cache.len() >= CAPACITY { + cache.clear(); + } + cache.insert(video_id.to_string(), metadata.clone()); + Some(metadata) + }) + .await + .clone(); + metadata.ok_or_else(|| { + AppError::ContentNotAvailable(format!("No details for video {video_id}")) + }) + } + + /// The video's category, which the recommendation engine clusters by. + pub async fn get_video_category(&self, video_id: &str) -> AppResult> { + Ok(self + .get_video_metadata(video_id) + .await + .ok() + .and_then(|metadata| metadata.category)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn an_mweb_answer_gives_length_views_date_and_category() { + let response = json!({ + "playabilityStatus": { "status": "UNPLAYABLE" }, + "videoDetails": { "title": "Never Gonna Give You Up", "author": "Rick Astley", + "channelId": "UCuAXFkgsw1L7xaCfnd5JJOw", "lengthSeconds": "213", "viewCount": "1830673075" }, + "microformat": { "playerMicroformatRenderer": { + "publishDate": "2009-10-24T23:57:33-07:00", "category": "Music" } } + }); + let metadata = parse_video_metadata(&response).expect("metadata"); + assert_eq!(metadata.duration_seconds, Some(213)); + assert_eq!(metadata.view_count, Some(1_830_673_075)); + assert_eq!(metadata.published_at_ms, Some(1_256_453_853_000)); + assert_eq!(metadata.category.as_deref(), Some("Music")); + assert!(!metadata.is_live); + } + + #[test] + fn a_live_stream_has_no_length_and_a_removed_video_no_metadata() { + let live = json!({ + "videoDetails": { "lengthSeconds": "121601512", "viewCount": "651847115" }, + "microformat": { "playerMicroformatRenderer": { "publishDate": "2022-07-12", + "liveBroadcastDetails": { "isLiveNow": true } } } + }); + let metadata = parse_video_metadata(&live).expect("metadata"); + assert!(metadata.is_live); + assert_eq!(metadata.duration_seconds, None); + assert_eq!(metadata.published_at_ms, Some(1_657_584_000_000)); + assert!( + parse_video_metadata(&json!({ "playabilityStatus": { "status": "ERROR" } })).is_none() + ); + } +} diff --git a/src-tauri/src/commands/youtube.rs b/src-tauri/src/commands/youtube.rs index 0fb6a843..d962cad2 100644 --- a/src-tauri/src/commands/youtube.rs +++ b/src-tauri/src/commands/youtube.rs @@ -9,7 +9,7 @@ use crate::models::comment::CommentsResponse; use crate::models::explore::{ExploreGrid, ExplorePage, ExploreTarget}; use crate::models::playlist::PlaylistDetailsResponse; use crate::models::search::{SearchVideosRequest, SearchVideosResponse}; -use crate::models::video::{RelatedPage, StreamInfo, VideoDetails, VideoSummary}; +use crate::models::video::{RelatedPage, StreamInfo, VideoDetails, VideoMetadata, VideoSummary}; use crate::security::validation::{ validate_browse_id, validate_channel_id, validate_page_token, validate_search_query, validate_video_id, @@ -367,6 +367,20 @@ pub async fn get_related_videos( Ok(related) } +/// A listed video's length, views, upload date and category, from one small +/// request that stays out of the playback caches. +#[tauri::command] +pub async fn get_video_metadata( + video_id: String, + youtube_service: State<'_, YoutubeService>, +) -> Result { + validate_video_id(&video_id).map_err(ErrorResponse::from)?; + youtube_service + .get_video_metadata(&video_id) + .await + .map_err(ErrorResponse::from) +} + /// The video's category, for the recommendation engine. A small request of /// its own, so the watch page asks only once playback has settled. #[tauri::command] diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 3c6f71c2..1c1c28a3 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -116,8 +116,8 @@ use commands::youtube::{ get_post_comments, get_related_videos, get_return_youtube_dislike, get_sabr_debug_state, get_search_suggestions, get_sponsorblock_segments, get_stream_info, get_subscription_rotation_feed, get_subscription_rss_feed, get_video_category, - get_video_details, parse_subscription_export, resolve_channel_id, search_videos, - stream_subscription_rss_feed, submit_sponsorblock_segment, + get_video_details, get_video_metadata, parse_subscription_export, resolve_channel_id, + search_videos, stream_subscription_rss_feed, submit_sponsorblock_segment, }; use services::discord_presence::DiscordPresence; use services::music_service::MusicService; @@ -485,6 +485,7 @@ pub fn run() { get_related_videos, get_more_related_videos, get_video_category, + get_video_metadata, get_stream_info, get_channel_details, get_channel_page, diff --git a/src-tauri/src/models/video.rs b/src-tauri/src/models/video.rs index 882d4f00..aca23d16 100644 --- a/src-tauri/src/models/video.rs +++ b/src-tauri/src/models/video.rs @@ -156,6 +156,22 @@ pub struct RelatedPage { pub continuation: Option, } +/// What a listed video is, from one small player request: enough to fill in a +/// stored track without the full details lookup. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VideoMetadata { + pub title: String, + pub channel_name: String, + pub channel_id: Option, + pub duration_seconds: Option, + pub view_count: Option, + /// The upload time, exact, in epoch milliseconds. + pub published_at_ms: Option, + pub is_live: bool, + pub category: Option, +} + /// The two facts a feed needs to fill in for a listed video, without the full /// details lookup and its `next` request. #[derive(Debug, Clone, Default)] diff --git a/src-tauri/src/services/youtube_service.rs b/src-tauri/src/services/youtube_service.rs index 944defec..77669a6b 100644 --- a/src-tauri/src/services/youtube_service.rs +++ b/src-tauri/src/services/youtube_service.rs @@ -10,7 +10,7 @@ use crate::models::live_chat::LiveChatResponse; use crate::models::playlist::PlaylistDetailsResponse; use crate::models::search::{SearchVideosRequest, SearchVideosResponse}; use crate::models::shorts::{ShortItem, ShortsFeed}; -use crate::models::video::{RelatedPage, StreamInfo, VideoBasics, VideoDetails}; +use crate::models::video::{RelatedPage, StreamInfo, VideoBasics, VideoDetails, VideoMetadata}; #[derive(Clone)] pub struct YoutubeService { @@ -42,6 +42,10 @@ impl YoutubeService { self.extractor.get_related_videos(video_id).await } + pub async fn get_video_metadata(&self, video_id: &str) -> AppResult { + self.extractor.get_video_metadata(video_id).await + } + pub async fn get_video_category(&self, video_id: &str) -> AppResult> { self.extractor.get_video_category(video_id).await } diff --git a/src/lib/api/youtube.ts b/src/lib/api/youtube.ts index dbcc90cc..91b38c4d 100644 --- a/src/lib/api/youtube.ts +++ b/src/lib/api/youtube.ts @@ -2,6 +2,7 @@ import type { SearchVideosRequest, SearchVideosResponse, VideoDetails, + VideoMetadata, StreamInfo, ChannelAbout, ChannelDetails, @@ -127,6 +128,12 @@ export async function getMoreRelatedVideos(continuation: string): Promise("get_more_related_videos", { continuation }); } +/** A listed video's length, views, upload date and category, from one small request. */ +export async function getVideoMetadata(videoId: string): Promise { + if (!(await isTauriEnv())) throw new Error("Video metadata requires the desktop app."); + return invokeBackend("get_video_metadata", { videoId }); +} + /** The video's category, from a small request of its own. */ export async function getVideoCategory(videoId: string): Promise { if (!(await isTauriEnv())) return null; diff --git a/src/lib/playlistEnrichment.test.ts b/src/lib/playlistEnrichment.test.ts new file mode 100644 index 00000000..b4f5fbf0 --- /dev/null +++ b/src/lib/playlistEnrichment.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { applyVideoMetadata, needsEnrichment } from "./playlistEnrichment"; +import type { VideoMetadata, VideoSummary } from "../types/video"; + +const NOW = Date.UTC(2026, 9, 26, 12); + +const video = (id: string, extra: Partial = {}): VideoSummary => ({ + id, + title: id, + channelName: "chan", + ...extra, +}); + +const metadata = (extra: Partial = {}): VideoMetadata => ({ + title: "", + channelName: "", + isLive: false, + ...extra, +}); + +const complete = { durationSeconds: 120, viewCountText: "1K views", publishedText: "2 days ago" }; + +describe("needsEnrichment", () => { + it("flags a track missing a duration, view count, or publish date", () => { + expect(needsEnrichment(video("full", complete))).toBe(false); + expect(needsEnrichment(video("v", { ...complete, durationSeconds: null }))).toBe(true); + expect(needsEnrichment(video("v", { ...complete, viewCountText: null }))).toBe(true); + expect(needsEnrichment(video("v", { ...complete, publishedText: null }))).toBe(true); + }); + + it("treats a live stream as having its duration", () => { + expect(needsEnrichment(video("live", { ...complete, durationSeconds: null, isLive: true }))) + .toBe(false); + }); +}); + +describe("applyVideoMetadata", () => { + it("fills a stub with the length, exact views and exact upload time", () => { + const uploaded = Date.UTC(2023, 9, 26, 12); + const refreshed = new Map([["v", metadata({ durationSeconds: 213, viewCount: 1_830_673_075, publishedAtMs: uploaded })]]); + + const [next] = applyVideoMetadata([video("v")], refreshed, NOW); + + expect(next?.durationSeconds).toBe(213); + expect(next?.viewCount).toBe(1_830_673_075); + expect(next?.viewCountText).toBe("1.8B views"); + expect(next?.publishedAt).toBe(uploaded); + expect(next?.publishedText).toBe("3 years ago"); + expect(next && needsEnrichment(next)).toBe(false); + }); + + it("keeps a stored value the metadata lacks", () => { + const videos = [video("v", { durationSeconds: 300, publishedText: "2 days ago" })]; + const refreshed = new Map([["v", metadata({ viewCount: 2_000_000 })]]); + + const [next] = applyVideoMetadata(videos, refreshed, NOW); + + expect(next?.title).toBe("v"); + expect(next?.durationSeconds).toBe(300); + expect(next?.publishedText).toBe("2 days ago"); + expect(next?.viewCountText).toBe("2M views"); + }); + + it("marks a stream on air as live and one that has ended as not", () => { + const live = applyVideoMetadata([video("l")], new Map([["l", metadata({ isLive: true })]]), NOW); + expect(live[0]?.isLive).toBe(true); + const ended = applyVideoMetadata([video("e", { isLive: true })], new Map([["e", metadata({ durationSeconds: 5400 })]]), NOW); + expect(ended[0]?.isLive).toBe(false); + }); + + it("returns the same array when nothing changes", () => { + const videos = [video("v", { ...complete, viewCount: 1000, viewCountText: "1K views" }), video("other")]; + const refreshed = new Map([["v", metadata({ durationSeconds: 120, viewCount: 1000 })]]); + + expect(applyVideoMetadata(videos, refreshed, NOW)).toBe(videos); + }); +}); diff --git a/src/lib/playlistEnrichment.ts b/src/lib/playlistEnrichment.ts new file mode 100644 index 00000000..d0d037d0 --- /dev/null +++ b/src/lib/playlistEnrichment.ts @@ -0,0 +1,59 @@ +import { formatYouTubeRelativeTime } from "./publishedDate"; +import { formatCount } from "./utils"; +import type { VideoMetadata, VideoSummary } from "../types/video"; + +// Mirrors Flow-Android's PlaylistDetailViewModel stub enrichment. +export const ENRICHMENT_TRACK_LIMIT = 50; +export const ENRICHMENT_CHUNK_SIZE = 5; +export const ENRICHMENT_CHUNK_DELAY_MS = 300; + +/** Missing a duration, view count or publish date. A live stream has no duration to fetch. */ +export const needsEnrichment = (video: VideoSummary): boolean => + !(video.isLive || (video.durationSeconds ?? 0) > 0) || + !video.viewCountText || + !video.publishedText; + +const asNullable = (value: T | undefined) => value ?? null; + +/** + * Fresh metadata wins, like Android's updateVideoMetadata; fields it lacks keep their stored + * value. The upload time is stored exactly, so sorting by date never depends on the language a + * date was written in, and the shown "3 years ago" is worked out from it. + */ +export function applyVideoMetadata( + videos: VideoSummary[], + refreshed: Map, + now = Date.now(), +): VideoSummary[] { + let changed = false; + + const merged = videos.map((video) => { + const metadata = refreshed.get(video.id); + if (!metadata) return video; + + const views = metadata.viewCount; + const publishedAt = metadata.publishedAtMs; + const next: VideoSummary = { + ...video, + title: metadata.title || video.title, + channelName: metadata.channelName || video.channelName, + channelId: metadata.channelId ?? video.channelId ?? null, + durationSeconds: metadata.durationSeconds || video.durationSeconds || null, + // A stream that has ended has a length; one still on air keeps its live flag. + isLive: metadata.isLive ? true : video.isLive && metadata.durationSeconds ? false : video.isLive ?? null, + viewCount: views ?? video.viewCount ?? null, + viewCountText: views != null ? `${formatCount(views)} views` : video.viewCountText ?? null, + publishedAt: publishedAt ?? video.publishedAt ?? null, + publishedText: publishedAt ? formatYouTubeRelativeTime(publishedAt, now) : video.publishedText ?? null, + }; + + const differs = (Object.keys(next) as Array).some( + (key) => asNullable(next[key]) !== asNullable(video[key]), + ); + if (!differs) return video; + changed = true; + return next; + }); + + return changed ? merged : videos; +} diff --git a/src/lib/playlistLibrary.test.ts b/src/lib/playlistLibrary.test.ts index 579d2f6b..45c39410 100644 --- a/src/lib/playlistLibrary.test.ts +++ b/src/lib/playlistLibrary.test.ts @@ -3,12 +3,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { VideoSummary } from "../types/video"; const settings = new Map(); +const setSetting = vi.fn(async (key: string, value: string) => { + settings.set(key, value); +}); vi.mock("./api/db", () => ({ getSetting: async (key: string) => settings.get(key) ?? null, - setSetting: async (key: string, value: string) => { - settings.set(key, value); - }, + setSetting: (key: string, value: string) => setSetting(key, value), })); const getPlaylistDetails = vi.fn(); vi.mock("./api/youtube", () => ({ @@ -21,6 +22,7 @@ import { addVideoToWatchLater, getStoredPlaylistById, savePlaylistToLibrary, + updateStoredPlaylistTracks, } from "./playlistLibrary"; const NOW = Date.UTC(2026, 9, 1, 12); @@ -31,33 +33,88 @@ const video = (id: string, extra: Partial = {}): VideoSummary => ( id, title: id, channelName: "chan", + thumbnailUrl: `thumb-${id}`, ...extra, }); -const addedAt = async (playlistId: string) => - Object.fromEntries( - (await getStoredPlaylistById(playlistId))?.tracks.map((track) => [track.id, track.addedAtInPlaylist ?? null]) ?? [], - ); - -beforeEach(() => { - vi.useFakeTimers({ now: NOW }); +const seed = (playlists: unknown[]) => { settings.clear(); + setSetting.mockClear(); getPlaylistDetails.mockReset(); - settings.set( - "user_playlists", - JSON.stringify([ + settings.set("user_playlists", JSON.stringify(playlists)); +}; + +describe("updateStoredPlaylistTracks", () => { + beforeEach(() => { + seed([ + { + id: WATCH_LATER_PLAYLIST_ID, + name: "Watch Later", + tracks: [video("a"), video("b"), video("c")], + createdAt: "2024-01-01T00:00:00.000Z", + source: "Owned", + }, + { + id: SAVED_ID, + name: "Saved", + tracks: [video("a")], + createdAt: "2024-01-01T00:00:00.000Z", + source: "Saved", + thumbnailUrl: "card-thumb", + videoCountText: "500 videos", + }, + ]); + }); + + it("skips the write when an updater returns the stored tracks unchanged", async () => { + await updateStoredPlaylistTracks(WATCH_LATER_PLAYLIST_ID, (tracks) => tracks); + expect(setSetting).not.toHaveBeenCalled(); + }); + + it("derives the thumbnail and count from the new tracks", async () => { + const updated = await updateStoredPlaylistTracks(WATCH_LATER_PLAYLIST_ID, [video("c")]); + expect(updated?.thumbnailUrl).toBe("thumb-c"); + expect(updated?.videoCountText).toBe("1 video"); + }); + + it("keeps a saved playlist's full count text when its first-page snapshot changes", async () => { + const updated = await updateStoredPlaylistTracks(SAVED_ID, [video("x"), video("y")]); + expect(updated?.tracks.map((track) => track.id)).toEqual(["x", "y"]); + expect(updated?.thumbnailUrl).toBe("thumb-x"); + expect(updated?.videoCountText).toBe("500 videos"); + }); + + it("runs changes made at the same time one after another, so neither undoes the other", async () => { + await Promise.all([ + addVideoToWatchLater(video("new")), + updateStoredPlaylistTracks(WATCH_LATER_PLAYLIST_ID, (tracks) => + tracks.map((track) => (track.id === "a" ? { ...track, durationSeconds: 99 } : track))), + ]); + const stored = await getStoredPlaylistById(WATCH_LATER_PLAYLIST_ID); + expect(stored?.tracks.map((track) => track.id)).toEqual(["new", "a", "b", "c"]); + expect(stored?.tracks.find((track) => track.id === "a")?.durationSeconds).toBe(99); + }); +}); + +describe("add times", () => { + const addedAt = async (playlistId: string) => + Object.fromEntries( + (await getStoredPlaylistById(playlistId))?.tracks.map((track) => [track.id, track.addedAtInPlaylist ?? null]) ?? [], + ); + + beforeEach(() => { + vi.useFakeTimers({ now: NOW }); + seed([ { id: WATCH_LATER_PLAYLIST_ID, name: "Watch Later", tracks: [video("old")], source: "Owned" }, { id: OWNED_ID, name: "Mine", tracks: [video("old")], source: "Owned" }, { id: SAVED_ID, name: "Saved", tracks: [video("kept", { addedAtInPlaylist: 1_000 })], source: "Saved" }, - ]), - ); -}); + ]); + }); -afterEach(() => { - vi.useRealTimers(); -}); + afterEach(() => { + vi.useRealTimers(); + }); -describe("add times", () => { it("stamps a video added to Watch Later", async () => { await addVideoToWatchLater(video("new")); expect(await addedAt(WATCH_LATER_PLAYLIST_ID)).toEqual({ new: NOW, old: null }); diff --git a/src/lib/playlistLibrary.ts b/src/lib/playlistLibrary.ts index 84f1b13f..f7748f3e 100644 --- a/src/lib/playlistLibrary.ts +++ b/src/lib/playlistLibrary.ts @@ -169,6 +169,19 @@ const withRequiredPlaylists = (playlists: StoredPlaylist[]) => { ]; }; +let libraryWrites: Promise = Promise.resolve(); + +/** + * Every change reads the stored library and writes it back whole, so two at once would undo + * one another: a reorder saved while an enrichment chunk is saving, or two adds in a row. They + * run one at a time instead. A change must not start another from inside itself. + */ +function withLibraryWrite(change: () => Promise): Promise { + const run = libraryWrites.then(change, change); + libraryWrites = run.catch(() => undefined); + return run; +} + export const loadStoredPlaylists = async () => { const playlistsJson = await getSetting(PLAYLISTS_SETTING_KEY); if (!playlistsJson) return withRequiredPlaylists([]); @@ -185,8 +198,8 @@ export const getStoredPlaylistById = async (playlistId: string) => { export const updateStoredPlaylistTracks = async ( playlistId: string, - tracks: VideoSummary[], -) => { + tracks: VideoSummary[] | ((current: VideoSummary[]) => VideoSummary[]), +) => withLibraryWrite(async () => { const playlists = await loadStoredPlaylists(); const index = playlists.findIndex((stored) => stored.id === playlistId); if (index === -1) return null; @@ -194,15 +207,23 @@ export const updateStoredPlaylistTracks = async ( const current = playlists[index]; if (!current) return null; - const videoCount = tracks.length; + const nextTracks = typeof tracks === "function" ? tracks(current.tracks) : tracks; + if (nextTracks === current.tracks) return current; + + const videoCount = nextTracks.length; const updated: StoredPlaylist = normalizePlaylist({ ...current, id: current.id, name: current.name, - tracks, - thumbnailUrl: tracks[0]?.thumbnailUrl ?? current.thumbnailUrl ?? null, - videoCount: videoCount > 0 ? videoCount : null, - videoCountText: videoCount > 0 ? formatVideoCountText(videoCount) : null, + tracks: nextTracks, + thumbnailUrl: nextTracks[0]?.thumbnailUrl ?? current.thumbnailUrl ?? null, + // A saved playlist may store only part of what YouTube lists; keep its remote count. + ...(current.source === "Saved" + ? {} + : { + videoCount: videoCount > 0 ? videoCount : null, + videoCountText: videoCount > 0 ? formatVideoCountText(videoCount) : null, + }), }); const nextPlaylists = playlists.map((stored, storedIndex) => ( @@ -211,7 +232,7 @@ export const updateStoredPlaylistTracks = async ( await persistStoredPlaylists(nextPlaylists); return updated; -}; +}); export const persistStoredPlaylists = async (playlists: StoredPlaylist[]) => { const now = Date.now(); @@ -234,9 +255,6 @@ export const isPlaylistInLibrary = async (playlistId: string) => { export const savePlaylistToLibrary = async (playlist: PlaylistSummary) => { if (isProtectedPlaylistId(playlist.id)) return loadStoredPlaylists(); - const playlists = await loadStoredPlaylists(); - const existingPlaylist = playlists.find((storedPlaylist) => storedPlaylist.id === playlist.id); - let details: Awaited> | null = null; try { details = await getPlaylistDetails(playlist.id); @@ -244,10 +262,7 @@ export const savePlaylistToLibrary = async (playlist: PlaylistSummary) => { console.warn("Failed to fetch playlist details while saving", error); } - const addedAt = Date.now(); - const existingAddedAt = new Map( - (existingPlaylist?.tracks ?? []).map((track) => [track.id, track.addedAtInPlaylist]), - ); + // The pages are fetched before taking the library, so a slow save never holds up other changes. let fetchedVideos = details?.videos ?? []; if (details?.nextPageToken) { try { @@ -256,65 +271,76 @@ export const savePlaylistToLibrary = async (playlist: PlaylistSummary) => { console.warn("Failed to fetch every playlist page while saving", error); } } - const fetchedTracks = fetchedVideos.map((track) => ({ - ...track, - addedAtInPlaylist: existingAddedAt.get(track.id) ?? addedAt, - })); - const parsedCount = parseVideoCountFromText(playlist.videoCountText); - const videoCount = details?.videoCount - ?? (fetchedTracks.length > 0 ? fetchedTracks.length : null) - ?? parsedCount - ?? existingPlaylist?.videoCount - ?? 0; - - const thumbnailUrl = playlist.thumbnailUrl - ?? fetchedTracks[0]?.thumbnailUrl - ?? existingPlaylist?.thumbnailUrl - ?? null; - - const resolvedName = resolvePlaylistTitle( - playlist.title, - details?.title, - existingPlaylist?.sourceTitle, - existingPlaylist?.name, - ); - - const savedPlaylist: StoredPlaylist = { - id: playlist.id, - name: resolvedName, - sourceTitle: playlist.title.trim() || existingPlaylist?.sourceTitle || resolvedName, - description: details?.description?.trim() || existingPlaylist?.description || "Saved playlist", - tracks: fetchedTracks.length > 0 ? fetchedTracks : (existingPlaylist?.tracks ?? []), - createdAt: existingPlaylist?.createdAt ?? new Date().toISOString(), - source: "Saved", - thumbnailUrl, - videoCount: videoCount > 0 ? videoCount : null, - videoCountText: playlist.videoCountText ?? (videoCount > 0 ? formatVideoCountText(videoCount) : null), - }; - const updatedPlaylists = existingPlaylist - ? playlists.map((storedPlaylist) => ( - storedPlaylist.id === playlist.id ? savedPlaylist : storedPlaylist - )) - : [...playlists, savedPlaylist]; - - await persistStoredPlaylists(updatedPlaylists); - return updatedPlaylists; + return withLibraryWrite(async () => { + const playlists = await loadStoredPlaylists(); + const existingPlaylist = playlists.find((storedPlaylist) => storedPlaylist.id === playlist.id); + const addedAt = Date.now(); + const existingAddedAt = new Map( + (existingPlaylist?.tracks ?? []).map((track) => [track.id, track.addedAtInPlaylist]), + ); + const fetchedTracks = fetchedVideos.map((track) => ({ + ...track, + addedAtInPlaylist: existingAddedAt.get(track.id) ?? addedAt, + })); + const parsedCount = parseVideoCountFromText(playlist.videoCountText); + const videoCount = details?.videoCount + ?? (fetchedTracks.length > 0 ? fetchedTracks.length : null) + ?? parsedCount + ?? existingPlaylist?.videoCount + ?? 0; + + const thumbnailUrl = playlist.thumbnailUrl + ?? fetchedTracks[0]?.thumbnailUrl + ?? existingPlaylist?.thumbnailUrl + ?? null; + + const resolvedName = resolvePlaylistTitle( + playlist.title, + details?.title, + existingPlaylist?.sourceTitle, + existingPlaylist?.name, + ); + + const savedPlaylist: StoredPlaylist = { + id: playlist.id, + name: resolvedName, + sourceTitle: playlist.title.trim() || existingPlaylist?.sourceTitle || resolvedName, + description: details?.description?.trim() || existingPlaylist?.description || "Saved playlist", + tracks: fetchedTracks.length > 0 ? fetchedTracks : (existingPlaylist?.tracks ?? []), + createdAt: existingPlaylist?.createdAt ?? new Date().toISOString(), + source: "Saved", + thumbnailUrl, + videoCount: videoCount > 0 ? videoCount : null, + videoCountText: playlist.videoCountText ?? (videoCount > 0 ? formatVideoCountText(videoCount) : null), + }; + + const updatedPlaylists = existingPlaylist + ? playlists.map((storedPlaylist) => ( + storedPlaylist.id === playlist.id ? savedPlaylist : storedPlaylist + )) + : [...playlists, savedPlaylist]; + + await persistStoredPlaylists(updatedPlaylists); + return updatedPlaylists; + }); }; export const removePlaylistFromLibrary = async (playlistId: string) => { if (isProtectedPlaylistId(playlistId)) return loadStoredPlaylists(); - const playlists = await loadStoredPlaylists(); - const updatedPlaylists = playlists.filter((playlist) => playlist.id !== playlistId); - await persistStoredPlaylists(updatedPlaylists); - return updatedPlaylists; + return withLibraryWrite(async () => { + const playlists = await loadStoredPlaylists(); + const updatedPlaylists = playlists.filter((playlist) => playlist.id !== playlistId); + await persistStoredPlaylists(updatedPlaylists); + return updatedPlaylists; + }); }; export const createStoredPlaylist = async ( name: string, description = "", -) => { +) => withLibraryWrite(async () => { const playlists = await loadStoredPlaylists(); const playlist: StoredPlaylist = normalizePlaylist({ id: `playlist-${Date.now()}`, @@ -330,7 +356,7 @@ export const createStoredPlaylist = async ( await persistStoredPlaylists([...playlists, playlist]); return playlist; -}; +}); export const addVideoToStoredPlaylist = async ( playlistId: string, @@ -338,33 +364,35 @@ export const addVideoToStoredPlaylist = async ( ) => { if (isProtectedPlaylistId(playlistId)) return addVideoToWatchLater(video); - const playlists = await loadStoredPlaylists(); - let updatedPlaylist: StoredPlaylist | null = null; - const nextPlaylists = playlists.map((playlist) => { - if (playlist.id !== playlistId || playlist.source !== "Owned") return playlist; - if (playlist.tracks.some((track) => track.id === video.id)) { - updatedPlaylist = playlist; - return playlist; - } - - const tracks = [...playlist.tracks, { ...video, addedAtInPlaylist: Date.now() }]; - updatedPlaylist = normalizePlaylist({ - ...playlist, - tracks, - thumbnailUrl: playlist.thumbnailUrl ?? video.thumbnailUrl ?? null, - videoCount: tracks.length, - videoCountText: formatVideoCountText(tracks.length), + return withLibraryWrite(async () => { + const playlists = await loadStoredPlaylists(); + let updatedPlaylist: StoredPlaylist | null = null; + const nextPlaylists = playlists.map((playlist) => { + if (playlist.id !== playlistId || playlist.source !== "Owned") return playlist; + if (playlist.tracks.some((track) => track.id === video.id)) { + updatedPlaylist = playlist; + return playlist; + } + + const tracks = [...playlist.tracks, { ...video, addedAtInPlaylist: Date.now() }]; + updatedPlaylist = normalizePlaylist({ + ...playlist, + tracks, + thumbnailUrl: playlist.thumbnailUrl ?? video.thumbnailUrl ?? null, + videoCount: tracks.length, + videoCountText: formatVideoCountText(tracks.length), + }); + return updatedPlaylist; }); - return updatedPlaylist; - }); - if (!updatedPlaylist) return null; + if (!updatedPlaylist) return null; - await persistStoredPlaylists(nextPlaylists); - return updatedPlaylist; + await persistStoredPlaylists(nextPlaylists); + return updatedPlaylist; + }); }; -export const addVideoToWatchLater = async (video: VideoSummary) => { +export const addVideoToWatchLater = async (video: VideoSummary) => withLibraryWrite(async () => { const playlists = await loadStoredPlaylists(); const nextPlaylists = playlists.map((playlist) => { if (playlist.id !== WATCH_LATER_PLAYLIST_ID) return playlist; @@ -382,9 +410,9 @@ export const addVideoToWatchLater = async (video: VideoSummary) => { await persistStoredPlaylists(nextPlaylists); return getStoredPlaylistById(WATCH_LATER_PLAYLIST_ID); -}; +}); -export const removeVideoFromWatchLater = async (videoId: string) => { +export const removeVideoFromWatchLater = async (videoId: string) => withLibraryWrite(async () => { const playlists = await loadStoredPlaylists(); const nextPlaylists = playlists.map((playlist) => { if (playlist.id !== WATCH_LATER_PLAYLIST_ID) return playlist; @@ -401,4 +429,4 @@ export const removeVideoFromWatchLater = async (videoId: string) => { await persistStoredPlaylists(nextPlaylists); return getStoredPlaylistById(WATCH_LATER_PLAYLIST_ID); -}; +}); diff --git a/src/lib/playlistMetadataChecks.test.ts b/src/lib/playlistMetadataChecks.test.ts new file mode 100644 index 00000000..08fe282b --- /dev/null +++ b/src/lib/playlistMetadataChecks.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; + +import { METADATA_CHECK_COOLDOWN_MS, pruneChecks } from "./playlistMetadataChecks"; + +describe("pruneChecks", () => { + it("drops checks past their cooldown and keeps the rest", () => { + const now = 10 * METADATA_CHECK_COOLDOWN_MS; + const checks = new Map([ + ["fresh", now - 1000], + ["expired", now - METADATA_CHECK_COOLDOWN_MS - 1], + ]); + expect([...pruneChecks(checks, now).keys()]).toEqual(["fresh"]); + }); +}); diff --git a/src/lib/playlistMetadataChecks.ts b/src/lib/playlistMetadataChecks.ts new file mode 100644 index 00000000..0906afad --- /dev/null +++ b/src/lib/playlistMetadataChecks.ts @@ -0,0 +1,44 @@ +import { getSetting, setSetting } from "./api/db"; + +/** + * Tracks whose details could not be filled in: removed or private videos, and ones that came back + * still missing a field (a premiere has no length yet). Each is left alone for a week, so they + * don't take the per-visit budget from tracks that can still be completed. A track that was + * filled in is not recorded, so one a sync later resets to a stub is looked up again. + */ +const SETTING_KEY = "playlist_metadata_checks"; +export const METADATA_CHECK_COOLDOWN_MS = 7 * 24 * 60 * 60 * 1000; +const MAX_CHECKS = 5000; + +let checks: Promise> | null = null; + +function load(): Promise> { + checks ??= getSetting(SETTING_KEY) + .then((raw) => new Map(Object.entries(raw ? (JSON.parse(raw) as Record) : {}))) + .catch(() => new Map()); + return checks; +} + +/** The checks still within their cooldown, newest kept when there are too many. */ +export function pruneChecks(all: Map, now: number): Map { + const live = [...all].filter(([, at]) => now - at < METADATA_CHECK_COOLDOWN_MS); + live.sort((a, b) => b[1] - a[1]); + return new Map(live.slice(0, MAX_CHECKS)); +} + +export async function recentlyCheckedIds(now = Date.now()): Promise> { + return new Set(pruneChecks(await load(), now).keys()); +} + +export async function rememberChecks(ids: string[], now = Date.now()): Promise { + if (ids.length === 0) return; + const all = await load(); + for (const id of ids) all.set(id, now); + const kept = pruneChecks(all, now); + checks = Promise.resolve(kept); + try { + await setSetting(SETTING_KEY, JSON.stringify(Object.fromEntries(kept))); + } catch (error) { + console.warn("Failed to remember checked playlist tracks", error); + } +} diff --git a/src/lib/usePlaylistDetails.ts b/src/lib/usePlaylistDetails.ts index d0f9b1e9..b9f68f10 100644 --- a/src/lib/usePlaylistDetails.ts +++ b/src/lib/usePlaylistDetails.ts @@ -1,15 +1,24 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { getPlaylistDetails } from "./api/youtube"; +import { getPlaylistDetails, getVideoMetadata } from "./api/youtube"; import { formatVideoCountText, getStoredPlaylistById, isProtectedPlaylistId, normalizePlaylist, resolvePlaylistTitle, + updateStoredPlaylistTracks, type StoredPlaylist, } from "./playlistLibrary"; import { appendUniqueVideos, loadRemainingPlaylistVideos } from "./playlistPages"; -import type { VideoSummary } from "../types/video"; +import { + applyVideoMetadata, + ENRICHMENT_CHUNK_DELAY_MS, + ENRICHMENT_CHUNK_SIZE, + ENRICHMENT_TRACK_LIMIT, + needsEnrichment, +} from "./playlistEnrichment"; +import { recentlyCheckedIds, rememberChecks } from "./playlistMetadataChecks"; +import type { VideoMetadata, VideoSummary } from "../types/video"; export interface PlaylistDetailsMeta { id: string; @@ -46,6 +55,8 @@ export function usePlaylistDetails(playlistId: string | undefined) { setError(null); setNextPageToken(null); setLoadMoreFailed(false); + // Cleared so enrichment never pairs the old list with the new playlist. + setVideos([]); try { const stored = await getStoredPlaylistById(playlistId); @@ -172,6 +183,78 @@ export function usePlaylistDetails(playlistId: string | undefined) { } }, [playlistId, nextPageToken, videos]); + // Per playlist, since this hook outlives a switch; a pass stops once its playlist leaves the screen. + // Tracks this visit has dealt with, looked up or skipped, and how many lookups it has made. + const consideredTracks = useRef(new Set()); + const lookupsThisVisit = useRef(0); + const enrichmentRunning = useRef(null); + const activePlaylistId = useRef(playlistId); + const latestVideos = useRef(videos); + latestVideos.current = videos; + useEffect(() => { + activePlaylistId.current = playlistId; + consideredTracks.current = new Set(); + lookupsThisVisit.current = 0; + return () => { + activePlaylistId.current = undefined; + }; + }, [playlistId]); + + const enrichTracks = useCallback((id: string, tracks: VideoSummary[]) => { + // The visit's budget: at most ENRICHMENT_TRACK_LIMIT lookups, however many passes it takes. + const budget = ENRICHMENT_TRACK_LIMIT - lookupsThisVisit.current; + if (budget <= 0 || enrichmentRunning.current === id) return; + const candidates = tracks.filter((track) => needsEnrichment(track) && !consideredTracks.current.has(track.id)); + if (candidates.length === 0) return; + enrichmentRunning.current = id; + let lookedUp = 0; + + void (async () => { + try { + const checked = await recentlyCheckedIds(); + for (const track of candidates) { + if (checked.has(track.id)) consideredTracks.current.add(track.id); + } + const targets = candidates.filter((track) => !checked.has(track.id)).slice(0, budget); + for (const track of targets) consideredTracks.current.add(track.id); + lookupsThisVisit.current += targets.length; + lookedUp = targets.length; + + for (let start = 0; start < targets.length; start += ENRICHMENT_CHUNK_SIZE) { + if (activePlaylistId.current !== id) return; + const chunk = targets.slice(start, start + ENRICHMENT_CHUNK_SIZE); + const refreshed = new Map(); + for (const track of chunk) { + try { + refreshed.set(track.id, await getVideoMetadata(track.id)); + } catch { + // Private or removed videos are expected in old playlists. + } + } + if (refreshed.size > 0) { + setVideos((previous) => applyVideoMetadata(previous, refreshed)); + // Applied to what's stored now, so a reorder or removal meanwhile survives. + await updateStoredPlaylistTracks(id, (stored) => applyVideoMetadata(stored, refreshed)); + } + // Failed or still incomplete: leave it alone for a while instead of on every visit. + const enriched = applyVideoMetadata(chunk, refreshed); + await rememberChecks(enriched.filter(needsEnrichment).map((track) => track.id)); + await new Promise((resolve) => setTimeout(resolve, ENRICHMENT_CHUNK_DELAY_MS)); + } + } catch (enrichError) { + console.warn("Failed to save enriched playlist tracks", enrichError); + } finally { + if (enrichmentRunning.current === id) enrichmentRunning.current = null; + } + // Pages loaded while the pass ran are looked at now, within what is left of the budget. + if (lookedUp > 0 && activePlaylistId.current === id) enrichTracks(id, latestVideos.current); + })(); + }, []); + + useEffect(() => { + if (playlistId && storedPlaylist?.id === playlistId) enrichTracks(playlistId, videos); + }, [videos, playlistId, storedPlaylist?.id, enrichTracks]); + return { loading, error, diff --git a/src/pages/PlaylistDetailsPage.tsx b/src/pages/PlaylistDetailsPage.tsx index fa9c8a1f..8a7598ae 100644 --- a/src/pages/PlaylistDetailsPage.tsx +++ b/src/pages/PlaylistDetailsPage.tsx @@ -55,19 +55,15 @@ export function PlaylistDetailsPage({ loadMoreFailed, loadMore, loadAllVideos, + setVideos, } = usePlaylistDetails(playlistId); usePublishTitle(meta?.title); const [storedSortOrder, setStoredSortOrder] = useState(() => playlistId ? loadPlaylistSortOrder(playlistId) : null, ); - const [manualVideos, setManualVideos] = useState([]); const [savedInLibrary, setSavedInLibrary] = useState(false); - useEffect(() => { - setManualVideos(videos); - }, [videos]); - useEffect(() => { setStoredSortOrder(playlistId ? loadPlaylistSortOrder(playlistId) : null); }, [playlistId]); @@ -94,11 +90,11 @@ export function PlaylistDetailsPage({ }, [sortOrder, hasMore, loadAllVideos]); const displayVideos = useMemo( - () => sortPlaylistVideos(manualVideos, sortOrder, isProtected), - [manualVideos, sortOrder, isProtected], + () => sortPlaylistVideos(videos, sortOrder, isProtected), + [videos, sortOrder, isProtected], ); - const leadVideo = manualVideos[0]; + const leadVideo = videos[0]; const heroThumbnailUrl = leadVideo?.thumbnailUrl ?? meta?.thumbnailUrl ?? null; const openCollectionDownload = useDownloadStore((state) => state.openCollection); @@ -119,7 +115,7 @@ export function PlaylistDetailsPage({ }; const handleReorder = async (nextVideos: VideoSummary[]) => { - setManualVideos(nextVideos); + setVideos(nextVideos); if (storedPlaylist) { await updateStoredPlaylistTracks(storedPlaylist.id, nextVideos); } @@ -161,7 +157,7 @@ export function PlaylistDetailsPage({ try { const playlists = await savePlaylistToLibrary(summary); const saved = playlists.find((playlist) => playlist.id === meta.id); - if (saved) setManualVideos(saved.tracks); + if (saved) setVideos(saved.tracks); setSavedInLibrary(true); showToast({ variant: "success", @@ -261,7 +257,7 @@ export function PlaylistDetailsPage({ />