From b972cc46e73e88c1960a63a9d0d08dce6d23bdd4 Mon Sep 17 00:00:00 2001 From: Jiwenji Date: Fri, 19 Jun 2026 23:18:27 -0400 Subject: [PATCH] missav scraper --- extensions/MissAv/MissAV123Scraper.csproj | 21 + .../MissAv/MissAV123ScraperExtension.cs | 453 ++++++++++++++++++ extensions/MissAv/extension.json | 22 + 3 files changed, 496 insertions(+) create mode 100644 extensions/MissAv/MissAV123Scraper.csproj create mode 100644 extensions/MissAv/MissAV123ScraperExtension.cs create mode 100644 extensions/MissAv/extension.json diff --git a/extensions/MissAv/MissAV123Scraper.csproj b/extensions/MissAv/MissAV123Scraper.csproj new file mode 100644 index 0000000..0d0be60 --- /dev/null +++ b/extensions/MissAv/MissAV123Scraper.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + true + 0.0.32 + $(CoveMinVersion) + $(CoveMinVersion) + false + false + + + + + + + + falseruntime + falseruntime + diff --git a/extensions/MissAv/MissAV123ScraperExtension.cs b/extensions/MissAv/MissAV123ScraperExtension.cs new file mode 100644 index 0000000..37cb210 --- /dev/null +++ b/extensions/MissAv/MissAV123ScraperExtension.cs @@ -0,0 +1,453 @@ +using System.Globalization; +using System.Net; +using System.Text.RegularExpressions; +using Cove.Core.DTOs; +using Cove.Plugins; +using Microsoft.Extensions.DependencyInjection; + +namespace Cove.Extensions.CommunityScrapers; + +public sealed class MissAV123ScraperExtension : IScraperProvider +{ + private const string ExtensionId = "cove.community.scrapers.missav123"; + private const string VideoScraperId = "cove.community.scrapers.missav123/video"; + + private const string CurrentHost = "123av.com"; + + private static readonly string[] UrlPatterns = ["missav123.to/*", "*.missav123.to/*", "123av.com/*", "*.123av.com/*"]; + + private static readonly ScraperDescriptor VideoScraper = new( + VideoScraperId, + "MissAV123 Video", + ScraperEntity.Video, + ScraperCapabilities.ByUrl | ScraperCapabilities.ByName | ScraperCapabilities.ByFragment | ScraperCapabilities.ByQueryFragment, + UrlPatterns, + ScraperRiskLevel.NetworkOnly, + ["123av.com", "missav123.to"]); + + private IServiceProvider? _services; + + public string Id => ExtensionId; + public string Name => "MissAV123 Scraper"; + public string Version => "1.0.0"; + public string? Description => "Extracts scene metadata from MissAV123/123AV pages."; + public string? Author => null; + public string? Url => "https://github.com/yourcove/communityscrapers"; + public string? IconUrl => null; + public IReadOnlyList Categories => [ExtensionCategories.Scraper, ExtensionCategories.Metadata, "video"]; + + public void ConfigureServices(IServiceCollection services, ExtensionContext context) + { + } + + public Task InitializeAsync(IServiceProvider services, CancellationToken ct = default) + { + _services = services; + return Task.CompletedTask; + } + + public IReadOnlyList GetScrapers() => [VideoScraper]; + + public async Task ScrapeVideoAsync(ScraperRequest request, CancellationToken ct) + { + if (!string.Equals(request.ScraperId, VideoScraperId, StringComparison.OrdinalIgnoreCase)) + return null; + + var url = ResolveVideoUrl(request.Input); + if (string.IsNullOrWhiteSpace(url) || !IsMissAV123Url(url)) + return null; + + var html = await GetStringAsync(url, ct); + return ParseVideo(html, url); + } + + public async Task> SearchVideosAsync(ScraperRequest request, CancellationToken ct) + { + if (!string.Equals(request.ScraperId, VideoScraperId, StringComparison.OrdinalIgnoreCase) || string.IsNullOrWhiteSpace(request.Input)) + return []; + + var code = ExtractCode(request.Input); + if (!string.IsNullOrWhiteSpace(code)) + { + try + { + var directUrl = BuildVideoUrl(code); + var directHtml = await GetStringAsync(directUrl, ct); + var directResult = ParseVideo(directHtml, directUrl); + if (directResult != null) + return [directResult]; + } + catch (HttpRequestException) + { + } + } + + var url = $"https://{CurrentHost}/en/search?keyword={Uri.EscapeDataString(request.Input.Trim())}"; + var html = await GetStringAsync(url, ct); + return ParseSearch(html).ToList(); + } + + private async Task GetStringAsync(string url, CancellationToken ct) + { + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"); + request.Headers.AcceptLanguage.ParseAdd("en-US"); + request.Headers.AcceptLanguage.ParseAdd("en;q=0.9"); + using var response = await GetHttpClient().SendAsync(request, ct); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStringAsync(ct); + } + + private HttpClient GetHttpClient() + => _services?.GetRequiredService().CreateClient(Id) + ?? throw new InvalidOperationException("The MissAV123 scraper has not been initialized."); + + private static ScrapedVideoDto? ParseVideo(string html, string fallbackUrl) + { + var canonicalUrl = ExtractDTagAttribute(html, "WatchShareBox", "url") ?? ExtractLinkHref(html, "canonical") ?? fallbackUrl; + canonicalUrl = NormalizeKnownUrl(canonicalUrl); + var details = ExtractTextReadMore(html); + var genreTags = ExtractAnchorTextsFromInfoRow(html, "Genres"); + var extraTags = ExtractAnchorTextsFromInfoRow(html, "Tags"); + var seriesTags = ExtractAnchorTextsFromInfoRow(html, "Series"); + var tags = genreTags.Concat(extraTags).Concat(seriesTags).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + var performers = ExtractAnchorTextsFromInfoRow(html, "Cast"); + var studio = ExtractAnchorTextsFromInfoRow(html, "Maker").FirstOrDefault(); + + var title = ExtractElementByExactClass(html, "h1", "watch__title") + ?? ExtractElementByExactClass(html, "h1", "title") + ?? CleanTitle(ExtractHtmlTitle(html)); + if (string.IsNullOrWhiteSpace(title) + && string.IsNullOrWhiteSpace(canonicalUrl) + && performers.Count == 0 + && tags.Count == 0 + && string.IsNullOrWhiteSpace(studio)) + { + return null; + } + + return new ScrapedVideoDto + { + SourceScraperId = VideoScraperId, + Title = title, + Urls = string.IsNullOrWhiteSpace(canonicalUrl) ? [fallbackUrl] : [canonicalUrl], + Code = ExtractCode(canonicalUrl) ?? ExtractCode(fallbackUrl), + Date = ParseDate(ExtractTextFromInfoRow(html, "Release date") ?? ExtractTextFromMetaLabel(html, "Released date:")), + Details = details, + ImageUrl = ExtractImageUrl(html), + StudioName = studio, + PerformerNames = performers, + TagNames = tags, + }; + } + + private static IReadOnlyList ParseSearch(string html) + { + var results = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (Match block in Regex.Matches(html, @"(?is)]*\bclass\s*=\s*(['""])[^'""]*\bcard\b[^'""]*\1)[^>]*>(?.*?)(?=]*\bclass\s*=\s*(['""])[^'""]*\bcard\b||]*\bsrc\s*=\s*(['""])(?.*?)\1"), + }); + } + + return results; + } + + private static string? ResolveVideoUrl(VideoScrapeInput input) + { + var url = input.Url ?? input.Urls.FirstOrDefault(IsMissAV123Url); + if (!string.IsNullOrWhiteSpace(url)) + return url.Trim(); + + var code = FirstNonEmpty(input.Code, ExtractCode(input.Title)); + if (string.IsNullOrWhiteSpace(code)) + { + foreach (var file in input.Files) + { + code = ExtractCode(file.Path); + if (!string.IsNullOrWhiteSpace(code)) + break; + } + } + + return string.IsNullOrWhiteSpace(code) ? null : BuildVideoUrl(code); + } + + private static bool IsMissAV123Url(string? url) + => !string.IsNullOrWhiteSpace(url) + && Uri.TryCreate(url, UriKind.Absolute, out var uri) + && (IsHost(uri, "missav123.to") || IsHost(uri, CurrentHost)); + + private static string BuildVideoUrl(string code) => $"https://{CurrentHost}/en/v/{NormalizeCode(code)}"; + + private static List ExtractAnchorTextsFromInfoRow(string html, string label) + { + var content = ExtractInfoRowContent(html, label); + if (string.IsNullOrWhiteSpace(content)) + return []; + + return Regex.Matches(content, @"(?is)]*>(?.*?)") + .Select(match => CleanHtml(match.Groups["text"].Value)) + .Where(text => !string.IsNullOrWhiteSpace(text)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string? ExtractTextFromInfoRow(string html, string label) + { + var content = ExtractInfoRowContent(html, label); + if (string.IsNullOrWhiteSpace(content)) + return null; + + var dd = ExtractElementText(content, "dd"); + return string.IsNullOrWhiteSpace(dd) ? null : dd; + } + + private static string? ExtractInfoRowContent(string html, string label) + { + foreach (Match match in Regex.Matches(html, @"(?is)]*\bclass\s*=\s*(['""])[^'""]*\bwatch__info-row\b[^'""]*\1)[^>]*>(?.*?)")) + { + var content = match.Groups["content"].Value; + var dt = ExtractElementText(content, "dt"); + if (string.Equals(dt, label, StringComparison.OrdinalIgnoreCase)) + return content; + } + + return null; + } + + private static List ExtractAnchorTextsFromMetaLabel(string html, string label) + { + var content = ExtractMetaLabelContent(html, label); + if (string.IsNullOrWhiteSpace(content)) + return []; + + return Regex.Matches(content, @"(?is)]*>(?.*?)") + .Select(match => CleanHtml(match.Groups["text"].Value)) + .Where(text => !string.IsNullOrWhiteSpace(text)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string? ExtractTextFromMetaLabel(string html, string label) + { + var content = ExtractMetaLabelContent(html, label); + return string.IsNullOrWhiteSpace(content) ? null : ExtractElementText(content, "span"); + } + + private static string? ExtractMetaLabelContent(string html, string label) + { + foreach (Match match in Regex.Matches(html, @"(?is)]*>(?.*?]*>.*?.*?)")) + { + var content = match.Groups["content"].Value; + if (Regex.IsMatch(content, $@"(?is)]*>\s*{Regex.Escape(label)}\s*")) + return content; + } + + return null; + } + + private static string? ExtractTextReadMore(string html) + { + var match = Regex.Match(html, @"(?is)]*\bsrc\s*=\s*(['""])TextReadMore\1)[^>]*>.*?]*\bclass\s*=\s*(['""])content\2)[^>]*>(?.*?)"); + return match.Success ? CleanHtml(match.Groups["text"].Value) : null; + } + + private static string? ExtractImageUrl(string html) + { + var cover = ExtractDTagAttributeById(html, "player", "cover"); + if (!string.IsNullOrWhiteSpace(cover)) + return cover; + + foreach (Match match in Regex.Matches(html, @"(?is)]*(?:background-image|plyr__poster|player))[^>]*\bstyle\s*=\s*(['""])(?