From a0e8544ecdff528a3fefce82ba2153d96f515877 Mon Sep 17 00:00:00 2001 From: make-it-fun Date: Wed, 22 Jul 2026 09:25:32 -0700 Subject: [PATCH] scraper: replace blocklisted default User-Agent (fixes RealJam VR / PornCorn VR 403) realjamvr.com and porncornvr.com reject XBVR's default User-Agent "Chrome/73.0.3683.103" with HTTP 403. Both scrapers therefore fail on every scheduled run with Error visiting https://realjamvr.com/scenes Forbidden Error visiting https://porncornvr.com/scenes Forbidden and ingest nothing. The failure is easy to miss: the scraper logs the error, reports "Finished", and the sites simply stop producing new scenes. The block is keyed to the EXACT literal version string, not to "old browser" heuristics. Verified live against realjamvr.com: Chrome/73.0.3683.103 -> 403 (the shipped default) Chrome/73.0.3683.104 -> 200 (one digit different) Chrome/73.0.0.0 -> 200 Chrome/120.0.0.0 -> 200 Because the default is compiled in, every XBVR install sends that string verbatim, which makes it a reliable fingerprint for the client - and that is what makes it worth blocklisting. Nothing short of changing the string works; this is not something a user can configure around. Bump it to Chrome/120.0.0.0, in line with the Chrome/108 UA already used by the imageproxy in pkg/server/server.go. Verified after the change: both scene listings return 200, and full scrapes of both sites complete normally and save scenes again. --- pkg/scrape/scrape.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/scrape/scrape.go b/pkg/scrape/scrape.go index c9e88def8..44004da9a 100644 --- a/pkg/scrape/scrape.go +++ b/pkg/scrape/scrape.go @@ -20,7 +20,21 @@ import ( var log = &common.Log -var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" +// UserAgent is sent by every scraper and by the HTML/JSON trailer-source scrapes. +// +// It must not be left on a string that studios have blocklisted. The previous value, +// "Chrome/73.0.3683.103", is rejected outright (HTTP 403) by realjamvr.com and porncornvr.com, +// which silently breaks both scrapers: every scheduled run fails with +// "Error visiting https:///scenes Forbidden" and no scenes are ingested. +// +// The block is on the EXACT literal version string, not on "old browser" heuristics. Verified +// live: "Chrome/73.0.3683.104" (one digit different) and "Chrome/73.0.0.0" both return 200 from +// the same host that 403s "Chrome/73.0.3683.103". Because that exact string is XBVR's shipped +// default, every install sends it verbatim, making it a reliable fingerprint for the client - +// which is precisely what makes it worth blocklisting. Keep this a realistic, current browser UA; +// if a studio starts 403ing again, check for a blocklisted-fingerprint 403 before assuming the +// site itself changed. +var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" func createCollector(domains ...string) *colly.Collector { c := colly.NewCollector(