Fix open provider and following issues

Author: raccommode

Dockerfile

@@ -29,12 +29,14 @@ COPY app ./app
 COPY static ./static
 COPY README.md ./
 COPY docker/entrypoint.sh /usr/local/bin/pstreamrec-entrypoint
-RUN chmod +x /usr/local/bin/pstreamrec-entrypoint
+RUN chmod +x /usr/local/bin/pstreamrec-entrypoint && \
+    test -x /usr/local/bin/pstreamrec-entrypoint && \
+    PSTREAMREC_ENTRYPOINT_TESTING=1 /usr/local/bin/pstreamrec-entrypoint
 
 # Create data volume for recordings
 VOLUME ["/data"]
 
 EXPOSE 8080
 
-ENTRYPOINT ["pstreamrec-entrypoint"]
+ENTRYPOINT ["/usr/local/bin/pstreamrec-entrypoint"]
 CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers"]

app/main.py

@@ -2,7 +2,7 @@
 import mimetypes
 from pathlib import Path
 from typing import Any, Optional, Tuple
-from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparse
+from urllib.parse import parse_qsl, quote, unquote, urlencode, urljoin, urlparse, urlunparse
 import os
 import asyncio
 import aiohttp
@@ -805,6 +805,11 @@ def _canonical_stream_url(source_type: str, channel_username: str, channel_url:
         return f"https://www.cam4.com/{channel}"
     if normalized_source == "chaturbate":
         return f"https://chaturbate.com/{channel}/"
+    try:
+        if provider_registry.has(normalized_source):
+            return provider_registry.get(normalized_source).canonical_url(channel_username)
+    except Exception:
+        pass
     return raw_url
 
 
@@ -5428,12 +5433,18 @@ async def update_model_volume(username: str, body: ModelVolumeBody):
 @app.post("/api/models")
 async def add_model(model: dict):
     """Ajoute un modèle dans SQLite"""
-    username = model.get('username')
+    raw_username = str(model.get('username') or "").strip()
+    source_from_url = _source_type_from_url(raw_username)
+    username = (
+        _normalize_live_channel_username(raw_username, raw_username)
+        if raw_username.startswith(("http://", "https://"))
+        else raw_username
+    )
     if not username:
         raise HTTPException(status_code=400, detail="Username requis")
 
     requested_source = _normalize_source_type(
-        model.get("sourceType") or model.get("source_type")
+        model.get("sourceType") or model.get("source_type") or source_from_url
     )
     source_type = requested_source or await _infer_source_type(username)
     if source_type not in _available_source_types():

app/providers/browser.py

@@ -1955,7 +1955,6 @@ async def _resolve_stripchat_public_hls(
         target: str,
         max_height: Optional[int] = None,
     ) -> Optional[ResolvedStream]:
-        del max_height
         username = self._stripchat_username_from_target(target)
         if not username:
             return None
@@ -1983,10 +1982,11 @@ async def _resolve_stripchat_public_hls(
                 f"https://edge-hls.{host}/hls/{model_id}/master/{model_id}_auto.m3u8"
                 f"{self._stripchat_master_playlist_query()}"
             )
-            if await self._stripchat_probe_hls_playlist(playlist_url, headers):
+            stream_url = await self._stripchat_validated_hls_url(playlist_url, headers, max_height)
+            if stream_url:
                 item = self._stripchat_model_item(model) or {}
                 return ResolvedStream(
-                    url=playlist_url,
+                    url=stream_url,
                     headers=headers,
                     source_type=self.source_type,
                     is_live=True,
@@ -1999,6 +1999,58 @@ async def _resolve_stripchat_public_hls(
 
         raise ProviderOfflineError(f"Aucun HLS Stripchat public valide pour {username}")
 
+    async def _stripchat_validated_hls_url(
+        self,
+        playlist_url: str,
+        headers: dict[str, str],
+        max_height: Optional[int],
+    ) -> Optional[str]:
+        if not max_height or max_height <= 0:
+            return playlist_url if await self._stripchat_probe_hls_playlist(playlist_url, headers) else None
+
+        playlist_text = await self._stripchat_fetch_hls_playlist(playlist_url, headers)
+        if not playlist_text:
+            return None
+        return self._stripchat_variant_url_for_height(playlist_url, playlist_text, max_height) or playlist_url
+
+    @staticmethod
+    def _stripchat_variant_url_for_height(
+        playlist_url: str,
+        playlist_text: str,
+        max_height: Optional[int],
+    ) -> Optional[str]:
+        if not max_height or max_height <= 0:
+            return None
+
+        variants: list[dict[str, object]] = []
+        pending_height = 0
+        for raw_line in (playlist_text or "").splitlines():
+            line = raw_line.strip()
+            if not line:
+                continue
+            if line.startswith("#EXT-X-STREAM-INF"):
+                match = re.search(r"RESOLUTION=\d+x(\d+)", line, re.IGNORECASE)
+                pending_height = int(match.group(1)) if match else 0
+                continue
+            if line.startswith("#"):
+                continue
+            if pending_height:
+                variants.append({
+                    "height": pending_height,
+                    "url": urljoin(playlist_url, line),
+                })
+            pending_height = 0
+
+        if not variants:
+            return None
+
+        eligible = [item for item in variants if int(item["height"]) <= max_height]
+        if eligible:
+            selected = max(eligible, key=lambda item: int(item["height"]))
+        else:
+            selected = min(variants, key=lambda item: int(item["height"]))
+        return str(selected["url"])
+
     def _stripchat_master_playlist_query(self) -> str:
         params = {
             "minHeight": os.getenv("PSTREAMREC_STRIPCHAT_MIN_HEIGHT", "240"),
@@ -2089,7 +2141,7 @@ def walk(value: object) -> None:
             hosts.append(host)
         return hosts
 
-    async def _stripchat_probe_hls_playlist(self, playlist_url: str, headers: dict[str, str]) -> bool:
+    async def _stripchat_fetch_hls_playlist(self, playlist_url: str, headers: dict[str, str]) -> Optional[str]:
         try:
             timeout = aiohttp.ClientTimeout(total=int(os.getenv("PSTREAMREC_STRIPCHAT_HLS_PROBE_TIMEOUT", "12") or "12"))
             async with aiohttp_client_session(timeout=timeout) as session:
@@ -2102,14 +2154,17 @@ async def _stripchat_probe_hls_playlist(self, playlist_url: str, headers: dict[s
                     if resp.status in (401, 403):
                         raise ProviderAuthError("Flux Stripchat refuse ou session requise")
                     if resp.status >= 400:
-                        return False
-                    head = await resp.content.read(512)
+                        return None
+                    text = await resp.text(errors="ignore")
         except ProviderError:
             raise
         except Exception as exc:
             logger.debug("Stripchat HLS probe failed", url=playlist_url, error=str(exc))
-            return False
-        return head.lstrip().startswith(b"#EXTM3U")
+            return None
+        return text if text.lstrip().startswith("#EXTM3U") else None
+
+    async def _stripchat_probe_hls_playlist(self, playlist_url: str, headers: dict[str, str]) -> bool:
+        return bool(await self._stripchat_fetch_hls_playlist(playlist_url, headers))
 
     async def _stripchat_user_by_username(self, username: str) -> dict[str, object]:
         payload = await self._stripchat_api_json(

app/services/chaturbate_api.py

@@ -4,6 +4,7 @@
 """
 
 import asyncio
+import json
 import re
 import time
 from html import unescape
@@ -642,8 +643,126 @@ def _parse_followed_html(cls, html: str) -> List[Dict[str, Any]]:
                 "gender": "",
                 "num_followers": 0,
             })
+        models.extend(cls._parse_followed_embedded_json(html))
+        return cls._dedupe_followed_items(models)
+
+    @classmethod
+    def _parse_followed_embedded_json(cls, html: str) -> List[Dict[str, Any]]:
+        models: List[Dict[str, Any]] = []
+        scripts = re.findall(r"<script\b[^>]*>(.*?)</script>", html or "", re.IGNORECASE | re.DOTALL)
+        for script in scripts:
+            text = unescape(script or "").strip()
+            if not text or len(text) > 2_000_000:
+                continue
+            json_values = cls._extract_json_values(text)
+            for value in json_values:
+                cls._collect_followed_json_models(value, models)
         return models
 
+    @classmethod
+    def _extract_json_values(cls, text: str) -> List[Any]:
+        values: List[Any] = []
+        decoder = json.JSONDecoder()
+        candidates = []
+        stripped = text.strip()
+        if stripped.startswith(("{", "[")):
+            candidates.append(stripped)
+        for match in re.finditer(r"=\s*({.*?});", text, re.DOTALL):
+            candidates.append(match.group(1))
+
+        for candidate in candidates:
+            try:
+                value, _ = decoder.raw_decode(candidate)
+            except Exception:
+                continue
+            values.append(value)
+        return values
+
+    @classmethod
+    def _collect_followed_json_models(cls, value: Any, models: List[Dict[str, Any]]) -> None:
+        if isinstance(value, list):
+            for item in value:
+                cls._collect_followed_json_models(item, models)
+            return
+        if not isinstance(value, dict):
+            return
+
+        item = cls._followed_json_item(value)
+        if item:
+            models.append(item)
+
+        for child in value.values():
+            if isinstance(child, (dict, list)):
+                cls._collect_followed_json_models(child, models)
+
+    @classmethod
+    def _followed_json_item(cls, item: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        username = str(
+            item.get("username")
+            or item.get("room")
+            or item.get("room_slug")
+            or item.get("slug")
+            or ""
+        ).strip().strip("/")
+        if not username or not re.match(r"^[A-Za-z0-9_]+$", username):
+            return None
+
+        has_followed_shape = any(
+            key in item
+            for key in (
+                "is_online",
+                "isOnline",
+                "current_show",
+                "room_status",
+                "num_users",
+                "viewers",
+                "thumbnail",
+                "thumbnail_url",
+                "img",
+            )
+        )
+        if not has_followed_shape:
+            return None
+
+        room_status = str(
+            item.get("room_status")
+            or item.get("roomStatus")
+            or item.get("current_show")
+            or item.get("status")
+            or ""
+        ).strip().lower()
+        is_private = room_status in {"private", "group", "ticket", "hidden"}
+        is_online = bool(item.get("is_online", item.get("isOnline", room_status == "public"))) and not is_private
+        thumbnail = cls._normalize_thumbnail(
+            item.get("thumbnail_url") or item.get("thumbnail") or item.get("img"),
+            username,
+        )
+        return {
+            "username": username,
+            "display_name": str(item.get("display_name") or item.get("displayName") or username),
+            "is_online": is_online,
+            "viewers": cls._as_int(item.get("viewers", item.get("num_users", 0))) if is_online else 0,
+            "thumbnail_url": thumbnail,
+            "room_status": room_status or ("public" if is_online else "offline"),
+            "tags": cls._normalize_tags(item.get("tags", [])),
+            "subject": str(item.get("subject") or item.get("room_subject") or ""),
+            "gender": str(item.get("gender") or ""),
+            "num_followers": cls._as_int(item.get("num_followers"), 0),
+        }
+
+    @staticmethod
+    def _dedupe_followed_items(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        deduped: List[Dict[str, Any]] = []
+        seen = set()
+        for item in items:
+            username = str(item.get("username") or "").strip()
+            key = username.lower()
+            if not username or key in seen:
+                continue
+            seen.add(key)
+            deduped.append(item)
+        return deduped
+
     @staticmethod
     def _parse_room_item(item: Dict[str, Any], is_online: bool = True) -> Dict[str, Any]:
         """Parse a room item from the roomlist API into our model format."""