feat(youtube): SABR extractor support

extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java

@@ -45,6 +45,7 @@ public final class NewPipe {
     private static Downloader downloader;
     private static Localization preferredLocalization;
     private static ContentCountry preferredContentCountry;
+    private static boolean forceSabr;
 
     private NewPipe() {
 
@@ -161,6 +162,14 @@ public static void setPreferredContentCountry(final ContentCountry preferredCont
         NewPipe.preferredContentCountry = preferredContentCountry;
     }
 
+    public static boolean isForceSabr() {
+        return forceSabr;
+    }
+
+    public static void setForceSabr(final boolean forceSabr) {
+        NewPipe.forceSabr = forceSabr;
+    }
+
     public static void trustEveryone() {
         try {
             HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier(){

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java

@@ -7,6 +7,7 @@
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -165,6 +166,24 @@ public Response post(final String url,
                 .build());
     }
 
+    /**
+     * Like {@link #post(String, Map, byte[], Localization)} but returns the body as a stream instead
+     * of a buffered {@code byte[]}, so a large response (e.g. a SABR media batch, 50-150MB at 4K) is
+     * not held whole in memory. The default falls back to the buffered {@code post()} and wraps its
+     * body; an implementation that can stream (e.g. over okhttp) should override this. Caller closes.
+     */
+    public StreamingResponse postStreaming(final String url,
+                                           @Nullable final Map<String, List<String>> headers,
+                                           @Nullable final byte[] dataToSend,
+                                           @Nullable final Localization localization)
+            throws IOException, ReCaptchaException {
+        final Response response = post(url, headers, dataToSend, localization);
+        final byte[] raw = response.rawResponseBody() == null
+                ? new byte[0] : response.rawResponseBody();
+        return new StreamingResponse(response.responseCode(), response.responseHeaders(),
+                new ByteArrayInputStream(raw));
+    }
+
     public CancellableCall postAsync(final String url,
                           @Nullable final Map<String, List<String>> headers,
                           @Nullable final byte[] dataToSend,

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/StreamingResponse.java

@@ -0,0 +1,57 @@
+package org.schabi.newpipe.extractor.downloader;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A streaming HTTP response: the body is exposed as an {@link InputStream} instead of a buffered
+ * {@code byte[]}, so a large response (e.g. a SABR media batch, which can be 50-150MB at 4K) is not
+ * held whole in memory. The caller MUST {@link #close()} it to release the underlying connection.
+ */
+public class StreamingResponse implements Closeable {
+    private final int responseCode;
+    @Nonnull
+    private final Map<String, List<String>> responseHeaders;
+    @Nonnull
+    private final InputStream body;
+
+    public StreamingResponse(final int responseCode,
+                             @Nullable final Map<String, List<String>> responseHeaders,
+                             @Nonnull final InputStream body) {
+        this.responseCode = responseCode;
+        this.responseHeaders = responseHeaders == null ? Collections.emptyMap() : responseHeaders;
+        this.body = body;
+    }
+
+    public int responseCode() {
+        return responseCode;
+    }
+
+    @Nonnull
+    public InputStream body() {
+        return body;
+    }
+
+    /** First value for a header name (case-insensitive), or {@code null}. */
+    @Nullable
+    public String getHeader(final String name) {
+        for (final Map.Entry<String, List<String>> entry : responseHeaders.entrySet()) {
+            final String key = entry.getKey();
+            if (key != null && key.equalsIgnoreCase(name) && !entry.getValue().isEmpty()) {
+                return entry.getValue().get(0);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+        body.close();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1512,7 +1512,8 @@ public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
     @Nonnull
     public static JsonBuilder<JsonObject> prepareSafariJsonBuilder(
             @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry) {
+            @Nonnull final ContentCountry contentCountry)
+            throws IOException, ExtractionException {
         return JsonObject.builder()
                 .object("context")
                     .object("client")
@@ -1522,7 +1523,7 @@ public static JsonBuilder<JsonObject> prepareSafariJsonBuilder(
                         .value("gl", contentCountry.getCountryCode())
                         .value("userAgent", SAFARI_USER_AGENT)
                         .value("clientName", "WEB")
-                        .value("clientVersion", SAFARI_CLIENT_VERSION)
+                        .value("clientVersion", getClientVersion())
                     .end()
                 .end();
     }
@@ -1533,7 +1534,8 @@ public static byte[] createSafariPlayerBody(
             @Nonnull final ContentCountry contentCountry,
             @Nonnull final String videoId,
             @Nonnull final Integer sts,
-            @Nonnull final String contentPlaybackNonce) {
+            @Nonnull final String contentPlaybackNonce)
+            throws IOException, ExtractionException {
         return JsonWriter.string(
                         prepareSafariJsonBuilder(localization, contentCountry)
                                 .object("playbackContext")
@@ -1559,7 +1561,7 @@ public static CancellableCall getSafariPostResponseAsync(final String endpoint,
         headers.put("Content-Type", singletonList("application/json"));
         headers.put("User-Agent", singletonList(SAFARI_USER_AGENT));
         headers.put("X-YouTube-Client-Name", singletonList("1"));
-        headers.put("X-Youtube-Client-Version", singletonList(SAFARI_CLIENT_VERSION));
+        headers.put("X-Youtube-Client-Version", singletonList(getClientVersion()));
 
         addLoggedInHeaders(headers);
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -785,6 +785,11 @@ private static String getManifestUrl(@Nonnull final String manifestType,
      * This method collects all audio, video, and video-only streams,
      * then performs batch deobfuscation in one request.
      */
+    // TEMP (deep SABR testing): route every video through the real SABR pipeline (via
+    // serverAbrStreamingUrl). Set false for production. With it false, SABR only fills the
+    // SABR-only/no-HLS gap that upstream otherwise throws ContentNotSupportedException on.
+    // Controlled by the Force SABR advanced preference on the client side.
+
     private void ensureStreamsAreCached() throws ExtractionException {
         if (streamsCached) {
             return;
@@ -793,6 +798,16 @@ private void ensureStreamsAreCached() throws ExtractionException {
         assertPageFetched();
         final String videoId = getId();
 
+        // SABR-only responses carry no per-format URLs: build session-based SABR streams instead
+        // of the classic URL/DASH/HLS path. The client drives a YoutubeSabrSession from these.
+        if (streamType != StreamType.LIVE_STREAM
+                && (NewPipe.isForceSabr()
+                    || (isSabrOnlyResponse() && getHlsManifestUrlFromStreamingData().isEmpty()))) {
+            buildSabrStreams();
+            streamsCached = true;
+            return;
+        }
+
         // Collect all ItagInfo objects from all stream types
         final List<ItagInfo> allItagInfos = new ArrayList<>();
         final int audioStartIndex = 0;
@@ -874,6 +889,146 @@ private void ensureStreamsAreCached() throws ExtractionException {
         }
     }
 
+    /**
+     * Build session-based SABR streams from a SABR-only response.
+     *
+     * <p>SABR adaptiveFormats carry no per-format URL: each stream is marked with
+     * {@link DeliveryMethod#SABR}, its {@code content} is the serverAbrStreamingUrl (for reference),
+     * and {@code isUrl} is false. The client drives a {@code YoutubeSabrSession} from the videoId and
+     * the selected itag to fetch media.</p>
+     */
+    private void buildSabrStreams() {
+        cachedAudioStreams = new ArrayList<>();
+        cachedVideoStreams = new ArrayList<>();
+        cachedVideoOnlyStreams = new ArrayList<>();
+
+        final JsonObject streamingData = getSabrStreamingData();
+        if (streamingData == null) {
+            return;
+        }
+        final String serverAbrStreamingUrl =
+                streamingData.getString("serverAbrStreamingUrl", EMPTY_STRING);
+        final JsonArray adaptiveFormats = streamingData.getArray(ADAPTIVE_FORMATS);
+        if (adaptiveFormats == null) {
+            return;
+        }
+
+        for (int i = 0; i < adaptiveFormats.size(); i++) {
+            final JsonObject formatData = adaptiveFormats.getObject(i);
+            try {
+                final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
+                fillSabrItagItem(itagItem, formatData);
+                final String id = String.valueOf(itagItem.id);
+
+                if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
+                    final AudioStream.Builder builder = new AudioStream.Builder()
+                            .setContent(serverAbrStreamingUrl, false)
+                            .setMediaFormat(itagItem.getMediaFormat())
+                            .setAverageBitrate(itagItem.getAverageBitrate())
+                            .setItagItem(itagItem)
+                            .setDeliveryMethod(DeliveryMethod.SABR);
+                    // Multi-track audio: the same itag is served once per language. Carry the track
+                    // info so the player can show a language selector, and key the id on (itag,
+                    // track) so the languages aren't collapsed into one by the dedup below.
+                    String streamId = id;
+                    if (formatData.has("audioTrack")) {
+                        final JsonObject audioTrack = formatData.getObject("audioTrack");
+                        if (audioTrack.has("id")) {
+                            final String trackId = audioTrack.getString("id");
+                            final String displayName = audioTrack.getString("displayName");
+                            final String langPart = trackId.split("\\.")[0];
+                            final boolean isOriginal = displayName != null
+                                    && (displayName.contains("original")
+                                        || displayName.contains("yokuqala"));
+                            builder.setAudioTrackId(trackId)
+                                    .setAudioTrackName(displayName != null ? displayName
+                                            : (isOriginal ? langPart + " (original)" : langPart))
+                                    .setAudioLocale(langPart.split("-")[0]);
+                            streamId = id + "-" + trackId;
+                        }
+                    }
+                    final String audioStreamId = streamId;
+                    final AudioStream stream = builder.setId(audioStreamId).build();
+                    // Dedup by id (itag, or itag+track when multi-track), not Stream.equalStats: all
+                    // SABR formats share the same MediaFormat/delivery, so equalStats would collapse
+                    // every bitrate/codec to one.
+                    if (cachedAudioStreams.stream().noneMatch(s -> audioStreamId.equals(s.getId()))) {
+                        cachedAudioStreams.add(stream);
+                    }
+                } else if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
+                    final String resolution = itagItem.getResolutionString();
+                    final VideoStream stream = new VideoStream.Builder()
+                            .setId(id)
+                            .setContent(serverAbrStreamingUrl, false)
+                            .setMediaFormat(itagItem.getMediaFormat())
+                            .setIsVideoOnly(true)
+                            .setItagItem(itagItem)
+                            .setResolution(resolution != null ? resolution : EMPTY_STRING)
+                            .setDeliveryMethod(DeliveryMethod.SABR)
+                            .build();
+                    if (cachedVideoOnlyStreams.stream().noneMatch(s -> id.equals(s.getId()))) {
+                        cachedVideoOnlyStreams.add(stream);
+                    }
+                }
+            } catch (final Exception e) {
+                // Skip unknown itags or malformed formats; do not fail the whole extraction.
+            }
+        }
+
+        Collections.sort(cachedAudioStreams,
+                Comparator.comparingInt(AudioStream::getBitrate).reversed());
+    }
+
+    @Nullable
+    private JsonObject getSabrStreamingData() {
+        for (final JsonObject streamingData : Arrays.asList(
+                webStreamingData, safariStreamingData, androidStreamingData,
+                tvHtml5SimplyEmbedStreamingData)) {
+            if (streamingData != null
+                    && streamingData.getArray(ADAPTIVE_FORMATS) != null
+                    && !streamingData.getArray(ADAPTIVE_FORMATS).isEmpty()) {
+                return streamingData;
+            }
+        }
+        return null;
+    }
+
+    private static void fillSabrItagItem(@Nonnull final ItagItem itagItem,
+                                         @Nonnull final JsonObject formatData) {
+        final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
+        final String codec = mimeType.contains("codecs") ? mimeType.split("\"")[1] : EMPTY_STRING;
+
+        itagItem.setBitrate(formatData.getInt("bitrate"));
+        itagItem.setWidth(formatData.getInt("width"));
+        itagItem.setHeight(formatData.getInt("height"));
+        if (formatData.has("initRange")) {
+            final JsonObject initRange = formatData.getObject("initRange");
+            itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
+            itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
+        }
+        if (formatData.has("indexRange")) {
+            final JsonObject indexRange = formatData.getObject("indexRange");
+            itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
+            itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
+        }
+        itagItem.setQuality(formatData.getString("quality"));
+        itagItem.setCodec(codec);
+        final int fps = formatData.getInt("fps", -1);
+        if (fps != -1) {
+            itagItem.setFps(fps);
+        }
+        if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
+            if (formatData.has("audioSampleRate")) {
+                itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate")));
+            }
+            itagItem.setAudioChannels(formatData.getInt("audioChannels", 2));
+        }
+        itagItem.setContentLength(Long.parseLong(formatData.getString("contentLength",
+                String.valueOf(CONTENT_LENGTH_UNKNOWN))));
+        itagItem.setApproxDurationMs(Long.parseLong(formatData.getString("approxDurationMs",
+                String.valueOf(APPROX_DURATION_MS_UNKNOWN))));
+    }
+
     private void tryExtractHlsStreams(final String videoId) throws ExtractionException {
         final String hlsManifestUrl = getHlsManifestUrlFromStreamingData();
         if (hlsManifestUrl.isEmpty()) {
@@ -1428,12 +1583,8 @@ public void onSuccess(Response response) throws ExtractionException {
         checkPlayabilityStatus(playerResponse.getObject("playabilityStatus"), videoId);
         setStreamType();
 
-        if (streamType != StreamType.LIVE_STREAM && isSabrOnlyResponse()
-                && getHlsManifestUrlFromStreamingData().isEmpty()) {
-            throw new ContentNotSupportedException(
-                    "YouTube returned SABR-only streaming data without usable stream URLs. "
-                    + "Try logging in to get HLS fallback streams.");
-        }
+        // SABR-only responses are no longer a hard failure: ensureStreamsAreCached() builds
+        // session-based SABR streams (DeliveryMethod.SABR) from the adaptiveFormats instead.
     }
 
     private boolean isSabrOnlyResponse() {