[YouTube] Use poTokens where needed and rework JSON requests

Author: Stypox

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java

@@ -21,8 +21,10 @@
  * These tokens may have a role in triggering the sign in requirement.
  * </p>
  *
- * @implNote This interface is expected to be thread-safe,
- * as it may be accessed by multiple threads.
+ * <p>
+ * <b>Implementations of this interface are expected to be thread-safe, as they may be accessed by
+ * multiple threads.</b>
+ * </p>
  */
 public interface PoTokenProvider {
 
@@ -35,11 +37,77 @@ public interface PoTokenProvider {
      * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
      * </p>
      *
+     * <p>
+     * Note that YouTube desktop website generates two poTokens:
+     * - one for the player requests poTokens, using the videoId as the minter value;
+     * - one for the streaming URLs, using a visitor data for logged-out users.
+     * </p>
+     *
      * @return a {@link PoTokenResult} specific to the WEB InnerTube client
      */
     @Nullable
-    PoTokenResult getWebClientPoToken();
+    PoTokenResult getWebClientPoToken(String videoId);
 
+    /**
+     * Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER
+     * InnerTube client.
+     *
+     * <p>
+     * To be generated and valid, poTokens from this client must be generated using Google's
+     * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
+     * should be added to adaptive/DASH streaming URLs with the {@code pot} parameter and do not
+     * seem to be mandatory for now.
+     * </p>
+     *
+     * <p>
+     * As of writing, like the YouTube desktop website previously did, it generates only one
+     * poToken, sent in player requests and streaming URLs, using a visitor data for logged-out
+     * users.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client
+     */
+    @Nullable
+    PoTokenResult getWebEmbedClientPoToken(String videoId);
+
+    /**
+     * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client.
+     *
+     * <p>
+     * Implementation details are not known, the app uses DroidGuard, a native virtual machine
+     * ran by Google Play Services for which its code is updated pretty frequently.
+     * </p>
+     *
+     * <p>
+     * As of writing, DroidGuard seem to check for the Android app signature and package ID, as
+     * unrooted YouTube patched with reVanced doesn't work without spoofing another InnerTube
+     * client while the rooted version works without any client spoofing.
+     * </p>
+     *
+     * <p>
+     * There should be only poToken needed, for the player requests.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the ANDROID InnerTube client
+     */
+    @Nullable
+    PoTokenResult getAndroidClientPoToken(String videoId);
+
+    /**
+     * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client.
+     *
+     * <p>
+     * Implementation details are not really known, the app seem to use something called
+     * iosGuard which should be something similar to Android's DroidGuard. It may rely on Apple's
+     * attestation APIs.
+     * </p>
+     *
+     * <p>
+     * There should be only poToken needed, for the player requests.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the IOS InnerTube client
+     */
     @Nullable
-    PoTokenResult getAndroidClientPoToken();
+    PoTokenResult getIosClientPoToken(String videoId);
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java

@@ -1,22 +1,40 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.util.Objects;
 
 public final class PoTokenResult {
 
     /**
-     * The visitor data associated with a poToken.
+     * The visitor data associated with a {@code poToken}.
      */
+    @Nonnull
     public final String visitorData;
 
     /**
-     * The poToken, a Protobuf object encoded as a base 64 string.
+     * The {@code poToken} of a player request, a Protobuf object encoded as a base 64 string.
      */
-    public final String poToken;
+    @Nonnull
+    public final String playerRequestPoToken;
 
-    public PoTokenResult(@Nonnull final String visitorData, @Nonnull final String poToken) {
+    /**
+     * The {@code poToken} to be appended to streaming URLs, a Protobuf object encoded as a base
+     * 64 string.
+     *
+     * <p>
+     * It may be required on some clients such as HTML5 ones and may also differ from the player
+     * request {@code poToken}.
+     * </p>
+     */
+    @Nullable
+    public final String streamingDataPoToken;
+
+    public PoTokenResult(@Nonnull final String visitorData,
+                         @Nonnull final String playerRequestPoToken,
+                         @Nullable final String streamingDataPoToken) {
         this.visitorData = Objects.requireNonNull(visitorData);
-        this.poToken = Objects.requireNonNull(poToken);
+        this.playerRequestPoToken = Objects.requireNonNull(playerRequestPoToken);
+        this.streamingDataPoToken = streamingDataPoToken;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -25,9 +25,7 @@
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
@@ -1062,43 +1060,6 @@ public static JsonObject getJsonPostResponse(final String endpoint,
                         + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
     }
 
-    public static JsonObject getJsonAndroidPostResponse(
-            final String endpoint,
-            final byte[] body,
-            @Nonnull final Localization localization,
-            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
-        return getMobilePostResponse(endpoint, body, localization,
-                getAndroidUserAgent(localization), endPartOfUrlRequest);
-    }
-
-    public static JsonObject getJsonIosPostResponse(
-            final String endpoint,
-            final byte[] body,
-            @Nonnull final Localization localization,
-            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
-        return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization),
-                endPartOfUrlRequest);
-    }
-
-    private static JsonObject getMobilePostResponse(
-            final String endpoint,
-            final byte[] body,
-            @Nonnull final Localization localization,
-            @Nonnull final String userAgent,
-            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
-        final var headers = Map.of("User-Agent", List.of(userAgent),
-                "X-Goog-Api-Format-Version", List.of("2"));
-
-        final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?"
-                + DISABLE_PRETTY_PRINT_PARAMETER;
-
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(
-                getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
-                                ? baseEndpointUrl
-                                : baseEndpointUrl + endPartOfUrlRequest,
-                        headers, body, localization)));
-    }
-
     @Nonnull
     public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
             @Nonnull final Localization localization,
@@ -1145,152 +1106,6 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
         // @formatter:on
     }
 
-    @Nonnull
-    public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
-            @Nullable final String visitorData) {
-        // @formatter:off
-        final JsonBuilder<JsonObject> builder = JsonObject.builder()
-                .object("context")
-                    .object("client")
-                        .value("clientName", "ANDROID")
-                        .value("clientVersion", ANDROID_CLIENT_VERSION)
-                        .value("platform", "MOBILE")
-                        .value("osName", "Android")
-                        .value("osVersion", "14")
-                        /*
-                        A valid Android SDK version is required to be sure to get a valid player
-                        response
-                        If this parameter is not provided, the player response is replaced by an
-                        error saying the message "The following content is not available on this
-                        app. Watch this content on the latest version on YouTube" (it was
-                        previously a 5-minute video with this message)
-                        See https://github.com/TeamNewPipe/NewPipe/issues/8713
-                        The Android SDK version corresponding to the Android version used in
-                        requests is sent
-                        */
-                        .value("androidSdkVersion", 34)
-                        .value("hl", localization.getLocalizationCode())
-                        .value("gl", contentCountry.getCountryCode())
-                        .value("utcOffsetMinutes", 0);
-
-        if (visitorData != null) {
-            builder.value("visitorData", visitorData);
-        }
-
-        builder.end()
-                    .object("request")
-                        .array("internalExperimentFlags")
-                        .end()
-                        .value("useSsl", true)
-                    .end()
-                    .object("user")
-                        // TODO: provide a way to enable restricted mode with:
-                        //  .value("enableSafetyMode", boolean)
-                        .value("lockedSafetyMode", false)
-                    .end()
-                .end();
-        // @formatter:on
-        return builder;
-    }
-
-    @Nonnull
-    public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry) {
-        // @formatter:off
-        return JsonObject.builder()
-                .object("context")
-                    .object("client")
-                        .value("clientName", "IOS")
-                        .value("clientVersion", IOS_CLIENT_VERSION)
-                        .value("deviceMake",  "Apple")
-                        // Device model is required to get 60fps streams
-                        .value("deviceModel", IOS_DEVICE_MODEL)
-                        .value("platform", "MOBILE")
-                        .value("osName", "iOS")
-                        .value("osVersion", IOS_OS_VERSION)
-                        .value("visitorData", randomVisitorData(contentCountry))
-                        .value("hl", localization.getLocalizationCode())
-                        .value("gl", contentCountry.getCountryCode())
-                        .value("utcOffsetMinutes", 0)
-                    .end()
-                    .object("request")
-                        .array("internalExperimentFlags")
-                        .end()
-                        .value("useSsl", true)
-                    .end()
-                    .object("user")
-                        // TODO: provide a way to enable restricted mode with:
-                        //  .value("enableSafetyMode", boolean)
-                        .value("lockedSafetyMode", false)
-                    .end()
-                .end();
-        // @formatter:on
-    }
-
-    @Nonnull
-    public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
-            @Nonnull final String videoId) {
-        // @formatter:off
-        return JsonObject.builder()
-                .object("context")
-                    .object("client")
-                        .value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER")
-                        .value("clientVersion", TVHTML5_CLIENT_VERSION)
-                        .value("clientScreen", "EMBED")
-                        .value("platform", "TV")
-                        .value("hl", localization.getLocalizationCode())
-                        .value("gl", contentCountry.getCountryCode())
-                        .value("utcOffsetMinutes", 0)
-                    .end()
-                    .object("thirdParty")
-                        .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
-                    .end()
-                    .object("request")
-                        .array("internalExperimentFlags")
-                        .end()
-                        .value("useSsl", true)
-                    .end()
-                    .object("user")
-                        // TODO: provide a way to enable restricted mode with:
-                        //  .value("enableSafetyMode", boolean)
-                        .value("lockedSafetyMode", false)
-                    .end()
-                .end();
-        // @formatter:on
-    }
-
-    @Nonnull
-    public static byte[] createTvHtml5EmbedPlayerBody(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
-            @Nonnull final String videoId,
-            @Nonnull final Integer sts,
-            @Nonnull final String contentPlaybackNonce) {
-        // @formatter:off
-        return JsonWriter.string(
-                prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
-                    .object("playbackContext")
-                        .object("contentPlaybackContext")
-                            // Signature timestamp from the JavaScript base player is needed to get
-                            // working obfuscated URLs
-                            .value("signatureTimestamp", sts)
-                            .value("referer", "https://www.youtube.com/watch?v=" + videoId)
-                        .end()
-                    .end()
-                    .value(CPN, contentPlaybackNonce)
-                    .value(VIDEO_ID, videoId)
-                    .value(CONTENT_CHECK_OK, true)
-                    .value(RACY_CHECK_OK, true)
-                    .done())
-                    .getBytes(StandardCharsets.UTF_8);
-        // @formatter:on
-    }
-
     /**
      * Get the user-agent string used as the user-agent for InnerTube requests with the Android
      * client.
@@ -1371,7 +1186,7 @@ public static Map<String, List<String>> getClientInfoHeaders()
      *
      * @param url The URL to be set as the origin and referrer.
      */
-    private static Map<String, List<String>> getOriginReferrerHeaders(@Nonnull final String url) {
+    static Map<String, List<String>> getOriginReferrerHeaders(@Nonnull final String url) {
         final var urlList = List.of(url);
         return Map.of("Origin", urlList, "Referer", urlList);
     }
@@ -1383,8 +1198,8 @@ private static Map<String, List<String>> getOriginReferrerHeaders(@Nonnull final
      * @param name The X-YouTube-Client-Name value.
      * @param version X-YouTube-Client-Version value.
      */
-    private static Map<String, List<String>> getClientHeaders(@Nonnull final String name,
-                                                              @Nonnull final String version) {
+    static Map<String, List<String>> getClientHeaders(@Nonnull final String name,
+                                                      @Nonnull final String version) {
         return Map.of("X-YouTube-Client-Name", List.of(name),
                 "X-YouTube-Client-Version", List.of(version));
     }