Adress requested changes

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -23,7 +23,6 @@
 import java.net.URL;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
@@ -65,6 +64,11 @@ private YoutubeParsingHelper() {
     public static final String CPN = "cpn";
     public static final String VIDEO_ID = "videoId";
 
+    /**
+     * Seed that will be used for video tests, in order to mock video requests.
+     */
+    private static final long SEED_FOR_VIDEOS_TESTS = 3000;
+
     private static final String HARDCODED_CLIENT_VERSION = "2.20220114.01.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
 
@@ -82,13 +86,17 @@ private YoutubeParsingHelper() {
     private static boolean keyAndVersionExtracted = false;
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
     private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();
+
     private static final String[] INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES =
             {"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
                     "innertube_context_client_version\":\"([0-9\\.]+?)\"",
                     "client.version=([0-9\\.]+)"};
     private static final String[] INNERTUBE_API_KEY_REGEXES =
             {"INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"",
                     "innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\""};
+    private static final String[] INITIAL_DATA_REGEXES =
+            {"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});",
+                    "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});"};
     private static final String INNERTUBE_CLIENT_NAME_REGEX =
             "INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),";
 
@@ -98,13 +106,24 @@ private YoutubeParsingHelper() {
     private static Random numberGenerator = new Random();
 
     /**
-     * <code>PENDING+</code> means that the user did not yet submit their choices.
+     * {@code PENDING+} means that the user did not yet submit their choices.
+     *
+     * <p>
      * Therefore, YouTube & Google should not track the user, because they did not give consent.
+     * </p>
+     *
+     * <p>
      * The three digits at the end can be random, but are required.
+     * </p>
      */
     private static final String CONSENT_COOKIE_VALUE = "PENDING+";
+
     /**
-     * Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
+     * YouTube {@code CONSENT} cookie.
+     *
+     * <p>
+     * Should prevent redirect to {@code consent.youtube.com}.
+     * </p>
      */
     private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
 
@@ -312,17 +331,10 @@ public static String extractVideoIdFromMixId(@Nonnull final String playlistId)
         }
     }
 
-    public static JsonObject getInitialData(final String html) throws ParsingException {
+    private static JsonObject getInitialData(final String html) throws ParsingException {
         try {
-            try {
-                final String initialData = Parser.matchGroup1(
-                        "window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
-                return JsonParser.object().from(initialData);
-            } catch (final Parser.RegexException e) {
-                final String initialData = Parser.matchGroup1(
-                        "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
-                return JsonParser.object().from(initialData);
-            }
+            return JsonParser.object().from(getStringResultFromRegexArray(html,
+                    INITIAL_DATA_REGEXES, 1));
         } catch (final JsonParserException | Parser.RegexException e) {
             throw new ParsingException("Could not get ytInitialData", e);
         }
@@ -445,7 +457,7 @@ private static void extractClientVersionAndKeyFromHtmlSearchResultsPage()
             key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
         } catch (final Parser.RegexException e) {
             throw new ParsingException(
-                    "Could not extract YouTube WEB InnerTube client version and API key from HTML search results page");
+                    "Could not extract YouTube WEB InnerTube client version and API key from HTML search results page", e);
         }
         keyAndVersionExtracted = true;
     }
@@ -609,8 +621,7 @@ public static String[] getYoutubeMusicKey()
             final String response = getDownloader().get(url, headers).responseBody();
                 musicClientVersion = getStringResultFromRegexArray(response,
                         INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-                musicKey = getStringResultFromRegexArray(response,
-                        INNERTUBE_API_KEY_REGEXES, 1);
+                musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
                 musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
         } catch (final Exception e) {
             final String url = "https://music.youtube.com/";
@@ -696,10 +707,11 @@ public static String getUrlFromNavigationEndpoint(@Nonnull final JsonObject navi
     }
 
     /**
-     * Get the text from a JSON object that has either a simpleText or a runs array.
+     * Get the text from a JSON object that has either a {@code simpleText} or a {@code runs}
+     * array.
      *
      * @param textObject JSON object to get the text from
-     * @param html       whether to return HTML, by parsing the navigationEndpoint
+     * @param html       whether to return HTML, by parsing the {@code navigationEndpoint}
      * @return text in the JSON object or {@code null}
      */
     @Nullable
@@ -1343,15 +1355,7 @@ public static String unescapeDocument(@Nonnull final String doc) {
      */
     @Nonnull
     public static String generateContentPlaybackNonce() {
-        final SecureRandom random = new SecureRandom();
-        final StringBuilder stringBuilder = new StringBuilder();
-
-        for (int i = 0; i < 16; i++) {
-            stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt(
-                    (random.nextInt(128) + 1) & 63));
-        }
-
-        return stringBuilder.toString();
+        return randomStringFromAlphabet(CONTENT_PLAYBACK_NONCE_ALPHABET, 16);
     }
 
     /**
@@ -1367,14 +1371,23 @@ public static String generateContentPlaybackNonce() {
      */
     @Nonnull
     public static String generateTParameter() {
-        final SecureRandom random = new SecureRandom();
-        final StringBuilder stringBuilder = new StringBuilder();
-
-        for (int i = 0; i < 12; i++) {
-            stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt(
-                    (random.nextInt(128) + 1) & 63));
-        }
+        return randomStringFromAlphabet(CONTENT_PLAYBACK_NONCE_ALPHABET, 12);
+    }
 
-        return stringBuilder.toString();
+    /**
+     * Set the seed for video tests.
+     *
+     * <p>
+     * This seed will be used to generate the same {@code t} and {@code cpn} values between
+     * different execution of tests so mocks can be used for stream tests.
+     * </p>
+     *
+     * <p>
+     * This method will call {@link Utils#setSecureRandomSeed(long)} with the
+     * {@link #SEED_FOR_VIDEOS_TESTS value}.
+     * </p>
+     */
+    public static void setSeedForVideoTests() {
+        Utils.setSecureRandomSeed(SEED_FOR_VIDEOS_TESTS);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -8,6 +8,7 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.security.SecureRandom;
 import java.util.*;
 import java.util.regex.Pattern;
 
@@ -19,16 +20,23 @@ public class Utils {
     public static final String EMPTY_STRING = "";
     private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
     private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
+    private static final SecureRandom random = new SecureRandom();
 
     private Utils() {
         // no instance
     }
 
     /**
-     * Remove all non-digit characters from a string.<p>
-     * Examples:<p>
-     * <ul><li>1 234 567 views -&gt; 1234567</li>
-     * <li>$31,133.124 -&gt; 31133124</li></ul>
+     * Remove all non-digit characters from a string.
+     *
+     * <p>
+     * Examples:
+     * </p>
+     *
+     * <ul>
+     *     <li>1 234 567 views -&gt; 1234567</li>
+     *     <li>$31,133.124 -&gt; 31133124</li>
+     * </ul>
      *
      * @param toRemove string to remove non-digit chars
      * @return a string that contains only digits
@@ -39,8 +47,12 @@ public static String removeNonDigitCharacters(@Nonnull final String toRemove) {
     }
 
     /**
-     * <p>Convert a mixed number word to a long.</p>
-     * <p>Examples:</p>
+     * Convert a mixed number word to a long.
+     *
+     * <p>
+     * Examples:
+     * </p>
+     *
      * <ul>
      *     <li>123 -&gt; 123</li>
      *     <li>1.23K -&gt; 1230</li>
@@ -49,16 +61,18 @@ public static String removeNonDigitCharacters(@Nonnull final String toRemove) {
      *
      * @param numberWord string to be converted to a long
      * @return a long
-     * @throws NumberFormatException
-     * @throws ParsingException
+     * @throws NumberFormatException if the string does not contain a parsable double
+     *                               by {@link Double#parseDouble(String)}
+     * @throws ParsingException      if a number could not be found in the string provided
      */
     public static long mixedNumberWordToLong(final String numberWord)
             throws NumberFormatException, ParsingException {
         String multiplier = "";
         try {
             multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
-        } catch (ParsingException ignored) {
+        } catch (final ParsingException ignored) {
         }
+
         final double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)",
                 numberWord).replace(",", "."));
         switch (multiplier.toUpperCase()) {
@@ -107,11 +121,15 @@ public static String replaceHttpWithHttps(final String url) {
 
     /**
      * Get the value of a URL-query by name.
-     * If a url-query is give multiple times, only the value of the first query is returned
+     *
+     * <p>
+     * If an url-query is give multiple times, only the value of the first query is returned.
+     * </p>
      *
      * @param url           the url to be used
      * @param parameterName the pattern that will be used to check the url
-     * @return a string that contains the value of the query parameter or null if nothing was found
+     * @return a string that contains the value of the query parameter or {@code null} if nothing
+     * was found
      */
     @Nullable
     public static String getQueryValue(@Nonnull final URL url,
@@ -149,11 +167,14 @@ public static String getQueryValue(@Nonnull final URL url,
     }
 
     /**
-     * converts a string to a URL-Object.
-     * defaults to HTTP if no protocol is given
+     * Convert a string to a {@link URL URL object}.
+     *
+     * <p>
+     * Defaults to HTTP if no protocol is given.
+     * </p>
      *
      * @param url the string to be converted to a URL-Object
-     * @return a URL-Object containing the url
+     * @return a {@link URL URL object} containing the url
      */
     @Nonnull
     public static URL stringToURL(final String url) throws MalformedURLException {
@@ -245,11 +266,12 @@ public static boolean isNullOrEmpty(final String str) {
     }
 
     /**
-     * Checks if a collection is null or empty.
+     * Check if a collection is null or empty.
      *
      * <p>
      * This method can be also used for {@link com.grack.nanojson.JsonArray JsonArray}s.
      * </p>
+     *
      * @param collection the collection on which check if it's null or empty
      * @return whether the collection is null or empty
      */
@@ -258,11 +280,12 @@ public static boolean isNullOrEmpty(final Collection<?> collection) {
     }
 
     /**
-     * Checks if a {@link Map map} is null or empty.
+     * Check if a {@link Map map} is null or empty.
      *
      * <p>
      * This method can be also used for {@link com.grack.nanojson.JsonObject JsonObject}s.
      * </p>
+     *
      * @param map the {@link Map map} on which check if it's null or empty
      * @return whether the {@link Map map} is null or empty
      */
@@ -386,6 +409,7 @@ public static String getStringResultFromRegexArray(@Nonnull final String input,
             } catch (final Parser.RegexException ignored) {
             }
         }
+
         if (result == null) {
             throw new Parser.RegexException("No regex matched the input on group " + group);
         }
@@ -419,9 +443,49 @@ public static String getStringResultFromRegexArray(@Nonnull final String input,
             } catch (final Parser.RegexException ignored) {
             }
         }
+
         if (result == null) {
             throw new Parser.RegexException("No regex matched the input on group " + group);
         }
         return result;
     }
+
+    /**
+     * Generate a random string using the secure random device {@link #random}.
+     *
+     * <p>
+     * {@link #setRandomSeed(long)} might be useful when mocking tests.
+     * </p>
+     *
+     * @param alphabet the characters' alphabet to use
+     * @param length   the length of the returned string
+     * @return a random string of the requested length made of only characters from the provided
+     * alphabet
+     */
+    @Nonnull
+    public static String randomStringFromAlphabet(final String alphabet, final int length) {
+        final StringBuilder stringBuilder = new StringBuilder();
+        for (int i = 0; i < length; ++i) {
+            stringBuilder.append(alphabet.charAt(random.nextInt(alphabet.length())));
+        }
+        return stringBuilder.toString();
+    }
+
+    /**
+     * Seed the secure random device used for {@link #randomStringFromAlphabet(String, int)}.
+     *
+     * <p>
+     * Use this in tests so that they can be mocked as the same random numbers are always
+     * generated.
+     * </p>
+     *
+     * <p>
+     * This is not intended to be used outside of tests.
+     * </p>
+     *
+     * @param seed the seed to pass to {@link SecureRandom#setSeed(long)}
+     */
+    public static void setSecureRandomSeed(final long seed) {
+        random.setSeed(seed);
+    }
 }