Skip to content

Commit c00f9e6

Browse files
litetexlitetex
committed
Improved test and randomness
* Use the existing RNG inside ``YouTubeParsingHelper`` * Deduplicated test-setup for YT tests * Minor improvements
1 parent a44467a commit c00f9e6

20 files changed

Lines changed: 257 additions & 323 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 38 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3+
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
4+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
5+
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
6+
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
7+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
8+
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
9+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
10+
311
import com.grack.nanojson.JsonArray;
412
import com.grack.nanojson.JsonBuilder;
513
import com.grack.nanojson.JsonObject;
@@ -9,12 +17,17 @@
917

1018
import org.schabi.newpipe.extractor.MetaInfo;
1119
import org.schabi.newpipe.extractor.downloader.Response;
12-
import org.schabi.newpipe.extractor.exceptions.*;
20+
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
21+
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
22+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
23+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
24+
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
1325
import org.schabi.newpipe.extractor.localization.ContentCountry;
1426
import org.schabi.newpipe.extractor.localization.Localization;
1527
import org.schabi.newpipe.extractor.stream.Description;
1628
import org.schabi.newpipe.extractor.utils.JsonUtils;
1729
import org.schabi.newpipe.extractor.utils.Parser;
30+
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
1831
import org.schabi.newpipe.extractor.utils.Utils;
1932

2033
import java.io.IOException;
@@ -23,18 +36,24 @@
2336
import java.net.URL;
2437
import java.net.URLDecoder;
2538
import java.nio.charset.StandardCharsets;
39+
import java.security.SecureRandom;
2640
import java.time.LocalDate;
2741
import java.time.OffsetDateTime;
2842
import java.time.ZoneOffset;
2943
import java.time.format.DateTimeParseException;
30-
import java.util.*;
44+
import java.util.ArrayList;
45+
import java.util.Arrays;
46+
import java.util.Collections;
47+
import java.util.HashMap;
48+
import java.util.List;
49+
import java.util.Map;
50+
import java.util.Objects;
51+
import java.util.Optional;
52+
import java.util.Random;
3153

3254
import javax.annotation.Nonnull;
3355
import javax.annotation.Nullable;
3456

35-
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
36-
import static org.schabi.newpipe.extractor.utils.Utils.*;
37-
3857
/*
3958
* Created by Christian Schabesberger on 02.03.16.
4059
*
@@ -64,11 +83,6 @@ private YoutubeParsingHelper() {
6483
public static final String CPN = "cpn";
6584
public static final String VIDEO_ID = "videoId";
6685

67-
/**
68-
* Seed that will be used for video tests, in order to mock video requests.
69-
*/
70-
private static final long SEED_FOR_VIDEOS_TESTS = 3000;
71-
7286
private static final String HARDCODED_CLIENT_VERSION = "2.20220114.01.00";
7387
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
7488

@@ -103,7 +117,7 @@ private YoutubeParsingHelper() {
103117
private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
104118
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
105119

106-
private static Random numberGenerator = new Random();
120+
private static Random numberGenerator = new SecureRandom();
107121

108122
/**
109123
* {@code PENDING+} means that the user did not yet submit their choices.
@@ -481,12 +495,12 @@ public static String getClientVersion() throws IOException, ExtractionException
481495

482496
if (keyAndVersionExtracted) {
483497
return clientVersion;
484-
} else {
485-
if (areHardcodedClientVersionAndKeyValid()) {
486-
clientVersion = HARDCODED_CLIENT_VERSION;
487-
return clientVersion;
488-
}
489498
}
499+
if (areHardcodedClientVersionAndKeyValid()) {
500+
clientVersion = HARDCODED_CLIENT_VERSION;
501+
return clientVersion;
502+
}
503+
490504
throw new ExtractionException("Could not get YouTube WEB client version");
491505
}
492506

@@ -509,11 +523,10 @@ public static String getKey() throws IOException, ExtractionException {
509523

510524
if (keyAndVersionExtracted) {
511525
return key;
512-
} else {
513-
if (areHardcodedClientVersionAndKeyValid()) {
514-
key = HARDCODED_KEY;
515-
return key;
516-
}
526+
}
527+
if (areHardcodedClientVersionAndKeyValid()) {
528+
key = HARDCODED_KEY;
529+
return key;
517530
}
518531

519532
// The ANDROID API key is also valid with the WEB client so return it if we couldn't
@@ -1355,7 +1368,8 @@ public static String unescapeDocument(@Nonnull final String doc) {
13551368
*/
13561369
@Nonnull
13571370
public static String generateContentPlaybackNonce() {
1358-
return randomStringFromAlphabet(CONTENT_PLAYBACK_NONCE_ALPHABET, 16);
1371+
return RandomStringFromAlphabetGenerator.generate(
1372+
CONTENT_PLAYBACK_NONCE_ALPHABET, 16, numberGenerator);
13591373
}
13601374

13611375
/**
@@ -1371,23 +1385,7 @@ public static String generateContentPlaybackNonce() {
13711385
*/
13721386
@Nonnull
13731387
public static String generateTParameter() {
1374-
return randomStringFromAlphabet(CONTENT_PLAYBACK_NONCE_ALPHABET, 12);
1375-
}
1376-
1377-
/**
1378-
* Set the seed for video tests.
1379-
*
1380-
* <p>
1381-
* This seed will be used to generate the same {@code t} and {@code cpn} values between
1382-
* different execution of tests so mocks can be used for stream tests.
1383-
* </p>
1384-
*
1385-
* <p>
1386-
* This method will call {@link Utils#setSecureRandomSeed(long)} with the
1387-
* {@link #SEED_FOR_VIDEOS_TESTS value}.
1388-
* </p>
1389-
*/
1390-
public static void setSeedForVideoTests() {
1391-
Utils.setSecureRandomSeed(SEED_FOR_VIDEOS_TESTS);
1388+
return RandomStringFromAlphabetGenerator.generate(
1389+
CONTENT_PLAYBACK_NONCE_ALPHABET, 12, numberGenerator);
13921390
}
13931391
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/RandomStringFromAlphabetGenerator.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package org.schabi.newpipe.extractor.utils;
2+
3+
import java.util.Random;
4+
5+
import javax.annotation.Nonnull;
6+
7+
/**
8+
* Generates a random string from a predefined alphabet.
9+
*/
10+
public final class RandomStringFromAlphabetGenerator {
11+
private RandomStringFromAlphabetGenerator() {
12+
// No impl
13+
}
14+
15+
/**
16+
* Generate a random string from an alphabet.
17+
*
18+
* @param alphabet the characters' alphabet to use
19+
* @param length the length of the returned string (> 0)
20+
* @param random {@link Random} (or better {@link java.security.SecureRandom}) used for generating the random string
21+
* @return a random string of the requested length made of only characters from the provided
22+
* alphabet
23+
*/
24+
@Nonnull
25+
public static String generate(
26+
final String alphabet,
27+
final int length,
28+
final Random random) {
29+
final StringBuilder stringBuilder = new StringBuilder(length);
30+
for (int i = 0; i < length; i++) {
31+
stringBuilder.append(alphabet.charAt(random.nextInt(alphabet.length())));
32+
}
33+
return stringBuilder.toString();
34+
}
35+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java‎

Lines changed: 10 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,22 @@
22

33
import org.schabi.newpipe.extractor.exceptions.ParsingException;
44

5-
import javax.annotation.Nonnull;
6-
import javax.annotation.Nullable;
75
import java.io.UnsupportedEncodingException;
86
import java.net.MalformedURLException;
97
import java.net.URL;
108
import java.net.URLDecoder;
11-
import java.security.SecureRandom;
12-
import java.util.*;
9+
import java.util.ArrayList;
10+
import java.util.Arrays;
11+
import java.util.Collection;
12+
import java.util.Iterator;
13+
import java.util.LinkedList;
14+
import java.util.List;
15+
import java.util.Map;
1316
import java.util.regex.Pattern;
1417

18+
import javax.annotation.Nonnull;
19+
import javax.annotation.Nullable;
20+
1521
public class Utils {
1622

1723
public static final String HTTP = "http://";
@@ -20,7 +26,6 @@ public class Utils {
2026
public static final String EMPTY_STRING = "";
2127
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
2228
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
23-
private static final SecureRandom random = new SecureRandom();
2429

2530
private Utils() {
2631
// no instance
@@ -449,43 +454,4 @@ public static String getStringResultFromRegexArray(@Nonnull final String input,
449454
}
450455
return result;
451456
}
452-
453-
/**
454-
* Generate a random string using the secure random device {@link #random}.
455-
*
456-
* <p>
457-
* {@link #setRandomSeed(long)} might be useful when mocking tests.
458-
* </p>
459-
*
460-
* @param alphabet the characters' alphabet to use
461-
* @param length the length of the returned string
462-
* @return a random string of the requested length made of only characters from the provided
463-
* alphabet
464-
*/
465-
@Nonnull
466-
public static String randomStringFromAlphabet(final String alphabet, final int length) {
467-
final StringBuilder stringBuilder = new StringBuilder();
468-
for (int i = 0; i < length; ++i) {
469-
stringBuilder.append(alphabet.charAt(random.nextInt(alphabet.length())));
470-
}
471-
return stringBuilder.toString();
472-
}
473-
474-
/**
475-
* Seed the secure random device used for {@link #randomStringFromAlphabet(String, int)}.
476-
*
477-
* <p>
478-
* Use this in tests so that they can be mocked as the same random numbers are always
479-
* generated.
480-
* </p>
481-
*
482-
* <p>
483-
* This is not intended to be used outside of tests.
484-
* </p>
485-
*
486-
* @param seed the seed to pass to {@link SecureRandom#setSeed(long)}
487-
*/
488-
public static void setSecureRandomSeed(final long seed) {
489-
random.setSeed(seed);
490-
}
491457
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeStaticTestUtil.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package org.schabi.newpipe.extractor.services.youtube;
2+
3+
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
4+
5+
import java.util.Random;
6+
7+
/**
8+
* Utility class for keeping YT test stateless.
9+
*/
10+
public final class YouTubeStaticTestUtil {
11+
private YouTubeStaticTestUtil() {
12+
// No impl
13+
}
14+
15+
/**
16+
* Clears static YT states.
17+
*/
18+
public static void ensureStateless() {
19+
YoutubeParsingHelper.resetClientVersionAndKey();
20+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
21+
YoutubeStreamExtractor.resetDeobfuscationCode();
22+
}
23+
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.assertFalse;
5+
import static org.junit.jupiter.api.Assertions.assertThrows;
6+
import static org.junit.jupiter.api.Assertions.assertTrue;
7+
import static org.junit.jupiter.api.Assertions.fail;
8+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
9+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
10+
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
11+
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
12+
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
13+
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
14+
315
import org.junit.jupiter.api.BeforeAll;
416
import org.junit.jupiter.api.Test;
517
import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -15,13 +27,6 @@
1527
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
1628

1729
import java.io.IOException;
18-
import java.util.Random;
19-
20-
import static org.junit.jupiter.api.Assertions.*;
21-
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
22-
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
23-
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
24-
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
2530

2631
/**
2732
* Test for {@link ChannelExtractor}
@@ -33,8 +38,7 @@ public class YoutubeChannelExtractorTest {
3338
public static class NotAvailable {
3439
@BeforeAll
3540
public static void setUp() throws IOException {
36-
YoutubeParsingHelper.resetClientVersionAndKey();
37-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
41+
YouTubeStaticTestUtil.ensureStateless();
3842
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
3943
}
4044

@@ -130,8 +134,7 @@ public void accountTerminatedCopyrightFetch() throws Exception {
130134
public static class NotSupported {
131135
@BeforeAll
132136
public static void setUp() throws IOException {
133-
YoutubeParsingHelper.resetClientVersionAndKey();
134-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
137+
YouTubeStaticTestUtil.ensureStateless();
135138
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
136139
}
137140

@@ -149,8 +152,7 @@ public static class Gronkh implements BaseChannelExtractorTest {
149152

150153
@BeforeAll
151154
public static void setUp() throws Exception {
152-
YoutubeParsingHelper.resetClientVersionAndKey();
153-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
155+
YouTubeStaticTestUtil.ensureStateless();
154156
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
155157
extractor = (YoutubeChannelExtractor) YouTube
156158
.getChannelExtractor("http://www.youtube.com/user/Gronkh");
@@ -246,8 +248,7 @@ public static class VSauce implements BaseChannelExtractorTest {
246248

247249
@BeforeAll
248250
public static void setUp() throws Exception {
249-
YoutubeParsingHelper.resetClientVersionAndKey();
250-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
251+
YouTubeStaticTestUtil.ensureStateless();
251252
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
252253
extractor = (YoutubeChannelExtractor) YouTube
253254
.getChannelExtractor("https://www.youtube.com/user/Vsauce");
@@ -342,8 +343,7 @@ public static class Kurzgesagt implements BaseChannelExtractorTest {
342343

343344
@BeforeAll
344345
public static void setUp() throws Exception {
345-
YoutubeParsingHelper.resetClientVersionAndKey();
346-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
346+
YouTubeStaticTestUtil.ensureStateless();
347347
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
348348
extractor = (YoutubeChannelExtractor) YouTube
349349
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
@@ -459,8 +459,7 @@ public static class CaptainDisillusion implements BaseChannelExtractorTest {
459459

460460
@BeforeAll
461461
public static void setUp() throws Exception {
462-
YoutubeParsingHelper.resetClientVersionAndKey();
463-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
462+
YouTubeStaticTestUtil.ensureStateless();
464463
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
465464
extractor = (YoutubeChannelExtractor) YouTube
466465
.getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
@@ -554,8 +553,7 @@ public static class RandomChannel implements BaseChannelExtractorTest {
554553

555554
@BeforeAll
556555
public static void setUp() throws Exception {
557-
YoutubeParsingHelper.resetClientVersionAndKey();
558-
YoutubeParsingHelper.setNumberGenerator(new Random(1));
556+
YouTubeStaticTestUtil.ensureStateless();
559557
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
560558
extractor = (YoutubeChannelExtractor) YouTube
561559
.getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");

0 commit comments

Comments
 (0)