Skip to content

Commit 21e579b

Browse files
AudricVFireMasterK
AudricV
authored and
FireMasterK
committed
[PATCH] [YouTube] Remove age-restricted videos workaround, start poTokens support
1 parent 8e92227 commit 21e579b

5 files changed

Lines changed: 342 additions & 207 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package org.schabi.newpipe.extractor.services.youtube;
2+
3+
import javax.annotation.Nullable;
4+
5+
/**
6+
* An interface to provide poTokens to YouTube player requests.
7+
*
8+
* <p>
9+
* On some major clients, YouTube requires that the integrity of the device passes some checks to
10+
* allow playback.
11+
* </p>
12+
*
13+
* <p>
14+
* These checks involve running codes to verify the integrity and using their result to generate a
15+
* poToken (which likely stands for proof of origin token), using a visitor data ID for logged-out
16+
* users.
17+
* </p>
18+
*
19+
* <p>
20+
* These tokens may have a role in triggering the sign in requirement.
21+
* </p>
22+
*/
23+
public interface PoTokenProvider {
24+
25+
/**
26+
* Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client.
27+
*
28+
* <p>
29+
* To be generated and valid, poTokens from this client must be generated using Google's
30+
* BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
31+
* must be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
32+
* </p>
33+
*
34+
* @return a {@link PoTokenResult} specific to the WEB InnerTube client
35+
*/
36+
@Nullable
37+
PoTokenResult getWebClientPoToken();
38+
39+
@Nullable
40+
PoTokenResult getAndroidClientPoToken();
41+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package org.schabi.newpipe.extractor.services.youtube;
2+
3+
import javax.annotation.Nonnull;
4+
import java.util.Objects;
5+
6+
public final class PoTokenResult {
7+
8+
/**
9+
* The visitor data associated with a poToken.
10+
*/
11+
public final String visitorData;
12+
13+
/**
14+
* The poToken, a Protobuf object encoded as a base 64 string.
15+
*/
16+
public final String poToken;
17+
18+
public PoTokenResult(@Nonnull final String visitorData, @Nonnull final String poToken) {
19+
this.visitorData = Objects.requireNonNull(visitorData);
20+
this.poToken = Objects.requireNonNull(poToken);
21+
}
22+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1200,9 +1200,10 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
12001200
@Nonnull
12011201
public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
12021202
@Nonnull final Localization localization,
1203-
@Nonnull final ContentCountry contentCountry) {
1203+
@Nonnull final ContentCountry contentCountry,
1204+
@Nullable final String visitorData) {
12041205
// @formatter:off
1205-
return JsonObject.builder()
1206+
final JsonBuilder<JsonObject> builder = JsonObject.builder()
12061207
.object("context")
12071208
.object("client")
12081209
.value("clientName", "ANDROID")
@@ -1224,8 +1225,13 @@ public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
12241225
.value("androidSdkVersion", 34)
12251226
.value("hl", localization.getLocalizationCode())
12261227
.value("gl", contentCountry.getCountryCode())
1227-
.value("utcOffsetMinutes", 0)
1228-
.end()
1228+
.value("utcOffsetMinutes", 0);
1229+
1230+
if (visitorData != null) {
1231+
builder.value("visitorData", visitorData);
1232+
}
1233+
1234+
builder.end()
12291235
.object("request")
12301236
.array("internalExperimentFlags")
12311237
.end()
@@ -1238,6 +1244,7 @@ public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
12381244
.end()
12391245
.end();
12401246
// @formatter:on
1247+
return builder;
12411248
}
12421249

12431250
@Nonnull
@@ -1308,26 +1315,6 @@ public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
13081315
// @formatter:on
13091316
}
13101317

1311-
@Nonnull
1312-
public static JsonObject getWebPlayerResponse(
1313-
@Nonnull final Localization localization,
1314-
@Nonnull final ContentCountry contentCountry,
1315-
@Nonnull final String videoId) throws IOException, ExtractionException {
1316-
final byte[] body = JsonWriter.string(
1317-
prepareDesktopJsonBuilder(localization, contentCountry)
1318-
.value(VIDEO_ID, videoId)
1319-
.value(CONTENT_CHECK_OK, true)
1320-
.value(RACY_CHECK_OK, true)
1321-
.done())
1322-
.getBytes(StandardCharsets.UTF_8);
1323-
final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
1324-
+ "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
1325-
1326-
return JsonUtils.toJsonObject(getValidJsonResponseBody(
1327-
getDownloader().postWithContentTypeJson(
1328-
url, getYouTubeHeaders(), body, localization)));
1329-
}
1330-
13311318
@Nonnull
13321319
public static byte[] createTvHtml5EmbedPlayerBody(
13331320
@Nonnull final Localization localization,

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package org.schabi.newpipe.extractor.services.youtube;
2+
3+
import com.grack.nanojson.JsonObject;
4+
import com.grack.nanojson.JsonWriter;
5+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
6+
import org.schabi.newpipe.extractor.localization.ContentCountry;
7+
import org.schabi.newpipe.extractor.localization.Localization;
8+
import org.schabi.newpipe.extractor.utils.JsonUtils;
9+
10+
import javax.annotation.Nonnull;
11+
import java.io.IOException;
12+
import java.nio.charset.StandardCharsets;
13+
14+
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
15+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
16+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
17+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
18+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
19+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
20+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
21+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
22+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
23+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
24+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
25+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
26+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
27+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
28+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
29+
30+
public final class YoutubeStreamHelper {
31+
32+
private static final String STREAMING_DATA = "streamingData";
33+
private static final String PLAYER = "player";
34+
private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions";
35+
private static final String PO_TOKEN = "poToken";
36+
37+
private YoutubeStreamHelper() {
38+
}
39+
40+
@Nonnull
41+
public static JsonObject getWebMetadataPlayerResponse(
42+
@Nonnull final Localization localization,
43+
@Nonnull final ContentCountry contentCountry,
44+
@Nonnull final String videoId) throws IOException, ExtractionException {
45+
final byte[] body = JsonWriter.string(
46+
prepareDesktopJsonBuilder(localization, contentCountry)
47+
.value(VIDEO_ID, videoId)
48+
.value(CONTENT_CHECK_OK, true)
49+
.value(RACY_CHECK_OK, true)
50+
.done())
51+
.getBytes(StandardCharsets.UTF_8);
52+
final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
53+
+ "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
54+
55+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
56+
getDownloader().postWithContentTypeJson(
57+
url, getYouTubeHeaders(), body, localization)));
58+
}
59+
60+
@Nonnull
61+
public static JsonObject getWebFullPlayerResponse(
62+
@Nonnull final Localization localization,
63+
@Nonnull final ContentCountry contentCountry,
64+
@Nonnull final String videoId,
65+
@Nonnull final PoTokenResult webPoTokenResult) throws IOException, ExtractionException {
66+
final byte[] body = JsonWriter.string(
67+
prepareDesktopJsonBuilder(localization, contentCountry, webPoTokenResult.visitorData)
68+
.value(VIDEO_ID, videoId)
69+
.value(CONTENT_CHECK_OK, true)
70+
.value(RACY_CHECK_OK, true)
71+
.object(SERVICE_INTEGRITY_DIMENSIONS)
72+
.value(PO_TOKEN, webPoTokenResult.poToken)
73+
.end()
74+
.done())
75+
.getBytes(StandardCharsets.UTF_8);
76+
final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
77+
78+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
79+
getDownloader().postWithContentTypeJson(
80+
url, getYouTubeHeaders(), body, localization)));
81+
}
82+
83+
public static JsonObject getAndroidPlayerResponse(@Nonnull final ContentCountry contentCountry,
84+
@Nonnull final Localization localization,
85+
@Nonnull final String videoId,
86+
@Nonnull final String androidCpn,
87+
@Nonnull final PoTokenResult androidPoTokenResult)
88+
throws IOException, ExtractionException {
89+
final byte[] mobileBody = JsonWriter.string(
90+
prepareAndroidMobileJsonBuilder(localization, contentCountry, androidPoTokenResult.visitorData)
91+
.value(VIDEO_ID, videoId)
92+
.value(CPN, androidCpn)
93+
.value(CONTENT_CHECK_OK, true)
94+
.value(RACY_CHECK_OK, true)
95+
.object(SERVICE_INTEGRITY_DIMENSIONS)
96+
.value(PO_TOKEN, androidPoTokenResult.poToken)
97+
.end()
98+
.done())
99+
.getBytes(StandardCharsets.UTF_8);
100+
101+
return getJsonAndroidPostResponse(
102+
"player",
103+
mobileBody,
104+
localization,
105+
"&t=" + generateTParameter() + "&id=" + videoId);
106+
}
107+
108+
public static JsonObject getAndroidReelPlayerResponse(@Nonnull final ContentCountry contentCountry,
109+
@Nonnull final Localization localization,
110+
@Nonnull final String videoId,
111+
@Nonnull final String androidCpn)
112+
throws IOException, ExtractionException {
113+
final byte[] mobileBody = JsonWriter.string(
114+
prepareAndroidMobileJsonBuilder(localization, contentCountry, null)
115+
.object("playerRequest")
116+
.value(VIDEO_ID, videoId)
117+
.end()
118+
.value("disablePlayerResponse", false)
119+
.value(VIDEO_ID, videoId)
120+
.value(CPN, androidCpn)
121+
.value(CONTENT_CHECK_OK, true)
122+
.value(RACY_CHECK_OK, true)
123+
.done())
124+
.getBytes(StandardCharsets.UTF_8);
125+
126+
final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(
127+
"reel/reel_item_watch",
128+
mobileBody,
129+
localization,
130+
"&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse");
131+
132+
return androidPlayerResponse.getObject("playerResponse");
133+
}
134+
135+
public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry,
136+
@Nonnull final Localization localization,
137+
@Nonnull final String videoId,
138+
@Nonnull final String iosCpn)
139+
throws IOException, ExtractionException {
140+
final byte[] mobileBody = JsonWriter.string(
141+
prepareIosMobileJsonBuilder(localization, contentCountry)
142+
.value(VIDEO_ID, videoId)
143+
.value(CPN, iosCpn)
144+
.value(CONTENT_CHECK_OK, true)
145+
.value(RACY_CHECK_OK, true)
146+
.done())
147+
.getBytes(StandardCharsets.UTF_8);
148+
149+
return getJsonIosPostResponse(PLAYER,
150+
mobileBody, localization, "&t=" + generateTParameter()
151+
+ "&id=" + videoId);
152+
}
153+
}

0 commit comments

Comments
 (0)