Skip to content

Commit 5794eb2

Browse files
AudricVAudricV
committed
Use the youtubei API for YouTube playlists
1 parent c97a19d commit 5794eb2

3 files changed

Lines changed: 56 additions & 30 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ private static boolean isGoogleURL(String url) {
9595
final String host = u.getHost();
9696
return host.startsWith("google.") || host.startsWith("m.google.")
9797
|| host.startsWith("www.google.");
98-
} catch (MalformedURLException e) {
98+
} catch (final MalformedURLException e) {
9999
return false;
100100
}
101101
}
@@ -207,10 +207,10 @@ public static String getFeedUrlFrom(final String channelIdOrUser) {
207207
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
208208
try {
209209
return OffsetDateTime.parse(textualUploadDate);
210-
} catch (DateTimeParseException e) {
210+
} catch (final DateTimeParseException e) {
211211
try {
212212
return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
213-
} catch (DateTimeParseException e1) {
213+
} catch (final DateTimeParseException e1) {
214214
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1);
215215
}
216216
}
@@ -277,11 +277,11 @@ public static JsonObject getInitialData(final String html) throws ParsingExcepti
277277
try {
278278
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
279279
return JsonParser.object().from(initialData);
280-
} catch (Parser.RegexException e) {
280+
} catch (final Parser.RegexException e) {
281281
final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
282282
return JsonParser.object().from(initialData);
283283
}
284-
} catch (JsonParserException | Parser.RegexException e) {
284+
} catch (final JsonParserException | Parser.RegexException e) {
285285
throw new ParsingException("Could not get ytInitialData", e);
286286
}
287287
}
@@ -342,7 +342,7 @@ private static void extractClientVersionAndKey() throws IOException, ExtractionE
342342
clientVersion = contextClientVersion;
343343
break;
344344
}
345-
} catch (Parser.RegexException ignored) {
345+
} catch (final Parser.RegexException ignored) {
346346
}
347347
}
348348

@@ -352,10 +352,10 @@ private static void extractClientVersionAndKey() throws IOException, ExtractionE
352352

353353
try {
354354
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
355-
} catch (Parser.RegexException e) {
355+
} catch (final Parser.RegexException e) {
356356
try {
357357
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
358-
} catch (Parser.RegexException ignored) {
358+
} catch (final Parser.RegexException ignored) {
359359
}
360360
}
361361
}
@@ -468,7 +468,7 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept
468468
String key;
469469
try {
470470
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
471-
} catch (Parser.RegexException e) {
471+
} catch (final Parser.RegexException e) {
472472
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
473473
}
474474

@@ -477,10 +477,10 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept
477477
String clientVersion;
478478
try {
479479
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
480-
} catch (Parser.RegexException e) {
480+
} catch (final Parser.RegexException e) {
481481
try {
482482
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
483-
} catch (Parser.RegexException ee) {
483+
} catch (final Parser.RegexException ee) {
484484
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
485485
}
486486
}
@@ -491,7 +491,7 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept
491491

492492

493493
@Nullable
494-
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
494+
public static String getUrlFromNavigationEndpoint(final JsonObject navigationEndpoint) throws ParsingException {
495495
if (navigationEndpoint.has("urlEndpoint")) {
496496
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
497497
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
@@ -508,7 +508,7 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
508508
String url;
509509
try {
510510
url = URLDecoder.decode(param.split("=")[1], UTF_8);
511-
} catch (UnsupportedEncodingException e) {
511+
} catch (final UnsupportedEncodingException e) {
512512
return null;
513513
}
514514
return url;
@@ -662,6 +662,31 @@ public static Response getResponse(final String url, final Localization localiza
662662
return response;
663663
}
664664

665+
public static String extractCookieValue(final String cookieName, final Response response) {
666+
final List<String> cookies = response.responseHeaders().get("set-cookie");
667+
int startIndex;
668+
String result = "";
669+
for (final String cookie : cookies) {
670+
startIndex = cookie.indexOf(cookieName);
671+
if (startIndex != -1) {
672+
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
673+
cookie.indexOf(";", startIndex));
674+
}
675+
}
676+
return result;
677+
}
678+
679+
public static JsonObject getJsonPostResponse(final String endpoint,
680+
final byte[] body,
681+
final Localization localization)
682+
throws IOException, ExtractionException {
683+
684+
final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/"
685+
+ endpoint + "?key=" + getKey(), new HashMap<>(), body, localization);
686+
687+
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
688+
}
689+
665690
public static JsonArray getJsonResponse(final String url, final Localization localization)
666691
throws IOException, ExtractionException {
667692
Map<String, List<String>> headers = new HashMap<>();
@@ -885,7 +910,7 @@ private static MetaInfo getClarificationRendererContent(final JsonObject clarifi
885910
metaInfo.addUrl(new URL(url));
886911
final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource"));
887912
metaInfo.addUrlText(description == null ? url : description);
888-
} catch (MalformedURLException e) {
913+
} catch (final MalformedURLException e) {
889914
throw new ParsingException("Could not get metadata info secondary URL", e);
890915
}
891916
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ public String getId() throws ParsingException {
147147
public String getName() throws ParsingException {
148148
try {
149149
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title");
150-
} catch (Exception e) {
150+
} catch (final Exception e) {
151151
throw new ParsingException("Could not get channel name", e);
152152
}
153153
}
@@ -159,7 +159,7 @@ public String getAvatarUrl() throws ParsingException {
159159
.getArray("thumbnails").getObject(0).getString("url");
160160

161161
return fixThumbnailUrl(url);
162-
} catch (Exception e) {
162+
} catch (final Exception e) {
163163
throw new ParsingException("Could not get avatar", e);
164164
}
165165
}
@@ -175,7 +175,7 @@ public String getBannerUrl() throws ParsingException {
175175
}
176176

177177
return fixThumbnailUrl(url);
178-
} catch (Exception e) {
178+
} catch (final Exception e) {
179179
throw new ParsingException("Could not get banner", e);
180180
}
181181
}
@@ -184,7 +184,7 @@ public String getBannerUrl() throws ParsingException {
184184
public String getFeedUrl() throws ParsingException {
185185
try {
186186
return YoutubeParsingHelper.getFeedUrlFrom(getId());
187-
} catch (Exception e) {
187+
} catch (final Exception e) {
188188
throw new ParsingException("Could not get feed url", e);
189189
}
190190
}
@@ -207,7 +207,7 @@ public long getSubscriberCount() throws ParsingException {
207207
public String getDescription() throws ParsingException {
208208
try {
209209
return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description");
210-
} catch (Exception e) {
210+
} catch (final Exception e) {
211211
throw new ParsingException("Could not get channel description", e);
212212
}
213213
}
@@ -293,7 +293,7 @@ private Page getNextPageFrom(final JsonObject continuations) throws IOException,
293293
.done())
294294
.getBytes(UTF_8);
295295

296-
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
296+
return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key=" + getKey(),
297297
body);
298298
}
299299

@@ -314,7 +314,7 @@ private JsonObject collectStreamsFrom(StreamInfoItemsCollector collector, JsonAr
314314

315315
JsonObject continuation = null;
316316

317-
for (Object object : videos) {
317+
for (final Object object : videos) {
318318
final JsonObject video = (JsonObject) object;
319319
if (video.has("gridVideoRenderer")) {
320320
collector.commit(new YoutubeStreamInfoItemExtractor(
@@ -344,7 +344,7 @@ private JsonObject getVideoTab() throws ParsingException {
344344
.getArray("tabs");
345345
JsonObject videoTab = null;
346346

347-
for (Object tab : tabs) {
347+
for (final Object tab : tabs) {
348348
if (((JsonObject) tab).has("tabRenderer")) {
349349
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
350350
videoTab = ((JsonObject) tab).getObject("tabRenderer");

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import javax.annotation.Nullable;
3030

3131
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
32-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
3333
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
3434
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
3535
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@@ -50,11 +50,13 @@ public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHa
5050

5151
@Override
5252
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
53-
final String url = getUrl() + "&pbj=1";
53+
final byte[] body = JsonWriter.string(prepareJsonBuilder()
54+
.value("browseId", "VL" + getId())
55+
.value("params", "wgYCCAA%3D") // show unavailable videos
56+
.done())
57+
.getBytes(UTF_8);
5458

55-
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
56-
57-
initialData = initialAjaxJson.getObject(1).getObject("response");
59+
initialData = getJsonPostResponse("browse", body, getExtractorLocalization());
5860
YoutubeParsingHelper.defaultAlertsCheck(initialData);
5961

6062
playlistInfo = getPlaylistInfo();
@@ -251,9 +253,8 @@ private Page getNextPageFrom(final JsonArray contents) throws IOException, Extra
251253
.done())
252254
.getBytes(UTF_8);
253255

254-
return new Page(
255-
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
256-
body);
256+
return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key="
257+
+ getKey(), body);
257258
} else {
258259
return null;
259260
}

0 commit comments

Comments
 (0)