Skip to content

Commit a12c69d

Browse files
AudricVAudricV
committed
Use the youtubei API for YouTube channels
1 parent 5794eb2 commit a12c69d

3 files changed

Lines changed: 122 additions & 35 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 119 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
1010
import org.schabi.newpipe.extractor.downloader.Downloader;
1111
import org.schabi.newpipe.extractor.downloader.Response;
12+
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
1213
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
1314
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1415
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -26,7 +27,7 @@
2627
import javax.annotation.Nonnull;
2728

2829
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
29-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
30+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
3031
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
3132
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
3233
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
@@ -72,35 +73,106 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
7273
*/
7374
private String redirectedChannelId;
7475

75-
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
76+
public YoutubeChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
7677
super(service, linkHandler);
7778
}
7879

7980
@Override
80-
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
81-
String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
82-
JsonArray ajaxJson = null;
81+
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
82+
final String channel_path = super.getId();
83+
final String[] channelInfo = channel_path.split("/");
84+
String id = "";
85+
// If the url is an URL with /user, we need to use navigation/resolve_url endpoint of the
86+
// youtubei API to get the channel id. Otherwise, we couldn't get information about the
87+
// channel associated with this username, if there is one.
88+
if (channelInfo[0].equals("user")) {
89+
final byte[] body = JsonWriter.string(prepareJsonBuilder()
90+
.value("url", "https://www.youtube.com/" + channel_path)
91+
.done())
92+
.getBytes(UTF_8);
93+
94+
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
95+
body, getExtractorLocalization());
96+
97+
System.out.println(jsonResponse.toString());
98+
99+
if (jsonResponse.has("error")) {
100+
if (jsonResponse.getInt("code") == 404) {
101+
throw new ContentNotAvailableException("No channel associated with this user"
102+
+ "exists");
103+
} else {
104+
throw new ContentNotAvailableException("Got error:\""
105+
+ jsonResponse.getString("status") + "\""
106+
+ jsonResponse.getString("message"));
107+
}
108+
}
109+
110+
final JsonObject endpoint = jsonResponse.getObject("endpoint");
111+
112+
final String webPageType = endpoint.getObject("commandMetadata")
113+
.getObject("webCommandMetadata")
114+
.getString("webPageType", EMPTY_STRING);
115+
116+
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
117+
final String browseId = browseEndpoint.getString("browseId", EMPTY_STRING);
118+
119+
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
120+
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
121+
&& !browseId.isEmpty()) {
122+
if (!browseId.startsWith("UC")) {
123+
throw new ExtractionException("Redirected id is not pointing to a channel");
124+
}
125+
126+
id = browseId;
127+
redirectedChannelId = browseId;
128+
}
129+
} else {
130+
id = channelInfo[1];
131+
}
132+
JsonObject ajaxJson = null;
83133

84134
int level = 0;
85135
while (level < 3) {
86-
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());
136+
final byte[] body = JsonWriter.string(prepareJsonBuilder()
137+
.value("browseId", id)
138+
.value("params", "EgZ2aWRlb3M%3D") // equals to videos
139+
.done())
140+
.getBytes(UTF_8);
141+
142+
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
143+
getExtractorLocalization());
144+
145+
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
146+
final int errorCode = jsonResponse.getObject("error").getInt("code");
147+
if (errorCode == 400) {
148+
throw new ContentNotAvailableException("This channel doesn't exists");
149+
} else {
150+
throw new ContentNotAvailableException("Got error:\""
151+
+ jsonResponse.getString("status") + "\""
152+
+ jsonResponse.getString("message"));
153+
}
154+
}
87155

88-
final JsonObject endpoint = jsonResponse.getObject(1).getObject("response")
89-
.getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction")
156+
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
157+
.getObject(0)
158+
.getObject("navigateAction")
90159
.getObject("endpoint");
91160

92-
final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata")
161+
final String webPageType = endpoint.getObject("commandMetadata")
162+
.getObject("webCommandMetadata")
93163
.getString("webPageType", EMPTY_STRING);
94164

95-
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);
165+
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
166+
EMPTY_STRING);
96167

97168
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
98-
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") && !browseId.isEmpty()) {
169+
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
170+
&& !browseId.isEmpty()) {
99171
if (!browseId.startsWith("UC")) {
100172
throw new ExtractionException("Redirected id is not pointing to a channel");
101173
}
102174

103-
url = "https://www.youtube.com/channel/" + browseId + "/videos?pbj=1&view=0&flow=grid";
175+
id = browseId;
104176
redirectedChannelId = browseId;
105177
level++;
106178
} else {
@@ -113,7 +185,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
113185
throw new ExtractionException("Could not fetch initial JSON data");
114186
}
115187

116-
initialData = ajaxJson.getObject(1).getObject("response");
188+
initialData = ajaxJson;
117189
YoutubeParsingHelper.defaultAlertsCheck(initialData);
118190
}
119191

@@ -122,15 +194,16 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
122194
public String getUrl() throws ParsingException {
123195
try {
124196
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId());
125-
} catch (ParsingException e) {
197+
} catch (final ParsingException e) {
126198
return super.getUrl();
127199
}
128200
}
129201

130202
@Nonnull
131203
@Override
132204
public String getId() throws ParsingException {
133-
final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
205+
final String channelId = initialData.getObject("header")
206+
.getObject("c4TabbedHeaderRenderer")
134207
.getString("channelId", EMPTY_STRING);
135208

136209
if (!channelId.isEmpty()) {
@@ -146,7 +219,8 @@ public String getId() throws ParsingException {
146219
@Override
147220
public String getName() throws ParsingException {
148221
try {
149-
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title");
222+
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
223+
.getString("title");
150224
} catch (final Exception e) {
151225
throw new ParsingException("Could not get channel name", e);
152226
}
@@ -155,8 +229,9 @@ public String getName() throws ParsingException {
155229
@Override
156230
public String getAvatarUrl() throws ParsingException {
157231
try {
158-
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
159-
.getArray("thumbnails").getObject(0).getString("url");
232+
String url = initialData.getObject("header")
233+
.getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails")
234+
.getObject(0).getString("url");
160235

161236
return fixThumbnailUrl(url);
162237
} catch (final Exception e) {
@@ -167,8 +242,9 @@ public String getAvatarUrl() throws ParsingException {
167242
@Override
168243
public String getBannerUrl() throws ParsingException {
169244
try {
170-
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
171-
.getArray("thumbnails").getObject(0).getString("url");
245+
String url = initialData.getObject("header")
246+
.getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails")
247+
.getObject(0).getString("url");
172248

173249
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
174250
return null;
@@ -191,11 +267,13 @@ public String getFeedUrl() throws ParsingException {
191267

192268
@Override
193269
public long getSubscriberCount() throws ParsingException {
194-
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer");
270+
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header")
271+
.getObject("c4TabbedHeaderRenderer");
195272
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
196273
try {
197-
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText")));
198-
} catch (NumberFormatException e) {
274+
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
275+
.getObject("subscriberCountText")));
276+
} catch (final NumberFormatException e) {
199277
throw new ParsingException("Could not get subscriber count", e);
200278
}
201279
} else {
@@ -206,7 +284,8 @@ public long getSubscriberCount() throws ParsingException {
206284
@Override
207285
public String getDescription() throws ParsingException {
208286
try {
209-
return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description");
287+
return initialData.getObject("metadata").getObject("channelMetadataRenderer")
288+
.getString("description");
210289
} catch (final Exception e) {
211290
throw new ParsingException("Could not get channel description", e);
212291
}
@@ -229,7 +308,8 @@ public String getParentChannelAvatarUrl() throws ParsingException {
229308

230309
@Override
231310
public boolean isVerified() throws ParsingException {
232-
final JsonArray badges = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
311+
final JsonArray badges = initialData.getObject("header")
312+
.getObject("c4TabbedHeaderRenderer")
233313
.getArray("badges");
234314

235315
return YoutubeParsingHelper.isVerified(badges);
@@ -243,11 +323,13 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
243323
Page nextPage = null;
244324

245325
if (getVideoTab() != null) {
246-
final JsonObject gridRenderer = getVideoTab().getObject("content").getObject("sectionListRenderer")
326+
final JsonObject gridRenderer = getVideoTab().getObject("content")
327+
.getObject("sectionListRenderer")
247328
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
248329
.getArray("contents").getObject(0).getObject("gridRenderer");
249330

250-
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer.getArray("items"));
331+
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
332+
.getArray("items"));
251333

252334
nextPage = getNextPageFrom(continuation);
253335
}
@@ -275,7 +357,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
275357
.getObject(0)
276358
.getObject("appendContinuationItemsAction");
277359

278-
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
360+
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
361+
.getArray("continuationItems"));
279362

280363
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
281364
}
@@ -286,7 +369,8 @@ private Page getNextPageFrom(final JsonObject continuations) throws IOException,
286369
}
287370

288371
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
289-
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
372+
final String continuation = continuationEndpoint.getObject("continuationCommand")
373+
.getString("token");
290374

291375
final byte[] body = JsonWriter.string(prepareJsonBuilder()
292376
.value("continuation", continuation)
@@ -305,7 +389,8 @@ private Page getNextPageFrom(final JsonObject continuations) throws IOException,
305389
* @return the continuation object
306390
* @throws ParsingException if an error happened while extracting
307391
*/
308-
private JsonObject collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
392+
private JsonObject collectStreamsFrom(final StreamInfoItemsCollector collector,
393+
final JsonArray videos) throws ParsingException {
309394
collector.reset();
310395

311396
final String uploaderName = getName();
@@ -340,13 +425,15 @@ public String getUploaderUrl() {
340425
private JsonObject getVideoTab() throws ParsingException {
341426
if (this.videoTab != null) return this.videoTab;
342427

343-
JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
428+
JsonArray tabs = initialData.getObject("contents")
429+
.getObject("twoColumnBrowseResultsRenderer")
344430
.getArray("tabs");
345431
JsonObject videoTab = null;
346432

347433
for (final Object tab : tabs) {
348434
if (((JsonObject) tab).has("tabRenderer")) {
349-
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
435+
if (((JsonObject) tab).getObject("tabRenderer").getString("title",
436+
EMPTY_STRING).equals("Videos")) {
350437
videoTab = ((JsonObject) tab).getObject("tabRenderer");
351438
break;
352439
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, Extr
157157
.object("client")
158158
.value("hl", "en")
159159
.value("gl", getExtractorContentCountry().getCountryCode())
160-
.value("clientName", "WEB")
160+
.value("clientName", "1")
161161
.value("clientVersion", getClientVersion())
162162
.value("utcOffsetMinutes", 0)
163163
.end()
@@ -231,7 +231,7 @@ private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throw
231231
final String token = continuationItemRenderer.getObject("continuationEndpoint")
232232
.getObject("continuationCommand").getString("token");
233233

234-
final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
234+
final String url = "https://youtubei.googleapis.com/youtubei/v1/search?key=" + getKey();
235235

236236
return new Page(url, token);
237237
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,7 @@ public void testRelatedItems() throws Exception {
619619
public void testMoreRelatedItems() {
620620
try {
621621
defaultTestMoreItems(extractor);
622-
} catch (Throwable ignored) {
622+
} catch (final Throwable ignored) {
623623
return;
624624
}
625625

0 commit comments

Comments
 (0)