Skip to content

Commit 4ca4b83

Browse files
AudricVAudricV
committed
Adress some requested changes + extract correctly PeerTube audio streams
1 parent 18d3e46 commit 4ca4b83

2 files changed

Lines changed: 146 additions & 38 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java‎

Lines changed: 134 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
3939
private JsonObject json;
4040
private final List<SubtitlesStream> subtitles = new ArrayList<>();
4141

42-
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException {
42+
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler)
43+
throws ParsingException {
4344
super(service, linkHandler);
4445
this.baseUrl = getBaseUrl();
4546
}
@@ -72,20 +73,19 @@ public Description getDescription() throws ParsingException {
7273
String text;
7374
try {
7475
text = JsonUtils.getString(json, "description");
75-
} catch (ParsingException e) {
76+
} catch (final ParsingException e) {
7677
return Description.emptyDescription;
7778
}
7879
if (text.length() == 250 && text.substring(247).equals("...")) {
79-
//if description is shortened, get full description
80+
// If description is shortened, get full description
8081
final Downloader dl = NewPipe.getDownloader();
8182
try {
8283
final Response response = dl.get(baseUrl
8384
+ PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
8485
+ getId() + "/description");
8586
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
8687
text = JsonUtils.getString(jsonObject, "description");
87-
} catch (ReCaptchaException | IOException | JsonParserException e) {
88-
e.printStackTrace();
88+
} catch (final IOException | ReCaptchaException | JsonParserException ignored) {
8989
}
9090
}
9191
return new Description(text, Description.MARKDOWN);
@@ -108,8 +108,8 @@ public long getLength() {
108108

109109
@Override
110110
public long getTimeStamp() throws ParsingException {
111-
final long timestamp =
112-
getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
111+
final long timestamp = getTimestampSeconds(
112+
"((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
113113

114114
if (timestamp == -2) {
115115
// regex for timestamp was not found
@@ -139,7 +139,8 @@ public long getDislikeCount() {
139139
public String getUploaderUrl() throws ParsingException {
140140
final String name = JsonUtils.getString(json, "account.name");
141141
final String host = JsonUtils.getString(json, "account.host");
142-
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
142+
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl)
143+
.getUrl();
143144
}
144145

145146
@Nonnull
@@ -159,7 +160,7 @@ public String getUploaderAvatarUrl() {
159160
String value;
160161
try {
161162
value = JsonUtils.getString(json, "account.avatar.path");
162-
} catch (Exception e) {
163+
} catch (final Exception e) {
163164
value = "/client/assets/images/default-avatar.png";
164165
}
165166
return baseUrl + value;
@@ -183,7 +184,7 @@ public String getSubChannelAvatarUrl() {
183184
String value;
184185
try {
185186
value = JsonUtils.getString(json, "channel.avatar.path");
186-
} catch (Exception e) {
187+
} catch (final Exception e) {
187188
value = "/client/assets/images/default-avatar.png";
188189
}
189190
return baseUrl + value;
@@ -192,29 +193,111 @@ public String getSubChannelAvatarUrl() {
192193
@Nonnull
193194
@Override
194195
public String getDashMpdUrl() {
195-
return "";
196+
return EMPTY_STRING;
196197
}
197198

198199
@Nonnull
199200
@Override
200201
public String getHlsUrl() {
202+
assertPageFetched();
201203
return json.getObject("files").getString("playlistUrl", EMPTY_STRING);
202204
}
203205

204206
@Override
205-
public List<AudioStream> getAudioStreams() {
206-
return Collections.emptyList();
207+
public List<AudioStream> getAudioStreams() throws ParsingException {
208+
assertPageFetched();
209+
final List<AudioStream> audioStreams = new ArrayList<>();
210+
// Non-HLS streams
211+
try {
212+
audioStreams.addAll(getAudioStreamsFromArray(json.getArray("files"), ""));
213+
} catch (final Exception ignored) {
214+
}
215+
// HLS streams
216+
try {
217+
final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
218+
for (final Object p : streamingPlaylists) {
219+
if (!(p instanceof JsonObject)) continue;
220+
final JsonObject playlist = (JsonObject) p;
221+
final String playlistUrl = playlist.getString("playlistUrl");
222+
audioStreams.addAll(getAudioStreamsFromArray(playlist.getArray("files"),
223+
playlistUrl));
224+
}
225+
} catch (final Exception e) {
226+
throw new ParsingException("Could not get video streams", e);
227+
}
228+
229+
return audioStreams;
230+
}
231+
232+
@Nonnull
233+
private List<AudioStream> getAudioStreamsFromArray(final JsonArray streams,
234+
final String playlistUrl)
235+
throws ParsingException {
236+
try {
237+
final List<AudioStream> audioStreams = new ArrayList<>();
238+
for (final Object s : streams) {
239+
if (!(s instanceof JsonObject)) continue;
240+
final JsonObject stream = (JsonObject) s;
241+
final String url;
242+
final String idSuffix;
243+
if (stream.has("fileDownloadUrl")) {
244+
url = JsonUtils.getString(stream, "fileDownloadUrl");
245+
idSuffix = "fileDownloadUrl";
246+
} else {
247+
url = JsonUtils.getString(stream, "fileUrl");
248+
idSuffix = "fileUrl";
249+
}
250+
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
251+
final String resolution = JsonUtils.getString(stream, "resolution.label");
252+
if (!resolution.equalsIgnoreCase("Audio")) {
253+
// Not an audio stream
254+
continue;
255+
}
256+
final String extension = url.substring(url.lastIndexOf(".") + 1);
257+
final MediaFormat format = MediaFormat.getFromSuffix(extension);
258+
final String id = resolution + "-" + extension;
259+
audioStreams.add(new AudioStream(
260+
id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP,
261+
url,
262+
true,
263+
format,
264+
DeliveryMethod.PROGRESSIVE_HTTP,
265+
-1));
266+
audioStreams.add(new AudioStream(
267+
id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT,
268+
torrentUrl,
269+
true,
270+
format,
271+
DeliveryMethod.TORRENT,
272+
-1));
273+
if (!isNullOrEmpty(playlistUrl)) {
274+
final String hlsStreamUrl = playlistUrl.replace("master",
275+
JsonUtils.getNumber(stream, "resolution.id").toString());
276+
audioStreams.add(new AudioStream(
277+
id + "-" + DeliveryMethod.HLS,
278+
hlsStreamUrl,
279+
true,
280+
format,
281+
DeliveryMethod.HLS,
282+
-1));
283+
}
284+
}
285+
return audioStreams;
286+
} catch (final Exception e) {
287+
throw new ParsingException("Could not get audio streams from array", e);
288+
}
207289
}
208290

209291
@Override
210292
public List<VideoStream> getVideoStreams() throws ExtractionException {
211293
assertPageFetched();
212294
final List<VideoStream> videoStreams = new ArrayList<>();
213-
// mp4
295+
// Non-HLS streams
214296
try {
215297
videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files"), ""));
216-
} catch (Exception ignored) { }
217-
// HLS
298+
} catch (final Exception ignored) {
299+
}
300+
// HLS streams
218301
try {
219302
final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
220303
for (final Object p : streamingPlaylists) {
@@ -224,7 +307,7 @@ public List<VideoStream> getVideoStreams() throws ExtractionException {
224307
videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files"),
225308
playlistUrl));
226309
}
227-
} catch (Exception e) {
310+
} catch (final Exception e) {
228311
throw new ParsingException("Could not get video streams", e);
229312
}
230313

@@ -237,7 +320,6 @@ private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams,
237320
throws ParsingException {
238321
try {
239322
final List<VideoStream> videoStreams = new ArrayList<>();
240-
final boolean playlistUrlNullOrEmpty = isNullOrEmpty(playlistUrl);
241323
for (final Object s : streams) {
242324
if (!(s instanceof JsonObject)) continue;
243325
final JsonObject stream = (JsonObject) s;
@@ -252,6 +334,10 @@ private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams,
252334
}
253335
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
254336
final String resolution = JsonUtils.getString(stream, "resolution.label");
337+
if (resolution.equalsIgnoreCase("Audio")) {
338+
// Not a video stream
339+
continue;
340+
}
255341
final String extension = url.substring(url.lastIndexOf(".") + 1);
256342
final MediaFormat format = MediaFormat.getFromSuffix(extension);
257343
final String id = resolution + "-" + extension;
@@ -273,7 +359,7 @@ private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams,
273359
resolution,
274360
false,
275361
null));
276-
if (!playlistUrlNullOrEmpty) {
362+
if (!isNullOrEmpty(playlistUrl)) {
277363
final String hlsStreamUrl = playlistUrl.replace("master",
278364
JsonUtils.getNumber(stream, "resolution.id").toString());
279365
videoStreams.add(new VideoStream(
@@ -337,7 +423,8 @@ public StreamInfoItemsCollector getRelatedItems() throws IOException, Extraction
337423
if (Utils.isBlank(apiUrl)) {
338424
return null;
339425
} else {
340-
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
426+
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
427+
getServiceId());
341428
getStreamsFromApi(collector, apiUrl);
342429
return collector;
343430
}
@@ -354,8 +441,8 @@ public List<String> getTags() {
354441
public String getSupportInfo() {
355442
try {
356443
return JsonUtils.getString(json, "support");
357-
} catch (ParsingException e) {
358-
return "";
444+
} catch (final ParsingException e) {
445+
return EMPTY_STRING;
359446
}
360447
}
361448

@@ -371,24 +458,27 @@ public List<MetaInfo> getMetaInfo() {
371458
return Collections.emptyList();
372459
}
373460

374-
private String getRelatedItemsUrl(final List<String> tags) throws UnsupportedEncodingException {
461+
@Nonnull
462+
private String getRelatedItemsUrl(@Nonnull final List<String> tags)
463+
throws UnsupportedEncodingException {
375464
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
376465
final StringBuilder params = new StringBuilder();
377466
params.append("start=0&count=8&sort=-createdAt");
378467
for (final String tag : tags) {
379468
params.append("&tagsOneOf=");
380469
params.append(URLEncoder.encode(tag, UTF_8));
381470
}
382-
return url + "?" + params.toString();
471+
return url + "?" + params;
383472
}
384473

385-
private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl) throws ReCaptchaException, IOException, ParsingException {
474+
private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl)
475+
throws IOException, ReCaptchaException, ParsingException {
386476
final Response response = getDownloader().get(apiUrl);
387477
JsonObject relatedVideosJson = null;
388478
if (response != null && !Utils.isBlank(response.responseBody())) {
389479
try {
390480
relatedVideosJson = JsonParser.object().from(response.responseBody());
391-
} catch (JsonParserException e) {
481+
} catch (final JsonParserException e) {
392482
throw new ParsingException("Could not parse json data for related videos", e);
393483
}
394484
}
@@ -398,20 +488,24 @@ private void getStreamsFromApi(final StreamInfoItemsCollector collector, final S
398488
}
399489
}
400490

401-
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject json) throws ParsingException {
491+
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
492+
final JsonObject json) throws ParsingException {
402493
final JsonArray contents;
403494
try {
404495
contents = (JsonArray) JsonUtils.getValue(json, "data");
405-
} catch (Exception e) {
406-
throw new ParsingException("unable to extract related videos", e);
496+
} catch (final Exception e) {
497+
throw new ParsingException("Could not extract related videos", e);
407498
}
408499

409500
for (final Object c : contents) {
410501
if (c instanceof JsonObject) {
411502
final JsonObject item = (JsonObject) c;
412-
final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
503+
final PeertubeStreamInfoItemExtractor extractor =
504+
new PeertubeStreamInfoItemExtractor(item, baseUrl);
413505
// Do not add the same stream in related streams
414-
if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
506+
if (!extractor.getUrl().equals(getUrl())) {
507+
collector.commit(extractor);
508+
}
415509
}
416510
}
417511
}
@@ -422,12 +516,14 @@ public String getErrorMessage() {
422516
}
423517

424518
@Override
425-
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
426-
final Response response = downloader.get(baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId());
519+
public void onFetchPage(@Nonnull final Downloader downloader)
520+
throws IOException, ExtractionException {
521+
final Response response = downloader.get(baseUrl
522+
+ PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId());
427523
if (response != null) {
428524
setInitialData(response.responseBody());
429525
} else {
430-
throw new ExtractionException("Unable to extract PeerTube channel data");
526+
throw new ExtractionException("Could not extract PeerTube channel data");
431527
}
432528

433529
loadSubtitles();
@@ -436,11 +532,11 @@ public void onFetchPage(final Downloader downloader) throws IOException, Extract
436532
private void setInitialData(final String responseBody) throws ExtractionException {
437533
try {
438534
json = JsonParser.object().from(responseBody);
439-
} catch (JsonParserException e) {
440-
throw new ExtractionException("Unable to extract PeerTube stream data", e);
535+
} catch (final JsonParserException e) {
536+
throw new ExtractionException("Could not extract PeerTube stream data", e);
441537
}
442538
if (json == null) {
443-
throw new ExtractionException("Unable to extract PeerTube stream data");
539+
throw new ExtractionException("Could not extract PeerTube stream data");
444540
}
445541
PeertubeParsingHelper.validate(json);
446542
}
@@ -515,7 +611,7 @@ public String getLicence() throws ParsingException {
515611
public Locale getLanguageInfo() {
516612
try {
517613
return new Locale(JsonUtils.getString(json, "language.id"));
518-
} catch (ParsingException e) {
614+
} catch (final ParsingException e) {
519615
return null;
520616
}
521617
}

0 commit comments

Comments
 (0)