Skip to content

Get IOS HLS manifest through async means #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
FireMasterK wants to merge 1 commit into from
Closed

Get IOS HLS manifest through async means #732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,6 +849,28 @@ public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
// @formatter:on
}

@Nonnull
public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "IOS")
.value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}

@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopEmbedVideoJsonBuilder(
@Nonnull final Localization localization,
Expand Down
73 changes: 50 additions & 23 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,16 @@
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;

import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;

import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
import org.schabi.newpipe.extractor.exceptions.PaidContentException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.exceptions.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.localization.*;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
Expand All @@ -44,11 +31,11 @@
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.*;

/*
* Created by Christian Schabesberger on 06.08.15.
Expand Down Expand Up @@ -97,6 +84,7 @@ public static class DeobfuscateException extends ParsingException {
private JsonObject desktopStreamingData;
@Nullable
private JsonObject mobileStreamingData;
private JsonObject mobileIosStreamingData;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
Expand Down Expand Up @@ -504,13 +492,19 @@ public String getDashMpdUrl() throws ParsingException {
public String getHlsUrl() throws ParsingException {
assertPageFetched();

String hlsManifestUrl = EMPTY_STRING;

if (desktopStreamingData != null) {
return desktopStreamingData.getString("hlsManifestUrl");
} else if (mobileStreamingData != null) {
return mobileStreamingData.getString("hlsManifestUrl");
} else {
return EMPTY_STRING;
hlsManifestUrl = desktopStreamingData.getString("hlsManifestUrl", EMPTY_STRING);
}
if (mobileStreamingData != null && hlsManifestUrl.isEmpty()) {
Comment on lines +499 to +500
Copy link
Copy Markdown
Member

@Stypox Stypox Nov 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These could as well be else ifs, can't they?

hlsManifestUrl = mobileStreamingData.getString("hlsManifestUrl", EMPTY_STRING);
}
if (mobileIosStreamingData != null && hlsManifestUrl.isEmpty()) {
hlsManifestUrl = mobileIosStreamingData.getString("hlsManifestUrl", EMPTY_STRING);
}

return hlsManifestUrl;
}

@Override
Expand Down Expand Up @@ -719,6 +713,13 @@ public void onFetchPage(@Nonnull final Downloader downloader)
.done())
.getBytes(UTF_8);

CompletableFuture iosTask = CompletableFuture.supplyAsync(() -> {
Copy link
Copy Markdown
Member

@Stypox Stypox Nov 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure this is a good solution: what if the task hasn't finished when getHlsUrl() is called? If the extractor was all built with async means it would integrate good, but in this case there is nothing waiting for the async task to complete. Also, futures are not used anywhere else in the extractor, by doing a fast search.

try {
fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) { }
return null;
});

// Put the sts string if we already know it so we don't have to fetch again the player
// endpoint of the desktop internal API if something went wrong when parsing the Android
// API.
Expand Down Expand Up @@ -783,6 +784,10 @@ public void onFetchPage(@Nonnull final Downloader downloader)
if (isCipherProtectedContent()) {
fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId);
}

try {
iosTask.get();
} catch (InterruptedException | ExecutionException ignored) { }
}

private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
Expand Down Expand Up @@ -864,6 +869,28 @@ private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry,
}
}

/**
* Fetch the IOS Mobile API and assign the streaming data to the mobileStreamingData JSON
* object.
*/
private void fetchIosMobileJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
throws IOException, ExtractionException {
final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder(
localization, contentCountry)
.value("videoId", videoId)
.done())
.getBytes(UTF_8);
final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player",
mobileBody, contentCountry, localization);

final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData");
if (!isNullOrEmpty(streamingData)) {
mobileIosStreamingData = streamingData;
}
}

/**
* Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to
* the {@code desktopStreamingData} JSON object.
Expand Down
20 changes: 20 additions & 0 deletions ...g/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,4 +470,24 @@ public void testGetLicence() throws ParsingException {
assertEquals("Creative Commons Attribution licence (reuse allowed)", extractor.getLicence());
}
}

public static class IosHlsManifest {
private static final String ID = "M4gD1WSo5mA";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;

@BeforeClass
public static void setUp() throws Exception {
YoutubeStreamExtractor.resetDeobfuscationCode();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ios_hls"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}

@Test
public void testGetHlsManifest() throws ParsingException {
assertNotNull(extractor.getHlsUrl());
}
}
}
Loading