Skip to content

[YouTube] Fix serialization of Videos channel tab when it is already fetched #1222

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AudricV merged 1 commit into from
Sep 29, 2024
Merged

[YouTube] Fix serialization of Videos channel tab when it is already fetched #1222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 25 additions & 23 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.Optional;

Expand Down Expand Up @@ -233,7 +234,7 @@ private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject json
* properties.
* </p>
*/
public static final class ChannelHeader {
public static final class ChannelHeader implements Serializable {

/**
* Types of supported YouTube channel headers.
Expand Down Expand Up @@ -294,27 +295,27 @@ public enum HeaderType {
*/
public final HeaderType headerType;

private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
public ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
this.json = json;
this.headerType = headerType;
}
}

/**
* Get a channel header as an {@link Optional} it if exists.
* Get a channel header it if exists.
*
* @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
* if no supported header has been found
* @return a {@link ChannelHeader} or {@code null} if no supported header has been found
*/
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nullable
public static ChannelHeader getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject(HEADER);

if (header.has(C4_TABBED_HEADER_RENDERER)) {
return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED))
.orElse(null);
} else if (header.has(CAROUSEL_HEADER_RENDERER)) {
return header.getObject(CAROUSEL_HEADER_RENDERER)
.getArray(CONTENTS)
Expand All @@ -324,17 +325,20 @@ public static Optional<ChannelHeader> getChannelHeader(
.filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
.findFirst()
.map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL))
.orElse(null);
} else if (header.has("pageHeaderRenderer")) {
return Optional.of(header.getObject("pageHeaderRenderer"))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE))
.orElse(null);
} else if (header.has("interactiveTabbedHeaderRenderer")) {
return Optional.of(header.getObject("interactiveTabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json,
ChannelHeader.HeaderType.INTERACTIVE_TABBED));
} else {
return Optional.empty();
ChannelHeader.HeaderType.INTERACTIVE_TABBED))
.orElse(null);
}

return null;
}

/**
Expand Down Expand Up @@ -418,20 +422,18 @@ public static boolean isChannelVerified(@Nonnull final ChannelHeader channelHead
* If the ID cannot still be get, the fallback channel ID, if provided, will be used.
* </p>
*
* @param header the channel header
* @param channelHeader the channel header
* @param fallbackChannelId the fallback channel ID, which can be null
* @return the ID of the channel
* @throws ParsingException if the channel ID cannot be got from the channel header, the
* channel response and the fallback channel ID
*/
@Nonnull
public static String getChannelId(
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> header,
@Nullable final ChannelHeader channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final String fallbackChannelId) throws ParsingException {
if (header.isPresent()) {
final ChannelHeader channelHeader = header.get();
if (channelHeader != null) {
switch (channelHeader.headerType) {
case C4_TABBED:
final String channelId = channelHeader.json.getObject(HEADER)
Expand Down Expand Up @@ -486,10 +488,9 @@ public static String getChannelId(
}

@Nonnull
public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final JsonObject channelAgeGateRenderer)
public static String getChannelName(@Nullable final ChannelHeader channelHeader,
@Nullable final JsonObject channelAgeGateRenderer,
@Nonnull final JsonObject jsonResponse)
throws ParsingException {
if (channelAgeGateRenderer != null) {
final String title = channelAgeGateRenderer.getString("channelTitle");
Expand All @@ -506,7 +507,8 @@ public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrPara
return metadataRendererTitle;
}

return channelHeader.map(header -> {
return Optional.ofNullable(channelHeader)
.map(header -> {
final JsonObject channelJson = header.json;
switch (header.headerType) {
case PAGE:
Expand Down
135 changes: 74 additions & 61 deletions ...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {

private JsonObject jsonResponse;

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<ChannelHeader> channelHeader;
@Nullable
private ChannelHeader channelHeader;

private String channelId;

Expand Down Expand Up @@ -132,7 +132,7 @@ public String getId() throws ParsingException {
public String getName() throws ParsingException {
assertPageFetched();
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse, channelAgeGateRenderer);
channelHeader, channelAgeGateRenderer, jsonResponse);
}

@Nonnull
Expand All @@ -146,40 +146,40 @@ public List<Image> getAvatars() throws ParsingException {
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}

return channelHeader.map(header -> {
switch (header.headerType) {
case PAGE:
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);

if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
}

if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
return Optional.ofNullable(channelHeader)
.map(header -> {
switch (header.headerType) {
case PAGE:
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);

if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
}

if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}

// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}

// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);

case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}
})
})
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}
Expand All @@ -192,7 +192,8 @@ public List<Image> getBanners() {
return List.of();
}

return channelHeader.map(header -> {
return Optional.ofNullable(channelHeader)
.map(header -> {
if (header.headerType == HeaderType.PAGE) {
final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);
Expand Down Expand Up @@ -235,16 +236,14 @@ public long getSubscriberCount() throws ParsingException {
return UNKNOWN_SUBSCRIBER_COUNT;
}

if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();

if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
if (channelHeader != null) {
if (channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
// No subscriber count is available on interactiveTabbedHeaderRenderer header
return UNKNOWN_SUBSCRIBER_COUNT;
}

final JsonObject headerJson = header.json;
if (header.headerType == HeaderType.PAGE) {
final JsonObject headerJson = channelHeader.json;
if (channelHeader.headerType == HeaderType.PAGE) {
return getSubscriberCountFromPageChannelHeader(headerJson);
}

Expand Down Expand Up @@ -321,19 +320,17 @@ public String getDescription() throws ParsingException {
}

try {
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(header.json.getObject("description"));
}
if (channelHeader != null
&& channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(channelHeader.json.getObject("description"));
}

return jsonResponse.getObject(METADATA)
Expand Down Expand Up @@ -368,8 +365,12 @@ public boolean isVerified() throws ParsingException {
return false;
}

return YoutubeChannelHelper.isChannelVerified(channelHeader.orElseThrow(() ->
new ParsingException("Could not get verified status")));
if (channelHeader == null) {
throw new ParsingException(
"Could not get channel verified status, no channel header has been extracted");
}

return YoutubeChannelHelper.isChannelVerified(channelHeader);
}

@Nonnull
Expand Down Expand Up @@ -421,6 +422,19 @@ private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws Parsin

final String urlSuffix = urlParts[urlParts.length - 1];

/*
Make a copy of the channelHeader member to avoid keeping a reference to
this YoutubeChannelExtractor instance which would prevent serialization of
the ReadyChannelTabListLinkHandler instance created above
*/
final ChannelHeader channelHeaderCopy;
if (channelHeader == null) {
channelHeaderCopy = null;
} else {
channelHeaderCopy = new ChannelHeader(channelHeader.json,
channelHeader.headerType);
}

switch (urlSuffix) {
case "videos":
// Since the Videos tab has already its contents fetched, make
Expand All @@ -431,9 +445,8 @@ private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws Parsin
channelId,
ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, channelHeader,
name, id, url)));

service, linkHandler, tabRenderer,
channelHeaderCopy, name, id, url)));
break;
case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS);
Expand Down
27 changes: 15 additions & 12 deletions .../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,11 @@
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {

@Nullable
protected YoutubeChannelHelper.ChannelHeader channelHeader;

private JsonObject jsonResponse;
private String channelId;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
protected Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;

public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
Expand Down Expand Up @@ -104,9 +105,9 @@ public String getId() throws ParsingException {
}

protected String getChannelName() throws ParsingException {
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse));
return YoutubeChannelHelper.getChannelName(channelHeader,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse),
jsonResponse);
}

@Nonnull
Expand Down Expand Up @@ -140,11 +141,14 @@ public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionEx
}
}

final VerifiedStatus verifiedStatus = channelHeader.flatMap(header ->
YoutubeChannelHelper.isChannelVerified(header)
? Optional.of(VerifiedStatus.VERIFIED)
: Optional.of(VerifiedStatus.UNVERIFIED))
.orElse(VerifiedStatus.UNKNOWN);
final VerifiedStatus verifiedStatus;
if (channelHeader == null) {
verifiedStatus = VerifiedStatus.UNKNOWN;
} else {
verifiedStatus = YoutubeChannelHelper.isChannelVerified(channelHeader)
? VerifiedStatus.VERIFIED
: VerifiedStatus.UNVERIFIED;
}

// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
Expand Down Expand Up @@ -462,8 +466,7 @@ public static final class VideosTabExtractor extends YoutubeChannelTabExtractor
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
final Optional<YoutubeChannelHelper.ChannelHeader> channelHeader,
@Nullable final YoutubeChannelHelper.ChannelHeader channelHeader,
final String channelName,
final String channelId,
final String channelUrl) {
Expand Down