Skip to content

Use BufferedReader#lines() #1246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
litetex merged 4 commits into from
Jan 27, 2025
Merged

Use BufferedReader#lines() #1246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0352659
Use BufferedReader#lines()
Isira-Seneviratne Nov 28, 2024
e475b29
Restore URL check
Isira-Seneviratne Dec 7, 2024
625e9f9
Remove public modifier from tests
litetex Jan 27, 2025
222d869
Polish: Use ternary instead of if
litetex Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 23 additions & 64 deletions ...rg/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

Expand Down Expand Up @@ -110,7 +113,7 @@ public List<SubscriptionItem> fromZipInputStream(@Nonnull final InputStream cont

// Return it only if it has items (it exits early if it's the wrong file
// format), otherwise try the next file
if (csvItems.size() > 0) {
if (!csvItems.isEmpty()) {
return csvItems;
}
} catch (final ExtractionException e) {
Expand Down Expand Up @@ -138,69 +141,25 @@ public List<SubscriptionItem> fromCsvInputStream(@Nonnull final InputStream cont
// The first line is always a header
// Header names are different based on the locale
// Fortunately the data is always the same order no matter what locale

int currentLine = 0;
String line = "";

try (BufferedReader br = new BufferedReader(new InputStreamReader(contentInputStream))) {
final List<SubscriptionItem> subscriptionItems = new ArrayList<>();

// ignore header and skip first line
currentLine = 1;
line = br.readLine();

while ((line = br.readLine()) != null) {
currentLine++;

// Exit early if we've read the first few lines and we haven't added any items
// It's likely we're in the wrong file
if (currentLine > 5 && subscriptionItems.size() == 0) {
break;
}

// First comma
final int i1 = line.indexOf(",");
if (i1 == -1) {
continue;
}

// Second comma
final int i2 = line.indexOf(",", i1 + 1);
if (i2 == -1) {
continue;
}

// Third comma or line length
int i3 = line.indexOf(",", i2 + 1);
if (i3 == -1) {
i3 = line.length();
}

// Channel URL from second entry
final String channelUrl = line
.substring(i1 + 1, i2)
.replace("http://", "https://");
if (!channelUrl.startsWith(BASE_CHANNEL_URL)) {
continue;
}

// Channel title from third entry
final String channelTitle = line.substring(i2 + 1, i3);

final SubscriptionItem newItem
= new SubscriptionItem(service.getServiceId(), channelUrl, channelTitle);
subscriptionItems.add(newItem);
}

return subscriptionItems;
} catch (final IOException e) {
if (line == null) {
line = "<null>";
} else if (line.length() > 10) {
line = line.substring(0, 10) + "...";
}
throw new InvalidSourceException("Error reading CSV file on line = \"" + line
+ "\", line number = " + currentLine, e);
try (var reader = new BufferedReader(new InputStreamReader(contentInputStream))) {
return reader.lines()
.skip(1) // ignore header and skip first line
.map(line -> line.split(","))
.filter(values -> values.length >= 3)
.map(values -> {
// Channel URL from second entry
final String channelUrl = values[1].replace("http://", "https://");
return channelUrl.startsWith(BASE_CHANNEL_URL)
? new SubscriptionItem(
service.getServiceId(),
channelUrl,
values[2]) // Channel title from third entry
: null;
})
.filter(Objects::nonNull)
.collect(Collectors.toUnmodifiableList());
} catch (final UncheckedIOException | IOException e) {
throw new InvalidSourceException("Error reading CSV file", e);
}
}
}
30 changes: 15 additions & 15 deletions .../java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@
/**
* Test for {@link YoutubeSubscriptionExtractor}
*/
public class YoutubeSubscriptionExtractorTest {

class YoutubeSubscriptionExtractorTest {

private static YoutubeSubscriptionExtractor subscriptionExtractor;
private static LinkHandlerFactory urlHandler;
Expand All @@ -41,7 +40,7 @@ public static void setupClass() {
}

@Test
public void testFromInputStream() throws Exception {
void testFromInputStream() throws Exception {
final List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(
new FileInputStream(resolveTestResource("youtube_takeout_import_test.json")));
assertEquals(7, subscriptionItems.size());
Expand All @@ -55,14 +54,14 @@ public void testFromInputStream() throws Exception {
}

@Test
public void testEmptySourceException() throws Exception {
void testEmptySourceException() throws Exception {
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
assertTrue(items.isEmpty());
}

@Test
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
void testSubscriptionWithEmptyTitleInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
Expand All @@ -74,7 +73,7 @@ public void testSubscriptionWithEmptyTitleInSource() throws Exception {
}

@Test
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
void testSubscriptionWithInvalidUrlInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
Expand All @@ -87,8 +86,8 @@ public void testSubscriptionWithInvalidUrlInSource() throws Exception {
}

@Test
public void testInvalidSourceException() {
List<String> invalidList = Arrays.asList(
void testInvalidSourceException() {
final List<String> invalidList = Arrays.asList(
"<xml><notvalid></notvalid></xml>",
"<opml><notvalid></notvalid></opml>",
"{\"a\":\"b\"}",
Expand All @@ -100,13 +99,14 @@ public void testInvalidSourceException() {
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
"gibberish");

for (String invalidContent : invalidList) {
for (final String invalidContent : invalidList) {
try {
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
final byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {
boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException;
final boolean correctType =
e instanceof SubscriptionExtractor.InvalidSourceException;
if (!correctType) {
e.printStackTrace();
}
Expand All @@ -117,7 +117,7 @@ public void testInvalidSourceException() {

private static void assertSubscriptionItems(final List<SubscriptionItem> subscriptionItems)
throws Exception {
assertTrue(subscriptionItems.size() > 0);
assertTrue(!subscriptionItems.isEmpty());

for (final SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
Expand All @@ -128,7 +128,7 @@ private static void assertSubscriptionItems(final List<SubscriptionItem> subscri
}

@Test
public void fromZipInputStream() throws Exception {
void fromZipInputStream() throws Exception {
final List<String> zipPaths = Arrays.asList(
"youtube_takeout_import_test_1.zip",
"youtube_takeout_import_test_2.zip"
Expand All @@ -144,13 +144,13 @@ public void fromZipInputStream() throws Exception {
}

@Test
public void fromCsvInputStream() throws Exception {
void fromCsvInputStream() throws Exception {
final List<String> csvPaths = Arrays.asList(
"youtube_takeout_import_test_1.csv",
"youtube_takeout_import_test_2.csv"
);

for (String path : csvPaths)
for (final String path : csvPaths)
{
final File file = resolveTestResource(path);
final FileInputStream fileInputStream = new FileInputStream(file);
Expand Down