Make RecordingDownloader more resillient against ReCaptchaExceptions

Implement a max number of requests per minute to prevent hitting reate limits and triggering ReCaptchaExceptions. This slows down the RecordingDownloader significantly and can be adjusted if needed. A request ist retried once when facing a ReCaptchaException.

Author: TobiGr

extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java

@@ -15,6 +15,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Random;
 
 import javax.annotation.Nonnull;
 
@@ -46,6 +47,18 @@ class RecordingDownloader extends Downloader {
     private int index = 0;
     private final String path;
 
+    // try to prevent ReCaptchaExceptions / rate limits by tracking and throttling the requests
+    /**
+     * The maximum number of requests per minutes which are executed
+     * by the {@link RecordingDownloader}.
+     * The values can be adjusted when executing the downloader and running into problems.
+     * <p>TODO: Allow adjusting the value by setting a param in the gradle command</p>
+     */
+    private static final int MAX_REQUESTS_PER_MINUTE = 30;
+    private static final long[] requestTimes = new long[MAX_REQUESTS_PER_MINUTE];
+    private static int requestTimesCursor = -1;
+    private static final Random throttleRandom = new Random();
+
     /**
      * Creates the folder described by {@code stringPath} if it does not exists.
      * Deletes existing files starting with {@link RecordingDownloader#FILE_NAME_PREFIX}.
@@ -69,6 +82,45 @@ public RecordingDownloader(final String stringPath) throws IOException {
     @Override
     public Response execute(@Nonnull final Request request) throws IOException,
             ReCaptchaException {
+
+        // Delay the execution if the max number of requests per minute is reached
+        final long currentTime = System.currentTimeMillis();
+        // the cursor points to the latest request time and the next position is the oldest one
+        final int oldestRequestTimeCursor = (requestTimesCursor + 1) % 12;
+        final long oldestRequestTime = requestTimes[oldestRequestTimeCursor];
+        if (oldestRequestTime + 60_000 >= currentTime) {
+            try {
+                // sleep at least until the oldest request is 60s old, but not more than 60s
+                final int minSleepTime = (int) (currentTime - oldestRequestTime);
+                Thread.sleep(minSleepTime + throttleRandom.nextInt(60_000 - minSleepTime));
+            } catch (InterruptedException e) {
+                System.err.println("Error while throttling the RecordingDownloader.");
+                e.printStackTrace();
+            }
+        }
+        requestTimesCursor = oldestRequestTimeCursor; // the oldest value needs to be overridden
+        requestTimes[requestTimesCursor] = System.currentTimeMillis();
+
+        // Handle ReCaptchaExceptions by retrying the request once after a while
+        try {
+            return executeRequest(request);
+        } catch (ReCaptchaException e) {
+            try {
+                System.out.println("Throttling the RecordingDownloader to handle a ReCaptchaException. Sleeping for 35-60 seconds.");
+                Thread.sleep(35_000 + throttleRandom.nextInt(25_000));
+            } catch (InterruptedException ie) {
+                System.err.println("Error while throttling the RecordingDownloader "
+                        + "to handle a ReCaptchaException");
+                ie.printStackTrace();
+                e.printStackTrace();
+            }
+            return executeRequest(request);
+        }
+    }
+
+    @Nonnull
+    private Response executeRequest(@Nonnull final Request request) throws IOException,
+            ReCaptchaException {
         final Downloader downloader = DownloaderTestImpl.getInstance();
         Response response = downloader.execute(request);
         String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");