Block the URLFrontier queue on Retry-After via the queue stream

core/src/main/java/org/apache/stormcrawler/util/RetryAfterParser.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.util;
+
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Parses the value of a <a
+ * href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After">Retry-After</a> HTTP
+ * response header, as sent by servers alongside a 429 (Too Many Requests) or 503 (Service
+ * Unavailable) response to request a back-off.
+ */
+public final class RetryAfterParser {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RetryAfterParser.class);
+
+    /**
+     * Formatter for the HTTP-date form of the Retry-After header, e.g. {@code Wed, 21 Oct 2015
+     * 07:28:00 GMT}. {@link DateTimeFormatter} is immutable and thread-safe, unlike {@code
+     * SimpleDateFormat}, which matters as this is shared across threads.
+     */
+    private static final DateTimeFormatter HTTP_DATE_FORMATTER =
+            DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ROOT)
+                    .withZone(ZoneOffset.UTC);
+
+    /** Pre-compiled matcher for the numeric (delta-seconds) form of the Retry-After header. */
+    private static final Pattern RETRY_AFTER_SECONDS = Pattern.compile("[0-9]+");
+
+    /**
+     * Upper bound on the length of a Retry-After header value we will attempt to parse. A
+     * delta-seconds value is at most 19 digits and an HTTP date ~29 chars; anything longer is
+     * rejected to bound the work done on this attacker-controlled value.
+     */
+    private static final int MAX_RETRY_AFTER_VALUE_LENGTH = 64;
+
+    private RetryAfterParser() {}
+
+    /**
+     * Parses a Retry-After header value into a delay expressed in milliseconds, relative to now.
+     * The value is either a number of seconds (e.g. {@code 120}) or an HTTP date (e.g. {@code Wed,
+     * 21 Oct 2015 07:28:00 GMT}).
+     *
+     * @return the delay in milliseconds (possibly {@code 0}), or {@code -1} if the header is
+     *     absent, malformed, in the past or implausibly long.
+     */
+    public static long parseDelay(String retryAfter) {
+        if (StringUtils.isBlank(retryAfter)) {
+            return -1;
+        }
+        retryAfter = retryAfter.trim();
+        // a valid Retry-After is either a delta-seconds value or an HTTP date;
+        // reject anything implausibly long to bound the work done on this
+        // attacker-controlled header value
+        if (retryAfter.length() > MAX_RETRY_AFTER_VALUE_LENGTH) {
+            return -1;
+        }
+        // delay expressed as a number of seconds
+        if (RETRY_AFTER_SECONDS.matcher(retryAfter).matches()) {
+            try {
+                return Math.multiplyExact(Long.parseLong(retryAfter), 1000L);
+            } catch (NumberFormatException | ArithmeticException e) {
+                // value too large to fit in a long, or its conversion to ms
+                // overflows - ignore
+                return -1;
+            }
+        }
+        // delay expressed as an HTTP date (IMF-fixdate form only, as produced by
+        // virtually all servers; the legacy RFC 850 / asctime forms are not accepted)
+        try {
+            Instant date = Instant.from(HTTP_DATE_FORMATTER.parse(retryAfter));
+            long delay = date.toEpochMilli() - System.currentTimeMillis();
+            return delay > 0 ? delay : -1;
+        } catch (DateTimeException e) {
+            LOG.debug("Invalid Retry-After header value: {}", retryAfter);
+            return -1;
+        }
+    }
+}

core/src/test/java/org/apache/stormcrawler/util/RetryAfterParserTest.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+
+/** Unit and randomised (fuzz) coverage for {@link RetryAfterParser#parseDelay(String)}. */
+class RetryAfterParserTest {
+
+    private static final DateTimeFormatter HTTP_DATE =
+            DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ROOT)
+                    .withZone(ZoneOffset.UTC);
+
+    @Test
+    void parseRetryAfterSeconds() {
+        assertEquals(120_000L, RetryAfterParser.parseDelay("120"));
+        // surrounding whitespace is tolerated
+        assertEquals(120_000L, RetryAfterParser.parseDelay("  120 "));
+        // zero is a valid (if pointless) delay and is not treated as "absent"
+        assertEquals(0L, RetryAfterParser.parseDelay("0"));
+    }
+
+    @Test
+    void parseRetryAfterFutureDate() {
+        Instant future = Instant.now().plusSeconds(300);
+        long delay = RetryAfterParser.parseDelay(HTTP_DATE.format(future));
+        // allow a small tolerance for the time elapsed during the call
+        assertTrue(delay > 250_000 && delay <= 300_000, "delay was " + delay);
+    }
+
+    @Test
+    void parseRetryAfterPastDateReturnsMinusOne() {
+        Instant past = Instant.now().minusSeconds(300);
+        assertEquals(-1L, RetryAfterParser.parseDelay(HTTP_DATE.format(past)));
+    }
+
+    @Test
+    void parseRetryAfterBlankOrMalformedReturnsMinusOne() {
+        assertEquals(-1L, RetryAfterParser.parseDelay(null));
+        assertEquals(-1L, RetryAfterParser.parseDelay(""));
+        assertEquals(-1L, RetryAfterParser.parseDelay("   "));
+        assertEquals(-1L, RetryAfterParser.parseDelay("not-a-date"));
+        assertEquals(-1L, RetryAfterParser.parseDelay("12.5"));
+        assertEquals(-1L, RetryAfterParser.parseDelay("-5"));
+    }
+
+    @Test
+    void parseRetryAfterOverlongValueReturnsMinusOne() {
+        // an implausibly long header value is rejected without further parsing
+        String tooLong = "1".repeat(65);
+        assertEquals(-1L, RetryAfterParser.parseDelay(tooLong));
+    }
+
+    @Test
+    void parseRetryAfterParseOverflowReturnsMinusOne() {
+        // too large to fit in a long at all
+        assertEquals(-1L, RetryAfterParser.parseDelay("999999999999999999999999999"));
+    }
+
+    @Test
+    void parseRetryAfterMillisecondOverflowReturnsMinusOne() {
+        // parses as a valid long, but * 1000 overflows a long (regression: was a
+        // silent overflow yielding a negative delay)
+        assertEquals(-1L, RetryAfterParser.parseDelay("3687950425865536959"));
+        assertEquals(-1L, RetryAfterParser.parseDelay(Long.toString(Long.MAX_VALUE)));
+    }
+
+    @Test
+    void parseRetryAfterRandomSecondsRoundTrip() {
+        Random rnd = new Random(7);
+        for (int i = 0; i < 1_000; i++) {
+            long secs = rnd.nextInt(1_000_000);
+            assertEquals(secs * 1000L, RetryAfterParser.parseDelay(Long.toString(secs)));
+        }
+    }
+
+    @Test
+    void parseRetryAfterNeverThrowsForRandomInput() {
+        Random rnd = new Random(42);
+        for (int i = 0; i < 10_000; i++) {
+            String input = randomRetryAfterValue(rnd);
+            long result = RetryAfterParser.parseDelay(input);
+            // contract: never throws, never returns anything below -1
+            assertTrue(result >= -1L, "value " + result + " for input <" + input + ">");
+        }
+    }
+
+    /**
+     * Produces a wide spectrum of inputs: digit runs (including values that overflow a long or
+     * overflow when converted to ms), real HTTP dates (past and future), blanks, random printable
+     * garbage and nulls.
+     */
+    private static String randomRetryAfterValue(Random rnd) {
+        switch (rnd.nextInt(5)) {
+            case 0:
+                int len = 1 + rnd.nextInt(40);
+                StringBuilder digits = new StringBuilder();
+                for (int i = 0; i < len; i++) {
+                    digits.append((char) ('0' + rnd.nextInt(10)));
+                }
+                return digits.toString();
+            case 1:
+                long offset = (rnd.nextBoolean() ? 1L : -1L) * rnd.nextInt(1_000_000);
+                return HTTP_DATE.format(Instant.now().plusSeconds(offset));
+            case 2:
+                return "";
+            case 3:
+                int glen = rnd.nextInt(30);
+                StringBuilder garbage = new StringBuilder();
+                for (int i = 0; i < glen; i++) {
+                    garbage.append((char) (32 + rnd.nextInt(95)));
+                }
+                return garbage.toString();
+            default:
+                return null;
+        }
+    }
+}

external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java

@@ -43,4 +43,12 @@ private Constants() {}
     public static final String URLFRONTIER_UPDATER_MAX_MESSAGES_KEY =
             "urlfrontier.updater.max.messages";
     public static final String URLFRONTIER_CRAWL_ID_KEY = "urlfrontier.crawlid";
+
+    /**
+     * Maximum delay in seconds honoured when a server requests a back-off via the Retry-After HTTP
+     * response header. {@code -1} disables the cap. Defaults to 86400 (24h).
+     */
+    public static final String URLFRONTIER_MAX_RETRY_AFTER_KEY = "urlfrontier.max.retry.after";
+
+    public static final int URLFRONTIER_MAX_RETRY_AFTER_DEFAULT = 86400;
 }