fix: make the fetch order actually take effect, and quieten its failure paths

fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/DepthFirstUrlQueueOrder.java

@@ -27,6 +27,9 @@
  * <p>
  * The approximation is bounded by the polling fetch size: a batch is fetched, then fully
  * consumed before the next one, so URLs discovered mid-batch wait for the following batch.
+ * A crawl whose frontier fits in a single batch therefore proceeds level by level no matter
+ * which order is selected. Lowering the queue service's polling fetch size tightens the
+ * approximation, at the cost of one queue query per that many URLs.
  * </p>
  */
 public class DepthFirstUrlQueueOrder implements UrlQueueOrder {

fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/SequentialUrlQueueOrder.java

@@ -23,6 +23,14 @@
 
 /**
  * Fetches queued URLs by descending weight, then by discovery order. This is the default.
+ *
+ * <p>
+ * Weight is the primary key, so a {@code UrlQueueWeigher} takes effect under this order
+ * without any {@code crawl.order} setting; discovery order only decides between entries of
+ * equal weight. Weights are uniform out of the box, which leaves discovery order as the
+ * effective sort. Use {@link WeightFirstUrlQueueOrder} instead when entries of equal weight
+ * should not be held to discovery order.
+ * </p>
  */
 public class SequentialUrlQueueOrder implements UrlQueueOrder {
 

fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/WeightFirstUrlQueueOrder.java

@@ -23,6 +23,20 @@
 
 /**
  * Fetches queued URLs by descending weight only, leaving ties in the index order.
+ *
+ * <p>
+ * This differs from the default {@link SequentialUrlQueueOrder} only in what happens between
+ * entries of equal weight: that order falls back to discovery order, this one lets the search
+ * engine return them however it likes. Choose it when a queue carries a large backlog scored
+ * by a {@code UrlQueueWeigher} and only the score should decide what is crawled next, with no
+ * bias towards whatever was discovered first.
+ * </p>
+ *
+ * <p>
+ * Without a weigher every entry is at the default weight, every entry ties, and the fetch
+ * order is whatever the index hands back - so this order only means something once weights
+ * differ.
+ * </p>
  */
 public class WeightFirstUrlQueueOrder implements UrlQueueOrder {
 

fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java

@@ -75,8 +75,15 @@ public class OpenSearchUrlQueueService extends AbstractCrawlerService implements
 
     /**
      * The number of URLs to fetch when polling.
+     *
+     * <p>
+     * A batch is handed out in full before the queue is queried again, so this is also how
+     * often the {@link UrlQueueOrder} is re-evaluated. A large batch makes every order
+     * degenerate towards the discovery order, because the URLs found while a batch is being
+     * consumed cannot be considered until the next one.
+     * </p>
      */
-    protected int pollingFetchSize = 1000;
+    protected int pollingFetchSize = 100;
 
     /**
      * The maximum size of the crawling queue.

fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java

@@ -575,6 +575,44 @@ public void test_poll_followsTheConfiguredOrder() {
         }
     }
 
+    @Test
+    public void test_poll_reevaluatesTheOrderOncePerBatch() {
+        final String sessionId = "batch-session";
+        final long base = System.currentTimeMillis();
+        insertUrlQueue(sessionId, "http://www.example.com/shallow1", 1, base);
+        insertUrlQueue(sessionId, "http://www.example.com/shallow2", 1, base + 1L);
+
+        final int defaultFetchSize = urlQueueService.pollingFetchSize;
+        urlQueueService.setUrlQueueOrder(new DepthFirstUrlQueueOrder());
+        urlQueueService.setPollingFetchSize(2);
+        try {
+            // Both shallow URLs are fetched as one batch. Equal depth, so the newer wins.
+            assertEquals("http://www.example.com/shallow2", urlQueueService.poll(sessionId).getUrl());
+
+            // A deeper URL turns up while that batch is still being handed out.
+            insertUrlQueue(sessionId, "http://www.example.com/deep", 5, base + 2L);
+
+            // The batch is drained before the queue is consulted again, so the deeper URL
+            // waits even though the order asks for the deepest first.
+            assertEquals("http://www.example.com/shallow1", urlQueueService.poll(sessionId).getUrl());
+            assertEquals("http://www.example.com/deep", urlQueueService.poll(sessionId).getUrl());
+        } finally {
+            urlQueueService.setUrlQueueOrder(new SequentialUrlQueueOrder());
+            urlQueueService.setPollingFetchSize(defaultFetchSize);
+            urlQueueService.clearCache();
+        }
+    }
+
+    private void insertUrlQueue(final String sessionId, final String url, final int depth, final long createTime) {
+        final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
+        urlQueue.setSessionId(sessionId);
+        urlQueue.setUrl(url);
+        urlQueue.setCreateTime(createTime);
+        urlQueue.setDepth(depth);
+        urlQueue.setMethod("GET");
+        urlQueueService.insert(urlQueue);
+    }
+
     @Test
     public void test_di_registersTheBuiltInOrders() {
         for (final String name : new String[] { "sequentialUrlQueueOrder", "randomUrlQueueOrder", "depthFirstUrlQueueOrder",

fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java

@@ -473,8 +473,11 @@ protected void offerChildUrls(final List<UrlQueue<?>> childList) {
         try {
             urlQueueWeigher.apply(crawlerContext.sessionId, childList);
         } catch (final Exception e) {
-            logger.warn("Failed to apply weigher {} to child URLs. Falling back to inherited weights.",
-                    urlQueueWeigher.getClass().getName(), e);
+            logger.warn("Failed to apply weigher {} to {} child URL(s). Queueing them with whatever weights it left behind.",
+                    urlQueueWeigher.getClass().getName(), childList.size());
+            if (logger.isDebugEnabled()) {
+                logger.debug("Weigher {} failed.", urlQueueWeigher.getClass().getName(), e);
+            }
         }
         urlQueueService.offerAll(crawlerContext.sessionId, childList);
     }

fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java

@@ -279,8 +279,11 @@ protected void storeChildUrls(final CrawlerContext crawlerContext, final Set<Req
             try {
                 urlQueueWeigher.apply(crawlerContext.getSessionId(), childList);
             } catch (final Exception e) {
-                logger.warn("Failed to apply weigher {} to child URLs. Falling back to inherited weights.",
-                        urlQueueWeigher.getClass().getName(), e);
+                logger.warn("Failed to apply weigher {} to child URLs of {}. Queueing them with whatever weights it left behind.",
+                        urlQueueWeigher.getClass().getName(), url);
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Weigher {} failed.", urlQueueWeigher.getClass().getName(), e);
+                }
             }
             CrawlingParameterUtil.getUrlQueueService().offerAll(crawlerContext.getSessionId(), childList);
         }