* The approximation is bounded by the polling fetch size: a batch is fetched, then fully * consumed before the next one, so URLs discovered mid-batch wait for the following batch. + * A crawl whose frontier fits in a single batch therefore proceeds level by level no matter + * which order is selected. Lowering the queue service's polling fetch size tightens the + * approximation, at the cost of one queue query per that many URLs. *
*/ public class DepthFirstUrlQueueOrder implements UrlQueueOrder { diff --git a/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/SequentialUrlQueueOrder.java b/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/SequentialUrlQueueOrder.java index e32d81a6..fc478d5e 100644 --- a/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/SequentialUrlQueueOrder.java +++ b/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/SequentialUrlQueueOrder.java @@ -23,6 +23,14 @@ /** * Fetches queued URLs by descending weight, then by discovery order. This is the default. + * + *+ * Weight is the primary key, so a {@code UrlQueueWeigher} takes effect under this order + * without any {@code crawl.order} setting; discovery order only decides between entries of + * equal weight. Weights are uniform out of the box, which leaves discovery order as the + * effective sort. Use {@link WeightFirstUrlQueueOrder} instead when entries of equal weight + * should not be held to discovery order. + *
*/ public class SequentialUrlQueueOrder implements UrlQueueOrder { diff --git a/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/WeightFirstUrlQueueOrder.java b/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/WeightFirstUrlQueueOrder.java index e71b37ff..36f7cf83 100644 --- a/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/WeightFirstUrlQueueOrder.java +++ b/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/order/impl/WeightFirstUrlQueueOrder.java @@ -23,6 +23,20 @@ /** * Fetches queued URLs by descending weight only, leaving ties in the index order. + * + *+ * This differs from the default {@link SequentialUrlQueueOrder} only in what happens between + * entries of equal weight: that order falls back to discovery order, this one lets the search + * engine return them however it likes. Choose it when a queue carries a large backlog scored + * by a {@code UrlQueueWeigher} and only the score should decide what is crawled next, with no + * bias towards whatever was discovered first. + *
+ * + *+ * Without a weigher every entry is at the default weight, every entry ties, and the fetch + * order is whatever the index hands back - so this order only means something once weights + * differ. + *
*/ public class WeightFirstUrlQueueOrder implements UrlQueueOrder { diff --git a/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java b/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java index 11059853..491306c4 100644 --- a/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java +++ b/fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java @@ -75,8 +75,15 @@ public class OpenSearchUrlQueueService extends AbstractCrawlerService implements /** * The number of URLs to fetch when polling. + * + *+ * A batch is handed out in full before the queue is queried again, so this is also how + * often the {@link UrlQueueOrder} is re-evaluated. A large batch makes every order + * degenerate towards the discovery order, because the URLs found while a batch is being + * consumed cannot be considered until the next one. + *
*/ - protected int pollingFetchSize = 1000; + protected int pollingFetchSize = 100; /** * The maximum size of the crawling queue. diff --git a/fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java b/fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java index f8d8eba9..1d4b7c29 100644 --- a/fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java +++ b/fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java @@ -575,6 +575,44 @@ public void test_poll_followsTheConfiguredOrder() { } } + @Test + public void test_poll_reevaluatesTheOrderOncePerBatch() { + final String sessionId = "batch-session"; + final long base = System.currentTimeMillis(); + insertUrlQueue(sessionId, "http://www.example.com/shallow1", 1, base); + insertUrlQueue(sessionId, "http://www.example.com/shallow2", 1, base + 1L); + + final int defaultFetchSize = urlQueueService.pollingFetchSize; + urlQueueService.setUrlQueueOrder(new DepthFirstUrlQueueOrder()); + urlQueueService.setPollingFetchSize(2); + try { + // Both shallow URLs are fetched as one batch. Equal depth, so the newer wins. + assertEquals("http://www.example.com/shallow2", urlQueueService.poll(sessionId).getUrl()); + + // A deeper URL turns up while that batch is still being handed out. + insertUrlQueue(sessionId, "http://www.example.com/deep", 5, base + 2L); + + // The batch is drained before the queue is consulted again, so the deeper URL + // waits even though the order asks for the deepest first. + assertEquals("http://www.example.com/shallow1", urlQueueService.poll(sessionId).getUrl()); + assertEquals("http://www.example.com/deep", urlQueueService.poll(sessionId).getUrl()); + } finally { + urlQueueService.setUrlQueueOrder(new SequentialUrlQueueOrder()); + urlQueueService.setPollingFetchSize(defaultFetchSize); + urlQueueService.clearCache(); + } + } + + private void insertUrlQueue(final String sessionId, final String url, final int depth, final long createTime) { + final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue(); + urlQueue.setSessionId(sessionId); + urlQueue.setUrl(url); + urlQueue.setCreateTime(createTime); + urlQueue.setDepth(depth); + urlQueue.setMethod("GET"); + urlQueueService.insert(urlQueue); + } + @Test public void test_di_registersTheBuiltInOrders() { for (final String name : new String[] { "sequentialUrlQueueOrder", "randomUrlQueueOrder", "depthFirstUrlQueueOrder", diff --git a/fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java b/fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java index 70330080..f86d2075 100644 --- a/fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java +++ b/fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java @@ -473,8 +473,11 @@ protected void offerChildUrls(final List