skip callbacks when a hoster is unreachable & cleanup/refactoring

crawlers/lib/crawl.py

@@ -6,11 +6,14 @@
 import uuid
 from typing import List, Generator
 from flask import current_app
+from requests import ConnectionError, Timeout, TooManyRedirects
 
 from crawlers.constants import BLOCK_KEY_CALLBACK_URL
 
 from crawlers.lib.platforms.i_crawler import ICrawler
 from crawlers.lib.platforms import platforms
+from urllib3.exceptions import MaxRetryError
+from requests.exceptions import RequestException
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +57,11 @@ def process_block_url(session, block_url) -> None:
             f"skip crawl - no callback_url found! - key: {BLOCK_KEY_CALLBACK_URL}, block_data: {block_data}"
         )
     else:
-        repos = run_block(block_data)
+        try:
+            repos = run_block(block_data)
+        except (MaxRetryError, ConnectionError, Timeout, TooManyRedirects):
+            logger.exception("hosting service not reachable - no indexer callback issued")
+            return
         _hoster_session_request(
             "PUT", session, url=block_data[BLOCK_KEY_CALLBACK_URL], json=repos
         )
@@ -76,8 +83,9 @@ def crawl(platform: ICrawler) -> Generator[List[dict], None, None]:
         else:
             # right now we dont want to emit failures (via yield) because that will send empty results back
             # to the indexer, which can trigger a state reset (i.e. reached end, start over).
-            # TODO deal with failures - what are they?
+            # - complete connection failures and such should be handled via raised exceptions within crawlers!
             pass
+
     logger.debug(f"END block: {platform.type} - final state: {platform.state}")
 
 

crawlers/lib/platforms/__init__.py

@@ -1,15 +1,14 @@
-from typing import Dict, Any, Type, Union
+from typing import Dict
 from crawlers.lib.platforms.i_crawler import ICrawler
 from crawlers.lib.platforms.gitea import GiteaCrawler
 from crawlers.lib.platforms.gitlab import GitLabCrawler
 from crawlers.lib.platforms.bitbucket import BitBucketCrawler
-from crawlers.lib.platforms.github import GitHubV4Crawler, GitHubRESTCrawler
+from crawlers.lib.platforms.github import GitHubV4Crawler
 
 platforms: Dict[str, ICrawler] = {
     GiteaCrawler.type: GiteaCrawler,
     GitLabCrawler.type: GitLabCrawler,
     GitHubV4Crawler.type: GitHubV4Crawler,
-    GitHubRESTCrawler.type: GitHubRESTCrawler,
     BitBucketCrawler.type: BitBucketCrawler,
 }
 

crawlers/lib/platforms/bitbucket.py

@@ -1,7 +1,7 @@
 import logging
 import time
 import requests
-from typing import List, Tuple
+from typing import List, Tuple, Union
 from urllib.parse import urljoin
 
 from crawlers.lib.platforms.i_crawler import ICrawler
@@ -65,7 +65,7 @@ def crawl(self, state: dict = None) -> Tuple[bool, List[dict], dict]:
                 logger.error(e)
                 logger.error(e.response.reason)
                 logger.error(e.response.text)
-                return False, [], {}
+                return False, [], {}, e
 
             response_json = response.json()
             repos = response_json['values']

crawlers/lib/platforms/gitea.py

@@ -1,5 +1,8 @@
 import logging
-from typing import List, Tuple
+from typing import List, Tuple, Union
+
+from requests import ConnectionError, Timeout, TooManyRedirects
+from urllib3.exceptions import MaxRetryError
 
 from crawlers.constants import GITEA_PER_PAGE_MAX, DEFAULT_REQUEST_TIMEOUT
 from crawlers.lib.platforms.i_crawler import ICrawler
@@ -40,6 +43,10 @@ def crawl(self, state: dict = None) -> Tuple[bool, List[dict], dict]:
                                    f"- response not ok, status: {response.status_code}")
                     return False, [], state  # nr.1 - we skip rest of this block, hope we get it next time
                 result = response.json()
+            except (MaxRetryError, ConnectionError, Timeout, TooManyRedirects) as e:
+                logger.exception(f"{self} - crawler cannot reach hoster")
+                # we re-raise these, as we want to avoid returning empty results to the indexer
+                raise e
             except Exception as e:
                 logger.exception(f"(skipping block chunk) gitea crawler crashed")
                 return False, [], state  # nr.2 - we skip rest of this block, hope we get it next time

crawlers/lib/platforms/github/__init__.py

@@ -1,2 +1 @@
 from .github_v4 import GitHubV4Crawler
-from .github_rest import GitHubRESTCrawler