Add celery config to adjust retry settings

docker-compose-dev.yml

@@ -23,7 +23,7 @@ services:
   redis:
     image: redis:latest
     ports:
-      - "6379:6379"
+      - "6355:6379"
 
 
   # Used for formatting the volume as ext4. Otherwise, some minio operations may fail.

rainfall/celeryconfig.py

@@ -0,0 +1,10 @@
+import os
+
+broker_url = os.environ['REDIS_URL'],
+result_backend = os.environ['REDIS_URL']
+# Don't use a pool, connect every time. There seems to be an issue with
+# connections to Redis getting stale.
+broker_pool_limit = None
+# Retry forever if Redis is unreachable.
+broker_connection_max_retries = None
+broker_connection_retry_on_startup = True

rainfall/main.py

@@ -23,9 +23,8 @@
 log = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
-task_app = Celery('tasks',
-                  broker_url=os.environ['REDIS_URL'],
-                  result_backend=os.environ['REDIS_URL'])
+task_app = Celery('tasks')
+task_app.config_from_object('rainfall.celeryconfig')
 
 
 @task_app.task