Use sane Celery defaults to prevent tasks from being delayed (#3267)

superset/cli.py

@@ -194,7 +194,7 @@ def worker(workers):
         celery_app.conf.update(
             CELERYD_CONCURRENCY=config.get("SUPERSET_CELERY_WORKERS"))
 
-    worker = celery_worker.worker(app=celery_app)
+    worker = celery_app.Worker(optimization='fair')
     worker.run()
 
 

superset/config.py

@@ -245,6 +245,8 @@ class CeleryConfig(object):
   CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite'
   CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
   CELERYD_LOG_LEVEL = 'DEBUG'
+  CELERYD_PREFETCH_MULTIPLIER = 1
+  CELERY_ACKS_LATE = True
 CELERY_CONFIG = CeleryConfig
 """
 CELERY_CONFIG = None