diff --git a/Dockerfile b/Dockerfile index bbc8640d4..9d6a61bef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,10 +123,27 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] # Dev image... ###################################################################### FROM lean AS dev +ARG GECKODRIVER_VERSION=v0.28.0 +ARG FIREFOX_VERSION=88.0 COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/ USER root + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends libnss3 libdbus-glib-1-2 libgtk-3-0 libx11-xcb1 + +# Install GeckoDriver WebDriver +RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O /tmp/geckodriver.tar.gz && \ + tar xvfz /tmp/geckodriver.tar.gz -C /tmp && \ + mv /tmp/geckodriver /usr/local/bin/geckodriver && \ + rm /tmp/geckodriver.tar.gz + +# Install Firefox +RUN wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O /opt/firefox.tar.bz2 && \ + tar xvf /opt/firefox.tar.bz2 -C /opt && \ + ln -s /opt/firefox/firefox /usr/local/bin/firefox + # Cache everything for dev purposes... RUN cd /app \ && pip install --no-cache -r requirements/docker.txt \ diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index c88f3ab99..a3ca8b1ec 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -20,11 +20,12 @@ # development environments. Also note that superset_config_docker.py is imported # as a final step as a means to override "defaults" configured here # - import logging import os +from datetime import timedelta from cachelib.file import FileSystemCache +from celery.schedules import crontab logger = logging.getLogger() @@ -70,13 +71,31 @@ RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab") class CeleryConfig(object): BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" - CELERY_IMPORTS = ("superset.sql_lab",) + CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks") CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}" - CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}} - CELERY_TASK_PROTOCOL = 1 + CELERYD_LOG_LEVEL = "DEBUG" + CELERYD_PREFETCH_MULTIPLIER = 1 + CELERY_ACKS_LATE = False + CELERYBEAT_SCHEDULE = { + "reports.scheduler": { + "task": "reports.scheduler", + "schedule": crontab(minute="*", hour="*"), + }, + "reports.prune_log": { + "task": "reports.prune_log", + "schedule": crontab(minute=10, hour=0), + }, + } CELERY_CONFIG = CeleryConfig + +FEATURE_FLAGS = {"ALERT_REPORTS": True} +ALERT_REPORTS_NOTIFICATION_DRY_RUN = True +WEBDRIVER_BASEURL = "http://superset:8088/" +# The base URL for the email report hyperlinks. +WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL + SQLLAB_CTAS_NO_LIMIT = True # diff --git a/superset/config.py b/superset/config.py index 3511d67d1..2a2705571 100644 --- a/superset/config.py +++ b/superset/config.py @@ -956,6 +956,9 @@ ALERT_REPORTS_WORKING_TIME_OUT_LAG = 10 # if ALERT_REPORTS_WORKING_TIME_OUT_KILL is True, set a celery hard timeout # Equal to working timeout + ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG = 1 +# If set to true no notification is sent, the worker will just log a message. +# Useful for debugging +ALERT_REPORTS_NOTIFICATION_DRY_RUN = False # A custom prefix to use on all Alerts & Reports emails EMAIL_REPORTS_SUBJECT_PREFIX = "[Report] " diff --git a/superset/reports/commands/execute.py b/superset/reports/commands/execute.py index befd0b233..cc7222e30 100644 --- a/superset/reports/commands/execute.py +++ b/superset/reports/commands/execute.py @@ -270,9 +270,10 @@ class BaseReportState: csv=csv_data, ) - @staticmethod def _send( - notification_content: NotificationContent, recipients: List[ReportRecipients] + self, + notification_content: NotificationContent, + recipients: List[ReportRecipients], ) -> None: """ Sends a notification to all recipients @@ -283,7 +284,14 @@ class BaseReportState: for recipient in recipients: notification = create_notification(recipient, notification_content) try: - notification.send() + if app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"]: + logger.info( + "Would send notification for alert %s, to %s", + self._report_schedule.name, + recipient.recipient_config_json, + ) + else: + notification.send() except NotificationError as ex: # collect notification errors but keep processing them notification_errors.append(str(ex)) diff --git a/tests/reports/commands_tests.py b/tests/reports/commands_tests.py index 8392d7417..bc0bf792a 100644 --- a/tests/reports/commands_tests.py +++ b/tests/reports/commands_tests.py @@ -568,7 +568,7 @@ def test_email_chart_report_schedule( screenshot_mock, email_mock, create_report_email_chart, ): """ - ExecuteReport Command: Test chart email report schedule with + ExecuteReport Command: Test chart email report schedule with screenshot """ # setup screenshot mock screenshot_mock.return_value = SCREENSHOT_FILE @@ -596,6 +596,29 @@ def test_email_chart_report_schedule( assert_log(ReportState.SUCCESS) +@pytest.mark.usefixtures( + "load_birth_names_dashboard_with_slices", "create_report_email_chart" +) +@patch("superset.reports.notifications.email.send_email_smtp") +@patch("superset.utils.screenshots.ChartScreenshot.get_screenshot") +def test_email_chart_report_dry_run( + screenshot_mock, email_mock, create_report_email_chart, +): + """ + ExecuteReport Command: Test chart email report schedule dry run + """ + # setup screenshot mock + screenshot_mock.return_value = SCREENSHOT_FILE + app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"] = True + with freeze_time("2020-01-01T00:00:00Z"): + AsyncExecuteReportScheduleCommand( + TEST_ID, create_report_email_chart.id, datetime.utcnow() + ).run() + + email_mock.assert_not_called() + app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"] = False + + @pytest.mark.usefixtures( "load_birth_names_dashboard_with_slices", "create_report_email_chart_with_csv" )