diff --git a/pyproject.toml b/pyproject.toml index 50c83f4b7..453134f9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ dependencies = [ "pyyaml>=6.0.0, <7.0.0", "PyJWT>=2.4.0, <3.0", "redis>=4.6.0, <5.0", - "selenium>=3.141.0, <4.10.0", + "selenium>=4.14.0, <5.0", "shillelagh[gsheetsapi]>=1.2.18, <2.0", "shortid", "sshtunnel>=0.4.0, <0.5", diff --git a/requirements/base.txt b/requirements/base.txt index c0cade94a..821655c17 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -16,7 +16,9 @@ attrs==24.2.0 # via # cattrs # jsonschema + # outcome # requests-cache + # trio babel==2.16.0 # via flask-babel backoff==2.2.1 @@ -42,7 +44,9 @@ cattrs==24.1.2 celery==5.4.0 # via apache-superset (pyproject.toml) certifi==2024.8.30 - # via requests + # via + # requests + # selenium cffi==1.17.1 # via # cryptography @@ -93,7 +97,10 @@ email-validator==2.2.0 et-xmlfile==2.0.0 # via openpyxl exceptiongroup==1.2.2 - # via cattrs + # via + # cattrs + # trio + # trio-websocket flask==2.3.3 # via # apache-superset (pyproject.toml) @@ -149,8 +156,11 @@ greenlet==3.0.3 # -r requirements/base.in # apache-superset (pyproject.toml) # shillelagh + # sqlalchemy gunicorn==23.0.0 # via apache-superset (pyproject.toml) +h11==0.14.0 + # via wsproto hashids==1.3.1 # via apache-superset (pyproject.toml) holidays==0.25 @@ -161,8 +171,13 @@ idna==3.10 # via # email-validator # requests + # trio importlib-metadata==8.5.0 - # via apache-superset (pyproject.toml) + # via + # apache-superset (pyproject.toml) + # flask + # markdown + # shillelagh importlib-resources==6.4.5 # via limits isodate==0.7.2 @@ -228,6 +243,8 @@ openpyxl==3.1.5 # via pandas ordered-set==4.1.0 # via flask-limiter +outcome==1.3.0.post0 + # via trio packaging==24.2 # via # apache-superset (pyproject.toml) @@ -283,6 +300,8 @@ pyparsing==3.2.0 # via apache-superset (pyproject.toml) pyrsistent==0.20.0 # via jsonschema +pysocks==1.7.1 + # via urllib3 python-dateutil==2.9.0.post0 # via # apache-superset (pyproject.toml) @@ -319,7 +338,7 @@ rich==13.9.4 # via flask-limiter rsa==4.9 # via google-auth -selenium==3.141.0 +selenium==4.27.1 # via apache-superset (pyproject.toml) shillelagh==1.2.18 # via apache-superset (pyproject.toml) @@ -335,6 +354,10 @@ six==1.16.0 # wtforms-json slack-sdk==3.33.4 # via apache-superset (pyproject.toml) +sniffio==1.3.1 + # via trio +sortedcontainers==2.4.0 + # via trio sqlalchemy==1.4.54 # via # apache-superset (pyproject.toml) @@ -356,14 +379,22 @@ sshtunnel==0.4.0 # via apache-superset (pyproject.toml) tabulate==0.8.10 # via apache-superset (pyproject.toml) +trio==0.28.0 + # via + # selenium + # trio-websocket +trio-websocket==0.11.1 + # via selenium typing-extensions==4.12.2 # via # apache-superset (pyproject.toml) # alembic # cattrs # flask-limiter + # kombu # limits # rich + # selenium # shillelagh tzdata==2024.2 # via @@ -385,6 +416,8 @@ vine==5.1.0 # kombu wcwidth==0.2.13 # via prompt-toolkit +websocket-client==1.8.0 + # via selenium werkzeug==3.1.3 # via # -r requirements/base.in @@ -394,6 +427,8 @@ werkzeug==3.1.3 # flask-login wrapt==1.17.0 # via deprecated +wsproto==1.2.0 + # via trio-websocket wtforms==3.2.1 # via # apache-superset (pyproject.toml) @@ -409,6 +444,8 @@ xlsxwriter==3.0.9 # apache-superset (pyproject.toml) # pandas zipp==3.21.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources zstandard==0.23.0 # via flask-compress diff --git a/requirements/development.txt b/requirements/development.txt index 246f9e6cb..3b2203f46 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -27,7 +27,9 @@ attrs==24.2.0 # -c requirements/base.txt # cattrs # jsonschema + # outcome # requests-cache + # trio babel==2.16.0 # via # -c requirements/base.txt @@ -77,6 +79,7 @@ certifi==2024.8.30 # via # -c requirements/base.txt # requests + # selenium cffi==1.17.1 # via # -c requirements/base.txt @@ -177,6 +180,8 @@ exceptiongroup==1.2.2 # -c requirements/base.txt # cattrs # pytest + # trio + # trio-websocket filelock==3.12.2 # via virtualenv flask==2.3.3 @@ -312,6 +317,7 @@ greenlet==3.0.3 # apache-superset # gevent # shillelagh + # sqlalchemy grpcio==1.68.0 # via # apache-superset @@ -323,6 +329,10 @@ gunicorn==23.0.0 # via # -c requirements/base.txt # apache-superset +h11==0.14.0 + # via + # -c requirements/base.txt + # wsproto hashids==1.3.1 # via # -c requirements/base.txt @@ -343,14 +353,19 @@ idna==3.10 # -c requirements/base.txt # email-validator # requests + # trio importlib-metadata==8.5.0 # via # -c requirements/base.txt # apache-superset + # flask + # markdown + # shillelagh importlib-resources==6.4.5 # via # -c requirements/base.txt # limits + # matplotlib # prophet iniconfig==2.0.0 # via pytest @@ -479,6 +494,10 @@ ordered-set==4.1.0 # via # -c requirements/base.txt # flask-limiter +outcome==1.3.0.post0 + # via + # -c requirements/base.txt + # trio packaging==24.2 # via # -c requirements/base.txt @@ -629,6 +648,10 @@ pyrsistent==0.20.0 # via # -c requirements/base.txt # jsonschema +pysocks==1.7.1 + # via + # -c requirements/base.txt + # urllib3 pytest==7.4.4 # via # apache-superset @@ -716,7 +739,7 @@ rsa==4.9 # google-auth ruff==0.8.0 # via apache-superset -selenium==3.141.0 +selenium==4.27.1 # via # -c requirements/base.txt # apache-superset @@ -751,6 +774,14 @@ slack-sdk==3.33.4 # via # -c requirements/base.txt # apache-superset +sniffio==1.3.1 + # via + # -c requirements/base.txt + # trio +sortedcontainers==2.4.0 + # via + # -c requirements/base.txt + # trio sqlalchemy==1.4.54 # via # -c requirements/base.txt @@ -799,6 +830,15 @@ tqdm==4.67.1 # prophet trino==0.330.0 # via apache-superset +trio==0.28.0 + # via + # -c requirements/base.txt + # selenium + # trio-websocket +trio-websocket==0.11.1 + # via + # -c requirements/base.txt + # selenium typing-extensions==4.12.2 # via # -c requirements/base.txt @@ -806,8 +846,10 @@ typing-extensions==4.12.2 # apache-superset # cattrs # flask-limiter + # kombu # limits # rich + # selenium # shillelagh tzdata==2024.2 # via @@ -840,6 +882,10 @@ wcwidth==0.2.13 # via # -c requirements/base.txt # prompt-toolkit +websocket-client==1.8.0 + # via + # -c requirements/base.txt + # selenium werkzeug==3.1.3 # via # -c requirements/base.txt @@ -851,6 +897,10 @@ wrapt==1.17.0 # via # -c requirements/base.txt # deprecated +wsproto==1.2.0 + # via + # -c requirements/base.txt + # trio-websocket wtforms==3.2.1 # via # -c requirements/base.txt @@ -875,6 +925,7 @@ zipp==3.21.0 # via # -c requirements/base.txt # importlib-metadata + # importlib-resources zope-event==5.0 # via gevent zope-interface==5.4.0 diff --git a/superset/config.py b/superset/config.py index 7ab30185b..a77ab15f2 100644 --- a/superset/config.py +++ b/superset/config.py @@ -1488,7 +1488,10 @@ WEBDRIVER_WINDOW = { WEBDRIVER_AUTH_FUNC = None # Any config options to be passed as-is to the webdriver -WEBDRIVER_CONFIGURATION: dict[Any, Any] = {"service_log_path": "/dev/null"} +WEBDRIVER_CONFIGURATION = { + "options": {"capabilities": {}, "preferences": {}}, + "service": {"log_output": "/dev/null", "service_args": [], "port": 0, "env": {}}, +} # Additional args to be passed as arguments to the config object # Note: If using Chrome, you'll want to add the "--marionette" arg. diff --git a/superset/utils/webdriver.py b/superset/utils/webdriver.py index 727f72130..c8e46581e 100644 --- a/superset/utils/webdriver.py +++ b/superset/utils/webdriver.py @@ -21,9 +21,11 @@ import logging from abc import ABC, abstractmethod from enum import Enum from time import sleep -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING from flask import current_app +from packaging import version +from selenium import __version__ as selenium_version from selenium.common.exceptions import ( StaleElementReferenceException, TimeoutException, @@ -31,6 +33,7 @@ from selenium.common.exceptions import ( ) from selenium.webdriver import chrome, firefox, FirefoxProfile from selenium.webdriver.common.by import By +from selenium.webdriver.common.service import Service from selenium.webdriver.remote.webdriver import WebDriver from selenium.webdriver.support import expected_conditions as EC # noqa: N812 from selenium.webdriver.support.ui import WebDriverWait @@ -246,13 +249,16 @@ class WebDriverSelenium(WebDriverProxy): def create(self) -> WebDriver: pixel_density = current_app.config["WEBDRIVER_WINDOW"].get("pixel_density", 1) if self._driver_type == "firefox": - driver_class = firefox.webdriver.WebDriver + driver_class: type[WebDriver] = firefox.webdriver.WebDriver + service_class: type[Service] = firefox.service.Service options = firefox.options.Options() profile = FirefoxProfile() profile.set_preference("layout.css.devPixelsPerPx", str(pixel_density)) - kwargs: dict[Any, Any] = {"options": options, "firefox_profile": profile} + options.profile = profile + kwargs = {"options": options} elif self._driver_type == "chrome": driver_class = chrome.webdriver.WebDriver + service_class = chrome.service.Service options = chrome.options.Options() options.add_argument(f"--force-device-scale-factor={pixel_density}") options.add_argument(f"--window-size={self._window[0]},{self._window[1]}") @@ -261,15 +267,41 @@ class WebDriverSelenium(WebDriverProxy): raise Exception( # pylint: disable=broad-exception-raised f"Webdriver name ({self._driver_type}) not supported" ) - # Prepare args for the webdriver init - # Add additional configured options - for arg in current_app.config["WEBDRIVER_OPTION_ARGS"]: + # Prepare args for the webdriver init + for arg in list(current_app.config["WEBDRIVER_OPTION_ARGS"]): options.add_argument(arg) - kwargs.update(current_app.config["WEBDRIVER_CONFIGURATION"]) - logger.debug("Init selenium driver") + # Add additional configured webdriver options + webdriver_conf = dict(current_app.config["WEBDRIVER_CONFIGURATION"]) + if version.parse(selenium_version) < version.parse("4.10.0"): + kwargs |= webdriver_conf + else: + driver_opts = dict( + webdriver_conf.get("options", {"capabilities": {}, "preferences": {}}) + ) + driver_srv = dict( + webdriver_conf.get( + "service", + { + "log_output": "/dev/null", + "service_args": [], + "port": 0, + "env": {}, + }, + ) + ) + for name, value in driver_opts.get("capabilities", {}).items(): + options.set_capability(name, value) + if hasattr(options, "profile"): + for name, value in driver_opts.get("preferences", {}).items(): + options.profile.set_preference(str(name), value) + kwargs |= { + "service": service_class(**driver_srv), + } + + logger.debug("Init selenium driver") return driver_class(**kwargs) def auth(self, user: User) -> WebDriver: