chore(deps): bump selenium 4.14.0+ (#25933)

Co-authored-by: Evan Rusackas <evan@rusackas.com>

pyproject.toml

@@ -86,7 +86,7 @@ dependencies = [
     "pyyaml>=6.0.0, <7.0.0",
     "PyJWT>=2.4.0, <3.0",
     "redis>=4.6.0, <5.0",
-    "selenium>=3.141.0, <4.10.0",
+    "selenium>=4.14.0, <5.0",
     "shillelagh[gsheetsapi]>=1.2.18, <2.0",
     "shortid",
     "sshtunnel>=0.4.0, <0.5",

requirements/base.txt

@@ -16,7 +16,9 @@ attrs==24.2.0
     # via
     #   cattrs
     #   jsonschema
+    #   outcome
     #   requests-cache
+    #   trio
 babel==2.16.0
     # via flask-babel
 backoff==2.2.1
@@ -42,7 +44,9 @@ cattrs==24.1.2
 celery==5.4.0
     # via apache-superset (pyproject.toml)
 certifi==2024.8.30
-    # via requests
+    # via
+    #   requests
+    #   selenium
 cffi==1.17.1
     # via
     #   cryptography
@@ -93,7 +97,10 @@ email-validator==2.2.0
 et-xmlfile==2.0.0
     # via openpyxl
 exceptiongroup==1.2.2
-    # via cattrs
+    # via
+    #   cattrs
+    #   trio
+    #   trio-websocket
 flask==2.3.3
     # via
     #   apache-superset (pyproject.toml)
@@ -149,8 +156,11 @@ greenlet==3.0.3
     #   -r requirements/base.in
     #   apache-superset (pyproject.toml)
     #   shillelagh
+    #   sqlalchemy
 gunicorn==23.0.0
     # via apache-superset (pyproject.toml)
+h11==0.14.0
+    # via wsproto
 hashids==1.3.1
     # via apache-superset (pyproject.toml)
 holidays==0.25
@@ -161,8 +171,13 @@ idna==3.10
     # via
     #   email-validator
     #   requests
+    #   trio
 importlib-metadata==8.5.0
-    # via apache-superset (pyproject.toml)
+    # via
+    #   apache-superset (pyproject.toml)
+    #   flask
+    #   markdown
+    #   shillelagh
 importlib-resources==6.4.5
     # via limits
 isodate==0.7.2
@@ -228,6 +243,8 @@ openpyxl==3.1.5
     # via pandas
 ordered-set==4.1.0
     # via flask-limiter
+outcome==1.3.0.post0
+    # via trio
 packaging==24.2
     # via
     #   apache-superset (pyproject.toml)
@@ -283,6 +300,8 @@ pyparsing==3.2.0
     # via apache-superset (pyproject.toml)
 pyrsistent==0.20.0
     # via jsonschema
+pysocks==1.7.1
+    # via urllib3
 python-dateutil==2.9.0.post0
     # via
     #   apache-superset (pyproject.toml)
@@ -319,7 +338,7 @@ rich==13.9.4
     # via flask-limiter
 rsa==4.9
     # via google-auth
-selenium==3.141.0
+selenium==4.27.1
     # via apache-superset (pyproject.toml)
 shillelagh==1.2.18
     # via apache-superset (pyproject.toml)
@@ -335,6 +354,10 @@ six==1.16.0
     #   wtforms-json
 slack-sdk==3.33.4
     # via apache-superset (pyproject.toml)
+sniffio==1.3.1
+    # via trio
+sortedcontainers==2.4.0
+    # via trio
 sqlalchemy==1.4.54
     # via
     #   apache-superset (pyproject.toml)
@@ -356,14 +379,22 @@ sshtunnel==0.4.0
     # via apache-superset (pyproject.toml)
 tabulate==0.8.10
     # via apache-superset (pyproject.toml)
+trio==0.28.0
+    # via
+    #   selenium
+    #   trio-websocket
+trio-websocket==0.11.1
+    # via selenium
 typing-extensions==4.12.2
     # via
     #   apache-superset (pyproject.toml)
     #   alembic
     #   cattrs
     #   flask-limiter
+    #   kombu
     #   limits
     #   rich
+    #   selenium
     #   shillelagh
 tzdata==2024.2
     # via
@@ -385,6 +416,8 @@ vine==5.1.0
     #   kombu
 wcwidth==0.2.13
     # via prompt-toolkit
+websocket-client==1.8.0
+    # via selenium
 werkzeug==3.1.3
     # via
     #   -r requirements/base.in
@@ -394,6 +427,8 @@ werkzeug==3.1.3
     #   flask-login
 wrapt==1.17.0
     # via deprecated
+wsproto==1.2.0
+    # via trio-websocket
 wtforms==3.2.1
     # via
     #   apache-superset (pyproject.toml)
@@ -409,6 +444,8 @@ xlsxwriter==3.0.9
     #   apache-superset (pyproject.toml)
     #   pandas
 zipp==3.21.0
-    # via importlib-metadata
+    # via
+    #   importlib-metadata
+    #   importlib-resources
 zstandard==0.23.0
     # via flask-compress

requirements/development.txt

@@ -27,7 +27,9 @@ attrs==24.2.0
     #   -c requirements/base.txt
     #   cattrs
     #   jsonschema
+    #   outcome
     #   requests-cache
+    #   trio
 babel==2.16.0
     # via
     #   -c requirements/base.txt
@@ -77,6 +79,7 @@ certifi==2024.8.30
     # via
     #   -c requirements/base.txt
     #   requests
+    #   selenium
 cffi==1.17.1
     # via
     #   -c requirements/base.txt
@@ -177,6 +180,8 @@ exceptiongroup==1.2.2
     #   -c requirements/base.txt
     #   cattrs
     #   pytest
+    #   trio
+    #   trio-websocket
 filelock==3.12.2
     # via virtualenv
 flask==2.3.3
@@ -312,6 +317,7 @@ greenlet==3.0.3
     #   apache-superset
     #   gevent
     #   shillelagh
+    #   sqlalchemy
 grpcio==1.68.0
     # via
     #   apache-superset
@@ -323,6 +329,10 @@ gunicorn==23.0.0
     # via
     #   -c requirements/base.txt
     #   apache-superset
+h11==0.14.0
+    # via
+    #   -c requirements/base.txt
+    #   wsproto
 hashids==1.3.1
     # via
     #   -c requirements/base.txt
@@ -343,14 +353,19 @@ idna==3.10
     #   -c requirements/base.txt
     #   email-validator
     #   requests
+    #   trio
 importlib-metadata==8.5.0
     # via
     #   -c requirements/base.txt
     #   apache-superset
+    #   flask
+    #   markdown
+    #   shillelagh
 importlib-resources==6.4.5
     # via
     #   -c requirements/base.txt
     #   limits
+    #   matplotlib
     #   prophet
 iniconfig==2.0.0
     # via pytest
@@ -479,6 +494,10 @@ ordered-set==4.1.0
     # via
     #   -c requirements/base.txt
     #   flask-limiter
+outcome==1.3.0.post0
+    # via
+    #   -c requirements/base.txt
+    #   trio
 packaging==24.2
     # via
     #   -c requirements/base.txt
@@ -629,6 +648,10 @@ pyrsistent==0.20.0
     # via
     #   -c requirements/base.txt
     #   jsonschema
+pysocks==1.7.1
+    # via
+    #   -c requirements/base.txt
+    #   urllib3
 pytest==7.4.4
     # via
     #   apache-superset
@@ -716,7 +739,7 @@ rsa==4.9
     #   google-auth
 ruff==0.8.0
     # via apache-superset
-selenium==3.141.0
+selenium==4.27.1
     # via
     #   -c requirements/base.txt
     #   apache-superset
@@ -751,6 +774,14 @@ slack-sdk==3.33.4
     # via
     #   -c requirements/base.txt
     #   apache-superset
+sniffio==1.3.1
+    # via
+    #   -c requirements/base.txt
+    #   trio
+sortedcontainers==2.4.0
+    # via
+    #   -c requirements/base.txt
+    #   trio
 sqlalchemy==1.4.54
     # via
     #   -c requirements/base.txt
@@ -799,6 +830,15 @@ tqdm==4.67.1
     #   prophet
 trino==0.330.0
     # via apache-superset
+trio==0.28.0
+    # via
+    #   -c requirements/base.txt
+    #   selenium
+    #   trio-websocket
+trio-websocket==0.11.1
+    # via
+    #   -c requirements/base.txt
+    #   selenium
 typing-extensions==4.12.2
     # via
     #   -c requirements/base.txt
@@ -806,8 +846,10 @@ typing-extensions==4.12.2
     #   apache-superset
     #   cattrs
     #   flask-limiter
+    #   kombu
     #   limits
     #   rich
+    #   selenium
     #   shillelagh
 tzdata==2024.2
     # via
@@ -840,6 +882,10 @@ wcwidth==0.2.13
     # via
     #   -c requirements/base.txt
     #   prompt-toolkit
+websocket-client==1.8.0
+    # via
+    #   -c requirements/base.txt
+    #   selenium
 werkzeug==3.1.3
     # via
     #   -c requirements/base.txt
@@ -851,6 +897,10 @@ wrapt==1.17.0
     # via
     #   -c requirements/base.txt
     #   deprecated
+wsproto==1.2.0
+    # via
+    #   -c requirements/base.txt
+    #   trio-websocket
 wtforms==3.2.1
     # via
     #   -c requirements/base.txt
@@ -875,6 +925,7 @@ zipp==3.21.0
     # via
     #   -c requirements/base.txt
     #   importlib-metadata
+    #   importlib-resources
 zope-event==5.0
     # via gevent
 zope-interface==5.4.0

superset/config.py

@@ -1488,7 +1488,10 @@ WEBDRIVER_WINDOW = {
 WEBDRIVER_AUTH_FUNC = None
 
 # Any config options to be passed as-is to the webdriver
-WEBDRIVER_CONFIGURATION: dict[Any, Any] = {"service_log_path": "/dev/null"}
+WEBDRIVER_CONFIGURATION = {
+    "options": {"capabilities": {}, "preferences": {}},
+    "service": {"log_output": "/dev/null", "service_args": [], "port": 0, "env": {}},
+}
 
 # Additional args to be passed as arguments to the config object
 # Note: If using Chrome, you'll want to add the "--marionette" arg.

superset/utils/webdriver.py

@@ -21,9 +21,11 @@ import logging
 from abc import ABC, abstractmethod
 from enum import Enum
 from time import sleep
-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING
 
 from flask import current_app
+from packaging import version
+from selenium import __version__ as selenium_version
 from selenium.common.exceptions import (
     StaleElementReferenceException,
     TimeoutException,
@@ -31,6 +33,7 @@ from selenium.common.exceptions import (
 )
 from selenium.webdriver import chrome, firefox, FirefoxProfile
 from selenium.webdriver.common.by import By
+from selenium.webdriver.common.service import Service
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.support import expected_conditions as EC  # noqa: N812
 from selenium.webdriver.support.ui import WebDriverWait
@@ -246,13 +249,16 @@ class WebDriverSelenium(WebDriverProxy):
     def create(self) -> WebDriver:
         pixel_density = current_app.config["WEBDRIVER_WINDOW"].get("pixel_density", 1)
         if self._driver_type == "firefox":
-            driver_class = firefox.webdriver.WebDriver
+            driver_class: type[WebDriver] = firefox.webdriver.WebDriver
+            service_class: type[Service] = firefox.service.Service
             options = firefox.options.Options()
             profile = FirefoxProfile()
             profile.set_preference("layout.css.devPixelsPerPx", str(pixel_density))
-            kwargs: dict[Any, Any] = {"options": options, "firefox_profile": profile}
+            options.profile = profile
+            kwargs = {"options": options}
         elif self._driver_type == "chrome":
             driver_class = chrome.webdriver.WebDriver
+            service_class = chrome.service.Service
             options = chrome.options.Options()
             options.add_argument(f"--force-device-scale-factor={pixel_density}")
             options.add_argument(f"--window-size={self._window[0]},{self._window[1]}")
@@ -261,15 +267,41 @@ class WebDriverSelenium(WebDriverProxy):
             raise Exception(  # pylint: disable=broad-exception-raised
                 f"Webdriver name ({self._driver_type}) not supported"
             )
-        # Prepare args for the webdriver init
 
-        # Add additional configured options
+        # Prepare args for the webdriver init
-        for arg in current_app.config["WEBDRIVER_OPTION_ARGS"]:
+        for arg in list(current_app.config["WEBDRIVER_OPTION_ARGS"]):
             options.add_argument(arg)
 
-        kwargs.update(current_app.config["WEBDRIVER_CONFIGURATION"])
+        # Add additional configured webdriver options
-        logger.debug("Init selenium driver")
+        webdriver_conf = dict(current_app.config["WEBDRIVER_CONFIGURATION"])
 
+        if version.parse(selenium_version) < version.parse("4.10.0"):
+            kwargs |= webdriver_conf
+        else:
+            driver_opts = dict(
+                webdriver_conf.get("options", {"capabilities": {}, "preferences": {}})
+            )
+            driver_srv = dict(
+                webdriver_conf.get(
+                    "service",
+                    {
+                        "log_output": "/dev/null",
+                        "service_args": [],
+                        "port": 0,
+                        "env": {},
+                    },
+                )
+            )
+            for name, value in driver_opts.get("capabilities", {}).items():
+                options.set_capability(name, value)
+            if hasattr(options, "profile"):
+                for name, value in driver_opts.get("preferences", {}).items():
+                    options.profile.set_preference(str(name), value)
+            kwargs |= {
+                "service": service_class(**driver_srv),
+            }
+
+        logger.debug("Init selenium driver")
         return driver_class(**kwargs)
 
     def auth(self, user: User) -> WebDriver: