chore(🦾): bump python et-xmlfile 1.1.0 -> 2.0.0 & remove pyhive[hive] from requirements/development.in (#31040)

Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>

.github/actions/setup-backend/action.yml

@@ -42,7 +42,7 @@ runs:
     - name: Install dependencies
       run: |
         if [ "${{ inputs.install-superset }}" = "true" ]; then
-          sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
+          sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev build-essential
           pip install --upgrade pip setuptools wheel
           if [ "${{ inputs.requirements-type }}" = "dev" ]; then
             pip install -r requirements/development.txt

.github/workflows/superset-python-presto-hive.yml

@@ -142,6 +142,7 @@ jobs:
       - name: Python unit tests (PostgreSQL)
         if: steps.check.outputs.python
         run: |
+          pip install -e .[hive]
           ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
       - name: Upload code coverage
         uses: codecov/codecov-action@v4

pyproject.toml

@@ -69,7 +69,7 @@ dependencies = [
     "nh3>=0.2.11, <0.3",
     "numpy==1.23.5",
     "packaging",
-    "pandas[performance]>=2.0.3, <2.1",
+    "pandas[excel,performance]>=2.0.3, <2.1",
     "parsedatetime",
     "paramiko>=3.4.0",
     "pgsanity",

requirements/base.txt

@@ -85,6 +85,8 @@ cryptography==42.0.8
     #   apache-superset
     #   paramiko
     #   pyopenssl
+defusedxml==0.7.1
+    # via odfpy
 deprecated==1.2.14
     # via limits
 deprecation==2.1.0
@@ -93,6 +95,8 @@ dnspython==2.6.1
     # via email-validator
 email-validator==2.1.1
     # via flask-appbuilder
+et-xmlfile==2.0.0
+    # via openpyxl
 exceptiongroup==1.2.2
     # via cattrs
 flask==2.3.3
@@ -231,6 +235,10 @@ numpy==1.23.5
     #   numexpr
     #   pandas
     #   pyarrow
+odfpy==1.4.1
+    # via pandas
+openpyxl==3.1.5
+    # via pandas
 ordered-set==4.1.0
     # via flask-limiter
 packaging==23.2
@@ -243,7 +251,7 @@ packaging==23.2
     #   marshmallow
     #   marshmallow-sqlalchemy
     #   shillelagh
-pandas[performance]==2.0.3
+pandas[excel,performance]==2.0.3
     # via apache-superset
 paramiko==3.4.0
     # via
@@ -306,6 +314,8 @@ pytz==2024.2
     #   croniter
     #   flask-babel
     #   pandas
+pyxlsb==1.0.10
+    # via pandas
 pyyaml==6.0.1
     # via
     #   apache-superset
@@ -404,8 +414,12 @@ wtforms==3.2.1
     #   wtforms-json
 wtforms-json==0.3.5
     # via apache-superset
+xlrd==2.0.1
+    # via pandas
 xlsxwriter==3.0.9
-    # via apache-superset
+    # via
+    #   apache-superset
+    #   pandas
 zipp==3.19.0
     # via importlib-metadata
 zstandard==0.22.0

requirements/development.in

@@ -17,4 +17,4 @@
 # under the License.
 #
 -r base.in
--e .[development,bigquery,cors,druid,gevent,gsheets,hive,mysql,playwright,postgres,presto,prophet,trino,thumbnails]
+-e .[development,bigquery,cors,druid,gevent,gsheets,mysql,playwright,postgres,presto,prophet,trino,thumbnails]

requirements/development.txt

@@ -1,4 +1,4 @@
-# SHA1:c186006a3f82c8775e1039f37c52309f6c858197
+# SHA1:dc767a7288b56c785b0cd3c38e95e7b5e66be1ac
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -13,23 +13,17 @@
 astroid==3.1.0
     # via pylint
 boto3==1.34.112
-    # via
-    #   apache-superset
-    #   dataflows-tabulator
+    # via apache-superset
 botocore==1.34.112
     # via
     #   boto3
     #   s3transfer
 build==1.2.1
     # via pip-tools
-cached-property==1.5.2
-    # via tableschema
 cfgv==3.3.1
     # via pre-commit
 chardet==5.1.0
-    # via
-    #   dataflows-tabulator
-    #   tox
+    # via tox
 cmdstanpy==1.1.0
     # via prophet
 contourpy==1.0.7
@@ -38,8 +32,6 @@ coverage[toml]==7.2.5
     # via pytest-cov
 cycler==0.11.0
     # via matplotlib
-dataflows-tabulator==1.54.3
-    # via tableschema
 db-dtypes==1.2.0
     # via pandas-gbq
 dill==0.3.8
@@ -48,8 +40,6 @@ distlib==0.3.8
     # via virtualenv
 docker==7.0.0
     # via apache-superset
-et-xmlfile==1.1.0
-    # via openpyxl
 filelock==3.12.2
     # via
     #   tox
@@ -103,8 +93,6 @@ grpcio-status==1.60.1
     # via google-api-core
 identify==2.5.36
     # via pre-commit
-ijson==3.2.3
-    # via dataflows-tabulator
 iniconfig==2.0.0
     # via pytest
 isort==5.12.0
@@ -113,16 +101,12 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsonlines==4.0.0
-    # via dataflows-tabulator
 jsonschema-spec==0.1.6
     # via openapi-spec-validator
 kiwisolver==1.4.5
     # via matplotlib
 lazy-object-proxy==1.10.0
     # via openapi-spec-validator
-linear-tsv==1.1.0
-    # via dataflows-tabulator
 matplotlib==3.9.0
     # via prophet
 mccabe==0.7.0
@@ -137,8 +121,6 @@ openapi-schema-validator==0.4.4
     # via openapi-spec-validator
 openapi-spec-validator==0.5.6
     # via apache-superset
-openpyxl==3.1.2
-    # via dataflows-tabulator
 pandas-gbq==0.19.1
     # via apache-superset
 parameterized==0.9.0
@@ -178,8 +160,6 @@ psutil==6.0.0
     # via apache-superset
 psycopg2-binary==2.9.6
     # via apache-superset
-pure-sasl==0.6.2
-    # via thrift-sasl
 pydata-google-auth==1.7.0
     # via pandas-gbq
 pydruid==0.6.9
@@ -215,8 +195,6 @@ requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3339-validator==0.1.4
     # via openapi-schema-validator
-rfc3986==2.0.0
-    # via tableschema
 ruff==0.4.5
     # via apache-superset
 s3transfer==0.10.1
@@ -227,14 +205,6 @@ sqloxide==0.1.43
     # via apache-superset
 statsd==4.0.1
     # via apache-superset
-tableschema==1.20.10
-    # via apache-superset
-thrift==0.16.0
-    # via
-    #   apache-superset
-    #   thrift-sasl
-thrift-sasl==0.4.3
-    # via apache-superset
 tomli==2.0.1
     # via
     #   build
@@ -259,18 +229,12 @@ trino==0.328.0
     # via apache-superset
 tzlocal==5.2
     # via trino
-unicodecsv==0.14.1
-    # via
-    #   dataflows-tabulator
-    #   tableschema
 virtualenv==20.23.1
     # via
     #   pre-commit
     #   tox
 wheel==0.43.0
     # via pip-tools
-xlrd==2.0.1
-    # via dataflows-tabulator
 zope-event==5.0
     # via gevent
 zope-interface==5.4.0

tests/integration_tests/conftest.py

@@ -122,7 +122,10 @@ def setup_sample_data() -> Any:
     # relying on `tests.integration_tests.test_app.app` leveraging an `app` fixture
     # which is purposely scoped to the function level to ensure tests remain idempotent.
     with app.app_context():
-        setup_presto_if_needed()
+        try:
+            setup_presto_if_needed()
+        except Exception:
+            pass
 
         from superset.examples.css_templates import load_css_templates
 

tests/integration_tests/db_engine_specs/hive_tests.py

@@ -16,6 +16,8 @@
 # under the License.
 # isort:skip_file
 from unittest import mock
+import unittest
+from .base_tests import SupersetTestCase
 
 import pytest
 import pandas as pd
@@ -154,6 +156,9 @@ def test_df_to_sql_if_exists_fail(mock_g):
 
 
 @mock.patch("superset.db_engine_specs.hive.g", spec={})
+@unittest.skipUnless(
+    SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
+)
 def test_df_to_sql_if_exists_fail_with_schema(mock_g):
     mock_g.user = True
     mock_database = mock.MagicMock()
@@ -290,6 +295,9 @@ def test_upload_to_s3_success(client):
     app.config = config
 
 
+@unittest.skipUnless(
+    SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
+)
 def test_fetch_data_query_error():
     from TCLIService import ttypes
 
@@ -301,6 +309,9 @@ def test_fetch_data_query_error():
         HiveEngineSpec.fetch_data(cursor)
 
 
+@unittest.skipUnless(
+    SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
+)
 @mock.patch("superset.db_engine_specs.base.BaseEngineSpec.fetch_data")
 def test_fetch_data_programming_error(fetch_data_mock):
     from pyhive.exc import ProgrammingError
@@ -310,6 +321,9 @@ def test_fetch_data_programming_error(fetch_data_mock):
     assert HiveEngineSpec.fetch_data(cursor) == []
 
 
+@unittest.skipUnless(
+    SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
+)
 @mock.patch("superset.db_engine_specs.base.BaseEngineSpec.fetch_data")
 def test_fetch_data_success(fetch_data_mock):
     return_value = ["a", "b"]

tests/integration_tests/model_tests.py

@@ -54,6 +54,9 @@ class TestDatabaseModel(SupersetTestCase):
     @unittest.skipUnless(
         SupersetTestCase.is_module_installed("requests"), "requests not installed"
     )
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
+    )
     def test_database_schema_presto(self):
         sqlalchemy_uri = "presto://presto.airbnb.io:8080/hive/default"
         model = Database(database_name="test_database", sqlalchemy_uri=sqlalchemy_uri)
@@ -108,7 +111,7 @@ class TestDatabaseModel(SupersetTestCase):
             assert "core_db" == db
 
     @unittest.skipUnless(
-        SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
+        SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
     )
     def test_database_schema_mysql(self):
         sqlalchemy_uri = "mysql://root@localhost/superset"
@@ -123,7 +126,7 @@ class TestDatabaseModel(SupersetTestCase):
             assert "staging" == db
 
     @unittest.skipUnless(
-        SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
+        SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
     )
     def test_database_impersonate_user(self):
         uri = "mysql://root@localhost"
@@ -142,6 +145,9 @@ class TestDatabaseModel(SupersetTestCase):
                 assert example_user.username != username
 
     @mock.patch("superset.models.core.create_engine")
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
+    )
     def test_impersonate_user_presto(self, mocked_create_engine):
         uri = "presto://localhost"
         principal_user = security_manager.find_user(username="gamma")
@@ -190,7 +196,7 @@ class TestDatabaseModel(SupersetTestCase):
             }
 
     @unittest.skipUnless(
-        SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
+        SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
     )
     @mock.patch("superset.models.core.create_engine")
     def test_adjust_engine_params_mysql(self, mocked_create_engine):
@@ -245,6 +251,12 @@ class TestDatabaseModel(SupersetTestCase):
             assert call_args[1]["connect_args"]["user"] == "gamma"
 
     @mock.patch("superset.models.core.create_engine")
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
+    )
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
+    )
     def test_impersonate_user_hive(self, mocked_create_engine):
         uri = "hive://localhost"
         principal_user = security_manager.find_user(username="gamma")
@@ -293,6 +305,9 @@ class TestDatabaseModel(SupersetTestCase):
             }
 
     @pytest.mark.usefixtures("load_energy_table_with_slice")
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
+    )
     def test_select_star(self):
         db = get_example_database()
         table_name = "energy_usage"