diff --git a/docs/docs/databases/bigquery.mdx b/docs/docs/databases/bigquery.mdx index 6d3ba0750..7ea993ae5 100644 --- a/docs/docs/databases/bigquery.mdx +++ b/docs/docs/databases/bigquery.mdx @@ -8,7 +8,7 @@ version: 1 ## Google BigQuery The recommended connector library for BigQuery is -[pybigquery](https://github.com/mxmzdlv/pybigquery). +[sqlalchemy-bigquery](https://github.com/googleapis/python-bigquery-sqlalchemy). ### Install BigQuery Driver @@ -16,7 +16,7 @@ Follow the steps [here](/docs/databases/docker-add-drivers) about how to install new database drivers when setting up Superset locally via docker-compose. ``` -echo "pybigquery" >> ./docker/requirements-local.txt +echo "sqlalchemy-bigquery" >> ./docker/requirements-local.txt ``` ### Connecting to BigQuery diff --git a/docs/docs/databases/installing-database-drivers.mdx b/docs/docs/databases/installing-database-drivers.mdx index 62244b3d6..a31a31178 100644 --- a/docs/docs/databases/installing-database-drivers.mdx +++ b/docs/docs/databases/installing-database-drivers.mdx @@ -35,7 +35,7 @@ A list of some of the recommended packages. | [Apache Spark SQL](/docs/databases/spark-sql) | `pip install pyhive` | `hive://hive@{hostname}:{port}/{database}` | | [Ascend.io](/docs/databases/ascend) | `pip install impyla` | `ascend://{username}:{password}@{hostname}:{port}/{database}?auth_mechanism=PLAIN;use_ssl=true` | | [Azure MS SQL](/docs/databases/sql-server) | `pip install pymssql` | `mssql+pymssql://UserName@presetSQL:TestPassword@presetSQL.database.windows.net:1433/TestSchema` | -| [Big Query](/docs/databases/bigquery) | `pip install pybigquery` | `bigquery://{project_id}` | +| [Big Query](/docs/databases/bigquery) | `pip install sqlalchemy-bigquery` | `bigquery://{project_id}` | | [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb://{username}:{password}@{hostname}:{port}/{database}` | | [CockroachDB](/docs/databases/cockroachdb) | `pip install cockroachdb` | `cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable` | | [Dremio](/docs/databases/dremio) | `pip install sqlalchemy_dremio` | `dremio://user:pwd@host:31010/` | diff --git a/docs/docs/installation/running-on-kubernetes.mdx b/docs/docs/installation/running-on-kubernetes.mdx index f7e568e52..1b75e0f5c 100644 --- a/docs/docs/installation/running-on-kubernetes.mdx +++ b/docs/docs/installation/running-on-kubernetes.mdx @@ -132,7 +132,7 @@ bootstrapScript: | #!/bin/bash pip install psycopg2==2.9.1 \ redis==3.2.1 \ - pybigquery==2.26.0 \ + sqlalchemy-bigquery==1.5.0 \ elasticsearch-dbapi==0.2.5 &&\ if [ ! -f ~/bootstrap ]; then echo "Running Superset with uid {{ .Values.runAsUser }}" > ~/bootstrap; fi ``` diff --git a/requirements/testing.txt b/requirements/testing.txt index 8068f3718..219b8c3ed 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -14,7 +14,7 @@ # -r requirements/testing.in astroid==2.6.6 # via pylint -cachetools==4.2.4 +cachetools==5.2.0 # via google-auth coverage==5.5 # via pytest-cov @@ -24,47 +24,47 @@ flask-testing==0.8.1 # via -r requirements/testing.in freezegun==1.1.0 # via -r requirements/testing.in -google-api-core[grpc]==2.2.1 +google-api-core[grpc]==2.11.0 # via # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core - # pybigquery -google-auth==2.2.1 + # sqlalchemy-bigquery +google-auth==2.14.1 # via # google-api-core # google-auth-oauthlib # google-cloud-core # pandas-gbq - # pybigquery + # sqlalchemy-bigquery # pydata-google-auth -google-auth-oauthlib==0.4.6 +google-auth-oauthlib==0.7.1 # via # pandas-gbq # pydata-google-auth -google-cloud-bigquery[bqstorage,pandas]==2.29.0 +google-cloud-bigquery[bqstorage,pandas]==3.4.0 # via # apache-superset # pandas-gbq - # pybigquery -google-cloud-bigquery-storage==2.9.1 + # sqlalchemy-bigquery +google-cloud-bigquery-storage==2.16.2 # via google-cloud-bigquery -google-cloud-core==2.1.0 +google-cloud-core==2.3.2 # via google-cloud-bigquery -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.1.0 +google-resumable-media==2.4.0 # via google-cloud-bigquery -googleapis-common-protos==1.53.0 +googleapis-common-protos==1.57.0 # via # google-api-core # grpcio-status -grpcio==1.41.1 +grpcio==1.51.1 # via # google-api-core # google-cloud-bigquery # grpcio-status -grpcio-status==1.41.1 +grpcio-status==1.51.1 # via google-api-core iniconfig==1.1.1 # via pytest @@ -72,27 +72,27 @@ isort==5.9.3 # via pylint lazy-object-proxy==1.6.0 # via astroid -libcst==0.3.21 +libcst==0.4.9 # via google-cloud-bigquery-storage mccabe==0.6.1 # via pylint mypy-extensions==0.4.3 # via typing-inspect -oauthlib==3.1.1 +oauthlib==3.2.2 # via requests-oauthlib openapi-schema-validator==0.1.5 # via openapi-spec-validator openapi-spec-validator==0.3.1 # via -r requirements/testing.in -pandas-gbq==0.15.0 +pandas-gbq==0.18.1 # via apache-superset parameterized==0.8.1 # via -r requirements/testing.in -proto-plus==1.19.7 +proto-plus==1.22.1 # via # google-cloud-bigquery # google-cloud-bigquery-storage -protobuf==3.19.1 +protobuf==4.21.10 # via # google-api-core # google-cloud-bigquery @@ -105,9 +105,9 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pybigquery==0.10.2 +sqlalchemy-bigquery==1.5.0 # via apache-superset -pydata-google-auth==1.2.0 +pydata-google-auth==1.4.0 # via pandas-gbq pyfakefs==4.5.6 # via -r requirements/testing.in @@ -124,9 +124,9 @@ pytest-cov==2.12.1 # via -r requirements/testing.in pytest-mock==3.6.1 # via -r requirements/testing.in -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via google-auth-oauthlib -rsa==4.7.2 +rsa==4.9 # via google-auth statsd==3.3.0 # via -r requirements/testing.in diff --git a/setup.cfg b/setup.cfg index 7f2f83c18..c4cd568c5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pybigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml +known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml multi_line_output = 3 order_by_type = false diff --git a/setup.py b/setup.py index 81d5789d8..969cb215a 100644 --- a/setup.py +++ b/setup.py @@ -126,9 +126,9 @@ setup( "athena": ["pyathena[pandas]>=2, <3"], "aurora-data-api": ["preset-sqlalchemy-aurora-data-api>=0.2.8,<0.3"], "bigquery": [ - "pandas_gbq>=0.10.0", - "pybigquery>=0.4.10", - "google-cloud-bigquery>=2.4.0", + "pandas-gbq>=0.18.1", + "sqlalchemy-bigquery>=1.5.0", + "google-cloud-bigquery>=3.4.0", ], "clickhouse": ["clickhouse-connect>=0.4.6, <0.5"], "cockroachdb": ["cockroachdb>=0.3.5, <0.4"], diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index bcaec8f6e..f3ed3d9d2 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -105,13 +105,13 @@ class BigQueryEngineSpec(BaseEngineSpec): """ https://www.python.org/dev/peps/pep-0249/#arraysize - raw_connections bypass the pybigquery query execution context and deal with + raw_connections bypass the sqlalchemy-bigquery query execution context and deal with raw dbapi connection directly. If this value is not set, the default value is set to 1, as described here, https://googlecloudplatform.github.io/google-cloud-python/latest/_modules/google/cloud/bigquery/dbapi/cursor.html#Cursor - The default value of 5000 is derived from the pybigquery. - https://github.com/mxmzdlv/pybigquery/blob/d214bb089ca0807ca9aaa6ce4d5a01172d40264e/pybigquery/sqlalchemy_bigquery.py#L102 + The default value of 5000 is derived from the sqlalchemy-bigquery. + https://github.com/googleapis/python-bigquery-sqlalchemy/blob/4e17259088f89eac155adc19e0985278a29ecf9c/sqlalchemy_bigquery/base.py#L762 """ arraysize = 5000 diff --git a/tests/integration_tests/db_engine_specs/bigquery_tests.py b/tests/integration_tests/db_engine_specs/bigquery_tests.py index 4e09077f5..5f0819258 100644 --- a/tests/integration_tests/db_engine_specs/bigquery_tests.py +++ b/tests/integration_tests/db_engine_specs/bigquery_tests.py @@ -356,7 +356,7 @@ class TestBigQueryDbEngineSpec(TestDbEngineSpec): ] @mock.patch("superset.models.core.Database.db_engine_spec", BigQueryEngineSpec) - @mock.patch("pybigquery._helpers.create_bigquery_client", mock.Mock) + @mock.patch("sqlalchemy_bigquery._helpers.create_bigquery_client", mock.Mock) @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_calculated_column_in_order_by(self): table = self.get_table(name="birth_names") diff --git a/tests/unit_tests/db_engine_specs/test_bigquery.py b/tests/unit_tests/db_engine_specs/test_bigquery.py index 4a5c74153..dda5fbf0e 100644 --- a/tests/unit_tests/db_engine_specs/test_bigquery.py +++ b/tests/unit_tests/db_engine_specs/test_bigquery.py @@ -19,10 +19,10 @@ import json -from pybigquery.sqlalchemy_bigquery import BigQueryDialect from pytest_mock import MockFixture from sqlalchemy import select from sqlalchemy.sql import sqltypes +from sqlalchemy_bigquery import BigQueryDialect def test_get_fields() -> None: