iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: Bump bigquery libs (#21466)

This commit is contained in:
EugeneTorap 2022-12-02 13:48:36 +03:00 committed by GitHub
parent 93cdcf1d92
commit 7bc5f04368
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 37 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/docs/databases/bigquery.mdx
View File

@ -8,7 +8,7 @@ version: 1
## Google BigQuery
The recommended connector library for BigQuery is
[pybigquery](https://github.com/mxmzdlv/pybigquery).
[sqlalchemy-bigquery](https://github.com/googleapis/python-bigquery-sqlalchemy).
### Install BigQuery Driver
@ -16,7 +16,7 @@ Follow the steps [here](/docs/databases/docker-add-drivers) about how to
install new database drivers when setting up Superset locally via docker-compose.
```
echo "pybigquery" >> ./docker/requirements-local.txt
echo "sqlalchemy-bigquery" >> ./docker/requirements-local.txt
```
### Connecting to BigQuery

2
docs/docs/databases/installing-database-drivers.mdx
View File

@ -35,7 +35,7 @@ A list of some of the recommended packages.
| [Apache Spark SQL](/docs/databases/spark-sql) | `pip install pyhive` | `hive://hive@{hostname}:{port}/{database}` |
| [Ascend.io](/docs/databases/ascend) | `pip install impyla` | `ascend://{username}:{password}@{hostname}:{port}/{database}?auth_mechanism=PLAIN;use_ssl=true` |
| [Azure MS SQL](/docs/databases/sql-server) | `pip install pymssql` | `mssql+pymssql://UserName@presetSQL:TestPassword@presetSQL.database.windows.net:1433/TestSchema` |
| [Big Query](/docs/databases/bigquery) | `pip install pybigquery` | `bigquery://{project_id}` |
| [Big Query](/docs/databases/bigquery) | `pip install sqlalchemy-bigquery` | `bigquery://{project_id}` |
| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb://{username}:{password}@{hostname}:{port}/{database}` |
| [CockroachDB](/docs/databases/cockroachdb) | `pip install cockroachdb` | `cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable` |
| [Dremio](/docs/databases/dremio) | `pip install sqlalchemy_dremio` | `dremio://user:pwd@host:31010/` |

2
docs/docs/installation/running-on-kubernetes.mdx
View File

@ -132,7 +132,7 @@ bootstrapScript: |
#!/bin/bash
pip install psycopg2==2.9.1 \
redis==3.2.1 \
pybigquery==2.26.0 \
sqlalchemy-bigquery==1.5.0 \
elasticsearch-dbapi==0.2.5 &&\
if [ ! -f ~/bootstrap ]; then echo "Running Superset with uid {{ .Values.runAsUser }}" > ~/bootstrap; fi
```

48
requirements/testing.txt
View File

@ -14,7 +14,7 @@
# -r requirements/testing.in
astroid==2.6.6
# via pylint
cachetools==4.2.4
cachetools==5.2.0
# via google-auth
coverage==5.5
# via pytest-cov
@ -24,47 +24,47 @@ flask-testing==0.8.1
# via -r requirements/testing.in
freezegun==1.1.0
# via -r requirements/testing.in
google-api-core[grpc]==2.2.1
google-api-core[grpc]==2.11.0
# via
# google-cloud-bigquery
# google-cloud-bigquery-storage
# google-cloud-core
# pybigquery
google-auth==2.2.1
# sqlalchemy-bigquery
google-auth==2.14.1
# via
# google-api-core
# google-auth-oauthlib
# google-cloud-core
# pandas-gbq
# pybigquery
# sqlalchemy-bigquery
# pydata-google-auth
google-auth-oauthlib==0.4.6
google-auth-oauthlib==0.7.1
# via
# pandas-gbq
# pydata-google-auth
google-cloud-bigquery[bqstorage,pandas]==2.29.0
google-cloud-bigquery[bqstorage,pandas]==3.4.0
# via
# apache-superset
# pandas-gbq
# pybigquery
google-cloud-bigquery-storage==2.9.1
# sqlalchemy-bigquery
google-cloud-bigquery-storage==2.16.2
# via google-cloud-bigquery
google-cloud-core==2.1.0
google-cloud-core==2.3.2
# via google-cloud-bigquery
google-crc32c==1.3.0
google-crc32c==1.5.0
# via google-resumable-media
google-resumable-media==2.1.0
google-resumable-media==2.4.0
# via google-cloud-bigquery
googleapis-common-protos==1.53.0
googleapis-common-protos==1.57.0
# via
# google-api-core
# grpcio-status
grpcio==1.41.1
grpcio==1.51.1
# via
# google-api-core
# google-cloud-bigquery
# grpcio-status
grpcio-status==1.41.1
grpcio-status==1.51.1
# via google-api-core
iniconfig==1.1.1
# via pytest
@ -72,27 +72,27 @@ isort==5.9.3
# via pylint
lazy-object-proxy==1.6.0
# via astroid
libcst==0.3.21
libcst==0.4.9
# via google-cloud-bigquery-storage
mccabe==0.6.1
# via pylint
mypy-extensions==0.4.3
# via typing-inspect
oauthlib==3.1.1
oauthlib==3.2.2
# via requests-oauthlib
openapi-schema-validator==0.1.5
# via openapi-spec-validator
openapi-spec-validator==0.3.1
# via -r requirements/testing.in
pandas-gbq==0.15.0
pandas-gbq==0.18.1
# via apache-superset
parameterized==0.8.1
# via -r requirements/testing.in
proto-plus==1.19.7
proto-plus==1.22.1
# via
# google-cloud-bigquery
# google-cloud-bigquery-storage
protobuf==3.19.1
protobuf==4.21.10
# via
# google-api-core
# google-cloud-bigquery
@ -105,9 +105,9 @@ pyasn1==0.4.8
# rsa
pyasn1-modules==0.2.8
# via google-auth
pybigquery==0.10.2
sqlalchemy-bigquery==1.5.0
# via apache-superset
pydata-google-auth==1.2.0
pydata-google-auth==1.4.0
# via pandas-gbq
pyfakefs==4.5.6
# via -r requirements/testing.in
@ -124,9 +124,9 @@ pytest-cov==2.12.1
# via -r requirements/testing.in
pytest-mock==3.6.1
# via -r requirements/testing.in
requests-oauthlib==1.3.0
requests-oauthlib==1.3.1
# via google-auth-oauthlib
rsa==4.7.2
rsa==4.9
# via google-auth
statsd==3.3.0
# via -r requirements/testing.in

2
setup.cfg
View File

@ -30,7 +30,7 @@ combine_as_imports = true
include_trailing_comma = true
line_length = 88
known_first_party = superset
known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pybigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml
known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml
multi_line_output = 3
order_by_type = false

6
setup.py
View File

@ -126,9 +126,9 @@ setup(
"athena": ["pyathena[pandas]>=2, <3"],
"aurora-data-api": ["preset-sqlalchemy-aurora-data-api>=0.2.8,<0.3"],
"bigquery": [
"pandas_gbq>=0.10.0",
"pybigquery>=0.4.10",
"google-cloud-bigquery>=2.4.0",
"pandas-gbq>=0.18.1",
"sqlalchemy-bigquery>=1.5.0",
"google-cloud-bigquery>=3.4.0",
],
"clickhouse": ["clickhouse-connect>=0.4.6, <0.5"],
"cockroachdb": ["cockroachdb>=0.3.5, <0.4"],

6
superset/db_engine_specs/bigquery.py
View File

@ -105,13 +105,13 @@ class BigQueryEngineSpec(BaseEngineSpec):
"""
https://www.python.org/dev/peps/pep-0249/#arraysize
raw_connections bypass the pybigquery query execution context and deal with
raw_connections bypass the sqlalchemy-bigquery query execution context and deal with
raw dbapi connection directly.
If this value is not set, the default value is set to 1, as described here,
https://googlecloudplatform.github.io/google-cloud-python/latest/_modules/google/cloud/bigquery/dbapi/cursor.html#Cursor
The default value of 5000 is derived from the pybigquery.
https://github.com/mxmzdlv/pybigquery/blob/d214bb089ca0807ca9aaa6ce4d5a01172d40264e/pybigquery/sqlalchemy_bigquery.py#L102
The default value of 5000 is derived from the sqlalchemy-bigquery.
https://github.com/googleapis/python-bigquery-sqlalchemy/blob/4e17259088f89eac155adc19e0985278a29ecf9c/sqlalchemy_bigquery/base.py#L762
"""
arraysize = 5000

2
tests/integration_tests/db_engine_specs/bigquery_tests.py
View File

@ -356,7 +356,7 @@ class TestBigQueryDbEngineSpec(TestDbEngineSpec):
]
@mock.patch("superset.models.core.Database.db_engine_spec", BigQueryEngineSpec)
@mock.patch("pybigquery._helpers.create_bigquery_client", mock.Mock)
@mock.patch("sqlalchemy_bigquery._helpers.create_bigquery_client", mock.Mock)
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
def test_calculated_column_in_order_by(self):
table = self.get_table(name="birth_names")

2
tests/unit_tests/db_engine_specs/test_bigquery.py
View File

@ -19,10 +19,10 @@
import json
from pybigquery.sqlalchemy_bigquery import BigQueryDialect
from pytest_mock import MockFixture
from sqlalchemy import select
from sqlalchemy.sql import sqltypes
from sqlalchemy_bigquery import BigQueryDialect
def test_get_fields() -> None: