iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: bump pyarrow (#10733) 

* chore: bump pandas an pyarrow

* remove unsupported kwargs

* fix types

* move pandas back to 1.0 branch due to possible MultiIndex regression
This commit is contained in:
Ville Brofeldt 2020-08-31 19:55:53 +03:00 committed by GitHub
parent 234b6bbba9
commit 4392c4608f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 12 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
requirements/base.txt
View File

@ -9,16 +9,16 @@
aiohttp==3.6.2 # via slackclient
alembic==1.4.2 # via flask-migrate
amqp==2.6.1 # via kombu
apispec[yaml]==3.3.1 # via flask-appbuilder
apispec[yaml]==3.3.2 # via flask-appbuilder
async-timeout==3.0.1 # via aiohttp
attrs==20.1.0 # via aiohttp, jsonschema
babel==2.8.0 # via flask-babel
backoff==1.10.0 # via apache-superset
billiard==3.6.3.0 # via celery
bleach==3.1.5 # via apache-superset
boto3==1.14.48 # via tabulator
botocore==1.17.48 # via boto3, s3transfer
brotli==1.0.7 # via flask-compress
boto3==1.14.51 # via tabulator
botocore==1.17.51 # via boto3, s3transfer
brotli==1.0.9 # via flask-compress
cached-property==1.5.1 # via tableschema
cachelib==0.1.1 # via apache-superset
celery==4.4.7 # via apache-superset
@ -29,7 +29,7 @@ click==7.1.2 # via apache-superset, flask, flask-appbuilder, tables
colorama==0.4.3 # via apache-superset, flask-appbuilder
contextlib2==0.6.0.post1 # via apache-superset
croniter==0.3.34 # via apache-superset
cryptography==3.0 # via apache-superset
cryptography==3.1 # via apache-superset
dataclasses==0.6 # via apache-superset
decorator==4.4.2 # via retry
defusedxml==0.6.0 # via python3-openid
@ -41,7 +41,7 @@ flask-appbuilder==3.0.1 # via apache-superset
flask-babel==1.0.0 # via flask-appbuilder
flask-caching==1.9.0 # via apache-superset
flask-compress==1.5.0 # via apache-superset
flask-cors==3.0.8 # via apache-superset
flask-cors==3.0.9 # via apache-superset
flask-jwt-extended==3.24.1 # via flask-appbuilder
flask-login==0.4.1 # via flask-appbuilder
flask-migrate==2.5.3 # via apache-superset
@ -88,7 +88,7 @@ polyline==1.4.0 # via apache-superset
prison==0.1.3 # via flask-appbuilder
psycopg2-binary==2.8.5 # via apache-superset
py==1.9.0 # via retry
pyarrow==0.17.1 # via apache-superset
pyarrow==1.0.1 # via apache-superset
pycparser==2.20 # via cffi
pydruid==0.6.1 # via apache-superset
pyhive[hive]==0.6.3 # via apache-superset

4
requirements/testing.txt
View File

@ -22,13 +22,13 @@ isort==5.4.2 # via pylint
jedi==0.17.2 # via ipython
lazy-object-proxy==1.4.3 # via astroid
mccabe==0.6.1 # via pylint
more-itertools==8.4.0 # via pytest
more-itertools==8.5.0 # via pytest
openapi-spec-validator==0.2.9 # via -r requirements/testing.in
parameterized==0.7.4 # via -r requirements/testing.in
parso==0.7.1 # via jedi
pexpect==4.8.0 # via ipython
pickleshare==0.7.5 # via ipython
prompt-toolkit==3.0.6 # via ipython
prompt-toolkit==3.0.7 # via ipython
ptyprocess==0.6.0 # via pexpect
pygments==2.6.1 # via ipython
pyhive[hive,presto]==0.6.3 # via -r requirements/testing.in, apache-superset

4
setup.py
View File

@ -90,14 +90,14 @@ setup(
"isodate",
"markdown>=3.0",
"msgpack>=1.0.0, <1.1",
"pandas>=1.0.3, <1.1",
"pandas>=1.0.5, <1.1",
"parsedatetime",
"pathlib2",
"polyline",
"python-dateutil",
"python-dotenv",
"python-geohash",
"pyarrow>=0.17.0, <0.18",
"pyarrow>=1.0.1, <1.1",
"pyyaml>=5.1",
"retry>=0.9.2",
"selenium>=3.141.0",

13
superset/db_engine_specs/base.py
View File

@ -441,17 +441,6 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
parsed_query = sql_parse.ParsedQuery(sql)
return parsed_query.set_or_update_query_limit(limit)
@staticmethod
def excel_to_df(**kwargs: Any) -> pd.DataFrame:
""" Read excel into Pandas DataFrame
:param kwargs: params to be passed to DataFrame.read_excel
:return: Pandas DataFrame containing data from excel
"""
kwargs["encoding"] = "utf-8"
kwargs["iterator"] = True
df = pd.read_excel(**kwargs)
return df
@staticmethod
def csv_to_df(**kwargs: Any) -> pd.DataFrame:
""" Read csv into Pandas DataFrame
@ -521,7 +510,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
Create table from contents of a excel. Note: this method does not create
metadata for the table.
"""
df = cls.excel_to_df(io=filename, **excel_to_df_kwargs,)
df = pd.read_excel(io=filename, **excel_to_df_kwargs)
engine = cls.get_engine(database)
if table.schema:
# only add schema when it is preset and non empty