iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: remove root dir from ZIP bundle (#11805) 

* fix: remove root dir from ZIP bundle

* Fix lint

* Fix tests

* Improve tests

* Fix dashboard as well
This commit is contained in:
Beto Dealmeida 2020-11-25 11:47:48 -08:00 committed by GitHub
parent 4cbf482194
commit c354e7e0ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 99 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
superset/charts/api.py
View File

@ -61,6 +61,7 @@ from superset.charts.schemas import (
thumbnail_query_schema,
)
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import remove_root
from superset.constants import RouteMethod
from superset.exceptions import SupersetSecurityException
from superset.extensions import event_logger
@ -869,12 +870,12 @@ class ChartRestApi(BaseSupersetModelRestApi):
500:
$ref: '#/components/responses/500'
"""
upload = request.files.get("file")
upload = request.files.get("formData")
if not upload:
return self.response_400()
with ZipFile(upload) as bundle:
contents = {
file_name: bundle.read(file_name).decode()
remove_root(file_name): bundle.read(file_name).decode()
for file_name in bundle.namelist()
}

12
superset/commands/importers/v1/utils.py
View File

@ -14,6 +14,7 @@
# under the License.
import logging
from pathlib import Path
from typing import Any, Dict
import yaml
@ -28,6 +29,13 @@ IMPORT_VERSION = "1.0.0"
logger = logging.getLogger(__name__)
def remove_root(file_path: str) -> str:
"""Remove the first directory of a path"""
full_path = Path(file_path)
relative_path = Path(*full_path.parts[1:])
return str(relative_path)
class MetadataSchema(Schema):
version = fields.String(required=True, validate=validate.Equal(IMPORT_VERSION))
type = fields.String(required=True)
@ -39,14 +47,14 @@ def load_yaml(file_name: str, content: str) -> Dict[str, Any]:
try:
return yaml.safe_load(content)
except yaml.parser.ParserError:
logger.exception("Invalid YAML in %s", METADATA_FILE_NAME)
logger.exception("Invalid YAML in %s", file_name)
raise ValidationError({file_name: "Not a valid YAML file"})
def load_metadata(contents: Dict[str, str]) -> Dict[str, str]:
"""Apply validation and load a metadata file"""
if METADATA_FILE_NAME not in contents:
# if the contents ahve no METADATA_FILE_NAME this is probably
# if the contents have no METADATA_FILE_NAME this is probably
# a original export without versioning that should not be
# handled by this command
raise IncorrectVersionError(f"Missing {METADATA_FILE_NAME}")

5
superset/dashboards/api.py
View File

@ -30,6 +30,7 @@ from werkzeug.wsgi import FileWrapper
from superset import is_feature_enabled, thumbnail_cache
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import remove_root
from superset.constants import RouteMethod
from superset.dashboards.commands.bulk_delete import BulkDeleteDashboardCommand
from superset.dashboards.commands.create import CreateDashboardCommand
@ -687,12 +688,12 @@ class DashboardRestApi(BaseSupersetModelRestApi):
500:
$ref: '#/components/responses/500'
"""
upload = request.files.get("file")
upload = request.files.get("formData")
if not upload:
return self.response_400()
with ZipFile(upload) as bundle:
contents = {
file_name: bundle.read(file_name).decode()
remove_root(file_name): bundle.read(file_name).decode()
for file_name in bundle.namelist()
}

5
superset/databases/api.py
View File

@ -36,6 +36,7 @@ from sqlalchemy.exc import (
from superset import event_logger
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import remove_root
from superset.constants import RouteMethod
from superset.databases.commands.create import CreateDatabaseCommand
from superset.databases.commands.delete import DeleteDatabaseCommand
@ -766,12 +767,12 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
500:
$ref: '#/components/responses/500'
"""
upload = request.files.get("file")
upload = request.files.get("formData")
if not upload:
return self.response_400()
with ZipFile(upload) as bundle:
contents = {
file_name: bundle.read(file_name).decode()
remove_root(file_name): bundle.read(file_name).decode()
for file_name in bundle.namelist()
}

5
superset/datasets/api.py
View File

@ -29,6 +29,7 @@ from marshmallow import ValidationError
from superset import event_logger, is_feature_enabled
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.v1.utils import remove_root
from superset.connectors.sqla.models import SqlaTable
from superset.constants import RouteMethod
from superset.databases.filters import DatabaseFilter
@ -634,12 +635,12 @@ class DatasetRestApi(BaseSupersetModelRestApi):
500:
$ref: '#/components/responses/500'
"""
upload = request.files.get("file")
upload = request.files.get("formData")
if not upload:
return self.response_400()
with ZipFile(upload) as bundle:
contents = {
file_name: bundle.read(file_name).decode()
remove_root(file_name): bundle.read(file_name).decode()
for file_name in bundle.namelist()
}

24
tests/charts/api_tests.py
View File

@ -1187,18 +1187,20 @@ class TestChartApi(SupersetTestCase, ApiOwnersTestCaseMixin):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("chart_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(chart_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"chart_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open("chart_export/datasets/imported_dataset.yaml", "w") as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
with bundle.open("charts/imported_chart.yaml", "w") as fp:
with bundle.open("chart_export/charts/imported_chart.yaml", "w") as fp:
fp.write(yaml.safe_dump(chart_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "chart_export.zip"),
"formData": (buf, "chart_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
@ -1233,18 +1235,20 @@ class TestChartApi(SupersetTestCase, ApiOwnersTestCaseMixin):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("chart_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(dataset_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"chart_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open("chart_export/datasets/imported_dataset.yaml", "w") as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
with bundle.open("charts/imported_chart.yaml", "w") as fp:
with bundle.open("chart_export/charts/imported_chart.yaml", "w") as fp:
fp.write(yaml.safe_dump(chart_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "chart_export.zip"),
"formData": (buf, "chart_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))

36
tests/dashboards/api_tests.py
View File

@ -1096,20 +1096,26 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("dashboard_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(dashboard_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"dashboard_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"dashboard_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
with bundle.open("charts/imported_chart.yaml", "w") as fp:
with bundle.open("dashboard_export/charts/imported_chart.yaml", "w") as fp:
fp.write(yaml.safe_dump(chart_config).encode())
with bundle.open("dashboards/imported_dashboard.yaml", "w") as fp:
with bundle.open(
"dashboard_export/dashboards/imported_dashboard.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dashboard_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "dashboard_export.zip"),
"formData": (buf, "dashboard_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
@ -1147,20 +1153,26 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("dashboard_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(dataset_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"dashboard_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"dashboard_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
with bundle.open("charts/imported_chart.yaml", "w") as fp:
with bundle.open("dashboard_export/charts/imported_chart.yaml", "w") as fp:
fp.write(yaml.safe_dump(chart_config).encode())
with bundle.open("dashboards/imported_dashboard.yaml", "w") as fp:
with bundle.open(
"dashboard_export/dashboards/imported_dashboard.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dashboard_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "dashboard_export.zip"),
"formData": (buf, "dashboard_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))

24
tests/databases/api_tests.py
View File

@ -836,16 +836,20 @@ class TestDatabaseApi(SupersetTestCase):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("database_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(database_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"database_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"database_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "database_export.zip"),
"formData": (buf, "database_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
@ -876,16 +880,20 @@ class TestDatabaseApi(SupersetTestCase):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("database_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(dataset_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"database_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"database_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "database_export.zip"),
"formData": (buf, "database_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))

34
tests/datasets/api_tests.py
View File

@ -1185,16 +1185,20 @@ class TestDatasetApi(SupersetTestCase):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("dataset_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(dataset_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"dataset_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"dataset_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "dataset_export.zip"),
"formData": (buf, "dataset_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
@ -1225,16 +1229,20 @@ class TestDatasetApi(SupersetTestCase):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("metadata.yaml", "w") as fp:
with bundle.open("dataset_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(database_metadata_config).encode())
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"dataset_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"dataset_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "dataset_export.zip"),
"formData": (buf, "dataset_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
@ -1253,14 +1261,18 @@ class TestDatasetApi(SupersetTestCase):
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("databases/imported_database.yaml", "w") as fp:
with bundle.open(
"dataset_export/databases/imported_database.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(database_config).encode())
with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
with bundle.open(
"dataset_export/datasets/imported_dataset.yaml", "w"
) as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"file": (buf, "dataset_export.zip"),
"formData": (buf, "dataset_export.zip"),
}
rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))