Getting ready to gut panoramix
This commit is contained in:
Maxime
parent
736e102cae
commit
28c0292920
111
app/models.py
111
app/models.py
|
|
@ -3,6 +3,8 @@ from pydruid import client
|
|||
from datetime import timedelta
|
||||
from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean, DateTime
|
||||
from sqlalchemy import create_engine, MetaData
|
||||
from sqlalchemy import Table as sqlaTable
|
||||
from sqlalchemy.orm import relationship
|
||||
from app import get_session
|
||||
from dateutil.parser import parse
|
||||
|
|
@ -11,6 +13,102 @@ import logging
|
|||
import json
|
||||
import requests
|
||||
|
||||
from app import db
|
||||
|
||||
class Queryable(object):
|
||||
@property
|
||||
def column_names(self):
|
||||
return sorted([c.column_name for c in self.columns])
|
||||
|
||||
@property
|
||||
def groupby_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.groupby])
|
||||
|
||||
@property
|
||||
def filterable_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.filterable])
|
||||
|
||||
class Database(Model, AuditMixin):
|
||||
__tablename__ = 'databases'
|
||||
id = Column(Integer, primary_key=True)
|
||||
database_name = Column(String(256), unique=True)
|
||||
sqlalchemy_uri = Column(String(1024))
|
||||
|
||||
def __repr__(self):
|
||||
return self.database_name
|
||||
|
||||
|
||||
class Table(Model, AuditMixin, Queryable):
|
||||
__tablename__ = 'tables'
|
||||
id = Column(Integer, primary_key=True)
|
||||
table_name = Column(String(256), unique=True)
|
||||
default_endpoint = Column(Text)
|
||||
database_id = Column(
|
||||
String(256), ForeignKey('databases.id'))
|
||||
database = relationship(
|
||||
'Database', backref='tables', foreign_keys=[database_id])
|
||||
|
||||
@property
|
||||
def table_link(self):
|
||||
url = "/panoramix/table/{}/".format(self.id)
|
||||
return '<a href="{url}">{self.table_name}</a>'.format(**locals())
|
||||
|
||||
@property
|
||||
def metrics_combo(self):
|
||||
return sorted(
|
||||
[
|
||||
(
|
||||
'sum__{}'.format(m.column_name),
|
||||
'SUM({})'.format(m.column_name),
|
||||
)
|
||||
for m in self.columns if m.sum],
|
||||
key=lambda x: x[1])
|
||||
|
||||
def fetch_metadata(self):
|
||||
engine = create_engine(self.database.sqlalchemy_uri)
|
||||
meta = MetaData()
|
||||
table = sqlaTable(
|
||||
self.table_name, meta, autoload=True, autoload_with=engine)
|
||||
TC = TableColumn
|
||||
for col in table.columns:
|
||||
dbcol = (
|
||||
db.session
|
||||
.query(TC)
|
||||
.filter(TC.table==self)
|
||||
.filter(TC.column_name==col.name)
|
||||
.first()
|
||||
)
|
||||
db.session.flush()
|
||||
if not dbcol:
|
||||
dbcol = TableColumn(column_name=col.name)
|
||||
if str(col.type) in ('VARCHAR', 'STRING'):
|
||||
dbcol.groupby = True
|
||||
dbcol.filterable = True
|
||||
self.columns.append(dbcol)
|
||||
|
||||
dbcol.type = str(col.type)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
class TableColumn(Model, AuditMixin):
|
||||
__tablename__ = 'table_columns'
|
||||
id = Column(Integer, primary_key=True)
|
||||
table_id = Column(
|
||||
String(256),
|
||||
ForeignKey('tables.id'))
|
||||
table = relationship('Table', backref='columns')
|
||||
column_name = Column(String(256))
|
||||
is_dttm = Column(Boolean, default=True)
|
||||
is_active = Column(Boolean, default=True)
|
||||
type = Column(String(32), default='')
|
||||
groupby = Column(Boolean, default=False)
|
||||
count_distinct = Column(Boolean, default=False)
|
||||
sum = Column(Boolean, default=False)
|
||||
max = Column(Boolean, default=False)
|
||||
min = Column(Boolean, default=False)
|
||||
filterable = Column(Boolean, default=False)
|
||||
description = Column(Text, default='')
|
||||
|
||||
|
||||
class Cluster(Model, AuditMixin):
|
||||
__tablename__ = 'clusters'
|
||||
|
|
@ -46,7 +144,7 @@ class Cluster(Model, AuditMixin):
|
|||
# logging.exception(e)
|
||||
# logging.error("Failed at syncing " + datasource)
|
||||
|
||||
class Datasource(Model, AuditMixin):
|
||||
class Datasource(Model, AuditMixin, Queryable):
|
||||
__tablename__ = 'datasources'
|
||||
id = Column(Integer, primary_key=True)
|
||||
datasource_name = Column(String(256), unique=True)
|
||||
|
|
@ -130,17 +228,6 @@ class Datasource(Model, AuditMixin):
|
|||
col_obj.generate_metrics()
|
||||
#session.commit()
|
||||
|
||||
@property
|
||||
def column_names(self):
|
||||
return sorted([c.column_name for c in self.columns])
|
||||
|
||||
@property
|
||||
def groupby_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.groupby])
|
||||
|
||||
@property
|
||||
def filterable_column_names(self):
|
||||
return sorted([c.column_name for c in self.columns if c.filterable])
|
||||
|
||||
|
||||
class Metric(Model):
|
||||
|
|
|
|||
80
app/views.py
80
app/views.py
|
|
@ -27,6 +27,22 @@ class DeleteMixin(object):
|
|||
return redirect(self.get_redirect())
|
||||
|
||||
|
||||
class TableColumnInlineView(CompactCRUDMixin, ModelView):
|
||||
datamodel = SQLAInterface(models.TableColumn)
|
||||
can_delete = False
|
||||
edit_columns = [
|
||||
'column_name', 'description', 'table', 'groupby', 'filterable',
|
||||
'count_distinct', 'sum', 'min', 'max']
|
||||
list_columns = [
|
||||
'column_name', 'type', 'groupby', 'count_distinct',
|
||||
'sum', 'min', 'max']
|
||||
page_size = 100
|
||||
list_columns = [
|
||||
'column_name', 'type', 'groupby', 'count_distinct',
|
||||
'sum', 'min', 'max']
|
||||
appbuilder.add_view_no_menu(TableColumnInlineView)
|
||||
|
||||
|
||||
class ColumnInlineView(CompactCRUDMixin, ModelView):
|
||||
datamodel = SQLAInterface(models.Column)
|
||||
edit_columns = [
|
||||
|
|
@ -80,6 +96,39 @@ appbuilder.add_view(
|
|||
category_icon='fa-cogs',)
|
||||
|
||||
|
||||
class DatabaseView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Database)
|
||||
list_columns = ['database_name']
|
||||
add_columns = ['database_name', 'sqlalchemy_uri']
|
||||
edit_columns = add_columns
|
||||
|
||||
appbuilder.add_view(
|
||||
DatabaseView,
|
||||
"Databases",
|
||||
icon="fa-database",
|
||||
category="Admin",
|
||||
category_icon='fa-cogs',)
|
||||
|
||||
|
||||
class TableView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Table)
|
||||
list_columns = ['table_link', 'database']
|
||||
add_columns = ['table_name', 'database']
|
||||
edit_columns = add_columns
|
||||
related_views = [TableColumnInlineView]
|
||||
|
||||
def post_insert(self, table):
|
||||
table.fetch_metadata()
|
||||
|
||||
def post_update(self, table):
|
||||
table.fetch_metadata()
|
||||
|
||||
appbuilder.add_view(
|
||||
TableView,
|
||||
"Tables",
|
||||
icon='fa-table',)
|
||||
|
||||
|
||||
class DatasourceModelView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Datasource)
|
||||
list_columns = [
|
||||
|
|
@ -101,8 +150,7 @@ class DatasourceModelView(ModelView, DeleteMixin):
|
|||
appbuilder.add_view(
|
||||
DatasourceModelView,
|
||||
"Druid Datasources",
|
||||
icon="fa-cube",
|
||||
category_icon='fa-envelope')
|
||||
icon="fa-cube")
|
||||
|
||||
|
||||
@app.route('/health')
|
||||
|
|
@ -116,6 +164,34 @@ def ping():
|
|||
|
||||
|
||||
class Panoramix(BaseView):
|
||||
@has_access
|
||||
@permission_name('tables')
|
||||
@expose("/table/<table_id>/")
|
||||
def table(self, table_id):
|
||||
|
||||
table = (
|
||||
db.session
|
||||
.query(models.Table)
|
||||
.filter_by(id=table_id)
|
||||
.first()
|
||||
)
|
||||
viz_type = request.args.get("viz_type")
|
||||
if not viz_type and table.default_endpoint:
|
||||
return redirect(table.default_endpoint)
|
||||
if not viz_type:
|
||||
viz_type = "table"
|
||||
obj = viz.viz_types[viz_type](
|
||||
table,
|
||||
form_data=request.args, view=self)
|
||||
if request.args.get("json"):
|
||||
return Response(
|
||||
json.dumps(obj.get_query(), indent=4),
|
||||
status=200,
|
||||
mimetype="application/json")
|
||||
if obj.df is None or obj.df.empty:
|
||||
return obj.render_no_data()
|
||||
return obj.render()
|
||||
|
||||
@has_access
|
||||
@permission_name('datasources')
|
||||
@expose("/datasource/<datasource_name>/")
|
||||
|
|
|
|||
Loading…
Reference in New Issue