diff --git a/app/models.py b/app/models.py index 1a7630a0a..49eb37014 100644 --- a/app/models.py +++ b/app/models.py @@ -3,6 +3,8 @@ from pydruid import client from datetime import timedelta from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean, DateTime +from sqlalchemy import create_engine, MetaData +from sqlalchemy import Table as sqlaTable from sqlalchemy.orm import relationship from app import get_session from dateutil.parser import parse @@ -11,6 +13,102 @@ import logging import json import requests +from app import db + +class Queryable(object): + @property + def column_names(self): + return sorted([c.column_name for c in self.columns]) + + @property + def groupby_column_names(self): + return sorted([c.column_name for c in self.columns if c.groupby]) + + @property + def filterable_column_names(self): + return sorted([c.column_name for c in self.columns if c.filterable]) + +class Database(Model, AuditMixin): + __tablename__ = 'databases' + id = Column(Integer, primary_key=True) + database_name = Column(String(256), unique=True) + sqlalchemy_uri = Column(String(1024)) + + def __repr__(self): + return self.database_name + + +class Table(Model, AuditMixin, Queryable): + __tablename__ = 'tables' + id = Column(Integer, primary_key=True) + table_name = Column(String(256), unique=True) + default_endpoint = Column(Text) + database_id = Column( + String(256), ForeignKey('databases.id')) + database = relationship( + 'Database', backref='tables', foreign_keys=[database_id]) + + @property + def table_link(self): + url = "/panoramix/table/{}/".format(self.id) + return '{self.table_name}'.format(**locals()) + + @property + def metrics_combo(self): + return sorted( + [ + ( + 'sum__{}'.format(m.column_name), + 'SUM({})'.format(m.column_name), + ) + for m in self.columns if m.sum], + key=lambda x: x[1]) + + def fetch_metadata(self): + engine = create_engine(self.database.sqlalchemy_uri) + meta = MetaData() + table = sqlaTable( + self.table_name, meta, autoload=True, autoload_with=engine) + TC = TableColumn + for col in table.columns: + dbcol = ( + db.session + .query(TC) + .filter(TC.table==self) + .filter(TC.column_name==col.name) + .first() + ) + db.session.flush() + if not dbcol: + dbcol = TableColumn(column_name=col.name) + if str(col.type) in ('VARCHAR', 'STRING'): + dbcol.groupby = True + dbcol.filterable = True + self.columns.append(dbcol) + + dbcol.type = str(col.type) + db.session.commit() + + +class TableColumn(Model, AuditMixin): + __tablename__ = 'table_columns' + id = Column(Integer, primary_key=True) + table_id = Column( + String(256), + ForeignKey('tables.id')) + table = relationship('Table', backref='columns') + column_name = Column(String(256)) + is_dttm = Column(Boolean, default=True) + is_active = Column(Boolean, default=True) + type = Column(String(32), default='') + groupby = Column(Boolean, default=False) + count_distinct = Column(Boolean, default=False) + sum = Column(Boolean, default=False) + max = Column(Boolean, default=False) + min = Column(Boolean, default=False) + filterable = Column(Boolean, default=False) + description = Column(Text, default='') + class Cluster(Model, AuditMixin): __tablename__ = 'clusters' @@ -46,7 +144,7 @@ class Cluster(Model, AuditMixin): # logging.exception(e) # logging.error("Failed at syncing " + datasource) -class Datasource(Model, AuditMixin): +class Datasource(Model, AuditMixin, Queryable): __tablename__ = 'datasources' id = Column(Integer, primary_key=True) datasource_name = Column(String(256), unique=True) @@ -130,17 +228,6 @@ class Datasource(Model, AuditMixin): col_obj.generate_metrics() #session.commit() - @property - def column_names(self): - return sorted([c.column_name for c in self.columns]) - - @property - def groupby_column_names(self): - return sorted([c.column_name for c in self.columns if c.groupby]) - - @property - def filterable_column_names(self): - return sorted([c.column_name for c in self.columns if c.filterable]) class Metric(Model): diff --git a/app/views.py b/app/views.py index 561802c18..c3ab428a6 100644 --- a/app/views.py +++ b/app/views.py @@ -27,6 +27,22 @@ class DeleteMixin(object): return redirect(self.get_redirect()) +class TableColumnInlineView(CompactCRUDMixin, ModelView): + datamodel = SQLAInterface(models.TableColumn) + can_delete = False + edit_columns = [ + 'column_name', 'description', 'table', 'groupby', 'filterable', + 'count_distinct', 'sum', 'min', 'max'] + list_columns = [ + 'column_name', 'type', 'groupby', 'count_distinct', + 'sum', 'min', 'max'] + page_size = 100 + list_columns = [ + 'column_name', 'type', 'groupby', 'count_distinct', + 'sum', 'min', 'max'] +appbuilder.add_view_no_menu(TableColumnInlineView) + + class ColumnInlineView(CompactCRUDMixin, ModelView): datamodel = SQLAInterface(models.Column) edit_columns = [ @@ -80,6 +96,39 @@ appbuilder.add_view( category_icon='fa-cogs',) +class DatabaseView(ModelView, DeleteMixin): + datamodel = SQLAInterface(models.Database) + list_columns = ['database_name'] + add_columns = ['database_name', 'sqlalchemy_uri'] + edit_columns = add_columns + +appbuilder.add_view( + DatabaseView, + "Databases", + icon="fa-database", + category="Admin", + category_icon='fa-cogs',) + + +class TableView(ModelView, DeleteMixin): + datamodel = SQLAInterface(models.Table) + list_columns = ['table_link', 'database'] + add_columns = ['table_name', 'database'] + edit_columns = add_columns + related_views = [TableColumnInlineView] + + def post_insert(self, table): + table.fetch_metadata() + + def post_update(self, table): + table.fetch_metadata() + +appbuilder.add_view( + TableView, + "Tables", + icon='fa-table',) + + class DatasourceModelView(ModelView, DeleteMixin): datamodel = SQLAInterface(models.Datasource) list_columns = [ @@ -101,8 +150,7 @@ class DatasourceModelView(ModelView, DeleteMixin): appbuilder.add_view( DatasourceModelView, "Druid Datasources", - icon="fa-cube", - category_icon='fa-envelope') + icon="fa-cube") @app.route('/health') @@ -116,6 +164,34 @@ def ping(): class Panoramix(BaseView): + @has_access + @permission_name('tables') + @expose("/table//") + def table(self, table_id): + + table = ( + db.session + .query(models.Table) + .filter_by(id=table_id) + .first() + ) + viz_type = request.args.get("viz_type") + if not viz_type and table.default_endpoint: + return redirect(table.default_endpoint) + if not viz_type: + viz_type = "table" + obj = viz.viz_types[viz_type]( + table, + form_data=request.args, view=self) + if request.args.get("json"): + return Response( + json.dumps(obj.get_query(), indent=4), + status=200, + mimetype="application/json") + if obj.df is None or obj.df.empty: + return obj.render_no_data() + return obj.render() + @has_access @permission_name('datasources') @expose("/datasource//")