diff --git a/app/models.py b/app/models.py
index 1a7630a0a..49eb37014 100644
--- a/app/models.py
+++ b/app/models.py
@@ -3,6 +3,8 @@ from pydruid import client
from datetime import timedelta
from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn
from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean, DateTime
+from sqlalchemy import create_engine, MetaData
+from sqlalchemy import Table as sqlaTable
from sqlalchemy.orm import relationship
from app import get_session
from dateutil.parser import parse
@@ -11,6 +13,102 @@ import logging
import json
import requests
+from app import db
+
+class Queryable(object):
+ @property
+ def column_names(self):
+ return sorted([c.column_name for c in self.columns])
+
+ @property
+ def groupby_column_names(self):
+ return sorted([c.column_name for c in self.columns if c.groupby])
+
+ @property
+ def filterable_column_names(self):
+ return sorted([c.column_name for c in self.columns if c.filterable])
+
+class Database(Model, AuditMixin):
+ __tablename__ = 'databases'
+ id = Column(Integer, primary_key=True)
+ database_name = Column(String(256), unique=True)
+ sqlalchemy_uri = Column(String(1024))
+
+ def __repr__(self):
+ return self.database_name
+
+
+class Table(Model, AuditMixin, Queryable):
+ __tablename__ = 'tables'
+ id = Column(Integer, primary_key=True)
+ table_name = Column(String(256), unique=True)
+ default_endpoint = Column(Text)
+ database_id = Column(
+ String(256), ForeignKey('databases.id'))
+ database = relationship(
+ 'Database', backref='tables', foreign_keys=[database_id])
+
+ @property
+ def table_link(self):
+ url = "/panoramix/table/{}/".format(self.id)
+ return '{self.table_name}'.format(**locals())
+
+ @property
+ def metrics_combo(self):
+ return sorted(
+ [
+ (
+ 'sum__{}'.format(m.column_name),
+ 'SUM({})'.format(m.column_name),
+ )
+ for m in self.columns if m.sum],
+ key=lambda x: x[1])
+
+ def fetch_metadata(self):
+ engine = create_engine(self.database.sqlalchemy_uri)
+ meta = MetaData()
+ table = sqlaTable(
+ self.table_name, meta, autoload=True, autoload_with=engine)
+ TC = TableColumn
+ for col in table.columns:
+ dbcol = (
+ db.session
+ .query(TC)
+ .filter(TC.table==self)
+ .filter(TC.column_name==col.name)
+ .first()
+ )
+ db.session.flush()
+ if not dbcol:
+ dbcol = TableColumn(column_name=col.name)
+ if str(col.type) in ('VARCHAR', 'STRING'):
+ dbcol.groupby = True
+ dbcol.filterable = True
+ self.columns.append(dbcol)
+
+ dbcol.type = str(col.type)
+ db.session.commit()
+
+
+class TableColumn(Model, AuditMixin):
+ __tablename__ = 'table_columns'
+ id = Column(Integer, primary_key=True)
+ table_id = Column(
+ String(256),
+ ForeignKey('tables.id'))
+ table = relationship('Table', backref='columns')
+ column_name = Column(String(256))
+ is_dttm = Column(Boolean, default=True)
+ is_active = Column(Boolean, default=True)
+ type = Column(String(32), default='')
+ groupby = Column(Boolean, default=False)
+ count_distinct = Column(Boolean, default=False)
+ sum = Column(Boolean, default=False)
+ max = Column(Boolean, default=False)
+ min = Column(Boolean, default=False)
+ filterable = Column(Boolean, default=False)
+ description = Column(Text, default='')
+
class Cluster(Model, AuditMixin):
__tablename__ = 'clusters'
@@ -46,7 +144,7 @@ class Cluster(Model, AuditMixin):
# logging.exception(e)
# logging.error("Failed at syncing " + datasource)
-class Datasource(Model, AuditMixin):
+class Datasource(Model, AuditMixin, Queryable):
__tablename__ = 'datasources'
id = Column(Integer, primary_key=True)
datasource_name = Column(String(256), unique=True)
@@ -130,17 +228,6 @@ class Datasource(Model, AuditMixin):
col_obj.generate_metrics()
#session.commit()
- @property
- def column_names(self):
- return sorted([c.column_name for c in self.columns])
-
- @property
- def groupby_column_names(self):
- return sorted([c.column_name for c in self.columns if c.groupby])
-
- @property
- def filterable_column_names(self):
- return sorted([c.column_name for c in self.columns if c.filterable])
class Metric(Model):
diff --git a/app/views.py b/app/views.py
index 561802c18..c3ab428a6 100644
--- a/app/views.py
+++ b/app/views.py
@@ -27,6 +27,22 @@ class DeleteMixin(object):
return redirect(self.get_redirect())
+class TableColumnInlineView(CompactCRUDMixin, ModelView):
+ datamodel = SQLAInterface(models.TableColumn)
+ can_delete = False
+ edit_columns = [
+ 'column_name', 'description', 'table', 'groupby', 'filterable',
+ 'count_distinct', 'sum', 'min', 'max']
+ list_columns = [
+ 'column_name', 'type', 'groupby', 'count_distinct',
+ 'sum', 'min', 'max']
+ page_size = 100
+ list_columns = [
+ 'column_name', 'type', 'groupby', 'count_distinct',
+ 'sum', 'min', 'max']
+appbuilder.add_view_no_menu(TableColumnInlineView)
+
+
class ColumnInlineView(CompactCRUDMixin, ModelView):
datamodel = SQLAInterface(models.Column)
edit_columns = [
@@ -80,6 +96,39 @@ appbuilder.add_view(
category_icon='fa-cogs',)
+class DatabaseView(ModelView, DeleteMixin):
+ datamodel = SQLAInterface(models.Database)
+ list_columns = ['database_name']
+ add_columns = ['database_name', 'sqlalchemy_uri']
+ edit_columns = add_columns
+
+appbuilder.add_view(
+ DatabaseView,
+ "Databases",
+ icon="fa-database",
+ category="Admin",
+ category_icon='fa-cogs',)
+
+
+class TableView(ModelView, DeleteMixin):
+ datamodel = SQLAInterface(models.Table)
+ list_columns = ['table_link', 'database']
+ add_columns = ['table_name', 'database']
+ edit_columns = add_columns
+ related_views = [TableColumnInlineView]
+
+ def post_insert(self, table):
+ table.fetch_metadata()
+
+ def post_update(self, table):
+ table.fetch_metadata()
+
+appbuilder.add_view(
+ TableView,
+ "Tables",
+ icon='fa-table',)
+
+
class DatasourceModelView(ModelView, DeleteMixin):
datamodel = SQLAInterface(models.Datasource)
list_columns = [
@@ -101,8 +150,7 @@ class DatasourceModelView(ModelView, DeleteMixin):
appbuilder.add_view(
DatasourceModelView,
"Druid Datasources",
- icon="fa-cube",
- category_icon='fa-envelope')
+ icon="fa-cube")
@app.route('/health')
@@ -116,6 +164,34 @@ def ping():
class Panoramix(BaseView):
+ @has_access
+ @permission_name('tables')
+ @expose("/table//")
+ def table(self, table_id):
+
+ table = (
+ db.session
+ .query(models.Table)
+ .filter_by(id=table_id)
+ .first()
+ )
+ viz_type = request.args.get("viz_type")
+ if not viz_type and table.default_endpoint:
+ return redirect(table.default_endpoint)
+ if not viz_type:
+ viz_type = "table"
+ obj = viz.viz_types[viz_type](
+ table,
+ form_data=request.args, view=self)
+ if request.args.get("json"):
+ return Response(
+ json.dumps(obj.get_query(), indent=4),
+ status=200,
+ mimetype="application/json")
+ if obj.df is None or obj.df.empty:
+ return obj.render_no_data()
+ return obj.render()
+
@has_access
@permission_name('datasources')
@expose("/datasource//")