Adding a command to load a sample dataset
This commit is contained in:
Maxime Beauchemin
parent
4edbbd350d
commit
e0d6d20993
10
README.md
10
README.md
|
|
@ -7,7 +7,7 @@ and interactive.
|
|||
Buzz Phrases
|
||||
------------
|
||||
|
||||
* Analytics at the speed of thought!
|
||||
* Analytics at the speed of thought!
|
||||
* Instantaneous learning curve
|
||||
* Realtime analytics when querying [Druid.io](http://druid.io)
|
||||
* Extentsible to infinity
|
||||
|
|
@ -67,11 +67,11 @@ pip install panoramix
|
|||
# Create an admin user
|
||||
fabmanager create-admin --app panoramix
|
||||
|
||||
# Clone the github repo
|
||||
git clone https://github.com/mistercrunch/panoramix.git
|
||||
# Load some data to play with
|
||||
panoramix load_examples
|
||||
|
||||
# Start the web server
|
||||
panoramix
|
||||
# Start the development web server
|
||||
panoramix runserver -d
|
||||
```
|
||||
|
||||
After installation, you should be able to point your browser to the right
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@ from flask.ext.script import Manager
|
|||
from panoramix import app, config
|
||||
from subprocess import Popen
|
||||
from flask.ext.migrate import MigrateCommand
|
||||
from panoramix import db
|
||||
from flask.ext.appbuilder import Base
|
||||
from sqlalchemy import Column, Integer, String
|
||||
from panoramix import config, models
|
||||
import csv
|
||||
import gzip
|
||||
|
||||
|
||||
manager = Manager(app)
|
||||
|
|
@ -33,9 +39,66 @@ def runserver(debug, port):
|
|||
Popen(cmd, shell=True).wait()
|
||||
|
||||
@manager.command
|
||||
def load_examples(self):
|
||||
def load_examples():
|
||||
"""Loads a set of Slices and Dashboards and a supporting dataset """
|
||||
print("Loading examples")
|
||||
print("Loading examples into {}".format(db))
|
||||
class BirthNames(Base):
|
||||
__tablename__ = "birth_names"
|
||||
id = Column(Integer, primary_key=True)
|
||||
state = Column(String(10))
|
||||
year = Column(Integer)
|
||||
name = Column(String(128))
|
||||
num = Column(Integer)
|
||||
ds = Column(String(20))
|
||||
gender = Column(String(10))
|
||||
try:
|
||||
BirthNames.__table__.drop(db.engine)
|
||||
except:
|
||||
pass
|
||||
Base.metadata.create_all(db.engine)
|
||||
session = db.session()
|
||||
|
||||
with gzip.open(config.basedir + '/data/birth_names.csv.gz') as f:
|
||||
bb_csv = csv.reader(f)
|
||||
for i, (state, year, name, num, gender) in enumerate(bb_csv):
|
||||
if i == 0 or not name or name=="\xc2\xa0":
|
||||
continue
|
||||
if num == "NA":
|
||||
num = 0
|
||||
ds = str(year) + '-01-01'
|
||||
session.add(
|
||||
BirthNames(
|
||||
state=state, year=year,
|
||||
ds=ds,
|
||||
name=name, num=num, gender=gender))
|
||||
if i % 1000 == 0:
|
||||
print("{} loaded out of 502619 rows".format(i))
|
||||
session.commit()
|
||||
session.commit()
|
||||
print("Done loading table!")
|
||||
DB = models.Database
|
||||
dbobj = session.query(DB).filter_by(database_name='main').first()
|
||||
if not dbobj:
|
||||
dbobj = DB()
|
||||
dbobj.database_name = "main"
|
||||
dbobj.sqlalchemy_uri = config.SQLALCHEMY_DATABASE_URI
|
||||
session.merge(dbobj)
|
||||
session.commit()
|
||||
|
||||
TBL = models.Table
|
||||
obj = session.query(TBL).filter_by(table_name='birth_names').first()
|
||||
if not obj:
|
||||
obj = TBL()
|
||||
obj.table_name = 'birth_names'
|
||||
obj.main_dttm_col = 'ds'
|
||||
obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=one+day&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table"
|
||||
obj.database = dbobj
|
||||
obj.fetch_metadata()
|
||||
session.merge(obj)
|
||||
session.commit()
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
manager.run()
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -24,11 +24,23 @@ import textwrap
|
|||
|
||||
from panoramix import db, get_session, config, utils
|
||||
from panoramix.viz import viz_types
|
||||
from sqlalchemy.ext.declarative import declared_attr
|
||||
|
||||
QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration'])
|
||||
|
||||
|
||||
class Slice(Model, AuditMixin):
|
||||
class AuditMixinNullable(AuditMixin):
|
||||
@declared_attr
|
||||
def created_by_fk(cls):
|
||||
return Column(Integer, ForeignKey('ab_user.id'),
|
||||
default=cls.get_user_id, nullable=True)
|
||||
@declared_attr
|
||||
def changed_by_fk(cls):
|
||||
return Column(Integer, ForeignKey('ab_user.id'),
|
||||
default=cls.get_user_id, onupdate=cls.get_user_id, nullable=True)
|
||||
|
||||
|
||||
class Slice(Model, AuditMixinNullable):
|
||||
"""A slice is essentially a report or a view on data"""
|
||||
__tablename__ = 'slices'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
|
@ -75,6 +87,10 @@ class Slice(Model, AuditMixin):
|
|||
"{self.datasource_id}/".format(self=self))
|
||||
return href(d)
|
||||
|
||||
@property
|
||||
def edit_url(self):
|
||||
return "/slicemodelview/edit/{}".format(self.id)
|
||||
|
||||
@property
|
||||
def slice_link(self):
|
||||
url = self.slice_url
|
||||
|
|
@ -101,7 +117,7 @@ dashboard_slices = Table('dashboard_slices', Model.metadata,
|
|||
)
|
||||
|
||||
|
||||
class Dashboard(Model, AuditMixin):
|
||||
class Dashboard(Model, AuditMixinNullable):
|
||||
"""A dash to slash"""
|
||||
__tablename__ = 'dashboards'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
|
@ -146,7 +162,7 @@ class Queryable(object):
|
|||
return sorted([c.column_name for c in self.columns if c.filterable])
|
||||
|
||||
|
||||
class Database(Model, AuditMixin):
|
||||
class Database(Model, AuditMixinNullable):
|
||||
__tablename__ = 'dbs'
|
||||
id = Column(Integer, primary_key=True)
|
||||
database_name = Column(String(250), unique=True)
|
||||
|
|
@ -166,15 +182,13 @@ class Database(Model, AuditMixin):
|
|||
autoload_with=self.get_sqla_engine())
|
||||
|
||||
|
||||
class Table(Model, Queryable, AuditMixin):
|
||||
class Table(Model, Queryable, AuditMixinNullable):
|
||||
type = "table"
|
||||
|
||||
__tablename__ = 'tables'
|
||||
id = Column(Integer, primary_key=True)
|
||||
table_name = Column(String(255), unique=True)
|
||||
main_datetime_column_id = Column(Integer, ForeignKey('table_columns.id'))
|
||||
main_datetime_column = relationship(
|
||||
'TableColumn', foreign_keys=[main_datetime_column_id])
|
||||
table_name = Column(String(250), unique=True)
|
||||
main_dttm_col = Column(String(250))
|
||||
default_endpoint = Column(Text)
|
||||
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
|
||||
database = relationship(
|
||||
|
|
@ -308,8 +322,11 @@ class Table(Model, Queryable, AuditMixin):
|
|||
extras=None):
|
||||
|
||||
qry_start_dttm = datetime.now()
|
||||
if not self.main_dttm_col:
|
||||
raise Exception(
|
||||
"Datetime column not provided as part table configuration")
|
||||
timestamp = literal_column(
|
||||
self.main_datetime_column.column_name).label('timestamp')
|
||||
self.main_dttm_col).label('timestamp')
|
||||
metrics_exprs = [
|
||||
literal_column(m.expression).label(m.metric_name)
|
||||
for m in self.metrics if m.metric_name in metrics]
|
||||
|
|
@ -420,7 +437,7 @@ class Table(Model, Queryable, AuditMixin):
|
|||
self.columns.append(dbcol)
|
||||
|
||||
if not any_date_col and 'date' in datatype.lower():
|
||||
any_date_col = dbcol
|
||||
any_date_col = col.name
|
||||
|
||||
if dbcol.sum:
|
||||
metrics.append(M(
|
||||
|
|
@ -464,18 +481,18 @@ class Table(Model, Queryable, AuditMixin):
|
|||
m = (
|
||||
db.session.query(M)
|
||||
.filter(M.metric_name == metric.metric_name)
|
||||
.filter(M.table == self)
|
||||
.filter(M.table_id == self.id)
|
||||
.first()
|
||||
)
|
||||
metric.table = self
|
||||
metric.table_id = self.id
|
||||
if not m:
|
||||
db.session.add(metric)
|
||||
db.session.commit()
|
||||
if not self.main_datetime_column:
|
||||
self.main_datetime_column = any_date_col
|
||||
if not self.main_dttm_col:
|
||||
self.main_dttm_col = any_date_col
|
||||
|
||||
|
||||
class SqlMetric(Model, AuditMixin):
|
||||
class SqlMetric(Model, AuditMixinNullable):
|
||||
__tablename__ = 'sql_metrics'
|
||||
id = Column(Integer, primary_key=True)
|
||||
metric_name = Column(String(512))
|
||||
|
|
@ -488,7 +505,7 @@ class SqlMetric(Model, AuditMixin):
|
|||
description = Column(Text)
|
||||
|
||||
|
||||
class TableColumn(Model, AuditMixin):
|
||||
class TableColumn(Model, AuditMixinNullable):
|
||||
__tablename__ = 'table_columns'
|
||||
id = Column(Integer, primary_key=True)
|
||||
table_id = Column(Integer, ForeignKey('tables.id'))
|
||||
|
|
@ -513,7 +530,7 @@ class TableColumn(Model, AuditMixin):
|
|||
return self.type in ('LONG', 'DOUBLE', 'FLOAT')
|
||||
|
||||
|
||||
class Cluster(Model, AuditMixin):
|
||||
class Cluster(Model, AuditMixinNullable):
|
||||
__tablename__ = 'clusters'
|
||||
id = Column(Integer, primary_key=True)
|
||||
cluster_name = Column(String(250), unique=True)
|
||||
|
|
@ -560,7 +577,7 @@ class Datasource(Model, AuditMixin, Queryable):
|
|||
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
||||
owner = relationship('User', backref='datasources', foreign_keys=[user_id])
|
||||
cluster_name = Column(
|
||||
String(255), ForeignKey('clusters.cluster_name'))
|
||||
String(250), ForeignKey('clusters.cluster_name'))
|
||||
cluster = relationship(
|
||||
'Cluster', backref='datasources', foreign_keys=[cluster_name])
|
||||
|
||||
|
|
@ -783,7 +800,7 @@ class Metric(Model):
|
|||
return obj
|
||||
|
||||
|
||||
class Column(Model, AuditMixin):
|
||||
class Column(Model, AuditMixinNullable):
|
||||
__tablename__ = 'columns'
|
||||
id = Column(Integer, primary_key=True)
|
||||
datasource_name = Column(
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@
|
|||
<div class="col-md-4 text-right" style="position: relative;">
|
||||
<a href="{{ slice.slice_url }}"><i class="fa fa-play"></i></a>
|
||||
<a class="refresh"><i class="fa fa-refresh"></i></a>
|
||||
<a><i class="fa fa-gear"></i></a>
|
||||
<a href="{{ slice.edit_url }}"><i class="fa fa-gear"></i></a>
|
||||
<a class="closewidget"><i class="fa fa-close"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -26,8 +26,7 @@
|
|||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<div id="{{ viz.token }}" style="display: none;">
|
||||
</div>
|
||||
<div id="{{ viz.token }}" style="display: none;"></div>
|
||||
<img src="{{ url_for("static", filename="loading.gif") }}" class="loading">
|
||||
{% endif %}
|
||||
{% endmacro %}
|
||||
|
|
@ -37,14 +36,22 @@
|
|||
<script>
|
||||
var url = "{{ viz.get_url(async="true", standalone="true", skip_libs="true")|safe }}";
|
||||
console.log(url);
|
||||
$("#{{ viz.token }}").load(url, function(){
|
||||
var table = $('table').DataTable({
|
||||
paging: false,
|
||||
searching: false,
|
||||
});
|
||||
$("#{{ viz.token }}").show();
|
||||
table.column('-1').order( 'desc' ).draw();
|
||||
$("#{{ viz.token }} img.loading").hide();
|
||||
var token = $("#{{ viz.token }}");
|
||||
token.load(url, function(response, status, xhr){
|
||||
if(status=="error"){
|
||||
var err = '<div class="alert alert-danger">' + xhr.responseText + '</div>';
|
||||
token.html(err);
|
||||
token.show();
|
||||
}
|
||||
else{
|
||||
var table = $('table').DataTable({
|
||||
paging: false,
|
||||
searching: false,
|
||||
});
|
||||
table.column('-1').order( 'desc' ).draw();
|
||||
}
|
||||
token.show();
|
||||
token.parent().find("img.loading").hide();
|
||||
});
|
||||
</script>
|
||||
{% endif %}
|
||||
|
|
|
|||
|
|
@ -86,6 +86,44 @@ class MetricInlineView(CompactCRUDMixin, ModelView):
|
|||
appbuilder.add_view_no_menu(MetricInlineView)
|
||||
|
||||
|
||||
class DatabaseView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Database)
|
||||
list_columns = ['database_name']
|
||||
add_columns = ['database_name', 'sqlalchemy_uri']
|
||||
edit_columns = add_columns
|
||||
|
||||
appbuilder.add_view(
|
||||
DatabaseView,
|
||||
"Databases",
|
||||
icon="fa-database",
|
||||
category="Sources",
|
||||
category_icon='fa-cogs',)
|
||||
|
||||
|
||||
class TableView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Table)
|
||||
list_columns = ['table_link', 'database']
|
||||
add_columns = ['table_name', 'database', 'default_endpoint']
|
||||
edit_columns = [
|
||||
'table_name', 'database', 'main_dttm_col', 'default_endpoint']
|
||||
related_views = [TableColumnInlineView, SqlMetricInlineView]
|
||||
|
||||
def post_add(self, table):
|
||||
table.fetch_metadata()
|
||||
|
||||
def post_update(self, table):
|
||||
table.fetch_metadata()
|
||||
|
||||
appbuilder.add_view(
|
||||
TableView,
|
||||
"Tables",
|
||||
category="Sources",
|
||||
icon='fa-table',)
|
||||
|
||||
|
||||
appbuilder.add_separator("Sources")
|
||||
|
||||
|
||||
class ClusterModelView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Cluster)
|
||||
add_columns = [
|
||||
|
|
@ -100,13 +138,16 @@ appbuilder.add_view(
|
|||
ClusterModelView,
|
||||
"Druid Clusters",
|
||||
icon="fa-server",
|
||||
category="Admin",
|
||||
category="Sources",
|
||||
category_icon='fa-cogs',)
|
||||
|
||||
|
||||
class SliceModelView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Slice)
|
||||
list_columns = ['slice_link', 'viz_type', 'datasource', 'created_by']
|
||||
edit_columns = [
|
||||
'slice_name', 'viz_type', 'druid_datasource', 'table',
|
||||
'dashboards', 'params']
|
||||
|
||||
appbuilder.add_view(
|
||||
SliceModelView,
|
||||
|
|
@ -131,40 +172,6 @@ appbuilder.add_view(
|
|||
category_icon='',)
|
||||
|
||||
|
||||
class DatabaseView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Database)
|
||||
list_columns = ['database_name']
|
||||
add_columns = ['database_name', 'sqlalchemy_uri']
|
||||
edit_columns = add_columns
|
||||
|
||||
appbuilder.add_view(
|
||||
DatabaseView,
|
||||
"Databases",
|
||||
icon="fa-database",
|
||||
category="Admin",
|
||||
category_icon='fa-cogs',)
|
||||
|
||||
|
||||
class TableView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Table)
|
||||
list_columns = ['table_link', 'database']
|
||||
add_columns = ['table_name', 'database', 'default_endpoint']
|
||||
edit_columns = [
|
||||
'table_name', 'database', 'main_datetime_column', 'default_endpoint']
|
||||
related_views = [TableColumnInlineView, SqlMetricInlineView]
|
||||
|
||||
def post_add(self, table):
|
||||
table.fetch_metadata()
|
||||
|
||||
def post_update(self, table):
|
||||
table.fetch_metadata()
|
||||
|
||||
appbuilder.add_view(
|
||||
TableView,
|
||||
"Tables",
|
||||
icon='fa-table',)
|
||||
|
||||
|
||||
class DatasourceModelView(ModelView, DeleteMixin):
|
||||
datamodel = SQLAInterface(models.Datasource)
|
||||
list_columns = [
|
||||
|
|
@ -186,6 +193,7 @@ class DatasourceModelView(ModelView, DeleteMixin):
|
|||
appbuilder.add_view(
|
||||
DatasourceModelView,
|
||||
"Druid Datasources",
|
||||
category="Sources",
|
||||
icon="fa-cubes")
|
||||
|
||||
|
||||
|
|
@ -241,7 +249,14 @@ class Panoramix(BaseView):
|
|||
status=status,
|
||||
mimetype="application/json")
|
||||
else:
|
||||
return self.render_template("panoramix/viz.html", viz=obj)
|
||||
try:
|
||||
resp = self.render_template("panoramix/viz.html", viz=obj)
|
||||
except Exception as e:
|
||||
return Response(
|
||||
str(e),
|
||||
status=500,
|
||||
mimetype="application/json")
|
||||
return resp
|
||||
|
||||
@has_access
|
||||
@expose("/save_dash/<dashboard_id>/", methods=['GET', 'POST'])
|
||||
|
|
@ -303,8 +318,10 @@ class Panoramix(BaseView):
|
|||
.filter(models.Dashboard.id == id_)
|
||||
.first()
|
||||
)
|
||||
pos_dict = {
|
||||
int(o['slice_id']):o for o in json.loads(dashboard.position_json)}
|
||||
pos_dict = {}
|
||||
if dashboard.position_json:
|
||||
pos_dict = {
|
||||
int(o['slice_id']):o for o in json.loads(dashboard.position_json)}
|
||||
return self.render_template(
|
||||
"panoramix/dashboard.html", dashboard=dashboard,
|
||||
pos_dict=pos_dict)
|
||||
|
|
@ -342,6 +359,6 @@ appbuilder.add_view_no_menu(Panoramix)
|
|||
appbuilder.add_link(
|
||||
"Refresh Druid Metadata",
|
||||
href='/panoramix/refresh_datasources/',
|
||||
category='Admin',
|
||||
category='Sources',
|
||||
category_icon='fa-cogs',
|
||||
icon="fa-cog")
|
||||
|
|
|
|||
Loading…
Reference in New Issue