iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Starting over

This commit is contained in:
Maxime 2015-07-15 17:12:32 +00:00
parent 4cb380d9ba
commit 8431800bd6
41 changed files with 504 additions and 416 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1 +1,2 @@
*.pyc *.pyc
tmp

86
base.html
View File

@ -1,86 +0,0 @@
{% import 'admin/layout.html' as layout with context -%}
{% import 'admin/static.html' as admin_static with context %}
<!DOCTYPE html>
<html>
<head>
<title>{% block title %}{% if admin_view.category %}{{ admin_view.category }} - {% endif %}{{ admin_view.name }} - {{ admin_view.admin.name }}{% endblock %}</title>
{% block head_meta %}
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="">
<meta name="author" content="">
{% endblock %}
{% block head_css %}
<link href="{{ admin_static.url(filename='bootstrap/bootstrap3/css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='bootstrap-theme.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='main.css') }}" rel="stylesheet">
<link href="{{ admin_static.url(filename='admin/css/bootstrap3/admin.css') }}" rel="stylesheet">
<link rel="stylesheet" href="{{url_for('static', filename='select2-bootstrap.css')}}">
<link rel="stylesheet" href="{{url_for('static', filename='select2.min.css')}}">
{% endblock %}
{% block head %}
{% endblock %}
{% block head_tail %}
{% endblock %}
</head>
<body>
{% block page_body %}
<div class="container">
<nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
<div class="container">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#admin-navbar-collapse">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
{% block brand %}
<a class="navbar-brand" href="#">{{ admin_view.admin.name }}</a>
{% endblock %}
</div>
<!-- navbar content -->
<div class="collapse navbar-collapse" id="admin-navbar-collapse">
{% block main_menu %}
<ul class="nav navbar-nav">
{{ layout.menu() }}
</ul>
{% endblock %}
{% block menu_links %}
<ul class="nav navbar-right">
{{ layout.menu_links() }}
</ul>
{% endblock %}
{% block access_control %}
{% endblock %}
<ul class="nav navbar-nav navbar-right">
</ul>
</div>
</div>
</nav>
{% block messages %}
{{ layout.messages() }}
{% endblock %}
{% set render_ctx = h.resolve_ctx() %}
{% block body %}{% endblock %}
</div>
{% endblock %}
{% block tail_js %}
<script src="{{ admin_static.url(filename='vendor/jquery-2.1.1.min.js') }}" type="text/javascript"></script>
<script src="{{ admin_static.url(filename='bootstrap/bootstrap3/js/bootstrap.min.js') }}" type="text/javascript"></script>
<script src="{{ admin_static.url(filename='vendor/moment-2.8.4.min.js') }}" type="text/javascript"></script>
<script src="{{ admin_static.url(filename='vendor/select2/select2.min.js') }}" type="text/javascript"></script>
{% endblock %}
{% block tail %}
{% endblock %}
</body>
</html>

1
init.sh
View File

@ -1 +0,0 @@
export PYTHONPATH=/home/maxime_beauchemin/code/panoramix:/home/maxime_beauchemin/code/pydruid

9
panoramix/.gitignore vendored Normal file
View File

@ -0,0 +1,9 @@
*.pyc
app.db
./tmp
./build/*
./.idea
./.idea/*
env
venv
*.sublime*

15
panoramix/README.rst Normal file
View File

@ -0,0 +1,15 @@
Base Skeleton to start your application using Flask-AppBuilder
--------------------------------------------------------------
- Install it::
pip install flask-appbuilder
git clone https://github.com/dpgaspar/Flask-AppBuilder-Skeleton.git
- Run it::
fabmanager run
That's it!!

3
panoramix/TODO.md Normal file
View File

@ -0,0 +1,3 @@
# TODO
* Default URL params per datasource
* Get config metrics to work

0
panoramix/__init__.py
View File

251
panoramix/app.py
View File

@ -1,251 +0,0 @@
from dateutil.parser import parse
from datetime import timedelta
from flask import Flask, request, Blueprint, url_for, Markup
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.admin import Admin, BaseView, expose, AdminIndexView
from panoramix import settings, viz, models
from flask_bootstrap import Bootstrap
from wtforms import Form, SelectMultipleField, SelectField, TextField
from wtforms.fields import Field
import pandas as pd
from flask_admin.contrib import sqla
pd.set_option('display.max_colwidth', -1)
client = settings.get_pydruid_client()
class OmgWtForm(Form):
field_order = (
'viz_type', 'granularity', 'since', 'group_by', 'limit')
def fields(self):
fields = []
for field in self.field_order:
if hasattr(self, field):
obj = getattr(self, field)
if isinstance(obj, Field):
fields.append(getattr(self, field))
return fields
def form_factory(datasource, form_args=None):
grain = ['all', 'none', 'minute', 'hour', 'day']
limits = [0, 5, 10, 25, 50, 100, 500]
if form_args:
limit = form_args.get("limit")
try:
limit = int(limit)
if limit not in limits:
limits.append(limit)
limits = sorted(limits)
except:
pass
class QueryForm(OmgWtForm):
viz_type = SelectField(
'Viz',
choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()])
metric = SelectField(
'Metric', choices=[(m, m) for m in datasource.metrics])
groupby = SelectMultipleField(
'Group by', choices=[
(s, s) for s in datasource.groupby_column_names])
granularity = SelectField(
'Time Granularity', choices=[(g, g) for g in grain])
since = SelectField(
'Since', choices=[(s, s) for s in settings.since_l.keys()],
default="all")
limit = SelectField(
'Limit', choices=[(s, s) for s in limits])
for i in range(10):
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names]))
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]]))
setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super"))
return QueryForm
"""
bp = Blueprint(
'panoramix', __name__,
template_folder='templates',
static_folder='static')
"""
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI
db = SQLAlchemy(app)
app.secret_key = "monkeys"
#app.register_blueprint(bp, url_prefix='/panoramix')
Bootstrap(app)
admin = Admin(
app, name = "Panoramix",
template_mode='bootstrap3')
class Datasource(db.Model):
__tablename__ = 'datasources'
id = db.Column(db.Integer, primary_key=True)
datasource_name = db.Column(db.String(256), unique=True)
is_featured = db.Column(db.Boolean, default=False)
is_hidden = db.Column(db.Boolean, default=False)
description = db.Column(db.Text)
created_dttm = db.Column(db.DateTime, default=db.func.now())
@property
def metrics(self):
return [col.column_name for col in self.columns if not col.groupby]
@classmethod
def latest_metadata(cls, name):
results = client.time_boundary(datasource=name)
max_time = results[0]['result']['maxTime']
max_time = parse(max_time)
intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
intervals += (max_time + timedelta(seconds=1)).isoformat()
segment_metadata = client.segment_metadata(
datasource=name,
intervals=intervals)
return segment_metadata[-1]['columns']
@classmethod
def sync_to_db(cls, name):
datasource = cls.query.filter_by(datasource_name=name).first()
if not datasource:
db.session.add(cls(datasource_name=name))
cols = cls.latest_metadata(name)
for col in cols:
col_obj = Column.query.filter_by(datasource_name=name, column_name=col).first()
datatype = cols[col]['type']
if not col_obj:
col_obj = Column(datasource_name=name, column_name=col)
db.session.add(col_obj)
if datatype == "STRING":
col_obj.groupby = True
col_obj.filterable = True
if col_obj:
col_obj.type = cols[col]['type']
db.session.commit()
@property
def column_names(self):
return sorted([c.column_name for c in self.columns])
@property
def groupby_column_names(self):
return sorted([c.column_name for c in self.columns if c.groupby])
@property
def filterable_column_names(self):
return sorted([c.column_name for c in self.columns if c.filterable])
class Column(db.Model):
__tablename__ = 'columns'
id = db.Column(db.Integer, primary_key=True)
datasource_name = db.Column(
db.String(256),
db.ForeignKey('datasources.datasource_name'))
column_name = db.Column(db.String(256))
is_active = db.Column(db.Boolean, default=True)
type = db.Column(db.String(32))
groupby = db.Column(db.Boolean, default=False)
count_distinct = db.Column(db.Boolean, default=False)
sum = db.Column(db.Boolean, default=False)
max = db.Column(db.Boolean, default=False)
min = db.Column(db.Boolean, default=False)
filterable = db.Column(db.Boolean, default=False)
datasource = db.relationship('Datasource',
backref=db.backref('columns', lazy='dynamic'))
def __repr__(self):
return self.column_name
class JsUdf(db.Model):
__tablename__ = 'udfs'
id = db.Column(db.Integer, primary_key=True)
datasource_name = db.Column(
db.String(256),
db.ForeignKey('datasources.datasource_name'))
udf_name = db.Column(db.String(256))
column_list = db.Column(db.String(1024))
code = db.Column(db.Text)
datasource = db.relationship('Datasource',
backref=db.backref('udfs', lazy='dynamic'))
def datasource_link(v, c, m, p):
url = '/admin/datasourceview/datasource/{}/'.format(m.datasource_name)
return Markup('<a href="{url}">{m.datasource_name}</a>'.format(**locals()))
class DatasourceAdmin(sqla.ModelView):
inline_models = (Column, JsUdf,)
column_formatters = dict(datasource_name=datasource_link)
class DatasourceView(BaseView):
@expose('/')
def index(self):
return ""
@expose("/datasource/<datasource_name>/")
def datasource(self, datasource_name):
viz_type = request.args.get("viz_type", "table")
datasource = (
Datasource
.query
.filter_by(datasource_name=datasource_name)
.first()
)
obj = viz.viz_types[viz_type](
datasource,
form_class=form_factory(datasource, request.args),
form_data=request.args,
admin_view=self)
if obj.df is None or obj.df.empty:
return obj.render_no_data()
return obj.render()
@expose("/datasources/")
def datasources(self):
import requests
import json
endpoint = (
"http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
"{COORDINATOR_BASE_ENDPOINT}/datasources"
).format(**settings.__dict__)
datasources = json.loads(requests.get(endpoint).text)
for datasource in datasources:
Datasource.sync_to_db(datasource)
return json.dumps(datasources, indent=4)
@expose("/datasource_metadata/<name>/")
def datasource_metadata(name):
import requests
import json
endpoint = (
"http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
"{COORDINATOR_BASE_ENDPOINT}/datasource"
).format(**settings.__dict__)
return str(datasources)
admin.add_view(DatasourceView(name="Datasource"))
if __name__ == '__main__':
db.create_all()
admin.add_view(DatasourceAdmin(Datasource, db.session, name="Datasources"))
app.debug = True
app.run(host='0.0.0.0', port=settings.FLASK_APP_PORT)

34
panoramix/app/__init__.py Normal file
View File

@ -0,0 +1,34 @@
import logging
from flask import Flask
from flask.ext.appbuilder import SQLA, AppBuilder
"""
Logging configuration
"""
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
logging.getLogger().setLevel(logging.DEBUG)
app = Flask(__name__)
app.config.from_object('config')
db = SQLA(app)
appbuilder = AppBuilder(
app, db.session, base_template='panoramix/base.html')
#appbuilder.app_name = 'Panoramix'
"""
from sqlalchemy.engine import Engine
from sqlalchemy import event
#Only include this for SQLLite constraints
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
# Will force sqllite contraint foreign keys
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
"""
from app import views

122
panoramix/app/models.py Normal file
View File

@ -0,0 +1,122 @@
from flask.ext.appbuilder import Model
from datetime import datetime, timedelta
from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn, ImageColumn
from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean
from sqlalchemy.orm import relationship
from app import db, utils
from dateutil.parser import parse
"""
You can use the extra Flask-AppBuilder fields and Mixin's
AuditMixin will add automatic timestamp of created and modified by who
"""
client = utils.get_pydruid_client()
class Datasource(Model, AuditMixin):
__tablename__ = 'datasources'
id = Column(Integer, primary_key=True)
datasource_name = Column(String(256), unique=True)
is_featured = Column(Boolean, default=False)
is_hidden = Column(Boolean, default=False)
description = Column(Text)
columns = relationship('Column', backref='datasource')
udfs = relationship('JavascriptUdf', backref='datasource')
@property
def metrics(self):
return [col.column_name for col in self.columns if not col.groupby]
def __repr__(self):
return self.datasource_name
@property
def datasource_link(self):
url = "/panoramix/datasource/{}/".format(self.datasource_name)
return '<a href="{url}">{self.datasource_name}</a>'.format(**locals())
@classmethod
def latest_metadata(cls, name):
results = client.time_boundary(datasource=name)
max_time = results[0]['result']['maxTime']
max_time = parse(max_time)
intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
intervals += (max_time + timedelta(seconds=1)).isoformat()
segment_metadata = client.segment_metadata(
datasource=name,
intervals=intervals)
return segment_metadata[-1]['columns']
@classmethod
def sync_to_db(cls, name):
datasource = db.session.query(cls).filter_by(datasource_name=name).first()
if not datasource:
db.session.add(cls(datasource_name=name))
cols = cls.latest_metadata(name)
for col in cols:
col_obj = (
db.session
.query(Column)
.filter_by(datasource_name=name, column_name=col)
.first()
)
datatype = cols[col]['type']
if not col_obj:
col_obj = Column(datasource_name=name, column_name=col)
db.session.add(col_obj)
if datatype == "STRING":
col_obj.groupby = True
col_obj.filterable = True
if col_obj:
col_obj.type = cols[col]['type']
db.session.commit()
@property
def column_names(self):
return sorted([c.column_name for c in self.columns])
@property
def groupby_column_names(self):
return sorted([c.column_name for c in self.columns if c.groupby])
@property
def filterable_column_names(self):
return sorted([c.column_name for c in self.columns if c.filterable])
class JavascriptUdf(Model, AuditMixin):
__tablename__ = 'udfs'
id = Column(Integer, primary_key=True)
datasource_name = Column(
String(256),
ForeignKey('datasources.datasource_name'))
udf_name = Column(String(256))
column_list = Column(String(1024))
code = Column(Text)
def __repr__(self):
return self.udf_name
class Column(Model, AuditMixin):
__tablename__ = 'columns'
id = Column(Integer, primary_key=True)
datasource_name = Column(
String(256),
ForeignKey('datasources.datasource_name'))
column_name = Column(String(256))
is_active = Column(Boolean, default=True)
type = Column(String(32))
groupby = Column(Boolean, default=False)
count_distinct = Column(Boolean, default=False)
sum = Column(Boolean, default=False)
max = Column(Boolean, default=False)
min = Column(Boolean, default=False)
filterable = Column(Boolean, default=False)
def __repr__(self):
return self.column_name

0
panoramix/static/bootstrap-theme.css → panoramix/app/static/bootstrap-theme.css vendored
View File

0
panoramix/static/chaudron.png → panoramix/app/static/chaudron.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

0
panoramix/static/favicon.png → panoramix/app/static/favicon.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

0
panoramix/static/highcharts.js → panoramix/app/static/highcharts.js
View File

0
panoramix/static/main.css → panoramix/app/static/main.css
View File

0
panoramix/static/panoramix.jpg → panoramix/app/static/panoramix.jpg
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 165 KiB

After

Width:  |  Height:  |  Size: 165 KiB

0
panoramix/static/panoramix.png → panoramix/app/static/panoramix.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.5 MiB

After

Width:  |  Height:  |  Size: 1.5 MiB

0
panoramix/static/select2-bootstrap.css → panoramix/app/static/select2-bootstrap.css
View File

0
panoramix/static/select2.min.css → panoramix/app/static/select2.min.css vendored
View File

0
panoramix/static/select2.min.js → panoramix/app/static/select2.min.js vendored
View File

0
panoramix/static/serpe.jpg → panoramix/app/static/serpe.jpg
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 121 KiB

After

Width:  |  Height:  |  Size: 121 KiB

0
panoramix/static/tux_panoramix.png → panoramix/app/static/tux_panoramix.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 73 KiB

After

Width:  |  Height:  |  Size: 73 KiB

2
panoramix/app/templates/index.html Normal file
View File

@ -0,0 +1,2 @@
{% extends "appbuilder/baselayout.html" %}

8
panoramix/app/templates/panoramix/base.html Normal file
View File

@ -0,0 +1,8 @@
{% extends "appbuilder/baselayout.html" %}
{% block head_css %}
{{super()}}
<link rel="shortcut icon" href="{{ url_for('static', filename='chaudron.png') }}">
<style>
</style>
{% endblock %}

22
panoramix/templates/panoramix/datasource.html → panoramix/app/templates/panoramix/datasource.html
View File

@ -1,5 +1,5 @@
{% extends "panoramix/base.html" %} {% extends "panoramix/base.html" %}
{% block styles %} {% block head_css %}
{{super()}} {{super()}}
<style> <style>
form .row { form .row {
@ -12,22 +12,22 @@ form .col {
} }
</style> </style>
{% endblock %} {% endblock %}
{% block body %} {% block content %}
<div class="container"> <div class="container">
<div class="col-md-3"> <div class="col-md-3">
<h3> <h3>
{{ datasource.datasource_name }} {{ datasource.datasource_name }}
<a href="/admin/datasource/edit/?id={{ datasource.id }}"><span class="glyphicon glyphicon-edit"></span></a> <a href="/datasourcemodelview/edit/{{ datasource.id }}"><span class="glyphicon glyphicon-edit"></span></a>
</h3> </h3>
<hr> <hr>
<form method="GET"> <form method="GET">
<div>{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control") }}</div> <div>{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control select2") }}</div>
<div>{{ form.metric.label }}: {{ form.metric(class_="form-control select2") }}</div> <div>{{ form.metric.label }}: {{ form.metric(class_="form-control select2") }}</div>
<div>{{ form.granularity.label }}: {{ form.granularity(class_="form-control") }}</div> <div>{{ form.granularity.label }}: {{ form.granularity(class_="form-control select2") }}</div>
<div>{{ form.since.label }}: {{ form.since(class_="form-control") }}</div> <div>{{ form.since.label }}: {{ form.since(class_="form-control select2") }}</div>
<div>{{ form.groupby.label }}: {{ form.groupby(class_="form-control") }}</div> <div>{{ form.groupby.label }}: {{ form.groupby(class_="form-control select2") }}</div>
<div>{{ form.limit.label }}: {{ form.limit(class_="form-control") }}</div> <div>{{ form.limit.label }}: {{ form.limit(class_="form-control select2") }}</div>
<hr> <hr>
<h4>Filters</h4> <h4>Filters</h4>
<div id="filters"> <div id="filters">
@ -78,12 +78,12 @@ form .col {
</div> </div>
{% endblock %} {% endblock %}
{% block tail %} {% block tail_js %}
{{ super() }} {{ super() }}
<script> <script>
$( document ).ready(function() { $( document ).ready(function() {
//`:$(".select2").select2(); $(".select2").select2();
// $(".select2_tags").select2({tags: true}); $(".select2_tags").select2({tags: true});
}); });
</script> </script>
{% endblock %} {% endblock %}

0
panoramix/templates/panoramix/no_data.html → panoramix/app/templates/panoramix/no_data.html
View File

0
panoramix/templates/panoramix/noadmin.html → panoramix/app/templates/panoramix/noadmin.html
View File

0
panoramix/templates/panoramix/viz_highcharts.html → panoramix/app/templates/panoramix/viz_highcharts.html
View File

0
panoramix/templates/panoramix/viz_table.html → panoramix/app/templates/panoramix/viz_table.html
View File

18
panoramix/app/utils.py Normal file
View File

@ -0,0 +1,18 @@
import config
from datetime import timedelta
since_l = {
'1hour': timedelta(hours=1),
'1day': timedelta(days=1),
'7days': timedelta(days=7),
'28days': timedelta(days=28),
'all': timedelta(days=365*100)
}
def get_pydruid_client():
from pydruid import client
return client.PyDruid(
"http://{0}:{1}/".format(config.DRUID_HOST, config.DRUID_PORT),
config.DRUID_BASE_ENDPOINT)

129
panoramix/app/views.py Normal file
View File

@ -0,0 +1,129 @@
from flask import request, redirect, flash
from flask.ext.appbuilder.models.sqla.interface import SQLAInterface
from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose
from app import appbuilder, db, models, viz, utils
import config
from wtforms import Form, SelectMultipleField, SelectField, TextField
from wtforms.fields import Field
from datetime import timedelta
class OmgWtForm(Form):
field_order = (
'viz_type', 'granularity', 'since', 'group_by', 'limit')
def fields(self):
fields = []
for field in self.field_order:
if hasattr(self, field):
obj = getattr(self, field)
if isinstance(obj, Field):
fields.append(getattr(self, field))
return fields
def form_factory(datasource, form_args=None):
grain = ['all', 'none', 'minute', 'hour', 'day']
limits = [0, 5, 10, 25, 50, 100, 500]
if form_args:
limit = form_args.get("limit")
try:
limit = int(limit)
if limit not in limits:
limits.append(limit)
limits = sorted(limits)
except:
pass
class QueryForm(OmgWtForm):
viz_type = SelectField(
'Viz',
choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()])
metric = SelectField(
'Metric', choices=[(m, m) for m in datasource.metrics])
groupby = SelectMultipleField(
'Group by', choices=[
(s, s) for s in datasource.groupby_column_names])
granularity = SelectField(
'Time Granularity', choices=[(g, g) for g in grain])
since = SelectField(
'Since', choices=[(s, s) for s in utils.since_l.keys()],
default="all")
limit = SelectField(
'Limit', choices=[(s, s) for s in limits])
for i in range(10):
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names]))
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]]))
setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super"))
return QueryForm
class ColumnInlineView(CompactCRUDMixin, ModelView):
datamodel = SQLAInterface(models.Column)
edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
can_delete = False
appbuilder.add_view_no_menu(ColumnInlineView)
class JavascriptUdfInlineView(CompactCRUDMixin, ModelView):
datamodel = SQLAInterface(models.JavascriptUdf)
edit_columns = ['udf_name', 'column_list', 'code']
appbuilder.add_view_no_menu(JavascriptUdfInlineView)
class DatasourceModelView(ModelView):
datamodel = SQLAInterface(models.Datasource)
list_columns = ['datasource_link', 'is_featured' ]
related_views = [ColumnInlineView, JavascriptUdfInlineView]
edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden']
page_size = 100
appbuilder.add_view(
DatasourceModelView,
"Datasources",
icon="fa-cube",
category_icon='fa-envelope')
class Panoramix(BaseView):
@expose("/datasource/<datasource_name>/")
def datasource(self, datasource_name):
viz_type = request.args.get("viz_type", "table")
datasource = (
db.session
.query(models.Datasource)
.filter_by(datasource_name=datasource_name)
.first()
)
obj = viz.viz_types[viz_type](
datasource,
form_class=form_factory(datasource, request.args),
form_data=request.args, view=self)
if obj.df is None or obj.df.empty:
return obj.render_no_data()
return obj.render()
@expose("/refresh_datasources/")
def datasources(self):
import requests
import json
endpoint = (
"http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/"
"{COORDINATOR_BASE_ENDPOINT}/datasources"
).format(**config.__dict__)
datasources = json.loads(requests.get(endpoint).text)
for datasource in datasources:
models.Datasource.sync_to_db(datasource)
flash("Refreshed metadata from Druid!", 'info')
return redirect("/datasourcemodelview/list/")
appbuilder.add_view_no_menu(Panoramix)
appbuilder.add_link(
"Refresh Metadata",
href='/panoramix/refresh_datasources/',
category='Admin',
icon="fa-cogs")
db.create_all()

52
panoramix/viz.py → panoramix/app/viz.py
View File

@ -3,9 +3,10 @@ from datetime import datetime
from flask import render_template, flash from flask import render_template, flash
import pandas as pd import pandas as pd
from pandas_highcharts.core import serialize from pandas_highcharts.core import serialize
from panoramix import settings
from pydruid.utils import aggregators as agg from pydruid.utils import aggregators as agg
from collections import OrderedDict from collections import OrderedDict
from app import utils
import config
CHART_ARGS = { CHART_ARGS = {
@ -18,13 +19,13 @@ CHART_ARGS = {
class BaseViz(object): class BaseViz(object):
verbose_name = "Base Viz" verbose_name = "Base Viz"
template = "panoramix/datasource.html" template = "panoramix/datasource.html"
def __init__(self, datasource, form_class, form_data, admin_view): def __init__(self, datasource, form_class, form_data, view):
self.datasource = datasource self.datasource = datasource
self.form_class = form_class self.form_class = form_class
self.form_data = form_data self.form_data = form_data
self.metric = form_data.get('metric') self.metric = form_data.get('metric')
self.admin_view = admin_view
self.df = self.bake_query() self.df = self.bake_query()
self.view = view
if self.df is not None: if self.df is not None:
self.df.timestamp = pd.to_datetime(self.df.timestamp) self.df.timestamp = pd.to_datetime(self.df.timestamp)
self.df_prep() self.df_prep()
@ -68,9 +69,9 @@ class BaseViz(object):
granularity = args.get("granularity") granularity = args.get("granularity")
metric = "count" metric = "count"
limit = int( limit = int(
args.get("limit", settings.ROW_LIMIT)) or settings.ROW_LIMIT args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT
since = args.get("since", "all") since = args.get("since", "all")
from_dttm = (datetime.now() - settings.since_l[since]).isoformat() from_dttm = (datetime.now() - utils.since_l[since]).isoformat()
d = { d = {
'datasource': ds.datasource_name, 'datasource': ds.datasource_name,
'granularity': granularity or 'all', 'granularity': granularity or 'all',
@ -92,7 +93,7 @@ class BaseViz(object):
return d return d
def bake_query(self): def bake_query(self):
client = settings.get_pydruid_client() client = utils.get_pydruid_client()
client.groupby(**self.query_obj()) client.groupby(**self.query_obj())
return client.export_pandas() return client.export_pandas()
@ -108,7 +109,7 @@ class BaseViz(object):
def render(self, *args, **kwargs): def render(self, *args, **kwargs):
form = self.form_class(self.form_data) form = self.form_class(self.form_data)
return self.admin_view.render( return self.view.render_template(
self.template, form=form, viz=self, datasource=self.datasource, self.template, form=form, viz=self, datasource=self.datasource,
*args, **kwargs) *args, **kwargs)
@ -159,29 +160,30 @@ class TimeSeriesViz(HighchartsViz):
""" """
Doing a 2 phase query where we limit the number of series. Doing a 2 phase query where we limit the number of series.
""" """
client = settings.get_pydruid_client() client = utils.get_pydruid_client()
qry = self.query_obj() qry = self.query_obj()
qry['granularity'] = "all" qry['granularity'] = "all"
client.groupby(**qry) client.groupby(**qry)
df = client.export_pandas() df = client.export_pandas()
dims = qry['dimensions'] if not df is None:
filters = [] dims = qry['dimensions']
for index, row in df.iterrows(): filters = []
fields = [] for index, row in df.iterrows():
for dim in dims: fields = []
f = Filter.build_filter(Dimension(dim) == row[dim]) for dim in dims:
fields.append(f) f = Filter.build_filter(Dimension(dim) == row[dim])
if len(fields) > 1: fields.append(f)
filters.append(Filter.build_filter(Filter(type="and", fields=fields))) if len(fields) > 1:
elif fields: filters.append(Filter.build_filter(Filter(type="and", fields=fields)))
filters.append(fields[0]) elif fields:
filters.append(fields[0])
qry = self.query_obj() qry = self.query_obj()
if filters: if filters:
ff = Filter(type="or", fields=filters) ff = Filter(type="or", fields=filters)
qry['filter'] = ff qry['filter'] = ff
del qry['limit_spec'] del qry['limit_spec']
client.groupby(**qry) client.groupby(**qry)
return client.export_pandas() return client.export_pandas()

3
panoramix/babel/babel.cfg Normal file
View File

@ -0,0 +1,3 @@
[python: **.py]
[jinja2: **/templates/**.html]
encoding = utf-8

1
panoramix/babel/messages.pot Normal file
View File

@ -0,0 +1 @@

117
panoramix/config.py Normal file
View File

@ -0,0 +1,117 @@
import os
from flask_appbuilder.security.manager import AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH
basedir = os.path.abspath(os.path.dirname(__file__))
#---------------------------------------------------------
# Panoramix specifix config
#---------------------------------------------------------
ROW_LIMIT = 5000
DRUID_HOST = '10.181.47.80'
DRUID_PORT = 8080
DRUID_BASE_ENDPOINT = 'druid/v2'
COORDINATOR_HOST = '10.168.176.249'
COORDINATOR_PORT = '8080'
COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1'
#---------------------------------------------------------
# Your App secret key
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'
# The SQLAlchemy connection string.
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'app.db')
#SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp'
#SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp'
# Flask-WTF flag for CSRF
CSRF_ENABLED = True
#------------------------------
# GLOBALS FOR APP Builder
#------------------------------
# Uncomment to setup Your App name
APP_NAME = "Panoramix"
# Uncomment to setup Setup an App icon
#APP_ICON = "static/img/logo.jpg"
#----------------------------------------------------
# AUTHENTICATION CONFIG
#----------------------------------------------------
# The authentication type
# AUTH_OID : Is for OpenID
# AUTH_DB : Is for database (username/password()
# AUTH_LDAP : Is for LDAP
# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server
AUTH_TYPE = AUTH_DB
# Uncomment to setup Full admin role name
#AUTH_ROLE_ADMIN = 'Admin'
# Uncomment to setup Public role name, no authentication needed
#AUTH_ROLE_PUBLIC = 'Public'
# Will allow user self registration
#AUTH_USER_REGISTRATION = True
# The default user self registration role
#AUTH_USER_REGISTRATION_ROLE = "Public"
# When using LDAP Auth, setup the ldap server
#AUTH_LDAP_SERVER = "ldap://ldapserver.new"
# Uncomment to setup OpenID providers example for OpenID authentication
#OPENID_PROVIDERS = [
# { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' },
# { 'name': 'AOL', 'url': 'http://openid.aol.com/<username>' },
# { 'name': 'Flickr', 'url': 'http://www.flickr.com/<username>' },
# { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }]
#---------------------------------------------------
# Babel config for translations
#---------------------------------------------------
# Setup default language
BABEL_DEFAULT_LOCALE = 'en'
# Your application default translation path
BABEL_DEFAULT_FOLDER = 'translations'
# The allowed translation for you app
LANGUAGES = {
'en': {'flag':'gb', 'name':'English'},
'pt': {'flag':'pt', 'name':'Portuguese'},
'pt_BR': {'flag':'br', 'name': 'Pt Brazil'},
'es': {'flag':'es', 'name':'Spanish'},
'de': {'flag':'de', 'name':'German'},
'zh': {'flag':'cn', 'name':'Chinese'},
'ru': {'flag':'ru', 'name':'Russian'}
}
#---------------------------------------------------
# Image and file configuration
#---------------------------------------------------
# The file upload folder, when using models with files
UPLOAD_FOLDER = basedir + '/app/static/uploads/'
# The image upload folder, when using models with images
IMG_UPLOAD_FOLDER = basedir + '/app/static/uploads/'
# The image upload url, when using models with images
IMG_UPLOAD_URL = '/static/uploads/'
# Setup image size default is (300, 200, True)
#IMG_SIZE = (300, 200, True)
# Theme configuration
# these are located on static/appbuilder/css/themes
# you can create your own and easily use them placing them on the same dir structure to override
APP_THEME = "bootstrap-theme.css" # default bootstrap
#APP_THEME = "cerulean.css"
#APP_THEME = "amelia.css"
#APP_THEME = "cosmo.css"
#APP_THEME = "cyborg.css"
#APP_THEME = "flatly.css"
#APP_THEME = "journal.css"
#APP_THEME = "readable.css"
#APP_THEME = "simplex.css"
#APP_THEME = "slate.css"
#APP_THEME = "spacelab.css"
#APP_THEME = "united.css"
#APP_THEME = "yeti.css"

0
panoramix/models.py
View File

4
panoramix/run.py Normal file
View File

@ -0,0 +1,4 @@
from app import app
app.run(host='0.0.0.0', port=8081, debug=True)

28
panoramix/settings.py
View File

@ -1,28 +0,0 @@
from datetime import timedelta
FLASK_APP_PORT = 8088
ROW_LIMIT = 10000
SQLALCHEMY_DATABASE_URI = "sqlite:////tmp/panoramix.db"
DRUID_HOST = '10.181.47.80'
DRUID_PORT = 8080
DRUID_BASE_ENDPOINT = 'druid/v2'
COORDINATOR_HOST = '10.168.176.249'
COORDINATOR_PORT = '8080'
COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1'
since_l = {
'1hour': timedelta(hours=1),
'1day': timedelta(days=1),
'7days': timedelta(days=7),
'28days': timedelta(days=28),
'all': timedelta(days=365*100)
}
def get_pydruid_client():
from pydruid import client
return client.PyDruid(
"http://{0}:{1}/".format(DRUID_HOST, DRUID_PORT),
DRUID_BASE_ENDPOINT)

2
panoramix/templates/index.html
View File

@ -1,2 +0,0 @@
{% extends "admin/base.html" %}

2
panoramix/templates/panoramix/base.html
View File

@ -1,2 +0,0 @@
{% extends "index.html" %}

10
requirements.txt
View File

@ -1,10 +0,0 @@
flask
flask-admin
flask-bootstrap
flask-sqlalchemy
pandas
pandas-highcharts
pydruid
python-dateutil
requests
wtforms