diff --git a/.gitignore b/.gitignore index 00b1aa5fc..700ce0ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +babel .DS_Store .coverage _build diff --git a/babel/babel.cfg b/babel/babel.cfg deleted file mode 100644 index 70e23ac63..000000000 --- a/babel/babel.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[python: **.py] -[jinja2: **/templates/**.html] -encoding = utf-8 diff --git a/babel/messages.pot b/babel/messages.pot deleted file mode 100644 index 8b1378917..000000000 --- a/babel/messages.pot +++ /dev/null @@ -1 +0,0 @@ - diff --git a/panoramix/__init__.py b/panoramix/__init__.py index e23539f69..ff9baaa61 100644 --- a/panoramix/__init__.py +++ b/panoramix/__init__.py @@ -33,4 +33,4 @@ appbuilder = AppBuilder( sm = appbuilder.sm get_session = appbuilder.get_session -from panoramix import config, views +from panoramix import config, views # noqa diff --git a/panoramix/assets/javascripts/featured.js b/panoramix/assets/javascripts/featured.js index 8fa875098..cd549dcc9 100644 --- a/panoramix/assets/javascripts/featured.js +++ b/panoramix/assets/javascripts/featured.js @@ -1,9 +1,10 @@ var $ = window.$ = require('jquery'); var jQuery = window.jQuery = $; +var px = require('./modules/panoramix.js'); +require('bootstrap'); require('datatables'); require('../node_modules/datatables-bootstrap3-plugin/media/css/datatables-bootstrap3.css'); -require('bootstrap'); $(document).ready(function () { $('#dataset-table').DataTable({ @@ -13,5 +14,6 @@ $(document).ready(function () { ] }); $('#dataset-table_info').remove(); + //$('input[type=search]').addClass('form-control'); # TODO get search box to look nice $('#dataset-table').show(); }); diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index a95084ad9..e6d7a3723 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -7,6 +7,7 @@ from subprocess import Popen from flask.ext.script import Manager from panoramix import app from flask.ext.migrate import MigrateCommand +import panoramix from panoramix import db from panoramix import data, utils @@ -49,7 +50,7 @@ def runserver(debug, port, timeout, workers): @manager.command def init(): """Inits the Panoramix application""" - utils.init() + utils.init(panoramix) @manager.option( '-s', '--sample', action='store_true', @@ -58,6 +59,8 @@ def load_examples(sample): """Loads a set of Slices and Dashboards and a supporting dataset """ print("Loading examples into {}".format(db)) + data.load_css_templates() + print("Loading [World Bank's Health Nutrition and Population Stats]") data.load_world_bank_health_n_pop() diff --git a/panoramix/config.py b/panoramix/config.py index ed5314e75..4980a06cf 100644 --- a/panoramix/config.py +++ b/panoramix/config.py @@ -25,7 +25,7 @@ CUSTOM_SECURITY_MANAGER = None # --------------------------------------------------------- # Your App secret key -SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' +SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' # noqa # The SQLAlchemy connection string. SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/panoramix.db' @@ -48,7 +48,7 @@ SHOW_STACKTRACE = True APP_NAME = "Panoramix" # Uncomment to setup Setup an App icon -APP_ICON = "/static/img/chaudron_white.png" +# APP_ICON = "/static/img/something.png" # Druid query timezone # tz.tzutc() : Using utc timezone @@ -113,6 +113,6 @@ IMG_UPLOAD_URL = '/static/uploads/' # IMG_SIZE = (300, 200, True) try: - from panoramix_config import * + from panoramix_config import * # noqa except Exception: pass diff --git a/panoramix/data/__init__.py b/panoramix/data/__init__.py index 08a0e95eb..9ba1c8755 100644 --- a/panoramix/data/__init__.py +++ b/panoramix/data/__init__.py @@ -274,33 +274,14 @@ def load_world_bank_health_n_pop(): dash = Dash( dashboard_title=dash_name, position_json=json.dumps(l, indent=4), + slug="world_health", ) for s in slices: dash.slices.append(s) db.session.commit() -def load_birth_names(): - session = db.session - with gzip.open(os.path.join(DATA_FOLDER, 'birth_names.json.gz')) as f: - pdf = pd.read_json(f) - pdf.ds = pd.to_datetime(pdf.ds, unit='ms') - pdf.to_sql( - 'birth_names', - db.engine, - if_exists='replace', - chunksize=500, - dtype={ - 'ds': DateTime, - 'gender': String(16), - 'state': String(10), - 'name': String(255), - }, - index=False) - l = [] - print("Done loading table!") - print("-" * 80) - +def load_css_templates(): print('Creating default CSS templates') CSS = models.CssTemplate @@ -400,6 +381,27 @@ def load_birth_names(): db.session.merge(obj) db.session.commit() + +def load_birth_names(): + with gzip.open(os.path.join(DATA_FOLDER, 'birth_names.json.gz')) as f: + pdf = pd.read_json(f) + pdf.ds = pd.to_datetime(pdf.ds, unit='ms') + pdf.to_sql( + 'birth_names', + db.engine, + if_exists='replace', + chunksize=500, + dtype={ + 'ds': DateTime, + 'gender': String(16), + 'state': String(10), + 'name': String(255), + }, + index=False) + l = [] + print("Done loading table!") + print("-" * 80) + print("Creating table reference") obj = db.session.query(TBL).filter_by(table_name='birth_names').first() if not obj: @@ -500,12 +502,15 @@ def load_birth_names(): defaults, viz_type="markup", markup_type="html", code="""\ -
-

Birth Names Dashboard

-

The source dataset came from [here]

- -
- """ +
+

Birth Names Dashboard

+

+ The source dataset came from + [here] +

+ +
+""" )), Slice( slice_name="Name Cloud", @@ -531,7 +536,7 @@ def load_birth_names(): merge_slice(slc) print("Creating a dashboard") - dash = session.query(Dash).filter_by(dashboard_title="Births").first() + dash = db.session.query(Dash).filter_by(dashboard_title="Births").first() if dash: db.session.delete(dash) @@ -608,7 +613,8 @@ def load_birth_names(): dash = Dash( dashboard_title="Births", position_json=json.dumps(l, indent=4), + slug="births", ) for s in slices: dash.slices.append(s) - session.commit() + db.session.commit() diff --git a/panoramix/data/countries.py b/panoramix/data/countries.py index 0a39d316b..5955fba18 100644 --- a/panoramix/data/countries.py +++ b/panoramix/data/countries.py @@ -1,3 +1,7 @@ +""" +This module contains data related to countries and is used for geo mapping +""" + countries = [ { "name": "Angola", diff --git a/panoramix/forms.py b/panoramix/forms.py index 1d8bf565a..e58c63ee2 100644 --- a/panoramix/forms.py +++ b/panoramix/forms.py @@ -1,20 +1,21 @@ from wtforms import ( - Field, Form, SelectMultipleField, SelectField, TextField, TextAreaField, + Form, SelectMultipleField, SelectField, TextField, TextAreaField, BooleanField, IntegerField, HiddenField) from wtforms import validators, widgets from copy import copy from panoramix import app -from six import string_types from collections import OrderedDict config = app.config class BetterBooleanField(BooleanField): + """ Fixes behavior of html forms omitting non checked (which doesn't distinguish False from NULL/missing ) If value is unchecked, this hidden fills in False value """ + def __call__(self, **kwargs): html = super(BetterBooleanField, self).__call__(**kwargs) html += u''.format(self.name) @@ -22,9 +23,9 @@ class BetterBooleanField(BooleanField): class SelectMultipleSortableField(SelectMultipleField): - """ - Works along with select2sortable to preserves the sort order - """ + + """Works along with select2sortable to preserves the sort order""" + def iter_choices(self): d = OrderedDict() for value, label in self.choices: @@ -39,6 +40,9 @@ class SelectMultipleSortableField(SelectMultipleField): class FreeFormSelect(widgets.Select): + + """A WTF widget that allows for free form entry""" + def __call__(self, field, **kwargs): kwargs.setdefault('id', field.id) if self.multiple: @@ -54,13 +58,20 @@ class FreeFormSelect(widgets.Select): html.append('') return widgets.HTMLString(''.join(html)) + class FreeFormSelectField(SelectField): + + """ A WTF SelectField that allows for free form input """ + widget = FreeFormSelect() def pre_validate(self, form): return class OmgWtForm(Form): + + """Panoramixification of the WTForm Form object""" + fieldsets = {} css_classes = dict() @@ -74,6 +85,7 @@ class OmgWtForm(Form): class FormFactory(object): + """Used to create the forms in the explore view dynamically""" series_limits = [0, 5, 10, 25, 50, 100, 500] fieltype_class = { SelectField: 'select2', @@ -231,12 +243,15 @@ class FormFactory(object): ]), description="Charge in the force layout"), 'granularity_sqla': SelectField( - 'Time Column', default=datasource.main_dttm_col, + 'Time Column', + default=datasource.main_dttm_col or datasource.any_dttm_col, choices=self.choicify(datasource.dttm_cols), description=( - "The time granularity for the visualization. Note that you " + "The time column for the visualization. Note that you " "can define arbitrary expression that return a DATETIME " - "column in the table editor")), + "column in the table editor. Also note that the " + "filter bellow is applied against this column or " + "expression")), 'resample_rule': FreeFormSelectField( 'Resample Rule', default='', choices=self.choicify(('1T', '1H', '1D', '7D', '1M', '1AS')), @@ -347,7 +362,9 @@ class FormFactory(object): "complex expression, parenthesis and anything else " "supported by the backend it is directed towards.")), 'compare_lag': TextField('Comparison Period Lag', - description="Based on granularity, number of time periods to compare against"), + description=( + "Based on granularity, number of time periods to " + "compare against")), 'compare_suffix': TextField('Comparison suffix', description="Suffix to apply after the percentage display"), 'x_axis_format': FreeFormSelectField('X axis format', @@ -356,7 +373,8 @@ class FormFactory(object): ('smart_date', 'Adaptative formating'), ("%m/%d/%Y", '"%m/%d/%Y" | 01/14/2019'), ("%Y-%m-%d", '"%Y-%m-%d" | 2019-01-14'), - ("%Y-%m-%d %H:%M:%S", '"%Y-%m-%d %H:%M:%S" | 2019-01-14 01:32:10'), + ("%Y-%m-%d %H:%M:%S", + '"%Y-%m-%d %H:%M:%S" | 2019-01-14 01:32:10'), ("%H:%M:%S", '"%H:%M:%S" | 01:32:10'), ], description="D3 format syntax for y axis " @@ -474,12 +492,10 @@ class FormFactory(object): def choicify(l): return [("{}".format(obj), "{}".format(obj)) for obj in l] - def get_form(self, previous=False): - px_form_fields = self.field_dict + def get_form(self): viz = self.viz - datasource = viz.datasource field_css_classes = {} - for name, obj in px_form_fields.items(): + for name, obj in self.field_dict.items(): field_css_classes[name] = ['form-control'] s = self.fieltype_class.get(obj.field_class) if s: @@ -489,7 +505,7 @@ class FormFactory(object): field_css_classes[field] += ['input-sm'] class QueryForm(OmgWtForm): - fieldsets = copy(viz.fieldsetizer()) + fieldsets = copy(viz.fieldsets) css_classes = field_css_classes standalone = HiddenField() async = HiddenField() @@ -501,7 +517,7 @@ class FormFactory(object): collapsed_fieldsets = HiddenField() viz_type = self.field_dict.get('viz_type') - filter_cols = datasource.filterable_column_names or [''] + filter_cols = viz.datasource.filterable_column_names or [''] for i in range(10): setattr(QueryForm, 'flt_col_' + str(i), SelectField( 'Filter 1', @@ -514,18 +530,16 @@ class FormFactory(object): setattr( QueryForm, 'flt_eq_' + str(i), TextField("Super", default='')) - for fieldset in viz.fieldsetizer(): - for ff in fieldset['fields']: - if ff: - if isinstance(ff, string_types): - ff = [ff] - for s in ff: - if s: - setattr(QueryForm, s, px_form_fields[s]) + for field in viz.flat_form_fields(): + setattr(QueryForm, field, self.field_dict[field]) + + def add_to_form(attrs): + for attr in attrs: + setattr(QueryForm, attr, self.field_dict[attr]) # datasource type specific form elements - if datasource.__class__.__name__ == 'SqlaTable': + if viz.datasource.__class__.__name__ == 'SqlaTable': QueryForm.fieldsets += ({ 'label': 'SQL', 'fields': ['where', 'having'], @@ -533,12 +547,36 @@ class FormFactory(object): "This section exposes ways to include snippets of " "SQL in your query"), },) - setattr(QueryForm, 'where', px_form_fields['where']) - setattr(QueryForm, 'having', px_form_fields['having']) + add_to_form(('where', 'having')) + grains = viz.datasource.database.grains() - if 'granularity' in viz.flat_form_fields(): - setattr( - QueryForm, - 'granularity', px_form_fields['granularity_sqla']) - field_css_classes['granularity'] = ['form-control', 'select2'] + if not viz.datasource.any_dttm_col: + return QueryForm + if grains: + time_fields = ('granularity_sqla', 'time_grain_sqla') + self.field_dict['time_grain_sqla'] = SelectField( + 'Time Grain', + choices=self.choicify((grain.name for grain in grains)), + default="Time Column", + description=( + "The time granularity for the visualization. This " + "applies a date transformation to alter " + "your time column and defines a new time granularity." + "The options here are defined on a per database " + "engine basis in the Panoramix source code")) + add_to_form(time_fields) + field_css_classes['time_grain_sqla'] = ['form-control', 'select2'] + else: + time_fields = 'granularity_sqla' + add_to_form((time_fields, )) + add_to_form(('since', 'until')) + QueryForm.fieldsets = ({ + 'label': 'Time', + 'fields': ( + time_fields, + ('since', 'until'), + ), + 'description': "Time related form attributes", + },) + tuple(QueryForm.fieldsets) + field_css_classes['granularity'] = ['form-control', 'select2'] return QueryForm diff --git a/panoramix/migrations/versions/18e88e1cc004_making_audit_nullable.py b/panoramix/migrations/versions/18e88e1cc004_making_audit_nullable.py index bd5f850cc..0143aad58 100644 --- a/panoramix/migrations/versions/18e88e1cc004_making_audit_nullable.py +++ b/panoramix/migrations/versions/18e88e1cc004_making_audit_nullable.py @@ -15,85 +15,84 @@ import sqlalchemy as sa def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.alter_column('clusters', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('clusters', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - try: + op.alter_column( + 'clusters', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column( + 'clusters', 'created_on', + existing_type=sa.DATETIME(), nullable=True) op.drop_constraint(None, 'columns', type_='foreignkey') op.drop_constraint(None, 'columns', type_='foreignkey') op.drop_column('columns', 'created_on') op.drop_column('columns', 'created_by_fk') op.drop_column('columns', 'changed_on') op.drop_column('columns', 'changed_by_fk') + op.alter_column('css_templates', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('css_templates', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('dashboards', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('dashboards', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.create_unique_constraint(None, 'dashboards', ['slug']) + op.alter_column('datasources', 'changed_by_fk', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('datasources', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('datasources', 'created_by_fk', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('datasources', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('dbs', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('dbs', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('slices', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('slices', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('sql_metrics', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('sql_metrics', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('table_columns', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('table_columns', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('tables', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('tables', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('url', 'changed_on', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('url', 'created_on', + existing_type=sa.DATETIME(), + nullable=True) + ### end Alembic commands ### except: pass - op.alter_column('css_templates', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('css_templates', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('dashboards', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('dashboards', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.create_unique_constraint(None, 'dashboards', ['slug']) - op.alter_column('datasources', 'changed_by_fk', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('datasources', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('datasources', 'created_by_fk', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('datasources', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('dbs', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('dbs', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('slices', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('slices', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('sql_metrics', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('sql_metrics', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('table_columns', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('table_columns', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('tables', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('tables', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('url', 'changed_on', - existing_type=sa.DATETIME(), - nullable=True) - op.alter_column('url', 'created_on', - existing_type=sa.DATETIME(), - nullable=True) - ### end Alembic commands ### def downgrade(): diff --git a/panoramix/models.py b/panoramix/models.py index 410d0c905..05f4a670d 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -1,6 +1,7 @@ from copy import deepcopy, copy from collections import namedtuple from datetime import timedelta, datetime +import functools import json import logging from six import string_types @@ -8,7 +9,7 @@ import sqlparse import requests from dateutil.parser import parse -from flask import flash +from flask import flash, request, g from flask.ext.appbuilder import Model from flask.ext.appbuilder.models.mixins import AuditMixin import pandas as pd @@ -39,31 +40,39 @@ class AuditMixinNullable(AuditMixin): changed_on = Column( DateTime, default=datetime.now, onupdate=datetime.now, nullable=True) + @declared_attr def created_by_fk(cls): return Column(Integer, ForeignKey('ab_user.id'), default=cls.get_user_id, nullable=True) + @declared_attr def changed_by_fk(cls): return Column(Integer, ForeignKey('ab_user.id'), default=cls.get_user_id, onupdate=cls.get_user_id, nullable=True) + @property def created_by_(self): return '{}'.format(self.created_by or '') - @property + + @property # noqa def changed_by_(self): return '{}'.format(self.changed_by or '') class Url(Model, AuditMixinNullable): + """Used for the short url feature""" + __tablename__ = 'url' id = Column(Integer, primary_key=True) url = Column(Text) class CssTemplate(Model, AuditMixinNullable): + """CSS templates for dashboards""" + __tablename__ = 'css_templates' id = Column(Integer, primary_key=True) template_name = Column(String(250)) @@ -71,7 +80,9 @@ class CssTemplate(Model, AuditMixinNullable): class Slice(Model, AuditMixinNullable): + """A slice is essentially a report or a view on data""" + __tablename__ = 'slices' id = Column(Integer, primary_key=True) slice_name = Column(String(250)) @@ -154,17 +165,6 @@ class Slice(Model, AuditMixinNullable): return '{self.slice_name}'.format( url=url, self=self) - @property - def js_files(self): - return viz_types[self.viz_type].js_files - - @property - def css_files(self): - return viz_types[self.viz_type].css_files - - def get_viz(self): - pass - dashboard_slices = Table('dashboard_slices', Model.metadata, Column('id', Integer, primary_key=True), @@ -174,7 +174,9 @@ dashboard_slices = Table('dashboard_slices', Model.metadata, class Dashboard(Model, AuditMixinNullable): + """A dash to slash""" + __tablename__ = 'dashboards' id = Column(Integer, primary_key=True) dashboard_title = Column(String(500)) @@ -203,20 +205,6 @@ class Dashboard(Model, AuditMixinNullable): def dashboard_link(self): return '{self.dashboard_title}'.format(self=self) - @property - def js_files(self): - l = [] - for o in self.slices: - l += [f for f in o.js_files if f not in l] - return l - - @property - def css_files(self): - l = [] - for o in self.slices: - l += o.css_files - return list(set(l)) - @property def json_data(self): d = { @@ -267,6 +255,37 @@ class Database(Model, AuditMixinNullable): def safe_sqlalchemy_uri(self): return self.sqlalchemy_uri + def grains(self): + + """Defines time granularity database-specific expressions. The idea + here is to make it easy for users to change the time grain form a + datetime (maybe the source grain is arbitrary timestamps, daily + or 5 minutes increments) to another, "truncated" datetime. Since + each database has slightly different but similar datetime functions, + this allows a mapping between database engines and actual functions. + """ + + Grain = namedtuple('Grain', 'name function') + DB_TIME_GRAINS = { + 'presto': ( + Grain('Time Column', '{col}'), + Grain('week', "date_trunc('week', {col})"), + Grain('month', "date_trunc('month', {col})"), + ), + 'mysql': ( + Grain('Time Column', '{col}'), + Grain('day', 'DATE({col})'), + Grain('week', 'DATE_SUB({col}, INTERVAL DAYOFWEEK({col}) - 1 DAY)'), + Grain('month', 'DATE_SUB({col}, INTERVAL DAYOFMONTH({col}) - 1 DAY)'), + ), + } + for db_type, grains in DB_TIME_GRAINS.items(): + if self.sqlalchemy_uri.startswith(db_type): + return grains + + def grains_dict(self): + return {grain.name: grain for grain in self.grains()} + def get_table(self, table_name): meta = MetaData() return Table( @@ -345,6 +364,12 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): l.append(self.main_dttm_col) return l + @property + def any_dttm_col(self): + cols = self.dttm_cols + if cols: + return cols[0] + @property def html(self): t = ((c.column_name, c.type) for c in self.columns) @@ -386,8 +411,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): self, groupby, metrics, granularity, from_dttm, to_dttm, - limit_spec=None, - filter=None, + filter=None, # noqa is_timeseries=True, timeseries_limit=15, row_limit=None, inner_from_dttm=None, inner_to_dttm=None, @@ -400,14 +424,21 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): cols = {col.column_name: col for col in self.columns} qry_start_dttm = datetime.now() - if not self.main_dttm_col: + + if not granularity and is_timeseries: raise Exception( - "Datetime column not provided as part table configuration") - dttm_expr = cols[granularity].expression - if dttm_expr: + "Datetime column not provided as part table configuration " + "and is required by this type of chart") + if granularity: + dttm_expr = cols[granularity].expression or granularity + + # Transforming time grain into an expression based on configuration + time_grain_sqla = extras.get('time_grain_sqla') + if time_grain_sqla: + udf = self.database.grains_dict().get(time_grain_sqla, '{col}') + dttm_expr = udf.function.format(col=dttm_expr) timestamp = literal_column(dttm_expr).label('timestamp') - else: - timestamp = literal_column(granularity).label('timestamp') + metrics_exprs = [ literal_column(m.expression).label(m.metric_name) for m in self.metrics if m.metric_name in metrics] @@ -455,16 +486,17 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): if not columns: qry = qry.group_by(*groupby_exprs) - tf = '%Y-%m-%d %H:%M:%S.%f' - time_filter = [ - timestamp >= from_dttm.strftime(tf), - timestamp <= to_dttm.strftime(tf), - ] - inner_time_filter = copy(time_filter) - if inner_from_dttm: - inner_time_filter[0] = timestamp >= inner_from_dttm.strftime(tf) - if inner_to_dttm: - inner_time_filter[1] = timestamp <= inner_to_dttm.strftime(tf) + if granularity: + tf = '%Y-%m-%d %H:%M:%S.%f' + time_filter = [ + timestamp >= from_dttm.strftime(tf), + timestamp <= to_dttm.strftime(tf), + ] + inner_time_filter = copy(time_filter) + if inner_from_dttm: + inner_time_filter[0] = timestamp >= inner_from_dttm.strftime(tf) + if inner_to_dttm: + inner_time_filter[1] = timestamp <= inner_to_dttm.strftime(tf) where_clause_and = [] having_clause_and = [] for col, op, eq in filter: @@ -483,7 +515,8 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): where_clause_and += [text(extras['where'])] if extras and 'having' in extras: having_clause_and += [text(extras['having'])] - qry = qry.where(and_(*(time_filter + where_clause_and))) + if granularity: + qry = qry.where(and_(*(time_filter + where_clause_and))) qry = qry.having(and_(*having_clause_and)) if groupby: qry = qry.order_by(desc(main_metric_expr)) @@ -813,8 +846,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): self, groupby, metrics, granularity, from_dttm, to_dttm, - limit_spec=None, - filter=None, + filter=None, # noqa is_timeseries=True, timeseries_limit=None, row_limit=None, @@ -888,7 +920,9 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): pre_qry['limit_spec'] = { "type": "default", "limit": timeseries_limit, - 'intervals': inner_from_dttm.isoformat() + '/' + inner_to_dttm.isoformat(), + 'intervals': ( + inner_from_dttm.isoformat() + '/' + + inner_to_dttm.isoformat()), "columns": [{ "dimension": metrics[0] if metrics else self.metrics[0], "direction": "descending", @@ -902,7 +936,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): if df is not None and not df.empty: dims = qry['dimensions'] filters = [] - for index, row in df.iterrows(): + for _, row in df.iterrows(): fields = [] for dim in dims: f = Filter.build_filter(Dimension(dim) == row[dim]) @@ -970,6 +1004,29 @@ class Log(Model): user = relationship('User', backref='logs', foreign_keys=[user_id]) dttm = Column(DateTime, default=func.now()) + @classmethod + def log_this(cls, f): + """Decorator to log user actions""" + @functools.wraps(f) + def wrapper(*args, **kwargs): + user_id = None + if g.user: + user_id = g.user.id + d = request.args.to_dict() + d.update(kwargs) + log = cls( + action=f.__name__, + json=json.dumps(d), + dashboard_id=d.get('dashboard_id') or None, + slice_id=d.get('slice_id') or None, + user_id=user_id) + db.session.add(log) + db.session.commit() + return f(*args, **kwargs) + return wrapper + + + class DruidMetric(Model): __tablename__ = 'metrics' diff --git a/panoramix/static/img/chaudron.png b/panoramix/static/img/chaudron.png deleted file mode 100644 index c1dd4ed74..000000000 Binary files a/panoramix/static/img/chaudron.png and /dev/null differ diff --git a/panoramix/static/img/chaudron_white.png b/panoramix/static/img/chaudron_white.png deleted file mode 100644 index 8e634c8b2..000000000 Binary files a/panoramix/static/img/chaudron_white.png and /dev/null differ diff --git a/panoramix/static/img/panoramix.jpg b/panoramix/static/img/panoramix.jpg deleted file mode 100644 index b6eb231ca..000000000 Binary files a/panoramix/static/img/panoramix.jpg and /dev/null differ diff --git a/panoramix/static/img/panoramix.png b/panoramix/static/img/panoramix.png deleted file mode 100644 index 3ce04c348..000000000 Binary files a/panoramix/static/img/panoramix.png and /dev/null differ diff --git a/panoramix/static/img/tux_panoramix.png b/panoramix/static/img/tux_panoramix.png deleted file mode 100644 index 3e2d5f102..000000000 Binary files a/panoramix/static/img/tux_panoramix.png and /dev/null differ diff --git a/panoramix/templates/panoramix/base.html b/panoramix/templates/panoramix/base.html index 03ab1a42e..b075d52be 100644 --- a/panoramix/templates/panoramix/base.html +++ b/panoramix/templates/panoramix/base.html @@ -3,6 +3,7 @@ {% block head_css %} {{super()}} + {% endblock %} {% block head_js %} diff --git a/panoramix/templates/panoramix/basic.html b/panoramix/templates/panoramix/basic.html index dcd4ca832..8d0cba178 100644 --- a/panoramix/templates/panoramix/basic.html +++ b/panoramix/templates/panoramix/basic.html @@ -8,6 +8,7 @@ {% block head_meta %}{% endblock %} {% block head_css %} + {% endblock %} {% block head_js %} diff --git a/panoramix/templates/panoramix/featured.html b/panoramix/templates/panoramix/featured.html index 0b33f6ed9..36e37a9e7 100644 --- a/panoramix/templates/panoramix/featured.html +++ b/panoramix/templates/panoramix/featured.html @@ -1,4 +1,10 @@ {% extends "panoramix/basic.html" %} + +{% block head_js %} + {{ super() }} + +{% endblock %} + {% block body %}
@@ -34,7 +40,3 @@
{% endblock %} -{% block tail_js %} - {{ super() }} - -{% endblock %} diff --git a/panoramix/utils.py b/panoramix/utils.py index dee370441..d24409264 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -1,16 +1,14 @@ from datetime import datetime -import functools import hashlib +import functools import json import logging from dateutil.parser import parse from sqlalchemy.types import TypeDecorator, TEXT -from flask import g, request, Markup from markdown import markdown as md import parsedatetime - -from panoramix import db +from flask_appbuilder.security.sqla import models as ab_models class memoized(object): @@ -64,12 +62,14 @@ def parse_human_datetime(s): True >>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date() True - >>> parse_human_datetime('one year ago').date() == (datetime.now() - relativedelta(years=1) ).date() + >>> year_ago_1 = parse_human_datetime('one year ago').date() + >>> year_ago_2 = (datetime.now() - relativedelta(years=1) ).date() + >>> year_ago_1 == year_ago_2 True """ try: dttm = parse(s) - except: + except Exception: try: cal = parsedatetime.Calendar() dttm = dttm_from_timtuple(cal.parse(s)[0]) @@ -154,14 +154,13 @@ class ColorFactory(object): return self.BNB_COLORS[i % len(self.BNB_COLORS)] -def init(): +def init(panoramix): """ Inits the Panoramix application with security roles and such """ - from panoramix import appbuilder - from panoramix import models - from flask_appbuilder.security.sqla import models as ab_models - sm = appbuilder.sm + db = panoramix.db + models = panoramix.models + sm = panoramix.appbuilder.sm alpha = sm.add_role("Alpha") admin = sm.add_role("Admin") @@ -178,7 +177,6 @@ def init(): sm.add_permission_role(admin, perm) gamma = sm.add_role("Gamma") for perm in perms: - s = perm.permission.name if( perm.view_menu.name not in ( 'ResetPasswordView', @@ -205,30 +203,6 @@ def init(): merge_perm(sm, 'datasource_access', table_perm) -def log_this(f): - ''' - Decorator to log user actions - ''' - @functools.wraps(f) - def wrapper(*args, **kwargs): - user_id = None - if g.user: - user_id = g.user.id - from panoramix import models - d = request.args.to_dict() - d.update(kwargs) - log = models.Log( - action=f.__name__, - json=json.dumps(d), - dashboard_id=d.get('dashboard_id') or None, - slice_id=d.get('slice_id') or None, - user_id=user_id) - db.session.add(log) - db.session.commit() - return f(*args, **kwargs) - return wrapper - - def datetime_f(dttm): """ Formats datetime to take less room is recent diff --git a/panoramix/views.py b/panoramix/views.py index 6951b65dc..9518bb8c8 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -20,14 +20,15 @@ from sqlalchemy.sql.expression import TextAsFrom from panoramix import appbuilder, db, models, viz, utils, app, sm, ascii_art config = app.config +log_this = models.Log.log_this -def validate_json(form, field): +def validate_json(form, field): # noqa try: json.loads(field.data) except Exception as e: logging.exception(e) - raise ValidationError("Json isn't valid") + raise ValidationError("json isn't valid") class DeleteMixin(object): @@ -161,7 +162,9 @@ class TableModelView(PanoramixModelView, DeleteMixin): base_order = ('changed_on','desc') description_columns = { 'offset': "Timezone offset (in hours) for this datasource", - 'description': Markup("Supports markdown"), + 'description': Markup( + "Supports " + "markdown"), } def post_add(self, table): @@ -303,7 +306,9 @@ class DruidDatasourceModelView(PanoramixModelView, DeleteMixin): base_order = ('datasource_name', 'asc') description_columns = { 'offset': "Timezone offset (in hours) for this datasource", - 'description': Markup("Supports markdown"), + 'description': Markup( + "Supports markdown"), } def post_add(self, datasource): @@ -332,7 +337,7 @@ def ping(): class R(BaseView): - @utils.log_this + @log_this @expose("/") def index(self, url_id): url = db.session.query(models.Url).filter_by(id=url_id).first() @@ -343,7 +348,7 @@ class R(BaseView): flash("URL to nowhere...", "danger") return redirect('/') - @utils.log_this + @log_this @expose("/shortner/", methods=['POST', 'GET']) def shortner(self): url = request.form.get('data') @@ -361,7 +366,7 @@ class Panoramix(BaseView): @has_access @expose("/explore///") @expose("/datasource///") # Legacy url - @utils.log_this + @log_this def explore(self, datasource_type, datasource_id): if datasource_type == "table": datasource = ( @@ -561,8 +566,8 @@ class Panoramix(BaseView): dash = qry.first() # Hack to log the dashboard_id properly, even when getting a slug - @utils.log_this - def dashboard(**kwargs): + @log_this + def dashboard(**kwargs): # noqa pass dashboard(dashboard_id=dash.id) @@ -578,7 +583,7 @@ class Panoramix(BaseView): @has_access @expose("/sql//") - @utils.log_this + @log_this def sql(self, database_id): mydb = db.session.query( models.Database).filter_by(id=database_id).first() @@ -594,7 +599,7 @@ class Panoramix(BaseView): @has_access @expose("/table///") - @utils.log_this + @log_this def table(self, database_id, table_name): mydb = db.session.query( models.Database).filter_by(id=database_id).first() @@ -612,7 +617,7 @@ class Panoramix(BaseView): @has_access @expose("/select_star///") - @utils.log_this + @log_this def select_star(self, database_id, table_name): mydb = db.session.query( models.Database).filter_by(id=database_id).first() @@ -627,7 +632,7 @@ class Panoramix(BaseView): @has_access @expose("/runsql/", methods=['POST', 'GET']) - @utils.log_this + @log_this def runsql(self): session = db.session() limit = 1000 diff --git a/panoramix/viz.py b/panoramix/viz.py index 3c7fa7709..1973aefca 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -26,8 +26,6 @@ class BaseViz(object): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'metrics', 'groupby', ) },) @@ -81,23 +79,16 @@ class BaseViz(object): s = Markup(s) return s - def fieldsetizer(self): - """ - Makes form_fields support either a list approach or a fieldsets - approach - """ - return self.fieldsets - @classmethod def flat_form_fields(cls): l = set() for d in cls.fieldsets: for obj in d['fields']: - if isinstance(obj, (tuple, list)): - l |= {a for a in obj} + if obj and isinstance(obj, (tuple, list)): + l |= {a for a in obj if a} elif obj: l.add(obj) - return l + return tuple(l) def reassignments(self): pass @@ -111,7 +102,7 @@ class BaseViz(object): d.update(kwargs) # Remove unchecked checkboxes because HTML is weird like that for key in d.keys(): - if d[key] == False: + if d[key] is False: del d[key] href = Href( '/panoramix/explore/{self.datasource.type}/' @@ -174,7 +165,8 @@ class BaseViz(object): form_data = self.form_data groupby = form_data.get("groupby") or [] metrics = form_data.get("metrics") or ['count'] - granularity = form_data.get("granularity") + granularity = \ + form_data.get("granularity") or form_data.get("granularity_sqla") limit = int(form_data.get("limit", 0)) row_limit = int( form_data.get("row_limit", config.get("ROW_LIMIT"))) @@ -193,6 +185,7 @@ class BaseViz(object): extras = { 'where': form_data.get("where", ''), 'having': form_data.get("having", ''), + 'time_grain_sqla': form_data.get("time_grain_sqla", ''), } d = { 'granularity': granularity, @@ -259,10 +252,8 @@ class TableViz(BaseViz): verbose_name = "Table View" fieldsets = ( { - 'label': None, + 'label': "Chart Options", 'fields': ( - 'granularity', - ('since', 'until'), 'row_limit', ('include_search', None), ) @@ -322,8 +313,6 @@ class PivotTableViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'groupby', 'columns', 'metrics', @@ -409,8 +398,6 @@ class WordCloudViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'series', 'metric', 'limit', ('size_from', 'size_to'), 'rotation', @@ -447,8 +434,6 @@ class BubbleViz(NVD3Viz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'series', 'entity', 'x', 'y', 'size', 'limit', @@ -515,8 +500,6 @@ class BigNumberViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'metric', 'compare_lag', 'compare_suffix', @@ -567,7 +550,6 @@ class NVD3TimeSeriesViz(NVD3Viz): { 'label': None, 'fields': ( - 'granularity', ('since', 'until'), 'metrics', 'groupby', 'limit', ), @@ -743,8 +725,6 @@ class DistributionPieViz(NVD3Viz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'metrics', 'groupby', 'limit', ('donut', 'show_legend'), @@ -777,12 +757,6 @@ class DistributionBarViz(DistributionPieViz): is_timeseries = False fieldsets = ( { - 'label': None, - 'fields': ( - 'granularity', - ('since', 'until'), - ) - }, { 'label': 'Chart Options', 'fields': ( 'groupby', @@ -803,7 +777,7 @@ class DistributionBarViz(DistributionPieViz): } def query_obj(self): - d = super(DistributionPieViz, self).query_obj() + d = super(DistributionPieViz, self).query_obj() # noqa fd = self.form_data d['is_timeseries'] = False gb = fd.get('groupby') or [] @@ -818,7 +792,7 @@ class DistributionBarViz(DistributionPieViz): return d def get_df(self, query_obj=None): - df = super(DistributionPieViz, self).get_df(query_obj) + df = super(DistributionPieViz, self).get_df(query_obj) # noqa fd = self.form_data row = df.groupby(self.groupby).sum()[self.metrics[0]].copy() @@ -863,8 +837,6 @@ class SunburstViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'groupby', 'metric', 'secondary_metric', 'row_limit', @@ -925,8 +897,6 @@ class SankeyViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'groupby', 'metric', 'row_limit', @@ -962,8 +932,6 @@ class DirectedForceViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'groupby', 'metric', 'row_limit', @@ -1004,8 +972,6 @@ class WorldMapViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'entity', 'country_fieldtype', 'metric', @@ -1077,8 +1043,6 @@ class FilterBoxViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'groupby', 'metric', ) @@ -1138,8 +1102,6 @@ class ParallelCoordinatesViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'series', 'metrics', 'secondary_metric', @@ -1170,8 +1132,6 @@ class HeatmapViz(BaseViz): { 'label': None, 'fields': ( - 'granularity', - ('since', 'until'), 'all_columns_x', 'all_columns_y', 'metric', diff --git a/tests/core_tests.py b/tests/core_tests.py index 7101b5dd7..8fae4f55d 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -5,6 +5,7 @@ import unittest os.environ['PANORAMIX_CONFIG'] = 'tests.panoramix_test_config' from flask.ext.testing import LiveServerTestCase, TestCase +import panoramix from panoramix import app, db, models, utils BASE_DIR = app.config.get("BASE_DIR") cli = imp.load_source('cli', BASE_DIR + "/bin/panoramix") @@ -21,7 +22,7 @@ class LiveTest(TestCase): pass def test_init(self): - utils.init() + utils.init(panoramix) def test_load_examples(self): cli.load_examples(sample=True)