From 5f20a080f408fe21dc799cbf45cb3f860f91440c Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 11 Sep 2015 15:32:42 -0700 Subject: [PATCH] Setting up alembic --- .gitignore | 1 + panoramix/__init__.py | 9 ++- panoramix/config.py | 2 + panoramix/migrations/README | 1 + panoramix/migrations/alembic.ini | 45 +++++++++++++++ panoramix/migrations/env.py | 89 +++++++++++++++++++++++++++++ panoramix/migrations/script.py.mako | 22 +++++++ panoramix/models.py | 20 +++++-- panoramix/utils.py | 18 ++++++ panoramix/views.py | 2 - requirements.txt | 2 +- setup.py | 4 +- 12 files changed, 204 insertions(+), 11 deletions(-) create mode 100755 panoramix/migrations/README create mode 100644 panoramix/migrations/alembic.ini create mode 100755 panoramix/migrations/env.py create mode 100755 panoramix/migrations/script.py.mako diff --git a/.gitignore b/.gitignore index ec315860f..03bd6e268 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +build *.db tmp panoramix_config.py diff --git a/panoramix/__init__.py b/panoramix/__init__.py index e71c24f99..487994078 100644 --- a/panoramix/__init__.py +++ b/panoramix/__init__.py @@ -1,6 +1,11 @@ import logging +import os from flask import Flask from flask.ext.appbuilder import SQLA, AppBuilder, IndexView +from flask.ext.migrate import Migrate +from panoramix import config + +APP_DIR = os.path.dirname(__file__) # Logging configuration logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s') @@ -9,6 +14,7 @@ logging.getLogger().setLevel(logging.DEBUG) app = Flask(__name__) app.config.from_object('panoramix.config') db = SQLA(app) +migrate = Migrate(app, db, directory=APP_DIR + "/migrations") class MyIndexView(IndexView): @@ -16,7 +22,8 @@ class MyIndexView(IndexView): appbuilder = AppBuilder( app, db.session, base_template='panoramix/base.html', - indexview=MyIndexView) + indexview=MyIndexView, + security_manager_class=config.CUSTOM_SECURITY_MANAGER) get_session = appbuilder.get_session from panoramix import views diff --git a/panoramix/config.py b/panoramix/config.py index 4ea6e34e1..703b24469 100644 --- a/panoramix/config.py +++ b/panoramix/config.py @@ -19,6 +19,8 @@ ROW_LIMIT = 50000 WEBSERVER_THREADS = 8 PANORAMIX_WEBSERVER_PORT = 8088 + +CUSTOM_SECURITY_MANAGER = None # --------------------------------------------------------- # Your App secret key diff --git a/panoramix/migrations/README b/panoramix/migrations/README new file mode 100755 index 000000000..98e4f9c44 --- /dev/null +++ b/panoramix/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/panoramix/migrations/alembic.ini b/panoramix/migrations/alembic.ini new file mode 100644 index 000000000..f8ed4801f --- /dev/null +++ b/panoramix/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/panoramix/migrations/env.py b/panoramix/migrations/env.py new file mode 100755 index 000000000..83114cab0 --- /dev/null +++ b/panoramix/migrations/env.py @@ -0,0 +1,89 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging +from panoramix import db, models +from flask.ext.appbuilder import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +from flask import current_app +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.readthedocs.org/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + #compare_type=True, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/panoramix/migrations/script.py.mako b/panoramix/migrations/script.py.mako new file mode 100755 index 000000000..95702017e --- /dev/null +++ b/panoramix/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/panoramix/models.py b/panoramix/models.py index 7b154314c..18358611c 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -8,6 +8,7 @@ from pydruid import client from pydruid.utils.filters import Dimension, Filter from sqlalchemy import ( Column, Integer, String, ForeignKey, Text, Boolean, DateTime) +from panoramix.utils import JSONEncodedDict from sqlalchemy import Table as sqlaTable from sqlalchemy import create_engine, MetaData, desc, select, and_ from sqlalchemy.orm import relationship @@ -27,6 +28,15 @@ import config QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration']) +class Slice(Model, AuditMixin): + """A slice is essentially a report or a view on data""" + __tablename__ = 'slices' + id = Column(Integer, primary_key=True) + params = Column(JSONEncodedDict) + datasource = Column(String(250)) + viz_type = Column(String(250)) + + class Queryable(object): @property def column_names(self): @@ -44,7 +54,7 @@ class Queryable(object): class Database(Model, AuditMixin): __tablename__ = 'dbs' id = Column(Integer, primary_key=True) - database_name = Column(String(255), unique=True) + database_name = Column(String(250), unique=True) sqlalchemy_uri = Column(String(1024)) def __repr__(self): @@ -406,7 +416,7 @@ class TableColumn(Model, AuditMixin): class Cluster(Model, AuditMixin): __tablename__ = 'clusters' id = Column(Integer, primary_key=True) - cluster_name = Column(String(255), unique=True) + cluster_name = Column(String(250), unique=True) coordinator_host = Column(String(256)) coordinator_port = Column(Integer) coordinator_endpoint = Column(String(256)) @@ -441,7 +451,7 @@ class Datasource(Model, AuditMixin, Queryable): __tablename__ = 'datasources' id = Column(Integer, primary_key=True) - datasource_name = Column(String(255), unique=True) + datasource_name = Column(String(250), unique=True) is_featured = Column(Boolean, default=False) is_hidden = Column(Boolean, default=False) description = Column(Text) @@ -655,7 +665,7 @@ class Metric(Model): verbose_name = Column(String(1024)) metric_type = Column(String(32)) datasource_name = Column( - String(256), + String(250), ForeignKey('datasources.datasource_name')) datasource = relationship('Datasource', backref='metrics') json = Column(Text) @@ -674,7 +684,7 @@ class Column(Model, AuditMixin): __tablename__ = 'columns' id = Column(Integer, primary_key=True) datasource_name = Column( - String(256), + String(250), ForeignKey('datasources.datasource_name')) datasource = relationship('Datasource', backref='columns') column_name = Column(String(256)) diff --git a/panoramix/utils.py b/panoramix/utils.py index bc83ddd29..78536c710 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -1,4 +1,6 @@ from datetime import datetime +from sqlalchemy.types import TypeDecorator, TEXT +import json import parsedatetime @@ -33,3 +35,19 @@ def parse_human_timedelta(s): d = datetime( d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec) return d - dttm + + + +class JSONEncodedDict(TypeDecorator): + """Represents an immutable structure as a json-encoded string.""" + impl = TEXT + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value diff --git a/panoramix/views.py b/panoramix/views.py index a5cfb07e5..982089b38 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -262,5 +262,3 @@ appbuilder.add_link( category='Admin', category_icon='fa-cogs', icon="fa-cog") - -db.create_all() diff --git a/requirements.txt b/requirements.txt index 73d0e8889..eb06eeb91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ flask -flask-alembic +flask-migrate flask-appbuilder gunicorn mysql-python diff --git a/setup.py b/setup.py index ee30c5344..1f6627d93 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,13 @@ setup( "and druid.io"), version=version, packages=find_packages(), - package_data={'': ['panoramix/alembic.ini']}, + package_data={'': ['panoramix/migrations/alembic.ini']}, include_package_data=True, zip_safe=False, scripts=['panoramix/bin/panoramix'], install_requires=[ 'flask-appbuilder>=1.4.5', - 'flask-alembic>=1.2.1', + 'flask-migrate>=1.5.1', 'gunicorn>=19.3.0', 'pandas>=0.16.2', 'pydruid>=0.2.2',