Merge pull request #5 from hailocab/permission_system

Permission system

bin/run

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+source .env
+"$@"

bin/test_multithreading.py

@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+    models.create_db(True, False)
+
+    print "Creating data source..."
+    data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+    print "Clear jobs/hashes:"
+    redis_connection.delete("jobs")
+    query_hashes = redis_connection.keys("query_hash_*")
+    if query_hashes:
+        redis_connection.delete(*query_hashes)
+
+    starting_query_results_count = models.QueryResult.select().count()
+    jobs_count = 5000
+    workers_count = 10
+
+    print "Creating jobs..."
+    for i in xrange(jobs_count):
+        query = "SELECT {}".format(i)
+        print "Inserting: {}".format(query)
+        data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+                             data_source=data_source)
+
+    print "Starting workers..."
+    workers = data_manager.start_workers(workers_count)
+
+    print "Waiting for jobs to be done..."
+    keep_waiting = True
+    while keep_waiting:
+        results_count = models.QueryResult.select().count() - starting_query_results_count
+        print "QueryResults: {}".format(results_count)
+        time.sleep(5)
+        if results_count == jobs_count:
+            print "Yay done..."
+            keep_waiting = False
+
+    data_manager.stop_workers()
+
+    qr_count = 0
+    for qr in models.QueryResult.select():
+        number = int(qr.query.split()[1])
+        data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+        if number != data_number:
+            print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+        qr_count += 1
+
+    print "Verified {} query results.".format(qr_count)
+
+    print "Done."

manage.py

@@ -33,7 +33,7 @@ def runworkers():
 
         logging.info("Cleaning old workers: %s", old_workers)
 
-        data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
+        data_manager.start_workers(settings.WORKERS_COUNT)
         logging.info("Workers started.")
 
         while True:
@@ -52,6 +52,15 @@ def runworkers():
 def make_shell_context():
     return dict(app=app, db=db, models=models)
 
+@manager.command
+def check_settings():
+    from types import ModuleType
+
+    for name in dir(settings):
+        item = getattr(settings, name)
+        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+            print "{} = {}".format(name, item)
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
@@ -71,18 +80,23 @@ def drop_tables():
 @users_manager.option('name', help="User's full name")
 @users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
 @users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
-def create(email, name, is_admin=False, google_auth=False):
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--groups', dest='groups', default=['default'], help="Comma seperated list of groups (leave blank for default).")
+def create(email, name, groups, is_admin=False, google_auth=False, password=None):
     print "Creating user (%s, %s)..." % (email, name)
     print "Admin: %r" % is_admin
     print "Login with Google Auth: %r\n" % google_auth
+    if isinstance(groups, basestring) and len(groups) > 0:
+        groups = groups.split(',')
+    else:
+        groups = ['default']
 
-    permissions = models.User.DEFAULT_PERMISSIONS
     if is_admin:
-        permissions += ['admin']
+        groups += ['admin']
 
-    user = models.User(email=email, name=name, permissions=permissions)
+    user = models.User(email=email, name=name, groups=groups)
     if not google_auth:
-        password = prompt_pass("Password")
+        password = password or prompt_pass("Password")
         user.hash_password(password)
 
     try:
@@ -101,8 +115,4 @@ def delete(email):
 manager.add_command("import", import_manager)
 
 if __name__ == '__main__':
-    channel = logging.StreamHandler()
-    logging.getLogger().addHandler(channel)
-    logging.getLogger().setLevel(settings.LOG_LEVEL)
-
     manager.run()

migrations/add_view_query_permission.py

@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+    previous_default_permissions.remove('view_query')
+    models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+    db.close_db(None)

migrations/create_data_sources.py

@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.DataSource.table_exists():
+        print "Creating data_sources table..."
+        models.DataSource.create_table()
+
+        default_data_source = models.DataSource.create(name="Default",
+                                                       type=settings.CONNECTION_ADAPTER,
+                                                       options=settings.CONNECTION_STRING)
+    else:
+        default_data_source = models.DataSource.select().first()
+
+    migrator = Migrator(db.database)
+    models.Query.data_source.null = True
+    models.QueryResult.data_source.null = True
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    print "Updating data source to existing one..."
+    models.Query.update(data_source=default_data_source.id).execute()
+    models.QueryResult.update(data_source=default_data_source.id).execute()
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+    db.close_db(None)

migrations/permissions_migration.py

@@ -0,0 +1,24 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+
+    if not models.Group.table_exists():
+        print "Creating groups table..."
+        models.Group.create_table()
+
+    with db.database.transaction():
+        models.Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+        models.Group.insert(name='default', permissions=models.Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
+
+        migrator.drop_column(models.User, 'permissions')
+        migrator.add_column(models.User, models.User.groups, 'groups')
+
+        models.User.update(groups=['admin', 'default']).where(models.User.is_admin == True).execute()
+        models.User.update(groups=['default']).where(models.User.is_admin == False).execute()
+
+    db.close_db(None)

rd_ui/app/index.html

@@ -14,6 +14,7 @@
     <link rel="stylesheet" href="/bower_components/gridster/dist/jquery.gridster.css">
     <link rel="stylesheet" href="/bower_components/pivottable/examples/pivot.css">
     <link rel="stylesheet" href="/bower_components/cornelius/src/cornelius.css">
+    <link rel="stylesheet" href="/bower_components/select2/select2.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -35,7 +36,7 @@
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <span ng-repeat="(name, group) in groupedDashboards">
@@ -51,11 +52,11 @@
                     <li ng-repeat="dashboard in otherDashboards">
                         <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard')"></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard') && (groupedDashboards.length > 0 || otherDashboards.length > 0)"></li>
                     <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
@@ -106,6 +107,8 @@
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
+<script src="/bower_components/select2/select2.js"></script>
+<script src="/bower_components/angular-ui-select2/src/select2.js"></script>
 
 <script src="/scripts/ng_highchart.js"></script>
 <script src="/scripts/ng_smart_table.js"></script>
@@ -116,17 +119,22 @@
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
 <script src="/scripts/app.js"></script>
 <script src="/scripts/services/services.js"></script>
+<script src="/scripts/services/resources.js"></script>
 <script src="/scripts/services/notifications.js"></script>
 <script src="/scripts/services/dashboards.js"></script>
 <script src="/scripts/controllers/controllers.js"></script>
+<script src="/scripts/controllers/dashboard.js"></script>
 <script src="/scripts/controllers/admin_controllers.js"></script>
 <script src="/scripts/controllers/query_view.js"></script>
+<script src="/scripts/controllers/query_source.js"></script>
 <script src="/scripts/visualizations/base.js"></script>
 <script src="/scripts/visualizations/chart.js"></script>
 <script src="/scripts/visualizations/cohort.js"></script>
 <script src="/scripts/visualizations/table.js"></script>
 <script src="/scripts/visualizations/pivot.js"></script>
-<script src="/scripts/directives.js"></script>
+<script src="/scripts/directives/directives.js"></script>
+<script src="/scripts/directives/query_directives.js"></script>
+<script src="/scripts/directives/dashboard_directives.js"></script>
 <script src="/scripts/filters.js"></script>
 <!-- endbuild -->
 

rd_ui/app/scripts/app.js

@@ -8,6 +8,7 @@ angular.module('redash', [
     'redash.visualization',
     'ui.codemirror',
     'highchart',
+    'ui.select2',
     'angular-growl',
     'angularMoment',
     'ui.bootstrap',
@@ -17,16 +18,9 @@ angular.module('redash', [
 ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
     function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
 
-        function getQuery(Query, $q, $route) {
-            var defer = $q.defer();
-
-            Query.get({
-                'id': $route.current.params.queryId
-            }, function(query) {
-                defer.resolve(query);
-            });
-
-            return defer.promise;
+        function getQuery(Query, $route) {
+            var query = Query.get({'id': $route.current.params.queryId });
+            return query.$promise;
         };
 
         $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
@@ -43,39 +37,29 @@ angular.module('redash', [
             reloadOnSearch: false
         });
         $routeProvider.when('/queries/new', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
             reloadOnSearch: false,
             resolve: {
-                'viewSource': function isViewSource() {
-                    return true;
-                }
+                'query': ['Query', function newQuery(Query) {
+                    return Query.newQuery();
+                }]
             }
         });
-        // TODO
-        // we should have 2 controllers: queryViewCtrl and queryEditCtrl
         $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/queryview.html',
+            templateUrl: '/views/query.html',
             controller: 'QueryViewCtrl',
             reloadOnSearch: false,
             resolve: {
-                'query': ['Query', '$q', '$route', getQuery]
+                'query': ['Query', '$route', getQuery]
             }
         });
-        $routeProvider.when('/queries/:queryId/fiddle', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
-        });
         $routeProvider.when('/queries/:queryId/source', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
             reloadOnSearch: false,
             resolve: {
-                'query': ['Query', '$q', '$route', getQuery],
-                'viewSource': function isViewSource() {
-                    return true;
-                }
+                'query': ['Query', '$route', getQuery]
             }
         });
         $routeProvider.when('/admin/status', {
@@ -90,9 +74,6 @@ angular.module('redash', [
             redirectTo: '/'
         });
 
-        Highcharts.setOptions({
-            colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
-                        "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
-        });
+
     }
 ]);