Browse Source

Hue-883 [core] Improved support for transactional databases

- Fixed oozie constraint relationships: Workflow was pointing to Start, now points to Node.
- Added transactional assistance to useradmin and jobsub.
- Beeswax query server configuration default for port is 0.
- Useradmin is depedent on all other apps to migrate dynamically.
  Ensures permissions are added last.
- MySQL (InnoDB) and PostgreSQL should work.
abec 13 years ago
parent
commit
fcbb211

+ 0 - 0
apps/about/src/about/models.py


+ 1 - 1
apps/beeswax/src/beeswax/migrations/0003_auto__add_field_queryhistory_server_name__add_field_queryhistory_serve.py

@@ -15,7 +15,7 @@ class Migration(SchemaMigration):
         db.add_column('beeswax_queryhistory', 'server_host', self.gf('django.db.models.fields.CharField')(default='', max_length=128), keep_default=False)
         db.add_column('beeswax_queryhistory', 'server_host', self.gf('django.db.models.fields.CharField')(default='', max_length=128), keep_default=False)
 
 
         # Adding field 'QueryHistory.server_port'
         # Adding field 'QueryHistory.server_port'
-        db.add_column('beeswax_queryhistory', 'server_port', self.gf('django.db.models.fields.SmallIntegerField')(default=''), keep_default=False)
+        db.add_column('beeswax_queryhistory', 'server_port', self.gf('django.db.models.fields.SmallIntegerField')(default=0), keep_default=False)
 
 
         # Changing field 'QueryHistory.query'
         # Changing field 'QueryHistory.query'
         db.alter_column('beeswax_queryhistory', 'query', self.gf('django.db.models.fields.TextField')())
         db.alter_column('beeswax_queryhistory', 'query', self.gf('django.db.models.fields.TextField')())

+ 14 - 2
apps/jobsub/src/jobsub/migrations/0002_auto__add_ooziestreamingaction__add_oozieaction__add_oozieworkflow__ad.py

@@ -35,6 +35,10 @@ LOG = logging.getLogger(__name__)
 class Migration(SchemaMigration):
 class Migration(SchemaMigration):
     
     
     def forwards(self, orm):
     def forwards(self, orm):
+        """
+        Added custom transaction processing for transactional DBMS.
+        If a DDL operation fails, the entire transaction fails and all future commands are ignored.
+        """
         
         
         # Adding model 'OozieStreamingAction'
         # Adding model 'OozieStreamingAction'
         db.create_table('jobsub_ooziestreamingaction', (
         db.create_table('jobsub_ooziestreamingaction', (
@@ -101,19 +105,27 @@ class Migration(SchemaMigration):
         # Adding field 'CheckForSetup.setup_level'
         # Adding field 'CheckForSetup.setup_level'
         db.add_column('jobsub_checkforsetup', 'setup_level', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
         db.add_column('jobsub_checkforsetup', 'setup_level', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
     
     
+        # The next sequence may fail... so they should have their own transactions.
+        db.commit_transaction()
+
         # Delete legacy tables. Note that this only applies to Hue 1.x installations
         # Delete legacy tables. Note that this only applies to Hue 1.x installations
+        db.start_transaction()
         try:
         try:
             db.delete_table('jobsub_submission')
             db.delete_table('jobsub_submission')
             remove_content_type('jobsub', 'submission')
             remove_content_type('jobsub', 'submission')
+            db.commit_transaction()
         except Exception, ex:
         except Exception, ex:
-            pass    # Table doesn't exist. Ok.
+            db.rollback_transaction()
 
 
+        db.start_transaction()
         try:
         try:
             db.delete_table('jobsub_serversubmissionstate')
             db.delete_table('jobsub_serversubmissionstate')
             remove_content_type('jobsub', 'serversubmissionstate')
             remove_content_type('jobsub', 'serversubmissionstate')
+            db.commit_transaction()
         except Exception, ex:
         except Exception, ex:
-            pass    # Table doesn't exist. Ok.
+            db.rollback_transaction()
 
 
+        db.start_transaction()
         hue1_to_hue2_data_migration()
         hue1_to_hue2_data_migration()
 
 
 
 

+ 2 - 2
apps/oozie/src/oozie/migrations/0001_initial.py

@@ -25,11 +25,11 @@ class Migration(SchemaMigration):
         # Adding model 'Workflow'
         # Adding model 'Workflow'
         db.create_table('oozie_workflow', (
         db.create_table('oozie_workflow', (
             ('job_xml', self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True)),
             ('job_xml', self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True)),
-            ('end', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='end_workflow', null=True, to=orm['oozie.End'])),
+            ('end', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='end_workflow', null=True, to=orm['oozie.Node'])),
             ('is_single', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True)),
             ('is_single', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True)),
             ('job_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Job'], unique=True, primary_key=True)),
             ('job_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Job'], unique=True, primary_key=True)),
             ('job_properties', self.gf('django.db.models.fields.TextField')(default='[]')),
             ('job_properties', self.gf('django.db.models.fields.TextField')(default='[]')),
-            ('start', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='start_workflow', null=True, to=orm['oozie.Start'])),
+            ('start', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='start_workflow', null=True, to=orm['oozie.Node'])),
         ))
         ))
         db.send_create_signal('oozie', ['Workflow'])
         db.send_create_signal('oozie', ['Workflow'])
 
 

+ 42 - 0
apps/useradmin/src/useradmin/migrations/0001_permissions_and_profiles.py

@@ -4,9 +4,36 @@ from south.db import db
 from south.v2 import DataMigration
 from south.v2 import DataMigration
 from django.db import models
 from django.db import models
 
 
+from desktop.lib.classes import classproperty
+
 from useradmin.models import UserProfile
 from useradmin.models import UserProfile
 
 
 class Migration(DataMigration):
 class Migration(DataMigration):
+    depends_on = (
+      # List of dependencies filled by the depends_on classproperty
+    )
+
+    def _depends_on(cls):
+      """
+      Ensure useradmin is migrated last by placing its migrations after all other migrations
+      """
+      from desktop.appmanager import DESKTOP_MODULES
+      import pkgutil
+
+      dependent_migrations = []
+      for module in DESKTOP_MODULES:
+        if module.name not in __name__:
+          try:
+            package = module.module.migrations
+            for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
+              dependent_migrations.append((module.name, modname))
+          except AttributeError:
+            # No migrations for this iteration
+            pass
+
+      return tuple(dependent_migrations)
+
+    depends_on = classproperty(_depends_on)
     
     
     def forwards(self, orm):
     def forwards(self, orm):
         """
         """
@@ -37,12 +64,17 @@ class Migration(DataMigration):
               up.creation_method = UserProfile.CreationMethod.HUE
               up.creation_method = UserProfile.CreationMethod.HUE
             up.save()
             up.save()
         except Exception:
         except Exception:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'UserProfile'
           # Adding model 'UserProfile'
           db.create_table('useradmin_userprofile', (
           db.create_table('useradmin_userprofile', (
               ('home_directory', self.gf('django.db.models.fields.CharField')(max_length=1024, null=True)),
               ('home_directory', self.gf('django.db.models.fields.CharField')(max_length=1024, null=True)),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], unique=True)),
               ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], unique=True)),
           ))
           ))
+          db.commit_transaction()
+          db.start_transaction()
           db.send_create_signal('useradmin', ['UserProfile'])
           db.send_create_signal('useradmin', ['UserProfile'])
 
 
         try:
         try:
@@ -51,17 +83,25 @@ class Migration(DataMigration):
           db.create_index('useradmin_grouppermission', ['group_id'])
           db.create_index('useradmin_grouppermission', ['group_id'])
           db.create_index('useradmin_grouppermission', ['hue_permission_id'])
           db.create_index('useradmin_grouppermission', ['hue_permission_id'])
         except Exception:
         except Exception:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'GroupPermission'
           # Adding model 'GroupPermission'
           db.create_table('useradmin_grouppermission', (
           db.create_table('useradmin_grouppermission', (
               ('hue_permission', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['useradmin.HuePermission'])),
               ('hue_permission', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['useradmin.HuePermission'])),
               ('group', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.Group'])),
               ('group', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.Group'])),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
           ))
           ))
+          db.commit_transaction()
+          db.start_transaction()
           db.send_create_signal('useradmin', ['GroupPermission'])
           db.send_create_signal('useradmin', ['GroupPermission'])
 
 
         try:
         try:
           db.rename_table('userman_desktoppermission', 'useradmin_huepermission')
           db.rename_table('userman_desktoppermission', 'useradmin_huepermission')
         except Exception:
         except Exception:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'HuePermission'
           # Adding model 'HuePermission'
           db.create_table('useradmin_huepermission', (
           db.create_table('useradmin_huepermission', (
               ('action', self.gf('django.db.models.fields.CharField')(max_length=100)),
               ('action', self.gf('django.db.models.fields.CharField')(max_length=100)),
@@ -69,6 +109,8 @@ class Migration(DataMigration):
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('description', self.gf('django.db.models.fields.CharField')(max_length=255)),
               ('description', self.gf('django.db.models.fields.CharField')(max_length=255)),
           ))
           ))
+          db.commit_transaction()
+          db.start_transaction()
           db.send_create_signal('useradmin', ['HuePermission'])
           db.send_create_signal('useradmin', ['HuePermission'])
     
     
     def backwards(self, orm):
     def backwards(self, orm):

+ 3 - 0
apps/useradmin/src/useradmin/migrations/0002_add_ldap_support.py

@@ -19,6 +19,9 @@ class Migration(DataMigration):
           db.rename_table('userman_ldapgroup', 'useradmin_ldapgroup')
           db.rename_table('userman_ldapgroup', 'useradmin_ldapgroup')
           db.delete_column('useradmin_ldapgroup', 'hidden')
           db.delete_column('useradmin_ldapgroup', 'hidden')
         except Exception, e:
         except Exception, e:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'LdapGroup'
           # Adding model 'LdapGroup'
           db.create_table('useradmin_ldapgroup', (
           db.create_table('useradmin_ldapgroup', (
               ('group', self.gf('django.db.models.fields.related.ForeignKey')(related_name='group', to=orm['auth.Group'])),
               ('group', self.gf('django.db.models.fields.related.ForeignKey')(related_name='group', to=orm['auth.Group'])),

+ 44 - 33
apps/useradmin/src/useradmin/models.py

@@ -213,34 +213,45 @@ def update_app_permissions(**kwargs):
   the best thing we can do, since some apps might not
   the best thing we can do, since some apps might not
   have models, but nonetheless, "syncdb" is typically
   have models, but nonetheless, "syncdb" is typically
   run when apps are installed.
   run when apps are installed.
+
+  This code executes when useradmin is sync'd since useradmin should be sync'd last.
   """
   """
   # Map app->action->HuePermission.
   # Map app->action->HuePermission.
-  current = {}
-  try:
-    for dp in HuePermission.objects.all():
-      current.setdefault(dp.app, {})[dp.action] = dp
-  except:
-    return
-
-  updated = 0
-  uptodate = 0
-  added = [ ]
-
-  for app_obj in appmanager.DESKTOP_APPS:
-    app = app_obj.name
-    actions = set([("access", "Launch this application")])
-    actions.update(getattr(app_obj.settings, "PERMISSION_ACTIONS", []))
-
-    if app not in current:
-      current[app] = {}
-
-    for action, description in actions:
-      c = current[app].get(action)
-      if c:
-        if c.description != description:
-          c.description = description
-          c.save()
-          updated += 1
+
+  # Only execute for useradmin app since useradmin is sync'd last.
+  # The HuePermission model needs to be sync'd for the following code to work.
+  # Since all apps should have been sync'd before useradmin, referencing them
+  # here is functional.
+  app = kwargs['app']
+  if app.__package__ == 'useradmin':
+    current = {}
+    try:
+      for dp in HuePermission.objects.all():
+        current.setdefault(dp.app, {})[dp.action] = dp
+    except:
+      return
+
+    updated = 0
+    uptodate = 0
+    added = [ ]
+
+    for app_obj in appmanager.DESKTOP_APPS:
+      app = app_obj.name
+      actions = set([("access", "Launch this application")])
+      actions.update(getattr(app_obj.settings, "PERMISSION_ACTIONS", []))
+
+      if app not in current:
+        current[app] = {}
+
+      for action, description in actions:
+        c = current[app].get(action)
+        if c:
+          if c.description != description:
+            c.description = description
+            c.save()
+            updated += 1
+          else:
+            uptodate += 1
         else:
         else:
           uptodate += 1
           uptodate += 1
       else:
       else:
@@ -248,15 +259,15 @@ def update_app_permissions(**kwargs):
         new_dp.save()
         new_dp.save()
         added.append(new_dp)
         added.append(new_dp)
 
 
-  # Add all hue permissions to default group.
-  default_group = get_default_user_group()
-  if default_group:
-    for new_dp in added:
-      GroupPermission.objects.create(group=default_group, hue_permission=new_dp)
+    # Add all hue permissions to default group.
+    default_group = get_default_user_group()
+    if default_group:
+      for new_dp in added:
+        GroupPermission.objects.create(group=default_group, hue_permission=new_dp)
 
 
-  available = HuePermission.objects.count()
+    available = HuePermission.objects.count()
 
 
-  LOG.info("HuePermissions: %d added, %d updated, %d up to date, %d stale" %
+    LOG.info("HuePermissions: %d added, %d updated, %d up to date, %d stale" %
            (len(added),
            (len(added),
             updated,
             updated,
             uptodate,
             uptodate,

+ 21 - 0
desktop/core/src/desktop/lib/classes.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class classproperty(property):
+    def __get__(self, cls, owner):
+        return classmethod(self.fget).__get__(None, owner)()