Răsfoiți Sursa

Hue-883 [core] Improved support for transactional databases

- Fixed oozie constraint relationships: Workflow was pointing to Start, now points to Node.
- Added transactional assistance to useradmin and jobsub.
- Beeswax query server configuration default for port is 0.
- Useradmin is depedent on all other apps to migrate dynamically.
  Ensures permissions are added last.
- MySQL (InnoDB) and PostgreSQL should work.
abec 13 ani în urmă
părinte
comite
fcbb211

+ 0 - 0
apps/about/src/about/models.py


+ 1 - 1
apps/beeswax/src/beeswax/migrations/0003_auto__add_field_queryhistory_server_name__add_field_queryhistory_serve.py

@@ -15,7 +15,7 @@ class Migration(SchemaMigration):
         db.add_column('beeswax_queryhistory', 'server_host', self.gf('django.db.models.fields.CharField')(default='', max_length=128), keep_default=False)
 
         # Adding field 'QueryHistory.server_port'
-        db.add_column('beeswax_queryhistory', 'server_port', self.gf('django.db.models.fields.SmallIntegerField')(default=''), keep_default=False)
+        db.add_column('beeswax_queryhistory', 'server_port', self.gf('django.db.models.fields.SmallIntegerField')(default=0), keep_default=False)
 
         # Changing field 'QueryHistory.query'
         db.alter_column('beeswax_queryhistory', 'query', self.gf('django.db.models.fields.TextField')())

+ 14 - 2
apps/jobsub/src/jobsub/migrations/0002_auto__add_ooziestreamingaction__add_oozieaction__add_oozieworkflow__ad.py

@@ -35,6 +35,10 @@ LOG = logging.getLogger(__name__)
 class Migration(SchemaMigration):
     
     def forwards(self, orm):
+        """
+        Added custom transaction processing for transactional DBMS.
+        If a DDL operation fails, the entire transaction fails and all future commands are ignored.
+        """
         
         # Adding model 'OozieStreamingAction'
         db.create_table('jobsub_ooziestreamingaction', (
@@ -101,19 +105,27 @@ class Migration(SchemaMigration):
         # Adding field 'CheckForSetup.setup_level'
         db.add_column('jobsub_checkforsetup', 'setup_level', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
     
+        # The next sequence may fail... so they should have their own transactions.
+        db.commit_transaction()
+
         # Delete legacy tables. Note that this only applies to Hue 1.x installations
+        db.start_transaction()
         try:
             db.delete_table('jobsub_submission')
             remove_content_type('jobsub', 'submission')
+            db.commit_transaction()
         except Exception, ex:
-            pass    # Table doesn't exist. Ok.
+            db.rollback_transaction()
 
+        db.start_transaction()
         try:
             db.delete_table('jobsub_serversubmissionstate')
             remove_content_type('jobsub', 'serversubmissionstate')
+            db.commit_transaction()
         except Exception, ex:
-            pass    # Table doesn't exist. Ok.
+            db.rollback_transaction()
 
+        db.start_transaction()
         hue1_to_hue2_data_migration()
 
 

+ 2 - 2
apps/oozie/src/oozie/migrations/0001_initial.py

@@ -25,11 +25,11 @@ class Migration(SchemaMigration):
         # Adding model 'Workflow'
         db.create_table('oozie_workflow', (
             ('job_xml', self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True)),
-            ('end', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='end_workflow', null=True, to=orm['oozie.End'])),
+            ('end', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='end_workflow', null=True, to=orm['oozie.Node'])),
             ('is_single', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True)),
             ('job_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Job'], unique=True, primary_key=True)),
             ('job_properties', self.gf('django.db.models.fields.TextField')(default='[]')),
-            ('start', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='start_workflow', null=True, to=orm['oozie.Start'])),
+            ('start', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='start_workflow', null=True, to=orm['oozie.Node'])),
         ))
         db.send_create_signal('oozie', ['Workflow'])
 

+ 42 - 0
apps/useradmin/src/useradmin/migrations/0001_permissions_and_profiles.py

@@ -4,9 +4,36 @@ from south.db import db
 from south.v2 import DataMigration
 from django.db import models
 
+from desktop.lib.classes import classproperty
+
 from useradmin.models import UserProfile
 
 class Migration(DataMigration):
+    depends_on = (
+      # List of dependencies filled by the depends_on classproperty
+    )
+
+    def _depends_on(cls):
+      """
+      Ensure useradmin is migrated last by placing its migrations after all other migrations
+      """
+      from desktop.appmanager import DESKTOP_MODULES
+      import pkgutil
+
+      dependent_migrations = []
+      for module in DESKTOP_MODULES:
+        if module.name not in __name__:
+          try:
+            package = module.module.migrations
+            for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
+              dependent_migrations.append((module.name, modname))
+          except AttributeError:
+            # No migrations for this iteration
+            pass
+
+      return tuple(dependent_migrations)
+
+    depends_on = classproperty(_depends_on)
     
     def forwards(self, orm):
         """
@@ -37,12 +64,17 @@ class Migration(DataMigration):
               up.creation_method = UserProfile.CreationMethod.HUE
             up.save()
         except Exception:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'UserProfile'
           db.create_table('useradmin_userprofile', (
               ('home_directory', self.gf('django.db.models.fields.CharField')(max_length=1024, null=True)),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], unique=True)),
           ))
+          db.commit_transaction()
+          db.start_transaction()
           db.send_create_signal('useradmin', ['UserProfile'])
 
         try:
@@ -51,17 +83,25 @@ class Migration(DataMigration):
           db.create_index('useradmin_grouppermission', ['group_id'])
           db.create_index('useradmin_grouppermission', ['hue_permission_id'])
         except Exception:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'GroupPermission'
           db.create_table('useradmin_grouppermission', (
               ('hue_permission', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['useradmin.HuePermission'])),
               ('group', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.Group'])),
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
           ))
+          db.commit_transaction()
+          db.start_transaction()
           db.send_create_signal('useradmin', ['GroupPermission'])
 
         try:
           db.rename_table('userman_desktoppermission', 'useradmin_huepermission')
         except Exception:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'HuePermission'
           db.create_table('useradmin_huepermission', (
               ('action', self.gf('django.db.models.fields.CharField')(max_length=100)),
@@ -69,6 +109,8 @@ class Migration(DataMigration):
               ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
               ('description', self.gf('django.db.models.fields.CharField')(max_length=255)),
           ))
+          db.commit_transaction()
+          db.start_transaction()
           db.send_create_signal('useradmin', ['HuePermission'])
     
     def backwards(self, orm):

+ 3 - 0
apps/useradmin/src/useradmin/migrations/0002_add_ldap_support.py

@@ -19,6 +19,9 @@ class Migration(DataMigration):
           db.rename_table('userman_ldapgroup', 'useradmin_ldapgroup')
           db.delete_column('useradmin_ldapgroup', 'hidden')
         except Exception, e:
+          db.rollback_transaction()  
+          db.start_transaction()
+
           # Adding model 'LdapGroup'
           db.create_table('useradmin_ldapgroup', (
               ('group', self.gf('django.db.models.fields.related.ForeignKey')(related_name='group', to=orm['auth.Group'])),

+ 44 - 33
apps/useradmin/src/useradmin/models.py

@@ -213,34 +213,45 @@ def update_app_permissions(**kwargs):
   the best thing we can do, since some apps might not
   have models, but nonetheless, "syncdb" is typically
   run when apps are installed.
+
+  This code executes when useradmin is sync'd since useradmin should be sync'd last.
   """
   # Map app->action->HuePermission.
-  current = {}
-  try:
-    for dp in HuePermission.objects.all():
-      current.setdefault(dp.app, {})[dp.action] = dp
-  except:
-    return
-
-  updated = 0
-  uptodate = 0
-  added = [ ]
-
-  for app_obj in appmanager.DESKTOP_APPS:
-    app = app_obj.name
-    actions = set([("access", "Launch this application")])
-    actions.update(getattr(app_obj.settings, "PERMISSION_ACTIONS", []))
-
-    if app not in current:
-      current[app] = {}
-
-    for action, description in actions:
-      c = current[app].get(action)
-      if c:
-        if c.description != description:
-          c.description = description
-          c.save()
-          updated += 1
+
+  # Only execute for useradmin app since useradmin is sync'd last.
+  # The HuePermission model needs to be sync'd for the following code to work.
+  # Since all apps should have been sync'd before useradmin, referencing them
+  # here is functional.
+  app = kwargs['app']
+  if app.__package__ == 'useradmin':
+    current = {}
+    try:
+      for dp in HuePermission.objects.all():
+        current.setdefault(dp.app, {})[dp.action] = dp
+    except:
+      return
+
+    updated = 0
+    uptodate = 0
+    added = [ ]
+
+    for app_obj in appmanager.DESKTOP_APPS:
+      app = app_obj.name
+      actions = set([("access", "Launch this application")])
+      actions.update(getattr(app_obj.settings, "PERMISSION_ACTIONS", []))
+
+      if app not in current:
+        current[app] = {}
+
+      for action, description in actions:
+        c = current[app].get(action)
+        if c:
+          if c.description != description:
+            c.description = description
+            c.save()
+            updated += 1
+          else:
+            uptodate += 1
         else:
           uptodate += 1
       else:
@@ -248,15 +259,15 @@ def update_app_permissions(**kwargs):
         new_dp.save()
         added.append(new_dp)
 
-  # Add all hue permissions to default group.
-  default_group = get_default_user_group()
-  if default_group:
-    for new_dp in added:
-      GroupPermission.objects.create(group=default_group, hue_permission=new_dp)
+    # Add all hue permissions to default group.
+    default_group = get_default_user_group()
+    if default_group:
+      for new_dp in added:
+        GroupPermission.objects.create(group=default_group, hue_permission=new_dp)
 
-  available = HuePermission.objects.count()
+    available = HuePermission.objects.count()
 
-  LOG.info("HuePermissions: %d added, %d updated, %d up to date, %d stale" %
+    LOG.info("HuePermissions: %d added, %d updated, %d up to date, %d stale" %
            (len(added),
             updated,
             uptodate,

+ 21 - 0
desktop/core/src/desktop/lib/classes.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class classproperty(property):
+    def __get__(self, cls, owner):
+        return classmethod(self.fget).__get__(None, owner)()