浏览代码

[oozie] Refactor oozie_setup command

Workaround for HDFS-3491
i18n of the messages
Raise exception if error during copy
Romain Rigaux 13 年之前
父节点
当前提交
338be2d

+ 19 - 15
apps/oozie/src/oozie/management/commands/oozie_setup.py

@@ -21,6 +21,7 @@ import posixpath
 
 
 from django.core import management
 from django.core import management
 from django.core.management.base import NoArgsCommand
 from django.core.management.base import NoArgsCommand
+from django.utils.translation import ugettext as _
 
 
 from hadoop import cluster
 from hadoop import cluster
 
 
@@ -36,28 +37,27 @@ class Command(NoArgsCommand):
     remote_fs = cluster.get_hdfs()
     remote_fs = cluster.get_hdfs()
     remote_dir = Workflow.objects.create_data_dir(remote_fs)
     remote_dir = Workflow.objects.create_data_dir(remote_fs)
 
 
-    # Copy sample binaries
+    # Copy examples binaries
     for demo in ('lib', 'pig'):
     for demo in ('lib', 'pig'):
       local_dir = posixpath.join(LOCAL_SAMPLE_DIR.get(), demo)
       local_dir = posixpath.join(LOCAL_SAMPLE_DIR.get(), demo)
       remote_data_dir = posixpath.join(remote_dir, demo)
       remote_data_dir = posixpath.join(remote_dir, demo)
-      LOG.info('Copying workflows %s to %s\n' % (local_dir, remote_data_dir))
+      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
+                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
       copy_dir(local_dir, remote_fs, remote_data_dir)
       copy_dir(local_dir, remote_fs, remote_data_dir)
 
 
     # Copy sample data
     # Copy sample data
     local_dir = LOCAL_SAMPLE_DATA_DIR.get()
     local_dir = LOCAL_SAMPLE_DATA_DIR.get()
     remote_data_dir = posixpath.join(remote_dir, 'data')
     remote_data_dir = posixpath.join(remote_dir, 'data')
-    LOG.info('Copying data %s to %s\n' % (local_dir, remote_data_dir))
+    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
+                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
     copy_dir(local_dir, remote_fs, remote_data_dir)
     copy_dir(local_dir, remote_fs, remote_data_dir)
 
 
     # Load jobs
     # Load jobs
     management.call_command('loaddata', 'apps/oozie/src/oozie/fixtures/initial_data.json', verbosity=2)
     management.call_command('loaddata', 'apps/oozie/src/oozie/fixtures/initial_data.json', verbosity=2)
 
 
-  def has_been_setup(self):
-    return False
-
 
 
 def copy_dir(local_dir, remote_fs, remote_dir, mode=755):
 def copy_dir(local_dir, remote_fs, remote_dir, mode=755):
-  remote_fs.mkdir(remote_dir, mode=mode)
+  remote_fs.do_as_user(remote_fs.DEFAULT_USER, remote_fs.mkdir, remote_dir, mode=mode)
 
 
   for f in os.listdir(local_dir):
   for f in os.listdir(local_dir):
     local_src = os.path.join(local_dir, f)
     local_src = os.path.join(local_dir, f)
@@ -69,21 +69,25 @@ CHUNK_SIZE = 1024 * 1024
 
 
 def copy_file(local_src, remote_fs, remote_dst):
 def copy_file(local_src, remote_fs, remote_dst):
   if remote_fs.exists(remote_dst):
   if remote_fs.exists(remote_dst):
-    LOG.info('%s already exists.  Skipping.' % remote_dst)
+    LOG.info(_('%(remote_dst)s already exists.  Skipping.') % {'remote_dst': remote_dst})
     return
     return
   else:
   else:
-    LOG.info('%s does not exist. trying to copy' % remote_dst)
+    LOG.info(_('%(remote_dst)s does not exist. Trying to copy') % {'remote_dst': remote_dst})
 
 
   if os.path.isfile(local_src):
   if os.path.isfile(local_src):
     src = file(local_src)
     src = file(local_src)
     try:
     try:
-      remote_fs.create(remote_dst, permission=01755)
-      chunk = src.read(CHUNK_SIZE)
-      while chunk:
-        remote_fs.append(remote_dst, chunk)
+      try:
+        remote_fs.do_as_user(remote_fs.DEFAULT_USER, remote_fs.create, remote_dst, permission=01755)
         chunk = src.read(CHUNK_SIZE)
         chunk = src.read(CHUNK_SIZE)
-      LOG.info('Copied %s -> %s' % (local_src, remote_dst))
+        while chunk:
+          remote_fs.do_as_user(remote_fs.DEFAULT_USER, remote_fs.append, remote_dst, chunk)
+          chunk = src.read(CHUNK_SIZE)
+        LOG.info(_('Copied %s -> %s') % (local_src, remote_dst))
+      except:
+        LOG.error(_('Copying %s -> %s failed') % (local_src, remote_dst))
+        raise
     finally:
     finally:
       src.close()
       src.close()
   else:
   else:
-    LOG.info('Skipping %s (not a file)' % local_src)
+    LOG.info(_('Skipping %s (not a file)') % local_src)

+ 1 - 0
apps/oozie/src/oozie/models.py

@@ -157,6 +157,7 @@ class WorkflowManager(models.Manager):
             fs.create_home_dir(remote_home_dir)
             fs.create_home_dir(remote_home_dir)
           # Shared by all the users
           # Shared by all the users
           fs.mkdir(directory, 01777)
           fs.mkdir(directory, 01777)
+          fs.chmod(directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
     finally:
     finally:
       fs.setuser(user)
       fs.setuser(user)
 
 

+ 6 - 4
apps/oozie/src/oozie/templates/editor/list_workflows.mako

@@ -34,7 +34,9 @@ ${ layout.menubar(section='workflows') }
   <div class="well hueWell">
   <div class="well hueWell">
     <div class="btn-group pull-right">
     <div class="btn-group pull-right">
       <a href="${ url('oozie:create_workflow') }" class="btn">${ _('Create') }</a>
       <a href="${ url('oozie:create_workflow') }" class="btn">${ _('Create') }</a>
-      <a href="#installSamples" data-toggle="modal" class="btn">${ _('Install examples') }</a>
+      % if currentuser.is_superuser:
+        <a href="#installSamples" data-toggle="modal" class="btn">${ _('Setup App') }</a>
+      % endif
     </div>
     </div>
 
 
     <div class="row-fluid">
     <div class="row-fluid">
@@ -139,13 +141,13 @@ ${ layout.menubar(section='workflows') }
 </div>
 </div>
 
 
 <div id="installSamples" class="modal hide fade">
 <div id="installSamples" class="modal hide fade">
-  <form id="installSamplesForm" action="${url('oozie:install_examples')}" method="POST">
+  <form id="installSamplesForm" action="${url('oozie:setup_app')}" method="POST">
     <div class="modal-header">
     <div class="modal-header">
       <a href="#" class="close" data-dismiss="modal">&times;</a>
       <a href="#" class="close" data-dismiss="modal">&times;</a>
-      <h3>${ _('Install samples?') }</h3>
+      <h3>${ _('Setup the workspaces and examples?') }</h3>
     </div>
     </div>
     <div class="modal-body">
     <div class="modal-body">
-      ${ _('It will take a few seconds to install.') }
+      ${ _('Hue is going to re-create the workspaces and re-install the examples...') }
     </div>
     </div>
     <div class="modal-footer">
     <div class="modal-footer">
       <input type="submit" class="btn primary" value="${ _('Yes') }"/>
       <input type="submit" class="btn primary" value="${ _('Yes') }"/>

+ 2 - 2
apps/oozie/src/oozie/tests.py

@@ -660,8 +660,8 @@ class TestEditor:
     data = json.loads(response.content)
     data = json.loads(response.content)
     assert_equal(0, data['status'], data['data'])
     assert_equal(0, data['status'], data['data'])
 
 
-  def test_install_examples(self):
-    self.c.post(reverse('oozie:install_examples'))
+  def test_setup_app(self):
+    self.c.post(reverse('oozie:setup_app'))
 
 
 
 
 # Utils
 # Utils

+ 1 - 1
apps/oozie/src/oozie/urls.py

@@ -51,7 +51,7 @@ urlpatterns = patterns(
   url(r'^workflow_parameters/(?P<workflow>\d+)$', 'get_workflow_parameters', name='workflow_parameters'),
   url(r'^workflow_parameters/(?P<workflow>\d+)$', 'get_workflow_parameters', name='workflow_parameters'),
   url(r'^list_history$', 'list_history', name='list_history'),
   url(r'^list_history$', 'list_history', name='list_history'),
   url(r'^list_history/(?P<record_id>[-\w]+)$', 'list_history_record', name='list_history_record'),
   url(r'^list_history/(?P<record_id>[-\w]+)$', 'list_history_record', name='list_history_record'),
-  url(r'^install_examples/$', 'install_examples', name='install_examples'),
+  url(r'^setup_app/$', 'setup_app', name='setup_app'),
 )
 )
 
 
 urlpatterns += patterns(
 urlpatterns += patterns(

+ 5 - 5
apps/oozie/src/oozie/views/editor.py

@@ -172,7 +172,7 @@ def check_action_edition_permission(view_func):
 
 
 
 
 def list_workflows(request, job_type='workflow'):
 def list_workflows(request, job_type='workflow'):
-  show_install_examples = True
+  show_setup_app = True
 
 
   if job_type == 'coordinators':
   if job_type == 'coordinators':
     data = Coordinator.objects
     data = Coordinator.objects
@@ -191,7 +191,7 @@ def list_workflows(request, job_type='workflow'):
   return render(template, request, {
   return render(template, request, {
     'jobs': list(data),
     'jobs': list(data),
     'currentuser': request.user,
     'currentuser': request.user,
-    'show_install_examples': show_install_examples,
+    'show_setup_app': show_setup_app,
   })
   })
 
 
 
 
@@ -679,13 +679,13 @@ def list_history_record(request, record_id):
   })
   })
 
 
 
 
-def install_examples(request):
+def setup_app(request):
   if request.method != 'POST':
   if request.method != 'POST':
     raise PopupException(_('A POST request is required.'))
     raise PopupException(_('A POST request is required.'))
   try:
   try:
     oozie_setup.Command().handle_noargs()
     oozie_setup.Command().handle_noargs()
-    request.info(_('Examples installed!'))
+    request.info(_('Workspace and examples installed!'))
   except WebHdfsException, e:
   except WebHdfsException, e:
-    raise PopupException(_('The examples could not be installed.'), detail=e)
+    raise PopupException(_('The app setup could complete.'), detail=e)
   return redirect(reverse('oozie:list_workflows'))
   return redirect(reverse('oozie:list_workflows'))