ソースを参照

[jobsub] HTTP 500 trying to install Job Design samples

Try to create the home dir of the default hue user (where
the samples are saved)
Replace 500 error by an error message
Romain Rigaux 13 年 前
コミット
f6355ef3b5

+ 9 - 2
apps/jobsub/src/jobsub/management/commands/jobsub_setup.py

@@ -29,6 +29,7 @@ from django.core.management.base import NoArgsCommand
 from django.contrib.auth.models import User
 
 from hadoop import cluster
+from hadoop.fs.hadoopfs import Hdfs
 import hadoop.conf
 import jobsub.conf
 
@@ -47,16 +48,22 @@ class Command(NoArgsCommand):
       remote_fs.setuser(remote_fs.DEFAULT_USER)
     LOG.info("Using remote fs: %s" % str(remote_fs))
 
+    # Create remote home directory if needed
+    remote_data_dir = jobsub.conf.REMOTE_DATA_DIR.get()
+    remote_home_dir = Hdfs.join('/user', remote_fs.user)
+    if remote_data_dir.startswith(remote_home_dir):
+      remote_fs.create_home_dir(remote_home_dir)
+
     # Copy over examples/
     for dirname in ("examples",):
       local_dir = os.path.join(jobsub.conf.LOCAL_DATA_DIR.get(), dirname)
-      remote_dir = posixpath.join(jobsub.conf.REMOTE_DATA_DIR.get(), dirname)
+      remote_dir = posixpath.join(remote_data_dir, dirname)
       copy_dir(local_dir, remote_fs, remote_dir)
 
     # Copy over sample data
     copy_dir(jobsub.conf.SAMPLE_DATA_DIR.get(),
       remote_fs,
-      posixpath.join(jobsub.conf.REMOTE_DATA_DIR.get(), "sample_data"))
+      posixpath.join(remote_data_dir, "sample_data"))
 
     # Write out the models too
     fixture_path = os.path.join(os.path.dirname(__file__), "..", "..", "fixtures", "example_data.xml")

+ 3 - 0
apps/jobsub/src/jobsub/templates/list_designs.mako

@@ -128,6 +128,9 @@ ${layout.menubar(section='designs')}
             <a href="#" class="close" data-dismiss="modal">&times;</a>
             <h3>Install sample job designs?</h3>
         </div>
+        <div class="modal-body">
+          It will take a few seconds to install.
+        </div>
         <div class="modal-footer">
             <input type="submit" class="btn primary" value="Yes"/>
             <a href="#" class="btn secondary" data-dismiss="modal">No</a>

+ 6 - 1
apps/jobsub/src/jobsub/views.py

@@ -38,6 +38,8 @@ from desktop.lib.django_util import render, PopupException, extract_field_data
 from desktop.lib.rest.http_client import RestException
 from desktop.log.access import access_warn
 
+from hadoop.fs.exceptions import WebHdfsException
+
 from jobsub import models, submit
 from jobsub.management.commands import jobsub_setup
 from jobsub.oozie_lib.oozie_api import get_oozie
@@ -265,7 +267,10 @@ def setup(request):
   """Installs jobsub examples."""
   if request.method != "POST":
     raise PopupException('Please use a POST request to install the examples.')
-  jobsub_setup.Command().handle_noargs()
+  try:
+    jobsub_setup.Command().handle_noargs()
+  except WebHdfsException, e:
+    raise PopupException('The examples could not be installed.', detail=e)
   return redirect(urlresolvers.reverse(list_designs))
 
 

+ 14 - 0
desktop/libs/hadoop/src/hadoop/fs/test_webhdfs.py

@@ -52,6 +52,20 @@ def test_webhdfs():
   finally:
     fs.remove("/fortest.txt")
 
+@attr('requires_hadoop')
+def test_webhdfs_functions():
+  """
+  Tests advanced file system operations.
+  """
+  cluster = pseudo_hdfs4.shared_cluster()
+  fs = cluster.fs
+  fs.setuser(cluster.superuser)
+
+  # Create home dir
+  fs.create_home_dir("/user/test")
+  assert_true(fs.isdir("/user/test"))
+  fs.remove("/user/test")
+
 @attr('requires_hadoop')
 def test_seek():
   """Test for DESKTOP-293 - ensure seek works in python2.4"""

+ 18 - 0
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -416,6 +416,24 @@ class WebHdfs(Hdfs):
   def get_hdfs_path(self, path):
     return posixpath.join(self.fs_defaultfs, path.lstrip('/'))
 
+  def create_home_dir(self, home_path=None):
+    if home_path is None:
+      home_path = self.get_home_dir()
+
+    if not self.exists(home_path):
+      user = self.user
+      try:
+        self.setuser(self.superuser)
+        self.mkdir(home_path)
+        self.chmod(home_path, 0755)
+        self.chown(home_path, user, user)
+      except IOError, e:
+        msg = 'Failed to create home dir ("%s") as superuser %s' %\
+              (home_path, self.superuser)
+        LOG.exception(msg)
+        raise PopupException(msg, detail=e)
+      finally:
+        self.setuser(user)
 
   def _invoke_with_redirect(self, method, path, params=None, data=None):
     """