Browse Source

[jobsub] HTTP 500 trying to install Job Design samples

Try to create the home dir of the default hue user (where
the samples are saved)
Replace 500 error by an error message
Romain Rigaux 13 years ago
parent
commit
f6355ef3b5

+ 9 - 2
apps/jobsub/src/jobsub/management/commands/jobsub_setup.py

@@ -29,6 +29,7 @@ from django.core.management.base import NoArgsCommand
 from django.contrib.auth.models import User
 from django.contrib.auth.models import User
 
 
 from hadoop import cluster
 from hadoop import cluster
+from hadoop.fs.hadoopfs import Hdfs
 import hadoop.conf
 import hadoop.conf
 import jobsub.conf
 import jobsub.conf
 
 
@@ -47,16 +48,22 @@ class Command(NoArgsCommand):
       remote_fs.setuser(remote_fs.DEFAULT_USER)
       remote_fs.setuser(remote_fs.DEFAULT_USER)
     LOG.info("Using remote fs: %s" % str(remote_fs))
     LOG.info("Using remote fs: %s" % str(remote_fs))
 
 
+    # Create remote home directory if needed
+    remote_data_dir = jobsub.conf.REMOTE_DATA_DIR.get()
+    remote_home_dir = Hdfs.join('/user', remote_fs.user)
+    if remote_data_dir.startswith(remote_home_dir):
+      remote_fs.create_home_dir(remote_home_dir)
+
     # Copy over examples/
     # Copy over examples/
     for dirname in ("examples",):
     for dirname in ("examples",):
       local_dir = os.path.join(jobsub.conf.LOCAL_DATA_DIR.get(), dirname)
       local_dir = os.path.join(jobsub.conf.LOCAL_DATA_DIR.get(), dirname)
-      remote_dir = posixpath.join(jobsub.conf.REMOTE_DATA_DIR.get(), dirname)
+      remote_dir = posixpath.join(remote_data_dir, dirname)
       copy_dir(local_dir, remote_fs, remote_dir)
       copy_dir(local_dir, remote_fs, remote_dir)
 
 
     # Copy over sample data
     # Copy over sample data
     copy_dir(jobsub.conf.SAMPLE_DATA_DIR.get(),
     copy_dir(jobsub.conf.SAMPLE_DATA_DIR.get(),
       remote_fs,
       remote_fs,
-      posixpath.join(jobsub.conf.REMOTE_DATA_DIR.get(), "sample_data"))
+      posixpath.join(remote_data_dir, "sample_data"))
 
 
     # Write out the models too
     # Write out the models too
     fixture_path = os.path.join(os.path.dirname(__file__), "..", "..", "fixtures", "example_data.xml")
     fixture_path = os.path.join(os.path.dirname(__file__), "..", "..", "fixtures", "example_data.xml")

+ 3 - 0
apps/jobsub/src/jobsub/templates/list_designs.mako

@@ -128,6 +128,9 @@ ${layout.menubar(section='designs')}
             <a href="#" class="close" data-dismiss="modal">&times;</a>
             <a href="#" class="close" data-dismiss="modal">&times;</a>
             <h3>Install sample job designs?</h3>
             <h3>Install sample job designs?</h3>
         </div>
         </div>
+        <div class="modal-body">
+          It will take a few seconds to install.
+        </div>
         <div class="modal-footer">
         <div class="modal-footer">
             <input type="submit" class="btn primary" value="Yes"/>
             <input type="submit" class="btn primary" value="Yes"/>
             <a href="#" class="btn secondary" data-dismiss="modal">No</a>
             <a href="#" class="btn secondary" data-dismiss="modal">No</a>

+ 6 - 1
apps/jobsub/src/jobsub/views.py

@@ -38,6 +38,8 @@ from desktop.lib.django_util import render, PopupException, extract_field_data
 from desktop.lib.rest.http_client import RestException
 from desktop.lib.rest.http_client import RestException
 from desktop.log.access import access_warn
 from desktop.log.access import access_warn
 
 
+from hadoop.fs.exceptions import WebHdfsException
+
 from jobsub import models, submit
 from jobsub import models, submit
 from jobsub.management.commands import jobsub_setup
 from jobsub.management.commands import jobsub_setup
 from jobsub.oozie_lib.oozie_api import get_oozie
 from jobsub.oozie_lib.oozie_api import get_oozie
@@ -265,7 +267,10 @@ def setup(request):
   """Installs jobsub examples."""
   """Installs jobsub examples."""
   if request.method != "POST":
   if request.method != "POST":
     raise PopupException('Please use a POST request to install the examples.')
     raise PopupException('Please use a POST request to install the examples.')
-  jobsub_setup.Command().handle_noargs()
+  try:
+    jobsub_setup.Command().handle_noargs()
+  except WebHdfsException, e:
+    raise PopupException('The examples could not be installed.', detail=e)
   return redirect(urlresolvers.reverse(list_designs))
   return redirect(urlresolvers.reverse(list_designs))
 
 
 
 

+ 14 - 0
desktop/libs/hadoop/src/hadoop/fs/test_webhdfs.py

@@ -52,6 +52,20 @@ def test_webhdfs():
   finally:
   finally:
     fs.remove("/fortest.txt")
     fs.remove("/fortest.txt")
 
 
+@attr('requires_hadoop')
+def test_webhdfs_functions():
+  """
+  Tests advanced file system operations.
+  """
+  cluster = pseudo_hdfs4.shared_cluster()
+  fs = cluster.fs
+  fs.setuser(cluster.superuser)
+
+  # Create home dir
+  fs.create_home_dir("/user/test")
+  assert_true(fs.isdir("/user/test"))
+  fs.remove("/user/test")
+
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_seek():
 def test_seek():
   """Test for DESKTOP-293 - ensure seek works in python2.4"""
   """Test for DESKTOP-293 - ensure seek works in python2.4"""

+ 18 - 0
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -416,6 +416,24 @@ class WebHdfs(Hdfs):
   def get_hdfs_path(self, path):
   def get_hdfs_path(self, path):
     return posixpath.join(self.fs_defaultfs, path.lstrip('/'))
     return posixpath.join(self.fs_defaultfs, path.lstrip('/'))
 
 
+  def create_home_dir(self, home_path=None):
+    if home_path is None:
+      home_path = self.get_home_dir()
+
+    if not self.exists(home_path):
+      user = self.user
+      try:
+        self.setuser(self.superuser)
+        self.mkdir(home_path)
+        self.chmod(home_path, 0755)
+        self.chown(home_path, user, user)
+      except IOError, e:
+        msg = 'Failed to create home dir ("%s") as superuser %s' %\
+              (home_path, self.superuser)
+        LOG.exception(msg)
+        raise PopupException(msg, detail=e)
+      finally:
+        self.setuser(user)
 
 
   def _invoke_with_redirect(self, method, path, params=None, data=None):
   def _invoke_with_redirect(self, method, path, params=None, data=None):
     """
     """