Browse Source

HUE-716 [jobsub] A normal user can't run his own designs

The owner can submit/delete/edit his own designs.
The admin can delete/edit any design.

The jobsub data dir could be created in two ways:
  - installing the samples
  - submitting a design
which were not following the same conventions. This is refactored
and a test was added.
Romain Rigaux 13 years ago
parent
commit
ecb7577e8e

+ 4 - 6
apps/jobsub/src/jobsub/management/commands/jobsub_setup.py

@@ -32,6 +32,7 @@ from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
 import hadoop.conf
 import jobsub.conf
+from jobsub.submit import Submission
 
 LOG = logging.getLogger(__name__)
 
@@ -48,11 +49,8 @@ class Command(NoArgsCommand):
       remote_fs.setuser(remote_fs.DEFAULT_USER)
     LOG.info("Using remote fs: %s" % str(remote_fs))
 
-    # Create remote home directory if needed
-    remote_data_dir = jobsub.conf.REMOTE_DATA_DIR.get()
-    remote_home_dir = Hdfs.join('/user', remote_fs.user)
-    if remote_data_dir.startswith(remote_home_dir):
-      remote_fs.create_home_dir(remote_home_dir)
+    # Create remote data directory if needed
+    remote_data_dir = Submission.create_data_dir(remote_fs)
 
     # Copy over examples/
     for dirname in ("examples",):
@@ -60,7 +58,7 @@ class Command(NoArgsCommand):
       remote_dir = posixpath.join(remote_data_dir, dirname)
       copy_dir(local_dir, remote_fs, remote_dir)
 
-    # Copy over sample data
+    # Copy over sample_data/
     copy_dir(jobsub.conf.SAMPLE_DATA_DIR.get(),
       remote_fs,
       posixpath.join(remote_data_dir, "sample_data"))

+ 24 - 6
apps/jobsub/src/jobsub/submit.py

@@ -25,6 +25,7 @@ import logging
 from desktop.lib import django_mako
 from desktop.lib.django_util import PopupException
 import hadoop.cluster
+from hadoop.fs.hadoopfs import Hdfs
 
 from jobsub import conf, models
 from jobsub.oozie_lib.oozie_api import get_oozie
@@ -173,17 +174,34 @@ class Submission(object):
 
 
   def _create_deployment_dir(self, path):
-    # Make the REMOTE_DATA_DIR, and have it owned by hue
-    data_repo = conf.REMOTE_DATA_DIR.get()
-    if not self._fs.exists(data_repo):
-      # Parent directories should be 0755. But the data dir should be 01777.
-      self._do_as(self._fs.DEFAULT_USER, self._fs.mkdir, data_repo, 0755)
-      self._do_as(self._fs.DEFAULT_USER, self._fs.chmod, data_repo, 01777)
+    # Make sure the root data dir exists
+    self.create_data_dir(self._fs)
 
     # The actual deployment dir should be 0711 owned by the user
     self._do_as(self._username, self._fs.mkdir, path, 0711)
 
 
+  @classmethod
+  def create_data_dir(cls, fs):
+    # If needed, create the remote home and data directories
+    remote_data_dir = conf.REMOTE_DATA_DIR.get()
+    user = fs.user
+
+    try:
+      fs.setuser(fs.DEFAULT_USER)
+      if not fs.exists(remote_data_dir):
+        remote_home_dir = Hdfs.join('/user', fs.user)
+        if remote_data_dir.startswith(remote_home_dir):
+          # Home is 755
+          fs.create_home_dir(remote_home_dir)
+        # Shared by all the users
+        fs.mkdir(remote_data_dir, 01777)
+    finally:
+      fs.setuser(user)
+
+    return remote_data_dir
+
+
   def _get_deployment_dir(self):
     """Return the workflow deployment directory"""
     if self._fs is None:

+ 15 - 15
apps/jobsub/src/jobsub/templates/list_designs.mako

@@ -65,22 +65,22 @@ ${layout.menubar(section='designs')}
                     <td>${design.description}</td>
                     <td nowrap="nowrap">${date(design.last_modified)} ${time(design.last_modified).replace("p.m.","PM").replace("a.m.","AM")}</td>
                     <td nowrap="nowrap" class="right">
-                      %if currentuser.is_superuser:
-                        %if currentuser.username == design.owner.username:
-                          <a title="Edit ${design.name}" class="btn small"
-                              href="${ url('jobsub.views.edit_design', design_id=design.id) }">Edit</a>
-                          <a title="Submit ${design.name}" class="btn small submitConfirmation"
-                              alt="Submit ${design.name} to the cluster"
-                              href="javascript:void(0)"
-                              data-param-url="${ url('jobsub.views.get_design_params', design_id=design.id) }"
-                              data-submit-url="${ url('jobsub.views.submit_design', design_id=design.id) }">Submit</a>
-                        %endif
+                    %if currentuser.username == design.owner.username:
+                        <a title="Submit ${design.name}" class="btn small submitConfirmation"
+                           alt="Submit ${design.name} to the cluster"
+                           href="javascript:void(0)"
+                           data-param-url="${ url('jobsub.views.get_design_params', design_id=design.id) }"
+                           data-submit-url="${ url('jobsub.views.submit_design', design_id=design.id) }">Submit</a>
+                        <a title="Edit ${design.name}" class="btn small"
+                           href="${ url('jobsub.views.edit_design', design_id=design.id) }">Edit</a>
+                    %endif%
+                    %if currentuser.is_superuser or currentuser.username == design.owner.username:
                         <a title="Delete ${design.name}" class="btn small deleteConfirmation"
-                            alt="Are you sure you want to delete ${design.name}?"
-                            href="javascript:void(0)"
-                            data-confirmation-url="${ url('jobsub.views.delete_design', design_id=design.id) }">Delete</a>
-                      %endif
-                      <a title="Clone ${design.name}" class="btn small" href="${ url('jobsub.views.clone_design', design_id=design.id) }">Clone</a>
+                           alt="Are you sure you want to delete ${design.name}?"
+                           href="javascript:void(0)"
+                           data-confirmation-url="${ url('jobsub.views.delete_design', design_id=design.id) }">Delete</a>
+                    %endif
+                        <a title="Clone ${design.name}" class="btn small" href="${ url('jobsub.views.clone_design', design_id=design.id) }">Clone</a>
                     </td>
                 </tr>
             %endfor

+ 40 - 1
apps/jobsub/src/jobsub/tests.py

@@ -37,10 +37,14 @@ from django.contrib.auth.models import User
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
 
+from jobsub import conf
+from jobsub.management.commands import jobsub_setup
 from jobsub.models import JobDesign
 from jobsub.parameterization import recursive_walk, find_variables, substitute_variables
 
 from hadoop import mini_cluster
+from hadoop import pseudo_hdfs4
+from hadoop.fs.hadoopfs import Hdfs
 import hadoop
 
 def test_recursive_walk():
@@ -299,7 +303,42 @@ def test_job_submission():
   finally:
     cluster.shutdown()
     jobsubd.exit()
-    
+
+@attr('requires_hadoop')
+def test_jobsub_setup():
+  # User 'test' triggers the setup of the examples.
+  # 'hue' home will be deleted, the examples installed in the new one
+  # and 'test' will try to access them.
+  cluster = pseudo_hdfs4.shared_cluster()
+  cluster.fs.setuser('test')
+
+  username = 'hue'
+  home_dir = '/user/%s/' % username
+  finish = conf.REMOTE_DATA_DIR.set_for_testing('%s/jobsub' % home_dir)
+
+  try:
+    data_dir = conf.REMOTE_DATA_DIR.get()
+    cluster.fs.setuser(cluster.fs.superuser)
+    if cluster.fs.exists(home_dir):
+      cluster.fs.rmtree(home_dir)
+    cluster.fs.setuser('test')
+
+    jobsub_setup.Command().handle()
+
+    cluster.fs.setuser('test')
+    stats = cluster.fs.stats(home_dir)
+    assert_equal(stats['user'], username)
+    assert_equal(oct(stats['mode']), '040755') #04 because is a dir
+
+    stats = cluster.fs.stats(data_dir)
+    assert_equal(stats['user'], username)
+    assert_equal(oct(stats['mode']), '041777')
+
+    stats = cluster.fs.listdir_stats(data_dir)
+    assert_equal(len(stats), 2) # 2 files inside
+  finally:
+    finish()
+
 @attr('requires_hadoop')
 def test_jobsub_setup_and_samples():
   """

+ 1 - 0
apps/jobsub/src/jobsub/views.py

@@ -268,6 +268,7 @@ def setup(request):
   if request.method != "POST":
     raise PopupException('Please use a POST request to install the examples.')
   try:
+    # Warning: below will modify fs.user
     jobsub_setup.Command().handle_noargs()
   except WebHdfsException, e:
     raise PopupException('The examples could not be installed.', detail=e)