Эх сурвалжийг харах

[oozie] Create workspace directory if needed

Fix tests
Romain Rigaux 12 жил өмнө
parent
commit
248259f

+ 132 - 135
apps/beeswax/src/beeswax/templates/watch_results.mako

@@ -250,153 +250,150 @@ ${layout.menubar(section='query')}
 %endif.resultTable
 
 
-
-
 <script type="text/javascript" charset="utf-8">
-
-    $(document).ready(function () {
-      var dataTable = $(".resultTable").dataTable({
-        "bPaginate": false,
-        "bLengthChange": false,
-        "bInfo": false,
-        "oLanguage": {
-            "sEmptyTable": "${_('No data available')}",
-            "sZeroRecords": "${_('No matching records')}",
-        },
-        "fnDrawCallback": function( oSettings ) {
-          $(".resultTable").jHueTableExtender({
-            hintElement: "#jumpToColumnAlert",
-            fixedHeader: true,
-            firstColumnTooltip: true
-          });
-        }
-      });
-      $(".dataTables_wrapper").css("min-height", "0");
-      $(".dataTables_filter").hide();
-      $("input[name='save_target']").change(function () {
-        $("#fieldRequired").addClass("hide");
-        $("input[name='target_dir']").removeClass("fieldError");
-        $("input[name='target_table']").removeClass("fieldError");
-        if ($(this).val().indexOf("HDFS") > -1) {
-          $("input[name='target_table']").addClass("hide");
-          $("input[name='target_dir']").removeClass("hide");
-          $(".fileChooserBtn").removeClass("hide");
-        }
-        else {
-          $("input[name='target_table']").removeClass("hide");
-          $("input[name='target_dir']").addClass("hide");
-          $(".fileChooserBtn").addClass("hide");
-        }
+$(document).ready(function () {
+  var dataTable = $(".resultTable").dataTable({
+    "bPaginate": false,
+    "bLengthChange": false,
+    "bInfo": false,
+    "oLanguage": {
+        "sEmptyTable": "${_('No data available')}",
+        "sZeroRecords": "${_('No matching records')}",
+    },
+    "fnDrawCallback": function( oSettings ) {
+      $(".resultTable").jHueTableExtender({
+        hintElement: "#jumpToColumnAlert",
+        fixedHeader: true,
+        firstColumnTooltip: true
       });
-
-      $("#saveBtn").click(function () {
-        if ($("input[name='save_target']:checked").val().indexOf("HDFS") > -1) {
-          if ($.trim($("input[name='target_dir']").val()) == "") {
-            $("#fieldRequired").removeClass("hide");
-            $("input[name='target_dir']").addClass("fieldError");
-            return false;
-          }
-        }
-        else {
-          if ($.trim($("input[name='target_table']").val()) == "") {
-            $("#fieldRequired").removeClass("hide");
-            $("input[name='target_table']").addClass("fieldError");
-            return false;
-          }
-        }
-        $("#saveForm").submit();
-      });
-
-
-      $("input[name='target_dir']").after(getFileBrowseButton($("input[name='target_dir']")));
-
-      function getFileBrowseButton(inputElement) {
-        return $("<a>").addClass("btn").addClass("fileChooserBtn").addClass("hide").text("..").click(function (e) {
-          e.preventDefault();
-          $("#fileChooserModal").jHueFileChooser({
-            onFolderChange:function (filePath) {
-              inputElement.val(filePath);
-            },
-            onFolderChoose:function (filePath) {
-              inputElement.val(filePath);
-              $("#fileChooserModal").slideUp();
-            },
-            createFolder:false,
-            uploadFile:false,
-            selectFolder:true,
-            initialPath:$.trim(inputElement.val())
-          });
-          $("#fileChooserModal").slideDown();
-        });
+    }
+  });
+  
+  $(".dataTables_wrapper").css("min-height", "0");
+  $(".dataTables_filter").hide();
+  
+  $("input[name='save_target']").change(function () {
+    $("#fieldRequired").addClass("hide");
+    $("input[name='target_dir']").removeClass("fieldError");
+    $("input[name='target_table']").removeClass("fieldError");
+    if ($(this).val().indexOf("HDFS") > -1) {
+      $("input[name='target_table']").addClass("hide");
+      $("input[name='target_dir']").removeClass("hide");
+      $(".fileChooserBtn").removeClass("hide");
+    }
+    else {
+      $("input[name='target_table']").removeClass("hide");
+      $("input[name='target_dir']").addClass("hide");
+      $(".fileChooserBtn").addClass("hide");
+    }
+  });
+
+  $("#saveBtn").click(function () {
+    if ($("input[name='save_target']:checked").val().indexOf("HDFS") > -1) {
+      if ($.trim($("input[name='target_dir']").val()) == "") {
+        $("#fieldRequired").removeClass("hide");
+        $("input[name='target_dir']").addClass("fieldError");
+        return false;
       }
+    }
+    else {
+      if ($.trim($("input[name='target_table']").val()) == "") {
+        $("#fieldRequired").removeClass("hide");
+        $("input[name='target_table']").addClass("fieldError");
+        return false;
+      }
+    }
+    $("#saveForm").submit();
+  });
 
-      $("#collapse").click(function () {
-        $(".sidebar-nav").parent().css("margin-left", "-31%");
-        $("#expand").show().css("top", $(".sidebar-nav i").position().top + "px");
-        $(".sidebar-nav").parent().next().removeClass("span9").addClass("span12").addClass("noLeftMargin");
-      });
-      $("#expand").click(function () {
-        $(this).hide();
-        $(".sidebar-nav").parent().next().removeClass("span12").addClass("span9").removeClass("noLeftMargin");
-        $(".sidebar-nav").parent().css("margin-left", "0");
-      });
-
-
-
-      resizeLogs();
 
-      $(window).resize(function () {
-        resizeLogs();
-      });
+  $("input[name='target_dir']").after(getFileBrowseButton($("input[name='target_dir']")));
 
-      $("a[href='#log']").on("shown", function () {
-        resizeLogs();
+  function getFileBrowseButton(inputElement) {
+    return $("<a>").addClass("btn").addClass("fileChooserBtn").addClass("hide").text("..").click(function (e) {
+      e.preventDefault();
+      $("#fileChooserModal").jHueFileChooser({
+        onFolderChange:function (filePath) {
+          inputElement.val(filePath);
+        },
+        onFolderChoose:function (filePath) {
+          inputElement.val(filePath);
+          $("#fileChooserModal").slideUp();
+        },
+        createFolder:false,
+        uploadFile:false,
+        selectFolder:true,
+        initialPath:$.trim(inputElement.val())
       });
+      $("#fileChooserModal").slideDown();
+    });
+  }
 
-      function resizeLogs() {
-        $("#log pre").css("overflow", "auto").height($(window).height() - $("#log pre").position().top - 40);
-      }
+  $("#collapse").click(function () {
+    $(".sidebar-nav").parent().css("margin-left", "-31%");
+    $("#expand").show().css("top", $(".sidebar-nav i").position().top + "px");
+    $(".sidebar-nav").parent().next().removeClass("span9").addClass("span12").addClass("noLeftMargin");
+  });
+  $("#expand").click(function () {
+    $(this).hide();
+    $(".sidebar-nav").parent().next().removeClass("span12").addClass("span9").removeClass("noLeftMargin");
+    $(".sidebar-nav").parent().css("margin-left", "0");
+  });
+
+  resizeLogs();
+
+  $(window).resize(function () {
+    resizeLogs();
+  });
+
+  $("a[href='#log']").on("shown", function () {
+    resizeLogs();
+  });
+
+  function resizeLogs() {
+    $("#log pre").css("overflow", "auto").height($(window).height() - $("#log pre").position().top - 40);
+  }
 
-      % if app_name == 'impala':
-        % if not download:
-          $("#collapse").click();
-          $(".sidebar-nav, #expand").hide();
-        % elif not error:
-          $("table").replaceWith("${ _('Download results from the left.') }");
-        % endif
-      % endif
-
-      // enable infinite scroll instead of pagination
-      var _nextJsonSet = "${ next_json_set }";
-      var _hasMore = ${ has_more and 'true' or 'false' };
-      var _dt = $("div.dataTables_wrapper");
-      _dt.on("scroll", function (e) {
-        if (_dt.scrollTop() + _dt.outerHeight() + 20 > _dt[0].scrollHeight && !_dt.data("isLoading") && _hasMore) {
-          _dt.data("isLoading", true);
-          _dt.animate({opacity: '0.55'}, 200);
-          $(".spinner").show();
-          $.getJSON(_nextJsonSet, function (data) {
-            _hasMore = data.has_more;
-            if (!_hasMore) {
-              $(".noMore").removeClass("hide");
-            }
-            _nextJsonSet = data.next_json_set;
-            var _cnt = 0;
-            for (var i = 0; i < data.results.length; i++) {
-              var row = data.results[i];
-              row.unshift(data.start_row + _cnt);
-              dataTable.fnAddData(row);
-              _cnt++;
-            }
-            _dt.data("isLoading", false);
-            _dt.animate({opacity: '1'}, 50);
-            $(".spinner").hide();
-          });
+  % if app_name == 'impala':
+    % if not download:
+      $("#collapse").click();
+      $(".sidebar-nav, #expand").hide();
+    % elif not error:
+      $("table").replaceWith("${ _('Download results from the left.') }");
+    % endif
+  % endif
+
+  // enable infinite scroll instead of pagination
+  var _nextJsonSet = "${ next_json_set }";
+  var _hasMore = ${ has_more and 'true' or 'false' };
+  var _dt = $("div.dataTables_wrapper");
+  _dt.on("scroll", function (e) {
+    if (_dt.scrollTop() + _dt.outerHeight() + 20 > _dt[0].scrollHeight && !_dt.data("isLoading") && _hasMore) {
+      _dt.data("isLoading", true);
+      _dt.animate({opacity: '0.55'}, 200);
+      $(".spinner").show();
+      $.getJSON(_nextJsonSet, function (data) {
+        _hasMore = data.has_more;
+        if (!_hasMore) {
+          $(".noMore").removeClass("hide");
+        }
+        _nextJsonSet = data.next_json_set;
+        var _cnt = 0;
+        for (var i = 0; i < data.results.length; i++) {
+          var row = data.results[i];
+          row.unshift(data.start_row + _cnt);
+          dataTable.fnAddData(row);
+          _cnt++;
         }
+        _dt.data("isLoading", false);
+        _dt.animate({opacity: '1'}, 50);
+        $(".spinner").hide();
       });
-      _dt.jHueScrollUp();
+    }
+  });
 
-    });
+  _dt.jHueScrollUp();
+});
 </script>
 
 ${ commonfooter(messages) | n,unicode }

+ 2 - 1
apps/beeswax/src/beeswax/views.py

@@ -587,7 +587,8 @@ def view_results(request, id, first_row=0):
     'context_param': context_param,
     'expired': expired,
     'app_name': app_name,
-    'download': download
+    'download': download,
+    'next_json_set': None
   }
 
   if not error:

+ 6 - 22
apps/oozie/src/oozie/management/commands/oozie_setup.py

@@ -25,10 +25,10 @@ from django.db import transaction
 from django.utils.translation import ugettext as _
 
 from hadoop import cluster
-from hadoop.fs.hadoopfs import Hdfs
-from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 
-from oozie.conf import LOCAL_SAMPLE_DATA_DIR, LOCAL_SAMPLE_DIR, REMOTE_SAMPLE_DIR
+from oozie.conf import LOCAL_SAMPLE_DATA_DIR, LOCAL_SAMPLE_DIR,\
+  REMOTE_SAMPLE_DIR
+from liboozie.submittion import create_directories
 
 
 LOG = logging.getLogger(__name__)
@@ -36,8 +36,9 @@ LOG = logging.getLogger(__name__)
 
 class Command(NoArgsCommand):
   def handle_noargs(self, **options):
-    fs = cluster.get_hdfs()
-    remote_dir = create_directories(fs)
+    fs = cluster.get_hdfs()    
+    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
+    remote_dir = REMOTE_SAMPLE_DIR.get()
 
     # Copy examples binaries
     for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
@@ -61,20 +62,3 @@ class Command(NoArgsCommand):
     except User.DoesNotExist:
       sample_user = User.objects.create(username=USERNAME, password='!', is_active=False, is_superuser=False, id=1100713, pk=1100713)
     management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
-
-
-def create_directories(fs):
-  # If needed, create the remote home, deployment and data directories
-  directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())
-
-  for directory in directories:
-    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
-      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
-      if directory.startswith(remote_home_dir):
-        # Home is 755
-        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
-      # Shared by all the users
-      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
-      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
-
-  return REMOTE_SAMPLE_DIR.get()

+ 2 - 1
apps/oozie/src/oozie/models.py

@@ -42,6 +42,7 @@ from hadoop.fs.exceptions import WebHdfsException
 
 from hadoop.fs.hadoopfs import Hdfs
 from liboozie.submittion import Submission
+from liboozie.submittion import create_directories
 
 from oozie.conf import REMOTE_SAMPLE_DIR, SHARE_JOBS
 from oozie.management.commands import oozie_setup
@@ -216,7 +217,7 @@ class WorkflowManager(models.Manager):
     self.check_workspace(workflow, fs)
 
   def check_workspace(self, workflow, fs):
-    oozie_setup.create_directories(fs)
+    create_directories(fs)
 
     if workflow.is_shared:
       perms = 0755

+ 1 - 1
desktop/core/static/js/Source/jHue/jquery.scrollup.js

@@ -23,7 +23,7 @@
  *   options:
  *    - threshold: (default 100) value in pixels, scroll amount before the link appears
  */
-;
+
 (function ($, window, document, undefined) {
 
   var pluginName = "jHueScrollUp",

+ 0 - 10
desktop/libs/liboozie/src/liboozie/conf.py

@@ -46,19 +46,9 @@ def config_validator():
   Called by core check_config() view.
   """
   from hadoop.cluster import get_all_hdfs
-#  from liboozie.oozie_api import get_oozie
 
   res = []
 
-#  status = 'down'
-#  try:
-#    status = str(get_oozie().get_oozie_status())
-#  except:
-#    pass
-#  if 'NORMAL' not in status:
-#    res.append((status, _('The Oozie server is not available')))
-
-
   class ConfigMock:
     def __init__(self, value): self.value = value
     def get(self): return self.value

+ 20 - 3
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -24,11 +24,10 @@ from django.utils.translation import ugettext as _
 from desktop.lib.exceptions_renderable import PopupException
 from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
-from liboozie.oozie_api import get_oozie
 
+from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 
-
 LOG = logging.getLogger(__name__)
 
 
@@ -184,7 +183,10 @@ class Submission(object):
     Return the job deployment directory in HDFS, creating it if necessary.
     The actual deployment dir should be 0711 owned by the user
     """
-    if self.user != self.job.owner:
+    # Automatic setup of the required directories if needed
+    create_directories(self.fs)
+
+    if self.user != self.job.owner:      
       path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
       self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
     else:
@@ -261,3 +263,18 @@ class Submission(object):
       LOG.warn("Failed to clean up workflow deployment directory for "
                "%s (owner %s). Caused by: %s",
                self.job.name, self.user, ex)
+
+
+def create_directories(fs, directory_list=[]):
+  # If needed, create the remote home, deployment and data directories
+  directories = [REMOTE_DEPLOYMENT_DIR.get()] + directory_list
+
+  for directory in directories:
+    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
+      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
+      if directory.startswith(remote_home_dir):
+        # Home is 755
+        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
+      # Shared by all the users
+      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
+      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491