Bläddra i källkod

[oozie] Create workspace directory if needed

Fix tests
Romain Rigaux 12 år sedan
förälder
incheckning
248259f878

+ 132 - 135
apps/beeswax/src/beeswax/templates/watch_results.mako

@@ -250,153 +250,150 @@ ${layout.menubar(section='query')}
 %endif.resultTable
 %endif.resultTable
 
 
 
 
-
-
 <script type="text/javascript" charset="utf-8">
 <script type="text/javascript" charset="utf-8">
-
-    $(document).ready(function () {
-      var dataTable = $(".resultTable").dataTable({
-        "bPaginate": false,
-        "bLengthChange": false,
-        "bInfo": false,
-        "oLanguage": {
-            "sEmptyTable": "${_('No data available')}",
-            "sZeroRecords": "${_('No matching records')}",
-        },
-        "fnDrawCallback": function( oSettings ) {
-          $(".resultTable").jHueTableExtender({
-            hintElement: "#jumpToColumnAlert",
-            fixedHeader: true,
-            firstColumnTooltip: true
-          });
-        }
-      });
-      $(".dataTables_wrapper").css("min-height", "0");
-      $(".dataTables_filter").hide();
-      $("input[name='save_target']").change(function () {
-        $("#fieldRequired").addClass("hide");
-        $("input[name='target_dir']").removeClass("fieldError");
-        $("input[name='target_table']").removeClass("fieldError");
-        if ($(this).val().indexOf("HDFS") > -1) {
-          $("input[name='target_table']").addClass("hide");
-          $("input[name='target_dir']").removeClass("hide");
-          $(".fileChooserBtn").removeClass("hide");
-        }
-        else {
-          $("input[name='target_table']").removeClass("hide");
-          $("input[name='target_dir']").addClass("hide");
-          $(".fileChooserBtn").addClass("hide");
-        }
+$(document).ready(function () {
+  var dataTable = $(".resultTable").dataTable({
+    "bPaginate": false,
+    "bLengthChange": false,
+    "bInfo": false,
+    "oLanguage": {
+        "sEmptyTable": "${_('No data available')}",
+        "sZeroRecords": "${_('No matching records')}",
+    },
+    "fnDrawCallback": function( oSettings ) {
+      $(".resultTable").jHueTableExtender({
+        hintElement: "#jumpToColumnAlert",
+        fixedHeader: true,
+        firstColumnTooltip: true
       });
       });
-
-      $("#saveBtn").click(function () {
-        if ($("input[name='save_target']:checked").val().indexOf("HDFS") > -1) {
-          if ($.trim($("input[name='target_dir']").val()) == "") {
-            $("#fieldRequired").removeClass("hide");
-            $("input[name='target_dir']").addClass("fieldError");
-            return false;
-          }
-        }
-        else {
-          if ($.trim($("input[name='target_table']").val()) == "") {
-            $("#fieldRequired").removeClass("hide");
-            $("input[name='target_table']").addClass("fieldError");
-            return false;
-          }
-        }
-        $("#saveForm").submit();
-      });
-
-
-      $("input[name='target_dir']").after(getFileBrowseButton($("input[name='target_dir']")));
-
-      function getFileBrowseButton(inputElement) {
-        return $("<a>").addClass("btn").addClass("fileChooserBtn").addClass("hide").text("..").click(function (e) {
-          e.preventDefault();
-          $("#fileChooserModal").jHueFileChooser({
-            onFolderChange:function (filePath) {
-              inputElement.val(filePath);
-            },
-            onFolderChoose:function (filePath) {
-              inputElement.val(filePath);
-              $("#fileChooserModal").slideUp();
-            },
-            createFolder:false,
-            uploadFile:false,
-            selectFolder:true,
-            initialPath:$.trim(inputElement.val())
-          });
-          $("#fileChooserModal").slideDown();
-        });
+    }
+  });
+  
+  $(".dataTables_wrapper").css("min-height", "0");
+  $(".dataTables_filter").hide();
+  
+  $("input[name='save_target']").change(function () {
+    $("#fieldRequired").addClass("hide");
+    $("input[name='target_dir']").removeClass("fieldError");
+    $("input[name='target_table']").removeClass("fieldError");
+    if ($(this).val().indexOf("HDFS") > -1) {
+      $("input[name='target_table']").addClass("hide");
+      $("input[name='target_dir']").removeClass("hide");
+      $(".fileChooserBtn").removeClass("hide");
+    }
+    else {
+      $("input[name='target_table']").removeClass("hide");
+      $("input[name='target_dir']").addClass("hide");
+      $(".fileChooserBtn").addClass("hide");
+    }
+  });
+
+  $("#saveBtn").click(function () {
+    if ($("input[name='save_target']:checked").val().indexOf("HDFS") > -1) {
+      if ($.trim($("input[name='target_dir']").val()) == "") {
+        $("#fieldRequired").removeClass("hide");
+        $("input[name='target_dir']").addClass("fieldError");
+        return false;
       }
       }
+    }
+    else {
+      if ($.trim($("input[name='target_table']").val()) == "") {
+        $("#fieldRequired").removeClass("hide");
+        $("input[name='target_table']").addClass("fieldError");
+        return false;
+      }
+    }
+    $("#saveForm").submit();
+  });
 
 
-      $("#collapse").click(function () {
-        $(".sidebar-nav").parent().css("margin-left", "-31%");
-        $("#expand").show().css("top", $(".sidebar-nav i").position().top + "px");
-        $(".sidebar-nav").parent().next().removeClass("span9").addClass("span12").addClass("noLeftMargin");
-      });
-      $("#expand").click(function () {
-        $(this).hide();
-        $(".sidebar-nav").parent().next().removeClass("span12").addClass("span9").removeClass("noLeftMargin");
-        $(".sidebar-nav").parent().css("margin-left", "0");
-      });
-
-
-
-      resizeLogs();
 
 
-      $(window).resize(function () {
-        resizeLogs();
-      });
+  $("input[name='target_dir']").after(getFileBrowseButton($("input[name='target_dir']")));
 
 
-      $("a[href='#log']").on("shown", function () {
-        resizeLogs();
+  function getFileBrowseButton(inputElement) {
+    return $("<a>").addClass("btn").addClass("fileChooserBtn").addClass("hide").text("..").click(function (e) {
+      e.preventDefault();
+      $("#fileChooserModal").jHueFileChooser({
+        onFolderChange:function (filePath) {
+          inputElement.val(filePath);
+        },
+        onFolderChoose:function (filePath) {
+          inputElement.val(filePath);
+          $("#fileChooserModal").slideUp();
+        },
+        createFolder:false,
+        uploadFile:false,
+        selectFolder:true,
+        initialPath:$.trim(inputElement.val())
       });
       });
+      $("#fileChooserModal").slideDown();
+    });
+  }
 
 
-      function resizeLogs() {
-        $("#log pre").css("overflow", "auto").height($(window).height() - $("#log pre").position().top - 40);
-      }
+  $("#collapse").click(function () {
+    $(".sidebar-nav").parent().css("margin-left", "-31%");
+    $("#expand").show().css("top", $(".sidebar-nav i").position().top + "px");
+    $(".sidebar-nav").parent().next().removeClass("span9").addClass("span12").addClass("noLeftMargin");
+  });
+  $("#expand").click(function () {
+    $(this).hide();
+    $(".sidebar-nav").parent().next().removeClass("span12").addClass("span9").removeClass("noLeftMargin");
+    $(".sidebar-nav").parent().css("margin-left", "0");
+  });
+
+  resizeLogs();
+
+  $(window).resize(function () {
+    resizeLogs();
+  });
+
+  $("a[href='#log']").on("shown", function () {
+    resizeLogs();
+  });
+
+  function resizeLogs() {
+    $("#log pre").css("overflow", "auto").height($(window).height() - $("#log pre").position().top - 40);
+  }
 
 
-      % if app_name == 'impala':
-        % if not download:
-          $("#collapse").click();
-          $(".sidebar-nav, #expand").hide();
-        % elif not error:
-          $("table").replaceWith("${ _('Download results from the left.') }");
-        % endif
-      % endif
-
-      // enable infinite scroll instead of pagination
-      var _nextJsonSet = "${ next_json_set }";
-      var _hasMore = ${ has_more and 'true' or 'false' };
-      var _dt = $("div.dataTables_wrapper");
-      _dt.on("scroll", function (e) {
-        if (_dt.scrollTop() + _dt.outerHeight() + 20 > _dt[0].scrollHeight && !_dt.data("isLoading") && _hasMore) {
-          _dt.data("isLoading", true);
-          _dt.animate({opacity: '0.55'}, 200);
-          $(".spinner").show();
-          $.getJSON(_nextJsonSet, function (data) {
-            _hasMore = data.has_more;
-            if (!_hasMore) {
-              $(".noMore").removeClass("hide");
-            }
-            _nextJsonSet = data.next_json_set;
-            var _cnt = 0;
-            for (var i = 0; i < data.results.length; i++) {
-              var row = data.results[i];
-              row.unshift(data.start_row + _cnt);
-              dataTable.fnAddData(row);
-              _cnt++;
-            }
-            _dt.data("isLoading", false);
-            _dt.animate({opacity: '1'}, 50);
-            $(".spinner").hide();
-          });
+  % if app_name == 'impala':
+    % if not download:
+      $("#collapse").click();
+      $(".sidebar-nav, #expand").hide();
+    % elif not error:
+      $("table").replaceWith("${ _('Download results from the left.') }");
+    % endif
+  % endif
+
+  // enable infinite scroll instead of pagination
+  var _nextJsonSet = "${ next_json_set }";
+  var _hasMore = ${ has_more and 'true' or 'false' };
+  var _dt = $("div.dataTables_wrapper");
+  _dt.on("scroll", function (e) {
+    if (_dt.scrollTop() + _dt.outerHeight() + 20 > _dt[0].scrollHeight && !_dt.data("isLoading") && _hasMore) {
+      _dt.data("isLoading", true);
+      _dt.animate({opacity: '0.55'}, 200);
+      $(".spinner").show();
+      $.getJSON(_nextJsonSet, function (data) {
+        _hasMore = data.has_more;
+        if (!_hasMore) {
+          $(".noMore").removeClass("hide");
+        }
+        _nextJsonSet = data.next_json_set;
+        var _cnt = 0;
+        for (var i = 0; i < data.results.length; i++) {
+          var row = data.results[i];
+          row.unshift(data.start_row + _cnt);
+          dataTable.fnAddData(row);
+          _cnt++;
         }
         }
+        _dt.data("isLoading", false);
+        _dt.animate({opacity: '1'}, 50);
+        $(".spinner").hide();
       });
       });
-      _dt.jHueScrollUp();
+    }
+  });
 
 
-    });
+  _dt.jHueScrollUp();
+});
 </script>
 </script>
 
 
 ${ commonfooter(messages) | n,unicode }
 ${ commonfooter(messages) | n,unicode }

+ 2 - 1
apps/beeswax/src/beeswax/views.py

@@ -587,7 +587,8 @@ def view_results(request, id, first_row=0):
     'context_param': context_param,
     'context_param': context_param,
     'expired': expired,
     'expired': expired,
     'app_name': app_name,
     'app_name': app_name,
-    'download': download
+    'download': download,
+    'next_json_set': None
   }
   }
 
 
   if not error:
   if not error:

+ 6 - 22
apps/oozie/src/oozie/management/commands/oozie_setup.py

@@ -25,10 +25,10 @@ from django.db import transaction
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
 from hadoop import cluster
 from hadoop import cluster
-from hadoop.fs.hadoopfs import Hdfs
-from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 
 
-from oozie.conf import LOCAL_SAMPLE_DATA_DIR, LOCAL_SAMPLE_DIR, REMOTE_SAMPLE_DIR
+from oozie.conf import LOCAL_SAMPLE_DATA_DIR, LOCAL_SAMPLE_DIR,\
+  REMOTE_SAMPLE_DIR
+from liboozie.submittion import create_directories
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -36,8 +36,9 @@ LOG = logging.getLogger(__name__)
 
 
 class Command(NoArgsCommand):
 class Command(NoArgsCommand):
   def handle_noargs(self, **options):
   def handle_noargs(self, **options):
-    fs = cluster.get_hdfs()
-    remote_dir = create_directories(fs)
+    fs = cluster.get_hdfs()    
+    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
+    remote_dir = REMOTE_SAMPLE_DIR.get()
 
 
     # Copy examples binaries
     # Copy examples binaries
     for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
     for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
@@ -61,20 +62,3 @@ class Command(NoArgsCommand):
     except User.DoesNotExist:
     except User.DoesNotExist:
       sample_user = User.objects.create(username=USERNAME, password='!', is_active=False, is_superuser=False, id=1100713, pk=1100713)
       sample_user = User.objects.create(username=USERNAME, password='!', is_active=False, is_superuser=False, id=1100713, pk=1100713)
     management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
     management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
-
-
-def create_directories(fs):
-  # If needed, create the remote home, deployment and data directories
-  directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())
-
-  for directory in directories:
-    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
-      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
-      if directory.startswith(remote_home_dir):
-        # Home is 755
-        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
-      # Shared by all the users
-      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
-      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
-
-  return REMOTE_SAMPLE_DIR.get()

+ 2 - 1
apps/oozie/src/oozie/models.py

@@ -42,6 +42,7 @@ from hadoop.fs.exceptions import WebHdfsException
 
 
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.hadoopfs import Hdfs
 from liboozie.submittion import Submission
 from liboozie.submittion import Submission
+from liboozie.submittion import create_directories
 
 
 from oozie.conf import REMOTE_SAMPLE_DIR, SHARE_JOBS
 from oozie.conf import REMOTE_SAMPLE_DIR, SHARE_JOBS
 from oozie.management.commands import oozie_setup
 from oozie.management.commands import oozie_setup
@@ -216,7 +217,7 @@ class WorkflowManager(models.Manager):
     self.check_workspace(workflow, fs)
     self.check_workspace(workflow, fs)
 
 
   def check_workspace(self, workflow, fs):
   def check_workspace(self, workflow, fs):
-    oozie_setup.create_directories(fs)
+    create_directories(fs)
 
 
     if workflow.is_shared:
     if workflow.is_shared:
       perms = 0755
       perms = 0755

+ 1 - 1
desktop/core/static/js/Source/jHue/jquery.scrollup.js

@@ -23,7 +23,7 @@
  *   options:
  *   options:
  *    - threshold: (default 100) value in pixels, scroll amount before the link appears
  *    - threshold: (default 100) value in pixels, scroll amount before the link appears
  */
  */
-;
+
 (function ($, window, document, undefined) {
 (function ($, window, document, undefined) {
 
 
   var pluginName = "jHueScrollUp",
   var pluginName = "jHueScrollUp",

+ 0 - 10
desktop/libs/liboozie/src/liboozie/conf.py

@@ -46,19 +46,9 @@ def config_validator():
   Called by core check_config() view.
   Called by core check_config() view.
   """
   """
   from hadoop.cluster import get_all_hdfs
   from hadoop.cluster import get_all_hdfs
-#  from liboozie.oozie_api import get_oozie
 
 
   res = []
   res = []
 
 
-#  status = 'down'
-#  try:
-#    status = str(get_oozie().get_oozie_status())
-#  except:
-#    pass
-#  if 'NORMAL' not in status:
-#    res.append((status, _('The Oozie server is not available')))
-
-
   class ConfigMock:
   class ConfigMock:
     def __init__(self, value): self.value = value
     def __init__(self, value): self.value = value
     def get(self): return self.value
     def get(self): return self.value

+ 20 - 3
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -24,11 +24,10 @@ from django.utils.translation import ugettext as _
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
 from hadoop import cluster
 from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.hadoopfs import Hdfs
-from liboozie.oozie_api import get_oozie
 
 
+from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 
 
-
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
@@ -184,7 +183,10 @@ class Submission(object):
     Return the job deployment directory in HDFS, creating it if necessary.
     Return the job deployment directory in HDFS, creating it if necessary.
     The actual deployment dir should be 0711 owned by the user
     The actual deployment dir should be 0711 owned by the user
     """
     """
-    if self.user != self.job.owner:
+    # Automatic setup of the required directories if needed
+    create_directories(self.fs)
+
+    if self.user != self.job.owner:      
       path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
       path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
       self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
       self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
     else:
     else:
@@ -261,3 +263,18 @@ class Submission(object):
       LOG.warn("Failed to clean up workflow deployment directory for "
       LOG.warn("Failed to clean up workflow deployment directory for "
                "%s (owner %s). Caused by: %s",
                "%s (owner %s). Caused by: %s",
                self.job.name, self.user, ex)
                self.job.name, self.user, ex)
+
+
+def create_directories(fs, directory_list=[]):
+  # If needed, create the remote home, deployment and data directories
+  directories = [REMOTE_DEPLOYMENT_DIR.get()] + directory_list
+
+  for directory in directories:
+    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
+      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
+      if directory.startswith(remote_home_dir):
+        # Home is 755
+        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
+      # Shared by all the users
+      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
+      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491