浏览代码

HUE-1089 [beeswax] Loading data can show a 500 error

Ajaxify load data popup and fix the tests
Romain Rigaux 12 年之前
父节点
当前提交
d170972

+ 22 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -140,6 +140,28 @@ class Dbms:
     return self.execute_statement(hql)
 
 
+  def load_data(self, database, table, form, design):
+    hql = "LOAD DATA INPATH"
+    hql += " '%s'" % form.cleaned_data['path']
+    if form.cleaned_data['overwrite']:
+      hql += " OVERWRITE"
+    hql += " INTO TABLE "
+    hql += "`%s.%s`" % (database, table.name,)
+    if form.partition_columns:
+      hql += " PARTITION ("
+      vals = []
+      for key, column_name in form.partition_columns.iteritems():
+        vals.append("%s='%s'" % (column_name, form.cleaned_data[key]))
+      hql += ", ".join(vals)
+      hql += ")"
+
+    query = hql_query(hql, database)
+    design.data = query.dumps()
+    design.save()
+
+    return self.execute_query(query, design)
+
+
   def drop_tables(self, database, tables, design):
     hql = []
 

+ 15 - 90
apps/beeswax/src/beeswax/templates/describe_table.mako

@@ -59,7 +59,7 @@ ${layout.menubar(section='tables')}
             <div class="well sidebar-nav">
                 <ul class="nav nav-list">
                     <li class="nav-header">${_('Actions')}</li>
-                    <li><a href="#importData" data-toggle="modal">${_('Import Data')}</a></li>
+                    <li><a href="#" id="import-data-btn">${_('Import Data')}</a></li>
                     <li><a href="${ url(app_name + ':read_table', database=database, table=table.name) }">${_('Browse Data')}</a></li>
                     <li><a href="#dropTable" data-toggle="modal">${_('Drop')} ${view_or_table_noun}</a></li>
                     <li><a href="${ table.hdfs_link }" rel="${ table.path_location }">${_('View File Location')}</a></li>
@@ -153,101 +153,18 @@ ${layout.menubar(section='tables')}
 </div>
 
 
+<div id="import-data-modal" class="modal hide fade"></div>
 
-<div id="importData" class="modal hide fade">
-    <form method="POST" action="${ url(app_name + ':load_table', database=database, table=table.name) }" class="form-horizontal">
-        <div class="modal-header">
-            <a href="#" class="close" data-dismiss="modal">&times;</a>
-            <h3>${_('Import data')}</h3>
-        </div>
-        <div class="modal-body">
-
-            <div class="control-group">
-                ${comps.bootstrapLabel(load_form["path"])}
-                <div class="controls">
-                    ${comps.field(load_form["path"], placeholder="/user/user_name/data_dir/file", klass="pathChooser input-xlarge", file_chooser=True, show_errors=False)}
-                </div>
-            </div>
-
-            <div id="filechooser"></div>
-
-            % for pf in load_form.partition_columns:
-                <div class="control-group">
-                     ${comps.bootstrapLabel(load_form[pf])}
-                     <div class="controls">
-                       ${comps.field(load_form[pf], render_default=True, attrs={'klass': 'input-xlarge'})}
-                    </div>
-                </div>
-            % endfor
-
-            <div class="control-group">
-              <div class="controls">
-                <label class="checkbox">
-                    <input type="checkbox" name="overwrite"/> ${_('Overwrite existing data')}
-                  </label>
-                </div>
-            </div>
-
-            <p class="muted"><em>${_("Note that loading data will move data from its location into the table's storage location.")}</em></p>
-        </div>
-
-        <div class="modal-footer">
-            <a href="#" class="btn" data-dismiss="modal">${_('Cancel')}</a>
-            <input type="submit" class="btn btn-primary" value="${_('Submit')}"/>
-        </div>
-    </form>
 </div>
-</div>
-
- <style>
-   #filechooser {
-     display: none;
-     min-height: 100px;
-     height: 250px;
-     overflow-y: scroll;
-     margin-top: 10px;
-   }
 
+<style>
    .sampleTable td, .sampleTable th {
      white-space: nowrap;
    }
+</style>
 
-   .form-horizontal .controls {
-     margin-left: 0;
-   }
-
-   .form-horizontal .control-label {
-     width: auto;
-     padding-right: 10px;
-   }
- </style>
-
- <script type="text/javascript" charset="utf-8">
+<script type="text/javascript" charset="utf-8">
    $(document).ready(function () {
-
-     $(".fileChooserBtn").click(function(e){
-       e.preventDefault();
-       var _destination = $(this).attr("data-filechooser-destination");
-       $("#filechooser").jHueFileChooser({
-         initialPath: $("input[name='"+_destination+"']").val(),
-         onFileChoose: function(filePath){
-           $("input[name='"+_destination+"']").val(filePath);
-           $("#filechooser").slideUp();
-         },
-         onFolderChange: function (filePath) {
-           $("input[name='"+_destination+"']").val(filePath);
-         },
-         onFolderChoose: function (filePath) {
-           $("input[name='"+_destination+"']").val(filePath);
-           $("#filechooser").slideUp();
-         },
-         createFolder: false,
-         selectFolder: true,
-         uploadFile: true
-       });
-       $("#filechooser").slideDown();
-     });
-
      $(".datatables").dataTable({
        "bPaginate": false,
        "bLengthChange": false,
@@ -283,7 +200,15 @@ ${layout.menubar(section='tables')}
        });
      })
 
+    $("#import-data-btn").click(function () {
+      $.get("${ url(app_name + ':load_table', database=database, table=table.name) }", function (response) {
+          $("#import-data-modal").html(response['data']);
+          $("#import-data-modal").modal("show");
+        }
+      );
+    });
+
    });
- </script>
+</script>
 
- ${ commonfooter(messages) | n,unicode }
+${ commonfooter(messages) | n,unicode }

+ 131 - 0
apps/beeswax/src/beeswax/templates/load_data_popup.mako

@@ -0,0 +1,131 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+<%!
+from django.utils.translation import ugettext as _
+%>
+
+
+<%namespace name="comps" file="beeswax_components.mako" />
+
+
+
+<form method="POST" class="form-horizontal" id="load-data-form">
+    <div class="modal-header">
+        <a href="#" class="close" data-dismiss="modal">&times;</a>
+        <h3>${_('Import data')}</h3>
+    </div>
+    <div class="modal-body">
+
+        <div class="control-group">
+            ${comps.bootstrapLabel(load_form["path"])}
+            <div class="controls">
+                ${comps.field(load_form["path"], placeholder="/user/user_name/data_dir/file", klass="pathChooser input-xlarge", file_chooser=True, show_errors=True)}
+            </div>
+        </div>
+
+        <div id="filechooser"></div>
+
+        % for pf in load_form.partition_columns:
+            <div class="control-group">
+                 ${comps.bootstrapLabel(load_form[pf])}
+                 <div class="controls">
+                   ${comps.field(load_form[pf], render_default=True, attrs={'klass': 'input-xlarge'})}
+                </div>
+            </div>
+        % endfor
+
+        <div class="control-group">
+          <div class="controls">
+            <label class="checkbox">
+                <input type="checkbox" name="overwrite"/> ${_('Overwrite existing data')}
+              </label>
+            </div>
+        </div>
+
+        <p class="alert alert-warning">${_("Note that loading data will move data from its location into the table's storage location.")}</p>
+        <p id="load-data-error" class="alert alert-error hide"></p>
+    </div>
+
+    <div class="modal-footer">
+        <a href="#" class="btn" data-dismiss="modal">${_('Cancel')}</a>
+        <a href="#" class="btn btn-primary" id="load-data-submit-btn">${_('Submit')}</a>
+    </div>
+</form>
+
+
+<style>
+   #filechooser {
+     display: none;
+     min-height: 100px;
+     height: 250px;
+     overflow-y: scroll;
+     margin-top: 10px;
+   }
+
+   .form-horizontal .controls {
+     margin-left: 0;
+   }
+
+   .form-horizontal .control-label {
+     width: auto;
+     padding-right: 10px;
+   }
+</style>
+
+<script type="text/javascript" charset="utf-8">
+  $(document).ready(function () {
+     $(".fileChooserBtn").click(function(e){
+       e.preventDefault();
+       var _destination = $(this).attr("data-filechooser-destination");
+       $("#filechooser").jHueFileChooser({
+         initialPath: $("input[name='"+_destination+"']").val(),
+         onFileChoose: function(filePath){
+           $("input[name='"+_destination+"']").val(filePath);
+           $("#filechooser").slideUp();
+         },
+         onFolderChange: function (filePath) {
+           $("input[name='"+_destination+"']").val(filePath);
+         },
+         onFolderChoose: function (filePath) {
+           $("input[name='"+_destination+"']").val(filePath);
+           $("#filechooser").slideUp();
+         },
+         createFolder: false,
+         selectFolder: true,
+         uploadFile: true
+       });
+       $("#filechooser").slideDown();
+     });
+
+   $("#load-data-submit-btn").click(function(e){
+     $.post("${ url(app_name + ':load_table', database=database, table=table.name) }",
+       $("#load-data-form").serialize(),
+        function (response) {
+          if (response['status'] != 0) {
+            if (response['status'] == 1) {
+              $('#load-data-error').html(response['data']);
+              $('#load-data-error').show();
+            } else {
+              $('#import-data-modal').html(response['data']);
+            }
+          } else {
+            window.location.replace(response['data']);
+          }
+        }
+     );
+    });
+  });
+</script>

+ 20 - 16
apps/beeswax/src/beeswax/tests.py

@@ -16,6 +16,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+try:
+  import json
+except ImportError:
+  import simplejson as json
 import cStringIO
 import gzip
 import logging
@@ -782,26 +786,24 @@ for x in sys.stdin:
     about whether a table is partitioned.
     """
     # Check that view works
-    resp = self.client.get("/beeswax/table/default/test")
-    assert_true(resp.context["load_form"])
+    resp = self.client.get("/beeswax/table/default/test/load")
+    assert_true('Path' in resp.content)
 
     # Try the submission
-    try:    
-      self.client.post("/beeswax/table/default/test/load", dict(path="/tmp/foo", overwrite=True))
-    except:
-      pass
+    self.client.post("/beeswax/table/default/test/load", dict(path="/tmp/foo", overwrite=True))
     query = QueryHistory.objects.latest('id')
-    
-    assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' OVERWRITE INTO TABLE `default.test`", query.query, resp.context)
+
+    assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' OVERWRITE INTO TABLE `default.test`", query.query)
 
     resp = self.client.post("/beeswax/table/default/test/load", dict(path="/tmp/foo", overwrite=False))
-    assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' INTO TABLE `default.test`",
-        resp.context["form"].query.initial["query"])
+    query = QueryHistory.objects.latest('id')
+    assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' INTO TABLE `default.test`", query.query)
 
     # Try it with partitions
     resp = self.client.post("/beeswax/table/default/test_partitions/load", dict(path="/tmp/foo", partition_0="alpha", partition_1="beta"))
+    query = QueryHistory.objects.latest('id')
     assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' INTO TABLE `default.test_partitions` PARTITION (baz='alpha', boom='beta')",
-        resp.context["form"].query.initial["query"])
+        query.query)
 
 
   def test_save_results_to_dir(self):
@@ -1229,15 +1231,17 @@ for x in sys.stdin:
 
     _make_query(client, 'SELECT', name='my query history', submission_type='Save')
     design = SavedQuery.objects.get(name='my query history')
-  
+
     for i in range(25):
       client.get('/beeswax/clone_design/%s' % (design.id,))
-  
+
     resp = client.get('/beeswax/list_designs')
-    assert_true(len(resp.context['page'].object_list) >=20)
+    ids_page_1 = set([query.id for query in resp.context['page'].object_list])
     resp = client.get('/beeswax/list_designs?q-page=2')
-    assert_true(len(resp.context['page'].object_list) > 1)
-  
+    ids_page_2 = set([query.id for query in resp.context['page'].object_list])
+    for id in ids_page_2:
+      assert_true(id not in ids_page_1)
+
     SavedQuery.objects.filter(name='my query history').delete()
 
 

+ 27 - 27
apps/beeswax/src/beeswax/views.py

@@ -325,12 +325,9 @@ def describe_table(request, database, table):
   except BeeswaxException, ex:
     error_message, logs = expand_exception(ex, db)
 
-  load_form = LoadDataForm(table)
-
   return render("describe_table.mako", request, {
       'table': table,
       'sample': table_data and table_data.rows(),
-      'load_form': load_form,
       'error_message': error_message,
       'database': database,
   })
@@ -374,35 +371,38 @@ def read_table(request, database, table):
 
 
 def load_table(request, database, table):
-  table_obj = dbms.get(request.user).get_table(database, table)
+  app_name = get_app_name(request)
+  db = dbms.get(request.user)
+  table = db.get_table(database, table)
+  response = {'status': -1, 'data': 'None'}
 
   if request.method == "POST":
-    form = beeswax.forms.LoadDataForm(table_obj, request.POST)
-    if form.is_valid():
-      # TODO(philip/todd): When PathField might refer to non-HDFS,
-      # we need a pathfield.is_local function.
-      hql = "LOAD DATA INPATH"
-      hql += " '%s'" % form.cleaned_data['path']
-      if form.cleaned_data['overwrite']:
-        hql += " OVERWRITE"
-      hql += " INTO TABLE "
-      hql += "`%s.%s`" % (database, table,)
-      if form.partition_columns:
-        hql += " PARTITION ("
-        vals = []
-        for key, column_name in form.partition_columns.iteritems():
-          vals.append("%s='%s'" % (column_name, form.cleaned_data[key]))
-        hql += ", ".join(vals)
-        hql += ")"
-
-      on_success_url = reverse(get_app_name(request) + ':describe_table', kwargs={'database': database, 'table': table})
-      query = hql_query(hql, database=database)
+    load_form = beeswax.forms.LoadDataForm(table, request.POST)
+
+    if load_form.is_valid():
+      on_success_url = reverse(get_app_name(request) + ':describe_table', kwargs={'database': database, 'table': table.name})
       try:
-        return execute_directly(request, query, on_success_url=on_success_url)
+        design = SavedQuery.create_empty(app_name=app_name, owner=request.user)
+        query_history = db.load_data(database, table, load_form, design)
+        url = reverse(app_name + ':watch_query', args=[query_history.id]) + '?on_success_url=' + on_success_url
+        response['status'] = 0
+        response['data'] = url
       except Exception, e:
-        raise PopupException(_("Can't load the data"), detail=e)
+        response['status'] = 1
+        response['data'] = _("Can't load the data: ") + str(e)
   else:
-    raise PopupException(_('Requires a POST'))
+    load_form = LoadDataForm(table)
+
+  if response['status'] == -1:
+    popup = render('load_data_popup.mako', request, {
+                     'table': table,
+                     'load_form': load_form,
+                     'database': database,
+                     'app_name': app_name
+                 }, force_template=True).content
+    response['data'] = popup
+
+  return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
 def describe_partitions(request, database, table):