Przeglądaj źródła

HUE-3315 [editor] Save large result on HDFS

Romain Rigaux 9 lat temu
rodzic
commit
9f9515c787

+ 10 - 0
desktop/core/src/desktop/templates/ko_components.mako

@@ -568,6 +568,16 @@ from desktop.views import _ko
                   <input data-bind="value: savePath" type="text" name="target_table" class="input-xlarge" placeholder="${_('Table name or <database>.<table>')}">
                 </span>
               </div>
+              <div class="controls">
+                <label class="radio">
+                  <input data-bind="checked: saveTarget" type="radio" name="save-results-type" value="hdfs-directory">
+                  &nbsp;${ _('Big Query in HDFS') }
+                </label>
+                <span data-bind="visible: saveTarget() == 'hdfs-directory'">
+                  <input data-bind="value: savePath" type="text" name="target_dir" placeholder="${_('Path to directory')}" class="folderChooser">
+                  <i class="fa fa-question-circle" id="hdfs-directory-help"></i>
+                </span>
+              </div>
             </div>
           </fieldset>
         </form>

+ 4 - 0
desktop/libs/notebook/src/notebook/api.py

@@ -478,5 +478,9 @@ def export_result(request):
     notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
     response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=save_as_table&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
     response['status'] = 0
+  elif data_format == 'hdfs-directory':
+    notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
+    response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=insert_as_query&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
+    response['status'] = 0
 
   return JsonResponse(response)

+ 2 - 0
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -159,3 +159,5 @@ class Api(object):
   def export_data_as_hdfs_file(self, snippet, target_file, overwrite): raise NotImplementedError()
 
   def export_data_as_table(self, snippet, destination): raise NotImplementedError()
+
+  def export_large_data_to_hdfs(self, snippet, destination): raise NotImplementedError()

+ 24 - 5
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -349,11 +349,10 @@ class HS2Api(Api):
     db = self._get_db(snippet)
 
     response = self._get_current_statement(db, snippet)
+    query = self._prepare_hql_query(snippet, response.pop('statement'))
 
-    statement = response.pop('statement')
-
-    if not statement.strip().lower().startswith('select'):
-      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': statement})
+    if not query.hql_query.strip().lower().startswith('select'):
+      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
 
     database = snippet.get('database') or 'default'
     table = destination
@@ -361,7 +360,27 @@ class HS2Api(Api):
     if '.' in table:
       database, table = table.split('.', 1)
 
-    hql = 'CREATE TABLE `%s`.`%s` AS %s' % (database, table, statement)
+    db.use(query.database)
+
+    hql = 'CREATE TABLE `%s`.`%s` AS %s' % (database, table, query.hql_query)
     success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
 
     return hql, success_url
+
+
+  def export_large_data_to_hdfs(self, snippet, destination):
+    db = self._get_db(snippet)
+
+    response = self._get_current_statement(db, snippet)
+    query = self._prepare_hql_query(snippet, response.pop('statement'))
+
+    if not query.hql_query.strip().lower().startswith('select'):
+      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
+
+    db.use(query.database)
+
+    hql = "INSERT OVERWRITE DIRECTORY '%s' %s" % (destination, query.hql_query)
+    success_url = '/filebrowser/view=%s' % destination
+
+    return hql, success_url
+

+ 3 - 0
desktop/libs/notebook/src/notebook/views.py

@@ -134,6 +134,9 @@ def execute_and_watch(request):
   if action == 'save_as_table':
     sql, success_url = api.export_data_as_table(snippet, destination)
     editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
+  elif action == 'insert_as_query':
+    sql, success_url = api.export_large_data_to_hdfs(snippet, destination)
+    editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
   else:
     raise PopupException(_('Action %s is unknown') % action)