Browse Source

HUE-5712 [metadata] Upload DDL and available stats of active tables and their columns

Romain Rigaux 8 years ago
parent
commit
07a7ed49ff

+ 1 - 0
apps/beeswax/src/beeswax/api.py

@@ -649,6 +649,7 @@ def _get_sample_data(db, database, table, column):
   if table_obj.is_impala_only and db.client.query_server['server_name'] != 'impala':
   if table_obj.is_impala_only and db.client.query_server['server_name'] != 'impala':
     query_server = get_query_server_config('impala')
     query_server = get_query_server_config('impala')
     db = dbms.get(db.client.user, query_server)
     db = dbms.get(db.client.user, query_server)
+
   sample_data = db.get_sample(database, table_obj, column)
   sample_data = db.get_sample(database, table_obj, column)
   response = {'status': -1}
   response = {'status': -1}
 
 

+ 3 - 1
apps/beeswax/src/beeswax/server/dbms.py

@@ -825,7 +825,7 @@ class HiveServer2Dbms(object):
     hql = []
     hql = []
 
 
     for partition_spec in partition_specs:
     for partition_spec in partition_specs:
-        hql.append("ALTER TABLE `%s`.`%s` DROP IF EXISTS PARTITION (%s) PURGE" % (db_name, table_name, partition_spec))
+      hql.append("ALTER TABLE `%s`.`%s` DROP IF EXISTS PARTITION (%s) PURGE" % (db_name, table_name, partition_spec))
 
 
     query = hql_query(';'.join(hql), db_name)
     query = hql_query(';'.join(hql), db_name)
     design.data = query.dumps()
     design.data = query.dumps()
@@ -842,6 +842,7 @@ class HiveServer2Dbms(object):
 
 
     if handle:
     if handle:
       result = self.fetch(handle, rows=5000)
       result = self.fetch(handle, rows=5000)
+      self.close(handle)
 
 
     return result
     return result
 
 
@@ -859,6 +860,7 @@ class HiveServer2Dbms(object):
 
 
     if handle:
     if handle:
       result = self.fetch(handle, rows=5000)
       result = self.fetch(handle, rows=5000)
+      self.close(handle)
 
 
     return result
     return result
 
 

+ 11 - 4
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -21,7 +21,6 @@ import logging
 import struct
 import struct
 
 
 from django.http import Http404
 from django.http import Http404
-from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_POST
 from django.views.decorators.http import require_POST
 
 
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.django_util import JsonResponse
@@ -33,12 +32,13 @@ from notebook.models import Notebook
 from metadata.conf import NAVIGATOR
 from metadata.conf import NAVIGATOR
 from metadata.optimizer_client import OptimizerApi
 from metadata.optimizer_client import OptimizerApi
 
 
-
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
 try:
 try:
   from beeswax.api import get_table_stats
   from beeswax.api import get_table_stats
+  from beeswax.design import hql_query
+  from beeswax.server import dbms
 except ImportError, e:
 except ImportError, e:
   LOG.warn("Hive lib not enabled")
   LOG.warn("Hive lib not enabled")
 
 
@@ -367,8 +367,15 @@ def upload_table_stats(request):
             "avg_col_len": int(float(col_stats['avg_col_len'])) if col_stats['avg_col_len'] != '' else -1
             "avg_col_len": int(float(col_stats['avg_col_len'])) if col_stats['avg_col_len'] != '' else -1
           })
           })
 
 
-#       if with_ddl:
-#         table_ddl.append((original_query_id, execution_time, query_data['snippets'][0]['statement']))
+      if with_ddl:
+        db = dbms.get(request.user)
+        query = hql_query('SHOW CREATE TABLE `%(database)s`.`%(table)s`' % path)
+        handle = db.execute_and_wait(query, timeout_sec=5.0)
+
+        if handle:
+          result = db.fetch(handle, rows=5000)
+          db.close(handle)
+          table_ddls.append((0, 0, '\n'.join([row[0] for row in result.rows()])))
 
 
     except Exception, e:
     except Exception, e:
       LOG.exception('Skipping upload of %s: %s' % (db_table, e))
       LOG.exception('Skipping upload of %s: %s' % (db_table, e))

+ 1 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1473,7 +1473,7 @@ var EditorViewModel = (function() {
       logGA('load_table_stats');
       logGA('load_table_stats');
 
 
       $.post("/metadata/api/optimizer/upload/table_stats", {
       $.post("/metadata/api/optimizer/upload/table_stats", {
-    	db_tables: ko.mapping.toJSON(activeTables),
+        db_tables: ko.mapping.toJSON(activeTables),
         sourcePlatform: ko.mapping.toJSON(self.type()),
         sourcePlatform: ko.mapping.toJSON(self.type()),
         with_columns: ko.mapping.toJSON(true),
         with_columns: ko.mapping.toJSON(true),
         with_ddl: ko.mapping.toJSON(true)
         with_ddl: ko.mapping.toJSON(true)