فهرست منبع

HUE-3301 [editor] Add generic API support for fetching table sample data

Jenny Kim 9 سال پیش
والد
کامیت
108d6f9

+ 10 - 3
apps/beeswax/src/beeswax/api.py

@@ -624,12 +624,19 @@ def clear_history(request):
 
 @error_handler
 def get_sample_data(request, database, table):
-  query_server = dbms.get_query_server_config(get_app_name(request))
+  app_name = get_app_name(request)
+  query_server = get_query_server_config(app_name)
   db = dbms.get(request.user, query_server)
-  response = {'status': -1}
 
+  response = _get_sample_data(db, database, table)
+  return JsonResponse(response)
+
+
+def _get_sample_data(db, database, table):
   table_obj = db.get_table(database, table)
   sample_data = db.get_sample(database, table_obj)
+  response = {'status': -1}
+
   if sample_data:
     response['status'] = 0
     response['headers'] = sample_data.cols()
@@ -637,7 +644,7 @@ def get_sample_data(request, database, table):
   else:
     response['message'] = _('Failed to get sample data.')
 
-  return JsonResponse(response)
+  return response
 
 
 @error_handler

+ 1 - 2
apps/metastore/src/metastore/static/metastore/js/metastore.ko.js

@@ -214,10 +214,9 @@
       return;
     }
     self.assistHelper.fetchTableSample({
-      sourceType: "hive",
+      type: "hive",
       databaseName: self.metastoreTable.database.name,
       tableName: self.metastoreTable.name,
-      dataType: "json",
       successCallback: function (data) {
         self.rows(data.rows);
         self.headers(data.headers);

+ 1 - 2
desktop/core/src/desktop/static/desktop/js/assist/assistDbEntry.js

@@ -334,10 +334,9 @@
     $assistQuickLook.attr("style", "width: " + ($(window).width() - 120) + "px;margin-left:-" + (($(window).width() - 80) / 2) + "px!important;");
 
     self.assistDbSource.assistHelper.fetchTableSample({
-      sourceType: self.assistDbSource.sourceType === "hive" ? "beeswax" : self.assistDbSource.sourceType,
+      type: self.assistDbSource.sourceType,
       databaseName: databaseName,
       tableName: tableName,
-      dataType: "html",
       successCallback: function(data) {
         if (! data.rows) {
           data.rows = [];

+ 17 - 16
desktop/core/src/desktop/static/desktop/js/assist/assistHelper.js

@@ -25,6 +25,7 @@
   var TIME_TO_LIVE_IN_MILLIS = $.totalStorage('hue.cacheable.ttl.override') || $.totalStorage('hue.cacheable.ttl'); // 1 day by default, configurable with desktop.custom.cacheable_ttl in the .ini or $.totalStorage('hue.cacheable.ttl.override', 1234567890)
 
   var AUTOCOMPLETE_API_PREFIX = "/notebook/api/autocomplete/";
+  var SAMPLE_API_PREFIX = "/notebook/api/sample/";
   var DOCUMENTS_API = "/desktop/api2/doc/";
   var DOCUMENTS_SEARCH_API = "/desktop/api2/docs/";
   var HDFS_API_PREFIX = "/filebrowser/view=";
@@ -589,27 +590,27 @@
    *
    * @param {string} options.databaseName
    * @param {string} options.tableName
-   * @param {string} options.dataType - html or json
+   * @param {string} options.type
    */
   AssistHelper.prototype.fetchTableSample = function (options) {
     var self = this;
-    $.ajax({
-      url: "/" + (options.sourceType == "hive" ? "beeswax" : options.sourceType) + "/api/table/" + options.databaseName + "/" + options.tableName + "/sample",
-      data: {},
-      beforeSend: function (xhr) {
-        xhr.setRequestHeader("X-Requested-With", "Hue");
-      },
-      success: function (response) {
-        if (! self.successResponseIsError(response)) {
-          options.successCallback(response);
-        } else {
-          self.assistErrorCallback(options)(response);
-        }
-      },
-      error: self.assistErrorCallback(options)
-    });
+    var url = SAMPLE_API_PREFIX + options.databaseName + '/' + options.tableName;
+
+    $.post(url, {
+      notebook: {},
+      snippet: ko.mapping.toJSON({
+        type: options.type
+      }),
+    }, function (data) {
+      if (! self.successResponseIsError(data)) {
+        options.successCallback(data);
+      } else {
+        self.assistErrorCallback(options)(data);
+      }
+    }).fail(self.assistErrorCallback(options));
   };
 
+
   /**
    * @param {Object} options
    * @param {string} options.sourceType

+ 2 - 0
desktop/core/src/desktop/templates/assist.mako

@@ -674,9 +674,11 @@ from desktop.views import _ko
     <div id="assistQuickLook" class="modal hide fade">
       <div class="modal-header">
         <a href="#" class="close" data-dismiss="modal">&times;</a>
+        <!-- ko if: sourceType === 'hive' || sourceType === 'impala' -->
         <a class="tableLink pull-right" href="#" target="_blank" style="margin-right: 20px;margin-top:6px">
           <i class="fa fa-external-link"></i> ${ _('View more...') }
         </a>
+        <!-- /ko -->
         <h3>${_('Data sample for')} <span class="tableName"></span></h3>
       </div>
       <div class="modal-body" style="min-height: 100px">

+ 3 - 0
desktop/libs/librdbms/src/librdbms/server/dbms.py

@@ -97,6 +97,9 @@ class Rdbms(object):
   def get_columns(self, database, table_name, names_only=True):
     return self.client.get_columns(database, table_name, names_only)
 
+  def get_sample_data(self, database, table_name, limit=100):
+    return self.client.get_sample_data(database, table_name, limit)
+
   def execute_statement(self, statement):
     return self.client.execute_statement(statement)
 

+ 5 - 0
desktop/libs/librdbms/src/librdbms/server/mysql_lib.py

@@ -129,3 +129,8 @@ class MySQLClient(BaseRDMSClient):
     else:
       columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
     return columns
+
+
+  def get_sample_data(self, database, table, limit=100):
+    statement = "SELECT * FROM `%s`.`%s` LIMIT %d" % (database, table, limit)
+    return self.execute_statement(statement)

+ 4 - 0
desktop/libs/librdbms/src/librdbms/server/oracle_lib.py

@@ -107,3 +107,7 @@ class OracleClient(BaseRDMSClient):
     else:
       columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
     return columns
+
+  def get_sample_data(self, database, table, limit=100):
+    statement = 'SELECT * FROM "%s"."%s" LIMIT %d' % (database, table, limit)
+    return self.execute_statement(statement)

+ 4 - 0
desktop/libs/librdbms/src/librdbms/server/postgresql_lib.py

@@ -132,3 +132,7 @@ class PostgreSQLClient(BaseRDMSClient):
     else:
       columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
     return columns
+
+  def get_sample_data(self, database, table, limit=100):
+    statement = 'SELECT * FROM "%s"."%s" LIMIT %d' % (database, table, limit)
+    return self.execute_statement(statement)

+ 4 - 0
desktop/libs/librdbms/src/librdbms/server/sqlite_lib.py

@@ -106,3 +106,7 @@ class SQLiteClient(BaseRDMSClient):
     else:
       columns = [dict(name=row[1], type=row[2], comment='') for row in cursor.fetchall()]
     return columns
+
+  def get_sample_data(self, database, table, limit=100):
+    statement = 'SELECT * FROM %s LIMIT %d' % (table, limit)
+    return self.execute_statement(statement)

+ 22 - 0
desktop/libs/notebook/src/notebook/api.py

@@ -373,6 +373,28 @@ def autocomplete(request, server=None, database=None, table=None, column=None, n
   return JsonResponse(response)
 
 
+@require_POST
+@check_document_access_permission()
+@api_error_handler
+def get_sample_data(request, server=None, database=None, table=None):
+  response = {'status': -1}
+
+  # Passed by check_document_access_permission but unused by APIs
+  notebook = json.loads(request.POST.get('notebook', '{}'))
+  snippet = json.loads(request.POST.get('snippet', '{}'))
+
+  try:
+    sample_data = get_api(request, snippet).get_sample_data(snippet, database, table)
+    response.update(sample_data)
+  except QueryExpired, e:
+    LOG.debug('get_sample_data query expired: %s' % e)
+    pass
+
+  response['status'] = 0
+
+  return JsonResponse(response)
+
+
 @require_GET
 @api_error_handler
 def github_fetch(request):

+ 7 - 1
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -32,7 +32,7 @@ LOG = logging.getLogger(__name__)
 
 try:
   from beeswax import data_export
-  from beeswax.api import _autocomplete
+  from beeswax.api import _autocomplete, _get_sample_data
   from beeswax.design import hql_query, strip_trailing_semicolon, split_statements
   from beeswax import conf as beeswax_conf
   from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory, Session
@@ -258,6 +258,12 @@ class HS2Api(Api):
     return _autocomplete(db, database, table, column, nested)
 
 
+  @query_error_handler
+  def get_sample_data(self, snippet, database=None, table=None):
+    db = self._get_db(snippet)
+    return _get_sample_data(db, database, table)
+
+
   def _get_current_statement(self, db, snippet):
     # Multiquery, if not first statement or arrived to the last query
     statement_id = snippet['result']['handle'].get('statement_id', 0)

+ 36 - 19
desktop/libs/notebook/src/notebook/connectors/jdbc.py

@@ -135,26 +135,40 @@ class JdbcApi(Api):
       raise AuthenticationRequired()
 
     assist = Assist(self.db)
-    response = {'error': 0}
+    response = {'status': -1}
+
+    if database is None:
+      response['databases'] = assist.get_databases()
+    elif table is None:
+      response['tables'] = assist.get_tables(database)
+    else:
+      columns = assist.get_columns(database, table)
+      response['columns'] = [col[0] for col in columns]
+      response['extended_columns'] = [{
+        'name': col[0],
+        'type': col[1],
+        'comment': col[5]
+      } for col in columns]
+
+    response['status'] = 0
+    return response
 
-    try:
-      if database is None:
-        response['databases'] = assist.get_databases()
-      elif table is None:
-        response['tables'] = assist.get_tables(database)
-      else:
-        columns = assist.get_columns(database, table)
-        response['columns'] = [col[0] for col in columns]
-        response['extended_columns'] = [{
-            'name': col[0],
-            'type': col[1],
-            'comment': col[5]
-          } for col in columns
-        ]
-    except Exception, e:
-      LOG.warn('Autocomplete data fetching error: %s' % e)
-      response['code'] = -1
-      response['error'] = str(e)
+  @query_error_handler
+  def get_sample_data(self, snippet, database=None, table=None):
+    if self.db is None:
+      raise AuthenticationRequired()
+
+    assist = Assist(self.db)
+    response = {'status': -1}
+
+    sample_data, description = assist.get_sample_data(database, table)
+
+    if sample_data:
+      response['status'] = 0
+      response['headers'] = [col[0] for col in description] if description else []
+      response['rows'] = sample_data
+    else:
+      response['message'] = _('Failed to get sample data.')
 
     return response
 
@@ -179,3 +193,6 @@ class Assist():
   def get_columns(self, database, table):
     columns, description = query_and_fetch(self.db, 'SHOW COLUMNS FROM %s.%s' % (database, table))
     return columns
+
+  def get_sample_data(self, database, table):
+    return query_and_fetch(self.db, 'SELECT * FROM %s.%s' % (database, table))

+ 23 - 0
desktop/libs/notebook/src/notebook/connectors/rdbms.py

@@ -132,6 +132,26 @@ class RdbmsApi(Api):
     response['status'] = 0
     return response
 
+
+  @query_error_handler
+  def get_sample_data(self, snippet, database=None, table=None):
+    query_server = dbms.get_query_server_config(server=self.interpreter)
+    db = dbms.get(self.user, query_server)
+
+    assist = Assist(db)
+    response = {'status': -1}
+
+    sample_data = assist.get_sample_data(database, table)
+
+    if sample_data:
+      response['status'] = 0
+      response['headers'] = sample_data.columns
+      response['rows'] = list(sample_data.rows())
+    else:
+      response['message'] = _('Failed to get sample data.')
+
+    return response
+
   @query_error_handler
   def get_select_star_query(self, snippet, database, table):
     return "SELECT * FROM `%s`.`%s`" % (database, table)
@@ -150,3 +170,6 @@ class Assist():
 
   def get_columns(self, database, table):
     return self.db.get_columns(database, table, names_only=False)
+
+  def get_sample_data(self, database, table):
+    return self.db.get_sample_data(database, table)

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/tests/tests_hiveserver2.py

@@ -30,7 +30,7 @@ from beeswax.server import dbms
 from beeswax.test_base import get_query_server_config
 
 
-class TestNotebookApi(object):
+class TestHiveserver2Api(object):
 
   def setUp(self):
     self.client = make_logged_in_client(username="test", groupname="test", recreate=False, is_superuser=False)

+ 4 - 0
desktop/libs/notebook/src/notebook/urls.py

@@ -74,12 +74,16 @@ urlpatterns += patterns('notebook.api',
 
 # Assist API
 urlpatterns += patterns('notebook.api',
+  # HS2, RDBMS, JDBC
   url(r'^api/autocomplete/?$', 'autocomplete', name='api_autocomplete_databases'),
   url(r'^api/autocomplete/(?P<database>\w+)/?$', 'autocomplete', name='api_autocomplete_tables'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'autocomplete', name='api_autocomplete_column'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/(?P<nested>.+)/?$', 'autocomplete', name='api_autocomplete_nested'),
+  url(r'^api/sample/(?P<database>\w+)/(?P<table>\w+)/?$', 'get_sample_data', name='api_sample_data'),
+
   # SQLite
   url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/?$', 'autocomplete', name='api_autocomplete_tables'),
   url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
+  url(r'^api/sample/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'get_sample_data', name='api_sample_data'),
 )