Преглед на файлове

HUE-8570 [editor] Enable optional operation on the sample API endpoints

This enables distinct sample queries
Johan Ahlen преди 7 години
родител
ревизия
03a64a0

+ 2 - 2
apps/beeswax/src/beeswax/api.py

@@ -656,13 +656,13 @@ def get_sample_data(request, database, table, column=None):
   return JsonResponse(response)
 
 
-def _get_sample_data(db, database, table, column, async=False, cluster=None):
+def _get_sample_data(db, database, table, column, async=False, cluster=None, operation=None):
   table_obj = db.get_table(database, table)
   if table_obj.is_impala_only and db.client.query_server['server_name'] != 'impala':
     query_server = get_query_server_config('impala', cluster=cluster)
     db = dbms.get(db.client.user, query_server, cluster=cluster)
 
-  sample_data = db.get_sample(database, table_obj, column, generate_sql_only=async)
+  sample_data = db.get_sample(database, table_obj, column, generate_sql_only=async, operation=operation)
   response = {'status': -1}
 
   if sample_data:

+ 13 - 6
apps/beeswax/src/beeswax/server/dbms.py

@@ -359,23 +359,29 @@ class HiveServer2Dbms(object):
     return resp
 
 
-  def get_sample(self, database, table, column=None, nested=None, limit=100, generate_sql_only=False):
+  def get_sample(self, database, table, column=None, nested=None, limit=100, generate_sql_only=False, operation=None):
     result = None
     hql = None
 
     # Filter on max # of partitions for partitioned tables
     column = '`%s`' % column if column else '*'
     if table.partition_keys:
-      hql = self._get_sample_partition_query(database, table, column, limit)
+      hql = self._get_sample_partition_query(database, table, column, limit, operation)
     elif self.server_name == 'impala':
       if column or nested:
         from impala.dbms import ImpalaDbms
         select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
-        hql = 'SELECT %s FROM %s LIMIT %s;' % (select_clause, from_clause, limit)
+        if operation == 'distinct':
+          hql = 'SELECT DISTINCT %s FROM %s LIMIT %s;' % (select_clause, from_clause, limit)
+        else:
+          hql = 'SELECT %s FROM %s LIMIT %s;' % (select_clause, from_clause, limit)
       else:
         hql = "SELECT * FROM `%s`.`%s` LIMIT %s;" % (database, table.name, limit)
     else:
-      hql = "SELECT %s FROM `%s`.`%s` LIMIT %s;" % (column, database, table.name, limit)
+      if operation == 'distinct':
+        hql = "SELECT DISTINCT %s FROM `%s`.`%s` LIMIT %s;" % (column, database, table.name, limit)
+      else:
+        hql = "SELECT %s FROM `%s`.`%s` LIMIT %s;" % (column, database, table.name, limit)
       # TODO: Add nested select support for HS2
 
     if hql:
@@ -392,7 +398,7 @@ class HiveServer2Dbms(object):
     return result
 
 
-  def _get_sample_partition_query(self, database, table, column='*', limit=100):
+  def _get_sample_partition_query(self, database, table, column='*', limit=100, operation=None):
     max_parts = QUERY_PARTITIONS_LIMIT.get()
     partitions = self.get_partitions(database, table, partition_spec=None, max_parts=max_parts)
 
@@ -404,7 +410,8 @@ class HiveServer2Dbms(object):
     else:
       partition_clause = ''
 
-    return "SELECT %(column)s FROM `%(database)s`.`%(table)s` %(partition_clause)s LIMIT %(limit)s" % \
+    prefix = "SELECT DISTINCT " if operation == 'distinct' else 'SELECT'
+    return prefix + "%(column)s FROM `%(database)s`.`%(table)s` %(partition_clause)s LIMIT %(limit)s" % \
       {'column': column, 'database': database, 'table': table.name, 'partition_clause': partition_clause, 'limit': limit}
 
 

+ 2 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -584,8 +584,9 @@ def get_sample_data(request, server=None, database=None, table=None, column=None
   notebook = json.loads(request.POST.get('notebook', '{}'))
   snippet = json.loads(request.POST.get('snippet', '{}'))
   async = json.loads(request.POST.get('async', 'false'))
+  operation = json.loads(request.POST.get('operation', 'default'))
 
-  sample_data = get_api(request, snippet).get_sample_data(snippet, database, table, column, async=async)
+  sample_data = get_api(request, snippet).get_sample_data(snippet, database, table, column, async=async, operation=operation)
   response.update(sample_data)
 
   response['status'] = 0

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -411,7 +411,7 @@ class Api(object):
   def get_jobs(self, notebook, snippet, logs):
     return []
 
-  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False): raise NotImplementedError()
+  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None): raise NotImplementedError()
 
   def export_data_as_hdfs_file(self, snippet, target_file, overwrite): raise NotImplementedError()
 

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -470,10 +470,10 @@ class HS2Api(Api):
 
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False):
+  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None):
     try:
       db = self._get_db(snippet, async, cluster=self.cluster)
-      return _get_sample_data(db, database, table, column, async)
+      return _get_sample_data(db, database, table, column, async, operation=operation)
     except QueryServerException, ex:
       raise QueryError(ex.message)
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/jdbc.py

@@ -159,7 +159,7 @@ class JdbcApi(Api):
     return response
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False):
+  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None):
     if self.db is None:
       raise AuthenticationRequired()
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/rdbms.py

@@ -149,7 +149,7 @@ class RdbmsApi(Api):
 
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False):
+  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None):
     query_server = dbms.get_query_server_config(server=self.interpreter)
     db = dbms.get(self.user, query_server)
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/solr.py

@@ -159,7 +159,7 @@ class SolrApi(Api):
 
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False):
+  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None):
     from search.conf import SOLR_URL
     db = NativeSolrApi(SOLR_URL.get(), self.user)
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/sqlalchemyapi.py

@@ -221,7 +221,7 @@ class SqlAlchemyApi(Api):
 
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False):
+  def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None):
     inspector = inspect(self.engine)
 
     assist = Assist(inspector, self.engine)