Parcourir la source

HUE-3357 [editor] Update sample API to provide data for a column

POST /notebook/api/sample/<database>/<table>/<column>/

or for SQLite:

POST /notebook/api/sample/<server>/<database>/<table>/<column>/
Jenny Kim il y a 9 ans
Parent
commit
a61084d

+ 4 - 4
apps/beeswax/src/beeswax/api.py

@@ -623,18 +623,18 @@ def clear_history(request):
 
 
 @error_handler
-def get_sample_data(request, database, table):
+def get_sample_data(request, database, table, column=None):
   app_name = get_app_name(request)
   query_server = get_query_server_config(app_name)
   db = dbms.get(request.user, query_server)
 
-  response = _get_sample_data(db, database, table)
+  response = _get_sample_data(db, database, table, column)
   return JsonResponse(response)
 
 
-def _get_sample_data(db, database, table):
+def _get_sample_data(db, database, table, column):
   table_obj = db.get_table(database, table)
-  sample_data = db.get_sample(database, table_obj)
+  sample_data = db.get_sample(database, table_obj, column)
   response = {'status': -1}
 
   if sample_data:

+ 13 - 11
apps/beeswax/src/beeswax/server/dbms.py

@@ -311,24 +311,26 @@ class HiveServer2Dbms(object):
     return resp
 
 
-  def get_sample(self, database, table, column=None, nested=None):
+  def get_sample(self, database, table, column=None, nested=None, limit=100):
     result = None
     hql = None
 
-    limit = 100
-
-    if column or nested: # Could do column for any type, then nested with partitions
-      if self.server_name == 'impala':
+    if self.server_name == 'impala':
+      if column or nested:
         from impala.dbms import ImpalaDbms
         select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
         hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
+      else:
+        hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
     else:
       # Filter on max # of partitions for partitioned tables
       # Impala's SHOW PARTITIONS is different from Hive, so we only support Hive for now
-      if self.server_name != 'impala' and table.partition_keys:
-        hql = self._get_sample_partition_query(database, table, limit)
+      column = '`%s`' % column if column else '*'
+      if table.partition_keys:
+        hql = self._get_sample_partition_query(database, table, column, limit)
       else:
-        hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)
+        hql = "SELECT %s FROM `%s`.`%s` LIMIT %s" % (column, database, table.name, limit)
+        # TODO: Add nested select support for HS2
 
     if hql:
       query = hql_query(hql)
@@ -341,7 +343,7 @@ class HiveServer2Dbms(object):
     return result
 
 
-  def _get_sample_partition_query(self, database, table, limit):
+  def _get_sample_partition_query(self, database, table, column=None, limit=100):
     max_parts = QUERY_PARTITIONS_LIMIT.get()
     partitions = self.get_partitions(database, table, partition_spec=None, max_parts=max_parts)
 
@@ -353,8 +355,8 @@ class HiveServer2Dbms(object):
     else:
       partition_clause = ''
 
-    return "SELECT * FROM `%(database)s`.`%(table)s` %(partition_clause)s LIMIT %(limit)s" % \
-      {'database': database, 'table': table.name, 'partition_clause': partition_clause, 'limit': limit}
+    return "SELECT %(column)s FROM `%(database)s`.`%(table)s` %(partition_clause)s LIMIT %(limit)s" % \
+      {'column': column, 'database': database, 'table': table.name, 'partition_clause': partition_clause, 'limit': limit}
 
 
   def analyze_table(self, database, table):

+ 1 - 0
apps/beeswax/src/beeswax/urls.py

@@ -88,6 +88,7 @@ urlpatterns += patterns(
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/?$', 'describe_table', name='describe_table'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/indexes/?$', 'get_indexes', name='get_indexes'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/sample/?$', 'get_sample_data', name='get_sample_data'),
+  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/sample/?$', 'get_sample_data', name='get_sample_data_column'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/stats/(?P<column>\w+)?$', 'get_table_stats', name='get_table_stats'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/terms/(?P<column>\w+)/(?P<prefix>\w+)?$', 'get_top_terms', name='get_top_terms'),
 

+ 2 - 2
desktop/libs/librdbms/src/librdbms/server/dbms.py

@@ -97,8 +97,8 @@ class Rdbms(object):
   def get_columns(self, database, table_name, names_only=True):
     return self.client.get_columns(database, table_name, names_only)
 
-  def get_sample_data(self, database, table_name, limit=100):
-    return self.client.get_sample_data(database, table_name, limit)
+  def get_sample_data(self, database, table_name, column=None, limit=100):
+    return self.client.get_sample_data(database, table_name, column, limit)
 
   def execute_statement(self, statement):
     return self.client.execute_statement(statement)

+ 3 - 2
desktop/libs/librdbms/src/librdbms/server/mysql_lib.py

@@ -131,6 +131,7 @@ class MySQLClient(BaseRDMSClient):
     return columns
 
 
-  def get_sample_data(self, database, table, limit=100):
-    statement = "SELECT * FROM `%s`.`%s` LIMIT %d" % (database, table, limit)
+  def get_sample_data(self, database, table, column=None, limit=100):
+    column = '`%s`' % column if column else '*'
+    statement = "SELECT %s FROM `%s`.`%s` LIMIT %d" % (column, database, table, limit)
     return self.execute_statement(statement)

+ 3 - 2
desktop/libs/librdbms/src/librdbms/server/oracle_lib.py

@@ -108,6 +108,7 @@ class OracleClient(BaseRDMSClient):
       columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
     return columns
 
-  def get_sample_data(self, database, table, limit=100):
-    statement = 'SELECT * FROM "%s"."%s" LIMIT %d' % (database, table, limit)
+  def get_sample_data(self, database, table, column=None, limit=100):
+    column = '"%s"' % column  if column else '*'
+    statement = 'SELECT %s FROM "%s"."%s" LIMIT %d' % (column, database, table, limit)
     return self.execute_statement(statement)

+ 3 - 2
desktop/libs/librdbms/src/librdbms/server/postgresql_lib.py

@@ -133,6 +133,7 @@ class PostgreSQLClient(BaseRDMSClient):
       columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
     return columns
 
-  def get_sample_data(self, database, table, limit=100):
-    statement = 'SELECT * FROM "%s"."%s" LIMIT %d' % (database, table, limit)
+  def get_sample_data(self, database, table, column=None, limit=100):
+    column = '"%s"' % column if column else '*'
+    statement = 'SELECT %s FROM "%s"."%s" LIMIT %d' % (column, database, table, limit)
     return self.execute_statement(statement)

+ 3 - 2
desktop/libs/librdbms/src/librdbms/server/sqlite_lib.py

@@ -107,6 +107,7 @@ class SQLiteClient(BaseRDMSClient):
       columns = [dict(name=row[1], type=row[2], comment='') for row in cursor.fetchall()]
     return columns
 
-  def get_sample_data(self, database, table, limit=100):
-    statement = 'SELECT * FROM %s LIMIT %d' % (table, limit)
+  def get_sample_data(self, database, table, column=None, limit=100):
+    column = '`%s`' % column if column else '*'
+    statement = 'SELECT %s FROM `%s` LIMIT %d' % (column, table, limit)
     return self.execute_statement(statement)

+ 2 - 2
desktop/libs/notebook/src/notebook/api.py

@@ -378,14 +378,14 @@ def autocomplete(request, server=None, database=None, table=None, column=None, n
 @require_POST
 @check_document_access_permission()
 @api_error_handler
-def get_sample_data(request, server=None, database=None, table=None):
+def get_sample_data(request, server=None, database=None, table=None, column=None):
   response = {'status': -1}
 
   # Passed by check_document_access_permission but unused by APIs
   notebook = json.loads(request.POST.get('notebook', '{}'))
   snippet = json.loads(request.POST.get('snippet', '{}'))
 
-  sample_data = get_api(request, snippet).get_sample_data(snippet, database, table)
+  sample_data = get_api(request, snippet).get_sample_data(snippet, database, table, column)
   response.update(sample_data)
 
   response['status'] = 0

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -261,9 +261,9 @@ class HS2Api(Api):
 
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None):
+  def get_sample_data(self, snippet, database=None, table=None, column=None):
     db = self._get_db(snippet)
-    return _get_sample_data(db, database, table)
+    return _get_sample_data(db, database, table, column)
 
 
   @query_error_handler

+ 5 - 4
desktop/libs/notebook/src/notebook/connectors/jdbc.py

@@ -154,14 +154,14 @@ class JdbcApi(Api):
     return response
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None):
+  def get_sample_data(self, snippet, database=None, table=None, column=None):
     if self.db is None:
       raise AuthenticationRequired()
 
     assist = Assist(self.db)
     response = {'status': -1}
 
-    sample_data, description = assist.get_sample_data(database, table)
+    sample_data, description = assist.get_sample_data(database, table, column)
 
     if sample_data:
       response['status'] = 0
@@ -194,5 +194,6 @@ class Assist():
     columns, description = query_and_fetch(self.db, 'SHOW COLUMNS FROM %s.%s' % (database, table))
     return columns
 
-  def get_sample_data(self, database, table):
-    return query_and_fetch(self.db, 'SELECT * FROM %s.%s' % (database, table))
+  def get_sample_data(self, database, table, column=None):
+    column = column or '*'
+    return query_and_fetch(self.db, 'SELECT %s FROM %s.%s' % (column, database, table))

+ 4 - 4
desktop/libs/notebook/src/notebook/connectors/rdbms.py

@@ -134,14 +134,14 @@ class RdbmsApi(Api):
 
 
   @query_error_handler
-  def get_sample_data(self, snippet, database=None, table=None):
+  def get_sample_data(self, snippet, database=None, table=None, column=None):
     query_server = dbms.get_query_server_config(server=self.interpreter)
     db = dbms.get(self.user, query_server)
 
     assist = Assist(db)
     response = {'status': -1}
 
-    sample_data = assist.get_sample_data(database, table)
+    sample_data = assist.get_sample_data(database, table, column)
 
     if sample_data:
       response['status'] = 0
@@ -171,5 +171,5 @@ class Assist():
   def get_columns(self, database, table):
     return self.db.get_columns(database, table, names_only=False)
 
-  def get_sample_data(self, database, table):
-    return self.db.get_sample_data(database, table)
+  def get_sample_data(self, database, table, column=None):
+    return self.db.get_sample_data(database, table, column)

+ 42 - 11
desktop/libs/notebook/src/notebook/connectors/tests/tests_hiveserver2.py

@@ -110,29 +110,29 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
   def setup_class(cls):
     super(TestHiveserver2ApiWithHadoop, cls).setup_class(load_data=False)
 
+
   def setUp(self):
-    self.user = User.objects.get(username='test')
+    self.client.post('/beeswax/install_examples')
 
+    self.user = User.objects.get(username='test')
     grant_access("test", "test", "notebook")
 
     self.db = dbms.get(self.user, get_query_server_config())
     self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir, '/user/test')
     self.api = HS2Api(self.user)
 
-
-  def test_explain(self):
-    notebook_json = """
+    self.notebook_json = """
       {
         "uuid": "f5d6394d-364f-56e8-6dd3-b1c5a4738c52",
         "id": 1234,
         "sessions": [{"type": "hive", "properties": [], "id": null}]
       }
     """
-    statement = 'SELECT description, salary FROM sample_07 WHERE (sample_07.salary > 100000) ORDER BY salary DESC LIMIT 1000'
-    snippet_json = """
+    self.statement = 'SELECT description, salary FROM sample_07 WHERE (sample_07.salary > 100000) ORDER BY salary DESC LIMIT 1000'
+    self.snippet_json = """
       {
           "status": "running",
-          "database": "default",
+          "database": "%(database)s",
           "id": "d70d31ee-a62a-4854-b2b1-b852f6a390f5",
           "result": {
               "type": "table",
@@ -147,13 +147,44 @@ class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
               "settings": []
           }
       }
-    """ % {'statement': statement}
+    """ % {'database': self.db_name, 'statement': self.statement}
 
-    Document2.objects.create(id=1234, name='Test Hive Query', type='query-hive', owner=self.user, is_history=True, data=notebook_json)
+    doc, created = Document2.objects.get_or_create(
+      id=1234,
+      name='Test Hive Query',
+      type='query-hive',
+      owner=self.user,
+      is_history=True,
+      data=self.notebook_json)
 
-    response = self.client.post(reverse('notebook:explain'), {'notebook': notebook_json, 'snippet': snippet_json})
+
+  def test_explain(self):
+    response = self.client.post(reverse('notebook:explain'), {'notebook': self.notebook_json, 'snippet': self.snippet_json})
     data = json.loads(response.content)
 
     assert_equal(0, data['status'], data)
     assert_true('STAGE DEPENDENCIES' in data['explanation'], data)
-    assert_equal(statement, data['statement'], data)
+    assert_equal(self.statement, data['statement'], data)
+
+
+  def test_get_sample(self):
+    response = self.client.post(reverse('notebook:api_sample_data',
+      kwargs={'database': 'default', 'table': 'sample_07'}),
+      {'notebook': self.notebook_json, 'snippet': self.snippet_json})
+    data = json.loads(response.content)
+
+    assert_equal(0, data['status'], data)
+    assert_true('headers' in data)
+    assert_true('rows' in data)
+    assert_true(len(data['rows']) > 0)
+
+    response = self.client.post(reverse('notebook:api_sample_data_column',
+      kwargs={'database': 'default', 'table': 'sample_07', 'column': 'code'}),
+      {'notebook': self.notebook_json, 'snippet': self.snippet_json})
+    data = json.loads(response.content)
+
+    assert_equal(0, data['status'], data)
+    assert_true('headers' in data)
+    assert_equal(['code'], data['headers'])
+    assert_true('rows' in data)
+    assert_true(len(data['rows']) > 0)

+ 2 - 0
desktop/libs/notebook/src/notebook/urls.py

@@ -79,11 +79,13 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'autocomplete', name='api_autocomplete_column'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/(?P<nested>.+)/?$', 'autocomplete', name='api_autocomplete_nested'),
   url(r'^api/sample/(?P<database>\w+)/(?P<table>\w+)/?$', 'get_sample_data', name='api_sample_data'),
+  url(r'^api/sample/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'get_sample_data', name='api_sample_data_column'),
 
   # SQLite
   url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/?$', 'autocomplete', name='api_autocomplete_tables'),
   url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
   url(r'^api/sample/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'get_sample_data', name='api_sample_data'),
+  url(r'^api/sample/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'get_sample_data', name='api_sample_data_column'),
 )
 
 # Github