Преглед на файлове

[metastore] Add separate get_sample_data API endpoint and fix metastore tests

Jenny Kim преди 10 години
родител
ревизия
c012003f56

+ 9 - 0
apps/beeswax/src/beeswax/api.py

@@ -661,6 +661,15 @@ def describe_table(request, database, table):
     raise PopupException(_('Problem accessing table metadata'), detail=e)
 
 
+def get_sample_data(request, database, table):
+  try:
+    from metastore.views import get_sample_data
+    return get_sample_data(request, database, table)
+  except Exception, e:
+    LOG.exception('Failed to retrieve sample data for `%s`.`%s`' % (database, table))
+    raise PopupException(_('Problem accessing table metadata'), detail=e)
+
+
 def get_query_form(request):
   try:
     try:

+ 1 - 0
apps/beeswax/src/beeswax/urls.py

@@ -85,6 +85,7 @@ urlpatterns += patterns(
   url(r'^api/query/clear_history/$', 'clear_history', name='clear_history'),
 
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)$', 'describe_table', name='describe_table'),
+  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/sample$', 'get_sample_data', name='get_sample_data'),
   url(r'^api/analyze/(?P<database>\w+)/(?P<table>\w+)/(?P<columns>\w+)?$', 'analyze_table', name='analyze_table'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/stats/(?P<column>\w+)?$', 'get_table_stats', name='get_table_stats'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/terms/(?P<column>\w+)/(?P<prefix>\w+)?$', 'get_top_terms', name='get_top_terms'),

+ 33 - 33
apps/metastore/src/metastore/tests.py

@@ -106,56 +106,49 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     assert_equal(verify_history(self.client, fragment='test'), history_cnt, 'Implicit queries should not be saved in the history')
 
   def test_show_tables(self):
-    try:
-      hql = """
+    hql = """
         CREATE TABLE test_show_tables_1 (a int) COMMENT 'Test for show_tables';
         CREATE TABLE test_show_tables_2 (a int) COMMENT 'Test for show_tables';
         CREATE TABLE test_show_tables_3 (a int) COMMENT 'Test for show_tables';
       """
-      resp = _make_query(self.client, hql, database=self.db_name)
-      resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    resp = _make_query(self.client, hql, database=self.db_name)
+    resp = wait_for_query_to_finish(self.client, resp, max=30.0)
 
-      # Table should have been created
-      response = self.client.get("/metastore/tables/%s?filter=show_tables" % self.db_name)
-      assert_equal(200, response.status_code)
-      assert_equal(len(response.context['tables']), 3)
-      assert_equal(response.context['has_metadata'], True)
-      assert_true('name' in response.context["tables"][0])
-      assert_true('comment' in response.context["tables"][0])
-      assert_true('type' in response.context["tables"][0])
+    # Table should have been created
+    response = self.client.get("/metastore/tables/%s?filter=show_tables" % self.db_name)
+    assert_equal(200, response.status_code)
+    assert_equal(len(response.context['tables']), 3)
+    assert_true('name' in response.context["tables"][0])
+    assert_true('comment' in response.context["tables"][0])
+    assert_true('type' in response.context["tables"][0])
 
-      hql = """
+    hql = """
         CREATE TABLE test_show_tables_4 (a int) COMMENT 'Test for show_tables';
         CREATE TABLE test_show_tables_5 (a int) COMMENT 'Test for show_tables';
       """
-      resp = _make_query(self.client, hql, database=self.db_name)
-      resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+    resp = _make_query(self.client, hql, database=self.db_name)
+    resp = wait_for_query_to_finish(self.client, resp, max=30.0)
 
-      # Table should have been created
-      response = self.client.get("/metastore/tables/%s?filter=show_tables" % self.db_name)
-      assert_equal(200, response.status_code)
-      assert_equal(len(response.context['tables']), 5)
-      assert_equal(response.context['has_metadata'], False)
-      assert_true('name' in response.context["tables"][0])
-      assert_false('comment' in response.context["tables"][0], response.context["tables"])
-      assert_false('type' in response.context["tables"][0])
+    # Table should have been created
+    response = self.client.get("/metastore/tables/%s?filter=show_tables" % self.db_name)
+    assert_equal(200, response.status_code)
+    assert_equal(len(response.context['tables']), 5)
+    assert_true('name' in response.context["tables"][0])
+    assert_true('comment' in response.context["tables"][0])
+    assert_true('type' in response.context["tables"][0])
 
-      hql = """
+    hql = """
         CREATE INDEX test_index ON TABLE test_show_tables_1 (a) AS 'COMPACT' WITH DEFERRED REBUILD;
       """
-      resp = _make_query(self.client, hql, wait=True, local=False, max=30.0, database=self.db_name)
+    resp = _make_query(self.client, hql, wait=True, local=False, max=30.0, database=self.db_name)
 
-      # By default, index table should not appear in show tables view
-      response = self.client.get("/metastore/tables/%s" % self.db_name)
-      assert_equal(200, response.status_code)
-      assert_false('test_index' in response.context['tables'])
-    finally:
-      for reset in resets:
-        reset()
+    # By default, index table should not appear in show tables view
+    response = self.client.get("/metastore/tables/%s" % self.db_name)
+    assert_equal(200, response.status_code)
+    assert_false('test_index' in response.context['tables'])
 
   def test_describe_view(self):
     resp = self.client.get('/metastore/table/%s/myview' % self.db_name)
-    assert_equal(None, resp.context['sample'])
     assert_true(resp.context['table'].is_view)
     assert_true("View" in resp.content)
     assert_true("Drop View" in resp.content)
@@ -163,6 +156,13 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     assert_true(self.db_name in resp.content)
     assert_true("myview" in resp.content)
 
+  def test_get_sample_data(self):
+    resp = self.client.get("/metastore/table/%s/test_partitions/sample" % self.db_name)
+    json_resp = json.loads(resp.content)
+    assert_equal(0, json_resp['status'], json_resp)
+    assert_true('headers' in json_resp, json_resp)
+    assert_true('rows' in json_resp, json_resp)
+
   def test_describe_partitions(self):
     response = self.client.get("/metastore/table/%s/test_partitions" % self.db_name)
     assert_true("Show Partitions (2)" in response.content, response.content)

+ 1 - 0
apps/metastore/src/metastore/urls.py

@@ -28,6 +28,7 @@ urlpatterns = patterns('metastore.views',
   url(r'^tables/drop/(?P<database>\w+)$', 'drop_table', name='drop_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)$', 'describe_table', name='describe_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/alter$', 'alter_table', name='alter_table'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/sample$', 'get_sample_data', name='get_sample_data'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/metadata$', 'get_table_metadata', name='get_table_metadata'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', 'load_table', name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', 'read_table', name='read_table'),

+ 39 - 40
apps/metastore/src/metastore/views.py

@@ -185,7 +185,6 @@ def show_tables(request, database=None):
       'db_form': db_form,
       'search_filter': search_filter,
       'database': database,
-      'has_metadata': True,
       'table_names': json.dumps(table_names),
       'has_write_access': has_write_access(request.user),
     })
@@ -212,6 +211,24 @@ def get_table_metadata(request, database, table):
   return JsonResponse(response)
 
 
+def get_sample_data(request, database, table):
+  db = dbms.get(request.user)
+  response = {'status': -1, 'error_message': ''}
+  try:
+    table_obj = db.get_table(database, table)
+    sample_data = db.get_sample(database, table_obj)
+    response = {
+      'status': 0,
+      'headers': sample_data and sample_data.cols(),
+      'rows': sample_data and list(sample_data.rows())
+    }
+  except Exception, ex:
+    error_message, logs = dbms.expand_exception(ex, db)
+    response['error_message'] = error_message
+
+  return JsonResponse(response)
+
+
 def describe_table(request, database, table):
   app_name = get_app_name(request)
   query_server = get_query_server_config(app_name)
@@ -226,11 +243,7 @@ def describe_table(request, database, table):
     else:
       raise PopupException(_("Hive Error"), detail=e)
 
-  partitions = None
-  if app_name != 'impala' and table.partition_keys:
-    partitions = db.get_partitions(database, table, partition_spec=None, max_parts=None)
-
-  if request.REQUEST.get("format", "html") == "json" and request.REQUEST.get("sample", "false") == "false":
+  if request.REQUEST.get("format", "html") == "json":
     return JsonResponse({
         'status': 0,
         'name': table.name,
@@ -244,41 +257,27 @@ def describe_table(request, database, table):
         'details': table.details,
         'stats': table.stats
     })
+  else:  # Render HTML
+    renderable = "describe_table.mako"
 
-  renderable = "describe_table.mako"
-  if request.REQUEST.get("sample", "false") == "true":
-    renderable = "sample.mako"
-    if request.REQUEST.get("format", "html") == "json":
-      response = {'status': -1, 'error_message': ''}
-      error_message = ''
-      table_data = ''
-      try:
-        table_data = db.get_sample(database, table)
-        response.update({
-            'status': 0,
-            'headers': table_data and table_data.cols(),
-            'rows': table_data and list(table_data.rows())
-        })
-      except Exception, ex:
-        error_message, logs = dbms.expand_exception(ex, db)
-        response['error_message'] = error_message
-
-      return JsonResponse(response)
-
-  return render(renderable, request, {
-    'breadcrumbs': [{
-        'name': database,
-        'url': reverse('metastore:show_tables', kwargs={'database': database})
-      }, {
-        'name': str(table.name),
-        'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
-      },
-    ],
-    'table': table,
-    'database': database,
-    'has_write_access': has_write_access(request.user),
-    'partitions': partitions
-  })
+    partitions = None
+    if app_name != 'impala' and table.partition_keys:
+      partitions = db.get_partitions(database, table, partition_spec=None, max_parts=None)
+
+    return render(renderable, request, {
+      'breadcrumbs': [{
+          'name': database,
+          'url': reverse('metastore:show_tables', kwargs={'database': database})
+        }, {
+          'name': str(table.name),
+          'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
+        },
+      ],
+      'table': table,
+      'partitions': partitions,
+      'database': database,
+      'has_write_access': has_write_access(request.user),
+    })
 
 
 @check_has_write_access_permission

+ 2 - 5
desktop/core/src/desktop/static/desktop/js/assist/assistHelper.js

@@ -151,11 +151,8 @@
    */
   AssistHelper.prototype.fetchTableSample = function (options) {
     $.ajax({
-      url: "/" + (options.sourceType == "hive" ? "beeswax" : options.sourceType) + "/api/table/" + options.databaseName + "/" + options.tableName,
-      data: {
-        "sample": true,
-        "format" : options.dataType
-      },
+      url: "/" + (options.sourceType == "hive" ? "beeswax" : options.sourceType) + "/api/table/" + options.databaseName + "/" + options.tableName + "/sample",
+      data: {},
       beforeSend: function (xhr) {
         xhr.setRequestHeader("X-Requested-With", "Hue");
       },