فهرست منبع

[HUE-2939] Use SHOW DATABASES for more performant fetch databases, and lazily-fetch metadata

Jenny Kim 10 سال پیش
والد
کامیت
f154e94

+ 28 - 4
apps/beeswax/src/beeswax/server/dbms.py

@@ -125,8 +125,29 @@ class HiveServer2Dbms(object):
     self.server_name = self.client.query_server['server_name']
 
 
-  def get_databases(self):
-    return self.client.get_databases()
+  @classmethod
+  def to_matching_wildcard(cls, identifier=None):
+    cleaned = "*"
+    if identifier and identifier.strip() != "*":
+      cleaned = "*%s*" % identifier.strip().strip("*")
+    return cleaned
+
+
+  def get_databases(self, database_names='*'):
+    identifier = self.to_matching_wildcard(database_names)
+
+    hql = "SHOW DATABASES LIKE '%s'" % (identifier) # self.client.get_databases() is too slow
+    query = hql_query(hql)
+    timeout = SERVER_CONN_TIMEOUT.get()
+
+    handle = self.execute_and_wait(query, timeout_sec=timeout)
+
+    if handle:
+      result = self.fetch(handle, rows=5000)
+      self.close(handle)
+      return [name for database in result.rows() for name in database]
+    else:
+      return []
 
 
   def get_database(self, database):
@@ -134,11 +155,14 @@ class HiveServer2Dbms(object):
 
 
   def get_tables_meta(self, database='default', table_names='*'):
-    return self.client.get_tables_meta(database, table_names)
+    identifier = self.to_matching_wildcard(table_names)
+    return self.client.get_tables_meta(database, identifier)
 
 
   def get_tables(self, database='default', table_names='*'):
-    hql = "SHOW TABLES IN `%s` '%s'" % (database, table_names) # self.client.get_tables(database, table_names) is too slow
+    identifier = self.to_matching_wildcard(table_names)
+
+    hql = "SHOW TABLES IN `%s` '%s'" % (database, identifier) # self.client.get_tables(database, table_names) is too slow
     query = hql_query(hql)
     timeout = SERVER_CONN_TIMEOUT.get()
 

+ 12 - 2
apps/beeswax/src/beeswax/tests.py

@@ -1551,7 +1551,7 @@ for x in sys.stdin:
       resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
       resp = wait_for_query_to_finish(self.client, resp, max=180.0)
       resp = self.client.get("/metastore/databases/")
-      assert_true(db_name in resp.context["database_names"], resp)
+      assert_true(db_name in resp.context["databases"], resp)
 
       # Test for accented characters in 'comment'
       resp = self.client.post("/beeswax/create/database", {
@@ -1563,7 +1563,7 @@ for x in sys.stdin:
       resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
       resp = wait_for_query_to_finish(self.client, resp, max=180.0)
       resp = self.client.get("/metastore/databases/")
-      assert_true(db_name_accent in resp.context['database_names'], resp)
+      assert_true(db_name_accent in resp.context['databases'], resp)
     finally:
       make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db_name}, wait=True)
       make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db_name_accent}, wait=True)
@@ -2842,3 +2842,13 @@ def test_ssl_validate():
     finally:
       for reset in resets:
         reset()
+
+
+def test_to_matching_wildcard():
+    match_fn = dbms.HiveServer2Dbms.to_matching_wildcard
+
+    assert_equal(match_fn(None), '*')
+    assert_equal(match_fn(''), '*')
+    assert_equal(match_fn('*'), '*')
+    assert_equal(match_fn('test'), '*test*')
+    assert_equal(match_fn('test*'), '*test*')

+ 12 - 0
apps/metastore/src/metastore/conf.py

@@ -14,3 +14,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from django.utils.translation import ugettext_lazy as _
+
+from desktop.lib.conf import Config
+
+
+HS2_GET_TABLES_MAX = Config(
+  key="hs2_get_tables_max",
+  help=_("The max number of records in the result set permitted to do a HS2 GetTables call."),
+  type=int,
+  default=1000
+)

+ 31 - 11
apps/metastore/src/metastore/templates/databases.mako

@@ -43,7 +43,9 @@ ${ components.menubar() }
         <h1 class="card-heading simple">${ components.breadcrumbs(breadcrumbs) }</h1>
         <%actionbar:render>
           <%def name="search()">
-            <input id="filterInput" type="text" class="input-xlarge search-query" placeholder="${_('Search for database name')}">
+            <form id="searchQueryForm" action="${ url('metastore:databases') }" method="GET" class="inline">
+              <input id="filterInput" type="text" name="filter" class="input-xlarge search-query" value="${ search_filter }" placeholder="${_('Search for database name')}" />
+            </form>
           </%def>
 
           <%def name="actions()">
@@ -58,7 +60,6 @@ ${ components.menubar() }
             <tr>
               <th width="1%"><div class="hueCheckbox selectAll fa" data-selectables="databaseCheck"></div></th>
               <th>${_('Database Name')}</th>
-              <th>${_('Comment')}</th>
             </tr>
           </thead>
           <tbody>
@@ -66,14 +67,13 @@ ${ components.menubar() }
             <tr>
               <td data-row-selector-exclude="true" width="1%">
                 <div class="hueCheckbox databaseCheck fa"
-                   data-view-url="${ url('metastore:show_tables', database=database['db_name']) }"
-                   data-drop-name="${ database['db_name'] }"
+                   data-view-url="${ url('metastore:show_tables', database=database) }"
+                   data-drop-name="${ database }"
                    data-row-selector-exclude="true"></div>
               </td>
               <td>
-                <a href="${ url('metastore:show_tables', database=database['db_name']) }" data-row-selector="true">${ database['db_name'] }</a>
+                <a class="databaseLink" href="${ url('metastore:show_tables', database=database) }" data-row-selector="true">${ database }</a>
               </td>
-              <td>${ smart_unicode(database['comment']) }</td>
             </tr>
           % endfor
           </tbody>
@@ -107,7 +107,7 @@ ${ components.menubar() }
 <script type="text/javascript" charset="utf-8">
   $(document).ready(function () {
     var viewModel = {
-      availableDatabases: ko.observableArray(${ database_names | n,unicode }),
+      availableDatabases: ko.observableArray(${ databases_json | n,unicode }),
       chosenDatabases: ko.observableArray([])
     };
 
@@ -118,11 +118,9 @@ ${ components.menubar() }
       "bPaginate": false,
       "bLengthChange": false,
       "bInfo": false,
-      "bFilter": true,
       "aoColumns": [
         {"bSortable": false, "sWidth": "1%" },
         null,
-        null
       ],
       "oLanguage": {
         "sEmptyTable": "${_('No data available')}",
@@ -130,10 +128,17 @@ ${ components.menubar() }
       }
     });
 
-    $("#filterInput").keyup(function () {
-      databases.fnFilter($(this).val());
+    var _searchInputValue = $("#filterInput").val();
+
+    $("#filterInput").jHueDelayedInput(function(){
+      if ($("#filterInput").val() != _searchInputValue){
+        $("#searchQueryForm").submit();
+      }
     });
 
+    $("#filterInput").focus();
+    $("#filterInput").val(_searchInputValue); // set caret at the end of the field
+
     $("a[data-row-selector='true']").jHueRowSelector();
 
     $(".selectAll").click(function () {
@@ -159,6 +164,21 @@ ${ components.menubar() }
       toggleActions();
     });
 
+    $(".databaseLink").mouseover(function() {
+      var _link = $(this);
+      $.ajax({
+        type: "GET",
+        url: "/metastore/databases/" + $(this).text() + "/metadata",
+        dataType: "json",
+        data: {},
+        success: function (response) {
+          if (response && response.status == 0) {
+            _link.attr("title", response.data.comment).tooltip("show");
+          }
+        },
+      });
+    });
+
     function toggleActions() {
       $(".toolbarBtn").attr("disabled", "disabled");
       var selector = $(".hueCheckbox[checked='checked']");

+ 34 - 5
apps/metastore/src/metastore/templates/tables.mako

@@ -53,9 +53,10 @@ ${ components.menubar() }
         <h1 class="card-heading simple">${ components.breadcrumbs(breadcrumbs) }</h1>
         <%actionbar:render>
           <%def name="search()">
-            <input id="filterInput" type="text" class="input-xlarge search-query" placeholder="${_('Search for table name')}">
+            <form id="searchQueryForm" action="${ url('metastore:show_tables') }" method="GET" class="inline">
+              <input id="filterInput" type="text" name="filter" class="input-xlarge search-query" value="${ search_filter }" placeholder="${_('Search for table name')}" />
+            </form>
           </%def>
-
           <%def name="actions()">
             <button id="viewBtn" class="btn toolbarBtn" title="${_('Browse the selected table')}" disabled="disabled"><i class="fa fa-eye"></i> ${_('View')}</button>
             <button id="browseBtn" class="btn toolbarBtn" title="${_('Browse the selected table')}" disabled="disabled"><i class="fa fa-list"></i> ${_('Browse Data')}</button>
@@ -71,8 +72,10 @@ ${ components.menubar() }
                 <th width="1%"><div class="hueCheckbox selectAll fa" data-selectables="tableCheck"></div></th>
                 <th>&nbsp;</th>
                 <th>${_('Table Name')}</th>
+                % if has_metadata:
                 <th>${_('Comment')}</th>
                 <th>${_('Type')}</th>
+                % endif
               </tr>
             </thead>
             <tbody>
@@ -87,10 +90,12 @@ ${ components.menubar() }
                 </td>
                 <td class="row-selector-exclude"><a href="javascript:void(0)" data-table="${ table['name'] }"><i class="fa fa-bar-chart" title="${ _('View statistics') }"></i></a></td>
                 <td>
-                  <a href="${ url('metastore:describe_table', database=database, table=table['name']) }" data-row-selector="true">${ table['name'] }</a>
+                  <a class="tableLink" href="${ url('metastore:describe_table', database=database, table=table['name']) }" data-row-selector="true">${ table['name'] }</a>
                 </td>
+                % if has_metadata:
                 <td>${ smart_unicode(table['comment']) }</td>
                 <td>${ smart_unicode(table['type']) }</td>
+                % endif
               </tr>
             % endfor
             </tbody>
@@ -160,8 +165,10 @@ ${ components.menubar() }
         {"bSortable": false, "sWidth": "1%" },
         {"bSortable": false, "sWidth": "1%" },
         null,
+        % if has_metadata:
         null,
         null
+        % endif
       ],
       "oLanguage": {
         "sEmptyTable": "${_('No data available')}",
@@ -169,10 +176,17 @@ ${ components.menubar() }
       }
     });
 
-    $("#filterInput").keyup(function () {
-      tables.fnFilter($(this).val());
+    var _searchInputValue = $("#filterInput").val();
+
+    $("#filterInput").jHueDelayedInput(function(){
+      if ($("#filterInput").val() != _searchInputValue){
+        $("#searchQueryForm").submit();
+      }
     });
 
+    $("#filterInput").focus();
+    $("#filterInput").val(_searchInputValue); // set caret at the end of the field
+
     $("a[data-row-selector='true']").jHueRowSelector();
 
     $("a[data-table]").on("click", function () {
@@ -213,6 +227,21 @@ ${ components.menubar() }
       toggleActions();
     });
 
+      $(".tableLink").mouseover(function() {
+      var _link = $(this);
+      $.ajax({
+        type: "GET",
+        url: "/metastore/table/${database}/" + $(this).text() + "/metadata",
+        dataType: "json",
+        data: {},
+        success: function (response) {
+          if (response && response.status == 0) {
+            _link.attr("title", response.data.comment).tooltip("show");
+          }
+        },
+      });
+    });
+
     $(".tableCheck").click(function () {
       if ($(this).attr("checked")) {
         $(this).removeClass("fa-check").removeAttr("checked");

+ 42 - 2
apps/metastore/src/metastore/tests.py

@@ -30,6 +30,7 @@ from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod
 from desktop.lib.test_utils import add_permission, grant_access
 from hadoop.pseudo_hdfs4 import is_live_cluster
 from metastore import parser
+from metastore.conf import HS2_GET_TABLES_MAX
 from useradmin.models import HuePermission, GroupPermission, group_has_permission
 
 from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
@@ -75,8 +76,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
 
     # Default database should exist
     response = self.client.get("/metastore/databases")
-    assert_true('db_name' in response.context["databases"][0])
-    assert_true(self.db_name in response.context["database_names"])
+    assert_true(self.db_name in response.context["databases"])
 
     # Table should have been created
     response = self.client.get("/metastore/tables/")
@@ -109,6 +109,46 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
     # This should NOT go into the query history.
     assert_equal(verify_history(self.client, fragment='test'), history_cnt, 'Implicit queries should not be saved in the history')
 
+  def test_show_tables(self):
+    if is_live_cluster():
+      raise SkipTest('HUE-2902: Test is not re-entrant')
+
+    # Set max limit to 3
+    HS2_GET_TABLES_MAX.set_for_testing(3)
+
+    hql = """
+      CREATE TABLE test_show_tables_1 (a int) COMMENT 'Test for show_tables';
+      CREATE TABLE test_show_tables_2 (a int) COMMENT 'Test for show_tables';
+      CREATE TABLE test_show_tables_3 (a int) COMMENT 'Test for show_tables';
+    """
+    resp = _make_query(self.client, hql, database=self.db_name)
+    resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+
+    # Table should have been created
+    response = self.client.get("/metastore/tables/%s?filter=show_tables" % self.db_name)
+    assert_equal(200, response.status_code)
+    assert_equal(len(response.context['tables']), 3)
+    assert_equal(response.context['has_metadata'], True)
+    assert_true('name' in response.context["tables"][0])
+    assert_true('comment' in response.context["tables"][0])
+    assert_true('type' in response.context["tables"][0])
+
+    hql = """
+      CREATE TABLE test_show_tables_4 (a int) COMMENT 'Test for show_tables';
+      CREATE TABLE test_show_tables_5 (a int) COMMENT 'Test for show_tables';
+    """
+    resp = _make_query(self.client, hql, database=self.db_name)
+    resp = wait_for_query_to_finish(self.client, resp, max=30.0)
+
+    # Table should have been created
+    response = self.client.get("/metastore/tables/%s?filter=show_tables" % self.db_name)
+    assert_equal(200, response.status_code)
+    assert_equal(len(response.context['tables']), 5)
+    assert_equal(response.context['has_metadata'], False)
+    assert_true('name' in response.context["tables"][0])
+    assert_false('comment' in response.context["tables"][0], response.context["tables"])
+    assert_false('type' in response.context["tables"][0])
+
   def test_describe_view(self):
     if is_live_cluster():
       raise SkipTest('HUE-2902: Test is not re-entrant')

+ 2 - 0
apps/metastore/src/metastore/urls.py

@@ -22,10 +22,12 @@ urlpatterns = patterns('metastore.views',
 
   url(r'^databases/?$', 'databases', name='databases'),
   url(r'^databases/drop/?$', 'drop_database', name='drop_database'),
+  url(r'^databases/(?P<database>\w+)/metadata$', 'get_database_metadata', name='get_database_metadata'),
 
   url(r'^tables/(?P<database>\w+)?$', 'show_tables', name='show_tables'),
   url(r'^tables/drop/(?P<database>\w+)$', 'drop_table', name='drop_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)$', 'describe_table', name='describe_table'),
+  url(r'^table/(?P<database>\w+)/(?P<table>\w+)/metadata$', 'get_table_metadata', name='get_table_metadata'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/load$', 'load_table', name='load_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/read$', 'read_table', name='read_table'),
   url(r'^table/(?P<database>\w+)/(?P<table>\w+)/partitions$', 'describe_partitions', name='describe_partitions'),

+ 53 - 11
apps/metastore/src/metastore/views.py

@@ -33,6 +33,7 @@ from beeswax.models import SavedQuery, MetaInstall
 from beeswax.server import dbms
 from beeswax.server.dbms import get_query_server_config
 from filebrowser.views import location_to_url
+from metastore.conf import HS2_GET_TABLES_MAX
 from metastore.forms import LoadDataForm, DbForm
 from metastore.settings import DJANGO_APPS
 
@@ -40,7 +41,7 @@ from metastore.settings import DJANGO_APPS
 LOG = logging.getLogger(__name__)
 
 SAVE_RESULTS_CTAS_TIMEOUT = 300         # seconds
-
+HS2_GET_TABLES_MAX = HS2_GET_TABLES_MAX.get()
 
 def check_has_write_access_permission(view_func):
   """
@@ -63,18 +64,15 @@ Database Views
 """
 
 def databases(request):
-  db = dbms.get(request.user)
-  databases = []
-  database_names = db.get_databases()
+  search_filter = request.GET.get('filter', '')
 
-  for database in database_names:
-    db_metadata = db.get_database(database)
-    databases.append(db_metadata)
+  db = dbms.get(request.user)
+  databases = db.get_databases(search_filter)
 
   return render("databases.mako", request, {
     'breadcrumbs': [],
+    'search_filter': search_filter,
     'databases': databases,
-    'database_names': json.dumps(database_names),
     'databases_json': json.dumps(databases),
     'has_write_access': has_write_access(request.user),
   })
@@ -102,6 +100,20 @@ def drop_database(request):
     return render('confirm.mako', request, {'url': request.path, 'title': title})
 
 
+def get_database_metadata(request, database):
+  db = dbms.get(request.user)
+  response = {'status': -1, 'data': ''}
+  try:
+    db_metadata = db.get_database(database)
+    response['status'] = 0
+    response['data'] = db_metadata
+  except Exception, ex:
+    response['status'] = 1
+    response['data'] = _("Cannot get metadata for database: %s") % (database,)
+
+  return JsonResponse(response)
+
+
 """
 Table Views
 """
@@ -125,8 +137,19 @@ def show_tables(request, database=None):
     else:
       db_form = DbForm(initial={'database': database}, databases=databases)
 
-    tables = db.get_tables_meta(database=database)
-    table_names = [table['name'] for table in tables]
+    search_filter = request.GET.get('filter', '')
+
+    table_names = db.get_tables(database=database, table_names=search_filter)
+    tables = [{'name': table} for table in table_names]
+    has_metadata = False
+
+    if len(table_names) <= HS2_GET_TABLES_MAX:  # Only attempt to do a GetTables HS2 call for small result sets
+      try:
+        tables = db.get_tables_meta(database=database, table_names=search_filter)
+        table_names = [table['name'] for table in tables]
+        has_metadata = True
+      except Exception, ex:
+        LOG.exception('Unable to fetch table metadata')
   except Exception, e:
     raise PopupException(_('Failed to retrieve tables for database: %s' % database), detail=e)
 
@@ -139,15 +162,34 @@ def show_tables(request, database=None):
     ],
     'tables': tables,
     'db_form': db_form,
+    'search_filter': search_filter,
     'database': database,
+    'has_metadata': has_metadata,
     'table_names': json.dumps(table_names),
-    'tables_json': json.dumps(tables),
     'has_write_access': has_write_access(request.user),
   })
   resp.set_cookie("hueBeeswaxLastDatabase", database, expires=90)
   return resp
 
 
+def get_table_metadata(request, database, table):
+  db = dbms.get(request.user)
+  response = {'status': -1, 'data': ''}
+  try:
+    table_metadata = db.get_table(database, table)
+    response['status'] = 0
+    response['data'] = {
+      'comment': table_metadata.comment,
+      'hdfs_link': table_metadata.hdfs_link,
+      'is_view': table_metadata.is_view
+    }
+  except Exception, ex:
+    response['status'] = 1
+    response['data'] = _("Cannot get metadata for table: `%s`.`%s`") % (database, table)
+
+  return JsonResponse(response)
+
+
 def describe_table(request, database, table):
   app_name = get_app_name(request)
   query_server = get_query_server_config(app_name)

+ 0 - 1
desktop/core/src/desktop/conf.py

@@ -998,7 +998,6 @@ DJANGO_EMAIL_BACKEND = Config(
   default="django.core.mail.backends.smtp.EmailBackend"
 )
 
-
 def validate_ldap(user, config):
   res = []