瀏覽代碼

[HUE-2750] [metastore] Show comments in the database and table views

NOTE: Impala does not currently support DESCRIBE on databases: https://issues.cloudera.org/browse/IMPALA-2196

This only includes the changes to the metastore browser to enable comments to appear for databases and tables views. Additional columns that we could show for the database include: location, owner_name, owner_type, parameters. For tables, we can also show hdfs_link, path_location, properties, etc.

In order to enable beeswax assist to access this metadata, we can update the response in the autocomplete API for databases and tables (which currently only return names), but since this is a bigger change I'll wait for FE to determine if and how they would surface this on UI.
Jenny Kim 10 年之前
父節點
當前提交
23cca8f

+ 8 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -129,6 +129,14 @@ class HiveServer2Dbms(object):
     return self.client.get_databases()
 
 
+  def get_database(self, database):
+    return self.client.get_database(database)
+
+
+  def get_tables_meta(self, database='default', table_names='*'):
+    return self.client.get_tables_meta(database, table_names)
+
+
   def get_tables(self, database='default', table_names='*'):
     hql = "SHOW TABLES IN `%s` '%s'" % (database, table_names) # self.client.get_tables(database, table_names) is too slow
     query = hql_query(hql)

+ 46 - 3
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -593,6 +593,36 @@ class HiveServerClient:
     return HiveServerTRowSet(results.results, schema.schema).cols((col,))
 
 
+  def get_database(self, database):
+    if self.query_server['server_name'] == 'impala':
+      raise NotImplementedError(_("Impala has not implemented the 'DESCRIBE DATABASE' command: %(issue_ref)s") % {
+        'issue_ref': "https://issues.cloudera.org/browse/IMPALA-2196"
+      })
+
+    query = 'DESCRIBE DATABASE EXTENDED `%s`' % (database)
+
+    (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
+    self.close_operation(operation_handle)
+
+    cols = ('db_name', 'comment', 'location')
+
+    if len(HiveServerTRowSet(desc_results.results, desc_schema.schema).cols(cols)) != 1:
+      raise ValueError(_("%(query)s returned more than 1 row") % {'query': query})
+
+    return HiveServerTRowSet(desc_results.results, desc_schema.schema).cols(cols)[0]  # Should only contain one row
+
+
+  def get_tables_meta(self, database, table_names):
+    req = TGetTablesReq(schemaName=database, tableName=table_names)
+    res = self.call(self._client.GetTables, req)
+
+    results, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=5000)
+    self.close_operation(res.operationHandle)
+
+    cols = ('TABLE_NAME', 'TABLE_TYPE', 'REMARKS')
+    return HiveServerTRowSet(results.results, schema.schema).cols(cols)
+
+
   def get_tables(self, database, table_names):
     req = TGetTablesReq(schemaName=database, tableName=table_names)
     res = self.call(self._client.GetTables, req)
@@ -956,6 +986,22 @@ class HiveServerClientCompatible(object):
     return [table[col] for table in self._client.get_databases()]
 
 
+  def get_database(self, database):
+    return self._client.get_database(database)
+
+
+  def get_tables_meta(self, database, table_names):
+    tables = self._client.get_tables_meta(database, table_names)
+    massaged_tables = []
+    for table in tables:
+      massaged_tables.append({
+        'name': table['TABLE_NAME'],
+        'comment': table['REMARKS'],
+        'type': table['TABLE_TYPE'].capitalize()}
+      )
+    return massaged_tables
+
+
   def get_tables(self, database, table_names):
     tables = [table['TABLE_NAME'] for table in self._client.get_tables(database, table_names)]
     tables.sort()
@@ -983,9 +1029,6 @@ class HiveServerClientCompatible(object):
   def create_database(self, name, description): raise NotImplementedError()
 
 
-  def get_database(self, *args, **kwargs): raise NotImplementedError()
-
-
   def alter_table(self, dbname, tbl_name, new_tbl): raise NotImplementedError()
 
 

+ 2 - 2
apps/beeswax/src/beeswax/tests.py

@@ -1552,7 +1552,7 @@ for x in sys.stdin:
       resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
       resp = wait_for_query_to_finish(self.client, resp, max=180.0)
       resp = self.client.get("/metastore/databases/")
-      assert_true(db_name in resp.context['databases'], resp)
+      assert_true(db_name in resp.context["database_names"], resp)
 
       # Test for accented characters in 'comment'
       resp = self.client.post("/beeswax/create/database", {
@@ -1564,7 +1564,7 @@ for x in sys.stdin:
       resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
       resp = wait_for_query_to_finish(self.client, resp, max=180.0)
       resp = self.client.get("/metastore/databases/")
-      assert_true(db_name_accent in resp.context['databases'], resp)
+      assert_true(db_name_accent in resp.context['database_names'], resp)
     finally:
       make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db_name}, wait=True)
       make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db_name_accent}, wait=True)

+ 8 - 4
apps/metastore/src/metastore/templates/databases.mako

@@ -14,6 +14,7 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 <%!
+from desktop.lib.i18n import smart_unicode
 from desktop.views import commonheader, commonfooter
 from django.utils.translation import ugettext as _
 %>
@@ -57,6 +58,7 @@ ${ components.menubar() }
             <tr>
               <th width="1%"><div class="hueCheckbox selectAll fa" data-selectables="databaseCheck"></div></th>
               <th>${_('Database Name')}</th>
+              <th>${_('Comment')}</th>
             </tr>
           </thead>
           <tbody>
@@ -64,13 +66,14 @@ ${ components.menubar() }
             <tr>
               <td data-row-selector-exclude="true" width="1%">
                 <div class="hueCheckbox databaseCheck fa"
-                   data-view-url="${ url('metastore:show_tables', database=database) }"
-                   data-drop-name="${ database }"
+                   data-view-url="${ url('metastore:show_tables', database=database['db_name']) }"
+                   data-drop-name="${ database['db_name'] }"
                    data-row-selector-exclude="true"></div>
               </td>
               <td>
-                <a href="${ url('metastore:show_tables', database=database) }" data-row-selector="true">${ database }</a>
+                <a href="${ url('metastore:show_tables', database=database['db_name']) }" data-row-selector="true">${ database['db_name'] }</a>
               </td>
+              <td>${ smart_unicode(database['comment']) }</td>
             </tr>
           % endfor
           </tbody>
@@ -104,7 +107,7 @@ ${ components.menubar() }
 <script type="text/javascript" charset="utf-8">
   $(document).ready(function () {
     var viewModel = {
-      availableDatabases: ko.observableArray(${ databases_json | n,unicode }),
+      availableDatabases: ko.observableArray(${ database_names | n,unicode }),
       chosenDatabases: ko.observableArray([])
     };
 
@@ -118,6 +121,7 @@ ${ components.menubar() }
       "bFilter": true,
       "aoColumns": [
         {"bSortable": false, "sWidth": "1%" },
+        null,
         null
       ],
       "oLanguage": {

+ 13 - 6
apps/metastore/src/metastore/templates/tables.mako

@@ -14,6 +14,7 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 <%!
+from desktop.lib.i18n import smart_unicode
 from desktop.views import commonheader, commonfooter
 from django.utils.translation import ugettext as _
 %>
@@ -70,6 +71,8 @@ ${ components.menubar() }
                 <th width="1%"><div class="hueCheckbox selectAll fa" data-selectables="tableCheck"></div></th>
                 <th>&nbsp;</th>
                 <th>${_('Table Name')}</th>
+                <th>${_('Comment')}</th>
+                <th>${_('Type')}</th>
               </tr>
             </thead>
             <tbody>
@@ -77,15 +80,17 @@ ${ components.menubar() }
               <tr>
                 <td data-row-selector-exclude="true" width="1%">
                   <div class="hueCheckbox tableCheck fa"
-                       data-view-url="${ url('metastore:describe_table', database=database, table=table) }"
-                       data-browse-url="${ url('metastore:read_table', database=database, table=table) }"
-                       data-drop-name="${ table }"
+                       data-view-url="${ url('metastore:describe_table', database=database, table=table['name']) }"
+                       data-browse-url="${ url('metastore:read_table', database=database, table=table['name']) }"
+                       data-drop-name="${ table['name'] }"
                        data-row-selector-exclude="true"></div>
                 </td>
-                <td class="row-selector-exclude"><a href="javascript:void(0)" data-table="${ table }"><i class="fa fa-bar-chart" title="${ _('View statistics') }"></i></a></td>
+                <td class="row-selector-exclude"><a href="javascript:void(0)" data-table="${ table['name'] }"><i class="fa fa-bar-chart" title="${ _('View statistics') }"></i></a></td>
                 <td>
-                  <a href="${ url('metastore:describe_table', database=database, table=table) }" data-row-selector="true">${ table }</a>
+                  <a href="${ url('metastore:describe_table', database=database, table=table['name']) }" data-row-selector="true">${ table['name'] }</a>
                 </td>
+                <td>${ smart_unicode(table['comment']) }</td>
+                <td>${ smart_unicode(table['type']) }</td>
               </tr>
             % endfor
             </tbody>
@@ -139,7 +144,7 @@ ${ components.menubar() }
 
   $(document).ready(function () {
     var viewModel = {
-      availableTables: ko.observableArray(${ tables_json | n }),
+      availableTables: ko.observableArray(${ table_names | n }),
       chosenTables: ko.observableArray([])
     };
 
@@ -154,6 +159,8 @@ ${ components.menubar() }
       "aoColumns": [
         {"bSortable": false, "sWidth": "1%" },
         {"bSortable": false, "sWidth": "1%" },
+        null,
+        null,
         null
       ],
       "oLanguage": {

+ 4 - 2
apps/metastore/src/metastore/tests.py

@@ -74,7 +74,8 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
 
     # Default database should exist
     response = self.client.get("/metastore/databases")
-    assert_true(self.db_name in response.context["databases"])
+    assert_true('db_name' in response.context["databases"][0])
+    assert_true(self.db_name in response.context["database_names"])
 
     # Table should have been created
     response = self.client.get("/metastore/tables/")
@@ -82,7 +83,8 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
 
     # Switch databases
     response = self.client.get("/metastore/tables/%s" % self.db_name)
-    assert_true("test" in response.context["tables"])
+    assert_true('name' in response.context["tables"][0])
+    assert_true("test" in response.context["table_names"])
 
     # Should default to "default" database
     response = self.client.get("/metastore/tables/not_there")

+ 10 - 2
apps/metastore/src/metastore/views.py

@@ -64,11 +64,17 @@ Database Views
 
 def databases(request):
   db = dbms.get(request.user)
-  databases = db.get_databases()
+  databases = []
+  database_names = db.get_databases()
+
+  for database in database_names:
+    db_metadata = db.get_database(database)
+    databases.append(db_metadata)
 
   return render("databases.mako", request, {
     'breadcrumbs': [],
     'databases': databases,
+    'database_names': json.dumps(database_names),
     'databases_json': json.dumps(databases),
     'has_write_access': has_write_access(request.user),
   })
@@ -119,7 +125,8 @@ def show_tables(request, database=None):
     else:
       db_form = DbForm(initial={'database': database}, databases=databases)
 
-    tables = db.get_tables(database=database)
+    tables = db.get_tables_meta(database=database)
+    table_names = [table['name'] for table in tables]
   except Exception, e:
     raise PopupException(_('Failed to retrieve tables for database: %s' % database), detail=e)
 
@@ -133,6 +140,7 @@ def show_tables(request, database=None):
     'tables': tables,
     'db_form': db_form,
     'database': database,
+    'table_names': json.dumps(table_names),
     'tables_json': json.dumps(tables),
     'has_write_access': has_write_access(request.user),
   })