Browse Source

HUE-8155 [metastore] Add a relationships tab to the table browser page

Johan Ahlen 7 years ago
parent
commit
bff9c3ec31

File diff suppressed because it is too large
+ 0 - 0
apps/metastore/src/metastore/static/metastore/css/metastore.css


+ 89 - 2
apps/metastore/src/metastore/static/metastore/js/metastore.model.js

@@ -369,6 +369,8 @@ var MetastoreTable = (function () {
 
 
     self.optimizerStats = ko.observable();
     self.optimizerStats = ko.observable();
     self.optimizerDetails = ko.observable();
     self.optimizerDetails = ko.observable();
+    self.topJoins = ko.observable();
+    self.foreignJoinCols = ko.observable();
     self.navigatorMeta = ko.observable();
     self.navigatorMeta = ko.observable();
     self.relationshipsDetails = ko.observable();
     self.relationshipsDetails = ko.observable();
 
 
@@ -379,6 +381,7 @@ var MetastoreTable = (function () {
     self.loadingQueries = ko.observable(false);
     self.loadingQueries = ko.observable(false);
     self.loadingComment = ko.observable(false);
     self.loadingComment = ko.observable(false);
     self.loadingViewSql = ko.observable(false);
     self.loadingViewSql = ko.observable(false);
+    self.loadingTopJoins = ko.observable(false);
 
 
     self.columns = ko.observableArray();
     self.columns = ko.observableArray();
 
 
@@ -396,7 +399,7 @@ var MetastoreTable = (function () {
 
 
     self.refreshing = ko.pureComputed(function () {
     self.refreshing = ko.pureComputed(function () {
       return self.loadingDetails() || self.loadingColumns() || self.loadingQueries() || self.loadingComment() ||
       return self.loadingDetails() || self.loadingColumns() || self.loadingQueries() || self.loadingComment() ||
-        self.samples.loading() || self.partitions.loading() || self.loadingViewSql();
+        self.samples.loading() || self.partitions.loading() || self.loadingViewSql() || self.loadingTopJoins();
     });
     });
 
 
     self.partitionsCountLabel = ko.pureComputed(function () {
     self.partitionsCountLabel = ko.pureComputed(function () {
@@ -492,6 +495,90 @@ var MetastoreTable = (function () {
         self.loadingViewSql(true);
         self.loadingViewSql(true);
       }
       }
 
 
+      self.catalogEntry.getTopJoins().done(function (topJoins) {
+        if (topJoins && topJoins.values) {
+          var joins = [];
+          var foreignJoinCols = {};
+          var ownQidLower = self.catalogEntry.path.join('.').toLowerCase();
+          var ownNameLower = self.catalogEntry.name.toLowerCase();
+          var ownDbNameLower = self.database.catalogEntry.name.toLowerCase();
+
+          var joinIndex = {};
+          var joinColsIndex = {};
+          topJoins.values.forEach(function (topJoin) {
+            if (topJoin.tables.length === 2) {
+              topJoin.tables.forEach(function (table) {
+                var tableLower = table.toLowerCase();
+                if (tableLower !== ownQidLower && tableLower !== ownNameLower) {
+                  var name = tableLower.indexOf(ownDbNameLower + '.') === 0 ? table.substring(ownDbNameLower.length + 1) : table;
+                  if (!joinIndex[name]) {
+                    joinIndex[name] = {
+                      tableName: name,
+                      tablePath: table.split('.'),
+                      joinCols: [],
+                      queryCount: 0
+                    }
+                  }
+                  var join = joinIndex[name];
+                  join.queryCount += topJoin.totalQueryCount;
+
+                  topJoin.joinCols.forEach(function (joinCol) {
+                    var cleanCols = {
+                      queryCount: topJoin.totalQueryCount
+                    };
+                    if (joinCol.columns.length === 2) {
+                      joinCol.columns.forEach(function (col) {
+                        var colLower = col.toLowerCase();
+                        if (colLower.indexOf(ownQidLower + '.') === 0) {
+                          cleanCols.source = colLower.substring(ownQidLower.length + 1);
+                          cleanCols.sourcePath = col.split('.');
+                        } else if (colLower.indexOf(ownNameLower + '.') === 0) {
+                          cleanCols.source = colLower.substring(ownNameLower.length + 1);
+                          cleanCols.sourcePath = col.split('.');
+                          cleanCols.sourcePath.unshift(ownDbNameLower);
+                        } else if (colLower.indexOf(ownDbNameLower + '.') === 0) {
+                          cleanCols.target = colLower.substring(ownDbNameLower.length + 1);
+                          cleanCols.targetPath = col.split('.');
+                        } else {
+                          cleanCols.target = col;
+                          cleanCols.targetPath = col.split('.');
+                        }
+                      })
+                    }
+                    if (cleanCols.source && cleanCols.target) {
+                      if (joinColsIndex[ownQidLower + join.tableName + cleanCols.source + cleanCols.target]) {
+                        joinColsIndex[ownQidLower + join.tableName + cleanCols.source + cleanCols.target].queryCount += topJoin.totalQueryCount;
+                      } else {
+                        joinColsIndex[ownQidLower + join.tableName + cleanCols.source + cleanCols.target] = cleanCols;
+                        join.joinCols.push(cleanCols);
+                        foreignJoinCols[cleanCols.source] = { target: cleanCols.target, path: cleanCols.targetPath };
+                      }
+                    }
+                  })
+                }
+              });
+            }
+          });
+
+          Object.keys(joinIndex).forEach(function (key) {
+            var join = joinIndex[key];
+            if (join.joinCols.length) {
+              join.joinCols.sort(function (a, b) {
+                return b.queryCount - a.queryCount;
+              });
+              joins.push(join);
+            }
+          });
+          joins.sort(function (a, b) {
+            return b.queryCount - a.queryCount;
+          });
+          self.foreignJoinCols(foreignJoinCols);
+          self.topJoins(joins);
+        }
+      }).always(function () {
+        self.loadingTopJoins(false);
+      });
+
       self.loadingDetails(true);
       self.loadingDetails(true);
       self.catalogEntry.getAnalysis().done(function (analysis) {
       self.catalogEntry.getAnalysis().done(function (analysis) {
         self.tableDetails(analysis);
         self.tableDetails(analysis);
@@ -513,7 +600,7 @@ var MetastoreTable = (function () {
               self.viewSql(property.data_type)
               self.viewSql(property.data_type)
             }).always(function () {
             }).always(function () {
               self.loadingViewSql(false);
               self.loadingViewSql(false);
-            })
+            });
             return true;
             return true;
           }
           }
         });
         });

+ 13 - 0
apps/metastore/src/metastore/static/metastore/less/metastore.less

@@ -339,4 +339,17 @@
   .metastore-table {
   .metastore-table {
     margin-bottom: 5px;
     margin-bottom: 5px;
   }
   }
+
+  .metastore-join-column-table {
+    tr {
+      td {
+        border: none !important;
+        padding: 0;
+      }
+
+      td.metastore-join-arrow {
+        padding: 0 15px;
+      }
+    }
+  }
 }
 }

+ 37 - 90
apps/metastore/src/metastore/templates/metastore.mako

@@ -845,90 +845,41 @@ ${ components.menubar(is_embeddable) }
   <!-- /ko -->
   <!-- /ko -->
 </script>
 </script>
 
 
-<script type="text/html" id="metastore-joins-tab">
-  <table data-bind="visible: $data.length > 0" class="table table-condensed">
+<script type="text/html" id="metastore-relationships-tab">
+  <!-- ko hueSpinner: { spin: loadingTopJoins, inline: true } --><!-- /ko -->
+  <table data-bind="visible: !loadingTopJoins()" class="table table-condensed">
     <thead>
     <thead>
     <tr>
     <tr>
-      <th width="10%">${ _('Id') }</th>
-      <th width="10%">${ _('Popularity') }</th>
-      <th width="30%">${ _('Table Name') }</th>
-      <th>${ _('Join Column') }</th>
-      <th width="10%">${ _('Join counts') }</th>
+      <th>${ _('Table') }</th>
+      <th>${ _('Foreign keys') }</th>
     </tr>
     </tr>
     </thead>
     </thead>
-    <tbody data-bind="hueach: {data: $data, itemHeight: 32, scrollable: '${ MAIN_SCROLLABLE }', scrollableOffset: 200}">
+    <tbody>
+    <!-- ko if: topJoins().length === 0 -->
+    <tr>
+      <td colspan="2" style="font-style: italic;">${ _('No related tables found.') }</td>
+    </tr>
+    <!-- /ko -->
+    <!-- ko foreach: topJoins -->
     <tr>
     <tr>
-      <td data-bind="text: tableEid"></td>
-      <td style="height: 10px; width: 70px; margin-top:5px;" data-bind="attr: { 'title': joinpercent() }">
-        <div class="progress bar" style="background-color: #0B7FAD" data-bind="style: { 'width' : joinpercent() + '%' }"></div>
+      <td><a href="javscript:void(0);" data-bind="text: tableName, sqlContextPopover: { sourceType: $parents[1].catalogEntry.getSourceType(), path: tablePath, offset: { top: -3, left: 3 }}"></a></td>
+      <td>
+        <table class="metastore-join-column-table">
+          <tbody data-bind="foreach: joinCols">
+          <tr>
+            <td><a href="javscript:void(0);" data-bind="text: source, sqlContextPopover: { sourceType: $parents[2].catalogEntry.getSourceType(), path: sourcePath, offset: { top: -3, left: 3 }}"></a></td>
+            <td class="metastore-join-arrow"><i class="fa fa-long-arrow-right"></i></td>
+            <td><a href="javscript:void(0);" data-bind="text: target, sqlContextPopover: { sourceType: $parents[2].catalogEntry.getSourceType(), path: targetPath, offset: { top: -3, left: 3 }}"></a></td>
+          </tr>
+          </tbody>
+        </table>
       </td>
       </td>
-      <td><a data-bind="text: tableName, attr: { href: '/metastore/table/' + $root.database().catalogEntry.name + '/' + tableName() }"></a></td>
-      <td class="pointer"><code data-bind="text: joinColumns, click: $root.scrollToColumn"></code></td>
-      <td data-bind="text: numJoins"></td>
     </tr>
     </tr>
+    <!-- /ko -->
     </tbody>
     </tbody>
   </table>
   </table>
 </script>
 </script>
 
 
-<script type="text/html" id="metastore-relationships-tab">
-  <h4>${ _('Inputs') }</h4>
-  <div class="row-fluid">
-             <!-- ko foreach: inputs -->
-               <div data-bind="text: $data"></div>
-    <!-- /ko -->
-    <!-- ko if: inputs().length == 0 -->
-    ${ _('Not inputs') }
-    <!-- /ko -->
-          </div>
-
-  <br/>
-
-  <h4>${ _('Targets') }</h4>
-  <div class="row-fluid">
-           <!-- ko foreach: targets -->
-             <div data-bind="text: $data"></div>
-    <!-- /ko -->
-    <!-- ko if: targets().length == 0 -->
-    ${ _('Not targets') }
-    <!-- /ko -->
-          </div>
-
-  <br/>
-
-  <h4>${ _('Source query') }</h4>
-  <div class="row-fluid">
-            <!-- ko if: source_query().length > 0 -->
-              <code data-bind="text: source_query"></code>
-    <!-- /ko -->
-    <!-- ko if: source_query().length == 0 -->
-    ${ _('No source query') }
-    <!-- /ko -->
-          </div>
-
-  <br/>
-
-  <h4>${ _('Target queries') }</h4>
-  <div class="row-fluid">
-            <!-- ko foreach: target_queries -->
-              <div>
-                <code data-bind="text: $data"></code>
-              </div>
-    <!-- /ko -->
-    <!-- ko if: target_queries().length == 0 -->
-    ${ _('Not target queries') }
-    <!-- /ko -->
-          </div>
-
-  <br/>
-
-  <h4>${ _('Lineage') }</h4>
-  <div class="row-fluid">
-    <button class="btn toolbarBtn" title="${_('Open in Navigator ')}" data-bind="click: function () { window.open($root.navigatorUrl() + '?view=detailsView&id=' + id() + '&b=rFlCX&tab=lineage', '_blank'); }">
-      <i class="fa fa-skyatlas"></i> ${_('View in Navigator')}
-    </button>
-  </div>
-</script>
-
 <script type="text/html" id="metastore-describe-table">
 <script type="text/html" id="metastore-describe-table">
   <div class="clearfix"></div>
   <div class="clearfix"></div>
   <!-- ko template: 'metastore-main-description' --><!-- /ko -->
   <!-- ko template: 'metastore-main-description' --><!-- /ko -->
@@ -939,24 +890,24 @@ ${ components.menubar(is_embeddable) }
 
 
   <ul class="nav nav-tabs nav-tabs-border margin-top-10">
   <ul class="nav nav-tabs nav-tabs-border margin-top-10">
     <li data-bind="css: { 'active': $root.currentTab() === 'overview' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('overview'); }">${_('Overview')}</a></li>
     <li data-bind="css: { 'active': $root.currentTab() === 'overview' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('overview'); }">${_('Overview')}</a></li>
-    <!-- ko if: tableDetails() && tableDetails().partition_keys.length -->
-      <li data-bind="css: { 'active': $root.currentTab() === 'partitions' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('partitions'); }">${_('Partitions')} (<span data-bind="text: partitionsCountLabel"></span>)</a></li>
-    <!-- /ko -->
-    <li data-bind="css: { 'active': $root.currentTab() === 'sample' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('sample'); }">${_('Sample')} (<span data-bind="text: samples.rows().length"></span>)</a></li>
-##     <!-- ko if: $root.optimizerEnabled() -->
+    <!-- ko if: $root.optimizerEnabled() -->
+      <li data-bind="css: { 'active': $root.currentTab() === 'relationships' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('relationships'); }">${_('Relationships')}</a></li>
 ##       <!-- ko if: $root.database().table().optimizerDetails() -->
 ##       <!-- ko if: $root.database().table().optimizerDetails() -->
 ##       <li data-bind="css: { 'active': $root.currentTab() === 'permissions' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('permissions'); }">${_('Permissions')}</a></li>
 ##       <li data-bind="css: { 'active': $root.currentTab() === 'permissions' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('permissions'); }">${_('Permissions')}</a></li>
 ##       <li data-bind="css: { 'active': $root.currentTab() === 'queries' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('queries'); }">${_('Queries')} (<span data-bind="text: $root.database().table().optimizerDetails().queryCount"></span>)</a></li>
 ##       <li data-bind="css: { 'active': $root.currentTab() === 'queries' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('queries'); }">${_('Queries')} (<span data-bind="text: $root.database().table().optimizerDetails().queryCount"></span>)</a></li>
 ##       <li data-bind="css: { 'active': $root.currentTab() === 'joins' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('joins'); }">${_('Joins')} (<span data-bind="text: $root.database().table().optimizerDetails().joinCount"></span>)</a></li>
 ##       <li data-bind="css: { 'active': $root.currentTab() === 'joins' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('joins'); }">${_('Joins')} (<span data-bind="text: $root.database().table().optimizerDetails().joinCount"></span>)</a></li>
 ##       <!-- /ko -->
 ##       <!-- /ko -->
 ##       <!-- ko if: $root.database().table().relationshipsDetails() -->
 ##       <!-- ko if: $root.database().table().relationshipsDetails() -->
-##       <li data-bind="css: { 'active': $root.currentTab() === 'relationships' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('relationships'); }">${_('Relationships')} (<span data-bind="text: $root.database().table().relationshipsDetails().inputs().length + $root.database().table().relationshipsDetails().targets().length"></span>)</a></li>
 ##       <!-- /ko -->
 ##       <!-- /ko -->
-##     <!-- /ko -->
+    <!-- /ko -->
+    <!-- ko if: tableDetails() && tableDetails().partition_keys.length -->
+      <li data-bind="css: { 'active': $root.currentTab() === 'partitions' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('partitions'); }">${_('Partitions')} (<span data-bind="text: partitionsCountLabel"></span>)</a></li>
+    <!-- /ko -->
+    <li data-bind="css: { 'active': $root.currentTab() === 'sample' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('sample'); }">${_('Sample')} (<span data-bind="text: samples.rows().length"></span>)</a></li>
     <!-- ko if: catalogEntry.isView() -->
     <!-- ko if: catalogEntry.isView() -->
-    <li data-bind="css: { 'active' : $root.currentTab() === 'viewSql' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('viewSql'); }">${ _('View SQL') }</a></li>
+    <li data-bind="css: { 'active' : $root.currentTab() === 'viewSql' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('viewSql'); }">${ _('View SQL') }</a></li>
     <!-- /ko -->
     <!-- /ko -->
-    <li data-bind="css: { 'active' : $root.currentTab() === 'details' }"><a href="javascript: void(0);" data-bind="click: function(){ $root.currentTab('details'); }">${ _('Details') }</a></li>
+    <li data-bind="css: { 'active' : $root.currentTab() === 'details' }"><a href="javascript: void(0);" data-bind="click: function() { $root.currentTab('details'); }">${ _('Details') }</a></li>
   </ul>
   </ul>
 
 
   <div class="tab-content margin-top-10" style="border: none; overflow: hidden">
   <div class="tab-content margin-top-10" style="border: none; overflow: hidden">
@@ -965,6 +916,10 @@ ${ components.menubar(is_embeddable) }
         <!-- ko template: 'metastore-overview-tab' --><!-- /ko -->
         <!-- ko template: 'metastore-overview-tab' --><!-- /ko -->
       <!-- /ko -->
       <!-- /ko -->
 
 
+      <!-- ko if: $root.currentTab() === 'relationships' -->
+      <!-- ko template: { name: 'metastore-relationships-tab' } --><!-- /ko -->
+      <!-- /ko -->
+
       <!-- ko if: $root.currentTab() === 'partitions' -->
       <!-- ko if: $root.currentTab() === 'partitions' -->
         <!-- ko template: 'metastore-partitions-tab' --><!-- /ko -->
         <!-- ko template: 'metastore-partitions-tab' --><!-- /ko -->
       <!-- /ko -->
       <!-- /ko -->
@@ -981,14 +936,6 @@ ${ components.menubar(is_embeddable) }
         <!-- ko template: { name: 'metastore-queries-tab', data: $root.database().table() } --><!-- /ko -->
         <!-- ko template: { name: 'metastore-queries-tab', data: $root.database().table() } --><!-- /ko -->
       <!-- /ko -->
       <!-- /ko -->
 
 
-      <!-- ko if: $root.optimizerEnabled() && $root.currentTab() === 'joins' -->
-        <!-- ko template: { name: 'metastore-joins-tab', data: $root.database().table().optimizerDetails().joinTables } --><!-- /ko -->
-      <!-- /ko -->
-
-      <!-- ko if: $root.currentTab() === 'relationships' && $root.database().table().relationshipsDetails() -->
-        <!-- ko template: { name: 'metastore-relationships-tab', data: $root.database().table().relationshipsDetails() } --><!-- /ko -->
-      <!-- /ko -->
-
       <!-- ko if: $root.currentTab() === 'viewSql' -->
       <!-- ko if: $root.currentTab() === 'viewSql' -->
         <!-- ko template: 'metastore-view-sql-tab' --><!-- /ko -->
         <!-- ko template: 'metastore-view-sql-tab' --><!-- /ko -->
       <!-- /ko -->
       <!-- /ko -->

Some files were not shown because too many files changed in this diff