Bladeren bron

HUE-8330 [clusters] Add initial Connectors page

Romain Rigaux 6 jaren geleden
bovenliggende
commit
9ac286cd89

+ 19 - 1
desktop/core/src/desktop/conf.py

@@ -686,6 +686,24 @@ METRICS = ConfigSection(
   )
   )
 )
 )
 
 
+
+CONNECTORS = ConfigSection(
+  key='connectors',
+  help=_("""Configuration options for connectors to external services"""),
+  members=dict(
+    IS_ENABLED=Config(
+      key='is_enabled',
+      help=_('Enable connector page'),
+      default=False,
+      type=coerce_bool),
+   LIST=Config(
+      key='list',
+      default=['impala'],
+      type=coerce_csv),
+  )
+)
+
+
 DATABASE = ConfigSection(
 DATABASE = ConfigSection(
   key='database',
   key='database',
   help=_("""Configuration options for specifying the Desktop Database.
   help=_("""Configuration options for specifying the Desktop Database.
@@ -1915,7 +1933,7 @@ def config_validator(user):
   # Validate if oozie email server is active
   # Validate if oozie email server is active
   try:
   try:
     from oozie.views.editor2 import _is_oozie_mail_enabled
     from oozie.views.editor2 import _is_oozie_mail_enabled
-  
+
     if not _is_oozie_mail_enabled(user):
     if not _is_oozie_mail_enabled(user):
       res.append(('OOZIE_EMAIL_SERVER', unicode(_('Email notifications is disabled for Workflows and Jobs as SMTP server is localhost.'))))
       res.append(('OOZIE_EMAIL_SERVER', unicode(_('Email notifications is disabled for Workflows and Jobs as SMTP server is localhost.'))))
   except Exception, e:
   except Exception, e:

+ 9 - 0
desktop/core/src/desktop/js/onePageViewModel.js

@@ -517,6 +517,15 @@ class OnePageViewModel {
           });
           });
         }
         }
       },
       },
+      {
+        url: '/desktop/connectors',
+        app: function() {
+          self.loadApp('connectors');
+          self.getActiveAppViewModel(viewModel => {
+            viewModel.fetchConnectors();
+          });
+        }
+      },
       {
       {
         url: '/desktop/download_logs',
         url: '/desktop/download_logs',
         app: function() {
         app: function() {

+ 10 - 0
desktop/core/src/desktop/lib/connectors/lib/impala.py

@@ -0,0 +1,10 @@
+class ImpalaConnector():
+  NAME = 'impala'
+
+  VERSION = 1
+  APP = 'notebook'
+  INTERFACE = 'hiveserver2'
+  PROPERTIES = {
+    'server_host': '',
+    'server_port': '',
+  }

+ 51 - 0
desktop/core/src/desktop/lib/connectors/models.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from django.urls import reverse, NoReverseMatch
+from django.db import connection, models, transaction
+from django.db.models import Q
+from django.db.models.query import QuerySet
+from django.utils.translation import ugettext as _, ugettext_lazy as _t
+
+
+class ImpalaConnector():
+  NAME = 'impala'
+
+  VERSION = 1
+  APP = 'notebook'
+  INTERFACE = 'hiveserver2'
+  PROPERTIES = {
+    'server_host': '',
+    'server_port': '',
+  }
+
+
+class Connectors(models.Model):
+
+  name = models.CharField(default='', max_length=255)
+  description = models.TextField(default='')
+  uuid = models.CharField(default=uuid_default, max_length=36, db_index=True)
+
+  category = models.CharField(max_length=32, db_index=True, help_text=_t('Type of connector, e.g. query, browser, catalog...'))
+  interface = models.CharField(max_length=32, db_index=True, help_text=_t('Type of connector, e.g. hiveserver2'))
+  type = models.CharField(max_length=32, db_index=True, help_text=_t('Type of connector, e.g. hive-tez, '))
+
+  data = models.TextField(default='{}')
+
+  last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))

+ 24 - 0
desktop/core/src/desktop/lib/connectors/urls.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.conf.urls import url
+
+from desktop.lib.connectors import views
+
+urlpatterns = [
+  url(r'^$', views.index, name='desktop.lib.connectors.views.index'),
+]

+ 48 - 0
desktop/core/src/desktop/lib/connectors/views.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from desktop.lib.django_util import JsonResponse, render
+
+
+LOG = logging.getLogger(__name__)
+
+
+def index(request):
+  CONNECTORS = {
+    "timestamp": "2019-04-05T23:36:47.533981",
+    "metric": [
+      {"category": "Query Engines", "values": [
+        {"name": "Impala"},
+        {"name": "SQL Database"},
+        {"name": "Hive"}
+      ]},
+      {"category": "Browsers", "values": [{"name": "HDFS"}, {"name": "S3"}, {"name": "ADLS"}]},
+      {"category": "Catalogs", "values": [{"name": "Navigator"}, {"name": "Atlas"}]},
+      {"category": "Optimizers", "values": [{"name": "Optimizer"}]},
+      {"category": "Schedulers", "values": [{"name": "Oozie"}, {"name": "Celery"}]},
+      {"category": "Apps", "values": []},
+      {"category": "Plugins", "values": []},
+    ]
+  }
+
+  if request.is_ajax():
+    return JsonResponse(CONNECTORS)
+  else:
+    return render("connectors.mako", request, {'connectors': json.dumps(CONNECTORS), 'is_embeddable': request.GET.get('is_embeddable', False)})

+ 11 - 3
desktop/core/src/desktop/models.py

@@ -1564,7 +1564,7 @@ class ClusterConfig():
   """
   """
   Configuration of the apps and engines that each individual user sees on the core Hue.
   Configuration of the apps and engines that each individual user sees on the core Hue.
   Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
   Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
-  
+
   TODO: rename to HueConfig
   TODO: rename to HueConfig
   TODO: get list of contexts dynamically
   TODO: get list of contexts dynamically
   """
   """
@@ -1584,6 +1584,12 @@ class ClusterConfig():
     editors = app_config.get('editor')
     editors = app_config.get('editor')
     main_button_action = self.get_main_quick_action(app_config)
     main_button_action = self.get_main_quick_action(app_config)
 
 
+    # Actually same references in editors
+    interpreters = [interpreter for interpreter in editors['interpreters'] if not interpreter['is_catalog']]
+    interpreter_names = [interpreter['type'] for interpreter in interpreters]
+    editors['interpreters'] = interpreters
+    editors['interpreter_names'] = interpreter_names
+
     if main_button_action.get('is_sql'):
     if main_button_action.get('is_sql'):
       default_sql_interpreter = main_button_action['type']
       default_sql_interpreter = main_button_action['type']
     else:
     else:
@@ -1670,7 +1676,8 @@ class ClusterConfig():
         'buttonName': _('Query'),
         'buttonName': _('Query'),
         'tooltip': _('%s Query') % interpreter['type'].title(),
         'tooltip': _('%s Query') % interpreter['type'].title(),
         'page': '/editor/?type=%(type)s' % interpreter,
         'page': '/editor/?type=%(type)s' % interpreter,
-        'is_sql': interpreter['is_sql']
+        'is_sql': interpreter['is_sql'],
+        'is_catalog': interpreter['is_catalog']
       })
       })
 
 
     if SHOW_NOTEBOOKS.get() and ANALYTIC_DB not in self.cluster_type:
     if SHOW_NOTEBOOKS.get() and ANALYTIC_DB not in self.cluster_type:
@@ -1685,7 +1692,8 @@ class ClusterConfig():
         'buttonName': _('Notebook'),
         'buttonName': _('Notebook'),
         'tooltip': _('Notebook'),
         'tooltip': _('Notebook'),
         'page': '/notebook',
         'page': '/notebook',
-        'is_sql': False
+        'is_sql': False,
+        'is_catalog': False
       })
       })
 
 
     if interpreters:
     if interpreters:

+ 9 - 0
desktop/core/src/desktop/templates/about_layout.mako

@@ -16,7 +16,9 @@
 
 
 <%!
 <%!
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
+
 from desktop.auth.backend import is_admin
 from desktop.auth.backend import is_admin
+from desktop.conf import METRICS, CONNECTORS
 
 
 def is_selected(section, matcher):
 def is_selected(section, matcher):
   if section == matcher:
   if section == matcher:
@@ -44,15 +46,22 @@ def is_selected(section, matcher):
                 <li class="${is_selected(section, 'dump_config')}">
                 <li class="${is_selected(section, 'dump_config')}">
                   <a href="${ url('desktop.views.dump_config') }">${_('Configuration')}</a>
                   <a href="${ url('desktop.views.dump_config') }">${_('Configuration')}</a>
                 </li>
                 </li>
+                % if CONNECTORS.IS_ENABLED.get():
+                <li class="${is_selected(section, 'connectors')}">
+                  <a href="${ url('desktop.lib.connectors.views.index') }">${_('Connectors')}</a>
+                </li>
+                % endif
                 <li class="${is_selected(section, 'log_view')}">
                 <li class="${is_selected(section, 'log_view')}">
                   <a href="${ url('desktop.views.log_view') }">${_('Server Logs')}</a>
                   <a href="${ url('desktop.views.log_view') }">${_('Server Logs')}</a>
                 </li>
                 </li>
                 <li class="${is_selected(section, 'threads')}">
                 <li class="${is_selected(section, 'threads')}">
                   <a href="${ url('desktop.views.threads') }">${_('Threads')}</a>
                   <a href="${ url('desktop.views.threads') }">${_('Threads')}</a>
                 </li>
                 </li>
+                % if METRICS.ENABLE_WEB_METRICS.get():
                 <li class="${is_selected(section, 'metrics')}">
                 <li class="${is_selected(section, 'metrics')}">
                   <a href="${ url('desktop.lib.metrics.views.index') }">${_('Metrics')}</a>
                   <a href="${ url('desktop.lib.metrics.views.index') }">${_('Metrics')}</a>
                 </li>
                 </li>
+                % endif
               % endif
               % endif
             </ul>
             </ul>
           </div>
           </div>

+ 157 - 0
desktop/core/src/desktop/templates/connectors.mako

@@ -0,0 +1,157 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+<%!
+from django.utils.translation import ugettext as _
+
+from desktop import conf
+from desktop.views import commonheader, commonfooter
+%>
+
+<%
+MAIN_SCROLLABLE = is_embeddable and "'.page-content'" or "window"
+if conf.CUSTOM.BANNER_TOP_HTML.get():
+  TOP_SNAP = is_embeddable and "78px" or "106px"
+else:
+  TOP_SNAP = is_embeddable and "50px" or "106px"
+%>
+
+<%namespace name="actionbar" file="actionbar.mako" />
+<%namespace name="layout" file="about_layout.mako" />
+
+%if not is_embeddable:
+${ commonheader(_('Connectors'), "about", user, request) | n,unicode }
+%endif
+
+
+<script type="text/javascript">
+  (function () {
+    var ConnectorsViewModel = function () {
+      var self = this;
+
+      self.apiHelper = window.apiHelper;
+      self.metrics = ko.observableArray();
+      self.selectedMetric = ko.observable('All');
+      self.metricsFilter = ko.observable();
+
+      self.selectedMetrics = ko.pureComputed(function () {
+        return self.metrics().filter(function (metric) {
+          return self.selectedMetric() == 'All' || metric.category == self.selectedMetric();
+        });
+      });
+      self.filteredMetrics = ko.pureComputed(function () {
+        var metrics = self.selectedMetrics();
+
+        if (self.metricsFilter()) {
+          var lowerQuery = self.metricsFilter().toLowerCase();
+          var filteredMetrics = []
+          metrics.forEach(function (metric) {
+            var _metric = {"category": metric.category, "values": []};
+            _metric.values = metric.values.filter(function (subMetricKey) {
+              return subMetricKey.name.toLowerCase().indexOf(lowerQuery) !== -1;
+            });
+            if (_metric.values.length > 0){
+              filteredMetrics.push(_metric);
+            }
+          });
+          metrics = filteredMetrics;
+        }
+
+        return metrics;
+      });
+
+      self.fetchConnectors = function () {
+        self.apiHelper.simpleGet('/desktop/connectors/', {}, {successCallback: function (data) {
+          self.metrics(data.metric);
+        }});
+      };
+    }
+
+    $(document).ready(function () {
+      var viewModel = new ConnectorsViewModel();
+      ko.applyBindings(viewModel, $('#connectorsComponents')[0]);
+    });
+  })();
+</script>
+
+${layout.menubar(section='connectors')}
+
+<div id="connectorsComponents" class="container-fluid">
+  <div class="card card-small margin-top-10">
+    <div data-bind="dockable: { scrollable: ${ MAIN_SCROLLABLE }, jumpCorrection: 0,topSnap: '${ TOP_SNAP }', triggerAdjust: ${ is_embeddable and "0" or "106" }}">
+      <ul class="nav nav-pills">
+        <li data-bind="css: { 'active': $root.selectedMetric() === 'All' }">
+          <a href="javascript:void(0)" data-bind="text: 'All', click: function(){ $root.selectedMetric('All') }"></a>
+        </li>
+        <!-- ko foreach: metrics() -->
+        <li data-bind="css: { 'active': $root.selectedMetric() === $data.category }">
+          <a href="javascript:void(0)" data-bind="text: $data.category, click: function(){ $root.selectedMetric($data.category) }"></a>
+        </li>
+        <!-- /ko -->
+      </ul>
+      <input type="text" data-bind="clearable: metricsFilter, valueUpdate: 'afterkeydown'"
+          class="input-xlarge pull-right margin-bottom-10" placeholder="${ _('Filter metrics...') }">
+    </div>
+
+    <div class="margin-top-10">
+      <div data-bind="foreach: filteredMetrics()">
+        <h4 data-bind="text: category"></h4>
+        <table class="table table-condensed">
+          <thead>
+            <tr>
+              <th width="30%">${ _('Name') }</th>
+              <th>${ _('Value') }</th>
+            </tr>
+          </thead>
+          <!-- ko if: $data.values -->
+          <tbody data-bind="foreach: values">
+            <tr>
+              <td data-bind="text: name"></td>
+              <td data-bind="text: ''"></td>
+            </tr>
+          </tbody>
+          <!-- /ko -->
+          <!-- ko ifnot: $data.values -->
+          <tfoot>
+            <tr>
+              <td colspan="2">${ _('There are no metrics matching your filter') }</td>
+            </tr>
+          </tfoot>
+          <!-- /ko -->
+          </table>
+      </div>
+
+      <!-- ko if: filteredMetrics().length == 0 -->
+      <table class="table table-condensed">
+        <thead>
+          <tr>
+            <th width="30%">${ _('Name') }</th>
+            <th>${ _('Value') }</th>
+          </tr>
+        </thead>
+        <tfoot>
+          <tr>
+            <td colspan="2">${ _('There are no metrics matching your filter') }</td>
+          </tr>
+        </tfoot>
+      </table>
+      <!-- /ko -->
+  </div>
+</div>
+
+
+% if not is_embeddable:
+ ${ commonfooter(request, messages) | n,unicode }
+% endif

+ 5 - 4
desktop/core/src/desktop/templates/dump_config.mako

@@ -25,10 +25,11 @@ LOG = logging.getLogger(__name__)
 %>
 %>
 
 
 <%namespace name="layout" file="about_layout.mako" />
 <%namespace name="layout" file="about_layout.mako" />
-%if not is_embeddable:
-${ commonheader(_('Configuration'), "about", user, request, "70px") | n,unicode }
-%endif
-${ layout.menubar(section='dump_config') }
+
+% if not is_embeddable:
+  ${ commonheader(_('Configuration'), "about", user, request, "70px") | n,unicode }
+% endif
+  ${ layout.menubar(section='dump_config') }
 
 
 <style type="text/css">
 <style type="text/css">
   .card-heading .pull-right {
   .card-heading .pull-right {

+ 2 - 0
desktop/core/src/desktop/templates/hue.mako

@@ -417,6 +417,7 @@ ${ hueIcons.symbols() }
       <div id="embeddable_dump_config" class="embeddable"></div>
       <div id="embeddable_dump_config" class="embeddable"></div>
       <div id="embeddable_threads" class="embeddable"></div>
       <div id="embeddable_threads" class="embeddable"></div>
       <div id="embeddable_metrics" class="embeddable"></div>
       <div id="embeddable_metrics" class="embeddable"></div>
+      <div id="embeddable_connectors" class="embeddable"></div>
       <div id="embeddable_403" class="embeddable"></div>
       <div id="embeddable_403" class="embeddable"></div>
       <div id="embeddable_404" class="embeddable"></div>
       <div id="embeddable_404" class="embeddable"></div>
       <div id="embeddable_500" class="embeddable"></div>
       <div id="embeddable_500" class="embeddable"></div>
@@ -562,6 +563,7 @@ ${ smart_unicode(login_modal(request).content) | n,unicode }
     dump_config: { url: '/desktop/dump_config', title: '${_('Dump Configuration')}' },
     dump_config: { url: '/desktop/dump_config', title: '${_('Dump Configuration')}' },
     threads: { url: '/desktop/debug/threads', title: '${_('Threads')}' },
     threads: { url: '/desktop/debug/threads', title: '${_('Threads')}' },
     metrics: { url: '/desktop/metrics', title: '${_('Metrics')}' },
     metrics: { url: '/desktop/metrics', title: '${_('Metrics')}' },
+    connectors: { url: '/desktop/connectors', title: '${_('Connectors')}' },
     sqoop: { url: '/sqoop', title: '${_('Sqoop')}' },
     sqoop: { url: '/sqoop', title: '${_('Sqoop')}' },
     jobsub: { url: '/jobsub/not_available', title: '${_('Job Designer')}' },
     jobsub: { url: '/jobsub/not_available', title: '${_('Job Designer')}' },
     % if other_apps:
     % if other_apps:

+ 6 - 1
desktop/core/src/desktop/urls.py

@@ -36,7 +36,7 @@ from django.contrib import admin
 from django.views.static import serve
 from django.views.static import serve
 
 
 from desktop import appmanager
 from desktop import appmanager
-from desktop.conf import METRICS, USE_NEW_EDITOR, ENABLE_DJANGO_DEBUG_TOOL
+from desktop.conf import METRICS, USE_NEW_EDITOR, ENABLE_DJANGO_DEBUG_TOOL, CONNECTORS
 
 
 from desktop.auth import views as desktop_auth_views
 from desktop.auth import views as desktop_auth_views
 from desktop.settings import is_oidc_configured
 from desktop.settings import is_oidc_configured
@@ -193,6 +193,11 @@ if METRICS.ENABLE_WEB_METRICS.get():
     url(r'^desktop/metrics/', include('desktop.lib.metrics.urls'))
     url(r'^desktop/metrics/', include('desktop.lib.metrics.urls'))
   ]
   ]
 
 
+if CONNECTORS.IS_ENABLED.get():
+  dynamic_patterns += [
+    url(r'^desktop/connectors/', include('desktop.lib.connectors.urls'))
+  ]
+
 dynamic_patterns += [
 dynamic_patterns += [
   url(r'^admin/', include(admin.site.urls)),
   url(r'^admin/', include(admin.site.urls)),
 ]
 ]

+ 2 - 1
desktop/libs/notebook/src/notebook/conf.py

@@ -69,7 +69,8 @@ def get_ordered_interpreters(user=None):
       "type": i,
       "type": i,
       "interface": interpreters[i].INTERFACE.get(),
       "interface": interpreters[i].INTERFACE.get(),
       "options": interpreters[i].OPTIONS.get(),
       "options": interpreters[i].OPTIONS.get(),
-      "is_sql" : interpreters[i].INTERFACE.get() in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy"]
+      "is_sql": interpreters[i].INTERFACE.get() in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "hms"],
+      "is_catalog": interpreters[i].INTERFACE.get() in ["hms",]
     }
     }
     for i in reordered_interpreters
     for i in reordered_interpreters
   ]
   ]