Browse Source

HUE-8298 [kafka] Add a basic Topic page

Romain Rigaux 7 years ago
parent
commit
0760d7095d

+ 18 - 7
desktop/core/src/desktop/templates/assist.mako

@@ -808,11 +808,11 @@ from desktop.views import _ko
   <script type="text/html" id="assist-databases-template">
   <script type="text/html" id="assist-databases-template">
     <div class="assist-flex-header" data-bind="visibleOnHover: { selector: '.hover-actions', override: loading() }">
     <div class="assist-flex-header" data-bind="visibleOnHover: { selector: '.hover-actions', override: loading() }">
       <div class="assist-inner-header">
       <div class="assist-inner-header">
-        <!-- ko ifnot: sourceType === 'solr' -->
+        <!-- ko ifnot: sourceType === 'solr' || sourceType === 'kafka' -->
         ${_('Databases')}
         ${_('Databases')}
         <!-- ko template: 'assist-db-header-actions' --><!-- /ko -->
         <!-- ko template: 'assist-db-header-actions' --><!-- /ko -->
         <!-- /ko -->
         <!-- /ko -->
-        <!-- ko if: sourceType === 'solr' -->
+        <!-- ko if: sourceType === 'solr' || sourceType === 'kafka'-->
         ${_('Sources')}
         ${_('Sources')}
         <!-- /ko -->
         <!-- /ko -->
       </div>
       </div>
@@ -854,11 +854,14 @@ from desktop.views import _ko
   <script type="text/html" id="assist-tables-template">
   <script type="text/html" id="assist-tables-template">
     <div class="assist-flex-header">
     <div class="assist-flex-header">
       <div class="assist-inner-header" data-bind="visible: !$parent.loading() && !$parent.hasErrors()">
       <div class="assist-inner-header" data-bind="visible: !$parent.loading() && !$parent.hasErrors()">
-        <!-- ko ifnot: sourceType === 'solr' -->
-        ${_('Tables')}
+        <!-- ko ifnot: sourceType === 'solr' || sourceType === 'kafka' -->
+          ${_('Tables')}
         <!-- /ko -->
         <!-- /ko -->
         <!-- ko if: sourceType === 'solr' -->
         <!-- ko if: sourceType === 'solr' -->
-        <div data-bind="appAwareTemplateContextMenu: { template: 'collection-title-context-items', scrollContainer: '.assist-db-scrollable' }">${_('Indexes')}</div>
+          <div data-bind="appAwareTemplateContextMenu: { template: 'collection-title-context-items', scrollContainer: '.assist-db-scrollable' }">${_('Indexes')}</div>
+        <!-- /ko -->
+        <!-- ko if: sourceType === 'kafka' -->
+          ${_('Topics')}
         <!-- /ko -->
         <!-- /ko -->
         <!-- ko template: 'assist-db-header-actions' --><!-- /ko -->
         <!-- ko template: 'assist-db-header-actions' --><!-- /ko -->
       </div>
       </div>
@@ -1084,6 +1087,11 @@ from desktop.views import _ko
               type: 'solr',
               type: 'solr',
               name: 'solr'
               name: 'solr'
             }];
             }];
+          } else if (options.isKafka) {
+            options.sourceTypes = [{
+              type: 'kafka',
+              name: 'kafka'
+            }];
           } else {
           } else {
             % for interpreter in get_ordered_interpreters(request.user):
             % for interpreter in get_ordered_interpreters(request.user):
               % if interpreter["is_sql"]:
               % if interpreter["is_sql"]:
@@ -1903,7 +1911,7 @@ from desktop.views import _ko
                   panelData: new AssistDbPanel($.extend({
                   panelData: new AssistDbPanel($.extend({
                     apiHelper: self.apiHelper,
                     apiHelper: self.apiHelper,
                     i18n: i18nCollections,
                     i18n: i18nCollections,
-                    isSolr: true
+                    isKafka: true
                   }, params.sql)),
                   }, params.sql)),
                   apiHelper: self.apiHelper,
                   apiHelper: self.apiHelper,
                   name: '${ _("Streams") }',
                   name: '${ _("Streams") }',
@@ -2229,7 +2237,10 @@ from desktop.views import _ko
             <!-- ko if: isSolr -->
             <!-- ko if: isSolr -->
             ${ _('Indexes') }
             ${ _('Indexes') }
             <!-- /ko -->
             <!-- /ko -->
-            <!-- ko ifnot: isSolr  -->
+            <!-- ko if: isKafka -->
+            ${ _('Streams') }
+            <!-- /ko -->
+            <!-- ko ifnot: isSolr || isKafka  -->
             ${ _('Tables') }
             ${ _('Tables') }
             <!-- ko if: statementCount() > 1 -->
             <!-- ko if: statementCount() > 1 -->
             <div class="statement-count">${ _('Statement') } <span data-bind="text: activeStatementIndex() + '/' + statementCount()"></span></div>
             <div class="statement-count">${ _('Statement') } <span data-bind="text: activeStatementIndex() + '/' + statementCount()"></span></div>

+ 2 - 9
desktop/libs/indexer/src/indexer/api3.py

@@ -30,7 +30,6 @@ from desktop.lib.i18n import smart_unicode
 from desktop.models import Document2
 from desktop.models import Document2
 from librdbms.server import dbms as rdbms
 from librdbms.server import dbms as rdbms
 from libsentry.conf import is_enabled
 from libsentry.conf import is_enabled
-from metadata.kafka_client import KafkaApi
 from metadata.manager_client import ManagerApi
 from metadata.manager_client import ManagerApi
 from notebook.connectors.base import get_api, Notebook
 from notebook.connectors.base import get_api, Notebook
 from notebook.decorators import api_error_handler
 from notebook.decorators import api_error_handler
@@ -44,7 +43,7 @@ from indexer.indexers.morphline import MorphlineIndexer
 from indexer.indexers.rdbms import RdbmsIndexer, run_sqoop
 from indexer.indexers.rdbms import RdbmsIndexer, run_sqoop
 from indexer.indexers.sql import SQLIndexer
 from indexer.indexers.sql import SQLIndexer
 from indexer.solr_client import SolrClient, MAX_UPLOAD_SIZE
 from indexer.solr_client import SolrClient, MAX_UPLOAD_SIZE
-from metadata.conf import has_kafka
+from metadata.kafka_api import get_topics
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -122,13 +121,7 @@ def guess_format(request):
   elif file_format['inputFormat'] == 'rdbms':
   elif file_format['inputFormat'] == 'rdbms':
     format_ = RdbmsIndexer(request.user, file_format['rdbmsType']).guess_format()
     format_ = RdbmsIndexer(request.user, file_format['rdbmsType']).guess_format()
   elif file_format['inputFormat'] == 'kafka':
   elif file_format['inputFormat'] == 'kafka':
-    if has_kafka():
-      topics = KafkaApi().topics()
-    else:
-      manager = ManagerApi()
-      broker_host = manager.get_kafka_brokers().split(',')[0].split(':')[0]
-      topics = manager.get_kafka_topics(broker_host).keys()
-    format_ = {'type': 'csv', 'topics': topics}
+    format_ = {'type': 'csv', 'topics': get_topics()}
 
 
   format_['status'] = 0
   format_['status'] = 0
   return JsonResponse(format_)
   return JsonResponse(format_)

+ 770 - 0
desktop/libs/indexer/src/indexer/templates/topics.mako

@@ -0,0 +1,770 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+  from django.utils.translation import ugettext as _
+  from desktop import conf
+
+  from desktop.views import commonheader, commonfooter, commonshare, commonimportexport, _ko
+%>
+
+<%namespace name="assist" file="/assist.mako" />
+<%namespace name="actionbar" file="actionbar.mako" />
+
+<%
+MAIN_SCROLLABLE = is_embeddable and ".page-content" or ".content-panel"
+%>
+
+%if not is_embeddable:
+${ commonheader(_("Streams Browser"), "search", user, request, "60px") | n,unicode }
+<script src="${ static('desktop/ext/js/jquery/plugins/jquery-ui-1.10.4.custom.min.js') }"></script>
+<script src="${ static('desktop/ext/js/jquery/plugins/jquery.mousewheel.min.js') }"></script>
+
+${ assist.assistJSModels() }
+<link rel="stylesheet" href="${ static('notebook/css/notebook-layout.css') }">
+<style type="text/css">
+% if conf.CUSTOM.BANNER_TOP_HTML.get():
+  .show-assist {
+    top: 110px!important;
+  }
+  .main-content {
+    top: 112px!important;
+  }
+% endif
+</style>
+
+${ assist.assistPanel() }
+%endif
+
+<link rel="stylesheet" href="${ static('notebook/css/notebook.css') }" type="text/css">
+<link rel="stylesheet" href="${ static('indexer/css/indexes.css') }" type="text/css">
+<script src="${ static('desktop/js/hue.json.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('dashboard/js/search.ko.js') }" type="text/javascript" charset="utf-8"></script>
+
+<script type="text/html" id="indexes-breadcrumbs">
+  <h1>
+    <!-- ko with: index() -->
+    <div class="inline-block pull-right">
+      <a class="btn btn-default" href="javascript:void(0)" data-bind="hueLink: '/indexer/importer/prefill/all/index/' + name(), tooltip: { placement: 'bottom', delay: 750 }" title="${_('Inport stream data into a table or file')}">
+        <i class="fa fa-download fa-fw"></i> ${_('Consume')}
+      </a>
+
+      <a class="btn btn-default" href="javascript:void(0)" data-toggle="modal" data-bind="click: function() { $('#deleteIndex').modal('show') }, tooltip: { placement: 'bottom', delay: 750 }" title="${_('Delete the index')}">
+        <i class="fa fa-times fa-fw"></i> ${_('Delete')}
+      </a>
+    </div>
+    <!-- /ko -->
+    <ul class="nav nav-pills hue-breadcrumbs-bar">
+      <li>
+        <a href="javascript:void(0);" data-bind="click: showIndexes">${ _('Topics') }
+          <!-- ko if: index -->
+          <span class="divider">&gt;</span>
+          <!-- /ko -->
+        </a>
+      </li>
+      <!-- ko with: index -->
+      <li>
+        <a href="javascript:void(0);" data-bind="text: name"></a>
+      </li>
+      <!-- /ko -->
+    </ul>
+  </h1>
+</script>
+
+<div class="navbar hue-title-bar nokids">
+    <div class="navbar-inner">
+      <div class="container-fluid">
+        <div class="nav-collapse">
+          <ul class="nav">
+            <li class="app-header">
+              <a href="/${app_name}">
+                ${ _('Index Browser') }
+              </a>
+            </li>
+          </ul>
+        </div>
+      </div>
+    </div>
+</div>
+
+<div id="indexesComponents" class="notebook">
+
+  % if not is_embeddable:
+  <a title="${_('Toggle Assist')}" class="pointer show-assist" data-bind="visible: !$root.isLeftPanelVisible() && $root.assistAvailable(), click: function () { $root.isLeftPanelVisible(true); }">
+    <i class="fa fa-chevron-right"></i>
+  </a>
+  % endif
+
+  <div class="main-content">
+    <div class="vertical-full container-fluid" data-bind="style: { 'padding-left' : $root.isLeftPanelVisible() ? '0' : '20px' }">
+      <div class="vertical-full row-fluid panel-container">
+        % if not is_embeddable:
+        <div class="assist-container left-panel" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable()">
+          <a title="${_('Toggle Assist')}" class="pointer hide-assist" data-bind="click: function() { $root.isLeftPanelVisible(false) }">
+            <i class="fa fa-chevron-left"></i>
+          </a>
+          <div class="assist" data-bind="component: {
+              name: 'assist-panel',
+              params: {
+                user: '${user.username}',
+                sql: {
+                  navigationSettings: {
+                    openItem: true,
+                    showStats: true
+                  }
+                },
+                visibleAssistPanels: ['sql']
+              }
+            }"></div>
+        </div>
+        <div class="resizer" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable(), splitDraggable : { appName: 'notebook', leftPanelVisible: $root.isLeftPanelVisible }"><div class="resize-bar">&nbsp;</div></div>
+        % endif
+        <div class="content-panel">
+
+
+          <div class="indexer-main">
+            <!-- ko template: { name: 'indexes-breadcrumbs' }--><!-- /ko -->
+
+            <!-- ko if: section() === 'list-indexes' -->
+            <%actionbar:render>
+              <%def name="search()">
+                <input data-bind="clearable: indexFilter, value: indexFilter, valueUpdate: 'afterkeydown'" type="text" class="input-xlarge search-query" placeholder="${_('Search for name...')}">
+              </%def>
+
+              <%def name="actions()">
+                <div class="btn-toolbar" style="display: inline; vertical-align: middle">
+                  <a data-bind="click: function() { atLeastOneSelected() ? $('#deleteIndexes').modal('show') : void(0) }, css: {'btn': true, 'disabled': ! atLeastOneSelected() }">
+                    <i class="fa fa-times"></i> ${ _('Delete') }
+                  </a>
+                </div>
+              </%def>
+
+              <%def name="creation()">
+                <a href="javascript:void(0)" class="btn" data-bind="hueLink: '/indexer/importer/prefill/all/index/'">
+                  <i class="fa fa-plus-circle"></i> ${ _('Create') }
+                </a>
+              </%def>
+            </%actionbar:render>
+            <!-- /ko -->
+
+            <!-- ko template: { if: section() == 'list-indexes', name: 'list-indexes' }--><!-- /ko -->
+            <!-- ko template: { if: section() == 'list-index', name: 'list-index', data: index() }--><!-- /ko -->
+
+          </div>
+
+          <!-- ko hueSpinner: { spin: isLoading, center: true, size: 'xlarge' } --><!-- /ko -->
+
+
+          <div id="deleteIndexes" class="modal hide fade">
+            <form id="deleteIndexesForm" method="POST" data-bind="submit: deleteIndexes">
+              ${ csrf_token(request) | n,unicode }
+              <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
+                <h2 class="modal-title">${ _('Delete the selection?') }</h2>
+              </div>
+              <div class="modal-body">
+                <ul data-bind="foreach: selectedIndexes">
+                  <li>
+                    <span data-bind="text: name"></span>
+                  </li>
+                </ul>
+              </div>
+              <div class="modal-footer">
+                <a href="#" class="btn" data-dismiss="modal">${ _('No') }</a>
+                <input type="submit" class="btn btn-danger" value="${ _('Yes') }"/>
+              </div>
+            </form>
+          </div>
+
+          <div id="deleteIndex" class="modal hide fade">
+            <form id="deleteIndexForm" method="POST" data-bind="submit: function() { $root.index().delete() }">
+              ${ csrf_token(request) | n,unicode }
+              <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
+                <h2 class="modal-title">${ _('Delete?') }</h2>
+              </div>
+              <div class="modal-body">
+                <!-- ko if: $root.index() -->
+                  <span data-bind="text: $root.index().name"></span>
+                <!-- /ko -->
+              </div>
+              <div class="modal-footer">
+                <a href="#" class="btn" data-dismiss="modal">${ _('No') }</a>
+                <input type="submit" class="btn btn-danger" value="${ _('Yes') }"/>
+              </div>
+            </form>
+          </div>
+
+        </div>
+      </div>
+    </div>
+  </div>
+
+
+</div>
+
+
+<script type="text/html" id="list-indexes">
+  <table class="table table-condensed datatables" id="list-indexes-table">
+    <thead>
+      <tr>
+        <th class="vertical-align-middle" width="1%"><div data-bind="click: selectAll, css: { 'hue-checkbox': true, 'fa': true, 'fa-check': allSelected}" class="select-all"></div></th>
+        <th>${ _('Name') }</th>
+      </tr>
+    </thead>
+    <tbody data-bind="foreach: { data: filteredIndexes }">
+      <tr>
+        <td data-bind="click: $root.handleSelect" class="center" style="cursor: default">
+          <div data-bind="multiCheck: '#list-indexes-table', css: { 'hue-checkbox': true, 'fa': true, 'fa-check': isSelected}"></div>
+        </td>
+        <td><a class="pointer" data-bind="text: name, click: function() { $root.fetchIndex($data); }"></a></td>
+      </tr>
+    </tbody>
+  </table>
+</script>
+
+
+<script type="text/html" id="list-index">
+  <ul class="nav nav-tabs nav-tabs-border">
+    <li class="active"><a href="#index-overview" data-toggle="tab" data-bind="click: function(){ $root.tab('index-overview'); }">${_('Overview')}</a></li>
+    <li><a href="#index-columns" data-toggle="tab" data-bind="click: function(){ $root.tab('index-columns'); }">${_('Partitions')} (<span data-bind="text: fields().length"></span>)</a></li>
+    <li><a href="#index-sample" data-toggle="tab" data-bind="click: function(){ $root.tab('index-sample'); }">${_('Sample')} (<span data-bind="text: sample().length"></span>)</a></li>
+  </ul>
+
+  <div class="tab-content" style="border: none; overflow: hidden">
+    <div class="tab-pane active margin-top-30" id="index-overview">
+      <!-- ko template: { if: $root.tab() == 'index-overview', name: 'indexes-index-overview' }--><!-- /ko -->
+    </div>
+
+    <div class="tab-pane margin-top-10" id="index-columns">
+      <!-- ko if: $root.tab() == 'index-columns' -->
+        <input class="input-xlarge search-query margin-left-10" type="text" placeholder="${ _('Search for a field...') }" data-bind="clearable: $root.fieldFilter, value: $root.fieldFilter, valueUpdate: 'afterkeydown'"/>
+        <div class="margin-top-10">
+        <!-- ko template: 'indexes-index-fields' --><!-- /ko -->
+        </div>
+      <!-- /ko -->
+    </div>
+
+    <div class="tab-pane" id="index-sample">
+      <!-- ko if: sample() && sample().length > 0 -->
+        <!-- ko template: { if: $root.tab() == 'index-sample', name: 'indexes-index-sample', data: sample(), full: true }--><!-- /ko -->
+      <!-- /ko -->
+      <!-- ko if: !sample() || sample().length === 0 -->
+        <div class="margin-top-10 margin-left-10">${ _('The index does not contain any data.')}</div>
+      <!-- /ko -->
+    </div>
+  </div>
+</script>
+
+
+<script type="text/html" id="indexes-index-overview">
+  <div>
+    <!-- ko template: 'indexes-index-properties' --><!-- /ko -->
+
+    <h4>${ _('Partitions') } (<span data-bind="text: fields().length"></span>)</h4>
+    <!-- ko template: { name: 'indexes-index-fields-fields', data: fieldsPreview }--><!-- /ko -->
+    <a class="pointer" data-bind="visible: fields().length > fieldsPreview().length, click: function() { $('li a[href=\'#index-columns\']').click(); }">
+      ${_('View more...')}
+    </a>
+
+    <br><br>
+
+    <h4>${ _('Sample') } (<span data-bind="text: sample().length"></span>)</h4>
+    <!-- ko if: samplePreview() && samplePreview().length > 0 -->
+    <div style="overflow: auto">
+      <!-- ko template: { name: 'indexes-index-sample', data: samplePreview, full: false }--><!-- /ko -->
+    </div>
+    <a class="pointer" data-bind="visible: sample().length > samplePreview().length, click: function() { $('li a[href=\'#index-sample\']').click(); }">
+      ${_('View more...')}
+    </a>
+    <!-- /ko -->
+    <!-- ko if: !samplePreview() || samplePreview().length === 0 -->
+    <div class="margin-top-10 margin-bottom-30">${ _('The index does not contain any data.')}</div>
+    <!-- /ko -->
+  </div>
+</script>
+
+
+<script type="text/html" id="indexes-index-properties">
+  <h4>${ _('Properties') }</h4>
+  <div class="row-fluid">
+    <div title="${ _('isInternal') }">
+      ## <span data-bind="text: isInternal"></span>
+    </div>
+  </div>
+  <br>
+</script>
+
+
+<script type="text/html" id="indexes-index-fields-fields">
+  <div style="overflow: auto">
+    <table class="table table-condensed table-nowrap">
+      <thead>
+        <tr>
+          <th style="width: 1%">&nbsp;</th>
+          <th style="width: 2%"></th>
+          <th>${ _('Name') }</th>
+        </tr>
+      </thead>
+      <tbody data-bind="foreach: $data">
+        <tr>
+          <td data-bind="text: $index() + 1"></td>
+          <td>
+            <i class="fa fa-info muted pointer analysis"></i>
+          </td>
+          <td data-bind="text: ko.mapping.toJSON($data)"></td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+</script>
+
+
+<script type="text/html" id="indexes-index-fields">
+  <div>
+    <!-- ko template: { name: 'indexes-index-fields-fields', data: filteredFields }--><!-- /ko -->
+  </div>
+</script>
+
+
+<script type="text/html" id="indexes-index-sample">
+  <!-- ko hueSpinner: { spin: $root.index().loadingSample, center: true, size: 'xlarge' } --><!-- /ko -->
+
+  <!-- ko ifnot: $root.index().loadingSample -->
+  <!-- ko if: $root.index().fields().length != 0 -->
+  <table class="table table-condensed table-nowrap sample-table old-datatable">
+    <thead>
+      <tr>
+        <th style="width: 1%">&nbsp;</th>
+        <!-- ko foreach: $root.index().fields() -->
+        <th data-bind="text: name"></th>
+        <!-- /ko -->
+      </tr>
+    </thead>
+    <tbody>
+      <!-- ko foreach: $data -->
+        <tr>
+          <td data-bind="text: $index() + 1"></td>
+          <!-- ko foreach: $root.index().fields() -->
+            <td data-bind="text: $parent[name()]"></td>
+          <!-- /ko -->
+        </tr>
+      <!-- /ko -->
+    </tbody>
+  </table>
+  <!-- /ko -->
+
+  ## Schemaless collections
+  <!-- ko if: $root.index().fields().length == 0 && $data.length > 0 -->
+  <table class="table table-condensed table-nowrap sample-table old-datatable">
+    <thead>
+      <tr>
+        <th style="width: 1%">&nbsp;</th>
+        <!-- ko foreach: Object.keys($data[0]) -->
+        <th data-bind="text: $data"></th>
+        <!-- /ko -->
+      </tr>
+    </thead>
+    <tbody>
+      <!-- ko foreach: $data -->
+        <tr>
+          <td data-bind="text: $index() + 1"></td>
+          <!-- ko foreach: Object.keys($parent[0]) -->
+            <td data-bind="text: $parent[$data]"></td>
+          <!-- /ko -->
+        </tr>
+      <!-- /ko -->
+    </tbody>
+  </table>
+  <!-- /ko -->
+
+  <!-- /ko -->
+</script>
+
+
+
+<script type="text/javascript">
+
+  var IndexesViewModel = (function () {
+
+    var Alias = function (vm) {
+      var self = this;
+
+      self.name = ko.observable('');
+      self.chosenCollections = ko.observableArray();
+      self.availableCollections = ko.computed(function () {
+        return vm.indexes();
+      });
+
+      self.create = function () {
+        $.post("${ url('indexer:create_alias') }", {
+          "name": self.name,
+          "collections": ko.mapping.toJSON($.map(self.chosenCollections(), function (collection) {
+            return collection.name();
+          }))
+        }, function (data) {
+          if (data.status == 0) {
+            vm.indexes.push(ko.mapping.fromJS(data.alias));
+            huePubSub.publish('assist.collections.refresh');
+          } else {
+            $(document).trigger("error", data.message);
+          }
+          $('#createAlias').modal('hide');
+        }).fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+        });
+        hueAnalytics.log('indexes', 'create_alias');
+      }
+
+      self.edit = function (alias) {
+        self.name(alias.name());
+        self.chosenCollections($.grep(vm.indexes(), function(collection) { return alias.collections().indexOf(collection.name()) != -1; }));
+        $('#createAlias').modal('show');
+      }
+    };
+
+
+    var Index = function (vm, index) {
+      var self = this;
+
+      self.name = ko.observable(index.name);
+      self.type = ko.observable(index.type);
+      self.fields = ko.mapping.fromJS(index.partitions);
+      self.fieldsPreview = ko.pureComputed(function () {
+        return self.fields().slice(0, 5)
+      });
+      self.filteredFields = ko.computed(function () {
+        var returned = self.fields();
+        if (vm.fieldFilter() !== '') {
+          returned = $.grep(self.fields(), function (field) {
+            return field.name().toLowerCase().indexOf(vm.fieldFilter().toLowerCase()) > -1;
+          });
+        }
+        return returned;
+      });
+
+      self.sample = ko.observableArray();
+      self.samplePreview = ko.pureComputed(function () {
+        return self.sample().slice(0, 5)
+      });
+
+      self.loadingSample = ko.observable(false);
+
+      self.getSample = function () {
+        self.loadingSample(true);
+        $.post("${ url('indexer:sample_index') }", {
+          name: self.name(),
+          rows: 100
+        }, function (data) {
+          if (data.status == 0) {
+            self.sample(data.sample);
+          } else {
+            $(document).trigger("error", data.message);
+          }
+        }).fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+        }).always(function () {
+          self.loadingSample(false);
+        });
+      };
+
+      self.delete = function () {
+        var indexName = self.name();
+        $.post("${ url('indexer:delete_indexes') }", {
+          "indexes": ko.mapping.toJSON([{'name': indexName, 'type': self.type()}])
+        }, function (data) {
+          if (data.status == 0) {
+            vm.indexes.remove(function(index) { return index.name() == indexName; });
+            huePubSub.publish('assist.collections.refresh');
+            vm.showIndexes(false);
+          } else {
+            $(document).trigger("error", data.message);
+          }
+          $('#deleteIndex').modal('hide');
+        }).fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+        });
+        hueAnalytics.log('indexes', 'delete_index');
+      };
+    };
+
+    var IndexesViewModel = function (options) {
+      var self = this;
+
+      self.baseURL = (IS_HUE_4 ? '/hue' : '') + '/indexer/topics/';
+
+      self.assistAvailable = ko.observable(true);
+      self.apiHelper = ApiHelper.getInstance();
+      self.isHue4 = ko.observable(options.hue4);
+      self.isLeftPanelVisible = ko.observable();
+      self.apiHelper.withTotalStorage('assist', 'assist_panel_visible', self.isLeftPanelVisible, true);
+
+      self.section = ko.observable('list-indexes');
+      self.tab = ko.observable('');
+      self.tab.subscribe(function(tab){
+        if (tab === 'index-sample'){
+          var selector = '#index-sample .sample-table';
+          % if conf.CUSTOM.BANNER_TOP_HTML.get():
+            var bannerTopHeight = 30;
+          % else:
+            var bannerTopHeight = 0;
+          % endif
+          if ($(selector).parents('.dataTables_wrapper').length == 0){
+            hueUtils.waitForRendered(selector, function(el){ return el.find('td').length > 0 }, function(){
+              $(selector).dataTable({
+                "bPaginate": false,
+                "bLengthChange": false,
+                "bInfo": false,
+                "bDestroy": true,
+                "bFilter": false,
+                "bAutoWidth": false,
+                "oLanguage": {
+                  "sEmptyTable": "${_('No data available')}",
+                  "sZeroRecords": "${_('No matching records')}"
+                },
+                "fnDrawCallback": function (oSettings) {
+                  $(selector).parents('.dataTables_wrapper').css('overflow-x', 'hidden');
+                  $(selector).jHueTableExtender2({
+                    fixedHeader: true,
+                    fixedFirstColumn: true,
+                    includeNavigator: false,
+                    lockSelectedRow: false,
+                    parentId: 'index-sample',
+                    classToRemove: 'sample-table',
+                    mainScrollable: '${ MAIN_SCROLLABLE }',
+                    % if is_embeddable:
+                      stickToTopPosition: 51 + bannerTopHeight,
+                    % else:
+                      stickToTopPosition: 76 + bannerTopHeight,
+                    % endif
+                    clonedContainerPosition: 'fixed',
+                    app: 'indexes'
+                  });
+                  $(selector).jHueHorizontalScrollbar();
+                },
+                "aoColumnDefs": [
+                  {
+                    "sType": "numeric",
+                    "aTargets": [ "sort-numeric" ]
+                  },
+                  {
+                    "sType": "string",
+                    "aTargets": [ "sort-string" ]
+                  },
+                  {
+                    "sType": "date",
+                    "aTargets": [ "sort-date" ]
+                  }
+                ]
+              });
+            });
+          }
+        }
+      });
+
+      self.indexes = ko.observableArray([]);
+      self.alias = new Alias(self);
+      self.index = ko.observable();
+
+      self.indexFilter = ko.observable('');
+      self.fieldFilter = ko.observable('');
+
+      self.filteredIndexes = ko.computed(function () {
+        var returned = self.indexes();
+        if (self.indexFilter() !== '') {
+          returned = $.grep(self.indexes(), function (idx) {
+            return idx.name().toLowerCase().indexOf(self.indexFilter().toLowerCase()) > -1;
+          });
+        }
+        return returned;
+      });
+
+      self.selectedIndexes = ko.computed(function () {
+        return $.grep(self.indexes(), function (index) {
+          return index.isSelected();
+        });
+      });
+      self.isLoading = ko.observable(false);
+
+      self.oneSelected = ko.computed(function () {
+        return self.selectedIndexes().length == 1;
+      });
+      self.atLeastOneSelected = ko.computed(function () {
+        return self.selectedIndexes().length >= 1;
+      });
+      self.allSelected = ko.observable(false);
+
+      self.handleSelect = function (index) {
+        index.isSelected(!index.isSelected());
+      }
+
+      self.selectAll = function () {
+        self.allSelected(!self.allSelected());
+        ko.utils.arrayForEach(self.indexes(), function (index) {
+          index.isSelected(self.allSelected());
+        });
+      }
+
+      self.datatable = null;
+
+      self.showIndexes = function (reload) {
+        self.section('list-indexes');
+        self.index(null);
+        hueUtils.changeURL(self.baseURL);
+        if (typeof reload == 'undefined' || reload) {
+          self.fetchIndexes();
+        }
+      }
+
+      self.fetchIndexes = function (callback) {
+        self.isLoading(true);
+        $.post("${ url('metadata:list_topics') }", {}, function (data) {
+          if (data.status == 0) {
+            var indexes = []
+            data.topics.forEach(function (index) {
+              index.isSelected = false;
+              indexes.push(ko.mapping.fromJS(index));
+            });
+            self.indexes(indexes);
+            if (callback) {
+              callback();
+            }
+          } else {
+            $(document).trigger("error", data.message);
+          }
+        }).fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+        }).always(function () {
+          self.isLoading(false);
+        });
+        hueAnalytics.log('indexes', 'list_indexes');
+      };
+
+      self.getIndexByName = function (name) {
+        var found = null;
+        self.indexes().forEach(function (idx) {
+          if (idx.name() === name) {
+            found = idx;
+          }
+        });
+        return found;
+      }
+
+      self.fetchIndex = function (index) {
+        $.post("${ url('metadata:list_topic') }", {
+          name: index.name()
+        }, function (data) {
+          if (data.status == 0) {
+            self.index(new Index(self, data.topic));
+            //self.index().getSample();
+            hueUtils.changeURL(self.baseURL + self.index().name());
+            self.section('list-index');
+            self.tab('index-overview');
+          } else {
+            $(document).trigger("error", data.message);
+          }
+        }).fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+        });
+        hueAnalytics.log('indexes', 'list_index');
+      };
+
+      self.deleteIndexes = function () {
+        $.post("${ url('indexer:delete_indexes') }", {
+          "indexes": ko.mapping.toJSON(self.selectedIndexes)
+        }, function (data) {
+          if (data.status == 0) {
+            self.indexes.removeAll(self.selectedIndexes());
+            huePubSub.publish('assist.collections.refresh');
+          } else {
+            $(document).trigger("error", data.message);
+          }
+          $('#deleteIndexes').modal('hide');
+        }).fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+        });
+        hueAnalytics.log('indexes', 'delete_indexes');
+      };
+
+      self.showContextPopover = function (field, event) {
+        var $source = $(event.target);
+        var offset = $source.offset();
+
+        huePubSub.publish('context.popover.show', {
+          data: {
+            type: 'collection',
+            identifierChain: [
+              {}, // empty, needed by the context popover
+              ko.mapping.toJS(self.index),
+              ko.mapping.toJS(field)
+            ]
+          },
+          showInAssistEnabled: true,
+          orientation: 'right',
+          pinEnabled: false,
+          source: {
+            element: event.target,
+            left: offset.left,
+            top: offset.top - 3,
+            right: offset.left + $source.width() + 1,
+            bottom: offset.top + $source.height() - 3
+          }
+        });
+      };
+
+    };
+    return IndexesViewModel;
+  })();
+
+
+  (function () {
+    $(document).ready(function () {
+      var options = {
+        user: '${ user.username }',
+        % if is_embeddable:
+          hue4: true,
+        % endif
+        index: '${ index }'
+      };
+      var viewModel = new IndexesViewModel(options);
+      ko.applyBindings(viewModel, $('#indexesComponents')[0]);
+
+      huePubSub.subscribe('open.index', function (index) {
+        var foundIndex = viewModel.getIndexByName(index);
+        if (foundIndex) {
+          viewModel.fetchIndex(foundIndex);
+        }
+      }, 'indexes');
+
+      viewModel.fetchIndexes(function () {
+        if (options.index) {
+          var foundIndex = viewModel.getIndexByName(options.index);
+          if (foundIndex) {
+            viewModel.fetchIndex(foundIndex);
+          }
+          else {
+            $.jHueNotify.error('${ _('The specified index has not been found') }')
+            viewModel.showIndexes();
+          }
+        }
+      });
+    });
+  })();
+</script>
+
+%if not is_embeddable:
+${ commonfooter(request, messages) | n,unicode }
+%endif

+ 7 - 0
desktop/libs/indexer/src/indexer/urls.py

@@ -40,6 +40,13 @@ else:
     url(r'^indexes/$', 'indexes', name='indexes'),
     url(r'^indexes/$', 'indexes', name='indexes'),
   )
   )
 
 
+# Kafka
+urlpatterns += patterns('indexer.views',
+    url(r'^topics/$', 'topics', name='topics'),
+    url(r'^topics/(?P<index>[^/]+)/?$', 'topics', name='topics'),
+)
+
+
 urlpatterns += patterns('indexer.solr_api',
 urlpatterns += patterns('indexer.solr_api',
   # V2
   # V2
   url(r'^api/aliases/create/$', 'create_alias', name='create_alias'),
   url(r'^api/aliases/create/$', 'create_alias', name='create_alias'),

+ 10 - 0
desktop/libs/indexer/src/indexer/views.py

@@ -53,6 +53,16 @@ def indexes(request, index=''):
   })
   })
 
 
 
 
+def topics(request, index=''):
+  if not request.user.has_hue_permission(action="access", app='search'):
+    raise PopupException(_('Missing permission.'), error_code=403)
+
+  return render('topics.mako', request, {
+    'is_embeddable': request.GET.get('is_embeddable', False),
+    'index': index
+  })
+
+
 def indexer(request):
 def indexer(request):
   if not request.user.has_hue_permission(action="access", app='search'):
   if not request.user.has_hue_permission(action="access", app='search'):
     raise PopupException(_('Missing permission.'), error_code=403)
     raise PopupException(_('Missing permission.'), error_code=403)

+ 108 - 0
desktop/libs/metadata/src/metadata/kafka_api.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -- coding: utf-8 --
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from metadata.manager_client import ManagerApi
+from metadata.kafka_client import KafkaApi
+
+try:
+  from collections import OrderedDict
+except ImportError:
+  from ordereddict import OrderedDict # Python 2.6
+
+from django.http import Http404
+from django.utils.html import escape
+from django.utils.translation import ugettext as _
+from django.views.decorators.http import require_POST
+
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.i18n import force_unicode
+
+from metadata.conf import has_navigator, has_kafka
+from metadata.navigator_client import NavigatorApiException
+
+
+LOG = logging.getLogger(__name__)
+
+
+def error_handler(view_fn):
+  def decorator(*args, **kwargs):
+    status = 500
+    response = {
+      'message': ''
+    }
+
+    try:
+      if has_navigator(args[0].user): # TODO
+        return view_fn(*args, **kwargs)
+      else:
+        raise NavigatorApiException('Navigator API is not configured.')
+    except NavigatorApiException, e:
+      try:
+        response['message'] = json.loads(e.message)
+      except Exception:
+        response['message'] = force_unicode(e.message)
+    except Exception, e:
+      message = force_unicode(e)
+      response['message'] = message
+      LOG.exception(message)
+
+    return JsonResponse(response, status=status)
+  return decorator
+
+
+@error_handler
+def list_topics(request):
+  return JsonResponse({
+    'status': 0,
+    'topics': [{'name': topic} for topic in get_topics()]
+  })
+
+
+@error_handler
+def list_topic(request):
+  name = request.POST.get('name')
+
+  topic = get_topic(name)
+  topic['name'] = name
+  topic['type'] = 'topic'
+
+  return JsonResponse({
+    'status': 0,
+    'topic': topic
+  })
+
+
+def get_topics():
+  if has_kafka():
+    return KafkaApi().topics()
+  else:
+    manager = ManagerApi()
+    broker_host = manager.get_kafka_brokers().split(',')[0].split(':')[0]
+    return manager.get_kafka_topics(broker_host).keys()
+
+
+def get_topic(name):
+  if has_kafka() and False:
+    pass
+  else:
+    manager = ManagerApi()
+    broker_host = manager.get_kafka_brokers().split(',')[0].split(':')[0]
+    return manager.get_kafka_topics(broker_host)[name]

+ 1 - 0
desktop/libs/metadata/src/metadata/manager_client.py

@@ -90,6 +90,7 @@ class ManagerApi(object):
 
 
       return ','.join(brokers_hosts)
       return ','.join(brokers_hosts)
     except RestException, e:
     except RestException, e:
+      return 'hue.com:9092'
       raise ManagerApiException(e)
       raise ManagerApiException(e)
 
 
 
 

+ 5 - 1
desktop/libs/metadata/src/metadata/urls.py

@@ -69,7 +69,11 @@ urlpatterns += patterns('metadata.manager_api',
   url(r'^api/manager/hello/?$', 'hello', name='hello'),
   url(r'^api/manager/hello/?$', 'hello', name='hello'),
 )
 )
 
 
-
+# Kafka API
+urlpatterns += patterns('metadata.kafka_api',
+  url(r'^api/topics/list/$', 'list_topics', name='list_topics'),
+  url(r'^api/topic/list/$', 'list_topic', name='list_topic'),
+)
 
 
 # Workload Analytics API
 # Workload Analytics API
 urlpatterns += patterns('metadata.workload_analytics_api',
 urlpatterns += patterns('metadata.workload_analytics_api',

+ 11 - 0
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -245,6 +245,14 @@ def get_api(request, snippet):
         'options': {},
         'options': {},
         'is_sql': False
         'is_sql': False
       }]
       }]
+    elif snippet['type'] == 'kafka':
+      interpreter = [{
+        'name': 'kafka',
+        'type': 'kafka',
+        'interface': 'kafka',
+        'options': {},
+        'is_sql': False
+      }]      
     elif snippet['type'] == 'solr':
     elif snippet['type'] == 'solr':
       interpreter = [{
       interpreter = [{
         'name': 'solr',
         'name': 'solr',
@@ -292,6 +300,9 @@ def get_api(request, snippet):
   elif interface == 'hbase':
   elif interface == 'hbase':
     from notebook.connectors.hbase import HBaseApi
     from notebook.connectors.hbase import HBaseApi
     return HBaseApi(request.user)
     return HBaseApi(request.user)
+  elif interface == 'kafka':
+    from notebook.connectors.kafka import KafkaApi
+    return KafkaApi(request.user)
   elif interface == 'pig':
   elif interface == 'pig':
     return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4
     return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4
   else:
   else:

+ 57 - 0
desktop/libs/notebook/src/notebook/connectors/kafka.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import logging
+
+from django.core.urlresolvers import reverse
+from django.utils.translation import ugettext as _
+
+from desktop.lib.i18n import force_unicode
+from metadata.kafka_api import get_topics
+
+from notebook.connectors.base import Api, QueryError
+
+
+LOG = logging.getLogger(__name__)
+
+
+def query_error_handler(func):
+  def decorator(*args, **kwargs):
+    try:
+      return func(*args, **kwargs)
+    except Exception, e:
+      message = force_unicode(str(e))
+      raise QueryError(message)
+  return decorator
+
+
+class KafkaApi(Api):
+
+  @query_error_handler
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+    response = {}
+
+    try:
+      response['databases'] = get_topics()
+    except Exception, e:
+      LOG.warn('Autocomplete data fetching error: %s' % e)
+      response['code'] = 500
+      response['error'] = e.message
+
+    return response