Explorar o código

[indexer] Skeleton for providing a path to an HDFS file and previewing it

Move API v2 to its own file.
Romain Rigaux %!s(int64=10) %!d(string=hai) anos
pai
achega
27e9292fd7

+ 1 - 82
desktop/libs/indexer/src/indexer/api.py

@@ -24,14 +24,11 @@ from django.utils.translation import ugettext as _
 
 
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
-from libsolr.api import SolrApi
-from search.conf import SOLR_URL, SECURITY_ENABLED
 from search.models import Collection
 from search.models import Collection
 
 
 from indexer.controller import CollectionManagerController
 from indexer.controller import CollectionManagerController
-from indexer.controller2 import CollectionController
 from indexer.utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, \
 from indexer.utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, \
-  get_field_types, get_default_fields
+  get_field_types
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -277,81 +274,3 @@ def collections_data(request, collection):
     response['message'] = _('Unsupported source %s') % source
     response['message'] = _('Unsupported source %s') % source
 
 
   return JsonResponse(response)
   return JsonResponse(response)
-
-
-# V2 API
-
-def create_collection(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  name = request.POST.get('name')
-
-  if name:
-    searcher = CollectionController(request.user)
-
-    try:
-      collection = searcher.create_collection(name,
-                                              request.POST.get('fields', get_default_fields()),
-                                              request.POST.get('uniqueKeyField', 'id'),
-                                              request.POST.get('df', 'text'))
-
-      response['status'] = 0
-      response['collection'] = collection
-      response['message'] = _('Collection created!')
-    except Exception, e:
-      response['message'] = _('Collection could not be created: %s') % e
-  else:
-    response['message'] = _('Collection requires a name field.')
-
-  return JsonResponse(response)
-
-
-def create_or_edit_alias(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  alias = request.POST.get('alias', '')
-  collections = json.loads(request.POST.get('collections', '[]'))
-
-  api = SolrApi(SOLR_URL.get(), request.user, SECURITY_ENABLED.get())
-
-  try:
-    api.create_or_modify_alias(alias, collections)
-    response['status'] = 0
-    response['message'] = _('Alias created or modified!')
-  except Exception, e:
-    response['message'] = _('Alias could not be created or modified: %s') % e
-
-  return JsonResponse(response)
-
-
-def delete_indexes(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  indexes = json.loads(request.POST.get('indexes', '[]'))
-
-  if not indexes:
-    response['message'] = _('No indexes to remove.')
-  else:
-    searcher = CollectionController(request.user)
-
-    for index in indexes:
-      if index['type'] == 'collection':
-        searcher.delete_collection(index['name'])
-      elif index['type'] == 'alias':
-        searcher.delete_alias(index['name'])
-      else:
-        LOG.warn('We could not delete: %s' % index)
-
-    response['status'] = 0
-    response['message'] = _('Indexes removed!')
-
-  return JsonResponse(response)

+ 154 - 0
desktop/libs/indexer/src/indexer/api2.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.utils.translation import ugettext as _
+
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.exceptions_renderable import PopupException
+from libsolr.api import SolrApi
+from search.conf import SOLR_URL, SECURITY_ENABLED
+
+from indexer.controller2 import CollectionController
+from indexer.utils import get_default_fields
+import csv
+
+
+LOG = logging.getLogger(__name__)
+
+
+def create_collection(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  name = request.POST.get('name')
+
+  if name:
+    searcher = CollectionController(request.user)
+
+    try:
+      collection = searcher.create_collection(name,
+                                              request.POST.get('fields', get_default_fields()),
+                                              request.POST.get('uniqueKeyField', 'id'),
+                                              request.POST.get('df', 'text'))
+
+      response['status'] = 0
+      response['collection'] = collection
+      response['message'] = _('Collection created!')
+    except Exception, e:
+      response['message'] = _('Collection could not be created: %s') % e
+  else:
+    response['message'] = _('Collection requires a name field.')
+
+  return JsonResponse(response)
+
+
+def create_or_edit_alias(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  alias = request.POST.get('alias', '')
+  collections = json.loads(request.POST.get('collections', '[]'))
+
+  api = SolrApi(SOLR_URL.get(), request.user, SECURITY_ENABLED.get())
+
+  try:
+    api.create_or_modify_alias(alias, collections)
+    response['status'] = 0
+    response['message'] = _('Alias created or modified!')
+  except Exception, e:
+    response['message'] = _('Alias could not be created or modified: %s') % e
+
+  return JsonResponse(response)
+
+
+def delete_indexes(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  indexes = json.loads(request.POST.get('indexes', '[]'))
+
+  if not indexes:
+    response['message'] = _('No indexes to remove.')
+  else:
+    searcher = CollectionController(request.user)
+
+    for index in indexes:
+      if index['type'] == 'collection':
+        searcher.delete_collection(index['name'])
+      elif index['type'] == 'alias':
+        searcher.delete_alias(index['name'])
+      else:
+        LOG.warn('We could not delete: %s' % index)
+
+    response['status'] = 0
+    response['message'] = _('Indexes removed!')
+
+  return JsonResponse(response)
+
+
+
+def create_wizard_get_sample(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  wizard = json.loads(request.POST.get('wizard', '{}'))
+
+  f = request.fs.open(wizard['path'])
+
+  response['status'] = 0
+  response['data'] = _read_csv(f)
+
+  return JsonResponse(response)
+
+
+def create_wizard_create(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  wizard = json.loads(request.POST.get('wizard', '{}'))
+
+  f = request.fs.open(wizard['path'])
+
+  response['status'] = 0
+  response['data'] = _read_csv(f)
+
+  return JsonResponse(response)
+
+
+def _read_csv(f):
+  content = f.read(1024 * 1024)
+
+  dialect = csv.Sniffer().sniff(content)
+  lines = content.splitlines()[:5]
+  reader = csv.reader(lines, delimiter=dialect.delimiter)
+  
+  return [row for row in reader]
+
+

+ 131 - 6
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -44,6 +44,9 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { collection.showCreateModal(true) }">
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { collection.showCreateModal(true) }">
         <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
         <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
       </a>
       </a>
+      <a href="javascript:void(0)" class="btn" data-bind="click: function() { createWizard.show(true) }">
+        <i class="fa fa-plus-circle"></i> ${ _('Create collection from a file') }
+      </a>      
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
         <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
         <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
       </a>
       </a>
@@ -97,6 +100,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
   </div>
   </div>
 </script>
 </script>
 
 
+
 <!-- ko template: 'create-alias' --><!-- /ko -->
 <!-- ko template: 'create-alias' --><!-- /ko -->
 
 
 <script type="text/html" id="create-alias">
 <script type="text/html" id="create-alias">
@@ -114,19 +118,73 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
   </div>
   </div>
 </script>
 </script>
 
 
-<script type="text/html" id="create-collection-from-file">
-  <div class="snippet-settings" data-bind="visible: alias.showCreateModal">
 
 
-    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
-      <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
+<!-- ko template: 'create-collection-wizard' --><!-- /ko -->
+
+<script type="text/html" id="create-collection-wizard">
+  <div class="snippet-settings" data-bind="visible: createWizard.show">
+
+    ${ _('Name') } <input data-bind="value: createWizard.name"></input>
+    
+    <!-- ko if: createWizard.name() -->
+    <select data-bind="options: createWizard.availableWizards, value: createWizard.wizard, optionsText: 'name'" size="5"></select>
+
+    <span data-bind="template: { name: 'create-collection-from-file', data: createWizard.wizard }"></span>
+    <span data-bind="template: { name: 'create-collection-from-hive', data: createWizard.wizard }"></span>
+    
+    <ul data-bind="foreach: createWizard.wizard().sample">
+      <li>
+        <div data-bind="foreach: $data">
+          <span data-bind="text: $data"></span>
+        </div>
+        <a rel="tooltip" data-placement="bottom" data-bind="css: {'btn': true}" class="btn" data-original-title="Edit">
+          <i class="fa fa-pencil"></i>
+        </a>
+      </li>
+    </ul>
+
+    <a href="javascript:void(0)" class="btn" data-bind="click: createWizard.getSample">
+      <i class="fa fa-list-alt"></i> ${ _('Get Sample') }
     </a>
     </a>
-    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(false) }">
+
+    <!-- /ko -->
+
+    <br/>
+
+    <a href="javascript:void(0)" class="btn" data-bind="visible: createWizard.showCreate, click: createWizard.create">
+      <i class="fa fa-plus-circle"></i> ${ _('Create') }
+    </a>
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { createWizard.show(false) }">
       <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
       <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
     </a>
     </a>
   </div>
   </div>
 </script>
 </script>
 
 
 
 
+<script type="text/html" id="create-collection-from-file">
+  <!-- ko if: name() == 'file' -->
+    <div class="snippet-settings" data-bind="visible: show">
+
+      ${ _('Path') } <input data-bind="value: path"></input>
+      <select data-bind="visible: path, options: availableFormats, value: format" size="5"></select>
+
+    </div>
+  <!-- /ko -->
+</script>
+
+
+<script type="text/html" id="create-collection-from-hive">
+  <!-- ko if: name() == 'hive' -->
+    <div class="snippet-settings" data-bind="visible: show">
+
+      ${ _('Database') } <input data-bind="value: database"></input>
+      ${ _('Table') } <input data-bind="value: table"></input>
+
+    </div>
+  <!-- /ko -->
+</script>
+
+
 <div class="hueOverlay" data-bind="visible: isLoading">
 <div class="hueOverlay" data-bind="visible: isLoading">
   <!--[if lte IE 9]>
   <!--[if lte IE 9]>
     <img src="${ static('desktop/art/spinner-big.gif') }" />
     <img src="${ static('desktop/art/spinner-big.gif') }" />
@@ -199,13 +257,79 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
     }
     }
     
     
     self.edit = function(alias) {
     self.edit = function(alias) {
-      self.name(alias.name());console.log(alias.collections());
+      self.name(alias.name());
       self.chosenCollections(alias.collections());
       self.chosenCollections(alias.collections());
 
 
       self.showCreateModal(true);
       self.showCreateModal(true);
     }
     }
   };
   };
 
 
+  var FileWizard = function (vm) {
+    var self = this;
+
+    self.name = ko.observable('file');
+    self.sample = ko.observableArray();
+    self.show = ko.observable(false);
+
+    self.path = ko.observable('');
+    self.format = ko.observable('csv');
+    self.availableFormats = ko.observableArray(['csv', 'log', 'apache logs', 'mailbox']);
+  };
+
+  var HiveWizard = function (vm) {
+    var self = this;
+
+    self.name = ko.observable('hive');
+    self.show = ko.observable(false);
+
+    self.database = ko.observable('');
+    self.table = ko.observable('');
+  };
+
+  var CreateWizard = function (vm) {
+    var self = this;
+
+    self.show = ko.observable(false);
+    self.showCreate = ko.observable(false);
+    
+    self.fileWizard = new FileWizard(vm);
+    self.hiveWizard = new HiveWizard(vm);    
+
+    self.name = ko.observable('');
+    self.wizard = ko.observable();
+    self.wizard.subscribe(function(val) {
+      val.show(true);
+    });
+    self.wizard(self.fileWizard);
+    self.availableWizards = ko.observableArray([self.fileWizard, self.hiveWizard]);
+
+    self.getSample = function() {
+      $.post("${ url('indexer:create_wizard_get_sample') }", {
+        "wizard": ko.mapping.toJSON(self.wizard)
+      }, function(resp) {
+        self.wizard().sample(resp.data);
+        self.showCreate(true);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+      });
+    }
+
+    self.create = function() {
+      $.post("${ url('indexer:create_wizard_create') }", {
+        "wizard": ko.mapping.toJSON(self.wizard)
+      }, function(resp) {
+        self.wizard().sample(resp.data);
+        self.showCreate(true);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+      });
+    }
+
+    self.edit = function() {
+      self.show(true);
+    }
+  };
+
   var Editor = function () {
   var Editor = function () {
     var self = this;
     var self = this;
 
 
@@ -213,6 +337,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 
     self.collection = new Collection(self);
     self.collection = new Collection(self);
     self.alias = new Alias(self);
     self.alias = new Alias(self);
+    self.createWizard = new CreateWizard(self);
 
 
     self.selectedJobs = ko.computed(function() {
     self.selectedJobs = ko.computed(function() {
       return $.grep(self.indexes(), function(index) { return index.isSelected(); });
       return $.grep(self.indexes(), function(index) { return index.isSelected(); });

+ 7 - 2
desktop/libs/indexer/src/indexer/urls.py

@@ -34,9 +34,14 @@ urlpatterns += patterns('indexer.api',
   url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
   url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
   url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
   url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
   url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
   url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
+)
+
 
 
+urlpatterns += patterns('indexer.api2',
   # V2
   # V2
   url(r'^api/v2/collections/create/$', 'create_collection', name='create_collection'),
   url(r'^api/v2/collections/create/$', 'create_collection', name='create_collection'),
   url(r'^api/alias/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/alias/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
-  url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes')
-)
+  url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes'),
+  url(r'^api/indexes/create_wizard_get_sample/$', 'create_wizard_get_sample', name='create_wizard_get_sample'),
+  url(r'^api/indexes/create_wizard_create/$', 'create_wizard_create', name='create_wizard_create')
+)