Browse Source

[indexer] Get rid of internal collection management

Make wizard for upload file
Remove all hue colleciton management
Abraham Elmahrek 11 years ago
parent
commit
a29f5620f7

+ 0 - 29
apps/search/src/search/models.py

@@ -366,15 +366,6 @@ class Collection(models.Model):
     if 'collection' in properties_python:
       if 'showFieldList' not in properties_python['collection']['template']:
         properties_python['collection']['template']['showFieldList'] = True
-    if 'field_order' not in properties_python:
-      properties_python['field_order'] = []
-    if 'data_type' not in properties_python:
-      properties_python['data_type'] = 'separated'
-    if properties_python['data_type'] == 'separated':
-      if 'separator' not in properties_python:
-        properties_python['separator'] = ','
-      if 'quote_character' not in properties_python:
-        properties_python['quote_character'] = '"'
     return properties_python
 
   def update_properties(self, post_data):
@@ -396,26 +387,6 @@ class Collection(models.Model):
     properties_['autocomplete'] = autocomplete
     self.properties = json.dumps(properties_)
 
-  @property
-  def field_order(self):
-    return self.properties_dict['field_order']
-
-  @field_order.setter
-  def field_order(self, field_order):
-    properties_ = self.properties_dict
-    properties_['field_order'] = field_order
-    self.properties = json.dumps(properties_)
-
-  @property
-  def data_type(self):
-    return self.properties_dict['data_type']
-
-  @data_type.setter
-  def data_type(self, data_type):
-    properties_ = self.properties_dict
-    properties_['data_type'] = data_type
-    self.properties = json.dumps(properties_)
-
   @property
   def icon(self):
     if self.name == 'twitter_demo':

+ 21 - 60
desktop/libs/indexer/src/indexer/api.py

@@ -117,16 +117,6 @@ def collections(request):
   for collection in solr_collections:
     massaged_collections.append({
       'name': collection,
-      'solr': True,
-      'hue': collection in hue_collections_map
-    })
-    if collection in hue_collections_map:
-      del hue_collections_map[collection]
-  for collection in hue_collections_map:
-    massaged_collections.append({
-      'name': collection,
-      'solr': False,
-      'hue': True
     })
   response = {
     'status': 0,
@@ -149,22 +139,17 @@ def collections_create(request):
 
     # Create instance directory, collection, and add fields
     searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'))
-    hue_collection, created = Collection.objects.get_or_create(name=collection.get('name'), solr_properties='{}', is_enabled=True, user=request.user)
-    properties_dict = hue_collection.properties_dict
-    properties_dict['data_type'] = request.POST.get('type')
-    properties_dict['field_order'] = [field['name'] for field in collection.get('fields', [])]
-    if properties_dict['data_type'] == 'separated':
-      properties_dict['separator'] = request.POST.get('separator', ',')
-      properties_dict['quote_character'] = request.POST.get('quote', '"')
-    hue_collection.properties = json.dumps(properties_dict)
-    hue_collection.save()
 
     try:
       if request.POST.get('source') == 'file':
         # Index data
         searcher.update_data_from_hdfs(request.fs,
-                                       hue_collection,
-                                       request.POST.get('path'))
+                                       collection.get('name'),
+                                       collection.get('fields', []),
+                                       request.POST.get('path'),
+                                       request.POST.get('type'),
+                                       separator=request.POST.get('separator'),
+                                       quote_character=request.POST.get('quote'))
 
       elif request.POST.get('source') == 'hive':
         # Run a custom hive query and post data to collection
@@ -182,7 +167,6 @@ def collections_create(request):
       response['message'] = _('Collection created!')
     except:
       searcher.delete_collection(collection.get('name'))
-      hue_collection.delete()
       raise
   else:
     response['message'] = _('Collection missing.')
@@ -234,7 +218,6 @@ def collections_remove(request):
   if response.get('message', None) is None:
     searcher = CollectionManagerController(request.user)
     solr_collections = searcher.get_collections()
-    Collection.objects.filter(name__in=[collection.get('name') for collection in collections]).delete()
 
     for collection in collections:
       if collection.get('name') in solr_collections:
@@ -248,33 +231,24 @@ def collections_remove(request):
 
 
 @admin_only()
-def collections_fields_and_metadata(request, collection_or_core):
+def collections_fields(request, collection):
   if request.method != 'GET':
     raise PopupException(_('GET request required.'))
 
   response = {}
 
   searcher = CollectionManagerController(request.user)
-  unique_key, fields = searcher.get_fields(collection_or_core)
-
-  try:
-    # Sort fields by field order
-    hue_collection = Collection.objects.get(name=collection_or_core)
-    unknown_fields = list(set(fields.keys()) - set(hue_collection.properties_dict['field_order']))
-    field_order = hue_collection.properties_dict['field_order'] + unknown_fields
-    data_type = hue_collection.properties_dict['data_type']
-  except Collection.DoesNotExist:
-    data_type = 'separated'
+  unique_key, fields = searcher.get_fields(collection)
+
   response['status'] = 0
-  response['fields'] = [(field, fields[field]['type'], fields[field].get('indexed', None), fields[field].get('stored', None)) for field in field_order]
+  response['fields'] = [(field, fields[field]['type'], fields[field].get('indexed', None), fields[field].get('stored', None)) for field in fields]
   response['unique_key'] = unique_key
-  response['type'] = data_type
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
 @admin_only()
-def collections_update(request, collection_or_core):
+def collections_update(request, collection):
   if request.method != 'POST':
     raise PopupException(_('POST request required.'))
 
@@ -282,34 +256,20 @@ def collections_update(request, collection_or_core):
 
   collection = json.loads(request.POST.get('collection', '{}'))
 
-  try:
-    hue_collection = Collection.objects.get(name=collection_or_core)
-  except Collection.DoesNotExist:
-    raise Http404()
-
   if not collection:
     response['message'] = _('No collection to update.')
 
   if response.get('message', None) is None:
     searcher = CollectionManagerController(request.user)
-    searcher.update_collection(collection_or_core, collection.get('fields', []))
-
-    # Update metadata
-    properties_dict = hue_collection.properties_dict
-    properties_dict['field_order'] = [field['name'] for field in collection.get('fields', [])]
-    if 'type' in request.POST:
-      properties_dict['data_type'] = request.POST.get('type')
-    hue_collection.properties = json.dumps(properties_dict)
-    hue_collection.save()
+    searcher.update_collection(collection.get('name'), collection.get('fields', []))
 
     response['status'] = 0
     response['message'] = _('Collection updated!')
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
-
 @admin_only()
-def collections_data(request, collection_or_core):
+def collections_data(request, collection):
   if request.method != 'POST':
     raise PopupException(_('POST request required.'))
 
@@ -317,18 +277,19 @@ def collections_data(request, collection_or_core):
 
   source = request.POST.get('source')
 
-  try:
-    hue_collection = Collection.objects.get(name=collection_or_core)
-  except Collection.DoesNotExist:
-    raise Http404()
-
   if source == 'file':
     searcher = CollectionManagerController(request.user)
 
-    searcher.update_data_from_hdfs(request.fs, hue_collection, request.POST.get('path'))
+    searcher.update_data_from_hdfs(request.fs,
+                                   collection,
+                                   None,
+                                   request.POST.get('path'),
+                                   request.POST.get('type'),
+                                   separator=request.POST.get('separator'),
+                                   quote_character=request.POST.get('quote'))
 
     response['status'] = 0
-    response['message'] = _('Collections updated!')
+    response['message'] = _('Index imported!')
   else:
     response['message'] = _('Unsupported source %s') % source
 

+ 8 - 10
desktop/libs/indexer/src/indexer/controller.py

@@ -31,7 +31,7 @@ from indexer import conf, utils
 
 
 LOG = logging.getLogger(__name__)
-MAX_UPLOAD_SIZE = 1024*1024 # 1 MB
+MAX_UPLOAD_SIZE = 10*1024*1024 # 10 MB
 ALLOWED_FIELD_ATTRIBUTES = set(['name', 'type', 'indexed', 'stored'])
 FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored')]
 
@@ -152,7 +152,7 @@ class CollectionManagerController(object):
 
     api.add_fields(name, new_fields_filtered)
 
-  def update_data_from_hdfs(self, fs, hue_collection, path, indexing_strategy='upload'):
+  def update_data_from_hdfs(self, fs, collection_or_core_name, fields, path, data_type='separated', indexing_strategy='upload', **kwargs):
     """
     Add hdfs path contents to index
     """
@@ -163,24 +163,22 @@ class CollectionManagerController(object):
         raise PopupException(_('File size is too large to handle!'))
       else:
         # Get fields for filtering
-        unique_key, fields = self.get_fields(hue_collection.name)
+        unique_key, fields = self.get_fields(collection_or_core_name)
         fields = [{'name': field, 'type': fields[field]['type']} for field in fields]
 
-        properties_dict = hue_collection.properties_dict
-
         fh = fs.open(path)
-        if properties_dict['data_type'] == 'log':
+        if data_type == 'log':
           # Transform to JSON then update
           data = json.dumps([value for value in utils.field_values_from_log(fh, fields)])
           content_type = 'json'
-        elif properties_dict['data_type'] == 'separated':
+        elif data_type == 'separated':
           # 'data' first line should be headers.
-          data = json.dumps([value for value in utils.field_values_from_separated_file(fh, properties_dict['separator'], properties_dict['quote_character'], fields)])
+          data = json.dumps([value for value in utils.field_values_from_separated_file(fh, kwargs.get('separator', ','), kwargs.get('quote_character', '"'), fields)])
           content_type = 'json'
         else:
-          raise PopupException(_('Could not update index. Unknown type %s') % properties_dict['data_type'])
+          raise PopupException(_('Could not update index. Unknown type %s') % data_type)
         fh.close()
-      if not api.update(hue_collection.name, data, content_type=content_type):
+      if not api.update(collection_or_core_name, data, content_type=content_type):
         raise PopupException(_('Could not update index. Check error logs for more info.'))
     else:
       raise PopupException(_('Could not update index. Indexing strategy %s not supported.') % indexing_strategy)

+ 113 - 52
desktop/libs/indexer/src/indexer/templates/collections.mako

@@ -145,7 +145,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
       <div class="row-fluid">
         <div class="span12">
           <p>
-            <ul id="collections" data-bind="template: {'name': 'collection-template', 'foreach': filteredCollections, 'afterRender': afterCollectionListRender}"></ul>
+            <ul id="collections" data-bind="template: {'name': 'collection-template', 'foreach': filteredCollections}"></ul>
           </p>
         </div>
       </div>
@@ -155,29 +155,24 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
 </script>
 
 <script id="collection-template" type="text/html">
-<li class="collection-row-container" data-bind="click: $parent.toggleCollectionSelect.bind($parent), clickBubble: false, css: {'hue-only': !hasSolrCollection(), 'solr-only': !hasHueCollection(), 'solr-and-hue': hasHueCollection() && hasSolrCollection()}" title="${ _('Click to edit') }">
+<li class="collection-row-container" data-bind="click: $parent.toggleCollectionSelect.bind($parent), clickBubble: false" title="${ _('Click to edit') }">
   <div data-bind="css: {'selected': $parent.collections()[$index()].selected()}" class="collection-row">
     <div class="pull-right" style="margin-top: 10px;margin-right: 10px;">
       <a data-bind="click: $parent.removeCollections, clickBubble: false" href="javascript:void(0);"><i class="fa fa-times"></i> ${_('Delete')}</a>
     </div>
 
-    <!-- ko if: hasHueCollection() && hasSolrCollection() -->
     <form class="pull-right" style="margin-top: 10px;margin-right: 10px;">
-      <div style="display: none">
-        <input class="fileChooser">
-      </div>
-      <a data-bind="click: chooseFileToIndex, clickBubble: false" href="javascript:void(0);"><i class="fa fa-upload"></i> ${_('Upload')}</a>
+      <a data-bind="routie: 'upload/' + name()" href="javascript:void(0);"><i class="fa fa-upload"></i> ${_('Upload')}</a>
     </form>
     <div class="pull-right" style="margin-top: 10px;margin-right: 10px;">
       <a data-bind="routie: 'edit/' + name()" href="javascript:void(0);"><i class="fa fa-pencil"></i> ${_('Edit')}</a>
     </div>
-    <!-- /ko -->
 
-    <!-- ko ifnot: hasHueCollection() -->
-    <div class="pull-right" style="margin-top: 10px;margin-right: 10px;">
-      <a data-bind="click: $parent.importCollection" href="javascript:void(0);"><i class="fa fa-pencil"></i> ${_('Create')}</a>
-    </div>
-    <!-- /ko -->
+    ## <!-- ko ifnot: hasHueCollection() -->
+    ## <div class="pull-right" style="margin-top: 10px;margin-right: 10px;">
+    ##   <a data-bind="click: $parent.importCollection" href="javascript:void(0);"><i class="fa fa-pencil"></i> ${_('Create')}</a>
+    ## </div>
+    ##<!-- /ko -->
     <h4><i class="fa fa-list"></i> <span data-bind="text: name"></span></h4>
   </div>
 </li>
@@ -191,7 +186,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
     <h1 class="card-heading simple">${_("Create collection from file")}</h1>
     <div class="card-body" data-bind="if: wizard.currentPage()">
       <form class="form form-horizontal">
-        <div data-bind="template: { 'name': wizard.currentPage().name}"></div>
+        <div data-bind="template: { 'name': wizard.currentPage().name, 'afterRender': afterRender}"></div>
         <br style="clear:both" />
         <br style="clear:both" />
         <a data-bind="routie: 'create/wizard/' + wizard.previousUrl(), visible: wizard.hasPrevious" class="btn btn-info" href="javascript:void(0)">${_('Previous')}</a>
@@ -202,10 +197,9 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
   </div>
 </div>
 </script>
-<!--/ Create by file -->
 
-<!-- Wizard -->
-<script type="text/html" id="collection-data">
+<!-- Create wizard -->
+<script type="text/html" id="create-collection-data">
   <div class="control-group" data-bind="css: {'error': collection.name.errors().length > 0}">
     <label for="name" class="control-label">${_("Name")}</label>
     <div class="controls">
@@ -228,7 +222,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
   </div>
 </script>
 
-<script type="text/html" id="collection-data-separated">
+<script type="text/html" id="create-collection-data-separated">
   <div class="control-group" data-bind="css: {'error': fieldSeparator.errors().length > 0}">
     <label for="separator" class="control-label">${_("Separator")}</label>
     <div class="controls">
@@ -244,7 +238,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
   </div>
 </script>
 
-<script type="text/html" id="collection-data-morphlines">
+<script type="text/html" id="create-collection-data-morphlines">
   <div class="control-group" data-bind="css: {'error': morphlines.name.errors().length > 0}">
     <label for="name" class="control-label">${_("Morphlines config name")}</label>
 
@@ -262,7 +256,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
   </div>
 </script>
 
-<script type="text/html" id="collection-fields">
+<script type="text/html" id="create-collection-fields">
   <table class="table">
     <thead>
       <tr>
@@ -275,8 +269,8 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
         <th width="50%"></th>
       </tr>
     </thead>
-    <tbody data-bind="sortable: collection.fields">
-      <tr data-bind="css: {'error': name.errors().length > 0}" class="ko_container editable">
+    <tbody data-bind="foreach: collection.fields">
+      <tr data-bind="css: {'error': name.errors().length > 0}" class="editable">
         <td data-bind="editableText: name">
           <span class="pull-left fa fa-pencil"></span>
         </td>
@@ -296,7 +290,8 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
   <br />
   <a data-bind="click: collection.newField" href="javascript:void(0)" class="btn btn-info"><i class="fa fa-plus"></i>&nbsp;${_("Add field")}</a>
 </script>
-<!--/ Wizard -->
+<!--/ Create wizard -->
+<!--/ Create by file -->
 
 <!-- Edit collection page -->
 <script id="edit-page" type="text/html">
@@ -305,14 +300,6 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
     <h1 class="card-heading simple">${_("Edit collection")}</h1>
     <div class="card-body">
       <form class="form">
-        <div class="control-group" data-bind="css: {'error': sourceType.errors().length > 0}">
-          <label for="name" class="control-label">${_("Source type")}</label>
-          <div class="controls">
-            <select data-bind="options: sourceTypes, value: sourceType" name="type"></select>
-          </div>
-        </div>
-        <br />
-        <br />
         <table class="table">
           <thead>
             <tr>
@@ -325,7 +312,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
               <th width="50%"></th>
             </tr>
           </thead>
-          <tbody data-bind="sortable: { 'data': ko.utils.arrayFilter(collection().fields(), function(field) { return field.saved() }) }">
+          <tbody data-bind="foreach: ko.utils.arrayFilter(collection().fields(), function(field) { return field.saved() })">
             <tr class="ko_container">
               <td data-bind="text: name"></td>
               <td data-bind="text: type"></td>
@@ -336,7 +323,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
               <td></td>
             </tr>
           </tbody>
-          <tbody data-bind="sortable: { 'data': ko.utils.arrayFilter(collection().fields(), function(field) { return !field.saved() }) }">
+          <tbody data-bind="foreach: ko.utils.arrayFilter(collection().fields(), function(field) { return !field.saved() })">
             <tr data-bind="css: {'error': name.errors().length > 0}"  class="ko_container editable">
               <td data-bind="editableText: name">
                 <span class="pull-left fa fa-pencil"></span>
@@ -361,6 +348,57 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
 </div>
 </script>
 
+<!-- Upload wizard -->
+<script id="upload-page" type="text/html">
+<div data-bind="with: edit" class="span12">
+  <div class="card wizard">
+    <h1 class="card-heading simple">${_("Index data from file")}</h1>
+    <div class="card-body" data-bind="if: wizard.currentPage()">
+      <form class="form form-horizontal">
+        <div data-bind="template: { 'name': wizard.currentPage().name, 'afterRender': afterRender}"></div>
+        <br style="clear:both" />
+        <br style="clear:both" />
+        <a data-bind="routie: 'upload/' + collection().name() + '/' + wizard.previousUrl(), visible: wizard.hasPrevious" class="btn btn-info" href="javascript:void(0)">${_('Previous')}</a>
+        <a data-bind="routie: 'upload/' + collection().name() + '/' + wizard.nextUrl(), visible: wizard.hasNext" class="btn btn-info" href="javascript:void(0)">${_('Next')}</a>
+        <a data-bind="click: addData, visible: !wizard.hasNext()" class="btn btn-info" href="javascript:void(0)">${_('Finish')}</a>
+      </form>
+    </div>
+  </div>
+</div>
+</script>
+
+<script type="text/html" id="upload-collection-data">
+  <div class="control-group" data-bind="css: {'error': file.errors().length > 0}">
+    <label for="name" class="control-label">${_("Files")}</label>
+    <div class="controls">
+      <input data-bind="value: file" type="text" class="span7 fileChooser" placeholder="/user/foo/data.csv"/>
+    </div>
+  </div>
+
+  <div class="control-group" data-bind="css: {'error': sourceType.errors().length > 0}">
+    <label for="name" class="control-label">${_("Source type")}</label>
+    <div class="controls">
+      <select data-bind="options: sourceTypes, value: sourceType" name="type"></select>
+    </div>
+  </div>
+</script>
+
+<script type="text/html" id="upload-collection-data-separated">
+  <div class="control-group" data-bind="css: {'error': fieldSeparator.errors().length > 0}">
+    <label for="separator" class="control-label">${_("Separator")}</label>
+    <div class="controls">
+      <select data-bind="options: fieldSeparators, value: fieldSeparator" name="separator"></select>
+    </div>
+  </div>
+
+  <div class="control-group" data-bind="css: {'error': fieldQuoteCharacter.errors().length > 0}">
+    <label for="quote" class="control-label">${_("Quote character")}</label>
+    <div class="controls">
+      <select data-bind="options: fieldQuoteCharacters, value: fieldQuoteCharacter" name="quote"></select>
+    </div>
+  </div>
+</script>
+<!--/ Wizard -->
 <!--/ Edit collection page -->
 
 
@@ -374,6 +412,10 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
 <script src="/indexer/static/js/collections.js" type="text/javascript" charset="utf-8"></script>
 
 <script type="text/javascript">
+function afterRender() {
+  $(".fileChooser:not(:has(~ button))").after(getFileBrowseButton($(".fileChooser:not(:has(~ button))")));
+}
+
 function validateAndUpdateCollection() {
   if (validateFields(ko.unwrap(vm.edit.collection))) {
     vm.edit.updateCollection().done(function(data) {
@@ -385,18 +427,6 @@ function validateAndUpdateCollection() {
   return false;
 }
 
-function afterCollectionListRender(elements) {
-  $(elements).find(".fileChooser:not(:has(~ button))").change(function(e) {
-    var context = ko.contextFor(e.target);
-    vm.manage.addData(context.$data, $(e.target).val());
-  });
-  addFileBrowseButton();
-}
-
-function addFileBrowseButton() {
-  $(".fileChooser:not(:has(~ button))").after(getFileBrowseButton($(".fileChooser:not(:has(~ button))")));
-}
-
 function chooseFileToIndex($data, e) {
   $(e.target).siblings().find('button').click();
 }
@@ -427,11 +457,11 @@ function validateFields(collection) {
 }
 
 var vm = new CollectionsViewModel();
-var root = vm.create.wizard.getPage('name', 'collection-data', 'separated', validateFileAndNameAndType);
-vm.create.wizard.getPage('separated', 'collection-data-separated', 'fields', validateFetchFields);
-vm.create.wizard.getPage('morphlines', 'collection-data-morphlines', 'fields', validateFetchFields);
-vm.create.wizard.getPage('fields', 'collection-fields', null, function() { return validateFields(vm.create.collection) });
-vm.create.wizard.rootPage(root);
+var create_root = vm.create.wizard.getPage('name', 'create-collection-data', 'separated', validateFileAndNameAndType);
+vm.create.wizard.getPage('separated', 'create-collection-data-separated', 'fields', validateFetchFields);
+vm.create.wizard.getPage('morphlines', 'create-collection-data-morphlines', 'fields', validateFetchFields);
+vm.create.wizard.getPage('fields', 'create-collection-fields', null, function() { return validateFields(vm.create.collection) });
+vm.create.wizard.rootPage(create_root);
 vm.create.wizard.currentPage(vm.create.wizard.rootPage());
 
 vm.create.sourceType.subscribe(function(value) {
@@ -442,6 +472,22 @@ vm.create.sourceType.subscribe(function(value) {
   }
 });
 
+var edit_root = vm.edit.wizard.getPage('data', 'upload-collection-data', 'separated', function() {return validateNotNull(vm.edit.file, "${ _('File path is missing') }")});
+vm.edit.wizard.getPage('separated', 'upload-collection-data-separated', null, null);
+vm.edit.wizard.rootPage(edit_root);
+vm.edit.wizard.currentPage(vm.edit.wizard.rootPage());
+
+vm.edit.sourceType.subscribe(function(value) {
+  if (!value) {
+    // Weird bug where sourceType disappears when switching between pages
+    vm.edit.sourceType(vm.edit.wizard.getPage('data').next());
+  } else if (value == 'log') {
+    vm.edit.wizard.getPage('data').next(null);
+  } else {
+    vm.edit.wizard.getPage('data').next(value);
+  }
+});
+
 routie({
   "": function() {
     routie('manage');
@@ -453,7 +499,7 @@ routie({
     vm.page('create-page');
     routie('create/wizard');
   },
-  "create/wizard": function(step) {
+  "create/wizard": function() {
     vm.page('create-page');
     routie('create/wizard/' + vm.create.wizard.currentPage().url());
   },
@@ -461,7 +507,6 @@ routie({
     vm.page('create-page');
     vm.create.wizard.setPageByUrl(step);
     routie('create/wizard/' + vm.create.wizard.currentPage().url());
-    $(".fileChooser:not(:has(~ button))").after(getFileBrowseButton($(".fileChooser:not(:has(~ button))")));
   },
   "edit/:name": function(name) {
     ko.utils.arrayForEach(vm.manage.collections(), function(collection) {
@@ -474,6 +519,22 @@ routie({
       vm.page('edit-page');
     }
   },
+  "upload/:name": function(name) {
+    routie('upload/' + name + '/' + vm.edit.wizard.currentPage().url());
+  },
+  "upload/:name/:step": function(name, step) {
+    ko.utils.arrayForEach(vm.manage.collections(), function(collection) {
+      collection.selected(ko.unwrap(collection).name() == name);
+    });
+    if (vm.manage.selectedCollections().length == 0) {
+      routie('manage');
+    } else {
+      vm.edit.collection(vm.manage.selectedCollections()[0]());
+      vm.page('upload-page');
+    }
+    vm.edit.wizard.setPageByUrl(step);
+    routie('upload/' + name + '/' + vm.create.wizard.currentPage().url());
+  },
   "*": function() {
     routie('manage');
   },

+ 3 - 3
desktop/libs/indexer/src/indexer/urls.py

@@ -27,7 +27,7 @@ urlpatterns += patterns('indexer.api',
   url(r'^api/collections/create/$', 'collections_create', name='api_collections_create'),
   url(r'^api/collections/import/$', 'collections_import', name='api_collections_import'),
   url(r'^api/collections/remove/$', 'collections_remove', name='api_collections_remove'),
-  url(r'^api/collections/(?P<collection_or_core>\w+)/metadata/$', 'collections_fields_and_metadata', name='api_collections_metadata'),
-  url(r'^api/collections/(?P<collection_or_core>\w+)/update/$', 'collections_update', name='api_collections_update'),
-  url(r'^api/collections/(?P<collection_or_core>\w+)/data/$', 'collections_data', name='api_collections_data')
+  url(r'^api/collections/(?P<collection>\w+)/fields/$', 'collections_fields', name='api_collections_fields'),
+  url(r'^api/collections/(?P<collection>\w+)/update/$', 'collections_update', name='api_collections_update'),
+  url(r'^api/collections/(?P<collection>\w+)/data/$', 'collections_data', name='api_collections_data')
 )

+ 4 - 2
desktop/libs/indexer/src/indexer/utils.py

@@ -178,11 +178,11 @@ def field_values_from_log(fh, fields=[ {'name': 'message', 'type': 'text_general
     message_key = 'message'
   else:
     try:
-      timestamp_key = next(filter(lambda field: field['type'] in DATE_FIELD_TYPES, fields))['name']
+      timestamp_key = next(iter(filter(lambda field: field['type'] in DATE_FIELD_TYPES, fields)))['name']
     except:
       timestamp_key = None
     try:
-      message_key = next(filter(lambda field: field['type'] in TEXT_FIELD_TYPES, fields))['name']
+      message_key = next(iter(filter(lambda field: field['type'] in TEXT_FIELD_TYPES, fields)))['name']
     except:
       message_key = None
 
@@ -204,12 +204,14 @@ def field_values_from_log(fh, fields=[ {'name': 'message', 'type': 'text_general
       buf = content[:last_newline]
       content = content[last_newline+1:]
       for row in value_generator(buf):
+        # print row
         yield row
     prev = fh.read()
     content += prev
 
   if content:
     for row in value_generator(content):
+      # print row
       yield row
 
 

+ 36 - 17
desktop/libs/indexer/static/js/collections.js

@@ -310,11 +310,19 @@ var EditCollectionViewModel = function() {
 
   // Models
   self.collection = ko.observable();
+  self.source = ko.observable(SOURCES[0]).extend({'errors': null});
+  self.fieldSeparator = ko.observable().extend({'errors': null});
+  self.fieldQuoteCharacter = ko.observable().extend({'errors': null});
+  self.file = ko.observable().extend({'errors': null});
   self.sourceType = ko.observable().extend({'errors': null});
 
   // UI
+  self.wizard = new Wizard();
+  self.sources = ko.mapping.fromJS(SOURCES);
   self.fieldTypes = ko.mapping.fromJS(FIELD_TYPES);
   self.sourceTypes = ko.mapping.fromJS(SOURCE_TYPES);
+  self.fieldSeparators = ko.mapping.fromJS(FIELD_SEPARATORS);
+  self.fieldQuoteCharacters = ko.mapping.fromJS(FIELD_QUOTE_CHARACTERS);
   self.isLoading = ko.observable();
 
   self.collection.subscribe(function(collection) {
@@ -326,7 +334,7 @@ var EditCollectionViewModel = function() {
   self.fetchFields = function() {
     if (self.collection()) {
       self.isLoading(true);
-      return $.get("/indexer/api/collections/" + self.collection().name() + "/metadata/").done(function(data) {
+      return $.get("/indexer/api/collections/" + self.collection().name() + "/fields/").done(function(data) {
         if (data.status == 0) {
           self.collection().fields(inferFields(data.fields, self.collection()));
           self.collection().uniqueKeyField(data.unique_key);
@@ -362,6 +370,33 @@ var EditCollectionViewModel = function() {
       self.isLoading(false);
     });
   };
+
+  self.addData = function() {
+    self.isLoading(true);
+    switch(self.source()) {
+      case 'file':
+      var data = ko.mapping.toJS(self.collection);
+      return $.post("/indexer/api/collections/" + self.collection().name() + "/data/", {
+        'collection': ko.mapping.toJSON(data),
+        'source': self.source(),
+        'type': self.sourceType(),
+        'path': self.file(),
+        'separator': self.fieldSeparator(),
+        'quote': self.fieldQuoteCharacter()
+      }).done(function(data) {
+        if (data.status == 0) {
+          window.location.href = '/indexer';
+        } else {
+          self.isLoading(false);
+          $(document).trigger("error", data.message);
+        }
+      })
+      .fail(function (xhr, textStatus, errorThrown) {
+        self.isLoading(false);
+        $(document).trigger("error", xhr.responseText);
+      });
+    }
+  };
 };
 
 
@@ -411,22 +446,6 @@ var ManageCollectionsViewModel = function() {
     return ko.unwrap(obj).name().indexOf(filter) != -1;
   };
 
-  self.addData = function(collection, path) {
-    self.isLoading(true);
-    return $.post("/indexer/api/collections/" + collection.name() + "/data/", {
-      'collection': ko.mapping.toJSON(collection),
-      'source': 'file',
-      'path': path
-    }).done(function(data) {
-      self.isLoading(false);
-      $(document).trigger("info", data.message);
-    })
-    .fail(function (xhr, textStatus, errorThrown) {
-      self.isLoading(false);
-      $(document).trigger("error", xhr.responseText);
-    });
-  };
-
   self.fetchCollections = function() {
     self.isLoading(true);
     return $.get("/indexer/api/collections/").done(function(data) {

+ 8 - 9
desktop/libs/indexer/static/js/lib.js

@@ -127,6 +127,7 @@ var Wizard = function() {
   self.currentPage = ko.observable(self.rootPage());
   // Stack of previous pages.
   self.previousPages = ko.observableArray();
+  self.pages = {};
 
   self.hasPrevious = ko.computed(function() {
     return self.previousPages().length > 0;
@@ -197,17 +198,15 @@ var Wizard = function() {
       }
     }
   };
-};
 
-ko.utils.extend(Wizard.prototype, {
-  'pages': {},
-  'getPage': function(url, name, next, validate_fn) {
-    if (!this.pages[url]) {
-      this.pages[url] = new Page(url, name, next, validate_fn);
+  self.getPage = function(url, name, next, validate_fn) {
+    self.pages;
+    if (!self.pages[url]) {
+      self.pages[url] = new Page(url, name, next, validate_fn);
     }
-    return this.pages[url];
-  }
-});
+    return self.pages[url];
+  };
+};
 
 // End Wizard