浏览代码

[indexer] Skeleton for listing collections, cores and aliases

Romain Rigaux 10 年之前
父节点
当前提交
dac5461

+ 1 - 2
apps/oozie/src/oozie/views/dashboard.py

@@ -1029,8 +1029,7 @@ def check_job_access_permission(request, job_id, **kwargs):
       msg = _("Error accessing Oozie job %s.") % (job_id,)
       LOG.exception(msg)
 
-      raise PopupException(msg,
-                           detail=ex._headers['oozie-error-message', ''])
+      raise PopupException(msg, detail=ex._headers['oozie-error-message', ''])
 
   if request.user.is_superuser \
       or oozie_job.user == request.user.username \

+ 170 - 0
desktop/libs/indexer/src/indexer/controller2.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+# -- coding: utf-8 --
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.utils.translation import ugettext as _
+
+from desktop.lib.exceptions_renderable import PopupException
+from libsolr.api import SolrApi
+from libzookeeper.conf import ENSEMBLE
+from search.conf import SOLR_URL, SECURITY_ENABLED
+
+from desktop.lib.i18n import smart_str
+
+
+LOG = logging.getLogger(__name__)
+MAX_UPLOAD_SIZE = 100 * 1024 * 1024 # 100 MB
+ALLOWED_FIELD_ATTRIBUTES = set(['name', 'type', 'indexed', 'stored'])
+FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored')]
+ZK_SOLR_CONFIG_NAMESPACE = 'configs'
+
+
+def get_solr_ensemble():
+  return '%s/solr' % ENSEMBLE.get()
+
+
+class CollectionController(object):
+  """
+  Glue the models to the views.
+  """
+  def __init__(self, user):
+    self.user = user
+    self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
+
+#  def _format_flags(self, fields):
+#    for field_name, field in fields.items():
+#      for index in range(0, len(FLAGS)):
+#        flags = FLAGS[index]
+#        field[flags[1]] = field['flags'][index] == FLAGS[index][0]
+#    return fields
+#
+  def is_solr_cloud_mode(self):
+    if not hasattr(self, '_solr_cloud_mode'):
+      try:
+        self.api.collections()
+        setattr(self, '_solr_cloud_mode', True)
+      except Exception, e:
+        LOG.info('Non SolrCloud server: %s' % e)
+        setattr(self, '_solr_cloud_mode', False)
+    return getattr(self, '_solr_cloud_mode')
+
+#  def collection_exists(self, collection):
+#    return collection in self.get_collections()
+
+  def get_indexes(self):
+    indexes = []
+
+    try:
+      if self.is_solr_cloud_mode():
+        collections = self.api.collections2()
+        for name in collections:
+          indexes.append({'name': name, 'type': 'collection'})
+
+      solr_cores = self.api.cores()
+      for name in solr_cores:
+        indexes.append({'name': name, 'type': 'core'})
+
+      solr_aliases = self.api.aliases()
+      for name in solr_aliases:
+        indexes.append({'name': name, 'type': 'alias', 'collections': solr_aliases[name]})
+
+    except Exception, e:
+      msg = _('Solr server could not be contacted properly: %s') % e
+      LOG.warn(msg)
+      raise PopupException(msg, detail=smart_str(e))
+
+    return indexes
+
+#  def get_fields(self, collection_or_core_name):
+#    try:
+#      field_data = self.api.fields(collection_or_core_name)
+#      fields = self._format_flags(field_data['schema']['fields'])
+#    except:
+#      LOG.exception(_('Could not fetch fields for collection %s.') % collection_or_core_name)
+#      raise PopupException(_('Could not fetch fields for collection %s. See logs for more info.') % collection_or_core_name)
+#
+#    try:
+#      uniquekey = self.api.uniquekey(collection_or_core_name)
+#    except:
+#      LOG.exception(_('Could not fetch unique key for collection %s.') % collection_or_core_name)
+#      raise PopupException(_('Could not fetch unique key for collection %s. See logs for more info.') % collection_or_core_name)
+#
+#    return uniquekey, fields
+#
+#  def create_collection(self, name, fields, unique_key_field='id', df='text'):
+#    """
+#    Create solr collection or core and instance dir.
+#    Create schema.xml file so that we can set UniqueKey field.
+#    """
+#    if self.is_solr_cloud_mode():
+#      # solrcloud mode
+#
+#      # Need to remove path afterwards
+#      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
+#
+#      zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
+#      root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
+#      config_root_path = '%s/%s' % (solr_config_path, 'conf')
+#      try:
+#        zc.copy_path(root_node, config_root_path)
+#      except Exception, e:
+#        zc.delete_path(root_node)
+#        raise PopupException(_('Error in copying Solr configurations.'), detail=e)
+#
+#      # Don't want directories laying around
+#      shutil.rmtree(tmp_path)
+#
+#      if not self.api.create_collection(name):
+#        # Delete instance directory if we couldn't create a collection.
+#        try:
+#          zc.delete_path(root_node)
+#        except Exception, e:
+#          raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
+#    else:
+#      # Non-solrcloud mode
+#      # Create instance directory locally.
+#      instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
+#      if os.path.exists(instancedir):
+#        raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
+#      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
+#      shutil.move(solr_config_path, instancedir)
+#      shutil.rmtree(tmp_path)
+#
+#      if not self.api.create_core(name, instancedir):
+#        # Delete instance directory if we couldn't create a collection.
+#        shutil.rmtree(instancedir)
+#        raise PopupException(_('Could not create collection. Check error logs for more info.'))
+#
+#  def delete_collection(self, name, core):
+#    if core:
+#      raise PopupException(_('Cannot remove Solr cores.'))
+#
+#    if self.api.remove_collection(name):
+#      # Delete instance directory.
+#      try:
+#        root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
+#        zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
+#        zc.delete_path(root_node)
+#      except Exception, e:
+#        # Re-create collection so that we don't have an orphan config
+#        self.api.add_collection(name)
+#        raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
+#    else:
+#      raise PopupException(_('Could not remove collection. Check error logs for more info.'))

+ 617 - 0
desktop/libs/indexer/src/indexer/static/indexer/js/indexes.ko.js

@@ -0,0 +1,617 @@
+
+// Defaults
+var SOURCES = [
+  'file',
+  // 'hbase',
+  // 'hive'
+];
+
+var SOURCE_TYPES = [
+  // 'log',
+  'separated',
+  // 'morphlines'
+];
+
+var FIELD_TYPES = [
+  "alphaOnlySort",
+  "ancestor_path",
+  "boolean",
+  "currency",
+  "date",
+  "descendent_path",
+  "double",
+  "float",
+  "int",
+  "location",
+  "location_rpt",
+  "long",
+  "lowercase",
+  "pdate",
+  "pdouble",
+  "pfloat",
+  "pint",
+  "plong",
+  "point",
+  "random",
+  "string",
+  "tdate",
+  "tdouble",
+  "text_ar",
+  "text_bg",
+  "text_ca",
+  "text_char_norm",
+  "text_cjk",
+  "text_cz",
+  "text_da",
+  "text_de",
+  "text_el",
+  "text_en",
+  "text_en_splitting",
+  "text_en_splitting_tight",
+  "text_es",
+  "text_eu",
+  "text_fa",
+  "text_fi",
+  "text_fr",
+  "text_ga",
+  "text_general",
+  "text_general_rev",
+  "text_gl",
+  "text_greek",
+  "text_hi",
+  "text_hu",
+  "text_hy",
+  "text_id",
+  "text_it",
+  "text_ja",
+  "text_lv",
+  "text_nl",
+  "text_no",
+  "text_pt",
+  "text_ro",
+  "text_ru",
+  "text_sv",
+  "text_th",
+  "text_tr",
+  "text_ws",
+  "tfloat",
+  "tint",
+  "tlong"
+];
+
+var FIELD_SEPARATORS = [
+  ',',
+  '\t',
+];
+
+var FIELD_QUOTE_CHARACTERS = [
+  '"',
+  "'"
+];
+
+// View Models
+
+var HiveViewModel = function() {
+  var self = this;
+
+  self.databases = ko.observableArray();
+  self.database = ko.observable().extend({'errors': null});
+  self.table = ko.observable().extend({'errors': null});
+
+  self.database.subscribe(function() {
+    self.loadTables();
+  });
+
+  self.table.subscribe(function() {
+    self.loadColumns();
+  });
+
+  self.loadDatabases = function() {
+    return $.get("/beeswax/api/autocomplete").done(function(data) {
+      if (data.databases.length > 0) {
+        self.databases($.map(data.databases, function(database) { return new HiveDatabase(database); }));
+        self.database(self.databases()[0]);
+      }
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  self.loadTables = function() {
+    return $.get("/beeswax/api/autocomplete/" + self.database().name()).done(function(data) {
+      if (data.tables.length > 0) {
+        self.database().tables($.map(data.tables, function(table) { return new HiveTable(table); }));
+        self.table(self.database().tables()[0]);
+      }
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  self.loadColumns = function() {
+    return $.get("/beeswax/api/autocomplete/" + self.database().name() + '/' + self.table().name()).done(function(data) {
+      if (data.columns.length > 0) {
+        self.table().columns(data.columns);
+      }
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  // self.loadDatabases();
+};
+
+var HBaseViewModel = function() {
+  var self = this;
+
+  self.clusters = ko.observableArray();
+  self.cluster = ko.observable().extend({'errors': null});
+  self.table = ko.observable().extend({'errors': null});
+  self.indexMapping = {};
+
+  self.cluster.subscribe(function() {
+    self.loadTables();
+  });
+
+  self.loadClusters = function() {
+    return $.get("/hbase/api/getClusters").done(function(data) {
+      if (data.data.length > 0) {
+        self.clusters($.map(data.data, function(cluster) { return new HBaseCluster(cluster.name); }));
+        self.cluster(self.clusters()[0]);
+      }
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  self.loadTables = function() {
+    return $.get("/hbase/api/getTableList/" + self.cluster().name()).done(function(data) {
+      if (data.data.length > 0) {
+        // @TODO(Abe): Check if enabled
+        self.cluster().tables($.map(data.data, function(table) { return new HBaseTable(table.name); }));
+        self.table(self.cluster().tables()[0]);
+      }
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  // self.loadClusters();
+};
+
+var CreateCollectionViewModel = function() {
+  var self = this;
+
+  // Models
+  self.collection = new Collection();
+
+  self.source = ko.observable(SOURCES[0]).extend({'errors': null});
+  self.fieldSeparator = ko.observable().extend({'errors': null});
+  self.fieldQuoteCharacter = ko.observable().extend({'errors': null});
+  self.file = ko.observable().extend({'errors': null});
+  self.sourceType = ko.observable().extend({'errors': null});
+  self.morphlines = ko.mapping.fromJS({'name': 'message', 'expression': '%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST:hostname} %{DATA:program}(?:\[%{POSINT:pid}\])?: %{GREEDYDATA:msg}'});
+  self.morphlines.name = self.morphlines.name.extend({'errors': null});
+  self.morphlines.expression = self.morphlines.expression.extend({'errors': null});
+
+  // UI
+  self.hive = new HiveViewModel();
+  self.hbase = new HBaseViewModel();
+  self.wizard = new Wizard();
+  self.isLoading = ko.observable();
+  self.sources = ko.mapping.fromJS(SOURCES);
+  self.fieldTypes = ko.mapping.fromJS(FIELD_TYPES);
+  self.fieldSeparators = ko.mapping.fromJS(FIELD_SEPARATORS);
+  self.fieldQuoteCharacters = ko.mapping.fromJS(FIELD_QUOTE_CHARACTERS);
+  self.sourceTypes = ko.mapping.fromJS(SOURCE_TYPES);
+
+  self.parseFields = function() {
+    if (self.source() == 'file') {
+      self.isLoading(true);
+      return $.post("/indexer/api/fields/parse/", {
+        'source': self.source(),
+        'type': self.sourceType(),
+        'path': self.file(),
+        'separator': self.fieldSeparator(),
+        'quote': self.fieldQuoteCharacter(),
+        'morphlines': ko.mapping.toJSON(self.morphlines)
+      }).done(function(data) {
+        if (data.status == 0) {
+          self.collection.fields(inferFields(data.data, self.collection));
+          chooseUniqueKey(self.collection);
+          chooseDefaultField(self.collection);
+        } else {
+          $(document).trigger("error", data.message);
+        }
+        self.isLoading(false);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+        self.isLoading(false);
+      });
+    }
+  };
+
+  self.save = function() {
+    if (self.wizard.currentPage().validate()) {
+      self.isLoading(true);
+      var collection = ko.mapping.toJS(self.collection);
+      collection.fields = ko.utils.arrayMap(collection.fields, function(field) {
+        delete field['uniqueKeyField'];
+        return field;
+      });
+      switch(self.source()) {
+        case 'file':
+        return $.post("/indexer/api/collections/create/", {
+          'collection': ko.mapping.toJSON(collection),
+          'type': self.sourceType(),
+          'path': self.file(),
+          'source': self.source(),
+          'separator': self.fieldSeparator(),
+          'quote': self.fieldQuoteCharacter()
+        }).done(function(data) {
+          if (data.status == 0) {
+            window.location.href = '/indexer';
+          } else {
+            self.isLoading(false);
+            $(document).trigger("error", data.message);
+          }
+        })
+        .fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+          self.isLoading(false);
+        });
+
+        case 'hive':
+        return $.post("/indexer/api/collections/create/", {
+          'collection': ko.toJSON(collection),
+          'database': self.hive.database().name(),
+          'table': self.hive.table().name(),
+          'source': self.source()
+        }).done(function(data) {
+          if (data.status == 0) {
+            window.location.href = '/indexer';
+          } else {
+            $(document).trigger("error", data.message);
+          }
+          self.isLoading(false);
+        })
+        .fail(function (xhr, textStatus, errorThrown) {
+          $(document).trigger("error", xhr.responseText);
+          self.isLoading(false);
+        });
+      }
+    }
+  };
+};
+
+
+var EditCollectionViewModel = function() {
+  var self = this;
+
+  // Models
+  self.collection = ko.observable();
+  self.source = ko.observable(SOURCES[0]).extend({'errors': null});
+  self.fieldSeparator = ko.observable().extend({'errors': null});
+  self.fieldQuoteCharacter = ko.observable().extend({'errors': null});
+  self.file = ko.observable().extend({'errors': null});
+  self.sourceType = ko.observable().extend({'errors': null});
+
+  // UI
+  self.wizard = new Wizard();
+  self.sources = ko.mapping.fromJS(SOURCES);
+  self.fieldTypes = ko.mapping.fromJS(FIELD_TYPES);
+  self.sourceTypes = ko.mapping.fromJS(SOURCE_TYPES);
+  self.fieldSeparators = ko.mapping.fromJS(FIELD_SEPARATORS);
+  self.fieldQuoteCharacters = ko.mapping.fromJS(FIELD_QUOTE_CHARACTERS);
+  self.isLoading = ko.observable();
+
+  self.fetchFields = function() {
+    if (self.collection()) {
+      self.isLoading(true);
+      return $.get("/indexer/api/collections/" + self.collection().name() + "/fields/").done(function(data) {
+        if (data.status == 0) {
+          self.collection().fields(inferFields(data.fields, self.collection()));
+          self.collection().uniqueKeyField(data.unique_key);
+          self.sourceType(data.type);
+        } else {
+          $(document).trigger("error", data.message);
+        }
+        self.isLoading(false);
+      })
+      .fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+        self.isLoading(false);
+      });
+    }
+  };
+
+  self.removeCollection = function(collection) {
+    self.isLoading(true);
+    var data = [ko.mapping.toJS(self.collection)];
+    return $.post("/indexer/api/collections/remove/", {
+      'collections': ko.mapping.toJSON(data)
+    }).done(function(data) {
+      if (data.status == 0) {
+        window.location.href = '/indexer';
+      } else {
+        $(document).trigger("error", data.message);
+      }
+      self.isLoading(false);
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+      self.isLoading(false);
+    });
+  };
+
+  self.updateCollection = function() {
+    self.isLoading(true);
+    var data = ko.mapping.toJS(self.collection);
+    return $.post("/indexer/api/collections/" + self.collection().name() + "/update/", {
+      'collection': ko.mapping.toJSON(data),
+      'type': self.sourceType()
+    }).done(function(data) {
+      if (data.status == 0) {
+        $(document).trigger("info", data.message);
+      } else {
+        $(document).trigger("error", data.message);
+      }
+      self.isLoading(false);
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+      self.isLoading(false);
+    });
+  };
+
+  self.addData = function() {
+    self.isLoading(true);
+    switch(self.source()) {
+      case 'file':
+      var data = ko.mapping.toJS(self.collection);
+      return $.post("/indexer/api/collections/" + self.collection().name() + "/data/", {
+        'collection': ko.mapping.toJSON(data),
+        'source': self.source(),
+        'type': self.sourceType(),
+        'path': self.file(),
+        'separator': self.fieldSeparator(),
+        'quote': self.fieldQuoteCharacter()
+      }).done(function(data) {
+        if (data.status == 0) {
+          window.location.href = '/indexer';
+        } else {
+          self.isLoading(false);
+          $(document).trigger("error", data.message);
+        }
+      })
+      .fail(function (xhr, textStatus, errorThrown) {
+        self.isLoading(false);
+        $(document).trigger("error", xhr.responseText);
+      });
+    }
+  };
+};
+
+
+var ManageCollectionsViewModel = function() {
+  var self = this;
+
+  // Models
+  self.collections = ko.observableArray();
+
+  // UI
+  self.isLoading = ko.observable();
+  self.hasLoadedOnce = ko.observable(false);
+  self.showCores = ko.observable(false);
+  self.filteredCollections = ko.observableArray();
+  self.displayCollections = ko.computed(function() {
+    return ko.utils.arrayFilter(self.filteredCollections(), function(collection) {
+      return ! self.hasCloudCollections() || self.showCores() || ! ko.unwrap(collection).isCoreOnly();
+    });
+  });
+  self.selectedCollections = ko.computed(function() {
+    return ko.utils.arrayFilter(self.collections(), function(collection) {
+      return collection.selected();
+    });
+  });
+  self.selectedCloudCollections = ko.computed(function() {
+    return ko.utils.arrayFilter(self.selectedCollections(), function(collection) {
+      return ! ko.unwrap(collection).isCoreOnly();
+    });
+  });
+  self.hasCloudCollections = ko.computed(function() {
+    var _arr = ko.utils.arrayFilter(self.collections(), function(collection) {
+      return ! ko.unwrap(collection).isCoreOnly();
+    });
+    return _arr.length > 0;
+  });
+
+  self.toggleSelectAll = function() {
+    var direction = ! self.selectedCollections().length;
+    ko.utils.arrayForEach(self.filteredCollections(), function(collection) {
+      collection.selected(direction);
+    });
+  };
+
+  self.toggleCollectionSelect = function(collection, e) {
+    ko.utils.arrayForEach(self.collections(), function(other_collection) {
+      if (ko.unwrap(other_collection).name() == collection.name()) {
+        other_collection.selected(! other_collection.selected());
+      }
+    });
+  };
+
+  self.importCollection = function(collection, e) {
+    self.isLoading(true);
+    return $.post("/indexer/api/collections/import/", {
+      'collection': ko.mapping.toJSON(collection)
+    }).done(function(data) {
+      self.isLoading(false);
+      $(document).trigger("info", data.message);
+      self.fetchCollections();
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      self.isLoading(false);
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  self.filterTest = function(obj, filter) {
+    return ko.unwrap(obj).name().indexOf(filter) != -1;
+  };
+
+  self.fetchCollections = function() {
+    self.isLoading(true);
+    return $.get("/indexer/api/collections/").done(function(data) {
+      if (data.status == 0) {
+        var collections = [];
+        ko.utils.arrayForEach(data.collections, function(collection) {
+          var new_collection = ko.observable(new Collection(collection.name)).extend({'selectable': null});
+          new_collection().hasHueCollection(collection.hue);
+          new_collection().hasSolrCollection(collection.solr);
+          new_collection().isCoreOnly(collection.isCoreOnly);
+          new_collection().isAlias(collection.isAlias);
+          new_collection().collections(collection.collections);
+          collections.push(new_collection);
+        });
+        self.collections(collections);
+        self.hasLoadedOnce(true);
+      } else {
+        $(document).trigger("error", data.message);
+      }
+      self.isLoading(false);
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+      self.isLoading(false);
+    });
+  };
+
+  self.removeCollections = function(collection) {
+    self.isLoading(true);
+    var collections = self.selectedCloudCollections();
+    if (collections.length == 0) {
+      collections = ($.isArray(collection)) ? collection : [collection];
+    }
+    return $.post("/indexer/api/collections/remove/", {
+      'collections': ko.mapping.toJSON(collections)
+    }).done(function(data) {
+      if (data.status == 0) {
+        // Remove collections
+        collection_names = ko.utils.arrayMap(collections, function(collection) {
+          return ko.unwrap(collection).name();
+        });
+        var index = 0;
+        var remove = [];
+        ko.utils.arrayForEach(self.collections(), function(collection) {
+          collection = ko.unwrap(collection);
+          if (collection_names.indexOf(collection.name()) != -1) {
+            remove.push(index);
+          }
+          index++;
+        });
+        remove.reverse();
+        ko.utils.arrayForEach(remove, function(index) {
+          self.collections.splice(index, 1);
+        });
+      } else {
+        $(document).trigger("error", data.message);
+      }
+      self.isLoading(false);
+    })
+    .fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+      self.isLoading(false);
+    });
+  };
+};
+
+
+var CollectionsViewModel = function(config) {
+  var self = this;
+
+  var _config = config || {};
+
+  // Models
+  self.page = ko.observable();
+  self.breadcrumb = ko.observable();
+  self.breadcrumb.list = ko.computed(function() {
+    var breadcrum_config = _config.breadcrumb || {};
+    var labels = breadcrum_config.labels || {};
+    var skip = breadcrum_config.skip || [];
+    var breadcrums = [];
+    var crum = '';
+    breadcrums.push({
+      'url': '/',
+      'label': labels[crum] || crum,
+      'crum': crum
+    });
+    if (self.breadcrumb()) {
+      var crums = [];
+      var newcrums = ko.utils.arrayMap( self.breadcrumb().split('/'), function(crum) {
+        crums.push(crum);
+        if (skip.indexOf(crum) != -1) {
+          return null;
+        }
+        return {
+          'url': crums.join('/'),
+          'label': labels[crum] || crum,
+          'crum': crum
+        };
+      });
+      breadcrums.push.apply( breadcrums, ko.utils.arrayFilter(newcrums, function(breadcrumb) {
+        return !!breadcrumb;
+      }) );
+    }
+    return breadcrums;
+  });
+
+  // UI
+  self.create = new CreateCollectionViewModel();
+  self.manage = new ManageCollectionsViewModel();
+  self.edit = new EditCollectionViewModel();
+  self.isLoading = ko.computed(function() {
+    return self.create.isLoading() || self.manage.isLoading() || self.edit.isLoading();
+  });
+};
+
+
+// Utils
+
+function getFileBrowseButton(inputElement) {
+  return $("<button>").addClass("btn").addClass("fileChooserBtn").text("..").click(function (e) {
+    e.preventDefault();
+    $("#filechooser").jHueFileChooser({
+      initialPath: inputElement.val(),
+      onFileChoose: function (filePath) {
+        inputElement.val(filePath);
+        inputElement.trigger("change");
+        $("#chooseFile").modal("hide");
+      },
+      selectFolder: false,
+      createFolder: false
+    });
+    $("#chooseFile").modal("show");
+  });
+}
+
+
+function inferFields(field_data, collection) {
+  var fields = [];
+  $.each(field_data, function(index, value) {
+    // 0 => name
+    // 1 => type
+    // 2 => required
+    // 3 => indexed
+    // 4 => stored
+    var field = new Field(collection, value[0], value[1], value[2], value[3]);
+    field.saved(true);
+    fields.push(field);
+  });
+  return fields;
+}

+ 263 - 0
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -0,0 +1,263 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+  from desktop.views import commonheader, commonfooter, commonshare, commonimportexport
+  from django.utils.translation import ugettext as _
+%>
+<%namespace name="actionbar" file="actionbar.mako" />
+
+${ commonheader(_("Solr Indexes"), "spark", user, "60px") | n,unicode }
+
+
+<div class="container-fluid">
+  <div class="card card-small">
+  <h1 class="card-heading simple">${ _('Solr Indexes') }</h1>
+
+  <%actionbar:render>
+    <%def name="search()">
+      <input id="filterInput" type="text" class="input-xlarge search-query" placeholder="${_('Search for name, description, etc...')}">
+    </%def>
+
+    <%def name="actions()">
+      <div class="btn-toolbar" style="display: inline; vertical-align: middle">
+        <a data-bind="click: function() { atLeastOneSelected() ? $('#deleteNotebook').modal('show') : void(0) }, css: {'btn': true, 'disabled': ! atLeastOneSelected() }">
+          <i class="fa fa-times"></i> ${ _('Delete') }
+        </a>
+      </div>
+    </%def>
+
+    <%def name="creation()">
+      <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
+        <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
+      </a>
+      <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
+        <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
+      </a>
+    </%def>
+  </%actionbar:render>
+
+
+  <table id="notebookTable" class="table datatables">
+    <thead>
+      <tr>
+        <th width="1%"><div data-bind="click: selectAll, css: {hueCheckbox: true, 'fa': true, 'fa-check': allSelected}" class="select-all"></div></th>
+        <th>${ _('Name') }</th>
+        <th>${ _('Type') }</th>
+        <th>${ _('Collections') }</th>
+      </tr>
+    </thead>
+    <tbody data-bind="foreach: { data: jobs }">
+      <tr>
+        <td data-bind="click: $root.handleSelect" class="center" style="cursor: default" data-row-selector-exclude="true">
+          <div data-bind="css: { 'hueCheckbox': true, 'fa': true, 'fa-check': isSelected }" data-row-selector-exclude="true"></div>
+          ## <a data-bind="attr: { 'href': '${ url('spark:notebook') }?notebook=' + id() }" data-row-selector="true"></a>
+        </td>
+        <td data-bind="text: name"></td>
+        <td data-bind="text: type"></td>
+        <td data-bind="text: type"></td>
+      </tr>
+    </tbody>
+  </table>
+
+  </div>
+</div>
+
+<!-- ko template: 'create-alias' --><!-- /ko -->
+
+<script type="text/html" id="create-alias">
+  <div class="snippet-settings" data-bind="visible: alias.showCreateModal">
+   
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
+      <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
+    </a>
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(false) }">
+      <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
+    </a>
+  </div>
+</script>
+
+<script type="text/html" id="create-collection">
+  <div class="snippet-settings" data-bind="visible: alias.showCreateModal">
+   
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
+      <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
+    </a>
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(false) }">
+      <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
+    </a>
+  </div>
+</script>
+
+<script type="text/html" id="create-collection-from-file">
+  <div class="snippet-settings" data-bind="visible: alias.showCreateModal">
+   
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
+      <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
+    </a>
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(false) }">
+      <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
+    </a>
+  </div>
+</script>
+
+
+<div class="hueOverlay" data-bind="visible: isLoading">
+  <!--[if lte IE 9]>
+    <img src="${ static('desktop/art/spinner-big.gif') }" />
+  <![endif]-->
+  <!--[if !IE]> -->
+    <i class="fa fa-spinner fa-spin"></i>
+  <!-- <![endif]-->
+</div>
+
+<div id="submit-notebook-modal" class="modal hide"></div>
+
+<div id="deleteNotebook" class="modal hide fade">
+  <form id="deleteNotebookForm" method="POST" data-bind="submit: delete2">
+    ${ csrf_token(request) | n,unicode }
+    <div class="modal-header">
+      <a href="#" class="close" data-dismiss="modal">&times;</a>
+      <h3 id="deleteNotebookMessage">${ _('Delete the selected notebook(s)?') }</h3>
+    </div>
+    <div class="modal-footer">
+      <a href="#" class="btn" data-dismiss="modal">${ _('No') }</a>
+      <input type="submit" class="btn btn-danger" value="${ _('Yes') }"/>
+    </div>
+  </form>
+</div>
+
+
+<script src="${ static('desktop/ext/js/datatables-paging-0.1.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('desktop/ext/js/knockout.min.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('desktop/ext/js/knockout-mapping.min.js') }" type="text/javascript" charset="utf-8"></script>
+
+
+<script type="text/javascript" charset="utf-8">
+  var AliasVM = function (vm) {
+    var self = this;
+    
+    self.formName = ko.observable(false);
+    self.formCollections = ko.observable(['c1', 'c2']); // todo get collection list
+
+    self.showCreateModal = ko.observable(false);
+  };
+  
+  var Editor = function () {
+    var self = this;
+
+    self.jobs = ko.mapping.fromJS(${ indexes_json | n });
+
+    self.alias = new AliasVM(self);
+
+    self.selectedJobs = ko.computed(function() {
+      return $.grep(self.jobs(), function(job) { return job.isSelected(); });
+    });
+    self.isLoading = ko.observable(false);
+
+    self.oneSelected = ko.computed(function() {
+      return self.selectedJobs().length == 1;
+    });
+    self.atLeastOneSelected = ko.computed(function() {
+      return self.selectedJobs().length >= 1;
+    });
+    self.allSelected = ko.observable(false);
+
+    self.handleSelect = function(notebook) {
+      notebook.isSelected(! notebook.isSelected());
+    }
+
+    self.selectAll = function() {
+      self.allSelected(! self.allSelected());
+      ko.utils.arrayForEach(self.jobs(), function (job) {
+        job.isSelected(self.allSelected());
+      });
+    }
+
+    self.datatable = null;
+
+    self.delete2 = function() {
+      $.post("${ url('spark:delete') }", {
+        "notebooks": ko.mapping.toJSON(self.selectedJobs)
+      }, function() {
+        window.location.reload();
+        $('#deleteNotebook').modal('hide');
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+      });
+    };
+  };
+
+  var viewModel;
+
+  $(document).ready(function () {
+    viewModel = new Editor();
+    ko.applyBindings(viewModel);
+
+    var oTable = $("#notebookTable").dataTable({
+      "sPaginationType":"bootstrap",
+      'iDisplayLength':50,
+      "bLengthChange":false,
+      "sDom": "<'row'r>t<'row-fluid'<'dt-pages'p><'dt-records'i>>",
+      "aoColumns":[
+        { "bSortable":false },
+        null,
+        null,
+        null,
+        { "sSortDataType":"dom-sort-value", "sType":"numeric" }
+      ],
+      "aaSorting":[
+        [4, 'desc'],
+        [1, 'asc' ]
+      ],
+      "oLanguage":{
+        "sEmptyTable":"${_('No data available')}",
+        "sInfo":"${_('Showing _START_ to _END_ of _TOTAL_ entries')}",
+        "sInfoEmpty":"${_('Showing 0 to 0 of 0 entries')}",
+        "sInfoFiltered":"${_('(filtered from _MAX_ total entries)')}",
+        "sZeroRecords":"${_('No matching records')}",
+        "oPaginate":{
+          "sFirst":"${_('First')}",
+          "sLast":"${_('Last')}",
+          "sNext":"${_('Next')}",
+          "sPrevious":"${_('Previous')}"
+        },
+        "bDestroy": true
+      },
+      "fnDrawCallback":function (oSettings) {
+        $("a[data-row-selector='true']").jHueRowSelector();
+      }
+    });
+
+    viewModel.datatable = oTable;
+
+    $("#filterInput").keydown(function (e) {
+      if (e.which == 13) {
+        e.preventDefault();
+        return false;
+      }
+    });
+
+    $("#filterInput").keyup(function () {
+      oTable.fnFilter($(this).val());
+    });
+
+    $("a[data-row-selector='true']").jHueRowSelector();
+  });
+</script>
+
+
+${ commonfooter(messages) | n,unicode }

+ 4 - 0
desktop/libs/indexer/src/indexer/urls.py

@@ -20,6 +20,9 @@ from django.conf.urls import patterns, url
 urlpatterns = patterns('indexer.views',
   url(r'^$', 'collections', name='collections'),
   url(r'^install_examples$', 'install_examples', name='install_examples'),
+  
+  # V2
+  url(r'^indexes/$', 'indexes', name='indexes'),
 )
 
 urlpatterns += patterns('indexer.api',
@@ -32,6 +35,7 @@ urlpatterns += patterns('indexer.api',
   url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
   url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
 
+  # V2
   url(r'^api/alias/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/alias/delete/$', 'delete_alias', name='delete_alias')
 )

+ 14 - 0
desktop/libs/indexer/src/indexer/views.py

@@ -16,11 +16,13 @@
 # limitations under the License.
 
 import logging
+import json
 
 from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import JsonResponse, render
 
+from indexer.controller2 import CollectionController
 from indexer.management.commands import indexer_setup
 
 
@@ -31,6 +33,18 @@ def collections(request, is_redirect=False):
   return render('collections.mako', request, {})
 
 
+def indexes(request):
+  searcher = CollectionController(request.user)
+  indexes = searcher.get_indexes()
+  
+  for index in indexes:
+    index['isSelected'] = False
+
+  return render('indexes.mako', request, {
+      'indexes_json': json.dumps(indexes),
+  })
+
+
 def install_examples(request, is_redirect=False):
   result = {'status': -1, 'message': ''}
 

+ 11 - 1
desktop/libs/libsolr/src/libsolr/api.py

@@ -418,7 +418,7 @@ class SolrApi(object):
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
-  def collections(self):
+  def collections(self): # To drop, used in indexer v1
     try:
       params = self._get_params() + (
           ('detail', 'true'),
@@ -430,6 +430,16 @@ class SolrApi(object):
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+  def collections2(self):
+    try:
+      params = self._get_params() + (
+          ('action', 'LIST'),
+          ('wt', 'json'),
+      )
+      return self._root.get('admin/collections', params=params)['collections']
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
   def aliases(self):
     try:
       params = self._get_params() + (