Просмотр исходного кода

HUE-2127 [indexer] Select a default field

When creating a collection, you can choose the default field.
The edit view doesn't show the default field because there
didn't seem to be a good way to get this information.
Abraham Elmahrek 11 лет назад
Родитель
Сommit
86f0395

+ 1 - 1
desktop/libs/indexer/src/indexer/api.py

@@ -139,7 +139,7 @@ def collections_create(request):
     searcher = CollectionManagerController(request.user)
     searcher = CollectionManagerController(request.user)
 
 
     # Create instance directory, collection, and add fields
     # Create instance directory, collection, and add fields
-    searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'))
+    searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'), collection.get('df'))
 
 
     try:
     try:
       if request.POST.get('source') == 'file':
       if request.POST.get('source') == 'file':

+ 2 - 2
desktop/libs/indexer/src/indexer/controller.py

@@ -78,13 +78,13 @@ class CollectionManagerController(object):
 
 
     return uniquekey, fields
     return uniquekey, fields
 
 
-  def create_collection(self, name, fields, unique_key_field='id'):
+  def create_collection(self, name, fields, unique_key_field='id', df='text'):
     """
     """
     Create solr collection and instance dir.
     Create solr collection and instance dir.
     Create schema.xml file so that we can set UniqueKey field.
     Create schema.xml file so that we can set UniqueKey field.
     """
     """
     # Need to remove path afterwards
     # Need to remove path afterwards
-    tmp_path, solr_config_path = utils.copy_config_with_fields_and_unique_key(fields, unique_key_field)
+    tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df)
 
 
     # Create instance directory.
     # Create instance directory.
     process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--create", name, solr_config_path],
     process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--create", name, solr_config_path],

+ 9 - 5
desktop/libs/indexer/src/indexer/management/commands/indexer_setup.py

@@ -46,7 +46,8 @@ class Command(NoArgsCommand):
     self._setup_collection_from_csv({
     self._setup_collection_from_csv({
       'name': 'twitter_demo',
       'name': 'twitter_demo',
       'fields': self._parse_fields(path),
       'fields': self._parse_fields(path),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'text'
     }, path)
     }, path)
     LOG.info(_("Twitter collection successfully installed"))
     LOG.info(_("Twitter collection successfully installed"))
 
 
@@ -55,7 +56,8 @@ class Command(NoArgsCommand):
     self._setup_collection_from_csv({
     self._setup_collection_from_csv({
       'name': 'yelp_demo',
       'name': 'yelp_demo',
       'fields': self._parse_fields(path),
       'fields': self._parse_fields(path),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'text'
     }, path)
     }, path)
     LOG.info(_("Yelp collection successfully installed"))
     LOG.info(_("Yelp collection successfully installed"))
 
 
@@ -64,7 +66,8 @@ class Command(NoArgsCommand):
     self._setup_collection_from_csv({
     self._setup_collection_from_csv({
       'name': 'jobs_demo',
       'name': 'jobs_demo',
       'fields': self._parse_fields(path),
       'fields': self._parse_fields(path),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'description'
     }, path)
     }, path)
     LOG.info(_("Jobs collection successfully installed"))
     LOG.info(_("Jobs collection successfully installed"))
 
 
@@ -76,13 +79,14 @@ class Command(NoArgsCommand):
         'region_code': 'string',
         'region_code': 'string',
         'referer': 'string'
         'referer': 'string'
       }),
       }),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'record'
     }, path)
     }, path)
     LOG.info(_("Logs collection successfully installed"))
     LOG.info(_("Logs collection successfully installed"))
 
 
   def _setup_collection_from_csv(self, collection, path, separator=',', quote_character='"'):
   def _setup_collection_from_csv(self, collection, path, separator=',', quote_character='"'):
     if not self.searcher.collection_exists(collection['name']):
     if not self.searcher.collection_exists(collection['name']):
-      self.searcher.create_collection(collection['name'], collection['fields'], collection['uniqueKeyField'])
+      self.searcher.create_collection(collection['name'], collection['fields'], collection['uniqueKeyField'], collection['df'])
 
 
     try:
     try:
       hdfs_path = '/tmp/%s' % uuid.uuid4()
       hdfs_path = '/tmp/%s' % uuid.uuid4()

+ 7 - 0
desktop/libs/indexer/src/indexer/templates/collections.mako

@@ -294,6 +294,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
         <th width="0%" class="nowrap">${_('Indexed')}</th>
         <th width="0%" class="nowrap">${_('Indexed')}</th>
         <th width="0%" class="nowrap">${_('Stored')}</th>
         <th width="0%" class="nowrap">${_('Stored')}</th>
         <th width="0%" class="nowrap">${_('Unique Key')}</th>
         <th width="0%" class="nowrap">${_('Unique Key')}</th>
+        <th width="0%" class="nowrap">${_('Default Field')}</th>
         <th width="50%"></th>
         <th width="50%"></th>
       </tr>
       </tr>
     </thead>
     </thead>
@@ -317,6 +318,12 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
             <span class="fa" data-bind="css: {'fa-check': uniqueKeyField}">
             <span class="fa" data-bind="css: {'fa-check': uniqueKeyField}">
           </p>
           </p>
         </td>
         </td>
+        <td>
+          <p class="text-center">
+            <input data-bind="checked: df, visible: !df()" name="unique-key" type="checkbox" />
+            <span class="fa" data-bind="css: {'fa-check': df}">
+          </p>
+        </td>
         <td>
         <td>
           <a data-bind="click: remove, visible: editable" href="javascript:void(0)" class="btn"><i class="fa fa-minus"></i></a>
           <a data-bind="click: remove, visible: editable" href="javascript:void(0)" class="btn"><i class="fa fa-minus"></i></a>
         </td>
         </td>

+ 44 - 26
desktop/libs/indexer/src/indexer/utils.py

@@ -47,38 +47,56 @@ DEFAULT_FIELD = {
   'required': 'true'
   'required': 'true'
 }
 }
 
 
-def schema_xml_with_fields(schema_xml, fields):
-  fields_xml = ''
-  for field in fields:
-    field_dict = DEFAULT_FIELD.copy()
-    field_dict.update(field)
-    fields_xml += FIELD_XML_TEMPLATE % field_dict + '\n'
-  return force_unicode(force_unicode(schema_xml).replace(u'<!-- REPLACE FIELDS -->', force_unicode(fields_xml)))
-
-def schema_xml_with_unique_key_field(schema_xml, unique_key_field):
-  return force_unicode(force_unicode(schema_xml).replace(u'<!-- REPLACE UNIQUE KEY -->', force_unicode(unique_key_field)))
-
-def schema_xml_with_fields_and_unique_key(schema_xml, fields, unique_key_field):
-  return schema_xml_with_unique_key_field(schema_xml_with_fields(schema_xml, fields), unique_key_field)
-
-def example_schema_xml_with_fields_and_unique_key(fields, unique_key_field):
-  # Get complete schema.xml
-  with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
-    return schema_xml_with_fields_and_unique_key(f.read(), fields, unique_key_field)
-
-def copy_config_with_fields_and_unique_key(fields, unique_key_field):
-  # Get complete schema.xml
-  with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
-    schema_xml = schema_xml_with_fields_and_unique_key(f.read(), fields, unique_key_field)
+class SchemaXml(object):
+  def __init__(self, xml):
+    self.xml = xml
 
 
+  def uniqueKeyField(self, unique_key_field):
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE UNIQUE KEY -->', force_unicode(unique_key_field)))
+
+  def fields(self, fields):
+    fields_xml = ''
+    for field in fields:
+      field_dict = DEFAULT_FIELD.copy()
+      field_dict.update(field)
+      fields_xml += FIELD_XML_TEMPLATE % field_dict + '\n'
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE FIELDS -->', force_unicode(fields_xml)))
+
+
+class SolrConfigXml(object):
+  def __init__(self, xml):
+    self.xml = xml
+
+  def defaultField(self, df):
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df)))
+
+
+def copy_configs(fields, unique_key_field, df):
   # Create temporary copy of solr configs
   # Create temporary copy of solr configs
   tmp_path = tempfile.mkdtemp()
   tmp_path = tempfile.mkdtemp()
   solr_config_path = os.path.join(tmp_path, os.path.basename(conf.CONFIG_TEMPLATE_PATH.get()))
   solr_config_path = os.path.join(tmp_path, os.path.basename(conf.CONFIG_TEMPLATE_PATH.get()))
   shutil.copytree(conf.CONFIG_TEMPLATE_PATH.get(), solr_config_path)
   shutil.copytree(conf.CONFIG_TEMPLATE_PATH.get(), solr_config_path)
 
 
-  # Write complete schema.xml to copy
-  with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
-    f.write(smart_str(schema_xml))
+  if fields or unique_key_field:
+    # Get complete schema.xml
+    with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
+      schemaxml = SchemaXml(f.read())
+      schemaxml.uniqueKeyField(unique_key_field)
+      schemaxml.fields(fields)
+
+    # Write complete schema.xml to copy
+    with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
+      f.write(smart_str(schemaxml.xml))
+
+  if df:
+    # Get complete solrconfig.xml
+    with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/solrconfig.xml')) as f:
+      solrconfigxml = SolrConfigXml(f.read())
+      solrconfigxml.defaultField(df)
+
+    # Write complete solrconfig.xml to copy
+    with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
+      f.write(smart_str(solrconfigxml.xml))
 
 
   return tmp_path, solr_config_path
   return tmp_path, solr_config_path
 
 

+ 1 - 0
desktop/libs/indexer/static/js/collections.js

@@ -224,6 +224,7 @@ var CreateCollectionViewModel = function() {
         if (data.status == 0) {
         if (data.status == 0) {
           self.collection.fields(inferFields(data.data, self.collection));
           self.collection.fields(inferFields(data.data, self.collection));
           chooseUniqueKey(self.collection);
           chooseUniqueKey(self.collection);
+          chooseDefaultField(self.collection);
         } else {
         } else {
           $(document).trigger("error", data.message);
           $(document).trigger("error", data.message);
         }
         }

+ 43 - 0
desktop/libs/indexer/static/js/lib.js

@@ -24,6 +24,7 @@ var Collection = function(name) {
 
 
   // Metadata
   // Metadata
   self.uniqueKeyField = ko.observable().extend({'errors': null});
   self.uniqueKeyField = ko.observable().extend({'errors': null});
+  self.df = ko.observable().extend({'errors': null});
   self.hasHueCollection = ko.observable(true).extend({'errors': null});
   self.hasHueCollection = ko.observable(true).extend({'errors': null});
   self.hasSolrCollection = ko.observable(true).extend({'errors': null});
   self.hasSolrCollection = ko.observable(true).extend({'errors': null});
 
 
@@ -71,6 +72,15 @@ var Field = function(collection, name, type, required, indexed, stored) {
       self.indexed(true);
       self.indexed(true);
     }
     }
   });
   });
+  self.df = ko.computed({
+    'read': function() {
+      return collection.df() == self.name();
+    },
+    'write': function(value) {
+      collection.df(self.name());
+      self.indexed(true);
+    }
+  });
   self.editable = ko.computed(function() {
   self.editable = ko.computed(function() {
     return !self.uniqueKeyField() && self.name() != '_version_';
     return !self.uniqueKeyField() && self.name() != '_version_';
   });
   });
@@ -331,6 +341,39 @@ function chooseUniqueKey(collection) {
   fieldChooser(collection.fields());
   fieldChooser(collection.fields());
 }
 }
 
 
+function chooseDefaultField(collection) {
+  function fieldChooser(fields) {
+    if (fields.length > 0) {
+      fields[0].df(true);
+      return true;
+    }
+    return false;
+  }
+
+  // Find a field named "text"
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.name().toLowerCase() == 'text';
+  }))) return;
+
+  // Find a text field
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.type().toLowerCase().substring(0, 4) == 'text';
+  }))) return;
+
+  // Find a string field
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.type().toLowerCase() == 'string';
+  }))) return;
+
+  // Find first indexed field
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.indexed();
+  }))) return;
+
+  // Choose a field
+  fieldChooser(collection.fields());
+}
+
 function getCharacterLabel(character) {
 function getCharacterLabel(character) {
   var LABELS = {
   var LABELS = {
     '\t': '\\t'
     '\t': '\\t'