Selaa lähdekoodia

HUE-5304 [indexer] Auto conf with Solr 4 or 6

Romain Rigaux 8 vuotta sitten
vanhempi
commit
067b959

+ 17 - 12
desktop/libs/indexer/src/indexer/api3.py

@@ -181,12 +181,16 @@ def importer_submit(request):
         fields += [{"name": unique_key_field, "type": "string"}]
         kwargs['rowid'] = unique_key_field
 
-      client.create_index(
-          name=index_name,
-          fields=fields,
-          unique_key_field=unique_key_field,
-          df=df
-      )
+      try:
+        client.create_index(
+            name=index_name,
+            fields=fields,
+            unique_key_field=unique_key_field,
+            df=df
+        )
+      except Exception, e:
+        if not 'already exists' in str(e):
+          raise e
 
       data = request.fs.read(source["path"], 0, MAX_UPLOAD_SIZE)
       client.index(name=index_name, data=data, **kwargs)
@@ -251,12 +255,13 @@ def _index(request, file_format, collection_name, query=None, start_time=None, l
   client = SolrClient(user=request.user)
   try:
     client.get_index_schema(collection_name)
-  except SolrClientException:
-    client.create_index(
-      name=collection_name,
-      fields=request.POST.get('fields', schema_fields),
-      unique_key_field=unique_field
-    )
+  except Exception, e:
+    if 'not available' in str(e):
+      client.create_index(
+        name=collection_name,
+        fields=request.POST.get('fields', schema_fields),
+        unique_key_field=unique_field
+      )
 
   if file_format['inputFormat'] == 'table':
     db = dbms.get(request.user)

+ 1 - 1
desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -102,7 +102,7 @@ class MorphlineIndexer(object):
           u'log4j.properties',
           u'--go-live',
           u'--zk-host',
-          ENSEMBLE.get(),
+          ENSEMBLE.get() + '/solr',
           u'--collection',
           collection_name,
           input_path,

+ 19 - 3
desktop/libs/indexer/src/indexer/solr_client.py

@@ -43,7 +43,9 @@ ALLOWED_FIELD_ATTRIBUTES = set(['name', 'type', 'indexed', 'stored'])
 FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored'), ('M', 'multivalued')]
 ZK_SOLR_CONFIG_NAMESPACE = 'configs'
 IS_SOLR_CLOUD = None
-
+IS_SOLR_6_PLUS = None
+IS_SOLR_WITH_HDFS = None
+IS_CDH_SOLR = None
 
 
 class SolrClientException(Exception):
@@ -109,7 +111,7 @@ class SolrClient(object):
           'stored': field.get('stored', True)
         } for field in fields
       ]
-      self.api.add_fields(name, fields)
+      #self.api.add_fields(name, fields)
     else:
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
 
@@ -129,7 +131,10 @@ class SolrClient(object):
       try:
         root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
-        zc.copy_path(root_node, config_root_path)
+        if not zc.path_exists(root_node):
+          zc.copy_path(root_node, config_root_path)
+        else:
+          LOG.warn('Config %s already existing.' % name)
       except Exception, e:
         if zc.path_exists(root_node):
           zc.delete_path(root_node)
@@ -196,3 +201,14 @@ class SolrClient(object):
 
   def delete_alias(self, name):
     return self.api.delete_alias(name)
+
+  # Used by morphline indexer
+  def get_index_schema(self, index_name):
+    try:
+      field_data = self.api.fields(index_name)
+      fields = self._format_flags(field_data['schema']['fields'])
+      uniquekey = self.api.uniquekey(index_name)
+      return uniquekey, fields
+    except Exception, e:
+      LOG.exception(e.message)
+      raise SolrClientException(_("Error in getting schema information for index '%s'" % index_name))

+ 8 - 6
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -657,6 +657,10 @@ ${ assist.assistPanel() }
               <label class="checkbox inline-block" title="${ _('Execute a cluster job to index a large dataset.') }">
                 <input type="checkbox" data-bind="checked: indexerRunJob"> ${_('Index with a job')}
               </label>
+
+              <label for="path" class="control-label" data-bind="visible: indexerRunJob"><div>${ _('Libs') }</div>
+                <input type="text" class="form-control path input-xxlarge" data-bind="value: indexerJobLibPath, filechooser: indexerJobLibPath, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
+              </label>
             </div>
 
             <div class="control-group">
@@ -686,12 +690,6 @@ ${ assist.assistPanel() }
                 </label>
               </div>
 
-              <div class="control-group" data-bind="visible: indexerRunJob">
-                <label for="path" class="control-label" data-bind="indexerRunJob"><div>${ _('Libs') }</div>
-                  <input type="text" class="form-control path input-xxlarge" data-bind="value: indexerJobLibPath, filechooser: indexerJobLibPath, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
-                </label>
-              </div>
-
               <div class="control-group">
                 <label for="indexerNumShards" class="control-label"><div>${ _('Num shards') }</div>
                   <input type="number" class="input-small" placeholder="1" data-bind="value: indexerNumShards">
@@ -1722,6 +1720,7 @@ ${ assist.assistPanel() }
         if (! self.readyToIndex()) {
           return;
         }
+        $(".jHueNotify").hide();
 
 %if not is_embeddable:
         self.indexingStarted(true);
@@ -1736,6 +1735,9 @@ ${ assist.assistPanel() }
             $(document).trigger("error", resp.message);
             self.indexingStarted(false);
             self.isIndexing(false);
+          } else if (resp.on_success_url) {
+            $.jHueNotify.info("${ _('Creation success.') }");
+            huePubSub.publish('open.link', resp.on_success_url);
           } else {
             self.showCreate(true);
             self.editorId(resp.history_id);

+ 1 - 1
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -452,7 +452,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     self.deleteIndexes = function() {
       $.post("${ url('indexer:delete_indexes') }", {
         "indexes": ko.mapping.toJSON(self.selectedIndexes)
-      }, function() {
+      }, function(data) {
         if (data.status == 0) {
           window.location.reload();
         } else {

+ 1 - 1
desktop/libs/indexer/src/indexer/utils.py

@@ -113,7 +113,7 @@ def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
 
     # Use template depending on type of Solr
     solr_config_name = 'solrconfig.xml'
-    if get_properties().get('solr', {}).get('analytics'):
+    if get_properties().get('solr', {}).get('analytics') and False:
       if FS_STORAGE.get() != 'hdfs':
         solr_config_name = 'solrconfig.xml.solr6NonHdfs'
       else: