浏览代码

HUE-5304 [indexer] Auto conf with Solr in HDFS mode or not

Romain Rigaux 8 年之前
父节点
当前提交
4c34fa2bae

+ 2 - 1
desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -25,6 +25,7 @@ from django.utils.translation import ugettext as _
 from mako.lookup import TemplateLookup
 
 from desktop.models import Document2
+from libsolr.conf import SOLR_ZK_PATH
 from libzookeeper.conf import ENSEMBLE
 from notebook.connectors.base import get_api
 from notebook.models import Notebook, make_notebook
@@ -102,7 +103,7 @@ class MorphlineIndexer(object):
           u'log4j.properties',
           u'--go-live',
           u'--zk-host',
-          ENSEMBLE.get() + '/solr',
+          ENSEMBLE.get() + SOLR_ZK_PATH.get(),
           u'--collection',
           collection_name,
           input_path,

+ 51 - 13
desktop/libs/indexer/src/indexer/solr_client.py

@@ -42,10 +42,10 @@ MAX_UPLOAD_SIZE = 100 * 1024 * 1024 # 100 MB
 ALLOWED_FIELD_ATTRIBUTES = set(['name', 'type', 'indexed', 'stored'])
 FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored'), ('M', 'multivalued')]
 ZK_SOLR_CONFIG_NAMESPACE = 'configs'
-IS_SOLR_CLOUD = None
-IS_SOLR_6_PLUS = None
-IS_SOLR_WITH_HDFS = None
-IS_CDH_SOLR = None
+
+_IS_SOLR_CLOUD = None
+_IS_SOLR_6_OR_MORE = None
+_IS_SOLR_WITH_HDFS = None
 
 
 class SolrClientException(Exception):
@@ -58,14 +58,16 @@ class SolrClient(object):
     self.user = user
     self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
 
+  def _fillup_properties(self):
+    global _IS_SOLR_CLOUD
+    global _IS_SOLR_6_OR_MORE
+    global _IS_SOLR_WITH_HDFS
 
-  def is_solr_cloud_mode(self):
-    global IS_SOLR_CLOUD
-
-    if IS_SOLR_CLOUD is None:
-      IS_SOLR_CLOUD = self.api.info_system().get('mode', 'solrcloud') == 'solrcloud'
+    properties = self.api.info_system()
 
-    return IS_SOLR_CLOUD
+    _IS_SOLR_CLOUD = properties.get('mode', 'solrcloud') == 'solrcloud'
+    _IS_SOLR_6_OR_MORE = not str(properties.get('lucene', {}).get('solr-spec-version')).startswith('4.')
+    _IS_SOLR_WITH_HDFS = 'solr.hdfs.home' in str(properties.get('jvm', {}).get('jmx'))
 
 
   def get_indexes(self, include_cores=False):
@@ -111,7 +113,8 @@ class SolrClient(object):
           'stored': field.get('stored', True)
         } for field in fields
       ]
-      #self.api.add_fields(name, fields)
+      if self.is_solr_six():
+        self.api.add_fields(name, fields)
     else:
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
 
@@ -126,7 +129,14 @@ class SolrClient(object):
 
   def _create_cloud_config(self, name, fields, unique_key_field, df):
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
-      tmp_path, solr_config_path = copy_configs(fields=fields, unique_key_field=unique_key_field, df=df, solr_cloud_mode=True)
+      tmp_path, solr_config_path = copy_configs(
+          fields=fields,
+          unique_key_field=unique_key_field,
+          df=df,
+          solr_cloud_mode=True,
+          is_solr_six_or_more=self.is_solr_six_or_more(),
+          is_solr_hdfs_mode=self.is_solr_with_hdfs()
+      )
 
       try:
         root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
@@ -194,7 +204,7 @@ class SolrClient(object):
   def list_configs(self):
     return self.api.configs()
 
-  
+
   def list_schema(self, index_name):
     return self.api.get_schema(index_name)
 
@@ -202,6 +212,34 @@ class SolrClient(object):
   def delete_alias(self, name):
     return self.api.delete_alias(name)
 
+
+  def is_solr_cloud_mode(self):
+    global _IS_SOLR_CLOUD
+
+    if _IS_SOLR_CLOUD is None:
+      self._fillup_properties()
+
+    return _IS_SOLR_CLOUD
+
+
+  def is_solr_six_or_more(self):
+    global _IS_SOLR_6_OR_MORE
+
+    if _IS_SOLR_6_OR_MORE is None:
+      self._fillup_properties()
+
+    return _IS_SOLR_6_OR_MORE
+
+
+  def is_solr_with_hdfs(self):
+    global _IS_SOLR_WITH_HDFS
+
+    if _IS_SOLR_WITH_HDFS is None:
+      self._fillup_properties()
+
+    return _IS_SOLR_WITH_HDFS
+
+
   # Used by morphline indexer
   def get_index_schema(self, index_name):
     try:

+ 7 - 6
desktop/libs/indexer/src/indexer/utils.py

@@ -32,9 +32,7 @@ from django.conf import settings
 from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import force_unicode, smart_str
-from dashboard.conf import get_properties
 from libsentry.conf import is_enabled as is_sentry_enabled
-from libsolr.conf import FS_STORAGE
 
 from indexer import conf
 from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES, DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
@@ -90,7 +88,7 @@ class SolrConfigXml(object):
     self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df) if df is not None else ''))
 
 
-def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
+def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True, is_solr_six_or_more=False, is_solr_hdfs_mode=True):
   # Create temporary copy of solr configs
   tmp_path = tempfile.mkdtemp()
 
@@ -113,13 +111,16 @@ def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
 
     # Use template depending on type of Solr
     solr_config_name = 'solrconfig.xml'
-    if get_properties().get('solr', {}).get('analytics') and False:
-      if FS_STORAGE.get() != 'hdfs':
+
+    if is_solr_six_or_more:
+      if is_solr_hdfs_mode:
         solr_config_name = 'solrconfig.xml.solr6NonHdfs'
       else:
         solr_config_name = 'solrconfig.xml.solr6'
+
     if is_sentry_enabled():
-      solr_config_name = 'solrconfig.xml.secure'
+      solr_config_name += '.secure'
+
     solrconfig = 'conf/%s' % solr_config_name
 
     # Get complete solrconfig.xml

+ 2 - 2
desktop/libs/libsolr/src/libsolr/conf.py

@@ -42,8 +42,8 @@ def zkensemble_path():
   """
   try:
     parsed = urlparse(ENSEMBLE.get())
-    if parsed.port == 9983:
-      return ENSEMBLE.get()
+    if parsed.port == 9983: # Standalone Solr cloud
+      return ''
   except:
     LOG.warn('Failed to get Zookeeper ensemble path')