Explorar o código

HUE-5304 [libsolr] Copy solrconfig.xml accordingly to the type of Solr server

Romain Rigaux %!s(int64=8) %!d(string=hai) anos
pai
achega
377b9b5

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1229,6 +1229,9 @@
   # Default path to Solr in ZooKeeper.
   ## solr_zk_path=/solr
 
+  # If Solr is saving its data on HDFS (hdfs) or local filesystem (fs).
+  ## fa_storage=hdfs
+
 
 ###########################################################################
 # Settings to configure Solr Indexer

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1231,6 +1231,9 @@
   # Default path to Solr in ZooKeeper.
   ## solr_zk_path=/solr
 
+  # If Solr is saving its data on HDFS (hdfs) or local filesystem (fs).
+  ## fa_storage=hdfs
+
 
 ###########################################################################
 # Settings to configure Solr Indexer

+ 0 - 47
desktop/libs/indexer/src/data/solrconfigs/solrcloud/conf/solrconfig.xml2.solr6NonHdfs → desktop/libs/indexer/src/data/solrconfigs/solrcloud/conf/solrconfig.xml.solr6NonHdfs

@@ -116,26 +116,6 @@
        solr.RAMDirectoryFactory is memory based, not
        persistent, and doesn't work with replication.
     -->
-    <!-- 
-  <directoryFactory name="DirectoryFactory" class="org.apache.solr.core.HdfsDirectoryFactory">
-    <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
-    <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
-    <str name="solr.hdfs.security.kerberos.enabled">${solr.hdfs.security.kerberos.enabled:false}</str>
-    <str name="solr.hdfs.security.kerberos.keytabfile">${solr.hdfs.security.kerberos.keytabfile:}</str>
-    <str name="solr.hdfs.security.kerberos.principal">${solr.hdfs.security.kerberos.principal:}</str>
-    <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
-
-    <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
-    <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
-    <bool name="solr.hdfs.blockcache.direct.memory.allocation">${solr.hdfs.blockcache.direct.memory.allocation:true}</bool>
-    <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:16384}</int>
-    <bool name="solr.hdfs.blockcache.read.enabled">${solr.hdfs.blockcache.read.enabled:true}</bool>
-    <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
-    <bool name="solr.hdfs.nrtcachingdirectory.enable">${solr.hdfs.nrtcachingdirectory.enable:true}</bool>
-    <int name="solr.hdfs.nrtcachingdirectory.maxmergesizemb">${solr.hdfs.nrtcachingdirectory.maxmergesizemb:16}</int>
-    <int name="solr.hdfs.nrtcachingdirectory.maxcachedmb">${solr.hdfs.nrtcachingdirectory.maxcachedmb:192}</int>
-  </directoryFactory>
-   -->
 
   <!-- The CodecFactory for defining the format of the inverted index.
        The default implementation is SchemaCodecFactory, which is the official Lucene
@@ -253,9 +233,6 @@
          More details on the nuances of each LockFactory...
          http://wiki.apache.org/lucene-java/AvailableLockFactories
     -->
-    <!--
-      <lockType>${solr.lock.type:hdfs}</lockType>
-    -->
 
     <!-- Unlock On Startup
 
@@ -1464,30 +1441,6 @@
   </requestHandler>
 
 
-  <!-- Query Elevation Component
-
-       http://wiki.apache.org/solr/QueryElevationComponent
-
-       a search component that enables you to configure the top
-       results for a given query regardless of the normal lucene
-       scoring.
-    -->
-  <searchComponent name="elevator" class="solr.QueryElevationComponent" >
-    <!-- pick a fieldType to analyze queries -->
-    <str name="queryFieldType">string</str>
-    <str name="config-file">elevate.xml</str>
-  </searchComponent>
-
-  <!-- A request handler for demonstrating the elevator component -->
-  <requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
-    <lst name="defaults">
-      <str name="echoParams">explicit</str>
-      <str name="df">text</str>
-    </lst>
-    <arr name="last-components">
-      <str>elevator</str>
-    </arr>
-  </requestHandler>
 
   <!-- Highlighting Component
 

+ 5 - 3
desktop/libs/indexer/src/indexer/solr_api.py

@@ -84,14 +84,16 @@ def create_index(request):
 
   name = request.POST.get('name')
   fields = json.loads(request.POST.get('fields', '[]'))
-  unique_key_field = request.POST.get('name')
-  df = request.POST.get('name')
+  unique_key_field = request.POST.get('unique_key_field')
+  df = request.POST.get('df')
 
   client = SolrClient(request.user)
 
   collection = client.create_index(
       name=name,
-      fields=request.POST.get('fields', fields, unique_key_field=unique_key_field, df=df),
+      fields=fields,
+      unique_key_field=unique_key_field,
+      df=df
   )
 
   response['status'] = 0

+ 6 - 8
desktop/libs/indexer/src/indexer/solr_client.py

@@ -23,10 +23,12 @@ import shutil
 
 from django.utils.translation import ugettext as _
 
+from dashboard.conf import get_properties
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
 from libsolr.api import SolrApi
-from libsentry.conf import is_enabled
+from libsolr.conf import FS_STORAGE
+from libsentry.conf import is_enabled as is_sentry_enabled
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
 
@@ -120,7 +122,7 @@ class SolrClient(object):
     if self.is_solr_cloud_mode():
       if config_name is None:
         self._create_cloud_config(name, fields, unique_key_field, df) # Create config set
-      
+
       self.api.create_collection2(name, config_name=config_name)
       fields = [{
           'name': field['name'],
@@ -135,16 +137,12 @@ class SolrClient(object):
 
   def _create_cloud_config(self, name, fields, unique_key_field, df):
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
-      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
+      tmp_path, solr_config_path = copy_configs(fields=fields, unique_key_field=unique_key_field, df=df, solr_cloud_mode=True)
 
       try:
         root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         zc.copy_path(root_node, config_root_path)
-
-        if is_enabled():
-          with open(os.path.join(config_root_path, 'solrconfig.xml.secure')) as f:
-            zc.set(os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read())
       except Exception, e:
         if zc.path_exists(root_node):
           zc.delete_path(root_node)
@@ -185,7 +183,7 @@ class SolrClient(object):
       raise PopupException(_('Cannot remove non-Solr cloud cores.'))
 
     result = self.api.delete_collection(name)
-        
+
     if result['status'] == 0:
       # Delete instance directory.
 #       try:

+ 23 - 15
desktop/libs/indexer/src/indexer/utils.py

@@ -32,10 +32,12 @@ from django.conf import settings
 from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import force_unicode, smart_str
-from libsentry.conf import is_enabled
+from libsentry.conf import is_enabled as is_sentry_enabled
 
 from indexer import conf
 from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES, DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
+from dashboard.conf import get_properties
+from libsolr.conf import FS_STORAGE
 
 
 LOG = logging.getLogger(__name__)
@@ -84,8 +86,8 @@ class SolrConfigXml(object):
   def __init__(self, xml):
     self.xml = xml
 
-  def defaultField(self, df):
-    self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df)))
+  def defaultField(self, df=None):
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df) if df is not None else ''))
 
 
 def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
@@ -109,18 +111,24 @@ def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
       with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
         f.write(smart_str(schemaxml.xml))
 
-    if df:
-      # Use secure template
-      solrconfig = 'conf/solrconfig.xml%s' % ('.secure' if is_enabled() else '')
-
-      # Get complete solrconfig.xml
-      with open(os.path.join(config_template_path, solrconfig)) as f:
-        solrconfigxml = SolrConfigXml(f.read())
-        solrconfigxml.defaultField(df)
-
-      # Write complete solrconfig.xml to copy
-      with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
-        f.write(smart_str(solrconfigxml.xml))
+    # Use template depending on type of Solr
+    solr_config_name = 'solrconfig.xml'
+    if get_properties().get('solr', {}).get('analytics'):
+      if FS_STORAGE.get() != 'hdfs':
+        solr_config_name = 'solrconfig.xml.solr6NonHdfs'
+      else:
+        solr_config_name = 'solrconfig.xml.solr6'
+    if is_sentry_enabled():
+      solr_config_name = 'solrconfig.xml.secure'      
+    solrconfig = 'conf/%s' % solr_config_name
+
+    # Get complete solrconfig.xml
+    with open(os.path.join(config_template_path, solrconfig)) as f:
+      solrconfigxml = SolrConfigXml(f.read())
+      solrconfigxml.defaultField(df)
+
+    with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
+      f.write(smart_str(solrconfigxml.xml))
     return tmp_path, solr_config_path
   except Exception:
     # Don't leak the tempdir if there was an exception.

+ 7 - 0
desktop/libs/libsolr/src/libsolr/conf.py

@@ -56,3 +56,10 @@ SOLR_ZK_PATH = Config(
   dynamic_default=zkensemble_path,
   type=str
 )
+
+FS_STORAGE = Config(
+  key="fa_storage",
+  help=_t("If Solr is saving its data on HDFS (hdfs) or local filesystem (fs)."),
+  default='hdfs',
+  type=str
+)