Sfoglia il codice sorgente

HUE-3941 [search] Use sentry base solr config for examples when Sentry is on

Romain Rigaux 9 anni fa
parent
commit
961e1526ce

+ 41 - 8
desktop/libs/indexer/src/data/solrconfigs/solrcloud/conf/solrconfig.xml.secure

@@ -35,7 +35,7 @@
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
   -->
-  <luceneMatchVersion>4.4</luceneMatchVersion>
+  <luceneMatchVersion>4.10.3</luceneMatchVersion>
 
   <!-- <lib/> directives can be used to instruct Solr to load an Jars
        identified and use them to resolve any "plugins" specified in
@@ -116,21 +116,28 @@
        solr.RAMDirectoryFactory is memory based, not
        persistent, and doesn't work with replication.
     -->
-  <directoryFactory name="DirectoryFactory" class="org.apache.solr.core.HdfsDirectoryFactory">
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:org.apache.solr.core.HdfsDirectoryFactory}">
     <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
     <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
     <str name="solr.hdfs.security.kerberos.enabled">${solr.hdfs.security.kerberos.enabled:false}</str>
     <str name="solr.hdfs.security.kerberos.keytabfile">${solr.hdfs.security.kerberos.keytabfile:}</str>
     <str name="solr.hdfs.security.kerberos.principal">${solr.hdfs.security.kerberos.principal:}</str>
     <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+    <!-- Enable/Disable using one global cache for all SolrCores.
+    The settings used will be from the first HdfsDirectoryFactory created. -->
+    <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
     <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
     <bool name="solr.hdfs.blockcache.direct.memory.allocation">${solr.hdfs.blockcache.direct.memory.allocation:true}</bool>
     <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:16384}</int>
     <bool name="solr.hdfs.blockcache.read.enabled">${solr.hdfs.blockcache.read.enabled:true}</bool>
     <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
+    <!-- the buffercount is actually the total size in bytes to used for buffer caching -->
+    <int name="solr.hdfs.blockcache.bufferstore.buffercount">${solr.hdfs.blockcache.bufferstore.buffercount:0}</int>
     <bool name="solr.hdfs.nrtcachingdirectory.enable">${solr.hdfs.nrtcachingdirectory.enable:true}</bool>
     <int name="solr.hdfs.nrtcachingdirectory.maxmergesizemb">${solr.hdfs.nrtcachingdirectory.maxmergesizemb:16}</int>
     <int name="solr.hdfs.nrtcachingdirectory.maxcachedmb">${solr.hdfs.nrtcachingdirectory.maxcachedmb:192}</int>
+    <!-- HDFS Block Locality Reporter can be toggled on and off -->
+    <bool name="solr.hdfs.locality.metrics.enabled">${solr.hdfs.locality.metrics.enabled:false}</bool>
   </directoryFactory>
 
   <!-- The CodecFactory for defining the format of the inverted index.
@@ -355,6 +362,7 @@
                 solr data directory.  -->
     <updateLog>
       <str name="dir">${solr.ulog.dir:}</str>
+      <int name="tlogDfsReplication">${solr.ulog.tlogDfsReplication:3}</int>
     </updateLog>
 
     <!-- AutoCommit
@@ -389,7 +397,7 @@
          faster and more near-realtime friendly than a hard commit.
       -->
      <autoSoftCommit>
-       <maxTime>${solr.autoSoftCommit.maxTime:1000}</maxTime>
+       <maxTime>${solr.autoSoftCommit.maxTime:15000}</maxTime>
      </autoSoftCommit>
 
 
@@ -876,6 +884,7 @@
       -->
       <arr name="first-components">
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
       </arr>
 
     </requestHandler>
@@ -890,6 +899,7 @@
      </lst>
      <arr name="first-components">
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
       </arr>
   </requestHandler>
 
@@ -897,7 +907,7 @@
   <!-- realtime get handler, guaranteed to return the latest stored fields of
        any document, without the need to commit or open a new searcher.  The
        current implementation relies on the updateLog feature being enabled. -->
-  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+  <requestHandler name="/get" class="solr.SecureRealTimeGetHandler">
      <lst name="defaults">
        <str name="omitHeader">true</str>
        <str name="wt">json</str>
@@ -905,6 +915,7 @@
      </lst>
      <arr name="first-components">
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
      </arr>
   </requestHandler>
 
@@ -1002,6 +1013,7 @@
 
       <arr name="first-components">
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
       </arr>
 
      <!-- append spellchecking to our list of components -->
@@ -1035,7 +1047,7 @@
          on each Update Request
       -->
 
-       <lst name="defaults">
+       <lst name="invariants">
          <str name="update.chain">updateIndexAuthorization</str>
        </lst>
 
@@ -1046,6 +1058,8 @@
   <requestHandler name="/update/json" class="solr.JsonUpdateRequestHandler">
         <lst name="defaults">
          <str name="stream.contentType">application/json</str>
+       </lst>
+       <lst name="invariants">
          <str name="update.chain">updateIndexAuthorization</str>
        </lst>
 
@@ -1053,6 +1067,8 @@
   <requestHandler name="/update/csv" class="solr.CSVRequestHandler">
         <lst name="defaults">
          <str name="stream.contentType">application/csv</str>
+       </lst>
+       <lst name="invariants">
          <str name="update.chain">updateIndexAuthorization</str>
        </lst>
 
@@ -1338,6 +1354,21 @@
   <searchComponent name="queryIndexAuthorization" class="org.apache.solr.handler.component.QueryIndexAuthorizationComponent" >
   </searchComponent>
 
+  <searchComponent name="queryDocAuthorization" class="org.apache.solr.handler.component.QueryDocAuthorizationComponent" >
+    <!-- Set to true to enabled document-level authorization -->
+    <bool name="enabled">false</bool>
+
+    <!-- Field where the auth tokens are stored in the document -->
+    <str name="sentryAuthField">sentry_auth</str>
+
+    <!-- Auth token defined to allow any role to access the document.
+         Uncomment to enable. -->
+    <!--<str name="allRolesToken">*</str>-->
+  </searchComponent>
+
+  <searchComponent name="secureGet" class="org.apache.solr.handler.component.SecureRealTimeGetComponent" >
+  </searchComponent>
+
   <!-- A request handler for demonstrating the spellcheck component.
 
        NOTE: This is purely as an example.  The whole purpose of the
@@ -1395,6 +1426,7 @@
     </lst>
     <arr name="first-components">
       <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
     </arr>
     <arr name="last-components">
       <str>tvComponent</str>
@@ -1506,6 +1538,7 @@
     </lst>
     <arr name="first-components">
       <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
     </arr>
     <arr name="last-components">
       <str>clustering</str>
@@ -1527,10 +1560,9 @@
       <bool name="terms">true</bool>
       <bool name="distrib">false</bool>
     </lst>
-    <arr name="first-components">
-      <str>queryIndexAuthorization</str>
-    </arr>
     <arr name="components">
+      <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
       <str>terms</str>
     </arr>
   </requestHandler>
@@ -1558,6 +1590,7 @@
     </lst>
     <arr name="first-components">
       <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
     </arr>
     <arr name="last-components">
       <str>elevator</str>

+ 6 - 4
desktop/libs/indexer/src/indexer/api2.py

@@ -45,10 +45,12 @@ def create_index(request):
     searcher = IndexController(request.user)
 
     try:
-      collection = searcher.create_index(name,
-                                         request.POST.get('fields', get_default_fields()),
-                                         request.POST.get('uniqueKeyField', 'id'),
-                                         request.POST.get('df', 'text'))
+      collection = searcher.create_index(
+          name,
+          request.POST.get('fields', get_default_fields()),
+          request.POST.get('uniqueKeyField', 'id'),
+          request.POST.get('df', 'text')
+      )
 
       response['status'] = 0
       response['collection'] = collection

+ 2 - 1
desktop/libs/indexer/src/indexer/controller.py

@@ -28,10 +28,10 @@ from libsolr.api import SolrApi
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
+from search.models import Collection2
 
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file
-from search.models import Collection2
 
 
 LOG = logging.getLogger(__name__)
@@ -160,6 +160,7 @@ class CollectionManagerController(object):
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         try:
           zc.copy_path(root_node, config_root_path)
+
         except Exception, e:
           zc.delete_path(root_node)
           raise PopupException(_('Error in copying Solr configurations.'), detail=e)

+ 5 - 0
desktop/libs/indexer/src/indexer/controller2.py

@@ -27,6 +27,7 @@ from desktop.lib.i18n import smart_str
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.utils import copy_configs
 from libsolr.api import SolrApi
+from libsentry.conf import is_enabled
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
@@ -117,6 +118,10 @@ class IndexController(object):
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         zc.copy_path(root_node, config_root_path)
 
+        if is_enabled():
+          with open(os.path.join(config_root_path, 'solrconfig.xml.secure')) as f:
+            zc.set(os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read())
+
         if not self.api.create_collection(name):
           raise Exception('Failed to create collection: %s' % name)
       except Exception, e:

+ 7 - 2
desktop/libs/indexer/src/indexer/utils.py

@@ -32,11 +32,13 @@ from django.conf import settings
 from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import force_unicode, smart_str
+from libsentry.conf import is_enabled
 
 from indexer import conf
 from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
                            DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
 
+
 LOG = logging.getLogger(__name__)
 TIMESTAMP_PATTERN = '\[([\w\d\s\-\/\:\+]*?)\]'
 FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" />'
@@ -48,6 +50,7 @@ DEFAULT_FIELD = {
   'required': 'true'
 }
 
+
 def get_config_template_path(solr_cloud_mode):
   if solr_cloud_mode:
     return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'solrcloud')
@@ -101,15 +104,17 @@ def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
         f.write(smart_str(schemaxml.xml))
 
     if df:
+      # Use secure template
+      solrconfig = 'conf/solrconfig.xml%s' % ('.secure' if is_enabled() else '')
+
       # Get complete solrconfig.xml
-      with open(os.path.join(config_template_path, 'conf/solrconfig.xml')) as f:
+      with open(os.path.join(config_template_path, solrconfig)) as f:
         solrconfigxml = SolrConfigXml(f.read())
         solrconfigxml.defaultField(df)
 
       # Write complete solrconfig.xml to copy
       with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
         f.write(smart_str(solrconfigxml.xml))
-
     return tmp_path, solr_config_path
   except Exception:
     # Don't leak the tempdir if there was an exception.

+ 11 - 0
desktop/libs/libsentry/src/libsentry/conf.py

@@ -15,11 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 
 from django.utils.translation import ugettext_lazy as _t
 from desktop.lib.conf import Config
 
+LOG = logging.getLogger(__name__)
+
 
 HOSTNAME=Config(
   key='hostname',
@@ -40,3 +43,11 @@ SENTRY_CONF_DIR = Config(
   help=_t('Sentry configuration directory, where sentry-site.xml is located.'),
   default=os.environ.get("SENTRY_CONF_DIR", '/etc/sentry/conf')
 )
+
+
+def is_enabled():
+  try:
+    from search.conf import SECURITY_ENABLED
+    return SECURITY_ENABLED.get()
+  except ImportError, e:
+    LOG.warn("Search app is not enabled")

+ 4 - 0
desktop/libs/libzookeeper/src/libzookeeper/models.py

@@ -86,6 +86,10 @@ class ZookeeperClient(object):
     return self.zk.exists(namespace) is not None
 
 
+  def set(self, path, value, version=-1):
+    return self.zk.set(path, value, version)
+
+
   def copy_path(self, namespace, filepath):
     if self.read_only:
       raise ReadOnlyClientException('Cannot execute copy_path when read_only is set to True.')