Эх сурвалжийг харах

HUE-3941 [search] Use sentry base solr config for examples when Sentry is on

Romain Rigaux 9 жил өмнө
parent
commit
961e1526ce

+ 41 - 8
desktop/libs/indexer/src/data/solrconfigs/solrcloud/conf/solrconfig.xml.secure

@@ -35,7 +35,7 @@
        that you fully re-index after changing this setting as it can
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
        affect both how text is indexed and queried.
   -->
   -->
-  <luceneMatchVersion>4.4</luceneMatchVersion>
+  <luceneMatchVersion>4.10.3</luceneMatchVersion>
 
 
   <!-- <lib/> directives can be used to instruct Solr to load an Jars
   <!-- <lib/> directives can be used to instruct Solr to load an Jars
        identified and use them to resolve any "plugins" specified in
        identified and use them to resolve any "plugins" specified in
@@ -116,21 +116,28 @@
        solr.RAMDirectoryFactory is memory based, not
        solr.RAMDirectoryFactory is memory based, not
        persistent, and doesn't work with replication.
        persistent, and doesn't work with replication.
     -->
     -->
-  <directoryFactory name="DirectoryFactory" class="org.apache.solr.core.HdfsDirectoryFactory">
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:org.apache.solr.core.HdfsDirectoryFactory}">
     <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
     <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
     <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
     <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
     <str name="solr.hdfs.security.kerberos.enabled">${solr.hdfs.security.kerberos.enabled:false}</str>
     <str name="solr.hdfs.security.kerberos.enabled">${solr.hdfs.security.kerberos.enabled:false}</str>
     <str name="solr.hdfs.security.kerberos.keytabfile">${solr.hdfs.security.kerberos.keytabfile:}</str>
     <str name="solr.hdfs.security.kerberos.keytabfile">${solr.hdfs.security.kerberos.keytabfile:}</str>
     <str name="solr.hdfs.security.kerberos.principal">${solr.hdfs.security.kerberos.principal:}</str>
     <str name="solr.hdfs.security.kerberos.principal">${solr.hdfs.security.kerberos.principal:}</str>
     <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
     <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+    <!-- Enable/Disable using one global cache for all SolrCores.
+    The settings used will be from the first HdfsDirectoryFactory created. -->
+    <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
     <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
     <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
     <bool name="solr.hdfs.blockcache.direct.memory.allocation">${solr.hdfs.blockcache.direct.memory.allocation:true}</bool>
     <bool name="solr.hdfs.blockcache.direct.memory.allocation">${solr.hdfs.blockcache.direct.memory.allocation:true}</bool>
     <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:16384}</int>
     <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:16384}</int>
     <bool name="solr.hdfs.blockcache.read.enabled">${solr.hdfs.blockcache.read.enabled:true}</bool>
     <bool name="solr.hdfs.blockcache.read.enabled">${solr.hdfs.blockcache.read.enabled:true}</bool>
     <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
     <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
+    <!-- the buffercount is actually the total size in bytes to used for buffer caching -->
+    <int name="solr.hdfs.blockcache.bufferstore.buffercount">${solr.hdfs.blockcache.bufferstore.buffercount:0}</int>
     <bool name="solr.hdfs.nrtcachingdirectory.enable">${solr.hdfs.nrtcachingdirectory.enable:true}</bool>
     <bool name="solr.hdfs.nrtcachingdirectory.enable">${solr.hdfs.nrtcachingdirectory.enable:true}</bool>
     <int name="solr.hdfs.nrtcachingdirectory.maxmergesizemb">${solr.hdfs.nrtcachingdirectory.maxmergesizemb:16}</int>
     <int name="solr.hdfs.nrtcachingdirectory.maxmergesizemb">${solr.hdfs.nrtcachingdirectory.maxmergesizemb:16}</int>
     <int name="solr.hdfs.nrtcachingdirectory.maxcachedmb">${solr.hdfs.nrtcachingdirectory.maxcachedmb:192}</int>
     <int name="solr.hdfs.nrtcachingdirectory.maxcachedmb">${solr.hdfs.nrtcachingdirectory.maxcachedmb:192}</int>
+    <!-- HDFS Block Locality Reporter can be toggled on and off -->
+    <bool name="solr.hdfs.locality.metrics.enabled">${solr.hdfs.locality.metrics.enabled:false}</bool>
   </directoryFactory>
   </directoryFactory>
 
 
   <!-- The CodecFactory for defining the format of the inverted index.
   <!-- The CodecFactory for defining the format of the inverted index.
@@ -355,6 +362,7 @@
                 solr data directory.  -->
                 solr data directory.  -->
     <updateLog>
     <updateLog>
       <str name="dir">${solr.ulog.dir:}</str>
       <str name="dir">${solr.ulog.dir:}</str>
+      <int name="tlogDfsReplication">${solr.ulog.tlogDfsReplication:3}</int>
     </updateLog>
     </updateLog>
 
 
     <!-- AutoCommit
     <!-- AutoCommit
@@ -389,7 +397,7 @@
          faster and more near-realtime friendly than a hard commit.
          faster and more near-realtime friendly than a hard commit.
       -->
       -->
      <autoSoftCommit>
      <autoSoftCommit>
-       <maxTime>${solr.autoSoftCommit.maxTime:1000}</maxTime>
+       <maxTime>${solr.autoSoftCommit.maxTime:15000}</maxTime>
      </autoSoftCommit>
      </autoSoftCommit>
 
 
 
 
@@ -876,6 +884,7 @@
       -->
       -->
       <arr name="first-components">
       <arr name="first-components">
         <str>queryIndexAuthorization</str>
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
       </arr>
       </arr>
 
 
     </requestHandler>
     </requestHandler>
@@ -890,6 +899,7 @@
      </lst>
      </lst>
      <arr name="first-components">
      <arr name="first-components">
         <str>queryIndexAuthorization</str>
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
       </arr>
       </arr>
   </requestHandler>
   </requestHandler>
 
 
@@ -897,7 +907,7 @@
   <!-- realtime get handler, guaranteed to return the latest stored fields of
   <!-- realtime get handler, guaranteed to return the latest stored fields of
        any document, without the need to commit or open a new searcher.  The
        any document, without the need to commit or open a new searcher.  The
        current implementation relies on the updateLog feature being enabled. -->
        current implementation relies on the updateLog feature being enabled. -->
-  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+  <requestHandler name="/get" class="solr.SecureRealTimeGetHandler">
      <lst name="defaults">
      <lst name="defaults">
        <str name="omitHeader">true</str>
        <str name="omitHeader">true</str>
        <str name="wt">json</str>
        <str name="wt">json</str>
@@ -905,6 +915,7 @@
      </lst>
      </lst>
      <arr name="first-components">
      <arr name="first-components">
         <str>queryIndexAuthorization</str>
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
      </arr>
      </arr>
   </requestHandler>
   </requestHandler>
 
 
@@ -1002,6 +1013,7 @@
 
 
       <arr name="first-components">
       <arr name="first-components">
         <str>queryIndexAuthorization</str>
         <str>queryIndexAuthorization</str>
+        <str>queryDocAuthorization</str>
       </arr>
       </arr>
 
 
      <!-- append spellchecking to our list of components -->
      <!-- append spellchecking to our list of components -->
@@ -1035,7 +1047,7 @@
          on each Update Request
          on each Update Request
       -->
       -->
 
 
-       <lst name="defaults">
+       <lst name="invariants">
          <str name="update.chain">updateIndexAuthorization</str>
          <str name="update.chain">updateIndexAuthorization</str>
        </lst>
        </lst>
 
 
@@ -1046,6 +1058,8 @@
   <requestHandler name="/update/json" class="solr.JsonUpdateRequestHandler">
   <requestHandler name="/update/json" class="solr.JsonUpdateRequestHandler">
         <lst name="defaults">
         <lst name="defaults">
          <str name="stream.contentType">application/json</str>
          <str name="stream.contentType">application/json</str>
+       </lst>
+       <lst name="invariants">
          <str name="update.chain">updateIndexAuthorization</str>
          <str name="update.chain">updateIndexAuthorization</str>
        </lst>
        </lst>
 
 
@@ -1053,6 +1067,8 @@
   <requestHandler name="/update/csv" class="solr.CSVRequestHandler">
   <requestHandler name="/update/csv" class="solr.CSVRequestHandler">
         <lst name="defaults">
         <lst name="defaults">
          <str name="stream.contentType">application/csv</str>
          <str name="stream.contentType">application/csv</str>
+       </lst>
+       <lst name="invariants">
          <str name="update.chain">updateIndexAuthorization</str>
          <str name="update.chain">updateIndexAuthorization</str>
        </lst>
        </lst>
 
 
@@ -1338,6 +1354,21 @@
   <searchComponent name="queryIndexAuthorization" class="org.apache.solr.handler.component.QueryIndexAuthorizationComponent" >
   <searchComponent name="queryIndexAuthorization" class="org.apache.solr.handler.component.QueryIndexAuthorizationComponent" >
   </searchComponent>
   </searchComponent>
 
 
+  <searchComponent name="queryDocAuthorization" class="org.apache.solr.handler.component.QueryDocAuthorizationComponent" >
+    <!-- Set to true to enabled document-level authorization -->
+    <bool name="enabled">false</bool>
+
+    <!-- Field where the auth tokens are stored in the document -->
+    <str name="sentryAuthField">sentry_auth</str>
+
+    <!-- Auth token defined to allow any role to access the document.
+         Uncomment to enable. -->
+    <!--<str name="allRolesToken">*</str>-->
+  </searchComponent>
+
+  <searchComponent name="secureGet" class="org.apache.solr.handler.component.SecureRealTimeGetComponent" >
+  </searchComponent>
+
   <!-- A request handler for demonstrating the spellcheck component.
   <!-- A request handler for demonstrating the spellcheck component.
 
 
        NOTE: This is purely as an example.  The whole purpose of the
        NOTE: This is purely as an example.  The whole purpose of the
@@ -1395,6 +1426,7 @@
     </lst>
     </lst>
     <arr name="first-components">
     <arr name="first-components">
       <str>queryIndexAuthorization</str>
       <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
     </arr>
     </arr>
     <arr name="last-components">
     <arr name="last-components">
       <str>tvComponent</str>
       <str>tvComponent</str>
@@ -1506,6 +1538,7 @@
     </lst>
     </lst>
     <arr name="first-components">
     <arr name="first-components">
       <str>queryIndexAuthorization</str>
       <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
     </arr>
     </arr>
     <arr name="last-components">
     <arr name="last-components">
       <str>clustering</str>
       <str>clustering</str>
@@ -1527,10 +1560,9 @@
       <bool name="terms">true</bool>
       <bool name="terms">true</bool>
       <bool name="distrib">false</bool>
       <bool name="distrib">false</bool>
     </lst>
     </lst>
-    <arr name="first-components">
-      <str>queryIndexAuthorization</str>
-    </arr>
     <arr name="components">
     <arr name="components">
+      <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
       <str>terms</str>
       <str>terms</str>
     </arr>
     </arr>
   </requestHandler>
   </requestHandler>
@@ -1558,6 +1590,7 @@
     </lst>
     </lst>
     <arr name="first-components">
     <arr name="first-components">
       <str>queryIndexAuthorization</str>
       <str>queryIndexAuthorization</str>
+      <str>queryDocAuthorization</str>
     </arr>
     </arr>
     <arr name="last-components">
     <arr name="last-components">
       <str>elevator</str>
       <str>elevator</str>

+ 6 - 4
desktop/libs/indexer/src/indexer/api2.py

@@ -45,10 +45,12 @@ def create_index(request):
     searcher = IndexController(request.user)
     searcher = IndexController(request.user)
 
 
     try:
     try:
-      collection = searcher.create_index(name,
-                                         request.POST.get('fields', get_default_fields()),
-                                         request.POST.get('uniqueKeyField', 'id'),
-                                         request.POST.get('df', 'text'))
+      collection = searcher.create_index(
+          name,
+          request.POST.get('fields', get_default_fields()),
+          request.POST.get('uniqueKeyField', 'id'),
+          request.POST.get('df', 'text')
+      )
 
 
       response['status'] = 0
       response['status'] = 0
       response['collection'] = collection
       response['collection'] = collection

+ 2 - 1
desktop/libs/indexer/src/indexer/controller.py

@@ -28,10 +28,10 @@ from libsolr.api import SolrApi
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.models import ZookeeperClient
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
 from search.conf import SOLR_URL, SECURITY_ENABLED
+from search.models import Collection2
 
 
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file
-from search.models import Collection2
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -160,6 +160,7 @@ class CollectionManagerController(object):
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         try:
         try:
           zc.copy_path(root_node, config_root_path)
           zc.copy_path(root_node, config_root_path)
+
         except Exception, e:
         except Exception, e:
           zc.delete_path(root_node)
           zc.delete_path(root_node)
           raise PopupException(_('Error in copying Solr configurations.'), detail=e)
           raise PopupException(_('Error in copying Solr configurations.'), detail=e)

+ 5 - 0
desktop/libs/indexer/src/indexer/controller2.py

@@ -27,6 +27,7 @@ from desktop.lib.i18n import smart_str
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.utils import copy_configs
 from indexer.utils import copy_configs
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
+from libsentry.conf import is_enabled
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.models import ZookeeperClient
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
 from search.conf import SOLR_URL, SECURITY_ENABLED
@@ -117,6 +118,10 @@ class IndexController(object):
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         config_root_path = '%s/%s' % (solr_config_path, 'conf')
         zc.copy_path(root_node, config_root_path)
         zc.copy_path(root_node, config_root_path)
 
 
+        if is_enabled():
+          with open(os.path.join(config_root_path, 'solrconfig.xml.secure')) as f:
+            zc.set(os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read())
+
         if not self.api.create_collection(name):
         if not self.api.create_collection(name):
           raise Exception('Failed to create collection: %s' % name)
           raise Exception('Failed to create collection: %s' % name)
       except Exception, e:
       except Exception, e:

+ 7 - 2
desktop/libs/indexer/src/indexer/utils.py

@@ -32,11 +32,13 @@ from django.conf import settings
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
 from desktop.lib.i18n import force_unicode, smart_str
 from desktop.lib.i18n import force_unicode, smart_str
+from libsentry.conf import is_enabled
 
 
 from indexer import conf
 from indexer import conf
 from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
 from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
                            DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
                            DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
 
 
+
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 TIMESTAMP_PATTERN = '\[([\w\d\s\-\/\:\+]*?)\]'
 TIMESTAMP_PATTERN = '\[([\w\d\s\-\/\:\+]*?)\]'
 FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" />'
 FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" />'
@@ -48,6 +50,7 @@ DEFAULT_FIELD = {
   'required': 'true'
   'required': 'true'
 }
 }
 
 
+
 def get_config_template_path(solr_cloud_mode):
 def get_config_template_path(solr_cloud_mode):
   if solr_cloud_mode:
   if solr_cloud_mode:
     return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'solrcloud')
     return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'solrcloud')
@@ -101,15 +104,17 @@ def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
         f.write(smart_str(schemaxml.xml))
         f.write(smart_str(schemaxml.xml))
 
 
     if df:
     if df:
+      # Use secure template
+      solrconfig = 'conf/solrconfig.xml%s' % ('.secure' if is_enabled() else '')
+
       # Get complete solrconfig.xml
       # Get complete solrconfig.xml
-      with open(os.path.join(config_template_path, 'conf/solrconfig.xml')) as f:
+      with open(os.path.join(config_template_path, solrconfig)) as f:
         solrconfigxml = SolrConfigXml(f.read())
         solrconfigxml = SolrConfigXml(f.read())
         solrconfigxml.defaultField(df)
         solrconfigxml.defaultField(df)
 
 
       # Write complete solrconfig.xml to copy
       # Write complete solrconfig.xml to copy
       with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
       with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
         f.write(smart_str(solrconfigxml.xml))
         f.write(smart_str(solrconfigxml.xml))
-
     return tmp_path, solr_config_path
     return tmp_path, solr_config_path
   except Exception:
   except Exception:
     # Don't leak the tempdir if there was an exception.
     # Don't leak the tempdir if there was an exception.

+ 11 - 0
desktop/libs/libsentry/src/libsentry/conf.py

@@ -15,11 +15,14 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
+import logging
 import os
 import os
 
 
 from django.utils.translation import ugettext_lazy as _t
 from django.utils.translation import ugettext_lazy as _t
 from desktop.lib.conf import Config
 from desktop.lib.conf import Config
 
 
+LOG = logging.getLogger(__name__)
+
 
 
 HOSTNAME=Config(
 HOSTNAME=Config(
   key='hostname',
   key='hostname',
@@ -40,3 +43,11 @@ SENTRY_CONF_DIR = Config(
   help=_t('Sentry configuration directory, where sentry-site.xml is located.'),
   help=_t('Sentry configuration directory, where sentry-site.xml is located.'),
   default=os.environ.get("SENTRY_CONF_DIR", '/etc/sentry/conf')
   default=os.environ.get("SENTRY_CONF_DIR", '/etc/sentry/conf')
 )
 )
+
+
+def is_enabled():
+  try:
+    from search.conf import SECURITY_ENABLED
+    return SECURITY_ENABLED.get()
+  except ImportError, e:
+    LOG.warn("Search app is not enabled")

+ 4 - 0
desktop/libs/libzookeeper/src/libzookeeper/models.py

@@ -86,6 +86,10 @@ class ZookeeperClient(object):
     return self.zk.exists(namespace) is not None
     return self.zk.exists(namespace) is not None
 
 
 
 
+  def set(self, path, value, version=-1):
+    return self.zk.set(path, value, version)
+
+
   def copy_path(self, namespace, filepath):
   def copy_path(self, namespace, filepath):
     if self.read_only:
     if self.read_only:
       raise ReadOnlyClientException('Cannot execute copy_path when read_only is set to True.')
       raise ReadOnlyClientException('Cannot execute copy_path when read_only is set to True.')