Эх сурвалжийг харах

HUE-5304 [solr] Add tests about dynamically retrieving Solr properties

Romain Rigaux 8 жил өмнө
parent
commit
0d2ab88

+ 7 - 5
desktop/libs/indexer/src/indexer/controller.py

@@ -31,7 +31,7 @@ from libsolr.api import SolrApi
 from libzookeeper.models import ZookeeperClient
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
 from search.conf import SOLR_URL, SECURITY_ENABLED
 
 
-from indexer.conf import CORE_INSTANCE_DIR, get_solr_ensemble
+from indexer.conf import CORE_INSTANCE_DIR
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file
 from indexer.solr_client import SolrClient
 from indexer.solr_client import SolrClient
 
 
@@ -141,10 +141,10 @@ class CollectionManagerController(object):
       self._create_non_solr_cloud_collection(name, fields, unique_key_field, df)
       self._create_non_solr_cloud_collection(name, fields, unique_key_field, df)
 
 
   def _create_solr_cloud_collection(self, name, fields, unique_key_field, df):
   def _create_solr_cloud_collection(self, name, fields, unique_key_field, df):
-    with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
-      root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
+    client = SolrClient(self.user)
 
 
-      client = SolrClient(self.user)
+    with ZookeeperClient(hosts=client.get_zookeeper_host(), read_only=False) as zc:
+      root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
 
 
       tmp_path, solr_config_path = copy_configs(
       tmp_path, solr_config_path = copy_configs(
           fields=fields,
           fields=fields,
@@ -200,6 +200,8 @@ class CollectionManagerController(object):
     Delete solr collection/core and instance dir
     Delete solr collection/core and instance dir
     """
     """
     api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
     api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
+    client = SolrClient(self.user)
+
     if core:
     if core:
       raise PopupException(_('Cannot remove Solr cores.'))
       raise PopupException(_('Cannot remove Solr cores.'))
 
 
@@ -207,7 +209,7 @@ class CollectionManagerController(object):
       # Delete instance directory.
       # Delete instance directory.
       try:
       try:
         root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
         root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
-        with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
+        with ZookeeperClient(hosts=client.get_zookeeper_host(), read_only=False) as zc:
           zc.delete_path(root_node)
           zc.delete_path(root_node)
       except Exception, e:
       except Exception, e:
         # Re-create collection so that we don't have an orphan config
         # Re-create collection so that we don't have an orphan config

+ 44 - 15
desktop/libs/indexer/src/indexer/solr_client.py

@@ -45,6 +45,7 @@ ZK_SOLR_CONFIG_NAMESPACE = 'configs'
 _IS_SOLR_CLOUD = None
 _IS_SOLR_CLOUD = None
 _IS_SOLR_6_OR_MORE = None
 _IS_SOLR_6_OR_MORE = None
 _IS_SOLR_WITH_HDFS = None
 _IS_SOLR_WITH_HDFS = None
+_ZOOKEEPER_HOST = None
 
 
 
 
 class SolrClientException(Exception):
 class SolrClientException(Exception):
@@ -53,20 +54,9 @@ class SolrClientException(Exception):
 
 
 class SolrClient(object):
 class SolrClient(object):
 
 
-  def __init__(self, user):
+  def __init__(self, user, api=None):
     self.user = user
     self.user = user
-    self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
-
-  def _fillup_properties(self):
-    global _IS_SOLR_CLOUD
-    global _IS_SOLR_6_OR_MORE
-    global _IS_SOLR_WITH_HDFS
-
-    properties = self.api.info_system()
-
-    _IS_SOLR_CLOUD = properties.get('mode', 'solrcloud') == 'solrcloud'
-    _IS_SOLR_6_OR_MORE = not str(properties.get('lucene', {}).get('solr-spec-version')).startswith('4.')
-    _IS_SOLR_WITH_HDFS = 'solr.hdfs.home' in str(properties.get('jvm', {}).get('jmx'))
+    self.api = api if api is not None else SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
 
 
 
 
   def get_indexes(self, include_cores=False):
   def get_indexes(self, include_cores=False):
@@ -127,7 +117,7 @@ class SolrClient(object):
 
 
 
 
   def _create_cloud_config(self, name, fields, unique_key_field, df):
   def _create_cloud_config(self, name, fields, unique_key_field, df):
-    with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
+    with ZookeeperClient(hosts=self.get_zookeeper_host(), read_only=False) as zc:
       tmp_path, solr_config_path = copy_configs(
       tmp_path, solr_config_path = copy_configs(
           fields=fields,
           fields=fields,
           unique_key_field=unique_key_field,
           unique_key_field=unique_key_field,
@@ -185,7 +175,7 @@ class SolrClient(object):
       if not keep_config:
       if not keep_config:
         try:
         try:
           root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
           root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
-          with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
+          with ZookeeperClient(hosts=self.get_zookeeper_host(), read_only=False) as zc:
             zc.delete_path(root_node)
             zc.delete_path(root_node)
         except Exception, e:
         except Exception, e:
           # Re-create collection so that we don't have an orphan config
           # Re-create collection so that we don't have an orphan config
@@ -239,6 +229,45 @@ class SolrClient(object):
     return _IS_SOLR_WITH_HDFS
     return _IS_SOLR_WITH_HDFS
 
 
 
 
+  def get_zookeeper_host(self):
+    global _ZOOKEEPER_HOST
+
+    if _ZOOKEEPER_HOST is None:
+      self._fillup_properties()
+
+    return _ZOOKEEPER_HOST
+
+
+  def _fillup_properties(self):
+    global _IS_SOLR_CLOUD
+    global _IS_SOLR_6_OR_MORE
+    global _IS_SOLR_WITH_HDFS
+    global _ZOOKEEPER_HOST
+
+    properties = self.api.info_system()
+
+    _IS_SOLR_CLOUD = properties.get('mode', 'solrcloud') == 'solrcloud'
+    _IS_SOLR_6_OR_MORE = not str(properties.get('lucene', {}).get('solr-spec-version')).startswith('4.')
+    _IS_SOLR_WITH_HDFS = False
+    _ZOOKEEPER_HOST = properties.get('zkHost', get_solr_ensemble())
+
+    command_line_args = properties.get('jvm', {}).get('jmx', {}).get('commandLineArgs', [])
+    for command_line_arg in command_line_args:
+      if not _IS_SOLR_WITH_HDFS and 'solr.hdfs.home' in command_line_arg:
+        _IS_SOLR_WITH_HDFS = True
+      if '-DzkHost=' in command_line_arg:
+        _ZOOKEEPER_HOST = command_line_arg.split('-DzkHost=', 1)[1]
+
+
+  def _reset_properties(self):
+    global _IS_SOLR_CLOUD
+    global _IS_SOLR_6_OR_MORE
+    global _IS_SOLR_WITH_HDFS
+    global _ZOOKEEPER_HOST
+
+    _IS_SOLR_CLOUD = _IS_SOLR_6_OR_MORE = _IS_SOLR_6_OR_MORE = _IS_SOLR_WITH_HDFS = _ZOOKEEPER_HOST = None
+
+
   # Used by morphline indexer
   # Used by morphline indexer
   def get_index_schema(self, index_name):
   def get_index_schema(self, index_name):
     try:
     try:

Файлын зөрүү хэтэрхий том тул дарагдсан байна
+ 68 - 0
desktop/libs/indexer/src/indexer/solr_client_tests.py


+ 2 - 1
desktop/libs/indexer/src/indexer/tests.py

@@ -29,7 +29,8 @@ from hadoop.pseudo_hdfs4 import is_live_cluster, get_db_prefix
 from libsolr import conf as libsolr_conf
 from libsolr import conf as libsolr_conf
 from libzookeeper import conf as libzookeeper_conf
 from libzookeeper import conf as libzookeeper_conf
 
 
-from indexer.controller import get_solr_ensemble, CollectionManagerController
+from indexer.conf import get_solr_ensemble
+from indexer.controller import CollectionManagerController
 
 
 
 
 def test_get_ensemble():
 def test_get_ensemble():

Энэ ялгаанд хэт олон файл өөрчлөгдсөн тул зарим файлыг харуулаагүй болно