|
|
@@ -35,7 +35,7 @@ from search.conf import SOLR_URL, SECURITY_ENABLED
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
MAX_UPLOAD_SIZE = 100 * 1024 * 1024 # 100 MB
|
|
|
ALLOWED_FIELD_ATTRIBUTES = set(['name', 'type', 'indexed', 'stored'])
|
|
|
-FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored')]
|
|
|
+FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored'), ('M', 'multivalued')]
|
|
|
ZK_SOLR_CONFIG_NAMESPACE = 'configs'
|
|
|
|
|
|
|
|
|
@@ -43,7 +43,11 @@ def get_solr_ensemble():
|
|
|
return '%s/solr' % ENSEMBLE.get()
|
|
|
|
|
|
|
|
|
-class CollectionController(object):
|
|
|
+class IndexControllerException(Exception):
|
|
|
+ pass
|
|
|
+
|
|
|
+
|
|
|
+class IndexController(object):
|
|
|
"""
|
|
|
Glue the models to the views.
|
|
|
"""
|
|
|
@@ -51,13 +55,7 @@ class CollectionController(object):
|
|
|
self.user = user
|
|
|
self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
|
|
|
|
|
|
-# def _format_flags(self, fields):
|
|
|
-# for field_name, field in fields.items():
|
|
|
-# for index in range(0, len(FLAGS)):
|
|
|
-# flags = FLAGS[index]
|
|
|
-# field[flags[1]] = field['flags'][index] == FLAGS[index][0]
|
|
|
-# return fields
|
|
|
-#
|
|
|
+
|
|
|
def is_solr_cloud_mode(self):
|
|
|
if not hasattr(self, '_solr_cloud_mode'):
|
|
|
try:
|
|
|
@@ -96,67 +94,63 @@ class CollectionController(object):
|
|
|
|
|
|
return indexes
|
|
|
|
|
|
- def create_collection(self, name, fields, unique_key_field='id', df='text'):
|
|
|
+
|
|
|
+ def create_index(self, name, fields, unique_key_field='id', df='text'):
|
|
|
"""
|
|
|
Create solr collection or core and instance dir.
|
|
|
Create schema.xml file so that we can set UniqueKey field.
|
|
|
"""
|
|
|
if self.is_solr_cloud_mode():
|
|
|
- # Need to remove path afterwards
|
|
|
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
|
|
|
|
|
|
- zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
|
|
|
- root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
|
|
|
- config_root_path = '%s/%s' % (solr_config_path, 'conf')
|
|
|
try:
|
|
|
+ zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
|
|
|
+ root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
|
|
|
+ config_root_path = '%s/%s' % (solr_config_path, 'conf')
|
|
|
zc.copy_path(root_node, config_root_path)
|
|
|
- except Exception, e:
|
|
|
- zc.delete_path(root_node)
|
|
|
- raise PopupException(_('Error in copying Solr configurations.'), detail=e)
|
|
|
-
|
|
|
- # Don't want directories laying around
|
|
|
- shutil.rmtree(tmp_path)
|
|
|
|
|
|
- if not self.api.create_collection(name):
|
|
|
- # Delete instance directory if we couldn't create a collection.
|
|
|
- try:
|
|
|
+ if not self.api.create_collection(name):
|
|
|
+ raise Exception('Failed to create collection: %s' % name)
|
|
|
+ except Exception, e:
|
|
|
+ if zc.path_exists(root_node):
|
|
|
+ # Remove the root node from Zookeeper
|
|
|
zc.delete_path(root_node)
|
|
|
- except Exception, e:
|
|
|
- raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
|
|
|
+ raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
|
|
|
+ finally:
|
|
|
+ # Remove tmp config directory
|
|
|
+ shutil.rmtree(tmp_path)
|
|
|
+
|
|
|
else: # Non-solrcloud mode
|
|
|
# Create instance directory locally.
|
|
|
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
|
|
|
+
|
|
|
if os.path.exists(instancedir):
|
|
|
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
|
|
|
- tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
|
|
|
- shutil.move(solr_config_path, instancedir)
|
|
|
- shutil.rmtree(tmp_path)
|
|
|
|
|
|
- if not self.api.create_core(name, instancedir):
|
|
|
- # Delete instance directory if we couldn't create a collection.
|
|
|
+ try:
|
|
|
+ tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
|
|
|
+ shutil.move(solr_config_path, instancedir)
|
|
|
+ shutil.rmtree(tmp_path)
|
|
|
+
|
|
|
+ if not self.api.create_core(name, instancedir):
|
|
|
+ raise Exception('Failed to create core: %s' % name)
|
|
|
+ except Exception, e:
|
|
|
+ raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
|
|
|
+ finally:
|
|
|
+ # Delete instance directory if we couldn't create the core.
|
|
|
shutil.rmtree(instancedir)
|
|
|
- raise PopupException(_('Could not create collection. Check error logs for more info.'))
|
|
|
|
|
|
return name
|
|
|
-
|
|
|
-# def get_fields(self, collection_or_core_name):
|
|
|
-# try:
|
|
|
-# field_data = self.api.fields(collection_or_core_name)
|
|
|
-# fields = self._format_flags(field_data['schema']['fields'])
|
|
|
-# except:
|
|
|
-# LOG.exception(_('Could not fetch fields for collection %s.') % collection_or_core_name)
|
|
|
-# raise PopupException(_('Could not fetch fields for collection %s. See logs for more info.') % collection_or_core_name)
|
|
|
-#
|
|
|
-# try:
|
|
|
-# uniquekey = self.api.uniquekey(collection_or_core_name)
|
|
|
-# except:
|
|
|
-# LOG.exception(_('Could not fetch unique key for collection %s.') % collection_or_core_name)
|
|
|
-# raise PopupException(_('Could not fetch unique key for collection %s. See logs for more info.') % collection_or_core_name)
|
|
|
-#
|
|
|
-# return uniquekey, fields
|
|
|
#
|
|
|
|
|
|
- def delete_collection(self, name):
|
|
|
+ def delete_index(self, name):
|
|
|
+ """
|
|
|
+ Delete solr collection/core and instance dir
|
|
|
+ """
|
|
|
+ # TODO: Implement deletion of local Solr cores
|
|
|
+ if not self.is_solr_cloud_mode():
|
|
|
+ raise PopupException(_('Cannot remove non-Solr cloud cores.'))
|
|
|
+
|
|
|
if self.api.remove_collection(name):
|
|
|
# Delete instance directory.
|
|
|
try:
|
|
|
@@ -170,5 +164,28 @@ class CollectionController(object):
|
|
|
else:
|
|
|
raise PopupException(_('Could not remove collection. Check error logs for more info.'))
|
|
|
|
|
|
+
|
|
|
+ def get_index_schema(self, index_name):
|
|
|
+ """
|
|
|
+ Returns a tuple of the unique key and schema fields for a given index
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ field_data = self.api.fields(index_name)
|
|
|
+ fields = self._format_flags(field_data['schema']['fields'])
|
|
|
+ uniquekey = self.api.uniquekey(index_name)
|
|
|
+ return uniquekey, fields
|
|
|
+ except Exception, e:
|
|
|
+ LOG.exception(e.message)
|
|
|
+ raise IndexControllerException(_("Error in getting schema information for index '%s'" % index_name))
|
|
|
+
|
|
|
+
|
|
|
def delete_alias(self, name):
|
|
|
return self.api.delete_alias(name)
|
|
|
+
|
|
|
+
|
|
|
+ def _format_flags(self, fields):
|
|
|
+ for field_name, field in fields.items():
|
|
|
+ for index in range(0, len(FLAGS)):
|
|
|
+ flags = FLAGS[index]
|
|
|
+ field[flags[1]] = field['flags'][index] == FLAGS[index][0]
|
|
|
+ return fields
|