|
|
@@ -20,10 +20,13 @@ import json
|
|
|
import logging
|
|
|
import urllib
|
|
|
|
|
|
+from django.utils.translation import ugettext as _
|
|
|
+
|
|
|
from desktop.lib.exceptions_renderable import PopupException
|
|
|
from desktop.lib.rest.http_client import HttpClient, RestException
|
|
|
from desktop.lib.rest import resource
|
|
|
-from django.utils.translation import ugettext as _
|
|
|
+
|
|
|
+from search.conf import EMPTY_QUERY, SECURITY_ENABLED
|
|
|
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
@@ -39,7 +42,7 @@ class SolrApi(object):
|
|
|
"""
|
|
|
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
|
|
|
"""
|
|
|
- def __init__(self, solr_url, user, security_enabled=False):
|
|
|
+ def __init__(self, solr_url, user, security_enabled=SECURITY_ENABLED.get()):
|
|
|
self._url = solr_url
|
|
|
self._user = user
|
|
|
self._client = HttpClient(self._url, logger=LOG)
|
|
|
@@ -53,17 +56,89 @@ class SolrApi(object):
|
|
|
return (('doAs', self._user ),)
|
|
|
return (('user.name', DEFAULT_USER), ('doAs', self._user),)
|
|
|
|
|
|
- @classmethod
|
|
|
- def _get_json(cls, response):
|
|
|
- if type(response) != dict:
|
|
|
- # Got 'plain/text' mimetype instead of 'application/json'
|
|
|
- try:
|
|
|
- response = json.loads(response)
|
|
|
- except ValueError, e:
|
|
|
- # Got some null bytes in the response
|
|
|
- LOG.error('%s: %s' % (unicode(e), repr(response)))
|
|
|
- response = json.loads(response.replace('\x00', ''))
|
|
|
- return response
|
|
|
+
|
|
|
+ def query(self, collection, query):
|
|
|
+ solr_query = {}
|
|
|
+
|
|
|
+ solr_query['collection'] = collection['name']
|
|
|
+ solr_query['rows'] = min(int(collection['template']['rows'] or 10), 1000)
|
|
|
+ solr_query['start'] = min(int(query['start']), 10000)
|
|
|
+
|
|
|
+ q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
|
|
|
+
|
|
|
+ params = self._get_params() + (
|
|
|
+ ('q', 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']])),
|
|
|
+ ('wt', 'json'),
|
|
|
+ ('rows', solr_query['rows']),
|
|
|
+ ('start', solr_query['start']),
|
|
|
+ )
|
|
|
+
|
|
|
+ if any(collection['facets']):
|
|
|
+ params += (
|
|
|
+ ('facet', 'true'),
|
|
|
+ ('facet.mincount', 0),
|
|
|
+ ('facet.limit', 10),
|
|
|
+ )
|
|
|
+ for facet in collection['facets']:
|
|
|
+ if facet['type'] == 'query':
|
|
|
+ params += (('facet.query', '%s' % facet['field']),)
|
|
|
+ elif facet['type'] == 'range':
|
|
|
+ params += tuple([
|
|
|
+ ('facet.range', '{!ex=%s}%s' % (facet['field'], facet['field'])),
|
|
|
+ ('f.%s.facet.range.start' % facet['field'], facet['properties']['start']),
|
|
|
+ ('f.%s.facet.range.end' % facet['field'], facet['properties']['end']),
|
|
|
+ ('f.%s.facet.range.gap' % facet['field'], facet['properties']['gap']),
|
|
|
+ ('f.%s.facet.mincount' % facet['field'], facet['properties']['mincount']),]
|
|
|
+ )
|
|
|
+ elif facet['type'] == 'field':
|
|
|
+ params += (
|
|
|
+ ('facet.field', '{!ex=%s}%s' % (facet['field'], facet['field'])),
|
|
|
+ ('f.%s.facet.limit' % facet['field'], int(facet['properties'].get('limit', 10)) + 1),
|
|
|
+ ('f.%s.facet.mincount' % facet['field'], int(facet['properties']['mincount'])),
|
|
|
+ )
|
|
|
+
|
|
|
+ for fq in query['fqs']:
|
|
|
+ if fq['type'] == 'field':
|
|
|
+ # This does not work if spaces in Solr:
|
|
|
+ # params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),)
|
|
|
+ f = []
|
|
|
+ for _filter in fq['filter']:
|
|
|
+ if _filter is not None and ' ' in _filter:
|
|
|
+ f.append('%s:"%s"' % (fq['field'], _filter))
|
|
|
+ else:
|
|
|
+ f.append('{!field f=%s}%s' % (fq['field'], _filter))
|
|
|
+ params += (('fq', urllib.unquote(utf_quoter('{!tag=%s}' % fq['field'] + ' '.join(f)))),)
|
|
|
+ elif fq['type'] == 'range':
|
|
|
+ params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([urllib.unquote(utf_quoter('%s:[%s TO %s}' % (fq['field'], f['from'], f['to']))) for f in fq['properties']])),)
|
|
|
+
|
|
|
+ if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
|
|
|
+ fields = collection['template']['fieldsSelected'] + [collection['idField']] if collection['idField'] else []
|
|
|
+ params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))),)
|
|
|
+ else:
|
|
|
+ params += (('fl', '*'),)
|
|
|
+
|
|
|
+ params += (
|
|
|
+ ('hl', 'true'),
|
|
|
+ ('hl.fl', '*'),
|
|
|
+ ('hl.snippets', 3)
|
|
|
+ )
|
|
|
+
|
|
|
+ if collection['template']['fieldsSelected']:
|
|
|
+ fields = []
|
|
|
+ for field in collection['template']['fieldsSelected']:
|
|
|
+ attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
|
|
|
+ if attribute_field:
|
|
|
+ if attribute_field[0]['sort']['direction']:
|
|
|
+ fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
|
|
|
+ if fields:
|
|
|
+ params += (
|
|
|
+ ('sort', ','.join(fields)),
|
|
|
+ )
|
|
|
+
|
|
|
+ response = self._root.get('%(collection)s/select' % solr_query, params)
|
|
|
+
|
|
|
+ return self._get_json(response)
|
|
|
+
|
|
|
|
|
|
def suggest(self, solr_query, hue_core):
|
|
|
try:
|
|
|
@@ -78,6 +153,7 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
def collections(self):
|
|
|
try:
|
|
|
params = self._get_params() + (
|
|
|
@@ -85,16 +161,18 @@ class SolrApi(object):
|
|
|
('path', '/clusterstate.json'),
|
|
|
)
|
|
|
response = self._root.get('zookeeper', params=params)
|
|
|
- return json.loads(response['znode']['data'])
|
|
|
+ return json.loads(response['znode'].get('data', '{}'))
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
def collection_or_core(self, hue_collection):
|
|
|
if hue_collection.is_core_only:
|
|
|
return self.core(hue_collection.name)
|
|
|
else:
|
|
|
return self.collection(hue_collection.name)
|
|
|
|
|
|
+
|
|
|
def collection(self, name):
|
|
|
try:
|
|
|
collections = self.collections()
|
|
|
@@ -102,52 +180,6 @@ class SolrApi(object):
|
|
|
except Exception, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
- def create_collection(self, name, shards=1, replication=1):
|
|
|
- try:
|
|
|
- params = (
|
|
|
- ('action', 'CREATE'),
|
|
|
- ('name', name),
|
|
|
- ('numShards', shards),
|
|
|
- ('replicationFactor', replication),
|
|
|
- ('collection.configName', name),
|
|
|
- ('wt', 'json')
|
|
|
- )
|
|
|
-
|
|
|
- response = self._root.post('admin/collections', params=params)
|
|
|
- if 'success' in response:
|
|
|
- return True
|
|
|
- else:
|
|
|
- LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
|
|
|
- return False
|
|
|
- except RestException, e:
|
|
|
- raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
-
|
|
|
- def remove_collection(self, name, replication=1):
|
|
|
- try:
|
|
|
- params = (
|
|
|
- ('action', 'DELETE'),
|
|
|
- ('name', name),
|
|
|
- ('replicationFactor', replication),
|
|
|
- ('wt', 'json')
|
|
|
- )
|
|
|
-
|
|
|
- response = self._root.post('admin/collections', params=params)
|
|
|
- if 'success' in response:
|
|
|
- return True
|
|
|
- else:
|
|
|
- LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
|
|
|
- return False
|
|
|
- except RestException, e:
|
|
|
- raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
-
|
|
|
- def add_fields(self, collection, fields):
|
|
|
- try:
|
|
|
-
|
|
|
- response = self._root.post('%s/schema/fields' % collection, data=json.dumps(fields))
|
|
|
-
|
|
|
- return response
|
|
|
- except RestException, e:
|
|
|
- raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
def cores(self):
|
|
|
try:
|
|
|
@@ -158,6 +190,7 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
def core(self, core):
|
|
|
try:
|
|
|
params = self._get_params() + (
|
|
|
@@ -168,6 +201,7 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
def schema(self, core):
|
|
|
try:
|
|
|
params = self._get_params() + (
|
|
|
@@ -178,6 +212,7 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
def fields(self, core, dynamic=False):
|
|
|
try:
|
|
|
params = self._get_params() + (
|
|
|
@@ -191,6 +226,116 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
+ def luke(self, core):
|
|
|
+ try:
|
|
|
+ params = self._get_params() + (
|
|
|
+ ('wt', 'json'),
|
|
|
+ )
|
|
|
+ response = self._root.get('%(core)s/admin/luke' % {'core': core}, params=params)
|
|
|
+ return self._get_json(response)
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
+ def schema_fields(self, core):
|
|
|
+ try:
|
|
|
+ params = self._get_params() + (
|
|
|
+ ('wt', 'json'),
|
|
|
+ )
|
|
|
+ response = self._root.get('%(core)s/schema/fields' % {'core': core}, params=params)
|
|
|
+ return self._get_json(response)
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
+ def stats(self, core, fields):
|
|
|
+ try:
|
|
|
+ params = (
|
|
|
+ ('q', EMPTY_QUERY.get()),
|
|
|
+ ('wt', 'json'),
|
|
|
+ ('rows', 0),
|
|
|
+ ('stats', 'true'),
|
|
|
+ )
|
|
|
+ params += tuple([('stats.field', field) for field in fields])
|
|
|
+ response = self._root.get('%(core)s/select' % {'core': core}, params=params)
|
|
|
+ return self._get_json(response)
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
+ def get(self, core, doc_id):
|
|
|
+ try:
|
|
|
+ params = (
|
|
|
+ ('id', doc_id),
|
|
|
+ ('wt', 'json'),
|
|
|
+ )
|
|
|
+ response = self._root.get('%(core)s/get' % {'core': core}, params=params)
|
|
|
+ return self._get_json(response)
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def _get_json(cls, response):
|
|
|
+ if type(response) != dict:
|
|
|
+ # Got 'plain/text' mimetype instead of 'application/json'
|
|
|
+ try:
|
|
|
+ response = json.loads(response)
|
|
|
+ except ValueError, e:
|
|
|
+ # Got some null bytes in the response
|
|
|
+ LOG.error('%s: %s' % (unicode(e), repr(response)))
|
|
|
+ response = json.loads(response.replace('\x00', ''))
|
|
|
+ return response
|
|
|
+
|
|
|
+
|
|
|
+ def create_collection(self, name, shards=1, replication=1):
|
|
|
+ try:
|
|
|
+ params = (
|
|
|
+ ('action', 'CREATE'),
|
|
|
+ ('name', name),
|
|
|
+ ('numShards', shards),
|
|
|
+ ('replicationFactor', replication),
|
|
|
+ ('collection.configName', name),
|
|
|
+ ('wt', 'json')
|
|
|
+ )
|
|
|
+
|
|
|
+ response = self._root.post('admin/collections', contenttype='application/text', params=params)
|
|
|
+ if 'success' in response:
|
|
|
+ return True
|
|
|
+ else:
|
|
|
+ LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
|
|
|
+ return False
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
+ def remove_collection(self, name, replication=1):
|
|
|
+ try:
|
|
|
+ params = (
|
|
|
+ ('action', 'DELETE'),
|
|
|
+ ('name', name),
|
|
|
+ ('replicationFactor', replication),
|
|
|
+ ('wt', 'json')
|
|
|
+ )
|
|
|
+
|
|
|
+ response = self._root.post('admin/collections', params=params)
|
|
|
+ if 'success' in response:
|
|
|
+ return True
|
|
|
+ else:
|
|
|
+ LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
|
|
|
+ return False
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
+ def add_fields(self, collection, fields):
|
|
|
+ try:
|
|
|
+ return self._root.post('%s/schema/fields' % collection, data=json.dumps(fields))
|
|
|
+ except RestException, e:
|
|
|
+ raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
+
|
|
|
+
|
|
|
def uniquekey(self, collection):
|
|
|
try:
|
|
|
params = self._get_params() + (
|
|
|
@@ -201,6 +346,7 @@ class SolrApi(object):
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|
|
|
|
|
|
+
|
|
|
def update(self, collection_or_core_name, data, content_type='csv'):
|
|
|
try:
|
|
|
if content_type == 'csv':
|
|
|
@@ -218,10 +364,8 @@ class SolrApi(object):
|
|
|
else:
|
|
|
LOG.error("Could not update index for %s. Unsupported content type %s. Allowed content types: csv" % (collection_or_core_name, content_type))
|
|
|
return False
|
|
|
- response = self._root.post('%s/update' % collection_or_core_name,
|
|
|
- contenttype=content_type,
|
|
|
- params=params,
|
|
|
- data=data)
|
|
|
+
|
|
|
+ self._root.post('%s/update' % collection_or_core_name, contenttype=content_type, params=params, data=data)
|
|
|
return True
|
|
|
except RestException, e:
|
|
|
raise PopupException(e, title=_('Error while accessing Solr'))
|