فهرست منبع

HUE-2135 [search] Support non Solr cloud mode

Merge duplicated SolrApi
Search and index listing is compatible with non Solr Cloud mode
Cannot create a core in indexer in non Solr Cloud mode
Romain Rigaux 11 سال پیش
والد
کامیت
1453bd2c35

+ 0 - 232
apps/search/src/search/api.py

@@ -16,7 +16,6 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-import json
 import logging
 import logging
 import numbers
 import numbers
 import urllib
 import urllib
@@ -26,19 +25,10 @@ from datetime import datetime, timedelta
 from math import log
 from math import log
 from time import mktime
 from time import mktime
 
 
-from desktop.lib.exceptions_renderable import PopupException
-from desktop.lib.rest.http_client import RestException
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
-from libsolr.api import SolrApi as BaseSolrApi
-
-from search.conf import EMPTY_QUERY, SECURITY_ENABLED
-
-
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
-DEFAULT_USER = 'hue'
-
 
 
 def utf_quoter(what):
 def utf_quoter(what):
   return urllib.quote(unicode(what).encode('utf-8'), safe='~@#$&()*!+=:;,.?/\'')
   return urllib.quote(unicode(what).encode('utf-8'), safe='~@#$&()*!+=:;,.?/\'')
@@ -56,7 +46,6 @@ def _guess_range_facet(widget_type, solr_api, collection, facet_field, propertie
     stat_facet = stats_json['stats']['stats_fields'][facet_field]
     stat_facet = stats_json['stats']['stats_fields'][facet_field]
     is_date = False
     is_date = False
 
 
-    # to refactor
     if isinstance(stat_facet['min'], numbers.Number):
     if isinstance(stat_facet['min'], numbers.Number):
       stats_min = int(stat_facet['min']) # Cast floats to int currently
       stats_min = int(stat_facet['min']) # Cast floats to int currently
       stats_max = int(stat_facet['max'])
       stats_max = int(stat_facet['max'])
@@ -188,224 +177,3 @@ def _zoom_range_facet(solr_api, collection, facet, direction='out'):
   properties = {}
   properties = {}
   _guess_range_facet(facet['widgetType'], solr_api, collection, facet['field'], properties)
   _guess_range_facet(facet['widgetType'], solr_api, collection, facet['field'], properties)
   return properties
   return properties
-
-
-class SolrApi(BaseSolrApi):
-  """
-  http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
-  """
-  def __init__(self, solr_url, user):
-    super(SolrApi, self).__init__(solr_url, user, SECURITY_ENABLED.get())
-
-  def _get_params(self):
-    if self.security_enabled:
-      return (('doAs', self._user ),)
-    return (('user.name', DEFAULT_USER), ('doAs', self._user),)
-
-  #@demo_handler
-  def query(self, collection, query):
-    solr_query = {}
-
-    solr_query['collection'] = collection['name']
-    solr_query['rows'] = min(int(collection['template']['rows'] or 10), 1000)
-    solr_query['start'] = min(int(query['start']), 10000)
-
-    q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
-
-    params = self._get_params() + (
-        ('q', 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']])),
-        ('wt', 'json'),
-        ('rows', solr_query['rows']),
-        ('start', solr_query['start']),
-    )
-
-    if any(collection['facets']):
-      params += (
-        ('facet', 'true'),
-        ('facet.mincount', 0),
-        ('facet.limit', 10),
-      )
-      for facet in collection['facets']:
-        if facet['type'] == 'query':
-          params += (('facet.query', '%s' % facet['field']),)
-        elif facet['type'] == 'range':
-          params += tuple([
-             ('facet.range', '{!ex=%s}%s' % (facet['field'], facet['field'])),
-             ('f.%s.facet.range.start' % facet['field'], facet['properties']['start']),
-             ('f.%s.facet.range.end' % facet['field'], facet['properties']['end']),
-             ('f.%s.facet.range.gap' % facet['field'], facet['properties']['gap']),
-             ('f.%s.facet.mincount' % facet['field'], facet['properties']['mincount']),]
-          )
-        elif facet['type'] == 'field':
-          params += (
-              ('facet.field', '{!ex=%s}%s' % (facet['field'], facet['field'])),
-              ('f.%s.facet.limit' % facet['field'], int(facet['properties'].get('limit', 10)) + 1),
-              ('f.%s.facet.mincount' % facet['field'], int(facet['properties']['mincount'])),
-          )
-
-    for fq in query['fqs']:
-      if fq['type'] == 'field':
-        # This does not work if spaces in Solr:
-        # params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),)
-        f = []
-        for _filter in fq['filter']:
-          if _filter is not None and ' ' in _filter:
-            f.append('%s:"%s"' % (fq['field'], _filter))
-          else:
-            f.append('{!field f=%s}%s' % (fq['field'], _filter))
-        params += (('fq', urllib.unquote(utf_quoter('{!tag=%s}' % fq['field'] + ' '.join(f)))),)
-      elif fq['type'] == 'range':
-        params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([urllib.unquote(utf_quoter('%s:[%s TO %s}' % (fq['field'], f['from'], f['to']))) for f in fq['properties']])),)
-
-    if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
-      fields = collection['template']['fieldsSelected'] + [collection['idField']] if collection['idField'] else []
-      params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))),)
-    else:
-      params += (('fl', '*'),)
-
-    params += (
-      ('hl', 'true'),
-      ('hl.fl', '*'),
-      ('hl.snippets', 3)
-    )
-
-    if collection['template']['fieldsSelected']:
-      fields = []
-      for field in collection['template']['fieldsSelected']:
-        attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
-        if attribute_field:
-          if attribute_field[0]['sort']['direction']:
-            fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
-      if fields:
-        params += (
-          ('sort', ','.join(fields)),
-        )
-
-    response = self._root.get('%(collection)s/select' % solr_query, params)
-
-    return self._get_json(response)
-
-
-  def suggest(self, solr_query, hue_core):
-    try:
-      params = self._get_params() + (
-          ('q', solr_query['q']),
-          ('wt', 'json'),
-      )
-      response = self._root.get('%(collection)s/suggest' % solr_query, params)
-      if type(response) != dict:
-        response = json.loads(response)
-      return response
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def collections(self):
-    try:
-      params = self._get_params() + (
-          ('detail', 'true'),
-          ('path', '/clusterstate.json'),
-      )
-      response = self._root.get('zookeeper', params=params)
-      return json.loads(response['znode'].get('data', '{}'))
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def collection_or_core(self, hue_collection):
-    if hue_collection.is_core_only:
-      return self.core(hue_collection.name)
-    else:
-      return self.collection(hue_collection.name)
-
-  def collection(self, name):
-    try:
-      collections = self.collections()
-      return collections[name]
-    except Exception, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def cores(self):
-    try:
-      params = self._get_params() + (
-          ('wt', 'json'),
-      )
-      return self._root.get('admin/cores', params=params)['status']
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def core(self, core):
-    try:
-      params = self._get_params() + (
-          ('wt', 'json'),
-          ('core', core),
-      )
-      return self._root.get('admin/cores', params=params)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def schema(self, core):
-    try:
-      params = self._get_params() + (
-          ('wt', 'json'),
-          ('file', 'schema.xml'),
-      )
-      return self._root.get('%(core)s/admin/file' % {'core': core}, params=params)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def fields(self, core, dynamic=False):
-    try:
-      params = self._get_params() + (
-          ('wt', 'json'),
-          ('fl', '*'),
-      )
-      if not dynamic:
-        params += (('show', 'schema'),)
-      response = self._root.get('%(core)s/admin/luke' % {'core': core}, params=params)
-      return self._get_json(response)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def luke(self, core):
-    try:
-      params = self._get_params() + (
-          ('wt', 'json'),
-      )
-      response = self._root.get('%(core)s/admin/luke' % {'core': core}, params=params)
-      return self._get_json(response)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def schema_fields(self, core):
-    try:
-      params = self._get_params() + (
-          ('wt', 'json'),
-      )
-      response = self._root.get('%(core)s/schema/fields' % {'core': core}, params=params)
-      return self._get_json(response)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def stats(self, core, fields):
-    try:
-      params = (
-          ('q', EMPTY_QUERY.get()),
-          ('wt', 'json'),
-          ('rows', 0),
-          ('stats', 'true'),
-      )
-      params += tuple([('stats.field', field) for field in fields])
-      response = self._root.get('%(core)s/select' % {'core': core}, params=params)
-      return self._get_json(response)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def get(self, core, doc_id):
-    try:
-      params = (
-          ('id', doc_id),
-          ('wt', 'json'),
-      )
-      response = self._root.get('%(core)s/get' % {'core': core}, params=params)
-      return self._get_json(response)
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))

تفاوت فایلی نمایش داده نمی شود زیرا این فایل بسیار بزرگ است
+ 0 - 44
apps/search/src/search/examples.py


+ 2 - 6
apps/search/src/search/models.py

@@ -28,18 +28,14 @@ from django.core.urlresolvers import reverse
 
 
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.i18n import smart_unicode
 
 
-from search.api import SolrApi
+from libsolr.api import SolrApi
+
 from search.conf import SOLR_URL
 from search.conf import SOLR_URL
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
-"""
-Collection model contains all the data now.
-"""
-
-
 # Deprecated
 # Deprecated
 class Facet(models.Model):
 class Facet(models.Model):
   _ATTRIBUTES = ['properties', 'fields', 'ranges', 'dates', 'charts', 'order']
   _ATTRIBUTES = ['properties', 'fields', 'ranges', 'dates', 'charts', 'order']

+ 10 - 4
apps/search/src/search/search_controller.py

@@ -18,12 +18,12 @@
 
 
 import logging
 import logging
 
 
-from desktop.lib.exceptions_renderable import PopupException
+from django.utils.translation import ugettext as _
+
+from libsolr.api import SolrApi
 
 
-from search.api import SolrApi
 from search.conf import SOLR_URL
 from search.conf import SOLR_URL
 from search.models import Collection
 from search.models import Collection
-from django.utils.translation import ugettext as _
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -93,4 +93,10 @@ class SearchController(object):
     return SolrApi(SOLR_URL.get(), self.user).collections()
     return SolrApi(SOLR_URL.get(), self.user).collections()
 
 
   def get_all_indexes(self):
   def get_all_indexes(self):
-    return self.get_solr_collection().keys() + SolrApi(SOLR_URL.get(), self.user).cores().keys()
+    indexes = []
+    try:
+      indexes = self.get_solr_collection().keys()
+    except:
+      pass
+    return indexes + SolrApi(SOLR_URL.get(), self.user).cores().keys()
+

+ 1 - 1
apps/search/src/search/urls.py

@@ -22,7 +22,7 @@ urlpatterns = patterns('search.views',
   url(r'^search$', 'search', name='search'),
   url(r'^search$', 'search', name='search'),
   url(r'^save$', 'save', name='save'),
   url(r'^save$', 'save', name='save'),
   url(r'^new_search', 'new_search', name='new_search'),
   url(r'^new_search', 'new_search', name='new_search'),
-  url(r'^browse/(?P<name>\w+)', 'browse', name='browse'),
+  url(r'^browse/(?P<name>.+)', 'browse', name='browse'),
   url(r'^download$', 'download', name='download'),
   url(r'^download$', 'download', name='download'),
 
 
   url(r'^admin/collections$', 'admin_collections', name='admin_collections'),
   url(r'^admin/collections$', 'admin_collections', name='admin_collections'),

+ 7 - 8
apps/search/src/search/views.py

@@ -20,15 +20,19 @@ import logging
 
 
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
 from django.http import HttpResponse
 from django.http import HttpResponse
-from django.utils.encoding import smart_str
+from django.utils.encoding import smart_str, force_unicode
 from django.utils.html import escape
 from django.utils.html import escape
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 from django.shortcuts import redirect
 from django.shortcuts import redirect
 
 
 from desktop.lib.django_util import render
 from desktop.lib.django_util import render
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.rest.http_client import RestException
+
+from libsolr.api import SolrApi
+from indexer.management.commands import indexer_setup
 
 
-from search.api import SolrApi, _guess_gap, _zoom_range_facet, _new_range_facet
+from search.api import _guess_gap, _zoom_range_facet, _new_range_facet
 from search.conf import SOLR_URL
 from search.conf import SOLR_URL
 from search.data_export import download as export_download
 from search.data_export import download as export_download
 from search.decorators import allow_admin_only
 from search.decorators import allow_admin_only
@@ -36,11 +40,6 @@ from search.management.commands import search_setup
 from search.models import Collection, augment_solr_response, augment_solr_exception
 from search.models import Collection, augment_solr_response, augment_solr_exception
 from search.search_controller import SearchController
 from search.search_controller import SearchController
 
 
-from indexer.management.commands import indexer_setup
-
-from django.utils.encoding import force_unicode
-from desktop.lib.rest.http_client import RestException
-
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -96,7 +95,7 @@ def new_search(request):
 
 
 @allow_admin_only
 @allow_admin_only
 def browse(request, name):
 def browse(request, name):
-  collections = SearchController(request.user).get_solr_collection().keys()
+  collections = SearchController(request.user).get_all_indexes()
   if not collections:
   if not collections:
     return no_collections(request)
     return no_collections(request)
 
 

+ 7 - 14
desktop/libs/indexer/src/indexer/api.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 # limitations under the License.
 
 
 from functools import wraps
 from functools import wraps
+
 import itertools
 import itertools
 import json
 import json
 import logging
 import logging
@@ -29,9 +30,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from search.models import Collection
 from search.models import Collection
 
 
 from controller import CollectionManagerController
 from controller import CollectionManagerController
-from utils import fields_from_log,\
-                  field_values_from_separated_file, get_type_from_morphline_type,\
-                  get_field_types
+from utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, get_field_types
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -111,18 +110,12 @@ def parse_fields(request):
 def collections(request):
 def collections(request):
   searcher = CollectionManagerController(request.user)
   searcher = CollectionManagerController(request.user)
   solr_collections = searcher.get_collections()
   solr_collections = searcher.get_collections()
-  hue_collections_map = {}
-  for collection in Collection.objects.all():
-    hue_collections_map[collection.name] = collection
-  massaged_collections = []
-  for collection in solr_collections:
-    massaged_collections.append({
-      'name': collection,
-    })
+
   response = {
   response = {
     'status': 0,
     'status': 0,
-    'collections': list(massaged_collections)
+    'collections': [{'name': collection} for collection in solr_collections]
   }
   }
+
   return HttpResponse(json.dumps(response), mimetype="application/json")
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
 
 
@@ -166,8 +159,8 @@ def collections_create(request):
 
 
       response['status'] = 0
       response['status'] = 0
       response['message'] = _('Collection created!')
       response['message'] = _('Collection created!')
-    except:
-      searcher.delete_collection(collection.get('name'))
+    except Exception, e:
+      LOG.error(e)
       raise
       raise
   else:
   else:
     response['message'] = _('Collection missing.')
     response['message'] = _('Collection missing.')

+ 6 - 4
desktop/libs/indexer/src/indexer/controller.py

@@ -55,9 +55,10 @@ class CollectionManagerController(object):
 
 
   def get_collections(self):
   def get_collections(self):
     try:
     try:
-      solr_collections = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()).collections()
+      from search.search_controller import SearchController
+      return SearchController(self.user).get_all_indexes()
     except Exception, e:
     except Exception, e:
-      LOG.warn('No Zookeeper servlet running on Solr server: %s' % e)
+      LOG.warn('Error get_collections: %s' % e)
       solr_collections = []
       solr_collections = []
 
 
     return solr_collections
     return solr_collections
@@ -95,8 +96,6 @@ class CollectionManagerController(object):
                                  'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
                                  'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
                                })
                                })
     status = process.wait()
     status = process.wait()
-
-    # Don't want directories laying around
     shutil.rmtree(tmp_path)
     shutil.rmtree(tmp_path)
 
 
     if status != 0:
     if status != 0:
@@ -104,6 +103,7 @@ class CollectionManagerController(object):
       raise PopupException(_('Could not create instance directory. Check error logs for more info.'))
       raise PopupException(_('Could not create instance directory. Check error logs for more info.'))
 
 
     api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
     api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
+
     if not api.create_collection(name):
     if not api.create_collection(name):
       # Delete instance directory.
       # Delete instance directory.
       process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--delete", name],
       process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--delete", name],
@@ -113,6 +113,7 @@ class CollectionManagerController(object):
                                    'SOLR_HOME': conf.SOLR_HOME.get(),
                                    'SOLR_HOME': conf.SOLR_HOME.get(),
                                    'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
                                    'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
                                  })
                                  })
+
       if process.wait() != 0:
       if process.wait() != 0:
         LOG.error("Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate())
         LOG.error("Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate())
       raise PopupException(_('Could not create collection. Check error logs for more info.'))
       raise PopupException(_('Could not create collection. Check error logs for more info.'))
@@ -160,6 +161,7 @@ class CollectionManagerController(object):
     Add hdfs path contents to index
     Add hdfs path contents to index
     """
     """
     api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
     api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
+
     if indexing_strategy == 'upload':
     if indexing_strategy == 'upload':
       stats = fs.stats(path)
       stats = fs.stats(path)
       if stats.size > MAX_UPLOAD_SIZE:
       if stats.size > MAX_UPLOAD_SIZE:

+ 3 - 3
desktop/libs/indexer/src/indexer/urls.py

@@ -28,7 +28,7 @@ urlpatterns += patterns('indexer.api',
   url(r'^api/collections/create/$', 'collections_create', name='api_collections_create'),
   url(r'^api/collections/create/$', 'collections_create', name='api_collections_create'),
   url(r'^api/collections/import/$', 'collections_import', name='api_collections_import'),
   url(r'^api/collections/import/$', 'collections_import', name='api_collections_import'),
   url(r'^api/collections/remove/$', 'collections_remove', name='api_collections_remove'),
   url(r'^api/collections/remove/$', 'collections_remove', name='api_collections_remove'),
-  url(r'^api/collections/(?P<collection>\w+)/fields/$', 'collections_fields', name='api_collections_fields'),
-  url(r'^api/collections/(?P<collection>\w+)/update/$', 'collections_update', name='api_collections_update'),
-  url(r'^api/collections/(?P<collection>\w+)/data/$', 'collections_data', name='api_collections_data')
+  url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
+  url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
+  url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data')
 )
 )

+ 208 - 64
desktop/libs/libsolr/src/libsolr/api.py

@@ -20,10 +20,13 @@ import json
 import logging
 import logging
 import urllib
 import urllib
 
 
+from django.utils.translation import ugettext as _
+
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest import resource
 from desktop.lib.rest import resource
-from django.utils.translation import ugettext as _
+
+from search.conf import EMPTY_QUERY, SECURITY_ENABLED
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -39,7 +42,7 @@ class SolrApi(object):
   """
   """
   http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
   http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
   """
   """
-  def __init__(self, solr_url, user, security_enabled=False):
+  def __init__(self, solr_url, user, security_enabled=SECURITY_ENABLED.get()):
     self._url = solr_url
     self._url = solr_url
     self._user = user
     self._user = user
     self._client = HttpClient(self._url, logger=LOG)
     self._client = HttpClient(self._url, logger=LOG)
@@ -53,17 +56,89 @@ class SolrApi(object):
       return (('doAs', self._user ),)
       return (('doAs', self._user ),)
     return (('user.name', DEFAULT_USER), ('doAs', self._user),)
     return (('user.name', DEFAULT_USER), ('doAs', self._user),)
 
 
-  @classmethod
-  def _get_json(cls, response):
-    if type(response) != dict:
-      # Got 'plain/text' mimetype instead of 'application/json'
-      try:
-        response = json.loads(response)
-      except ValueError, e:
-        # Got some null bytes in the response
-        LOG.error('%s: %s' % (unicode(e), repr(response)))
-        response = json.loads(response.replace('\x00', ''))
-    return response
+
+  def query(self, collection, query):
+    solr_query = {}
+
+    solr_query['collection'] = collection['name']
+    solr_query['rows'] = min(int(collection['template']['rows'] or 10), 1000)
+    solr_query['start'] = min(int(query['start']), 10000)
+
+    q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
+
+    params = self._get_params() + (
+        ('q', 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']])),
+        ('wt', 'json'),
+        ('rows', solr_query['rows']),
+        ('start', solr_query['start']),
+    )
+
+    if any(collection['facets']):
+      params += (
+        ('facet', 'true'),
+        ('facet.mincount', 0),
+        ('facet.limit', 10),
+      )
+      for facet in collection['facets']:
+        if facet['type'] == 'query':
+          params += (('facet.query', '%s' % facet['field']),)
+        elif facet['type'] == 'range':
+          params += tuple([
+             ('facet.range', '{!ex=%s}%s' % (facet['field'], facet['field'])),
+             ('f.%s.facet.range.start' % facet['field'], facet['properties']['start']),
+             ('f.%s.facet.range.end' % facet['field'], facet['properties']['end']),
+             ('f.%s.facet.range.gap' % facet['field'], facet['properties']['gap']),
+             ('f.%s.facet.mincount' % facet['field'], facet['properties']['mincount']),]
+          )
+        elif facet['type'] == 'field':
+          params += (
+              ('facet.field', '{!ex=%s}%s' % (facet['field'], facet['field'])),
+              ('f.%s.facet.limit' % facet['field'], int(facet['properties'].get('limit', 10)) + 1),
+              ('f.%s.facet.mincount' % facet['field'], int(facet['properties']['mincount'])),
+          )
+
+    for fq in query['fqs']:
+      if fq['type'] == 'field':
+        # This does not work if spaces in Solr:
+        # params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),)
+        f = []
+        for _filter in fq['filter']:
+          if _filter is not None and ' ' in _filter:
+            f.append('%s:"%s"' % (fq['field'], _filter))
+          else:
+            f.append('{!field f=%s}%s' % (fq['field'], _filter))
+        params += (('fq', urllib.unquote(utf_quoter('{!tag=%s}' % fq['field'] + ' '.join(f)))),)
+      elif fq['type'] == 'range':
+        params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([urllib.unquote(utf_quoter('%s:[%s TO %s}' % (fq['field'], f['from'], f['to']))) for f in fq['properties']])),)
+
+    if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
+      fields = collection['template']['fieldsSelected'] + [collection['idField']] if collection['idField'] else []
+      params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))),)
+    else:
+      params += (('fl', '*'),)
+
+    params += (
+      ('hl', 'true'),
+      ('hl.fl', '*'),
+      ('hl.snippets', 3)
+    )
+
+    if collection['template']['fieldsSelected']:
+      fields = []
+      for field in collection['template']['fieldsSelected']:
+        attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
+        if attribute_field:
+          if attribute_field[0]['sort']['direction']:
+            fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
+      if fields:
+        params += (
+          ('sort', ','.join(fields)),
+        )
+
+    response = self._root.get('%(collection)s/select' % solr_query, params)
+
+    return self._get_json(response)
+
 
 
   def suggest(self, solr_query, hue_core):
   def suggest(self, solr_query, hue_core):
     try:
     try:
@@ -78,6 +153,7 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
   def collections(self):
   def collections(self):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -85,16 +161,18 @@ class SolrApi(object):
           ('path', '/clusterstate.json'),
           ('path', '/clusterstate.json'),
       )
       )
       response = self._root.get('zookeeper', params=params)
       response = self._root.get('zookeeper', params=params)
-      return json.loads(response['znode']['data'])
+      return json.loads(response['znode'].get('data', '{}'))
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
   def collection_or_core(self, hue_collection):
   def collection_or_core(self, hue_collection):
     if hue_collection.is_core_only:
     if hue_collection.is_core_only:
       return self.core(hue_collection.name)
       return self.core(hue_collection.name)
     else:
     else:
       return self.collection(hue_collection.name)
       return self.collection(hue_collection.name)
 
 
+
   def collection(self, name):
   def collection(self, name):
     try:
     try:
       collections = self.collections()
       collections = self.collections()
@@ -102,52 +180,6 @@ class SolrApi(object):
     except Exception, e:
     except Exception, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
-  def create_collection(self, name, shards=1, replication=1):
-    try:
-      params = (
-        ('action', 'CREATE'),
-        ('name', name),
-        ('numShards', shards),
-        ('replicationFactor', replication),
-        ('collection.configName', name),
-        ('wt', 'json')
-      )
-
-      response = self._root.post('admin/collections', params=params)
-      if 'success' in response:
-        return True
-      else:
-        LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
-        return False
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def remove_collection(self, name, replication=1):
-    try:
-      params = (
-        ('action', 'DELETE'),
-        ('name', name),
-        ('replicationFactor', replication),
-        ('wt', 'json')
-      )
-
-      response = self._root.post('admin/collections', params=params)
-      if 'success' in response:
-        return True
-      else:
-        LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
-        return False
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
-  def add_fields(self, collection, fields):
-    try:
-      
-      response = self._root.post('%s/schema/fields' % collection, data=json.dumps(fields))
-
-      return response
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
 
 
   def cores(self):
   def cores(self):
     try:
     try:
@@ -158,6 +190,7 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
   def core(self, core):
   def core(self, core):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -168,6 +201,7 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
   def schema(self, core):
   def schema(self, core):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -178,6 +212,7 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
   def fields(self, core, dynamic=False):
   def fields(self, core, dynamic=False):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -191,6 +226,116 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
+  def luke(self, core):
+    try:
+      params = self._get_params() + (
+          ('wt', 'json'),
+      )
+      response = self._root.get('%(core)s/admin/luke' % {'core': core}, params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def schema_fields(self, core):
+    try:
+      params = self._get_params() + (
+          ('wt', 'json'),
+      )
+      response = self._root.get('%(core)s/schema/fields' % {'core': core}, params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def stats(self, core, fields):
+    try:
+      params = (
+          ('q', EMPTY_QUERY.get()),
+          ('wt', 'json'),
+          ('rows', 0),
+          ('stats', 'true'),
+      )
+      params += tuple([('stats.field', field) for field in fields])
+      response = self._root.get('%(core)s/select' % {'core': core}, params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def get(self, core, doc_id):
+    try:
+      params = (
+          ('id', doc_id),
+          ('wt', 'json'),
+      )
+      response = self._root.get('%(core)s/get' % {'core': core}, params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  @classmethod
+  def _get_json(cls, response):
+    if type(response) != dict:
+      # Got 'plain/text' mimetype instead of 'application/json'
+      try:
+        response = json.loads(response)
+      except ValueError, e:
+        # Got some null bytes in the response
+        LOG.error('%s: %s' % (unicode(e), repr(response)))
+        response = json.loads(response.replace('\x00', ''))
+    return response
+
+
+  def create_collection(self, name, shards=1, replication=1):
+    try:
+      params = (
+        ('action', 'CREATE'),
+        ('name', name),
+        ('numShards', shards),
+        ('replicationFactor', replication),
+        ('collection.configName', name),
+        ('wt', 'json')
+      )
+
+      response = self._root.post('admin/collections', contenttype='application/text', params=params)
+      if 'success' in response:
+        return True
+      else:
+        LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
+        return False
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def remove_collection(self, name, replication=1):
+    try:
+      params = (
+        ('action', 'DELETE'),
+        ('name', name),
+        ('replicationFactor', replication),
+        ('wt', 'json')
+      )
+
+      response = self._root.post('admin/collections', params=params)
+      if 'success' in response:
+        return True
+      else:
+        LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
+        return False
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def add_fields(self, collection, fields):
+    try:
+      return self._root.post('%s/schema/fields' % collection, data=json.dumps(fields))
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
   def uniquekey(self, collection):
   def uniquekey(self, collection):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -201,6 +346,7 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
   def update(self, collection_or_core_name, data, content_type='csv'):
   def update(self, collection_or_core_name, data, content_type='csv'):
     try:
     try:
       if content_type == 'csv':
       if content_type == 'csv':
@@ -218,10 +364,8 @@ class SolrApi(object):
       else:
       else:
         LOG.error("Could not update index for %s. Unsupported content type %s. Allowed content types: csv" % (collection_or_core_name, content_type))
         LOG.error("Could not update index for %s. Unsupported content type %s. Allowed content types: csv" % (collection_or_core_name, content_type))
         return False
         return False
-      response = self._root.post('%s/update' % collection_or_core_name,
-                                 contenttype=content_type,
-                                 params=params,
-                                 data=data)
+
+      self._root.post('%s/update' % collection_or_core_name, contenttype=content_type, params=params, data=data)
       return True
       return True
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))

برخی فایل ها در این مقایسه diff نمایش داده نمی شوند زیرا تعداد فایل ها بسیار زیاد است