Преглед изворни кода

HUE-5833 [dashboard] Unify dashboard logic into its own library

Romain Rigaux пре 9 година
родитељ
комит
2a0801e
66 измењених фајлова са 1813 додато и 1120 уклоњено
  1. 2 2
      apps/impala/src/impala/dashboard_api.py
  2. 0 8
      apps/search/src/search/conf.py
  3. 53 0
      apps/search/src/search/dashboard_api.py
  4. 2 770
      apps/search/src/search/models.py
  5. 3 3
      apps/search/src/search/tests.py
  6. 9 3
      apps/search/src/search/urls.py
  7. 1 232
      apps/search/src/search/views.py
  8. 2 1
      desktop/Makefile
  9. 1 1
      desktop/core/src/desktop/templates/common_header.mako
  10. 9 8
      desktop/core/src/desktop/templates/responsive.mako
  11. 36 0
      desktop/libs/dashboard/Makefile
  12. 2 0
      desktop/libs/dashboard/babel.cfg
  13. 20 0
      desktop/libs/dashboard/hueversion.py
  14. 29 0
      desktop/libs/dashboard/setup.py
  15. 15 0
      desktop/libs/dashboard/src/dashboard/__init__.py
  16. 7 6
      desktop/libs/dashboard/src/dashboard/api.py
  17. 37 0
      desktop/libs/dashboard/src/dashboard/conf.py
  18. 1 35
      desktop/libs/dashboard/src/dashboard/dashboard_api.py
  19. 0 0
      desktop/libs/dashboard/src/dashboard/data_export.py
  20. 0 0
      desktop/libs/dashboard/src/dashboard/decorators.py
  21. 0 0
      desktop/libs/dashboard/src/dashboard/facet_builder.py
  22. 801 0
      desktop/libs/dashboard/src/dashboard/models.py
  23. 1 1
      desktop/libs/dashboard/src/dashboard/search_controller.py
  24. 23 0
      desktop/libs/dashboard/src/dashboard/settings.py
  25. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/bird_gray_32.png
  26. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_logs.png
  27. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_logs_48.png
  28. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_search_24.png
  29. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_search_48.png
  30. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_twitter.png
  31. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_twitter_48.png
  32. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_yelp.png
  33. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_yelp_48.png
  34. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/remove.png
  35. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/reply.png
  36. BIN
      desktop/libs/dashboard/src/dashboard/static/dashboard/art/retweet.png
  37. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/css/admin.css
  38. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/css/admin_mobile.css
  39. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/css/search.css
  40. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/css/search_mobile.css
  41. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/help/index.html
  42. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/img/clear.png
  43. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/img/loading.gif
  44. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/js/collections.ko.js
  45. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/js/create-collections.ko.js
  46. 20 20
      desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js
  47. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.utils.js
  48. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/templates/logs.jpg
  49. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/templates/restaurant.jpg
  50. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/templates/templates.xml
  51. 0 0
      desktop/libs/dashboard/src/dashboard/static/dashboard/templates/twitter.jpg
  52. 1 1
      desktop/libs/dashboard/src/dashboard/templates/admin_collections.mako
  53. 1 1
      desktop/libs/dashboard/src/dashboard/templates/admin_collections_m.mako
  54. 10 10
      desktop/libs/dashboard/src/dashboard/templates/common_admin_collections.mako
  55. 6 6
      desktop/libs/dashboard/src/dashboard/templates/common_search.mako
  56. 5 2
      desktop/libs/dashboard/src/dashboard/templates/macros.mako
  57. 2 3
      desktop/libs/dashboard/src/dashboard/templates/no_collections.mako
  58. 1 1
      desktop/libs/dashboard/src/dashboard/templates/search.mako
  59. 2 1
      desktop/libs/dashboard/src/dashboard/templates/search_embeddable.mako
  60. 1 1
      desktop/libs/dashboard/src/dashboard/templates/search_m.mako
  61. 398 0
      desktop/libs/dashboard/src/dashboard/tests.py
  62. 52 0
      desktop/libs/dashboard/src/dashboard/urls.py
  63. 256 0
      desktop/libs/dashboard/src/dashboard/views.py
  64. 1 1
      desktop/libs/indexer/src/indexer/controller.py
  65. 1 1
      desktop/libs/indexer/src/indexer/settings.py
  66. 2 2
      desktop/libs/libsolr/src/libsolr/api.py

+ 2 - 2
apps/impala/src/impala/dashboard_api.py

@@ -31,8 +31,8 @@ from libsolr.api import GAPS
 from notebook.models import make_notebook
 from notebook.models import make_notebook
 from notebook.connectors.base import get_api, OperationTimeout
 from notebook.connectors.base import get_api, OperationTimeout
 
 
-from search.models import Collection2, augment_response
-from search.facet_builder import _compute_range_facet
+from dashboard.models import Collection2, augment_response
+from dashboard.facet_builder import _compute_range_facet
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)

+ 0 - 8
apps/search/src/search/conf.py

@@ -17,7 +17,6 @@
 
 
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
 
 
-from desktop.conf import is_hue4
 from desktop.lib.conf import Config, coerce_bool
 from desktop.lib.conf import Config, coerce_bool
 
 
 
 
@@ -42,10 +41,3 @@ LATEST = Config(
   help=_("Use latest Solr 5.2+ features."),
   help=_("Use latest Solr 5.2+ features."),
   default=False,
   default=False,
   type=coerce_bool)
   type=coerce_bool)
-
-ENABLE_SQL = Config(
-  key="enable_sql",
-  help=_("Offer to use SQL engines to compute the dashboards."),
-  dynamic_default=is_hue4,
-  private=True,
-  type=coerce_bool)

+ 53 - 0
apps/search/src/search/dashboard_api.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from dashboard.dashboard_api import DashboardApi
+from dashboard.models import augment_solr_response
+from dashboard.search_controller import SearchController
+from libsolr.api import SolrApi
+
+from search.conf import SOLR_URL
+
+
+class SearchApi(DashboardApi):
+
+  def __init__(self, user):
+    DashboardApi.__init__(self, user)
+    self.api = SolrApi(SOLR_URL.get(), self.user)
+
+  def query(self, collection, query, facet=None):
+    response = self.api.query(collection, query)
+    return augment_solr_response(response, collection, query)
+
+  def datasets(self, show_all=False):
+    return SearchController(self.user).get_all_indexes(show_all=show_all)
+
+  def fields(self, collection):
+    return self.api.fields(collection)
+
+  def schema_fields(self, collection):
+    return self.api.fields(collection)
+
+  def luke(self, collection):
+    return self.api.luke(collection)
+
+  def stats(self, collection, field, query=None, facet=''):
+    return self.api.stats(collection, field, query, facet)
+
+  def get(self, collection, doc_id):
+    return self.api.get(collection, doc_id)

+ 2 - 770
apps/search/src/search/models.py

@@ -15,26 +15,19 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-import collections
-import itertools
 import json
 import json
 import logging
 import logging
-import numbers
 import re
 import re
 
 
 from django.contrib.auth.models import User
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
 from django.db import models
 from django.db import models
 from django.utils.html import escape
 from django.utils.html import escape
-from django.utils.translation import ugettext as _, ugettext_lazy as _t
-
-from desktop.lib.i18n import smart_unicode, smart_str
-from desktop.models import get_data_link
+from django.utils.translation import ugettext_lazy as _t
 
 
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
-from notebook.conf import get_ordered_interpreters
 
 
-from search.conf import SOLR_URL, LATEST, ENABLE_SQL
+from search.conf import SOLR_URL
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -444,764 +437,3 @@ class Collection(models.Model):
               "size":12,"name": facet['label'], "id":facet_id, "widgetType": "facet-widget",
               "size":12,"name": facet['label'], "id":facet_id, "widgetType": "facet-widget",
               "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"
               "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"
           })
           })
-
-
-class Collection2(object):
-
-  def __init__(self, user, name='Default', data=None, document=None, engine='solr'):
-    self.document = document
-
-    if document is not None:
-      self.data = json.loads(document.data)
-    elif data is not None:
-      self.data = json.loads(data)
-    else:
-      self.data = {
-          'collection': self.get_default(user, name, engine),
-          'layout': []
-      }
-
-  def get_json(self, user):
-    props = self.data
-
-    if self.document is not None:
-      props['collection']['id'] = self.document.id
-      props['collection']['label'] = self.document.name
-      props['collection']['description'] = self.document.description
-
-    # For backward compatibility
-    if 'rows' not in props['collection']['template']:
-      props['collection']['template']['rows'] = 25
-    if 'showGrid' not in props['collection']['template']:
-      props['collection']['template']['showGrid'] = True
-    if 'showChart' not in props['collection']['template']:
-      props['collection']['template']['showChart'] = False
-    if 'chartSettings' not in props['collection']['template']:
-      props['collection']['template']['chartSettings'] = {
-        'chartType': 'bars',
-        'chartSorting': 'none',
-        'chartScatterGroup': None,
-        'chartScatterSize': None,
-        'chartScope': 'world',
-        'chartX': None,
-        'chartYSingle': None,
-        'chartYMulti': [],
-        'chartData': [],
-        'chartMapLabel': None,
-      }
-    if 'enabled' not in props['collection']:
-      props['collection']['enabled'] = True
-    if 'engine' not in props['collection']:
-      props['collection']['engine'] = 'solr'
-    if 'leafletmap' not in props['collection']['template']:
-      props['collection']['template']['leafletmap'] = {'latitudeField': None, 'longitudeField': None, 'labelField': None}
-    if 'timeFilter' not in props['collection']:
-      props['collection']['timeFilter'] = {
-        'field': '',
-        'type': 'rolling',
-        'value': 'all',
-        'from': '',
-        'to': '',
-        'truncate': True
-      }
-    if 'suggest' not in props['collection']:
-      props['collection']['suggest'] = {'enabled': False, 'dictionary': ''}
-    for field in props['collection']['template']['fieldsAttributes']:
-      if 'type' not in field:
-        field['type'] = 'string'
-    if 'nested' not in props['collection'] and LATEST.get():
-      props['collection']['nested'] = {
-        'enabled': False,
-        'schema': []
-      }
-
-    for facet in props['collection']['facets']:
-      properties = facet['properties']
-      if 'gap' in properties and not 'initial_gap' in properties:
-        properties['initial_gap'] = properties['gap']
-      if 'start' in properties and not 'initial_start' in properties:
-        properties['initial_start'] = properties['start']
-      if 'end' in properties and not 'initial_end' in properties:
-        properties['initial_end'] = properties['end']
-      if 'domain' not in properties:
-        properties['domain'] = {'blockParent': [], 'blockChildren': []}
-
-      if facet['widgetType'] == 'histogram-widget':
-        if 'timelineChartType' not in properties:
-          properties['timelineChartType'] = 'bar'
-        if 'enableSelection' not in properties:
-          properties['enableSelection'] = True
-        if 'extraSeries' not in properties:
-          properties['extraSeries'] = []
-
-      if facet['widgetType'] == 'map-widget' and facet['type'] == 'field':
-        facet['type'] = 'pivot'
-        properties['facets'] = []
-        properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
-
-    if 'qdefinitions' not in props['collection']:
-      props['collection']['qdefinitions'] = []
-
-    return json.dumps(props)
-
-  def get_default(self, user, name, engine='solr'):
-    fields = self.fields_data(user, name, engine)
-    id_field = [field['name'] for field in fields if field.get('isId')]
-
-    if id_field:
-      id_field = id_field[0]
-    else:
-      id_field = '' # Schemaless might not have an id
-
-    TEMPLATE = {
-      "extracode": escape("<style type=\"text/css\">\nem {\n  font-weight: bold;\n  background-color: yellow;\n}</style>\n\n<script>\n</script>"),
-      "highlighting": [""],
-      "properties": {"highlighting_enabled": True},
-      "template": """
-      <div class="row-fluid">
-        <div class="row-fluid">
-          <div class="span12">%s</div>
-        </div>
-        <br/>
-      </div>""" % ' '.join(['{{%s}}' % field['name'] for field in fields]),
-      "isGridLayout": True,
-      "showFieldList": True,
-      "showGrid": True,
-      "showChart": False,
-      "chartSettings" : {
-        'chartType': 'bars',
-        'chartSorting': 'none',
-        'chartScatterGroup': None,
-        'chartScatterSize': None,
-        'chartScope': 'world',
-        'chartX': None,
-        'chartYSingle': None,
-        'chartYMulti': [],
-        'chartData': [],
-        'chartMapLabel': None,
-      },
-      "fieldsAttributes": [self._make_gridlayout_header_field(field) for field in fields],
-      "fieldsSelected": [],
-      "leafletmap": {'latitudeField': None, 'longitudeField': None, 'labelField': None},
-      "rows": 25,
-    }
-
-    FACETS = []
-
-    return {
-      'id': None,
-      'name': name,
-      'engine': engine,
-      'label': name,
-      'enabled': False,
-      'template': TEMPLATE,
-      'facets': FACETS,
-      'fields': fields,
-      'idField': id_field,
-    }
-
-  @classmethod
-  def _make_field(cls, field, attributes):
-    return {
-        'name': str(escape(field)),
-        'type': str(attributes.get('type', '')),
-        'isId': attributes.get('required') and attributes.get('uniqueKey'),
-        'isDynamic': 'dynamicBase' in attributes
-    }
-
-  @classmethod
-  def _make_gridlayout_header_field(cls, field, isDynamic=False):
-    return {'name': field['name'], 'type': field['type'], 'sort': {'direction': None}, 'isDynamic': isDynamic}
-
-  @classmethod
-  def _make_luke_from_schema_fields(cls, schema_fields):
-    return dict([
-        (f['name'], {
-              'copySources': [],
-              'type': f['type'],
-              'required': True,
-              'uniqueKey': f.get('uniqueKey'),
-              'flags': u'%s-%s-----OF-----l' % ('I' if f['indexed'] else '-', 'S' if f['stored'] else '-'), u'copyDests': []
-        })
-        for f in schema_fields['fields']
-    ])
-
-  def get_absolute_url(self):
-    return reverse('search:index') + '?collection=%s' % self.id
-
-  def fields(self, user):
-    return sorted([str(field.get('name', '')) for field in self.fields_data(user)])
-
-  def fields_data(self, user, name, engine='solr'):
-    from search.api_engines import get_engine
-    api = get_engine(user, engine)
-    try:
-      schema_fields = api.fields(name)
-      schema_fields = schema_fields['schema']['fields']
-    except Exception, e:
-      LOG.warn('/luke call did not succeed: %s' % e)
-      fields = api.schema_fields(name)
-      schema_fields = Collection2._make_luke_from_schema_fields(fields)
-
-    return sorted([self._make_field(field, attributes) for field, attributes in schema_fields.iteritems()])
-
-  def update_data(self, post_data):
-    data_dict = self.data
-
-    data_dict.update(post_data)
-
-    self.data = data_dict
-
-  @property
-  def autocomplete(self):
-    return self.data['autocomplete']
-
-  @autocomplete.setter
-  def autocomplete(self, autocomplete):
-    properties_ = self.data
-    properties_['autocomplete'] = autocomplete
-    self.data = json.dumps(properties_)
-
-  @classmethod
-  def get_field_list(cls, collection):
-    if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
-      fields = set(collection['template']['fieldsSelected'] + ([collection['idField']] if collection['idField'] else []))
-      # Add field if needed
-      if collection['template']['leafletmap'].get('latitudeField'):
-        fields.add(collection['template']['leafletmap']['latitudeField'])
-      if collection['template']['leafletmap'].get('longitudeField'):
-        fields.add(collection['template']['leafletmap']['longitudeField'])
-      if collection['template']['leafletmap'].get('labelField'):
-        fields.add(collection['template']['leafletmap']['labelField'])
-      return list(fields)
-    else:
-      return ['*']
-
-def get_facet_field(category, field, facets):
-  if category in ('nested', 'function'):
-    id_pattern = '%(id)s'
-  else:
-    id_pattern = '%(field)s-%(id)s'
-
-  facets = filter(lambda facet: facet['type'] == category and id_pattern % facet == field, facets)
-
-  if facets:
-    return facets[0]
-  else:
-    return None
-
-def pairwise2(field, fq_filter, iterable):
-  pairs = []
-  selected_values = [f['value'] for f in fq_filter]
-  a, b = itertools.tee(iterable)
-  for element in a:
-    pairs.append({
-        'cat': field,
-        'value': element,
-        'count': next(a),
-        'selected': element in selected_values,
-        'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element])
-    })
-  return pairs
-
-def range_pair(field, cat, fq_filter, iterable, end, collection_facet):
-  # e.g. counts":["0",17430,"1000",1949,"2000",671,"3000",404,"4000",243,"5000",165],"gap":1000,"start":0,"end":6000}
-  pairs = []
-  selected_values = [f['value'] for f in fq_filter]
-  is_single_unit_gap = re.match('^[\+\-]?1[A-Za-z]*$', str(collection_facet['properties']['gap'])) is not None
-  is_up = collection_facet['properties']['sort'] == 'asc'
-
-  if collection_facet['properties']['sort'] == 'asc' and (collection_facet['type'] == 'range-up' or collection_facet['properties'].get('type') == 'range-up'):
-    prev = None
-    n = []
-    for e in iterable:
-      if prev is not None:
-        n.append(e)
-        n.append(prev)
-        prev = None
-      else:
-        prev = e
-    iterable = n
-    iterable.reverse()
-
-  a, to = itertools.tee(iterable)
-  next(to, None)
-  counts = iterable[1::2]
-  total_counts = counts.pop(0) if collection_facet['properties']['sort'] == 'asc' else 0
-
-  for element in a:
-    next(to, None)
-    to_value = next(to, end)
-    count = next(a)
-
-    pairs.append({
-        'field': field, 'from': element, 'value': count, 'to': to_value, 'selected': element in selected_values,
-        'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element]),
-        'is_single_unit_gap': is_single_unit_gap,
-        'total_counts': total_counts,
-        'is_up': is_up
-    })
-    total_counts += counts.pop(0) if counts else 0
-
-  if collection_facet['properties']['sort'] == 'asc' and collection_facet['type'] != 'range-up' and collection_facet['properties'].get('type') != 'range-up':
-    pairs.reverse()
-
-  return pairs
-
-
-def augment_solr_response(response, collection, query):
-  augmented = response
-  augmented['normalized_facets'] = []
-  NAME = '%(field)s-%(id)s'
-  normalized_facets = []
-
-  selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
-
-  if response and response.get('facet_counts'):
-    for facet in collection['facets']:
-      category = facet['type']
-
-      if category == 'field' and response['facet_counts']['facet_fields']:
-        name = NAME % facet
-        collection_facet = get_facet_field(category, name, collection['facets'])
-        counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name])
-        if collection_facet['properties']['sort'] == 'asc':
-          counts.reverse()
-        facet = {
-          'id': collection_facet['id'],
-          'field': facet['field'],
-          'type': category,
-          'label': collection_facet['label'],
-          'counts': counts,
-        }
-        normalized_facets.append(facet)
-      elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']:
-        name = NAME % facet
-        collection_facet = get_facet_field(category, name, collection['facets'])
-        counts = response['facet_counts']['facet_ranges'][name]['counts']
-        end = response['facet_counts']['facet_ranges'][name]['end']
-        counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet)
-        facet = {
-          'id': collection_facet['id'],
-          'field': facet['field'],
-          'type': category,
-          'label': collection_facet['label'],
-          'counts': counts,
-          'extraSeries': []
-        }
-        normalized_facets.append(facet)
-      elif category == 'query' and response['facet_counts']['facet_queries']:
-        for name, value in response['facet_counts']['facet_queries'].iteritems():
-          collection_facet = get_facet_field(category, name, collection['facets'])
-          facet = {
-            'id': collection_facet['id'],
-            'query': name,
-            'type': category,
-            'label': name,
-            'counts': value,
-          }
-          normalized_facets.append(facet)
-      elif category == 'pivot':
-        name = NAME % facet
-        if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']:
-          if facet['properties']['scope'] == 'stack':
-            count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values)
-          else:
-            count = response['facet_counts']['facet_pivot'][name]
-            _augment_pivot_nd(facet['id'], count, selected_values)
-        else:
-          count = []
-        facet = {
-          'id': facet['id'],
-          'field': name,
-          'type': category,
-          'label': name,
-          'counts': count,
-        }
-        normalized_facets.append(facet)
-
-  if response and response.get('facets'):
-    for facet in collection['facets']:
-      category = facet['type']
-      name = facet['id'] # Nested facets can only have one name
-
-      if category == 'function' and name in response['facets']:
-        value = response['facets'][name]
-        collection_facet = get_facet_field(category, name, collection['facets'])
-        facet = {
-          'id': collection_facet['id'],
-          'query': name,
-          'type': category,
-          'label': name,
-          'counts': value,
-        }
-        normalized_facets.append(facet)
-      elif category == 'nested' and name in response['facets']:
-        value = response['facets'][name]
-        collection_facet = get_facet_field(category, name, collection['facets'])
-        extraSeries = []
-        counts = response['facets'][name]['buckets']
-
-        cols = ['%(field)s' % facet, 'count(%(field)s)' % facet]
-        last_x_col = 0
-        last_xx_col = 0
-        for i, f in enumerate(facet['properties']['facets']):
-          if f['aggregate']['function'] == 'count':
-            cols.append(f['field'])
-            last_xx_col = last_x_col
-            last_x_col = i + 2
-          cols.append(SolrApi._get_aggregate_function(f))
-        rows = []
-
-        # For dim in dimensions
-
-        # Number or Date range
-        if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
-          dimension = 3 if collection_facet['properties']['isDate'] else 1
-          # Single dimension or dimension 2 with analytics
-          if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1:
-            column = 'count'
-            if len(collection_facet['properties']['facets']) == 1:
-              agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
-              legend = agg_keys[0].split(':', 2)[1]
-              column = agg_keys[0]
-            else:
-              legend = facet['field'] # 'count(%s)' % legend
-              agg_keys = [column]
-
-            _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-
-            counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
-            counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet)
-          else:
-            # Dimension 1 with counts and 2 with analytics
-            agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
-            agg_keys.sort(key=lambda a: a[4:])
-
-            if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
-              agg_keys.insert(0, 'count')
-            counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-
-            _series = collections.defaultdict(list)
-
-            for row in rows:
-              for i, cell in enumerate(row):
-                if i > last_x_col:
-                  legend = cols[i]
-                  if last_xx_col != last_x_col:
-                    legend = '%s %s' % (cols[i], row[last_x_col])
-                  _series[legend].append(row[last_xx_col])
-                  _series[legend].append(cell)
-
-            for name, val in _series.iteritems():
-              _c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet)
-              extraSeries.append({'counts': _c, 'label': name})
-            counts = []
-        elif collection_facet['properties'].get('isOldPivot'):
-          facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count']
-
-          column = 'count'
-          agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
-          agg_keys.sort(key=lambda a: a[4:])
-
-          if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
-            agg_keys.insert(0, 'count')
-          counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-
-          #_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
-          dimension = len(facet_fields)
-        elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
-          # Dimension 1 with 1 count or agg
-          dimension = 1
-
-          column = 'count'
-          if len(collection_facet['properties']['facets']) == 1:
-            agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
-            legend = agg_keys[0].split(':', 2)[1]
-            column = agg_keys[0]
-          else:
-            legend = facet['field']
-            agg_keys = [column]
-
-          _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-
-          counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
-          counts = pairwise2(legend, selected_values.get(facet['id'], []), counts)
-        else:
-          # Dimension 2 with analytics or 1 with N aggregates
-          dimension = 2
-          agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
-          agg_keys.sort(key=lambda a: a[4:])
-
-          if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
-            agg_keys.insert(0, 'count')
-          counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-          actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']])
-
-          counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts)
-
-        num_bucket = response['facets'][name]['numBuckets'] if 'numBuckets' in response['facets'][name] else len(response['facets'][name])
-        facet = {
-          'id': collection_facet['id'],
-          'field': facet['field'],
-          'type': category,
-          'label': collection_facet['label'],
-          'counts': counts,
-          'extraSeries': extraSeries,
-          'dimension': dimension,
-          'response': {'response': {'start': 0, 'numFound': num_bucket}}, # Todo * nested buckets + offsets
-          'docs': [dict(zip(cols, row)) for row in rows],
-          'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols]
-        }
-
-        normalized_facets.append(facet)
-
-    # Remove unnecessary facet data
-    if response:
-      response.pop('facet_counts')
-      response.pop('facets')
-
-  augment_response(collection, query, response)
-
-  if normalized_facets:
-    augmented['normalized_facets'].extend(normalized_facets)
-
-  return augmented
-
-
-def augment_response(collection, query, response):
-  # HTML escaping
-  if not query.get('download'):
-    id_field = collection.get('idField', '')
-
-    for doc in response['response']['docs']:
-      for field, value in doc.iteritems():
-        if isinstance(value, numbers.Number):
-          escaped_value = value
-        elif field == '_childDocuments_': # Nested documents
-          escaped_value = value
-        elif isinstance(value, list): # Multivalue field
-          escaped_value = [smart_unicode(escape(val), errors='replace') for val in value]
-        else:
-          value = smart_unicode(value, errors='replace')
-          escaped_value = escape(value)
-        doc[field] = escaped_value
-
-      link = None
-      if 'link-meta' in doc:
-        meta = json.loads(doc['link-meta'])
-        link = get_data_link(meta)
-      elif 'link' in doc:
-        meta = {'type': 'link', 'link': doc['link']}
-        link = get_data_link(meta)
-
-      doc['externalLink'] = link
-      doc['details'] = []
-      doc['hueId'] = smart_unicode(doc.get(id_field, ''))
-
-  highlighted_fields = response.get('highlighting', {}).keys()
-  if highlighted_fields and not query.get('download'):
-    id_field = collection.get('idField')
-    if id_field:
-      for doc in response['response']['docs']:
-        if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields:
-          highlighting = response['highlighting'][smart_unicode(doc[id_field])]
-
-          if highlighting:
-            escaped_highlighting = {}
-            for field, hls in highlighting.iteritems():
-              _hls = [escape(smart_unicode(hl, errors='replace')).replace('&lt;em&gt;', '<em>').replace('&lt;/em&gt;', '</em>') for hl in hls]
-              escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls
-
-            doc.update(escaped_highlighting)
-    else:
-      response['warning'] = _("The Solr schema requires an id field for performing the result highlighting")
-
-
-def _augment_pivot_2d(name, facet_id, counts, selected_values):
-  values = set()
-
-  for dimension in counts:
-    for pivot in dimension['pivot']:
-      values.add(pivot['value'])
-
-  values = sorted(list(values))
-  augmented = []
-
-  for dimension in counts:
-    count = {}
-    pivot_field = ''
-    for pivot in dimension['pivot']:
-      count[pivot['value']] = pivot['count']
-      pivot_field = pivot['field']
-    for val in values:
-      fq_values = [dimension['value'], val]
-      fq_fields = [dimension['field'], pivot_field]
-      fq_filter = selected_values.get(facet_id, [])
-      _selected_values = [f['value'] for f in fq_filter]
-
-      augmented.append({
-          "count": count.get(val, 0),
-          "value": val,
-          "cat": dimension['value'],
-          'selected': fq_values in _selected_values,
-          'exclude': all([f['exclude'] for f in fq_filter if f['value'] == val]),
-          'fq_fields': fq_fields,
-          'fq_values': fq_values,
-      })
-
-  return augmented
-
-
-def _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows):
-  fq_fields = []
-  fq_values = []
-  fq_filter = []
-  _selected_values = [f['value'] for f in selected_values.get(facet['id'], [])]
-  _fields = [facet['field']] + [facet['field'] for facet in facet['properties']['facets']]
-
-  return __augment_stats_2d(counts, facet['field'], fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys, rows)
-
-
-# Clear one dimension
-def __augment_stats_2d(counts, label, fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys, rows):
-  augmented = []
-
-  for bucket in counts: # For each dimension, go through each bucket and pick up the counts or aggregates, then go recursively in the next dimension
-    val = bucket['val']
-    count = bucket['count']
-    dim_row = [val]
-
-    _fq_fields = fq_fields + _fields[0:1]
-    _fq_values = fq_values + [val]
-
-    for agg_key in agg_keys:
-      if agg_key == 'count':
-        dim_row.append(count)
-        augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
-      elif agg_key.startswith('agg_'):
-        label = fq_values[0] if len(_fq_fields) >= 2 else agg_key.split(':', 2)[1]
-        if agg_keys.index(agg_key) == 0: # One count by dimension
-          dim_row.append(count)
-        dim_row.append(bucket[agg_key])
-        augmented.append(_get_augmented(bucket[agg_key], val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
-      else:
-        augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values)) # Needed?
-
-        # Go rec
-        _agg_keys = [key for key, value in bucket[agg_key]['buckets'][0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
-        _agg_keys.sort(key=lambda a: a[4:])
-
-        if not _agg_keys or len(_agg_keys) == 1 and _agg_keys[0].lower().startswith('dim_'):
-          _agg_keys.insert(0, 'count')
-        next_dim = []
-        new_rows = []
-        augmented += __augment_stats_2d(bucket[agg_key]['buckets'], val, _fq_fields, _fq_values, fq_filter, _selected_values, _fields[1:], _agg_keys, next_dim)
-        for row in next_dim:
-          new_rows.append(dim_row + row)
-        dim_row = new_rows
-
-    if dim_row and type(dim_row[0]) == list:
-      rows.extend(dim_row)
-    else:
-      rows.append(dim_row)
-
-  return augmented
-
-
-def _get_augmented(count, val, label, fq_values, fq_fields, fq_filter, _selected_values):
-  return {
-      "count": count,
-      "value": val,
-      "cat": label,
-      'selected': fq_values in _selected_values,
-      'exclude': all([f['exclude'] for f in fq_filter if f['value'] == val]),
-      'fq_fields': fq_fields,
-      'fq_values': fq_values
-  }
-
-
-def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''):
-  for c in counts:
-    fq_fields = (fields if fields else []) + [c['field']]
-    fq_values = (values if values else []) + [smart_str(c['value'])]
-
-    if 'pivot' in c:
-      _augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values)
-
-    fq_filter = selected_values.get(facet_id, [])
-    _selected_values = [f['value'] for f in fq_filter]
-    c['selected'] = fq_values in _selected_values
-    c['exclude'] = False
-    c['fq_fields'] = fq_fields
-    c['fq_values'] = fq_values
-
-
-def _convert_nested_to_augmented_pivot_nd(facet_fields, facet_id, counts, selected_values, fields='', values='', dimension=2):
-  for c in counts['buckets']:
-    c['field'] = facet_fields[0]
-    fq_fields = (fields if fields else []) + [c['field']]
-    fq_values = (values if values else []) + [smart_str(c['val'])]
-    c['value'] = c.pop('val')
-    bucket = 'd%s' % dimension
-
-    if bucket in c:
-      next_dimension = facet_fields[1:]
-      if next_dimension:
-        _convert_nested_to_augmented_pivot_nd(next_dimension, facet_id, c[bucket], selected_values, fq_fields, fq_values, dimension=dimension+1)
-        c['pivot'] = c.pop(bucket)['buckets']
-      else:
-        c['count'] = c.pop(bucket)
-
-    fq_filter = selected_values.get(facet_id, [])
-    _selected_values = [f['value'] for f in fq_filter]
-    c['selected'] = fq_values in _selected_values
-    c['exclude'] = False
-    c['fq_fields'] = fq_fields
-    c['fq_values'] = fq_values
-
-
-def get_engines(user):
-  engines = [{'name': _('index (Solr)'), 'type': 'solr'}]
-
-  if ENABLE_SQL.get():
-    engines += [{
-          'name': _('table (%s)') % interpreter['name'],
-          'type': interpreter['type'],
-          'async': interpreter['interface'] == 'hiveserver2'
-        }
-        for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms')
-    ]
-
-  return engines
-
-
-def augment_solr_exception(response, collection):
-  response.update(
-  {
-    "facet_counts": {
-    },
-    "highlighting": {
-    },
-    "normalized_facets": [
-      {
-        "field": facet['field'],
-        "counts": [],
-        "type": facet['type'],
-        "label": facet['label']
-      }
-      for facet in collection['facets']
-    ],
-    "responseHeader": {
-      "status": -1,
-      "QTime": 0,
-      "params": {
-      }
-    },
-    "response": {
-      "start": 0,
-      "numFound": 0,
-      "docs": [
-      ]
-    }
-  })

+ 3 - 3
apps/search/src/search/tests.py

@@ -28,9 +28,9 @@ from desktop.lib.test_utils import grant_access
 from desktop.lib.rest import resource
 from desktop.lib.rest import resource
 from desktop.models import Document2
 from desktop.models import Document2
 
 
-from search.facet_builder import _round_number_range
-from search.models import Collection2
-from search.search_controller import SearchController
+from dashboard.facet_builder import _round_number_range
+from dashboard.models import Collection2
+from dashboard.search_controller import SearchController
 
 
 
 
 QUERY = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
 QUERY = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}

+ 9 - 3
apps/search/src/search/urls.py

@@ -17,7 +17,15 @@
 
 
 from django.conf.urls import patterns, url
 from django.conf.urls import patterns, url
 
 
+
 urlpatterns = patterns('search.views',
 urlpatterns = patterns('search.views',
+  url(r'^install_examples$', 'install_examples', name='install_examples'),
+)
+
+
+# Those are all deprecated and dashboard.urls.py is the new reference.
+
+urlpatterns += patterns('dashboard.views',
   url(r'^$', 'index', name='index'),
   url(r'^$', 'index', name='index'),
   url(r'^m$', 'index_m', name='index_m'),
   url(r'^m$', 'index_m', name='index_m'),
   url(r'^embeddable$', 'index_embeddable', name='index_embeddable'),
   url(r'^embeddable$', 'index_embeddable', name='index_embeddable'),
@@ -31,12 +39,10 @@ urlpatterns = patterns('search.views',
   url(r'^admin/collections$', 'admin_collections', name='admin_collections'),
   url(r'^admin/collections$', 'admin_collections', name='admin_collections'),
   url(r'^admin/collection_delete$', 'admin_collection_delete', name='admin_collection_delete'),
   url(r'^admin/collection_delete$', 'admin_collection_delete', name='admin_collection_delete'),
   url(r'^admin/collection_copy$', 'admin_collection_copy', name='admin_collection_copy'),
   url(r'^admin/collection_copy$', 'admin_collection_copy', name='admin_collection_copy'),
-
-  url(r'^install_examples$', 'install_examples', name='install_examples'),
 )
 )
 
 
 
 
-urlpatterns += patterns('search.api',
+urlpatterns += patterns('dashboard.api',
   url(r'^search$', 'search', name='search'),
   url(r'^search$', 'search', name='search'),
   url(r'^suggest/$', 'query_suggest', name='query_suggest'),
   url(r'^suggest/$', 'query_suggest', name='query_suggest'),
   url(r'^index/fields/dynamic$', 'index_fields_dynamic', name='index_fields_dynamic'),
   url(r'^index/fields/dynamic$', 'index_fields_dynamic', name='index_fields_dynamic'),

+ 1 - 232
apps/search/src/search/views.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Licensed to Cloudera, Inc. under one
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # distributed with this work for additional information
@@ -15,250 +14,20 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-import json
 import logging
 import logging
 
 
-from django.utils.html import escape
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
-from desktop.conf import USE_NEW_EDITOR
-from desktop.lib.django_util import JsonResponse, render
+from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
-from desktop.models import Document2, Document
-
 from indexer.management.commands import indexer_setup
 from indexer.management.commands import indexer_setup
 
 
-from search.api_engines import get_engine
-from search.conf import LATEST
-from search.decorators import allow_owner_only
 from search.management.commands import search_setup
 from search.management.commands import search_setup
-from search.models import Collection2, get_engines
-from search.search_controller import SearchController, can_edit_index
-
-from django.core.urlresolvers import reverse
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
-DEFAULT_LAYOUT = [
-     {"size":2,"rows":[{"widgets":[]}],"drops":["temp"],"klass":"card card-home card-column span2"},
-     {"size":10,"rows":[{"widgets":[
-         {"size":12,"name":"Filter Bar","widgetType":"filter-widget", "id":"99923aef-b233-9420-96c6-15d48293532b",
-          "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]},
-                        {"widgets":[
-         {"size":12,"name":"Grid Results","widgetType":"resultset-widget", "id":"14023aef-b233-9420-96c6-15d48293532b",
-          "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
-        "drops":["temp"],"klass":"card card-home card-column span10"},
-]
-
-
-def index(request, is_mobile=False, is_embeddable=False):
-  hue_collections = SearchController(request.user).get_search_collections()
-  collection_id = request.GET.get('collection')
-
-  if not hue_collections or not collection_id:
-    return admin_collections(request, True, is_mobile)
-
-  try:
-    collection_doc = Document2.objects.get(id=collection_id)
-    if USE_NEW_EDITOR.get():
-      collection_doc.can_read_or_exception(request.user)
-    else:
-      collection_doc.doc.get().can_read_or_exception(request.user)
-    collection = Collection2(request.user, document=collection_doc)
-  except Exception, e:
-    raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
-
-  query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
-
-  if request.method == 'GET':
-    if 'q' in request.GET:
-      query['qs'][0]['q'] = request.GET.get('q')
-    if 'qd' in request.GET:
-      query['qd'] = request.GET.get('qd')
-
-  template = 'search.mako'
-  if is_mobile:
-    template = 'search_m.mako'
-  if is_embeddable:
-    template = 'search_embeddable.mako'
-
-  return render(template, request, {
-    'collection': collection,
-    'query': json.dumps(query),
-    'initial': json.dumps({
-        'collections': [],
-        'layout': DEFAULT_LAYOUT,
-        'is_latest': LATEST.get(),
-        'engines': get_engines(request.user)
-    }),
-    'is_owner': collection_doc.doc.get().can_write(request.user),
-    'can_edit_index': can_edit_index(request.user),
-    'mobile': is_mobile,
-  })
-
-def index_m(request):
-  return index(request, True)
-
-def index_embeddable(request):
-  return index(request, False, True)
-
-def new_search(request, is_embeddable=False):
-  engine = request.GET.get('engine', 'solr')
-  collections = get_engine(request.user, engine).datasets()
-  if not collections:
-    return no_collections(request)
-
-  collection = Collection2(user=request.user, name=collections[0], engine=engine)
-  query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
-
-  template = 'search.mako'
-  if is_embeddable:
-    template = 'search_embeddable.mako'
-
-  return render(template, request, {
-    'collection': collection,
-    'query': query,
-    'initial': json.dumps({
-        'collections': collections,
-        'layout': DEFAULT_LAYOUT,
-        'is_latest': LATEST.get(),
-        'engines': get_engines(request.user)
-     }),
-    'is_owner': True,
-    'can_edit_index': can_edit_index(request.user)
-  })
-
-def new_search_embeddable(request):
-  return new_search(request, True)
-
-def browse(request, name, is_mobile=False):
-  collections = SearchController(request.user).get_all_indexes()
-  if not collections:
-    return no_collections(request)
-
-  collection = Collection2(user=request.user, name=name)
-  query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
-
-  template = 'search.mako'
-  if is_mobile:
-    template = 'search_m.mako'
-
-  return render(template, request, {
-    'collection': collection,
-    'query': query,
-    'initial': json.dumps({
-      'autoLoad': True,
-      'collections': collections,
-      'layout': [
-          {"size":12,"rows":[{"widgets":[
-              {"size":12,"name":"Grid Results","id":"52f07188-f30f-1296-2450-f77e02e1a5c0","widgetType":"resultset-widget",
-               "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
-          "drops":["temp"],"klass":"card card-home card-column span10"}
-      ],
-      'is_latest': LATEST.get(),
-      'engines': get_engines(request.user)
-    }),
-    'is_owner': True,
-    'can_edit_index': can_edit_index(request.user),
-    'mobile': is_mobile
-  })
-
-
-def browse_m(request, name):
-  return browse(request, name, True)
-
-
-@allow_owner_only
-def save(request):
-  response = {'status': -1}
-
-  collection = json.loads(request.POST.get('collection', '{}'))
-  layout = json.loads(request.POST.get('layout', '{}'))
-
-  collection['template']['extracode'] = escape(collection['template']['extracode'])
-
-  if collection:
-    if collection['id']:
-      dashboard_doc = Document2.objects.get(id=collection['id'])
-    else:
-      dashboard_doc = Document2.objects.create(name=collection['name'], uuid=collection['uuid'], type='search-dashboard', owner=request.user, description=collection['label'])
-      Document.objects.link(dashboard_doc, owner=request.user, name=collection['name'], description=collection['label'], extra='search-dashboard')
-
-    dashboard_doc.update_data({
-        'collection': collection,
-        'layout': layout
-    })
-    dashboard_doc1 = dashboard_doc.doc.get()
-    dashboard_doc.name = dashboard_doc1.name = collection['label']
-    dashboard_doc.description = dashboard_doc1.description = collection['description']
-    dashboard_doc.save()
-    dashboard_doc1.save()
-
-    response['status'] = 0
-    response['id'] = dashboard_doc.id
-    response['message'] = _('Page saved !')
-  else:
-    response['message'] = _('There is no collection to search.')
-
-  return JsonResponse(response)
-
-
-def no_collections(request):
-  return render('no_collections.mako', request, {})
-
-
-def admin_collections(request, is_redirect=False, is_mobile=False):
-  existing_hue_collections = SearchController(request.user).get_search_collections()
-
-  if request.GET.get('format') == 'json':
-    collections = []
-    for collection in existing_hue_collections:
-      massaged_collection = collection.to_dict()
-      if request.GET.get('is_mobile'):
-        massaged_collection['absoluteUrl'] = reverse('search:index_m') + '?collection=%s' % collection.id
-      massaged_collection['isOwner'] = collection.doc.get().can_write(request.user)
-      collections.append(massaged_collection)
-    return JsonResponse(collections, safe=False)
-
-  template = 'admin_collections.mako'
-  if is_mobile:
-    template = 'admin_collections_m.mako'
-
-  return render(template, request, {
-    'is_embeddable': request.GET.get('is_embeddable', False),
-    'existing_hue_collections': existing_hue_collections,
-    'is_redirect': is_redirect
-  })
-
-
-def admin_collection_delete(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  collections = json.loads(request.POST.get('collections'))
-  searcher = SearchController(request.user)
-  response = {
-    'result': searcher.delete_collections([collection['id'] for collection in collections])
-  }
-
-  return JsonResponse(response)
-
-
-def admin_collection_copy(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  collections = json.loads(request.POST.get('collections'))
-  searcher = SearchController(request.user)
-  response = {
-    'result': searcher.copy_collections([collection['id'] for collection in collections])
-  }
-
-  return JsonResponse(response)
-
-
 def install_examples(request):
 def install_examples(request):
   result = {'status': -1, 'message': ''}
   result = {'status': -1, 'message': ''}
 
 

+ 2 - 1
desktop/Makefile

@@ -52,7 +52,8 @@ APPS := core \
 	libs/libsolr \
 	libs/libsolr \
 	libs/libzookeeper \
 	libs/libzookeeper \
 	libs/metadata \
 	libs/metadata \
-	libs/notebook
+	libs/notebook \
+	libs/dashboard
 
 
 .PHONY: default
 .PHONY: default
 default:: hue syncdb
 default:: hue syncdb

+ 1 - 1
desktop/core/src/desktop/templates/common_header.mako

@@ -417,7 +417,7 @@ ${ hueIcons.symbols() }
        </li>
        </li>
        % endif
        % endif
        % if 'search' in apps:
        % if 'search' in apps:
-         <% from search.search_controller import SearchController %>
+         <% from dashboard.search_controller import SearchController %>
          <% controller = SearchController(user) %>
          <% controller = SearchController(user) %>
          <% collections = controller.get_shared_search_collections() %>
          <% collections = controller.get_shared_search_collections() %>
          % if not collections:
          % if not collections:

+ 9 - 8
desktop/core/src/desktop/templates/responsive.mako

@@ -21,6 +21,7 @@
   from desktop.lib.i18n import smart_unicode
   from desktop.lib.i18n import smart_unicode
   from desktop.views import login_modal
   from desktop.views import login_modal
 
 
+  from dashboard.conf import IS_ENABLED as IS_DASHBOARD_ENABLED
   from metadata.conf import has_optimizer, OPTIMIZER
   from metadata.conf import has_optimizer, OPTIMIZER
 %>
 %>
 
 
@@ -122,8 +123,8 @@ ${ hueIcons.symbols() }
           % if 'impala' in apps and 'beeswax' not in apps: ## impala requires beeswax anyway
           % if 'impala' in apps and 'beeswax' not in apps: ## impala requires beeswax anyway
             <li><a href="javascript: void(0)" data-bind="click: function(){ onePageViewModel.changeEditorType('impala'); onePageViewModel.currentApp('editor') }"><img src="${ static(apps['impala'].icon_path) }" class="app-icon"/> ${_('Impala Query')}</a></li>
             <li><a href="javascript: void(0)" data-bind="click: function(){ onePageViewModel.changeEditorType('impala'); onePageViewModel.currentApp('editor') }"><img src="${ static(apps['impala'].icon_path) }" class="app-icon"/> ${_('Impala Query')}</a></li>
           % endif
           % endif
-          % if 'search' in apps:
-            <li><a href="javascript: void(0)" data-bind="click: function(){ onePageViewModel.currentApp('search') }"><i class="fa fa-fw fa-area-chart"></i> ${ _('Dashboard') }</a></li>
+          % if IS_DASHBOARD_ENABLED.get():
+            <li><a href="javascript: void(0)" data-bind="click: function(){ onePageViewModel.currentApp('dashboard') }"><i class="fa fa-fw fa-area-chart"></i> ${ _('Dashboard') }</a></li>
           % endif
           % endif
           <li><a href="javascript: void(0)" data-bind="click: function(){ onePageViewModel.currentApp('notebook') }"><i class="fa fa-fw fa-file-text-o inline-block"></i> ${ _('Presentation') }</a></li>
           <li><a href="javascript: void(0)" data-bind="click: function(){ onePageViewModel.currentApp('notebook') }"><i class="fa fa-fw fa-file-text-o inline-block"></i> ${ _('Presentation') }</a></li>
           % if 'oozie' in apps:
           % if 'oozie' in apps:
@@ -300,7 +301,7 @@ ${ hueIcons.symbols() }
         <li class="header" style="padding-left: 4px; border-bottom: 1px solid #DDD; padding-bottom: 3px;">${ _('Analyse') }</li>
         <li class="header" style="padding-left: 4px; border-bottom: 1px solid #DDD; padding-bottom: 3px;">${ _('Analyse') }</li>
         <li data-bind="click: function () { onePageViewModel.currentApp('home') }"><a href="javascript: void(0);">Home</a></li>
         <li data-bind="click: function () { onePageViewModel.currentApp('home') }"><a href="javascript: void(0);">Home</a></li>
         <li data-bind="click: function () { onePageViewModel.changeEditorType('hive'); onePageViewModel.currentApp('editor') }"><a href="javascript: void(0);">Editor</a></li>
         <li data-bind="click: function () { onePageViewModel.changeEditorType('hive'); onePageViewModel.currentApp('editor') }"><a href="javascript: void(0);">Editor</a></li>
-        <li data-bind="click: function () { onePageViewModel.currentApp('search') }"><a href="javascript: void(0);">Dashboard</a></li>
+        <li data-bind="click: function () { onePageViewModel.currentApp('dashboard') }"><a href="javascript: void(0);">Dashboard</a></li>
         <li data-bind="click: function () { onePageViewModel.currentApp('notebook') }"><a href="javascript: void(0);">Report</a></li>
         <li data-bind="click: function () { onePageViewModel.currentApp('notebook') }"><a href="javascript: void(0);">Report</a></li>
         <li data-bind="click: function () { onePageViewModel.currentApp('oozie_workflow') }"><a href="javascript: void(0);">Workflows</a></li>
         <li data-bind="click: function () { onePageViewModel.currentApp('oozie_workflow') }"><a href="javascript: void(0);">Workflows</a></li>
         <li class="header">&nbsp;</li>
         <li class="header">&nbsp;</li>
@@ -363,7 +364,7 @@ ${ hueIcons.symbols() }
       <div id="embeddable_editor" class="embeddable"></div>
       <div id="embeddable_editor" class="embeddable"></div>
       <div id="embeddable_notebook" class="embeddable"></div>
       <div id="embeddable_notebook" class="embeddable"></div>
       <div id="embeddable_metastore" class="embeddable"></div>
       <div id="embeddable_metastore" class="embeddable"></div>
-      <div id="embeddable_search" class="embeddable"></div>
+      <div id="embeddable_dashboard" class="embeddable"></div>
       <div id="embeddable_oozie_workflow" class="embeddable"></div>
       <div id="embeddable_oozie_workflow" class="embeddable"></div>
       <div id="embeddable_oozie_coordinator" class="embeddable"></div>
       <div id="embeddable_oozie_coordinator" class="embeddable"></div>
       <div id="embeddable_oozie_bundle" class="embeddable"></div>
       <div id="embeddable_oozie_bundle" class="embeddable"></div>
@@ -581,7 +582,7 @@ ${ assist.assistPanel() }
           editor: '/notebook/editor_embeddable',
           editor: '/notebook/editor_embeddable',
           notebook: '/notebook/notebook_embeddable',
           notebook: '/notebook/notebook_embeddable',
           metastore: '/metastore/tables/?is_embeddable=true',
           metastore: '/metastore/tables/?is_embeddable=true',
-          search: '/search/embeddable/new_search',
+          dashboard: '/dashboard/embeddable/new_search',
           oozie_workflow: '/oozie/editor/workflow/new/?is_embeddable=true',
           oozie_workflow: '/oozie/editor/workflow/new/?is_embeddable=true',
           oozie_coordinator: '/oozie/editor/coordinator/new/?is_embeddable=true',
           oozie_coordinator: '/oozie/editor/coordinator/new/?is_embeddable=true',
           oozie_bundle: '/oozie/editor/bundle/new/?is_embeddable=true',
           oozie_bundle: '/oozie/editor/bundle/new/?is_embeddable=true',
@@ -591,7 +592,7 @@ ${ assist.assistPanel() }
           fileviewer: 'filebrowser/view=',
           fileviewer: 'filebrowser/view=',
           home: '/home_embeddable',
           home: '/home_embeddable',
           indexer: '/indexer/indexer/?is_embeddable=true',
           indexer: '/indexer/indexer/?is_embeddable=true',
-          collections: '/search/admin/collections?is_embeddable=true',
+          collections: '/dashboard/admin/collections?is_embeddable=true',
           indexes: '/indexer/?is_embeddable=true',
           indexes: '/indexer/?is_embeddable=true',
           importer: '/indexer/importer/?is_embeddable=true',
           importer: '/indexer/importer/?is_embeddable=true',
         };
         };
@@ -683,8 +684,8 @@ ${ assist.assistPanel() }
           } else if (href.startsWith('/pig')){
           } else if (href.startsWith('/pig')){
             self.changeEditorType('pig');
             self.changeEditorType('pig');
             self.currentApp('editor');
             self.currentApp('editor');
-          } else if (href.startsWith('/search')){
-            self.currentApp('search');
+          } else if (href.startsWith('/dashboard')){
+            self.currentApp('dashboard');
           } else if (href.startsWith('/oozie/editor/workflow/new')){
           } else if (href.startsWith('/oozie/editor/workflow/new')){
             self.currentApp('oozie_workflow');
             self.currentApp('oozie_workflow');
           } else if (href.startsWith('/oozie/editor/coordinator/new')){
           } else if (href.startsWith('/oozie/editor/coordinator/new')){

+ 36 - 0
desktop/libs/dashboard/Makefile

@@ -0,0 +1,36 @@
+#
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+ifeq ($(ROOT),)
+  $(error "Error: Expect the environment variable $$ROOT to point to the Desktop installation")
+endif
+
+include $(ROOT)/Makefile.sdk
+
+default::
+	@echo '  env-install    : Install into virtual-env'
+
+#
+# env-install
+#   Install app into the virtual environment.
+#
+.PHONY: env-install
+env-install: compile ext-env-install
+	@echo '--- Installing $(APP_NAME) into virtual-env'
+	@$(ENV_PYTHON) setup.py develop -N -q

+ 2 - 0
desktop/libs/dashboard/babel.cfg

@@ -0,0 +1,2 @@
+[python: src/dashboard/**.py]
+[mako: src/dashboard/templates/**.mako]

+ 20 - 0
desktop/libs/dashboard/hueversion.py

@@ -0,0 +1,20 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file should be the one source of truth for for versions within HUE.
+# It is at least included by each of the default hue app's setup.py.
+
+VERSION="3.12.0"

+ 29 - 0
desktop/libs/dashboard/setup.py

@@ -0,0 +1,29 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from setuptools import setup, find_packages
+from hueversion import VERSION
+
+setup(
+      name = "dashboard",
+      version = VERSION,
+      author = "Hue",
+      url = 'http://github.com/cloudera/hue',
+      description = "Drag & Drop and Visualization of data",
+      packages = find_packages('src'),
+      package_dir = {'': 'src'},
+      install_requires = ['setuptools', 'desktop'],
+      entry_points = { 'desktop.sdk.application': 'dashboard=dashboard' },
+)

+ 15 - 0
desktop/libs/dashboard/src/dashboard/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 7 - 6
apps/search/src/search/api.py → desktop/libs/dashboard/src/dashboard/api.py

@@ -27,13 +27,14 @@ from desktop.lib.rest.http_client import RestException
 
 
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
 
 
-from search.api_engines import get_engine
 from search.conf import SOLR_URL
 from search.conf import SOLR_URL
-from search.data_export import download as export_download
-from search.decorators import allow_viewer_only
-from search.facet_builder import _guess_gap, _zoom_range_facet, _new_range_facet
-from search.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception
-from search.search_controller import can_edit_index
+
+from dashboard.dashboard_api import get_engine
+from dashboard.data_export import download as export_download
+from dashboard.decorators import allow_viewer_only
+from dashboard.facet_builder import _guess_gap, _zoom_range_facet, _new_range_facet
+from dashboard.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception
+from dashboard.search_controller import can_edit_index
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)

+ 37 - 0
desktop/libs/dashboard/src/dashboard/conf.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.utils.translation import ugettext_lazy as _
+
+from desktop.conf import is_hue4
+from desktop.lib.conf import Config, coerce_bool
+
+
+IS_ENABLED = Config(
+  key="is_enabled",
+  help=_("Activate the app in the menu."),
+  dynamic_default=is_hue4,
+  private=True,
+  type=coerce_bool)
+
+# TODO [[interfaces]] instead
+IS_SQL_ENABLED = Config(
+  key="is_sql_enabled",
+  help=_("Offer to use SQL engines to compute the dashboards."),
+  dynamic_default=is_hue4,
+  private=True,
+  type=coerce_bool)

+ 1 - 35
apps/search/src/search/api_engines.py → desktop/libs/dashboard/src/dashboard/dashboard_api.py

@@ -17,12 +17,6 @@
 
 
 import logging
 import logging
 
 
-from libsolr.api import SolrApi
-
-from search.conf import SOLR_URL
-from search.models import augment_solr_response
-from search.search_controller import SearchController
-
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -35,6 +29,7 @@ def get_engine(user, engine='solr'):
     from impala.dashboard_api import SQLApi
     from impala.dashboard_api import SQLApi
     return SQLApi(user, engine)
     return SQLApi(user, engine)
   else:
   else:
+    from search.dashboard_api import SearchApi
     return SearchApi(user)
     return SearchApi(user)
 
 
 
 
@@ -64,32 +59,3 @@ class DashboardApi(object):
   def fetch_result(self, collection, query, facet=None): pass
   def fetch_result(self, collection, query, facet=None): pass
 
 
   def get(self, collection, doc_id): pass
   def get(self, collection, doc_id): pass
-
-
-class SearchApi(DashboardApi):
-
-  def __init__(self, user):
-    DashboardApi.__init__(self, user)
-    self.api = SolrApi(SOLR_URL.get(), self.user)
-
-  def query(self, collection, query, facet=None):
-    response = self.api.query(collection, query)
-    return augment_solr_response(response, collection, query)
-
-  def datasets(self, show_all=False):
-    return SearchController(self.user).get_all_indexes(show_all=show_all)
-
-  def fields(self, collection):
-    return self.api.fields(collection)
-
-  def schema_fields(self, collection):
-    return self.api.fields(collection)
-
-  def luke(self, collection):
-    return self.api.luke(collection)
-
-  def stats(self, collection, field, query=None, facet=''):
-    return self.api.stats(collection, field, query, facet)
-
-  def get(self, collection, doc_id):
-    return self.api.get(collection, doc_id)

+ 0 - 0
apps/search/src/search/data_export.py → desktop/libs/dashboard/src/dashboard/data_export.py


+ 0 - 0
apps/search/src/search/decorators.py → desktop/libs/dashboard/src/dashboard/decorators.py


+ 0 - 0
apps/search/src/search/facet_builder.py → desktop/libs/dashboard/src/dashboard/facet_builder.py


+ 801 - 0
desktop/libs/dashboard/src/dashboard/models.py

@@ -0,0 +1,801 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import itertools
+import json
+import logging
+import numbers
+import re
+
+from django.core.urlresolvers import reverse
+from django.utils.html import escape
+from django.utils.translation import ugettext as _
+
+from desktop.lib.i18n import smart_unicode, smart_str
+from desktop.models import get_data_link
+
+from libsolr.api import SolrApi
+from notebook.conf import get_ordered_interpreters
+
+from search.conf import LATEST
+
+from dashboard.conf import IS_SQL_ENABLED
+from dashboard.dashboard_api import get_engine
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Collection2(object):
+
+  def __init__(self, user, name='Default', data=None, document=None, engine='solr'):
+    self.document = document
+
+    if document is not None:
+      self.data = json.loads(document.data)
+    elif data is not None:
+      self.data = json.loads(data)
+    else:
+      self.data = {
+          'collection': self.get_default(user, name, engine),
+          'layout': []
+      }
+
+  def get_json(self, user):
+    props = self.data
+
+    if self.document is not None:
+      props['collection']['id'] = self.document.id
+      props['collection']['label'] = self.document.name
+      props['collection']['description'] = self.document.description
+
+    # For backward compatibility
+    if 'rows' not in props['collection']['template']:
+      props['collection']['template']['rows'] = 25
+    if 'showGrid' not in props['collection']['template']:
+      props['collection']['template']['showGrid'] = True
+    if 'showChart' not in props['collection']['template']:
+      props['collection']['template']['showChart'] = False
+    if 'chartSettings' not in props['collection']['template']:
+      props['collection']['template']['chartSettings'] = {
+        'chartType': 'bars',
+        'chartSorting': 'none',
+        'chartScatterGroup': None,
+        'chartScatterSize': None,
+        'chartScope': 'world',
+        'chartX': None,
+        'chartYSingle': None,
+        'chartYMulti': [],
+        'chartData': [],
+        'chartMapLabel': None,
+      }
+    if 'enabled' not in props['collection']:
+      props['collection']['enabled'] = True
+    if 'engine' not in props['collection']:
+      props['collection']['engine'] = 'solr'
+    if 'leafletmap' not in props['collection']['template']:
+      props['collection']['template']['leafletmap'] = {'latitudeField': None, 'longitudeField': None, 'labelField': None}
+    if 'timeFilter' not in props['collection']:
+      props['collection']['timeFilter'] = {
+        'field': '',
+        'type': 'rolling',
+        'value': 'all',
+        'from': '',
+        'to': '',
+        'truncate': True
+      }
+    if 'suggest' not in props['collection']:
+      props['collection']['suggest'] = {'enabled': False, 'dictionary': ''}
+    for field in props['collection']['template']['fieldsAttributes']:
+      if 'type' not in field:
+        field['type'] = 'string'
+    if 'nested' not in props['collection'] and LATEST.get():
+      props['collection']['nested'] = {
+        'enabled': False,
+        'schema': []
+      }
+
+    for facet in props['collection']['facets']:
+      properties = facet['properties']
+      if 'gap' in properties and not 'initial_gap' in properties:
+        properties['initial_gap'] = properties['gap']
+      if 'start' in properties and not 'initial_start' in properties:
+        properties['initial_start'] = properties['start']
+      if 'end' in properties and not 'initial_end' in properties:
+        properties['initial_end'] = properties['end']
+      if 'domain' not in properties:
+        properties['domain'] = {'blockParent': [], 'blockChildren': []}
+
+      if facet['widgetType'] == 'histogram-widget':
+        if 'timelineChartType' not in properties:
+          properties['timelineChartType'] = 'bar'
+        if 'enableSelection' not in properties:
+          properties['enableSelection'] = True
+        if 'extraSeries' not in properties:
+          properties['extraSeries'] = []
+
+      if facet['widgetType'] == 'map-widget' and facet['type'] == 'field':
+        facet['type'] = 'pivot'
+        properties['facets'] = []
+        properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
+
+    if 'qdefinitions' not in props['collection']:
+      props['collection']['qdefinitions'] = []
+
+    return json.dumps(props)
+
+  def get_default(self, user, name, engine='solr'):
+    fields = self.fields_data(user, name, engine)
+    id_field = [field['name'] for field in fields if field.get('isId')]
+
+    if id_field:
+      id_field = id_field[0]
+    else:
+      id_field = '' # Schemaless might not have an id
+
+    TEMPLATE = {
+      "extracode": escape("<style type=\"text/css\">\nem {\n  font-weight: bold;\n  background-color: yellow;\n}</style>\n\n<script>\n</script>"),
+      "highlighting": [""],
+      "properties": {"highlighting_enabled": True},
+      "template": """
+      <div class="row-fluid">
+        <div class="row-fluid">
+          <div class="span12">%s</div>
+        </div>
+        <br/>
+      </div>""" % ' '.join(['{{%s}}' % field['name'] for field in fields]),
+      "isGridLayout": True,
+      "showFieldList": True,
+      "showGrid": True,
+      "showChart": False,
+      "chartSettings" : {
+        'chartType': 'bars',
+        'chartSorting': 'none',
+        'chartScatterGroup': None,
+        'chartScatterSize': None,
+        'chartScope': 'world',
+        'chartX': None,
+        'chartYSingle': None,
+        'chartYMulti': [],
+        'chartData': [],
+        'chartMapLabel': None,
+      },
+      "fieldsAttributes": [self._make_gridlayout_header_field(field) for field in fields],
+      "fieldsSelected": [],
+      "leafletmap": {'latitudeField': None, 'longitudeField': None, 'labelField': None},
+      "rows": 25,
+    }
+
+    FACETS = []
+
+    return {
+      'id': None,
+      'name': name,
+      'engine': engine,
+      'label': name,
+      'enabled': False,
+      'template': TEMPLATE,
+      'facets': FACETS,
+      'fields': fields,
+      'idField': id_field,
+    }
+
+  @classmethod
+  def _make_field(cls, field, attributes):
+    return {
+        'name': str(escape(field)),
+        'type': str(attributes.get('type', '')),
+        'isId': attributes.get('required') and attributes.get('uniqueKey'),
+        'isDynamic': 'dynamicBase' in attributes
+    }
+
+  @classmethod
+  def _make_gridlayout_header_field(cls, field, isDynamic=False):
+    return {'name': field['name'], 'type': field['type'], 'sort': {'direction': None}, 'isDynamic': isDynamic}
+
+  @classmethod
+  def _make_luke_from_schema_fields(cls, schema_fields):
+    return dict([
+        (f['name'], {
+              'copySources': [],
+              'type': f['type'],
+              'required': True,
+              'uniqueKey': f.get('uniqueKey'),
+              'flags': u'%s-%s-----OF-----l' % ('I' if f['indexed'] else '-', 'S' if f['stored'] else '-'), u'copyDests': []
+        })
+        for f in schema_fields['fields']
+    ])
+
+  def get_absolute_url(self):
+    return reverse('search:index') + '?collection=%s' % self.id
+
+  def fields(self, user):
+    return sorted([str(field.get('name', '')) for field in self.fields_data(user)])
+
+  def fields_data(self, user, name, engine='solr'):
+    api = get_engine(user, engine)
+    try:
+      schema_fields = api.fields(name)
+      schema_fields = schema_fields['schema']['fields']
+    except Exception, e:
+      LOG.warn('/luke call did not succeed: %s' % e)
+      fields = api.schema_fields(name)
+      schema_fields = Collection2._make_luke_from_schema_fields(fields)
+
+    return sorted([self._make_field(field, attributes) for field, attributes in schema_fields.iteritems()])
+
+  def update_data(self, post_data):
+    data_dict = self.data
+
+    data_dict.update(post_data)
+
+    self.data = data_dict
+
+  @property
+  def autocomplete(self):
+    return self.data['autocomplete']
+
+  @autocomplete.setter
+  def autocomplete(self, autocomplete):
+    properties_ = self.data
+    properties_['autocomplete'] = autocomplete
+    self.data = json.dumps(properties_)
+
+  @classmethod
+  def get_field_list(cls, collection):
+    if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
+      fields = set(collection['template']['fieldsSelected'] + ([collection['idField']] if collection['idField'] else []))
+      # Add field if needed
+      if collection['template']['leafletmap'].get('latitudeField'):
+        fields.add(collection['template']['leafletmap']['latitudeField'])
+      if collection['template']['leafletmap'].get('longitudeField'):
+        fields.add(collection['template']['leafletmap']['longitudeField'])
+      if collection['template']['leafletmap'].get('labelField'):
+        fields.add(collection['template']['leafletmap']['labelField'])
+      return list(fields)
+    else:
+      return ['*']
+
+def get_facet_field(category, field, facets):
+  if category in ('nested', 'function'):
+    id_pattern = '%(id)s'
+  else:
+    id_pattern = '%(field)s-%(id)s'
+
+  facets = filter(lambda facet: facet['type'] == category and id_pattern % facet == field, facets)
+
+  if facets:
+    return facets[0]
+  else:
+    return None
+
+def pairwise2(field, fq_filter, iterable):
+  pairs = []
+  selected_values = [f['value'] for f in fq_filter]
+  a, b = itertools.tee(iterable)
+  for element in a:
+    pairs.append({
+        'cat': field,
+        'value': element,
+        'count': next(a),
+        'selected': element in selected_values,
+        'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element])
+    })
+  return pairs
+
+def range_pair(field, cat, fq_filter, iterable, end, collection_facet):
+  # e.g. counts":["0",17430,"1000",1949,"2000",671,"3000",404,"4000",243,"5000",165],"gap":1000,"start":0,"end":6000}
+  pairs = []
+  selected_values = [f['value'] for f in fq_filter]
+  is_single_unit_gap = re.match('^[\+\-]?1[A-Za-z]*$', str(collection_facet['properties']['gap'])) is not None
+  is_up = collection_facet['properties']['sort'] == 'asc'
+
+  if collection_facet['properties']['sort'] == 'asc' and (collection_facet['type'] == 'range-up' or collection_facet['properties'].get('type') == 'range-up'):
+    prev = None
+    n = []
+    for e in iterable:
+      if prev is not None:
+        n.append(e)
+        n.append(prev)
+        prev = None
+      else:
+        prev = e
+    iterable = n
+    iterable.reverse()
+
+  a, to = itertools.tee(iterable)
+  next(to, None)
+  counts = iterable[1::2]
+  total_counts = counts.pop(0) if collection_facet['properties']['sort'] == 'asc' else 0
+
+  for element in a:
+    next(to, None)
+    to_value = next(to, end)
+    count = next(a)
+
+    pairs.append({
+        'field': field, 'from': element, 'value': count, 'to': to_value, 'selected': element in selected_values,
+        'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element]),
+        'is_single_unit_gap': is_single_unit_gap,
+        'total_counts': total_counts,
+        'is_up': is_up
+    })
+    total_counts += counts.pop(0) if counts else 0
+
+  if collection_facet['properties']['sort'] == 'asc' and collection_facet['type'] != 'range-up' and collection_facet['properties'].get('type') != 'range-up':
+    pairs.reverse()
+
+  return pairs
+
+
+def augment_solr_response(response, collection, query):
+  augmented = response
+  augmented['normalized_facets'] = []
+  NAME = '%(field)s-%(id)s'
+  normalized_facets = []
+
+  selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
+
+  if response and response.get('facet_counts'):
+    for facet in collection['facets']:
+      category = facet['type']
+
+      if category == 'field' and response['facet_counts']['facet_fields']:
+        name = NAME % facet
+        collection_facet = get_facet_field(category, name, collection['facets'])
+        counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name])
+        if collection_facet['properties']['sort'] == 'asc':
+          counts.reverse()
+        facet = {
+          'id': collection_facet['id'],
+          'field': facet['field'],
+          'type': category,
+          'label': collection_facet['label'],
+          'counts': counts,
+        }
+        normalized_facets.append(facet)
+      elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']:
+        name = NAME % facet
+        collection_facet = get_facet_field(category, name, collection['facets'])
+        counts = response['facet_counts']['facet_ranges'][name]['counts']
+        end = response['facet_counts']['facet_ranges'][name]['end']
+        counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet)
+        facet = {
+          'id': collection_facet['id'],
+          'field': facet['field'],
+          'type': category,
+          'label': collection_facet['label'],
+          'counts': counts,
+          'extraSeries': []
+        }
+        normalized_facets.append(facet)
+      elif category == 'query' and response['facet_counts']['facet_queries']:
+        for name, value in response['facet_counts']['facet_queries'].iteritems():
+          collection_facet = get_facet_field(category, name, collection['facets'])
+          facet = {
+            'id': collection_facet['id'],
+            'query': name,
+            'type': category,
+            'label': name,
+            'counts': value,
+          }
+          normalized_facets.append(facet)
+      elif category == 'pivot':
+        name = NAME % facet
+        if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']:
+          if facet['properties']['scope'] == 'stack':
+            count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values)
+          else:
+            count = response['facet_counts']['facet_pivot'][name]
+            _augment_pivot_nd(facet['id'], count, selected_values)
+        else:
+          count = []
+        facet = {
+          'id': facet['id'],
+          'field': name,
+          'type': category,
+          'label': name,
+          'counts': count,
+        }
+        normalized_facets.append(facet)
+
+  if response and response.get('facets'):
+    for facet in collection['facets']:
+      category = facet['type']
+      name = facet['id'] # Nested facets can only have one name
+
+      if category == 'function' and name in response['facets']:
+        value = response['facets'][name]
+        collection_facet = get_facet_field(category, name, collection['facets'])
+        facet = {
+          'id': collection_facet['id'],
+          'query': name,
+          'type': category,
+          'label': name,
+          'counts': value,
+        }
+        normalized_facets.append(facet)
+      elif category == 'nested' and name in response['facets']:
+        value = response['facets'][name]
+        collection_facet = get_facet_field(category, name, collection['facets'])
+        extraSeries = []
+        counts = response['facets'][name]['buckets']
+
+        cols = ['%(field)s' % facet, 'count(%(field)s)' % facet]
+        last_x_col = 0
+        last_xx_col = 0
+        for i, f in enumerate(facet['properties']['facets']):
+          if f['aggregate']['function'] == 'count':
+            cols.append(f['field'])
+            last_xx_col = last_x_col
+            last_x_col = i + 2
+          cols.append(SolrApi._get_aggregate_function(f))
+        rows = []
+
+        # For dim in dimensions
+
+        # Number or Date range
+        if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
+          dimension = 3 if collection_facet['properties']['isDate'] else 1
+          # Single dimension or dimension 2 with analytics
+          if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1:
+            column = 'count'
+            if len(collection_facet['properties']['facets']) == 1:
+              agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
+              legend = agg_keys[0].split(':', 2)[1]
+              column = agg_keys[0]
+            else:
+              legend = facet['field'] # 'count(%s)' % legend
+              agg_keys = [column]
+
+            _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
+
+            counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
+            counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet)
+          else:
+            # Dimension 1 with counts and 2 with analytics
+            agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
+            agg_keys.sort(key=lambda a: a[4:])
+
+            if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
+              agg_keys.insert(0, 'count')
+            counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
+
+            _series = collections.defaultdict(list)
+
+            for row in rows:
+              for i, cell in enumerate(row):
+                if i > last_x_col:
+                  legend = cols[i]
+                  if last_xx_col != last_x_col:
+                    legend = '%s %s' % (cols[i], row[last_x_col])
+                  _series[legend].append(row[last_xx_col])
+                  _series[legend].append(cell)
+
+            for name, val in _series.iteritems():
+              _c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet)
+              extraSeries.append({'counts': _c, 'label': name})
+            counts = []
+        elif collection_facet['properties'].get('isOldPivot'):
+          facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count']
+
+          column = 'count'
+          agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
+          agg_keys.sort(key=lambda a: a[4:])
+
+          if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
+            agg_keys.insert(0, 'count')
+          counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
+
+          #_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
+          dimension = len(facet_fields)
+        elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
+          # Dimension 1 with 1 count or agg
+          dimension = 1
+
+          column = 'count'
+          if len(collection_facet['properties']['facets']) == 1:
+            agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
+            legend = agg_keys[0].split(':', 2)[1]
+            column = agg_keys[0]
+          else:
+            legend = facet['field']
+            agg_keys = [column]
+
+          _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
+
+          counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
+          counts = pairwise2(legend, selected_values.get(facet['id'], []), counts)
+        else:
+          # Dimension 2 with analytics or 1 with N aggregates
+          dimension = 2
+          agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
+          agg_keys.sort(key=lambda a: a[4:])
+
+          if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
+            agg_keys.insert(0, 'count')
+          counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
+          actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']])
+
+          counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts)
+
+        num_bucket = response['facets'][name]['numBuckets'] if 'numBuckets' in response['facets'][name] else len(response['facets'][name])
+        facet = {
+          'id': collection_facet['id'],
+          'field': facet['field'],
+          'type': category,
+          'label': collection_facet['label'],
+          'counts': counts,
+          'extraSeries': extraSeries,
+          'dimension': dimension,
+          'response': {'response': {'start': 0, 'numFound': num_bucket}}, # Todo * nested buckets + offsets
+          'docs': [dict(zip(cols, row)) for row in rows],
+          'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols]
+        }
+
+        normalized_facets.append(facet)
+
+    # Remove unnecessary facet data
+    if response:
+      response.pop('facet_counts')
+      response.pop('facets')
+
+  augment_response(collection, query, response)
+
+  if normalized_facets:
+    augmented['normalized_facets'].extend(normalized_facets)
+
+  return augmented
+
+
+def augment_response(collection, query, response):
+  # HTML escaping
+  if not query.get('download'):
+    id_field = collection.get('idField', '')
+
+    for doc in response['response']['docs']:
+      for field, value in doc.iteritems():
+        if isinstance(value, numbers.Number):
+          escaped_value = value
+        elif field == '_childDocuments_': # Nested documents
+          escaped_value = value
+        elif isinstance(value, list): # Multivalue field
+          escaped_value = [smart_unicode(escape(val), errors='replace') for val in value]
+        else:
+          value = smart_unicode(value, errors='replace')
+          escaped_value = escape(value)
+        doc[field] = escaped_value
+
+      link = None
+      if 'link-meta' in doc:
+        meta = json.loads(doc['link-meta'])
+        link = get_data_link(meta)
+      elif 'link' in doc:
+        meta = {'type': 'link', 'link': doc['link']}
+        link = get_data_link(meta)
+
+      doc['externalLink'] = link
+      doc['details'] = []
+      doc['hueId'] = smart_unicode(doc.get(id_field, ''))
+
+  highlighted_fields = response.get('highlighting', {}).keys()
+  if highlighted_fields and not query.get('download'):
+    id_field = collection.get('idField')
+    if id_field:
+      for doc in response['response']['docs']:
+        if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields:
+          highlighting = response['highlighting'][smart_unicode(doc[id_field])]
+
+          if highlighting:
+            escaped_highlighting = {}
+            for field, hls in highlighting.iteritems():
+              _hls = [escape(smart_unicode(hl, errors='replace')).replace('&lt;em&gt;', '<em>').replace('&lt;/em&gt;', '</em>') for hl in hls]
+              escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls
+
+            doc.update(escaped_highlighting)
+    else:
+      response['warning'] = _("The Solr schema requires an id field for performing the result highlighting")
+
+
+def _augment_pivot_2d(name, facet_id, counts, selected_values):
+  values = set()
+
+  for dimension in counts:
+    for pivot in dimension['pivot']:
+      values.add(pivot['value'])
+
+  values = sorted(list(values))
+  augmented = []
+
+  for dimension in counts:
+    count = {}
+    pivot_field = ''
+    for pivot in dimension['pivot']:
+      count[pivot['value']] = pivot['count']
+      pivot_field = pivot['field']
+    for val in values:
+      fq_values = [dimension['value'], val]
+      fq_fields = [dimension['field'], pivot_field]
+      fq_filter = selected_values.get(facet_id, [])
+      _selected_values = [f['value'] for f in fq_filter]
+
+      augmented.append({
+          "count": count.get(val, 0),
+          "value": val,
+          "cat": dimension['value'],
+          'selected': fq_values in _selected_values,
+          'exclude': all([f['exclude'] for f in fq_filter if f['value'] == val]),
+          'fq_fields': fq_fields,
+          'fq_values': fq_values,
+      })
+
+  return augmented
+
+
+def _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows):
+  fq_fields = []
+  fq_values = []
+  fq_filter = []
+  _selected_values = [f['value'] for f in selected_values.get(facet['id'], [])]
+  _fields = [facet['field']] + [facet['field'] for facet in facet['properties']['facets']]
+
+  return __augment_stats_2d(counts, facet['field'], fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys, rows)
+
+
+# Clear one dimension
+def __augment_stats_2d(counts, label, fq_fields, fq_values, fq_filter, _selected_values, _fields, agg_keys, rows):
+  augmented = []
+
+  for bucket in counts: # For each dimension, go through each bucket and pick up the counts or aggregates, then go recursively in the next dimension
+    val = bucket['val']
+    count = bucket['count']
+    dim_row = [val]
+
+    _fq_fields = fq_fields + _fields[0:1]
+    _fq_values = fq_values + [val]
+
+    for agg_key in agg_keys:
+      if agg_key == 'count':
+        dim_row.append(count)
+        augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
+      elif agg_key.startswith('agg_'):
+        label = fq_values[0] if len(_fq_fields) >= 2 else agg_key.split(':', 2)[1]
+        if agg_keys.index(agg_key) == 0: # One count by dimension
+          dim_row.append(count)
+        dim_row.append(bucket[agg_key])
+        augmented.append(_get_augmented(bucket[agg_key], val, label, _fq_values, _fq_fields, fq_filter, _selected_values))
+      else:
+        augmented.append(_get_augmented(count, val, label, _fq_values, _fq_fields, fq_filter, _selected_values)) # Needed?
+
+        # Go rec
+        _agg_keys = [key for key, value in bucket[agg_key]['buckets'][0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
+        _agg_keys.sort(key=lambda a: a[4:])
+
+        if not _agg_keys or len(_agg_keys) == 1 and _agg_keys[0].lower().startswith('dim_'):
+          _agg_keys.insert(0, 'count')
+        next_dim = []
+        new_rows = []
+        augmented += __augment_stats_2d(bucket[agg_key]['buckets'], val, _fq_fields, _fq_values, fq_filter, _selected_values, _fields[1:], _agg_keys, next_dim)
+        for row in next_dim:
+          new_rows.append(dim_row + row)
+        dim_row = new_rows
+
+    if dim_row and type(dim_row[0]) == list:
+      rows.extend(dim_row)
+    else:
+      rows.append(dim_row)
+
+  return augmented
+
+
+def _get_augmented(count, val, label, fq_values, fq_fields, fq_filter, _selected_values):
+  return {
+      "count": count,
+      "value": val,
+      "cat": label,
+      'selected': fq_values in _selected_values,
+      'exclude': all([f['exclude'] for f in fq_filter if f['value'] == val]),
+      'fq_fields': fq_fields,
+      'fq_values': fq_values
+  }
+
+
+def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''):
+  for c in counts:
+    fq_fields = (fields if fields else []) + [c['field']]
+    fq_values = (values if values else []) + [smart_str(c['value'])]
+
+    if 'pivot' in c:
+      _augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values)
+
+    fq_filter = selected_values.get(facet_id, [])
+    _selected_values = [f['value'] for f in fq_filter]
+    c['selected'] = fq_values in _selected_values
+    c['exclude'] = False
+    c['fq_fields'] = fq_fields
+    c['fq_values'] = fq_values
+
+
+def _convert_nested_to_augmented_pivot_nd(facet_fields, facet_id, counts, selected_values, fields='', values='', dimension=2):
+  for c in counts['buckets']:
+    c['field'] = facet_fields[0]
+    fq_fields = (fields if fields else []) + [c['field']]
+    fq_values = (values if values else []) + [smart_str(c['val'])]
+    c['value'] = c.pop('val')
+    bucket = 'd%s' % dimension
+
+    if bucket in c:
+      next_dimension = facet_fields[1:]
+      if next_dimension:
+        _convert_nested_to_augmented_pivot_nd(next_dimension, facet_id, c[bucket], selected_values, fq_fields, fq_values, dimension=dimension+1)
+        c['pivot'] = c.pop(bucket)['buckets']
+      else:
+        c['count'] = c.pop(bucket)
+
+    fq_filter = selected_values.get(facet_id, [])
+    _selected_values = [f['value'] for f in fq_filter]
+    c['selected'] = fq_values in _selected_values
+    c['exclude'] = False
+    c['fq_fields'] = fq_fields
+    c['fq_values'] = fq_values
+
+
+def get_engines(user):
+  engines = [{'name': _('index (Solr)'), 'type': 'solr'}]
+
+  if IS_SQL_ENABLED.get():
+    engines += [{
+          'name': _('table (%s)') % interpreter['name'],
+          'type': interpreter['type'],
+          'async': interpreter['interface'] == 'hiveserver2'
+        }
+        for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms')
+    ]
+
+  return engines
+
+
+def augment_solr_exception(response, collection):
+  response.update(
+  {
+    "facet_counts": {
+    },
+    "highlighting": {
+    },
+    "normalized_facets": [
+      {
+        "field": facet['field'],
+        "counts": [],
+        "type": facet['type'],
+        "label": facet['label']
+      }
+      for facet in collection['facets']
+    ],
+    "responseHeader": {
+      "status": -1,
+      "QTime": 0,
+      "params": {
+      }
+    },
+    "response": {
+      "start": 0,
+      "numFound": 0,
+      "docs": [
+      ]
+    }
+  })

+ 1 - 1
apps/search/src/search/search_controller.py → desktop/libs/dashboard/src/dashboard/search_controller.py

@@ -25,7 +25,7 @@ from desktop.models import Document2, Document, SAMPLE_USER_OWNERS
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
 
 
 from search.conf import SOLR_URL
 from search.conf import SOLR_URL
-from search.models import Collection2
+from dashboard.models import Collection2
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)

+ 23 - 0
desktop/libs/dashboard/src/dashboard/settings.py

@@ -0,0 +1,23 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DJANGO_APPS = ["dashboard"]
+NICE_NAME = "Analytics Dashboards"
+MENU_INDEX = -1
+ICON = "dashboard/art/icon_search_48.png"
+
+REQUIRES_HADOOP = False
+IS_URL_NAMESPACED = True

BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/bird_gray_32.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_logs.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_logs_48.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_search_24.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_search_48.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_twitter.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_twitter_48.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_yelp.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/icon_yelp_48.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/remove.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/reply.png


BIN
desktop/libs/dashboard/src/dashboard/static/dashboard/art/retweet.png


+ 0 - 0
apps/search/src/search/static/search/css/admin.css → desktop/libs/dashboard/src/dashboard/static/dashboard/css/admin.css


+ 0 - 0
apps/search/src/search/static/search/css/admin_mobile.css → desktop/libs/dashboard/src/dashboard/static/dashboard/css/admin_mobile.css


+ 0 - 0
apps/search/src/search/static/search/css/search.css → desktop/libs/dashboard/src/dashboard/static/dashboard/css/search.css


+ 0 - 0
apps/search/src/search/static/search/css/search_mobile.css → desktop/libs/dashboard/src/dashboard/static/dashboard/css/search_mobile.css


+ 0 - 0
apps/search/src/search/static/search/help/index.html → desktop/libs/dashboard/src/dashboard/static/dashboard/help/index.html


+ 0 - 0
apps/search/src/search/static/search/img/clear.png → desktop/libs/dashboard/src/dashboard/static/dashboard/img/clear.png


+ 0 - 0
apps/search/src/search/static/search/img/loading.gif → desktop/libs/dashboard/src/dashboard/static/dashboard/img/loading.gif


+ 0 - 0
apps/search/src/search/static/search/js/collections.ko.js → desktop/libs/dashboard/src/dashboard/static/dashboard/js/collections.ko.js


+ 0 - 0
apps/search/src/search/static/search/js/create-collections.ko.js → desktop/libs/dashboard/src/dashboard/static/dashboard/js/create-collections.ko.js


+ 20 - 20
apps/search/src/search/static/search/js/search.ko.js → desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js

@@ -388,7 +388,7 @@ var FieldAnalysis = function (vm, fieldName, fieldType) {
   self.getTerms = function () {
   self.getTerms = function () {
     self.isLoading(true);
     self.isLoading(true);
     self.terms.data.removeAll();
     self.terms.data.removeAll();
-    $.post("/search/get_terms", {
+    $.post("/dashboard/get_terms", {
       collection: ko.mapping.toJSON(vm.collection),
       collection: ko.mapping.toJSON(vm.collection),
       analysis: ko.mapping.toJSON(self)
       analysis: ko.mapping.toJSON(self)
     }, function (data) {
     }, function (data) {
@@ -415,7 +415,7 @@ var FieldAnalysis = function (vm, fieldName, fieldType) {
   self.getStats = function () {
   self.getStats = function () {
     self.stats.data.removeAll();
     self.stats.data.removeAll();
     self.isLoading(true);
     self.isLoading(true);
-    $.post("/search/get_stats", {
+    $.post("/dashboard/get_stats", {
       collection: ko.mapping.toJSON(vm.collection),
       collection: ko.mapping.toJSON(vm.collection),
       query: ko.mapping.toJSON(vm.query),
       query: ko.mapping.toJSON(vm.query),
       analysis: ko.mapping.toJSON(self)
       analysis: ko.mapping.toJSON(self)
@@ -502,7 +502,7 @@ var Collection = function (vm, collection) {
 	if (val == 'fixed' && self.timeFilter.from().length == 0) {
 	if (val == 'fixed' && self.timeFilter.from().length == 0) {
       $.ajax({
       $.ajax({
         type: "POST",
         type: "POST",
-        url: "/search/get_range_facet",
+        url: "/dashboard/get_range_facet",
         data: {
         data: {
           collection: ko.mapping.toJSON(self),
           collection: ko.mapping.toJSON(self),
           facet: ko.mapping.toJSON({widgetType: 'facet-widget', field: self.timeFilter.field()}),
           facet: ko.mapping.toJSON({widgetType: 'facet-widget', field: self.timeFilter.field()}),
@@ -831,7 +831,7 @@ var Collection = function (vm, collection) {
     self.removeFacet(function(){return facet_json.widget_id});
     self.removeFacet(function(){return facet_json.widget_id});
     logGA('add_facet/' + facet_json.widgetType);
     logGA('add_facet/' + facet_json.widgetType);
 
 
-    $.post("/search/template/new_facet", {
+    $.post("/dashboard/template/new_facet", {
         "collection": ko.mapping.toJSON(self),
         "collection": ko.mapping.toJSON(self),
         "id": facet_json.widget_id,
         "id": facet_json.widget_id,
         "label": facet_json.name,
         "label": facet_json.name,
@@ -1118,7 +1118,7 @@ var Collection = function (vm, collection) {
   });
   });
 
 
   self.switchCollection = function() {
   self.switchCollection = function() {
-    $.post("/search/get_collection", {
+    $.post("/dashboard/get_collection", {
         name: self.name(),
         name: self.name(),
         engine: self.engine()
         engine: self.engine()
     }, function (data) {
     }, function (data) {
@@ -1179,7 +1179,7 @@ var Collection = function (vm, collection) {
   }
   }
 
 
   self.syncFields = function() {
   self.syncFields = function() {
-    $.post("/search/get_collection", {
+    $.post("/dashboard/get_collection", {
         name: self.name(),
         name: self.name(),
         engine: self.engine()
         engine: self.engine()
       }, function (data) {
       }, function (data) {
@@ -1194,7 +1194,7 @@ var Collection = function (vm, collection) {
   };
   };
 
 
   self.syncDynamicFields = function () {
   self.syncDynamicFields = function () {
-    $.post("/search/index/fields/dynamic", {
+    $.post("/dashboard/index/fields/dynamic", {
         name: self.name(),
         name: self.name(),
         engine: self.engine()
         engine: self.engine()
       }, function (data) {
       }, function (data) {
@@ -1210,7 +1210,7 @@ var Collection = function (vm, collection) {
   };
   };
 
 
   self.getNestedDocuments = function () {
   self.getNestedDocuments = function () {
-    $.post("/search/index/fields/nested_documents", {
+    $.post("/dashboard/index/fields/nested_documents", {
         collection: ko.mapping.toJSON(self),
         collection: ko.mapping.toJSON(self),
         engine: self.engine()
         engine: self.engine()
       }, function (data) {
       }, function (data) {
@@ -1303,7 +1303,7 @@ var Collection = function (vm, collection) {
 
 
     $.ajax({
     $.ajax({
       type: "POST",
       type: "POST",
-      url: "/search/get_range_facet",
+      url: "/dashboard/get_range_facet",
       data: {
       data: {
         collection: ko.mapping.toJSON(self),
         collection: ko.mapping.toJSON(self),
         facet: ko.mapping.toJSON(facet),
         facet: ko.mapping.toJSON(facet),
@@ -1397,7 +1397,7 @@ var NewTemplate = function (vm, initial) {
 
 
   self.syncCollections = function () {
   self.syncCollections = function () {
     vm.isSyncingCollections(true);
     vm.isSyncingCollections(true);
-    $.post("/search/get_collections", {
+    $.post("/dashboard/get_collections", {
         collection: ko.mapping.toJSON(vm.collection),
         collection: ko.mapping.toJSON(vm.collection),
         show_all: ko.mapping.toJSON(vm.showCores)
         show_all: ko.mapping.toJSON(vm.showCores)
       }, function (data) {
       }, function (data) {
@@ -1738,7 +1738,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
     if (! self.collection.async()) {
     if (! self.collection.async()) {
       self._loadResults(facet, facet.queryResult().asyncResult());
       self._loadResults(facet, facet.queryResult().asyncResult());
     } else {
     } else {
-      $.post("/search/search", {
+      $.post("/dashboard/search", {
           collection: ko.mapping.toJSON(self.collection),
           collection: ko.mapping.toJSON(self.collection),
           query: ko.mapping.toJSON(self.query),
           query: ko.mapping.toJSON(self.query),
           facet: ko.mapping.toJSON(facet),
           facet: ko.mapping.toJSON(facet),
@@ -1809,7 +1809,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       }
       }
 
 
       multiQs = $.map(queries, function (qdata) {
       multiQs = $.map(queries, function (qdata) {
-        return $.post("/search/get_timeline", {
+        return $.post("/dashboard/get_timeline", {
           collection: ko.mapping.toJSON(self.collection),
           collection: ko.mapping.toJSON(self.collection),
           query: ko.mapping.toJSON(self.query),
           query: ko.mapping.toJSON(self.query),
           facet: ko.mapping.toJSON(facet),
           facet: ko.mapping.toJSON(facet),
@@ -1829,7 +1829,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
       });
       });
 
 
       multiQs = $.map(self.collection.facets(), function(facet) {
       multiQs = $.map(self.collection.facets(), function(facet) {
-        return $.post("/search/search", {
+        return $.post("/dashboard/search", {
             collection: ko.mapping.toJSON(self.collection),
             collection: ko.mapping.toJSON(self.collection),
             query: ko.mapping.toJSON(self.query),
             query: ko.mapping.toJSON(self.query),
             facet: ko.mapping.toJSON(facet),
             facet: ko.mapping.toJSON(facet),
@@ -1860,7 +1860,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
 
 
 
 
     $.when.apply($, [
     $.when.apply($, [
-      $.post("/search/search", {
+      $.post("/dashboard/search", {
           collection: ko.mapping.toJSON(self.collection),
           collection: ko.mapping.toJSON(self.collection),
           query: ko.mapping.toJSON(self.query),
           query: ko.mapping.toJSON(self.query),
           layout: ko.mapping.toJSON(self.columns)
           layout: ko.mapping.toJSON(self.columns)
@@ -2047,7 +2047,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   }
   }
 
 
   self.suggest = function (query, callback) {
   self.suggest = function (query, callback) {
-    $.post("/search/suggest/", {
+    $.post("/dashboard/suggest/", {
       collection: ko.mapping.toJSON(self.collection),
       collection: ko.mapping.toJSON(self.collection),
       query: query
       query: query
     }, function (data) {
     }, function (data) {
@@ -2103,7 +2103,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   }
   }
 
 
   self.getDocument = function (doc, callback) {
   self.getDocument = function (doc, callback) {
-    $.post("/search/get_document", {
+    $.post("/dashboard/get_document", {
       collection: ko.mapping.toJSON(self.collection),
       collection: ko.mapping.toJSON(self.collection),
       id: doc.id
       id: doc.id
     }, function (data) {
     }, function (data) {
@@ -2146,7 +2146,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   };
   };
 
 
   self.updateDocument = function (doc) {
   self.updateDocument = function (doc) {
-    $.post("/search/update_document", {
+    $.post("/dashboard/update_document", {
       collection: ko.mapping.toJSON(self.collection),
       collection: ko.mapping.toJSON(self.collection),
       document: ko.mapping.toJSON(doc),
       document: ko.mapping.toJSON(doc),
       id: doc.id
       id: doc.id
@@ -2200,7 +2200,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   }
   }
 
 
   self.save = function () {
   self.save = function () {
-    $.post("/search/save", {
+    $.post("/dashboard/save", {
       collection: ko.mapping.toJSON(self.collection),
       collection: ko.mapping.toJSON(self.collection),
       layout: ko.mapping.toJSON(self.columns)
       layout: ko.mapping.toJSON(self.columns)
     }, function (data) {
     }, function (data) {
@@ -2208,7 +2208,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
         self.collection.id(data.id);
         self.collection.id(data.id);
         $(document).trigger("info", data.message);
         $(document).trigger("info", data.message);
         if (window.location.search.indexOf("collection") == -1) {
         if (window.location.search.indexOf("collection") == -1) {
-          hueUtils.changeURL('/search/?collection=' + data.id);
+          hueUtils.changeURL('/dashboard/?collection=' + data.id);
         }
         }
       }
       }
       else {
       else {
@@ -2223,6 +2223,6 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
 
 
 function logGA(page) {
 function logGA(page) {
   if (typeof trackOnGA == 'function') {
   if (typeof trackOnGA == 'function') {
-    trackOnGA('search/' + page);
+    trackOnGA('dashboard/' + page);
   }
   }
 }
 }

+ 0 - 0
apps/search/src/search/static/search/js/search.utils.js → desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.utils.js


+ 0 - 0
apps/search/src/search/static/search/templates/logs.jpg → desktop/libs/dashboard/src/dashboard/static/dashboard/templates/logs.jpg


+ 0 - 0
apps/search/src/search/static/search/templates/restaurant.jpg → desktop/libs/dashboard/src/dashboard/static/dashboard/templates/restaurant.jpg


+ 0 - 0
apps/search/src/search/static/search/templates/templates.xml → desktop/libs/dashboard/src/dashboard/static/dashboard/templates/templates.xml


+ 0 - 0
apps/search/src/search/static/search/templates/twitter.jpg → desktop/libs/dashboard/src/dashboard/static/dashboard/templates/twitter.jpg


+ 1 - 1
apps/search/src/search/templates/admin_collections.mako → desktop/libs/dashboard/src/dashboard/templates/admin_collections.mako

@@ -22,7 +22,7 @@
 <%namespace name="common_admin_collections" file="common_admin_collections.mako" />
 <%namespace name="common_admin_collections" file="common_admin_collections.mako" />
 
 
 %if not is_embeddable:
 %if not is_embeddable:
-${ commonheader(_('Search'), "search", user, request, "29px") | n,unicode }
+${ commonheader(_('Dashboard'), "dashboard", user, request, "29px") | n,unicode }
 %endif
 %endif
 
 
 ${ common_admin_collections.page_structure() }
 ${ common_admin_collections.page_structure() }

+ 1 - 1
apps/search/src/search/templates/admin_collections_m.mako → desktop/libs/dashboard/src/dashboard/templates/admin_collections_m.mako

@@ -21,7 +21,7 @@
 
 
 <%namespace name="common_admin_collections" file="common_admin_collections.mako" />
 <%namespace name="common_admin_collections" file="common_admin_collections.mako" />
 
 
-${ commonheader_m(_('Search'), "search", user, request, "29px") | n,unicode }
+${ commonheader_m(_('Dashboard'), "dashboard", user, request, "29px") | n,unicode }
 
 
 ${ common_admin_collections.page_structure(True) }
 ${ common_admin_collections.page_structure(True) }
 
 

+ 10 - 10
apps/search/src/search/templates/common_admin_collections.mako → desktop/libs/dashboard/src/dashboard/templates/common_admin_collections.mako

@@ -24,9 +24,9 @@
 
 
 <%def name="page_structure(is_mobile=False)">
 <%def name="page_structure(is_mobile=False)">
 
 
-<link rel="stylesheet" href="${ static('search/css/admin.css') }">
+<link rel="stylesheet" href="${ static('dashboard/css/admin.css') }">
 %if is_mobile:
 %if is_mobile:
-<link rel="stylesheet" href="${ static('search/css/admin_mobile.css') }">
+<link rel="stylesheet" href="${ static('dashboard/css/admin_mobile.css') }">
 <h3 style="text-align: center">${_('Dashboards')}</h3>
 <h3 style="text-align: center">${_('Dashboards')}</h3>
 %endif
 %endif
 
 
@@ -75,7 +75,7 @@
 
 
       <%def name="creation()">
       <%def name="creation()">
         %if not is_mobile:
         %if not is_mobile:
-        <a data-bind="visible: collections().length > 0 && !isLoading()" class="btn" href="${ url('search:new_search') }" title="${ _('Create a new dashboard') }">
+        <a data-bind="visible: collections().length > 0 && !isLoading()" class="btn" href="${ url('dashboard:new_search') }" title="${ _('Create a new dashboard') }">
           <i class="fa fa-plus-circle"></i> ${ _('Create') }
           <i class="fa fa-plus-circle"></i> ${ _('Create') }
         </a>
         </a>
         <a data-bind="visible: !isLoading(), click: function() { $('#import-documents').modal('show'); }" class="btn">
         <a data-bind="visible: !isLoading(), click: function() { $('#import-documents').modal('show'); }" class="btn">
@@ -87,13 +87,13 @@
 
 
     <div class="row-fluid" data-bind="visible: collections().length == 0 && !isLoading()">
     <div class="row-fluid" data-bind="visible: collections().length == 0 && !isLoading()">
       <div class="span10 offset1 center importBtn pointer">
       <div class="span10 offset1 center importBtn pointer">
-        <a href="${ url('search:new_search') }">
+        <a href="${ url('dashboard:new_search') }">
           <i class="fa fa-plus-circle waiting"></i>
           <i class="fa fa-plus-circle waiting"></i>
         </a>
         </a>
 
 
         <h1 class="emptyMessage">
         <h1 class="emptyMessage">
           ${ _('There are currently no dashboards defined.') }<br/>
           ${ _('There are currently no dashboards defined.') }<br/>
-          <a href="${ url('search:new_search') }">${ _('Click here to add') }</a> ${ _('one or more.') }</h1>
+          <a href="${ url('dashboard:new_search') }">${ _('Click here to add') }</a> ${ _('one or more.') }</h1>
         </h1>
         </h1>
       </div>
       </div>
     </div>
     </div>
@@ -164,7 +164,7 @@ ${ commonshare() | n,unicode }
 ${ commonimportexport(request) | n,unicode }
 ${ commonimportexport(request) | n,unicode }
 
 
 
 
-<script src="${ static('search/js/collections.ko.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('dashboard/js/collections.ko.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/share2.vm.js') }"></script>
 <script src="${ static('desktop/js/share2.vm.js') }"></script>
 
 
 
 
@@ -172,12 +172,12 @@ ${ commonimportexport(request) | n,unicode }
   var appProperties = {
   var appProperties = {
     labels: [],
     labels: [],
     %if is_mobile:
     %if is_mobile:
-    listCollectionsUrl: "${ url("search:admin_collections") }?format=json&is_mobile=true",
+    listCollectionsUrl: "${ url("dashboard:admin_collections") }?format=json&is_mobile=true",
     %else:
     %else:
-    listCollectionsUrl: "${ url("search:admin_collections") }?format=json",
+    listCollectionsUrl: "${ url("dashboard:admin_collections") }?format=json",
     %endif
     %endif
-    deleteUrl: "${ url("search:admin_collection_delete") }",
-    copyUrl: "${ url("search:admin_collection_copy") }",
+    deleteUrl: "${ url("dashboard:admin_collection_delete") }",
+    copyUrl: "${ url("dashboard:admin_collection_copy") }",
     indexerUrl: "/indexer/#link/"
     indexerUrl: "/indexer/#link/"
   }
   }
 
 

+ 6 - 6
apps/search/src/search/templates/common_search.mako → desktop/libs/dashboard/src/dashboard/templates/common_search.mako

@@ -78,7 +78,7 @@ from desktop.views import commonheader, commonfooter, _ko
     <a class="btn" href="javascript:void(0)" title="${ _('New') }" rel="tooltip" data-placement="bottom" data-bind="css: {'btn': true}, click: newSearch">
     <a class="btn" href="javascript:void(0)" title="${ _('New') }" rel="tooltip" data-placement="bottom" data-bind="css: {'btn': true}, click: newSearch">
       <i class="fa fa-file-o"></i>
       <i class="fa fa-file-o"></i>
     </a>
     </a>
-    <a class="btn" href="${ url('search:admin_collections') }" title="${ _('Dashboards') }" rel="tooltip" data-placement="bottom" data-bind="css: {'btn': true}">
+    <a class="btn" href="${ url('dashboard:admin_collections') }" title="${ _('Dashboards') }" rel="tooltip" data-placement="bottom" data-bind="css: {'btn': true}">
       <i class="fa fa-tags"></i>
       <i class="fa fa-tags"></i>
     </a>
     </a>
   </div>
   </div>
@@ -1478,7 +1478,7 @@ ${ dashboard.layout_skeleton() }
         <!-- /ko -->
         <!-- /ko -->
 
 
         <!-- ko if: $root.collection.engine() == 'solr' -->
         <!-- ko if: $root.collection.engine() == 'solr' -->
-        <form method="POST" action="${ url('search:download') }" style="display:inline">
+        <form method="POST" action="${ url('dashboard:download') }" style="display:inline">
           ${ csrf_token(request) | n,unicode }
           ${ csrf_token(request) | n,unicode }
           <input type="hidden" name="collection" data-bind="value: ko.mapping.toJSON($root.collection)"/>
           <input type="hidden" name="collection" data-bind="value: ko.mapping.toJSON($root.collection)"/>
           <input type="hidden" name="query" data-bind="value: ko.mapping.toJSON($root.query)"/>
           <input type="hidden" name="query" data-bind="value: ko.mapping.toJSON($root.query)"/>
@@ -2753,9 +2753,9 @@ ${ dashboard.layout_skeleton() }
 <span id="extra" data-bind="augmenthtml: $root.collection.template.extracode"></span>
 <span id="extra" data-bind="augmenthtml: $root.collection.template.extracode"></span>
 
 
 
 
-<link rel="stylesheet" href="${ static('search/css/search.css') }">
+<link rel="stylesheet" href="${ static('dashboard/css/search.css') }">
 %if is_mobile:
 %if is_mobile:
-<link rel="stylesheet" href="${ static('search/css/search_mobile.css') }">
+<link rel="stylesheet" href="${ static('dashboard/css/search_mobile.css') }">
 %endif
 %endif
 <link rel="stylesheet" href="${ static('desktop/ext/css/hue-filetypes.css') }">
 <link rel="stylesheet" href="${ static('desktop/ext/css/hue-filetypes.css') }">
 <link rel="stylesheet" href="${ static('desktop/ext/css/hue-charts.css') }">
 <link rel="stylesheet" href="${ static('desktop/ext/css/hue-charts.css') }">
@@ -2767,7 +2767,7 @@ ${ dashboard.layout_skeleton() }
 
 
 ${ dashboard.import_layout(True) }
 ${ dashboard.import_layout(True) }
 
 
-<script src="${ static('search/js/search.utils.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('dashboard/js/search.utils.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/jquery.textsqueezer.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/jquery.textsqueezer.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/ext/js/bootstrap-editable.min.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/ext/js/bootstrap-editable.min.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/ko.editable.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/js/ko.editable.js') }" type="text/javascript" charset="utf-8"></script>
@@ -2778,7 +2778,7 @@ ${ dashboard.import_layout(True) }
 <script src="${ static('desktop/ext/select2/select2.min.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/ext/select2/select2.min.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
 <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
 <script src="${ static('desktop/js/ko.selectize.js') }"></script>
 <script src="${ static('desktop/js/ko.selectize.js') }"></script>
-<script src="${ static('search/js/search.ko.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('dashboard/js/search.ko.js') }" type="text/javascript" charset="utf-8"></script>
 
 
 ${ dashboard.import_bindings() }
 ${ dashboard.import_bindings() }
 ${ dashboard.import_charts() }
 ${ dashboard.import_charts() }

+ 5 - 2
apps/search/src/search/templates/macros.mako → desktop/libs/dashboard/src/dashboard/templates/macros.mako

@@ -14,11 +14,14 @@
 ## See the License for the specific language governing permissions and
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 ## limitations under the License.
 <%!
 <%!
-from django.utils.translation import ugettext as _
-from itertools import izip
 import re
 import re
 import urllib
 import urllib
 
 
+from itertools import izip
+
+from django.utils.translation import ugettext as _
+
+
 # <http://github.com/mzsanford/twitter-text-java>
 # <http://github.com/mzsanford/twitter-text-java>
 
 
 AT_SIGNS = ur'[@\uff20]'
 AT_SIGNS = ur'[@\uff20]'

+ 2 - 3
apps/search/src/search/templates/no_collections.mako → desktop/libs/dashboard/src/dashboard/templates/no_collections.mako

@@ -21,10 +21,10 @@ from django.utils.translation import ugettext as _
 
 
 <%namespace name="macros" file="macros.mako" />
 <%namespace name="macros" file="macros.mako" />
 
 
-${ commonheader(_('Search'), "search", user, request, "120px") | n,unicode }
+${ commonheader(_('Dashboard'), "dashboard", user, request, "120px") | n,unicode }
 
 
-<style type="text/css">
 
 
+<style type="text/css">
   .waiting {
   .waiting {
     font-size: 196px;
     font-size: 196px;
     color: #DDD;
     color: #DDD;
@@ -35,7 +35,6 @@ ${ commonheader(_('Search'), "search", user, request, "120px") | n,unicode }
     color: #BBB;
     color: #BBB;
     line-height: 60px;
     line-height: 60px;
   }
   }
-
 </style>
 </style>
 
 
 <div class="container-fluid">
 <div class="container-fluid">

+ 1 - 1
apps/search/src/search/templates/search.mako → desktop/libs/dashboard/src/dashboard/templates/search.mako

@@ -24,7 +24,7 @@ from desktop import conf
 <%namespace name="common_search" file="common_search.mako" />
 <%namespace name="common_search" file="common_search.mako" />
 <%namespace name="notebookKoComponents" file="/common_notebook_ko_components.mako" />
 <%namespace name="notebookKoComponents" file="/common_notebook_ko_components.mako" />
 
 
-${ commonheader(_('Search'), "search", user, request, "80px") | n,unicode }
+${ commonheader(_('Dashboard'), "dashboard", user, request, "80px") | n,unicode }
 
 
 ${ notebookKoComponents.downloadSnippetResults() }
 ${ notebookKoComponents.downloadSnippetResults() }
 
 

+ 2 - 1
apps/search/src/search/templates/search_embeddable.mako → desktop/libs/dashboard/src/dashboard/templates/search_embeddable.mako

@@ -15,9 +15,10 @@
 ## limitations under the License.
 ## limitations under the License.
 
 
 <%!
 <%!
+from django.utils.translation import ugettext as _
+
 from desktop.views import commonheader, commonfooter, _ko
 from desktop.views import commonheader, commonfooter, _ko
 from desktop import conf
 from desktop import conf
-from django.utils.translation import ugettext as _
 %>
 %>
 
 
 <%namespace name="common_search" file="common_search.mako" />
 <%namespace name="common_search" file="common_search.mako" />

+ 1 - 1
apps/search/src/search/templates/search_m.mako → desktop/libs/dashboard/src/dashboard/templates/search_m.mako

@@ -24,7 +24,7 @@ from desktop import conf
 <%namespace name="common_search" file="common_search.mako" />
 <%namespace name="common_search" file="common_search.mako" />
 <%namespace name="notebookKoComponents" file="/common_notebook_ko_components.mako" />
 <%namespace name="notebookKoComponents" file="/common_notebook_ko_components.mako" />
 
 
-${ commonheader_m(_('Search'), "search", user, request, "80px") | n,unicode }
+${ commonheader_m(_('Dashboard'), "dashboard", user, request, "80px") | n,unicode }
 
 
 ${ notebookKoComponents.downloadSnippetResults() }
 ${ notebookKoComponents.downloadSnippetResults() }
 
 

Разлика између датотеке није приказан због своје велике величине
+ 398 - 0
desktop/libs/dashboard/src/dashboard/tests.py


+ 52 - 0
desktop/libs/dashboard/src/dashboard/urls.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.conf.urls import patterns, url
+
+urlpatterns = patterns('dashboard.views',
+  url(r'^$', 'index', name='index'),
+  url(r'^m$', 'index_m', name='index_m'),
+  url(r'^embeddable$', 'index_embeddable', name='index_embeddable'),
+  url(r'^save$', 'save', name='save'),
+  url(r'^new_search', 'new_search', name='new_search'),
+  url(r'^embeddable/new_search', 'new_search_embeddable', name='new_search_embeddable'),
+  url(r'^browse/(?P<name>.+)', 'browse', name='browse'),
+  url(r'^browse_m/(?P<name>.+)', 'browse_m', name='browse_m'),
+
+  # Admin
+  url(r'^admin/collections$', 'admin_collections', name='admin_collections'),
+  url(r'^admin/collection_delete$', 'admin_collection_delete', name='admin_collection_delete'),
+  url(r'^admin/collection_copy$', 'admin_collection_copy', name='admin_collection_copy'),
+)
+
+
+urlpatterns += patterns('dashboard.api',
+  url(r'^search$', 'search', name='search'),
+  url(r'^suggest/$', 'query_suggest', name='query_suggest'),
+  url(r'^index/fields/dynamic$', 'index_fields_dynamic', name='index_fields_dynamic'),
+  url(r'^index/fields/nested_documents', 'nested_documents', name='nested_documents'),
+  url(r'^template/new_facet$', 'new_facet', name='new_facet'),
+  url(r'^get_document$', 'get_document', name='get_document'),
+  url(r'^update_document$', 'update_document', name='update_document'),
+  url(r'^get_range_facet$', 'get_range_facet', name='get_range_facet'),
+  url(r'^download$', 'download', name='download'),
+  url(r'^get_timeline$', 'get_timeline', name='get_timeline'),
+  url(r'^get_collection$', 'get_collection', name='get_collection'),
+  url(r'^get_collections$', 'get_collections', name='get_collections'),
+  url(r'^get_stats$', 'get_stats', name='get_stats'),
+  url(r'^get_terms$', 'get_terms', name='get_terms'),
+)

+ 256 - 0
desktop/libs/dashboard/src/dashboard/views.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.utils.html import escape
+from django.utils.translation import ugettext as _
+
+from django.core.urlresolvers import reverse
+from desktop.conf import USE_NEW_EDITOR
+from desktop.lib.django_util import JsonResponse, render
+from desktop.lib.exceptions_renderable import PopupException
+from desktop.models import Document2, Document
+
+from search.conf import LATEST
+
+from dashboard.dashboard_api import get_engine
+from dashboard.decorators import allow_owner_only
+from dashboard.models import Collection2, get_engines
+from dashboard.search_controller import SearchController, can_edit_index
+
+
+LOG = logging.getLogger(__name__)
+
+
+DEFAULT_LAYOUT = [
+     {"size":2,"rows":[{"widgets":[]}],"drops":["temp"],"klass":"card card-home card-column span2"},
+     {"size":10,"rows":[{"widgets":[
+         {"size":12,"name":"Filter Bar","widgetType":"filter-widget", "id":"99923aef-b233-9420-96c6-15d48293532b",
+          "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]},
+                        {"widgets":[
+         {"size":12,"name":"Grid Results","widgetType":"resultset-widget", "id":"14023aef-b233-9420-96c6-15d48293532b",
+          "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
+        "drops":["temp"],"klass":"card card-home card-column span10"},
+]
+
+
+def index(request, is_mobile=False, is_embeddable=False):
+  hue_collections = SearchController(request.user).get_search_collections()
+  collection_id = request.GET.get('collection')
+
+  if not hue_collections or not collection_id:
+    return admin_collections(request, True, is_mobile)
+
+  try:
+    collection_doc = Document2.objects.get(id=collection_id)
+    if USE_NEW_EDITOR.get():
+      collection_doc.can_read_or_exception(request.user)
+    else:
+      collection_doc.doc.get().can_read_or_exception(request.user)
+    collection = Collection2(request.user, document=collection_doc)
+  except Exception, e:
+    raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
+
+  query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
+
+  if request.method == 'GET':
+    if 'q' in request.GET:
+      query['qs'][0]['q'] = request.GET.get('q')
+    if 'qd' in request.GET:
+      query['qd'] = request.GET.get('qd')
+
+  template = 'search.mako'
+  if is_mobile:
+    template = 'search_m.mako'
+  if is_embeddable:
+    template = 'search_embeddable.mako'
+
+  return render(template, request, {
+    'collection': collection,
+    'query': json.dumps(query),
+    'initial': json.dumps({
+        'collections': [],
+        'layout': DEFAULT_LAYOUT,
+        'is_latest': LATEST.get(),
+        'engines': get_engines(request.user)
+    }),
+    'is_owner': collection_doc.doc.get().can_write(request.user),
+    'can_edit_index': can_edit_index(request.user),
+    'mobile': is_mobile,
+  })
+
+def index_m(request):
+  return index(request, True)
+
+def index_embeddable(request):
+  return index(request, False, True)
+
+def new_search(request, is_embeddable=False):
+  engine = request.GET.get('engine', 'solr')
+  collections = get_engine(request.user, engine).datasets()
+  if not collections:
+    return no_collections(request)
+
+  collection = Collection2(user=request.user, name=collections[0], engine=engine)
+  query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
+
+  template = 'search.mako'
+  if is_embeddable:
+    template = 'search_embeddable.mako'
+
+  return render(template, request, {
+    'collection': collection,
+    'query': query,
+    'initial': json.dumps({
+        'collections': collections,
+        'layout': DEFAULT_LAYOUT,
+        'is_latest': LATEST.get(),
+        'engines': get_engines(request.user)
+     }),
+    'is_owner': True,
+    'can_edit_index': can_edit_index(request.user)
+  })
+
+def new_search_embeddable(request):
+  return new_search(request, True)
+
+def browse(request, name, is_mobile=False):
+  collections = SearchController(request.user).get_all_indexes()
+  if not collections:
+    return no_collections(request)
+
+  collection = Collection2(user=request.user, name=name)
+  query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
+
+  template = 'search.mako'
+  if is_mobile:
+    template = 'search_m.mako'
+
+  return render(template, request, {
+    'collection': collection,
+    'query': query,
+    'initial': json.dumps({
+      'autoLoad': True,
+      'collections': collections,
+      'layout': [
+          {"size":12,"rows":[{"widgets":[
+              {"size":12,"name":"Grid Results","id":"52f07188-f30f-1296-2450-f77e02e1a5c0","widgetType":"resultset-widget",
+               "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
+          "drops":["temp"],"klass":"card card-home card-column span10"}
+      ],
+      'is_latest': LATEST.get(),
+      'engines': get_engines(request.user)
+    }),
+    'is_owner': True,
+    'can_edit_index': can_edit_index(request.user),
+    'mobile': is_mobile
+  })
+
+
+def browse_m(request, name):
+  return browse(request, name, True)
+
+
+@allow_owner_only
+def save(request):
+  response = {'status': -1}
+
+  collection = json.loads(request.POST.get('collection', '{}'))
+  layout = json.loads(request.POST.get('layout', '{}'))
+
+  collection['template']['extracode'] = escape(collection['template']['extracode'])
+
+  if collection:
+    if collection['id']:
+      dashboard_doc = Document2.objects.get(id=collection['id'])
+    else:
+      dashboard_doc = Document2.objects.create(name=collection['name'], uuid=collection['uuid'], type='search-dashboard', owner=request.user, description=collection['label'])
+      Document.objects.link(dashboard_doc, owner=request.user, name=collection['name'], description=collection['label'], extra='search-dashboard')
+
+    dashboard_doc.update_data({
+        'collection': collection,
+        'layout': layout
+    })
+    dashboard_doc1 = dashboard_doc.doc.get()
+    dashboard_doc.name = dashboard_doc1.name = collection['label']
+    dashboard_doc.description = dashboard_doc1.description = collection['description']
+    dashboard_doc.save()
+    dashboard_doc1.save()
+
+    response['status'] = 0
+    response['id'] = dashboard_doc.id
+    response['message'] = _('Page saved !')
+  else:
+    response['message'] = _('There is no collection to search.')
+
+  return JsonResponse(response)
+
+
+def no_collections(request):
+  return render('no_collections.mako', request, {})
+
+
+def admin_collections(request, is_redirect=False, is_mobile=False):
+  existing_hue_collections = SearchController(request.user).get_search_collections()
+
+  if request.GET.get('format') == 'json':
+    collections = []
+    for collection in existing_hue_collections:
+      massaged_collection = collection.to_dict()
+      if request.GET.get('is_mobile'):
+        massaged_collection['absoluteUrl'] = reverse('search:index_m') + '?collection=%s' % collection.id
+      massaged_collection['isOwner'] = collection.doc.get().can_write(request.user)
+      collections.append(massaged_collection)
+    return JsonResponse(collections, safe=False)
+
+  template = 'admin_collections.mako'
+  if is_mobile:
+    template = 'admin_collections_m.mako'
+
+  return render(template, request, {
+    'is_embeddable': request.GET.get('is_embeddable', False),
+    'existing_hue_collections': existing_hue_collections,
+    'is_redirect': is_redirect
+  })
+
+
+def admin_collection_delete(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  collections = json.loads(request.POST.get('collections'))
+  searcher = SearchController(request.user)
+  response = {
+    'result': searcher.delete_collections([collection['id'] for collection in collections])
+  }
+
+  return JsonResponse(response)
+
+
+def admin_collection_copy(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  collections = json.loads(request.POST.get('collections'))
+  searcher = SearchController(request.user)
+  response = {
+    'result': searcher.copy_collections([collection['id'] for collection in collections])
+  }
+
+  return JsonResponse(response)

+ 1 - 1
desktop/libs/indexer/src/indexer/controller.py

@@ -26,12 +26,12 @@ from django.utils.translation import ugettext as _
 import tablib
 import tablib
 
 
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
+from dashboard.models import Collection2
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
 from libsolr.conf import SOLR_ZK_PATH
 from libsolr.conf import SOLR_ZK_PATH
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.models import ZookeeperClient
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
 from search.conf import SOLR_URL, SECURITY_ENABLED
-from search.models import Collection2
 
 
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.conf import CORE_INSTANCE_DIR
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file
 from indexer.utils import copy_configs, field_values_from_log, field_values_from_separated_file

+ 1 - 1
desktop/libs/indexer/src/indexer/settings.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-DJANGO_APPS = [ "indexer" ]
+DJANGO_APPS = ["indexer"]
 NICE_NAME = "Data Importer"
 NICE_NAME = "Data Importer"
 REQUIRES_HADOOP = False
 REQUIRES_HADOOP = False
 MENU_INDEX = 43
 MENU_INDEX = 43

+ 2 - 2
desktop/libs/libsolr/src/libsolr/api.py

@@ -31,9 +31,9 @@ from desktop.lib.conf import BoundConfig
 from desktop.lib.i18n import force_unicode
 from desktop.lib.i18n import force_unicode
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest import resource
 from desktop.lib.rest import resource
+from dashboard.facet_builder import _compute_range_facet
 
 
 from search.conf import EMPTY_QUERY, SECURITY_ENABLED
 from search.conf import EMPTY_QUERY, SECURITY_ENABLED
-from search.facet_builder import _compute_range_facet
 
 
 from libsolr.conf import SSL_CERT_CA_VERIFY
 from libsolr.conf import SSL_CERT_CA_VERIFY
 
 
@@ -215,7 +215,7 @@ class SolrApi(object):
 
 
     params += self._get_fq(collection, query)
     params += self._get_fq(collection, query)
 
 
-    from search.models import Collection2
+    from dashboard.models import Collection2
     fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection))))
     fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection))))
 
 
     nested_fields = self._get_nested_fields(collection)
     nested_fields = self._get_nested_fields(collection)

Неке датотеке нису приказане због велике количине промена