Просмотр исходного кода

HUE-3228 [dashboard] Move out of Dashboard the Solr specific code

Romain Rigaux 8 лет назад
Родитель
Сommit
8410459f7a

+ 1 - 0
apps/search/src/search/conf.py

@@ -36,6 +36,7 @@ SECURITY_ENABLED = Config(
   default=False,
   type=coerce_bool)
 
+# Unused: deprecated by dashboard
 LATEST = Config(
   key="latest",
   help=_("Use latest Solr 5.2+ features."),

+ 58 - 0
apps/search/src/search/controller.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from libsolr.api import SolrApi
+
+from search.conf import SOLR_URL
+
+
+LOG = logging.getLogger(__name__)
+
+
+class SearchController(object):
+
+  def __init__(self, user):
+    self.user = user
+  
+  def is_collection(self, collection_name):
+    return collection_name in self.get_solr_collections()
+
+  def is_core(self, core_name):
+    solr_cores = SolrApi(SOLR_URL.get(), self.user).cores()
+    return core_name in solr_cores
+
+  def get_solr_collections(self):
+    return SolrApi(SOLR_URL.get(), self.user).collections()
+
+  def get_all_indexes(self, show_all=False):
+    indexes = []
+    try:
+      indexes = self.get_solr_collections().keys()
+    except:
+      LOG.exception('failed to get indexes')
+
+    try:
+      indexes += SolrApi(SOLR_URL.get(), self.user).aliases().keys()
+    except:
+      LOG.exception('failed to get index aliases')
+
+    if show_all or not indexes:
+      return indexes + SolrApi(SOLR_URL.get(), self.user).cores().keys()
+    else:
+      return indexes

+ 6 - 2
apps/search/src/search/dashboard_api.py

@@ -15,13 +15,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 
 from dashboard.dashboard_api import DashboardApi
 from dashboard.models import augment_solr_response
-from dashboard.search_controller import SearchController
 from libsolr.api import SolrApi
 
 from search.conf import SOLR_URL
+from search.controller import SearchController
+
+
+LOG = logging.getLogger(__name__)
 
 
 class SearchApi(DashboardApi):
@@ -50,4 +54,4 @@ class SearchApi(DashboardApi):
     return self.api.stats(collection, field, query, facet)
 
   def get(self, collection, doc_id):
-    return self.api.get(collection, doc_id)
+    return self.api.get(collection, doc_id)

+ 10 - 3
desktop/conf.dist/hue.ini

@@ -781,6 +781,16 @@
   # interpreters_shown_on_wheel=
 
 
+###########################################################################
+# Settings to configure your Analytics Dashboards
+###########################################################################
+
+[dashboard]
+
+  # Use latest Solr 5+ functionalities like Analytics Facets and Nested Documents (warning: still in beta).
+  ## support_latest_solr=false
+
+
 ###########################################################################
 # Settings to configure your Hadoop cluster.
 ###########################################################################
@@ -1169,9 +1179,6 @@
   ## Query sent when no term is entered
   ## empty_query=*:*
 
-  # Use latest Solr 5.2+ features.
-  ## latest=false
-
 
 ###########################################################################
 # Settings to configure Solr API lib

+ 10 - 3
desktop/conf/pseudo-distributed.ini.tmpl

@@ -783,6 +783,16 @@
   # interpreters_shown_on_wheel=
 
 
+###########################################################################
+# Settings to configure your Analytics Dashboards
+###########################################################################
+
+[dashboard]
+
+  # Use latest Solr 5+ functionalities like Analytics Facets and Nested Documents (warning: still in beta).
+  ## support_latest_solr=false
+
+
 ###########################################################################
 # Settings to configure your Hadoop cluster.
 ###########################################################################
@@ -1171,9 +1181,6 @@
   ## Query sent when no term is entered
   ## empty_query=*:*
 
-  # Use latest Solr 5.2+ features.
-  ## latest=false
-
 
 ###########################################################################
 # Settings to configure Solr API lib

+ 2 - 2
desktop/core/src/desktop/templates/common_header.mako

@@ -417,8 +417,8 @@ ${ hueIcons.symbols() }
        </li>
        % endif
        % if 'search' in apps:
-         <% from dashboard.search_controller import SearchController %>
-         <% controller = SearchController(user) %>
+         <% from dashboard.controller import DashboardController %>
+         <% controller = DashboardController(user) %>
          <% collections = controller.get_shared_search_collections() %>
          % if not collections:
            <li>

+ 1 - 1
desktop/libs/dashboard/src/dashboard/api.py

@@ -34,7 +34,7 @@ from dashboard.data_export import download as export_download
 from dashboard.decorators import allow_viewer_only
 from dashboard.facet_builder import _guess_gap, _zoom_range_facet, _new_range_facet
 from dashboard.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception
-from dashboard.search_controller import can_edit_index
+from dashboard.controller import can_edit_index
 
 
 LOG = logging.getLogger(__name__)

+ 45 - 6
desktop/libs/dashboard/src/dashboard/conf.py

@@ -15,23 +15,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _t
 
 from desktop.conf import is_hue4
-from desktop.lib.conf import Config, coerce_bool
+from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_json_dict, coerce_bool
 
 
 IS_ENABLED = Config(
   key="is_enabled",
-  help=_("Activate the app in the menu."),
+  help=_t("Activate the app in the menu."),
   dynamic_default=is_hue4,
   private=True,
-  type=coerce_bool)
+  type=coerce_bool
+)
+
+# ANALYTICS_ENABLED 
+SUPPORT_LATEST_SOLR = Config(
+  key="support_latest_solr",
+  help=_t("Use latest Solr 5+ functionalities like Analytics Facets and Nested Documents (warning: still in beta)."),
+  default=True,
+  type=coerce_bool
+)
+
+NESTED_ENABLED = Config(
+  key="support_latest_solr",
+  help=_t("Use latest Solr 5+ functionalities like Analytics Facets and Nested Documents (warning: still in beta)."),
+  default=True,
+  type=coerce_bool
+)
 
 # TODO [[interfaces]] instead
 IS_SQL_ENABLED = Config(
   key="is_sql_enabled",
-  help=_("Offer to use SQL engines to compute the dashboards."),
+  help=_t("Offer to use SQL engines to compute the dashboards."),
   dynamic_default=is_hue4,
   private=True,
-  type=coerce_bool)
+  type=coerce_bool
+)
+
+INTERPRETERS = UnspecifiedConfigSection(
+  "connectors",
+  help="One entry for each type of snippet.",
+  each=ConfigSection(
+    help=_t("Define the name and how to connect and execute the language."),
+    members=dict(
+      ANALYTICS_SUPPORT=Config(
+          "name",
+          help=_t("The name of the snippet."),
+          default=False,
+          type=coerce_bool,
+      ),
+      NESTED_SUPPORT=Config(
+          "name",
+          help=_t("The name of the snippet."),
+          default=False,
+          type=coerce_bool,
+      ),
+    )
+  )
+)

+ 6 - 37
desktop/libs/dashboard/src/dashboard/search_controller.py → desktop/libs/dashboard/src/dashboard/controller.py

@@ -22,19 +22,15 @@ from django.db.models import Q
 
 from desktop.conf import USE_NEW_EDITOR
 from desktop.models import Document2, Document, SAMPLE_USER_OWNERS
-from libsolr.api import SolrApi
 
-from search.conf import SOLR_URL
 from dashboard.models import Collection2
 
 
 LOG = logging.getLogger(__name__)
 
 
-class SearchController(object):
-  """
-  Glue the models to the views.
-  """
+class DashboardController(object):
+
   def __init__(self, user):
     self.user = user
 
@@ -68,13 +64,13 @@ class SearchController(object):
 
   def get_icon(self, name):
     if name == 'Twitter':
-      return 'search/art/icon_twitter_48.png'
+      return 'dashboard/art/icon_twitter_48.png'
     elif name == 'Yelp Reviews':
-      return 'search/art/icon_yelp_48.png'
+      return 'dashboard/art/icon_yelp_48.png'
     elif name == 'Web Logs':
-      return 'search/art/icon_logs_48.png'
+      return 'dashboard/art/icon_logs_48.png'
     else:
-      return 'search/art/icon_search_48.png'
+      return 'dashboard/art/icon_search_48.png'
 
   def delete_collections(self, collection_ids):
     result = {'status': -1, 'message': ''}
@@ -116,33 +112,6 @@ class SearchController(object):
 
     return result
 
-  def is_collection(self, collection_name):
-    return collection_name in self.get_solr_collections()
-
-  def is_core(self, core_name):
-    solr_cores = SolrApi(SOLR_URL.get(), self.user).cores()
-    return core_name in solr_cores
-
-  def get_solr_collections(self):
-    return SolrApi(SOLR_URL.get(), self.user).collections()
-
-  def get_all_indexes(self, show_all=False):
-    indexes = []
-    try:
-      indexes = self.get_solr_collections().keys()
-    except:
-      LOG.exception('failed to get indexes')
-
-    try:
-      indexes += SolrApi(SOLR_URL.get(), self.user).aliases().keys()
-    except:
-      LOG.exception('failed to get index aliases')
-
-    if show_all or not indexes:
-      return indexes + SolrApi(SOLR_URL.get(), self.user).cores().keys()
-    else:
-      return indexes
-
 
 def can_edit_index(user):
   return user.is_superuser

+ 4 - 6
desktop/libs/dashboard/src/dashboard/models.py

@@ -32,9 +32,7 @@ from desktop.models import get_data_link
 from libsolr.api import SolrApi
 from notebook.conf import get_ordered_interpreters
 
-from search.conf import LATEST
-
-from dashboard.conf import IS_SQL_ENABLED
+from dashboard.conf import IS_SQL_ENABLED, SUPPORT_LATEST_SOLR
 from dashboard.dashboard_api import get_engine
 
 
@@ -104,7 +102,7 @@ class Collection2(object):
     for field in props['collection']['template']['fieldsAttributes']:
       if 'type' not in field:
         field['type'] = 'string'
-    if 'nested' not in props['collection'] and LATEST.get():
+    if 'nested' not in props['collection'] and SUPPORT_LATEST_SOLR.get():
       props['collection']['nested'] = {
         'enabled': False,
         'schema': []
@@ -763,8 +761,8 @@ def get_engines(user):
           'name': _('table (%s)') % interpreter['name'],
           'type': interpreter['type'],
           'async': interpreter['interface'] == 'hiveserver2'
-        }
-        for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms')
+      }
+      for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms')
     ]
 
   return engines

+ 1 - 0
desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js

@@ -461,6 +461,7 @@ var Collection = function (vm, collection) {
   self.queryResult = ko.observable(new QueryResult(self, {
     type: self.engine(),
   }));
+  //self.hasAnalytics = ko.observable(typeof collection.hasAnalytics != "undefined" && collection.hasAnalytics != null ? collection.hasAnalytics : false);
   self.nested = ko.mapping.fromJS(collection.nested);
   self.nestedNames = ko.computed(function() {
     function flatten(values) {

+ 19 - 19
desktop/libs/dashboard/src/dashboard/tests.py

@@ -30,7 +30,7 @@ from desktop.models import Document2
 
 from dashboard.facet_builder import _round_number_range
 from dashboard.models import Collection2
-from dashboard.search_controller import SearchController
+from dashboard.controller import DashboardController
 
 
 QUERY = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
@@ -157,8 +157,8 @@ class TestWithMockedSolr(TestSearchBase):
     assert_equal(-1, data['status'])
 
     # There are no collections with user_not_me
-    search_controller = SearchController(self.user_not_me)
-    hue_collections = search_controller.get_search_collections()
+    controller = DashboardController(self.user_not_me)
+    hue_collections = controller.get_search_collections()
     assert_true(len(hue_collections) == 0)
 
     # Share read perm by users
@@ -200,28 +200,28 @@ class TestWithMockedSolr(TestSearchBase):
     assert_true('docs' in data['response'], data)
 
     # For self.user_not_me
-    search_controller = SearchController(self.user_not_me)
-    hue_collections = search_controller.get_search_collections()
+    controller = DashboardController(self.user_not_me)
+    hue_collections = controller.get_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard')
 
-    hue_collections = search_controller.get_owner_search_collections()
+    hue_collections = controller.get_owner_search_collections()
     assert_equal(len(hue_collections), 0)
 
-    hue_collections = search_controller.get_shared_search_collections()
+    hue_collections = controller.get_shared_search_collections()
     assert_equal(len(hue_collections), 0)
 
     # For self.user
-    search_controller = SearchController(self.user)
-    hue_collections = search_controller.get_search_collections()
+    controller = DashboardController(self.user)
+    hue_collections = controller.get_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard')
 
-    hue_collections = search_controller.get_owner_search_collections()
+    hue_collections = controller.get_owner_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard')
 
-    hue_collections = search_controller.get_shared_search_collections()
+    hue_collections = controller.get_shared_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard')
 
@@ -261,28 +261,28 @@ class TestWithMockedSolr(TestSearchBase):
     assert_true(doc1.can_write(self.user_not_me))
 
     # For self.user_not_me
-    search_controller = SearchController(self.user_not_me)
-    hue_collections = search_controller.get_search_collections()
+    controller = DashboardController(self.user_not_me)
+    hue_collections = controller.get_search_collections()
     assert_equal(len(hue_collections), 2)
 
-    hue_collections = search_controller.get_owner_search_collections()
+    hue_collections = controller.get_owner_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard1')
 
-    hue_collections = search_controller.get_shared_search_collections()
+    hue_collections = controller.get_shared_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard1')
 
     # For self.user
-    search_controller = SearchController(self.user)
-    hue_collections = search_controller.get_search_collections()
+    controller = DashboardController(self.user)
+    hue_collections = controller.get_search_collections()
     assert_equal(len(hue_collections), 2)
 
-    hue_collections = search_controller.get_owner_search_collections()
+    hue_collections = controller.get_owner_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard')
 
-    hue_collections = search_controller.get_shared_search_collections()
+    hue_collections = controller.get_shared_search_collections()
     assert_equal(len(hue_collections), 1)
     assert_equal(hue_collections[0].name, 'test_dashboard')
 

+ 6 - 6
desktop/libs/dashboard/src/dashboard/views.py

@@ -32,7 +32,7 @@ from search.conf import LATEST
 from dashboard.dashboard_api import get_engine
 from dashboard.decorators import allow_owner_only
 from dashboard.models import Collection2, get_engines
-from dashboard.search_controller import SearchController, can_edit_index
+from dashboard.controller import DashboardController, can_edit_index
 
 
 LOG = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ DEFAULT_LAYOUT = [
 
 
 def index(request, is_mobile=False, is_embeddable=False):
-  hue_collections = SearchController(request.user).get_search_collections()
+  hue_collections = DashboardController(request.user).get_search_collections()
   collection_id = request.GET.get('collection')
 
   if not hue_collections or not collection_id:
@@ -131,7 +131,7 @@ def new_search_embeddable(request):
   return new_search(request, True)
 
 def browse(request, name, is_mobile=False):
-  collections = SearchController(request.user).get_all_indexes()
+  collections = DashboardController(request.user).get_all_indexes() # TODO convert
   if not collections:
     return no_collections(request)
 
@@ -207,7 +207,7 @@ def no_collections(request):
 
 
 def admin_collections(request, is_redirect=False, is_mobile=False):
-  existing_hue_collections = SearchController(request.user).get_search_collections()
+  existing_hue_collections = DashboardController(request.user).get_search_collections()
 
   if request.GET.get('format') == 'json':
     collections = []
@@ -235,7 +235,7 @@ def admin_collection_delete(request):
     raise PopupException(_('POST request required.'))
 
   collections = json.loads(request.POST.get('collections'))
-  searcher = SearchController(request.user)
+  searcher = DashboardController(request.user)
   response = {
     'result': searcher.delete_collections([collection['id'] for collection in collections])
   }
@@ -248,7 +248,7 @@ def admin_collection_copy(request):
     raise PopupException(_('POST request required.'))
 
   collections = json.loads(request.POST.get('collections'))
-  searcher = SearchController(request.user)
+  searcher = DashboardController(request.user)
   response = {
     'result': searcher.copy_collections([collection['id'] for collection in collections])
   }