浏览代码

HUE-5304 [indexer] Adding logic to turn on/off the new index page

Romain Rigaux 8 年之前
父节点
当前提交
2a75a64

+ 1 - 1
desktop/core/src/desktop/static/desktop/js/assist/assistDbEntry.js

@@ -473,7 +473,7 @@ var AssistDbEntry = (function () {
     var self = this;
     huePubSub.publish('open.link', '/hue/dashboard/browse/' + self.getDatabaseName() + '.' + self.getTableName() + '?engine=' + self.assistDbSource.sourceType);
   };
-  
+
   AssistDbEntry.prototype.openInMetastore = function () {
     var self = this;
     var url;

+ 5 - 2
desktop/core/src/desktop/templates/assist.mako

@@ -103,8 +103,11 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, get_ord
       </a>
     </li>
     % if HAS_SQL_ENABLED.get():
-    <li><a href="javascript: void(0);" data-bind="click: explore">
-      <!-- ko template: { name: 'app-icon-template', data: { icon: 'dashboard' } } --><!-- /ko --> ${ _('Open in Dashboard') }</a></li>
+    <li>
+      <a href="javascript: void(0);" data-bind="click: explore">
+        <!-- ko template: { name: 'app-icon-template', data: { icon: 'dashboard' } } --><!-- /ko --> ${ _('Open in Dashboard') }
+      </a>
+    </li>
     % endif
     <!-- /ko -->
     %if ENABLE_QUERY_BUILDER.get():

+ 4 - 2
desktop/core/src/desktop/templates/common_header.mako

@@ -481,12 +481,14 @@ ${ hueIcons.symbols() }
              <ul role="menu" class="dropdown-menu">
                <li><a href="${ url('search:new_search') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-plus" style="vertical-align: middle"></i> ${ _('Dashboard') }</a></li>
                <li><a href="${ url('search:admin_collections') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-tags" style="vertical-align: middle"></i>${ _('Dashboards') }</a></li>
-               <li><a href="${ url('indexer:collections') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-database" style="vertical-align: middle"></i> ${ _('Indexes') }</a></li>
                <%!
                  from indexer.conf import ENABLE_NEW_INDEXER
                %>
                % if ENABLE_NEW_INDEXER.get():
-                 <li><a href="${ url('indexer:indexer') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-plus" style="vertical-align: middle"></i> ${ _('Index') }</a></li>
+                 <li><a href="${ url('indexer:indexes') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-database" style="vertical-align: middle"></i> ${ _('Indexes') }</a></li>
+                 <li><a href="${ url('indexer:indexes') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-plus" style="vertical-align: middle"></i> ${ _('Index') }</a></li>
+               % else:
+                 <li><a href="${ url('indexer:collections') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-database" style="vertical-align: middle"></i> ${ _('Indexes') }</a></li>
                % endif
                <li class="divider"></li>
                % for collection in collections:

+ 13 - 0
desktop/libs/indexer/src/data/solrconfigs/solrcloud/conf/schema.xml

@@ -224,6 +224,9 @@
 
     <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
 
+    <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
+        geo="true" distErrPct="0.025" maxDistErr="0.000009" units="degrees" />
+
     <fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
 
     <!-- Arabic -->
@@ -557,6 +560,16 @@
       </analyzer>
     </fieldType>
 
+    <!-- Thai -->
+    <fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">
+      <analyzer>
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.ThaiWordFilterFactory"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_th.txt" />
+      </analyzer>
+    </fieldType>
+
     <!-- Turkish -->
     <fieldType name="text_tr" class="solr.TextField" positionIncrementGap="100">
       <analyzer>

+ 3 - 28
desktop/libs/indexer/src/indexer/solr_api.py

@@ -38,7 +38,7 @@ def api_error_handler(func):
     try:
       return func(*args, **kwargs)
     except Exception, e:
-      LOG.exception('Error running %s' % func)
+      LOG.exception('Error running %s' % func.__name__)
       response['status'] = -1
       response['message'] = smart_unicode(e)
     finally:
@@ -65,9 +65,9 @@ def list_indexes(request):
 @api_error_handler
 def list_index(request):
   response = {'status': -1}
-  
+
   name = request.POST.get('name')
-  
+
   client = SolrClient(user=request.user)
 
   response['schema'] = client.list_schema(name)
@@ -187,28 +187,3 @@ def list_configs(request):
   response['status'] = 0
 
   return JsonResponse(response)
-
-
-@require_POST
-@api_error_handler
-def design_schema(request, index):
-  result = {'status': -1, 'message': ''}
-
-  searcher = SolrClient(request.user)
-  unique_key, fields = searcher.get_index_schema(index)
-
-  result['status'] = 0
-  formatted_fields = []
-  for field in fields:
-    formatted_fields.append({
-      'name': field,
-      'type': fields[field]['type'],
-      'required': fields[field].get('required', None),
-      'indexed': fields[field].get('indexed', None),
-      'stored': fields[field].get('stored', None),
-      'multivalued': fields[field].get('multivalued', None),
-    })
-  result['fields'] = formatted_fields
-  result['unique_key'] = unique_key
-
-  return JsonResponse(result)

+ 12 - 36
desktop/libs/indexer/src/indexer/solr_client.py

@@ -98,10 +98,6 @@ class SolrClient(object):
 
 
   def create_index(self, name, fields, config_name=None, unique_key_field=None, df=None):
-    """
-    Create solr collection or core and instance dir.
-    Create schema.xml file so that we can set UniqueKey field.
-    """
     if self.is_solr_cloud_mode():
       if config_name is None:
         self._create_cloud_config(name, fields, unique_key_field, df)
@@ -120,15 +116,8 @@ class SolrClient(object):
 
   def index(self, name, data, content_type='csv', version=None, **kwargs):
     """
-    separator = ','
-    fieldnames = 'a,b,c' # header=true
-    skip 'a,b'
-    encapsulator="
-    escape=\
-    map
-    split
-    overwrite=true
-    rowid=id
+    e.g. Parameters: separator = ',', fieldnames = 'a,b,c', header=true, skip 'a,b', encapsulator="
+      escape=\, map, split, overwrite=true, rowid=id
     """
     return self.api.update(name, data, content_type=content_type, version=version, **kwargs)
 
@@ -168,15 +157,10 @@ class SolrClient(object):
     except Exception, e:
       raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
     finally:
-      # Delete instance directory if we couldn't create the core.
       shutil.rmtree(instancedir)
 
 
-  def delete_index(self, name):
-    """
-    Delete solr collection/core and instance dir
-    """
-    # TODO: Implement deletion of local Solr cores
+  def delete_index(self, name, keep_config=True):
     if not self.is_solr_cloud_mode():
       raise PopupException(_('Cannot remove non-Solr cloud cores.'))
 
@@ -184,15 +168,15 @@ class SolrClient(object):
 
     if result['status'] == 0:
       # Delete instance directory.
-#       try:
-#         root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
-#         with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
-#           zc.delete_path(root_node)
-#       except Exception, e:
-#         # Re-create collection so that we don't have an orphan config
-#         self.api.add_collection(name)
-#         raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
-      pass
+      if not keep_config:
+        try:
+          root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
+          with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
+            zc.delete_path(root_node)
+        except Exception, e:
+          # Re-create collection so that we don't have an orphan config
+          self.api.add_collection(name)
+          raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
     else:
       if not 'Cannot unload non-existent core' in json.dumps(result):
         raise PopupException(_('Could not remove collection: %(message)s') % result)
@@ -212,11 +196,3 @@ class SolrClient(object):
 
   def delete_alias(self, name):
     return self.api.delete_alias(name)
-
-
-  def _format_flags(self, fields):
-    for name, properties in fields.items():
-      for (code, value) in FLAGS:
-        if code in properties['flags']:
-          properties[value] = True  # Add a new key-value boolean for the decoded flag
-    return fields

+ 2 - 2
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -913,8 +913,8 @@ ${ assist.assistPanel() }
   <!-- ko if: operations().length > 0 -->
   <a class="pointer" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation to')} <span data-bind="text: name"></span></a>
   <!-- /ko -->
-  
-  <span data-bind="template: { name:'field-column-example' }"></span>
+
+  <span data-bind="template: { name: 'field-column-example' }"></span>
 </script>
 
 

+ 17 - 14
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -30,7 +30,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
   <ul class="nav nav-pills hue-breadcrumbs-bar" id="breadcrumbs">
     <li>
       <a href="javascript:void(0);" data-bind="click: function() { section('list-indexes'); }">${ _('Indexes') }
-        <!-- ko if: index --> 
+        <!-- ko if: index -->
         <span class="divider">&gt;</span>
         <!-- /ko -->
       </a>
@@ -46,7 +46,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 <div class="container-fluid">
   <div class="card card-small">
     <h1 class="card-heading simple">${ _('Index Browser') }</h1>
-    
+
     <!-- ko template: { name: 'indexes-breadcrumbs' }--><!-- /ko -->
 
     <%actionbar:render>
@@ -158,7 +158,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 <script type="text/html" id="indexes-index-overview">
   <div>
     Overview
-    
+
     <!-- ko template: 'indexes-index-properties' --><!-- /ko -->
 
     <!-- ko template: { name: 'indexes-index-fields-fields', data: fieldsPreview }--><!-- /ko -->
@@ -177,12 +177,9 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 <script type="text/html" id="indexes-index-properties">
   <h4>${ _('Properties') }</h4>
   <div class="row-fluid">
-    <div title="${ _('Type') }">
-      <i class="fa fa-fw fa-eye muted"></i> ${ _('Collection') }
-    </div>
     <div title="${ _('Unique Key') }">
       <i class="fa fa-fw fa-key muted"></i> <span data-bind="text: uniqueKey"></span>
-    </div>    
+    </div>
   </div>
 </script>
 
@@ -194,7 +191,9 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     <table id="indexTable" class="table datatables">
       <thead>
         <tr>
+          <th style="width: 1%">&nbsp;</th>
           <th width="1%"></th>
+          <th></th>
           <th>${ _('Name') }</th>
           <th>${ _('Type') }</th>
           <th>${ _('Required') }</th>
@@ -205,6 +204,10 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
       </thead>
       <tbody data-bind="foreach: $data">
         <tr>
+          <td data-bind="text: $index() + 1"></td>
+          <td>
+            <i class="fa fa-info muted pointer analysis"></i>
+          </td>
           <td>
             <div></div>
           </td>
@@ -224,13 +227,13 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 <script type="text/html" id="indexes-index-fields">
   <div>
     <!-- ko template: { name: 'indexes-index-fields-fields', data: fields }--><!-- /ko -->
-    
+
     Copy Fields
     <span data-bind="text: ko.mapping.toJSON(copyFields)"></span>
-  
-  
+
+
     Dynamic Fields
-    <span data-bind="text: ko.mapping.toJSON(dynamicFields)"></span> 
+    <span data-bind="text: ko.mapping.toJSON(dynamicFields)"></span>
   </div>
 </script>
 
@@ -238,7 +241,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 <script type="text/html" id="indexes-index-sample">
   <div>
     Sample
-    
+
     <a data-bind="click: $root.index().getSample">Load</a>
 
     <table class="table table-condensed table-nowrap sample-table">
@@ -346,12 +349,12 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     });
     self.dynamicFields = ko.mapping.fromJS(data.schema.dynamicFields);
     self.copyFields = ko.mapping.fromJS(data.schema.copyFields);
-    
+
     self.sample = ko.observableArray();
     self.samplePreview = ko.pureComputed(function() {
       return self.sample().splice(0, 5)
     });
-    
+
     self.getSample = function() {
       $.post("${ url('indexer:sample_index') }", {
         name: self.name(),

+ 28 - 24
desktop/libs/indexer/src/indexer/urls.py

@@ -17,32 +17,27 @@
 
 from django.conf.urls import patterns, url
 
+from indexer.conf import ENABLE_NEW_INDEXER
+
+
 urlpatterns = patterns('indexer.views',
-  url(r'^$', 'collections', name='collections'),
   url(r'^install_examples$', 'install_examples', name='install_examples'),
-  
-  # V2
-  url(r'^indexes/$', 'indexes', name='indexes'),
 
-  # V3
-  url(r'^indexer/$', 'indexer', name='indexer'),
   url(r'^importer/$', 'importer', name='importer'),
   url(r'^importer/prefill/(?P<source_type>[^/]+)/(?P<target_type>[^/]+)/(?P<target_path>[^/]+)?$', 'importer_prefill', name='importer_prefill'),
 )
 
-# Current v1
-urlpatterns += patterns('indexer.api',
-  url(r'^api/fields/parse/$', 'parse_fields', name='api_parse_fields'),
-  url(r'^api/autocomplete/$', 'autocomplete', name='api_autocomplete'),
-  url(r'^api/collections/$', 'collections', name='api_collections'),
-  url(r'^api/collections/create/$', 'collections_create', name='api_collections_create'),
-  url(r'^api/collections/import/$', 'collections_import', name='api_collections_import'),
-  url(r'^api/collections/remove/$', 'collections_remove', name='api_collections_remove'),
-  url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
-  url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
-  url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
-)
-
+if ENABLE_NEW_INDEXER.get():
+  urlpatterns += patterns('indexer.views',
+    url(r'^$', 'indexes', name='indexes'),
+    url(r'^indexes/$', 'indexes', name='indexes'),
+    url(r'^collections$', 'collections', name='collections'), # Old page
+  )
+else:
+  urlpatterns += patterns('indexer.views',
+    url(r'^$', 'collections', name='collections'),
+    url(r'^indexes/$', 'indexes', name='indexes'),
+  )
 
 urlpatterns += patterns('indexer.solr_api',
   # V2
@@ -56,11 +51,6 @@ urlpatterns += patterns('indexer.solr_api',
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
 )
 
-urlpatterns += patterns('indexer.solr_api',
-  url(r'^api/collections/delete/$', 'delete_collections', name='delete_collections'),
-)
-
-
 urlpatterns += patterns('indexer.api3',
   # Importer
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
@@ -68,3 +58,17 @@ urlpatterns += patterns('indexer.api3',
 
   url(r'^api/importer/submit', 'importer_submit', name='importer_submit')
 )
+
+
+# Deprecated
+urlpatterns += patterns('indexer.api',
+  url(r'^api/fields/parse/$', 'parse_fields', name='api_parse_fields'),
+  url(r'^api/autocomplete/$', 'autocomplete', name='api_autocomplete'),
+  url(r'^api/collections/$', 'collections', name='api_collections'),
+  url(r'^api/collections/create/$', 'collections_create', name='api_collections_create'),
+  url(r'^api/collections/import/$', 'collections_import', name='api_collections_import'),
+  url(r'^api/collections/remove/$', 'collections_remove', name='api_collections_remove'),
+  url(r'^api/collections/(?P<collection>[^/]+)/fields/$', 'collections_fields', name='api_collections_fields'),
+  url(r'^api/collections/(?P<collection>[^/]+)/update/$', 'collections_update', name='api_collections_update'),
+  url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
+)

+ 3 - 3
desktop/libs/indexer/src/indexer/utils.py

@@ -32,12 +32,12 @@ from django.conf import settings
 from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import force_unicode, smart_str
+from dashboard.conf import get_properties
 from libsentry.conf import is_enabled as is_sentry_enabled
+from libsolr.conf import FS_STORAGE
 
 from indexer import conf
 from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES, DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
-from dashboard.conf import get_properties
-from libsolr.conf import FS_STORAGE
 
 
 LOG = logging.getLogger(__name__)
@@ -119,7 +119,7 @@ def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
       else:
         solr_config_name = 'solrconfig.xml.solr6'
     if is_sentry_enabled():
-      solr_config_name = 'solrconfig.xml.secure'      
+      solr_config_name = 'solrconfig.xml.secure'
     solrconfig = 'conf/%s' % solr_config_name
 
     # Get complete solrconfig.xml

+ 0 - 7
desktop/libs/libsolr/src/libsolr/api.py

@@ -513,12 +513,6 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
-#   def add_fields(self, collection, fields):
-#     try:
-#       params = self._get_params()
-#       return self._root.post('%s/schema/fields' % collection, params=params, data=json.dumps(fields), contenttype='application/json')
-#     except RestException, e:
-#       raise PopupException(e, title=_('Error while accessing Solr'))
 
   def cores(self):
     try:
@@ -677,7 +671,6 @@ class SolrApi(object):
 
 
   def export(self, name, query, fl, sort, rows=100):
-    # /solr/demo6/export?user.name=hue&doAs=romain&q=code_s:[0%20TO%20400]&wt=json&rows=25&start=0&fl=code_s&sort=code_s+desc
     try:
       params = self._get_params() + (
           ('q', query),