浏览代码

HUE-5304 [indexer] Move morphline indexer into its own lib

Romain Rigaux 8 年之前
父节点
当前提交
cffca3b

+ 1 - 1
desktop/core/src/desktop/templates/assist.mako

@@ -585,7 +585,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, get_ord
   <script type="text/html" id="assist-collections-header-actions">
     <div class="assist-db-header-actions">
       <a class="inactive-action" href="javascript:void(0)" data-bind="click: $parent.toggleSearch, css: { 'blue' : $parent.isSearchVisible }"><i class="pointer fa fa-filter" title="${_('Filter')}"></i></a>
-      <a class="inactive-action" data-bind="hueLink: '/indexer#create'" title="${_('Create index')}" href="javascript:void(0)">
+      <a class="inactive-action" data-bind="hueLink: '/indexer/#create'" title="${_('Create index')}" href="javascript:void(0)">
         <i class="pointer fa fa-plus" title="${_('Create index')}"></i>
       </a>
       <a class="inactive-action" href="javascript:void(0)" data-bind="click: function () { huePubSub.publish('assist.collections.refresh'); }"><i class="pointer fa fa-refresh" data-bind="css: { 'fa-spin blue' : loading }" title="${_('Manual refresh')}"></i></a>

+ 1 - 1
desktop/libs/indexer/src/indexer/api3.py

@@ -32,7 +32,7 @@ from notebook.models import make_notebook
 from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
-from indexer.smart_indexer import Indexer
+from indexer.indexers.morphline import Indexer
 from indexer.solr_client import SolrClient, SolrClientException
 
 

+ 5 - 4
desktop/libs/indexer/src/indexer/file_format.py

@@ -26,7 +26,7 @@ from desktop.lib import i18n
 from indexer.argument import CheckboxArgument, TextDelimiterArgument
 from indexer.conf import ENABLE_NEW_INDEXER
 from indexer.fields import Field, guess_field_type_from_samples
-from indexer.operations import get_operator
+from indexer.morphline_operations import get_operator
 
 
 LOG = logging.getLogger(__name__)
@@ -41,15 +41,16 @@ def get_format_types():
 
   if ENABLE_NEW_INDEXER.get():
     formats.extend([
-      HueLogFormat,
       ApacheCombinedFormat,
-      RubyLogFormat,
       SyslogFormat,
-      ParquetFormat
+      HueLogFormat,
+      #RubyLogFormat,
+      #ParquetFormat
     ])
 
   return formats
 
+
 def get_file_indexable_format_types():
   return [format_ for format_ in get_format_types() if format_.is_file_indexable]
 

+ 15 - 0
desktop/libs/indexer/src/indexer/indexers/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 1 - 1
desktop/libs/indexer/src/indexer/smart_indexer.py → desktop/libs/indexer/src/indexer/indexers/morphline.py

@@ -33,7 +33,7 @@ from indexer.conf import CONFIG_INDEXING_TEMPLATES_PATH
 from indexer.conf import CONFIG_INDEXER_LIBS_PATH
 from indexer.fields import get_field_type
 from indexer.file_format import get_file_format_instance, get_file_format_class
-from indexer.operations import get_checked_args
+from indexer.morphline_operations import get_checked_args
 
 
 LOG = logging.getLogger(__name__)

+ 1 - 0
desktop/libs/indexer/src/indexer/operations.py → desktop/libs/indexer/src/indexer/indexers/morphline_operations.py

@@ -20,6 +20,7 @@ from indexer.argument import TextArgument, CheckboxArgument, MappingArgument
 
 
 class Operator():
+
   def __init__(self, name, args, output_type):
     self._name = name
     self._args = args

+ 2 - 2
desktop/libs/indexer/src/indexer/smart_indexer_tests.py → desktop/libs/indexer/src/indexer/indexers/morphline_tests.py

@@ -30,8 +30,8 @@ from indexer.conf import ENABLE_NEW_INDEXER
 from indexer.controller import CollectionManagerController
 from indexer.file_format import ApacheCombinedFormat, RubyLogFormat, HueLogFormat
 from indexer.fields import Field
-from indexer.operations import get_operator
-from indexer.smart_indexer import Indexer
+from indexer.indexers.morphline_operations import get_operator
+from indexer.indexers.morphline import Indexer
 
 
 LOG = logging.getLogger(__name__)

+ 20 - 27
desktop/libs/indexer/src/indexer/solr_api.py

@@ -103,10 +103,9 @@ def create_index(request):
   return JsonResponse(response)
 
 
+@require_POST
+@api_error_handler
 def delete_indexes(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
   response = {'status': -1}
 
   indexes = json.loads(request.POST.get('indexes', '[]'))
@@ -147,32 +146,26 @@ def create_alias(request):
   return JsonResponse(response)
 
 
+@require_POST
+@api_error_handler
 def design_schema(request, index):
-  if request.method == 'POST':
-    pass # TODO: Support POST for update?
-
   result = {'status': -1, 'message': ''}
 
-  try:
-    searcher = SolrClient(request.user)
-    unique_key, fields = searcher.get_index_schema(index)
-
-    result['status'] = 0
-    formatted_fields = []
-    for field in fields:
-      formatted_fields.append({
-        'name': field,
-        'type': fields[field]['type'],
-        'required': fields[field].get('required', None),
-        'indexed': fields[field].get('indexed', None),
-        'stored': fields[field].get('stored', None),
-        'multivalued': fields[field].get('multivalued', None),
-      })
-    result['fields'] = formatted_fields
-    result['unique_key'] = unique_key
-  except Exception, e:
-    result['message'] = _('Could not get index schema: %s') % e
+  searcher = SolrClient(request.user)
+  unique_key, fields = searcher.get_index_schema(index)
+
+  result['status'] = 0
+  formatted_fields = []
+  for field in fields:
+    formatted_fields.append({
+      'name': field,
+      'type': fields[field]['type'],
+      'required': fields[field].get('required', None),
+      'indexed': fields[field].get('indexed', None),
+      'stored': fields[field].get('stored', None),
+      'multivalued': fields[field].get('multivalued', None),
+    })
+  result['fields'] = formatted_fields
+  result['unique_key'] = unique_key
 
   return JsonResponse(result)
-
-

+ 5 - 3
desktop/libs/indexer/src/indexer/solr_client.py

@@ -23,11 +23,9 @@ import shutil
 
 from django.utils.translation import ugettext as _
 
-from dashboard.conf import get_properties
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
 from libsolr.api import SolrApi
-from libsolr.conf import FS_STORAGE
 from libsentry.conf import is_enabled as is_sentry_enabled
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
@@ -121,7 +119,7 @@ class SolrClient(object):
     """
     if self.is_solr_cloud_mode():
       if config_name is None:
-        self._create_cloud_config(name, fields, unique_key_field, df) # Create config set
+        self._create_cloud_config(name, fields, unique_key_field, df)
 
       self.api.create_collection2(name, config_name=config_name)
       fields = [{
@@ -200,6 +198,10 @@ class SolrClient(object):
         raise PopupException(_('Could not remove collection: %(message)s') % result)
 
 
+  def list_configs(self):    
+    return self.api.configs()
+
+
   def get_index_schema(self, index_name):
     """
     Returns a tuple of the unique key and schema fields for a given index

+ 1 - 1
desktop/libs/indexer/src/indexer/views.py

@@ -28,7 +28,7 @@ from indexer.solr_client import SolrClient
 from indexer.fields import FIELD_TYPES, Field
 from indexer.file_format import get_file_indexable_format_types
 from indexer.management.commands import indexer_setup
-from indexer.operations import OPERATORS
+from indexer.morphline_operations import OPERATORS
 
 
 LOG = logging.getLogger(__name__)