Forráskód Böngészése

HUE-4459 [indexer] Add morphline generation tests for each file format

Aaron Peddle 9 éve
szülő
commit
6de223d

+ 1 - 1
desktop/libs/indexer/src/data/oozie_workspace/parse_hue.conf

@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.from nose.tools import assert_equal
-<%page args="format_instance, format_class"/>
+<%page args="format_settings, format_class"/>
 
 {
   readMultiLine{

+ 3 - 3
desktop/libs/indexer/src/indexer/fields.py

@@ -39,18 +39,18 @@ class FieldType():
     return pattern.match(field)
 
 class Field(object):
-  def __init__(self, name, field_type):
+  def __init__(self, name, field_type, operations=[]):
     self.name = name
     self.field_type = field_type
     self.keep = True
-    self.operations = []
+    self.operations = operations
     self.required = False
 
   def to_dict(self):
     return {'name': self.name,
     'type': self.field_type,
     'keep': self.keep,
-    'operations': self.operations,
+    'operations': [operation.to_dict() for operation in self.operations],
     'required': self.required}
 
 FIELD_TYPES = [

+ 5 - 6
desktop/libs/indexer/src/indexer/file_format.py

@@ -28,9 +28,9 @@ LOG = logging.getLogger(__name__)
 def get_format_types():
   return [
     CSVFormat,
-    HueFormat,
+    HueLogFormat,
     ApacheCombinedFormat,
-    RubyLog
+    RubyLogFormat
   ]
 
 def _get_format_mapping():
@@ -101,7 +101,6 @@ class FileFormat(object):
       "args": [arg.to_dict() for arg in cls.get_arguments()],
       "description": cls.get_description(),
       "isCustomizable": cls.is_customizable(),
-      "parse_type": cls.get_parse_type()
     }
 
   @classmethod
@@ -120,7 +119,7 @@ class FileFormat(object):
     return []
 
   def get_format(self):
-    return {"type": self.get_name(), "parse_type": self.get_parse_type()}
+    return {"type": self.get_name()}
 
   def get_fields(self):
     obj = {}
@@ -155,7 +154,7 @@ class GrokkedFormat(FileFormat):
 
     return format_
 
-class HueFormat(GrokkedFormat):
+class HueLogFormat(GrokkedFormat):
   _name = "hue"
   _description = _("Hue Log File")
   _customizable = False
@@ -209,7 +208,7 @@ class ApacheCombinedFormat(GrokLineFormat):
   def fields(self):
     return self._fields
 
-class RubyLog(GrokLineFormat):
+class RubyLogFormat(GrokLineFormat):
   _name = "ruby_log"
   _description = _("Ruby Log")
   _customizable = False

+ 3 - 3
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -149,7 +149,7 @@ class Indexer(object):
 
     return matches[0].regex.replace('\\', '\\\\')
 
-  def generate_morphline_config(self, collection_name, data, uuid_name):
+  def generate_morphline_config(self, collection_name, data, uuid_name="__uuid"):
     """
     Input:
     data: {
@@ -175,8 +175,8 @@ class Indexer(object):
       "format_settings": data['format'],
       "format_class": get_file_format_class(data['format']['type']),
       "get_kept_args": get_checked_args,
-      "grok_dictionaries_location" : grok_dicts_loc if self.fs.exists(grok_dicts_loc) else None,
-      "geolite_db_location" : geolite_loc if self.fs.exists(geolite_loc) else None,
+      "grok_dictionaries_location" : grok_dicts_loc if self.fs and self.fs.exists(grok_dicts_loc) else None,
+      "geolite_db_location" : geolite_loc if self.fs and self.fs.exists(geolite_loc) else None,
       "zk_host": zkensemble()
     }
 

+ 86 - 32
desktop/libs/indexer/src/indexer/tests_indexer.py

@@ -16,7 +16,7 @@
 import StringIO
 import logging
 
-from nose.tools import assert_equal
+from nose.tools import assert_equal, assert_true
 from nose.plugins.skip import SkipTest
 
 from hadoop import cluster
@@ -25,20 +25,74 @@ from hadoop.pseudo_hdfs4 import is_live_cluster
 from indexer.smart_indexer import Indexer
 from indexer.controller import CollectionManagerController
 
+from indexer.file_format import ApacheCombinedFormat, RubyLogFormat, HueLogFormat
+
 LOG = logging.getLogger(__name__)
 
+def _test_fixed_type_format_generate_morphline(format_):
+  indexer = Indexer("test", None)
+  format_instance = format_()
+
+  morphline = indexer.generate_morphline_config("test_collection", {
+      "columns": [field.to_dict() for field in format_instance.fields],
+      "format": format_instance.get_format()
+    })
+
+  assert_true(isinstance(morphline, basestring))
+
+
 class IndexerTest():
   simpleCSVString = """id,Rating,Location,Name,Time
 1,5,San Francisco,Good Restaurant,8:30pm
 2,4,San Mateo,Cafe,11:30am
 3,3,Berkeley,Sauls,2:30pm
 """
-
-  def setup(self):
-    if not is_live_cluster():
-      raise SkipTest()
-
-  def test_guess_format(self):
+  simpleCSVFields = [
+    {
+      "name": "id",
+      "type": "long",
+      "operations": [],
+      "keep": True,
+      "required": False
+    },
+    {
+      "name": "Rating",
+      "type": "long",
+      "operations": [],
+      "keep": True,
+      "required": False
+    },
+    {
+      "name": "Location",
+      "type": "string",
+      "operations": [],
+      "keep": True,
+      "required": False
+    },
+    {
+      "name": "Name",
+      "type": "string",
+      "operations": [],
+      "keep": True,
+      "required": False
+    },
+    {
+      "name": "Time",
+      "type": "string",
+      "operations": [],
+      "keep": True,
+      "required": False
+    }
+  ]
+  simpleCSVFormat = {
+    'type': 'csv',
+    'fieldSeparator': ',',
+    'recordSeparator': '\n',
+    'hasHeader': True,
+    'quoteChar': '"'
+  }
+
+  def test_guess_csv_format(self):
     stream = StringIO.StringIO(IndexerTest.simpleCSVString)
     indexer = Indexer("test", None)
 
@@ -46,33 +100,12 @@ class IndexerTest():
 
     fields = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     # test format
-    assert_equal('csv', guessed_format['type'])
-    assert_equal(',', guessed_format['fieldSeparator'])
-    assert_equal('\n', guessed_format['recordSeparator'])
+    expected_format = self.simpleCSVFormat
+
+    assert_equal(expected_format, guessed_format)
 
     # test fields
-    expected_fields = [
-      {
-        "name": "id",
-        "type": "long"
-      },
-      {
-        "name": "Rating",
-        "type": "long"
-      },
-      {
-        "name": "Location",
-        "type": "string"
-      },
-      {
-        "name": "Name",
-        "type": "string"
-      },
-      {
-        "name": "Time",
-        "type": "string"
-      }
-    ]
+    expected_fields = self.simpleCSVFields
 
     for expected, actual in zip(expected_fields, fields):
       for key in ("name", "type"):
@@ -105,7 +138,28 @@ class IndexerTest():
     fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
     assert_equal(fields, [])
 
+  def test_generate_csv_morphline(self):
+    indexer = Indexer("test", None)
+    morphline =indexer.generate_morphline_config("test_collection", {
+        "columns": self.simpleCSVFields,
+        "format": self.simpleCSVFormat
+      })
+
+    assert_true(isinstance(morphline, basestring))
+
+  def test_generate_apache_combined_morphline(self):
+    _test_fixed_type_format_generate_morphline(ApacheCombinedFormat)
+
+  def test_generate_ruby_logs_morphline(self):
+    _test_fixed_type_format_generate_morphline(RubyLogFormat)
+
+  def test_generate_hue_log_morphline(self):
+    _test_fixed_type_format_generate_morphline(HueLogFormat)
+
   def test_end_to_end(self):
+    if not is_live_cluster():
+      raise SkipTest()
+
     fs = cluster.get_hdfs()
     collection_name = "test_collection"
     indexer = Indexer("test", fs)