浏览代码

HUE-4244 [indexer] First skeleton on scalable indexer

solr smart indexer end to end

smart_indexer refactor

smart indexer cleanup and refactor

code review smart indexer cleanup

basic ui end to end smart indexing

indexer auto updates fields and preview

parameterized

indexer treats empty string data as a valid string

ui grok option, collection name update and check on keypress, positive feedback on collection name available"

end to end grok

pre rebase cleanup

further pre rebase cleanup

grok dictionaries bug fix

operation and field types defined server side

cleanup and internationalization
peddle 9 年之前
父节点
当前提交
af51f4cb29

+ 4 - 1
apps/oozie/src/oozie/models2.py

@@ -28,6 +28,7 @@ from string import Template
 from django.core.urlresolvers import reverse
 from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
+from django.contrib.auth.models import User
 
 from desktop.conf import USE_DEFAULT_CONFIGURATION
 from desktop.lib import django_mako
@@ -67,7 +68,9 @@ class Job(object):
 
   @classmethod
   def get_workspace(cls, user):
-    return (REMOTE_SAMPLE_DIR.get() + '/hue-oozie-$TIME').replace('$USER', user.username).replace('$TIME', str(time.time()))
+    if type(user) is User:
+      user = user.username
+    return (REMOTE_SAMPLE_DIR.get() + '/hue-oozie-$TIME').replace('$USER', user).replace('$TIME', str(time.time()))
 
   @property
   def validated_name(self):

+ 27 - 0
desktop/libs/indexer/src/data/oozie_workspace/convert_date_operation.conf

@@ -0,0 +1,27 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+<%page args="field, operation"/>
+
+# field
+# operation
+{
+  convertTimestamp {
+    field : ${field['name']}
+    inputFormats : ["""${operation['settings']['format']}"""]
+    outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+    outputTimezone : UTC
+  }
+}

+ 27 - 0
desktop/libs/indexer/src/data/oozie_workspace/grok_operation.conf

@@ -0,0 +1,27 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+<%page args="field, operation"/>
+
+# field
+# operation
+{
+  grok {
+    dictionaryFiles : ["grok_dictionaries"]
+    expressions : {
+      ${field['name']}: """${operation['settings']['regexp']}"""
+    }
+  }
+}

+ 34 - 0
desktop/libs/indexer/src/data/oozie_workspace/log4j.properties

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+log4j.rootLogger=WARN, A1
+
+log4j.logger.org.apache.flume.sink=INFO
+#log4j.logger.org.apache.flume.sink.solr=DEBUG
+log4j.logger.org.apache.solr=INFO
+#log4j.logger.org.apache.solr.hadoop=DEBUG
+log4j.logger.org.kitesdk.morphline=TRACE
+#log4j.logger.org.apache.solr.morphline=DEBUG
+log4j.logger.org.apache.solr.update.processor.LogUpdateProcessor=WARN
+log4j.logger.org.apache.solr.core.SolrCore=WARN
+log4j.logger.org.apache.solr.search.SolrIndexSearcher=ERROR
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

+ 143 - 0
desktop/libs/indexer/src/data/oozie_workspace/morphline_template.conf

@@ -0,0 +1,143 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+SOLR_LOCATOR : {
+  # Name of solr collection
+  collection : "${collection_name}"
+
+  # ZooKeeper ensemble
+  zkHost : "${zk_host}"
+}
+
+morphlines : [
+  {
+    id : ${collection_name}
+    importCommands : ["org.kitesdk.**", "org.apache.solr.**"]
+    commands : [
+      {
+        readCSV {
+          separator : "${format_character(format['fieldSeparator'])}"
+          columns : [
+            % for field in fields[:num_base_fields]:
+            "${field['name']}"
+            %endfor
+          ]
+          quoteChar : "${format_character(format['quoteChar'])}"
+          ignoreFirstLine : "${'true' if format['hasHeader'] else 'false'}"
+          charset : UTF-8
+        }
+      }
+
+      {
+        downloadHdfsFile{
+          inputFiles:["${grok_dictionaries_location}"]
+        }
+      }
+
+
+      {
+        generateUUID {
+          field : ${uuid_name}
+          type : "nonSecure"
+        }
+      }
+
+      # process operations on fields
+      % for field in fields:
+        %for operation in field['operations']:
+          <%include file="${operation['type']}_operation.conf" args="field=field, operation=operation"/>
+        %endfor
+      %endfor
+
+
+      # require that all kept fields are present
+      % for field in fields:
+        % if field["keep"] and field["required"]:
+          {
+            if {
+              conditions : [
+                { equals { "${field['name']}" : [] } }
+              ]
+              then : [
+                { logError { format : "Ignoring record because it has no ${field['name']}: {}", args : ["@{}"] } }
+                { dropRecord {} }
+              ]
+            }
+          }
+        % endif
+      %endfor
+
+      # require that all kept fields match their expected type
+      % for field in fields:
+        %if field["keep"]:
+          {
+            if {
+              conditions : [
+                {
+                  grok {
+                    expressions : {
+                      "${field['name']}" : "${get_regex(field['type'])}"
+                    }
+                    extract : false
+
+                  }
+                }
+              ]
+              then : []
+              else : [
+                %if not field["required"]:
+                  # if didn't match because the field isn't present then we keep the record
+                  {
+                    if {
+                      conditions : [
+                          { equals { "${field['name']}" : [] } }
+                      ]
+                      then : []
+                      else : [
+                        { logError { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
+                        { dropRecord {} }
+                      ]
+                    }
+                  }
+                %else:
+                  { logError { format : "Ignoring record due to incorrect type for ${field['name']}: {}", args : ["@{}"] } }
+                  { dropRecord {} }
+                %endif
+              ]
+            }
+          }
+        %endif
+      %endfor
+
+      # remove excess fields
+      {
+        sanitizeUnknownSolrFields {
+          # Location from which to fetch Solr schema
+          solrLocator : <%text>${SOLR_LOCATOR}</%text>
+        }
+      }
+
+      # log the record at DEBUG level to SLF4J
+      { logDebug { format : "output record: {}", args : ["@{}"] } }
+
+      {
+        loadSolr {
+          solrLocator : <%text>${SOLR_LOCATOR}</%text>
+        }
+      }
+
+    ]
+  }
+]

+ 34 - 0
desktop/libs/indexer/src/data/oozie_workspace/split_operation.conf

@@ -0,0 +1,34 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+<%page args="field, operation"/>
+
+# field
+# operation
+{
+  split {
+    inputField : ${field['name']}
+    outputFields : [
+      % for field in operation['fields']:
+      "${field['name']}"
+      %endfor
+    ]
+    separator : "${operation['settings']['splitChar']}"
+    isRegex : false
+    #separator : """\s*,\s*"""
+    #isRegex : true
+    addEmptyStrings : false
+  }
+}

+ 46 - 0
desktop/libs/indexer/src/data/oozie_workspace/workflow.xml

@@ -0,0 +1,46 @@
+<!-- Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.import logging -->
+<workflow-app name="SolrFileIndexer" xmlns="uri:oozie:workflow:0.5">
+    <start to="java-212a"/>
+    <kill name="IndexFailed">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    <action name="java-212a">
+        <java>
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <main-class>org.apache.solr.hadoop.MapReduceIndexerTool</main-class>
+            <arg>--morphline-file</arg>
+            <arg>morphline.conf</arg>
+            <arg>--output-dir</arg>
+            <arg>${nameNode}${outputDir}</arg>
+            <arg>--log4j</arg>
+            <arg>log4j.properties</arg>
+            <arg>--verbose</arg>
+            <arg>--go-live</arg>
+            <arg>--zk-host</arg>
+            <arg>${zkHost}</arg>
+            <arg>--collection</arg>
+            <arg>${collectionName}</arg>
+            <arg>${nameNode}${filePath}</arg>
+            <file>morphline.conf#morphline.conf</file>
+            <file>log4j.properties#log4j.properties</file>
+        </java>
+        <ok to="End"/>
+        <error to="IndexFailed"/>
+    </action>
+    <end name="End"/>
+</workflow-app>

+ 0 - 8
desktop/libs/indexer/src/data/solrconfigs/solrcloud/conf/solrconfig.xml

@@ -146,14 +146,6 @@
        before upgrading to a newer version to avoid unnecessary reindexing.
   -->
   <codecFactory class="solr.SchemaCodecFactory"/>
-
-  <!-- To enable dynamic schema REST APIs, use the following for <schemaFactory>: -->
-
-       <schemaFactory class="ManagedIndexSchemaFactory">
-         <bool name="mutable">true</bool>
-         <str name="managedSchemaResourceName">managed-schema</str>
-       </schemaFactory>
-
 <!--
        When ManagedIndexSchemaFactory is specified, Solr will load the schema from
        he resource named in 'managedSchemaResourceName', rather than from schema.xml.

+ 96 - 0
desktop/libs/indexer/src/indexer/api3.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+
+from django.utils.translation import ugettext as _
+
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.exceptions_renderable import PopupException
+from libsolr.api import SolrApi
+from search.conf import SOLR_URL, SECURITY_ENABLED
+
+from indexer.controller2 import IndexController
+from indexer.utils import get_default_fields
+from hadoop import cluster
+from indexer.smart_indexer import Indexer
+from indexer.controller import CollectionManagerController
+
+LOG = logging.getLogger(__name__)
+
+def _escape_white_space_characters(s, inverse = False):
+  MAPPINGS = {
+    "\n":"\\n",
+    "\t":"\\t",
+    "\r":"\\r",
+    " ":"\\s"
+  }
+
+  to = 1 if inverse else 0
+  from_ = 0 if inverse else 1
+
+  for pair in MAPPINGS.iteritems():
+    s = s.replace(pair[to], pair[from_]).encode('utf-8')
+
+  return s
+
+def _convert_format(format_dict, inverse=False):
+  FIELDS = [
+    "fieldSeparator",
+    "quoteChar",
+    "recordSeparator"
+  ]
+
+  for field in FIELDS:
+    format_dict[field] = _escape_white_space_characters(format_dict[field], inverse)
+
+def guess_format(request):
+  file_format = json.loads(request.POST.get('fileFormat', '{}'))
+  indexer = Indexer(request.user, request.fs)
+  stream = request.fs.open(file_format["path"])
+  format_ = indexer.guess_format({"file":stream})
+  _convert_format(format_)
+  
+  return JsonResponse(format_)
+
+def guess_field_types(request):
+  file_format = json.loads(request.POST.get('fileFormat', '{}'))
+  indexer = Indexer(request.user, request.fs)
+  stream = request.fs.open(file_format["path"])
+  _convert_format(file_format["format"], inverse = True)
+  format_ = indexer.guess_field_types({"file":stream, "format":file_format['format']})
+
+  return JsonResponse(format_)
+
+def index_file(request):
+  file_format = json.loads(request.POST.get('fileFormat', '{}'))
+  _convert_format(file_format["format"], inverse = True)
+  collection_name = file_format["name"]
+  indexer = Indexer(request.user, request.fs)
+  unique_field = indexer.get_uuid_name(file_format)
+  schema_fields = [{"name": unique_field, "type": "string"}] + \
+    indexer.get_kept_field_list(file_format['columns'])
+
+  morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
+
+  collection_manager = CollectionManagerController(request.user.username)
+  if not collection_manager.collection_exists(collection_name):
+    collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)
+
+  job_id = indexer.run_morphline(collection_name, morphline, file_format["path"])
+
+  return JsonResponse({"jobId": job_id})

+ 14 - 0
desktop/libs/indexer/src/indexer/conf.py

@@ -79,6 +79,20 @@ CONFIG_TEMPLATE_PATH = Config(
   type=str,
   default=os.path.join(os.path.dirname(__file__), '..', 'data', 'solrconfigs'))
 
+CONFIG_INDEXING_TEMPLATES_PATH = Config(
+  key="config_oozie_workspace_path",
+  help=_t("oozie workspace template for indexing:"),
+  type=str,
+  default=os.path.join(os.path.dirname(__file__), '..', 'data', 'oozie_workspace')
+  )
+
+CONFIG_INDEXER_LIBS_PATH = Config(
+  key="config_indexer_libs_path",
+  help=_t("oozie workspace template for indexing:"),
+  type=str,
+  default='/tmp/smart_indexer_lib'
+  )
+
 SOLRCTL_PATH = Config(
   key="solrctl_path",
   help=_t("Location of the solrctl binary."),

+ 473 - 0
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -0,0 +1,473 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.import logging
+import os
+import csv
+import operator
+import itertools
+import re
+import logging
+
+from mako.lookup import TemplateLookup
+from mako.template import Template
+
+from liboozie.oozie_api import get_oozie
+from oozie.models2 import Job
+from liboozie.submission2 import Submission
+
+from indexer.conf import CONFIG_INDEXING_TEMPLATES_PATH
+from indexer.conf import CONFIG_INDEXER_LIBS_PATH
+from indexer.conf import zkensemble
+
+from collections import deque
+
+LOG = logging.getLogger(__name__)
+
+class Indexer(object):
+  def __init__(self, username, fs):
+    self.fs = fs
+    self.username = username
+
+  # TODO: This oozie job code shouldn't be in the indexer. What's a better spot for it?
+  def _upload_workspace(self, morphline):
+    hdfs_workspace_path = Job.get_workspace(self.username)
+    hdfs_morphline_path = os.path.join(hdfs_workspace_path, "morphline.conf")
+    hdfs_workflow_path = os.path.join(hdfs_workspace_path, "workflow.xml")
+    hdfs_log4j_properties_path = os.path.join(hdfs_workspace_path, "log4j.properties")
+
+    workflow_template_path = os.path.join(CONFIG_INDEXING_TEMPLATES_PATH.get(), "workflow.xml")
+    log4j_template_path = os.path.join(CONFIG_INDEXING_TEMPLATES_PATH.get(), "log4j.properties")
+
+    # create workspace on hdfs
+    self.fs.do_as_user(self.username, self.fs.mkdir, hdfs_workspace_path)
+
+    self.fs.do_as_user(self.username, self.fs.create, hdfs_morphline_path, data=morphline)
+    self.fs.do_as_user(self.username, self.fs.create, hdfs_workflow_path, data=open(workflow_template_path).read())
+    self.fs.do_as_user(self.username, self.fs.create, hdfs_log4j_properties_path, data=open(log4j_template_path).read())
+
+    return hdfs_workspace_path
+
+  def _schedule_oozie_job(self, workspace_path, collection_name, input_path):
+    oozie = get_oozie(self.username)
+
+    properties = {
+      "dryrun": "False",
+      "zkHost":  zkensemble(),
+      # these libs can be installed from here:
+      # https://drive.google.com/a/cloudera.com/folderview?id=0B1gZoK8Ae1xXc0sxSkpENWJ3WUU&usp=sharing
+      "oozie.libpath": CONFIG_INDEXER_LIBS_PATH.get(),
+      "security_enabled": "False",
+      "collectionName": collection_name,
+      "filePath": input_path,
+      "outputDir": "/user/%s/indexer" % self.username,
+      "workspacePath": workspace_path,
+      'oozie.wf.application.path': "${nameNode}%s" % workspace_path,
+      'user.name': self.username
+    }
+
+    submission = Submission(self.username, fs=self.fs, properties=properties)
+    job_id = submission.run(workspace_path)
+
+    return job_id
+
+  def run_morphline(self, collection_name, morphline, input_path):
+    workspace_path = self._upload_workspace(morphline)
+
+    job_id = self._schedule_oozie_job(workspace_path, collection_name, input_path)
+    return job_id
+
+  def guess_format(self, data):
+    """
+    Input:
+    data: {'type': 'file', 'path': '/user/hue/logs.csv'}
+    Output:
+    {'format':
+      {
+        type: 'csv',
+        fieldSeparator : ",",
+        recordSeparator: '\n',
+        quoteChar : "\""
+      },
+      'columns':
+        [
+          {name: business_id, type: string},
+          {name: cool, type: integer},
+          {name: date, type: date}
+          ]
+    }
+    """
+    file_format = FileFormat.get_instance(data['file'])
+    return file_format.get_format()
+
+  def guess_field_types(self, data):
+    file_format = FileFormat.get_instance(data['file'], data['format'])
+    return file_format.get_fields()
+
+  # Breadth first ordering of fields
+  def get_field_list(self, field_data):
+    fields = []
+
+    queue = deque(field_data)
+
+    while len(queue):
+      curr_field = queue.popleft()
+      fields.append(curr_field)
+
+      for operation in curr_field["operations"]:
+        for field in operation["fields"]:
+          queue.append(field)
+
+    return fields
+
+  def get_kept_field_list(self, field_data):
+    return [field for field in self.get_field_list(field_data) if field['keep']]
+
+  def get_uuid_name(self, format_):
+    base_name = "_uuid"
+
+    field_names = set([column['name'] for column in format_['columns']])
+
+    while base_name in field_names:
+      base_name = '_' + base_name
+
+    return base_name
+
+  @staticmethod
+  def _format_character(string):
+    string = string.replace('\\', '\\\\')
+    string = string.replace('"', '\\"')
+    string = string.replace('\t', '\\t')
+    string = string.replace('\n', '\\n')
+
+    return string
+
+  @staticmethod
+  def _get_regex_for_type(type_):
+    matches = filter(lambda field_type: field_type.name == type_, Field.TYPES)
+
+    return matches[0].regex.replace('\\', '\\\\')
+
+  def generate_morphline_config(self, collection_name, data, uuid_name):
+    """
+    Input:
+    data: {
+      'type': {'name': 'My New Collection!' format': 'csv', 'columns': [{'name': business_id, 'included': True', 'type': 'string'}, cool, date], fieldSeparator : ",", recordSeparator: '\n', quoteChar : "\""},
+      'transformation': [
+        'country_code': {'replace': {'FRA': 'FR, 'CAN': 'CA'..}}
+        'ip': {'geoIP': }
+      ]
+    }
+    Output:
+    Morphline content 'SOLR_LOCATOR : { ...}'
+    """
+
+    properties = {
+      "collection_name":collection_name,
+      "fields":self.get_field_list(data['columns']),
+      "num_base_fields": len(data['columns']),
+      "format_character":Indexer._format_character,
+      "uuid_name" : uuid_name,
+      "get_regex":Indexer._get_regex_for_type,
+      "format":data['format'],
+      "grok_dictionaries_location" : os.path.join(CONFIG_INDEXER_LIBS_PATH.get(), "grok_dictionaries"),
+      "zk_host": zkensemble()
+    }
+
+    oozie_workspace = CONFIG_INDEXING_TEMPLATES_PATH.get()
+
+    lookup = TemplateLookup(directories=[oozie_workspace])
+    morphline = lookup.get_template("morphline_template.conf").render(**properties)
+
+    return morphline
+
+class FieldType():
+  def __init__(self, name, regex):
+    self._name = name
+    self._regex = regex
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def regex(self):
+    return self._regex
+  
+ 
+  def matches(self, field):
+    pattern = re.compile(self._regex)
+
+    return pattern.match(field)
+
+class Operator():
+  def __init__(self, name, args):
+    self._name = name
+    self._args = args
+
+  def to_dict(self):
+    return {
+      "name": self._name,
+      "args": self._args
+    }
+
+class Field(object):
+  TYPES = [
+    FieldType('text', "^.{100,}$"),
+    FieldType('string', "^.*$"),
+    FieldType('double', "^[+-]?[0-9]+\\.?[0-9]+$"),
+    FieldType('long', "^(?:[+-]?(?:[0-9]+))$"),
+    FieldType('date', "[0-9]+-[0-9]+-[0-9]+T[0-9]+:[0-9]+:[0-9]+(\\.[0-9]*)?Z")
+  ]
+
+  OPERATORS = [
+    Operator(
+      name="split",
+      args=["splitChar"]
+      ),
+    Operator(
+      name="grok",
+      args=["regexp"]
+      ),
+    Operator(
+      name="convert_date",
+      args=["format"]
+      ),
+  ]
+
+  def __init__(self, name, field_type):
+    self._name = name
+    self._field_type = field_type
+    self._keep = True
+    self._operations = []
+    self._required = True
+
+  @staticmethod
+  def guess_type(samples):
+    guesses = [Field._guess_field_type(sample) for sample in samples]
+
+    return Field._pick_best(guesses)
+
+  @staticmethod
+  def _guess_field_type(field):
+    for field_type in Field.TYPES[::-1]:
+      if field_type.matches(field):
+        return field_type.name
+
+  @staticmethod
+  def _pick_best(types):
+    types = set(types)
+
+    for field in Field.TYPES:
+      if field.name in types:
+        return field.name
+    return "string"
+
+  @property
+  def required(self):
+    return self._required
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def field_type(self):
+    return self._field_type
+
+  @property
+  def keep(self):
+    return self._keep
+
+  @property
+  def operations(self):
+    return self._operations
+
+  def to_dict(self):
+    return {'name': self.name,
+    'type': self.field_type,
+    'keep': self.keep,
+    'operations': self.operations,
+    'required': self.required}
+
+class FileFormat(object):
+  @staticmethod
+  def get_instance(file_stream, format_=None):
+    return CSVFormat(file_stream, format_)
+
+  def __init__(self):
+    pass
+
+  @property
+  def format_(self):
+    pass
+
+  @property
+  def sample(self):
+    pass
+
+  @property
+  def fields(self):
+    return []
+
+  def get_format(self):
+    return self.format_
+
+  def get_fields(self):
+    obj = {}
+
+    obj['columns'] = [field.to_dict() for field in self.fields]
+    obj['sample'] = self.sample
+
+    return obj
+
+  def to_dict(self):
+    obj = {}
+
+    obj['format'] = self.format_
+    obj['columns'] = [field.to_dict() for field in self.fields]
+    obj['sample'] = self.sample
+
+    return obj
+
+class CSVFormat(FileFormat):
+  def __init__(self, file_stream, format_=None):
+    file_stream.seek(0)
+    sample = file_stream.read(1024*1024*5)
+    file_stream.seek(0)
+
+    if format_:
+      self._delimiter = format_["fieldSeparator"].encode('utf-8')
+      self._line_terminator = format_["recordSeparator"].encode('utf-8')
+      self._quote_char = format_["quoteChar"].encode('utf-8')
+      self._has_header = format_["hasHeader"]
+    else:
+      dialect, self._has_header = self._guess_dialect(sample)
+      self._delimiter = dialect.delimiter
+      self._line_terminator = dialect.lineterminator
+      self._quote_char = dialect.quotechar
+
+    # sniffer insists on \r\n even when \n. This is safer and good enough for a preview
+    self._line_terminator = self._line_terminator.replace("\r\n", "\n")
+
+    self._sample_rows = self._get_sample_rows(sample)
+    self._num_columns = self._guess_num_columns(self._sample_rows)
+
+    self._fields = self._guess_fields(sample)
+
+    super(CSVFormat, self).__init__()
+
+  @property
+  def sample(self):
+    return self._sample_rows
+
+  @property
+  def fields(self):
+    return self._fields
+
+  @property
+  def delimiter(self):
+    return self._delimiter
+
+  @property
+  def line_terminator(self):
+    return self._line_terminator
+
+  @property
+  def quote_char(self):
+    return self._quote_char
+
+  @property
+  def format_(self):
+    return {
+      "type":"csv",
+      "fieldSeparator":self.delimiter,
+      "recordSeparator":self.line_terminator,
+      "quoteChar":self.quote_char,
+      "hasHeader":self._has_header
+    }
+
+  def _guess_dialect(self, sample):
+    sniffer = csv.Sniffer()
+    dialect = sniffer.sniff(sample)
+    has_header = sniffer.has_header(sample)
+    return dialect, has_header
+
+  def _guess_num_columns(self, sample_rows):
+    counts = {}
+
+    for row in sample_rows:
+      num_columns = len(row)
+
+      if num_columns not in counts:
+        counts[num_columns] = 0
+      counts[num_columns] += 1
+
+    if counts:
+      num_columns_guess = max(counts.iteritems(), key=operator.itemgetter(1))[0]
+    else:
+      num_columns_guess = 0
+    return num_columns_guess
+
+  def _guess_field_types(self, sample_rows):
+    field_type_guesses = []
+
+    num_columns = self._num_columns
+
+    for col in range(num_columns):
+      column_samples = [sample_row[col] for sample_row in sample_rows if len(sample_row) > col]
+
+      field_type_guess = Field.guess_type(column_samples)
+      field_type_guesses.append(field_type_guess)
+
+    return field_type_guesses
+
+  def _get_sample_reader(self, sample):
+    if self.line_terminator != '\n':
+      sample = sample.replace('\n', '\\n')
+    return csv.reader(sample.split(self.line_terminator), delimiter=self.delimiter, quotechar=self.quote_char)
+
+  def _guess_field_names(self, sample):
+    reader = self._get_sample_reader(sample)
+
+    first_row = reader.next()
+
+    if self._has_header:
+      header = first_row
+    else:
+      header = ["field_%d" % (i+1) for i in range(self._num_columns)]
+
+    return header
+
+  def _get_sample_rows(self, sample):
+    NUM_SAMPLES = 5
+
+    header_offset = 1 if self._has_header else 0
+    reader = itertools.islice(self._get_sample_reader(sample), header_offset, NUM_SAMPLES + 1)
+
+    sample_rows = list(reader)
+    return sample_rows
+
+  def _guess_fields(self, sample):
+    header = self._guess_field_names(sample)
+    types = self._guess_field_types(self._sample_rows)
+
+    if len(header) == len(types):
+      fields = [Field(header[i], types[i]) for i in range(len(header))]
+    else:
+      # likely failed to guess correctly
+      LOG.warn("Guess field types failed - number of headers didn't match number of predicted types.")
+      fields = []
+
+    return fields

+ 369 - 0
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -0,0 +1,369 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+  from desktop.views import commonheader, commonfooter, commonshare, commonimportexport
+  from django.utils.translation import ugettext as _
+%>
+<%namespace name="actionbar" file="actionbar.mako" />
+
+${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
+
+
+<div class="container-fluid">
+  <div class="card card-small">
+  <h1 class="card-heading simple">${ _('Solr Indexer') }</h1>
+  </div>
+</div>
+
+
+<!-- ko template: 'create-index-wizard' --><!-- /ko -->
+
+<script type="text/html" id="create-index-wizard">
+  <div class="snippet-settings" data-bind="visible: createWizard.show" style="
+  text-align: center;">
+
+    <div class="control-group" data-bind="css: { error: createWizard.validName() === false, success: createWizard.validName()}">
+      <label for="collectionName" class="control-label">${ _('Name') }</label>
+      <div class="controls">
+        <input type="text" class="form-control" id = "collectionName" data-bind="value: createWizard.fileFormat().name, valueUpdate: 'afterkeydown'">
+        <span class="help-block" data-bind="visible: createWizard.validName() === true">${ _('Collection name available') }</span>
+        <span class="help-block" data-bind="visible: createWizard.validName() === false">${_('This collection already exists') }</span>
+      </div>
+    </div>
+
+    <div class="control-group">
+      <label for="path" class="control-label">${ _('Path') }</label>
+      <div class="controls">
+        <input type="text" class="form-control" id = "path" data-bind="value: createWizard.fileFormat().path">
+        <a style="margin-bottom:10px" href="javascript:void(0)" class="btn" data-bind="click: createWizard.guessFormat">${_('Guess Format')}</a>
+      </div>
+    </div>
+
+
+    <div data-bind="visible: createWizard.fileFormat().show">
+      <div data-bind="with: createWizard.fileFormat().format">
+          <h3>${_('File Type')}: <span data-bind="text: type"></span></h3>
+          <h4>${_('Has Header')}:</h4>
+          <input type="checkbox" data-bind="checked: hasHeader">
+
+          <h4>${_('Quote Character')}:</h4>
+          <input data-bind="value: quoteChar">
+          <h4>${_('Record Separator')}:</h4>
+          <input data-bind="value: recordSeparator">
+          <h4>${_('Field Separator')}:</h4>
+          <input data-bind="value: fieldSeparator">
+        </div>
+
+        <h3>${_('Fields')}</h3>
+        <div data-bind="foreach: createWizard.fileFormat().columns">
+          <div data-bind="template: { name:'field-template',data:$data}"></div>
+        </div>
+
+        <h3>${_('Preview')}</h3>
+        <table style="margin:auto;text-align:left">
+          <thead>
+            <tr data-bind="foreach: createWizard.fileFormat().columns">
+              <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
+            </tr>
+          </thead>
+          <tbody data-bind="foreach: createWizard.sample">
+            <tr data-bind="foreach: $data">
+              <td data-bind="visible: $root.createWizard.fileFormat().columns()[$index()].keep, text: $data">
+              </td>
+
+                <!-- ko with: $root.createWizard.fileFormat().columns()[$index()] -->
+                  <!-- ko template: 'output-generated-field-data-template' --> <!-- /ko -->
+                <!-- /ko -->
+            </tr>
+          </tbody>
+        </table>
+
+        <br><hr><br>
+
+        <a href="javascript:void(0)" class="btn" data-bind="visible: !createWizard.indexingStarted() , click: createWizard.indexFile, css: {disabled : !createWizard.validName()}">${_('Index File!')}</a>
+
+        <h4 class="error" data-bind="visible: !createWizard.validName()">${_('Collection needs a unique name')}</h4>
+
+        <a href="javascript:void(0)" class="btn btn-success" data-bind="visible: createWizard.jobId, attr: {           href: '/oozie/list_oozie_workflow/' + createWizard.jobId() }" target="_blank" title="${ _('Open') }">
+          ${_('View Indexing Status')}
+        </a>
+
+      </div>
+
+    <br/>
+  </div>
+</script>
+
+<script type="text/html" id="field-template">
+  <div>
+    <span>${_('Keep')}</span><input type="checkbox" data-bind="checked: keep">
+    <span>${_('Required')}</span><input type="checkbox" data-bind="checked: required">
+    <input type="text" data-bind="value: name"></input> - <select data-bind="options: $root.createWizard.fieldTypes, value: type"></select>
+    <button class="btn" data-bind="click: $root.createWizard.addOperation">${_('Add Operation')}</button>
+  </div>
+  <div data-bind="foreach: operations">
+    <div data-bind="template: { name:'operation-template',data:{operation: $data, list: $parent.operations}}"></div>
+  </div>
+</script>
+
+<script type="text/html" id="operation-template">
+  <div><select data-bind="options: $root.createWizard.operationTypes.map(function(o){return o.name});, value: operation.type"></select>
+  <!-- ko template: "operation-args-template" --><!-- /ko -->
+    <input type="number" data-bind="value: operation.numExpectedFields">
+    <button class="btn" data-bind="click: function(){$root.createWizard.removeOperation(operation, list)}">${_('remove')}</button>
+    <div style="padding-left:50px" data-bind="foreach: operation.fields">
+      <div data-bind="template: { name:'field-template',data:$data}"></div>
+    </div>
+
+  </div>
+</script>
+
+<script type="text/html" id="field-preview-header-template">
+
+  <th data-bind="visible: keep, text: name" style="padding-right:60px"></th>
+  <!-- ko foreach: operations -->
+    <!--ko foreach: fields -->
+      <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
+    <!-- /ko -->
+  <!--/ko -->
+</script>
+
+
+<script type="text/html" id="output-generated-field-data-template">
+  <!-- ko foreach: operations -->
+    <!--ko foreach: fields -->
+      <td data-bind="visible: keep">[[${_('generated')}]]</td>
+      <!-- ko template: 'output-generated-field-data-template' --><!-- /ko -->
+    <!-- /ko -->
+  <!--/ko -->
+</script>
+
+<script type="text/html" id="operation-args-template">
+  <!-- ko foreach: Object.keys(operation.settings()) -->
+  <input type="text" data-bind="value: $parent.operation.settings()[$data]">
+  <!-- /ko -->
+</script>
+
+
+<div class="hueOverlay" data-bind="visible: isLoading">
+  <!--[if lte IE 9]>
+    <img src="${ static('desktop/art/spinner-big.gif') }" />
+  <![endif]-->
+  <!--[if !IE]> -->
+    <i class="fa fa-spinner fa-spin"></i>
+  <!-- <![endif]-->
+</div>
+
+
+<script src="${ static('desktop/ext/js/datatables-paging-0.1.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('desktop/ext/js/knockout.min.js') }" type="text/javascript" charset="utf-8"></script>
+<script src="${ static('desktop/ext/js/knockout-mapping.min.js') }" type="text/javascript" charset="utf-8"></script>
+
+
+<script type="text/javascript" charset="utf-8">
+  var createDefaultField = function(){
+    return {
+      name: ko.observable("new_field"),
+      type: ko.observable("string"),
+      keep: ko.observable(true),
+      required: ko.observable(true),
+      operations: ko.observableArray([])
+    }
+  };
+
+  var Operation = function(type){
+    var constructSettings = function(type){
+      var settings = {};
+      var operation = viewModel.createWizard.operationTypes.find(function(currOperation){
+        return currOperation.name == type;
+      });
+
+      for(var i = 0; i < operation.args.length; i++){
+        settings[operation.args[i]] = ko.observable(operation.args[i]);
+      }
+      return settings;
+    };
+
+    var self = this;
+
+    self.type = ko.observable(type);
+    self.fields = ko.observableArray();
+
+    self.numExpectedFields = ko.observable(0);
+
+    self.numExpectedFields.subscribe(function(numExpectedFields){
+      if(numExpectedFields < self.fields().length){
+        self.fields(self.fields().slice(0,numExpectedFields));
+      }
+      else if (numExpectedFields > self.fields().length){
+        difference = numExpectedFields - self.fields().length;
+
+        for(var i = 0; i < difference; i++){
+          self.fields.push(createDefaultField());
+        }
+      }
+    });
+
+    self.settings = ko.observable(constructSettings(type));
+
+    self.type.subscribe(function(newType){
+      self.settings(constructSettings(newType));
+    });
+  }
+
+  var File_Format = function (vm) {
+    var self = this;
+
+
+    self.name = ko.observable('');
+    self.sample = ko.observableArray();
+    self.show = ko.observable(false);
+
+    self.path = ko.observable('/tmp/test.csv');
+    self.format = ko.observable();
+    self.columns = ko.observableArray();
+  };
+
+  var CreateWizard = function (vm) {
+    var self = this;
+    var guessFieldTypesXhr;
+
+    self.operationTypes = ${operators_json | n};
+
+    self.fieldTypes = ko.observableArray(${fields_json | n});
+
+    self.validName = ko.observable(null);
+
+    self.show = ko.observable(true);
+    self.showCreate = ko.observable(false);
+
+    self.fileFormat = ko.observable(new File_Format(vm));
+
+    self.sample = ko.observableArray();
+
+    self.jobId = ko.observable(null);
+
+    self.indexingStarted = ko.observable(false);
+
+    self.fileFormat().format.subscribe(function(){
+      self.fileFormat().format().quoteChar.subscribe(self.guessFieldTypes);
+      self.fileFormat().format().recordSeparator.subscribe(self.guessFieldTypes);
+      self.fileFormat().format().type.subscribe(self.guessFieldTypes);
+      self.fileFormat().format().hasHeader.subscribe(self.guessFieldTypes);
+      self.fileFormat().format().fieldSeparator.subscribe(self.guessFieldTypes);
+
+      self.guessFieldTypes();
+    });
+
+    self.fileFormat().name.subscribe(function(newName){
+      self.validName(viewModel.collectionNameAvailable(newName));
+    })
+
+    self.guessFormat = function() {
+      viewModel.isLoading(true);
+      $.post("${ url('indexer:guess_format') }", {
+        "fileFormat": ko.mapping.toJSON(self.fileFormat)
+      }, function(resp) {
+
+        self.fileFormat().format(ko.mapping.fromJS(resp));
+
+        self.fileFormat().show(true);
+
+        viewModel.isLoading(false);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+
+        viewModel.isLoading(false);
+      });
+    }
+
+    self.guessFieldTypes = function(){
+      if(guessFieldTypesXhr) guessFieldTypesXhr.abort();
+      guessFieldTypesXhr = $.post("${ url('indexer:guess_field_types') }",{
+        "fileFormat": ko.mapping.toJSON(self.fileFormat)
+      }, function(resp){
+        resp.columns.forEach(function(entry, i, arr){
+          arr[i] = ko.mapping.fromJS(entry);
+        });
+        self.fileFormat().columns(resp.columns);
+
+        self.sample(resp.sample);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+
+        viewModel.isLoading(false);
+      });;
+    };
+
+    self.indexFile = function() {
+      if(!self.validName()) return;
+
+      self.indexingStarted(true);
+
+      viewModel.isLoading(true);
+
+      $.post("${ url('indexer:index_file') }", {
+        "fileFormat": ko.mapping.toJSON(self.fileFormat)
+      }, function(resp) {
+        self.showCreate(true);
+        self.jobId(resp.jobId);
+        viewModel.isLoading(false);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+        viewModel.isLoading(false);
+      });
+    }
+
+    self.removeOperation = function(operation, operationList){
+      operationList.remove(operation);
+    }
+
+    self.addOperation = function(field){
+      field.operations.push(new Operation("split"));
+    }
+  };
+
+  var Editor = function () {
+    var self = this;
+
+    self.collections = ${ indexes_json | n }.filter(function(index){
+      return index.type == 'collection';
+    });;
+
+    self.createWizard = new CreateWizard(self);
+    self.isLoading = ko.observable(false);
+
+    self.collectionNameAvailable = function(name){
+      var matchingCollections = self.collections.filter(function(collection){
+        return collection.name == name;
+      });
+
+      return matchingCollections.length == 0;
+    }
+
+  };
+
+  var viewModel;
+
+  $(document).ready(function () {
+    viewModel = new Editor();
+    ko.applyBindings(viewModel);
+  });
+</script>
+
+
+${ commonfooter(request, messages) | n,unicode }

+ 119 - 0
desktop/libs/indexer/src/indexer/tests_indexer.py

@@ -0,0 +1,119 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+import StringIO
+import logging
+
+from nose.tools import assert_equal
+from nose.plugins.skip import SkipTest
+
+from hadoop import cluster
+from hadoop.pseudo_hdfs4 import is_live_cluster
+
+from indexer.smart_indexer import Indexer
+from indexer.controller import CollectionManagerController
+
+LOG = logging.getLogger(__name__)
+
+class IndexerTest():
+  simpleCSVString = """id,Rating,Location,Name,Time
+1,5,San Francisco,Good Restaurant,8:30pm
+2,4,San Mateo,Cafe,11:30am
+3,3,Berkeley,Sauls,2:30pm
+"""
+
+  def setup(self):
+    if not is_live_cluster():
+      raise SkipTest()
+
+  def test_guess_format(self):
+    stream = StringIO.StringIO(IndexerTest.simpleCSVString)
+    indexer = Indexer("hue", None)
+
+    guessed_format = indexer.guess_format({'file': stream})
+
+    fields = indexer.guess_field_types({"file":stream, "format": guessed_format})['columns']
+    # test format
+    assert_equal('csv', guessed_format['type'])
+    assert_equal(',', guessed_format['fieldSeparator'])
+    assert_equal('\n', guessed_format['recordSeparator'])
+
+    # test fields
+    expected_fields = [
+      {
+        "name": "id",
+        "type": "long"
+      },
+      {
+        "name": "Rating",
+        "type": "long"
+      },
+      {
+        "name": "Location",
+        "type": "string"
+      },
+      {
+        "name": "Name",
+        "type": "string"
+      },
+      {
+        "name": "Time",
+        "type": "string"
+      }
+    ]
+
+    for i in range(len(expected_fields)):
+      expected = expected_fields[i]
+      actual = fields[i]
+
+      for key in ("name", "type"):
+        assert_equal(expected[key], actual[key])
+
+  def test_end_to_end(self):
+    fs = cluster.get_hdfs()
+    collection_name = "test_collection"
+    indexer = Indexer("test", fs)
+    input_loc = "/tmp/test.csv"
+
+    # upload the test file to hdfs
+    fs.create(input_loc, data=IndexerTest.simpleCSVString, overwrite=True)
+
+    # open a filestream for the file on hdfs
+    stream = fs.open(input_loc)
+
+    # guess the format of the file
+    file_type_format = indexer.guess_format({'file': stream})
+
+    field_types = indexer.guess_field_types({"file":stream, "format": file_type_format})
+
+    format_ = field_types.copy()
+    format_['format'] = file_type_format
+
+    # find a field name available to use for the record's uuid
+    unique_field = indexer.get_uuid_name(format_)
+
+    # generate morphline
+    morphline = indexer.generate_morphline_config(collection_name, format_, unique_field)
+
+    schema_fields = [{"name": unique_field, "type": "string"}] + indexer.get_kept_field_list(format_['columns'])
+
+    # create the collection from the specified fields
+    collection_manager = CollectionManagerController("test")
+    if collection_manager.collection_exists(collection_name):
+      collection_manager.delete_collection(collection_name, None)
+    collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)
+
+    # index the file
+    indexer.run_morphline(collection_name, morphline, input_loc)

+ 10 - 0
desktop/libs/indexer/src/indexer/urls.py

@@ -23,6 +23,9 @@ urlpatterns = patterns('indexer.views',
   
   # V2
   url(r'^indexes/$', 'indexes', name='indexes'),
+
+  # V3
+  url(r'^indexer/$', 'indexer', name='indexer')
 )
 
 urlpatterns += patterns('indexer.api',
@@ -46,4 +49,11 @@ urlpatterns += patterns('indexer.api2',
   url(r'^api/indexes/create_wizard_get_sample/$', 'create_wizard_get_sample', name='create_wizard_get_sample'),
   url(r'^api/indexes/create_wizard_create/$', 'create_wizard_create', name='create_wizard_create'),
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
+)
+
+urlpatterns += patterns('indexer.api3',
+  # V3
+  url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
+  url(r'^api/indexer/index_file/$', 'index_file', name='index_file'),
+  url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
 )

+ 15 - 3
desktop/libs/indexer/src/indexer/views.py

@@ -24,7 +24,7 @@ from desktop.lib.django_util import JsonResponse, render
 
 from indexer.controller2 import IndexController
 from indexer.management.commands import indexer_setup
-
+from indexer.smart_indexer import Field
 
 LOG = logging.getLogger(__name__)
 
@@ -36,14 +36,26 @@ def collections(request, is_redirect=False):
 def indexes(request):
   searcher = IndexController(request.user)
   indexes = searcher.get_indexes()
-  
+
   for index in indexes:
     index['isSelected'] = False
 
   return render('indexes.mako', request, {
-      'indexes_json': json.dumps(indexes),
+      'indexes_json': json.dumps(indexes)
   })
 
+def indexer(request):
+  searcher = IndexController(request.user)
+  indexes = searcher.get_indexes()
+
+  for index in indexes:
+    index['isSelected'] = False
+
+  return render('indexer.mako', request, {
+      'indexes_json': json.dumps(indexes),
+      'fields_json' : json.dumps([field.name for field in Field.TYPES]),
+      'operators_json' : json.dumps([operator.to_dict() for operator in Field.OPERATORS])
+  })
 
 def install_examples(request, is_redirect=False):
   result = {'status': -1, 'message': ''}