Ver código fonte

HUE-7 [importer] Prevent creating table with names already existing

Add regexp table format.
Romain Rigaux 8 anos atrás
pai
commit
6793ea66e1

+ 8 - 4
desktop/libs/indexer/src/indexer/api3.py

@@ -26,12 +26,12 @@ from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document2
 from notebook.connectors.base import get_api, Notebook
+from notebook.models import make_notebook
 
 from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.smart_indexer import Indexer
-from notebook.models import make_notebook
 
 
 LOG = logging.getLogger(__name__)
@@ -232,12 +232,11 @@ def _create_table_from_a_file(request, source, destination):
     field_delimiter = destination['customFieldDelimiter']
     collection_delimiter = destination['customCollectionDelimiter']
     map_delimiter = destination['customMapDelimiter']
-    regexp_delimiter = destination['customRegexp']
   else:
     field_delimiter = ','
     collection_delimiter = r'\002'
     map_delimiter = r'\003'
-    regexp_delimiter = '.*'
+  regexp_delimiter = destination['customRegexp']
 
   file_format = 'TextFile'
   row_format = 'Delimited'
@@ -256,6 +255,11 @@ def _create_table_from_a_file(request, source, destination):
    "quoteChar"     = "'",
    "escapeChar"    = "\\\\"
    '''
+  elif table_format == 'regexp':
+    row_format = 'serde'
+    serde_name = 'org.apache.hadoop.hive.serde2.RegexSerDe'
+    serde_properties = '"input.regex" = "%s"' % regexp_delimiter
+
 
   if table_format in ('parquet', 'kudu'):
     if load_data:
@@ -340,7 +344,7 @@ def _create_table_from_a_file(request, source, destination):
   editor_type = 'impala' if table_format == 'kudu' else 'hive'
   on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
 
-  return make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready', database=database, on_success_url=on_success_url)
+  return make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql.strip(), status='ready', database=database, on_success_url=on_success_url)
 
 
 def _index(request, file_format, collection_name, query=None):

+ 3 - 3
desktop/libs/indexer/src/indexer/file_format.py

@@ -332,7 +332,7 @@ class CSVFormat(FileFormat):
   @classmethod
   def _guess_from_file_stream(cls, file_stream):
     sample = cls._get_sample(file_stream)
-
+    
     try:
       dialect, has_header = cls._guess_dialect(sample)
       delimiter = dialect.delimiter
@@ -343,7 +343,7 @@ class CSVFormat(FileFormat):
       return cls()
 
     return cls(**{
-      "delimiter":delimiter,
+      "delimiter": delimiter,
       "line_terminator": line_terminator,
       "quote_char": quote_char,
       "has_header": has_header,
@@ -353,7 +353,7 @@ class CSVFormat(FileFormat):
   @classmethod
   def _get_sample(cls, file_stream):
     file_stream.seek(0)
-    sample = '\n'.join(file_stream.read(1024*1024*5).splitlines())
+    sample = '\n'.join(file_stream.read(1024 * 1024 * 5).splitlines())
     file_stream.seek(0)
 
     return sample

+ 15 - 8
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -79,7 +79,6 @@ ${ assist.assistPanel() }
   }
 
   .step .card-heading.simple {
-    border-bottom: 1px solid #e5e5e5;
     font-size: 17px;
   }
 
@@ -470,7 +469,12 @@ ${ assist.assistPanel() }
                 ${ _('Create a new ') } <span data-bind="text: outputFormat"></span>
               </span>
               <span class="help-inline muted" data-bind="visible: isTargetExisting()">
-                ${ _('Adding data to the existing ') } <span data-bind="text: outputFormat"></span>
+                <!-- ko if: outputFormat() == 'index' -->
+                  ${ _('Adding data to the existing ') } <span data-bind="text: outputFormat"></span>
+                <!-- /ko -->
+                <!-- ko if: outputFormat() != 'index' -->
+                  <span data-bind="text: outputFormat"></span> ${ _('alredy exists.') } 
+                <!-- /ko -->
                 <a href="javascript:void(0)" data-bind="attr: { href: existingTargetUrl() }, text: name" target="_blank"></a>
               </span>
             </label>
@@ -531,13 +535,14 @@ ${ assist.assistPanel() }
                   <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { create: true, maxLength: 2 }, value: customMapDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
                 </label>
               </div>
-              <div class="control-group">
-                <label for="customRegexp" class="control-label"><div>${ _('Regexp') }</div>
-                  <input id="customRegexp"  type="text" data-bind="value: customRegexp">
-                </label>
-              </div>
             </span>
 
+            <div class="control-group" data-bind="visible: tableFormat() == 'regexp'">
+              <label for="customRegexp" class="control-label"><div>${ _('Regexp') }</div>
+                <input id="customRegexp" class="input-xxlarge" type="text" data-bind="value: customRegexp" placeholder='([^]*) ([^]*) ([^]*) (-|\\[^\\]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\".*\") ([^ \"]*|\".*\"))?'>
+              </label>
+            </div>
+
             <div class="control-group" data-bind="visible: tableFormat() == 'kudu'">
               <label for="kuduPks" class="control-label"><div>${ _('Primary keys') }</div>
                 ## At least one selected
@@ -1286,6 +1291,7 @@ ${ assist.assistPanel() }
           {'value': 'json', 'name': 'Json'},
           {'value': 'kudu', 'name': 'Kudu'},
           {'value': 'avro', 'name': 'Avro'},
+          {'value': 'regexp', 'name': 'Regexp'},
           {'value': 'rcfile', 'name': 'RCFile'},
           {'value': 'orc', 'name': 'ORC'},
           {'value': 'sequencefile', 'name': 'SequenceFile'}
@@ -1366,8 +1372,9 @@ ${ assist.assistPanel() }
       self.readyToIndex = ko.computed(function () {
         var validFields = self.destination.columns().length || self.destination.outputFormat() == 'database';
         var validDestination = self.destination.name().length > 0 && (['table', 'database'].indexOf(self.destination.outputFormat()) == -1 || /^[a-zA-Z0-9_]*$/.test(self.destination.name()));
+        var isTargetAlreadyExisting = ! self.destination.isTargetExisting() || self.destination.outputFormat() == 'index';
 
-        return validDestination && validFields;
+        return validDestination && validFields && isTargetAlreadyExisting;
       });
 
       self.formatTypeSubscribed = false;