Преглед изворни кода

HUE-7 [metastore] Notion of nested type in the frontend

Romain Rigaux пре 9 година
родитељ
комит
2d926bdfcd

+ 5 - 6
apps/beeswax/src/beeswax/create_table.py

@@ -103,7 +103,7 @@ DELIMITER_READABLE = {'\\001' : _('ctrl-As'),
                       '\\t'   : _('tabs'),
                       ','     : _('commas'),
                       ' '     : _('spaces')}
-FILE_READERS = [ ]
+FILE_READERS = []
 
 def import_wizard(request, database='default'):
   """
@@ -341,8 +341,7 @@ def _delim_preview(fs, file_form, encoding, file_types, delimiters):
 
 def _parse_fields(path, file_obj, encoding, filetypes, delimiters):
   """
-  _parse_fields(path, file_obj, encoding, filetypes, delimiters)
-                                  -> (delimiter, filetype, fields_list)
+  _parse_fields(path, file_obj, encoding, filetypes, delimiters) -> (delimiter, filetype, fields_list)
 
   Go through the list of ``filetypes`` (gzip, text) and stop at the first one
   that works for the data. Then apply the list of ``delimiters`` and pick the
@@ -351,7 +350,7 @@ def _parse_fields(path, file_obj, encoding, filetypes, delimiters):
 
   Return the best delimiter, filetype and the data broken down into rows of fields.
   """
-  file_readers = [ reader for reader in FILE_READERS if reader.TYPE in filetypes ]
+  file_readers = [reader for reader in FILE_READERS if reader.TYPE in filetypes]
 
   for reader in file_readers:
     LOG.debug("Trying %s for file: %s" % (reader.TYPE, path))
@@ -382,7 +381,7 @@ def _readfields(lines, delimiters):
     The score is always non-negative. The higher the better.
     """
     n_lines = len(fields_list)
-    len_list = [ len(fields) for fields in fields_list ]
+    len_list = [len(fields) for fields in fields_list]
 
     if not len_list:
       raise PopupException(_("Could not find any columns to import"))
@@ -394,7 +393,7 @@ def _readfields(lines, delimiters):
     avg_n_fields = sum(len_list) / n_lines
     sq_of_exp = avg_n_fields * avg_n_fields
 
-    len_list_sq = [ l * l for l in len_list ]
+    len_list_sq = [l * l for l in len_list]
     exp_of_sq = sum(len_list_sq) / n_lines
     var = exp_of_sq - sq_of_exp
     # Favour more fields

+ 5 - 1
desktop/libs/indexer/src/indexer/argument.py

@@ -13,8 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.import logging
+
 from django.utils.translation import ugettext as _
 
+
 class Argument():
   _type = None
   _default_value = None
@@ -35,21 +37,23 @@ class Argument():
   def default_value(self):
     return self._default_value
 
-
   def to_dict(self):
     return {"name": self._name, "type": self._type, "description": self._description}
 
   def get_default_arg_pair(self):
     return (self.name, self.default_value)
 
+
 class TextArgument(Argument):
   _type = "text"
   _default_value = ""
 
+
 class CheckboxArgument(Argument):
   _type = "checkbox"
   _default_value = False
 
+
 class MappingArgument(Argument):
   _type = "mapping"
 

+ 4 - 0
desktop/libs/indexer/src/indexer/fields.py

@@ -13,8 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.import logging
+
 import re
 
+
 class FieldType():
   def __init__(self, name, regex, heuristic_regex=None):
     self._name = name
@@ -59,6 +61,8 @@ class Field(object):
       'required': self.required,
       'multiValued': self.multi_valued,
       'showProperties': self.show_properties,
+      'nested': [],
+      'level': 0,
     }
 
 FIELD_TYPES = [

+ 22 - 24
desktop/libs/indexer/src/indexer/file_format.py

@@ -279,6 +279,20 @@ class CSVFormat(FileFormat):
   ]
   _extensions = ["csv", "tsv"]
 
+  def __init__(self, delimiter=',', line_terminator='\n', quote_char='"', has_header=False, sample="", fields=None):
+    self._delimiter = delimiter
+    self._line_terminator = line_terminator
+    self._quote_char = quote_char
+    self._has_header = has_header
+
+    # sniffer insists on \r\n even when \n. This is safer and good enough for a preview
+    self._line_terminator = self._line_terminator.replace("\r\n", "\n")
+    self._sample_rows = self._get_sample_rows(sample)
+    self._num_columns = self._guess_num_columns(self._sample_rows)
+    self._fields = fields if fields else self._guess_fields(sample)
+
+    super(CSVFormat, self).__init__()
+
   @staticmethod
   def format_character(string):
     string = string.replace('"', '\\"')
@@ -347,7 +361,7 @@ class CSVFormat(FileFormat):
     quote_char = format_["quoteChar"].encode('utf-8')
     has_header = format_["hasHeader"]
     return cls(**{
-      "delimiter":delimiter,
+      "delimiter": delimiter,
       "line_terminator": line_terminator,
       "quote_char": quote_char,
       "has_header": has_header,
@@ -361,23 +375,6 @@ class CSVFormat(FileFormat):
     else:
       return cls._guess_from_file_stream(file_stream)
 
-  def __init__(self, delimiter=',', line_terminator='\n', quote_char='"', has_header=False, sample="", fields=None):
-    self._delimiter = delimiter
-    self._line_terminator = line_terminator
-    self._quote_char = quote_char
-    self._has_header = has_header
-
-    # sniffer insists on \r\n even when \n. This is safer and good enough for a preview
-    self._line_terminator = self._line_terminator.replace("\r\n", "\n")
-
-    self._sample_rows = self._get_sample_rows(sample)
-
-    self._num_columns = self._guess_num_columns(self._sample_rows)
-
-    self._fields = fields if fields else self._guess_fields(sample)
-
-    super(CSVFormat, self).__init__()
-
   @property
   def sample(self):
     return self._sample_rows
@@ -401,10 +398,10 @@ class CSVFormat(FileFormat):
   def get_format(self):
     format_ = super(CSVFormat, self).get_format()
     specific_format = {
-      "fieldSeparator":self.delimiter,
-      "recordSeparator":self.line_terminator,
-      "quoteChar":self.quote_char,
-      "hasHeader":self._has_header
+      "fieldSeparator": self.delimiter,
+      "recordSeparator": self.line_terminator,
+      "quoteChar": self.quote_char,
+      "hasHeader": self._has_header
     }
     format_.update(specific_format)
 
@@ -452,7 +449,7 @@ class CSVFormat(FileFormat):
     if self._has_header:
       header = first_row
     else:
-      header = ["field_%d" % (i+1) for i in range(self._num_columns)]
+      header = ["field_%d" % (i + 1) for i in range(self._num_columns)]
 
     return header
 
@@ -479,6 +476,7 @@ class CSVFormat(FileFormat):
 
     return fields
 
+
 class HiveFormat(CSVFormat):
   FIELD_TYPE_TRANSLATE = {
     "BOOLEAN_TYPE": "string",
@@ -517,7 +515,7 @@ class HiveFormat(CSVFormat):
       fields.append(Field(
         name=field["name"],
         field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
-        ))
+      ))
 
     return cls(**{
       "delimiter":',',

+ 4 - 3
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -27,8 +27,8 @@ def col_type(col):
   return col["type"]
 %>\
 
+
 <%def name="column_list(table, columns)">\
-## Returns (foo int, bar string)-like data for columns
 (
 <% first = True %>\
 % for col in columns:
@@ -37,7 +37,7 @@ def col_type(col):
 %   else:
 ,
 %   endif
-  `${col["name"]|n}` ${col_type(col)|n} \
+  `${ col["name"] | n }` ${ col_type(col) | n } \
 %   if col.get("comment"):
 COMMENT "${col["comment"]|n}" \
 %   endif
@@ -47,7 +47,8 @@ COMMENT "${col["comment"]|n}" \
 % endif
 ) \
 </%def>\
-#########################
+
+
 CREATE \
 % if table.get("external", False):
 EXTERNAL \

+ 39 - 16
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -468,25 +468,40 @@ ${ assist.assistPanel() }
   <label>${ _('Name') }
     <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name">
   </label>
+
   <label>${ _('Type') }
-    <select class="input-small" data-bind="options: $root.createWizard.hiveFieldTypes, value: type"></select>
+  <select class="input-small" data-bind="options: $root.createWizard.hiveFieldTypes, value: type"></select>
+
+  <!-- ko if: type() == 'array' || type() == 'map' || type() == 'struct' -->
+    <div data-bind="template: { name: 'table-field-template', foreach: nested }">
+    </div>
+    <a data-bind="click: function() { nested.push(ko.mapping.fromJS({operations: [], nested: [], name: '', type: '', level: level() + 1})); }"><i class="fa fa-plus"></i></a>    
+  <!-- /ko -->
   </label>
 
-  <label data-bind="text: $root.createWizard.source.sample()[0][$index()]"></label>
-  <label data-bind="text: $root.createWizard.source.sample()[1][$index()]"></label>
+  ${_('Comment')}
 
-  <a href="javascript:void(0)" title="${ _('Show field properties') }" data-bind="click: function() {showProperties(! showProperties()) }">
-    <i class="fa fa-sliders"></i>
-  </a>
-  <span data-bind="visible: showProperties">
-    <label class="checkbox">
-      <input type="checkbox" data-bind="checked: unique"> ${_('Primary key')}
-    </label>
-  </span>
+  <!-- ko if: level() > 0 -->
+    <a data-bind="click: function() { $parent.nested.remove($data); }"><i class="fa fa-minus"></i></a>
+  <!-- /ko -->
 
-  <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Nested')}</a>
-  <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
-  ${_('Comment')}
+  <!-- ko if: level() == 0 -->
+    <label data-bind="text: $root.createWizard.source.sample()[0][$index()]"></label>
+    <label data-bind="text: $root.createWizard.source.sample()[1][$index()]"></label>
+  <!-- /ko -->
+</script>
+
+
+<script type="text/html" id="display-table-nested-field">
+  <!-- ko: if type() != 'array' -->
+    <select class="input-small" data-bind="options: $root.createWizard.hiveFieldTypes, value: type"></select>
+  <!-- /ko -->
+
+  <!-- ko: if type() == 'array' -->
+    <div data-bind="template: { name: 'display-table-nested-field', foreach: nested }">
+    </div>
+    <a data-bind="click: function() { nested.push('aa'); }"><i class="fa fa-plus"></i></a>
+  <!-- /ko -->
 </script>
 
 
@@ -544,7 +559,7 @@ ${ assist.assistPanel() }
 
 <script type="text/html" id="args-template">
   <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
-    <!-- ko template: {name: 'arg-'+argument.type, data:{description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
+    <!-- ko template: {name: 'arg-' + argument.type, data: {description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
   <!-- /ko -->
 </script>
 
@@ -897,7 +912,9 @@ ${ assist.assistPanel() }
           {'value': 'text', 'name': 'Text'},
           {'value': 'parquet', 'name': 'Parquet'},
           {'value': 'json', 'name': 'Json'},
-          {'value': 'kudu', 'name': 'Kudu'}
+          {'value': 'kudu', 'name': 'Kudu'},
+          {'value': 'orc', 'name': 'ORC'},
+          {'value': 'avro', 'name': 'Avro'}
       ]);
       self.ouputFormat = ko.observable('table');
 
@@ -1148,6 +1165,12 @@ ${ assist.assistPanel() }
         koField.operations.push(operation);
       });
 
+      koField.type.subscribe(function(newVal) {
+        if ((newVal == 'array' || newVal == 'map' || newVal == 'struct') && koField.nested().length == 0) {
+          koField.nested.push(ko.mapping.fromJS({operations: [], nested: [], name: '', type: '', level: koField.level() + 1}));
+        }
+      });
+
       return koField;
     }
 

+ 1 - 1
desktop/libs/indexer/src/indexer/views.py

@@ -80,7 +80,7 @@ def importer(request):
 
   return render('importer.mako', request, {
       'indexes_json': json.dumps(indexes),
-      'fields_json' : json.dumps({'solr': [field.name for field in FIELD_TYPES], 'hive': HIVE_PRIMITIVE_TYPES}),
+      'fields_json' : json.dumps({'solr': [field.name for field in FIELD_TYPES], 'hive': HIVE_TYPES}),
       'operators_json' : json.dumps([operator.to_dict() for operator in OPERATORS]),
       'file_types_json' : json.dumps([format_.format_info() for format_ in get_file_indexable_format_types()]),
       'default_field_type' : json.dumps(Field().to_dict())