Переглянути джерело

HUE-5129 [metastore] Backend for providing create table as Kudu option

Romain Rigaux 9 роки тому
батько
коміт
49fc79fc15

+ 32 - 13
desktop/libs/indexer/src/indexer/api3.py

@@ -174,22 +174,25 @@ def _create_table(request, source, destination):
   skip_header = True
   database = 'default'
   path = source['path']
-  table_format = 'parquet' #'text'
+  table_format = destination['tableFormat']
+  columns = destination['columns']
+  primary_keys = ['id']
 
   file_format = 'TextFile'
   sql = ''
   
   # if external and non text and load_data, both bath !=
   
-  
-  if table_format == 'parquet':
-    table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name # Or tmp table?
+  if load_data:
+    if table_format in ('parquet', 'kudu'):
+      table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name # Or tmp table?
     
   if external and not request.fs.isdir(path):
     path = request.fs.split(path)[0]
     # If dir not empty, create data dir %(filename)_table and move file there...
     
-    # Guess should accept a directory too
+    # Guess format should accept a directory too
+  # Kudu external table has extra prop
 
   sql += django_mako.render_to_string("gen/create_table_statement.mako", {
       'table': {
@@ -200,28 +203,44 @@ def _create_table(request, source, destination):
           'file_format': file_format,
           'external': external,
           'path': path, 
-          'skip_header': skip_header
+          'skip_header': skip_header,
+          'primary_keys': primary_keys if table_format == 'kudu' and not load_data else []
        },
-      'columns': destination['columns'],
+      'columns': columns,
       'partition_columns': [],
       'database': database
     }
   )
 
-  if not external and load_data:
+  if table_format == 'text' and not external and load_data:
     sql += "\n\nLOAD DATA INPATH '%s' INTO TABLE `%s`.`%s`;" % (path, database, table_name)
 
-  if table_format == 'parquet':
-    sql += '\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s` STORED AS %(file_format)s AS SELECT * FROM `%(database)s`.`%(table_name)s`;' % {
+  if table_format in ('parquet', 'kudu'):
+    file_format = table_format
+    if table_format == 'kudu':
+      columns_list = primary_keys + [col['name'] for col in destination['columns'] if col['name'] not in primary_keys]
+    else:
+      columns_list = ['*']
+    sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`
+      PRIMARY KEY (%(primary_keys)s)
+      DISTRIBUTE BY HASH INTO 16 BUCKETS
+      STORED AS %(file_format)s
+      AS SELECT %(columns_list)s
+      FROM `%(database)s`.`%(table_name)s`;''' % {
         'database': database,
         'final_table_name': final_table_name,
         'table_name': table_name,
-        'file_format': table_format
+        'file_format': file_format,
+        'columns_list': ', '.join(['`%s`' % col for col in columns_list]),
+        'primary_keys': ', '.join(primary_keys)
+    }
+    sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;' % {
+        'database': database,
+        'table_name': table_name
     }
-    sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;' % {'database': database, 'table_name': table_name}
 
   try:
-    editor_type = 'hive'
+    editor_type = 'impala' if table_format == 'kudu' else 'hive'
     # on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
     notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready', database=database)
 

+ 7 - 4
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -27,7 +27,7 @@ def col_type(col):
   return col["type"]
 %>\
 
-<%def name="column_list(columns)">\
+<%def name="column_list(table, columns)">\
 ## Returns (foo int, bar string)-like data for columns
 (
 <% first = True %>\
@@ -42,6 +42,9 @@ def col_type(col):
 COMMENT "${col["comment"]|n}" \
 %   endif
 % endfor
+% if table.get('primary_keys'):
+, PRIMARY KEY (${ ', '.join(table['primary_keys']) })
+% endif
 ) \
 </%def>\
 #########################
@@ -49,10 +52,10 @@ CREATE \
 % if table.get("external", False):
 EXTERNAL \
 % endif
-TABLE `${ '%s.%s' % (database, table["name"]) | n }`
-${column_list(columns)|n}
+TABLE ${ '`%s`.`%s`' % (database, table["name"]) | n }
+${ column_list(table, columns) | n }
 % if table["comment"]:
-COMMENT "${table["comment"] | n}"
+COMMENT "${table["comment"] | n }"
 % endif
 % if len(partition_columns) > 0:
 PARTITIONED BY ${column_list(partition_columns)|n}

+ 8 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -355,6 +355,7 @@ ${ assist.assistPanel() }
             <input type="checkbox"> ${_('Custom delimiters')}
           </label>
           ## field, coll map delimieters
+          ## regexp
 
           <label class="checkbox">
               <input type="checkbox" checked> ${_('Use headers')}
@@ -738,7 +739,7 @@ ${ assist.assistPanel() }
           self.getDocuments();
         }
       });
-      self.inputFormats = ko.observableArray(['file', 'text', 'table', 'query', 'dbms', 'nothing']);
+      self.inputFormats = ko.observableArray(['file', 'text', 'table', 'query', 'dbms', 'manual']);
 
       // File
       self.path = ko.observable('');
@@ -987,6 +988,12 @@ ${ assist.assistPanel() }
                 aceMode: 'ace/mode/hive',
                 snippetImage: '${ static("beeswax/art/icon_beeswax_48.png") }',
                 sqlDialect: true
+              },
+              impala: {
+                placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
+                aceMode: 'ace/mode/impala',
+                snippetImage: '${ static("impala/art/icon_impala_48.png") }',
+                sqlDialect: true
               }
             }
           });