Эх сурвалжийг харах

HUE-5988 [importer] Generate proper LOAD data for partioned table created from a file

Romain Rigaux 8 жил өмнө
parent
commit
e3dd0c4

+ 6 - 8
apps/beeswax/src/beeswax/server/dbms.py

@@ -536,18 +536,16 @@ class HiveServer2Dbms(object):
     return self.execute_statement(hql)
 
 
-  def load_data(self, database, table, form, design, generate_ddl_only=False):
+  def load_data(self, database, table, form_data, design, generate_ddl_only=False):
     hql = "LOAD DATA INPATH"
-    hql += " '%s'" % form.cleaned_data['path']
-    if form.cleaned_data['overwrite']:
+    hql += " '%(path)s'" % form_data
+    if form_data['overwrite']:
       hql += " OVERWRITE"
     hql += " INTO TABLE "
-    hql += "`%s`.`%s`" % (database, table.name,)
-    if form.partition_columns:
+    hql += "`%s`.`%s`" % (database, table)
+    if form_data['partition_columns']:
       hql += " PARTITION ("
-      vals = []
-      for key, column_name in form.partition_columns.iteritems():
-        vals.append("%s='%s'" % (column_name, form.cleaned_data[key]))
+      vals = ["%s='%s'" % (column_name, column_value) for column_name, column_value in form_data['partition_columns']]
       hql += ", ".join(vals)
       hql += ")"
 

+ 6 - 1
apps/metastore/src/metastore/views.py

@@ -442,7 +442,12 @@ def load_table(request, database, table):
       generate_ddl_only = request.POST.get('is_embeddable', 'false') == 'true'
       try:
         design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
-        query_history = db.load_data(database, table, load_form, design, generate_ddl_only=generate_ddl_only)
+        form_data = {
+          'path': load_form.cleaned_data['path'],
+          'overwrite': load_form.cleaned_data['overwrite'],
+          'partition_columns': [(column_name, load_form.cleaned_data[key]) for key, column_name in load_form.cleaned_data['partition_columns'].iteritems()],
+        }
+        query_history = db.load_data(database, table.name, form_data, design, generate_ddl_only=generate_ddl_only)
         if generate_ddl_only:
           job = make_notebook(
             name='Execute and watch',

+ 7 - 3
desktop/libs/indexer/src/indexer/api3.py

@@ -323,9 +323,13 @@ def _create_table_from_a_file(request, source, destination):
   )
 
   if table_format in ('text', 'json', 'csv', 'regexp') and not external and load_data:
-    sql += "\n\nLOAD DATA INPATH '%s' INTO TABLE `%s`.`%s`;" % (source_path, database, table_name)
-    # TODO partition_columns
-    # cf metastore load_data
+    form_data = {
+      'path': source_path,
+      'overwrite': False,
+      'partition_columns': [(partition['name'], partition['partitionValue']) for partition in partition_columns],
+    }
+    db = dbms.get(request.user)
+    sql += "\n\n%s;" % db.load_data(database, table_name, form_data, None, generate_ddl_only=True)
 
   if load_data and table_format in ('parquet', 'kudu'):
     file_format = table_format

Файлын зөрүү хэтэрхий том тул дарагдсан байна
+ 11 - 1
desktop/libs/indexer/src/indexer/create_table_tests.py


+ 1 - 1
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -57,7 +57,7 @@ COMMENT "${col["comment"]|n}" \
 % if table.get('primary_keys'):
 , PRIMARY KEY (${ ', '.join(table['primary_keys']) })
 % endif
-) \
+)\
 </%def>\
 
 <%def name="kudu_partition(partition)">

Энэ ялгаанд хэт олон файл өөрчлөгдсөн тул зарим файлыг харуулаагүй болно