|
@@ -46,23 +46,23 @@ class SQLIndexer(object):
|
|
|
database = 'default'
|
|
database = 'default'
|
|
|
table_name = destination['name']
|
|
table_name = destination['name']
|
|
|
final_table_name = table_name
|
|
final_table_name = table_name
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
table_format = destination['tableFormat']
|
|
table_format = destination['tableFormat']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
columns = destination['columns']
|
|
columns = destination['columns']
|
|
|
partition_columns = destination['partitionColumns']
|
|
partition_columns = destination['partitionColumns']
|
|
|
kudu_partition_columns = destination['kuduPartitionColumns']
|
|
kudu_partition_columns = destination['kuduPartitionColumns']
|
|
|
comment = destination['description']
|
|
comment = destination['description']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
source_path = source['path']
|
|
source_path = source['path']
|
|
|
external = not destination['useDefaultLocation']
|
|
external = not destination['useDefaultLocation']
|
|
|
external_path = destination['nonDefaultLocation']
|
|
external_path = destination['nonDefaultLocation']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
load_data = destination['importData']
|
|
load_data = destination['importData']
|
|
|
skip_header = destination['hasHeader']
|
|
skip_header = destination['hasHeader']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
primary_keys = destination['primaryKeys']
|
|
primary_keys = destination['primaryKeys']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if destination['useCustomDelimiters']:
|
|
if destination['useCustomDelimiters']:
|
|
|
field_delimiter = destination['customFieldDelimiter']
|
|
field_delimiter = destination['customFieldDelimiter']
|
|
|
collection_delimiter = destination['customCollectionDelimiter']
|
|
collection_delimiter = destination['customCollectionDelimiter']
|
|
@@ -72,21 +72,21 @@ class SQLIndexer(object):
|
|
|
collection_delimiter = r'\002'
|
|
collection_delimiter = r'\002'
|
|
|
map_delimiter = r'\003'
|
|
map_delimiter = r'\003'
|
|
|
regexp_delimiter = destination['customRegexp']
|
|
regexp_delimiter = destination['customRegexp']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
file_format = 'TextFile'
|
|
file_format = 'TextFile'
|
|
|
row_format = 'Delimited'
|
|
row_format = 'Delimited'
|
|
|
serde_name = ''
|
|
serde_name = ''
|
|
|
serde_properties = ''
|
|
serde_properties = ''
|
|
|
extra_create_properties = ''
|
|
extra_create_properties = ''
|
|
|
sql = ''
|
|
sql = ''
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if source['inputFormat'] == 'manual':
|
|
if source['inputFormat'] == 'manual':
|
|
|
load_data = False
|
|
load_data = False
|
|
|
source['format'] = {
|
|
source['format'] = {
|
|
|
'quoteChar': '"',
|
|
'quoteChar': '"',
|
|
|
'fieldSeparator': ','
|
|
'fieldSeparator': ','
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if table_format == 'json':
|
|
if table_format == 'json':
|
|
|
row_format = 'serde'
|
|
row_format = 'serde'
|
|
|
serde_name = 'org.apache.hive.hcatalog.data.JsonSerDe'
|
|
serde_name = 'org.apache.hive.hcatalog.data.JsonSerDe'
|
|
@@ -103,12 +103,12 @@ class SQLIndexer(object):
|
|
|
"quoteChar" = "%(quoteChar)s",
|
|
"quoteChar" = "%(quoteChar)s",
|
|
|
"escapeChar" = "\\\\"
|
|
"escapeChar" = "\\\\"
|
|
|
''' % source['format']
|
|
''' % source['format']
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
if table_format in ('parquet', 'kudu'):
|
|
if table_format in ('parquet', 'kudu'):
|
|
|
if load_data:
|
|
if load_data:
|
|
|
table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name
|
|
table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
|
|
sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
|
|
|
'database': database,
|
|
'database': database,
|
|
|
'table_name': table_name
|
|
'table_name': table_name
|
|
@@ -119,20 +119,20 @@ class SQLIndexer(object):
|
|
|
skip_header = False
|
|
skip_header = False
|
|
|
if table_format == 'kudu':
|
|
if table_format == 'kudu':
|
|
|
columns = [col for col in columns if col['name'] in primary_keys] + [col for col in columns if col['name'] not in primary_keys]
|
|
columns = [col for col in columns if col['name'] in primary_keys] + [col for col in columns if col['name'] not in primary_keys]
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if table_format == 'kudu':
|
|
if table_format == 'kudu':
|
|
|
collection_delimiter = None
|
|
collection_delimiter = None
|
|
|
map_delimiter = None
|
|
map_delimiter = None
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if external or (load_data and table_format in ('parquet', 'kudu')):
|
|
if external or (load_data and table_format in ('parquet', 'kudu')):
|
|
|
if not self.fs.isdir(external_path): # File selected
|
|
if not self.fs.isdir(external_path): # File selected
|
|
|
external_path, external_file_name = self.fs.split(external_path)
|
|
external_path, external_file_name = self.fs.split(external_path)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if len(self.fs.listdir(external_path)) > 1:
|
|
if len(self.fs.listdir(external_path)) > 1:
|
|
|
external_path = external_path + '/%s_table' % external_file_name # If dir not just the file, create data dir and move file there.
|
|
external_path = external_path + '/%s_table' % external_file_name # If dir not just the file, create data dir and move file there.
|
|
|
self.fs.mkdir(external_path)
|
|
self.fs.mkdir(external_path)
|
|
|
self.fs.rename(source_path, external_path)
|
|
self.fs.rename(source_path, external_path)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
sql += django_mako.render_to_string("gen/create_table_statement.mako", {
|
|
sql += django_mako.render_to_string("gen/create_table_statement.mako", {
|
|
|
'table': {
|
|
'table': {
|
|
|
'name': table_name,
|
|
'name': table_name,
|
|
@@ -155,7 +155,7 @@ class SQLIndexer(object):
|
|
|
'database': database
|
|
'database': database
|
|
|
}
|
|
}
|
|
|
)
|
|
)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if table_format in ('text', 'json', 'csv', 'regexp') and not external and load_data:
|
|
if table_format in ('text', 'json', 'csv', 'regexp') and not external and load_data:
|
|
|
form_data = {
|
|
form_data = {
|
|
|
'path': source_path,
|
|
'path': source_path,
|
|
@@ -164,7 +164,7 @@ class SQLIndexer(object):
|
|
|
}
|
|
}
|
|
|
db = dbms.get(self.user)
|
|
db = dbms.get(self.user)
|
|
|
sql += "\n\n%s;" % db.load_data(database, table_name, form_data, None, generate_ddl_only=True)
|
|
sql += "\n\n%s;" % db.load_data(database, table_name, form_data, None, generate_ddl_only=True)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if load_data and table_format in ('parquet', 'kudu'):
|
|
if load_data and table_format in ('parquet', 'kudu'):
|
|
|
file_format = table_format
|
|
file_format = table_format
|
|
|
if table_format == 'kudu':
|
|
if table_format == 'kudu':
|
|
@@ -196,11 +196,11 @@ class SQLIndexer(object):
|
|
|
'database': database,
|
|
'database': database,
|
|
|
'table_name': table_name
|
|
'table_name': table_name
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
editor_type = 'impala' if table_format == 'kudu' else destination['apiHelperType']
|
|
editor_type = 'impala' if table_format == 'kudu' else destination['apiHelperType']
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
|
|
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
return make_notebook(
|
|
return make_notebook(
|
|
|
name=_('Creating table %(database)s.%(table)s') % {'database': database, 'table': table_name},
|
|
name=_('Creating table %(database)s.%(table)s') % {'database': database, 'table': table_name},
|
|
|
editor_type=editor_type,
|
|
editor_type=editor_type,
|