|
|
@@ -20,19 +20,23 @@ from builtins import object
|
|
|
import logging
|
|
|
import sys
|
|
|
import urllib.request, urllib.error
|
|
|
+import uuid
|
|
|
|
|
|
from django.contrib.auth.models import User
|
|
|
from django.urls import reverse
|
|
|
from django.utils.translation import ugettext as _
|
|
|
|
|
|
from desktop.lib import django_mako
|
|
|
+from desktop.lib.exceptions_renderable import PopupException
|
|
|
+
|
|
|
from notebook.models import make_notebook
|
|
|
from azure.abfs.__init__ import abfspath
|
|
|
|
|
|
if sys.version_info[0] > 2:
|
|
|
- from urllib.parse import unquote as urllib_unquote
|
|
|
+ from urllib.parse import urlparse, unquote as urllib_unquote
|
|
|
else:
|
|
|
from urllib import unquote as urllib_unquote
|
|
|
+ from urlparse import urlparse
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
@@ -66,10 +70,14 @@ class SQLIndexer(object):
|
|
|
comment = destination['description']
|
|
|
|
|
|
source_path = urllib_unquote(source['path'])
|
|
|
+ load_data = destination['importData']
|
|
|
external = not destination['useDefaultLocation']
|
|
|
external_path = urllib_unquote(destination['nonDefaultLocation'])
|
|
|
|
|
|
- load_data = destination['importData']
|
|
|
+ editor_type = destination['sourceType']
|
|
|
+ is_transactional = destination['isTransactional']
|
|
|
+ default_transactional_type = 'insert_only' if destination['isInsertOnly'] else 'default'
|
|
|
+
|
|
|
skip_header = destination['hasHeader']
|
|
|
|
|
|
primary_keys = destination['primaryKeys']
|
|
|
@@ -115,8 +123,8 @@ class SQLIndexer(object):
|
|
|
"escapeChar" = "\\\\"
|
|
|
''' % source['format']
|
|
|
|
|
|
-
|
|
|
- if table_format in ('parquet', 'kudu'):
|
|
|
+ use_temp_table = table_format in ('parquet', 'orc', 'kudu') or is_transactional
|
|
|
+ if use_temp_table: # We'll be using a temp table to load data
|
|
|
if load_data:
|
|
|
table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name
|
|
|
|
|
|
@@ -135,18 +143,33 @@ class SQLIndexer(object):
|
|
|
collection_delimiter = None
|
|
|
map_delimiter = None
|
|
|
|
|
|
- if external or (load_data and table_format in ('parquet', 'kudu')):
|
|
|
+ if external or (load_data and table_format in ('parquet', 'orc', 'kudu')): # We'll use location to load data
|
|
|
if not self.fs.isdir(external_path): # File selected
|
|
|
external_path, external_file_name = self.fs.split(external_path)
|
|
|
|
|
|
if len(self.fs.listdir(external_path)) > 1:
|
|
|
- external_path = external_path + '/%s_table' % external_file_name # If dir not just the file, create data dir and move file there.
|
|
|
+ external_path = external_path + '/%s%s_table' % (external_file_name, str(uuid.uuid4())) # If dir not just the file, create data dir and move file there. Make sure it's unique.
|
|
|
self.fs.mkdir(external_path)
|
|
|
self.fs.rename(source_path, external_path)
|
|
|
-
|
|
|
+ elif load_data: # We'll use load data command
|
|
|
+ parent_path = self.fs.parent_path(source_path)
|
|
|
+ stats = self.fs.stats(parent_path)
|
|
|
+ split = urlparse(source_path)
|
|
|
+ # Only for HDFS, import data and non-external table
|
|
|
+ if split.scheme in ('', 'hdfs') and oct(stats["mode"])[-1] != '7':
|
|
|
+ user_scratch_dir = self.fs.get_home_dir() + '/.scratchdir/%s' % str(uuid.uuid4()) # Make sure it's unique.
|
|
|
+ self.fs.do_as_user(self.user, self.fs.mkdir, user_scratch_dir, 0o0777)
|
|
|
+ self.fs.do_as_user(self.user, self.fs.rename, source['path'], user_scratch_dir)
|
|
|
+ source_path = user_scratch_dir + '/' + source['path'].split('/')[-1]
|
|
|
+
|
|
|
if external_path.lower().startswith("abfs"): #this is to check if its using an ABFS path
|
|
|
- external_path = abfspath(external_path)
|
|
|
-
|
|
|
+ external_path = abfspath(external_path)
|
|
|
+
|
|
|
+ tbl_properties={}
|
|
|
+ if skip_header:
|
|
|
+ tbl_properties['skip.header.line.count'] = '1'
|
|
|
+ tbl_properties['transactional'] = 'false' # The temp table is not transactional, but final table can be if is_transactional. tbl_properties that don't exist in previous versions can safely be added without error
|
|
|
+
|
|
|
sql += django_mako.render_to_string("gen/create_table_statement.mako", {
|
|
|
'table': {
|
|
|
'name': table_name,
|
|
|
@@ -158,10 +181,10 @@ class SQLIndexer(object):
|
|
|
'serde_name': serde_name,
|
|
|
'serde_properties': serde_properties,
|
|
|
'file_format': file_format,
|
|
|
- 'external': external or load_data and table_format in ('parquet', 'kudu'),
|
|
|
+ 'external': external or load_data and table_format in ('parquet', 'orc', 'kudu'),
|
|
|
'path': external_path,
|
|
|
- 'skip_header': skip_header,
|
|
|
'primary_keys': primary_keys if table_format == 'kudu' and not load_data else [],
|
|
|
+ 'tbl_properties': tbl_properties
|
|
|
},
|
|
|
'columns': columns,
|
|
|
'partition_columns': partition_columns,
|
|
|
@@ -180,8 +203,8 @@ class SQLIndexer(object):
|
|
|
db = dbms.get(self.user, query_server=query_server_config)
|
|
|
sql += "\n\n%s;" % db.load_data(database, table_name, form_data, None, generate_ddl_only=True)
|
|
|
|
|
|
- if load_data and table_format in ('parquet', 'kudu'):
|
|
|
- file_format = table_format
|
|
|
+ if load_data and use_temp_table:
|
|
|
+ file_format = 'TextFile' if table_format == 'text' else table_format
|
|
|
if table_format == 'kudu':
|
|
|
columns_list = ['`%s`' % col for col in primary_keys + [col['name'] for col in destination['columns'] if col['name'] not in primary_keys and col['keep']]]
|
|
|
extra_create_properties = """PRIMARY KEY (%(primary_keys)s)
|
|
|
@@ -196,6 +219,9 @@ class SQLIndexer(object):
|
|
|
else:
|
|
|
columns_list = ['*']
|
|
|
extra_create_properties = 'STORED AS %(file_format)s' % {'file_format': file_format}
|
|
|
+ if is_transactional:
|
|
|
+ extra_create_properties += '\nTBLPROPERTIES("transactional"="true", "transactional_properties"="%s")' % default_transactional_type
|
|
|
+
|
|
|
sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`%(comment)s
|
|
|
%(extra_create_properties)s
|
|
|
AS SELECT %(columns_list)s
|
|
|
@@ -212,8 +238,6 @@ class SQLIndexer(object):
|
|
|
'table_name': table_name
|
|
|
}
|
|
|
|
|
|
- editor_type = 'impala' if table_format == 'kudu' else destination['sourceType']
|
|
|
-
|
|
|
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + '?source_type=' + source_type
|
|
|
|
|
|
return make_notebook(
|