Browse Source

[importer] Use full abfs path in load data SQL (#1818)

Ying Chen 4 years ago
parent
commit
b1962d5449

+ 5 - 1
apps/beeswax/src/beeswax/server/dbms.py

@@ -38,6 +38,7 @@ from desktop.settings import CACHES_HIVE_DISCOVERY_KEY
 from indexer.file_format import HiveFormat
 from libzookeeper import conf as libzookeeper_conf
 
+from azure.abfs import abfspath
 from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_SERVER_HOST, HIVE_HTTP_THRIFT_PORT, HIVE_METASTORE_HOST, \
     HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
     AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
@@ -806,7 +807,10 @@ class HiveServer2Dbms(object):
 
   def load_data(self, database, table, form_data, design, generate_ddl_only=False):
     hql = "LOAD DATA INPATH"
-    hql += " '%(path)s'" % form_data
+    source_path = "%(path)s" % form_data
+    if source_path.lower().startswith("abfs"): #this is to check if its using an ABFS path
+      source_path = abfspath(source_path)
+    hql += " '%s'" % source_path
     if form_data['overwrite']:
       hql += " OVERWRITE"
     hql += " INTO TABLE "

+ 24 - 0
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -25,6 +25,8 @@ from nose.tools import assert_equal, assert_true
 from desktop.lib.django_test_util import make_logged_in_client
 from useradmin.models import User
 
+from azure.conf import ABFS_CLUSTERS
+from beeswax.server import dbms
 from indexer.indexers.sql import SQLIndexer
 
 
@@ -579,3 +581,25 @@ def test_generate_create_empty_kudu_table():
   `vrfcn_city_lon` double , PRIMARY KEY (acct_client)
 )   STORED AS kudu TBLPROPERTIES("transactional" = "false")
 ;''' in sql, sql)
+
+
+def test_create_ddl_with_abfs():
+  finish = ABFS_CLUSTERS.set_for_testing(
+    {
+      'default': {
+        'fs_defaultfs': 'abfs://my-data@yingstorage.dfs.core.windows.net',
+        'webhdfs_url': 'https://yingstorage.dfs.core.windows.net'
+      }
+    }
+  )
+
+  form_data = {'path': u'abfs://my-data/test_data/cars.csv', 'partition_columns': [], 'overwrite': False}
+  sql = ''
+  request = MockRequest(fs=MockFs())
+  query_server_config = dbms.get_query_server_config(name='impala')
+  db = dbms.get(request.user, query_server=query_server_config)
+  try:
+    sql = "\n\n%s;" % db.load_data('default', 'cars', form_data, None, generate_ddl_only=True)
+  finally:
+    finish()
+  assert_true(u"\'abfs://my-data@yingstorage.dfs.core.windows.net/test_data/cars.csv\'" in sql)