Browse Source

HUE-9150 [importer] First Mock test for TestSQLIndexer with file_to_csv

Romain 5 years ago
parent
commit
4a996b591f

+ 3 - 1
apps/beeswax/src/beeswax/query_history.py

@@ -36,12 +36,14 @@ from desktop.lib import django_mako
 
 from useradmin.models import install_sample_user
 
+
 LOG = logging.getLogger(__name__)
 
 QUERY_HISTORY_CACHE_MAX_USER_COUNT = 10
 QUERY_HISTORY_CACHE_MAX_LENGTH_PER_USER = 25
 HAS_CREATED_TABLE = False
 
+
 class QueryHistory(object):
   def __init__(self, max_user=10, max_history_per_user=25):
     self.max_user=max_user
@@ -280,4 +282,4 @@ def _groupby(by_user, data):
         item[8] += row[8]
 
   results.sort(key=lambda result: result[1][0], reverse=True)
-  return results
+  return results

+ 3 - 1
desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -22,6 +22,8 @@ import sys
 import urllib.request, urllib.error
 import uuid
 
+from collections import OrderedDict
+
 from django.urls import reverse
 from django.utils.translation import ugettext as _
 
@@ -166,7 +168,7 @@ class SQLIndexer(object):
     if external_path.lower().startswith("abfs"): #this is to check if its using an ABFS path
       external_path = abfspath(external_path)
 
-    tbl_properties={}
+    tbl_properties = OrderedDict()
     if skip_header:
       tbl_properties['skip.header.line.count'] = '1'
     tbl_properties['transactional'] = 'false' # The temp table is not transactional, but final table can be if is_transactional. tbl_properties that don't exist in previous versions can safely be added without error

+ 74 - 0
desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -27,11 +27,85 @@ from useradmin.models import User
 
 from indexer.indexers.sql import SQLIndexer
 
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock, MagicMock
+else:
+  from mock import patch, Mock, MagicMock
+
+
 table_properties_py2 = '"transactional" = "false", "skip.header.line.count" = "1"'
 table_properties_py3 = '"skip.header.line.count" = "1", "transactional" = "false"'
 is_py3 = sys.version_info[0] > 2
 
 
+class TestSQLIndexer(object):
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="empty", recreate=True, is_superuser=False)
+    self.user = User.objects.get(username="test")
+
+
+  def test_create_table_from_a_file_to_csv(self):
+    fs = Mock(
+      stats=Mock(return_value={'mode': 0o0777})
+    )
+
+    def source_dict(key):
+      return {
+        'path': 'hdfs:///path/data.csv',
+        'format': {'quoteChar': '"', 'fieldSeparator': ','},
+        'sampleCols': [{u'operations': [], u'comment': u'', u'name': u'customers.id'}],
+        'sourceType': 'hive'
+      }.get(key, Mock())
+    source = MagicMock()
+    source.__getitem__.side_effect = source_dict
+
+    def destination_dict(key):
+      return {
+        'name': 'default.export_table',
+        'tableFormat': 'csv',
+        'importData': True,
+        'nonDefaultLocation': '/user/hue/customer_stats.csv',
+        'columns': [{'name': 'id', 'type': 'int'}],
+        'partitionColumns': [{'name': 'day', 'type': 'date', 'partitionValue': '20200101'}],
+        'description': 'No comment!',
+        'sourceType': 'hive-1'
+      }.get(key, Mock())
+    destination = MagicMock()
+    destination.__getitem__.side_effect = destination_dict
+
+    with patch('notebook.models.get_interpreter') as get_interpreter:
+      notebook = SQLIndexer(user=self.user, fs=fs).create_table_from_a_file(source, destination)
+
+    assert_equal(
+      [statement.strip() for statement in u'''DROP TABLE IF EXISTS `default`.`hue__tmp_export_table`;
+
+CREATE TABLE `default`.`hue__tmp_export_table`
+(
+  `id` int ) COMMENT "No comment!"
+PARTITIONED BY (
+  `day` date )
+ROW FORMAT   SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+  WITH SERDEPROPERTIES ("separatorChar" = ",",
+    "quoteChar"     = """,
+    "escapeChar"    = "\\\\"
+    )
+  STORED AS TextFile TBLPROPERTIES("skip.header.line.count" = "1", "transactional" = "false")
+;
+
+LOAD DATA INPATH 'hdfs:///path/data.csv' INTO TABLE `default`.`hue__tmp_export_table` PARTITION (day='20200101');
+
+CREATE TABLE `default`.`export_table` COMMENT "No comment!"
+        STORED AS csv
+TBLPROPERTIES("transactional"="true", "transactional_properties"="insert_only")
+        AS SELECT *
+        FROM `default`.`hue__tmp_export_table`;
+
+DROP TABLE IF EXISTS `default`.`hue__tmp_export_table`;'''.split(';')],
+    [statement.strip() for statement in notebook.get_data()['snippets'][0]['statement_raw'].split(';')]
+  )
+
+
 class MockRequest(object):
   def __init__(self, fs=None, user=None):
     self.fs = fs if fs is not None else MockFs()