|
|
@@ -155,7 +155,7 @@ class TestHiveServerClient():
|
|
|
results=Mock(
|
|
|
columns=[
|
|
|
# Dump of `EXPLAIN SELECT 1`
|
|
|
- Mock(stringVal=Mock(values=['Plan optimized by CBO.', '', 'Stage-0', ' Fetch Operator', '5 limit:-1' ], nulls='')),
|
|
|
+ Mock(stringVal=Mock(values=['Plan optimized by CBO.', '', 'Stage-0', ' Fetch Operator', '5 limit:-1'], nulls='')),
|
|
|
]
|
|
|
),
|
|
|
schema=Mock(
|
|
|
@@ -172,7 +172,7 @@ class TestHiveServerClient():
|
|
|
),
|
|
|
results=Mock(
|
|
|
columns=[
|
|
|
- Mock(stringVal=Mock(values=['Explain', ], nulls='')), # Fake but ok
|
|
|
+ Mock(stringVal=Mock(values=['Explain',], nulls='')), # Fake but ok
|
|
|
]
|
|
|
),
|
|
|
schema=Mock(
|
|
|
@@ -298,9 +298,11 @@ class TestHiveServerClient():
|
|
|
client.get_table(database='database', table_name='table_name')
|
|
|
except QueryServerException as e:
|
|
|
if sys.version_info[0] > 2:
|
|
|
- req_string = "TGetTablesReq(sessionHandle=TSessionHandle(sessionId=THandleIdentifier(guid=b'l\\xc4', secret=b'l\\xc4')), catalogName=None, schemaName='database', tableName='table_name', tableTypes=None)"
|
|
|
+ req_string = ("TGetTablesReq(sessionHandle=TSessionHandle(sessionId=THandleIdentifier(guid=b'l\\xc4', secret=b'l\\xc4')), "
|
|
|
+ "catalogName=None, schemaName='database', tableName='table_name', tableTypes=None)")
|
|
|
else:
|
|
|
- req_string = "TGetTablesReq(schemaName='database', sessionHandle=TSessionHandle(sessionId=THandleIdentifier(secret='1', guid='1')), tableName='table_name', tableTypes=None, catalogName=None)"
|
|
|
+ req_string = ("TGetTablesReq(schemaName='database', sessionHandle=TSessionHandle(sessionId=THandleIdentifier"
|
|
|
+ "(secret='1', guid='1')), tableName='table_name', tableTypes=None, catalogName=None)")
|
|
|
assert_equal(
|
|
|
"Bad status for request %s:\n%s" % (req_string, get_tables_res),
|
|
|
str(e)
|
|
|
@@ -315,9 +317,33 @@ class TestHiveServerTable():
|
|
|
desc_results = Mock(
|
|
|
columns=[
|
|
|
# Dump of `DESCRIBE FORMATTED table`
|
|
|
- Mock(stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', ], nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database:', 'OwnerType:',
|
|
|
+ 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '',
|
|
|
+ '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:',
|
|
|
+ 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'default', 'USER', 'hive',
|
|
|
+ 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL',
|
|
|
+ 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional',
|
|
|
+ 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",'\
|
|
|
+ '\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
]
|
|
|
)
|
|
|
desc_schema = Mock(
|
|
|
@@ -344,36 +370,63 @@ class TestHiveServerTable():
|
|
|
|
|
|
def test_cols_hive_tez(self):
|
|
|
|
|
|
- table_results = Mock()
|
|
|
- table_schema = Mock()
|
|
|
- desc_results = Mock(
|
|
|
- columns=[
|
|
|
- # Dump of `DESCRIBE FORMATTED table`
|
|
|
- Mock(stringVal=Mock(values=['code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database: ', 'OwnerType: ', 'Owner: ', 'CreateTime: ', 'LastAccessTime: ', 'Retention: ', 'Location: ', 'Table Type: ', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library: ', 'InputFormat: ', 'OutputFormat: ', 'Compressed: ', 'Num Buckets: ', 'Bucket Columns: ', 'Sort Columns: ', 'Storage Desc Params:', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['string', 'string', 'int', 'int', 'NULL', 'NULL', 'default ', 'USER ', 'hive ', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN ', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE ', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version ', 'numFiles ', 'numRows ', 'rawDataSize ', 'totalSize ', 'transactional ', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No ', '-1', '[] ', '[] ', 'NULL', 'serialization.format', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['', '', '', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'TRUE', 'insert_only ', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', ], nulls='')),
|
|
|
- ]
|
|
|
- )
|
|
|
- desc_schema = Mock(
|
|
|
- columns=[
|
|
|
- Mock(columnName='col_name'),
|
|
|
- Mock(columnName='data_type'),
|
|
|
- Mock(columnName='comment')
|
|
|
- ]
|
|
|
- )
|
|
|
+ table_results = Mock()
|
|
|
+ table_schema = Mock()
|
|
|
+ desc_results = Mock(
|
|
|
+ columns=[
|
|
|
+ # Dump of `DESCRIBE FORMATTED table`
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database: ',
|
|
|
+ 'OwnerType: ', 'Owner: ', 'CreateTime: ', 'LastAccessTime: ', 'Retention: ',
|
|
|
+ 'Location: ', 'Table Type: ', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '',
|
|
|
+ '# Storage Information', 'SerDe Library: ', 'InputFormat: ', 'OutputFormat: ', 'Compressed: ',
|
|
|
+ 'Num Buckets: ', 'Bucket Columns: ', 'Sort Columns: ', 'Storage Desc Params:',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'default ', 'USER ', 'hive ',
|
|
|
+ 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN ', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE ',
|
|
|
+ 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version ', 'numFiles ', 'numRows ',
|
|
|
+ 'rawDataSize ', 'totalSize ', 'transactional ', 'transactional_properties', 'transient_lastDdlTime',
|
|
|
+ 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No ', '-1', '[] ',
|
|
|
+ '[] ', 'NULL', 'serialization.format',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '', '', '', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":'\
|
|
|
+ '\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2',
|
|
|
+ '1', '822', '3288', '48445', 'TRUE', 'insert_only ', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', '1',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ desc_schema = Mock(
|
|
|
+ columns=[
|
|
|
+ Mock(columnName='col_name'),
|
|
|
+ Mock(columnName='data_type'),
|
|
|
+ Mock(columnName='comment')
|
|
|
+ ]
|
|
|
+ )
|
|
|
|
|
|
- table = HiveServerTable(
|
|
|
- table_results=table_results,
|
|
|
- table_schema=table_schema,
|
|
|
- desc_results=desc_results,
|
|
|
- desc_schema=desc_schema
|
|
|
- )
|
|
|
+ table = HiveServerTable(
|
|
|
+ table_results=table_results,
|
|
|
+ table_schema=table_schema,
|
|
|
+ desc_results=desc_results,
|
|
|
+ desc_schema=desc_schema
|
|
|
+ )
|
|
|
|
|
|
- assert_equal(len(table.cols), 4)
|
|
|
- assert_equal(table.cols[0], {'col_name': 'code', 'data_type': 'string', 'comment': ''})
|
|
|
- assert_equal(table.cols[1], {'col_name': 'description', 'data_type': 'string', 'comment': ''})
|
|
|
- assert_equal(table.cols[2], {'col_name': 'total_emp', 'data_type': 'int', 'comment': ''})
|
|
|
- assert_equal(table.cols[3], {'col_name': 'salary', 'data_type': 'int', 'comment': ''})
|
|
|
+ assert_equal(len(table.cols), 4)
|
|
|
+ assert_equal(table.cols[0], {'col_name': 'code', 'data_type': 'string', 'comment': ''})
|
|
|
+ assert_equal(table.cols[1], {'col_name': 'description', 'data_type': 'string', 'comment': ''})
|
|
|
+ assert_equal(table.cols[2], {'col_name': 'total_emp', 'data_type': 'int', 'comment': ''})
|
|
|
+ assert_equal(table.cols[3], {'col_name': 'salary', 'data_type': 'int', 'comment': ''})
|
|
|
|
|
|
|
|
|
def test_cols_hive_llap_upstream(self):
|
|
|
@@ -383,9 +436,32 @@ class TestHiveServerTable():
|
|
|
desc_results = Mock(
|
|
|
columns=[
|
|
|
# No empty line after headers
|
|
|
- Mock(stringVal=Mock(values=['# col_name', 'code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['data_type', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', ], nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', 'code', 'description', 'total_emp', 'salary', '', '# Detailed Table Information', 'Database:', 'OwnerType:',
|
|
|
+ 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '',
|
|
|
+ '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:',
|
|
|
+ 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019',
|
|
|
+ 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07',
|
|
|
+ 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize',
|
|
|
+ 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",'\
|
|
|
+ '\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
]
|
|
|
)
|
|
|
desc_schema = Mock(
|
|
|
@@ -412,193 +488,321 @@ class TestHiveServerTable():
|
|
|
|
|
|
def test_partition_keys_impala(self):
|
|
|
|
|
|
- table_results = Mock()
|
|
|
- table_schema = Mock()
|
|
|
- desc_results = Mock(
|
|
|
- columns=[
|
|
|
- # Dump of `DESCRIBE FORMATTED table`
|
|
|
- Mock(stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', '', 'date', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'NULL', 'string', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', ], nulls='')),
|
|
|
- ]
|
|
|
- )
|
|
|
- desc_schema = Mock(
|
|
|
- columns=[
|
|
|
- Mock(columnName='col_name'),
|
|
|
- Mock(columnName='data_type'),
|
|
|
- Mock(columnName='comment')
|
|
|
- ]
|
|
|
- )
|
|
|
+ table_results = Mock()
|
|
|
+ table_schema = Mock()
|
|
|
+ desc_results = Mock(
|
|
|
+ columns=[
|
|
|
+ # Dump of `DESCRIBE FORMATTED table`
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', '', 'date', '',
|
|
|
+ '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:',
|
|
|
+ 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information',
|
|
|
+ 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:',
|
|
|
+ 'Storage Desc Params:',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'NULL', 'string', 'NULL', 'NULL',
|
|
|
+ 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL',
|
|
|
+ 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional',
|
|
|
+ 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":'\
|
|
|
+ '{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445',
|
|
|
+ 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ desc_schema = Mock(
|
|
|
+ columns=[
|
|
|
+ Mock(columnName='col_name'),
|
|
|
+ Mock(columnName='data_type'),
|
|
|
+ Mock(columnName='comment')
|
|
|
+ ]
|
|
|
+ )
|
|
|
|
|
|
- table = HiveServerTable(
|
|
|
- table_results=table_results,
|
|
|
- table_schema=table_schema,
|
|
|
- desc_results=desc_results,
|
|
|
- desc_schema=desc_schema
|
|
|
- )
|
|
|
+ table = HiveServerTable(
|
|
|
+ table_results=table_results,
|
|
|
+ table_schema=table_schema,
|
|
|
+ desc_results=desc_results,
|
|
|
+ desc_schema=desc_schema
|
|
|
+ )
|
|
|
|
|
|
- assert_equal(len(table.partition_keys), 1)
|
|
|
- assert_equal(table.partition_keys[0].name, 'date')
|
|
|
- assert_equal(table.partition_keys[0].type, 'string')
|
|
|
- assert_equal(table.partition_keys[0].comment, 'NULL')
|
|
|
+ assert_equal(len(table.partition_keys), 1)
|
|
|
+ assert_equal(table.partition_keys[0].name, 'date')
|
|
|
+ assert_equal(table.partition_keys[0].type, 'string')
|
|
|
+ assert_equal(table.partition_keys[0].comment, 'NULL')
|
|
|
|
|
|
|
|
|
def test_partition_keys_hive(self):
|
|
|
|
|
|
- table_results = Mock()
|
|
|
- table_schema = Mock()
|
|
|
- desc_results = Mock(
|
|
|
- columns=[
|
|
|
- # Dump of `DESCRIBE FORMATTED table`
|
|
|
- # Note: missing blank line below '# Partition Information'
|
|
|
- Mock(stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', ], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', ], nulls='')),
|
|
|
- ]
|
|
|
- )
|
|
|
- desc_schema = Mock(
|
|
|
- columns=[
|
|
|
- Mock(columnName='col_name'),
|
|
|
- Mock(columnName='data_type'),
|
|
|
- Mock(columnName='comment')
|
|
|
- ]
|
|
|
- )
|
|
|
+ table_results = Mock()
|
|
|
+ table_schema = Mock()
|
|
|
+ desc_results = Mock(
|
|
|
+ columns=[
|
|
|
+ # Dump of `DESCRIBE FORMATTED table`
|
|
|
+ # Note: missing blank line below '# Partition Information'
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '',
|
|
|
+ '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:',
|
|
|
+ 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information',
|
|
|
+ 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:',
|
|
|
+ 'Storage Desc Params:',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default',
|
|
|
+ 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL',
|
|
|
+ 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional',
|
|
|
+ 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":'\
|
|
|
+ '\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only',
|
|
|
+ '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1',
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ desc_schema = Mock(
|
|
|
+ columns=[
|
|
|
+ Mock(columnName='col_name'),
|
|
|
+ Mock(columnName='data_type'),
|
|
|
+ Mock(columnName='comment')
|
|
|
+ ]
|
|
|
+ )
|
|
|
|
|
|
- table = HiveServerTable(
|
|
|
- table_results=table_results,
|
|
|
- table_schema=table_schema,
|
|
|
- desc_results=desc_results,
|
|
|
- desc_schema=desc_schema
|
|
|
- )
|
|
|
+ table = HiveServerTable(
|
|
|
+ table_results=table_results,
|
|
|
+ table_schema=table_schema,
|
|
|
+ desc_results=desc_results,
|
|
|
+ desc_schema=desc_schema
|
|
|
+ )
|
|
|
|
|
|
- assert_equal(len(table.partition_keys), 1)
|
|
|
- assert_equal(table.partition_keys[0].name, 'date')
|
|
|
- assert_equal(table.partition_keys[0].type, 'string')
|
|
|
- assert_equal(table.partition_keys[0].comment, '')
|
|
|
+ assert_equal(len(table.partition_keys), 1)
|
|
|
+ assert_equal(table.partition_keys[0].name, 'date')
|
|
|
+ assert_equal(table.partition_keys[0].type, 'string')
|
|
|
+ assert_equal(table.partition_keys[0].comment, '')
|
|
|
|
|
|
|
|
|
def test_single_primary_key_hive(self):
|
|
|
|
|
|
- table_results = Mock()
|
|
|
- table_schema = Mock()
|
|
|
- desc_results = Mock(
|
|
|
- columns=[
|
|
|
- # Dump of `DESCRIBE FORMATTED table`
|
|
|
- Mock(stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', '', '', '# Constraints', '', '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:', ''], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', 'NULL', 'NULL', 'NULL', 'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'NULL'], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL'], nulls='')),
|
|
|
- ]
|
|
|
- )
|
|
|
- desc_schema = Mock(
|
|
|
- columns=[
|
|
|
- Mock(columnName='col_name'),
|
|
|
- Mock(columnName='data_type'),
|
|
|
- Mock(columnName='comment')
|
|
|
- ]
|
|
|
- )
|
|
|
+ table_results = Mock()
|
|
|
+ table_schema = Mock()
|
|
|
+ desc_results = Mock(
|
|
|
+ columns=[
|
|
|
+ # Dump of `DESCRIBE FORMATTED table`
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '',
|
|
|
+ '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:',
|
|
|
+ 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information',
|
|
|
+ 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:',
|
|
|
+ 'Storage Desc Params:', '', '', '# Constraints', '', '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:', ''
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default',
|
|
|
+ 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL',
|
|
|
+ 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional',
|
|
|
+ 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'NULL'
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":'\
|
|
|
+ '\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only',
|
|
|
+ '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL'
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ desc_schema = Mock(
|
|
|
+ columns=[
|
|
|
+ Mock(columnName='col_name'),
|
|
|
+ Mock(columnName='data_type'),
|
|
|
+ Mock(columnName='comment')
|
|
|
+ ]
|
|
|
+ )
|
|
|
|
|
|
- table = HiveServerTable(
|
|
|
- table_results=table_results,
|
|
|
- table_schema=table_schema,
|
|
|
- desc_results=desc_results,
|
|
|
- desc_schema=desc_schema
|
|
|
- )
|
|
|
+ table = HiveServerTable(
|
|
|
+ table_results=table_results,
|
|
|
+ table_schema=table_schema,
|
|
|
+ desc_results=desc_results,
|
|
|
+ desc_schema=desc_schema
|
|
|
+ )
|
|
|
|
|
|
- assert_equal(len(table.primary_keys), 1)
|
|
|
- assert_equal(table.primary_keys[0].name, 'id1')
|
|
|
- assert_equal(table.primary_keys[0].type, 'NULL')
|
|
|
- assert_equal(table.primary_keys[0].comment, 'NULL')
|
|
|
+ assert_equal(len(table.primary_keys), 1)
|
|
|
+ assert_equal(table.primary_keys[0].name, 'id1')
|
|
|
+ assert_equal(table.primary_keys[0].type, 'NULL')
|
|
|
+ assert_equal(table.primary_keys[0].comment, 'NULL')
|
|
|
|
|
|
|
|
|
def test_multi_primary_keys_hive(self):
|
|
|
|
|
|
- table_results = Mock()
|
|
|
- table_schema = Mock()
|
|
|
- desc_results = Mock(
|
|
|
- columns=[
|
|
|
- # Dump of `DESCRIBE FORMATTED table`
|
|
|
- Mock(stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', '', '', '# Constraints', '', '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:', 'Column Name:'], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', 'NULL', 'NULL', 'NULL', 'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'id2 '], nulls='')),
|
|
|
- Mock(stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL'], nulls='')),
|
|
|
- ]
|
|
|
- )
|
|
|
- desc_schema = Mock(
|
|
|
- columns=[
|
|
|
- Mock(columnName='col_name'),
|
|
|
- Mock(columnName='data_type'),
|
|
|
- Mock(columnName='comment')
|
|
|
- ]
|
|
|
- )
|
|
|
+ table_results = Mock()
|
|
|
+ table_schema = Mock()
|
|
|
+ desc_results = Mock(
|
|
|
+ columns=[
|
|
|
+ # Dump of `DESCRIBE FORMATTED table`
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '',
|
|
|
+ '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:',
|
|
|
+ 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information',
|
|
|
+ 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:',
|
|
|
+ 'Storage Desc Params:', '', '', '# Constraints', '', '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:',
|
|
|
+ 'Column Name:'
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default',
|
|
|
+ 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL',
|
|
|
+ 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional',
|
|
|
+ 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'id2 '
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ Mock(stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":'\
|
|
|
+ '\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only',
|
|
|
+ '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL'
|
|
|
+ ],
|
|
|
+ nulls='')),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ desc_schema = Mock(
|
|
|
+ columns=[
|
|
|
+ Mock(columnName='col_name'),
|
|
|
+ Mock(columnName='data_type'),
|
|
|
+ Mock(columnName='comment')
|
|
|
+ ]
|
|
|
+ )
|
|
|
|
|
|
- table = HiveServerTable(
|
|
|
- table_results=table_results,
|
|
|
- table_schema=table_schema,
|
|
|
- desc_results=desc_results,
|
|
|
- desc_schema=desc_schema
|
|
|
- )
|
|
|
+ table = HiveServerTable(
|
|
|
+ table_results=table_results,
|
|
|
+ table_schema=table_schema,
|
|
|
+ desc_results=desc_results,
|
|
|
+ desc_schema=desc_schema
|
|
|
+ )
|
|
|
|
|
|
- assert_equal(len(table.primary_keys), 2)
|
|
|
- assert_equal(table.primary_keys[0].name, 'id1')
|
|
|
- assert_equal(table.primary_keys[0].type, 'NULL')
|
|
|
- assert_equal(table.primary_keys[0].comment, 'NULL')
|
|
|
+ assert_equal(len(table.primary_keys), 2)
|
|
|
+ assert_equal(table.primary_keys[0].name, 'id1')
|
|
|
+ assert_equal(table.primary_keys[0].type, 'NULL')
|
|
|
+ assert_equal(table.primary_keys[0].comment, 'NULL')
|
|
|
|
|
|
- assert_equal(table.primary_keys[1].name, 'id2')
|
|
|
- assert_equal(table.primary_keys[1].type, 'NULL')
|
|
|
- assert_equal(table.primary_keys[1].comment, 'NULL')
|
|
|
+ assert_equal(table.primary_keys[1].name, 'id2')
|
|
|
+ assert_equal(table.primary_keys[1].type, 'NULL')
|
|
|
+ assert_equal(table.primary_keys[1].comment, 'NULL')
|
|
|
|
|
|
|
|
|
def test_foreign_keys_hive(self):
|
|
|
|
|
|
- table_results = Mock()
|
|
|
- table_schema = Mock()
|
|
|
- desc_results = Mock(
|
|
|
- columns=[
|
|
|
- # Dump of `DESCRIBE FORMATTED table`
|
|
|
- Mock(
|
|
|
- stringVal=Mock(values=['# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '', '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:', 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information', 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:', 'Storage Desc Params:', '', '', '# Constraints', '',
|
|
|
- '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:', '',
|
|
|
- '# Foreign Keys', 'Table:', 'Constraint Name:', 'Parent Column Name:default.persons.id', ''
|
|
|
- ],
|
|
|
- nulls=''
|
|
|
- )
|
|
|
- ),
|
|
|
- Mock(
|
|
|
- stringVal=Mock(values=['data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL', 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0', 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL', 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional', 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL', 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format', 'NULL', 'NULL', 'NULL',
|
|
|
- 'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'NULL',
|
|
|
- 'NULL', 'default.businessunit', 'fk', 'Column Name:head', 'NULL'
|
|
|
- ],
|
|
|
- nulls=''
|
|
|
- )
|
|
|
- ),
|
|
|
- Mock(
|
|
|
- stringVal=Mock(values=['comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true', 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
- 'NULL', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
- 'NULL', 'NULL', 'NULL', 'Key Sequence:1', 'NULL',
|
|
|
- ],
|
|
|
- nulls=''
|
|
|
- )
|
|
|
- ),
|
|
|
- ]
|
|
|
- )
|
|
|
- desc_schema = Mock(
|
|
|
- columns=[
|
|
|
- Mock(columnName='col_name'),
|
|
|
- Mock(columnName='data_type'),
|
|
|
- Mock(columnName='comment')
|
|
|
- ]
|
|
|
- )
|
|
|
+ table_results = Mock()
|
|
|
+ table_schema = Mock()
|
|
|
+ desc_results = Mock(
|
|
|
+ columns=[
|
|
|
+ # Dump of `DESCRIBE FORMATTED table`
|
|
|
+ Mock(
|
|
|
+ stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ '# col_name', '', 'code', 'description', 'total_emp', 'salary', '', '# Partition Information', '# col_name', 'date', '',
|
|
|
+ '# Detailed Table Information', 'Database:', 'OwnerType:', 'Owner:', 'CreateTime:', 'LastAccessTime:', 'Retention:',
|
|
|
+ 'Location:', 'Table Type:', 'Table Parameters:', '', '', '', '', '', '', '', '', '', '', '# Storage Information',
|
|
|
+ 'SerDe Library:', 'InputFormat:', 'OutputFormat:', 'Compressed:', 'Num Buckets:', 'Bucket Columns:', 'Sort Columns:',
|
|
|
+ 'Storage Desc Params:', '', '', '# Constraints', '',
|
|
|
+ '# Primary Key', 'Table:', 'Constraint Name:', 'Column Name:', '',
|
|
|
+ '# Foreign Keys', 'Table:', 'Constraint Name:', 'Parent Column Name:default.persons.id', ''
|
|
|
+ ],
|
|
|
+ nulls=''
|
|
|
+ )
|
|
|
+ ),
|
|
|
+ Mock(
|
|
|
+ stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'data_type', 'NULL', 'string', 'string', 'int', 'int', 'NULL', 'NULL', 'data_type', 'string', 'NULL',
|
|
|
+ 'NULL', 'default', 'USER', 'hive', 'Mon Nov 04 07:44:10 PST 2019', 'UNKNOWN', '0',
|
|
|
+ 'hdfs://nightly7x-unsecure-1.vpc.cloudera.com:8020/warehouse/tablespace/managed/hive/sample_07', 'MANAGED_TABLE', 'NULL',
|
|
|
+ 'COLUMN_STATS_ACCURATE', 'bucketing_version', 'numFiles', 'numRows', 'rawDataSize', 'totalSize', 'transactional',
|
|
|
+ 'transactional_properties', 'transient_lastDdlTime', 'NULL', 'NULL',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
|
|
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', 'No', '-1', '[]', '[]', 'NULL', 'serialization.format',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'default.pk', 'pk_165400321_1572980510006_0', 'id1 ', 'NULL',
|
|
|
+ 'NULL', 'default.businessunit', 'fk', 'Column Name:head', 'NULL'
|
|
|
+ ],
|
|
|
+ nulls=''
|
|
|
+ )
|
|
|
+ ),
|
|
|
+ Mock(
|
|
|
+ stringVal=Mock(
|
|
|
+ values=[
|
|
|
+ 'comment', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'comment', '', 'NULL', 'NULL', 'NULL', 'NULL',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"code\":\"true\",'\
|
|
|
+ '\"description\":\"true\",\"salary\":\"true\",\"total_emp\":\"true\"}}', '2', '1', '822', '3288', '48445', 'true',
|
|
|
+ 'insert_only', '1572882268', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', '1',
|
|
|
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'NULL', 'Key Sequence:1',
|
|
|
+ 'NULL',
|
|
|
+ ],
|
|
|
+ nulls=''
|
|
|
+ )
|
|
|
+ ),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ desc_schema = Mock(
|
|
|
+ columns=[
|
|
|
+ Mock(columnName='col_name'),
|
|
|
+ Mock(columnName='data_type'),
|
|
|
+ Mock(columnName='comment')
|
|
|
+ ]
|
|
|
+ )
|
|
|
|
|
|
- table = HiveServerTable(
|
|
|
- table_results=table_results,
|
|
|
- table_schema=table_schema,
|
|
|
- desc_results=desc_results,
|
|
|
- desc_schema=desc_schema
|
|
|
- )
|
|
|
+ table = HiveServerTable(
|
|
|
+ table_results=table_results,
|
|
|
+ table_schema=table_schema,
|
|
|
+ desc_results=desc_results,
|
|
|
+ desc_schema=desc_schema
|
|
|
+ )
|
|
|
|
|
|
- assert_equal(len(table.foreign_keys), 1)
|
|
|
- assert_equal(table.foreign_keys[0].name, 'head') # 'from' column
|
|
|
- assert_equal(table.foreign_keys[0].type, 'default.persons.id') # 'to' column
|
|
|
- assert_equal(table.foreign_keys[0].comment, 'NULL')
|
|
|
+ assert_equal(len(table.foreign_keys), 1)
|
|
|
+ assert_equal(table.foreign_keys[0].name, 'head') # 'from' column
|
|
|
+ assert_equal(table.foreign_keys[0].type, 'default.persons.id') # 'to' column
|
|
|
+ assert_equal(table.foreign_keys[0].comment, 'NULL')
|
|
|
|
|
|
|
|
|
|
|
|
@@ -688,7 +892,7 @@ class TestSessionManagement():
|
|
|
with patch('beeswax.server.hive_server2_lib.thrift_util.get_client') as get_client:
|
|
|
with patch('beeswax.server.hive_server2_lib.HiveServerClient.open_session') as open_session:
|
|
|
with patch('beeswax.server.hive_server2_lib.Session.objects.get_tez_session') as get_tez_session:
|
|
|
- get_tez_session.side_effect=Exception('')
|
|
|
+ get_tez_session.side_effect = Exception('')
|
|
|
open_session.return_value = MagicMock(status_code=0)
|
|
|
fn = MagicMock(return_value=MagicMock(status=MagicMock(statusCode=0)))
|
|
|
req = MagicMock()
|