|
|
@@ -45,36 +45,36 @@ class TestStandardTables():
|
|
|
|
|
|
|
|
|
def test_install_queries_mysql(self):
|
|
|
- design_dict = {
|
|
|
- "name": "TestStandardTables Query",
|
|
|
- "desc": "Small query",
|
|
|
- "type": "2",
|
|
|
- "dialects": ["postgresql", "mysql", "presto"],
|
|
|
- "data": {
|
|
|
- "query": {
|
|
|
- "query": "SELECT 1",
|
|
|
- "type": 0,
|
|
|
- "email_notify": False,
|
|
|
- "is_parameterized": False,
|
|
|
- "database": "default"
|
|
|
- },
|
|
|
- "functions": [],
|
|
|
- "VERSION": "0.4.1",
|
|
|
- "file_resources": [],
|
|
|
- "settings": []
|
|
|
- }
|
|
|
+ design_dict = {
|
|
|
+ "name": "TestStandardTables Query",
|
|
|
+ "desc": "Small query",
|
|
|
+ "type": "2",
|
|
|
+ "dialects": ["postgresql", "mysql", "presto"],
|
|
|
+ "data": {
|
|
|
+ "query": {
|
|
|
+ "query": "SELECT 1",
|
|
|
+ "type": 0,
|
|
|
+ "email_notify": False,
|
|
|
+ "is_parameterized": False,
|
|
|
+ "database": "default"
|
|
|
+ },
|
|
|
+ "functions": [],
|
|
|
+ "VERSION": "0.4.1",
|
|
|
+ "file_resources": [],
|
|
|
+ "settings": []
|
|
|
}
|
|
|
- interpreter = {'type': 'mysql'}
|
|
|
+ }
|
|
|
+ interpreter = {'type': 'mysql'}
|
|
|
|
|
|
- design = SampleQuery(design_dict)
|
|
|
- assert_false(Document2.objects.filter(name='TestStandardTables Query').exists())
|
|
|
+ design = SampleQuery(design_dict)
|
|
|
+ assert_false(Document2.objects.filter(name='TestStandardTables Query').exists())
|
|
|
|
|
|
- with patch('notebook.models.get_interpreter') as get_interpreter:
|
|
|
- design.install(django_user=self.user, interpreter=interpreter)
|
|
|
+ with patch('notebook.models.get_interpreter') as get_interpreter:
|
|
|
+ design.install(django_user=self.user, interpreter=interpreter)
|
|
|
|
|
|
- assert_true(Document2.objects.filter(name='TestStandardTables Query').exists())
|
|
|
- query = Document2.objects.filter(name='TestStandardTables Query').get()
|
|
|
- assert_equal('query-mysql', query.type)
|
|
|
+ assert_true(Document2.objects.filter(name='TestStandardTables Query').exists())
|
|
|
+ query = Document2.objects.filter(name='TestStandardTables Query').get()
|
|
|
+ assert_equal('query-mysql', query.type)
|
|
|
|
|
|
|
|
|
class TestHiveServer2():
|
|
|
@@ -84,41 +84,43 @@ class TestHiveServer2():
|
|
|
self.user = User.objects.get(username="test")
|
|
|
|
|
|
def test_install_queries(self):
|
|
|
- design_dict = {
|
|
|
- "name": "TestBeswaxHiveTables Query",
|
|
|
- "desc": "Small query",
|
|
|
- "type": "0",
|
|
|
- "data": {
|
|
|
- "query": {
|
|
|
- "query": "SELECT 1",
|
|
|
- "type": 0,
|
|
|
- "email_notify": False,
|
|
|
- "is_parameterized": False,
|
|
|
- "database": "default"
|
|
|
- },
|
|
|
- "functions": [],
|
|
|
- "VERSION": "0.4.1",
|
|
|
- "file_resources": [],
|
|
|
- "settings": []
|
|
|
- }
|
|
|
+ design_dict = {
|
|
|
+ "name": "TestBeswaxHiveTables Query",
|
|
|
+ "desc": "Small query",
|
|
|
+ "type": "0",
|
|
|
+ "data": {
|
|
|
+ "query": {
|
|
|
+ "query": "SELECT 1",
|
|
|
+ "type": 0,
|
|
|
+ "email_notify": False,
|
|
|
+ "is_parameterized": False,
|
|
|
+ "database": "default"
|
|
|
+ },
|
|
|
+ "functions": [],
|
|
|
+ "VERSION": "0.4.1",
|
|
|
+ "file_resources": [],
|
|
|
+ "settings": []
|
|
|
}
|
|
|
- interpreter = {'type': 'hive'}
|
|
|
+ }
|
|
|
+ interpreter = {'type': 'hive'}
|
|
|
|
|
|
- design = SampleQuery(design_dict)
|
|
|
- assert_false(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
|
|
|
+ design = SampleQuery(design_dict)
|
|
|
+ assert_false(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
|
|
|
|
|
|
- with patch('notebook.models.get_interpreter') as get_interpreter:
|
|
|
- design.install(django_user=self.user, interpreter=interpreter)
|
|
|
+ with patch('notebook.models.get_interpreter') as get_interpreter:
|
|
|
+ design.install(django_user=self.user, interpreter=interpreter)
|
|
|
|
|
|
- assert_true(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
|
|
|
- query = Document2.objects.filter(name='TestBeswaxHiveTables Query').get()
|
|
|
- assert_equal('query-hive', query.type)
|
|
|
+ assert_true(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
|
|
|
+ query = Document2.objects.filter(name='TestBeswaxHiveTables Query').get()
|
|
|
+ assert_equal('query-hive', query.type)
|
|
|
|
|
|
|
|
|
def test_create_table_load_data_but_no_fs(self):
|
|
|
- table_data = {
|
|
|
+ table_data = {
|
|
|
"data_file": "sample_07.csv",
|
|
|
- "create_sql": "CREATE TABLE `sample_07` (\n `code` string ,\n `description` string ,\n `total_emp` int ,\n `salary` int )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n",
|
|
|
+ "create_sql":
|
|
|
+ """CREATE TABLE `sample_07` (\n `code` string ,\n `description` string ,\n `total_emp` int ,\n `salary` int )"""
|
|
|
+ """\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n""",
|
|
|
"table_name": "sample_07",
|
|
|
}
|
|
|
|
|
|
@@ -140,9 +142,11 @@ class TestTransactionalTables():
|
|
|
|
|
|
|
|
|
def test_load_sample_07_with_concurrency_support(self):
|
|
|
- table_data = {
|
|
|
+ table_data = {
|
|
|
"data_file": "sample_07.csv",
|
|
|
- "create_sql": "CREATE TABLE `sample_07` (\n `code` string ,\n `description` string ,\n `total_emp` int ,\n `salary` int )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n",
|
|
|
+ "create_sql":
|
|
|
+ """CREATE TABLE `sample_07` (\n `code` string ,\n `description` string ,\n `total_emp` int ,\n `salary` int )\n"""
|
|
|
+ """STORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n""",
|
|
|
"table_name": "sample_07",
|
|
|
"transactional": True
|
|
|
}
|
|
|
@@ -164,9 +168,29 @@ class TestTransactionalTables():
|
|
|
"`date`='2015-11-20'": "web_logs_3.csv",
|
|
|
"`date`='2015-11-21'": "web_logs_4.csv"
|
|
|
},
|
|
|
- "create_sql": "CREATE TABLE `web_logs` ( `_version_` bigint, `app` string, `bytes` int, `city` string, `client_ip` string, `code` smallint, `country_code` string, `country_code3` string, `country_name` string, `device_family` string, `extension` string, `latitude` float, `longitude` float, `method` string, `os_family` string, `os_major` string, `protocol` string, `record` string, `referer` string, `region_code` string, `request` string, `subapp` string, `time` string, `url` string, `user_agent` string, `user_agent_family` string, `user_agent_major` string, `id` string)\nPARTITIONED BY ( `date` string )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')",
|
|
|
+ "create_sql":
|
|
|
+ """CREATE TABLE `web_logs` ( `_version_` bigint, `app` string, `bytes` int, `city` string, `client_ip` string, """
|
|
|
+ """`code` smallint, `country_code` string, `country_code3` string, `country_name` string, `device_family` string, """
|
|
|
+ """ `extension` string, `latitude` float, `longitude` float, `method` string, `os_family` string, `os_major` string,"""
|
|
|
+ """ `protocol` string, `record` string, `referer` string, `region_code` string, `request` string, `subapp` string,"""
|
|
|
+ """ `time` string, `url` string, `user_agent` string, `user_agent_family` string, `user_agent_major` string,"""
|
|
|
+ """ `id` string)\nPARTITIONED BY ( `date` string )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true',"""
|
|
|
+ """ 'transactional_properties'='insert_only')""",
|
|
|
"table_name": "web_logs",
|
|
|
- "columns": [{"name": "_version_", "type": "bigint"}, {"name": "app", "type": "string"}, {"name": "bytes", "type": "int"}, {"name": "city", "type": "string"}, {"name": "client_ip", "type": "string"}, {"name": "code", "type": "smallint"}, {"name": "country_code", "type": "string"}, {"name": "country_code3", "type": "string"}, {"name": "country_name", "type": "string"}, {"name": "device_family", "type": "string"}, {"name": "extension", "type": "string"}, {"name": "latitude", "type": "float"}, {"name": "longitude", "type": "float"}, {"name": "method", "type": "string"}, {"name": "os_family", "type": "string"}, {"name": "os_major", "type": "string"}, {"name": "protocol", "type": "string"}, {"name": "record", "type": "string"}, {"name": "referer", "type": "string"}, {"name": "region_code", "type": "string"}, {"name": "request", "type": "string"}, {"name": "subapp", "type": "string"}, {"name": "time", "type": "string"}, {"name": "url", "type": "string"}, {"name": "user_agent", "type": "string"}, {"name": "user_agent_family", "type": "string"}, {"name": "user_agent_major", "type": "string"}, {"name": "id", "type": "string"}, {"name": "date", "type": "string"}],
|
|
|
+ "columns": [
|
|
|
+ {"name": "_version_", "type": "bigint"}, {"name": "app", "type": "string"}, {"name": "bytes", "type": "int"},
|
|
|
+ {"name": "city", "type": "string"}, {"name": "client_ip", "type": "string"}, {"name": "code", "type": "smallint"},
|
|
|
+ {"name": "country_code", "type": "string"}, {"name": "country_code3", "type": "string"},
|
|
|
+ {"name": "country_name", "type": "string"}, {"name": "device_family", "type": "string"},
|
|
|
+ {"name": "extension", "type": "string"}, {"name": "latitude", "type": "float"},
|
|
|
+ {"name": "longitude", "type": "float"}, {"name": "method", "type": "string"},
|
|
|
+ {"name": "os_family", "type": "string"}, {"name": "os_major", "type": "string"},
|
|
|
+ {"name": "protocol", "type": "string"}, {"name": "record", "type": "string"},
|
|
|
+ {"name": "referer", "type": "string"}, {"name": "region_code", "type": "string"},
|
|
|
+ {"name": "request", "type": "string"}, {"name": "subapp", "type": "string"},
|
|
|
+ {"name": "time", "type": "string"}, {"name": "url", "type": "string"},
|
|
|
+ {"name": "user_agent", "type": "string"}, {"name": "user_agent_family", "type": "string"},
|
|
|
+ {"name": "user_agent_major", "type": "string"}, {"name": "id", "type": "string"}, {"name": "date", "type": "string"}],
|
|
|
"transactional": True
|
|
|
}
|
|
|
|
|
|
@@ -180,9 +204,11 @@ class TestTransactionalTables():
|
|
|
|
|
|
|
|
|
def test_create_phoenix_table(self):
|
|
|
- table_data = {
|
|
|
+ table_data = {
|
|
|
"data_file": "./tables/us_population.csv",
|
|
|
- "create_sql": "CREATE TABLE IF NOT EXISTS us_population (\n state CHAR(2) NOT NULL,\n city VARCHAR NOT NULL,\n population BIGINT\n CONSTRAINT my_pk PRIMARY KEY (state, city)\n)\n",
|
|
|
+ "create_sql":
|
|
|
+ """CREATE TABLE IF NOT EXISTS us_population (\n state CHAR(2) NOT NULL,\n city VARCHAR NOT NULL,\n """
|
|
|
+ """population BIGINT\n CONSTRAINT my_pk PRIMARY KEY (state, city)\n)\n""",
|
|
|
"insert_sql": "UPSERT INTO us_population VALUES %(values)s",
|
|
|
"table_name": "us_population",
|
|
|
"dialects": ["phoenix"],
|