Explorar el Código

[notebook] Adding hive and impala dialects to their sample queries

Romain Rigaux hace 4 años
padre
commit
7ba88e00d7

+ 8 - 4
apps/beeswax/data/tables_standard.json

@@ -2,17 +2,20 @@
   {
     "data_file": "sample_07.csv",
     "create_sql": "CREATE TABLE `sample_07` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\nROW FORMAT DELIMITED\n  FIELDS TERMINATED BY '\t'\nSTORED AS TextFile\n",
-    "table_name": "sample_07"
+    "table_name": "sample_07",
+    "dialects": ["hive", "impala"]
   },
   {
     "data_file": "sample_08.csv",
     "create_sql": "CREATE TABLE `sample_08` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\nROW FORMAT DELIMITED\n  FIELDS TERMINATED BY '\t'\nSTORED AS TextFile\n",
-    "table_name": "sample_08"
+    "table_name": "sample_08",
+    "dialects": ["hive", "impala"]
   },
   {
     "data_file": "customers.parquet",
     "create_sql": "CREATE TABLE `customers` (\n `id` INT,\n `name` STRING,\n `email_preferences` STRUCT<`email_format`:STRING, `frequency`:STRING, `categories`:STRUCT<`promos`:BOOLEAN, `surveys`:BOOLEAN>>,\n `addresses` MAP<STRING, STRUCT<`street_1`:STRING, `street_2`:STRING, `city`:STRING, `state`:STRING, `zip_code`:STRING>>,\n `orders` ARRAY<STRUCT<`order_id`:STRING, `order_date`:STRING, `items`:ARRAY<STRUCT<`product_id`:INT, `sku`:STRING, `name`:STRING, `price`:DOUBLE, `qty`:INT>>>>)\nSTORED AS PARQUET",
-    "table_name": "customers"
+    "table_name": "customers",
+    "dialects": ["hive", "impala"]
   },
   {
     "partition_files": {
@@ -22,6 +25,7 @@
       "`date`='2015-11-21'": "web_logs_4.csv"
     },
     "create_sql": "CREATE TABLE `web_logs`  (  `_version_` bigint,   `app` string,   `bytes` smallint,   `city` string,   `client_ip` string,   `code` tinyint,   `country_code` string,   `country_code3` string,   `country_name` string,   `device_family` string,   `extension` string,   `latitude` float,   `longitude` float,   `method` string,   `os_family` string,   `os_major` string,   `protocol` string,   `record` string,   `referer` string,   `region_code` bigint,   `request` string,   `subapp` string,   `time` string,   `url` string,   `user_agent` string,   `user_agent_family` string,   `user_agent_major` string,   `id` string)\nPARTITIONED BY (  `date` string  )\nROW FORMAT DELIMITED \n FIELDS TERMINATED BY ','",
-    "table_name": "web_logs"
+    "table_name": "web_logs",
+    "dialects": ["hive", "impala"]
   }
 ]

+ 8 - 4
apps/beeswax/data/tables_transactional.json

@@ -3,19 +3,22 @@
     "data_file": "sample_07.csv",
     "create_sql": "CREATE TABLE `sample_07` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\nSTORED AS parquet\nTBLPROPERTIES ('transactional_properties'='insert_only')\n",
     "table_name": "sample_07",
-    "transactional": true
+    "transactional": true,
+    "dialects": ["hive", "impala"]
   },
   {
     "data_file": "sample_08.csv",
     "create_sql": "CREATE TABLE `sample_08` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\nSTORED AS parquet\nTBLPROPERTIES ('transactional_properties'='insert_only')\n",
     "table_name": "sample_08",
-    "transactional": true
+    "transactional": true,
+    "dialects": ["hive", "impala"]
   },
   {
     "data_file": "customers.parquet",
     "create_sql": "CREATE TABLE `customers` (\n `id` INT,\n `name` STRING,\n `email_preferences` STRUCT<`email_format`:STRING, `frequency`:STRING, `categories`:STRUCT<`promos`:BOOLEAN, `surveys`:BOOLEAN>>,\n `addresses` MAP<STRING, STRUCT<`street_1`:STRING, `street_2`:STRING, `city`:STRING, `state`:STRING, `zip_code`:STRING>>,\n `orders` ARRAY<STRUCT<`order_id`:STRING, `order_date`:STRING, `items`:ARRAY<STRUCT<`product_id`:INT, `sku`:STRING, `name`:STRING, `price`:DOUBLE, `qty`:INT>>>>)\nSTORED AS PARQUET\nTBLPROPERTIES ('transactional'='false')\n",
     "table_name": "customers",
-    "transactional": false
+    "transactional": false,
+    "dialects": ["hive", "impala"]
   },
   {
     "partition_files": {
@@ -27,6 +30,7 @@
     "create_sql": "CREATE TABLE `web_logs`  (  `_version_` bigint,   `app` string,   `bytes` int,   `city` string,   `client_ip` string,   `code` smallint,   `country_code` string,   `country_code3` string,   `country_name` string,   `device_family` string,   `extension` string,   `latitude` float,   `longitude` float,   `method` string,   `os_family` string,   `os_major` string,   `protocol` string,   `record` string,   `referer` string,   `region_code` string,   `request` string,   `subapp` string,   `time` string,   `url` string,   `user_agent` string,   `user_agent_family` string,   `user_agent_major` string,   `id` string)\nPARTITIONED BY (  `date` string  )\nSTORED AS parquet\nTBLPROPERTIES ('transactional_properties'='insert_only')",
     "table_name": "web_logs",
     "columns": [{"name": "_version_", "type": "bigint"}, {"name": "app", "type": "string"}, {"name": "bytes", "type": "int"}, {"name": "city", "type": "string"}, {"name": "client_ip", "type": "string"}, {"name": "code", "type": "smallint"}, {"name": "country_code", "type": "string"}, {"name": "country_code3", "type": "string"}, {"name": "country_name", "type": "string"}, {"name": "device_family", "type": "string"}, {"name": "extension", "type": "string"}, {"name": "latitude", "type": "float"}, {"name": "longitude", "type": "float"}, {"name": "method", "type": "string"}, {"name": "os_family", "type": "string"}, {"name": "os_major", "type": "string"}, {"name": "protocol", "type": "string"}, {"name": "record", "type": "string"}, {"name": "referer", "type": "string"}, {"name": "region_code", "type": "string"}, {"name": "request", "type": "string"}, {"name": "subapp", "type": "string"}, {"name": "time", "type": "string"}, {"name": "url", "type": "string"}, {"name": "user_agent", "type": "string"}, {"name": "user_agent_family", "type": "string"}, {"name": "user_agent_major", "type": "string"}, {"name": "id", "type": "string"}, {"name": "date", "type": "string"}],
-    "transactional": true
+    "transactional": true,
+    "dialects": ["hive", "impala"]
   }
 ]

+ 1 - 0
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples.py

@@ -75,6 +75,7 @@ class Command(BaseCommand):
     )
     exception = None
 
+    LOG.info('Installing %s samples %s in DB %s owned by user %s' % (dialect, tables, db_name, user))
 
     self.successes = []
     self.errors = []

+ 2 - 0
desktop/libs/notebook/src/notebook/management/commands/samples_setup.py

@@ -47,6 +47,8 @@ class Command(BaseCommand):
     )
 
   def handle(self, *args, **options):
+    LOG.info('Installing %s examples as %s' % (options.get('dialect') or 'all', options['username']))
+
     user = User.objects.get(username=options['username'])
     dialect = options.get('dialect')
 

+ 31 - 33
desktop/libs/notebook/src/notebook/views_tests.py

@@ -48,40 +48,38 @@ class TestInstallExamples():
       with patch('notebook.views.get_interpreter') as get_interpreter:
         with patch('notebook.connectors.base.get_ordered_interpreters') as get_ordered_interpreters:
           with patch('beeswax.management.commands.beeswax_install_examples.make_notebook') as make_notebook:
-            with patch('beeswax.management.commands.beeswax_install_examples.cluster.get_hdfs') as get_hdfs:
-              get_hdfs.return_value = None
 
-              ConnectorObjects.get = Mock(
-                return_value=Connector(
-                  id=10,
-                  name='MySql',
-                  dialect='mysql',
-                ),
-              )
-
-              get_interpreter.return_value = {'type': 10, 'dialect': 'mysql'}
-              get_ordered_interpreters.return_value = [
-                {
-                  'name': 'MySql',
-                  'type': 10,
-                  'dialect': 'mysql',
-                  'interface': 'sqlalchemy',
-                }
-              ]
-
-              resp = self.client.post(reverse('notebook:install_examples'), {'db_name': 'default', 'dialect': 'mysql'})
-              data = json.loads(resp.content)
-
-              assert_equal(0, data['status'], data)
-              assert_equal(
-                  'Query Sample: Salary Analysis mysql installed. '
-                  'Table default.employe_sample installed.',
-                  data['message'],
-                  data
-              )
-              assert_equal('', data['errorMessage'], data)
-
-              make_notebook.assert_called()
+            ConnectorObjects.get = Mock(
+              return_value=Connector(
+                id=10,
+                name='MySql',
+                dialect='mysql',
+              ),
+            )
+
+            get_interpreter.return_value = {'type': 10, 'dialect': 'mysql'}
+            get_ordered_interpreters.return_value = [
+              {
+                'name': 'MySql',
+                'type': 10,
+                'dialect': 'mysql',
+                'interface': 'sqlalchemy',
+              }
+            ]
+
+            resp = self.client.post(reverse('notebook:install_examples'), {'db_name': 'default', 'dialect': 'mysql'})
+            data = json.loads(resp.content)
+
+            assert_equal(0, data['status'], data)
+            assert_equal(
+                'Query Sample: Salary Analysis mysql installed. '
+                'Table default.employe_sample installed.',
+                data['message'],
+                data
+            )
+            assert_equal('', data['errorMessage'], data)
+
+            make_notebook.assert_called()
 
 
   def test_install_via_load_hive(self):