Sfoglia il codice sorgente

[notebook] Install all samples command

The goal is to support any SQL dialect and provide CREATE/INSERT
tables with also saved sample queries.

Add one iteration on the decoupling of example setups from hive
by not relying on has_concurrency() which is Hive specific only.

Still more to go but it is getting simpler/refactored with more tests.
Romain Rigaux 5 anni fa
parent
commit
bd46cdbcda

+ 6 - 4
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples.py

@@ -29,7 +29,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.conf import USE_NEW_EDITOR
 from desktop.models import Directory, Document, Document2, Document2Permission
 from hadoop import cluster
-from notebook.models import import_saved_beeswax_query, make_notebook
+from notebook.models import import_saved_beeswax_query, make_notebook, MockRequest
 from useradmin.models import get_default_user_group, install_sample_user, User
 
 from beeswax.design import hql_query
@@ -63,11 +63,12 @@ class Command(BaseCommand):
       db_name = options.get('db_name', 'default')
       interpreter = options.get('interpreter')  # Only when connectors are enabled. Later will deprecate `dialect`.
       user = options['user']
-      request = options['request']
+      request = options.get('request', MockRequest(user=user))
       self.queries = options.get('queries')
-      self.tables = options.get('tables')
+      self.tables = options.get('tables')  # Optional whitelist of table names
 
-    tables = 'tables.json' if dialect not in ('hive', 'impala') else (
+    tables = \
+        'tables.json' if dialect not in ('hive', 'impala') else (
         'tables_transactional.json' if has_concurrency_support() else
         'tables_standard.json'
     )
@@ -81,6 +82,7 @@ class Command(BaseCommand):
       self.install_queries(sample_user, dialect, interpreter=interpreter)
       self.install_tables(user, dialect, db_name, tables, interpreter=interpreter, request=request)
     except Exception as ex:
+      LOG.exception('Dialect %s sample install' % dialect)
       exception = ex
 
     if exception is not None:

+ 85 - 59
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples_tests.py

@@ -45,36 +45,36 @@ class TestStandardTables():
 
 
   def test_install_queries_mysql(self):
-      design_dict = {
-        "name": "TestStandardTables Query",
-        "desc": "Small query",
-        "type": "2",
-        "dialects": ["postgresql", "mysql", "presto"],
-        "data": {
-          "query": {
-            "query": "SELECT 1",
-            "type": 0,
-            "email_notify": False,
-            "is_parameterized": False,
-            "database": "default"
-          },
-          "functions": [],
-          "VERSION": "0.4.1",
-          "file_resources": [],
-          "settings": []
-        }
+    design_dict = {
+      "name": "TestStandardTables Query",
+      "desc": "Small query",
+      "type": "2",
+      "dialects": ["postgresql", "mysql", "presto"],
+      "data": {
+        "query": {
+          "query": "SELECT 1",
+          "type": 0,
+          "email_notify": False,
+          "is_parameterized": False,
+          "database": "default"
+        },
+        "functions": [],
+        "VERSION": "0.4.1",
+        "file_resources": [],
+        "settings": []
       }
-      interpreter = {'type': 'mysql'}
+    }
+    interpreter = {'type': 'mysql'}
 
-      design = SampleQuery(design_dict)
-      assert_false(Document2.objects.filter(name='TestStandardTables Query').exists())
+    design = SampleQuery(design_dict)
+    assert_false(Document2.objects.filter(name='TestStandardTables Query').exists())
 
-      with patch('notebook.models.get_interpreter') as get_interpreter:
-        design.install(django_user=self.user, interpreter=interpreter)
+    with patch('notebook.models.get_interpreter') as get_interpreter:
+      design.install(django_user=self.user, interpreter=interpreter)
 
-        assert_true(Document2.objects.filter(name='TestStandardTables Query').exists())
-        query = Document2.objects.filter(name='TestStandardTables Query').get()
-        assert_equal('query-mysql', query.type)
+      assert_true(Document2.objects.filter(name='TestStandardTables Query').exists())
+      query = Document2.objects.filter(name='TestStandardTables Query').get()
+      assert_equal('query-mysql', query.type)
 
 
 class TestHiveServer2():
@@ -84,41 +84,43 @@ class TestHiveServer2():
     self.user = User.objects.get(username="test")
 
   def test_install_queries(self):
-      design_dict = {
-        "name": "TestBeswaxHiveTables Query",
-        "desc": "Small query",
-        "type": "0",
-        "data": {
-          "query": {
-            "query": "SELECT 1",
-            "type": 0,
-            "email_notify": False,
-            "is_parameterized": False,
-            "database": "default"
-          },
-          "functions": [],
-          "VERSION": "0.4.1",
-          "file_resources": [],
-          "settings": []
-        }
+    design_dict = {
+      "name": "TestBeswaxHiveTables Query",
+      "desc": "Small query",
+      "type": "0",
+      "data": {
+        "query": {
+          "query": "SELECT 1",
+          "type": 0,
+          "email_notify": False,
+          "is_parameterized": False,
+          "database": "default"
+        },
+        "functions": [],
+        "VERSION": "0.4.1",
+        "file_resources": [],
+        "settings": []
       }
-      interpreter = {'type': 'hive'}
+    }
+    interpreter = {'type': 'hive'}
 
-      design = SampleQuery(design_dict)
-      assert_false(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
+    design = SampleQuery(design_dict)
+    assert_false(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
 
-      with patch('notebook.models.get_interpreter') as get_interpreter:
-        design.install(django_user=self.user, interpreter=interpreter)
+    with patch('notebook.models.get_interpreter') as get_interpreter:
+      design.install(django_user=self.user, interpreter=interpreter)
 
-        assert_true(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
-        query = Document2.objects.filter(name='TestBeswaxHiveTables Query').get()
-        assert_equal('query-hive', query.type)
+      assert_true(Document2.objects.filter(name='TestBeswaxHiveTables Query').exists())
+      query = Document2.objects.filter(name='TestBeswaxHiveTables Query').get()
+      assert_equal('query-hive', query.type)
 
 
   def test_create_table_load_data_but_no_fs(self):
-    table_data =   {
+    table_data = {
       "data_file": "sample_07.csv",
-      "create_sql": "CREATE TABLE `sample_07` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n",
+      "create_sql":
+        """CREATE TABLE `sample_07` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )"""
+        """\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n""",
       "table_name": "sample_07",
     }
 
@@ -140,9 +142,11 @@ class TestTransactionalTables():
 
 
   def test_load_sample_07_with_concurrency_support(self):
-    table_data =   {
+    table_data = {
       "data_file": "sample_07.csv",
-      "create_sql": "CREATE TABLE `sample_07` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n",
+      "create_sql":
+        """CREATE TABLE `sample_07` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\n"""
+        """STORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')\n""",
       "table_name": "sample_07",
       "transactional": True
     }
@@ -164,9 +168,29 @@ class TestTransactionalTables():
         "`date`='2015-11-20'": "web_logs_3.csv",
         "`date`='2015-11-21'": "web_logs_4.csv"
       },
-      "create_sql": "CREATE TABLE `web_logs`  (  `_version_` bigint,   `app` string,   `bytes` int,   `city` string,   `client_ip` string,   `code` smallint,   `country_code` string,   `country_code3` string,   `country_name` string,   `device_family` string,   `extension` string,   `latitude` float,   `longitude` float,   `method` string,   `os_family` string,   `os_major` string,   `protocol` string,   `record` string,   `referer` string,   `region_code` string,   `request` string,   `subapp` string,   `time` string,   `url` string,   `user_agent` string,   `user_agent_family` string,   `user_agent_major` string,   `id` string)\nPARTITIONED BY (  `date` string  )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')",
+      "create_sql":
+          """CREATE TABLE `web_logs`  (  `_version_` bigint,   `app` string,   `bytes` int,   `city` string,   `client_ip` string,   """
+          """`code` smallint,   `country_code` string,   `country_code3` string,   `country_name` string,   `device_family` string,  """
+          """ `extension` string,   `latitude` float,   `longitude` float,   `method` string,   `os_family` string,   `os_major` string,"""
+          """   `protocol` string,   `record` string,   `referer` string,   `region_code` string,   `request` string,   `subapp` string,"""
+          """   `time` string,   `url` string,   `user_agent` string,   `user_agent_family` string,   `user_agent_major` string,"""
+          """   `id` string)\nPARTITIONED BY (  `date` string  )\nSTORED AS parquet\nTBLPROPERTIES ('transactional'='true',"""
+          """ 'transactional_properties'='insert_only')""",
       "table_name": "web_logs",
-      "columns": [{"name": "_version_", "type": "bigint"}, {"name": "app", "type": "string"}, {"name": "bytes", "type": "int"}, {"name": "city", "type": "string"}, {"name": "client_ip", "type": "string"}, {"name": "code", "type": "smallint"}, {"name": "country_code", "type": "string"}, {"name": "country_code3", "type": "string"}, {"name": "country_name", "type": "string"}, {"name": "device_family", "type": "string"}, {"name": "extension", "type": "string"}, {"name": "latitude", "type": "float"}, {"name": "longitude", "type": "float"}, {"name": "method", "type": "string"}, {"name": "os_family", "type": "string"}, {"name": "os_major", "type": "string"}, {"name": "protocol", "type": "string"}, {"name": "record", "type": "string"}, {"name": "referer", "type": "string"}, {"name": "region_code", "type": "string"}, {"name": "request", "type": "string"}, {"name": "subapp", "type": "string"}, {"name": "time", "type": "string"}, {"name": "url", "type": "string"}, {"name": "user_agent", "type": "string"}, {"name": "user_agent_family", "type": "string"}, {"name": "user_agent_major", "type": "string"}, {"name": "id", "type": "string"}, {"name": "date", "type": "string"}],
+      "columns": [
+          {"name": "_version_", "type": "bigint"}, {"name": "app", "type": "string"}, {"name": "bytes", "type": "int"},
+          {"name": "city", "type": "string"}, {"name": "client_ip", "type": "string"}, {"name": "code", "type": "smallint"},
+          {"name": "country_code", "type": "string"}, {"name": "country_code3", "type": "string"},
+          {"name": "country_name", "type": "string"}, {"name": "device_family", "type": "string"},
+          {"name": "extension", "type": "string"}, {"name": "latitude", "type": "float"},
+          {"name": "longitude", "type": "float"}, {"name": "method", "type": "string"},
+          {"name": "os_family", "type": "string"}, {"name": "os_major", "type": "string"},
+          {"name": "protocol", "type": "string"}, {"name": "record", "type": "string"},
+          {"name": "referer", "type": "string"}, {"name": "region_code", "type": "string"},
+          {"name": "request", "type": "string"}, {"name": "subapp", "type": "string"},
+          {"name": "time", "type": "string"}, {"name": "url", "type": "string"},
+          {"name": "user_agent", "type": "string"}, {"name": "user_agent_family", "type": "string"},
+          {"name": "user_agent_major", "type": "string"}, {"name": "id", "type": "string"}, {"name": "date", "type": "string"}],
       "transactional": True
     }
 
@@ -180,9 +204,11 @@ class TestTransactionalTables():
 
 
   def test_create_phoenix_table(self):
-    table_data =   {
+    table_data = {
       "data_file": "./tables/us_population.csv",
-      "create_sql": "CREATE TABLE IF NOT EXISTS us_population (\n  state CHAR(2) NOT NULL,\n  city VARCHAR NOT NULL,\n  population BIGINT\n  CONSTRAINT my_pk PRIMARY KEY (state, city)\n)\n",
+      "create_sql":
+        """CREATE TABLE IF NOT EXISTS us_population (\n  state CHAR(2) NOT NULL,\n  city VARCHAR NOT NULL,\n  """
+        """population BIGINT\n  CONSTRAINT my_pk PRIMARY KEY (state, city)\n)\n""",
       "insert_sql": "UPSERT INTO us_population VALUES %(values)s",
       "table_name": "us_population",
       "dialects": ["phoenix"],

+ 1 - 2
desktop/libs/notebook/src/notebook/management/commands/notebook_setup.py

@@ -63,5 +63,4 @@ class Command(BaseCommand):
       LOG.info('Successfully installed sample notebook')
 
     from beeswax.management.commands.beeswax_install_examples import Command
-    app_name = 'beeswax'
-    Command().handle(app_name=app_name, user=user, tables='tables.json')
+    Command().handle(dialect='hive', user=user)

+ 71 - 0
desktop/libs/notebook/src/notebook/management/commands/samples_setup.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from django.core.management.base import BaseCommand
+
+from desktop.lib.connectors.models import _get_installed_connectors
+from beeswax.management.commands.beeswax_install_examples import Command as EditorCommand
+from useradmin.models import User
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  args = '<user>'
+  help = 'Install examples but do not overwrite them.'
+
+  def add_arguments(self, parser):
+    parser.add_argument(
+        '--username',
+        dest='username',
+        default='hue',
+        help='Hue username used to execute the command',
+    )
+    parser.add_argument(
+        '--dialect',
+        dest='dialect',
+        default=None,
+        help='Dialect name we want to install the samples, all if not specified',
+    )
+
+  def handle(self, *args, **options):
+    user = User.objects.get(username=options['username'])
+    dialect = options.get('dialect')
+
+    dialects = [
+      {
+        'id': connector['id'],
+        'dialect': connector['dialect']
+      }
+      for connector in _get_installed_connectors(category='editor')
+      if dialect is None or connector['dialect'] == dialect
+    ]
+
+    tables = None
+
+    for dialect in dialects:
+      EditorCommand().handle(
+        app_name=dialect['dialect'],
+        user=user,
+        tables=tables,
+        dialect=dialect['dialect'],
+        interpreter={'type': dialect['id']}
+      )