Эх сурвалжийг харах

HUE-8758 [editor] Refactoring for generic SQL samples queries

Romain 5 жил өмнө
parent
commit
62542d842e

+ 1 - 1
apps/about/src/about/templates/admin_wizard.mako

@@ -109,7 +109,7 @@ ${ layout.menubar(section='quick_start') }
 
 
               % if has_connectors():
               % if has_connectors():
                 <!-- ko foreach: connectors -->
                 <!-- ko foreach: connectors -->
-                  <!-- ko if: ['hive', 'impala'].indexOf(dialect) != -1 -->
+                  <!-- ko if: ['hive', 'impala', 'mysql', 'postgresql', 'presto'].indexOf(dialect) != -1 -->
                   <li>
                   <li>
                     <a href="javascript:void(0)" data-bind="click: $root.installConnectorDataExample">
                     <a href="javascript:void(0)" data-bind="click: $root.installConnectorDataExample">
                       <i class="fa fa-download"></i> <span data-bind="text: name"></span>
                       <i class="fa fa-download"></i> <span data-bind="text: name"></span>

+ 0 - 146
apps/beeswax/data/designs.json

@@ -1,146 +0,0 @@
-[
-   {
-      "name":"Sample: Top salary",
-      "desc":"Top salary 2007 above $100k",
-      "type":"0",
-      "data":{
-         "query":{
-            "query":"SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000",
-            "type":0,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Salary growth",
-      "desc":"Salary growth (sorted) from 2007-08",
-      "type":"0",
-      "data":{
-         "query":{
-            "query":"SELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000",
-            "type":0,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Job loss",
-      "desc":"Job loss among the top earners 2007-08",
-      "type":"0",
-      "data":{
-         "query":{
-            "query":"SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary\r\nFROM\r\n  sample_07 s07 JOIN \r\n  sample_08 s08\r\nON ( s07.code = s08.code )\r\nWHERE\r\n( s07.total_emp > s08.total_emp\r\n AND s07.salary > 100000 )\r\nORDER BY s07.salary DESC\nLIMIT 1000",
-            "type":0,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Customers",
-      "desc":"Email Survey Opt-Ins, Customers for Shipping ZIP Code, Total Amount per Order",
-      "type":"0",
-      "data":{
-         "query":{
-            "query": "-- Get email survey opt-in values for all customers\nSELECT\r\n  c.id,\r\n  c.name,\r\n  c.email_preferences.categories.surveys\r\nFROM customers c;\n\n\n\n-- Select customers for a given shipping ZIP Code\nSELECT\r\n  customers.id,\r\n  customers.name\r\nFROM customers\r\nWHERE customers.addresses['shipping'].zip_code = '76710';\n\n\n\n-- Compute total amount per order for all customers\nSELECT\r\n  c.id AS customer_id,\r\n  c.name AS customer_name,\r\n  ords.order_id AS order_id,\r\n  SUM(order_items.price * order_items.qty) AS total_amount\r\nFROM\r\n  customers c\r\nLATERAL VIEW EXPLODE(c.orders) o AS ords\r\nLATERAL VIEW EXPLODE(ords.items) i AS order_items\r\nGROUP BY c.id, c.name, ords.order_id;",
-            "type":0,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Top salary",
-      "desc":"Top salary 2007 above $100k",
-      "type":"1",
-      "data":{
-         "query":{
-            "query":"SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000",
-            "type":1,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Salary growth",
-      "desc":"Salary growth (sorted) from 2007-08",
-      "type":"1",
-      "data":{
-         "query":{
-            "query":"SELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000",
-            "type":1,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Job loss",
-      "desc":"Job loss among the top earners 2007-08",
-      "type":"1",
-      "data":{
-         "query":{
-            "query":"SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary\r\nFROM\r\n  sample_07 s07 JOIN \r\n  sample_08 s08\r\nON ( s07.code = s08.code )\r\nWHERE\r\n( s07.total_emp > s08.total_emp\r\n AND s07.salary > 100000 )\r\nORDER BY s07.salary DESC\nLIMIT 1000",
-            "type":0,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   },
-   {
-      "name":"Sample: Customers",
-      "desc":"Email Survey Opt-Ins, Customers for Shipping ZIP Code, Total Amount per Order",
-      "type":"1",
-      "data":{
-         "query":{
-            "query": "-- Get email survey opt-in values for all customers \nSELECT\r\n  c.id,\r\n  c.name,\r\n  c.email_preferences.categories.surveys\r\nFROM customers c;\n\n\n\n-- Select customers for a given shipping ZIP Code\nSELECT\r\n  c.id,\r\n  c.name\r\nFROM customers c, c.addresses a\r\nWHERE a.key = 'shipping' and a.zip_code = '76710';\n\n\n\n-- Compute total amount per order for all customers\nSELECT\r\n  c.id AS customer_id,\r\n c.name AS customer_name,\r\n  o.order_id,\r\n  v.total\r\nFROM\r\n  customers c,\r\n  c.orders o,\r\n  (SELECT SUM(price * qty) total FROM o.items) v;",
-            "type":1,
-            "email_notify":false,
-            "is_parameterized":false,
-            "database":"default"
-         },
-         "functions":[ ],
-         "VERSION":"0.4.1",
-         "file_resources":[ ],
-         "settings":[ ]
-      }
-   }
-]

+ 165 - 0
apps/beeswax/data/queries.json

@@ -0,0 +1,165 @@
+[
+  {
+    "name": "Sample: Salary Analysis",
+    "desc": "Top salary 2007 above $100k, Salary growth (sorted) from 2007-08",
+    "type": "0",
+    "dialects": ["postgresql", "mysql", "presto"],
+    "data": {
+      "query": {
+        "query": "SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000;\n\n\nSELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000;\n\n\n",
+        "type": 0,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Top salary",
+    "desc": "Top salary 2007 above $100k",
+    "type": "0",
+    "data": {
+      "query": {
+        "query": "SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000",
+        "type": 0,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Salary growth",
+    "desc": "Salary growth (sorted) from 2007-08",
+    "type": "0",
+    "data": {
+      "query": {
+        "query": "SELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000",
+        "type": 0,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Job loss",
+    "desc": "Job loss among the top earners 2007-08",
+    "type": "0",
+    "data": {
+      "query": {
+        "query": "SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary\r\nFROM\r\n  sample_07 s07 JOIN \r\n  sample_08 s08\r\nON ( s07.code = s08.code )\r\nWHERE\r\n( s07.total_emp > s08.total_emp\r\n AND s07.salary > 100000 )\r\nORDER BY s07.salary DESC\nLIMIT 1000",
+        "type": 0,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Customers",
+    "desc": "Email Survey Opt-Ins, Customers for Shipping ZIP Code, Total Amount per Order",
+    "type": "0",
+    "data": {
+      "query": {
+        "query": "-- Get email survey opt-in values for all customers\nSELECT\r\n  c.id,\r\n  c.name,\r\n  c.email_preferences.categories.surveys\r\nFROM customers c;\n\n\n\n-- Select customers for a given shipping ZIP Code\nSELECT\r\n  customers.id,\r\n  customers.name\r\nFROM customers\r\nWHERE customers.addresses['shipping'].zip_code = '76710';\n\n\n\n-- Compute total amount per order for all customers\nSELECT\r\n  c.id AS customer_id,\r\n  c.name AS customer_name,\r\n  ords.order_id AS order_id,\r\n  SUM(order_items.price * order_items.qty) AS total_amount\r\nFROM\r\n  customers c\r\nLATERAL VIEW EXPLODE(c.orders) o AS ords\r\nLATERAL VIEW EXPLODE(ords.items) i AS order_items\r\nGROUP BY c.id, c.name, ords.order_id;",
+        "type": 0,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Top salary",
+    "desc": "Top salary 2007 above $100k",
+    "type": "1",
+    "data": {
+      "query": {
+        "query": "SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000",
+        "type": 1,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Salary growth",
+    "desc": "Salary growth (sorted) from 2007-08",
+    "type": "1",
+    "data": {
+      "query": {
+        "query": "SELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000",
+        "type": 1,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Job loss",
+    "desc": "Job loss among the top earners 2007-08",
+    "type": "1",
+    "data": {
+      "query": {
+        "query": "SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary\r\nFROM\r\n  sample_07 s07 JOIN \r\n  sample_08 s08\r\nON ( s07.code = s08.code )\r\nWHERE\r\n( s07.total_emp > s08.total_emp\r\n AND s07.salary > 100000 )\r\nORDER BY s07.salary DESC\nLIMIT 1000",
+        "type": 0,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  },
+  {
+    "name": "Sample: Customers",
+    "desc": "Email Survey Opt-Ins, Customers for Shipping ZIP Code, Total Amount per Order",
+    "type": "1",
+    "data": {
+      "query": {
+        "query": "-- Get email survey opt-in values for all customers \nSELECT\r\n  c.id,\r\n  c.name,\r\n  c.email_preferences.categories.surveys\r\nFROM customers c;\n\n\n\n-- Select customers for a given shipping ZIP Code\nSELECT\r\n  c.id,\r\n  c.name\r\nFROM customers c, c.addresses a\r\nWHERE a.key = 'shipping' and a.zip_code = '76710';\n\n\n\n-- Compute total amount per order for all customers\nSELECT\r\n  c.id AS customer_id,\r\n c.name AS customer_name,\r\n  o.order_id,\r\n  v.total\r\nFROM\r\n  customers c,\r\n  c.orders o,\r\n  (SELECT SUM(price * qty) total FROM o.items) v;",
+        "type": 1,
+        "email_notify": false,
+        "is_parameterized": false,
+        "database": "default"
+      },
+      "functions": [],
+      "VERSION": "0.4.1",
+      "file_resources": [],
+      "settings": []
+    }
+  }
+]

+ 7 - 0
apps/beeswax/data/tables_standard.json

@@ -0,0 +1,7 @@
+[
+  {
+    "data_file": "employe_sample.csv",
+    "create_hql": "CREATE TABLE `employe_sample` (\n  `code` string ,\n  `description` string ,\n  `total_emp` int ,\n  `salary` int )\n",
+    "table_name": "sample_sample"
+  }
+]

+ 9 - 5
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples.py

@@ -29,6 +29,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.conf import USE_NEW_EDITOR
 from desktop.conf import USE_NEW_EDITOR
 from desktop.models import Directory, Document, Document2, Document2Permission
 from desktop.models import Directory, Document, Document2, Document2Permission
 from hadoop import cluster
 from hadoop import cluster
+
 from notebook.models import import_saved_beeswax_query
 from notebook.models import import_saved_beeswax_query
 from useradmin.models import get_default_user_group, install_sample_user, User
 from useradmin.models import get_default_user_group, install_sample_user, User
 
 
@@ -103,12 +104,12 @@ class Command(BaseCommand):
         raise InstallException(_('Could not install table: %s') % ex)
         raise InstallException(_('Could not install table: %s') % ex)
 
 
   def _install_queries(self, django_user, app_name, interpreter=None):
   def _install_queries(self, django_user, app_name, interpreter=None):
-    design_file = open(os.path.join(LOCAL_EXAMPLES_DATA_DIR.get(), 'designs.json'))
+    design_file = open(os.path.join(LOCAL_EXAMPLES_DATA_DIR.get(), 'queries.json'))
     design_list = json.load(design_file)
     design_list = json.load(design_file)
     design_file.close()
     design_file.close()
 
 
     # Filter design list to app-specific designs
     # Filter design list to app-specific designs
-    app_type = HQL if app_name == 'beeswax' else IMPALA
+    app_type = HQL if app_name == 'beeswax' else IMPALA if app_name == 'impala' else 'sql'
     design_list = [d for d in design_list if int(d['type']) == app_type]
     design_list = [d for d in design_list if int(d['type']) == app_type]
 
 
     for design_dict in design_list:
     for design_dict in design_list:
@@ -368,9 +369,10 @@ class SampleQuery(object):
         name=Document2.EXAMPLES_DIR
         name=Document2.EXAMPLES_DIR
       )
       )
 
 
+      document_type = self._document_type(self.type, interpreter)
       try:
       try:
         # Don't overwrite
         # Don't overwrite
-        doc2 = Document2.objects.get(owner=django_user, name=self.name, type=self._document_type(self.type), is_history=False)
+        doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
         # If document exists but has been trashed, recover from Trash
         # If document exists but has been trashed, recover from Trash
         if doc2.parent_directory != examples_dir:
         if doc2.parent_directory != examples_dir:
           doc2.parent_directory = examples_dir
           doc2.parent_directory = examples_dir
@@ -388,7 +390,7 @@ class SampleQuery(object):
           owner=django_user,
           owner=django_user,
           parent_directory=examples_dir,
           parent_directory=examples_dir,
           name=self.name,
           name=self.name,
-          type=self._document_type(self.type),
+          type=document_type,
           description=self.desc,
           description=self.desc,
           data=data
           data=data
         )
         )
@@ -398,10 +400,12 @@ class SampleQuery(object):
       LOG.info('Successfully installed sample query: %s' % (self.name,))
       LOG.info('Successfully installed sample query: %s' % (self.name,))
 
 
 
 
-  def _document_type(self, type):
+  def _document_type(self, type, interpreter=None):
     if type == HQL:
     if type == HQL:
       return 'query-hive'
       return 'query-hive'
     elif type == IMPALA:
     elif type == IMPALA:
       return 'query-impala'
       return 'query-impala'
+    elif interpreter:
+      return 'query-%(dialect)s' % interpreter
     else:
     else:
       return None
       return None

+ 52 - 5
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples_tests.py

@@ -24,19 +24,66 @@ from nose.tools import assert_equal, assert_not_equal, assert_true, assert_false
 from desktop.auth.backend import rewrite_user
 from desktop.auth.backend import rewrite_user
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import add_to_group, grant_access
 from desktop.lib.test_utils import add_to_group, grant_access
+from desktop.models import Document2
+from useradmin.models import User, install_sample_user
 
 
-from beeswax.management.commands.beeswax_install_examples import SampleTable, Command
-
-from django.contrib.auth.models import User
+from beeswax.management.commands.beeswax_install_examples import SampleTable, Command, SampleQuery
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
-  from unittest.mock import patch, Mock
+  from unittest.mock import patch, Mock, MagicMock
 else:
 else:
-  from mock import patch, Mock
+  from mock import patch, Mock, MagicMock
+
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
+class TestStandardTables():
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+    self.user = User.objects.get(username="test")
+
+  def test_install_queries(self):
+      design_dict = {
+        "name": "TestStandardTables Query",
+        "desc": "Small query",
+        "type": "0",
+        "dialects": ["postgresql", "mysql", "presto"],
+        "data": {
+          "query": {
+            "query": "SELECT 1",
+            "type": 0,
+            "email_notify": False,
+            "is_parameterized": False,
+            "database": "default"
+          },
+          "functions": [],
+          "VERSION": "0.4.1",
+          "file_resources": [],
+          "settings": []
+        }
+      }
+      interpreter = {'type': 'mysql'}
+
+      design = SampleQuery(design_dict)
+      assert_false(Document2.objects.filter(name='TestStandardTables Query').exists())
+
+      with patch('notebook.models.get_interpreter') as get_interpreter:
+        design.install(django_user=self.user, interpreter=interpreter)
+
+        assert_true(Document2.objects.filter(name='TestStandardTables Query').exists())
+        query = Document2.objects.filter(name='TestStandardTables Query').get()
+        assert_equal('query-hive', query.type)
+
+  # def test_install_queries(self):
+  #     sample_user = install_sample_user()
+  #     cmd = beeswax_install_examples.Command()
+  #     app_nam = 'mysql'
+  #     interpreter = Mock()
+  #     cmd._install_queries(sample_user, app_name, interpreter=interpreter)
+
+
 class TestTransactionalTables():
 class TestTransactionalTables():
 
 
   def setUp(self):
   def setUp(self):

+ 4 - 4
desktop/core/src/desktop/lib/test_utils.py

@@ -15,14 +15,14 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-from past.builtins import basestring
 import json
 import json
-from lxml import objectify, etree
 
 
-from django.contrib.auth.models import Group, User
-from useradmin.models import HuePermission, GroupPermission, get_default_user_group
+from past.builtins import basestring
+from lxml import objectify, etree
 
 
 from hadoop import cluster
 from hadoop import cluster
+from useradmin.models import Group, User, HuePermission, GroupPermission, get_default_user_group
+
 from desktop.lib import fsmanager
 from desktop.lib import fsmanager
 
 
 
 

+ 1 - 1
desktop/libs/notebook/src/notebook/views.py

@@ -382,7 +382,7 @@ def install_examples(request):
     try:
     try:
       connector = Connector.objects.get(id=request.POST.get('connector'))
       connector = Connector.objects.get(id=request.POST.get('connector'))
       if connector:
       if connector:
-        app_name = 'beeswax' if connector.dialect == 'hive' else 'impala'
+        app_name = 'beeswax' if connector.dialect == 'hive' else connector.dialect
         db_name = request.POST.get('db_name', 'default')
         db_name = request.POST.get('db_name', 'default')
         interpreter = get_interpreter(connector_type=connector.to_dict()['type'], user=request.user)
         interpreter = get_interpreter(connector_type=connector.to_dict()['type'], user=request.user)