Эх сурвалжийг харах

[slack] Query bank for Assistance V1 (#2154)

- Query bank extraction and surfacing it up in Slack
- Separate `assistant` directory with its own utils py
- Added unit tests
Harsh Gupta 4 жил өмнө
parent
commit
c240af0a10

+ 2 - 2
desktop/core/src/desktop/lib/botserver/api.py

@@ -57,8 +57,8 @@ def send_message(request):
     'ok': slack_response.get('ok'),
   })
 
-def _send_message(channel_info, message):
+def _send_message(channel_info, message=None, block_element=None):
   try:
-    return slack_client.chat_postMessage(channel=channel_info, text=message)
+    return slack_client.chat_postMessage(channel=channel_info, text=message, blocks=block_element)
   except Exception as e:
     raise PopupException(_("Error posting message in channel"), detail=e)

+ 1 - 1
desktop/core/src/desktop/lib/botserver/api_tests.py

@@ -79,5 +79,5 @@ class TestApi(object):
       data = json.loads(response.content)
 
       assert_equal(200, response.status_code)
-      chat_postMessage.assert_called_with(channel='channel-1', text='@api_user: message with link')
+      chat_postMessage.assert_called_with(channel='channel-1', text='@api_user: message with link', blocks=None)
       assert_true(data.get('ok'))

+ 45 - 1
desktop/core/src/desktop/lib/botserver/views.py

@@ -18,6 +18,7 @@
 import logging
 import json
 import sys
+import os
 from urllib.parse import urlsplit
 from tabulate import tabulate
 
@@ -33,6 +34,8 @@ from notebook.api import _fetch_result_data, _check_status, _execute_notebook
 from notebook.models import MockRequest, get_api
 from notebook.connectors.base import _get_snippet_name
 
+from metadata.assistant.queries_utils import get_all_queries
+
 from useradmin.models import User
 
 from django.http import HttpResponse
@@ -79,7 +82,7 @@ def parse_events(request, event):
   """
   channel_id = event.get('channel')
   user_id = event.get('user')
-  message_element = event['blocks'][0]['elements'] if event.get('blocks') else []
+  message_element = event['blocks'][0].get('elements', []) if event.get('blocks') else []
 
   if event.get('type') == 'message':
     handle_on_message(request.get_host(), request.is_secure(), channel_id, event.get('bot_id'), message_element, user_id)
@@ -87,6 +90,47 @@ def parse_events(request, event):
   if event.get('type') == 'link_shared':
     handle_on_link_shared(request.get_host(), channel_id, event.get('message_ts'), event.get('links'), user_id)
 
+  if event.get('type') == 'app_mention':
+    handle_on_app_mention(request.get_host(), channel_id, user_id, event.get('text'))
+
+
+def handle_on_app_mention(host_domain, channel_id, user_id, text):
+  if text and 'queries' in text:
+    slack_user = check_slack_user_permission(host_domain, user_id)
+    user = get_user(channel_id, slack_user)
+
+    handle_query_bank(channel_id, user_id)
+
+  
+def handle_query_bank(channel_id, user_id):
+  data = get_all_queries()
+
+  message_block = [
+    {
+      "type": "section",
+      "text": {
+        "type": "mrkdwn",
+        "text": "Hi <@{user}>, here is the list of all saved queries!".format(user=user_id)
+      }
+    },
+    {
+      "type": "divider"
+    },
+  ]
+
+  for query in data:
+    statement = query['data']['query']['statement']
+    query_element = {
+      "type": "section",
+      "text": {
+        "type": "mrkdwn",
+        "text": "*Name:* {name} \n *Statement:*\n ```{statement}```".format(name=query['name'], statement=statement)
+      },
+    }
+    message_block.append(query_element)
+
+  _send_message(channel_id, block_element=message_block)
+
 
 def handle_on_message(host_domain, is_http_secure, channel_id, bot_id, elements, user_id):
   # Ignore bot's own message since that will cause an infinite loop of messages if we respond.

+ 66 - 1
desktop/core/src/desktop/lib/botserver/views_tests.py

@@ -329,4 +329,69 @@ class TestBotServer(unittest.TestCase):
               }
             }
           }
-          assert_raises(PopupException, handle_on_link_shared, self.host_domain, "channel", "12.1", links, "<@user_id>")
+          assert_raises(PopupException, handle_on_link_shared, self.host_domain, "channel", "12.1", links, "<@user_id>")
+  
+  def test_handle_on_app_mention(self):
+    with patch('desktop.lib.botserver.views.check_slack_user_permission') as check_slack_user_permission:
+      with patch('desktop.lib.botserver.views.get_user') as get_user:
+        with patch('desktop.lib.botserver.views.handle_query_bank') as handle_query_bank:
+
+          channel_id = "channel"
+          user_id = "<@user_id>"
+
+          text = '@hue some message'
+          handle_on_app_mention(self.host_domain, channel_id, user_id, text)
+
+          assert_false(handle_query_bank.called)
+
+          text = '@hue queries'
+          handle_on_app_mention(self.host_domain, channel_id, user_id, text)
+
+          handle_query_bank.assert_called_with(channel_id, user_id)
+
+  def test_handle_query_bank(self):
+    with patch('desktop.lib.botserver.views.get_all_queries') as get_all_queries:
+      with patch('desktop.lib.botserver.views._send_message') as _send_message:
+
+        channel_id = "channel"
+        user_id = "<@user_id>"
+
+        get_all_queries.return_value = [
+          {
+            "name": "Test Query 1",
+            "data": {
+              "query": {
+                "statement": "SELECT 1"
+              }
+            }
+          },
+          {
+            "name": "Test Query 2",
+            "data": {
+              "query": {
+                "statement": "SELECT 2"
+              }
+            }
+          }
+        ]
+
+        test_query_block = [
+          {
+            'type': 'section',
+            'text': {'type': 'mrkdwn', 'text': 'Hi <@<@user_id>>, here is the list of all saved queries!'}
+          }, 
+          {
+            'type': 'divider'
+          }, 
+          {
+            'type': 'section',
+            'text': {'type': 'mrkdwn', 'text': '*Name:* Test Query 1 \n *Statement:*\n ```SELECT 1```'}
+          },
+          {
+            'type': 'section',
+            'text': {'type': 'mrkdwn', 'text': '*Name:* Test Query 2 \n *Statement:*\n ```SELECT 2```'}
+          }
+        ]
+        
+        handle_query_bank(channel_id, user_id)
+        _send_message.assert_called_with(channel_id, block_element=test_query_block)

+ 16 - 0
desktop/libs/metadata/src/metadata/assistant/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 82 - 0
desktop/libs/metadata/src/metadata/assistant/data/queries.json

@@ -0,0 +1,82 @@
+[
+  {
+    "name": "Sample: Salary Analysis",
+    "desc": "Top salary 2007 above $100k, Salary growth (sorted) from 2007-08",
+    "dialects": [
+      "postgresql",
+      "mysql",
+      "presto"
+    ],
+    "data": {
+      "query": {
+        "statement": "SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000;\n\n\nSELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000;\n"
+      }
+    }
+  },
+  {
+    "name": "Sample: Top salary",
+    "desc": "Top salary 2007 above $100k",
+    "dialects": [],
+    "data": {
+      "query": {
+        "statement": "SELECT sample_07.description, sample_07.salary\r\nFROM\r\n  sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000"
+      }
+    }
+  },
+  {
+    "name": "Sample: Salary growth",
+    "desc": "Salary growth (sorted) from 2007-08",
+    "dialects": [],
+    "data": {
+      "query": {
+        "statement": "SELECT s07.description, s07.salary, s08.salary,\r\n  s08.salary - s07.salary\r\nFROM\r\n  sample_07 s07 JOIN sample_08 s08\r\nON ( s07.code = s08.code)\r\nWHERE\r\n s07.salary < s08.salary\r\nORDER BY s08.salary-s07.salary DESC\r\nLIMIT 1000"
+      }
+    }
+  },
+  {
+    "name": "Sample: Job loss",
+    "desc": "Job loss among the top earners 2007-08",
+    "dialects": [],
+    "data": {
+      "query": {
+        "statement": "SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary\r\nFROM\r\n  sample_07 s07 JOIN \r\n  sample_08 s08\r\nON ( s07.code = s08.code )\r\nWHERE\r\n( s07.total_emp > s08.total_emp\r\n AND s07.salary > 100000 )\r\nORDER BY s07.salary DESC\nLIMIT 1000"
+      }
+    }
+  },
+  {
+    "name": "US City Population",
+    "desc": "Small samples of number of inhabitants in some US cities",
+    "dialects": [
+      "pheonix"
+    ],
+    "data": {
+      "query": {
+        "statement": "\nCREATE TABLE IF NOT EXISTS us_population (\n  state CHAR(2) NOT NULL,\n  city VARCHAR NOT NULL,\n  population BIGINT\n  CONSTRAINT my_pk PRIMARY KEY (state, city)\n);\n\n\nUPSERT INTO us_population VALUES ('NY','New York',8143197);\nUPSERT INTO us_population VALUES ('CA','Los Angeles',3844829);\nUPSERT INTO us_population VALUES ('IL','Chicago',2842518);\nUPSERT INTO us_population VALUES ('TX','Houston',2016582);\nUPSERT INTO us_population VALUES ('PA','Philadelphia',1463281);\nUPSERT INTO us_population VALUES ('AZ','Phoenix',1461575);\nUPSERT INTO us_population VALUES ('TX','San Antonio',1256509);\nUPSERT INTO us_population VALUES ('CA','San Diego',1255540);\nUPSERT INTO us_population VALUES ('TX','Dallas',1213825);\nUPSERT INTO us_population VALUES ('CA','San Jose',91233);\n\nSELECT\n  state as \"State\",\n  count(city) as \"City Count\",\n  sum(population) as \"Population Sum\"\nFROM\n  us_population\nGROUP BY\n  state\nORDER BY\n  sum(population) DESC\n;"
+      }
+    }
+  },
+  {
+    "name": "Query and live display a live stream of data",
+    "desc": "Simple select of auto generated data or via the user table backed by a Kafka topic",
+    "dialects": [
+      "flink"
+    ],
+    "data": {
+      "query": {
+        "statement": "\nCREATE TABLE datagen (\n  f_sequence INT,\n  f_random INT,\n  f_random_str STRING,\n  ts AS localtimestamp,\n  WATERMARK FOR ts AS ts\n) WITH (\n  'connector' = 'datagen',\n  'rows-per-second'='5',\n  'fields.f_sequence.kind'='sequence',\n  'fields.f_sequence.start'='1',\n  'fields.f_sequence.end'='1000',\n  'fields.f_random.min'='1',\n  'fields.f_random.max'='1000',\n  'fields.f_random_str.length'='10'\n)\n;\n\nSELECT *\nFROM datagen\nLIMIT 50\n;\n\n\n\nCREATE TABLE user_behavior (\n  user_id BIGINT,\n  item_id BIGINT,\n  category_id BIGINT,\n  behavior STRING,\n  ts TIMESTAMP(3),\n  proctime AS PROCTIME(),   -- generates processing-time attribute using computed column\n  WATERMARK FOR ts AS ts - INTERVAL '5' SECOND  -- defines watermark on ts column, marks ts as event-time attribute\n) WITH (\n  'connector' = 'kafka',  -- using kafka connector\n  'topic' = 'user_behavior',  -- kafka topic\n  'scan.startup.mode' = 'earliest-offset',  -- reading from the beginning\n  'properties.bootstrap.servers' = 'kafka:9094',  -- kafka broker address\n  'format' = 'json'  -- the data format is json\n)\n;\n\nSELECT * \nFROM user_behavior \nLIMIT 50\n;\n\n\nSELECT\n  HOUR(TUMBLE_START(ts, INTERVAL '1' HOUR)) as hour_of_day,\n  COUNT(*) as buy_cnt\nFROM\n  user_behavior\nWHERE\n  behavior = 'buy'\nGROUP BY\n  TUMBLE(ts, INTERVAL '1' HOUR)\n;\n  "
+      }
+    }
+  },
+  {
+    "name": "New York Taxi dataset Analysis",
+    "desc": "Regular SELECTs with custom Python UDF and create Machine Learning Model",
+    "dialects": [
+      "dasksql"
+    ],
+    "data": {
+      "query": {
+        "statement": "\nSELECT *\nFROM \"schema\".\"nyc-taxi\"\nLIMIT 100\n;\n\nSELECT\n    FLOOR(trip_distance / 5) * 5 AS \"distance\",\n    AVG(tip_amount) AS \"given tip\",\n    AVG(predict_price(total_amount, trip_distance, passenger_count)) AS \"predicted tip\"\nFROM \"nyc-taxi\"\nWHERE\n    trip_distance > 0 AND trip_distance < 50\nGROUP BY\n    FLOOR(trip_distance / 5) * 5\n;\n\nCREATE MODEL fare_estimator WITH (\n    model_class = 'sklearn.ensemble.GradientBoostingClassifier',\n    wrap_predict = True,\n    target_column = 'fare_amount'\n) AS (\n    SELECT trip_distance, fare_amount\n    FROM \"nyc-taxi\"\n    LIMIT 100\n)\n;\n"
+      }
+    }
+  }
+]

+ 34 - 0
desktop/libs/metadata/src/metadata/assistant/queries_utils.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+import sys
+import os
+
+if sys.version_info[0] > 2:
+  from django.utils.translation import gettext as _
+else:
+  from django.utils.translation import ugettext as _
+
+LOG = logging.getLogger(__name__)
+
+def get_all_queries():
+  with open(os.path.join(os.path.dirname(__file__), 'data/queries.json')) as file:
+    queries = json.load(file)
+
+    return queries