Browse Source

HUE-7738 [editor] Add function listing in autocomplete API

This is a first pass before v1. See the documentation for usage.

Note:
- only Hive can provide detailed function info
- by default Hive only provides function names

Still need to add:

Both:
- add other fields when possible, e.g. return type, signature, binary
type, is persistent
- show aggregate functions;
- show analytic functions;

Hive:
- DESCRIBE FUNCTION EXTENDED trunc

Impala
- include _impala_builtins per default
- include the active database

Other SQL
- TODO: either errors or try to return columns
- MySql e.g. show function status (or better check with SqlAlchemy)
Romain 5 years ago
parent
commit
4e69ffe3aa

+ 15 - 2
apps/beeswax/src/beeswax/api.py

@@ -103,11 +103,13 @@ def autocomplete(request, database=None, table=None, column=None, nested=None):
   return JsonResponse(response)
 
 
-def _autocomplete(db, database=None, table=None, column=None, nested=None, query=None, cluster=None):
+def _autocomplete(db, database=None, table=None, column=None, nested=None, query=None, cluster=None, operation='schema'):
   response = {}
 
   try:
-    if database is None:
+    if operation == 'functions':
+      response['functions'] = _get_functions(db, database)
+    elif database is None:
       response['databases'] = db.get_databases()
     elif table is None:
       tables_meta = db.get_tables_meta(database=database)
@@ -173,6 +175,17 @@ def _autocomplete(db, database=None, table=None, column=None, nested=None, query
   return response
 
 
+def _get_functions(db, database=None):
+  data = []
+
+  functions = db.get_functions(prefix=database)
+  if functions:
+    rows = escape_rows(functions.rows(), nulls_only=True)
+    data = [{'name': row[0]} for row in rows]
+
+  return data
+
+
 @error_handler
 def parameters(request, design_id=None):
   response = {'status': -1, 'message': ''}

+ 35 - 2
apps/beeswax/src/beeswax/api_tests.py

@@ -28,7 +28,7 @@ from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import add_to_group, grant_access
 from useradmin.models import User
 
-from beeswax.api import _autocomplete
+from beeswax.api import _autocomplete, get_functions
 
 
 if sys.version_info[0] > 2:
@@ -46,8 +46,8 @@ class TestApi():
     self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
     self.user = User.objects.get(username="test")
 
-  def test_autocomplete_time_out(self):
 
+  def test_autocomplete_time_out(self):
     get_tables_meta=Mock(
       side_effect=ReadTimeout("HTTPSConnectionPool(host='gethue.com', port=10001): Read timed out. (read timeout=120)")
     )
@@ -64,3 +64,36 @@ class TestApi():
         'error': "HTTPSConnectionPool(host='gethue.com', port=10001): Read timed out. (read timeout=120)"
       }
     )
+
+
+  def test_get_functions(self):
+    db = Mock(
+      get_functions=Mock(
+        return_value=Mock(
+          rows=Mock(
+            return_value=[{'name': 'f1'}, {'name': 'f2'}]
+          )
+        )
+      )
+    )
+
+    resp = get_functions(db)
+
+    assert_equal(
+      resp,
+      [{'name': 'f1'}, {'name': 'f2'}]
+    )
+
+
+  def test_get_functions(self):
+    with patch('beeswax.api._get_functions') as _get_functions:
+      db = Mock()
+      _get_functions.return_value = [
+        {'name': 'f1'}, {'name': 'f2'}, {'name': 'f3'}
+      ]
+      resp = _autocomplete(db, database='default', operation='functions')
+
+      assert_equal(
+        resp['functions'],
+        [{'name': 'f1'}, {'name': 'f2'}, {'name': 'f3'}]
+      )

+ 2 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -724,9 +724,10 @@ def autocomplete(request, server=None, database=None, table=None, column=None, n
   # Passed by check_document_access_permission but unused by APIs
   notebook = json.loads(request.POST.get('notebook', '{}'))
   snippet = json.loads(request.POST.get('snippet', '{}'))
+  action = request.POST.get('operation', 'schema')
 
   try:
-    autocomplete_data = get_api(request, snippet).autocomplete(snippet, database, table, column, nested)
+    autocomplete_data = get_api(request, snippet).autocomplete(snippet, database, table, column, nested, action)
     response.update(autocomplete_data)
   except QueryExpired as e:
     LOG.warn('Expired query seen: %s' % e)

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/altus_adb.py

@@ -81,7 +81,7 @@ class AltusAdbApi(Api):
     return []
 
 
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     url_path = '/notebook/api/autocomplete'
 
     if database is not None:

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -550,7 +550,7 @@ class Api(object):
   def get_log(self, notebook, snippet, startFrom=None, size=None):
     return 'No logs'
 
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     return {}
 
   def progress(self, notebook, snippet, logs=None):

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/flink_sql.py

@@ -201,7 +201,7 @@ class FlinkSqlApi(Api):
 
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     response = {}
 
     if database is None:

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/hbase.py

@@ -50,7 +50,7 @@ def query_error_handler(func):
 class HBaseApi(Api):
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     db = HbaseApi(self.user)
     cluster_name = database
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/hive_metastore.py

@@ -63,7 +63,7 @@ def query_error_handler(func):
 class HiveMetastoreApi(Api):
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     db = self._get_db(snippet, cluster=self.cluster)
 
     return _autocomplete(db, database, table, column, nested, query=None, cluster=self.cluster)

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -517,7 +517,7 @@ class HS2Api(Api):
 
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     db = self._get_db(snippet, interpreter=self.interpreter)
     query = None
 
@@ -531,7 +531,7 @@ class HS2Api(Api):
       query = self._get_current_statement(notebook, snippet)['statement']
       database, table = '', ''
 
-    resp = _autocomplete(db, database, table, column, nested, query=query, cluster=self.interpreter)
+    resp = _autocomplete(db, database, table, column, nested, query=query, cluster=self.interpreter, operation=operation)
 
     if resp.get('error'):
       resp['message'] = resp.pop('error')

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/jdbc.py

@@ -130,7 +130,7 @@ class JdbcApi(Api):
     return {'status': -1}
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     if self.db is None:
       raise AuthenticationRequired()
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/kafka.py

@@ -44,7 +44,7 @@ def query_error_handler(func):
 class KafkaApi(Api):
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     response = {}
 
     try:

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/ksql.py

@@ -95,7 +95,7 @@ class KSqlApi(Api):
 
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     response = {}
 
     db = self._get_db()

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/rdbms.py

@@ -114,7 +114,7 @@ class RdbmsApi(Api):
 
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     query_server = self._get_query_server()
     db = dbms.get(self.user, query_server)
 

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/solr.py

@@ -135,7 +135,7 @@ class SolrApi(Api):
 
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     from search.conf import SOLR_URL
     api = NativeSolrApi(SOLR_URL.get(), self.user.username)
     assist = Assist(self, self.user, api)

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/spark_shell.py

@@ -366,7 +366,7 @@ class SparkApi(Api):
     else:
       return self._get_standalone_jobs(logs)
 
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     response = {}
 
     # As booting a new SQL session is slow and we don't send the id of the current one in /autocomplete

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/sql_alchemy.py

@@ -335,7 +335,7 @@ class SqlAlchemyApi(Api):
 
 
   @query_error_handler
-  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
+  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
     engine = self._get_engine()
     inspector = inspect(engine)
 

+ 26 - 0
docs/docs-site/content/developer/api/_index.md

@@ -353,6 +353,32 @@ For nested columns:
       console.log(ko.mapping.toJSON(data));
     });
 
+#### Listing Functions
+
+Default functions:
+
+    $.post("/notebook/api/autocomplete/", {
+      "snippet": ko.mapping.toJSON({
+          type: "hive"
+      }),
+      "operation": "functions"
+    }, function(data) {
+      console.log(ko.mapping.toJSON(data));
+    });
+
+
+For a specific database:
+
+    $.post("/notebook/api/autocomplete/<DB>", {
+      "snippet": ko.mapping.toJSON({
+          type: "hive"
+      }),
+      "operation": "functions"
+    }, function(data) {
+      console.log(ko.mapping.toJSON(data));
+    });
+
+
 ### SQL Risk Optimization
 ### Data Browsing
 ### Workflow scheduling