ソースを参照

[notebook] Pick snippet interface according to the configuration

Romain Rigaux 10 年 前
コミット
72352d7

+ 9 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -20,6 +20,15 @@ from django.utils.translation import ugettext_lazy as _t
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection
 
 
 
 
+def get_interpreters():
+  return [{
+      "name": INTERPRETERS.get()[i].NAME.get(),
+      "type": i,
+      "interface": INTERPRETERS.get()[i].INTERFACE.get()}
+      for i in INTERPRETERS.get()
+  ]
+
+
 INTERPRETERS = UnspecifiedConfigSection(
 INTERPRETERS = UnspecifiedConfigSection(
   "interpreters",
   "interpreters",
   help="One entry for each type of snippet",
   help="One entry for each type of snippet",

+ 21 - 5
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -18,8 +18,13 @@
 import json
 import json
 import logging
 import logging
 
 
+from django.utils.translation import ugettext as _
+
+from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 from desktop.lib.i18n import force_unicode
 
 
+from notebook.conf import get_interpreters
+
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -73,17 +78,28 @@ class Notebook():
 
 
 
 
 def get_api(user, snippet):
 def get_api(user, snippet):
-  from notebook.connectors.hiveserver2 import HS2Api
-  from notebook.connectors.spark_batch import SparkBatchApi
+  from notebook.connectors.hiveserver2 import HS2Api  
+  from notebook.connectors.mysql import MySqlApi
+  from notebook.connectors.jdbc import JDBCApi
   from notebook.connectors.text import TextApi
   from notebook.connectors.text import TextApi
   from notebook.connectors.spark_shell import SparkApi
   from notebook.connectors.spark_shell import SparkApi
+  from notebook.connectors.spark_batch import SparkBatchApi
+
+  interface = [interpreter for interpreter in get_interpreters() if interpreter['type'] == snippet['type']]
+  if not interface:
+    raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
+  interface = interface[0]['interface']
 
 
-  if snippet['type'] in ('hive', 'impala', 'spark-sql'):
+  if interface == 'hiveserver2':
     return HS2Api(user)
     return HS2Api(user)
-  elif snippet['type'] in ('jar', 'py'):
+  elif interface == 'spark-submit':
     return SparkBatchApi(user)
     return SparkBatchApi(user)
-  elif snippet['type'] == 'text':
+  elif interface == 'text':
     return TextApi(user)
     return TextApi(user)
+  elif interface == 'mysql':
+    return MySqlApi(user)
+  elif interface == 'jdbc':
+    return JDBCApi(user)
   else:
   else:
     return SparkApi(user)
     return SparkApi(user)
 
 

+ 6 - 3
desktop/libs/notebook/src/notebook/connectors/jdbc.py

@@ -35,8 +35,8 @@ def query_error_handler(func):
       return func(*args, **kwargs)
       return func(*args, **kwargs)
     except Exception, e:
     except Exception, e:
       message = force_unicode(str(e))
       message = force_unicode(str(e))
-      if 'Invalid query handle' in message or 'Invalid OperationHandle' in message:
-        raise QueryExpired(e)
+      if 'Class com.mysql.jdbc.Driver not found' in message:
+        raise QueryExpired(_('%s: did you export CLASSPATH=$CLASSPATH:/usr/share/java/mysql.jar?') % message)
       else:
       else:
         raise QueryError(message)
         raise QueryError(message)
   return decorator
   return decorator
@@ -44,6 +44,9 @@ def query_error_handler(func):
 
 
 class JDBCApi(Api):
 class JDBCApi(Api):
 
 
+  # TODO
+  # async with queuing system
+  # impersonation / prompting for username/password
   def execute(self, notebook, snippet):
   def execute(self, notebook, snippet):
     user = 'root'
     user = 'root'
     password = 'root'
     password = 'root'
@@ -64,7 +67,7 @@ class JDBCApi(Api):
     curs = db.cursor()
     curs = db.cursor()
     curs.execute(snippet['statement'])
     curs.execute(snippet['statement'])
 
 
-    curs.description
+    print curs.description
     
     
     return {
     return {
       'result': curs.fetchmany(100)
       'result': curs.fetchmany(100)

+ 8 - 49
desktop/libs/notebook/src/notebook/connectors/mysql.py

@@ -16,14 +16,13 @@
 # limitations under the License.
 # limitations under the License.
 
 
 import logging
 import logging
-import re
 
 
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 from desktop.lib.i18n import force_unicode
 
 
-from notebook.connectors.base import Api, QueryError, QueryExpired
+from librdbms.server import dbms
 
 
-import jaydebeapi
+from notebook.connectors.base import Api, QueryError, QueryExpired
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -45,29 +44,14 @@ def query_error_handler(func):
 class MySqlApi(Api):
 class MySqlApi(Api):
 
 
   def execute(self, notebook, snippet):
   def execute(self, notebook, snippet):
-    user = 'root'
-    password = 'root'
-    
-    host = 'localhost'
-    port = 3306
-    database = 'test'
+    query_server = dbms.get_query_server_config(server='mysql')
+    db = dbms.get(self.user, query_server)
+    table = db.execute_statement()
     
     
-    autocommit = True
-    jclassname = "com.mysql.jdbc.Driver"
-    url = "jdbc:mysql://{host}:{port}/{database}".format(host=host, port=port, database=database)
-    driver_args = [url, user, password]
-    jars = None
-    libs = None
-    db = jaydebeapi.connect(jclassname, driver_args, jars=jars, libs=libs)
-    db.jconn.setAutoCommit(autocommit)
-    
-    curs = db.cursor()
-    curs.execute(snippet['statement'])
-
-    curs.description
+    print table
     
     
     return {
     return {
-      'result': curs.fetchmany(100)
+      'result': table
     }
     }
 
 
   @query_error_handler
   @query_error_handler
@@ -75,17 +59,7 @@ class MySqlApi(Api):
     return {'status': 'running'}
     return {'status': 'running'}
 
 
   def _fetch_result(self, cursor):
   def _fetch_result(self, cursor):
-
-    return {
-        'has_more': results.has_more,
-        'data': list(results.rows()),
-        'meta': [{
-          'name': column.name,
-          'type': column.type,
-          'comment': column.comment
-        } for column in results.data_table.cols()],
-        'type': 'table'
-    }
+    return {}
 
 
   @query_error_handler
   @query_error_handler
   def fetch_result_metadata(self):
   def fetch_result_metadata(self):
@@ -102,21 +76,6 @@ class MySqlApi(Api):
   def download(self, notebook, snippet, format):
   def download(self, notebook, snippet, format):
     raise PopupException('Downloading is not supported yet')
     raise PopupException('Downloading is not supported yet')
 
 
-  def _progress(self, snippet, logs):
-    if snippet['type'] == 'hive':
-      match = re.search('Total jobs = (\d+)', logs, re.MULTILINE)
-      total = (int(match.group(1)) if match else 1) * 2
-
-      started = logs.count('Starting Job')
-      ended = logs.count('Ended Job')
-
-      return int((started + ended) * 100 / total)
-    elif snippet['type'] == 'impala':
-      match = re.search('(\d+)% Complete', logs, re.MULTILINE)
-      return int(match.group(1)) if match else 0
-    else:
-      return 50
-
   @query_error_handler
   @query_error_handler
   def close_statement(self, snippet):
   def close_statement(self, snippet):
     return {'status': -1}
     return {'status': -1}

+ 1 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -406,7 +406,7 @@ var Snippet = function (vm, notebook, snippet) {
         self.result.handle(data.handle);
         self.result.handle(data.handle);
         self.result.hasResultset(data.handle.has_result_set);
         self.result.hasResultset(data.handle.has_result_set);
         self.checkStatus();
         self.checkStatus();
-      } else {
+      } else {console.log('aaa');
         self._ajaxError(data, self.execute);
         self._ajaxError(data, self.execute);
       }
       }
     }).fail(function (xhr, textStatus, errorThrown) {
     }).fail(function (xhr, textStatus, errorThrown) {

+ 2 - 2
desktop/libs/notebook/src/notebook/views.py

@@ -30,7 +30,7 @@ from notebook.decorators import check_document_access_permission, check_document
 from notebook.connectors.base import Notebook, get_api
 from notebook.connectors.base import Notebook, get_api
 from notebook.management.commands.notebook_setup import Command
 from notebook.management.commands.notebook_setup import Command
 from notebook.connectors.spark_shell import SparkApi
 from notebook.connectors.spark_shell import SparkApi
-from notebook.conf import INTERPRETERS
+from notebook.conf import get_interpreters
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ def notebook(request):
   return render('notebook.mako', request, {
   return render('notebook.mako', request, {
       'notebooks_json': json.dumps([notebook.get_data()]),
       'notebooks_json': json.dumps([notebook.get_data()]),
       'options_json': json.dumps({
       'options_json': json.dumps({
-          'languages': [{"name": INTERPRETERS.get()[i].NAME.get(), "type": i} for i in INTERPRETERS.get()],
+          'languages': get_interpreters(),
           'snippet_placeholders' : {
           'snippet_placeholders' : {
               'sql': _('Example: 1 + 1, or press CTRL + space'),
               'sql': _('Example: 1 + 1, or press CTRL + space'),
               'spark': _('Example: 1 + 1, or press CTRL + space'),
               'spark': _('Example: 1 + 1, or press CTRL + space'),