Преглед на файлове

[notebook] Add RDBMS interface and corresponding connectors to notebook

Jenny Kim преди 10 години
родител
ревизия
b704ede

+ 19 - 3
desktop/conf.dist/hue.ini

@@ -577,6 +577,10 @@
       name=Impala
       interface=hiveserver2
 
+    # [[[sparksql]]]
+    #   name=SparkSql
+    #   interface=hiveserver2
+
     [[[spark]]]
       name=Scala
       interface=livy
@@ -609,9 +613,21 @@
       name=Markdown
       interface=text
 
-    # [[[sparksql]]]
-    #   name=SparkSql
-    #   interface=hiveserver2
+    [[[mysql]]]
+      name = MySQL
+      interface=rdbms
+
+    [[[sqlite]]]
+      name = SQLite
+      interface=rdbms
+
+    [[[postgresql]]]
+      name = PostgreSQL
+      interface=rdbms
+
+    [[[oracle]]]
+      name = Oracle
+      interface=rdbms
 
     # [[[mysql]]]
     #   name=MySql JDBC

+ 19 - 3
desktop/conf/pseudo-distributed.ini.tmpl

@@ -581,6 +581,10 @@
       name=Impala
       interface=hiveserver2
 
+    # [[[sparksql]]]
+    #   name=SparkSql
+    #   interface=hiveserver2
+
     [[[spark]]]
       name=Scala
       interface=livy
@@ -613,9 +617,21 @@
       name=Markdown
       interface=text
 
-    # [[[sparksql]]]
-    #   name=SparkSql
-    #   interface=hiveserver2
+    [[[mysql]]]
+      name = MySQL
+      interface=rdbms
+
+    [[[sqlite]]]
+      name = SQLite
+      interface=rdbms
+
+    [[[postgresql]]]
+      name = PostgreSQL
+      interface=rdbms
+
+    [[[oracle]]]
+      name = Oracle
+      interface=rdbms
 
     # [[[mysql]]]
     #   name=MySql JDBC

+ 6 - 3
desktop/libs/librdbms/src/librdbms/server/dbms.py

@@ -88,14 +88,17 @@ class Rdbms(object):
   def get_databases(self):
     return self.client.get_databases()
 
-  def get_tables(self, database):
+  def get_tables(self, database, table_names=None):
     return self.client.get_tables(database)
 
   def get_table(self, database, table_name):
     return self.client.get_table(database, table_name)
 
-  def get_columns(self, database, table_name):
-    return self.client.get_columns(database, table_name)
+  def get_columns(self, database, table_name, names_only=True):
+    return self.client.get_columns(database, table_name, names_only)
+
+  def execute_statement(self, statement):
+    return self.client.execute_statement(statement)
 
   def execute_query(self, query, design):
     from beeswax.models import QueryHistory

+ 11 - 3
desktop/libs/librdbms/src/librdbms/server/mysql_lib.py

@@ -111,13 +111,21 @@ class MySQLClient(BaseRDMSClient):
 
   def get_tables(self, database, table_names=[]):
     cursor = self.connection.cursor()
-    cursor.execute("SHOW TABLES")
+    query = 'SHOW TABLES'
+    if table_names:
+      clause = ' OR '.join(["`Tables_in_%(database)s` LIKE '%%%(table)s%%'" % {'database': database, 'table': table} for table in table_names])
+      query += ' FROM `%(database)s` WHERE (%(clause)s)' % {'database': database, 'clause': clause}
+    cursor.execute(query)
     self.connection.commit()
     return [row[0] for row in cursor.fetchall()]
 
 
-  def get_columns(self, database, table):
+  def get_columns(self, database, table, names_only=True):
     cursor = self.connection.cursor()
     cursor.execute("SHOW COLUMNS FROM %s.%s" % (database, table))
     self.connection.commit()
-    return [row[0] for row in cursor.fetchall()]
+    if names_only:
+      columns = [row[0] for row in cursor.fetchall()]
+    else:
+      columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
+    return columns

+ 12 - 4
desktop/libs/librdbms/src/librdbms/server/oracle_lib.py

@@ -89,13 +89,21 @@ class OracleClient(BaseRDMSClient):
 
   def get_tables(self, database, table_names=[]):
     cursor = self.connection.cursor()
-    cursor.execute("SELECT table_name FROM all_tables")
+    query = "SELECT table_name FROM user_tables"
+    if table_names:
+      clause = ' OR '.join(["table_name LIKE '%%%(table)s%%'" % {'table': table} for table in table_names])
+      query += ' WHERE (%s)' % clause
+    cursor.execute(query)
     self.connection.commit()
     return [row[0] for row in cursor.fetchall()]
 
 
-  def get_columns(self, database, table):
+  def get_columns(self, database, table, names_only=True):
     cursor = self.connection.cursor()
-    cursor.execute("SELECT column_name FROM user_tab_cols WHERE table_name = '%s'" % table)
+    cursor.execute("SELECT column_name, data_type FROM user_tab_cols WHERE table_name = '%s'" % table)
     self.connection.commit()
-    return [row[0] for row in cursor.fetchall()]
+    if names_only:
+      columns = [row[0] for row in cursor.fetchall()]
+    else:
+      columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
+    return columns

+ 33 - 6
desktop/libs/librdbms/src/librdbms/server/postgresql_lib.py

@@ -85,23 +85,50 @@ class PostgreSQLClient(BaseRDMSClient):
     # List all the schemas in the database
     try:
       cursor = self.connection.cursor()
-      cursor.execute('SELECT schema_name FROM information_schema.schemata')
+      cursor.execute('SELECT nspname from pg_catalog.pg_namespace')
       self.connection.commit()
       return [row[0] for row in cursor.fetchall()]
     except Exception:
-      LOG.exception('Failed to select schema_name from information_schema')
+      LOG.exception('Failed to select nspname from pg_catalog.pg_namespace')
       return [self._conn_params['database']]
 
 
   def get_tables(self, database, table_names=[]):
     cursor = self.connection.cursor()
-    cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema='%s'" % database)
+    query = "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = '%s'" % database
+    if table_names:
+      clause = ' OR '.join(["tablename LIKE '%%%(table)s%%'" % {'table': table} for table in table_names])
+      query += ' AND (%s)' % clause
+    cursor.execute(query)
     self.connection.commit()
     return [row[0] for row in cursor.fetchall()]
 
 
-  def get_columns(self, database, table):
+  def get_columns(self, database, table, names_only=True):
     cursor = self.connection.cursor()
-    cursor.execute("SELECT column_name FROM information_schema.columns WHERE table_schema='%s' and table_name='%s'" % (database, table))
+    query = """
+      SELECT
+          a.attname as "name",
+          pg_catalog.format_type(a.atttypid, a.atttypmod) as "datatype"
+      FROM
+          pg_catalog.pg_attribute a
+      WHERE
+          a.attnum > 0
+          AND NOT a.attisdropped
+          AND a.attrelid = (
+              SELECT c.oid
+              FROM pg_catalog.pg_class c
+                  LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+              WHERE c.relname ~ '^(%(table)s)$'
+                  AND n.nspname = '%(database)s'
+                  AND pg_catalog.pg_table_is_visible(c.oid)
+          )
+    """ % {'table': table, 'database': database}
+
+    cursor.execute(query)
     self.connection.commit()
-    return [row[0] for row in cursor.fetchall()]
+    if names_only:
+      columns = [row[0] for row in cursor.fetchall()]
+    else:
+      columns = [dict(name=row[0], type=row[1], comment='') for row in cursor.fetchall()]
+    return columns

+ 11 - 3
desktop/libs/librdbms/src/librdbms/server/sqlite_lib.py

@@ -88,13 +88,21 @@ class SQLiteClient(BaseRDMSClient):
   def get_tables(self, database, table_names=[]):
     # Doesn't use database and only retrieves tables for database currently in use.
     cursor = self.connection.cursor()
-    cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    query = "SELECT name FROM sqlite_master WHERE type='table'"
+    if table_names:
+      clause = ' OR '.join(["tablename LIKE '%%%(table)s%%'" % {'table': table} for table in table_names])
+      query += ' AND (%s)' % clause
+    cursor.execute(query)
     self.connection.commit()
     return [row[0] for row in cursor.fetchall()]
 
 
-  def get_columns(self, database, table):
+  def get_columns(self, database, table, names_only=True):
     cursor = self.connection.cursor()
     cursor.execute("PRAGMA table_info(%s)" % table)
     self.connection.commit()
-    return [row[1] for row in cursor.fetchall()]
+    if names_only:
+      columns = [row[1] for row in cursor.fetchall()]
+    else:
+      columns = [dict(name=row[1], type=row[2], comment='') for row in cursor.fetchall()]
+    return columns

+ 1 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -349,7 +349,7 @@ def close_statement(request):
 @require_POST
 @check_document_access_permission()
 @api_error_handler
-def autocomplete(request, database=None, table=None, column=None, nested=None):
+def autocomplete(request, server=None, database=None, table=None, column=None, nested=None):
   response = {'status': -1}
 
   # Passed by check_document_access_permission but unused by APIs

+ 3 - 4
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -81,13 +81,12 @@ class Notebook(object):
 def get_api(user, snippet, fs, jt):
   from notebook.connectors.hiveserver2 import HS2Api
   from notebook.connectors.jdbc import JdbcApi
-  from notebook.connectors.mysql import MySqlApi
+  from notebook.connectors.rdbms import RdbmsApi
   from notebook.connectors.pig_batch import PigApi
   from notebook.connectors.spark_shell import SparkApi
   from notebook.connectors.spark_batch import SparkBatchApi
   from notebook.connectors.text import TextApi
 
-
   interpreter = [interpreter for interpreter in get_interpreters(user) if interpreter['type'] == snippet['type']]
   if not interpreter:
     raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
@@ -102,8 +101,8 @@ def get_api(user, snippet, fs, jt):
     return SparkBatchApi(user)
   elif interface == 'text' or interface == 'markdown':
     return TextApi(user)
-  elif interface == 'mysql':
-    return MySqlApi(user)
+  elif interface == 'rdbms':
+    return RdbmsApi(user, interpreter=snippet['type'])
   elif interface == 'jdbc':
     return JdbcApi(user, interpreter=interpreter)
   elif interface == 'pig':

+ 44 - 31
desktop/libs/notebook/src/notebook/connectors/mysql.py → desktop/libs/notebook/src/notebook/connectors/rdbms.py

@@ -21,7 +21,6 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 
 from librdbms.server import dbms
-from librdbms.server.mysql_lib import MySQLClient
 
 from notebook.connectors.base import Api, QueryError, QueryExpired
 
@@ -42,13 +41,14 @@ def query_error_handler(func):
   return decorator
 
 
-class MySqlApi(Api):
+class RdbmsApi(Api):
 
+  @query_error_handler
   def execute(self, notebook, snippet):
-    query_server = dbms.get_query_server_config(server='mysql')
-    db = MySQLClient(query_server, self.user)
+    query_server = dbms.get_query_server_config(server=self.interpreter)
+    db = dbms.get(self.user, query_server)
 
-    table = db.execute_statement(snippet['statement'])
+    table = db.execute_statement(snippet['statement'])  # TODO: execute statement stub in Rdbms
 
     data = table.rows()
     has_result_set = data is not None
@@ -56,6 +56,7 @@ class MySqlApi(Api):
     return {
       'sync': True,
       'has_result_set': has_result_set,
+      'modified_row_count': 0,
       'result': {
         'has_more': False,
         'data': data if has_result_set else [],
@@ -68,61 +69,73 @@ class MySqlApi(Api):
       }
     }
 
+
   @query_error_handler
   def check_status(self, notebook, snippet):
     return {'status': 'available'}
 
-  def _fetch_result(self, cursor):
-    return {}
+
+  @query_error_handler
+  def fetch_result(self, notebook, snippet, rows, start_over):
+    return {
+      'has_more': False,
+      'data': [],
+      'meta': [],
+      'type': 'table'
+    }
+
 
   @query_error_handler
   def fetch_result_metadata(self):
     pass
 
+
   @query_error_handler
   def cancel(self, notebook, snippet):
     return {'status': 0}
 
+
   @query_error_handler
   def get_log(self, notebook, snippet, startFrom=None, size=None):
     return 'No logs'
 
+
   def download(self, notebook, snippet, format):
     raise PopupException('Downloading is not supported yet')
 
+
   @query_error_handler
   def close_statement(self, snippet):
     return {'status': -1}
 
+
   @query_error_handler
   def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
-    query_server = dbms.get_query_server_config(server='mysql')
-    db = MySQLClient(query_server, self.user)
+    query_server = dbms.get_query_server_config(server=self.interpreter)
+    db = dbms.get(self.user, query_server)
 
     assist = Assist(db)
-    response = {'error': 0}
-
-    try:
-      if database is None:
-        response['databases'] = assist.get_databases()
-      elif table is None:
-        response['tables'] = assist.get_tables(database)
-      else:
-        columns = assist.get_columns(database, table)
-        response['columns'] = [col for col in columns]
-        response['extended_columns'] = [{
-            'name': col,
-            'type': '',
-            'comment': ''
-          } for col in columns
-        ]
-    except Exception, e:
-      LOG.warn('Autocomplete data fetching error: %s' % e)
-      response['code'] = -1
-      response['error'] = str(e)
-
+    response = {'status': -1}
+
+    if database is None:
+      response['databases'] = assist.get_databases()
+    elif table is None:
+      tables_meta = []
+      for t in assist.get_tables(database):
+        tables_meta.append({'name': t, 'type': 'Table', 'comment': ''})
+      response['tables_meta'] = tables_meta
+    else:
+      columns = assist.get_columns(database, table)
+      response['columns'] = [col['name'] for col in columns]
+      response['extended_columns'] = columns
+
+    response['status'] = 0
     return response
 
+  @query_error_handler
+  def get_select_star_query(self, snippet, database, table):
+    return "SELECT * FROM `%s`.`%s`" % (database, table)
+
 
 class Assist():
 
@@ -136,4 +149,4 @@ class Assist():
     return self.db.get_tables(database, table_names)
 
   def get_columns(self, database, table):
-    return self.db.get_columns(database, table)
+    return self.db.get_columns(database, table, names_only=False)

+ 13 - 1
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1698,12 +1698,18 @@ ${ require.config() }
           snippetIcon: 'fa-database',
           sqlDialect: true
         },
+        oracle: {
+          placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
+          aceMode: 'ace/mode/oracle',
+          snippetIcon: 'fa-database',
+          sqlDialect: true
+        },
         pig: {
           placeHolder: '${ _("Example: 1 + 1, or press CTRL + space") }',
           aceMode: 'ace/mode/pig',
           snippetImage: '${ static("pig/art/icon_pig_48.png") }'
         },
-        pgsql: {
+        postgresql: {
           placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
           aceMode: 'ace/mode/pgsql',
           snippetIcon: 'fa-database',
@@ -1727,6 +1733,12 @@ ${ require.config() }
           aceMode: 'ace/mode/scala',
           snippetImage: '${ static("spark/art/icon_spark_48.png") }'
         },
+        sqlite: {
+          placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
+          aceMode: 'ace/mode/sqlite',
+          snippetIcon: 'fa-database',
+          sqlDialect: true
+        },
         text: {
           placeHolder: '${ _('Type your text here') }',
           aceMode: 'ace/mode/text',

+ 3 - 0
desktop/libs/notebook/src/notebook/urls.py

@@ -79,4 +79,7 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'autocomplete', name='api_autocomplete_column'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/(?P<nested>.+)/?$', 'autocomplete', name='api_autocomplete_nested'),
+  # SQLite
+  url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/?$', 'autocomplete', name='api_autocomplete_tables'),
+  url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
 )

+ 4 - 5
desktop/libs/notebook/src/notebook/views.py

@@ -19,18 +19,17 @@ import json
 import logging
 
 from django.db.models import Q
-from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import render, JsonResponse
 from desktop.lib.json_utils import JSONEncoderForHTML
 from desktop.models import Document2, Document
 
-from notebook.decorators import check_document_access_permission, check_document_modify_permission
+from notebook.conf import get_interpreters
 from notebook.connectors.base import Notebook, get_api
-from notebook.management.commands.notebook_setup import Command
 from notebook.connectors.spark_shell import SparkApi
-from notebook.conf import get_interpreters
+from notebook.decorators import check_document_access_permission, check_document_modify_permission
+from notebook.management.commands.notebook_setup import Command
 from notebook.models import make_notebook
 
 
@@ -83,7 +82,7 @@ def editor(request):
     editor = Notebook()
     data = editor.get_data()
     data['name'] = 'Untitled %s Query' % editor_type.title()
-    data['type'] = 'query-%s' % editor_type
+    data['type'] = 'query-%s' % editor_type  # TODO: Add handling for non-SQL types
     editor.data = json.dumps(data)
 
   return render('editor.mako', request, {