Quellcode durchsuchen

HUE-5747 [librdms] Add column types to MySQL queries

Romain Rigaux vor 8 Jahren
Ursprung
Commit
6fff6435ac

+ 50 - 1
desktop/libs/librdbms/src/librdbms/server/mysql_lib.py

@@ -33,6 +33,7 @@ if (version < (1,2,1) or (version[:3] == (1, 2, 1) and
     raise ImproperlyConfigured("MySQLdb-1.2.1p2 or newer is required; you have %s" % Database.__version__)
 
 from django.utils.translation import ugettext as _
+from MySQLdb.converters import FIELD_TYPE
 
 from librdbms.server.rdbms_base_lib import BaseRDBMSDataTable, BaseRDBMSResult, BaseRDMSClient
 
@@ -46,6 +47,53 @@ class DataTable(BaseRDBMSDataTable): pass
 class Result(BaseRDBMSResult): pass
 
 
+def _convert_types(t):
+  if t == FIELD_TYPE.DECIMAL:
+    return 'DECIMAL_TYPE'
+  elif t == FIELD_TYPE.TINY:
+    return 'TINYINT_TYPE'
+  elif t == FIELD_TYPE.SHORT:
+    return 'SMALLINT_TYPE'
+  elif t == FIELD_TYPE.LONG:
+    return 'BIGINT_TYPE'
+  elif t == FIELD_TYPE.FLOAT:
+    return 'FLOAT_TYPE'
+  elif t == FIELD_TYPE.DOUBLE:
+    return 'DOUBLE_TYPE'
+  elif t == FIELD_TYPE.NULL:
+    return 'NULL_TYPE'
+  elif t == FIELD_TYPE.LONGLONG:
+    return 'BIGINT_TYPE'
+  elif t == FIELD_TYPE.INT24:
+    return 'INT_TYPE'
+  elif t == FIELD_TYPE.TIMESTAMP:
+    return 'TIMESTAMP_TYPE'
+  elif t == FIELD_TYPE.DATE:
+    return 'DATE_TYPE'
+  elif t == FIELD_TYPE.YEAR:
+    return 'INT_TYPE'
+  elif t == FIELD_TYPE.NEWDATE:
+    return 'DATE_TYPE'
+  elif t == FIELD_TYPE.VARCHAR:
+    return 'VARCHAR_TYPE'
+  elif t == FIELD_TYPE.BIT:
+    return 'BOOLEAN_TYPE'
+  elif t == FIELD_TYPE.NEWDECIMAL:
+    return 'DECIMAL_TYPE'
+  elif t == FIELD_TYPE.ENUM:
+    return 'INT_TYPE'
+  elif t == FIELD_TYPE.SET:
+    return 'ARRAY_TYPE'
+  elif t == FIELD_TYPE.TINY_BLOB:
+    return 'BINARY_TYPE'
+  elif t == FIELD_TYPE.MEDIUM_BLOB:
+    return 'BINARY_TYPE'
+  elif t == FIELD_TYPE.LONG_BLOB:
+    return 'BINARY_TYPE'
+  else:
+    return 'STRING_TYPE'
+
+
 class MySQLClient(BaseRDMSClient):
   """Same API as Beeswax"""
 
@@ -88,8 +136,9 @@ class MySQLClient(BaseRDMSClient):
     cursor = self.connection.cursor()
     cursor.execute(statement)
     self.connection.commit()
+
     if cursor.description:
-      columns = [column[0] for column in cursor.description]
+      columns = [{'name': column[0], 'type': _convert_types(column[1])} for column in cursor.description]
     else:
       columns = []
     return self.data_table_cls(cursor, columns)

+ 7 - 1
desktop/libs/librdbms/src/librdbms/server/rdbms_base_lib.py

@@ -24,7 +24,13 @@ LOG = logging.getLogger(__name__)
 class BaseRDBMSDataTable(object):
   def __init__(self, cursor, columns, fetch_size=1000):
     self.cursor = cursor
-    self.columns = columns
+    if columns and isinstance(columns[0], dict): # Bacward compatible for API without column metadata
+      self.columns_description = columns
+      self.columns = [col['name'] for col in columns]
+    else:
+      self.columns_description = [{'name': col} for col in columns]
+      self.columns = columns
+    self.columns_description = columns
     self.next = None
     self.startRowOffset = 0
     self.fetchSize = 1000

+ 5 - 4
desktop/libs/notebook/src/notebook/connectors/rdbms.py

@@ -53,7 +53,8 @@ class RdbmsApi(Api):
 
     data = list(table.rows())
     has_result_set = data is not None
-
+    print table.columns
+    print table.columns_description
     return {
       'sync': True,
       'has_result_set': has_result_set,
@@ -62,10 +63,10 @@ class RdbmsApi(Api):
         'has_more': False,
         'data': data if has_result_set else [],
         'meta': [{
-          'name': col,
-          'type': '',
+          'name': col['name'],
+          'type': col.get('type', ''),
           'comment': ''
-        } for col in table.columns] if has_result_set else [],
+        } for col in table.columns_description] if has_result_set else [],
         'type': 'table'
       }
     }

+ 4 - 4
desktop/libs/notebook/src/notebook/connectors/spark_shell.py

@@ -201,10 +201,10 @@ class SparkApi(Api):
     LOG.debug("Property Spark Conf kvp list from UI is: " + str(listitems))
     confDict = {}
     for i in range(len(listitems)):
-       kvp = listitems[i]
-       LOG.debug("Property Spark Conf key " + str(i) + " = " + str(kvp.get('key')))
-       LOG.debug("Property Spark Conf value " + str(i) + " = " + str(kvp.get('value')))
-       confDict[kvp.get('key')] = kvp.get('value')
+      kvp = listitems[i]
+      LOG.debug("Property Spark Conf key " + str(i) + " = " + str(kvp.get('key')))
+      LOG.debug("Property Spark Conf value " + str(i) + " = " + str(kvp.get('value')))
+      confDict[kvp.get('key')] = kvp.get('value')
     props['conf'] = confDict
     LOG.debug("Property Spark Conf dictionary is: " + str(confDict))