Browse Source

HUE-1713 [dbquery] Add SQLite backend

Name is required for SQLite backend.
Abraham Elmahrek 12 years ago
parent
commit
058d572

+ 4 - 1
apps/beeswax/src/beeswax/models.py

@@ -46,6 +46,7 @@ BEESWAX = 'beeswax'
 HIVE_SERVER2 = 'hiveserver2'
 HIVE_SERVER2 = 'hiveserver2'
 MYSQL = 'mysql'
 MYSQL = 'mysql'
 POSTGRESQL = 'postgresql'
 POSTGRESQL = 'postgresql'
+SQLITE = 'sqlite'
 QUERY_TYPES = (HQL, IMPALA, RDBMS) = range(3)
 QUERY_TYPES = (HQL, IMPALA, RDBMS) = range(3)
 
 
 
 
@@ -54,7 +55,9 @@ class QueryHistory(models.Model):
   Holds metadata about all queries that have been executed.
   Holds metadata about all queries that have been executed.
   """
   """
   STATE = Enum('submitted', 'running', 'available', 'failed', 'expired')
   STATE = Enum('submitted', 'running', 'available', 'failed', 'expired')
-  SERVER_TYPE = ((BEESWAX, 'Beeswax'), (HIVE_SERVER2, 'Hive Server 2'), (MYSQL, 'MySQL'), (POSTGRESQL, 'PostgreSQL'))
+  SERVER_TYPE = ((BEESWAX, 'Beeswax'), (HIVE_SERVER2, 'Hive Server 2'),
+                 (MYSQL, 'MySQL'), (POSTGRESQL, 'PostgreSQL'),
+                 (SQLITE, 'sqlite'))
 
 
   owner = models.ForeignKey(User, db_index=True)
   owner = models.ForeignKey(User, db_index=True)
   query = models.TextField()
   query = models.TextField()

+ 7 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -55,6 +55,10 @@ def get(user, query_server=None):
     from beeswax.server.postgresql_lib import PostgreSQLClient
     from beeswax.server.postgresql_lib import PostgreSQLClient
 
 
     return Rdbms(PostgreSQLClient(query_server, user), QueryHistory.SERVER_TYPE[2][0])
     return Rdbms(PostgreSQLClient(query_server, user), QueryHistory.SERVER_TYPE[2][0])
+  elif query_server['server_name'] in ('sqlite', 'sqlite3'):
+    from beeswax.server.sqlite_lib import SQLiteClient
+
+    return Rdbms(SQLiteClient(query_server, user), QueryHistory.SERVER_TYPE[2][0])
 
 
 
 
 def get_query_server_config(name='beeswax', server=None):
 def get_query_server_config(name='beeswax', server=None):
@@ -88,6 +92,9 @@ def get_query_server_config(name='beeswax', server=None):
         'password': RDBMS[name].PASSWORD.get(),
         'password': RDBMS[name].PASSWORD.get(),
         'alias': name
         'alias': name
       }
       }
+
+      if RDBMS[name].NAME.get():
+        query_server['name'] = RDBMS[name].NAME.get()
     else:
     else:
       query_server = {}
       query_server = {}
 
 

+ 12 - 4
apps/beeswax/src/beeswax/server/mysql_lib.py

@@ -57,18 +57,26 @@ class MySQLClient(BaseRDMSClient):
 
 
   @property
   @property
   def _conn_params(self):
   def _conn_params(self):
-    return {
+    params = {
       'user': self.query_server['username'],
       'user': self.query_server['username'],
       'passwd': self.query_server['password'],
       'passwd': self.query_server['password'],
       'host': self.query_server['server_host'],
       'host': self.query_server['server_host'],
       'port': self.query_server['server_port']
       'port': self.query_server['server_port']
     }
     }
 
 
+    if 'name' in self.query_server:
+      params['db'] = self.query_server['name']
+
+    return params
+
 
 
   def use(self, database):
   def use(self, database):
-    cursor = self.connection.cursor()
-    cursor.execute("USE %s" % database)
-    self.connection.commit()
+    if 'db' in self._conn_params and self._conn_params['db'] != database:
+      raise RuntimeError("Tried to use database %s when %s was specified." % (database, self._conn_params['db']))
+    else:
+      cursor = self.connection.cursor()
+      cursor.execute("USE %s" % database)
+      self.connection.commit()
 
 
 
 
   def execute_statement(self, statement):
   def execute_statement(self, statement):

+ 11 - 4
apps/beeswax/src/beeswax/server/postgresql_lib.py

@@ -48,18 +48,25 @@ class PostgreSQLClient(BaseRDMSClient):
 
 
   @property
   @property
   def _conn_params(self):
   def _conn_params(self):
-    return {
+    params = {
       'user': self.query_server['username'],
       'user': self.query_server['username'],
       'password': self.query_server['password'],
       'password': self.query_server['password'],
       'host': self.query_server['server_host'],
       'host': self.query_server['server_host'],
       'port': self.query_server['server_port'] == 0 and 5432 or self.query_server['server_port']
       'port': self.query_server['server_port'] == 0 and 5432 or self.query_server['server_port']
     }
     }
+    if 'name' in self.query_server:
+      params['database'] = self.query_server['name']
+    return params
 
 
 
 
   def use(self, database):
   def use(self, database):
-    conn_params = self._conn_params
-    conn_params['database'] = database
-    self.connection = Database.connect(**conn_params)
+    # No op if a database has been specified.
+    if 'database' in self._conn_params and self._conn_params['database'] != database:
+      raise RuntimeError("Tried to use database %s when %s was specified." % (database, self._conn_params['db']))
+    else:
+      conn_params = self._conn_params
+      conn_params['database'] = database
+      self.connection = Database.connect(**conn_params)
 
 
 
 
   def execute_statement(self, statement):
   def execute_statement(self, statement):

+ 92 - 0
apps/beeswax/src/beeswax/server/sqlite_lib.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+try:
+  try:
+    from pysqlite2 import dbapi2 as Database
+  except ImportError, e1:
+    from sqlite3 import dbapi2 as Database
+except ImportError, exc:
+  from django.core.exceptions import ImproperlyConfigured
+  raise ImproperlyConfigured("Error loading either pysqlite2 or sqlite3 modules (tried in that order): %s" % exc)
+
+from beeswax.server.rdbms_base_lib import BaseRDBMSDataTable, BaseRDBMSResult, BaseRDMSClient
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DataTable(BaseRDBMSDataTable): pass
+
+
+class Result(BaseRDBMSResult): pass
+
+
+class SQLiteClient(BaseRDMSClient):
+  """Same API as Beeswax"""
+
+  data_table_cls = DataTable
+  result_cls = Result
+
+  def __init__(self, *args, **kwargs):
+    super(SQLiteClient, self).__init__(*args, **kwargs)
+    self.connection = Database.connect(**self._conn_params)
+
+
+  @property
+  def _conn_params(self):
+    return {
+      'database': self.query_server['name'],
+      'detect_types': Database.PARSE_DECLTYPES | Database.PARSE_COLNAMES,
+    }
+
+
+  def use(self, database):
+    # Do nothing because SQLite has one database per path.
+    pass
+
+
+  def execute_statement(self, statement):
+    cursor = self.connection.cursor()
+    cursor.execute(statement)
+    self.connection.commit()
+    if cursor.description:
+      columns = [column[0] for column in cursor.description]
+    else:
+      columns = []
+    return self.data_table_cls(cursor, columns)
+
+
+  def get_databases(self):
+    return [self._conn_params['database']]
+
+
+  def get_tables(self, database, table_names):
+    # Doesn't use database and only retrieves tables for database currently in use.
+    cursor = self.connection.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    self.connection.commit()
+    return [row[0] for row in cursor.fetchall()]
+
+
+  def get_columns(self, database, table):
+    cursor = self.connection.cursor()
+    cursor.execute("PRAGMA table_info(%s)" % table)
+    self.connection.commit()
+    return [row[1] for row in cursor.fetchall()]

+ 14 - 1
apps/rdbms/src/rdbms/conf.py

@@ -15,11 +15,13 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-from django.utils.translation import ugettext_lazy as _t
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
 from desktop.lib.conf import Config, UnspecifiedConfigSection,\
 from desktop.lib.conf import Config, UnspecifiedConfigSection,\
                              ConfigSection, coerce_json_dict
                              ConfigSection, coerce_json_dict
 from desktop.conf import coerce_database
 from desktop.conf import coerce_database
 
 
+from rdbms.settings import NICE_NAME
+
 
 
 RDBMS = UnspecifiedConfigSection(
 RDBMS = UnspecifiedConfigSection(
   key="databases",
   key="databases",
@@ -33,6 +35,12 @@ RDBMS = UnspecifiedConfigSection(
         type=str,
         type=str,
         default=None,
         default=None,
       ),
       ),
+      NAME=Config(
+        key='name',
+        help=_t('Database name or path to database file. If provided, then choosing other databases will not be permitted.'),
+        type=str,
+        default='',
+      ),
       ENGINE=Config(
       ENGINE=Config(
         key='engine',
         key='engine',
         help=_t('Database engine, such as postgresql_psycopg2, mysql, or sqlite3.'),
         help=_t('Database engine, such as postgresql_psycopg2, mysql, or sqlite3.'),
@@ -76,6 +84,11 @@ RDBMS = UnspecifiedConfigSection(
 
 
 def config_validator(user):
 def config_validator(user):
   res = []
   res = []
+
+  for server in RDBMS:
+    if RDBMS[server].ENGINE.get().split('.')[-1] in ('sqlite', 'sqlite3') and not RDBMS[server].NAME.get():
+      res.append((RDBMS[server].NAME, _("Database name should not be empty for SQLite backends. The %s may not work correctly.") % NICE_NAME))
+
   return res
   return res
 
 
 
 

+ 1 - 0
desktop/conf.dist/hue.ini

@@ -635,6 +635,7 @@
     # mysql or postgresql
     # mysql or postgresql
     ## [[[mydb]]]
     ## [[[mydb]]]
     ## nice_name=
     ## nice_name=
+    ## name=
     ## engine=sqlite3
     ## engine=sqlite3
     ## host=
     ## host=
     ## port=
     ## port=

+ 1 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -640,6 +640,7 @@
     # mysql or postgresql
     # mysql or postgresql
     ## [[[mydb]]]
     ## [[[mydb]]]
     ## nice_name=
     ## nice_name=
+    ## name=
     ## engine=sqlite3
     ## engine=sqlite3
     ## host=
     ## host=
     ## port=
     ## port=