Browse Source

HUE-2962 [desktop] Add configuration property definition for HS2

Hive and Impala configuration classes
Add get_configurable_apps API endpoint
Add upgrade_properties method and test
Jenny Kim 9 years ago
parent
commit
8c88233

+ 8 - 1
apps/impala/src/impala/conf.py

@@ -22,7 +22,7 @@ import socket
 from django.utils.translation import ugettext_lazy as _t, ugettext as _
 from desktop.conf import default_ssl_cacerts, default_ssl_validate, AUTH_USERNAME as DEFAULT_AUTH_USERNAME,\
   AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD
-from desktop.lib.conf import ConfigSection, Config, coerce_bool, coerce_password_from_script
+from desktop.lib.conf import ConfigSection, Config, coerce_bool, coerce_csv, coerce_password_from_script
 from desktop.lib.exceptions import StructuredThriftTransportException
 
 from impala.settings import NICE_NAME
@@ -91,6 +91,13 @@ SESSION_TIMEOUT_S = Config(
   default=12 * 60 * 60
 )
 
+CONFIG_WHITELIST = Config(
+  key='config_whitelist',
+  default='debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s',
+  type=coerce_csv,
+  help=_t('A comma-separated list of white-listed Impala configuration properties that users are authorized to set.')
+)
+
 SSL = ConfigSection(
   key='ssl',
   help=_t('SSL configuration for the server.'),

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -889,6 +889,9 @@
   ## auth_username=hue
   ## auth_password=
 
+  # A comma-separated list of white-listed Impala configuration properties that users are authorized to set.
+  # config_whitelist=debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s
+
   [[ssl]]
     # SSL communication enabled for this server.
     ## enabled=false

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -893,6 +893,9 @@
   ## auth_username=hue
   ## auth_password=
 
+  # A comma-separated list of white-listed Impala configuration properties that users are authorized to set.
+  # config_whitelist=debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s
+
   [[ssl]]
     # SSL communication enabled for this server.
     ## enabled=false

+ 24 - 8
desktop/core/src/desktop/configuration/api.py

@@ -26,6 +26,8 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 from desktop.models import DefaultConfiguration
 
+from notebook.connectors.hiveserver2 import HiveConfiguration, ImpalaConfiguration
+
 
 LOG = logging.getLogger(__name__)
 
@@ -49,14 +51,28 @@ def api_error_handler(func):
 
 @api_error_handler
 def get_configurable_apps(request):
-  # TODO: dynamically register apps that are configurable and return here
-  app_configs = []
-  try:
-    from notebook.connectors.hiveserver2 import HS2Api
-    app_configs.append(HS2Api.HIVE_PROPERTIES)
-    app_configs.append(HS2Api.IMPALA_PROPERTIES)
-  except ImportError, e:
-    LOG.error('Failed to import notebook libs')
+  # TODO: Use metaclasses to self-register configurable apps
+  app_configs = {}
+  config_classes = [HiveConfiguration, ImpalaConfiguration]
+
+  for config_cls in config_classes:
+    if not hasattr(config_cls, 'APP_NAME') or not hasattr(config_cls, 'PROPERTIES'):
+      LOG.exception('Configurable classes must define APP_NAME and PROPERTIES.')
+    app_name = config_cls.APP_NAME
+    app_configs[app_name] = {
+      'properties': config_cls.PROPERTIES
+    }
+
+    # Get default config
+    if DefaultConfiguration.objects.filter(app=app_name, is_default=True).exists():
+      default_config = DefaultConfiguration.objects.get(app=app_name, is_default=True)
+      app_configs[app_name].update({'default': default_config.properties_list})
+
+    # Get group configs
+    if DefaultConfiguration.objects.filter(app=app_name, group__isnull=False).exists():
+      app_configs[app_name].update({'groups': {}})
+      for grp_config in DefaultConfiguration.objects.filter(app=app_name, group_isnull=False).all():
+        app_configs[app_name]['groups'].update({grp_config.group.id: grp_config.properties_list})
 
   return JsonResponse({
     'status': 0,

+ 13 - 5
desktop/core/src/desktop/configuration/tests.py

@@ -50,9 +50,17 @@ class TestDefaultConfiguration(object):
   def test_save_default_configuration(self):
     app = 'hive'
     is_default = True
-    properties = {
-        'settings': [{'key': 'hive.execution.engine', 'value': 'spark'}]
-    }
+    properties = [
+        {
+          "multiple": True,
+          "value": [],
+          "nice_name": "Settings",
+          "key": "settings",
+          "help_text": "Impala configuration properties.",
+          "type": "settings",
+          "options": []
+        }
+    ]
 
     # Create new default configuration
     configs = DefaultConfiguration.objects.filter(app=app, is_default=is_default)
@@ -67,7 +75,7 @@ class TestDefaultConfiguration(object):
     assert_true('configuration' in content, content)
 
     config = DefaultConfiguration.objects.get(app=app, is_default=is_default)
-    assert_equal(config.properties_dict, properties, config.properties_dict)
+    assert_equal(config.properties_list, properties, config.properties_list)
 
     # Update same default configuration
     properties = {
@@ -83,7 +91,7 @@ class TestDefaultConfiguration(object):
     assert_true('configuration' in content, content)
 
     config = DefaultConfiguration.objects.get(app=app, is_default=is_default)
-    assert_equal(config.properties_dict, properties, config.properties_dict)
+    assert_equal(config.properties_list, properties, config.properties_list)
 
 
   def test_get_default_configurations(self):

+ 4 - 5
desktop/core/src/desktop/models.py

@@ -39,7 +39,6 @@ from desktop import appmanager
 from desktop.lib.i18n import force_unicode
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.redaction import global_redaction_engine
-from notebook.models import make_notebook
 
 
 LOG = logging.getLogger(__name__)
@@ -106,7 +105,7 @@ class DefaultConfiguration(models.Model):
   Can be designated as default for all users by is_default flag, or for a specific group or user
   """
   app = models.CharField(max_length=32, null=False, db_index=True, help_text=_t('App that this configuration belongs to.'))
-  properties = models.TextField(default='{}', help_text=_t('JSON-formatted default properties values.'))
+  properties = models.TextField(default='[]', help_text=_t('JSON-formatted default properties values.'))
 
   is_default = models.BooleanField(default=False, db_index=True)
   group = models.ForeignKey(auth_models.Group, blank=True, null=True, db_index=True)
@@ -120,15 +119,15 @@ class DefaultConfiguration(models.Model):
 
 
   @property
-  def properties_dict(self):
+  def properties_list(self):
     if not self.properties:
-      self.properties = json.dumps({})
+      self.properties = json.dumps([])
     return json.loads(self.properties)
 
   def to_dict(self):
     return {
       'app': self.app,
-      'properties': self.properties_dict,
+      'properties': self.properties_list,
       'is_default': self.is_default,
       'group': self.group.name if self.group else None,
       'user': self.user.username if self.user else None

+ 9 - 6
desktop/libs/notebook/src/notebook/api.py

@@ -50,12 +50,6 @@ def create_session(request):
 
   properties = session.get('properties', [])
 
-  # If not properties look for previously used notebook session
-  if not properties:
-    old_session = [_session for _session in notebook['sessions'] if _session['type'] == session['type']]
-    if any(old_session) and 'properties' in old_session[0]:
-      properties = old_session[0]['properties']
-
   response['session'] = get_api(request, session).create_session(lang=session['type'], properties=properties)
   response['status'] = 0
 
@@ -342,6 +336,15 @@ def open_notebook(request):
   notebook_id = request.GET.get('notebook')
   notebook = Notebook(document=Document2.objects.get(id=notebook_id))
 
+  # Check session properties format and upgrade if necessary
+  data = notebook.get_data()
+  for session in data['sessions']:
+    api = get_api(request, session)
+    if 'type' in session and hasattr(api, 'upgrade_properties'):
+      properties = session.get('properties', None)
+      session['properties'] = api.upgrade_properties(session['type'], properties)
+  notebook.data = json.dumps(data)
+
   response['status'] = 0
   response['notebook'] = notebook.get_json()
   response['message'] = _('Notebook loaded successfully')

+ 145 - 53
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import logging
 import re
 import sys
@@ -28,6 +29,7 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
+from desktop.models import DefaultConfiguration
 
 from notebook.connectors.base import Api, QueryError, QueryExpired
 
@@ -38,6 +40,7 @@ LOG = logging.getLogger(__name__)
 try:
   from beeswax import data_export
   from beeswax.api import _autocomplete, _get_sample_data
+  from beeswax.conf import CONFIG_WHITELIST as hive_settings
   from beeswax.data_export import upload
   from beeswax.design import hql_query, strip_trailing_semicolon, split_statements
   from beeswax import conf as beeswax_conf
@@ -48,6 +51,12 @@ try:
 except ImportError, e:
   LOG.exception('Hive and HiveServer2 interfaces are not enabled')
 
+try:
+  from impala.conf import CONFIG_WHITELIST as impala_settings
+except ImportError, e:
+  LOG.warn("Impala app is not enabled")
+  impala_settings = None
+
 
 DEFAULT_HIVE_ENGINE = 'mr'
 
@@ -65,8 +74,61 @@ def query_error_handler(func):
   return decorator
 
 
+class HiveConfiguration(object):
+
+  APP_NAME = 'hive'
+
+  PROPERTIES = [
+    {
+      "multiple": True,
+      "value": [],
+      "nice_name": _("Files"),
+      "key": "files",
+      "help_text": _("Add one or more files, jars, or archives to the list of resources."),
+      "type": "hdfs-files"
+    }, {
+      "multiple": True,
+      "value": [],
+      "nice_name": _("Functions"),
+      "key": "functions",
+      "help_text": _("Add one or more registered UDFs (requires function name and fully-qualified class name)."),
+      "type": "functions"
+    }, {
+      "multiple": True,
+      "value": [],
+      "nice_name": _("Settings"),
+      "key": "settings",
+      "help_text": _("Hive and Hadoop configuration properties."),
+      "type": "settings",
+      "options": [config.lower() for config in hive_settings.get()]
+    }
+  ]
+
+
+class ImpalaConfiguration(object):
+
+  APP_NAME = 'impala'
+
+  PROPERTIES = [
+    {
+      "multiple": True,
+      "value": [],
+      "nice_name": _("Settings"),
+      "key": "settings",
+      "help_text": _("Impala configuration properties."),
+      "type": "settings",
+      "options": [config.lower() for config in impala_settings.get()] if impala_settings is not None else []
+    }
+  ]
+
+
 class HS2Api(Api):
 
+  @staticmethod
+  def get_properties(lang='hive'):
+    return ImpalaConfiguration.PROPERTIES if lang == 'impala' else HiveConfiguration.PROPERTIES
+
+
   @query_error_handler
   def create_session(self, lang='hive', properties=None):
     application = 'beeswax' if lang == 'hive' else lang
@@ -76,10 +138,17 @@ class HS2Api(Api):
     if session is None:
       session = dbms.get(self.user, query_server=get_query_server_config(name=lang)).open_session(self.user)
 
+    if not properties:
+      config = DefaultConfiguration.objects.get_configuration_for_user(app=lang, user=self.user)
+      if config is not None:
+        properties = config.properties_list
+      else:
+        properties = self.get_properties(lang)
+
     return {
         'type': lang,
         'id': session.id,
-        'properties': session.get_formatted_properties()
+        'properties': properties
     }
 
 
@@ -289,6 +358,81 @@ class HS2Api(Api):
     }
 
 
+  @query_error_handler
+  def export_data_as_hdfs_file(self, snippet, target_file, overwrite):
+    db = self._get_db(snippet)
+
+    handle = self._get_handle(snippet)
+
+    upload(target_file, handle, self.request.user, db, self.request.fs)
+
+    return '/filebrowser/view=%s' % target_file
+
+
+  def export_data_as_table(self, snippet, destination):
+    db = self._get_db(snippet)
+
+    response = self._get_current_statement(db, snippet)
+    query = self._prepare_hql_query(snippet, response.pop('statement'))
+
+    if not query.hql_query.strip().lower().startswith('select'):
+      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
+
+    database = snippet.get('database') or 'default'
+    table = destination
+
+    if '.' in table:
+      database, table = table.split('.', 1)
+
+    db.use(query.database)
+
+    hql = 'CREATE TABLE `%s`.`%s` AS %s' % (database, table, query.hql_query)
+    success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
+
+    return hql, success_url
+
+
+  def export_large_data_to_hdfs(self, snippet, destination):
+    db = self._get_db(snippet)
+
+    response = self._get_current_statement(db, snippet)
+    query = self._prepare_hql_query(snippet, response.pop('statement'))
+
+    if not query.hql_query.strip().lower().startswith('select'):
+      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
+
+    db.use(query.database)
+
+    hql = "INSERT OVERWRITE DIRECTORY '%s' %s" % (destination, query.hql_query)
+    success_url = '/filebrowser/view=%s' % destination
+
+    return hql, success_url
+
+
+  def upgrade_properties(self, lang='hive', properties=None):
+    upgraded_properties = copy.deepcopy(self.get_properties(lang))
+
+    # Check that current properties is a list of dictionary objects with 'key' and 'value' keys
+    if not isinstance(properties, list) or \
+      not all(isinstance(prop, dict) for prop in properties) or \
+      not all('key' in prop for prop in properties) or not all('value' in prop for prop in properties):
+      LOG.warn('Current properties are not formatted correctly, will replace with defaults.')
+      return upgraded_properties
+
+    valid_props_dict = dict((prop["key"], prop) for prop in upgraded_properties)
+    curr_props_dict = dict((prop['key'], prop) for prop in properties)
+
+    # Upgrade based on valid properties as needed
+    if set(valid_props_dict.keys()) != set(curr_props_dict.keys()):
+      settings = next((prop for prop in upgraded_properties if prop['key'] == 'settings'), None)
+      if settings is not None and isinstance(properties, list):
+        settings['value'] = properties
+    else:  # No upgrade needed so return existing properties
+      upgraded_properties = properties
+
+    return upgraded_properties
+
+
   def _get_hive_execution_engine(self, notebook, snippet):
     # Get hive.execution.engine from snippet properties, if none, then get from session
     properties = snippet['properties']
@@ -396,55 +540,3 @@ class HS2Api(Api):
       name = 'spark-sql'
 
     return dbms.get(self.user, query_server=get_query_server_config(name=name))
-
-
-  @query_error_handler
-  def export_data_as_hdfs_file(self, snippet, target_file, overwrite):
-    db = self._get_db(snippet)
-
-    handle = self._get_handle(snippet)
-
-    upload(target_file, handle, self.request.user, db, self.request.fs)
-
-    return '/filebrowser/view=%s' % target_file
-
-
-  def export_data_as_table(self, snippet, destination):
-    db = self._get_db(snippet)
-
-    response = self._get_current_statement(db, snippet)
-    query = self._prepare_hql_query(snippet, response.pop('statement'))
-
-    if not query.hql_query.strip().lower().startswith('select'):
-      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
-
-    database = snippet.get('database') or 'default'
-    table = destination
-
-    if '.' in table:
-      database, table = table.split('.', 1)
-
-    db.use(query.database)
-
-    hql = 'CREATE TABLE `%s`.`%s` AS %s' % (database, table, query.hql_query)
-    success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
-
-    return hql, success_url
-
-
-  def export_large_data_to_hdfs(self, snippet, destination):
-    db = self._get_db(snippet)
-
-    response = self._get_current_statement(db, snippet)
-    query = self._prepare_hql_query(snippet, response.pop('statement'))
-
-    if not query.hql_query.strip().lower().startswith('select'):
-      raise Exception(_('Only SELECT statements can be saved. Provided statement: %(query)s') % {'query': query.hql_query})
-
-    db.use(query.database)
-
-    hql = "INSERT OVERWRITE DIRECTORY '%s' %s" % (destination, query.hql_query)
-    success_url = '/filebrowser/view=%s' % destination
-
-    return hql, success_url
-

+ 64 - 0
desktop/libs/notebook/src/notebook/connectors/tests/tests_hiveserver2.py

@@ -104,6 +104,70 @@ class TestHiveserver2Api(object):
     assert_true("CREATE TEMPORARY FUNCTION myUpper AS 'org.hue.udf.MyUpper'" in config_statements, config_statements)
 
 
+  def test_upgrade_properties(self):
+    properties = None
+    # Verify that upgrade will return defaults if current properties not formatted as settings
+    upgraded_props = self.api.upgrade_properties(lang='hive', properties=properties)
+    assert_equal(upgraded_props, self.api.get_properties(lang='hive'))
+
+    # Verify that upgrade will save old properties and new settings
+    properties = [
+        {
+            'key': 'hive.execution.engine',
+            'value': 'mr'
+        },
+        {
+            'key': 'hive.exec.compress.output',
+            'value': False
+        }
+    ]
+    upgraded_props = self.api.upgrade_properties(lang='hive', properties=properties)
+    settings = next((prop for prop in upgraded_props if prop['key'] == 'settings'), None)
+    assert_equal(settings['value'], properties)
+
+    # Verify that already upgraded properties will be unchanged
+    properties = [
+        {
+            "multiple": True,
+            "value": [],
+            "nice_name": "Files",
+            "key": "files",
+            "help_text": "Add one or more files, jars, or archives to the list of resources.",
+            "type": "hdfs-files"
+        },
+        {
+            "multiple": True,
+            "value": [],
+            "nice_name": "Functions",
+            "key": "functions",
+            "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+            "type": "functions"
+        },
+        {
+            "multiple": True,
+            "value": [
+                {
+                    "key": "hive.execution.engine",
+                    "value": "spark"
+                }
+            ],
+            "nice_name": "Settings",
+            "key": "settings",
+            "help_text": "Hive and Hadoop configuration properties.",
+            "type": "settings",
+            "options": [
+                "hive.map.aggr",
+                "hive.exec.compress.output",
+                "hive.exec.parallel",
+                "hive.execution.engine",
+                "mapreduce.job.queuename"
+            ]
+        }
+    ]
+    upgraded_props = self.api.upgrade_properties(lang='hive', properties=properties)
+    assert_equal(upgraded_props, properties)
+
+
 class TestHiveserver2ApiWithHadoop(BeeswaxSampleProvider):
 
   @classmethod

+ 30 - 4
desktop/libs/notebook/src/notebook/models.py

@@ -51,18 +51,35 @@ def escape_rows(rows, nulls_only=False):
   return data
 
 
-def make_notebook(name='Browse', description='', editor_type='hive', statement='', status='ready', files=None, functions=None, settings=None):
+def make_notebook(name='Browse', description='', editor_type='hive', statement='', status='ready',
+                  files=None, functions=None, settings=None):
+
+  from notebook.connectors.hiveserver2 import HS2Api
+
   editor = Notebook()
 
+  properties = HS2Api.get_properties(editor_type)
+
+  if editor_type == 'hive':
+    if files is not None:
+      _update_property_value(properties, 'files', files)
+
+    if functions is not None:
+      _update_property_value(properties, 'functions', functions)
+
+    if settings is not None:
+      _update_property_value(properties, 'settings', settings)
+  elif editor_type == 'impala':
+    if settings is not None:
+      _update_property_value(properties, 'files', files)
+
   editor.data = json.dumps({
     'name': name,
     'description': description,
     'sessions': [
       {
          'type': editor_type,
-         'properties': [
-
-         ],
+         'properties': properties,
          'id': None
       }
     ],
@@ -121,3 +138,12 @@ def _convert_type(btype, bdata):
     return 'spark'
   else:
     return 'hive'
+
+
+def _update_property_value(properties, key, value):
+  """
+  Update property dict in list of properties where prop has "key": key, set "value": value
+  """
+  for prop in properties:
+    if prop['key'] == key:
+      prop.update({'value': value})