Browse Source

[notebook] Allow snippet properties files and functions to be passed from notebook

Hive/Impala snippets should set:

snippet['properties']['files'] = [{'type': 'JAR', 'path': '/user/jennykim/myudfs.jar'}]

snippet['properties']['functions'] = [{'name': 'myUpper', 'class_name': 'org.hue.udf.MyUpper'}]
Jenny Kim 10 years ago
parent
commit
7427e5562b

+ 26 - 15
apps/beeswax/src/beeswax/design.py

@@ -27,9 +27,10 @@ import urlparse
 
 
 import django.http
 import django.http
 from django import forms
 from django import forms
+from django.forms import ValidationError
+from django.utils.translation import ugettext as _
 
 
 from desktop.lib.django_forms import BaseSimpleFormSet, MultiForm
 from desktop.lib.django_forms import BaseSimpleFormSet, MultiForm
-from desktop.lib.django_mako import render_to_string
 from hadoop.cluster import get_hdfs
 from hadoop.cluster import get_hdfs
 
 
 
 
@@ -38,7 +39,7 @@ LOG = logging.getLogger(__name__)
 SERIALIZATION_VERSION = '0.4.1'
 SERIALIZATION_VERSION = '0.4.1'
 
 
 
 
-def hql_query(hql, database='default', query_type=None, settings=None):
+def hql_query(hql, database='default', query_type=None, settings=None, file_resources=None, functions=None):
   data_dict = HQLdesign.get_default_data_dict()
   data_dict = HQLdesign.get_default_data_dict()
 
 
   if not (isinstance(hql, str) or isinstance(hql, unicode)):
   if not (isinstance(hql, str) or isinstance(hql, unicode)):
@@ -50,8 +51,14 @@ def hql_query(hql, database='default', query_type=None, settings=None):
   if query_type:
   if query_type:
     data_dict['query']['type'] = query_type
     data_dict['query']['type'] = query_type
 
 
-  if settings is not None and HQLdesign.is_valid_settings(settings):
-    data_dict['query']['settings'] = settings
+  if settings is not None and HQLdesign.validate_properties('settings', settings, HQLdesign._SETTINGS_ATTRS):
+    data_dict['settings'] = settings
+
+  if file_resources is not None and HQLdesign.validate_properties('file resources', file_resources, HQLdesign._FILE_RES_ATTRS):
+    data_dict['file_resources'] = file_resources
+
+  if functions is not None and HQLdesign.validate_properties('functions', functions, HQLdesign._FUNCTIONS_ATTRS):
+    data_dict['functions'] = functions
 
 
   hql_design = HQLdesign()
   hql_design = HQLdesign()
   hql_design._data_dict = data_dict
   hql_design._data_dict = data_dict
@@ -154,16 +161,19 @@ class HQLdesign(object):
     return design
     return design
 
 
   @staticmethod
   @staticmethod
-  def is_valid_settings(settings):
-    is_valid = True
-    if isinstance(settings, list) and all(isinstance(item, dict) for item in settings):
-      for item in settings:
-        if not all(attr in item for attr in HQLdesign._SETTINGS_ATTRS):
-          is_valid = False
-          break
+  def validate_properties(property_type, properties, req_attr_list):
+    """
+    :param property_type: 'Settings', 'File Resources', or 'Functions'
+    :param properties: list of properties as dict
+    :param req_attr_list: list of attributes that are required keys for each dict item
+    """
+    if isinstance(properties, list) and all(isinstance(item, dict) for item in properties):
+      for item in properties:
+        if not all(attr in item for attr in req_attr_list):
+          raise ValidationError(_("Invalid %s, missing required attributes: %s.") % (property_type, ', '.join(req_attr_list)))
     else:
     else:
-      is_valid = False
-    return is_valid
+      raise ValidationError(_('Invalid settings, expected list of dict items.'))
+    return True
 
 
   def dumps(self):
   def dumps(self):
     """Returns the serialized form of the design in a string"""
     """Returns the serialized form of the design in a string"""
@@ -179,10 +189,11 @@ class HQLdesign(object):
         scheme = get_hdfs().fs_defaultfs
         scheme = get_hdfs().fs_defaultfs
       else:
       else:
         scheme = ''
         scheme = ''
-      configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path'], scheme=scheme)))
+      configuration.append('ADD %(type)s %(scheme)s%(path)s' % {'type': f['type'], 'path': f['path'], 'scheme': scheme})
 
 
     for f in self.functions:
     for f in self.functions:
-      configuration.append(render_to_string("hql_function.mako", f))
+      configuration.append("CREATE TEMPORARY FUNCTION %(name)s AS '%(class_name)s'" %
+                           {'name': f['name'], 'class_name': f['class_name']})
 
 
     return configuration
     return configuration
 
 

+ 0 - 16
apps/beeswax/src/beeswax/templates/hql_function.mako

@@ -1,16 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-CREATE TEMPORARY FUNCTION ${name} AS '${class_name}'

+ 0 - 17
apps/beeswax/src/beeswax/templates/hql_resource.mako

@@ -1,17 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-
-ADD ${type} ${scheme + path}

+ 3 - 3
apps/beeswax/src/beeswax/tests.py

@@ -2759,9 +2759,9 @@ class TestDesign():
     ]
     ]
 
 
     statements = design.get_configuration_statements()
     statements = design.get_configuration_statements()
-    assert_true(re.match('\nADD FILE hdfs://([^:]+):(\d+)my_file\n', statements[0]), statements[0])
-    assert_true(re.match('\nADD FILE hdfs://([^:]+):(\d+)/my_path/my_file\n', statements[1]), statements[1])
-    assert_equal('\nADD FILE s3://host/my_s3_file\n', statements[2])
+    assert_true(re.match('ADD FILE hdfs://([^:]+):(\d+)my_file', statements[0]), statements[0])
+    assert_true(re.match('ADD FILE hdfs://([^:]+):(\d+)/my_path/my_file', statements[1]), statements[1])
+    assert_equal('ADD FILE s3://host/my_s3_file', statements[2])
 
 
 
 
 def search_log_line(expected_log, all_logs):
 def search_log_line(expected_log, all_logs):

+ 3 - 1
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -92,8 +92,10 @@ class HS2Api(Api):
     statement = statements[statement_id]
     statement = statements[statement_id]
 
 
     settings = snippet['properties'].get('settings', None)
     settings = snippet['properties'].get('settings', None)
+    file_resources = snippet['properties'].get('files', None)
+    functions = snippet['properties'].get('functions', None)
 
 
-    query = hql_query(statement, query_type=QUERY_TYPES[0], settings=settings)
+    query = hql_query(statement, query_type=QUERY_TYPES[0], settings=settings, file_resources=file_resources, functions=functions)
 
 
     try:
     try:
       handle = db.client.query(query)
       handle = db.client.query(query)

+ 4 - 0
desktop/libs/notebook/src/notebook/decorators.py

@@ -18,6 +18,7 @@
 import json
 import json
 import logging
 import logging
 
 
+from django.forms import ValidationError
 from django.http import Http404
 from django.http import Http404
 from django.utils.functional import wraps
 from django.utils.functional import wraps
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
@@ -81,6 +82,9 @@ def api_error_handler(func):
       response['status'] = -3
       response['status'] = -3
     except AuthenticationRequired, e:
     except AuthenticationRequired, e:
       response['status'] = 401
       response['status'] = 401
+    except ValidationError, e:
+      response['status'] = -1
+      response['message'] = e.message
     except QueryError, e:
     except QueryError, e:
       LOG.exception('error running %s' % func)
       LOG.exception('error running %s' % func)
       response['status'] = 1
       response['status'] = 1

+ 1 - 0
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -130,6 +130,7 @@
     else if (snippetType == 'hive' || snippetType == 'impala') {
     else if (snippetType == 'hive' || snippetType == 'impala') {
       properties['settings'] = [];
       properties['settings'] = [];
       properties['files'] = [];
       properties['files'] = [];
+      properties['functions'] = [];
     }
     }
     else if (snippetType == 'pig') {
     else if (snippetType == 'pig') {
       properties['parameters'] = [];
       properties['parameters'] = [];