浏览代码

HUE-1494 [beeswax] Re-enable Hadoop tests

Works with Yarn and latest HiveServer2.
Romain Rigaux 12 年之前
父节点
当前提交
e0c7c39

+ 3 - 2
apps/beeswax/src/beeswax/conf.py

@@ -20,7 +20,7 @@ import sys
 
 from django.utils.translation import ugettext_lazy as _t, ugettext as _
 
-from desktop.lib.conf import Config, coerce_bool
+from desktop.lib.conf import Config
 
 from beeswax.settings import NICE_NAME
 
@@ -46,7 +46,8 @@ HIVE_CONF_DIR = Config(
 HIVE_SERVER_BIN = Config(
   key="hive_server_bin",
   help=_t("Path to HiveServer2 start script"),
-  default='/usr/lib/hive/bin/hiveserver2')
+  default='/usr/lib/hive/bin/hiveserver2',
+  private=True)
 
 LOCAL_EXAMPLES_DATA_DIR = Config(
   key='local_examples_data_dir',

+ 6 - 2
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -456,7 +456,7 @@ class HiveServerClient:
 
   def close_operation(self, operation_handle):
     req = TCloseOperationReq(operationHandle=operation_handle)
-    return self.call(self._client.CancelOperation, req)
+    return self.call(self._client.CloseOperation, req)
 
 
   def get_columns(self, database, table):
@@ -519,7 +519,7 @@ class HiveServerTableCompatible(HiveServerTable):
 
   @property
   def cols(self):
-    return [type('Col', (object,), {'name': col.get('col_name', ''),
+    return [type('Col', (object,), {'name': col.get('col_name', '').strip(),
                                     'type': col.get('data_type', ''),
                                     'comment': col.get('comment', ''), }) for col in HiveServerTable.cols.fget(self)]
 
@@ -620,6 +620,10 @@ class HiveServerClientCompatible:
     return self._client.cancel_operation(operationHandle)
 
 
+  def close(self, handle):
+    return self.close_operation(handle)
+
+
   def close_operation(self, handle):
     operationHandle = handle.get_rpc_handle()
     return self._client.close_operation(operationHandle)

+ 0 - 34
apps/beeswax/src/beeswax/templates/confirm.html

@@ -1,34 +0,0 @@
-{% comment %}
-Licensed to Cloudera, Inc. under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  Cloudera, Inc. licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-{% endcomment %}
-{% block content %}
-<form action="{{ url }}" method="POST">>
-<div class="modal-header">
-	<a href="#" class="close">&times;</a>
-	<h3>Confirm action</h3>
-</div>
-<div class="modal-body">
-  <div class="alert-message block-message warning">
-        {{title}}
-  </div>
-</div>
-<div class="modal-footer">
-	<input type="submit" class="btn primary" value="Yes"/>
-	<a href="#" class="btn secondary hideModal">No</a>
-</div>
-</form>
-{% endblock %}

+ 35 - 0
apps/beeswax/src/beeswax/templates/confirm.mako

@@ -0,0 +1,35 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+from django.utils.translation import ugettext as _
+%>
+
+<form action="{{ url }}" method="POST">>
+  <div class="modal-header">
+	<a href="javascript:void(0);" class="close">&times;</a>
+	<h3>${ _('Confirm action') }</h3>
+  </div>
+  <div class="modal-body">
+    <div class="alert-message block-message warning">
+      ${ title }
+    </div>
+  </div>
+  <div class="modal-footer">
+	<input type="submit" class="btn primary" value="${ _('Yes') }"/>
+	<a href="#" class="btn secondary hideModal">${ _('No') }</a>
+  </div>
+</form>

+ 10 - 4
apps/beeswax/src/beeswax/templates/watch_results.mako

@@ -125,11 +125,11 @@ ${layout.menubar(section='query')}
             <p>
             <ul class="nav nav-tabs">
               <li class="active"><a href="#results" data-toggle="tab">
-                  %if error:
+                  % if error:
                         ${_('Error')}
-                  %else:
+                  % else:
                         ${_('Results')}
-                  %endif
+                  % endif
               </a></li>
               <li><a href="#query" data-toggle="tab">${_('Query')}</a></li>
               <li><a href="#log" data-toggle="tab">${_('Log')}</a></li>
@@ -169,7 +169,13 @@ ${layout.menubar(section='query')}
               <tr>
                 <td>${ start_row + i }</td>
                 % for item in row:
-                  <td>${ smart_unicode(item, errors='ignore') }</td>
+                  <td>
+                    % if item is None:
+                      NULL
+                    % else:
+                      ${ smart_unicode(item, errors='ignore') }
+                    % endif
+                  </td>
                 % endfor
               </tr>
               % endfor

+ 29 - 24
apps/beeswax/src/beeswax/test_base.py

@@ -30,13 +30,13 @@ from django.contrib.auth.models import User
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.paths import get_run_root
 from hadoop import pseudo_hdfs4
-from nose.plugins.skip import SkipTest
 
 import beeswax.conf
 
 from beeswax.server.dbms import get_query_server_config
 from beeswax.server import dbms
 
+
 HIVE_SERVER_TEST_PORT = 6969
 _INITIALIZED = False
 _SHARED_HIVE_SERVER_PROCESS = None
@@ -50,12 +50,23 @@ LOG = logging.getLogger(__name__)
 def _start_server(cluster):
   args = [beeswax.conf.HIVE_SERVER_BIN.get()]
 
-  env = cluster.mr1_env.copy()
+  env = cluster._mr2_env.copy()
 
   env.update({
     'HIVE_CONF_DIR': beeswax.conf.HIVE_CONF_DIR.get(),
     'HIVE_SERVER2_THRIFT_PORT': str(HIVE_SERVER_TEST_PORT),
-    'AUX_CLASSPATH': '/usr/lib/hadoop-hdfs/hadoop-hdfs.jar:/usr/lib/hadoop/hadoop-auth.jar:/usr/lib/hadoop/hadoop-common.jar', # todo update
+    'HADOOP_MAPRED_HOME': get_run_root('ext/hadoop/hadoop') + '/share/hadoop/mapreduce',
+    # Links created in jenkins script.
+    # If missing classes when booting HS2, check here.
+    'AUX_CLASSPATH':
+       get_run_root('ext/hadoop/hadoop') + '/share/hadoop/hdfs/hadoop-hdfs.jar'
+       + ':' +
+       get_run_root('ext/hadoop/hadoop') + '/share/hadoop/common/lib/hadoop-auth.jar'
+       + ':' +
+       get_run_root('ext/hadoop/hadoop') + '/share/hadoop/common/hadoop-common.jar'
+       + ':' +
+       get_run_root('ext/hadoop/hadoop') + '/share/hadoop/mapreduce/hadoop-mapreduce-client-core.jar'
+       ,
     'HADOOP_CLASSPATH': '',
   })
 
@@ -63,7 +74,7 @@ def _start_server(cluster):
     env["JAVA_HOME"] = os.getenv("JAVA_HOME")
 
   LOG.info("Executing %s, env %s, cwd %s" % (repr(args), repr(env), cluster._tmpdir))
-  return subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir)#, stdin=subprocess.PIPE)
+  return subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
 
 
 def get_shared_beeswax_server():
@@ -73,7 +84,7 @@ def get_shared_beeswax_server():
 
     cluster = pseudo_hdfs4.shared_cluster()
 
-    HIVE_CONF = cluster._tmpdir + "/conf"
+    HIVE_CONF = cluster.hadoop_conf_dir
     finish = (
       beeswax.conf.HIVE_SERVER_HOST.set_for_testing("localhost"),
       beeswax.conf.HIVE_SERVER_PORT.set_for_testing(HIVE_SERVER_TEST_PORT),
@@ -92,8 +103,18 @@ def get_shared_beeswax_server():
   <description>JDBC connect string for a JDBC metastore</description>
 </property>
 
+ <property>
+   <name>hive.server2.enable.impersonation</name>
+   <value>false</value>
+ </property>
+
+<property>
+  <name>hive.querylog.location</name>
+  <value>%(querylog)s</value>
+</property>
+
 </configuration>
-""" % {'root': cluster._tmpdir}
+""" % {'root': cluster._tmpdir, 'querylog': cluster.log_dir + '/hive'}
 
     file(HIVE_CONF + '/hive-site.xml', 'w').write(default_xml)
 
@@ -125,23 +146,13 @@ def get_shared_beeswax_server():
           started = True
           break
         except Exception, e:
-          LOG.info('HiveServer2 server status not started yet: %s' % e)
+          LOG.info('HiveServer2 server status not started yet after: %s' % e)
           time.sleep(sleep)
           sleep *= 2
 
       if not started:
         raise Exception("Server took too long to come up.")
 
-      # Make sure /tmp is 0777
-      cluster.fs.setuser(cluster.superuser)
-      if not cluster.fs.isdir('/tmp'):
-        cluster.fs.mkdir('/tmp', 0777)
-      else:
-        cluster.fs.chmod('/tmp', 0777)
-
-      cluster.fs.chmod(cluster._tmpdir, 0777)
-      cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)
-
     def s():
       for f in finish:
         f()
@@ -156,7 +167,6 @@ REFRESH_RE = re.compile('<\s*meta\s+http-equiv="refresh"\s+content="\d*;([^"]*)"
 
 
 def wait_for_query_to_finish(client, response, max=30.0):
-  # logging.info(str(response.template.filename) + ": " + str(response.content))
   start = time.time()
   sleep_time = 0.05
   # We don't check response.template == "watch_wait.mako" here,
@@ -194,7 +204,7 @@ def make_query(client, query, submission_type="Execute",
     settings = []
   if local:
     # Tests run faster if not run against the real cluster.
-    settings.append(("mapred.job.tracker", "local"))
+    settings.append(('mapreduce.framework.name', 'local'))
 
   # Prepares arguments for the execute view.
   parameters = {
@@ -261,10 +271,6 @@ def verify_history(client, fragment, design=None, reverse=False):
     except KeyError:
       pass
 
-  # This could happen if we issue multiple requests in parallel.
-  # The capturing of Django response context is not thread safe.
-  # Also see:
-  #   http://docs.djangoproject.com/en/1.2/topics/testing/#testing-responses
   LOG.warn('Cannot find history size. Response context clobbered')
   return -1
 
@@ -275,7 +281,6 @@ class BeeswaxSampleProvider(object):
   """
   @classmethod
   def setup_class(cls):
-    raise SkipTest
     cls.cluster, shutdown = get_shared_beeswax_server()
     cls.client = make_logged_in_client()
     # Weird redirection to avoid binding nonsense.

+ 54 - 47
apps/beeswax/src/beeswax/tests.py

@@ -15,12 +15,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-try:
-  import json
-except ImportError:
-  import simplejson as json
+
 import cStringIO
 import gzip
+import json
 import logging
 import os
 import re
@@ -40,7 +38,7 @@ from django.core.urlresolvers import reverse
 
 from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
 from desktop.lib.django_test_util import assert_similar_pages
-from desktop.lib.test_utils import grant_access
+from desktop.lib.test_utils import grant_access, add_to_group
 
 import beeswax.create_table
 import beeswax.forms
@@ -90,11 +88,11 @@ def get_csv(client, result_response):
 
 
 class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
-  """Tests for beeswax that require a running Hadoop"""
   requires_hadoop = True
 
   def setUp(self):
     user = User.objects.get(username='test')
+    add_to_group('test')
     self.db = dbms.get(user, get_query_server_config())
 
   def _verify_query_state(self, state):
@@ -117,6 +115,9 @@ class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
     assert_true("Table test already exists" in response.context["error_message"])
 
   def test_configuration(self):
+    # No HS2 API
+    raise SkipTest
+
     params = {'server': 'default'}
 
     response = self.client.post("/beeswax/configuration", params)
@@ -148,7 +149,6 @@ for x in sys.stdin:
       resources=[("FILE", "/square.py")], local=False)
     response = wait_for_query_to_finish(self.client, response, max=180.0)
     assert_equal([['0'], ['1'], ['4'], ['9']], response.context["results"][0:4])
-    assert_true('converting to local %s/square.py' % self.cluster._fs_default_name in response.context["log"], response.context["log"])
 
   def test_query_with_setting(self):
     response = _make_query(self.client, "CREATE TABLE test2 AS SELECT foo+1 FROM test WHERE foo=4",
@@ -157,8 +157,10 @@ for x in sys.stdin:
     response = wait_for_query_to_finish(self.client, response, max=180.0)
     # Check that we actually got a compressed output
     files = self.cluster.fs.listdir("/user/hive/warehouse/test2")
-    assert_true(len(files) >= 1)
-    assert_true(files[0].endswith(".deflate"))
+    assert_true(len(files) >= 1, files)
+    assert_true(files[0].endswith(".deflate"), files[0])
+
+    raise SkipTest
     # And check that the name is right...
     assert_true("test_query_with_setting" in [ x.profile.name for x in self.cluster.jt.all_jobs().jobs ])
 
@@ -179,16 +181,16 @@ for x in sys.stdin:
     QUERY = """
       SELECT MIN(foo), MAX(foo), SUM(foo) FROM test;
     """
-    response = _make_query(self.client, QUERY)
+    response = _make_query(self.client, QUERY, local=False)
     assert_true(response.redirect_chain[0][0].startswith("http://testserver/beeswax/watch/"))
     # Check that we report this query as "running". (This query takes a while.)
     self._verify_query_state(beeswax.models.QueryHistory.STATE.running)
 
     response = wait_for_query_to_finish(self.client, response, max=180.0)
-    assert_equal(["0", "255", "32640"], response.context["results"][0])
-    # Because it happens that we're running this with mapred.job.tracker,
+    assert_equal([0, 255, 32640], response.context["results"][0], response.content)
+    # Because it happens that we're running this with local mode,
     # we won't see any hadoop jobs.
-    assert_equal(0, len(response.context["hadoop_jobs"]), "Shouldn't have found jobs.")
+    assert_equal(1, len(response.context["hadoop_jobs"]), response.context["hadoop_jobs"])
     self._verify_query_state(beeswax.models.QueryHistory.STATE.available)
 
 
@@ -199,12 +201,12 @@ for x in sys.stdin:
     response = _make_query(self.client, QUERY, name='select star', local=False)
     response = wait_for_query_to_finish(self.client, response)
     assert_equal(str(response.context['query_context'][0]), 'design')
-    assert_true("99</td>" in response.content)
+    assert_true("99" in response.content)
     assert_true(response.context["has_more"])
     response = self.client.get("/beeswax/results/%d/%d" % (response.context["query"].id, response.context["next_row"]))
-    assert_true("199</td>" in response.content)
+    assert_true("199" in response.content)
     response = self.client.get("/beeswax/results/%d/0" % (response.context["query"].id))
-    assert_true("99</td>" in response.content)
+    assert_true("99" in response.content)
     assert_equal(0, len(response.context["hadoop_jobs"]), "SELECT * shouldn't have started jobs.")
 
     # Download the data
@@ -220,7 +222,7 @@ for x in sys.stdin:
       udfs=[('my_sqrt', 'org.apache.hadoop.hive.ql.udf.UDFSqrt'),
             ('my_power', 'org.apache.hadoop.hive.ql.udf.UDFPower')], local=False)
     response = wait_for_query_to_finish(self.client, response, max=60.0)
-    assert_equal(["2.0", "256.0"], response.context["results"][0])
+    assert_equal([2.0, 256.0], response.context["results"][0])
     log = response.context['log']
     assert_true(search_log_line('ql.Driver', 'Total MapReduce jobs', log), 'Captured log from Driver in %s' % log)
     assert_true(search_log_line('exec.Task', 'Starting Job = job_', log), 'Captured log from MapRedTask in %s' % log)
@@ -250,13 +252,14 @@ for x in sys.stdin:
   def test_query_with_simple_errors(self):
     """Test handling syntax error"""
     def check_error_in_response(response):
-      assert_true("ParseException" in response.context["error_message"])
-      log = response.context['log']
+      assert_true("ParseException" in response.content, response.content)
+      page_context = [context for context in response.context if 'log' in context][0]
+      log = page_context['log']
       assert_true(len(log.split('\n')) > 10, 'Captured stack trace')
       assert_true('org.apache.hadoop.hive.ql.parse.ParseException: line' in log, 'Captured stack trace')
 
     hql = "SELECT KITTENS ARE TASTY"
-    resp = _make_query(self.client, hql, name='tasty kittens')
+    resp = _make_query(self.client, hql, name='tasty kittens', wait=True)
     check_error_in_response(resp)
     id = self._verify_query_state(beeswax.models.QueryHistory.STATE.failed)
 
@@ -366,9 +369,9 @@ for x in sys.stdin:
     response = self.client.post("/beeswax/execute_parameterized/%d" % design_id,
                                 {"parameterization-x": "'_this_is_not SQL ", "parameterization-y": str(2)},
                                 follow=True)
-    assert_true(any(["execute.mako" in _template.filename for _template in response.template]))
-    log = response.context["log"]
-    assert_true(search_log_line('ql.Driver', 'FAILED: ParseException', log), log)
+    response = wait_for_query_to_finish(self.client, response)
+    assert_true("ql.Driver" in response.content, response.content)
+    assert_true("FAILED: ParseException" in response.content, response.content)
 
     # Check multi DB with a non default DB
     response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", database='other_db')
@@ -376,7 +379,7 @@ for x in sys.stdin:
     design_id = response.context["design"].id
     response = self.client.post("/beeswax/execute_parameterized/%d" % design_id, {
                                 "parameterization-x": str(1), "parameterization-y": str(2)}, follow=True)
-    assert_equal('other_db', response.context['design'].get_design().query['database'])
+    assert_equal('other_db', response.context['query'].design.get_design().query['database'])
 
   def test_explain_query(self):
     c = self.client
@@ -546,7 +549,7 @@ for x in sys.stdin:
     # Should be CSV since we simply change the file extension and MIME type from CSV to XLS.
     translated_csv = xls_resp.content
     # It should have 257 lines (256 + header)
-    assert_equal(len(translated_csv.strip('\r\n').split('\r\n')), 257)
+    assert_equal(len(translated_csv.strip('\r\n').split('\r\n')), 257, translated_csv)
 
     # Get the result in csv.
     query = hql_query(hql)
@@ -555,7 +558,6 @@ for x in sys.stdin:
     assert_equal(csv_resp.content, translated_csv)
 
   def test_designs(self):
-    """Test design view and interaction"""
     cli = self.client
 
     # An auto hql design should be created, and it should ignore the given name and desc
@@ -579,7 +581,7 @@ for x in sys.stdin:
     # Test explicit save and use another DB
     query = 'MORE BOGUS JUNKS FROM test'
     exe_resp = _make_query(self.client, query, name='rubbish', submission_type='Save', database='other_db')
-    assert_true("error_message" not in exe_resp.context)
+    assert_true([context["error_message"] for context in exe_resp.context if 'error_message' in context][0] is None, exe_resp.context)
     resp = cli.get('/beeswax/list_designs')
     assert_true('rubbish' in resp.content, resp.content)
     nplusplus_designs = len(resp.context['page'].object_list)
@@ -644,6 +646,9 @@ for x in sys.stdin:
     _make_query(client_me, "select one", name='client query 1', submission_type='Save')
     _make_query(client_me, "select two", name='client query 2', submission_type='Save')
 
+    # TODO in HUE-1589
+    raise SkipTest
+
     finish = conf.SHARE_SAVED_QUERIES.set_for_testing(True)
     try:
       resp = client_me.get('/beeswax/list_designs')
@@ -796,7 +801,7 @@ for x in sys.stdin:
       # Check that data is right. The SELECT may not give us the whole table.
       resp = _make_query(self.client, 'SELECT * FROM %s' % (target_tbl,), wait=True, local=False)
       for i in xrange(90):
-        assert_equal([str(i), '0x%x' % (i,)], resp.context['results'][i])
+        assert_equal([i, '0x%x' % (i,)], resp.context['results'][i])
 
     TARGET_TBL_ROOT = 'test_copy'
 
@@ -812,6 +817,8 @@ for x in sys.stdin:
 
 
   def test_install_examples(self):
+    raise SkipTest
+
     assert_true(not beeswax.models.MetaInstall.get().installed_example)
 
     # Check popup
@@ -827,7 +834,7 @@ for x in sys.stdin:
 
     # New designs exists
     resp = self.client.get('/beeswax/list_designs')
-    assert_true('Sample: Job loss' in resp.content)
+    assert_true('Sample: Job loss' in resp.content, resp.content)
     assert_true('Sample: Salary growth' in resp.content)
     assert_true('Sample: Top salary' in resp.content)
 
@@ -865,7 +872,9 @@ for x in sys.stdin:
       'create': 'Create table',
     }, follow=True)
 
-    if "watch_wait.mako" in resp.template:
+    templates = [_template.filename for _template in resp.template]
+
+    if any(['watch_wait.mako' in template for template in templates]):
       assert_equal_mod_whitespace("""
           CREATE EXTERNAL TABLE `default.my_table`
           (
@@ -881,7 +890,8 @@ for x in sys.stdin:
       assert_true('on_success_url=%2Fmetastore%2Ftable%2Fdefault%2Fmy_table' in resp.context['fwd_params'], resp.context['fwd_params'])
     else:
       # Create was fast
-      assert_true('describe_table.mako' in resp.template, resp.template)
+      templates = [_template.filename for _template in resp.template]
+      assert_true(any(['describe_table.mako' in template for template in templates]), templates)
       assert_true('Table my_table' in resp.content, resp.content)
 
 
@@ -1089,8 +1099,8 @@ for x in sys.stdin:
     cols = resp.context['table'].cols
     assert_equal(len(cols), 3)
     assert_equal([ col.name for col in cols ], [ 'col_a', 'col_b', 'col_c' ])
-    assert_true("nada</td>" in resp.content)
-    assert_true("sp ace</td>" in resp.content)
+    assert_true("nada" in resp.content, resp.content)
+    assert_true("sp ace" in resp.content, resp.content)
 
 
   def test_create_database(self):
@@ -1101,11 +1111,13 @@ for x in sys.stdin:
       'use_default_location': True,
     }, follow=True)
 
-    if "watch_wait.mako" in resp.template:
+    templates = [_template.filename for _template in resp.template]
+
+    if [template for template in templates if "watch_wait.mako" in template]:
       assert_equal_mod_whitespace("CREATE DATABASE my_db COMMENT \"foo\"", resp.context['query'].query, resp.content)
     else:
       # Create was fast
-      assert_true('databases.mako' in resp.template, resp.template)
+      assert_true([template for template in templates if 'databases.mako' in template], templates)
 
     resp = wait_for_query_to_finish(self.client, resp, max=180.0)
     assert_true('my_db' in resp.context['databases'], resp)
@@ -1124,8 +1136,8 @@ for x in sys.stdin:
     assert_equal('beeswax', query_server['server_name'])
     assert_equal('localhost', query_server['server_host'])
     assert_equal(HIVE_SERVER_TEST_PORT, query_server['server_port'])
-    assert_equal('beeswax', query_server['server_type'])
-    assert_true(query_server['principal'].startswith('hue/'), query_server['principal'])
+    assert_equal('hiveserver2', query_server['server_type'])
+    assert_true(query_server['principal'] is None, query_server['principal']) # No default hive/HOST_@TEST.COM so far
 
 
   def test_select_multi_db(self):
@@ -1143,17 +1155,12 @@ for x in sys.stdin:
 
 
   def test_xss_html_escaping(self):
-    client = make_logged_in_client()
-
-    data = {
-        u'settings-next_form_id': [u'1'], u'settings-0-key': [u'"><script>alert(1);</script>'], u'button-submit': [u'Execute'],
-        u'functions-next_form_id': [u'0'], u'settings-0-value': [u'"><script>alert(1);</script>'], u'query-is_parameterized': [u'on'],
-        u'query-query': [u'query'], u'query-database': [u'default'], u'settings-0-_exists': [u'True'], u'file_resources-next_form_id': [u'0']
-     }
+    query = 'I love Hue'
+    settings = [('"><script>alert(1);</script>', '"><script>alert(1);</script>')]
+    response = _make_query(self.client, query, name='lovehue', submission_type='Save', settings=settings)
 
-    resp = client.post('/beeswax/execute/', data)
-    assert_false('"><script>alert(1);</script>' in resp.content, resp.content)
-    assert_true('&quot;&gt;&lt;script&gt;alert(1);&lt;/script&gt;' in resp.content, resp.content)
+    assert_false('"><script>alert(1);</script>' in response.content, response.content)
+    assert_true('&quot;&gt;&lt;script&gt;alert(1);&lt;/script&gt;' in response.content, response.content)
 
   def test_list_design_pagination(self):
     client = make_logged_in_client()

+ 8 - 13
apps/beeswax/src/beeswax/views.py

@@ -15,10 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+import json
 import logging
 import re
 
@@ -166,7 +163,7 @@ def delete_design(request):
         design.delete()
     return redirect(reverse(get_app_name(request) + ':list_designs'))
   else:
-    return render('confirm.html', request, dict(url=request.path, title=_('Delete design(s)?')))
+    return render('confirm.mako', request, {'url': request.path, 'title': _('Delete design(s)?')})
 
 
 def restore_design(request):
@@ -182,7 +179,7 @@ def restore_design(request):
       design.doc.get().restore_from_trash()
     return redirect(reverse(get_app_name(request) + ':list_designs'))
   else:
-    return render('confirm.html', request, dict(url=request.path, title=_('Restore design(s)?')))
+    return render('confirm.mako', request, {'url': request.path, 'title': _('Restore design(s)?')})
 
 
 def clone_design(request, design_id):
@@ -614,7 +611,8 @@ def view_results(request, id, first_row=0):
                 'rows': 0,
                 'columns': [],
                 'has_more': False,
-                'start_row': 0, })
+                'start_row': 0,
+            })
   data = []
   fetch_error = False
   error_message = ''
@@ -629,7 +627,7 @@ def view_results(request, id, first_row=0):
   context_param = request.GET.get('context', '')
   query_context = _parse_query_context(context_param)
 
-  # To remove in Hue 2.4
+  # To remove when Impala has start_over support
   download  = request.GET.get('download', '')
 
   # Update the status as expired should not be accessible
@@ -843,15 +841,12 @@ def query_done_cb(request, server_id):
   """
   A callback for query completion notification. When the query is done,
   BeeswaxServer notifies us by sending a GET request to this view.
-
-  This view should always return a 200 response, to reflect that the
-  notification is delivered to the right view.
   """
   message_template = '<html><head></head>%(message)s<body></body></html>'
   message = {'message': 'error'}
 
   try:
-    query_history = models.QueryHistory.objects.get(server_id=server_id)
+    query_history = QueryHistory.objects.get(server_id=server_id + '\n')
 
     # Update the query status
     query_history.set_to_available()
@@ -957,7 +952,7 @@ def authorized_get_history(request, query_history_id, owner_only=False, must_exi
       return None
 
   # Some queries don't have a design so are not linked to Document Model permission
-  if query_history.design is None:
+  if query_history.design is None or not query_history.design.doc.exists():
     if not request.user.is_superuser and request.user != query_history.owner:
       raise PopupException(_('Permission denied to read QueryHistory %(id)s') % {'id': query_history_id})
   else:

+ 7 - 1
apps/metastore/src/metastore/templates/describe_table.mako

@@ -121,7 +121,13 @@ ${ components.menubar() }
                   % for i, row in enumerate(sample):
                     <tr>
                     % for item in row:
-                      <td>${ smart_unicode(item, errors='ignore') }</td>
+                      <td>
+                        % if item is None:
+                          NULL
+                        % else:
+                          ${ smart_unicode(item, errors='ignore') }
+                        % endif
+                      </td>
                     % endfor
                     </tr>
                   % endfor

+ 2 - 2
apps/oozie/src/oozie/tests.py

@@ -1679,7 +1679,7 @@ class TestImportWorkflow04(OozieMockBase):
 
   def setUp(self):
     raise SkipTest
-    
+
     super(TestImportWorkflow04, self).setUp()
     self.setup_simple_workflow()
 
@@ -3111,7 +3111,7 @@ def create_workflow(client, user, workflow_dict=WORKFLOW_DICT):
 
   response = client.post(reverse('oozie:create_workflow'), workflow_dict, follow=True)
   assert_equal(200, response.status_code)
-  
+
   assert_equal(workflow_count + 1, Document.objects.available_docs(Workflow, user).count())
 
   wf = Document.objects.get_docs(user, Workflow).get(name=name, extra='').content_object

+ 4 - 1
desktop/core/src/desktop/lib/export_csvxls.py

@@ -130,9 +130,12 @@ class CSVformatter(Formatter):
 
   def format_row(self, row):
     # writerow will call our write() method
-    row = [smart_str(cell, self._encoding, strings_only=True, errors='replace') for cell in row]
+    row = [smart_str(self.nullify(cell), self._encoding, strings_only=True, errors='replace') for cell in row]
     self._csv_writer.writerow(row)
     return self._line
 
   def fini_doc(self):
     return ""
+
+  def nullify(self, cell):
+    return cell if cell is not None else 'NULL'

+ 5 - 3
tools/jenkins/build-functions

@@ -42,6 +42,7 @@ CDH_URL=${CDH_URL:-http://nightly.cloudera.com/cdh4/cdh/4/hadoop-2.0.0-cdh4.5.0-
 
 CDH_TGZ=$(basename $CDH_URL)
 CDH_VERSION=${CDH_TGZ/.tar.gz/}
+CDH_SHORT_VERSION=${CDH_VERSION/hadoop-/}
 CDH_CACHE="$HOME/.hue_cache/${CDH_TGZ}"
 CDH_MTIME_FILE="$HOME/.hue_cache/.cdh_mtime"
 
@@ -63,11 +64,12 @@ build_hadoop() {
   echo "Unpacking $CDH_CACHE to $HADOOP_DIR"
   tar -C $HADOOP_DIR -xzf $CDH_CACHE
   # For Hive
-  #ln -s "$HADOOP_DIR/${CDH_VERSION}/bin-mapreduce1" $HADOOP_MR1_HOME/bin
+  ln -sf $HADOOP_DIR/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*.jar $HADOOP_DIR/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-core.jar
+  ln -sf $HADOOP_DIR/hadoop/share/hadoop/common/hadoop-common-*-SNAPSHOT.jar $HADOOP_DIR/hadoop/share/hadoop/common/hadoop-common.jar
+  ln -sf $HADOOP_DIR/hadoop/share/hadoop/common/lib/hadoop-auth-*-SNAPSHOT.jar $HADOOP_DIR/hadoop/share/hadoop/common/lib/hadoop-auth.jar
+  ln -sf $HADOOP_DIR/hadoop/share/hadoop/hdfs/hadoop-hdfs-${CDH_SHORT_VERSION}.jar  $HADOOP_DIR/hadoop/share/hadoop/hdfs/hadoop-hdfs.jar
   # For MR2
-  #rm -f "$HADOOP_DIR/${CDH_VERSION}/share/hadoop/mapreduce"
   ln -sf "$HADOOP_DIR/${CDH_VERSION}/share/hadoop/mapreduce2" "$HADOOP_DIR/${CDH_VERSION}/share/hadoop/mapreduce"
-  #ln -sf "mapreduce" "$HADOOP_DIR/${CDH_VERSION}/share/hadoop/mapreduce"
   ln -s $HADOOP_DIR/${CDH_VERSION} $HADOOP_DIR/hadoop
 }