Bläddra i källkod

Beeswax configuration for tests, handling exceptions in a non-regressive manner

vinithra 15 år sedan
förälder
incheckning
e2e845b8eb

+ 1 - 1
apps/beeswax/beeswax_server.sh

@@ -36,7 +36,7 @@ BEESWAX_HIVE_LIB=$BEESWAX_ROOT/hive/lib
 
 echo \$HADOOP_HOME=$HADOOP_HOME
 
-export HADOOP_CLASSPATH=$(find $BEESWAX_HIVE_LIB -name "*.jar" | tr "\n" :):$HIVE_CONF_DIR
+export HADOOP_CLASSPATH=$(find $BEESWAX_HIVE_LIB -name "*.jar" | tr "\n" :):$HIVE_CONF_DIR:$BEESWAX_ROOT/../../desktop/libs/hadoop/static-group-mapping/java-lib/static-group-mapping-1.0.jar
 echo \$HADOOP_CLASSPATH=$HADOOP_CLASSPATH
 
 # Use HADOOP_CONF_DIR to preprend to classpath, to avoid fb303 conflict,

+ 96 - 94
apps/beeswax/java/src/com/cloudera/beeswax/BeeswaxServiceImpl.java

@@ -21,10 +21,12 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.UnsupportedEncodingException;
+import java.lang.reflect.UndeclaredThrowableException;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
+import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.security.SecureRandom;
 import java.text.SimpleDateFormat;
@@ -588,13 +590,39 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
     evicter.start();
   }
 
+
+  private <T> T doWithState(RunningQueryState state, PrivilegedExceptionAction<T> action)
+  throws BeeswaxException
+  {
+    try{
+      UserGroupInformation ugi = UserGroupInformation.createProxyUser(state.query.hadoop_user, UserGroupInformation.getLoginUser());
+      return ugi.doAs(action);
+    } catch (UndeclaredThrowableException e) {
+      if (e.getUndeclaredThrowable() instanceof PrivilegedActionException) {
+        Throwable bwe = e.getUndeclaredThrowable().getCause();
+        if (bwe instanceof BeeswaxException) {
+          LOG.error("Caught BeeswaxException", (BeeswaxException) bwe);
+          throw (BeeswaxException) bwe;
+        }
+      }
+      LOG.error("Caught unexpected exception.", e);
+      throw new BeeswaxException(e.getMessage(), state.handle.log_context, state.handle);
+    } catch (IOException e) {
+      LOG.error("Caught IOException", e);
+      throw new BeeswaxException(e.getMessage(), state.handle.log_context, state.handle);
+    } catch (InterruptedException e) {
+      LOG.error("Caught InterruptedException", e);
+      throw new BeeswaxException(e.getMessage(), state.handle.log_context, state.handle);
+    }
+  }
+
   /**
    * Submit a query and return a handle (QueryHandle). The query runs asynchronously.
    * Queries can be long-lasting, so we push the execution into a new state.
    * Compiling happens in the current context so we report errors early.
    */
   @Override
-  public QueryHandle query(final Query query) throws BeeswaxException {
+  public QueryHandle query(Query query) throws BeeswaxException {
     // First, create an id and reset the LogContext
     String uuid = UUID.randomUUID().toString();
     final QueryHandle handle = new QueryHandle(uuid, uuid);
@@ -603,37 +631,30 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
 
     // Make an administrative record
     final RunningQueryState state = new RunningQueryState(query, lc);
-
     try {
-      UserGroupInformation ugi = UserGroupInformation.createProxyUser(query.hadoop_user, UserGroupInformation.getLoginUser());
-      return ugi.doAs(new PrivilegedExceptionAction<QueryHandle>() {
-        public QueryHandle run() throws Exception {
-          state.setQueryHandle(handle);
-          runningQueries.put(handle.id, state);
-          state.initialize();
-          // All kinds of things can go wrong when we compile it. So catch all.
-          try {
-            state.compile();
-          } catch (BeeswaxException perr) {
-            state.saveException(perr);
-            throw perr;
-          } catch (Throwable t) {
-            state.saveException(t);
-            throw new BeeswaxException(t.toString(), handle.log_context, handle);
-          }
-          // Now spin off the query.
-          state.submitTo(executor, lc);
-          return handle;
-        }
-      });
-    } catch (IOException e) {
-      String errorMsg = "Error while creating proxy user";
-      LOG.error(errorMsg);
-      throw new BeeswaxException(errorMsg, handle.log_context, handle);
-    } catch (InterruptedException e) {
-      String errorMsg = "Error while submitting query";
-      LOG.error(errorMsg);
-      throw new BeeswaxException(errorMsg, handle.log_context, handle);
+      return doWithState(state,
+          new PrivilegedExceptionAction<QueryHandle>() {
+            public QueryHandle run() throws Exception {
+              state.setQueryHandle(handle);
+              runningQueries.put(handle.id, state);
+              state.initialize();
+              // All kinds of things can go wrong when we compile it. So catch all.
+              try {
+                state.compile();
+              } catch (BeeswaxException perr) {
+                state.saveException(perr);
+                throw perr;
+              } catch (Throwable t) {
+                state.saveException(t);
+                throw new BeeswaxException(t.toString(), handle.log_context, handle);
+              }
+              // Now spin off the query.
+              state.submitTo(executor, lc);
+              return handle;
+            }
+          });
+    } catch (BeeswaxException e) {
+      throw e;
     }
   }
 
@@ -660,37 +681,31 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
    * Get the query plan for a query.
    */
   @Override
-  public QueryExplanation explain(final Query query) throws BeeswaxException, TException {
+  public QueryExplanation explain(Query query) throws BeeswaxException, TException {
     final String contextName = UUID.randomUUID().toString();
     LogContext lc = LogContext.registerCurrentThread(contextName);
     final RunningQueryState state = new RunningQueryState(query, lc);
     try {
-      UserGroupInformation ugi = UserGroupInformation.createProxyUser(query.hadoop_user, UserGroupInformation.getLoginUser());
-      return ugi.doAs(new PrivilegedExceptionAction<QueryExplanation>() {
-        public QueryExplanation run() throws Exception {
-          state.initialize();
-          QueryExplanation exp;
-          // All kinds of things can go wrong when we compile it. So catch all.
-          try {
-            exp = state.explain();
-          } catch (BeeswaxException perr) {
-            throw perr;
-          } catch (Throwable t) {
-            throw new BeeswaxException(t.toString(), contextName, null);
-          }
-          // On success, we remove the LogContext
-          LogContext.destroyContext(contextName);
-          return exp;
-        }
-      });
-    } catch (IOException e) {
-      String errorMsg = "Error while creating proxy user";
-      LOG.error(errorMsg);
-      throw new BeeswaxException(errorMsg, state.handle.log_context, state.handle);
-    } catch (InterruptedException e) {
-      String errorMsg = "Error while submitting query";
-      LOG.error(errorMsg);
-      throw new BeeswaxException(errorMsg, state.handle.log_context, state.handle);
+      return doWithState(state,
+          new PrivilegedExceptionAction<QueryExplanation>() {
+            public QueryExplanation run() throws Exception {
+              state.initialize();
+              QueryExplanation exp;
+              // All kinds of things can go wrong when we compile it. So catch all.
+              try {
+                exp = state.explain();
+              } catch (BeeswaxException perr) {
+                throw perr;
+              } catch (Throwable t) {
+                throw new BeeswaxException(t.toString(), contextName, null);
+              }
+              // On success, we remove the LogContext
+              LogContext.destroyContext(contextName);
+              return exp;
+            }
+          });
+    } catch (BeeswaxException e) {
+      throw e;
     }
   }
 
@@ -702,31 +717,24 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
    * @param fromBeginning  If true, rewind to the first row. Otherwise fetch from last position.
    */
   @Override
-  public Results fetch(final QueryHandle handle, final boolean fromBeginning)
+  public Results fetch(QueryHandle handle, final boolean fromBeginning)
       throws QueryNotFoundException, BeeswaxException {
     LogContext.unregisterCurrentThread();
     validateHandle(handle);
     LogContext.registerCurrentThread(handle.log_context);
     final RunningQueryState state = runningQueries.get(handle.id);
     try {
-      UserGroupInformation ugi = UserGroupInformation.createProxyUser(state.query.hadoop_user, UserGroupInformation.getLoginUser());
-
-      return ugi.doAs(new PrivilegedExceptionAction<Results>() {
-        public Results run() throws Exception {
-          if (state == null) {
-            throw new QueryNotFoundException();
-          }
-          return state.fetch(fromBeginning);
-        }
-      });
-    } catch (IOException e) {
-      String errorMsg = "Error while creating proxy user";
-      LOG.error(errorMsg);
-      throw new BeeswaxException(errorMsg, handle.log_context, handle);
-    } catch (InterruptedException e) {
-      String errorMsg = "Error while submitting query";
-      LOG.error(errorMsg);
-      throw new BeeswaxException(errorMsg, handle.log_context, handle);
+      return doWithState(state,
+          new PrivilegedExceptionAction<Results>() {
+            public Results run() throws Exception {
+              if (state == null) {
+                throw new QueryNotFoundException();
+              }
+              return state.fetch(fromBeginning);
+            }
+          });
+    } catch (BeeswaxException e) {
+      throw e;
     }
   }
 
@@ -771,23 +779,17 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
     LogContext.registerCurrentThread(handle.log_context);
     final RunningQueryState state = runningQueries.get(handle.id);
     try {
-      UserGroupInformation ugi = UserGroupInformation.createProxyUser(state.query.hadoop_user, UserGroupInformation.getLoginUser());
-
-      return ugi.doAs(new PrivilegedExceptionAction<ResultsMetadata>() {
-        public ResultsMetadata run() throws Exception {
-          if (state == null) {
-            throw new QueryNotFoundException();
-          }
-          return state.getResultMetadata();
-        }
-      });
-    } catch (IOException e) {
-      String errorMsg = "Error while creating proxy user";
-      LOG.error(errorMsg);
-      throw new QueryNotFoundException();
-    } catch (InterruptedException e) {
-      String errorMsg = "Error while submitting query";
-      LOG.error(errorMsg);
+      return doWithState(state,
+          new PrivilegedExceptionAction<ResultsMetadata>() {
+            public ResultsMetadata run() throws Exception {
+              if (state == null) {
+                throw new QueryNotFoundException();
+              }
+              return state.getResultMetadata();
+            }
+          });
+    } catch (BeeswaxException e) {
+      LOG.error("Caught BeeswaxException.", e);
       throw new QueryNotFoundException();
     }
   }

+ 1 - 1
apps/beeswax/java/src/com/cloudera/beeswax/Server.java

@@ -157,7 +157,7 @@ public class Server {
    */
   private static void createDirectoriesAsNecessary() {
     try {
-      LOG.info("Classpath: " + System.getProperty("java.class.path"));
+      LOG.debug("Classpath: " + System.getProperty("java.class.path"));
       HiveConf conf = new HiveConf(Driver.class);
       FileSystem fs = FileSystem.get(conf);
       Path tmpDir = new Path("/tmp");

+ 6 - 0
apps/beeswax/src/beeswax/tests.py

@@ -214,6 +214,9 @@ for x in sys.stdin:
     assert_equal(257, response.content.count("\n"))
 
   def test_query_with_udf(self):
+    """
+    Testing query with udf
+    """
     response = _make_query(self.client, "SELECT my_sqrt(foo), my_power(foo, foo) FROM test WHERE foo=4",
       udfs=[('my_sqrt', 'org.apache.hadoop.hive.ql.udf.UDFSqrt'),
         ('my_power', 'org.apache.hadoop.hive.ql.udf.UDFPower')], local=False)
@@ -264,6 +267,9 @@ for x in sys.stdin:
     check_error_in_response(resp)
 
   def test_parameterization(self):
+    """
+    Test parameterization
+    """
     response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", is_parameterized=False)
     # Assert no parameterization was offered
     assert_equal("watch_wait.mako", response.template, "we should have seen the template for a query executing")

+ 4 - 4
apps/beeswax/src/beeswax/views.py

@@ -66,7 +66,7 @@ def show_tables(request):
 def describe_table(request, table):
   table_obj = db_utils.meta_client().get_table("default", table)
   # Show the first few rows
-  hql = "SELECT * FROM %s" % (table,)
+  hql = "SELECT * FROM `%s`" % (table,)
   query_msg = make_beeswax_query(request, hql)
   try:
     results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
@@ -93,7 +93,7 @@ def drop_table(request, table):
     title = "This may delete the underlying data as well as the metadata.  Drop table %s?" % table
     return render('confirm.html', request, dict(url=request.path, title=title))
   elif request.method == 'POST':
-    hql = "DROP TABLE %s" % (table,)
+    hql = "DROP TABLE `%s`" % (table,)
     query_msg = make_beeswax_query(request, hql)
     try:
       return execute_directly(request,
@@ -962,7 +962,7 @@ def _save_results_ctas(request, query_history, target_table, result_meta):
   """
   # Case 1: The results are straight from an existing table
   if result_meta.in_tablename:
-    hql = 'CREATE TABLE %s AS SELECT * FROM %s' % (target_table, result_meta.in_tablename)
+    hql = 'CREATE TABLE `%s` AS SELECT * FROM %s' % (target_table, result_meta.in_tablename)
     query_msg = make_beeswax_query(request, hql)
     # Display the CTAS running. Could take a long time.
     return execute_directly(request, query_msg, on_success_url=urlresolvers.reverse(show_tables))
@@ -1031,7 +1031,7 @@ def load_table(request, table):
       if form.cleaned_data['overwrite']:
         hql += " OVERWRITE"
       hql += " INTO TABLE "
-      hql += "%s" % (table,)
+      hql += "`%s`" % (table,)
       if len(form.partition_columns) > 0:
         hql += " PARTITION ("
         vals = []

+ 5 - 5
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -59,9 +59,9 @@ MAX_CLUSTER_STARTUP_TIME = 120.0
 
 # users and their groups which are used in Hue tests.
 TEST_USER_GROUP_MAPPING = {
-   'test': ['test','users'], 'chown_test': ['chown_test'],
+   'test': ['test','users','supergroup'], 'chown_test': ['chown_test'],
    'notsuperuser': ['notsuperuser'], 'gamma': ['gamma'],
-   'webui': ['webui']
+   'webui': ['webui'], 'hue': ['supergroup']
 }
 
 LOGGER=logging.getLogger(__name__)
@@ -124,8 +124,8 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
 
     write_config({'hadoop.security.group.mapping': 'org.apache.hadoop.security.StaticUserGroupMapping',
       'hadoop.security.static.group.mapping.file': tmppath('ugm.properties'),
-      'hadoop.proxyuser.'+self.superuser+'.groups': 'users',
-      'hadoop.proxyuser.'+self.superuser+'.hosts': 'localhost'}, tmppath('in-conf/core-site.xml'))
+      'hadoop.proxyuser.%s.groups' % (self.superuser,): 'users,supergroup',
+      'hadoop.proxyuser.%s.hosts' % (self.superuser,): 'localhost'}, tmppath('in-conf/core-site.xml'))
 
     hadoop_policy_keys = ['client', 'client.datanode', 'datanode', 'inter.datanode', 'namenode', 'inter.tracker', 'job.submission', 'task.umbilical', 'refresh.policy', 'admin.operations']
     hadoop_policy_config = {}
@@ -253,7 +253,7 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
     write_config(hadoop_policy_config, tmppath('conf/hadoop-policy.xml'))
 
     # Once the config is written out, we can start the 2NN.
-    args = [hadoop.conf.HADOOP_BIN.get(), 
+    args = [hadoop.conf.HADOOP_BIN.get(),
       '--config', self.config_dir,
       'secondarynamenode']