Переглянути джерело

HUE-326. Beeswax might be leaking file descriptors

Add cleanup mechanism for HiveHistory.

Subclass SessionState, implement a cleanup mehtod, use this new class in
RunningQueryState and set HiveHistory object to null once we are done
executing/explaining the query. HIVE-1508 is related to this issue.
Shrijeet Paliwal 15 роки тому
батько
коміт
c769c61d00

+ 19 - 6
apps/beeswax/java/src/main/java/com/cloudera/beeswax/BeeswaxServiceImpl.java

@@ -112,7 +112,7 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
   private class RunningQueryState {
     private QueryState state = QueryState.CREATED;
     // Thread local used by Hive quite a bit.
-    private SessionState sessionState;
+    private CleanableSessionState sessionState;
     private Throwable exception;
     private Driver driver;
     private ByteArrayOutputStream errStream = new ByteArrayOutputStream();
@@ -179,9 +179,13 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
     }
 
     synchronized public void compile() throws BeeswaxException {
-      assertState(QueryState.INITIALIZED);
-      checkedCompile();
-      state = QueryState.COMPILED;
+      try {
+        assertState(QueryState.INITIALIZED);
+        checkedCompile();
+        state = QueryState.COMPILED;
+      } finally {
+        cleanSessionState();
+      }
     }
 
     private void assertState(QueryState expected) {
@@ -267,8 +271,8 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
       }
       hiveConf.setClassLoader(loader);
       Thread.currentThread().setContextClassLoader(loader);
-      SessionState.start(hiveConf); // this is thread-local
-      this.sessionState = SessionState.get();
+      this.sessionState = new CleanableSessionState(hiveConf);
+      SessionState.start(this.sessionState);
 
       // If this work has a LogContext, associate the children output to the logContext
       OutputStream lcOutStream = null;
@@ -426,6 +430,7 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
       } finally {
         // Don't let folks re-use the state object.
         state = QueryState.FINISHED;
+        cleanSessionState();
       }
       return new QueryExplanation(sb.toString());
     }
@@ -446,6 +451,8 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
             materializeResults(r, fromBeginning);
           } catch (IOException e) {
             throw new BeeswaxException(e.toString(), logContext.getName(), handle);
+          } finally {
+            cleanSessionState();
           }
           break;
         case EXCEPTION:
@@ -498,6 +505,8 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
               } catch (Throwable t) {
                 LOG.error("Exception while processing query", t);
                 state.saveException(t);
+              } finally {
+                cleanSessionState();
               }
               return null;
             }
@@ -505,6 +514,10 @@ public class BeeswaxServiceImpl implements BeeswaxService.Iface {
         }
       });
     }
+
+    private void cleanSessionState() {
+      ((CleanableSessionState) SessionState.get()).destroyHiveHistory();
+    }
   }
 
 

+ 45 - 0
apps/beeswax/java/src/main/java/com/cloudera/beeswax/CleanableSessionState.java

@@ -0,0 +1,45 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.cloudera.beeswax;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.history.HiveHistory;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Cleanable object of type SessionState.
+ *
+ * This lets us clean object references held by
+ * {@link SessionState} (specifically {@link HiveHistory}.
+ * Cleanup is needed since via RunningQueryState object we keep
+ * a session alive for EVICTION_INTERVAL time period.
+ * Refer HUE-326.
+ *
+ */
+public class CleanableSessionState extends SessionState {
+
+  public CleanableSessionState() {
+    super();
+  }
+
+  public CleanableSessionState(HiveConf conf) {
+    super(conf);
+  }
+
+  public void destroyHiveHistory() {
+    this.hiveHist = null;
+  }
+}