Browse Source

HUE-676. [beeswax] BeeswaxServer OOME when hive loglevel set to DEBUG

The size is determined from the property "beeswax.log.context.size",
which can be set in beeswax_server.sh in HADOOP_OPTS. The default is 1MB.
bc Wong 13 năm trước cách đây
mục cha
commit
a5c21761b8

+ 78 - 0
apps/beeswax/java/src/main/java/com/cloudera/beeswax/LinkedStringBuffer.java

@@ -0,0 +1,78 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.cloudera.beeswax;
+
+import java.util.LinkedList;
+
+/**
+ * A linked string buffer with a capacity limit.
+ */
+public class LinkedStringBuffer {
+
+  private final LinkedList<String> list;
+  private final int capacity;
+  private int size;
+
+  /**
+   * Create a buffer with the specified capacity on the number of characters.
+   */
+  public LinkedStringBuffer(int capacity) {
+    this.capacity = capacity;
+    list = new LinkedList<String>();
+  }
+
+  /**
+   * @return Size (number of characters) in the buffer
+   */
+  public synchronized int size() {
+    return size;
+  }
+
+  /**
+   * Write to the buffer, which will remove previously written strings if
+   * we don't fit in capacity.
+   */
+  public synchronized void write(String data) {
+    list.add(data);
+    size += data.length();
+
+    // Trim from the front
+    while (size > capacity) {
+      String evicted = list.remove(0);
+      size -= evicted.length();
+    }
+  }
+
+  /**
+   * @return All the data in the buffer.
+   */
+  public synchronized String read() {
+    StringBuilder sb = new StringBuilder();
+    for (String s : list) {
+      sb.append(s);
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Remove all stored data.
+   */
+  public synchronized void clear() {
+    list.clear();
+    size = 0;
+  }
+}

+ 19 - 25
apps/beeswax/java/src/main/java/com/cloudera/beeswax/LogContext.java

@@ -15,14 +15,12 @@
 // limitations under the License.
 package com.cloudera.beeswax;
 
-import java.io.CharArrayWriter;
+import com.google.common.base.Charsets;
+
 import java.io.IOException;
 import java.io.OutputStream;
-import java.io.Writer;
 
 import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -39,6 +37,11 @@ import org.apache.log4j.PatternLayout;
  * to one context at any time). And their logs are all kept here.
  */
 public class LogContext {
+  private static final String BUFFER_SIZE =
+      System.getProperty("beeswax.log.context.size", "1048576");
+
+  private static final String HIVE_ENCODING = Charsets.UTF_8.name();
+
   /** This Logger's name is added to an exclusion list in LogDivertAppender */
   private static Logger LOG = Logger.getLogger(LogContext.class.getName());
 
@@ -54,7 +57,7 @@ public class LogContext {
   private static boolean lcIsInitialized = false;
 
   /** Where we keep the log */
-  private CharArrayWriter logStore;
+  private LinkedStringBuffer logBuffer;
 
   /** Name of the context */
   private String name;
@@ -66,28 +69,24 @@ public class LogContext {
    * The LogContextOutputStream helps translate a LogContext to an OutputStream.
    */
   private static class LogContextOutputStream extends OutputStream {
-    private Writer logStore;
+    private LinkedStringBuffer backingStore;
 
     /**
      * Create a LogContextOutputStream backed by the given Writer.
      */
-    public LogContextOutputStream(Writer logStore) {
+    public LogContextOutputStream(LinkedStringBuffer logBuffer) {
       super();
-      this.logStore = logStore;
+      backingStore = logBuffer;
     }
 
     @Override
     public void write(byte[] b) throws IOException {
-      synchronized (this.logStore) {
-        this.logStore.write(new String(b));
-      }
+      backingStore.write(new String(b, HIVE_ENCODING));
     }
 
     @Override
     public void write(byte[] b, int off, int len) throws IOException {
-      synchronized (this.logStore) {
-        this.logStore.write(new String(b, off, len));
-      }
+      backingStore.write(new String(b, off, len, HIVE_ENCODING));
     }
 
     @Override
@@ -138,8 +137,9 @@ public class LogContext {
    * Create a LogContext with the given name.
    */
   private LogContext(String name) {
+    int bufferSize = Integer.valueOf(BUFFER_SIZE);
     this.name = name;
-    this.logStore = new CharArrayWriter();
+    this.logBuffer = new LinkedStringBuffer(bufferSize);
     this.createTime = System.currentTimeMillis();
   }
 
@@ -235,34 +235,28 @@ public class LogContext {
    * Store the given log message
    */
   public void writeLog(String logMessage) {
-    synchronized (this.logStore) {
-      this.logStore.write(logMessage, 0, logMessage.length());
-    }
+    logBuffer.write(logMessage);
   }
 
   /**
    * Retrieve the log stored
    */
   public String readLog() {
-    synchronized (this.logStore) {
-      return this.logStore.toString();
-    }
+    return logBuffer.read();
   }
 
   /**
    * Reset the log stored
    */
   public void resetLog() {
-    synchronized (this.logStore) {
-      this.logStore.reset();
-    }
+    logBuffer.clear();
   }
 
   /**
    * Get an OutputStream that writes to this LogContext.
    */
   public OutputStream getOutputStream() {
-    return new LogContextOutputStream(this.logStore);
+    return new LogContextOutputStream(logBuffer);
   }
 
   public String getName() {

+ 56 - 0
apps/beeswax/java/src/test/java/com/cloudera/beeswax/LinkedStringBufferTest.java

@@ -0,0 +1,56 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.cloudera.beeswax;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class LinkedStringBufferTest {
+
+  @Test
+  public void testBuffer() {
+    LinkedStringBuffer buf = new LinkedStringBuffer(8);
+
+    // Normal write
+    buf.write("foo");
+    buf.write("bar");
+    assertEquals(6, buf.size());
+    assertEquals("foobar", buf.read());
+
+    // Eviction
+    buf.write("baz");
+    assertEquals(6, buf.size());
+    assertEquals("barbaz", buf.read());
+
+    buf.clear();
+    assertEquals(0, buf.size());
+    assertEquals("", buf.read());
+
+    // Just barely fit
+    String str8 = "12345678";
+    buf.write(str8);
+    assertEquals(8, buf.size());
+    assertEquals(str8, buf.read());
+
+    // Don't fit
+    String str9 = str8 + '9';
+    buf.write(str9);
+    assertEquals(0, buf.size());
+    assertEquals("", buf.read());
+  }
+}