Эх сурвалжийг харах

[notebook] Automatically start the DBProxy server if we have some JDBC snippets

Add a main blacklist flag in the config
enable_dbproxy_server=true

Move command to Nnotebook to make it discoverable.
Romain Rigaux 10 жил өмнө
parent
commit
49a29dc

+ 5 - 0
Makefile.vars

@@ -91,6 +91,11 @@ MAVEN_VERSION = $(DESKTOP_VERSION)-SNAPSHOT
 ##############################
 DESKTOP_PLUGIN_JAR := $(ROOT)/desktop/libs/hadoop/java-lib/hue-plugins-$(MAVEN_VERSION).jar
 
+##############################
+# Path to the desktop dbproxy jar
+##############################
+DB_PROXY_JAR := $(ROOT)/desktop/libs/librdbms/java-lib/dbproxy-1.0.jar
+
 ################################################
 # Internationalization
 ################################################

+ 16 - 0
apps/spark/src/spark/management/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 16 - 0
apps/spark/src/spark/management/commands/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -585,6 +585,10 @@
     #   options='{"url": "jdbc:mysql://localhost:3306/hue", "driver": "com.mysql.jdbc.Driver", "user": "root", "password": "root"}'
 
 
+  ## Main flag to override the automatic starting of the DBProxy server.
+  # enable_dbproxy_server=true
+
+
 ###########################################################################
 # Settings to configure your Hadoop cluster.
 ###########################################################################

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -593,6 +593,10 @@
     #   options='{"url": "jdbc:mysql://localhost:3306/hue", "driver": "com.mysql.jdbc.Driver", "user": "root", "password": "root"}'
 
 
+  ## Main flag to override the automatic starting of the DBProxy server.
+  # enable_dbproxy_server=true
+
+
 ###########################################################################
 # Settings to configure your Hadoop cluster.
 ###########################################################################

+ 4 - 4
desktop/libs/librdbms/Makefile

@@ -45,20 +45,20 @@ PLUGIN_JAVA_LIB := $(PLUGIN_DIR)/java-lib
 PLUGIN_JAVA_DIR := $(PLUGIN_DIR)/java
 BLD_DIR_PLUGINS := $(PLUGIN_DIR)/java/target
 
-compile: $(DESKTOP_PLUGIN_JAR)
+compile: $(DB_PROXY_JAR)
 
 # Build the plugin jar iff the source is present
 ifneq (,$(wildcard $(PLUGIN_JAVA_DIR)))
-$(DESKTOP_PLUGIN_JAR): $(shell find $(PLUGIN_JAVA_DIR) -type f)
+$(DB_PROXY_JAR): $(shell find $(PLUGIN_JAVA_DIR) -type f)
 	mkdir -p $(PLUGIN_JAVA_LIB)
 	@echo "--- Building DBProxy plugin"
 	cd $(PLUGIN_JAVA_DIR) && mvn clean install -DskipTests $(MAVEN_OPTIONS)
-	cp $(BLD_DIR_PLUGINS)/dbproxy-1.0-SNAPSHOT.jar $(DESKTOP_PLUGIN_JAR)
+	cp $(BLD_DIR_PLUGINS)/dbproxy-1.0-SNAPSHOT.jar $(DB_PROXY_JAR)
 
 clean::
 	rm -rf $(PLUGIN_JAVA_LIB)
 	cd $(PLUGIN_JAVA_DIR) && mvn clean $(MAVEN_OPTIONS) ||:
 else
-$(DESKTOP_PLUGIN_JAR):
+$(DB_PROXY_JAR):
 	$(error Cannot build DBProxy plugin without source)
 endif

+ 2 - 2
desktop/libs/librdbms/java/bin/dbproxy

@@ -18,9 +18,9 @@
 
 set -e
 
-export DBPROXY_HOME=$(cd $(dirname $0)/.. && pwd)
+export DBPROXY_HOME=$(cd $(dirname $0)/../.. && pwd)
 
-export CLASSPATH=$DBPROXY_HOME/target/dbproxy-1.0-SNAPSHOT.jar:$CLASSPATH
+export CLASSPATH=$DBPROXY_HOME/java-lib/dbproxy-1.0.jar:$CLASSPATH
 
 echo "JDBC drivers jars are expected in the CLASSPATH: '$CLASSPATH'"
 echo "Print port and wait"

+ 0 - 0
desktop/libs/librdbms/src/librdbms/management/__init__.py


+ 0 - 0
desktop/libs/librdbms/src/librdbms/management/commands/__init__.py


+ 6 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -58,3 +58,9 @@ INTERPRETERS = UnspecifiedConfigSection(
     )
   )
 )
+
+ENABLE_DBPROXY_SERVER = Config(
+  key="enable_dbproxy_server",
+  help=_t("Main flag to override the automatic starting of the DBProxy server."),
+  type=bool,
+  default=True)

+ 14 - 5
desktop/libs/librdbms/src/librdbms/management/commands/dbproxy_server.py → desktop/libs/notebook/src/notebook/management/commands/dbproxy_server.py

@@ -17,10 +17,8 @@
 
 import logging
 import os
-import subprocess
 
 from django.core.management.base import NoArgsCommand
-import spark.conf
 
 
 LOG = logging.getLogger(__name__)
@@ -35,10 +33,21 @@ class Command(NoArgsCommand):
 
   def handle(self, *args, **kwargs):
     env = os.environ.copy()
+#../../../../../librdbms/java/bin/dbproxy
+    cmd = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "librdbms", "java", "bin", "dbproxy")
 
-    args.append("com.cloudera.hue.livy.server.Main")
+#     classpath = env.get('CLASSPATH', '').split(os.pathsep)
+#     classpath += 
 
-    LOG.info("Executing %r (%r) (%r)" % (args[0], args, env))
+    args = [
+      #"java",
+    ]
+
+#     args.extend(("-cp", os.pathsep.join(classpath)))
+
+#     args.append("com.cloudera.hue.livy.server.Main")
+
+    LOG.info("Executing %r (%r) (%r)" % (args, args, env))
 
     # Use exec, so that this takes only one process.
-    os.execvpe(args[0], args, env)
+    os.execvpe(cmd, args, env)

+ 45 - 0
desktop/libs/notebook/src/notebook/monkey_patches.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start DBProxy server if we have some JDBC snippets
+
+from notebook.conf import get_interpreters, ENABLE_DBPROXY_SERVER
+
+
+def _start_livy_server():
+  import atexit
+  import subprocess
+  import sys
+  import time
+
+  p = subprocess.Popen([sys.executable, sys.argv[0], 'dbproxy_server'])
+
+  def cleanup():
+    p.terminate()
+    for _ in xrange(5):
+      if p.poll() == None:
+        time.sleep(1)
+      else:
+        break
+    else:
+      p.kill()
+
+  atexit.register(cleanup)
+
+
+if ENABLE_DBPROXY_SERVER.get() and [interpreter for interpreter in get_interpreters() if interpreter['interface'] == 'jdbc']:
+  _start_livy_server()

+ 11 - 0
desktop/libs/notebook/src/notebook/urls.py

@@ -17,6 +17,17 @@
 
 from django.conf.urls import patterns, url
 
+# FIXME: This could be replaced with hooking into the `AppConfig.ready()`
+# signal in Django 1.7:
+#
+# https://docs.djangoproject.com/en/1.7/ref/applications/#django.apps.AppConfig.ready
+#
+# For now though we have to load in the monkey patches here because we know
+# this file has been loaded after `desktop.settings` has been loaded.
+
+# Start DBProxy server
+import notebook.monkey_patches
+
 
 # Views
 urlpatterns = patterns('notebook.views',