Przeglądaj źródła

[spark] Add an option for configuring Spark SQL

Note: Except for simple selects, Spark SQL is still way behind the regular Hive API
Keeping it commented by default
Romain Rigaux 10 lat temu
rodzic
commit
d44466e

+ 9 - 0
apps/beeswax/src/beeswax/server/dbms.py

@@ -100,6 +100,15 @@ def get_query_server_config(name='beeswax', server=None):
         'auth_password': AUTH_PASSWORD.get()
         'auth_password': AUTH_PASSWORD.get()
     }
     }
 
 
+  if name == 'sparksql': # Spark SQL is almost the same as Hive
+    from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
+
+    query_server.update({
+        'server_name': 'sparksql',
+        'server_host': SPARK_SERVER_HOST.get(),
+        'server_port': SPARK_SERVER_PORT.get()
+    })
+
   debug_query_server = query_server.copy()
   debug_query_server = query_server.copy()
   debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
   debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
   LOG.debug("Query Server: %s" % debug_query_server)
   LOG.debug("Query Server: %s" % debug_query_server)

+ 3 - 0
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -541,6 +541,9 @@ class HiveServerClient:
     if self.query_server['server_name'] == 'beeswax': # All the time
     if self.query_server['server_name'] == 'beeswax': # All the time
       kwargs['configuration'].update({'hive.server2.proxy.user': user.username})
       kwargs['configuration'].update({'hive.server2.proxy.user': user.username})
 
 
+    if self.query_server['server_name'] == 'sparksql': # All the time
+      kwargs['configuration'].update({'hive.server2.proxy.user': user.username})
+
     req = TOpenSessionReq(**kwargs)
     req = TOpenSessionReq(**kwargs)
     res = self._client.OpenSession(req)
     res = self._client.OpenSession(req)
 
 

+ 13 - 0
apps/spark/src/spark/conf.py

@@ -67,6 +67,19 @@ START_LIVY_SERVER = Config(
   type=coerce_bool,
   type=coerce_bool,
   private=True)
   private=True)
 
 
+
+SQL_SERVER_HOST = Config(
+  key="sql_server_host",
+  help=_t("Host where SparkSQL server is running."),
+  default="localhost")
+
+SQL_SERVER_PORT = Config(
+  key="sql_server_port",
+  help=_t("Port the SparkSQL server runs on."),
+  default=10000,
+  type=int)
+
+
 def get_livy_server_url():
 def get_livy_server_url():
   return 'http://%s:%s' % (LIVY_SERVER_HOST.get(), LIVY_SERVER_PORT.get())
   return 'http://%s:%s' % (LIVY_SERVER_HOST.get(), LIVY_SERVER_PORT.get())
 
 

+ 10 - 0
desktop/conf.dist/hue.ini

@@ -559,6 +559,10 @@
     name=Text
     name=Text
     interface=text
     interface=text
 
 
+    # [[[sparksql]]]
+    # name=SparkSql
+    # interface=hiveserver2
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure your Hadoop cluster.
 # Settings to configure your Hadoop cluster.
@@ -783,6 +787,12 @@
   # If livy should use proxy users when submitting a job.
   # If livy should use proxy users when submitting a job.
   ## livy_impersonation_enabled=true
   ## livy_impersonation_enabled=true
 
 
+  # Host of the Sql Server
+  ## sql_server_host=localhost
+
+  # Port of the Sql Server
+  ## sql_server_port=10000
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure the Oozie app
 # Settings to configure the Oozie app

+ 10 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -559,6 +559,10 @@
     name=Text
     name=Text
     interface=text
     interface=text
 
 
+    # [[[sparksql]]]
+    # name=SparkSql
+    # interface=hiveserver2
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure your Hadoop cluster.
 # Settings to configure your Hadoop cluster.
@@ -783,6 +787,12 @@
   # If livy should use proxy users when submitting a job.
   # If livy should use proxy users when submitting a job.
   ## livy_impersonation_enabled=true
   ## livy_impersonation_enabled=true
 
 
+  # Host of the Sql Server
+  ## sql_server_host=localhost
+
+  # Port of the Sql Server
+  ## sql_server_port=10000
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure the Oozie app
 # Settings to configure the Oozie app