Эх сурвалжийг харах

[beeswax] Install Hive and Impala sample on a non local host

Manage Impala uploads in case impersonation if off.
Romain Rigaux 11 жил өмнө
parent
commit
df7ee86

+ 24 - 4
apps/beeswax/src/beeswax/management/commands/beeswax_install_examples.py

@@ -24,6 +24,7 @@ from django.contrib.auth.models import User
 from django.utils.translation import ugettext as _
 
 from desktop.models import Document
+from hadoop import cluster
 
 import beeswax.conf
 
@@ -102,6 +103,7 @@ class SampleTable(object):
     self.filename = data_dict['data_file']
     self.hql = data_dict['create_hql']
     self.query_server = get_query_server_config(app_name)
+    self.app_name = app_name
 
     # Sanity check
     self._data_dir = beeswax.conf.LOCAL_EXAMPLES_DATA_DIR.get()
@@ -117,7 +119,7 @@ class SampleTable(object):
 
   def create(self, django_user):
     """
-    Create in Hive. Raise InstallException on failure.
+    Create table in the Hive Metastore.
     """
     LOG.info('Creating table "%s"' % (self.name,))
     db = dbms.get(django_user, self.query_server)
@@ -142,17 +144,35 @@ class SampleTable(object):
 
   def load(self, django_user):
     """
-    Load data into table. Raise InstallException on failure.
+    Upload data to HDFS home of user then load (aka move) it into the Hive table (in the Hive metastore in HDFS).
     """
     LOAD_HQL = \
       """
-      LOAD DATA local INPATH
+      LOAD DATA INPATH
       '%(filename)s' OVERWRITE INTO TABLE %(tablename)s
       """
 
+    fs = cluster.get_hdfs()
+
+    if self.app_name == 'impala':
+      # Because Impala does not have impersonation on by default, we use a public destination for the upload.
+      from impala.conf import IMPERSONATION_ENABLED
+      if not IMPERSONATION_ENABLED.get():
+        tmp_public = '/tmp/public_hue_examples'
+        fs.do_as_user(django_user, fs.mkdir, tmp_public, '0777')
+        hdfs_root_destination = tmp_public
+    else:
+      hdfs_root_destination = fs.do_as_user(django_user, fs.get_home_dir)
+
+    hdfs_destination = os.path.join(hdfs_root_destination, self.name)
+
+    LOG.info('Uploading local data %s to HDFS table "%s"' % (self.name, hdfs_destination))
+    fs.do_as_user(django_user, fs.copyFromLocal, self._contents_file, hdfs_destination)
+
     LOG.info('Loading data into table "%s"' % (self.name,))
-    hql = LOAD_HQL % dict(tablename=self.name, filename=self._contents_file)
+    hql = LOAD_HQL % {'tablename': self.name, 'filename': hdfs_destination}
     query = hql_query(hql)
+
     try:
       results = dbms.get(django_user, self.query_server).execute_and_wait(query)
       if not results: