Bläddra i källkod

HUE-518 [beeswax] File Resources support other file systems

Romain Rigaux 12 år sedan
förälder
incheckning
dbebdca350

+ 8 - 2
apps/beeswax/src/beeswax/design.py

@@ -18,16 +18,18 @@
 """
 The HQLdesign class can (de)serialize a design to/from a QueryDict.
 """
-import json
 
+import json
 import logging
 import re
+import urlparse
 
 import django.http
 from django import forms
 
 from desktop.lib.django_forms import BaseSimpleFormSet, MultiForm
 from desktop.lib.django_mako import render_to_string
+from hadoop.cluster import get_hdfs
 
 
 LOG = logging.getLogger(__name__)
@@ -104,7 +106,11 @@ class HQLdesign(object):
     configuration = []
 
     for f in self.file_resources:
-      configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path'])))
+      if not urlparse.urlsplit(f['path']).scheme:
+        scheme = get_hdfs().fs_defaultfs
+      else:
+        scheme = ''
+      configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path'], scheme=scheme)))
 
     for f in self.functions:
       configuration.append(render_to_string("hql_function.mako", f))

+ 1 - 4
apps/beeswax/src/beeswax/templates/hql_resource.mako

@@ -13,8 +13,5 @@
 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
-<%!
-  import hadoop
-%>
 
-ADD ${type} ${hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.get() + path}
+ADD ${type} ${scheme + path}

+ 16 - 0
apps/beeswax/src/beeswax/tests.py

@@ -1585,6 +1585,22 @@ class TestWithMockedServer(object):
     assert_equal(0, sum([query_id in ids_page_1 for query_id in ids]))
 
 
+class TestDesign():
+
+  def test_hql_resource(self):
+    design = hql_query('SELECT')
+    design._data_dict['file_resources'] = [
+        {'type': 'FILE', 'path': 'my_file'},
+        {'type': 'FILE', 'path': '/my_path/my_file'},
+        {'type': 'FILE', 'path': 's3://host/my_s3_file'}
+    ]
+
+    assert_equal([
+        u'\nADD FILE hdfs://localhost:8020my_file\n', # Expected
+        u'\nADD FILE hdfs://localhost:8020/my_path/my_file\n',
+        u'\nADD FILE s3://host/my_s3_file\n'
+    ], design.get_configuration_statements())
+
 def search_log_line(component, expected_log, all_logs):
   """Checks if 'expected_log' can be found in one line of 'all_logs' outputed by the logging component 'component'."""
   return re.compile('.+?%(component)s(.+?)%(expected_log)s' % {'component': component, 'expected_log': expected_log}).search(all_logs)

+ 0 - 1
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -36,7 +36,6 @@ from hadoop.fs.webhdfs_types import WebHdfsStat, WebHdfsContentSummary
 from hadoop.conf import UPLOAD_CHUNK_SIZE
 
 import hadoop.conf
-import hadoop.core_site
 
 
 DEFAULT_HDFS_SUPERUSER = 'hdfs'