Explorar el Código

HUE-309. Hardcoded "default" hdfs cluster in config

bc Wong hace 15 años
padre
commit
9946e5c055

+ 11 - 0
desktop/core/src/desktop/lib/fsmanager.py

@@ -48,6 +48,17 @@ def get_filesystem(name):
   _init_filesystems()
   return _filesystems[name]
 
+def get_default_hdfs():
+  """
+  Return the (name, fs) for the default hdfs.
+  Return (None, None) if no hdfs cluster configured
+  """
+  _init_filesystems()
+  for name, fs in _filesystems.iteritems():
+    # Return the first HDFS encountered
+    if fs.uri.startswith('hdfs'):
+      return name, fs
+  return None, None
 
 def reset():
   """

+ 9 - 11
desktop/core/src/desktop/middleware.py

@@ -123,22 +123,20 @@ class ClusterMiddleware(object):
     if "fs" in view_kwargs:
       del view_kwargs["fs"]
 
-    if not fs_ref:
-      fs_ref = "default"
-    try:
-      request.fs = fsmanager.get_filesystem(fs_ref)
-      request.fs_ref = fs_ref
-    except KeyError:
-      if fs_ref == "default" and not has_hadoop:
-        request.fs = None
-      else:
-        raise
+    if fs_ref is None:
+      request.fs_ref, request.fs = fsmanager.get_default_hdfs()
+    else:
+      try:
+        request.fs = fsmanager.get_filesystem(fs_ref)
+        request.fs_ref = fs_ref
+      except KeyError:
+        raise KeyError('Cannot find filesystem called "%s"' % (fs_ref,))
 
     if request.user.is_authenticated() and request.fs is not None:
       request.fs.setuser(request.user.username)
 
     if request.user.is_authenticated() and has_hadoop:
-      request.jt = cluster.get_mrcluster()
+      request.jt = cluster.get_default_mrcluster()
       if request.jt is not None:
         request.jt.setuser(request.user.username)
     else:

+ 12 - 0
desktop/libs/hadoop/src/hadoop/cluster.py

@@ -59,6 +59,18 @@ def get_all_hdfs():
   return FS_CACHE
 
 MR_CACHE = None
+
+def get_default_mrcluster():
+  global MR_CACHE
+  try:
+    return get_mrcluster()
+  except KeyError:
+    # Return an arbitrary cluster
+    candidates = all_mrclusters()
+    if candidates:
+      return candidates.values()[0]
+    return None
+
 def get_mrcluster(identifier="default"):
   global MR_CACHE
   all_mrclusters()

+ 4 - 0
desktop/libs/hadoop/src/hadoop/fs/__init__.py

@@ -176,6 +176,10 @@ class LocalSubFileSystem(object):
   rmtree = _wrap(shutil.rmtree)
   chown = _wrap(os.chown, paths=[0], users=[1], groups=[2])
 
+  @property
+  def uri(self):
+    return self.name
+
   def stats(self, path, raise_on_fnf=True):
     path = self._resolve_path(path)
     try:

+ 23 - 0
desktop/libs/hadoop/src/hadoop/tests.py

@@ -26,6 +26,7 @@ from nose.plugins.attrib import attr
 import desktop.views
 
 from desktop.lib.django_test_util import make_logged_in_client
+from hadoop import cluster
 from hadoop import conf
 from hadoop import confparse
 from hadoop import mini_cluster
@@ -164,3 +165,25 @@ def test_config_validator_more():
     for old_conf in reset:
       old_conf()
     cluster.shutdown()
+
+
+def test_non_default_cluster():
+  NON_DEFAULT_NAME = 'non_default'
+  cluster.clear_caches()
+  reset = (
+    conf.HDFS_CLUSTERS.set_for_testing({ NON_DEFAULT_NAME: { } }),
+    conf.MR_CLUSTERS.set_for_testing({ NON_DEFAULT_NAME: { } }),
+  )
+  try:
+    # This is indeed the only hdfs/mr cluster
+    assert_equal(1, len(cluster.get_all_hdfs()))
+    assert_equal(1, len(cluster.all_mrclusters()))
+    assert_true(cluster.get_hdfs(NON_DEFAULT_NAME))
+    assert_true(cluster.get_mrcluster(NON_DEFAULT_NAME))
+
+    cli = make_logged_in_client()
+    # That we can get to a view without errors means that the middlewares work
+    cli.get('/about')
+  finally:
+    for old_conf in reset:
+      old_conf()