Jelajahi Sumber

HUE-8908 [abfs] Fix permissions, rename, add chmod (#967)

* Added feature where the full path of abfs stays as a the full path rather than changing whenever the user goes thorugh directories (not including the root directory)

* Fixed issue where the autocomplete requires an extra forward slash in front of a scheme to show all the results when manually typing in locations.
Changed the ABFS function stats to have an IOerror if the the path is not given correctly

* Changed where ABFS sidebar leads to. It now starts at the filesystem listed in the hue.ini when clicking the side bar.

* Fixed what is being displayed as ABFS file permission

* Re-enabled Chmod for ABFS, but also kept file permissions at ABFS root disabled

* Fixed issue where users cannot rename a file that had spaces
Slightly improved speed of program by performing less matches

* Move around methods and removed a LOG.debug
travisle22 6 tahun lalu
induk
melakukan
813bcd1e49

+ 1 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -1115,7 +1115,7 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE
         return self.isHdfs();
       });
       self.isPermissionEnabled = ko.pureComputed(function () {
-        return !self.isS3() && !self.isABFS();
+        return !self.isS3() && !self.isABFSRoot();
       });
       self.isReplicationEnabled = ko.pureComputed(function () {
         return self.isHdfs();

+ 3 - 3
desktop/core/src/desktop/js/sql/autocompleteResults.js

@@ -1330,13 +1330,13 @@ class AutocompleteResults {
 
       if (/^s3a:\/\//i.test(path)) {
         fetchFunction = 'fetchS3Path';
-        path = path.substring(5);
+        path = path.substring(4);
       } else if (/^adl:\/\//i.test(path)) {
         fetchFunction = 'fetchAdlsPath';
-        path = path.substring(5);
+        path = path.substring(4);
       } else if (/^abfs:\/\//i.test(path)) {
         fetchFunction = 'fetchAbfsPath';
-        path = path.substring(6);
+        path = path.substring(5);
       } else if (/^hdfs:\/\//i.test(path)) {
         path = path.substring(6);
       }

+ 3 - 1
desktop/core/src/desktop/models.py

@@ -1794,12 +1794,14 @@ class ClusterConfig(object):
       })
 
     if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
+      from azure.abfs.__init__ import get_home_dir_for_ABFS
+      
       interpreters.append({
         'type': 'abfs',
         'displayName': _('ABFS'),
         'buttonName': _('Browse'),
         'tooltip': _('ABFS'),
-        'page': '/filebrowser/view=' + urllib_quote('abfs://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote(get_home_dir_for_ABFS().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
 
     if 'metastore' in self.apps:

+ 30 - 18
desktop/libs/azure/src/azure/abfs/__init__.py

@@ -29,8 +29,7 @@ from azure.conf import get_default_abfs_fs
 
 LOG = logging.getLogger(__name__)
 
-ABFS_PATH_RE = re.compile('^/*[aA][bB][fF][sS]{1,2}://([$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9])(/(.*?)/?)?$')
-ABFS_PATH_FULL = re.compile('^/*[aA][bB][fF][sS]{1,2}://(([$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9])@[^.]*?\.dfs\.core\.windows\.net)(/(.*?)/?)?$')#bug here
+ABFS_PATH_RE = re.compile('^/*[aA][bB][fF][sS]{1,2}://([$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9])(@[^.]*?\.dfs\.core\.windows\.net)?(/(.*?)/?)?$') #check this first for problems
 ABFS_ROOT_S = 'abfss://'
 ABFS_ROOT = 'abfs://'
 ABFSACCOUNT_NAME = re.compile('^/*[aA][bB][fF][sS]{1,2}://[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9](@.*?)$')
@@ -41,21 +40,21 @@ def parse_uri(uri):
   Raises ValueError if invalid ABFS URI is passed.
   """
   match = ABFS_PATH_RE.match(uri)
-  direct_name = ''
-  base_direct_name = ''
-  file_system = ''
-  if match:
-    direct_name = match.group(3) or ''
-    base_direct_name = match.group(2) or ''
-    file_system = match.group(1)
-  else:
-    match = ABFS_PATH_FULL.match(uri)
-    if not match:
-      raise ValueError("Invalid ABFS URI: %s" % uri)
-    direct_name = match.group(4) or ''
-    base_direct_name = match.group(3) or ''
-    file_system = match.group(2)
-  return file_system, direct_name, base_direct_name
+  if not match:
+    raise ValueError("Invalid ABFS URI: %s" % uri)
+  direct_name = match.group(4) or ''
+  account_name_and_path = match.group(2) or ''
+  return match.group(1), direct_name, account_name_and_path
+
+def only_filesystem_and_account_name(uri):
+  """
+  Given a path returns only the filesystem and account name,
+  Returns uri if it doesn't match
+  """
+  match = ABFS_PATH_RE.match(uri)
+  if match and match.group(2):
+    return match.group(1) + match.group(2)
+  return uri
 
 def is_root(uri):
   """
@@ -105,13 +104,16 @@ def parent_path(path):
   """
   if is_root(path):
     return path
-  filesystem, directory_name, other = parse_uri(path)
+  filesystem, directory_name = parse_uri(path)[:2]
   parent = None
   if directory_name == "":
     if path.lower() == ABFS_ROOT_S:
       return ABFS_ROOT_S
     return ABFS_ROOT
   else:
+    x = only_filesystem_and_account_name(path)
+    if x !=path:
+      filesystem = x
     parent = '/'.join(directory_name.split('/')[:-1])
   if path.lower().startswith(ABFS_ROOT):
     return normpath(ABFS_ROOT + filesystem + '/' + parent)
@@ -162,6 +164,16 @@ def abfspath(path, fs_defaultfs = None):
   LOG.debug("%s" % path)
   return path
 
+def get_home_dir_for_ABFS():
+  """
+  Attempts to go to the directory set by the user in the configuration file. If not defaults to abfs:// 
+  """
+  try:
+    filesystem = parse_uri(get_default_abfs_fs())[0]
+    return "abfs://" + filesystem
+  except:
+    return 'abfs://'
+
 def abfsdatetime_to_timestamp(datetime):
   """
   Returns timestamp (seconds) by datetime string from ABFS API responses.

+ 28 - 9
desktop/libs/azure/src/azure/abfs/abfs.py

@@ -29,6 +29,7 @@ import re
 from math import ceil
 from posixpath import join
 from urllib.parse import urlparse
+from urllib import quote
 
 from hadoop.hdfs_site import get_umask_mode
 from hadoop.fs.exceptions import WebHdfsException
@@ -128,7 +129,6 @@ class ABFS(object):
     Test if a path exists
     """
     try:
-      #LOG.debug("checking existence")
       if ABFS.isroot(path):
         return True
       self.stats(path)
@@ -136,6 +136,8 @@ class ABFS(object):
       if e.code == 404:
         return False
       raise WebHdfsException
+    except IOError:
+      return False
     return True
 
   def stats(self, path, params=None, **kwargs):
@@ -145,7 +147,10 @@ class ABFS(object):
     """
     if ABFS.isroot(path):
       return ABFSStat.for_root(path)
-    file_system, dir_name = Init_ABFS.parse_uri(path)[:2]
+    try:
+      file_system, dir_name = Init_ABFS.parse_uri(path)[:2]
+    except:
+      raise IOError
     if dir_name == '':
       LOG.debug("Path being called is a Filesystem")
       return ABFSStat.for_filesystem(self._statsf(file_system, params, **kwargs), path)
@@ -160,7 +165,7 @@ class ABFS(object):
       LOG.warn("Path: %s is a Filesystem" % path)
       return self.listfilesystems_stats(params=None, **kwargs)
     dir_stats = []
-    file_system, directory_name = Init_ABFS.parse_uri(path)[:2]
+    file_system, directory_name, account = Init_ABFS.parse_uri(path)
     root = Init_ABFS.ABFS_ROOT
     if path.lower().startswith(Init_ABFS.ABFS_ROOT_S):
       root = Init_ABFS.ABFS_ROOT_S
@@ -173,6 +178,8 @@ class ABFS(object):
       params['directory'] = directory_name
     res = self._root._invoke("GET", file_system, params, headers=self._getheaders(), **kwargs)
     resp = self._root._format_response(res)
+    if account != '':
+      file_system = file_system + account
     for x in resp['paths']:
       dir_stats.append(ABFSStat.for_directory(res.headers, x, root + file_system + "/" + x['name']))
     return dir_stats
@@ -233,6 +240,12 @@ class ABFS(object):
     listofFileSystems = self.listfilesystems_stats(root = root, params = params)
     return [x.name for x in listofFileSystems]
   
+  @staticmethod
+  def get_home_dir():
+    """
+    Attempts to go to the directory set by the user in the configuration file. If not defaults to abfs:// 
+    """
+    return Init_ABFS.get_home_dir_for_ABFS()
   # Find or alter information about the URI path
   # --------------------------------
   @staticmethod
@@ -256,9 +269,6 @@ class ABFS(object):
     Normalizes a path
     """
     return Init_ABFS.normpath(path)
-
-  def open(self, path, option='r', *args, **kwargs):
-    return ABFSFile(self,path, option )
   
   @staticmethod
   def parent_path(path):
@@ -330,6 +340,12 @@ class ABFS(object):
       headers['range']= 'bytes=%s-%s' % (str(offset), str(int(offset) + int(length)))
     return self._root.get(path, headers = headers)
   
+  def open(self, path, option='r', *args, **kwargs):
+    """
+    Returns an ABFSFile object that pretends that a file is open
+    """
+    return ABFSFile(self,path, option )
+  
   # Alter Files
   # --------------------------------
   def append(self, path, data, offset=0):
@@ -428,11 +444,14 @@ class ABFS(object):
   
   def chmod(self, path, permissionNumber = None, *args, **kwargs):
     """
-    Set File Permissions (not implemented)
+    Set File Permissions (passing as an int converts said integer to octal. Passing as a string assumes the string is in octal)
     """
     header = {}
     if permissionNumber is not None:
-      header['x-ms-permissions'] = str(permissionNumber)
+      if isinstance(permissionNumber, basestring):
+        header['x-ms-permissions'] = str(permissionNumber)
+      else:
+        header['x-ms-permissions'] = oct(permissionNumber)
     self.setAccessControl(path, headers=header)
   
   def setAccessControl(self, path, headers, **kwargs):
@@ -490,7 +509,7 @@ class ABFS(object):
     Renames a file
     """ 
     LOG.debug("%s\n%s" % (old, new))
-    headers = {'x-ms-rename-source' : '/' + Init_ABFS.strip_scheme(old) }
+    headers = {'x-ms-rename-source' : '/' + quote(Init_ABFS.strip_scheme(old)) }
     try:
       self._create_path(new, headers=headers, overwrite=True)
     except WebHdfsException as e:

+ 16 - 0
desktop/libs/azure/src/azure/abfs/abfs_test.py

@@ -30,6 +30,7 @@ from nose.tools import assert_true, assert_false, assert_equal
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_to_group, add_permission, remove_from_group
 
+from azure.abfs.__init__ import abfspath
 from azure.abfs.abfs import ABFS
 from azure.active_directory import ActiveDirectory
 from azure.conf import ABFS_CLUSTERS,AZURE_ACCOUNTS, is_abfs_enabled
@@ -78,6 +79,10 @@ class ABFSTestBase(unittest.TestCase):
     LOG.debug("%s" % directory)
     assert_true(directory is not None, directory)
     
+    directory = self.client.listdir(abfspath(self.test_fs))
+    LOG.debug("%s" % directory)
+    assert_true(directory is not None, directory)
+    
     pathing = self.client._statsf(filesystems[276])
     LOG.debug("%s" % pathing)
     assert_true(pathing is not None, pathing)
@@ -171,8 +176,10 @@ class ABFSTestBase(unittest.TestCase):
     test_fs = self.test_fs
     test_dir = test_fs + '/test'
     test_dir2 = test_fs + '/test2'
+    test_dir3 = test_fs + '/test 3'
     test_file = test_fs + '/test.txt'
     test_file2 = test_fs + '/test2.txt'
+    test_file3 = test_fs + '/test 3.txt'
     
     self.client.mkdir(test_dir)
     assert_true(self.client.exists(test_dir))
@@ -190,6 +197,15 @@ class ABFSTestBase(unittest.TestCase):
     assert_false(self.client.exists(test_file))
     assert_true(self.client.exists(test_file2))
     
+    self.client.rename(test_dir2, test_dir3)
+    assert_false(self.client.exists(test_dir2))
+    assert_true(self.client.exists(test_dir3))
+    
+    self.client.rename(test_dir3, test_dir2)
+    assert_false(self.client.exists(test_dir3))
+    assert_true(self.client.exists(test_dir2))
+    
+    
   def test_chmod(self):
     test_dir = self.test_fs + '/test_chmod'
     self.client.mkdir(test_dir)

+ 28 - 9
desktop/libs/azure/src/azure/abfs/abfsstats.py

@@ -16,15 +16,17 @@
 from __future__ import absolute_import
 
 import stat
+import logging
 
 from azure.abfs.__init__ import strip_path, abfsdatetime_to_timestamp
 from django.utils.encoding import smart_str
 
+LOG = logging.getLogger(__name__)
+CHAR_TO_OCT = {"---": 0, "--x" : 1, "-w-": 2, "-wx": 3, "r--" : 4, "r-x" : 5, "rw-" : 6,"rwx": 7}
+
 class ABFSStat(object):
-  DIR_MODE = 0o777 | stat.S_IFDIR
-  FILE_MODE = 0o666 | stat.S_IFREG
 
-  def __init__(self, isDir, atime, mtime, size, path, owner = '', group = ''):
+  def __init__(self, isDir, atime, mtime, size, path, owner = '', group = '', mode = None):
     self.name = strip_path(path)
     self.path = path
     self.isDir = isDir
@@ -38,6 +40,11 @@ class ABFSStat(object):
     self.size = size
     self.user = owner
     self.group = group
+    self.mode = mode or (0o777 if isDir else 0o666)
+    if self.isDir:
+      self.mode |= stat.S_IFDIR
+    else:
+      self.mode |= stat.S_IFREG
     
   def __getitem__(self, key):
     try:
@@ -51,10 +58,6 @@ class ABFSStat(object):
     
   def __repr__(self):
     return smart_str("<abfsStat %s>" % (self.path,))
-    
-  @property
-  def mode(self):
-    return ABFSStat.DIR_MODE if self.isDir else ABFSStat.FILE_MODE
   
   @property
   def aclBit(self):
@@ -78,17 +81,33 @@ class ABFSStat(object):
       isDir = resp['isDirectory'] == 'true'
     except:
       isDir = False
-    return cls(isDir, headers['date'], resp['lastModified'], size, path, resp['owner'], resp['group'])
+    try:
+      permissions = ABFSStat.char_permissions_to_oct_permissions(resp['permissions'])
+    except:
+      permissions = None
+    return cls(isDir, headers['date'], resp['lastModified'], size, path, resp['owner'], resp['group'], mode = permissions)
   
   @classmethod
   def for_single(cls,resp, path):
     size = int(resp['Content-Length'])
     isDir = resp['x-ms-resource-type'] == 'directory'
-    return cls(isDir, resp['date'],resp['Last-Modified'], size, path, resp['x-ms-owner'], resp['x-ms-group'])
+    try:
+      permissions = ABFSStat.char_permissions_to_oct_permissions(resp['x-ms-permissions'])
+    except:
+      permissions = None
+    return cls(isDir, resp['date'],resp['Last-Modified'], size, path, resp['x-ms-owner'], resp['x-ms-group'], mode = permissions)
   
   @classmethod
   def for_filesystem(cls, resp, path):
     return cls(True, resp['date'], resp['Last-Modified'], 0, path)
+  
+  @staticmethod
+  def char_permissions_to_oct_permissions(permissions):
+    try:
+      octal_permissions = CHAR_TO_OCT[permissions[0:3]] * 64 + CHAR_TO_OCT[permissions[3:6]] * 8 + CHAR_TO_OCT[permissions[6:]]
+    except:
+      return None
+    return octal_permissions
     
   def to_json_dict(self):
     """