Browse Source

[ozone] Add serviceID in the ofs path

- Path now changes from ofs://vol1/buck1/... to ofs://serviceID/vol1/buck1/...

- We are actually mocking liststatus for ofs:// now to only show serviceID as an element. This enables in correct breadcrumbs and ofs valid path structure.
Harshg999 2 years ago
parent
commit
42abc4423f

+ 21 - 16
desktop/core/src/desktop/lib/fs/ozone/__init__.py

@@ -46,15 +46,15 @@ def normpath(path):
   return normalized
 
 
-def abspath(path, key):
+def abspath(path, key, ofs_serviceid):
   """
   Returns absolute URI, examples:
 
-  abspath('ofs://volume/bucket/key', key2') == 'ofs://volume/bucket/key/key2'
-  abspath('ofs://volume/bucket/key', 'ofs://volume/bucket2/key2') == 'ofs://volume/bucket2/key2'
+  abspath('ofs://ozone1/volume/bucket/key', key2') == 'ofs://ozone1/volume/bucket/key/key2'
+  abspath('ofs://ozone1/volume/bucket/key', 'ofs://ozone1/volume/bucket2/key2') == 'ofs://ozone1/volume/bucket2/key2'
   """
   if path.lower().startswith(OFS_ROOT):
-    key = join(path, key)
+    key = _serviceid_join(join(path, key), ofs_serviceid)
   else:
     key = normpath(join(path, key))
   return key
@@ -67,10 +67,15 @@ def join(*comp_list):
     except ValueError:
       return '/' if is_root(uri) else uri
   joined = posixpath.join(*list(map(_prep, comp_list)))
-  if joined and joined[0] == '/':
-    joined = 'ofs:/%s' % joined
   return joined
 
+def _serviceid_join(path, ofs_serviceid):
+  if path and (path == '/' or path.startswith('/' + ofs_serviceid)):
+    path = 'ofs:/' + path
+  elif path and not path.startswith(OFS_ROOT + ofs_serviceid + '/'):
+    path = OFS_ROOT + ofs_serviceid + '/' + path.lstrip('/')
+  
+  return path
 
 def _append_separator(path):
   if path and not path.endswith('/'):
@@ -80,13 +85,13 @@ def _append_separator(path):
 
 def parse_uri(uri):
   """
-  Returns tuple (volume_name, key_name, key_basename).
+  Returns tuple (service_id, key_name, key_basename).
   Raises ValueError if invalid OFS URI is passed.
   
-  ofs://volume1/bucket1/key1/key2 -> 
-  group1 -> volume1
-  group2 -> /bucket1/key1/key2
-  group3 -> bucket1/key1/key2
+  ofs://ozone1/volume1/bucket1/key1/key2 -> 
+  group1 -> ozone1
+  group2 -> /volume1/bucket1/key1/key2
+  group3 -> volume1/bucket1/key1/key2
   group4 -> key2
   """
   match = OFS_PATH_RE.match(uri)
@@ -97,14 +102,14 @@ def parse_uri(uri):
   return match.group(1), key_name, key_basename
 
 
-def parent_path(path):
+def parent_path(path, ofs_serviceid):
   parent_dir = _append_separator(path)
   if not is_root(parent_dir):
-    volume_name, key_name, key_basename = parse_uri(path)
-    if not key_basename:  # volume is top-level so return root
+    service_id, key_name, key_basename = parse_uri(path)
+    if not key_basename:  # service_id is top-level so return root
       parent_dir = OFS_ROOT
     else:
-      volume_path = '%s%s' % (OFS_ROOT, volume_name)
+      service_id_path = '%s%s' % (OFS_ROOT, service_id)
       key_path = '/'.join(key_name.split('/')[:-1])
-      parent_dir = abspath(volume_path, key_path)
+      parent_dir = abspath(service_id_path, key_path, ofs_serviceid)
   return parent_dir

+ 83 - 25
desktop/core/src/desktop/lib/fs/ozone/ofs.py

@@ -23,8 +23,10 @@ import logging
 import sys
 import threading
 
+from django.utils.encoding import smart_str
+
 from desktop.lib.rest import http_client, resource
-from desktop.lib.fs.ozone import OFS_ROOT, normpath, is_root, parent_path
+from desktop.lib.fs.ozone import OFS_ROOT, normpath, is_root, parent_path, _serviceid_join, join as ofs_join
 from desktop.lib.fs.ozone.ofsstat import OzoneFSStat
 from desktop.conf import PERMISSION_ACTION_OFS
 
@@ -85,10 +87,11 @@ class OzoneFS(WebHdfs):
     )
 
   def strip_normpath(self, path):
-    if path.startswith('ofs://'):
-      path = path[5:]
-    elif path.startswith('ofs:/'):
-      path = path[4:]
+    if path.startswith(OFS_ROOT + self._netloc):
+      path = path.split(OFS_ROOT + self._netloc)[1]
+    elif path.startswith('ofs:/' + self._netloc):
+      path = path.split('ofs:/' + self._netloc)[1]
+
     return path
 
   def normpath(self, path):
@@ -101,7 +104,7 @@ class OzoneFS(WebHdfs):
     return is_root(path)
 
   def parent_path(self, path):
-    return parent_path(path)
+    return parent_path(path, self._netloc)
 
   def listdir_stats(self, path, glob=None):
     """
@@ -109,31 +112,54 @@ class OzoneFS(WebHdfs):
 
     Get directory listing with stats.
     """
-    path = self.strip_normpath(path)
-    params = self._getparams()
-    if glob is not None:
-      params['filter'] = glob
-    params['op'] = 'LISTSTATUS'
-    headers = self._getheaders()
-    json = self._root.get(path, params, headers)
+    if path == OFS_ROOT:
+      json = self._handle_serviceid_path_status()
+    else:
+      path = self.strip_normpath(path)
+      params = self._getparams()
+
+      if glob is not None:
+        params['filter'] = glob
+      params['op'] = 'LISTSTATUS'
+      headers = self._getheaders()
+
+      json = self._root.get(path, params, headers)
+
     filestatus_list = json['FileStatuses']['FileStatus']
-    return [OzoneFSStat(st, path) for st in filestatus_list]
+    return [OzoneFSStat(st, path, self._netloc) for st in filestatus_list]
 
   def _stats(self, path):
     """
     This stats method returns None if the entry is not found.
     """
-    path = self.strip_normpath(path)
-    params = self._getparams()
-    params['op'] = 'GETFILESTATUS'
-    headers = self._getheaders()
-    try:
-      json = self._root.get(path, params, headers)
-      return OzoneFSStat(json['FileStatus'], path)
-    except WebHdfsException as ex:
-      if ex.server_exc == 'FileNotFoundException' or ex.code == 404:
-        return None
-      raise ex
+    if path == OFS_ROOT:
+      serviceid_path_status = self._handle_serviceid_path_status()['FileStatuses']['FileStatus'][0]
+      json = {'FileStatus': serviceid_path_status}
+    else:
+      path = self.strip_normpath(path)
+      params = self._getparams()
+      params['op'] = 'GETFILESTATUS'
+      headers = self._getheaders()
+
+      try:
+        json = self._root.get(path, params, headers)
+      except WebHdfsException as ex:
+        if ex.server_exc == 'FileNotFoundException' or ex.code == 404:
+          return None
+        raise ex
+    
+    return OzoneFSStat(json['FileStatus'], path, self._netloc)
+  
+  def _handle_serviceid_path_status(self):
+    json = {
+      'FileStatuses': {
+        'FileStatus': [{
+          'pathSuffix': self._netloc, 'type': 'DIRECTORY', 'length': 0, 'owner': '', 'group': '', 
+          'permission': '777', 'accessTime': 0, 'modificationTime': 0, 'blockSize': 0, 'replication': 0
+          }]
+        }
+      }
+    return json
   
   def stats(self, path):
     """
@@ -152,3 +178,35 @@ class OzoneFS(WebHdfs):
     Upload is done by the OFSFileUploadHandler
     """
     pass
+
+  def rename(self, old, new):
+    """rename(old, new)"""
+    old = self.strip_normpath(old)
+    if not self.is_absolute(new):
+      new = _serviceid_join(ofs_join(self.dirname(old), new), self._netloc)
+    new = self.strip_normpath(new)
+
+    params = self._getparams()
+    params['op'] = 'RENAME'
+    # Encode `new' because it's in the params
+    params['destination'] = smart_str(new)
+    headers = self._getheaders()
+
+    result = self._root.put(old, params, headers=headers)
+
+    if not result['boolean']:
+      raise IOError(_("Rename failed: %s -> %s") % (smart_str(old, errors='replace'), smart_str(new, errors='replace')))
+  
+  def rename_star(self, old_dir, new_dir):
+    """Equivalent to `mv old_dir/* new"""
+    if not self.isdir(old_dir):
+      raise IOError(errno.ENOTDIR, _("'%s' is not a directory") % old_dir)
+
+    if not self.exists(new_dir):
+      self.mkdir(new_dir)
+    elif not self.isdir(new_dir):
+      raise IOError(errno.ENOTDIR, _("'%s' is not a directory") % new_dir)
+  
+    ls = self.listdir(old_dir)
+    for dirent in ls:
+      self.rename(_serviceid_join(ofs_join(old_dir, dirent), self._netloc), _serviceid_join(ofs_join(new_dir, dirent), self._netloc))

+ 3 - 3
desktop/core/src/desktop/lib/fs/ozone/ofsstat.py

@@ -24,7 +24,7 @@ from django.utils.encoding import smart_str
 from hadoop.fs.hadoopfs import decode_fs_path
 from hadoop.fs.webhdfs_types import WebHdfsStat
 
-from desktop.lib.fs.ozone import join as ofs_join
+from desktop.lib.fs.ozone import _serviceid_join, join as ofs_join
 
 class OzoneFSStat(WebHdfsStat):
   """
@@ -33,9 +33,9 @@ class OzoneFSStat(WebHdfsStat):
   Modelled after org.apache.hadoop.fs.FileStatus
   """
 
-  def __init__(self, file_status, parent_path):
+  def __init__(self, file_status, parent_path, ofs_serviceid=''):
     super(OzoneFSStat, self).__init__(file_status, parent_path)
-    self.path = ofs_join(parent_path, self.name)
+    self.path = _serviceid_join(ofs_join(parent_path, self.name), ofs_serviceid)
 
   def __unicode__(self):
     return "[OzoneFSStat] %7s %8s %8s %12s %s%s" % (oct(self.mode), self.user, self.group, self.size, self.path, self.isDir and '/' or "")

+ 1 - 1
desktop/core/src/desktop/lib/fs/proxyfs.py

@@ -223,7 +223,7 @@ class ProxyFS(object):
 
     # All users will have access to Ozone root.
     if is_ofs_enabled():
-      LOG.debug('Creation of user home path is not supported in Ozone. Redirect to %s' % OFS_ROOT)
+      LOG.debug('Creation of user home path is not supported in Ozone.')
 
     # Get the new home_path for S3/ABFS when RAZ is enabled.
     if is_raz_s3():