Browse Source

HUE-9149 [fb] Fist unit test with mock of an empty directory

Romain 5 years ago
parent
commit
2e0c120de5

+ 17 - 20
apps/filebrowser/src/filebrowser/views.py

@@ -487,13 +487,9 @@ def listdir_paged(request, path):
     descending_param = request.GET.get('descending', None)
     descending_param = request.GET.get('descending', None)
     if sortby is not None:
     if sortby is not None:
         if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'):
         if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'):
-            logger.info("Invalid sort attribute '%s' for listdir." %
-                        (sortby,))
+            logger.info("Invalid sort attribute '%s' for listdir." % sortby)
         else:
         else:
-            all_stats = sorted(all_stats,
-                               key=operator.attrgetter(sortby),
-                               reverse=coerce_bool(descending_param))
-
+            all_stats = sorted(all_stats, key=operator.attrgetter(sortby), reverse=coerce_bool(descending_param))
 
 
     # Do pagination
     # Do pagination
     try:
     try:
@@ -520,12 +516,12 @@ def listdir_paged(request, path):
     current_stat = request.fs.stats(path)
     current_stat = request.fs.stats(path)
     # The 'path' field would be absolute, but we want its basename to be
     # The 'path' field would be absolute, but we want its basename to be
     # actually '.' for display purposes. Encode it since _massage_stats expects byte strings.
     # actually '.' for display purposes. Encode it since _massage_stats expects byte strings.
-    current_stat['path'] = path
-    current_stat['name'] = "."
+    current_stat.path = path
+    current_stat.name = "."
     shown_stats.insert(1, current_stat)
     shown_stats.insert(1, current_stat)
 
 
     if page:
     if page:
-      page.object_list = [ _massage_stats(request, stat_absolute_path(path, s)) for s in shown_stats ]
+      page.object_list = [_massage_stats(request, stat_absolute_path(path, s)) for s in shown_stats]
 
 
     is_trash_enabled = request.fs._get_scheme(path) == 'hdfs' and int(get_trash_interval()) > 0
     is_trash_enabled = request.fs._get_scheme(path) == 'hdfs' and int(get_trash_interval()) > 0
 
 
@@ -566,7 +562,7 @@ def scheme_absolute_path(root, path):
   return path
   return path
 
 
 def stat_absolute_path(path, stat):
 def stat_absolute_path(path, stat):
-  stat["path"] = scheme_absolute_path(path, stat["path"])
+  stat.path = scheme_absolute_path(path, stat.path)
   return stat
   return stat
 
 
 def _massage_stats(request, stats):
 def _massage_stats(request, stats):
@@ -574,17 +570,17 @@ def _massage_stats(request, stats):
     Massage a stats record as returned by the filesystem implementation
     Massage a stats record as returned by the filesystem implementation
     into the format that the views would like it in.
     into the format that the views would like it in.
     """
     """
-    path = stats['path']
+    path = stats.path
     normalized = request.fs.normpath(path)
     normalized = request.fs.normpath(path)
     return {
     return {
         'path': normalized, # Normally this should be quoted, but we only use this in POST request so we're ok. Changing this to quoted causes many issues.
         'path': normalized, # Normally this should be quoted, but we only use this in POST request so we're ok. Changing this to quoted causes many issues.
-        'name': stats['name'],
+        'name': stats.name,
         'stats': stats.to_json_dict(),
         'stats': stats.to_json_dict(),
-        'mtime': datetime.fromtimestamp(stats['mtime']).strftime('%B %d, %Y %I:%M %p') if stats['mtime'] else '',
-        'humansize': filesizeformat(stats['size']),
-        'type': filetype(stats['mode']),
-        'rwx': rwx(stats['mode'], stats['aclBit']),
-        'mode': stringformat(stats['mode'], "o"),
+        'mtime': datetime.fromtimestamp(stats.mtime).strftime('%B %d, %Y %I:%M %p') if stats.mtime else '',
+        'humansize': filesizeformat(stats.size),
+        'type': filetype(stats.mode),
+        'rwx': rwx(stats.mode, stats.aclBit),
+        'mode': stringformat(stats.mode, "o"),
         'url': '/filebrowser/view=' + urllib_quote(normalized.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS),
         'url': '/filebrowser/view=' + urllib_quote(normalized.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS),
         'is_sentry_managed': request.fs.is_sentry_managed(path)
         'is_sentry_managed': request.fs.is_sentry_managed(path)
     }
     }
@@ -721,11 +717,11 @@ def display(request, path):
         'dirname': dirname,
         'dirname': dirname,
         'mode': mode,
         'mode': mode,
         'compression': compression,
         'compression': compression,
-        'size': stats['size'],
+        'size': stats.size,
         'max_chunk_size': str(MAX_CHUNK_SIZE_BYTES)
         'max_chunk_size': str(MAX_CHUNK_SIZE_BYTES)
     }
     }
     data["filename"] = os.path.basename(path)
     data["filename"] = os.path.basename(path)
-    data["editable"] = stats['size'] < MAX_FILEEDITOR_SIZE
+    data["editable"] = stats.size < MAX_FILEEDITOR_SIZE
     if mode == "binary":
     if mode == "binary":
         # This might be the wrong thing for ?format=json; doing the
         # This might be the wrong thing for ?format=json; doing the
         # xxd'ing in javascript might be more compact, or sending a less
         # xxd'ing in javascript might be more compact, or sending a less
@@ -937,8 +933,9 @@ def detect_snappy(contents):
 def detect_parquet(fhandle):
 def detect_parquet(fhandle):
     """
     """
     Detect parquet from magic header bytes.
     Detect parquet from magic header bytes.
+    Python 2 only currently.
     """
     """
-    return parquet._check_header_magic_bytes(fhandle)
+    return False if sys.version_info[0] > 2 else parquet._check_header_magic_bytes(fhandle)
 
 
 
 
 def snappy_installed():
 def snappy_installed():

+ 45 - 6
apps/filebrowser/src/filebrowser/views_test.py

@@ -25,8 +25,8 @@ from builtins import object
 import json
 import json
 import logging
 import logging
 import os
 import os
-
 import re
 import re
+import stat
 import sys
 import sys
 import tempfile
 import tempfile
 
 
@@ -34,9 +34,7 @@ import urllib.request, urllib.error
 import urllib.parse
 import urllib.parse
 
 
 from time import sleep, time
 from time import sleep, time
-
 from avro import schema, datafile, io
 from avro import schema, datafile, io
-
 from aws.s3.s3fs import S3FileSystemException
 from aws.s3.s3fs import S3FileSystemException
 from aws.s3.s3test_utils import get_test_bucket
 from aws.s3.s3test_utils import get_test_bucket
 
 
@@ -68,6 +66,11 @@ else:
   from urllib import unquote as urllib_unquote
   from urllib import unquote as urllib_unquote
   open_file = file
   open_file = file
 
 
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock
+else:
+  from mock import patch, Mock
+
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -87,6 +90,44 @@ def cleanup_file(cluster, path):
     LOG.exception('failed to cleanup %s' % path)
     LOG.exception('failed to cleanup %s' % path)
 
 
 
 
+class TestFileBrowser():
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test_filebrowser", groupname='test_filebrowser', recreate=True, is_superuser=False)
+    self.user = User.objects.get(username="test_filebrowser")
+    grant_access(self.user.username, 'test_filebrowser', 'filebrowser')
+    add_to_group(self.user.username, 'test_filebrowser')
+
+  def test_listdir_paged(self):
+
+    with patch('desktop.middleware.fsmanager.get_filesystem') as get_filesystem:
+      with patch('filebrowser.views.snappy_installed') as snappy_installed:
+        snappy_installed.return_value = False
+        get_filesystem.return_value = Mock(
+          stats=Mock(
+            return_value=Mock(
+              isDir=True,
+              size=1024,
+              path=b'/',
+              mtime=None,
+              mode=stat.S_IFDIR
+            ),
+          ),
+          normpath=Mock(return_value='/'),
+          listdir_stats=Mock(
+            return_value=[]  # Add "Mock files here"
+          ),
+          superuser='hdfs',
+          supergroup='hdfs'
+        )
+
+        response = self.client.get('/filebrowser/view=')
+
+        assert_equal(200, response.status_code)
+        dir_listing = response.context[0]['files']
+        assert_equal(1, len(dir_listing))
+
+
 class TestFileBrowserWithHadoop(object):
 class TestFileBrowserWithHadoop(object):
   requires_hadoop = True
   requires_hadoop = True
   integration = True
   integration = True
@@ -655,9 +696,7 @@ class TestFileBrowserWithHadoop(object):
     """)
     """)
 
 
     f = self.cluster.fs.open(prefix + '/test-view.avro', "w")
     f = self.cluster.fs.open(prefix + '/test-view.avro', "w")
-    data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
-                                                writers_schema=test_schema,
-                                                codec='deflate')
+    data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(), writers_schema=test_schema, codec='deflate')
     dummy_datum = {
     dummy_datum = {
       'name': 'Test',
       'name': 'Test',
       'integer': 10,
       'integer': 10,

+ 3 - 6
desktop/libs/hadoop/src/hadoop/fs/webhdfs_types.py

@@ -57,9 +57,7 @@ class WebHdfsStat(object):
       self.mode |= stat.S_IFREG
       self.mode |= stat.S_IFREG
 
 
   def __unicode__(self):
   def __unicode__(self):
-    return "[WebHdfsStat] %7s %8s %8s %12s %s%s" % \
-        (oct(self.mode), self.user, self.group, self.size, self.path,
-         self.isDir and '/' or "")
+    return "[WebHdfsStat] %7s %8s %8s %12s %s%s" % (oct(self.mode), self.user, self.group, self.size, self.path, self.isDir and '/' or "")
 
 
   def __repr__(self):
   def __repr__(self):
     return smart_str("<WebHdfsStat %s>" % (self.path,))
     return smart_str("<WebHdfsStat %s>" % (self.path,))
@@ -75,9 +73,8 @@ class WebHdfsStat(object):
 
 
   def to_json_dict(self):
   def to_json_dict(self):
     """Returns a dictionary for easy serialization"""
     """Returns a dictionary for easy serialization"""
-    KEYS = ('path', 'size', 'atime', 'mtime', 'mode', 'user', 'group',
-            'blockSize', 'replication')
-    res = { }
+    KEYS = ('path', 'size', 'atime', 'mtime', 'mode', 'user', 'group', 'blockSize', 'replication')
+    res = {}
     for k in KEYS:
     for k in KEYS:
       res[k] = getattr(self, k)
       res[k] = getattr(self, k)
     return res
     return res