views_test.py 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067
  1. # -*- coding: utf-8 -*-
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #!/usr/bin/env python
  18. """
  19. Tests for filebrowser views
  20. """
  21. try:
  22. import json
  23. except ImportError:
  24. import simplejson as json
  25. import logging
  26. import os
  27. import re
  28. import urlparse
  29. from avro import schema, datafile, io
  30. from django.utils.encoding import smart_str
  31. from nose.plugins.attrib import attr
  32. from nose.plugins.skip import SkipTest
  33. from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal
  34. from desktop.lib.django_test_util import make_logged_in_client
  35. from desktop.lib.test_utils import grant_access
  36. from hadoop import pseudo_hdfs4
  37. from conf import MAX_SNAPPY_DECOMPRESSION_SIZE
  38. from lib.rwx import expand_mode
  39. from views import snappy_installed
  40. LOG = logging.getLogger(__name__)
  41. @attr('requires_hadoop')
  42. def test_remove():
  43. cluster = pseudo_hdfs4.shared_cluster()
  44. try:
  45. c = make_logged_in_client(cluster.superuser)
  46. cluster.fs.setuser(cluster.superuser)
  47. prefix = '/test-delete'
  48. PATH_1 = '/%s/1' % prefix
  49. PATH_2 = '/%s/2' % prefix
  50. PATH_3 = '/%s/3' % prefix
  51. cluster.fs.mkdir(prefix)
  52. cluster.fs.mkdir(PATH_1)
  53. cluster.fs.mkdir(PATH_2)
  54. cluster.fs.mkdir(PATH_3)
  55. assert_true(cluster.fs.exists(PATH_1))
  56. assert_true(cluster.fs.exists(PATH_2))
  57. assert_true(cluster.fs.exists(PATH_3))
  58. c.post('/filebrowser/rmtree', dict(path=[PATH_1]))
  59. assert_false(cluster.fs.exists(PATH_1))
  60. assert_true(cluster.fs.exists(PATH_2))
  61. assert_true(cluster.fs.exists(PATH_3))
  62. c.post('/filebrowser/rmtree', dict(path=[PATH_2, PATH_3]))
  63. assert_false(cluster.fs.exists(PATH_1))
  64. assert_false(cluster.fs.exists(PATH_2))
  65. assert_false(cluster.fs.exists(PATH_3))
  66. finally:
  67. try:
  68. cluster.fs.rmtree(prefix) # Clean up
  69. except:
  70. pass # Don't let cleanup errors mask earlier failures
  71. @attr('requires_hadoop')
  72. def test_move():
  73. cluster = pseudo_hdfs4.shared_cluster()
  74. try:
  75. c = make_logged_in_client(cluster.superuser)
  76. cluster.fs.setuser(cluster.superuser)
  77. prefix = '/test-move'
  78. PATH_1 = '%s/1' % prefix
  79. PATH_2 = '%s/2' % prefix
  80. SUB_PATH1_1 = '%s/1' % PATH_1
  81. SUB_PATH1_2 = '%s/2' % PATH_1
  82. SUB_PATH1_3 = '%s/3' % PATH_1
  83. SUB_PATH2_1 = '%s/1' % PATH_2
  84. SUB_PATH2_2 = '%s/2' % PATH_2
  85. SUB_PATH2_3 = '%s/3' % PATH_2
  86. cluster.fs.mkdir(prefix)
  87. cluster.fs.mkdir(PATH_1)
  88. cluster.fs.mkdir(PATH_2)
  89. cluster.fs.mkdir(SUB_PATH1_1)
  90. cluster.fs.mkdir(SUB_PATH1_2)
  91. cluster.fs.mkdir(SUB_PATH1_3)
  92. assert_true(cluster.fs.exists(SUB_PATH1_1))
  93. assert_true(cluster.fs.exists(SUB_PATH1_2))
  94. assert_true(cluster.fs.exists(SUB_PATH1_3))
  95. assert_false(cluster.fs.exists(SUB_PATH2_1))
  96. assert_false(cluster.fs.exists(SUB_PATH2_2))
  97. assert_false(cluster.fs.exists(SUB_PATH2_3))
  98. c.post('/filebrowser/move', dict(src_path=[SUB_PATH1_1], dest_path=PATH_2))
  99. assert_false(cluster.fs.exists(SUB_PATH1_1))
  100. assert_true(cluster.fs.exists(SUB_PATH1_2))
  101. assert_true(cluster.fs.exists(SUB_PATH1_3))
  102. assert_true(cluster.fs.exists(SUB_PATH2_1))
  103. assert_false(cluster.fs.exists(SUB_PATH2_2))
  104. assert_false(cluster.fs.exists(SUB_PATH2_3))
  105. c.post('/filebrowser/move', dict(src_path=[SUB_PATH1_2, SUB_PATH1_3], dest_path=PATH_2))
  106. assert_false(cluster.fs.exists(SUB_PATH1_1))
  107. assert_false(cluster.fs.exists(SUB_PATH1_2))
  108. assert_false(cluster.fs.exists(SUB_PATH1_3))
  109. assert_true(cluster.fs.exists(SUB_PATH2_1))
  110. assert_true(cluster.fs.exists(SUB_PATH2_2))
  111. assert_true(cluster.fs.exists(SUB_PATH2_3))
  112. finally:
  113. try:
  114. cluster.fs.rmtree(prefix) # Clean up
  115. except:
  116. pass # Don't let cleanup errors mask earlier failures
  117. @attr('requires_hadoop')
  118. def test_copy():
  119. cluster = pseudo_hdfs4.shared_cluster()
  120. try:
  121. c = make_logged_in_client(cluster.superuser)
  122. cluster.fs.setuser(cluster.superuser)
  123. prefix = '/test-copy'
  124. PATH_1 = '%s/1' % prefix
  125. PATH_2 = '%s/2' % prefix
  126. SUB_PATH1_1 = '%s/1' % PATH_1
  127. SUB_PATH1_2 = '%s/2' % PATH_1
  128. SUB_PATH1_3 = '%s/3' % PATH_1
  129. SUB_PATH2_1 = '%s/1' % PATH_2
  130. SUB_PATH2_2 = '%s/2' % PATH_2
  131. SUB_PATH2_3 = '%s/3' % PATH_2
  132. cluster.fs.mkdir(prefix)
  133. cluster.fs.mkdir(PATH_1)
  134. cluster.fs.mkdir(PATH_2)
  135. cluster.fs.mkdir(SUB_PATH1_1)
  136. cluster.fs.mkdir(SUB_PATH1_2)
  137. cluster.fs.mkdir(SUB_PATH1_3)
  138. assert_true(cluster.fs.exists(SUB_PATH1_1))
  139. assert_true(cluster.fs.exists(SUB_PATH1_2))
  140. assert_true(cluster.fs.exists(SUB_PATH1_3))
  141. assert_false(cluster.fs.exists(SUB_PATH2_1))
  142. assert_false(cluster.fs.exists(SUB_PATH2_2))
  143. assert_false(cluster.fs.exists(SUB_PATH2_3))
  144. c.post('/filebrowser/copy', dict(src_path=[SUB_PATH1_1], dest_path=PATH_2))
  145. assert_true(cluster.fs.exists(SUB_PATH1_1))
  146. assert_true(cluster.fs.exists(SUB_PATH1_2))
  147. assert_true(cluster.fs.exists(SUB_PATH1_3))
  148. assert_true(cluster.fs.exists(SUB_PATH2_1))
  149. assert_false(cluster.fs.exists(SUB_PATH2_2))
  150. assert_false(cluster.fs.exists(SUB_PATH2_3))
  151. c.post('/filebrowser/copy', dict(src_path=[SUB_PATH1_2, SUB_PATH1_3], dest_path=PATH_2))
  152. assert_true(cluster.fs.exists(SUB_PATH1_1))
  153. assert_true(cluster.fs.exists(SUB_PATH1_2))
  154. assert_true(cluster.fs.exists(SUB_PATH1_3))
  155. assert_true(cluster.fs.exists(SUB_PATH2_1))
  156. assert_true(cluster.fs.exists(SUB_PATH2_2))
  157. assert_true(cluster.fs.exists(SUB_PATH2_3))
  158. finally:
  159. try:
  160. cluster.fs.rmtree(prefix) # Clean up
  161. except:
  162. pass # Don't let cleanup errors mask earlier failures
  163. @attr('requires_hadoop')
  164. def test_mkdir_singledir():
  165. cluster = pseudo_hdfs4.shared_cluster()
  166. cluster.fs.setuser('test')
  167. c = make_logged_in_client()
  168. try:
  169. # We test that mkdir fails when a non-relative path is provided and a multi-level path is provided.
  170. success_path = 'mkdir_singledir'
  171. path_absolute = '/mkdir_singledir'
  172. path_fail = 'fail/foo'
  173. path_other_failure = 'fail#bar'
  174. prefix = '/tmp/test-filebrowser/'
  175. # Two of the following post requests should throw exceptions.
  176. # See https://issues.cloudera.org/browse/HUE-793.
  177. c.post('/filebrowser/mkdir', dict(path=prefix, name=path_fail))
  178. c.post('/filebrowser/mkdir', dict(path=prefix, name=path_other_failure))
  179. c.post('/filebrowser/mkdir', dict(path=prefix, name=path_absolute))
  180. c.post('/filebrowser/mkdir', dict(path=prefix, name=success_path))
  181. # Read the parent dir and make sure we created 'success_path' only.
  182. response = c.get('/filebrowser/view' + prefix)
  183. dir_listing = response.context['files']
  184. assert_equal(3, len(dir_listing))
  185. assert_equal(dir_listing[2]['name'], success_path)
  186. finally:
  187. try:
  188. cluster.fs.rmtree(prefix) # Clean up
  189. except:
  190. pass # Don't let cleanup errors mask earlier failures
  191. @attr('requires_hadoop')
  192. def test_touch():
  193. cluster = pseudo_hdfs4.shared_cluster()
  194. cluster.fs.setuser('test')
  195. c = make_logged_in_client()
  196. try:
  197. success_path = 'touch_file'
  198. path_absolute = '/touch_file'
  199. path_fail = 'touch_fail/file'
  200. prefix = '/tmp/test-filebrowser-touch/'
  201. cluster.fs.mkdir(prefix)
  202. resp = c.post('/filebrowser/touch', dict(path=prefix, name=path_fail))
  203. assert_equal(500, resp.status_code)
  204. resp = c.post('/filebrowser/touch', dict(path=prefix, name=path_absolute))
  205. assert_equal(500, resp.status_code)
  206. resp = c.post('/filebrowser/touch', dict(path=prefix, name=success_path))
  207. assert_equal(200, resp.status_code)
  208. # Read the parent dir and make sure we created 'success_path' only.
  209. response = c.get('/filebrowser/view' + prefix)
  210. file_listing = response.context['files']
  211. assert_equal(3, len(file_listing))
  212. assert_equal(file_listing[2]['name'], success_path)
  213. finally:
  214. try:
  215. cluster.fs.rmtree(prefix)
  216. except:
  217. pass
  218. @attr('requires_hadoop')
  219. def test_chmod():
  220. cluster = pseudo_hdfs4.shared_cluster()
  221. try:
  222. c = make_logged_in_client(cluster.superuser)
  223. cluster.fs.setuser(cluster.superuser)
  224. PATH = "/chmod_test"
  225. SUBPATH = PATH + '/test'
  226. cluster.fs.mkdir(SUBPATH)
  227. permissions = ('user_read', 'user_write', 'user_execute',
  228. 'group_read', 'group_write', 'group_execute',
  229. 'other_read', 'other_write', 'other_execute',
  230. 'sticky') # Order matters!
  231. # Get current mode, change mode, check mode
  232. # Start with checking current mode
  233. assert_not_equal(041777, int(cluster.fs.stats(PATH)["mode"]))
  234. # Setup post data
  235. permissions_dict = dict( zip(permissions, [True]*len(permissions)) )
  236. kwargs = {'path': [PATH]}
  237. kwargs.update(permissions_dict)
  238. # Set 1777, then check permissions of dirs
  239. response = c.post("/filebrowser/chmod", kwargs)
  240. assert_equal(041777, int(cluster.fs.stats(PATH)["mode"]))
  241. # Now do the above recursively
  242. assert_not_equal(041777, int(cluster.fs.stats(SUBPATH)["mode"]))
  243. kwargs['recursive'] = True
  244. response = c.post("/filebrowser/chmod", kwargs)
  245. assert_equal(041777, int(cluster.fs.stats(SUBPATH)["mode"]))
  246. # Test bulk chmod
  247. PATH_2 = u"/test-chmod2"
  248. PATH_3 = u"/test-chown3"
  249. cluster.fs.mkdir(PATH_2)
  250. cluster.fs.mkdir(PATH_3)
  251. kwargs['path'] = [PATH_2, PATH_3]
  252. assert_not_equal(041777, int(cluster.fs.stats(PATH_2)["mode"]))
  253. assert_not_equal(041777, int(cluster.fs.stats(PATH_3)["mode"]))
  254. c.post("/filebrowser/chmod", kwargs)
  255. assert_equal(041777, int(cluster.fs.stats(PATH_2)["mode"]))
  256. assert_equal(041777, int(cluster.fs.stats(PATH_3)["mode"]))
  257. finally:
  258. try:
  259. cluster.fs.rmtree(PATH) # Clean up
  260. cluster.fs.rmtree(PATH_2) # Clean up
  261. cluster.fs.rmtree(PATH_3) # Clean up
  262. except:
  263. pass # Don't let cleanup errors mask earlier failures
  264. @attr('requires_hadoop')
  265. def test_chmod_sticky():
  266. cluster = pseudo_hdfs4.shared_cluster()
  267. try:
  268. c = make_logged_in_client(cluster.superuser)
  269. cluster.fs.setuser(cluster.superuser)
  270. PATH = "/chmod_test"
  271. cluster.fs.mkdir(PATH)
  272. # Get current mode and make sure sticky bit is off
  273. mode = expand_mode( int(cluster.fs.stats(PATH)["mode"]) )
  274. assert_equal(False, mode[-1])
  275. # Setup post data
  276. permissions = ('user_read', 'user_write', 'user_execute',
  277. 'group_read', 'group_write', 'group_execute',
  278. 'other_read', 'other_write', 'other_execute',
  279. 'sticky') # Order matters!
  280. permissions_dict = dict(filter(lambda x: x[1], zip(permissions, mode)))
  281. permissions_dict['sticky'] = True
  282. kwargs = {'path': [PATH]}
  283. kwargs.update(permissions_dict)
  284. # Set sticky bit, then check sticky bit is on in hdfs
  285. response = c.post("/filebrowser/chmod", kwargs)
  286. mode = expand_mode( int(cluster.fs.stats(PATH)["mode"]) )
  287. assert_equal(True, mode[-1])
  288. # Unset sticky bit, then check sticky bit is off in hdfs
  289. del kwargs['sticky']
  290. response = c.post("/filebrowser/chmod", kwargs)
  291. mode = expand_mode( int(cluster.fs.stats(PATH)["mode"]) )
  292. assert_equal(False, mode[-1])
  293. finally:
  294. try:
  295. cluster.fs.rmtree(PATH) # Clean up
  296. except:
  297. pass # Don't let cleanup errors mask earlier failures
  298. @attr('requires_hadoop')
  299. def test_chown():
  300. cluster = pseudo_hdfs4.shared_cluster()
  301. # Only the Hadoop superuser really has carte blanche here
  302. c = make_logged_in_client(cluster.superuser)
  303. cluster.fs.setuser(cluster.superuser)
  304. PATH = u"/test-chown-en-Español"
  305. cluster.fs.mkdir(PATH)
  306. c.post("/filebrowser/chown", dict(path=[PATH], user="x", group="y"))
  307. assert_equal("x", cluster.fs.stats(PATH)["user"])
  308. assert_equal("y", cluster.fs.stats(PATH)["group"])
  309. c.post("/filebrowser/chown", dict(path=[PATH], user="__other__", user_other="z", group="y"))
  310. assert_equal("z", cluster.fs.stats(PATH)["user"])
  311. # Now check recursive
  312. SUBPATH = PATH + '/test'
  313. cluster.fs.mkdir(SUBPATH)
  314. c.post("/filebrowser/chown", dict(path=[PATH], user="x", group="y", recursive=True))
  315. assert_equal("x", cluster.fs.stats(SUBPATH)["user"])
  316. assert_equal("y", cluster.fs.stats(SUBPATH)["group"])
  317. c.post("/filebrowser/chown", dict(path=[PATH], user="__other__", user_other="z", group="y", recursive=True))
  318. assert_equal("z", cluster.fs.stats(SUBPATH)["user"])
  319. # Test bulk chown
  320. PATH_2 = u"/test-chown-en-Español2"
  321. PATH_3 = u"/test-chown-en-Español2"
  322. cluster.fs.mkdir(PATH_2)
  323. cluster.fs.mkdir(PATH_3)
  324. c.post("/filebrowser/chown", dict(path=[PATH_2, PATH_3], user="x", group="y", recursive=True))
  325. assert_equal("x", cluster.fs.stats(PATH_2)["user"])
  326. assert_equal("y", cluster.fs.stats(PATH_2)["group"])
  327. assert_equal("x", cluster.fs.stats(PATH_3)["user"])
  328. assert_equal("y", cluster.fs.stats(PATH_3)["group"])
  329. @attr('requires_hadoop')
  330. def test_rename():
  331. cluster = pseudo_hdfs4.shared_cluster()
  332. c = make_logged_in_client(cluster.superuser)
  333. cluster.fs.setuser(cluster.superuser)
  334. PREFIX = u"/test-rename/"
  335. NAME = u"test-rename-before"
  336. NEW_NAME = u"test-rename-after"
  337. cluster.fs.mkdir(PREFIX + NAME)
  338. op = "rename"
  339. # test for full path rename
  340. c.post("/filebrowser/rename", dict(src_path=PREFIX + NAME, dest_path=PREFIX + NEW_NAME))
  341. assert_true(cluster.fs.exists(PREFIX + NEW_NAME))
  342. # test for smart rename
  343. c.post("/filebrowser/rename", dict(src_path=PREFIX + NAME, dest_path=NEW_NAME))
  344. assert_true(cluster.fs.exists(PREFIX + NEW_NAME))
  345. @attr('requires_hadoop')
  346. def test_listdir():
  347. cluster = pseudo_hdfs4.shared_cluster()
  348. try:
  349. c = make_logged_in_client('test')
  350. # Delete user's home if there's already something there
  351. home = cluster.fs.do_as_user('test', cluster.fs.get_home_dir)
  352. if cluster.fs.exists(home):
  353. cluster.fs.do_as_superuser(cluster.fs.rmtree, home)
  354. response = c.get('/filebrowser/')
  355. # Since we deleted the home directory... home_directory context should be None.
  356. assert_false(response.context['home_directory'], response.context['home_directory'])
  357. cluster.fs.do_as_superuser(cluster.fs.mkdir, home)
  358. cluster.fs.do_as_superuser(cluster.fs.chown, home, 'test', 'test')
  359. # These paths contain non-ascii characters. Your editor will need the
  360. # corresponding font library to display them correctly.
  361. #
  362. # We test that mkdir can handle unicode strings as well as byte strings.
  363. # And even when the byte string can't be decoded properly (big5), the listdir
  364. # still succeeds.
  365. orig_paths = [
  366. u'greek-Ελληνικά',
  367. u'chinese-漢語',
  368. 'listdir%20.,<>~`!@$%^&()_-+="',
  369. ]
  370. prefix = home + '/test-filebrowser/'
  371. for path in orig_paths:
  372. c.post('/filebrowser/mkdir', dict(path=prefix, name=path))
  373. # Read the parent dir
  374. response = c.get('/filebrowser/view' + prefix)
  375. dir_listing = response.context['files']
  376. assert_equal(len(orig_paths) + 2, len(dir_listing))
  377. for dirent in dir_listing:
  378. path = dirent['name']
  379. if path in ('.', '..'):
  380. continue
  381. assert_true(path in orig_paths)
  382. # Drill down into the subdirectory
  383. url = urlparse.urlsplit(dirent['url'])[2]
  384. resp = c.get(url)
  385. # We are actually reading a directory
  386. assert_equal('.', resp.context['files'][0]['name'])
  387. assert_equal('..', resp.context['files'][1]['name'])
  388. # Test's home directory now exists. Should be returned.
  389. c = make_logged_in_client()
  390. response = c.get('/filebrowser/view' + prefix)
  391. assert_equal(response.context['home_directory'], home)
  392. # Test URL conflicts with filenames
  393. stat_dir = '%sstat/dir' % prefix
  394. cluster.fs.do_as_user('test', cluster.fs.mkdir, stat_dir)
  395. response = c.get('/filebrowser/view%s' % stat_dir)
  396. assert_equal(stat_dir, response.context['path'])
  397. response = c.get('/filebrowser/view/test-filebrowser/?default_to_home')
  398. assert_true(re.search('%s$' % home, response['Location']))
  399. # Test path relative to home directory
  400. cluster.fs.do_as_user('test', cluster.fs.mkdir, '%s/test_dir' % home)
  401. response = c.get('/filebrowser/home_relative_view/test_dir')
  402. assert_equal('%s/test_dir' % home, response.context['path'])
  403. finally:
  404. try:
  405. cluster.fs.do_as_superuser(cluster.fs.rmtree, prefix)
  406. except:
  407. pass # Don't let cleanup errors mask earlier failures
  408. @attr('requires_hadoop')
  409. def test_listdir_sort_and_filter():
  410. cluster = pseudo_hdfs4.shared_cluster()
  411. c = make_logged_in_client(cluster.superuser)
  412. cluster.fs.setuser(cluster.superuser)
  413. BASE = '/test_sort_and_filter'
  414. FUNNY_NAME = u'greek-Ελληνικά'
  415. try:
  416. cluster.fs.mkdir(BASE)
  417. # Create 10 files
  418. for i in range(1, 11):
  419. cluster.fs.create(cluster.fs.join(BASE, str(i)), data="foo" * i)
  420. # Create 1 funny name directory
  421. cluster.fs.mkdir(cluster.fs.join(BASE, FUNNY_NAME))
  422. # All 12 of the entries
  423. expect = [ '.', '..', FUNNY_NAME] + [ str(i) for i in range(1, 11) ]
  424. # Check pagination
  425. listing = c.get('/filebrowser/view' + BASE + '?pagesize=20').context['files']
  426. assert_equal(len(expect), len(listing))
  427. listing = c.get('/filebrowser/view' + BASE + '?pagesize=10').context['files']
  428. assert_equal(12, len(listing))
  429. listing = c.get('/filebrowser/view' + BASE + '?pagesize=10&pagenum=1').context['files']
  430. assert_equal(12, len(listing))
  431. listing = c.get('/filebrowser/view' + BASE + '?pagesize=10&pagenum=2').context['files']
  432. assert_equal(3, len(listing))
  433. # Check sorting (name)
  434. listing = c.get('/filebrowser/view' + BASE + '?sortby=name').context['files']
  435. assert_equal(sorted(expect), [ f['name'] for f in listing ])
  436. listing = c.get('/filebrowser/view' + BASE + '?sortby=name&descending=false').context['files']
  437. assert_equal(sorted(expect), [ f['name'] for f in listing ])
  438. listing = c.get('/filebrowser/view' + BASE + '?sortby=name&descending=true').context['files']
  439. assert_equal(".", listing[0]['name'])
  440. assert_equal("..", listing[1]['name'])
  441. assert_equal(FUNNY_NAME, listing[2]['name'])
  442. # Check sorting (size)
  443. listing = c.get('/filebrowser/view' + BASE + '?sortby=size').context['files']
  444. assert_equal(expect, [ f['name'] for f in listing ])
  445. # Check sorting (mtime)
  446. listing = c.get('/filebrowser/view' + BASE + '?sortby=mtime').context['files']
  447. assert_equal(".", listing[0]['name'])
  448. assert_equal("..", listing[1]['name'])
  449. assert_equal(FUNNY_NAME, listing[-1]['name'])
  450. # Check filter
  451. listing = c.get('/filebrowser/view' + BASE + '?filter=1').context['files']
  452. assert_equal(['.', '..', '1', '10'], [ f['name'] for f in listing ])
  453. listing = c.get('/filebrowser/view' + BASE + '?filter=' + FUNNY_NAME).context['files']
  454. assert_equal(['.', '..', FUNNY_NAME], [ f['name'] for f in listing ])
  455. # Check filter + sorting
  456. listing = c.get('/filebrowser/view' + BASE + '?filter=1&sortby=name&descending=true').context['files']
  457. assert_equal(['.', '..', '10', '1'], [ f['name'] for f in listing ])
  458. # Check filter + sorting + pagination
  459. listing = c.get('/filebrowser/view' + BASE + '?filter=1&sortby=name&descending=true&pagesize=1&pagenum=2').context['files']
  460. assert_equal(['.', '..', '1'], [ f['name'] for f in listing ])
  461. finally:
  462. try:
  463. cluster.fs.rmtree(BASE)
  464. except:
  465. pass # Don't let cleanup errors mask earlier failures
  466. @attr('requires_hadoop')
  467. def test_chooser():
  468. cluster = pseudo_hdfs4.shared_cluster()
  469. c = make_logged_in_client()
  470. # Note that the trailing slash is important. We ask for the root dir.
  471. resp = c.get('/filebrowser/chooser/?format=json')
  472. # We should get a json response
  473. dic = json.loads(resp.content)
  474. assert_equal('/', dic['current_dir_path'])
  475. assert_equal('/', dic['path'])
  476. @attr('requires_hadoop')
  477. def test_view_snappy_compressed_avro():
  478. if not snappy_installed():
  479. raise SkipTest
  480. import snappy
  481. cluster = pseudo_hdfs4.shared_cluster()
  482. try:
  483. c = make_logged_in_client()
  484. cluster.fs.setuser(cluster.superuser)
  485. if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
  486. cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
  487. cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
  488. test_schema = schema.parse("""
  489. {
  490. "name": "test",
  491. "type": "record",
  492. "fields": [
  493. { "name": "name", "type": "string" },
  494. { "name": "integer", "type": "int" }
  495. ]
  496. }
  497. """)
  498. # Cannot use StringIO with datafile writer!
  499. f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', "w")
  500. data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
  501. writers_schema=test_schema,
  502. codec='deflate')
  503. dummy_datum = {
  504. 'name': 'Test',
  505. 'integer': 10,
  506. }
  507. data_file_writer.append(dummy_datum)
  508. data_file_writer.close()
  509. fh = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', 'r')
  510. f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
  511. f.write(snappy.compress(fh.read()))
  512. f.close()
  513. fh.close()
  514. # Snappy compressed fail
  515. response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
  516. assert_true('Failed to decompress' in response.context['message'], response)
  517. # Snappy compressed succeed
  518. response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
  519. assert_equal('snappy_avro', response.context['view']['compression'])
  520. assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
  521. response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=snappy_avro')
  522. assert_equal('snappy_avro', response.context['view']['compression'])
  523. assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
  524. # Avro should also decompress snappy
  525. response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=avro')
  526. assert_equal('snappy_avro', response.context['view']['compression'])
  527. assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
  528. # Largest snappy compressed file
  529. finish = MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1)
  530. response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
  531. assert_true('File size is greater than allowed max snappy decompression size' in response.context['message'], response)
  532. finally:
  533. finish()
  534. try:
  535. cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
  536. except:
  537. pass # Don't let cleanup errors mask earlier failures
  538. @attr('requires_hadoop')
  539. def test_view_avro():
  540. cluster = pseudo_hdfs4.shared_cluster()
  541. try:
  542. c = make_logged_in_client()
  543. cluster.fs.setuser(cluster.superuser)
  544. if cluster.fs.isdir("/test-avro-filebrowser"):
  545. cluster.fs.rmtree('/test-avro-filebrowser/')
  546. cluster.fs.mkdir('/test-avro-filebrowser/')
  547. test_schema = schema.parse("""
  548. {
  549. "name": "test",
  550. "type": "record",
  551. "fields": [
  552. { "name": "name", "type": "string" },
  553. { "name": "integer", "type": "int" }
  554. ]
  555. }
  556. """)
  557. f = cluster.fs.open('/test-avro-filebrowser/test-view.avro', "w")
  558. data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
  559. writers_schema=test_schema,
  560. codec='deflate')
  561. dummy_datum = {
  562. 'name': 'Test',
  563. 'integer': 10,
  564. }
  565. data_file_writer.append(dummy_datum)
  566. data_file_writer.close()
  567. # autodetect
  568. response = c.get('/filebrowser/view/test-avro-filebrowser/test-view.avro')
  569. # (Note: we use eval here cause of an incompatibility issue between
  570. # the representation string of JSON dicts in simplejson vs. json)
  571. assert_equal(eval(response.context['view']['contents']), dummy_datum)
  572. # offsetting should work as well
  573. response = c.get('/filebrowser/view/test-avro-filebrowser/test-view.avro?offset=1')
  574. assert_equal('avro', response.context['view']['compression'])
  575. f = cluster.fs.open('/test-avro-filebrowser/test-view2.avro', "w")
  576. f.write("hello")
  577. f.close()
  578. # we shouldn't autodetect non avro files
  579. response = c.get('/filebrowser/view/test-avro-filebrowser/test-view2.avro')
  580. assert_equal(response.context['view']['contents'], "hello")
  581. # we should fail to do a bad thing if they specify compression when it's not set.
  582. response = c.get('/filebrowser/view/test-avro-filebrowser/test-view2.avro?compression=gzip')
  583. assert_true('Failed to decompress' in response.context['message'])
  584. finally:
  585. try:
  586. cluster.fs.rmtree('/test-avro-filebrowser/')
  587. except:
  588. pass # Don't let cleanup errors mask earlier failures
  589. @attr('requires_hadoop')
  590. def test_view_gz():
  591. cluster = pseudo_hdfs4.shared_cluster()
  592. try:
  593. c = make_logged_in_client()
  594. cluster.fs.setuser(cluster.superuser)
  595. if cluster.fs.isdir("/test-gz-filebrowser"):
  596. cluster.fs.rmtree('/test-gz-filebrowser/')
  597. cluster.fs.mkdir('/test-gz-filebrowser/')
  598. f = cluster.fs.open('/test-gz-filebrowser/test-view.gz', "w")
  599. sdf_string = '\x1f\x8b\x08\x082r\xf4K\x00\x03f\x00+NI\xe3\x02\x00\xad\x96b\xc4\x04\x00\x00\x00'
  600. f.write(sdf_string)
  601. f.close()
  602. response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip')
  603. assert_equal(response.context['view']['contents'], "sdf\n")
  604. # autodetect
  605. response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz')
  606. assert_equal(response.context['view']['contents'], "sdf\n")
  607. # offset should do nothing
  608. response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip&offset=1')
  609. assert_true("Offsets are not supported" in response.context['message'], response.context['message'])
  610. f = cluster.fs.open('/test-gz-filebrowser/test-view2.gz', "w")
  611. f.write("hello")
  612. f.close()
  613. # we shouldn't autodetect non gzip files
  614. response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz')
  615. assert_equal(response.context['view']['contents'], "hello")
  616. # we should fail to do a bad thing if they specify compression when it's not set.
  617. response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz?compression=gzip')
  618. assert_true("Failed to decompress" in response.context['message'])
  619. finally:
  620. try:
  621. cluster.fs.rmtree('/test-gz-filebrowser/')
  622. except:
  623. pass # Don't let cleanup errors mask earlier failures
  624. @attr('requires_hadoop')
  625. def test_view_i18n():
  626. cluster = pseudo_hdfs4.shared_cluster()
  627. try:
  628. cluster.fs.setuser(cluster.superuser)
  629. cluster.fs.mkdir('/test-filebrowser/')
  630. # Test viewing files in different encodings
  631. content = u'pt-Olá en-hello ch-你好 ko-안녕 ru-Здравствуйте'
  632. view_helper(cluster, 'utf-8', content)
  633. view_helper(cluster, 'utf-16', content)
  634. content = u'你好-big5'
  635. view_helper(cluster, 'big5', content)
  636. content = u'こんにちは-shift-jis'
  637. view_helper(cluster, 'shift_jis', content)
  638. content = u'안녕하세요-johab'
  639. view_helper(cluster, 'johab', content)
  640. # Test that the default view is home
  641. c = make_logged_in_client()
  642. response = c.get('/filebrowser/view/')
  643. assert_equal(response.context['path'], '/')
  644. cluster.fs.mkdir('/user/test')
  645. cluster.fs.chown("/user/test", "test", "test")
  646. response = c.get('/filebrowser/view/?default_to_home=1')
  647. assert_equal("http://testserver/filebrowser/view/user/test", response["location"])
  648. finally:
  649. try:
  650. cluster.fs.rmtree('/test-filebrowser/')
  651. cluster.fs.rmtree('/user/test')
  652. except Exception, ex:
  653. LOG.error('Failed to cleanup test directory: %s' % (ex,))
  654. @attr('requires_hadoop')
  655. def test_view_access():
  656. cluster = pseudo_hdfs4.shared_cluster()
  657. NO_PERM_DIR = u'/test-no-perm'
  658. try:
  659. c = make_logged_in_client()
  660. cluster.fs.setuser(cluster.superuser)
  661. cluster.fs.mkdir(NO_PERM_DIR, mode='700')
  662. response = c.get('/filebrowser/view/test-no-perm')
  663. assert_true('Cannot access' in response.context['message'])
  664. response = c.get('/filebrowser/view/test-does-not-exist')
  665. assert_true('Cannot access' in response.context['message'])
  666. finally:
  667. try:
  668. cluster.fs.rmtree(NO_PERM_DIR)
  669. except:
  670. pass # Don't let cleanup errors mask earlier failures
  671. @attr('requires_hadoop')
  672. def test_index():
  673. HOME_DIR = u'/user/test'
  674. NO_HOME_DIR = u'/user/no_home'
  675. c = make_logged_in_client()
  676. c_no_home = make_logged_in_client(username='no_home')
  677. cluster = pseudo_hdfs4.shared_cluster()
  678. if not cluster.fs.exists(HOME_DIR):
  679. cluster.fs.create_home_dir(HOME_DIR)
  680. assert_false(cluster.fs.exists(NO_HOME_DIR))
  681. response = c.get('/filebrowser', follow=True)
  682. assert_equal(HOME_DIR, response.context['path'])
  683. assert_equal(HOME_DIR, response.context['home_directory'])
  684. response = c_no_home.get('/filebrowser', follow=True)
  685. assert_equal('/', response.context['path'])
  686. assert_equal(None, response.context['home_directory'])
  687. def view_helper(cluster, encoding, content):
  688. """
  689. Write the content in the given encoding directly into the filesystem.
  690. Then try to view it and make sure the data is correct.
  691. """
  692. c = make_logged_in_client()
  693. filename = u'/test-filebrowser/test-view-carácter-internacional'
  694. bytestring = content.encode(encoding)
  695. try:
  696. f = cluster.fs.open(filename, "w")
  697. f.write(bytestring)
  698. f.close()
  699. response = c.get('/filebrowser/view%s?encoding=%s' % (filename, encoding))
  700. assert_equal(response.context['view']['contents'], content)
  701. response = c.get('/filebrowser/view%s?encoding=%s&end=8&begin=1' % (filename, encoding))
  702. assert_equal(response.context['view']['contents'],
  703. unicode(bytestring[0:8], encoding, errors='replace'))
  704. finally:
  705. try:
  706. cluster.fs.remove(filename)
  707. except Exception, ex:
  708. LOG.error('Failed to cleanup %s: %s' % (filename, ex))
  709. @attr('requires_hadoop')
  710. def test_edit_i18n():
  711. cluster = pseudo_hdfs4.shared_cluster()
  712. try:
  713. cluster.fs.setuser(cluster.superuser)
  714. cluster.fs.mkdir('/test-filebrowser/')
  715. # Test utf-8
  716. pass_1 = u'en-hello pt-Olá ch-你好 ko-안녕 ru-Здравствуйте'
  717. pass_2 = pass_1 + u'yi-העלא'
  718. edit_helper(cluster, 'utf-8', pass_1, pass_2)
  719. # Test utf-16
  720. edit_helper(cluster, 'utf-16', pass_1, pass_2)
  721. # Test cjk
  722. pass_1 = u'big5-你好'
  723. pass_2 = pass_1 + u'世界'
  724. edit_helper(cluster, 'big5', pass_1, pass_2)
  725. pass_1 = u'shift_jis-こんにちは'
  726. pass_2 = pass_1 + u'世界'
  727. edit_helper(cluster, 'shift_jis', pass_1, pass_2)
  728. pass_1 = u'johab-안녕하세요'
  729. pass_2 = pass_1 + u'세상'
  730. edit_helper(cluster, 'johab', pass_1, pass_2)
  731. finally:
  732. try:
  733. cluster.fs.rmtree('/test-filebrowser/')
  734. except Exception, ex:
  735. LOG.error('Failed to remove tree /test-filebrowser: %s' % (ex,))
  736. def edit_helper(cluster, encoding, contents_pass_1, contents_pass_2):
  737. """
  738. Put the content into the file with a specific encoding.
  739. """
  740. c = make_logged_in_client(cluster.superuser)
  741. # This path is non-normalized to test normalization too
  742. filename = u'//test-filebrowser//./test-edit-carácter-internacional'
  743. # File doesn't exist - should be empty
  744. edit_url = '/filebrowser/edit' + filename
  745. response = c.get(edit_url)
  746. assert_equal(response.context['form'].data['path'], filename)
  747. assert_equal(response.context['form'].data['contents'], "")
  748. # Just going to the edit page and not hitting save should not
  749. # create the file
  750. assert_false(cluster.fs.exists(filename))
  751. try:
  752. # Put some data in there and post
  753. response = c.post("/filebrowser/save", dict(
  754. path=filename,
  755. contents=contents_pass_1,
  756. encoding=encoding), follow=True)
  757. assert_equal(response.context['form'].data['path'], filename)
  758. assert_equal(response.context['form'].data['contents'], contents_pass_1)
  759. # File should now exist
  760. assert_true(cluster.fs.exists(filename))
  761. # And its contents should be what we expect
  762. f = cluster.fs.open(filename)
  763. assert_equal(f.read(), contents_pass_1.encode(encoding))
  764. assert_false('\r\n' in f.read()) # No CRLF line terminators
  765. f.close()
  766. # We should be able to overwrite the file with another save
  767. response = c.post("/filebrowser/save", dict(
  768. path=filename,
  769. contents=contents_pass_2,
  770. encoding=encoding), follow=True)
  771. assert_equal(response.context['form'].data['path'], filename)
  772. assert_equal(response.context['form'].data['contents'], contents_pass_2)
  773. f = cluster.fs.open(filename)
  774. assert_equal(f.read(), contents_pass_2.encode(encoding))
  775. assert_false('\r\n' in f.read()) # No CRLF line terminators
  776. f.close()
  777. # TODO(todd) add test for maintaining ownership/permissions
  778. finally:
  779. try:
  780. cluster.fs.remove(filename)
  781. except Exception, ex:
  782. LOG.error('Failed to remove %s: %s' % (smart_str(filename), ex))
  783. @attr('requires_hadoop')
  784. def test_upload_file():
  785. """Test file upload"""
  786. cluster = pseudo_hdfs4.shared_cluster()
  787. try:
  788. USER_NAME = 'test'
  789. HDFS_DEST_DIR = "/tmp/fb-upload-test"
  790. LOCAL_FILE = __file__
  791. HDFS_FILE = HDFS_DEST_DIR + '/' + os.path.basename(__file__)
  792. cluster.fs.setuser(USER_NAME)
  793. client = make_logged_in_client(USER_NAME)
  794. cluster.fs.do_as_superuser(cluster.fs.mkdir, HDFS_DEST_DIR)
  795. cluster.fs.do_as_superuser(cluster.fs.chown, HDFS_DEST_DIR, USER_NAME, USER_NAME)
  796. cluster.fs.do_as_superuser(cluster.fs.chmod, HDFS_DEST_DIR, 0700)
  797. stats = cluster.fs.stats(HDFS_DEST_DIR)
  798. assert_equal(stats['user'], USER_NAME)
  799. assert_equal(stats['group'], USER_NAME)
  800. # Just upload the current python file
  801. resp = client.post('/filebrowser/upload/file?dest=%s' % HDFS_DEST_DIR, # GET param avoids infinite looping
  802. dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
  803. response = json.loads(resp.content)
  804. assert_equal(0, response['status'], response)
  805. stats = cluster.fs.stats(HDFS_FILE)
  806. assert_equal(stats['user'], USER_NAME)
  807. assert_equal(stats['group'], USER_NAME)
  808. f = cluster.fs.open(HDFS_FILE)
  809. actual = f.read()
  810. expected = file(LOCAL_FILE).read()
  811. assert_equal(actual, expected)
  812. # Upload again and so fails because file already exits
  813. resp = client.post('/filebrowser/upload/file?dest=%s' % HDFS_DEST_DIR,
  814. dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
  815. response = json.loads(resp.content)
  816. assert_equal(-1, response['status'], response)
  817. assert_true('already exists' in response['data'], response)
  818. # Upload in / and fails because of missing permissions
  819. LOG.debug('HDFS superuser: %s' % cluster.fs._superuser)
  820. try:
  821. resp = client.post('/filebrowser/upload/file?dest=%s' % '/',
  822. dict(dest='/', hdfs_file=file(LOCAL_FILE)))
  823. response = json.loads(resp.content)
  824. assert_equal(-1, response['status'], response)
  825. assert_true('Permission denied' in response['data'], response)
  826. except AttributeError:
  827. # Seems like a Django bug.
  828. # StopFutureHandlers() does not seem to work in test mode as it continues to MemoryFileUploadHandler after perm issue and so fails.
  829. pass
  830. finally:
  831. try:
  832. cluster.fs.remove(HDFS_DEST_DIR)
  833. except Exception, ex:
  834. pass
  835. @attr('requires_hadoop')
  836. def test_upload_archive():
  837. """Test archive upload"""
  838. cluster = pseudo_hdfs4.shared_cluster()
  839. try:
  840. USER_NAME = 'test'
  841. HDFS_DEST_DIR = "/tmp/fb-upload-test"
  842. ZIP_FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.zip')
  843. HDFS_ZIP_FILE = HDFS_DEST_DIR + '/test.zip'
  844. HDFS_UNZIPPED_FILE = HDFS_DEST_DIR + '/test'
  845. cluster.fs.setuser(USER_NAME)
  846. client = make_logged_in_client(USER_NAME)
  847. cluster.fs.mkdir(HDFS_DEST_DIR)
  848. cluster.fs.chown(HDFS_DEST_DIR, USER_NAME)
  849. cluster.fs.chmod(HDFS_DEST_DIR, 0700)
  850. # Upload and unzip archive
  851. resp = client.post('/filebrowser/upload/archive?dest=%s' % HDFS_DEST_DIR,
  852. dict(dest=HDFS_DEST_DIR, archive=file(ZIP_FILE)))
  853. response = json.loads(resp.content)
  854. assert_equal(0, response['status'], response)
  855. assert_false(cluster.fs.exists(HDFS_ZIP_FILE))
  856. assert_true(cluster.fs.isdir(HDFS_UNZIPPED_FILE))
  857. assert_true(cluster.fs.isfile(HDFS_UNZIPPED_FILE + '/test.txt'))
  858. # Upload archive
  859. resp = client.post('/filebrowser/upload/file?dest=%s' % HDFS_DEST_DIR,
  860. dict(dest=HDFS_DEST_DIR, hdfs_file=file(ZIP_FILE)))
  861. response = json.loads(resp.content)
  862. assert_equal(0, response['status'], response)
  863. assert_true(cluster.fs.exists(HDFS_ZIP_FILE))
  864. finally:
  865. try:
  866. cluster.fs.remove(HDFS_DEST_DIR)
  867. except Exception, ex:
  868. pass