views.py 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #
  18. # Implements simple file system browsing operations.
  19. #
  20. # Useful resources:
  21. # django/views/static.py manages django's internal directory index
  22. import errno
  23. import logging
  24. import mimetypes
  25. import operator
  26. import posixpath
  27. import re
  28. import shutil
  29. import stat as stat_module
  30. import os
  31. try:
  32. import json
  33. except ImportError:
  34. import simplejson as json
  35. from datetime import datetime
  36. from django.contrib import messages
  37. from django.contrib.auth.models import User, Group
  38. from django.core import urlresolvers
  39. from django.template.defaultfilters import stringformat, filesizeformat
  40. from django.http import Http404, HttpResponse, HttpResponseNotModified
  41. from django.views.decorators.http import require_http_methods
  42. from django.views.static import was_modified_since
  43. from django.utils.functional import curry
  44. from django.utils.http import http_date, urlquote
  45. from django.utils.html import escape
  46. from cStringIO import StringIO
  47. from gzip import GzipFile
  48. from avro import datafile, io
  49. from desktop.lib import i18n, paginator
  50. from desktop.lib.conf import coerce_bool
  51. from desktop.lib.django_util import make_absolute, render, render_json, format_preserving_redirect
  52. from desktop.lib.exceptions_renderable import PopupException
  53. from filebrowser.conf import MAX_SNAPPY_DECOMPRESSION_SIZE
  54. from filebrowser.lib.archives import archive_factory
  55. from filebrowser.lib.rwx import filetype, rwx
  56. from filebrowser.lib import xxd
  57. from filebrowser.forms import RenameForm, UploadFileForm, UploadArchiveForm, MkDirForm, EditorForm, TouchForm,\
  58. RenameFormSet, RmTreeFormSet, ChmodFormSet, ChownFormSet, CopyFormSet, RestoreFormSet,\
  59. TrashPurgeForm
  60. from hadoop.core_site import get_trash_interval
  61. from hadoop.fs.hadoopfs import Hdfs
  62. from hadoop.fs.exceptions import WebHdfsException
  63. from django.utils.translation import ugettext as _
  64. DEFAULT_CHUNK_SIZE_BYTES = 1024 * 4 # 4KB
  65. MAX_CHUNK_SIZE_BYTES = 1024 * 1024 # 1MB
  66. DOWNLOAD_CHUNK_SIZE = 64 * 1024 * 1024 # 64MB
  67. # Defaults for "xxd"-style output.
  68. # Sentences refer to groups of bytes printed together, within a line.
  69. BYTES_PER_LINE = 16
  70. BYTES_PER_SENTENCE = 2
  71. # The maximum size the file editor will allow you to edit
  72. MAX_FILEEDITOR_SIZE = 256 * 1024
  73. logger = logging.getLogger(__name__)
  74. def index(request):
  75. # Redirect to home directory by default
  76. path = request.user.get_home_directory()
  77. if not request.fs.isdir(path):
  78. path = '/'
  79. return view(request, path)
  80. def _file_reader(fh):
  81. """Generator that reads a file, chunk-by-chunk."""
  82. while True:
  83. chunk = fh.read(DOWNLOAD_CHUNK_SIZE)
  84. if chunk == '':
  85. fh.close()
  86. break
  87. yield chunk
  88. def download(request, path):
  89. """
  90. Downloads a file.
  91. This is inspired by django.views.static.serve.
  92. """
  93. if not request.fs.exists(path):
  94. raise Http404(_("File not found: %(path)s") % {'path': escape(path)})
  95. if not request.fs.isfile(path):
  96. raise PopupException(_("'%(path)s' is not a file") % {'path': path})
  97. mimetype = mimetypes.guess_type(path)[0] or 'application/octet-stream'
  98. stats = request.fs.stats(path)
  99. mtime = stats['mtime']
  100. size = stats['size']
  101. if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), mtime, size):
  102. return HttpResponseNotModified()
  103. # TODO(philip): Ideally a with statement would protect from leaks,
  104. # but tricky to do here.
  105. fh = request.fs.open(path)
  106. response = HttpResponse(_file_reader(fh), mimetype=mimetype)
  107. response["Last-Modified"] = http_date(stats['mtime'])
  108. response["Content-Length"] = stats['size']
  109. response["Content-Disposition"] = "attachment"
  110. return response
  111. def view(request, path):
  112. """Dispatches viewing of a path to either index() or fileview(), depending on type."""
  113. # default_to_home is set in bootstrap.js
  114. if 'default_to_home' in request.GET:
  115. home_dir_path = request.user.get_home_directory()
  116. if request.fs.isdir(home_dir_path):
  117. return format_preserving_redirect(request, urlresolvers.reverse(view, kwargs=dict(path=home_dir_path)))
  118. # default_to_home is set in bootstrap.js
  119. if 'default_to_trash' in request.GET:
  120. if request.fs.isdir(request.fs.trash_path):
  121. return format_preserving_redirect(request, urlresolvers.reverse(view, kwargs=dict(path=request.fs.trash_path)))
  122. try:
  123. stats = request.fs.stats(path)
  124. if stats.isDir:
  125. return listdir_paged(request, path)
  126. else:
  127. return display(request, path)
  128. except (IOError, WebHdfsException), e:
  129. msg = _("Cannot access: %(path)s.") % {'path': escape(path)}
  130. if request.user.is_superuser and not request.user == request.fs.superuser:
  131. msg += _(' Note: you are a Hue admin but not a HDFS superuser (which is "%(superuser)s").') % {'superuser': request.fs.superuser}
  132. if request.is_ajax():
  133. exception = {
  134. 'error': msg
  135. }
  136. return render_json(exception)
  137. else:
  138. raise PopupException(msg , detail=e)
  139. def home_relative_view(request, path):
  140. home_dir_path = request.user.get_home_directory()
  141. if request.fs.exists(home_dir_path):
  142. path = '%s%s' % (home_dir_path, path)
  143. return view(request, path)
  144. def edit(request, path, form=None):
  145. """Shows an edit form for the given path. Path does not necessarily have to exist."""
  146. try:
  147. stats = request.fs.stats(path)
  148. except IOError, ioe:
  149. # A file not found is OK, otherwise re-raise
  150. if ioe.errno == errno.ENOENT:
  151. stats = None
  152. else:
  153. raise
  154. # Can't edit a directory
  155. if stats and stats['mode'] & stat_module.S_IFDIR:
  156. raise PopupException(_("Cannot edit a directory: %(path)s") % {'path': path})
  157. # Maximum size of edit
  158. if stats and stats['size'] > MAX_FILEEDITOR_SIZE:
  159. raise PopupException(_("File too big to edit: %(path)s") % {'path': path})
  160. if not form:
  161. encoding = request.REQUEST.get('encoding') or i18n.get_site_encoding()
  162. if stats:
  163. f = request.fs.open(path)
  164. try:
  165. try:
  166. current_contents = unicode(f.read(), encoding)
  167. except UnicodeDecodeError:
  168. raise PopupException(_("File is not encoded in %(encoding)s; cannot be edited: %(path)s") % {'encoding': encoding, 'path': path})
  169. finally:
  170. f.close()
  171. else:
  172. current_contents = u""
  173. form = EditorForm(dict(path=path, contents=current_contents, encoding=encoding))
  174. data = dict(
  175. exists=(stats is not None),
  176. form=form,
  177. path=path,
  178. filename=os.path.basename(path),
  179. dirname=os.path.dirname(path),
  180. breadcrumbs = parse_breadcrumbs(path))
  181. return render("edit.mako", request, data)
  182. def save_file(request):
  183. """
  184. The POST endpoint to save a file in the file editor.
  185. Does the save and then redirects back to the edit page.
  186. """
  187. form = EditorForm(request.POST)
  188. is_valid = form.is_valid()
  189. path = form.cleaned_data.get('path')
  190. if request.POST.get('save') == "Save As":
  191. if not is_valid:
  192. return edit(request, path, form=form)
  193. else:
  194. data = dict(form=form)
  195. return render("saveas.mako", request, data)
  196. if not path:
  197. raise PopupException("No path specified")
  198. if not is_valid:
  199. return edit(request, path, form=form)
  200. if request.fs.exists(path):
  201. _do_overwrite_save(request.fs, path,
  202. form.cleaned_data['contents'],
  203. form.cleaned_data['encoding'])
  204. else:
  205. _do_newfile_save(request.fs, path,
  206. form.cleaned_data['contents'],
  207. form.cleaned_data['encoding'])
  208. messages.info(request, _('Saved %(path)s.') % {'path': os.path.basename(path)})
  209. """ Changing path to reflect the request path of the JFrame that will actually be returned."""
  210. request.path = urlresolvers.reverse("filebrowser.views.edit", kwargs=dict(path=path))
  211. return edit(request, path, form)
  212. def _do_overwrite_save(fs, path, data, encoding):
  213. """
  214. Atomically (best-effort) save the specified data to the given path
  215. on the filesystem.
  216. TODO(todd) should this be in some fsutil.py?
  217. """
  218. # TODO(todd) Should probably do an advisory permissions check here to
  219. # see if we're likely to fail (eg make sure we own the file
  220. # and can write to the dir)
  221. # First write somewhat-kinda-atomically to a staging file
  222. # so that if we fail, we don't clobber the old one
  223. path_dest = path + "._hue_new"
  224. new_file = fs.open(path_dest, "w")
  225. try:
  226. try:
  227. new_file.write(data.encode(encoding))
  228. logging.info("Wrote to " + path_dest)
  229. finally:
  230. new_file.close()
  231. except Exception, e:
  232. # An error occurred in writing, we should clean up
  233. # the tmp file if it exists, before re-raising
  234. try:
  235. fs.remove(path_dest)
  236. except:
  237. pass
  238. raise e
  239. # Try to match the permissions and ownership of the old file
  240. cur_stats = fs.stats(path)
  241. try:
  242. fs.chmod(path_dest, stat_module.S_IMODE(cur_stats['mode']))
  243. except:
  244. logging.warn("Could not chmod new file %s to match old file %s" % (
  245. path_dest, path), exc_info=True)
  246. # but not the end of the world - keep going
  247. try:
  248. fs.chown(path_dest, cur_stats['user'], cur_stats['group'])
  249. except:
  250. logging.warn("Could not chown new file %s to match old file %s" % (
  251. path_dest, path), exc_info=True)
  252. # but not the end of the world - keep going
  253. # Now delete the old - nothing we can do here to recover
  254. fs.remove(path)
  255. # Now move the new one into place
  256. # If this fails, then we have no reason to assume
  257. # we can do anything to recover, since we know the
  258. # destination shouldn't already exist (we just deleted it above)
  259. fs.rename(path_dest, path)
  260. def _do_newfile_save(fs, path, data, encoding):
  261. """
  262. Save data to the path 'path' on the filesystem 'fs'.
  263. There must not be a pre-existing file at that path.
  264. """
  265. new_file = fs.open(path, "w")
  266. try:
  267. new_file.write(data.encode(encoding))
  268. finally:
  269. new_file.close()
  270. def parse_breadcrumbs(path):
  271. breadcrumbs_parts = Hdfs.normpath(path).split('/')
  272. i = 1
  273. breadcrumbs = [{'url': '', 'label': '/'}]
  274. while (i < len(breadcrumbs_parts)):
  275. breadcrumb_url = breadcrumbs[i - 1]['url'] + '/' + breadcrumbs_parts[i]
  276. if breadcrumb_url != '/':
  277. breadcrumbs.append({'url': breadcrumb_url, 'label': breadcrumbs_parts[i]})
  278. i = i + 1
  279. return breadcrumbs
  280. def listdir(request, path, chooser):
  281. """
  282. Implements directory listing (or index).
  283. Intended to be called via view().
  284. TODO: Remove?
  285. """
  286. if not request.fs.isdir(path):
  287. raise PopupException(_("Not a directory: %(path)s") % {'path': path})
  288. file_filter = request.REQUEST.get('file_filter', 'any')
  289. assert file_filter in ['any', 'file', 'dir']
  290. home_dir_path = request.user.get_home_directory()
  291. breadcrumbs = parse_breadcrumbs(path)
  292. data = {
  293. 'path': path,
  294. 'file_filter': file_filter,
  295. 'breadcrumbs': breadcrumbs,
  296. 'current_dir_path': path,
  297. # These could also be put in automatically via
  298. # http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
  299. # but manually seems cleaner, since we only need it here.
  300. 'current_request_path': request.path,
  301. 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
  302. 'cwd_set': True,
  303. 'is_superuser': request.user.username == request.fs.superuser,
  304. 'groups': request.user.username == request.fs.superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [],
  305. 'users': request.user.username == request.fs.superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [],
  306. 'superuser': request.fs.superuser,
  307. 'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False,) or (True,))[0]
  308. }
  309. stats = request.fs.listdir_stats(path)
  310. # Include parent dir, unless at filesystem root.
  311. if Hdfs.normpath(path) != posixpath.sep:
  312. parent_path = request.fs.join(path, "..")
  313. parent_stat = request.fs.stats(parent_path)
  314. # The 'path' field would be absolute, but we want its basename to be
  315. # actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
  316. parent_stat['path'] = parent_path
  317. stats.insert(0, parent_stat)
  318. data['files'] = [_massage_stats(request, stat) for stat in stats]
  319. if chooser:
  320. return render('chooser.mako', request, data)
  321. else:
  322. return render('listdir.mako', request, data)
  323. def _massage_page(page):
  324. return {
  325. 'number': page.number,
  326. 'num_pages': page.num_pages(),
  327. 'previous_page_number': page.previous_page_number(),
  328. 'next_page_number': page.next_page_number(),
  329. 'start_index': page.start_index(),
  330. 'end_index': page.end_index(),
  331. 'total_count': page.total_count()
  332. }
  333. def listdir_paged(request, path):
  334. """
  335. A paginated version of listdir.
  336. Query parameters:
  337. pagenum - The page number to show. Defaults to 1.
  338. pagesize - How many to show on a page. Defaults to 15.
  339. sortby=? - Specify attribute to sort by. Accepts:
  340. (type, name, atime, mtime, size, user, group)
  341. Defaults to name.
  342. descending - Specify a descending sort order.
  343. Default to false.
  344. filter=? - Specify a substring filter to search for in
  345. the filename field.
  346. """
  347. if not request.fs.isdir(path):
  348. raise PopupException("Not a directory: %s" % (path,))
  349. trash_enabled = get_trash_interval()
  350. pagenum = int(request.GET.get('pagenum', 1))
  351. pagesize = int(request.GET.get('pagesize', 30))
  352. home_dir_path = request.user.get_home_directory()
  353. breadcrumbs = parse_breadcrumbs(path)
  354. all_stats = request.fs.listdir_stats(path)
  355. # Filter first
  356. filter_str = request.GET.get('filter', None)
  357. if filter_str:
  358. filtered_stats = filter(lambda sb: filter_str in sb['name'], all_stats)
  359. all_stats = filtered_stats
  360. # Sort next
  361. sortby = request.GET.get('sortby', None)
  362. descending_param = request.GET.get('descending', None)
  363. if sortby is not None:
  364. if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'):
  365. logger.info("Invalid sort attribute '%s' for listdir." %
  366. (sortby,))
  367. else:
  368. all_stats = sorted(all_stats,
  369. key=operator.attrgetter(sortby),
  370. reverse=coerce_bool(descending_param))
  371. # Do pagination
  372. page = paginator.Paginator(all_stats, pagesize).page(pagenum)
  373. shown_stats = page.object_list
  374. # Include parent dir always as second option, unless at filesystem root.
  375. if Hdfs.normpath(path) != posixpath.sep:
  376. parent_path = request.fs.join(path, "..")
  377. parent_stat = request.fs.stats(parent_path)
  378. # The 'path' field would be absolute, but we want its basename to be
  379. # actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
  380. parent_stat['path'] = parent_path
  381. parent_stat['name'] = ".."
  382. shown_stats.insert(0, parent_stat)
  383. # Include same dir always as first option to see stats of the current folder
  384. current_stat = request.fs.stats(path)
  385. # The 'path' field would be absolute, but we want its basename to be
  386. # actually '.' for display purposes. Encode it since _massage_stats expects byte strings.
  387. current_stat['path'] = path
  388. current_stat['name'] = "."
  389. shown_stats.insert(0, current_stat)
  390. page.object_list = [ _massage_stats(request, s) for s in shown_stats ]
  391. data = {
  392. 'path': path,
  393. 'breadcrumbs': breadcrumbs,
  394. 'current_request_path': request.path,
  395. 'files': page.object_list,
  396. 'page': _massage_page(page),
  397. 'pagesize': pagesize,
  398. 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
  399. 'trash_enabled': trash_enabled,
  400. 'sortby': sortby,
  401. 'descending': descending_param,
  402. # The following should probably be deprecated
  403. 'cwd_set': True,
  404. 'file_filter': 'any',
  405. 'current_dir_path': path,
  406. 'is_fs_superuser': request.user.username == request.fs.superuser,
  407. 'is_superuser': request.user.username == request.fs.superuser,
  408. 'groups': request.user.username == request.fs.superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [],
  409. 'users': request.user.username == request.fs.superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [],
  410. 'superuser': request.fs.superuser
  411. }
  412. return render('listdir.mako', request, data)
  413. def chooser(request, path):
  414. """
  415. Returns the html to JFrame that will display a file prompt.
  416. Dispatches viewing of a path to either index() or fileview(), depending on type.
  417. """
  418. # default_to_home is set in bootstrap.js
  419. home_dir_path = request.user.get_home_directory()
  420. if 'default_to_home' in request.GET and request.fs.isdir(home_dir_path):
  421. return listdir(request, home_dir_path, True)
  422. if request.fs.isdir(path):
  423. return listdir(request, path, True)
  424. elif request.fs.isfile(path):
  425. return display(request, path)
  426. else:
  427. raise Http404(_("File not found: %(path)s") % {'path': escape(path)})
  428. def _massage_stats(request, stats):
  429. """
  430. Massage a stats record as returned by the filesystem implementation
  431. into the format that the views would like it in.
  432. """
  433. path = stats['path']
  434. normalized = Hdfs.normpath(path)
  435. return {
  436. 'path': normalized,
  437. 'name': stats['name'],
  438. 'stats': stats.to_json_dict(),
  439. 'mtime': datetime.fromtimestamp(stats['mtime']).strftime('%B %d, %Y %I:%M %p'),
  440. 'humansize': filesizeformat(stats['size']),
  441. 'type': filetype(stats['mode']),
  442. 'rwx': rwx(stats['mode']),
  443. 'mode': stringformat(stats['mode'], "o"),
  444. 'url': make_absolute(request, "view", dict(path=urlquote(normalized))),
  445. }
  446. def stat(request, path):
  447. """
  448. Returns just the generic stats of a file.
  449. Intended for use via AJAX (and hence doesn't provide
  450. an HTML view).
  451. """
  452. if not request.fs.exists(path):
  453. raise Http404(_("File not found: %(path)s") % {'path': escape(path)})
  454. stats = request.fs.stats(path)
  455. return render_json(_massage_stats(request, stats))
  456. def display(request, path):
  457. """
  458. Implements displaying part of a file.
  459. GET arguments are length, offset, mode, compression and encoding
  460. with reasonable defaults chosen.
  461. Note that display by length and offset are on bytes, not on characters.
  462. TODO(philip): Could easily built-in file type detection
  463. (perhaps using something similar to file(1)), as well
  464. as more advanced binary-file viewing capability (de-serialize
  465. sequence files, decompress gzipped text files, etc.).
  466. There exists a python-magic package to interface with libmagic.
  467. """
  468. if not request.fs.isfile(path):
  469. raise PopupException(_("Not a file: '%(path)s'") % {'path': path})
  470. stats = request.fs.stats(path)
  471. encoding = request.GET.get('encoding') or i18n.get_site_encoding()
  472. # I'm mixing URL-based parameters and traditional
  473. # HTTP GET parameters, since URL-based parameters
  474. # can't naturally be optional.
  475. # Need to deal with possibility that length is not present
  476. # because the offset came in via the toolbar manual byte entry.
  477. end = request.GET.get("end")
  478. if end:
  479. end = int(end)
  480. begin = request.GET.get("begin", 1)
  481. if begin:
  482. # Subtract one to zero index for file read
  483. begin = int(begin) - 1
  484. if end:
  485. offset = begin
  486. length = end - begin
  487. if begin >= end:
  488. raise PopupException(_("First byte to display must be before last byte to display."))
  489. else:
  490. length = int(request.GET.get("length", DEFAULT_CHUNK_SIZE_BYTES))
  491. # Display first block by default.
  492. offset = int(request.GET.get("offset", 0))
  493. mode = request.GET.get("mode")
  494. compression = request.GET.get("compression")
  495. if mode and mode not in ["binary", "text"]:
  496. raise PopupException(_("Mode must be one of 'binary' or 'text'."))
  497. if offset < 0:
  498. raise PopupException(_("Offset may not be less than zero."))
  499. if length < 0:
  500. raise PopupException(_("Length may not be less than zero."))
  501. if length > MAX_CHUNK_SIZE_BYTES:
  502. raise PopupException(_("Cannot request chunks greater than %(bytes)d bytes") % {'bytes': MAX_CHUNK_SIZE_BYTES})
  503. # Do not decompress in binary mode.
  504. if mode == 'binary':
  505. compression = 'none'
  506. # Read out based on meta.
  507. compression, offset, length, contents =\
  508. read_contents(compression, path, request.fs, offset, length)
  509. # Get contents as string for text mode, or at least try
  510. uni_contents = None
  511. if not mode or mode == 'text':
  512. uni_contents = unicode(contents, encoding, errors='replace')
  513. is_binary = uni_contents.find(i18n.REPLACEMENT_CHAR) != -1
  514. # Auto-detect mode
  515. if not mode:
  516. mode = is_binary and 'binary' or 'text'
  517. # Get contents as bytes
  518. if mode == "binary":
  519. xxd_out = list(xxd.xxd(offset, contents, BYTES_PER_LINE, BYTES_PER_SENTENCE))
  520. dirname = posixpath.dirname(path)
  521. # Start with index-like data:
  522. data = _massage_stats(request, request.fs.stats(path))
  523. # And add a view structure:
  524. data["success"] = True
  525. data["view"] = {
  526. 'offset': offset,
  527. 'length': length,
  528. 'end': offset + len(contents),
  529. 'dirname': dirname,
  530. 'mode': mode,
  531. 'compression': compression,
  532. 'size': stats['size'],
  533. 'max_chunk_size': str(MAX_CHUNK_SIZE_BYTES)
  534. }
  535. data["filename"] = os.path.basename(path)
  536. data["editable"] = stats['size'] < MAX_FILEEDITOR_SIZE
  537. if mode == "binary":
  538. # This might be the wrong thing for ?format=json; doing the
  539. # xxd'ing in javascript might be more compact, or sending a less
  540. # intermediate representation...
  541. logger.debug("xxd: " + str(xxd_out))
  542. data['view']['xxd'] = xxd_out
  543. data['view']['masked_binary_data'] = False
  544. else:
  545. data['view']['contents'] = uni_contents
  546. data['view']['masked_binary_data'] = is_binary
  547. data['breadcrumbs'] = parse_breadcrumbs(path)
  548. return render("display.mako", request, data)
  549. def read_contents(codec_type, path, fs, offset, length):
  550. """
  551. Reads contents of a passed path, by appropriately decoding the data.
  552. Arguments:
  553. codec_type - The type of codec to use to decode. (Auto-detected if None).
  554. path - The path of the file to read.
  555. fs - The FileSystem instance to use to read.
  556. offset - Offset to seek to before read begins.
  557. length - Amount of bytes to read after offset.
  558. Returns: A tuple of codec_type, offset, length and contents read.
  559. """
  560. contents = ''
  561. try:
  562. fhandle = fs.open(path)
  563. stats = fs.stats(path)
  564. # Auto codec detection for [gzip, avro, none]
  565. # Only done when codec_type is unset
  566. contents = fhandle.read(3)
  567. if not codec_type:
  568. codec_type = 'none'
  569. if path.endswith('.gz') and detect_gzip(contents):
  570. codec_type = 'gzip'
  571. offset = 0
  572. elif path.endswith('.avro'):
  573. if detect_avro(contents):
  574. codec_type = 'avro'
  575. elif snappy_installed():
  576. if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
  577. raise PopupException(_('Failed to validate snappy compressed file. File size is greater than allowed max snappy decompression size of %d') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
  578. if detect_snappy(contents + fhandle.read()):
  579. codec_type = 'snappy_avro'
  580. fhandle.seek(0)
  581. if codec_type == 'avro' and snappy_installed() and detect_snappy(fhandle.read()):
  582. fhandle.seek(0)
  583. codec_type = 'snappy_avro'
  584. if codec_type == 'gzip':
  585. contents = _read_gzip(fhandle, path, offset, length, stats)
  586. elif codec_type == 'avro':
  587. contents = _read_avro(fhandle, path, offset, length, stats)
  588. elif codec_type == 'snappy_avro':
  589. contents = _read_snappy_avro(fhandle, path, offset, length, stats)
  590. else:
  591. # for 'none' type.
  592. contents = _read_simple(fhandle, path, offset, length, stats)
  593. finally:
  594. fhandle.close()
  595. return (codec_type, offset, length, contents)
  596. def _decompress_snappy(compressed_content):
  597. try:
  598. import snappy
  599. return snappy.decompress(compressed_content)
  600. except Exception, e:
  601. raise PopupException(_('Failed to decompress snappy compressed file.'), detail=e)
  602. def _read_snappy_avro(fhandle, path, offset, length, stats):
  603. if not snappy_installed():
  604. raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed!'))
  605. if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
  606. raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
  607. return _read_avro(StringIO(_decompress_snappy(fhandle.read())), path, offset, length, stats)
  608. def _read_avro(fhandle, path, offset, length, stats):
  609. contents = ''
  610. try:
  611. fhandle.seek(offset)
  612. data_file_reader = datafile.DataFileReader(fhandle, io.DatumReader())
  613. contents_list = []
  614. read_start = fhandle.tell()
  615. # Iterate over the entire sought file.
  616. for datum in data_file_reader:
  617. read_length = fhandle.tell() - read_start
  618. if read_length > length and len(contents_list) > 0:
  619. break
  620. else:
  621. datum_str = str(datum) + "\n"
  622. contents_list.append(datum_str)
  623. data_file_reader.close()
  624. contents = "".join(contents_list)
  625. except:
  626. logging.warn("Could not read avro file at %s" % path, exc_info=True)
  627. raise PopupException(_("Failed to read Avro file."))
  628. return contents
  629. def _read_gzip(fhandle, path, offset, length, stats):
  630. contents = ''
  631. if offset and offset != 0:
  632. raise PopupException(_("Offsets are not supported with Gzip compression."))
  633. try:
  634. contents = GzipFile('', 'r', 0, StringIO(fhandle.read())).read(length)
  635. except:
  636. logging.warn("Could not decompress file at %s" % path, exc_info=True)
  637. raise PopupException(_("Failed to decompress file."))
  638. return contents
  639. def _read_simple(fhandle, path, offset, length, stats):
  640. contents = ''
  641. try:
  642. fhandle.seek(offset)
  643. contents = fhandle.read(length)
  644. except:
  645. logging.warn("Could not read file at %s" % path, exc_info=True)
  646. raise PopupException(_("Failed to read file."))
  647. return contents
  648. def detect_gzip(contents):
  649. '''This is a silly small function which checks to see if the file is Gzip'''
  650. return contents[:2] == '\x1f\x8b'
  651. def detect_avro(contents):
  652. '''This is a silly small function which checks to see if the file is Avro'''
  653. # Check if the first three bytes are 'O', 'b' and 'j'
  654. return contents[:3] == '\x4F\x62\x6A'
  655. def detect_snappy(contents):
  656. '''
  657. This is a silly small function which checks to see if the file is Snappy.
  658. It requires the entire contents of the compressed file.
  659. This will also return false if snappy decompression if we do not have the library available.
  660. '''
  661. try:
  662. import snappy
  663. return snappy.isValidCompressed(contents)
  664. except:
  665. return False
  666. def snappy_installed():
  667. '''Snappy is library that isn't supported by python2.4'''
  668. try:
  669. import snappy
  670. return True
  671. except:
  672. return False
  673. def _calculate_navigation(offset, length, size):
  674. """
  675. List of (offset, length, string) tuples for suggested navigation through the file.
  676. If offset is -1, then this option is already "selected". (Whereas None would
  677. be the natural pythonic way, Django's template syntax doesn't let us test
  678. against None (since its truth value is the same as 0).)
  679. By all means this logic ought to be in the template, but the template
  680. language is too limiting.
  681. """
  682. if offset == 0:
  683. first, prev = (-1, None, _("First Block")), (-1, None, _("Previous Block"))
  684. else:
  685. first, prev = (0, length, _("First Block")), (max(0, offset - length), length, _("Previous Block"))
  686. if offset + length >= size:
  687. next, last = (-1, None, _("Next Block")), (-1, None, _("Last Block"))
  688. else:
  689. # 1-off Reasoning: if length is the same as size, you want to start at 0.
  690. next, last = (offset + length, length, _("Next Block")), (max(0, size - length), length, _("Last Block"))
  691. return first, prev, next, last
  692. def default_initial_value_extractor(request, parameter_names):
  693. initial_values = {}
  694. for p in parameter_names:
  695. val = request.GET.get(p)
  696. if val:
  697. initial_values[p] = val
  698. return initial_values
  699. def formset_initial_value_extractor(request, parameter_names):
  700. """
  701. Builds a list of data that formsets should use by extending some fields to every object,
  702. whilst others are assumed to be received in order.
  703. Formsets should receive data that looks like this: [{'param1': <something>,...}, ...].
  704. The formsets should then handle construction on their own.
  705. """
  706. def _intial_value_extractor(request):
  707. if not submitted:
  708. return []
  709. # Build data with list of in order parameters receive in POST data
  710. # Size can be inferred from largest list returned in POST data
  711. data = []
  712. for param in submitted:
  713. i = 0
  714. for val in request.POST.getlist(param):
  715. if len(data) == i:
  716. data.append({})
  717. data[i][param] = val
  718. i += 1
  719. # Extend every data object with recurring params
  720. for kwargs in data:
  721. for recurrent in recurring:
  722. kwargs[recurrent] = request.POST.get(recurrent)
  723. initial_data = data
  724. return {'initial': initial_data}
  725. return _intial_value_extractor
  726. def default_arg_extractor(request, form, parameter_names):
  727. return [form.cleaned_data[p] for p in parameter_names]
  728. def formset_arg_extractor(request, formset, parameter_names):
  729. data = []
  730. for form in formset.forms:
  731. data_dict = {}
  732. for p in parameter_names:
  733. data_dict[p] = form.cleaned_data[p]
  734. data.append(data_dict)
  735. return data
  736. def default_data_extractor(request):
  737. return {'data': request.POST.copy()}
  738. def formset_data_extractor(recurring=[], submitted=[]):
  739. """
  740. Builds a list of data that formsets should use by extending some fields to every object,
  741. whilst others are assumed to be received in order.
  742. Formsets should receive data that looks like this: [{'param1': <something>,...}, ...].
  743. The formsets should then handle construction on their own.
  744. """
  745. def _data_extractor(request):
  746. if not submitted:
  747. return []
  748. # Build data with list of in order parameters receive in POST data
  749. # Size can be inferred from largest list returned in POST data
  750. data = []
  751. for param in submitted:
  752. i = 0
  753. for val in request.POST.getlist(param):
  754. if len(data) == i:
  755. data.append({})
  756. data[i][param] = val
  757. i += 1
  758. # Extend every data object with recurring params
  759. for kwargs in data:
  760. for recurrent in recurring:
  761. kwargs[recurrent] = request.POST.get(recurrent)
  762. initial = list(data)
  763. return {'initial': initial, 'data': data}
  764. return _data_extractor
  765. def generic_op(form_class, request, op, parameter_names, piggyback=None, template="fileop.mako", data_extractor=default_data_extractor, arg_extractor=default_arg_extractor, initial_value_extractor=default_initial_value_extractor, extra_params=None):
  766. """
  767. Generic implementation for several operations.
  768. @param form_class form to instantiate
  769. @param request incoming request, used for parameters
  770. @param op callable with the filesystem operation
  771. @param parameter_names list of form parameters that are extracted and then passed to op
  772. @param piggyback list of form parameters whose file stats to look up after the operation
  773. @param data_extractor function that extracts POST data to be used by op
  774. @param arg_extractor function that extracts args from a given form or formset
  775. @param initial_value_extractor function that extracts the initial values of a form or formset
  776. @param extra_params dictionary of extra parameters to send to the template for rendering
  777. """
  778. # Use next for non-ajax requests, when available.
  779. next = request.GET.get("next", request.POST.get("next", None))
  780. ret = dict({
  781. 'next': next
  782. })
  783. if extra_params is not None:
  784. ret['extra_params'] = extra_params
  785. for p in parameter_names:
  786. val = request.REQUEST.get(p)
  787. if val:
  788. ret[p] = val
  789. if request.method == 'POST':
  790. form = form_class(**data_extractor(request))
  791. ret['form'] = form
  792. if form.is_valid():
  793. args = arg_extractor(request, form, parameter_names)
  794. try:
  795. op(*args)
  796. except (IOError, WebHdfsException), e:
  797. msg = _("Cannot perform operation.")
  798. if request.user.is_superuser and not request.user == request.fs.superuser:
  799. msg += _(' Note: you are a Hue admin but not a HDFS superuser (which is "%(superuser)s").') \
  800. % {'superuser': request.fs.superuser}
  801. raise PopupException(msg, detail=e)
  802. if next:
  803. logging.debug("Next: %s" % next)
  804. # Doesn't need to be quoted: quoting is done by HttpResponseRedirect.
  805. return format_preserving_redirect(request, next)
  806. ret["success"] = True
  807. try:
  808. if piggyback:
  809. piggy_path = form.cleaned_data[piggyback]
  810. ret["result"] = _massage_stats(request, request.fs.stats(piggy_path))
  811. except Exception, e:
  812. # Hard to report these more naturally here. These happen either
  813. # because of a bug in the piggy-back code or because of a
  814. # race condition.
  815. logger.exception("Exception while processing piggyback data")
  816. ret["result_error"] = True
  817. ret['user'] = request.user
  818. return render(template, request, ret)
  819. else:
  820. # Initial parameters may be specified with get with the default extractor
  821. initial_values = initial_value_extractor(request, parameter_names)
  822. formset = form_class(initial=initial_values)
  823. ret['form'] = formset
  824. return render(template, request, ret)
  825. def rename(request):
  826. def smart_rename(src_path, dest_path):
  827. """If dest_path doesn't have a directory specified, use same dir."""
  828. if "#" in dest_path:
  829. raise PopupException(_("Could not rename folder \"%s\" to \"%s\": Hashes are not allowed in filenames." % (src_path, dest_path)))
  830. if "/" not in dest_path:
  831. src_dir = os.path.dirname(src_path)
  832. dest_path = os.path.join(src_dir, dest_path)
  833. request.fs.rename(src_path, dest_path)
  834. return generic_op(RenameForm, request, smart_rename, ["src_path", "dest_path"], None)
  835. def mkdir(request):
  836. def smart_mkdir(path, name):
  837. # Make sure only one directory is specified at a time.
  838. # No absolute directory specification allowed.
  839. if posixpath.sep in name or "#" in name:
  840. raise PopupException(_("Could not name folder \"%s\": Slashes or hashes are not allowed in filenames." % name))
  841. request.fs.mkdir(os.path.join(path, name))
  842. return generic_op(MkDirForm, request, smart_mkdir, ["path", "name"], "path")
  843. def touch(request):
  844. def smart_touch(path, name):
  845. # Make sure only the filename is specified.
  846. # No absolute path specification allowed.
  847. if posixpath.sep in name:
  848. raise PopupException(_("Could not name file \"%s\": Slashes are not allowed in filenames." % name))
  849. request.fs.create(os.path.join(path, name))
  850. return generic_op(TouchForm, request, smart_touch, ["path", "name"], "path")
  851. @require_http_methods(["POST"])
  852. def rmtree(request):
  853. recurring = []
  854. params = ["path"]
  855. def bulk_rmtree(*args, **kwargs):
  856. original = request.fs.setskiptrash('skip_trash' in request.GET)
  857. for arg in args:
  858. request.fs.do_as_user(request.user, request.fs.rmtree, arg['path'])
  859. request.fs.setskiptrash(original)
  860. return generic_op(RmTreeFormSet, request, bulk_rmtree, ["path"], None,
  861. data_extractor=formset_data_extractor(recurring, params),
  862. arg_extractor=formset_arg_extractor,
  863. initial_value_extractor=formset_initial_value_extractor)
  864. @require_http_methods(["POST"])
  865. def move(request):
  866. recurring = ['dest_path']
  867. params = ['src_path']
  868. def bulk_move(*args, **kwargs):
  869. for arg in args:
  870. request.fs.rename(arg['src_path'], arg['dest_path'])
  871. return generic_op(RenameFormSet, request, bulk_move, ["src_path", "dest_path"], None,
  872. data_extractor=formset_data_extractor(recurring, params),
  873. arg_extractor=formset_arg_extractor,
  874. initial_value_extractor=formset_initial_value_extractor)
  875. @require_http_methods(["POST"])
  876. def copy(request):
  877. recurring = ['dest_path']
  878. params = ['src_path']
  879. def bulk_copy(*args, **kwargs):
  880. for arg in args:
  881. request.fs.copy(arg['src_path'], arg['dest_path'], recursive=True, owner=request.user)
  882. return generic_op(CopyFormSet, request, bulk_copy, ["src_path", "dest_path"], None,
  883. data_extractor=formset_data_extractor(recurring, params),
  884. arg_extractor=formset_arg_extractor,
  885. initial_value_extractor=formset_initial_value_extractor)
  886. @require_http_methods(["POST"])
  887. def chmod(request):
  888. recurring = ["sticky", "user_read", "user_write", "user_execute", "group_read", "group_write", "group_execute", "other_read", "other_write", "other_execute"]
  889. params = ["path"]
  890. def bulk_chmod(*args, **kwargs):
  891. op = curry(request.fs.chmod, recursive=request.POST.get('recursive', False))
  892. for arg in args:
  893. op(arg['path'], arg['mode'])
  894. # mode here is abused: on input, it's a string, but when retrieved,
  895. # it's an int.
  896. return generic_op(ChmodFormSet, request, bulk_chmod, ['path', 'mode'], "path",
  897. data_extractor=formset_data_extractor(recurring, params),
  898. arg_extractor=formset_arg_extractor,
  899. initial_value_extractor=formset_initial_value_extractor)
  900. @require_http_methods(["POST"])
  901. def chown(request):
  902. # This is a bit clever: generic_op takes an argument (here, args), indicating
  903. # which POST parameters to pick out and pass to the given function.
  904. # We update that mapping based on whether or not the user selected "other".
  905. param_names = ["path", "user", "group"]
  906. if request.POST.get("user") == "__other__":
  907. param_names[1] = "user_other"
  908. if request.POST.get("group") == "__other__":
  909. param_names[2] = "group_other"
  910. recurring = ["user", "group", "user_other", "group_other"]
  911. params = ["path"]
  912. def bulk_chown(*args, **kwargs):
  913. op = curry(request.fs.chown, recursive=request.POST.get('recursive', False))
  914. for arg in args:
  915. varg = [arg[param] for param in param_names]
  916. op(*varg)
  917. return generic_op(ChownFormSet, request, bulk_chown, param_names, "path",
  918. data_extractor=formset_data_extractor(recurring, params),
  919. arg_extractor=formset_arg_extractor,
  920. initial_value_extractor=formset_initial_value_extractor)
  921. @require_http_methods(["POST"])
  922. def trash_restore(request):
  923. recurring = []
  924. params = ["path"]
  925. def bulk_restore(*args, **kwargs):
  926. for arg in args:
  927. request.fs.do_as_user(request.user, request.fs.restore, arg['path'])
  928. return generic_op(RestoreFormSet, request, bulk_restore, ["path"], None,
  929. data_extractor=formset_data_extractor(recurring, params),
  930. arg_extractor=formset_arg_extractor,
  931. initial_value_extractor=formset_initial_value_extractor)
  932. @require_http_methods(["POST"])
  933. def trash_purge(request):
  934. return generic_op(TrashPurgeForm, request, request.fs.purge_trash, [], None)
  935. def upload_file(request):
  936. """
  937. A wrapper around the actual upload view function to clean up the temporary file afterwards if it fails.
  938. Returns JSON.
  939. e.g. {'status' 0/1, data:'message'...}
  940. """
  941. response = {'status': -1, 'data': ''}
  942. if request.method == 'POST':
  943. try:
  944. resp = _upload_file(request)
  945. response.update(resp)
  946. except Exception, ex:
  947. response['data'] = str(ex)
  948. hdfs_file = request.FILES.get('hdfs_file')
  949. if hdfs_file:
  950. hdfs_file.remove()
  951. else:
  952. response['data'] = _('A POST request is required.')
  953. if response['status'] == 0:
  954. request.info(_('%(destination)s upload succeeded') % {'destination': response['path']})
  955. else:
  956. request.error(_('Upload failed: %(data)s') % {'data': response['data']})
  957. return HttpResponse(json.dumps(response), content_type="text/plain")
  958. def _upload_file(request):
  959. """
  960. Handles file uploaded by HDFSfileUploadHandler.
  961. The uploaded file is stored in HDFS at its destination with a .tmp suffix.
  962. We just need to rename it to the destination path.
  963. """
  964. form = UploadFileForm(request.POST, request.FILES)
  965. if form.is_valid():
  966. uploaded_file = request.FILES['hdfs_file']
  967. dest = form.cleaned_data['dest']
  968. if request.fs.isdir(dest) and posixpath.sep in uploaded_file.name:
  969. raise PopupException(_('Sorry, no "%(sep)s" in the filename %(name)s.' % {'sep': posixpath.sep, 'name': uploaded_file.name}))
  970. dest = request.fs.join(dest, uploaded_file.name)
  971. tmp_file = uploaded_file.get_temp_path()
  972. username = request.user.username
  973. try:
  974. # Remove tmp suffix of the file
  975. request.fs.do_as_user(username, request.fs.rename, tmp_file, dest)
  976. except IOError, ex:
  977. already_exists = False
  978. try:
  979. already_exists = request.fs.exists(dest)
  980. except Exception:
  981. pass
  982. if already_exists:
  983. msg = _('Destination %(name)s already exists.') % {'name': dest}
  984. else:
  985. msg = _('Copy to %(name)s failed: %(error)s') % {'name': dest, 'error': ex}
  986. raise PopupException(msg)
  987. return {
  988. 'status': 0,
  989. 'path': dest,
  990. 'result': _massage_stats(request, request.fs.stats(dest)),
  991. 'next': request.GET.get("next")
  992. }
  993. else:
  994. raise PopupException(_("Error in upload form: %s") % (form.errors,))
  995. def upload_archive(request):
  996. """
  997. A wrapper around the actual upload view function to clean up the temporary file afterwards.
  998. Returns JSON.
  999. e.g. {'status' 0/1, data:'message'...}
  1000. """
  1001. response = {'status': -1, 'data': ''}
  1002. if request.method == 'POST':
  1003. try:
  1004. try:
  1005. resp = _upload_archive(request)
  1006. response.update(resp)
  1007. except Exception, ex:
  1008. response['data'] = str(ex)
  1009. finally:
  1010. hdfs_file = request.FILES.get('hdfs_file')
  1011. if hdfs_file:
  1012. hdfs_file.remove()
  1013. else:
  1014. response['data'] = _('A POST request is required.')
  1015. if response['status'] == 0:
  1016. request.info(_('%(destination)s upload succeeded') % {'destination': response['path']})
  1017. else:
  1018. request.error(_('Upload failed: %(data)s') % {'data': response['data']})
  1019. return HttpResponse(json.dumps(response), content_type="text/plain")
  1020. def _upload_archive(request):
  1021. """
  1022. Handles archive upload.
  1023. The uploaded file is stored in memory.
  1024. We need to extract it and rename it.
  1025. """
  1026. form = UploadArchiveForm(request.POST, request.FILES)
  1027. if form.is_valid():
  1028. uploaded_file = request.FILES['archive']
  1029. # Always a dir
  1030. if request.fs.isdir(form.cleaned_data['dest']) and posixpath.sep in uploaded_file.name:
  1031. raise PopupException(_('Sorry, no "%(sep)s" in the filename %(name)s.' % {'sep': posixpath.sep, 'name': uploaded_file.name}))
  1032. dest = request.fs.join(form.cleaned_data['dest'], uploaded_file.name)
  1033. try:
  1034. # Extract if necessary
  1035. # Make sure dest path is without '.zip' extension
  1036. if dest.endswith('.zip'):
  1037. temp_path = archive_factory(uploaded_file).extract()
  1038. if not temp_path:
  1039. raise PopupException(_('Could not extract contents of file.'))
  1040. # Move the file to where it belongs
  1041. dest = dest[:-4]
  1042. request.fs.copyFromLocal(temp_path, dest)
  1043. shutil.rmtree(temp_path)
  1044. else:
  1045. raise PopupException(_('Could not interpret archive type.'))
  1046. except IOError, ex:
  1047. already_exists = False
  1048. try:
  1049. already_exists = request.fs.exists(dest)
  1050. except Exception:
  1051. pass
  1052. if already_exists:
  1053. msg = _('Destination %(name)s already exists.') % {'name': dest}
  1054. else:
  1055. msg = _('Copy to %(name)s failed: %(error)s') % {'name': dest, 'error': ex}
  1056. raise PopupException(msg)
  1057. return {
  1058. 'status': 0,
  1059. 'path': dest,
  1060. 'result': _massage_stats(request, request.fs.stats(dest)),
  1061. 'next': request.GET.get("next")
  1062. }
  1063. else:
  1064. raise PopupException(_("Error in upload form: %s") % (form.errors,))
  1065. def status(request):
  1066. status = request.fs.status()
  1067. data = {
  1068. # Beware: "messages" is special in the context browser.
  1069. 'msgs': status.get_messages(),
  1070. 'health': status.get_health(),
  1071. 'datanode_report': status.get_datanode_report(),
  1072. 'name': request.fs.name
  1073. }
  1074. return render("status.mako", request, data)
  1075. def location_to_url(location, strict=True):
  1076. """
  1077. If possible, returns a file browser URL to the location.
  1078. Location is a URI, if strict is True.
  1079. Python doesn't seem to have a readily-available URI-comparison
  1080. library, so this is quite hacky.
  1081. """
  1082. if location is None:
  1083. return None
  1084. split_path = Hdfs.urlsplit(location)
  1085. if strict and not split_path[1]:
  1086. # No netloc, not full url
  1087. return None
  1088. return urlresolvers.reverse("filebrowser.views.view", kwargs=dict(path=split_path[2]))
  1089. def truncate(toTruncate, charsToKeep=50):
  1090. """
  1091. Returns a string truncated to 'charsToKeep' length plus ellipses.
  1092. """
  1093. if len(toTruncate) > charsToKeep:
  1094. truncated = toTruncate[:charsToKeep] + "..."
  1095. return truncated
  1096. else:
  1097. return toTruncate