models.py 77 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. from __future__ import absolute_import
  18. from future import standard_library
  19. standard_library.install_aliases()
  20. from builtins import next
  21. from builtins import object
  22. import calendar
  23. import json
  24. import logging
  25. import os
  26. import sys
  27. import uuid
  28. from collections import OrderedDict
  29. from itertools import chain
  30. from django.db import connection, models, transaction
  31. from django.db.models import Q
  32. from django.db.models.query import QuerySet
  33. from django.contrib.auth.validators import UnicodeUsernameValidator
  34. from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey
  35. from django.contrib.contenttypes.models import ContentType
  36. from django.contrib.staticfiles.storage import staticfiles_storage
  37. from django.urls import reverse, NoReverseMatch
  38. from django.utils.translation import ugettext as _, ugettext_lazy as _t
  39. from dashboard.conf import get_engines, HAS_REPORT_ENABLED
  40. from kafka.conf import has_kafka
  41. from metadata.conf import get_optimizer_mode
  42. from notebook.conf import DEFAULT_LIMIT, SHOW_NOTEBOOKS, get_ordered_interpreters
  43. from useradmin.models import User, Group, get_organization
  44. from useradmin.organization import _fitered_queryset
  45. from desktop import appmanager
  46. from desktop.auth.backend import is_admin
  47. from desktop.conf import get_clusters, CLUSTER_ID, IS_MULTICLUSTER_ONLY, IS_K8S_ONLY, ENABLE_ORGANIZATIONS, ENABLE_PROMETHEUS,\
  48. has_connectors, TASK_SERVER, ENABLE_GIST, APP_BLACKLIST, ENABLE_SHARING
  49. from desktop.lib import fsmanager
  50. from desktop.lib.connectors.api import _get_installed_connectors
  51. from desktop.lib.connectors.models import Connector
  52. from desktop.lib.i18n import force_unicode
  53. from desktop.lib.exceptions_renderable import PopupException
  54. from desktop.lib.paths import get_run_root, SAFE_CHARACTERS_URI_COMPONENTS
  55. from desktop.redaction import global_redaction_engine
  56. from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH, HUE_DESKTOP_VERSION
  57. from filebrowser.conf import REMOTE_STORAGE_HOME
  58. if sys.version_info[0] > 2:
  59. from urllib.parse import quote as urllib_quote
  60. else:
  61. from urllib import quote as urllib_quote
  62. LOG = logging.getLogger(__name__)
  63. SAMPLE_USER_ID = 1100713
  64. SAMPLE_USER_INSTALL = 'hue'
  65. SAMPLE_USER_OWNERS = ['hue', 'sample']
  66. UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
  67. HUE_VERSION = None
  68. def uuid_default():
  69. return str(uuid.uuid4())
  70. def hue_version():
  71. global HUE_VERSION
  72. if HUE_VERSION is None:
  73. HUE_VERSION = HUE_DESKTOP_VERSION
  74. p = get_run_root('cloudera', 'cdh_version.properties')
  75. if os.path.exists(p):
  76. build_version = _version_from_properties(open(p))
  77. if build_version:
  78. HUE_VERSION = '%s - %s' % (HUE_VERSION, build_version)
  79. return HUE_VERSION
  80. def _version_from_properties(f):
  81. return dict(line.strip().split('=') for line in f.readlines() if len(line.strip().split('=')) == 2).get('cloudera.cdh.release')
  82. def get_sample_user_install(user):
  83. if ENABLE_ORGANIZATIONS.get():
  84. organization = get_organization(email=user.email)
  85. if organization.is_multi_user:
  86. return SAMPLE_USER_INSTALL + '@' + organization.domain
  87. else:
  88. return organization.domain
  89. else:
  90. return SAMPLE_USER_INSTALL
  91. ###################################################################################################
  92. # Custom Settings
  93. ###################################################################################################
  94. PREFERENCE_IS_WELCOME_TOUR_SEEN = 'is_welcome_tour_seen'
  95. if not ENABLE_ORGANIZATIONS.get():
  96. class HueUser(User):
  97. class Meta(object):
  98. proxy = True
  99. def __init__(self, *args, **kwargs):
  100. self._meta.get_field(
  101. 'username'
  102. ).validators[0] = UnicodeUsernameValidator()
  103. super(User, self).__init__(*args, **kwargs)
  104. class UserPreferences(models.Model):
  105. """
  106. Holds arbitrary key/value strings.
  107. Note: ideally user/jkeu should be unique together.
  108. """
  109. user = models.ForeignKey(User, on_delete=models.CASCADE)
  110. key = models.CharField(max_length=20)
  111. value = models.TextField(max_length=4096)
  112. class Settings(models.Model):
  113. collect_usage = models.BooleanField(db_index=True, default=True)
  114. tours_and_tutorials = models.BooleanField(db_index=True, default=True)
  115. @classmethod
  116. def get_settings(cls):
  117. settings, created = Settings.objects.get_or_create(id=1)
  118. return settings
  119. class DefaultConfigurationManager(models.Manager):
  120. def get_configuration_for_user(self, app, user):
  121. """
  122. :param app: app name
  123. :param user: User object
  124. :return: DefaultConfiguration for user, or first group found, or default for app, or None
  125. """
  126. try:
  127. return super(DefaultConfigurationManager, self).get(app=app, user=user)
  128. except DefaultConfiguration.DoesNotExist:
  129. pass
  130. configs = super(DefaultConfigurationManager, self).get_queryset().filter(app=app, groups__in=user.groups.all())
  131. if configs.count() > 0:
  132. return configs[0]
  133. try:
  134. return super(DefaultConfigurationManager, self).get(app=app, is_default=True)
  135. except DefaultConfiguration.DoesNotExist:
  136. pass
  137. return None
  138. class DefaultConfiguration(models.Model):
  139. """
  140. Default values for configuration properties for a given app/editor
  141. Can be designated as default for all users by is_default flag, or for a specific group or user
  142. """
  143. app = models.CharField(max_length=32, null=False, db_index=True, help_text=_t('App that this configuration belongs to.'))
  144. properties = models.TextField(default='[]', help_text=_t('JSON-formatted default properties values.'))
  145. is_default = models.BooleanField(default=False, db_index=True)
  146. groups = models.ManyToManyField(Group, db_index=True, db_table='defaultconfiguration_groups')
  147. user = models.ForeignKey(User, on_delete=models.CASCADE, blank=True, null=True, db_index=True)
  148. objects = DefaultConfigurationManager()
  149. class Meta(object):
  150. ordering = ["app", "-is_default", "user"]
  151. @property
  152. def properties_list(self):
  153. """
  154. :return: Deserialized properties as a list of property objects
  155. """
  156. if not self.properties:
  157. self.properties = json.dumps([])
  158. return json.loads(self.properties)
  159. @property
  160. def properties_dict(self):
  161. """
  162. :return: Deserialized properties as a dict of key: value pairs
  163. """
  164. if not self.properties:
  165. self.properties = json.dumps([])
  166. return dict((prop['key'], prop['value']) for prop in json.loads(self.properties))
  167. def to_dict(self):
  168. return {
  169. 'id': self.id,
  170. 'app': self.app,
  171. 'properties': self.properties_list,
  172. 'is_default': self.is_default,
  173. 'group_ids': [group.id for group in self.groups.all()],
  174. 'user': self.user.username if self.user else None
  175. }
  176. ###################################################################################################
  177. # Document1
  178. ###################################################################################################
  179. class DocumentTagManager(models.Manager):
  180. def get_tags(self, user):
  181. return self.filter(owner=user).distinct()
  182. def create_tag(self, owner, tag_name):
  183. if tag_name in DocumentTag.RESERVED:
  184. raise Exception(_("Can't add %s: it is a reserved tag.") % tag_name)
  185. else:
  186. tag, created = DocumentTag.objects.get_or_create(tag=tag_name, owner=owner)
  187. return tag
  188. def _get_tag(self, user, name):
  189. tag, created = DocumentTag.objects.get_or_create(owner=user, tag=name)
  190. return tag
  191. def get_default_tag(self, user):
  192. return self._get_tag(user, DocumentTag.DEFAULT)
  193. def get_trash_tag(self, user):
  194. return self._get_tag(user, DocumentTag.TRASH)
  195. def get_history_tag(self, user):
  196. return self._get_tag(user, DocumentTag.HISTORY)
  197. def get_example_tag(self, user):
  198. return self._get_tag(user, DocumentTag.EXAMPLE)
  199. def get_imported2_tag(self, user):
  200. return self._get_tag(user, DocumentTag.IMPORTED2)
  201. def tag(self, owner, doc_id, tag_name='', tag_id=None):
  202. try:
  203. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  204. if tag.tag in DocumentTag.RESERVED:
  205. raise Exception(_("Can't add %s: it is a reserved tag.") % tag)
  206. except DocumentTag.DoesNotExist:
  207. tag = self._get_tag(user=owner, name=tag_name)
  208. doc = Document.objects.get_doc_for_writing(doc_id, owner)
  209. doc.add_tag(tag)
  210. return tag
  211. def untag(self, tag_id, owner, doc_id):
  212. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  213. if tag.tag in DocumentTag.RESERVED:
  214. raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
  215. doc = Document.objects.get_doc_for_writing(doc_id, owner=owner)
  216. doc.remove_tag(tag)
  217. def delete_tag(self, tag_id, owner):
  218. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  219. default_tag = DocumentTag.objects.get_default_tag(owner)
  220. if tag.tag in DocumentTag.RESERVED:
  221. raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
  222. else:
  223. tag.delete()
  224. for doc in Document.objects.get_docs(owner).filter(tags=None):
  225. doc.add_tag(default_tag)
  226. def update_tags(self, owner, doc_id, tag_ids):
  227. doc = Document.objects.get_doc_for_writing(doc_id, owner)
  228. for tag in doc.tags.all():
  229. if tag.tag not in DocumentTag.RESERVED:
  230. doc.remove_tag(tag)
  231. for tag_id in tag_ids:
  232. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  233. if tag.tag not in DocumentTag.RESERVED:
  234. doc.add_tag(tag)
  235. return doc
  236. class DocumentTag(models.Model):
  237. """
  238. Reserved tags can't be manually removed by the user.
  239. """
  240. owner = models.ForeignKey(User, on_delete=models.CASCADE, db_index=True)
  241. tag = models.SlugField()
  242. DEFAULT = 'default' # Always there
  243. TRASH = 'trash' # There when the document is trashed
  244. HISTORY = 'history' # There when the document is a submission history
  245. EXAMPLE = 'example' # Hue examples
  246. IMPORTED2 = 'imported2' # Was imported to document2
  247. RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE, IMPORTED2)
  248. objects = DocumentTagManager()
  249. class Meta(object):
  250. unique_together = ('owner', 'tag')
  251. def __unicode__(self):
  252. return force_unicode('%s') % (self.tag,)
  253. class DocumentManager(models.Manager):
  254. def documents(self, user):
  255. return Document.objects.filter(
  256. Q(owner=user) |
  257. Q(documentpermission__users=user) |
  258. Q(documentpermission__groups__in=user.groups.all())
  259. ).defer('description', 'extra').distinct()
  260. def get_docs(self, user, model_class=None, extra=None, qfilter=None):
  261. docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
  262. if model_class is not None:
  263. ct = ContentType.objects.get_for_model(model_class)
  264. docs = docs.filter(content_type=ct)
  265. if extra is not None:
  266. docs = docs.filter(extra=extra)
  267. if qfilter is not None:
  268. docs = docs.filter(qfilter)
  269. return docs
  270. def get_doc_for_writing(self, doc_id, user):
  271. """Fetch a document and confirm that this user can write to it."""
  272. doc = Document.objects.documents(user).get(id=doc_id)
  273. doc.can_write_or_exception(user)
  274. return doc
  275. def trashed_docs(self, model_class, user):
  276. tag = DocumentTag.objects.get_trash_tag(user=user)
  277. return Document.objects.get_docs(user, model_class).filter(tags__in=[tag]).order_by('-last_modified')
  278. def trashed(self, model_class, user):
  279. docs = self.trashed_docs(model_class, user)
  280. return [job.content_object for job in docs if job.content_object]
  281. def available_docs(self, model_class, user, with_history=False):
  282. exclude = [DocumentTag.objects.get_trash_tag(user=user)]
  283. if not with_history:
  284. exclude.append(DocumentTag.objects.get_history_tag(user=user))
  285. return Document.objects.get_docs(user, model_class).exclude(tags__in=exclude).order_by('-last_modified')
  286. def history_docs(self, model_class, user):
  287. include = [DocumentTag.objects.get_history_tag(user=user)]
  288. exclude = [DocumentTag.objects.get_trash_tag(user=user)]
  289. return Document.objects.get_docs(user, model_class).filter(tags__in=include).exclude(tags__in=exclude).order_by('-last_modified')
  290. def available(self, model_class, user, with_history=False):
  291. docs = self.available_docs(model_class, user, with_history)
  292. return [doc.content_object for doc in docs if doc.content_object]
  293. def can_read_or_exception(self, user, doc_class, doc_id, exception_class=PopupException):
  294. if doc_id is None:
  295. return
  296. try:
  297. ct = ContentType.objects.get_for_model(doc_class)
  298. doc = Document.objects.get(object_id=doc_id, content_type=ct)
  299. if doc.can_read(user):
  300. return doc
  301. else:
  302. message = _("Permission denied. %(username)s does not have the permissions required to access document %(id)s") % \
  303. {'username': user.username, 'id': doc.id}
  304. raise exception_class(message)
  305. except Document.DoesNotExist:
  306. raise exception_class(_('Document %(id)s does not exist') % {'id': doc_id})
  307. def can_read(self, user, doc_class, doc_id):
  308. ct = ContentType.objects.get_for_model(doc_class)
  309. doc = Document.objects.get(object_id=doc_id, content_type=ct)
  310. return doc.can_read(user)
  311. def link(self, content_object, owner, name='', description='', extra=''):
  312. if not content_object.doc.exists():
  313. doc = Document.objects.create(
  314. content_object=content_object,
  315. owner=owner,
  316. name=name,
  317. description=description,
  318. extra=extra
  319. )
  320. tag = DocumentTag.objects.get_default_tag(user=owner)
  321. doc.tags.add(tag)
  322. return doc
  323. else:
  324. LOG.warn('Object %s already has documents: %s' % (content_object, content_object.doc.all()))
  325. return content_object.doc.all()[0]
  326. def sync(self, doc2_only=True):
  327. def find_jobs_with_no_doc(model):
  328. jobs = model.objects.filter(doc__isnull=True)
  329. if model == Document2:
  330. jobs = jobs.exclude(is_history=True)
  331. return jobs.select_related('owner')
  332. def find_oozie_jobs_with_no_doc(model):
  333. return model.objects.filter(doc__isnull=True).exclude(name__exact='').select_related('owner')
  334. table_names = connection.introspection.table_names()
  335. try:
  336. from oozie.models import Workflow, Coordinator, Bundle
  337. if \
  338. Workflow._meta.db_table in table_names or \
  339. Coordinator._meta.db_table in table_names or \
  340. Bundle._meta.db_table in table_names:
  341. with transaction.atomic():
  342. for job in chain(
  343. find_oozie_jobs_with_no_doc(Workflow),
  344. find_oozie_jobs_with_no_doc(Coordinator),
  345. find_oozie_jobs_with_no_doc(Bundle)):
  346. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description)
  347. if job.is_trashed:
  348. doc.send_to_trash()
  349. if job.is_shared:
  350. doc.share_to_default()
  351. if hasattr(job, 'managed'):
  352. if not job.managed:
  353. doc.extra = 'jobsub'
  354. doc.save()
  355. except Exception as e:
  356. LOG.exception('error syncing oozie')
  357. try:
  358. from beeswax.models import SavedQuery
  359. if SavedQuery._meta.db_table in table_names:
  360. with transaction.atomic():
  361. for job in find_jobs_with_no_doc(SavedQuery):
  362. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
  363. if job.is_trashed:
  364. doc.send_to_trash()
  365. except Exception as e:
  366. LOG.exception('error syncing beeswax')
  367. try:
  368. from pig.models import PigScript
  369. if PigScript._meta.db_table in table_names:
  370. with transaction.atomic():
  371. for job in find_jobs_with_no_doc(PigScript):
  372. Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
  373. except Exception as e:
  374. LOG.exception('error syncing pig')
  375. try:
  376. from search.models import Collection
  377. if Collection._meta.db_table in table_names:
  378. with transaction.atomic():
  379. for dashboard in Collection.objects.all():
  380. if 'collection' in dashboard.properties_dict:
  381. col_dict = dashboard.properties_dict['collection']
  382. if not 'uuid' in col_dict:
  383. _uuid = str(uuid.uuid4())
  384. col_dict['uuid'] = _uuid
  385. dashboard.update_properties({'collection': col_dict})
  386. if dashboard.owner is None:
  387. from useradmin.models import install_sample_user
  388. owner = install_sample_user()
  389. else:
  390. owner = dashboard.owner
  391. dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard',
  392. owner=owner, description=dashboard.label, data=dashboard.properties)
  393. Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label,
  394. extra='search-dashboard')
  395. dashboard.save()
  396. except Exception as e:
  397. LOG.exception('error syncing search')
  398. try:
  399. if Document2._meta.db_table in table_names:
  400. with transaction.atomic():
  401. for job in find_jobs_with_no_doc(Document2):
  402. if job.type == 'oozie-workflow2':
  403. extra = 'workflow2'
  404. elif job.type == 'oozie-coordinator2':
  405. extra = 'coordinator2'
  406. elif job.type == 'oozie-bundle2':
  407. extra = 'bundle2'
  408. elif job.type == 'notebook':
  409. extra = 'notebook'
  410. elif job.type == 'search-dashboard':
  411. extra = 'search-dashboard'
  412. else:
  413. extra = ''
  414. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description, extra=extra)
  415. except Exception as e:
  416. LOG.exception('error syncing Document2')
  417. if not doc2_only and Document._meta.db_table in table_names:
  418. # Make sure doc have at least a tag
  419. try:
  420. for doc in Document.objects.filter(tags=None):
  421. default_tag = DocumentTag.objects.get_default_tag(doc.owner)
  422. doc.tags.add(default_tag)
  423. except Exception as e:
  424. LOG.exception('error adding at least one tag to docs')
  425. # Make sure all the sample user documents are shared.
  426. try:
  427. with transaction.atomic():
  428. for doc in Document.objects.filter(owner__username__in=SAMPLE_USER_OWNERS):
  429. doc.share_to_default()
  430. tag = DocumentTag.objects.get_example_tag(user=doc.owner)
  431. doc.tags.add(tag)
  432. doc_last_modified = doc.last_modified
  433. doc.save()
  434. Document.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
  435. except Exception as e:
  436. LOG.exception('error sharing sample user documents')
  437. # For now remove the default tag from the examples
  438. try:
  439. for doc in Document.objects.filter(tags__tag=DocumentTag.EXAMPLE):
  440. default_tag = DocumentTag.objects.get_default_tag(doc.owner)
  441. doc.tags.remove(default_tag)
  442. except Exception as e:
  443. LOG.exception('error removing default tags')
  444. # ------------------------------------------------------------------------
  445. LOG.info('Looking for documents that have no object')
  446. # Delete documents with no object.
  447. try:
  448. with transaction.atomic():
  449. # First, delete all the documents that don't have a content type
  450. docs = Document.objects.filter(content_type=None)
  451. if docs:
  452. LOG.info('Deleting %s doc(s) that do not have a content type' % docs.count())
  453. docs.delete()
  454. # Next, it's possible that there are documents pointing at a non-existing
  455. # content_type. We need to do a left join to find these records, but we
  456. # can't do this directly in django. To get around writing wrap sql (which
  457. # might not be portable), we'll use an aggregate to count up all the
  458. # associated content_types, and delete the documents that have a count of
  459. # zero.
  460. #
  461. # Note we're counting `content_type__name` to force the join.
  462. docs = Document.objects \
  463. .values('id') \
  464. .annotate(content_type_count=models.Count('content_type__name')) \
  465. .filter(content_type_count=0)
  466. if docs:
  467. LOG.info('Deleting %s doc(s) that have invalid content types' % docs.count())
  468. docs.delete()
  469. # Finally we need to delete documents with no associated content object.
  470. # This is tricky because of our use of generic foreign keys. So to do
  471. # this a bit more efficiently, we'll start with a query of all the
  472. # documents, then step through each content type and and filter out all
  473. # the documents it's referencing from our document query. Messy, but it
  474. # works.
  475. # TODO: This can be several 100k entries for large databases.
  476. # Need to figure out a better way to handle this scenario.
  477. docs = Document.objects.all()
  478. for content_type in ContentType.objects.all():
  479. model_class = content_type.model_class()
  480. # Ignore any types that don't have a model.
  481. if model_class is None:
  482. continue
  483. # Ignore types that don't have a table yet.
  484. if model_class._meta.db_table not in table_names:
  485. continue
  486. # Ignore classes that don't have a 'doc'.
  487. if not hasattr(model_class, 'doc'):
  488. continue
  489. # First create a query that grabs all the document ids for this type.
  490. docs_from_content = model_class.objects.values('doc__id')
  491. # Next, filter these from our document query.
  492. docs = docs.exclude(id__in=docs_from_content)
  493. if docs.exists():
  494. LOG.info('Deleting %s documents' % docs.count())
  495. docs.delete()
  496. except Exception as e:
  497. LOG.exception('Error in sync while attempting to delete documents with no object: %s' % e)
  498. class Document(models.Model):
  499. owner = models.ForeignKey(User, on_delete=models.CASCADE, db_index=True, verbose_name=_t('Owner'),
  500. help_text=_t('User who can own the job.'), related_name='doc_owner')
  501. name = models.CharField(default='', max_length=255)
  502. description = models.TextField(default='')
  503. last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Last modified'))
  504. version = models.SmallIntegerField(default=1, verbose_name=_t('Schema version'))
  505. extra = models.TextField(default='')
  506. tags = models.ManyToManyField(DocumentTag, db_index=True)
  507. content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
  508. object_id = models.PositiveIntegerField()
  509. content_object = GenericForeignKey('content_type', 'object_id')
  510. objects = DocumentManager()
  511. class Meta(object):
  512. unique_together = ('content_type', 'object_id')
  513. def __unicode__(self):
  514. return force_unicode('%s %s %s') % (self.content_type, self.name, self.owner)
  515. def is_editable(self, user):
  516. """Deprecated by can_read"""
  517. return self.can_write(user)
  518. def can_edit_or_exception(self, user, exception_class=PopupException):
  519. """Deprecated by can_write_or_exception"""
  520. return self.can_write_or_exception(user, exception_class)
  521. def add_tag(self, tag):
  522. self.tags.add(tag)
  523. def remove_tag(self, tag):
  524. self.tags.remove(tag)
  525. def is_trashed(self):
  526. return DocumentTag.objects.get_trash_tag(user=self.owner) in self.tags.all()
  527. def is_historic(self):
  528. return DocumentTag.objects.get_history_tag(user=self.owner) in self.tags.all()
  529. def send_to_trash(self):
  530. tag = DocumentTag.objects.get_trash_tag(user=self.owner)
  531. self.tags.add(tag)
  532. def restore_from_trash(self):
  533. tag = DocumentTag.objects.get_trash_tag(user=self.owner)
  534. self.tags.remove(tag)
  535. def add_to_history(self):
  536. tag = DocumentTag.objects.get_history_tag(user=self.owner)
  537. self.tags.add(tag)
  538. def remove_from_history(self):
  539. tag = DocumentTag.objects.get_history_tag(user=self.owner)
  540. self.tags.remove(tag)
  541. def share_to_default(self, name='read'):
  542. DocumentPermission.objects.share_to_default(self, name=name)
  543. def can_read(self, user):
  544. return is_admin(user) or self.owner == user or Document.objects.get_docs(user).filter(id=self.id).exists()
  545. def can_write(self, user):
  546. perm = self.list_permissions('write')
  547. return is_admin(user) or self.owner == user or perm.groups.filter(id__in=user.groups.all()).exists() or user in perm.users.all()
  548. def can_read_or_exception(self, user, exception_class=PopupException):
  549. if self.can_read(user):
  550. return True
  551. else:
  552. raise exception_class(_("Document does not exist or you don't have the permission to access it."))
  553. def can_write_or_exception(self, user, exception_class=PopupException):
  554. if self.can_write(user):
  555. return True
  556. else:
  557. raise exception_class(_("Document does not exist or you don't have the permission to access it."))
  558. def copy(self, content_object, name, owner, description=None):
  559. if content_object:
  560. copy_doc = self
  561. copy_doc.pk = None
  562. copy_doc.id = None
  563. copy_doc.name = name
  564. copy_doc.owner = owner
  565. if description:
  566. copy_doc.description = description
  567. copy_doc = Document.objects.link(content_object,
  568. owner=copy_doc.owner,
  569. name=copy_doc.name,
  570. description=copy_doc.description,
  571. extra=copy_doc.extra)
  572. # Update reverse Document relation to new copy
  573. if content_object.doc.get():
  574. content_object.doc.get().delete()
  575. content_object.doc.add(copy_doc)
  576. copy_doc.save()
  577. return copy_doc
  578. else:
  579. raise PopupException(_("Document copy method requires a content_object argument."))
  580. @property
  581. def icon(self):
  582. apps = appmanager.get_apps_dict()
  583. try:
  584. if self.extra == 'workflow2':
  585. return staticfiles_storage.url('oozie/art/icon_oozie_workflow_48.png')
  586. elif self.extra == 'coordinator2':
  587. return staticfiles_storage.url('oozie/art/icon_oozie_coordinator_48.png')
  588. elif self.extra == 'bundle2':
  589. return staticfiles_storage.url('oozie/art/icon_oozie_bundle_48.png')
  590. elif self.extra == 'notebook':
  591. return staticfiles_storage.url('notebook/art/icon_notebook_48.png')
  592. elif self.extra.startswith('query'):
  593. if self.extra == 'query-impala':
  594. return staticfiles_storage.url(apps['impala'].icon_path)
  595. else:
  596. return staticfiles_storage.url(apps['beeswax'].icon_path)
  597. elif self.extra.startswith('search'):
  598. return staticfiles_storage.url('search/art/icon_search_48.png')
  599. elif self.content_type.app_label == 'beeswax':
  600. if self.extra == '0':
  601. return staticfiles_storage.url(apps['beeswax'].icon_path)
  602. elif self.extra == '3':
  603. return staticfiles_storage.url(apps['spark'].icon_path)
  604. else:
  605. return staticfiles_storage.url(apps['impala'].icon_path)
  606. elif self.content_type.app_label == 'oozie':
  607. if self.extra == 'jobsub':
  608. return staticfiles_storage.url(apps['jobsub'].icon_path)
  609. else:
  610. return staticfiles_storage.url(self.content_type.model_class().ICON)
  611. elif self.content_type.app_label in apps:
  612. return staticfiles_storage.url(apps[self.content_type.app_label].icon_path)
  613. else:
  614. return staticfiles_storage.url('desktop/art/icon_hue_48.png')
  615. except Exception as e:
  616. LOG.warn(force_unicode(e))
  617. return staticfiles_storage.url('desktop/art/icon_hue_48.png')
  618. def share(self, users, groups, name='read'):
  619. DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=True)
  620. def unshare(self, users, groups, name='read'):
  621. DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=False)
  622. def sync_permissions(self, perms_dict):
  623. """
  624. Set who else or which other group can interact with the document.
  625. Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
  626. """
  627. for name, perm in perms_dict.items():
  628. users = groups = None
  629. if perm.get('user_ids'):
  630. users = User.objects.in_bulk(perm.get('user_ids'))
  631. else:
  632. users = []
  633. if perm.get('group_ids'):
  634. groups = Group.objects.in_bulk(perm.get('group_ids'))
  635. else:
  636. groups = []
  637. DocumentPermission.objects.sync(document=self, name=name, users=users, groups=groups)
  638. def list_permissions(self, perm='read'):
  639. return DocumentPermission.objects.list(document=self, perm=perm)
  640. def to_dict(self):
  641. return {
  642. 'owner': self.owner.username,
  643. 'name': self.name,
  644. 'description': self.description,
  645. 'uuid': None, # no uuid == v1
  646. 'id': self.id,
  647. 'doc1_id': self.id,
  648. 'object_id': self.object_id,
  649. 'type': str(self.content_type),
  650. 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT),
  651. 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()),
  652. 'isSelected': False
  653. }
  654. class DocumentPermissionManager(models.Manager):
  655. def _check_perm(self, name):
  656. perms = (DocumentPermission.READ_PERM, DocumentPermission.WRITE_PERM)
  657. if name not in perms:
  658. perms_string = ' and '.join(', '.join(perms).rsplit(', ', 1))
  659. raise PopupException(_('Only %s permissions are supported, not %s.') % (perms_string, name))
  660. def share_to_default(self, document, name='read'):
  661. from useradmin.models import get_default_user_group # Remove build dependency
  662. self._check_perm(name)
  663. if name == DocumentPermission.WRITE_PERM:
  664. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.WRITE_PERM)
  665. else:
  666. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.READ_PERM)
  667. default_group = get_default_user_group()
  668. if default_group:
  669. perm.groups.add(default_group)
  670. def update(self, document, name='read', users=None, groups=None, add=True):
  671. self._check_perm(name)
  672. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
  673. if users is not None:
  674. if add:
  675. perm.users.add(*users)
  676. else:
  677. perm.users.remove(*users)
  678. if groups is not None:
  679. if add:
  680. perm.groups.add(*groups)
  681. else:
  682. perm.groups.remove(*groups)
  683. if not perm.users and not perm.groups:
  684. perm.delete()
  685. def sync(self, document, name='read', users=None, groups=None):
  686. self._check_perm(name)
  687. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
  688. if users is not None:
  689. perm.users = []
  690. perm.users = users
  691. perm.save()
  692. if groups is not None:
  693. perm.groups = []
  694. perm.groups = groups
  695. perm.save()
  696. if not users and not groups:
  697. perm.delete()
  698. def list(self, document, perm='read'):
  699. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=perm)
  700. return perm
  701. class DocumentPermission(models.Model):
  702. READ_PERM = 'read'
  703. WRITE_PERM = 'write'
  704. doc = models.ForeignKey(Document, on_delete=models.CASCADE)
  705. users = models.ManyToManyField(User, db_index=True, db_table='documentpermission_users')
  706. groups = models.ManyToManyField(Group, db_index=True, db_table='documentpermission_groups')
  707. perms = models.CharField(default=READ_PERM, max_length=10, choices=( # one perm
  708. (READ_PERM, 'read'),
  709. (WRITE_PERM, 'write'),
  710. ))
  711. objects = DocumentPermissionManager()
  712. class Meta(object):
  713. unique_together = ('doc', 'perms')
  714. ###################################################################################################
  715. # Document2
  716. ###################################################################################################
  717. class FilesystemException(Exception):
  718. pass
  719. class Document2QueryMixin(object):
  720. def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False, include_shared_links=False):
  721. """
  722. Returns all documents that are owned or shared with the user.
  723. :param perms: both, shared, owned. Defaults to both.
  724. :param include_history: boolean flag to return history documents. Defaults to False.
  725. :param include_trashed: boolean flag to return trashed documents. Defaults to True.
  726. :param include_managed: boolean flag to return documents generated by Hue. Defaults to False.
  727. :param include_shared_links: boolean flag to include documents shared by links. Defaults to False.
  728. """
  729. if perms == 'both':
  730. docs = self.filter(
  731. Q(owner=user) |
  732. Q(document2permission__users=user) |
  733. Q(document2permission__groups__in=user.groups.all()) |
  734. (Q(document2permission__is_link_on=include_shared_links) & Q(document2permission__is_link_on=True))
  735. )
  736. elif perms == 'shared':
  737. docs = self.filter(
  738. Q(document2permission__users=user) |
  739. Q(document2permission__groups__in=user.groups.all()) |
  740. (Q(document2permission__is_link_on=include_shared_links) & Q(document2permission__is_link_on=True))
  741. )
  742. else: # Only return documents owned by the user
  743. docs = self.filter(owner=user)
  744. if not include_history:
  745. docs = docs.exclude(is_history=True)
  746. if not include_managed:
  747. docs = docs.exclude(is_managed=True)
  748. if not include_trashed:
  749. docs = docs.exclude(is_trashed=True)
  750. return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
  751. def search_documents(self, types=None, search_text=None, order_by=None):
  752. """
  753. Search for documents based on type filters, search_text or order_by and return a queryset of document objects
  754. :param types: list of Document2 types (e.g. - query-hive, directory, etc)
  755. :param search_text: text to search on in the name and description fields
  756. :param order_by: order by field (e.g. -last_modified, type)
  757. """
  758. documents = self
  759. if types and isinstance(types, list) and types[0] != 'all':
  760. documents = documents.filter(type__in=types)
  761. if search_text:
  762. documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) |
  763. Q(search__icontains=search_text))
  764. if order_by: # TODO: Validate that order_by is a valid sort parameter
  765. documents = documents.order_by(order_by)
  766. return documents
  767. class Document2QuerySet(QuerySet, Document2QueryMixin):
  768. pass
  769. class Document2Manager(models.Manager, Document2QueryMixin):
  770. def get_queryset(self):
  771. """Make sure to restrict to only organization's documents"""
  772. return _fitered_queryset(
  773. Document2QuerySet(self.model, using=self._db),
  774. by_owner=True
  775. )
  776. # TODO prevent get() in favor of this
  777. def document(self, user, doc_id):
  778. return self.documents(user, include_trashed=True, include_history=True, include_shared_links=True).get(id=doc_id)
  779. def get_by_natural_key(self, uuid, version, is_history):
  780. return self.get(uuid=uuid, version=version, is_history=is_history)
  781. def get_by_uuid(self, user, uuid, perm_type='read'):
  782. """
  783. Since UUID is not a unique field, but part of a composite unique key, this returns the latest version by UUID.
  784. This should always be used in place of Document2.objects.get(uuid=) when a single document is expected.
  785. :param user: User to check permissions against
  786. :param uuid
  787. :param perm_type: permission type to check against
  788. """
  789. docs = self.filter(uuid=uuid).order_by('-last_modified')
  790. if not docs.exists():
  791. raise FilesystemException(_('Document with UUID %s not found.') % uuid)
  792. latest_doc = docs[0]
  793. if perm_type == 'write':
  794. latest_doc.can_write_or_exception(user)
  795. else:
  796. latest_doc.can_read_or_exception(user)
  797. return latest_doc
  798. def get_history(self, user, doc_type=None, connector_id=None, include_trashed=False):
  799. history = self.documents(
  800. user,
  801. perms='owned',
  802. include_history=True,
  803. include_trashed=include_trashed
  804. ).filter(is_history=True)
  805. if doc_type is not None:
  806. history = history.filter(type=doc_type)
  807. if connector_id is not None:
  808. history = history.filter(connector__id=connector_id)
  809. return history
  810. def get_tasks_history(self, user):
  811. return self.documents(
  812. user,
  813. perms='owned',
  814. include_history=True,
  815. include_trashed=False,
  816. include_managed=True
  817. ).filter(is_history=True, is_managed=True).exclude(name='pig-app-hue-script').exclude(type='oozie-workflow2')
  818. def get_home_directory(self, user):
  819. try:
  820. return self.get(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory')
  821. except Document2.DoesNotExist:
  822. return self.create_user_directories(user)
  823. except Document2.MultipleObjectsReturned:
  824. LOG.error('Multiple Home directories detected. Merging all into one.')
  825. home_dirs = list(self.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory').order_by('-last_modified'))
  826. parent_home_dir = home_dirs.pop()
  827. for dir in home_dirs:
  828. dir.children.exclude(name='.Trash').update(parent_directory=parent_home_dir)
  829. dir.delete()
  830. return parent_home_dir
  831. def get_gist_directory(self, user):
  832. home_dir = self.get_home_directory(user)
  833. gist_dir, created = Directory.objects.get_or_create(name=Document2.GIST_DIR, owner=user, parent_directory=home_dir)
  834. if created:
  835. LOG.info('Successfully created gist directory for user: %s' % user.username)
  836. return gist_dir
  837. def get_by_path(self, user, path):
  838. """
  839. This can be an expensive operation b/c we have to traverse the path tree, so if possible, request a document by UUID
  840. NOTE: get_by_path only works for the owner's documents since it is based off the user's home directory
  841. """
  842. cleaned_path = path.rstrip('/')
  843. doc = Document2.objects.get_home_directory(user)
  844. if cleaned_path:
  845. path_tokens = cleaned_path.split('/')[1:]
  846. for token in path_tokens:
  847. try:
  848. doc = doc.children.get(name=token)
  849. except Document2.DoesNotExist:
  850. raise FilesystemException(_('Requested invalid path for user %s: %s') % (user.username, path))
  851. except Document2.MultipleObjectsReturned:
  852. raise FilesystemException(_('Duplicate documents found for user %s at path: %s') % (user.username, path))
  853. doc.can_read_or_exception(user)
  854. return doc
  855. def create_user_directories(self, user):
  856. """
  857. Creates user home and Trash directories if they do not exist and move any orphan documents to home directory
  858. :param user: User object
  859. """
  860. # Edge-case if the user has a legacy home directory with path-name
  861. Directory.objects.filter(name='/', owner=user).update(name=Document2.HOME_DIR)
  862. # Get or create home and Trash directories for all users
  863. home_dir, created = Directory.objects.get_or_create(name=Document2.HOME_DIR, owner=user)
  864. if created:
  865. LOG.info('Successfully created home directory for user: %s' % user.username)
  866. trash_dir, created = Directory.objects.get_or_create(name=Document2.TRASH_DIR, owner=user, parent_directory=home_dir)
  867. if created:
  868. LOG.info('Successfully created trash directory for user: %s' % user.username)
  869. # For any directories or documents that do not have a parent directory, assign it to home directory
  870. count = 0
  871. for doc in Document2.objects.filter(owner=user).filter(parent_directory=None).exclude(id__in=[home_dir.id, trash_dir.id]):
  872. doc.parent_directory = home_dir
  873. doc.save()
  874. count += 1
  875. LOG.info("Moved %d documents to home directory for user: %s" % (count, user.username))
  876. return home_dir
  877. class Document2(models.Model):
  878. HOME_DIR = ''
  879. GIST_DIR = 'Gist'
  880. TRASH_DIR = '.Trash'
  881. EXAMPLES_DIR = 'examples'
  882. owner = models.ForeignKey(
  883. User,
  884. on_delete=models.CASCADE,
  885. db_index=True,
  886. verbose_name=_t('Owner'),
  887. help_text=_t('Creator.'),
  888. related_name='doc2_owner'
  889. )
  890. name = models.CharField(default='', max_length=255)
  891. description = models.TextField(default='')
  892. uuid = models.CharField(default=uuid_default, max_length=36, db_index=True)
  893. type = models.CharField(
  894. default='',
  895. max_length=32,
  896. db_index=True,
  897. help_text=_t('Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...')
  898. )
  899. connector = models.ForeignKey(
  900. Connector,
  901. on_delete=models.CASCADE,
  902. verbose_name=_t('Connector'),
  903. help_text=_t('Connector.'),
  904. blank=True,
  905. null=True,
  906. db_index=True
  907. )
  908. data = models.TextField(default='{}')
  909. extra = models.TextField(default='')
  910. search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
  911. # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
  912. last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
  913. version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
  914. is_history = models.BooleanField(default=False, db_index=True)
  915. is_managed = models.BooleanField( # Aka isTask
  916. default=False,
  917. db_index=True,
  918. verbose_name=_t('If managed under the cover by Hue and never by the user')
  919. )
  920. is_trashed = models.NullBooleanField(default=False, db_index=True, verbose_name=_t('True if trashed'))
  921. dependencies = models.ManyToManyField('self', symmetrical=False, related_name='dependents', db_index=True)
  922. parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
  923. doc = GenericRelation(Document, related_query_name='doc_doc') # Compatibility with Hue 3
  924. objects = Document2Manager()
  925. class Meta(object):
  926. unique_together = ('uuid', 'version', 'is_history')
  927. ordering = ["-last_modified", "name"]
  928. def __unicode__(self):
  929. res = '%s - %s - %s - %s' % (force_unicode(self.name), self.owner, self.type, self.uuid)
  930. return force_unicode(res)
  931. @property
  932. def data_dict(self):
  933. if not self.data:
  934. self.data = json.dumps({})
  935. data_python = json.loads(self.data)
  936. return data_python
  937. @property
  938. def path(self):
  939. quoted_name = urllib_quote(self.name.encode('utf-8'))
  940. if self.parent_directory:
  941. return '%s/%s' % (self.parent_directory.path, quoted_name)
  942. else:
  943. return quoted_name
  944. @property
  945. def dirname(self):
  946. return os.path.dirname(self.path) or '/'
  947. @property
  948. def is_directory(self):
  949. return self.type == 'directory'
  950. @property
  951. def is_home_directory(self):
  952. return self.is_directory and self.parent_directory == None and self.name == self.HOME_DIR
  953. @property
  954. def is_trash_directory(self):
  955. return self.is_directory and self.name == self.TRASH_DIR
  956. def natural_key(self):
  957. return (self.uuid, self.version, self.is_history)
  958. def copy(self, name, owner, description=None):
  959. copy_doc = self
  960. copy_doc.pk = None
  961. copy_doc.id = None
  962. copy_doc.uuid = uuid_default()
  963. copy_doc.name = name
  964. copy_doc.owner = owner
  965. if description:
  966. copy_doc.description = description
  967. copy_doc.save()
  968. return copy_doc
  969. def update_data(self, post_data):
  970. data_dict = self.data_dict
  971. data_dict.update(post_data)
  972. self.data = json.dumps(data_dict)
  973. def get_absolute_url(self):
  974. url = None
  975. try:
  976. if self.type == 'oozie-coordinator2':
  977. url = reverse('oozie:edit_coordinator') + '?coordinator=' + str(self.id)
  978. elif self.type == 'oozie-bundle2':
  979. url = reverse('oozie:edit_bundle') + '?bundle=' + str(self.id)
  980. elif self.type.startswith('query'):
  981. url = '/editor' + '?editor=' + str(self.id)
  982. elif self.type == 'directory':
  983. url = '/home2' + '?uuid=' + self.uuid
  984. elif self.type == 'gist':
  985. url = '/hue/gist?uuid=' + str(self.uuid)
  986. elif self.type == 'notebook':
  987. url = reverse('notebook:notebook') + '?notebook=' + str(self.id)
  988. elif self.type == 'search-dashboard':
  989. url = reverse('search:index') + '?collection=' + str(self.id)
  990. elif self.type == 'link-pigscript':
  991. url = reverse('pig:index') + '#edit/%s' % self.data_dict.get('object_id', '')
  992. elif self.type == 'link-workflow':
  993. url = '/jobsub/#edit-design/%s' % self.data_dict.get('object_id', '')
  994. else:
  995. url = reverse('oozie:edit_workflow') + '?workflow=' + str(self.id)
  996. except NoReverseMatch:
  997. LOG.warn('Could not perform reverse lookup for type %s, app may be blacklisted.' % self.type)
  998. return url
  999. def to_dict(self):
  1000. return {
  1001. 'owner': self.owner.username,
  1002. 'name': self.name,
  1003. 'path': self.path or '/',
  1004. 'description': self.description,
  1005. 'uuid': self.uuid,
  1006. 'id': self.id,
  1007. 'doc1_id': self.doc.get().id if self.doc.exists() else -1,
  1008. 'parent_uuid': self.parent_directory.uuid if self.parent_directory else None,
  1009. 'type': self.type,
  1010. 'perms': self._massage_permissions(),
  1011. 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT) if self.last_modified else None,
  1012. 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()) if self.last_modified else None,
  1013. 'is_managed': self.is_managed,
  1014. 'isSelected': False,
  1015. 'absoluteUrl': self.get_absolute_url(),
  1016. }
  1017. def get_history(self):
  1018. return self.dependencies.filter(is_history=True).order_by('-last_modified')
  1019. def add_to_history(self, user, data_dict):
  1020. doc_id = self.id # Need to copy as the clone messes it
  1021. history_doc = self.copy(name=self.name, owner=user)
  1022. history_doc.update_data({'history': data_dict})
  1023. history_doc.is_history = True
  1024. history_doc.last_modified = None
  1025. history_doc.save()
  1026. Document2.objects.get(id=doc_id).dependencies.add(history_doc)
  1027. return history_doc
  1028. def save(self, *args, **kwargs):
  1029. # Set document parent to home directory if parent directory isn't specified
  1030. if not self.parent_directory and not self.is_home_directory and not self.is_trash_directory:
  1031. home_dir = Document2.objects.get_home_directory(self.owner)
  1032. self.parent_directory = home_dir
  1033. # Run validations
  1034. self.validate()
  1035. # Redact query if needed
  1036. self._redact_query()
  1037. if self.search:
  1038. self.search = self.search[:DOCUMENT2_SEARCH_MAX_LENGTH]
  1039. super(Document2, self).save(*args, **kwargs)
  1040. # Inherit shared permissions from parent directory, must be done after save b/c new doc needs ID
  1041. self.inherit_permissions()
  1042. def validate(self):
  1043. # Validate home and Trash directories are only created once per user and cannot be created or modified after
  1044. if self.name in [Document2.HOME_DIR, Document2.TRASH_DIR] and self.type == 'directory' and \
  1045. Document2.objects.filter(name=self.name, owner=self.owner, type='directory').exists():
  1046. raise FilesystemException(_('Cannot create or modify directory with name: %s') % self.name)
  1047. # Validate that parent directory does not create cycle
  1048. if self._contains_cycle():
  1049. raise FilesystemException(_('Cannot save document %s under parent directory %s due to circular dependency') %
  1050. (self.name, self.parent_directory.uuid))
  1051. def inherit_permissions(self):
  1052. if self.parent_directory is not None:
  1053. parent_perms = Document2Permission.objects.filter(doc=self.parent_directory)
  1054. for perm in parent_perms:
  1055. self.share(self.owner, name=perm.perms, users=perm.users.all(), groups=perm.groups.all())
  1056. def move(self, directory, user):
  1057. if not directory.is_directory:
  1058. raise FilesystemException(_('Target with UUID %s is not a directory') % directory.uuid)
  1059. if directory.can_write_or_exception(user=user):
  1060. # Restore last_modified date after save
  1061. original_last_modified = self.last_modified
  1062. self.parent_directory = directory
  1063. self.save()
  1064. # Use update instead of save() so that last_modified date is not modified automatically
  1065. Document2.objects.filter(id=self.id).update(last_modified=original_last_modified)
  1066. return self
  1067. def trash(self):
  1068. try:
  1069. trash_dir = Directory.objects.get(name=self.TRASH_DIR, owner=self.owner)
  1070. self.move(trash_dir, self.owner)
  1071. self.is_trashed = True
  1072. self.save()
  1073. if self.is_directory:
  1074. children_ids = self._get_child_ids_recursively(self.id)
  1075. Document2.objects.filter(id__in=children_ids).update(is_trashed=True)
  1076. except Document2.MultipleObjectsReturned:
  1077. LOG.error('Multiple Trash directories detected. Merging all into one.')
  1078. trash_dirs = list(Directory.objects.filter(name=self.TRASH_DIR, owner=self.owner).order_by('-last_modified'))
  1079. parent_trash_dir = trash_dirs.pop()
  1080. for dir in trash_dirs:
  1081. dir.children.update(parent_directory=parent_trash_dir)
  1082. dir.delete()
  1083. self.move(parent_trash_dir, self.owner)
  1084. def restore(self):
  1085. # Currently restoring any doucment to /home
  1086. home_dir = Document2.objects.get_home_directory(self.owner)
  1087. self.move(home_dir, self.owner)
  1088. self.is_trashed = False
  1089. self.save()
  1090. if self.is_directory:
  1091. children_ids = self._get_child_ids_recursively(self.id)
  1092. Document2.objects.filter(id__in=children_ids).update(is_trashed=False)
  1093. def _get_child_ids_recursively(self, directory_id):
  1094. """
  1095. Returns the list of all children ids for a given directory id recursively, excluding history documents
  1096. """
  1097. directory = Directory.objects.get(id=directory_id)
  1098. children_ids = []
  1099. for child in directory.children.filter(is_history=False).filter(is_managed=False):
  1100. children_ids.append(child.id)
  1101. if child.is_directory:
  1102. children_ids.extend(self._get_child_ids_recursively(child.id))
  1103. return children_ids
  1104. def can_read(self, user):
  1105. return \
  1106. self.owner == user \
  1107. or is_admin(user) \
  1108. or any([perm.user_has_access(user) for perm in self.get_permissions('read')]) \
  1109. or self.can_write(user)
  1110. def can_read_or_exception(self, user):
  1111. if self.can_read(user):
  1112. return True
  1113. else:
  1114. raise PopupException(_("Document does not exist or you don't have the permission to access it."), error_code=401)
  1115. def can_write(self, user):
  1116. return \
  1117. self.owner == user \
  1118. or is_admin(user) \
  1119. or any([perm.user_has_access(user) for perm in self.get_permissions('write')]) \
  1120. or (self.parent_directory and self.parent_directory.can_write(user))
  1121. def can_write_or_exception(self, user):
  1122. if self.can_write(user):
  1123. return True
  1124. else:
  1125. raise PopupException(_("Document does not exist or you don't have the permission to access it."))
  1126. def get_permission(self, perm='read'):
  1127. try:
  1128. return Document2Permission.objects.get(doc=self, perms=perm)
  1129. except Document2Permission.DoesNotExist:
  1130. return None
  1131. def get_permissions(self, perm='read'):
  1132. '''
  1133. Return the sub permissions that make one of the two top privileges.
  1134. e.g. 'read' and 'link_read' perms give the global 'read' privilege.
  1135. '''
  1136. if perm == 'read':
  1137. perms = Q(perms=Document2Permission.READ_PERM) | (Q(perms=Document2Permission.LINK_READ_PERM) & Q(is_link_on=True))
  1138. elif perm == 'write':
  1139. perms = Q(perms=Document2Permission.WRITE_PERM) | (Q(perms=Document2Permission.LINK_WRITE_PERM) & Q(is_link_on=True))
  1140. else:
  1141. raise PopupException(_("Permission name %s is invalid.") % perm)
  1142. return Document2Permission.objects.filter(Q(doc=self) & perms)
  1143. def share(self, user, name='read', users=None, groups=None, is_link_on=False):
  1144. try:
  1145. with transaction.atomic():
  1146. self.update_permission(user, name, users, groups, is_link_on)
  1147. # For directories, update all children recursively with same permissions
  1148. for child in self.children.all():
  1149. child.share(user, name, users, groups, is_link_on)
  1150. except Exception as e:
  1151. raise PopupException(_("Failed to share document: %s") % e)
  1152. return self
  1153. def share_link(self, user, perm='read'):
  1154. if perm == 'read':
  1155. self.share(user, name=Document2Permission.LINK_READ_PERM, is_link_on=True)
  1156. self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=False)
  1157. elif perm == 'write':
  1158. self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=True)
  1159. else:
  1160. self.share(user, name=Document2Permission.LINK_READ_PERM, is_link_on=False)
  1161. self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=False)
  1162. return self
  1163. def update_permission(self, user, name='read', users=None, groups=None, is_link_on=False):
  1164. # Check if user has access to grant permissions
  1165. if users or groups:
  1166. if name == 'read' or name == 'link_read':
  1167. self.can_read_or_exception(user)
  1168. elif name == 'write' or name == 'link_write':
  1169. self.can_write_or_exception(user)
  1170. else:
  1171. raise ValueError(_('Invalid permission type: %s') % name)
  1172. perm, created = Document2Permission.objects.get_or_create(doc=self, perms=name)
  1173. perm.users = []
  1174. if users is not None:
  1175. perm.users = users
  1176. perm.groups = []
  1177. if groups is not None:
  1178. perm.groups = groups
  1179. perm.is_link_on = is_link_on
  1180. perm.save()
  1181. def _get_doc1(self, doc2_type=None):
  1182. if not doc2_type:
  1183. doc2_type = self.type
  1184. try:
  1185. doc = self.doc.get()
  1186. except Exception as e:
  1187. LOG.error('Exception when retrieving document object for saved query: %s' % e)
  1188. doc = Document.objects.link(
  1189. self,
  1190. owner=self.owner,
  1191. name=self.name,
  1192. description=self.description,
  1193. extra=doc2_type
  1194. )
  1195. return doc
  1196. def _massage_permissions(self):
  1197. """
  1198. Returns the permissions for a given document as a dictionary
  1199. """
  1200. permissions = {
  1201. 'read': {'users': [], 'groups': []},
  1202. 'write': {'users': [], 'groups': []},
  1203. 'link_sharing_on': False,
  1204. 'link_read': False,
  1205. 'link_write': False,
  1206. }
  1207. read_perm = self.get_permission(perm='read')
  1208. write_perm = self.get_permission(perm='write')
  1209. if read_perm:
  1210. permissions.update(read_perm.to_dict())
  1211. if write_perm:
  1212. permissions.update(write_perm.to_dict())
  1213. link_read_perm = self.get_permission(perm=Document2Permission.LINK_READ_PERM)
  1214. link_write_perm = self.get_permission(perm=Document2Permission.LINK_WRITE_PERM)
  1215. if link_read_perm:
  1216. permissions['link_read'] = link_read_perm.is_link_on
  1217. if link_write_perm:
  1218. permissions['link_write'] = link_write_perm.is_link_on
  1219. permissions['link_sharing_on'] = permissions['link_read'] or permissions['link_write']
  1220. return permissions
  1221. def _redact_query(self):
  1222. """
  1223. Optionally mask out the query from being saved to the database. This is because if the database contains sensitive
  1224. information like personally identifiable information, that information could be leaked into the Hue database and
  1225. logfiles.
  1226. """
  1227. if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
  1228. data_dict = self.data_dict
  1229. snippets = data_dict.get('snippets', [])
  1230. for snippet in snippets:
  1231. if snippet['type'] in ('hive', 'impala'): # TODO: Pull SQL types from canonical lookup
  1232. redacted_statement_raw = global_redaction_engine.redact(snippet['statement_raw'])
  1233. if snippet['statement_raw'] != redacted_statement_raw:
  1234. snippet['statement_raw'] = redacted_statement_raw
  1235. snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
  1236. snippet['is_redacted'] = True
  1237. self.data = json.dumps(data_dict)
  1238. self.search = global_redaction_engine.redact(self.search)
  1239. def _contains_cycle(self):
  1240. """
  1241. Uses Floyd's cycle-detection algorithm to detect a cycle (aka Tortoise and Hare)
  1242. https://en.wikipedia.org/wiki/Cycle_detection#Tortoise_and_hare
  1243. """
  1244. # Test base case where self.uuid == self.parent_directory.uuid first
  1245. if self.parent_directory is not None and self.parent_directory.uuid == self.uuid:
  1246. return True
  1247. slow = self
  1248. fast = self
  1249. while True:
  1250. slow = slow.parent_directory
  1251. if slow and slow.uuid == self.uuid:
  1252. slow = self
  1253. if fast.parent_directory is not None:
  1254. if fast.parent_directory.uuid == self.uuid:
  1255. fast = self.parent_directory.parent_directory
  1256. else:
  1257. fast = fast.parent_directory.parent_directory
  1258. else:
  1259. return False
  1260. if slow is None or fast is None:
  1261. return False
  1262. if slow == fast:
  1263. return True
  1264. class DirectoryManager(Document2Manager):
  1265. def get_queryset(self):
  1266. return super(DirectoryManager, self).get_queryset().filter(type='directory')
  1267. class Directory(Document2):
  1268. # e.g. name = '/' or '/dir1/dir2/f3'
  1269. objects = DirectoryManager()
  1270. class Meta(object):
  1271. proxy = True
  1272. def get_children_documents(self):
  1273. """
  1274. Returns the children documents for a given directory, excluding history documents
  1275. """
  1276. documents = self.children.filter(is_history=False).filter(is_managed=False) # TODO: perms
  1277. return documents
  1278. def get_children_and_shared_documents(self, user):
  1279. """
  1280. Returns the children and shared documents for a given directory, excluding history documents
  1281. """
  1282. # Get documents that are direct children, or shared with but not owned by the current user
  1283. documents = Document2.objects.filter(
  1284. Q(parent_directory=self) |
  1285. ((Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
  1286. ~Q(owner=user))
  1287. )
  1288. documents = documents.exclude(is_history=True).exclude(is_managed=True)
  1289. # Excluding all trashed docs across users
  1290. documents = documents.exclude(is_trashed=True)
  1291. # Optimizing roll up for /home by checking only with directories instead of all documents
  1292. # For all other directories roll up is done in _filter_documents()
  1293. directories_all = Document2.objects.filter(type='directory').exclude(id=self.id)
  1294. directories_inside_home = directories_all.filter(
  1295. (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
  1296. ~Q(owner=user)
  1297. )
  1298. documents = documents.exclude(parent_directory__in=directories_inside_home)
  1299. return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
  1300. def save(self, *args, **kwargs):
  1301. self.type = 'directory'
  1302. super(Directory, self).save(*args, **kwargs)
  1303. class Document2Permission(models.Model):
  1304. """
  1305. Combine either:
  1306. - regular perms (listed)
  1307. - link
  1308. """
  1309. READ_PERM = 'read'
  1310. WRITE_PERM = 'write'
  1311. COMMENT_PERM = 'comment'
  1312. LINK_READ_PERM = 'link_read'
  1313. LINK_WRITE_PERM = 'link_write'
  1314. doc = models.ForeignKey(Document2, on_delete=models.CASCADE)
  1315. users = models.ManyToManyField(User, db_index=True, db_table='documentpermission2_users')
  1316. groups = models.ManyToManyField(Group, db_index=True, db_table='documentpermission2_groups')
  1317. perms = models.CharField(default=READ_PERM, max_length=10, db_index=True, choices=( # One perm
  1318. (READ_PERM, 'read'),
  1319. (WRITE_PERM, 'write'),
  1320. (COMMENT_PERM, 'comment'), # Unused
  1321. (LINK_READ_PERM, 'link read'),
  1322. (LINK_WRITE_PERM, 'link write'),
  1323. ))
  1324. is_link_on = models.BooleanField(default=False)
  1325. class Meta(object):
  1326. unique_together = ('doc', 'perms')
  1327. def __str__(self):
  1328. return force_unicode('Document: %s, Name: %s, Users: %s, Groups: %s, is_link_on: %s') % (
  1329. self.doc, self.perms, self.users, self.groups, self.is_link_on
  1330. )
  1331. def to_dict(self):
  1332. return {
  1333. self.perms: {
  1334. 'users': [{'id': perm_user.id, 'username': perm_user.username} for perm_user in self.users.all()],
  1335. 'groups': [{'id': perm_group.id, 'name': perm_group.name} for perm_group in self.groups.all()]
  1336. }
  1337. }
  1338. def user_has_access(self, user):
  1339. """
  1340. Returns true if the given user has permissions based on users, groups or all flag.
  1341. """
  1342. return self.is_link_on or user in self.users.all() or self.groups.filter(id__in=user.groups.all()).exists()
  1343. def get_cluster_config(user):
  1344. return Cluster(user).get_app_config().get_config()
  1345. class ClusterConfig(object):
  1346. """
  1347. Configuration of the apps and engines that each individual user sees on the core Hue.
  1348. Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
  1349. TODO: rename to HueConfig
  1350. TODO: get list of contexts dynamically
  1351. """
  1352. def __init__(self, user, apps=None, cluster_type='ini'):
  1353. self.user = user
  1354. self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
  1355. self.cluster_type = cluster_type
  1356. def get_config(self):
  1357. app_config = self.get_apps()
  1358. editors = app_config.get('editor')
  1359. main_button_action = self.get_main_quick_action(app_config)
  1360. if main_button_action.get('is_sql'):
  1361. default_sql_interpreter = main_button_action['type']
  1362. else:
  1363. default_sql_interpreter = editors and editors['default_sql_interpreter']
  1364. return {
  1365. 'app_config': app_config,
  1366. 'main_button_action': main_button_action,
  1367. 'button_actions': [
  1368. app for app in [
  1369. editors,
  1370. app_config.get('dashboard'),
  1371. app_config.get('scheduler')
  1372. ]
  1373. if app is not None
  1374. ],
  1375. 'default_sql_interpreter': default_sql_interpreter,
  1376. 'cluster_type': self.cluster_type,
  1377. 'has_computes': self.cluster_type in ('altus', 'snowball'), # or any grouped engine connectors
  1378. 'hue_config': {
  1379. 'enable_sharing': ENABLE_SHARING.get()
  1380. }
  1381. }
  1382. def get_apps(self):
  1383. apps = OrderedDict([
  1384. app for app in [
  1385. ('editor', self._get_editor()),
  1386. ('dashboard', self._get_dashboard()),
  1387. ('catalogs', self._get_catalogs()),
  1388. ('browser', self._get_browser()),
  1389. ('scheduler', self._get_scheduler()),
  1390. ('sdkapps', self._get_sdk_apps()),
  1391. ('home', self._get_home()),
  1392. ] if app[1]
  1393. ])
  1394. return apps
  1395. def get_main_quick_action(self, apps):
  1396. if not apps:
  1397. raise PopupException(_('No permission to any app.'))
  1398. default_app = list(apps.values())[0]
  1399. default_interpreter = default_app.get('interpreters')
  1400. try:
  1401. user_preference = get_user_preferences(user=self.user, key='default_app')
  1402. if not user_preference:
  1403. raise UserPreferences.DoesNotExist()
  1404. user_default_app = json.loads(user_preference)
  1405. if apps.get(user_default_app['app']):
  1406. default_interpreter = []
  1407. default_app = apps[user_default_app['app']]
  1408. if default_app.get('interpreters'):
  1409. interpreters = [
  1410. interpreter
  1411. for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']
  1412. ]
  1413. if interpreters:
  1414. default_interpreter = interpreters
  1415. except UserPreferences.DoesNotExist:
  1416. pass
  1417. except Exception:
  1418. LOG.exception('Could not load back default app')
  1419. if default_interpreter:
  1420. return default_interpreter[0]
  1421. else:
  1422. return default_app
  1423. def _get_home(self):
  1424. return {
  1425. 'name': 'home',
  1426. 'displayName': _('Home'),
  1427. 'buttonName': _('Saved Queries') if has_connectors() else _('Documents'),
  1428. 'interpreters': [],
  1429. 'page': '/home'
  1430. }
  1431. def _get_editor(self):
  1432. interpreters = []
  1433. _interpreters = get_ordered_interpreters(self.user)
  1434. for interpreter in _interpreters:
  1435. if interpreter['interface'] != 'hms':
  1436. interpreters.append({
  1437. 'name': interpreter['name'],
  1438. 'type': interpreter['type'], # Connector v1
  1439. 'id': interpreter['type'],
  1440. 'displayName': interpreter['name'],
  1441. 'buttonName': _('Query'),
  1442. 'tooltip': _('%s Query') % interpreter['type'].title(),
  1443. 'optimizer': get_optimizer_mode(),
  1444. 'page': '/editor/?type=%(type)s' % interpreter,
  1445. 'is_sql': interpreter['is_sql'],
  1446. 'is_batchable': interpreter['dialect'] in ['hive', 'impala'] or interpreter['interface'] in ['oozie', 'sqlalchemy'],
  1447. 'dialect': interpreter['dialect'],
  1448. 'dialect_properties': interpreter.get('dialect_properties'),
  1449. })
  1450. if SHOW_NOTEBOOKS.get():
  1451. try:
  1452. first_non_sql_index = [interpreter['is_sql'] for interpreter in interpreters].index(False)
  1453. except ValueError:
  1454. first_non_sql_index = len(interpreters) if all([interpreter['is_sql'] for interpreter in interpreters]) else 0
  1455. interpreters.insert(first_non_sql_index, {
  1456. 'name': 'notebook',
  1457. 'type': 'notebook',
  1458. 'displayName': 'Notebook',
  1459. 'buttonName': _('Notebook'),
  1460. 'tooltip': _('Notebook'),
  1461. 'page': '/notebook',
  1462. 'is_sql': False,
  1463. 'dialect': 'notebook'
  1464. })
  1465. if interpreters:
  1466. return {
  1467. 'name': 'editor',
  1468. 'displayName': _('Editor'),
  1469. 'buttonName': _('Query'),
  1470. 'interpreters': interpreters,
  1471. 'default_limit': None if DEFAULT_LIMIT.get() == 0 else DEFAULT_LIMIT.get(),
  1472. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1473. 'page': interpreters[0]['page'],
  1474. 'default_sql_interpreter': next((interpreter['type'] for interpreter in interpreters if interpreter.get('is_sql')), 'hive')
  1475. }
  1476. else:
  1477. return None
  1478. def _get_catalogs(self):
  1479. '''
  1480. Any connector implementing Notebook Assist API.
  1481. Currently SQL connectors, Hive Metastore. Possibly Metadata Catalogs later.
  1482. '''
  1483. interpreters = []
  1484. _interpreters = get_ordered_interpreters(self.user)
  1485. for interpreter in _interpreters:
  1486. if interpreter['is_sql'] or interpreter['interface'] == 'hms':
  1487. interpreters.append({
  1488. 'name': interpreter['name'],
  1489. 'type': interpreter['type'],
  1490. 'id': interpreter['type'],
  1491. 'displayName': interpreter['name'],
  1492. 'buttonName': _('Query'),
  1493. 'tooltip': _('%s Query') % interpreter['type'].title(),
  1494. 'page': '/editor/?type=%(type)s' % interpreter,
  1495. 'is_sql': interpreter['is_sql'],
  1496. 'is_catalog': True,
  1497. })
  1498. return interpreters if interpreters else None
  1499. def _get_dashboard(self):
  1500. interpreters = get_engines(self.user)
  1501. _interpreters = []
  1502. if interpreters:
  1503. if HAS_REPORT_ENABLED.get():
  1504. _interpreters.append({
  1505. 'type': 'report',
  1506. 'id': 'report',
  1507. 'displayName': 'Report',
  1508. 'buttonName': 'Report',
  1509. 'page': '/dashboard/new_search?engine=report',
  1510. 'tooltip': _('Report'),
  1511. 'is_sql': False
  1512. })
  1513. _interpreters.extend([{
  1514. 'type': interpreter['type'],
  1515. 'id': interpreter['type'],
  1516. 'displayName': interpreter['type'].title(),
  1517. 'buttonName': interpreter['type'].title(),
  1518. 'page': '/dashboard/new_search?engine=%(type)s' % interpreter,
  1519. 'tooltip': _('%s Dashboard') % interpreter['type'].title(),
  1520. 'is_sql': True
  1521. } for interpreter in interpreters
  1522. ])
  1523. return {
  1524. 'name': 'dashboard',
  1525. 'displayName': _('Dashboard'),
  1526. 'buttonName': _('Dashboard'),
  1527. 'interpreters': _interpreters,
  1528. 'page': '/dashboard/new_search?engine=%(type)s' % interpreters[0]
  1529. }
  1530. else:
  1531. return None
  1532. def _get_browser(self):
  1533. interpreters = []
  1534. hdfs_connectors = []
  1535. if has_connectors():
  1536. hdfs_connectors = [
  1537. connector['nice_name'] for connector in _get_installed_connectors(category='browsers', dialect='hdfs', interface='rest')
  1538. ]
  1539. elif 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('hdfs', self.user):
  1540. hdfs_connectors.append(_('Files'))
  1541. remote_home_storage = REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() else None
  1542. for hdfs_connector in hdfs_connectors:
  1543. force_home = remote_home_storage and not remote_home_storage.startswith('/')
  1544. home_path = self.user.get_home_directory(force_home=force_home).encode('utf-8')
  1545. interpreters.append({
  1546. 'type': 'hdfs',
  1547. 'displayName': hdfs_connector,
  1548. 'buttonName': _('Browse'),
  1549. 'tooltip': hdfs_connector,
  1550. 'page': '/filebrowser/' + (
  1551. not self.user.is_anonymous() and
  1552. 'view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
  1553. )
  1554. })
  1555. if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('s3a', self.user):
  1556. home_path = remote_home_storage if remote_home_storage else 'S3A://'.encode('utf-8')
  1557. interpreters.append({
  1558. 'type': 's3',
  1559. 'displayName': _('S3'),
  1560. 'buttonName': _('Browse'),
  1561. 'tooltip': _('S3'),
  1562. 'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1563. })
  1564. if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('adl', self.user):
  1565. home_path = remote_home_storage if remote_home_storage else 'adl:/'.encode('utf-8')
  1566. interpreters.append({
  1567. 'type': 'adls',
  1568. 'displayName': _('ADLS'),
  1569. 'buttonName': _('Browse'),
  1570. 'tooltip': _('ADLS'),
  1571. 'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1572. })
  1573. if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
  1574. from azure.abfs.__init__ import get_home_dir_for_ABFS
  1575. home_path = remote_home_storage if remote_home_storage else get_home_dir_for_ABFS().encode('utf-8')
  1576. interpreters.append({
  1577. 'type': 'abfs',
  1578. 'displayName': _('ABFS'),
  1579. 'buttonName': _('Browse'),
  1580. 'tooltip': _('ABFS'),
  1581. 'page': '/filebrowser/view=' + urllib_quote(home_path, safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1582. })
  1583. if 'metastore' in self.apps:
  1584. interpreters.append({
  1585. 'type': 'tables',
  1586. 'displayName': _('Tables'),
  1587. 'buttonName': _('Browse'),
  1588. 'tooltip': _('Tables'),
  1589. 'page': '/metastore/tables'
  1590. })
  1591. if 'search' in self.apps:
  1592. interpreters.append({
  1593. 'type': 'indexes',
  1594. 'displayName': _('Indexes'),
  1595. 'buttonName': _('Dashboard'),
  1596. 'tooltip': _('Indexes'),
  1597. 'page': '/indexer/'
  1598. })
  1599. if 'jobbrowser' in self.apps:
  1600. from hadoop.cluster import get_default_yarncluster # Circular loop
  1601. from jobbrowser.conf import ENABLE_HIVE_QUERY_BROWSER
  1602. if get_default_yarncluster() or ENABLE_HIVE_QUERY_BROWSER.get():
  1603. interpreters.append({
  1604. 'type': 'yarn',
  1605. 'displayName': _('Jobs'),
  1606. 'buttonName': _('Jobs'),
  1607. 'tooltip': _('Jobs'),
  1608. 'page': '/jobbrowser/'
  1609. })
  1610. if has_kafka():
  1611. interpreters.append({
  1612. 'type': 'kafka',
  1613. 'displayName': _('Streams'),
  1614. 'buttonName': _('Browse'),
  1615. 'tooltip': _('Kafka'),
  1616. 'page': '/kafka/'
  1617. })
  1618. if 'hbase' in self.apps:
  1619. interpreters.append({
  1620. 'type': 'hbase',
  1621. 'displayName': _('HBase'),
  1622. 'buttonName': _('Browse'),
  1623. 'tooltip': _('HBase'),
  1624. 'page': '/hbase/'
  1625. })
  1626. if 'security' in self.apps:
  1627. interpreters.append({
  1628. 'type': 'security',
  1629. 'displayName': _('Security'),
  1630. 'buttonName': _('Browse'),
  1631. 'tooltip': _('Security'),
  1632. 'page': '/security/hive'
  1633. })
  1634. if 'indexer' in self.apps and 'filebrowser' in self.apps and self.user.has_hue_permission(action="access:importer", app="indexer") \
  1635. and 'importer' not in APP_BLACKLIST.get():
  1636. interpreters.append({
  1637. 'type': 'importer',
  1638. 'displayName': _('Importer'),
  1639. 'buttonName': _('Import'),
  1640. 'tooltip': _('Importer'),
  1641. 'page': '/indexer/importer'
  1642. })
  1643. if 'sqoop' in self.apps:
  1644. from sqoop.conf import IS_ENABLED
  1645. if IS_ENABLED.get():
  1646. interpreters.append({
  1647. 'type': 'sqoop',
  1648. 'displayName': _('Sqoop'),
  1649. 'buttonName': _('Browse'),
  1650. 'tooltip': _('Sqoop'),
  1651. 'page': '/sqoop'
  1652. })
  1653. if interpreters:
  1654. return {
  1655. 'name': 'browser',
  1656. 'displayName': _('Browsers'),
  1657. 'buttonName': _('Browse'),
  1658. 'interpreters': interpreters,
  1659. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1660. }
  1661. else:
  1662. return None
  1663. def _get_scheduler(self):
  1664. interpreters = []
  1665. if 'oozie' in self.apps and not (self.user.has_hue_permission(action="disable_editor_access", app="oozie") and not is_admin(self.user)):
  1666. interpreters.extend([{
  1667. 'type': 'oozie-workflow',
  1668. 'displayName': _('Workflow'),
  1669. 'buttonName': _('Schedule'),
  1670. 'tooltip': _('Workflow'),
  1671. 'page': '/oozie/editor/workflow/new/'
  1672. }, {
  1673. 'type': 'oozie-coordinator',
  1674. 'displayName': _('Schedule'),
  1675. 'buttonName': _('Schedule'),
  1676. 'tooltip': _('Schedule'),
  1677. 'page': '/oozie/editor/coordinator/new/'
  1678. }, {
  1679. 'type': 'oozie-bundle',
  1680. 'displayName': _('Bundle'),
  1681. 'buttonName': _('Schedule'),
  1682. 'tooltip': _('Bundle'),
  1683. 'page': '/oozie/editor/bundle/new/'
  1684. }
  1685. ])
  1686. if TASK_SERVER.BEAT_ENABLED.get():
  1687. interpreters.append({
  1688. 'type': 'celery-beat',
  1689. 'displayName': _('Scheduled Tasks'),
  1690. 'buttonName': _('Scheduled Tasks'),
  1691. 'tooltip': _('Scheduled Tasks'),
  1692. 'page': '/jobbrowser/'
  1693. }
  1694. )
  1695. if interpreters:
  1696. return {
  1697. 'name': 'oozie',
  1698. 'displayName': _('Scheduler'),
  1699. 'buttonName': _('Schedule'),
  1700. 'interpreters': interpreters,
  1701. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1702. 'page': interpreters[0]['page']
  1703. }
  1704. else:
  1705. return None
  1706. def _get_sdk_apps(self):
  1707. current_app, other_apps, apps_list = _get_apps(self.user)
  1708. interpreters = []
  1709. for other in other_apps:
  1710. interpreters.append({
  1711. 'type': other.display_name,
  1712. 'displayName': other.nice_name,
  1713. 'buttonName': other.nice_name,
  1714. 'tooltip': other.nice_name,
  1715. 'page': '/%s' % other.display_name
  1716. })
  1717. if interpreters:
  1718. return {
  1719. 'name': 'other',
  1720. 'displayName': _('Other Apps'),
  1721. 'buttonName': _('Other'),
  1722. 'interpreters': interpreters,
  1723. }
  1724. else:
  1725. return None
  1726. def get_hive_metastore_interpreters(self):
  1727. return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user)
  1728. if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
  1729. class Cluster(object):
  1730. def __init__(self, user):
  1731. self.user = user
  1732. self.clusters = get_clusters(user)
  1733. if IS_MULTICLUSTER_ONLY.get():
  1734. self.data = self.clusters['Altus'] # Backward compatibility
  1735. else:
  1736. self.data = list(self.clusters.values())[0] # Next: CLUSTER_ID.get() or user persisted
  1737. def get_type(self):
  1738. return self.data['type']
  1739. def get_config(self, name):
  1740. return self.clusters[name]
  1741. def get_app_config(self):
  1742. return ClusterConfig(self.user, cluster_type=self.get_type())
  1743. def _get_apps(user, section=None):
  1744. current_app = None
  1745. other_apps = []
  1746. if user.is_authenticated():
  1747. apps_list = appmanager.get_apps_dict(user)
  1748. apps = list(apps_list.values())
  1749. for app in apps:
  1750. if app.display_name not in [
  1751. 'beeswax', 'hive', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
  1752. 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'
  1753. ] and app.menu_index != -1:
  1754. other_apps.append(app)
  1755. if section == app.display_name:
  1756. current_app = app
  1757. else:
  1758. apps_list = []
  1759. return current_app, other_apps, apps_list
  1760. def get_user_preferences(user, key=None):
  1761. if key is not None:
  1762. try:
  1763. x = UserPreferences.objects.get(user=user, key=key)
  1764. return {key: x.value}
  1765. except UserPreferences.DoesNotExist:
  1766. return None
  1767. except UserPreferences.MultipleObjectsReturned:
  1768. for dup in UserPreferences.objects.filter(user=user, key=key)[1:]:
  1769. LOG.warn('Deleting UserPreferences duplicate %s' % dup)
  1770. dup.delete()
  1771. x = UserPreferences.objects.get(user=user, key=key)
  1772. return {key: x.value}
  1773. else:
  1774. return dict((x.key, x.value) for x in UserPreferences.objects.filter(user=user))
  1775. def set_user_preferences(user, key, value):
  1776. try:
  1777. x = UserPreferences.objects.get(user=user, key=key)
  1778. except UserPreferences.DoesNotExist:
  1779. x = UserPreferences(user=user, key=key)
  1780. x.value = value
  1781. x.save()
  1782. return x
  1783. def get_data_link(meta):
  1784. link = None
  1785. if not meta.get('type'):
  1786. pass
  1787. elif meta['type'] == 'hbase':
  1788. link = '/hbase/#Cluster/%(table)s/query/%(row_key)s' % meta
  1789. if 'col' in meta:
  1790. link += '[%(fam)s:%(col)s]' % meta
  1791. elif 'fam' in meta:
  1792. link += '[%(fam)s]' % meta
  1793. elif meta['type'] == 'hdfs':
  1794. link = '/filebrowser/view=%(path)s' % meta # Could add a byte #
  1795. elif meta['type'] == 'link':
  1796. link = meta['link']
  1797. elif meta['type'] == 'hive':
  1798. link = '/metastore/table/%(database)s/%(table)s' % meta # Could also add col=val
  1799. return link
  1800. def _get_gist_document(uuid):
  1801. return Document2.objects.get(uuid=uuid, type='gist') # Workaround until there is a share to all permission
  1802. def __paginate(page, limit, queryset):
  1803. if limit > 0:
  1804. offset = (page - 1) * limit
  1805. last = offset + limit
  1806. queryset = queryset.all()[offset:last]
  1807. return {
  1808. 'documents': queryset,
  1809. 'page': page,
  1810. 'limit': limit
  1811. }