models.py 75 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. from __future__ import absolute_import
  18. from future import standard_library
  19. standard_library.install_aliases()
  20. from builtins import next
  21. from builtins import object
  22. import calendar
  23. import json
  24. import logging
  25. import os
  26. import sys
  27. import uuid
  28. from collections import OrderedDict
  29. from itertools import chain
  30. from django.db import connection, models, transaction
  31. from django.db.models import Q
  32. from django.db.models.query import QuerySet
  33. from django.contrib.auth.validators import UnicodeUsernameValidator
  34. from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey
  35. from django.contrib.contenttypes.models import ContentType
  36. from django.contrib.staticfiles.storage import staticfiles_storage
  37. from django.urls import reverse, NoReverseMatch
  38. from django.utils.translation import ugettext as _, ugettext_lazy as _t
  39. from dashboard.conf import get_engines, HAS_REPORT_ENABLED
  40. from kafka.conf import has_kafka
  41. from notebook.conf import DEFAULT_LIMIT, SHOW_NOTEBOOKS, get_ordered_interpreters
  42. from useradmin.models import User, Group, get_organization
  43. from useradmin.organization import _fitered_queryset
  44. from desktop import appmanager
  45. from desktop.auth.backend import is_admin
  46. from desktop.conf import get_clusters, CLUSTER_ID, IS_MULTICLUSTER_ONLY, IS_K8S_ONLY, ENABLE_ORGANIZATIONS, ENABLE_PROMETHEUS,\
  47. has_connectors, TASK_SERVER, ENABLE_GIST, APP_BLACKLIST
  48. from desktop.lib import fsmanager
  49. from desktop.lib.connectors.api import _get_installed_connectors
  50. from desktop.lib.i18n import force_unicode
  51. from desktop.lib.exceptions_renderable import PopupException
  52. from desktop.lib.paths import get_run_root, SAFE_CHARACTERS_URI_COMPONENTS
  53. from desktop.redaction import global_redaction_engine
  54. from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH, HUE_DESKTOP_VERSION
  55. if sys.version_info[0] > 2:
  56. from urllib.parse import quote as urllib_quote
  57. else:
  58. from urllib import quote as urllib_quote
  59. LOG = logging.getLogger(__name__)
  60. SAMPLE_USER_ID = 1100713
  61. SAMPLE_USER_INSTALL = 'hue'
  62. SAMPLE_USER_OWNERS = ['hue', 'sample']
  63. UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
  64. HUE_VERSION = None
  65. def uuid_default():
  66. return str(uuid.uuid4())
  67. def hue_version():
  68. global HUE_VERSION
  69. if HUE_VERSION is None:
  70. HUE_VERSION = HUE_DESKTOP_VERSION
  71. p = get_run_root('cloudera', 'cdh_version.properties')
  72. if os.path.exists(p):
  73. build_version = _version_from_properties(open(p))
  74. if build_version:
  75. HUE_VERSION = '%s - %s' % (HUE_VERSION, build_version)
  76. return HUE_VERSION
  77. def _version_from_properties(f):
  78. return dict(line.strip().split('=') for line in f.readlines() if len(line.strip().split('=')) == 2).get('cloudera.cdh.release')
  79. def get_sample_user_install(user):
  80. if ENABLE_ORGANIZATIONS.get():
  81. organization = get_organization(email=user.email)
  82. if organization.is_multi_user:
  83. return SAMPLE_USER_INSTALL + '@' + organization.domain
  84. else:
  85. return organization.domain
  86. else:
  87. return SAMPLE_USER_INSTALL
  88. ###################################################################################################
  89. # Custom Settings
  90. ###################################################################################################
  91. PREFERENCE_IS_WELCOME_TOUR_SEEN = 'is_welcome_tour_seen'
  92. if not ENABLE_ORGANIZATIONS.get():
  93. class HueUser(User):
  94. class Meta(object):
  95. proxy = True
  96. def __init__(self, *args, **kwargs):
  97. self._meta.get_field(
  98. 'username'
  99. ).validators[0] = UnicodeUsernameValidator()
  100. super(User, self).__init__(*args, **kwargs)
  101. class UserPreferences(models.Model):
  102. """Holds arbitrary key/value strings."""
  103. user = models.ForeignKey(User)
  104. key = models.CharField(max_length=20)
  105. value = models.TextField(max_length=4096)
  106. class Settings(models.Model):
  107. collect_usage = models.BooleanField(db_index=True, default=True)
  108. tours_and_tutorials = models.BooleanField(db_index=True, default=True)
  109. @classmethod
  110. def get_settings(cls):
  111. settings, created = Settings.objects.get_or_create(id=1)
  112. return settings
  113. class DefaultConfigurationManager(models.Manager):
  114. def get_configuration_for_user(self, app, user):
  115. """
  116. :param app: app name
  117. :param user: User object
  118. :return: DefaultConfiguration for user, or first group found, or default for app, or None
  119. """
  120. try:
  121. return super(DefaultConfigurationManager, self).get(app=app, user=user)
  122. except DefaultConfiguration.DoesNotExist:
  123. pass
  124. configs = super(DefaultConfigurationManager, self).get_queryset().filter(app=app, groups__in=user.groups.all())
  125. if configs.count() > 0:
  126. return configs[0]
  127. try:
  128. return super(DefaultConfigurationManager, self).get(app=app, is_default=True)
  129. except DefaultConfiguration.DoesNotExist:
  130. pass
  131. return None
  132. class DefaultConfiguration(models.Model):
  133. """
  134. Default values for configuration properties for a given app/editor
  135. Can be designated as default for all users by is_default flag, or for a specific group or user
  136. """
  137. app = models.CharField(max_length=32, null=False, db_index=True, help_text=_t('App that this configuration belongs to.'))
  138. properties = models.TextField(default='[]', help_text=_t('JSON-formatted default properties values.'))
  139. is_default = models.BooleanField(default=False, db_index=True)
  140. groups = models.ManyToManyField(Group, db_index=True, db_table='defaultconfiguration_groups')
  141. user = models.ForeignKey(User, blank=True, null=True, db_index=True)
  142. objects = DefaultConfigurationManager()
  143. class Meta(object):
  144. ordering = ["app", "-is_default", "user"]
  145. @property
  146. def properties_list(self):
  147. """
  148. :return: Deserialized properties as a list of property objects
  149. """
  150. if not self.properties:
  151. self.properties = json.dumps([])
  152. return json.loads(self.properties)
  153. @property
  154. def properties_dict(self):
  155. """
  156. :return: Deserialized properties as a dict of key: value pairs
  157. """
  158. if not self.properties:
  159. self.properties = json.dumps([])
  160. return dict((prop['key'], prop['value']) for prop in json.loads(self.properties))
  161. def to_dict(self):
  162. return {
  163. 'id': self.id,
  164. 'app': self.app,
  165. 'properties': self.properties_list,
  166. 'is_default': self.is_default,
  167. 'group_ids': [group.id for group in self.groups.all()],
  168. 'user': self.user.username if self.user else None
  169. }
  170. ###################################################################################################
  171. # Document1
  172. ###################################################################################################
  173. class DocumentTagManager(models.Manager):
  174. def get_tags(self, user):
  175. return self.filter(owner=user).distinct()
  176. def create_tag(self, owner, tag_name):
  177. if tag_name in DocumentTag.RESERVED:
  178. raise Exception(_("Can't add %s: it is a reserved tag.") % tag_name)
  179. else:
  180. tag, created = DocumentTag.objects.get_or_create(tag=tag_name, owner=owner)
  181. return tag
  182. def _get_tag(self, user, name):
  183. tag, created = DocumentTag.objects.get_or_create(owner=user, tag=name)
  184. return tag
  185. def get_default_tag(self, user):
  186. return self._get_tag(user, DocumentTag.DEFAULT)
  187. def get_trash_tag(self, user):
  188. return self._get_tag(user, DocumentTag.TRASH)
  189. def get_history_tag(self, user):
  190. return self._get_tag(user, DocumentTag.HISTORY)
  191. def get_example_tag(self, user):
  192. return self._get_tag(user, DocumentTag.EXAMPLE)
  193. def get_imported2_tag(self, user):
  194. return self._get_tag(user, DocumentTag.IMPORTED2)
  195. def tag(self, owner, doc_id, tag_name='', tag_id=None):
  196. try:
  197. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  198. if tag.tag in DocumentTag.RESERVED:
  199. raise Exception(_("Can't add %s: it is a reserved tag.") % tag)
  200. except DocumentTag.DoesNotExist:
  201. tag = self._get_tag(user=owner, name=tag_name)
  202. doc = Document.objects.get_doc_for_writing(doc_id, owner)
  203. doc.add_tag(tag)
  204. return tag
  205. def untag(self, tag_id, owner, doc_id):
  206. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  207. if tag.tag in DocumentTag.RESERVED:
  208. raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
  209. doc = Document.objects.get_doc_for_writing(doc_id, owner=owner)
  210. doc.remove_tag(tag)
  211. def delete_tag(self, tag_id, owner):
  212. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  213. default_tag = DocumentTag.objects.get_default_tag(owner)
  214. if tag.tag in DocumentTag.RESERVED:
  215. raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
  216. else:
  217. tag.delete()
  218. for doc in Document.objects.get_docs(owner).filter(tags=None):
  219. doc.add_tag(default_tag)
  220. def update_tags(self, owner, doc_id, tag_ids):
  221. doc = Document.objects.get_doc_for_writing(doc_id, owner)
  222. for tag in doc.tags.all():
  223. if tag.tag not in DocumentTag.RESERVED:
  224. doc.remove_tag(tag)
  225. for tag_id in tag_ids:
  226. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  227. if tag.tag not in DocumentTag.RESERVED:
  228. doc.add_tag(tag)
  229. return doc
  230. class DocumentTag(models.Model):
  231. """
  232. Reserved tags can't be manually removed by the user.
  233. """
  234. owner = models.ForeignKey(User, db_index=True)
  235. tag = models.SlugField()
  236. DEFAULT = 'default' # Always there
  237. TRASH = 'trash' # There when the document is trashed
  238. HISTORY = 'history' # There when the document is a submission history
  239. EXAMPLE = 'example' # Hue examples
  240. IMPORTED2 = 'imported2' # Was imported to document2
  241. RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE, IMPORTED2)
  242. objects = DocumentTagManager()
  243. class Meta(object):
  244. unique_together = ('owner', 'tag')
  245. def __unicode__(self):
  246. return force_unicode('%s') % (self.tag,)
  247. class DocumentManager(models.Manager):
  248. def documents(self, user):
  249. return Document.objects.filter(
  250. Q(owner=user) |
  251. Q(documentpermission__users=user) |
  252. Q(documentpermission__groups__in=user.groups.all())
  253. ).defer('description', 'extra').distinct()
  254. def get_docs(self, user, model_class=None, extra=None, qfilter=None):
  255. docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
  256. if model_class is not None:
  257. ct = ContentType.objects.get_for_model(model_class)
  258. docs = docs.filter(content_type=ct)
  259. if extra is not None:
  260. docs = docs.filter(extra=extra)
  261. if qfilter is not None:
  262. docs = docs.filter(qfilter)
  263. return docs
  264. def get_doc_for_writing(self, doc_id, user):
  265. """Fetch a document and confirm that this user can write to it."""
  266. doc = Document.objects.documents(user).get(id=doc_id)
  267. doc.can_write_or_exception(user)
  268. return doc
  269. def trashed_docs(self, model_class, user):
  270. tag = DocumentTag.objects.get_trash_tag(user=user)
  271. return Document.objects.get_docs(user, model_class).filter(tags__in=[tag]).order_by('-last_modified')
  272. def trashed(self, model_class, user):
  273. docs = self.trashed_docs(model_class, user)
  274. return [job.content_object for job in docs if job.content_object]
  275. def available_docs(self, model_class, user, with_history=False):
  276. exclude = [DocumentTag.objects.get_trash_tag(user=user)]
  277. if not with_history:
  278. exclude.append(DocumentTag.objects.get_history_tag(user=user))
  279. return Document.objects.get_docs(user, model_class).exclude(tags__in=exclude).order_by('-last_modified')
  280. def history_docs(self, model_class, user):
  281. include = [DocumentTag.objects.get_history_tag(user=user)]
  282. exclude = [DocumentTag.objects.get_trash_tag(user=user)]
  283. return Document.objects.get_docs(user, model_class).filter(tags__in=include).exclude(tags__in=exclude).order_by('-last_modified')
  284. def available(self, model_class, user, with_history=False):
  285. docs = self.available_docs(model_class, user, with_history)
  286. return [doc.content_object for doc in docs if doc.content_object]
  287. def can_read_or_exception(self, user, doc_class, doc_id, exception_class=PopupException):
  288. if doc_id is None:
  289. return
  290. try:
  291. ct = ContentType.objects.get_for_model(doc_class)
  292. doc = Document.objects.get(object_id=doc_id, content_type=ct)
  293. if doc.can_read(user):
  294. return doc
  295. else:
  296. message = _("Permission denied. %(username)s does not have the permissions required to access document %(id)s") % \
  297. {'username': user.username, 'id': doc.id}
  298. raise exception_class(message)
  299. except Document.DoesNotExist:
  300. raise exception_class(_('Document %(id)s does not exist') % {'id': doc_id})
  301. def can_read(self, user, doc_class, doc_id):
  302. ct = ContentType.objects.get_for_model(doc_class)
  303. doc = Document.objects.get(object_id=doc_id, content_type=ct)
  304. return doc.can_read(user)
  305. def link(self, content_object, owner, name='', description='', extra=''):
  306. if not content_object.doc.exists():
  307. doc = Document.objects.create(
  308. content_object=content_object,
  309. owner=owner,
  310. name=name,
  311. description=description,
  312. extra=extra
  313. )
  314. tag = DocumentTag.objects.get_default_tag(user=owner)
  315. doc.tags.add(tag)
  316. return doc
  317. else:
  318. LOG.warn('Object %s already has documents: %s' % (content_object, content_object.doc.all()))
  319. return content_object.doc.all()[0]
  320. def sync(self, doc2_only=True):
  321. def find_jobs_with_no_doc(model):
  322. jobs = model.objects.filter(doc__isnull=True)
  323. if model == Document2:
  324. jobs = jobs.exclude(is_history=True)
  325. return jobs.select_related('owner')
  326. def find_oozie_jobs_with_no_doc(model):
  327. return model.objects.filter(doc__isnull=True).exclude(name__exact='').select_related('owner')
  328. table_names = connection.introspection.table_names()
  329. try:
  330. from oozie.models import Workflow, Coordinator, Bundle
  331. if \
  332. Workflow._meta.db_table in table_names or \
  333. Coordinator._meta.db_table in table_names or \
  334. Bundle._meta.db_table in table_names:
  335. with transaction.atomic():
  336. for job in chain(
  337. find_oozie_jobs_with_no_doc(Workflow),
  338. find_oozie_jobs_with_no_doc(Coordinator),
  339. find_oozie_jobs_with_no_doc(Bundle)):
  340. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description)
  341. if job.is_trashed:
  342. doc.send_to_trash()
  343. if job.is_shared:
  344. doc.share_to_default()
  345. if hasattr(job, 'managed'):
  346. if not job.managed:
  347. doc.extra = 'jobsub'
  348. doc.save()
  349. except Exception as e:
  350. LOG.exception('error syncing oozie')
  351. try:
  352. from beeswax.models import SavedQuery
  353. if SavedQuery._meta.db_table in table_names:
  354. with transaction.atomic():
  355. for job in find_jobs_with_no_doc(SavedQuery):
  356. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
  357. if job.is_trashed:
  358. doc.send_to_trash()
  359. except Exception as e:
  360. LOG.exception('error syncing beeswax')
  361. try:
  362. from pig.models import PigScript
  363. if PigScript._meta.db_table in table_names:
  364. with transaction.atomic():
  365. for job in find_jobs_with_no_doc(PigScript):
  366. Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
  367. except Exception as e:
  368. LOG.exception('error syncing pig')
  369. try:
  370. from search.models import Collection
  371. if Collection._meta.db_table in table_names:
  372. with transaction.atomic():
  373. for dashboard in Collection.objects.all():
  374. if 'collection' in dashboard.properties_dict:
  375. col_dict = dashboard.properties_dict['collection']
  376. if not 'uuid' in col_dict:
  377. _uuid = str(uuid.uuid4())
  378. col_dict['uuid'] = _uuid
  379. dashboard.update_properties({'collection': col_dict})
  380. if dashboard.owner is None:
  381. from useradmin.models import install_sample_user
  382. owner = install_sample_user()
  383. else:
  384. owner = dashboard.owner
  385. dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
  386. Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
  387. dashboard.save()
  388. except Exception as e:
  389. LOG.exception('error syncing search')
  390. try:
  391. if Document2._meta.db_table in table_names:
  392. with transaction.atomic():
  393. for job in find_jobs_with_no_doc(Document2):
  394. if job.type == 'oozie-workflow2':
  395. extra = 'workflow2'
  396. elif job.type == 'oozie-coordinator2':
  397. extra = 'coordinator2'
  398. elif job.type == 'oozie-bundle2':
  399. extra = 'bundle2'
  400. elif job.type == 'notebook':
  401. extra = 'notebook'
  402. elif job.type == 'search-dashboard':
  403. extra = 'search-dashboard'
  404. else:
  405. extra = ''
  406. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description, extra=extra)
  407. except Exception as e:
  408. LOG.exception('error syncing Document2')
  409. if not doc2_only and Document._meta.db_table in table_names:
  410. # Make sure doc have at least a tag
  411. try:
  412. for doc in Document.objects.filter(tags=None):
  413. default_tag = DocumentTag.objects.get_default_tag(doc.owner)
  414. doc.tags.add(default_tag)
  415. except Exception as e:
  416. LOG.exception('error adding at least one tag to docs')
  417. # Make sure all the sample user documents are shared.
  418. try:
  419. with transaction.atomic():
  420. for doc in Document.objects.filter(owner__username__in=SAMPLE_USER_OWNERS):
  421. doc.share_to_default()
  422. tag = DocumentTag.objects.get_example_tag(user=doc.owner)
  423. doc.tags.add(tag)
  424. doc_last_modified = doc.last_modified
  425. doc.save()
  426. Document.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
  427. except Exception as e:
  428. LOG.exception('error sharing sample user documents')
  429. # For now remove the default tag from the examples
  430. try:
  431. for doc in Document.objects.filter(tags__tag=DocumentTag.EXAMPLE):
  432. default_tag = DocumentTag.objects.get_default_tag(doc.owner)
  433. doc.tags.remove(default_tag)
  434. except Exception as e:
  435. LOG.exception('error removing default tags')
  436. # ------------------------------------------------------------------------
  437. LOG.info('Looking for documents that have no object')
  438. # Delete documents with no object.
  439. try:
  440. with transaction.atomic():
  441. # First, delete all the documents that don't have a content type
  442. docs = Document.objects.filter(content_type=None)
  443. if docs:
  444. LOG.info('Deleting %s doc(s) that do not have a content type' % docs.count())
  445. docs.delete()
  446. # Next, it's possible that there are documents pointing at a non-existing
  447. # content_type. We need to do a left join to find these records, but we
  448. # can't do this directly in django. To get around writing wrap sql (which
  449. # might not be portable), we'll use an aggregate to count up all the
  450. # associated content_types, and delete the documents that have a count of
  451. # zero.
  452. #
  453. # Note we're counting `content_type__name` to force the join.
  454. docs = Document.objects \
  455. .values('id') \
  456. .annotate(content_type_count=models.Count('content_type__name')) \
  457. .filter(content_type_count=0)
  458. if docs:
  459. LOG.info('Deleting %s doc(s) that have invalid content types' % docs.count())
  460. docs.delete()
  461. # Finally we need to delete documents with no associated content object.
  462. # This is tricky because of our use of generic foreign keys. So to do
  463. # this a bit more efficiently, we'll start with a query of all the
  464. # documents, then step through each content type and and filter out all
  465. # the documents it's referencing from our document query. Messy, but it
  466. # works.
  467. # TODO: This can be several 100k entries for large databases.
  468. # Need to figure out a better way to handle this scenario.
  469. docs = Document.objects.all()
  470. for content_type in ContentType.objects.all():
  471. model_class = content_type.model_class()
  472. # Ignore any types that don't have a model.
  473. if model_class is None:
  474. continue
  475. # Ignore types that don't have a table yet.
  476. if model_class._meta.db_table not in table_names:
  477. continue
  478. # Ignore classes that don't have a 'doc'.
  479. if not hasattr(model_class, 'doc'):
  480. continue
  481. # First create a query that grabs all the document ids for this type.
  482. docs_from_content = model_class.objects.values('doc__id')
  483. # Next, filter these from our document query.
  484. docs = docs.exclude(id__in=docs_from_content)
  485. if docs.exists():
  486. LOG.info('Deleting %s documents' % docs.count())
  487. docs.delete()
  488. except Exception as e:
  489. LOG.exception('Error in sync while attempting to delete documents with no object: %s' % e)
  490. class Document(models.Model):
  491. owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
  492. name = models.CharField(default='', max_length=255)
  493. description = models.TextField(default='')
  494. last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Last modified'))
  495. version = models.SmallIntegerField(default=1, verbose_name=_t('Schema version'))
  496. extra = models.TextField(default='')
  497. tags = models.ManyToManyField(DocumentTag, db_index=True)
  498. content_type = models.ForeignKey(ContentType)
  499. object_id = models.PositiveIntegerField()
  500. content_object = GenericForeignKey('content_type', 'object_id')
  501. objects = DocumentManager()
  502. class Meta(object):
  503. unique_together = ('content_type', 'object_id')
  504. def __unicode__(self):
  505. return force_unicode('%s %s %s') % (self.content_type, self.name, self.owner)
  506. def is_editable(self, user):
  507. """Deprecated by can_read"""
  508. return self.can_write(user)
  509. def can_edit_or_exception(self, user, exception_class=PopupException):
  510. """Deprecated by can_write_or_exception"""
  511. return self.can_write_or_exception(user, exception_class)
  512. def add_tag(self, tag):
  513. self.tags.add(tag)
  514. def remove_tag(self, tag):
  515. self.tags.remove(tag)
  516. def is_trashed(self):
  517. return DocumentTag.objects.get_trash_tag(user=self.owner) in self.tags.all()
  518. def is_historic(self):
  519. return DocumentTag.objects.get_history_tag(user=self.owner) in self.tags.all()
  520. def send_to_trash(self):
  521. tag = DocumentTag.objects.get_trash_tag(user=self.owner)
  522. self.tags.add(tag)
  523. def restore_from_trash(self):
  524. tag = DocumentTag.objects.get_trash_tag(user=self.owner)
  525. self.tags.remove(tag)
  526. def add_to_history(self):
  527. tag = DocumentTag.objects.get_history_tag(user=self.owner)
  528. self.tags.add(tag)
  529. def remove_from_history(self):
  530. tag = DocumentTag.objects.get_history_tag(user=self.owner)
  531. self.tags.remove(tag)
  532. def share_to_default(self, name='read'):
  533. DocumentPermission.objects.share_to_default(self, name=name)
  534. def can_read(self, user):
  535. return is_admin(user) or self.owner == user or Document.objects.get_docs(user).filter(id=self.id).exists()
  536. def can_write(self, user):
  537. perm = self.list_permissions('write')
  538. return is_admin(user) or self.owner == user or perm.groups.filter(id__in=user.groups.all()).exists() or user in perm.users.all()
  539. def can_read_or_exception(self, user, exception_class=PopupException):
  540. if self.can_read(user):
  541. return True
  542. else:
  543. raise exception_class(_("Document does not exist or you don't have the permission to access it."))
  544. def can_write_or_exception(self, user, exception_class=PopupException):
  545. if self.can_write(user):
  546. return True
  547. else:
  548. raise exception_class(_("Document does not exist or you don't have the permission to access it."))
  549. def copy(self, content_object, name, owner, description=None):
  550. if content_object:
  551. copy_doc = self
  552. copy_doc.pk = None
  553. copy_doc.id = None
  554. copy_doc.name = name
  555. copy_doc.owner = owner
  556. if description:
  557. copy_doc.description = description
  558. copy_doc = Document.objects.link(content_object,
  559. owner=copy_doc.owner,
  560. name=copy_doc.name,
  561. description=copy_doc.description,
  562. extra=copy_doc.extra)
  563. # Update reverse Document relation to new copy
  564. if content_object.doc.get():
  565. content_object.doc.get().delete()
  566. content_object.doc.add(copy_doc)
  567. copy_doc.save()
  568. return copy_doc
  569. else:
  570. raise PopupException(_("Document copy method requires a content_object argument."))
  571. @property
  572. def icon(self):
  573. apps = appmanager.get_apps_dict()
  574. try:
  575. if self.extra == 'workflow2':
  576. return staticfiles_storage.url('oozie/art/icon_oozie_workflow_48.png')
  577. elif self.extra == 'coordinator2':
  578. return staticfiles_storage.url('oozie/art/icon_oozie_coordinator_48.png')
  579. elif self.extra == 'bundle2':
  580. return staticfiles_storage.url('oozie/art/icon_oozie_bundle_48.png')
  581. elif self.extra == 'notebook':
  582. return staticfiles_storage.url('notebook/art/icon_notebook_48.png')
  583. elif self.extra.startswith('query'):
  584. if self.extra == 'query-impala':
  585. return staticfiles_storage.url(apps['impala'].icon_path)
  586. else:
  587. return staticfiles_storage.url(apps['beeswax'].icon_path)
  588. elif self.extra.startswith('search'):
  589. return staticfiles_storage.url('search/art/icon_search_48.png')
  590. elif self.content_type.app_label == 'beeswax':
  591. if self.extra == '0':
  592. return staticfiles_storage.url(apps['beeswax'].icon_path)
  593. elif self.extra == '3':
  594. return staticfiles_storage.url(apps['spark'].icon_path)
  595. else:
  596. return staticfiles_storage.url(apps['impala'].icon_path)
  597. elif self.content_type.app_label == 'oozie':
  598. if self.extra == 'jobsub':
  599. return staticfiles_storage.url(apps['jobsub'].icon_path)
  600. else:
  601. return staticfiles_storage.url(self.content_type.model_class().ICON)
  602. elif self.content_type.app_label in apps:
  603. return staticfiles_storage.url(apps[self.content_type.app_label].icon_path)
  604. else:
  605. return staticfiles_storage.url('desktop/art/icon_hue_48.png')
  606. except Exception as e:
  607. LOG.warn(force_unicode(e))
  608. return staticfiles_storage.url('desktop/art/icon_hue_48.png')
  609. def share(self, users, groups, name='read'):
  610. DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=True)
  611. def unshare(self, users, groups, name='read'):
  612. DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=False)
  613. def sync_permissions(self, perms_dict):
  614. """
  615. Set who else or which other group can interact with the document.
  616. Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
  617. """
  618. for name, perm in perms_dict.items():
  619. users = groups = None
  620. if perm.get('user_ids'):
  621. users = User.objects.in_bulk(perm.get('user_ids'))
  622. else:
  623. users = []
  624. if perm.get('group_ids'):
  625. groups = Group.objects.in_bulk(perm.get('group_ids'))
  626. else:
  627. groups = []
  628. DocumentPermission.objects.sync(document=self, name=name, users=users, groups=groups)
  629. def list_permissions(self, perm='read'):
  630. return DocumentPermission.objects.list(document=self, perm=perm)
  631. def to_dict(self):
  632. return {
  633. 'owner': self.owner.username,
  634. 'name': self.name,
  635. 'description': self.description,
  636. 'uuid': None, # no uuid == v1
  637. 'id': self.id,
  638. 'doc1_id': self.id,
  639. 'object_id': self.object_id,
  640. 'type': str(self.content_type),
  641. 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT),
  642. 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()),
  643. 'isSelected': False
  644. }
  645. class DocumentPermissionManager(models.Manager):
  646. def _check_perm(self, name):
  647. perms = (DocumentPermission.READ_PERM, DocumentPermission.WRITE_PERM)
  648. if name not in perms:
  649. perms_string = ' and '.join(', '.join(perms).rsplit(', ', 1))
  650. raise PopupException(_('Only %s permissions are supported, not %s.') % (perms_string, name))
  651. def share_to_default(self, document, name='read'):
  652. from useradmin.models import get_default_user_group # Remove build dependency
  653. self._check_perm(name)
  654. if name == DocumentPermission.WRITE_PERM:
  655. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.WRITE_PERM)
  656. else:
  657. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.READ_PERM)
  658. default_group = get_default_user_group()
  659. if default_group:
  660. perm.groups.add(default_group)
  661. def update(self, document, name='read', users=None, groups=None, add=True):
  662. self._check_perm(name)
  663. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
  664. if users is not None:
  665. if add:
  666. perm.users.add(*users)
  667. else:
  668. perm.users.remove(*users)
  669. if groups is not None:
  670. if add:
  671. perm.groups.add(*groups)
  672. else:
  673. perm.groups.remove(*groups)
  674. if not perm.users and not perm.groups:
  675. perm.delete()
  676. def sync(self, document, name='read', users=None, groups=None):
  677. self._check_perm(name)
  678. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
  679. if users is not None:
  680. perm.users = []
  681. perm.users = users
  682. perm.save()
  683. if groups is not None:
  684. perm.groups = []
  685. perm.groups = groups
  686. perm.save()
  687. if not users and not groups:
  688. perm.delete()
  689. def list(self, document, perm='read'):
  690. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=perm)
  691. return perm
  692. class DocumentPermission(models.Model):
  693. READ_PERM = 'read'
  694. WRITE_PERM = 'write'
  695. doc = models.ForeignKey(Document)
  696. users = models.ManyToManyField(User, db_index=True, db_table='documentpermission_users')
  697. groups = models.ManyToManyField(Group, db_index=True, db_table='documentpermission_groups')
  698. perms = models.CharField(default=READ_PERM, max_length=10, choices=( # one perm
  699. (READ_PERM, 'read'),
  700. (WRITE_PERM, 'write'),
  701. ))
  702. objects = DocumentPermissionManager()
  703. class Meta(object):
  704. unique_together = ('doc', 'perms')
  705. ###################################################################################################
  706. # Document2
  707. ###################################################################################################
  708. class FilesystemException(Exception):
  709. pass
  710. class Document2QueryMixin(object):
  711. def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False, include_shared_links=False):
  712. """
  713. Returns all documents that are owned or shared with the user.
  714. :param perms: both, shared, owned. Defaults to both.
  715. :param include_history: boolean flag to return history documents. Defaults to False.
  716. :param include_trashed: boolean flag to return trashed documents. Defaults to True.
  717. :param include_managed: boolean flag to return documents generated by Hue. Defaults to False.
  718. :param include_shared_links: boolean flag to include documents shared by links. Defaults to False.
  719. """
  720. if perms == 'both':
  721. docs = self.filter(
  722. Q(owner=user) |
  723. Q(document2permission__users=user) |
  724. Q(document2permission__groups__in=user.groups.all()) |
  725. (Q(document2permission__is_link_on=include_shared_links) & Q(document2permission__is_link_on=True))
  726. )
  727. elif perms == 'shared':
  728. docs = self.filter(
  729. Q(document2permission__users=user) |
  730. Q(document2permission__groups__in=user.groups.all()) |
  731. (Q(document2permission__is_link_on=include_shared_links) & Q(document2permission__is_link_on=True))
  732. )
  733. else: # Only return documents owned by the user
  734. docs = self.filter(owner=user)
  735. if not include_history:
  736. docs = docs.exclude(is_history=True)
  737. if not include_managed:
  738. docs = docs.exclude(is_managed=True)
  739. if not include_trashed:
  740. docs = docs.exclude(is_trashed=True)
  741. return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
  742. def search_documents(self, types=None, search_text=None, order_by=None):
  743. """
  744. Search for documents based on type filters, search_text or order_by and return a queryset of document objects
  745. :param types: list of Document2 types (e.g. - query-hive, directory, etc)
  746. :param search_text: text to search on in the name and description fields
  747. :param order_by: order by field (e.g. -last_modified, type)
  748. """
  749. documents = self
  750. if types and isinstance(types, list) and types[0] != 'all':
  751. documents = documents.filter(type__in=types)
  752. if search_text:
  753. documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) | Q(search__icontains=search_text))
  754. if order_by: # TODO: Validate that order_by is a valid sort parameter
  755. documents = documents.order_by(order_by)
  756. return documents
  757. class Document2QuerySet(QuerySet, Document2QueryMixin):
  758. pass
  759. class Document2Manager(models.Manager, Document2QueryMixin):
  760. def get_queryset(self):
  761. """Make sure to restrict to only organization's documents"""
  762. return _fitered_queryset(
  763. Document2QuerySet(self.model, using=self._db),
  764. by_owner=True
  765. )
  766. # TODO prevent get() in favor of this
  767. def document(self, user, doc_id):
  768. return self.documents(user, include_trashed=True, include_history=True, include_shared_links=True).get(id=doc_id)
  769. def get_by_natural_key(self, uuid, version, is_history):
  770. return self.get(uuid=uuid, version=version, is_history=is_history)
  771. def get_by_uuid(self, user, uuid, perm_type='read'):
  772. """
  773. Since UUID is not a unique field, but part of a composite unique key, this returns the latest version by UUID.
  774. This should always be used in place of Document2.objects.get(uuid=) when a single document is expected.
  775. :param user: User to check permissions against
  776. :param uuid
  777. :param perm_type: permission type to check against
  778. """
  779. docs = self.filter(uuid=uuid).order_by('-last_modified')
  780. if not docs.exists():
  781. raise FilesystemException(_('Document with UUID %s not found.') % uuid)
  782. latest_doc = docs[0]
  783. if perm_type == 'write':
  784. latest_doc.can_write_or_exception(user)
  785. else:
  786. latest_doc.can_read_or_exception(user)
  787. return latest_doc
  788. def get_history(self, user, doc_type, include_trashed=False):
  789. return self.documents(user, perms='owned', include_history=True, include_trashed=include_trashed).filter(type=doc_type, is_history=True)
  790. def get_tasks_history(self, user):
  791. return self.documents(user, perms='owned', include_history=True, include_trashed=False, include_managed=True).filter(is_history=True, is_managed=True).exclude(name='pig-app-hue-script').exclude(type='oozie-workflow2')
  792. def get_home_directory(self, user):
  793. try:
  794. return self.get(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory')
  795. except Document2.DoesNotExist:
  796. return self.create_user_directories(user)
  797. except Document2.MultipleObjectsReturned:
  798. LOG.error('Multiple Home directories detected. Merging all into one.')
  799. home_dirs = list(self.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory').order_by('-last_modified'))
  800. parent_home_dir = home_dirs.pop()
  801. for dir in home_dirs:
  802. dir.children.exclude(name='.Trash').update(parent_directory=parent_home_dir)
  803. dir.delete()
  804. return parent_home_dir
  805. def get_gist_directory(self, user):
  806. home_dir = self.get_home_directory(user)
  807. gist_dir, created = Directory.objects.get_or_create(name=Document2.GIST_DIR, owner=user, parent_directory=home_dir)
  808. if created:
  809. LOG.info('Successfully created gist directory for user: %s' % user.username)
  810. return gist_dir
  811. def get_by_path(self, user, path):
  812. """
  813. This can be an expensive operation b/c we have to traverse the path tree, so if possible, request a document by UUID
  814. NOTE: get_by_path only works for the owner's documents since it is based off the user's home directory
  815. """
  816. cleaned_path = path.rstrip('/')
  817. doc = Document2.objects.get_home_directory(user)
  818. if cleaned_path:
  819. path_tokens = cleaned_path.split('/')[1:]
  820. for token in path_tokens:
  821. try:
  822. doc = doc.children.get(name=token)
  823. except Document2.DoesNotExist:
  824. raise FilesystemException(_('Requested invalid path for user %s: %s') % (user.username, path))
  825. except Document2.MultipleObjectsReturned:
  826. raise FilesystemException(_('Duplicate documents found for user %s at path: %s') % (user.username, path))
  827. doc.can_read_or_exception(user)
  828. return doc
  829. def create_user_directories(self, user):
  830. """
  831. Creates user home and Trash directories if they do not exist and move any orphan documents to home directory
  832. :param user: User object
  833. """
  834. # Edge-case if the user has a legacy home directory with path-name
  835. Directory.objects.filter(name='/', owner=user).update(name=Document2.HOME_DIR)
  836. # Get or create home and Trash directories for all users
  837. home_dir, created = Directory.objects.get_or_create(name=Document2.HOME_DIR, owner=user)
  838. if created:
  839. LOG.info('Successfully created home directory for user: %s' % user.username)
  840. trash_dir, created = Directory.objects.get_or_create(name=Document2.TRASH_DIR, owner=user, parent_directory=home_dir)
  841. if created:
  842. LOG.info('Successfully created trash directory for user: %s' % user.username)
  843. # For any directories or documents that do not have a parent directory, assign it to home directory
  844. count = 0
  845. for doc in Document2.objects.filter(owner=user).filter(parent_directory=None).exclude(id__in=[home_dir.id, trash_dir.id]):
  846. doc.parent_directory = home_dir
  847. doc.save()
  848. count += 1
  849. LOG.info("Moved %d documents to home directory for user: %s" % (count, user.username))
  850. return home_dir
  851. class Document2(models.Model):
  852. HOME_DIR = ''
  853. GIST_DIR = 'Gist'
  854. TRASH_DIR = '.Trash'
  855. EXAMPLES_DIR = 'examples'
  856. owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Creator.'), related_name='doc2_owner')
  857. name = models.CharField(default='', max_length=255)
  858. description = models.TextField(default='')
  859. uuid = models.CharField(default=uuid_default, max_length=36, db_index=True)
  860. type = models.CharField(default='', max_length=32, db_index=True, help_text=_t('Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...'))
  861. data = models.TextField(default='{}')
  862. extra = models.TextField(default='')
  863. search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
  864. # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
  865. last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
  866. version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
  867. is_history = models.BooleanField(default=False, db_index=True)
  868. is_managed = models.BooleanField(default=False, db_index=True, verbose_name=_t('If managed under the cover by Hue and never by the user')) # Aka isTask
  869. is_trashed = models.NullBooleanField(default=False, db_index=True, verbose_name=_t('True if trashed'))
  870. dependencies = models.ManyToManyField('self', symmetrical=False, related_name='dependents', db_index=True)
  871. parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
  872. doc = GenericRelation(Document, related_query_name='doc_doc') # Compatibility with Hue 3
  873. objects = Document2Manager()
  874. class Meta(object):
  875. unique_together = ('uuid', 'version', 'is_history')
  876. ordering = ["-last_modified", "name"]
  877. def __unicode__(self):
  878. res = '%s - %s - %s - %s' % (force_unicode(self.name), self.owner, self.type, self.uuid)
  879. return force_unicode(res)
  880. @property
  881. def data_dict(self):
  882. if not self.data:
  883. self.data = json.dumps({})
  884. data_python = json.loads(self.data)
  885. return data_python
  886. @property
  887. def path(self):
  888. quoted_name = urllib_quote(self.name.encode('utf-8'))
  889. if self.parent_directory:
  890. return '%s/%s' % (self.parent_directory.path, quoted_name)
  891. else:
  892. return quoted_name
  893. @property
  894. def dirname(self):
  895. return os.path.dirname(self.path) or '/'
  896. @property
  897. def is_directory(self):
  898. return self.type == 'directory'
  899. @property
  900. def is_home_directory(self):
  901. return self.is_directory and self.parent_directory == None and self.name == self.HOME_DIR
  902. @property
  903. def is_trash_directory(self):
  904. return self.is_directory and self.name == self.TRASH_DIR
  905. def natural_key(self):
  906. return (self.uuid, self.version, self.is_history)
  907. def copy(self, name, owner, description=None):
  908. copy_doc = self
  909. copy_doc.pk = None
  910. copy_doc.id = None
  911. copy_doc.uuid = uuid_default()
  912. copy_doc.name = name
  913. copy_doc.owner = owner
  914. if description:
  915. copy_doc.description = description
  916. copy_doc.save()
  917. return copy_doc
  918. def update_data(self, post_data):
  919. data_dict = self.data_dict
  920. data_dict.update(post_data)
  921. self.data = json.dumps(data_dict)
  922. def get_absolute_url(self):
  923. url = None
  924. try:
  925. if self.type == 'oozie-coordinator2':
  926. url = reverse('oozie:edit_coordinator') + '?coordinator=' + str(self.id)
  927. elif self.type == 'oozie-bundle2':
  928. url = reverse('oozie:edit_bundle') + '?bundle=' + str(self.id)
  929. elif self.type.startswith('query'):
  930. url = '/editor' + '?editor=' + str(self.id)
  931. elif self.type == 'directory':
  932. url = '/home2' + '?uuid=' + self.uuid
  933. elif self.type == 'gist':
  934. url = '/hue/gist?uuid=' + str(self.uuid)
  935. elif self.type == 'notebook':
  936. url = reverse('notebook:notebook') + '?notebook=' + str(self.id)
  937. elif self.type == 'search-dashboard':
  938. url = reverse('search:index') + '?collection=' + str(self.id)
  939. elif self.type == 'link-pigscript':
  940. url = reverse('pig:index') + '#edit/%s' % self.data_dict.get('object_id', '')
  941. elif self.type == 'link-workflow':
  942. url = '/jobsub/#edit-design/%s' % self.data_dict.get('object_id', '')
  943. else:
  944. url = reverse('oozie:edit_workflow') + '?workflow=' + str(self.id)
  945. except NoReverseMatch:
  946. LOG.warn('Could not perform reverse lookup for type %s, app may be blacklisted.' % self.type)
  947. return url
  948. def to_dict(self):
  949. return {
  950. 'owner': self.owner.username,
  951. 'name': self.name,
  952. 'path': self.path or '/',
  953. 'description': self.description,
  954. 'uuid': self.uuid,
  955. 'id': self.id,
  956. 'doc1_id': self.doc.get().id if self.doc.exists() else -1,
  957. 'parent_uuid': self.parent_directory.uuid if self.parent_directory else None,
  958. 'type': self.type,
  959. 'perms': self._massage_permissions(),
  960. 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT) if self.last_modified else None,
  961. 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()) if self.last_modified else None,
  962. 'is_managed': self.is_managed,
  963. 'isSelected': False,
  964. 'absoluteUrl': self.get_absolute_url(),
  965. }
  966. def get_history(self):
  967. return self.dependencies.filter(is_history=True).order_by('-last_modified')
  968. def add_to_history(self, user, data_dict):
  969. doc_id = self.id # Need to copy as the clone messes it
  970. history_doc = self.copy(name=self.name, owner=user)
  971. history_doc.update_data({'history': data_dict})
  972. history_doc.is_history = True
  973. history_doc.last_modified = None
  974. history_doc.save()
  975. Document2.objects.get(id=doc_id).dependencies.add(history_doc)
  976. return history_doc
  977. def save(self, *args, **kwargs):
  978. # Set document parent to home directory if parent directory isn't specified
  979. if not self.parent_directory and not self.is_home_directory and not self.is_trash_directory:
  980. home_dir = Document2.objects.get_home_directory(self.owner)
  981. self.parent_directory = home_dir
  982. # Run validations
  983. self.validate()
  984. # Redact query if needed
  985. self._redact_query()
  986. if self.search:
  987. self.search = self.search[:DOCUMENT2_SEARCH_MAX_LENGTH]
  988. super(Document2, self).save(*args, **kwargs)
  989. # Inherit shared permissions from parent directory, must be done after save b/c new doc needs ID
  990. self.inherit_permissions()
  991. def validate(self):
  992. # Validate home and Trash directories are only created once per user and cannot be created or modified after
  993. if self.name in [Document2.HOME_DIR, Document2.TRASH_DIR] and self.type == 'directory' and \
  994. Document2.objects.filter(name=self.name, owner=self.owner, type='directory').exists():
  995. raise FilesystemException(_('Cannot create or modify directory with name: %s') % self.name)
  996. # Validate that parent directory does not create cycle
  997. if self._contains_cycle():
  998. raise FilesystemException(_('Cannot save document %s under parent directory %s due to circular dependency') %
  999. (self.name, self.parent_directory.uuid))
  1000. def inherit_permissions(self):
  1001. if self.parent_directory is not None:
  1002. parent_perms = Document2Permission.objects.filter(doc=self.parent_directory)
  1003. for perm in parent_perms:
  1004. self.share(self.owner, name=perm.perms, users=perm.users.all(), groups=perm.groups.all())
  1005. def move(self, directory, user):
  1006. if not directory.is_directory:
  1007. raise FilesystemException(_('Target with UUID %s is not a directory') % directory.uuid)
  1008. if directory.can_write_or_exception(user=user):
  1009. # Restore last_modified date after save
  1010. original_last_modified = self.last_modified
  1011. self.parent_directory = directory
  1012. self.save()
  1013. # Use update instead of save() so that last_modified date is not modified automatically
  1014. Document2.objects.filter(id=self.id).update(last_modified=original_last_modified)
  1015. return self
  1016. def trash(self):
  1017. try:
  1018. trash_dir = Directory.objects.get(name=self.TRASH_DIR, owner=self.owner)
  1019. self.move(trash_dir, self.owner)
  1020. self.is_trashed = True
  1021. self.save()
  1022. if self.is_directory:
  1023. children_ids = self._get_child_ids_recursively(self.id)
  1024. Document2.objects.filter(id__in=children_ids).update(is_trashed=True)
  1025. except Document2.MultipleObjectsReturned:
  1026. LOG.error('Multiple Trash directories detected. Merging all into one.')
  1027. trash_dirs = list(Directory.objects.filter(name=self.TRASH_DIR, owner=self.owner).order_by('-last_modified'))
  1028. parent_trash_dir = trash_dirs.pop()
  1029. for dir in trash_dirs:
  1030. dir.children.update(parent_directory=parent_trash_dir)
  1031. dir.delete()
  1032. self.move(parent_trash_dir, self.owner)
  1033. def restore(self):
  1034. # Currently restoring any doucment to /home
  1035. home_dir = Document2.objects.get_home_directory(self.owner)
  1036. self.move(home_dir, self.owner)
  1037. self.is_trashed = False
  1038. self.save()
  1039. if self.is_directory:
  1040. children_ids = self._get_child_ids_recursively(self.id)
  1041. Document2.objects.filter(id__in=children_ids).update(is_trashed=False)
  1042. def _get_child_ids_recursively(self, directory_id):
  1043. """
  1044. Returns the list of all children ids for a given directory id recursively, excluding history documents
  1045. """
  1046. directory = Directory.objects.get(id=directory_id)
  1047. children_ids = []
  1048. for child in directory.children.filter(is_history=False).filter(is_managed=False):
  1049. children_ids.append(child.id)
  1050. if child.is_directory:
  1051. children_ids.extend(self._get_child_ids_recursively(child.id))
  1052. return children_ids
  1053. def can_read(self, user):
  1054. return \
  1055. self.owner == user \
  1056. or is_admin(user) \
  1057. or any([perm.user_has_access(user) for perm in self.get_permissions('read')]) \
  1058. or self.can_write(user)
  1059. def can_read_or_exception(self, user):
  1060. if self.can_read(user):
  1061. return True
  1062. else:
  1063. raise PopupException(_("Document does not exist or you don't have the permission to access it."), error_code=401)
  1064. def can_write(self, user):
  1065. return \
  1066. self.owner == user \
  1067. or is_admin(user) \
  1068. or any([perm.user_has_access(user) for perm in self.get_permissions('write')]) \
  1069. or (self.parent_directory and self.parent_directory.can_write(user))
  1070. def can_write_or_exception(self, user):
  1071. if self.can_write(user):
  1072. return True
  1073. else:
  1074. raise PopupException(_("Document does not exist or you don't have the permission to access it."))
  1075. def get_permission(self, perm='read'):
  1076. try:
  1077. return Document2Permission.objects.get(doc=self, perms=perm)
  1078. except Document2Permission.DoesNotExist:
  1079. return None
  1080. def get_permissions(self, perm='read'):
  1081. '''
  1082. Return the sub permissions that make one of the two top privileges.
  1083. e.g. 'read' and 'link_read' perms give the global 'read' privilege.
  1084. '''
  1085. if perm == 'read':
  1086. perms = Q(perms=Document2Permission.READ_PERM) | (Q(perms=Document2Permission.LINK_READ_PERM) & Q(is_link_on=True))
  1087. elif perm == 'write':
  1088. perms = Q(perms=Document2Permission.WRITE_PERM) | (Q(perms=Document2Permission.LINK_WRITE_PERM) & Q(is_link_on=True))
  1089. else:
  1090. raise PopupException(_("Permission name %s is invalid.") % perm)
  1091. return Document2Permission.objects.filter(Q(doc=self) & perms)
  1092. def share(self, user, name='read', users=None, groups=None, is_link_on=False):
  1093. try:
  1094. with transaction.atomic():
  1095. self.update_permission(user, name, users, groups, is_link_on)
  1096. # For directories, update all children recursively with same permissions
  1097. for child in self.children.all():
  1098. child.share(user, name, users, groups, is_link_on)
  1099. except Exception as e:
  1100. raise PopupException(_("Failed to share document: %s") % e)
  1101. return self
  1102. def share_link(self, user, perm='read'):
  1103. if perm == 'read':
  1104. self.share(user, name=Document2Permission.LINK_READ_PERM, is_link_on=True)
  1105. self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=False)
  1106. elif perm == 'write':
  1107. self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=True)
  1108. else:
  1109. self.share(user, name=Document2Permission.LINK_READ_PERM, is_link_on=False)
  1110. self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=False)
  1111. return self
  1112. def update_permission(self, user, name='read', users=None, groups=None, is_link_on=False):
  1113. # Check if user has access to grant permissions
  1114. if users or groups:
  1115. if name == 'read' or name == 'link_read':
  1116. self.can_read_or_exception(user)
  1117. elif name == 'write' or name == 'link_write':
  1118. self.can_write_or_exception(user)
  1119. else:
  1120. raise ValueError(_('Invalid permission type: %s') % name)
  1121. perm, created = Document2Permission.objects.get_or_create(doc=self, perms=name)
  1122. perm.users = []
  1123. if users is not None:
  1124. perm.users = users
  1125. perm.groups = []
  1126. if groups is not None:
  1127. perm.groups = groups
  1128. perm.is_link_on = is_link_on
  1129. perm.save()
  1130. def _get_doc1(self, doc2_type=None):
  1131. if not doc2_type:
  1132. doc2_type = self.type
  1133. try:
  1134. doc = self.doc.get()
  1135. except Exception as e:
  1136. LOG.error('Exception when retrieving document object for saved query: %s' % e)
  1137. doc = Document.objects.link(
  1138. self,
  1139. owner=self.owner,
  1140. name=self.name,
  1141. description=self.description,
  1142. extra=doc2_type
  1143. )
  1144. return doc
  1145. def _massage_permissions(self):
  1146. """
  1147. Returns the permissions for a given document as a dictionary
  1148. """
  1149. permissions = {
  1150. 'read': {'users': [], 'groups': []},
  1151. 'write': {'users': [], 'groups': []},
  1152. 'link_sharing_on': False,
  1153. 'link_read': False,
  1154. 'link_write': False,
  1155. }
  1156. read_perm = self.get_permission(perm='read')
  1157. write_perm = self.get_permission(perm='write')
  1158. if read_perm:
  1159. permissions.update(read_perm.to_dict())
  1160. if write_perm:
  1161. permissions.update(write_perm.to_dict())
  1162. link_read_perm = self.get_permission(perm=Document2Permission.LINK_READ_PERM)
  1163. link_write_perm = self.get_permission(perm=Document2Permission.LINK_WRITE_PERM)
  1164. if link_read_perm:
  1165. permissions['link_read'] = link_read_perm.is_link_on
  1166. if link_write_perm:
  1167. permissions['link_write'] = link_write_perm.is_link_on
  1168. permissions['link_sharing_on'] = permissions['link_read'] or permissions['link_write']
  1169. return permissions
  1170. def _redact_query(self):
  1171. """
  1172. Optionally mask out the query from being saved to the database. This is because if the database contains sensitive
  1173. information like personally identifiable information, that information could be leaked into the Hue database and
  1174. logfiles.
  1175. """
  1176. if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
  1177. data_dict = self.data_dict
  1178. snippets = data_dict.get('snippets', [])
  1179. for snippet in snippets:
  1180. if snippet['type'] in ('hive', 'impala'): # TODO: Pull SQL types from canonical lookup
  1181. redacted_statement_raw = global_redaction_engine.redact(snippet['statement_raw'])
  1182. if snippet['statement_raw'] != redacted_statement_raw:
  1183. snippet['statement_raw'] = redacted_statement_raw
  1184. snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
  1185. snippet['is_redacted'] = True
  1186. self.data = json.dumps(data_dict)
  1187. self.search = global_redaction_engine.redact(self.search)
  1188. def _contains_cycle(self):
  1189. """
  1190. Uses Floyd's cycle-detection algorithm to detect a cycle (aka Tortoise and Hare)
  1191. https://en.wikipedia.org/wiki/Cycle_detection#Tortoise_and_hare
  1192. """
  1193. # Test base case where self.uuid == self.parent_directory.uuid first
  1194. if self.parent_directory is not None and self.parent_directory.uuid == self.uuid:
  1195. return True
  1196. slow = self
  1197. fast = self
  1198. while True:
  1199. slow = slow.parent_directory
  1200. if slow and slow.uuid == self.uuid:
  1201. slow = self
  1202. if fast.parent_directory is not None:
  1203. if fast.parent_directory.uuid == self.uuid:
  1204. fast = self.parent_directory.parent_directory
  1205. else:
  1206. fast = fast.parent_directory.parent_directory
  1207. else:
  1208. return False
  1209. if slow is None or fast is None:
  1210. return False
  1211. if slow == fast:
  1212. return True
  1213. class DirectoryManager(Document2Manager):
  1214. def get_queryset(self):
  1215. return super(DirectoryManager, self).get_queryset().filter(type='directory')
  1216. class Directory(Document2):
  1217. # e.g. name = '/' or '/dir1/dir2/f3'
  1218. objects = DirectoryManager()
  1219. class Meta(object):
  1220. proxy = True
  1221. def get_children_documents(self):
  1222. """
  1223. Returns the children documents for a given directory, excluding history documents
  1224. """
  1225. documents = self.children.filter(is_history=False).filter(is_managed=False) # TODO: perms
  1226. return documents
  1227. def get_children_and_shared_documents(self, user):
  1228. """
  1229. Returns the children and shared documents for a given directory, excluding history documents
  1230. """
  1231. # Get documents that are direct children, or shared with but not owned by the current user
  1232. documents = Document2.objects.filter(
  1233. Q(parent_directory=self) |
  1234. ( (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
  1235. ~Q(owner=user) )
  1236. )
  1237. documents = documents.exclude(is_history=True).exclude(is_managed=True)
  1238. # Excluding all trashed docs across users
  1239. documents = documents.exclude(is_trashed=True)
  1240. # Optimizing roll up for /home by checking only with directories instead of all documents
  1241. # For all other directories roll up is done in _filter_documents()
  1242. directories_all = Document2.objects.filter(type='directory').exclude(id=self.id)
  1243. directories_inside_home = directories_all.filter(
  1244. (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
  1245. ~Q(owner=user)
  1246. )
  1247. documents = documents.exclude(parent_directory__in=directories_inside_home)
  1248. return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
  1249. def save(self, *args, **kwargs):
  1250. self.type = 'directory'
  1251. super(Directory, self).save(*args, **kwargs)
  1252. class Document2Permission(models.Model):
  1253. """
  1254. Combine either:
  1255. - regular perms (listed)
  1256. - link
  1257. """
  1258. READ_PERM = 'read'
  1259. WRITE_PERM = 'write'
  1260. COMMENT_PERM = 'comment'
  1261. LINK_READ_PERM = 'link_read'
  1262. LINK_WRITE_PERM = 'link_write'
  1263. doc = models.ForeignKey(Document2)
  1264. users = models.ManyToManyField(User, db_index=True, db_table='documentpermission2_users')
  1265. groups = models.ManyToManyField(Group, db_index=True, db_table='documentpermission2_groups')
  1266. perms = models.CharField(default=READ_PERM, max_length=10, db_index=True, choices=( # One perm
  1267. (READ_PERM, 'read'),
  1268. (WRITE_PERM, 'write'),
  1269. (COMMENT_PERM, 'comment'), # Unused
  1270. (LINK_READ_PERM, 'link read'),
  1271. (LINK_WRITE_PERM, 'link write'),
  1272. ))
  1273. is_link_on = models.BooleanField(default=False)
  1274. class Meta(object):
  1275. unique_together = ('doc', 'perms')
  1276. def __str__(self):
  1277. return force_unicode('Document: %s, Name: %s, Users: %s, Groups: %s, is_link_on: %s') % (
  1278. self.doc, self.perms, self.users, self.groups, self.is_link_on
  1279. )
  1280. def to_dict(self):
  1281. return {
  1282. self.perms: {
  1283. 'users': [{'id': perm_user.id, 'username': perm_user.username} for perm_user in self.users.all()],
  1284. 'groups': [{'id': perm_group.id, 'name': perm_group.name} for perm_group in self.groups.all()]
  1285. }
  1286. }
  1287. def user_has_access(self, user):
  1288. """
  1289. Returns true if the given user has permissions based on users, groups or all flag.
  1290. """
  1291. return self.is_link_on or user in self.users.all() or self.groups.filter(id__in=user.groups.all()).exists()
  1292. def get_cluster_config(user):
  1293. return Cluster(user).get_app_config().get_config()
  1294. class ClusterConfig(object):
  1295. """
  1296. Configuration of the apps and engines that each individual user sees on the core Hue.
  1297. Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
  1298. TODO: rename to HueConfig
  1299. TODO: get list of contexts dynamically
  1300. """
  1301. def __init__(self, user, apps=None, cluster_type='ini'):
  1302. self.user = user
  1303. self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
  1304. self.cluster_type = cluster_type
  1305. def get_config(self):
  1306. app_config = self.get_apps()
  1307. editors = app_config.get('editor')
  1308. main_button_action = self.get_main_quick_action(app_config)
  1309. if main_button_action.get('is_sql'):
  1310. default_sql_interpreter = main_button_action['type']
  1311. else:
  1312. default_sql_interpreter = editors and editors['default_sql_interpreter']
  1313. return {
  1314. 'app_config': app_config,
  1315. 'main_button_action': main_button_action,
  1316. 'button_actions': [
  1317. app for app in [
  1318. editors,
  1319. app_config.get('dashboard'),
  1320. app_config.get('scheduler')
  1321. ] if app is not None
  1322. ],
  1323. 'default_sql_interpreter': default_sql_interpreter,
  1324. 'cluster_type': self.cluster_type,
  1325. 'has_computes': self.cluster_type in ('altus', 'snowball') # or any grouped engine connectors
  1326. }
  1327. def get_apps(self):
  1328. apps = OrderedDict([
  1329. app for app in [
  1330. ('editor', self._get_editor()),
  1331. ('dashboard', self._get_dashboard()),
  1332. ('catalogs', self._get_catalogs()),
  1333. ('browser', self._get_browser()),
  1334. ('scheduler', self._get_scheduler()),
  1335. ('sdkapps', self._get_sdk_apps()),
  1336. ('home', self._get_home()),
  1337. ] if app[1]
  1338. ])
  1339. return apps
  1340. def get_main_quick_action(self, apps):
  1341. if not apps:
  1342. raise PopupException(_('No permission to any app.'))
  1343. default_app = list(apps.values())[0]
  1344. default_interpreter = default_app.get('interpreters')
  1345. try:
  1346. user_default_app = json.loads(UserPreferences.objects.get(user=self.user, key='default_app').value)
  1347. if apps.get(user_default_app['app']):
  1348. default_interpreter = []
  1349. default_app = apps[user_default_app['app']]
  1350. if default_app.get('interpreters'):
  1351. interpreters = [interpreter for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']]
  1352. if interpreters:
  1353. default_interpreter = interpreters
  1354. except UserPreferences.DoesNotExist:
  1355. pass
  1356. except Exception:
  1357. LOG.exception('Could not load back default app')
  1358. if default_interpreter:
  1359. return default_interpreter[0]
  1360. else:
  1361. return default_app
  1362. def _get_home(self):
  1363. return {
  1364. 'name': 'home',
  1365. 'displayName': _('Home'),
  1366. 'buttonName': _('Saved Queries') if has_connectors() else _('Documents'),
  1367. 'interpreters': [],
  1368. 'page': '/home'
  1369. }
  1370. def _get_editor(self):
  1371. interpreters = []
  1372. _interpreters = get_ordered_interpreters(self.user)
  1373. for interpreter in _interpreters:
  1374. if interpreter['interface'] != 'hms':
  1375. interpreters.append({
  1376. 'name': interpreter['name'],
  1377. 'type': interpreter['type'],
  1378. 'displayName': interpreter['name'],
  1379. 'buttonName': _('Query'),
  1380. 'tooltip': _('%s Query') % interpreter['type'].title(),
  1381. 'page': '/editor/?type=%(type)s' % interpreter,
  1382. 'is_sql': interpreter['is_sql'],
  1383. 'dialect': interpreter['dialect'],
  1384. 'dialect_properties': interpreter.get('dialect_properties'),
  1385. })
  1386. if SHOW_NOTEBOOKS.get():
  1387. try:
  1388. first_non_sql_index = [interpreter['is_sql'] for interpreter in interpreters].index(False)
  1389. except ValueError:
  1390. first_non_sql_index = len(interpreters) if all([interpreter['is_sql'] for interpreter in interpreters]) else 0
  1391. interpreters.insert(first_non_sql_index, {
  1392. 'name': 'notebook',
  1393. 'type': 'notebook',
  1394. 'displayName': 'Notebook',
  1395. 'buttonName': _('Notebook'),
  1396. 'tooltip': _('Notebook'),
  1397. 'page': '/notebook',
  1398. 'is_sql': False,
  1399. 'dialect': 'notebook'
  1400. })
  1401. if interpreters:
  1402. return {
  1403. 'name': 'editor',
  1404. 'displayName': _('Editor'),
  1405. 'buttonName': _('Query'),
  1406. 'interpreters': interpreters,
  1407. 'default_limit': None if DEFAULT_LIMIT.get() == 0 else DEFAULT_LIMIT.get(),
  1408. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1409. 'page': interpreters[0]['page'],
  1410. 'default_sql_interpreter': next((interpreter['type'] for interpreter in interpreters if interpreter.get('is_sql')), 'hive')
  1411. }
  1412. else:
  1413. return None
  1414. def _get_catalogs(self):
  1415. '''
  1416. Any connector implementing Notebook Assist API.
  1417. Currently SQL connectors, Hive Metastore. Possibly Metadata Catalogs later.
  1418. '''
  1419. interpreters = []
  1420. _interpreters = get_ordered_interpreters(self.user)
  1421. for interpreter in _interpreters:
  1422. if interpreter['is_sql'] or interpreter['interface'] == 'hms':
  1423. interpreters.append({
  1424. 'name': interpreter['name'],
  1425. 'type': interpreter['type'],
  1426. 'displayName': interpreter['name'],
  1427. 'buttonName': _('Query'),
  1428. 'tooltip': _('%s Query') % interpreter['type'].title(),
  1429. 'page': '/editor/?type=%(type)s' % interpreter,
  1430. 'is_sql': interpreter['is_sql'],
  1431. 'is_catalog': True,
  1432. })
  1433. return interpreters if interpreters else None
  1434. def _get_dashboard(self):
  1435. interpreters = get_engines(self.user)
  1436. _interpreters = []
  1437. if interpreters:
  1438. if HAS_REPORT_ENABLED.get():
  1439. _interpreters.append({
  1440. 'type': 'report',
  1441. 'displayName': 'Report',
  1442. 'buttonName': 'Report',
  1443. 'page': '/dashboard/new_search?engine=report',
  1444. 'tooltip': _('Report'),
  1445. 'is_sql': False
  1446. })
  1447. _interpreters.extend([{
  1448. 'type': interpreter['type'],
  1449. 'displayName': interpreter['type'].title(),
  1450. 'buttonName': interpreter['type'].title(),
  1451. 'page': '/dashboard/new_search?engine=%(type)s' % interpreter,
  1452. 'tooltip': _('%s Dashboard') % interpreter['type'].title(),
  1453. 'is_sql': True
  1454. } for interpreter in interpreters
  1455. ])
  1456. return {
  1457. 'name': 'dashboard',
  1458. 'displayName': _('Dashboard'),
  1459. 'buttonName': _('Dashboard'),
  1460. 'interpreters': _interpreters,
  1461. 'page': '/dashboard/new_search?engine=%(type)s' % interpreters[0]
  1462. }
  1463. else:
  1464. return None
  1465. def _get_browser(self):
  1466. interpreters = []
  1467. hdfs_connectors = []
  1468. if has_connectors():
  1469. hdfs_connectors = [
  1470. connector['nice_name'] for connector in _get_installed_connectors(category='browsers', dialect='hdfs', interface='rest')
  1471. ]
  1472. elif 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('hdfs', self.user):
  1473. hdfs_connectors.append(_('Files'))
  1474. for hdfs_connector in hdfs_connectors:
  1475. interpreters.append({
  1476. 'type': 'hdfs',
  1477. 'displayName': hdfs_connector,
  1478. 'buttonName': _('Browse'),
  1479. 'tooltip': hdfs_connector,
  1480. 'page': '/filebrowser/' + (
  1481. not self.user.is_anonymous() and
  1482. 'view=' + urllib_quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
  1483. )
  1484. })
  1485. if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('s3a', self.user):
  1486. interpreters.append({
  1487. 'type': 's3',
  1488. 'displayName': _('S3'),
  1489. 'buttonName': _('Browse'),
  1490. 'tooltip': _('S3'),
  1491. 'page': '/filebrowser/view=' + urllib_quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1492. })
  1493. if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('adl', self.user):
  1494. interpreters.append({
  1495. 'type': 'adls',
  1496. 'displayName': _('ADLS'),
  1497. 'buttonName': _('Browse'),
  1498. 'tooltip': _('ADLS'),
  1499. 'page': '/filebrowser/view=' + urllib_quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1500. })
  1501. if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
  1502. from azure.abfs.__init__ import get_home_dir_for_ABFS
  1503. interpreters.append({
  1504. 'type': 'abfs',
  1505. 'displayName': _('ABFS'),
  1506. 'buttonName': _('Browse'),
  1507. 'tooltip': _('ABFS'),
  1508. 'page': '/filebrowser/view=' + urllib_quote(get_home_dir_for_ABFS().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1509. })
  1510. if 'metastore' in self.apps:
  1511. interpreters.append({
  1512. 'type': 'tables',
  1513. 'displayName': _('Tables'),
  1514. 'buttonName': _('Browse'),
  1515. 'tooltip': _('Tables'),
  1516. 'page': '/metastore/tables'
  1517. })
  1518. if 'search' in self.apps:
  1519. interpreters.append({
  1520. 'type': 'indexes',
  1521. 'displayName': _('Indexes'),
  1522. 'buttonName': _('Dashboard'),
  1523. 'tooltip': _('Indexes'),
  1524. 'page': '/indexer/'
  1525. })
  1526. if 'jobbrowser' in self.apps:
  1527. from hadoop.cluster import get_default_yarncluster # Circular loop
  1528. if get_default_yarncluster():
  1529. interpreters.append({
  1530. 'type': 'yarn',
  1531. 'displayName': _('Jobs'),
  1532. 'buttonName': _('Jobs'),
  1533. 'tooltip': _('Jobs'),
  1534. 'page': '/jobbrowser/'
  1535. })
  1536. if has_kafka():
  1537. interpreters.append({
  1538. 'type': 'kafka',
  1539. 'displayName': _('Streams'),
  1540. 'buttonName': _('Browse'),
  1541. 'tooltip': _('Kafka'),
  1542. 'page': '/kafka/'
  1543. })
  1544. if 'hbase' in self.apps:
  1545. interpreters.append({
  1546. 'type': 'hbase',
  1547. 'displayName': _('HBase'),
  1548. 'buttonName': _('Browse'),
  1549. 'tooltip': _('HBase'),
  1550. 'page': '/hbase/'
  1551. })
  1552. if 'security' in self.apps:
  1553. interpreters.append({
  1554. 'type': 'security',
  1555. 'displayName': _('Security'),
  1556. 'buttonName': _('Browse'),
  1557. 'tooltip': _('Security'),
  1558. 'page': '/security/hive'
  1559. })
  1560. if 'indexer' in self.apps and 'filebrowser' in self.apps and self.user.has_hue_permission(action="access:importer", app="indexer") \
  1561. and 'importer' not in APP_BLACKLIST.get():
  1562. interpreters.append({
  1563. 'type': 'importer',
  1564. 'displayName': _('Importer'),
  1565. 'buttonName': _('Import'),
  1566. 'tooltip': _('Importer'),
  1567. 'page': '/indexer/importer'
  1568. })
  1569. if 'sqoop' in self.apps:
  1570. from sqoop.conf import IS_ENABLED
  1571. if IS_ENABLED.get():
  1572. interpreters.append({
  1573. 'type': 'sqoop',
  1574. 'displayName': _('Sqoop'),
  1575. 'buttonName': _('Browse'),
  1576. 'tooltip': _('Sqoop'),
  1577. 'page': '/sqoop'
  1578. })
  1579. if interpreters:
  1580. return {
  1581. 'name': 'browser',
  1582. 'displayName': _('Browsers'),
  1583. 'buttonName': _('Browse'),
  1584. 'interpreters': interpreters,
  1585. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1586. }
  1587. else:
  1588. return None
  1589. def _get_scheduler(self):
  1590. interpreters = []
  1591. if 'oozie' in self.apps and not (self.user.has_hue_permission(action="disable_editor_access", app="oozie") and not is_admin(self.user)):
  1592. interpreters.extend([{
  1593. 'type': 'oozie-workflow',
  1594. 'displayName': _('Workflow'),
  1595. 'buttonName': _('Schedule'),
  1596. 'tooltip': _('Workflow'),
  1597. 'page': '/oozie/editor/workflow/new/'
  1598. }, {
  1599. 'type': 'oozie-coordinator',
  1600. 'displayName': _('Schedule'),
  1601. 'buttonName': _('Schedule'),
  1602. 'tooltip': _('Schedule'),
  1603. 'page': '/oozie/editor/coordinator/new/'
  1604. }, {
  1605. 'type': 'oozie-bundle',
  1606. 'displayName': _('Bundle'),
  1607. 'buttonName': _('Schedule'),
  1608. 'tooltip': _('Bundle'),
  1609. 'page': '/oozie/editor/bundle/new/'
  1610. }
  1611. ])
  1612. if TASK_SERVER.BEAT_ENABLED.get():
  1613. interpreters.append({
  1614. 'type': 'celery-beat',
  1615. 'displayName': _('Scheduled Tasks'),
  1616. 'buttonName': _('Scheduled Tasks'),
  1617. 'tooltip': _('Scheduled Tasks'),
  1618. 'page': '/jobbrowser/'
  1619. }
  1620. )
  1621. if interpreters:
  1622. return {
  1623. 'name': 'oozie',
  1624. 'displayName': _('Scheduler'),
  1625. 'buttonName': _('Schedule'),
  1626. 'interpreters': interpreters,
  1627. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1628. 'page': interpreters[0]['page']
  1629. }
  1630. else:
  1631. return None
  1632. def _get_sdk_apps(self):
  1633. current_app, other_apps, apps_list = _get_apps(self.user)
  1634. interpreters = []
  1635. for other in other_apps:
  1636. interpreters.append({
  1637. 'type': other.display_name,
  1638. 'displayName': other.nice_name,
  1639. 'buttonName': other.nice_name,
  1640. 'tooltip': other.nice_name,
  1641. 'page': '/%s' % other.display_name
  1642. })
  1643. if interpreters:
  1644. return {
  1645. 'name': 'other',
  1646. 'displayName': _('Other Apps'),
  1647. 'buttonName': _('Other'),
  1648. 'interpreters': interpreters,
  1649. }
  1650. else:
  1651. return None
  1652. def get_hive_metastore_interpreters(self):
  1653. return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
  1654. class Cluster(object):
  1655. def __init__(self, user):
  1656. self.user = user
  1657. self.clusters = get_clusters(user)
  1658. if IS_MULTICLUSTER_ONLY.get():
  1659. self.data = self.clusters['Altus'] # Backward compatibility
  1660. else:
  1661. self.data = list(self.clusters.values())[0] # Next: CLUSTER_ID.get() or user persisted
  1662. def get_type(self):
  1663. return self.data['type']
  1664. def get_config(self, name):
  1665. return self.clusters[name]
  1666. def get_app_config(self):
  1667. return ClusterConfig(self.user, cluster_type=self.get_type())
  1668. def _get_apps(user, section=None):
  1669. current_app = None
  1670. other_apps = []
  1671. if user.is_authenticated():
  1672. apps_list = appmanager.get_apps_dict(user)
  1673. apps = list(apps_list.values())
  1674. for app in apps:
  1675. if app.display_name not in [
  1676. 'beeswax', 'hive', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
  1677. 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'
  1678. ] and app.menu_index != -1:
  1679. other_apps.append(app)
  1680. if section == app.display_name:
  1681. current_app = app
  1682. else:
  1683. apps_list = []
  1684. return current_app, other_apps, apps_list
  1685. def get_user_preferences(user, key=None):
  1686. if key is not None:
  1687. try:
  1688. x = UserPreferences.objects.get(user=user, key=key)
  1689. return {key: x.value}
  1690. except UserPreferences.DoesNotExist:
  1691. return None
  1692. else:
  1693. return dict((x.key, x.value) for x in UserPreferences.objects.filter(user=user))
  1694. def set_user_preferences(user, key, value):
  1695. try:
  1696. x = UserPreferences.objects.get(user=user, key=key)
  1697. except UserPreferences.DoesNotExist:
  1698. x = UserPreferences(user=user, key=key)
  1699. x.value = value
  1700. x.save()
  1701. return x
  1702. def get_data_link(meta):
  1703. link = None
  1704. if not meta.get('type'):
  1705. pass
  1706. elif meta['type'] == 'hbase':
  1707. link = '/hbase/#Cluster/%(table)s/query/%(row_key)s' % meta
  1708. if 'col' in meta:
  1709. link += '[%(fam)s:%(col)s]' % meta
  1710. elif 'fam' in meta:
  1711. link += '[%(fam)s]' % meta
  1712. elif meta['type'] == 'hdfs':
  1713. link = '/filebrowser/view=%(path)s' % meta # Could add a byte #
  1714. elif meta['type'] == 'link':
  1715. link = meta['link']
  1716. elif meta['type'] == 'hive':
  1717. link = '/metastore/table/%(database)s/%(table)s' % meta # Could also add col=val
  1718. return link
  1719. def _get_gist_document(uuid):
  1720. return Document2.objects.get(uuid=uuid, type='gist') # Workaround until there is a share to all permission
  1721. def __paginate(page, limit, queryset):
  1722. if limit > 0:
  1723. offset = (page - 1) * limit
  1724. last = offset + limit
  1725. queryset = queryset.all()[offset:last]
  1726. return {
  1727. 'documents': queryset,
  1728. 'page': page,
  1729. 'limit': limit
  1730. }