models.py 69 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import calendar
  18. import json
  19. import logging
  20. import os
  21. import urllib
  22. import uuid
  23. from collections import OrderedDict
  24. from itertools import chain
  25. from django.contrib.auth import models as auth_models
  26. from django.contrib.auth.validators import UnicodeUsernameValidator
  27. from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey
  28. from django.contrib.contenttypes.models import ContentType
  29. from django.contrib.staticfiles.storage import staticfiles_storage
  30. from django.urls import reverse, NoReverseMatch
  31. from django.db import connection, models, transaction
  32. from django.db.models import Q
  33. from django.db.models.query import QuerySet
  34. from django.utils.translation import ugettext as _, ugettext_lazy as _t
  35. from settings import HUE_DESKTOP_VERSION
  36. from aws.conf import is_enabled as is_s3_enabled, has_s3_access
  37. from azure.conf import is_adls_enabled, has_adls_access
  38. from dashboard.conf import get_engines, HAS_REPORT_ENABLED
  39. from hadoop.conf import has_hdfs_enabled
  40. from kafka.conf import has_kafka
  41. from notebook.conf import SHOW_NOTEBOOKS, get_ordered_interpreters
  42. from desktop import appmanager
  43. from desktop.conf import get_clusters, CLUSTER_ID, IS_MULTICLUSTER_ONLY, IS_EMBEDDED, IS_K8S_ONLY
  44. from desktop.lib.i18n import force_unicode
  45. from desktop.lib.exceptions_renderable import PopupException
  46. from desktop.lib.paths import get_run_root, SAFE_CHARACTERS_URI_COMPONENTS
  47. from desktop.redaction import global_redaction_engine
  48. from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH
  49. from desktop.auth.backend import is_admin
  50. LOG = logging.getLogger(__name__)
  51. SAMPLE_USER_ID = 1100713
  52. SAMPLE_USER_INSTALL = 'hue'
  53. SAMPLE_USER_OWNERS = ['hue', 'sample']
  54. UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
  55. HUE_VERSION = None
  56. def uuid_default():
  57. return str(uuid.uuid4())
  58. def hue_version():
  59. global HUE_VERSION
  60. if HUE_VERSION is None:
  61. HUE_VERSION = HUE_DESKTOP_VERSION
  62. p = get_run_root('cloudera', 'cdh_version.properties')
  63. if os.path.exists(p):
  64. build_version = _version_from_properties(open(p))
  65. if build_version:
  66. HUE_VERSION = '%s - %s' % (HUE_VERSION, build_version)
  67. return HUE_VERSION
  68. def _version_from_properties(f):
  69. return dict(line.strip().split('=') for line in f.readlines() if len(line.strip().split('=')) == 2).get('cloudera.cdh.release')
  70. ###################################################################################################
  71. # Custom Settings
  72. ###################################################################################################
  73. PREFERENCE_IS_WELCOME_TOUR_SEEN = 'is_welcome_tour_seen'
  74. class HueUser(auth_models.User):
  75. class Meta:
  76. proxy = True
  77. def __init__(self, *args, **kwargs):
  78. self._meta.get_field(
  79. 'username'
  80. ).validators[0] = UnicodeUsernameValidator()
  81. super(auth_models.User, self).__init__(*args, **kwargs)
  82. class UserPreferences(models.Model):
  83. """Holds arbitrary key/value strings."""
  84. user = models.ForeignKey(auth_models.User)
  85. key = models.CharField(max_length=20)
  86. value = models.TextField(max_length=4096)
  87. class Settings(models.Model):
  88. collect_usage = models.BooleanField(db_index=True, default=True)
  89. tours_and_tutorials = models.BooleanField(db_index=True, default=True)
  90. @classmethod
  91. def get_settings(cls):
  92. settings, created = Settings.objects.get_or_create(id=1)
  93. return settings
  94. class DefaultConfigurationManager(models.Manager):
  95. def get_configuration_for_user(self, app, user):
  96. """
  97. :param app: app name
  98. :param user: User object
  99. :return: DefaultConfiguration for user, or first group found, or default for app, or None
  100. """
  101. try:
  102. return super(DefaultConfigurationManager, self).get(app=app, user=user)
  103. except DefaultConfiguration.DoesNotExist:
  104. pass
  105. configs = super(DefaultConfigurationManager, self).get_queryset().filter(app=app, groups__in=user.groups.all())
  106. if configs.count() > 0:
  107. return configs[0]
  108. try:
  109. return super(DefaultConfigurationManager, self).get(app=app, is_default=True)
  110. except DefaultConfiguration.DoesNotExist:
  111. pass
  112. return None
  113. class DefaultConfiguration(models.Model):
  114. """
  115. Default values for configuration properties for a given app/editor
  116. Can be designated as default for all users by is_default flag, or for a specific group or user
  117. """
  118. app = models.CharField(max_length=32, null=False, db_index=True, help_text=_t('App that this configuration belongs to.'))
  119. properties = models.TextField(default='[]', help_text=_t('JSON-formatted default properties values.'))
  120. is_default = models.BooleanField(default=False, db_index=True)
  121. groups = models.ManyToManyField(auth_models.Group, db_index=True, db_table='defaultconfiguration_groups')
  122. user = models.ForeignKey(auth_models.User, blank=True, null=True, db_index=True)
  123. objects = DefaultConfigurationManager()
  124. class Meta:
  125. ordering = ["app", "-is_default", "user"]
  126. @property
  127. def properties_list(self):
  128. """
  129. :return: Deserialized properties as a list of property objects
  130. """
  131. if not self.properties:
  132. self.properties = json.dumps([])
  133. return json.loads(self.properties)
  134. @property
  135. def properties_dict(self):
  136. """
  137. :return: Deserialized properties as a dict of key: value pairs
  138. """
  139. if not self.properties:
  140. self.properties = json.dumps([])
  141. return dict((prop['key'], prop['value']) for prop in json.loads(self.properties))
  142. def to_dict(self):
  143. return {
  144. 'id': self.id,
  145. 'app': self.app,
  146. 'properties': self.properties_list,
  147. 'is_default': self.is_default,
  148. 'group_ids': [group.id for group in self.groups.all()],
  149. 'user': self.user.username if self.user else None
  150. }
  151. ###################################################################################################
  152. # Document1
  153. ###################################################################################################
  154. class DocumentTagManager(models.Manager):
  155. def get_tags(self, user):
  156. return self.filter(owner=user).distinct()
  157. def create_tag(self, owner, tag_name):
  158. if tag_name in DocumentTag.RESERVED:
  159. raise Exception(_("Can't add %s: it is a reserved tag.") % tag_name)
  160. else:
  161. tag, created = DocumentTag.objects.get_or_create(tag=tag_name, owner=owner)
  162. return tag
  163. def _get_tag(self, user, name):
  164. tag, created = DocumentTag.objects.get_or_create(owner=user, tag=name)
  165. return tag
  166. def get_default_tag(self, user):
  167. return self._get_tag(user, DocumentTag.DEFAULT)
  168. def get_trash_tag(self, user):
  169. return self._get_tag(user, DocumentTag.TRASH)
  170. def get_history_tag(self, user):
  171. return self._get_tag(user, DocumentTag.HISTORY)
  172. def get_example_tag(self, user):
  173. return self._get_tag(user, DocumentTag.EXAMPLE)
  174. def get_imported2_tag(self, user):
  175. return self._get_tag(user, DocumentTag.IMPORTED2)
  176. def tag(self, owner, doc_id, tag_name='', tag_id=None):
  177. try:
  178. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  179. if tag.tag in DocumentTag.RESERVED:
  180. raise Exception(_("Can't add %s: it is a reserved tag.") % tag)
  181. except DocumentTag.DoesNotExist:
  182. tag = self._get_tag(user=owner, name=tag_name)
  183. doc = Document.objects.get_doc_for_writing(doc_id, owner)
  184. doc.add_tag(tag)
  185. return tag
  186. def untag(self, tag_id, owner, doc_id):
  187. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  188. if tag.tag in DocumentTag.RESERVED:
  189. raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
  190. doc = Document.objects.get_doc_for_writing(doc_id, owner=owner)
  191. doc.remove_tag(tag)
  192. def delete_tag(self, tag_id, owner):
  193. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  194. default_tag = DocumentTag.objects.get_default_tag(owner)
  195. if tag.tag in DocumentTag.RESERVED:
  196. raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
  197. else:
  198. tag.delete()
  199. for doc in Document.objects.get_docs(owner).filter(tags=None):
  200. doc.add_tag(default_tag)
  201. def update_tags(self, owner, doc_id, tag_ids):
  202. doc = Document.objects.get_doc_for_writing(doc_id, owner)
  203. for tag in doc.tags.all():
  204. if tag.tag not in DocumentTag.RESERVED:
  205. doc.remove_tag(tag)
  206. for tag_id in tag_ids:
  207. tag = DocumentTag.objects.get(id=tag_id, owner=owner)
  208. if tag.tag not in DocumentTag.RESERVED:
  209. doc.add_tag(tag)
  210. return doc
  211. class DocumentTag(models.Model):
  212. """
  213. Reserved tags can't be manually removed by the user.
  214. """
  215. owner = models.ForeignKey(auth_models.User, db_index=True)
  216. tag = models.SlugField()
  217. DEFAULT = 'default' # Always there
  218. TRASH = 'trash' # There when the document is trashed
  219. HISTORY = 'history' # There when the document is a submission history
  220. EXAMPLE = 'example' # Hue examples
  221. IMPORTED2 = 'imported2' # Was imported to document2
  222. RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE, IMPORTED2)
  223. objects = DocumentTagManager()
  224. class Meta:
  225. unique_together = ('owner', 'tag')
  226. def __unicode__(self):
  227. return force_unicode('%s') % (self.tag,)
  228. class DocumentManager(models.Manager):
  229. def documents(self, user):
  230. return Document.objects.filter(
  231. Q(owner=user) |
  232. Q(documentpermission__users=user) |
  233. Q(documentpermission__groups__in=user.groups.all())
  234. ).defer('description', 'extra').distinct()
  235. def get_docs(self, user, model_class=None, extra=None, qfilter=None):
  236. docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
  237. if model_class is not None:
  238. ct = ContentType.objects.get_for_model(model_class)
  239. docs = docs.filter(content_type=ct)
  240. if extra is not None:
  241. docs = docs.filter(extra=extra)
  242. if qfilter is not None:
  243. docs = docs.filter(qfilter)
  244. return docs
  245. def get_doc_for_writing(self, doc_id, user):
  246. """Fetch a document and confirm that this user can write to it."""
  247. doc = Document.objects.documents(user).get(id=doc_id)
  248. doc.can_write_or_exception(user)
  249. return doc
  250. def trashed_docs(self, model_class, user):
  251. tag = DocumentTag.objects.get_trash_tag(user=user)
  252. return Document.objects.get_docs(user, model_class).filter(tags__in=[tag]).order_by('-last_modified')
  253. def trashed(self, model_class, user):
  254. docs = self.trashed_docs(model_class, user)
  255. return [job.content_object for job in docs if job.content_object]
  256. def available_docs(self, model_class, user, with_history=False):
  257. exclude = [DocumentTag.objects.get_trash_tag(user=user)]
  258. if not with_history:
  259. exclude.append(DocumentTag.objects.get_history_tag(user=user))
  260. return Document.objects.get_docs(user, model_class).exclude(tags__in=exclude).order_by('-last_modified')
  261. def history_docs(self, model_class, user):
  262. include = [DocumentTag.objects.get_history_tag(user=user)]
  263. exclude = [DocumentTag.objects.get_trash_tag(user=user)]
  264. return Document.objects.get_docs(user, model_class).filter(tags__in=include).exclude(tags__in=exclude).order_by('-last_modified')
  265. def available(self, model_class, user, with_history=False):
  266. docs = self.available_docs(model_class, user, with_history)
  267. return [doc.content_object for doc in docs if doc.content_object]
  268. def can_read_or_exception(self, user, doc_class, doc_id, exception_class=PopupException):
  269. if doc_id is None:
  270. return
  271. try:
  272. ct = ContentType.objects.get_for_model(doc_class)
  273. doc = Document.objects.get(object_id=doc_id, content_type=ct)
  274. if doc.can_read(user):
  275. return doc
  276. else:
  277. message = _("Permission denied. %(username)s does not have the permissions required to access document %(id)s") % \
  278. {'username': user.username, 'id': doc.id}
  279. raise exception_class(message)
  280. except Document.DoesNotExist:
  281. raise exception_class(_('Document %(id)s does not exist') % {'id': doc_id})
  282. def can_read(self, user, doc_class, doc_id):
  283. ct = ContentType.objects.get_for_model(doc_class)
  284. doc = Document.objects.get(object_id=doc_id, content_type=ct)
  285. return doc.can_read(user)
  286. def link(self, content_object, owner, name='', description='', extra=''):
  287. if not content_object.doc.exists():
  288. doc = Document.objects.create(
  289. content_object=content_object,
  290. owner=owner,
  291. name=name,
  292. description=description,
  293. extra=extra
  294. )
  295. tag = DocumentTag.objects.get_default_tag(user=owner)
  296. doc.tags.add(tag)
  297. return doc
  298. else:
  299. LOG.warn('Object %s already has documents: %s' % (content_object, content_object.doc.all()))
  300. return content_object.doc.all()[0]
  301. def sync(self, doc2_only=True):
  302. def find_jobs_with_no_doc(model):
  303. jobs = model.objects.filter(doc__isnull=True)
  304. if model == Document2:
  305. jobs = jobs.exclude(is_history=True)
  306. return jobs.select_related('owner')
  307. def find_oozie_jobs_with_no_doc(model):
  308. return model.objects.filter(doc__isnull=True).exclude(name__exact='').select_related('owner')
  309. table_names = connection.introspection.table_names()
  310. try:
  311. from oozie.models import Workflow, Coordinator, Bundle
  312. if \
  313. Workflow._meta.db_table in table_names or \
  314. Coordinator._meta.db_table in table_names or \
  315. Bundle._meta.db_table in table_names:
  316. with transaction.atomic():
  317. for job in chain(
  318. find_oozie_jobs_with_no_doc(Workflow),
  319. find_oozie_jobs_with_no_doc(Coordinator),
  320. find_oozie_jobs_with_no_doc(Bundle)):
  321. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description)
  322. if job.is_trashed:
  323. doc.send_to_trash()
  324. if job.is_shared:
  325. doc.share_to_default()
  326. if hasattr(job, 'managed'):
  327. if not job.managed:
  328. doc.extra = 'jobsub'
  329. doc.save()
  330. except Exception, e:
  331. LOG.exception('error syncing oozie')
  332. try:
  333. from beeswax.models import SavedQuery
  334. if SavedQuery._meta.db_table in table_names:
  335. with transaction.atomic():
  336. for job in find_jobs_with_no_doc(SavedQuery):
  337. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
  338. if job.is_trashed:
  339. doc.send_to_trash()
  340. except Exception, e:
  341. LOG.exception('error syncing beeswax')
  342. try:
  343. from pig.models import PigScript
  344. if PigScript._meta.db_table in table_names:
  345. with transaction.atomic():
  346. for job in find_jobs_with_no_doc(PigScript):
  347. Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
  348. except Exception, e:
  349. LOG.exception('error syncing pig')
  350. try:
  351. from search.models import Collection
  352. if Collection._meta.db_table in table_names:
  353. with transaction.atomic():
  354. for dashboard in Collection.objects.all():
  355. if 'collection' in dashboard.properties_dict:
  356. col_dict = dashboard.properties_dict['collection']
  357. if not 'uuid' in col_dict:
  358. _uuid = str(uuid.uuid4())
  359. col_dict['uuid'] = _uuid
  360. dashboard.update_properties({'collection': col_dict})
  361. if dashboard.owner is None:
  362. from useradmin.models import install_sample_user
  363. owner = install_sample_user()
  364. else:
  365. owner = dashboard.owner
  366. dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
  367. Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
  368. dashboard.save()
  369. except Exception, e:
  370. LOG.exception('error syncing search')
  371. try:
  372. if Document2._meta.db_table in table_names:
  373. with transaction.atomic():
  374. for job in find_jobs_with_no_doc(Document2):
  375. if job.type == 'oozie-workflow2':
  376. extra = 'workflow2'
  377. elif job.type == 'oozie-coordinator2':
  378. extra = 'coordinator2'
  379. elif job.type == 'oozie-bundle2':
  380. extra = 'bundle2'
  381. elif job.type == 'notebook':
  382. extra = 'notebook'
  383. elif job.type == 'search-dashboard':
  384. extra = 'search-dashboard'
  385. else:
  386. extra = ''
  387. doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description, extra=extra)
  388. except Exception, e:
  389. LOG.exception('error syncing Document2')
  390. if not doc2_only and Document._meta.db_table in table_names:
  391. # Make sure doc have at least a tag
  392. try:
  393. for doc in Document.objects.filter(tags=None):
  394. default_tag = DocumentTag.objects.get_default_tag(doc.owner)
  395. doc.tags.add(default_tag)
  396. except Exception, e:
  397. LOG.exception('error adding at least one tag to docs')
  398. # Make sure all the sample user documents are shared.
  399. try:
  400. with transaction.atomic():
  401. for doc in Document.objects.filter(owner__username__in=SAMPLE_USER_OWNERS):
  402. doc.share_to_default()
  403. tag = DocumentTag.objects.get_example_tag(user=doc.owner)
  404. doc.tags.add(tag)
  405. doc_last_modified = doc.last_modified
  406. doc.save()
  407. Document.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
  408. except Exception, e:
  409. LOG.exception('error sharing sample user documents')
  410. # For now remove the default tag from the examples
  411. try:
  412. for doc in Document.objects.filter(tags__tag=DocumentTag.EXAMPLE):
  413. default_tag = DocumentTag.objects.get_default_tag(doc.owner)
  414. doc.tags.remove(default_tag)
  415. except Exception, e:
  416. LOG.exception('error removing default tags')
  417. # ------------------------------------------------------------------------
  418. LOG.info('Looking for documents that have no object')
  419. # Delete documents with no object.
  420. try:
  421. with transaction.atomic():
  422. # First, delete all the documents that don't have a content type
  423. docs = Document.objects.filter(content_type=None)
  424. if docs:
  425. LOG.info('Deleting %s doc(s) that do not have a content type' % docs.count())
  426. docs.delete()
  427. # Next, it's possible that there are documents pointing at a non-existing
  428. # content_type. We need to do a left join to find these records, but we
  429. # can't do this directly in django. To get around writing wrap sql (which
  430. # might not be portable), we'll use an aggregate to count up all the
  431. # associated content_types, and delete the documents that have a count of
  432. # zero.
  433. #
  434. # Note we're counting `content_type__name` to force the join.
  435. docs = Document.objects \
  436. .values('id') \
  437. .annotate(content_type_count=models.Count('content_type__name')) \
  438. .filter(content_type_count=0)
  439. if docs:
  440. LOG.info('Deleting %s doc(s) that have invalid content types' % docs.count())
  441. docs.delete()
  442. # Finally we need to delete documents with no associated content object.
  443. # This is tricky because of our use of generic foreign keys. So to do
  444. # this a bit more efficiently, we'll start with a query of all the
  445. # documents, then step through each content type and and filter out all
  446. # the documents it's referencing from our document query. Messy, but it
  447. # works.
  448. # TODO: This can be several 100k entries for large databases.
  449. # Need to figure out a better way to handle this scenario.
  450. docs = Document.objects.all()
  451. for content_type in ContentType.objects.all():
  452. model_class = content_type.model_class()
  453. # Ignore any types that don't have a model.
  454. if model_class is None:
  455. continue
  456. # Ignore types that don't have a table yet.
  457. if model_class._meta.db_table not in table_names:
  458. continue
  459. # Ignore classes that don't have a 'doc'.
  460. if not hasattr(model_class, 'doc'):
  461. continue
  462. # First create a query that grabs all the document ids for this type.
  463. docs_from_content = model_class.objects.values('doc__id')
  464. # Next, filter these from our document query.
  465. docs = docs.exclude(id__in=docs_from_content)
  466. if docs.exists():
  467. LOG.info('Deleting %s documents' % docs.count())
  468. docs.delete()
  469. except Exception, e:
  470. LOG.exception('Error in sync while attempting to delete documents with no object: %s' % e)
  471. class Document(models.Model):
  472. owner = models.ForeignKey(auth_models.User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
  473. name = models.CharField(default='', max_length=255)
  474. description = models.TextField(default='')
  475. last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Last modified'))
  476. version = models.SmallIntegerField(default=1, verbose_name=_t('Schema version'))
  477. extra = models.TextField(default='')
  478. tags = models.ManyToManyField(DocumentTag, db_index=True)
  479. content_type = models.ForeignKey(ContentType)
  480. object_id = models.PositiveIntegerField()
  481. content_object = GenericForeignKey('content_type', 'object_id')
  482. objects = DocumentManager()
  483. class Meta:
  484. unique_together = ('content_type', 'object_id')
  485. def __unicode__(self):
  486. return force_unicode('%s %s %s') % (self.content_type, self.name, self.owner)
  487. def is_editable(self, user):
  488. """Deprecated by can_read"""
  489. return self.can_write(user)
  490. def can_edit_or_exception(self, user, exception_class=PopupException):
  491. """Deprecated by can_write_or_exception"""
  492. return self.can_write_or_exception(user, exception_class)
  493. def add_tag(self, tag):
  494. self.tags.add(tag)
  495. def remove_tag(self, tag):
  496. self.tags.remove(tag)
  497. def is_trashed(self):
  498. return DocumentTag.objects.get_trash_tag(user=self.owner) in self.tags.all()
  499. def is_historic(self):
  500. return DocumentTag.objects.get_history_tag(user=self.owner) in self.tags.all()
  501. def send_to_trash(self):
  502. tag = DocumentTag.objects.get_trash_tag(user=self.owner)
  503. self.tags.add(tag)
  504. def restore_from_trash(self):
  505. tag = DocumentTag.objects.get_trash_tag(user=self.owner)
  506. self.tags.remove(tag)
  507. def add_to_history(self):
  508. tag = DocumentTag.objects.get_history_tag(user=self.owner)
  509. self.tags.add(tag)
  510. def remove_from_history(self):
  511. tag = DocumentTag.objects.get_history_tag(user=self.owner)
  512. self.tags.remove(tag)
  513. def share_to_default(self, name='read'):
  514. DocumentPermission.objects.share_to_default(self, name=name)
  515. def can_read(self, user):
  516. return is_admin(user) or self.owner == user or Document.objects.get_docs(user).filter(id=self.id).exists()
  517. def can_write(self, user):
  518. perm = self.list_permissions('write')
  519. return is_admin(user) or self.owner == user or perm.groups.filter(id__in=user.groups.all()).exists() or user in perm.users.all()
  520. def can_read_or_exception(self, user, exception_class=PopupException):
  521. if self.can_read(user):
  522. return True
  523. else:
  524. raise exception_class(_("Document does not exist or you don't have the permission to access it."))
  525. def can_write_or_exception(self, user, exception_class=PopupException):
  526. if self.can_write(user):
  527. return True
  528. else:
  529. raise exception_class(_("Document does not exist or you don't have the permission to access it."))
  530. def copy(self, content_object, name, owner, description=None):
  531. if content_object:
  532. copy_doc = self
  533. copy_doc.pk = None
  534. copy_doc.id = None
  535. copy_doc.name = name
  536. copy_doc.owner = owner
  537. if description:
  538. copy_doc.description = description
  539. copy_doc = Document.objects.link(content_object,
  540. owner=copy_doc.owner,
  541. name=copy_doc.name,
  542. description=copy_doc.description,
  543. extra=copy_doc.extra)
  544. # Update reverse Document relation to new copy
  545. if content_object.doc.get():
  546. content_object.doc.get().delete()
  547. content_object.doc.add(copy_doc)
  548. return copy_doc
  549. else:
  550. raise PopupException(_("Document copy method requires a content_object argument."))
  551. @property
  552. def icon(self):
  553. apps = appmanager.get_apps_dict()
  554. try:
  555. if self.extra == 'workflow2':
  556. return staticfiles_storage.url('oozie/art/icon_oozie_workflow_48.png')
  557. elif self.extra == 'coordinator2':
  558. return staticfiles_storage.url('oozie/art/icon_oozie_coordinator_48.png')
  559. elif self.extra == 'bundle2':
  560. return staticfiles_storage.url('oozie/art/icon_oozie_bundle_48.png')
  561. elif self.extra == 'notebook':
  562. return staticfiles_storage.url('notebook/art/icon_notebook_48.png')
  563. elif self.extra.startswith('query'):
  564. if self.extra == 'query-impala':
  565. return staticfiles_storage.url(apps['impala'].icon_path)
  566. else:
  567. return staticfiles_storage.url(apps['beeswax'].icon_path)
  568. elif self.extra.startswith('search'):
  569. return staticfiles_storage.url('search/art/icon_search_48.png')
  570. elif self.content_type.app_label == 'beeswax':
  571. if self.extra == '0':
  572. return staticfiles_storage.url(apps['beeswax'].icon_path)
  573. elif self.extra == '3':
  574. return staticfiles_storage.url(apps['spark'].icon_path)
  575. else:
  576. return staticfiles_storage.url(apps['impala'].icon_path)
  577. elif self.content_type.app_label == 'oozie':
  578. if self.extra == 'jobsub':
  579. return staticfiles_storage.url(apps['jobsub'].icon_path)
  580. else:
  581. return staticfiles_storage.url(self.content_type.model_class().ICON)
  582. elif self.content_type.app_label in apps:
  583. return staticfiles_storage.url(apps[self.content_type.app_label].icon_path)
  584. else:
  585. return staticfiles_storage.url('desktop/art/icon_hue_48.png')
  586. except Exception, e:
  587. LOG.warn(force_unicode(e))
  588. return staticfiles_storage.url('desktop/art/icon_hue_48.png')
  589. def share(self, users, groups, name='read'):
  590. DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=True)
  591. def unshare(self, users, groups, name='read'):
  592. DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=False)
  593. def sync_permissions(self, perms_dict):
  594. """
  595. Set who else or which other group can interact with the document.
  596. Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
  597. """
  598. for name, perm in perms_dict.iteritems():
  599. users = groups = None
  600. if perm.get('user_ids'):
  601. users = auth_models.User.objects.in_bulk(perm.get('user_ids'))
  602. else:
  603. users = []
  604. if perm.get('group_ids'):
  605. groups = auth_models.Group.objects.in_bulk(perm.get('group_ids'))
  606. else:
  607. groups = []
  608. DocumentPermission.objects.sync(document=self, name=name, users=users, groups=groups)
  609. def list_permissions(self, perm='read'):
  610. return DocumentPermission.objects.list(document=self, perm=perm)
  611. def to_dict(self):
  612. return {
  613. 'owner': self.owner.username,
  614. 'name': self.name,
  615. 'description': self.description,
  616. 'uuid': None, # no uuid == v1
  617. 'id': self.id,
  618. 'doc1_id': self.id,
  619. 'object_id': self.object_id,
  620. 'type': str(self.content_type),
  621. 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT),
  622. 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()),
  623. 'isSelected': False
  624. }
  625. class DocumentPermissionManager(models.Manager):
  626. def _check_perm(self, name):
  627. perms = (DocumentPermission.READ_PERM, DocumentPermission.WRITE_PERM)
  628. if name not in perms:
  629. perms_string = ' and '.join(', '.join(perms).rsplit(', ', 1))
  630. raise PopupException(_('Only %s permissions are supported, not %s.') % (perms_string, name))
  631. def share_to_default(self, document, name='read'):
  632. from useradmin.models import get_default_user_group # Remove build dependency
  633. self._check_perm(name)
  634. if name == DocumentPermission.WRITE_PERM:
  635. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.WRITE_PERM)
  636. else:
  637. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.READ_PERM)
  638. default_group = get_default_user_group()
  639. if default_group:
  640. perm.groups.add(default_group)
  641. def update(self, document, name='read', users=None, groups=None, add=True):
  642. self._check_perm(name)
  643. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
  644. if users is not None:
  645. if add:
  646. perm.users.add(*users)
  647. else:
  648. perm.users.remove(*users)
  649. if groups is not None:
  650. if add:
  651. perm.groups.add(*groups)
  652. else:
  653. perm.groups.remove(*groups)
  654. if not perm.users and not perm.groups:
  655. perm.delete()
  656. def sync(self, document, name='read', users=None, groups=None):
  657. self._check_perm(name)
  658. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
  659. if users is not None:
  660. perm.users = []
  661. perm.users = users
  662. perm.save()
  663. if groups is not None:
  664. perm.groups = []
  665. perm.groups = groups
  666. perm.save()
  667. if not users and not groups:
  668. perm.delete()
  669. def list(self, document, perm='read'):
  670. perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=perm)
  671. return perm
  672. class DocumentPermission(models.Model):
  673. READ_PERM = 'read'
  674. WRITE_PERM = 'write'
  675. doc = models.ForeignKey(Document)
  676. users = models.ManyToManyField(auth_models.User, db_index=True, db_table='documentpermission_users')
  677. groups = models.ManyToManyField(auth_models.Group, db_index=True, db_table='documentpermission_groups')
  678. perms = models.CharField(default=READ_PERM, max_length=10, choices=( # one perm
  679. (READ_PERM, 'read'),
  680. (WRITE_PERM, 'write'),
  681. ))
  682. objects = DocumentPermissionManager()
  683. class Meta:
  684. unique_together = ('doc', 'perms')
  685. ###################################################################################################
  686. # Document2
  687. ###################################################################################################
  688. class FilesystemException(Exception):
  689. pass
  690. class Document2QueryMixin(object):
  691. def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False):
  692. """
  693. Returns all documents that are owned or shared with the user.
  694. :param perms: both, shared, owned. Defaults to both.
  695. :param include_history: boolean flag to return history documents. Defaults to False.
  696. :param include_trashed: boolean flag to return trashed documents. Defaults to True.
  697. """
  698. if perms == 'both':
  699. docs = self.filter(
  700. Q(owner=user) |
  701. Q(document2permission__users=user) |
  702. Q(document2permission__groups__in=user.groups.all())
  703. )
  704. elif perms == 'shared':
  705. docs = self.filter(
  706. Q(document2permission__users=user) |
  707. Q(document2permission__groups__in=user.groups.all())
  708. )
  709. else: # only return documents owned by the user
  710. docs = self.filter(owner=user)
  711. if not include_history:
  712. docs = docs.exclude(is_history=True)
  713. if not include_managed:
  714. docs = docs.exclude(is_managed=True)
  715. if not include_trashed:
  716. docs = docs.exclude(is_trashed=True)
  717. return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
  718. def search_documents(self, types=None, search_text=None, order_by=None):
  719. """
  720. Search for documents based on type filters, search_text or order_by and return a queryset of document objects
  721. :param types: list of Document2 types (e.g. - query-hive, directory, etc)
  722. :param search_text: text to search on in the name and description fields
  723. :param order_by: order by field (e.g. -last_modified, type)
  724. """
  725. documents = self
  726. if types and isinstance(types, list) and types[0] != 'all':
  727. documents = documents.filter(type__in=types)
  728. if search_text:
  729. documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) | Q(search__icontains=search_text))
  730. if order_by: # TODO: Validate that order_by is a valid sort parameter
  731. documents = documents.order_by(order_by)
  732. return documents
  733. class Document2QuerySet(QuerySet, Document2QueryMixin):
  734. pass
  735. class Document2Manager(models.Manager, Document2QueryMixin):
  736. def get_queryset(self):
  737. return Document2QuerySet(self.model, using=self._db)
  738. # TODO prevent get() in favor of this
  739. def document(self, user, doc_id):
  740. return self.documents(user, include_trashed=True, include_history=True).get(id=doc_id)
  741. def get_by_natural_key(self, uuid, version, is_history):
  742. return self.get(uuid=uuid, version=version, is_history=is_history)
  743. def get_by_uuid(self, user, uuid, perm_type='read'):
  744. """
  745. Since UUID is not a unique field, but part of a composite unique key, this returns the latest version by UUID
  746. This should always be used in place of Document2.objects.get(uuid=) when a single document is expected
  747. :param user: User to check permissions against
  748. :param uuid
  749. :param perm_type: permission type to check against
  750. """
  751. docs = self.filter(uuid=uuid).order_by('-last_modified')
  752. if not docs.exists():
  753. raise FilesystemException(_('Document with UUID %s not found.') % uuid)
  754. latest_doc = docs[0]
  755. if perm_type == 'write':
  756. latest_doc.can_write_or_exception(user)
  757. else:
  758. latest_doc.can_read_or_exception(user)
  759. return latest_doc
  760. def get_history(self, user, doc_type, include_trashed=False):
  761. return self.documents(user, perms='owned', include_history=True, include_trashed=include_trashed).filter(type=doc_type, is_history=True)
  762. def get_tasks_history(self, user):
  763. return self.documents(user, perms='owned', include_history=True, include_trashed=False, include_managed=True).filter(is_history=True, is_managed=True).exclude(name='pig-app-hue-script').exclude(type='oozie-workflow2')
  764. def get_home_directory(self, user):
  765. try:
  766. return self.get(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory')
  767. except Document2.DoesNotExist:
  768. return self.create_user_directories(user)
  769. except Document2.MultipleObjectsReturned:
  770. LOG.error('Multiple Home directories detected. Merging all into one.')
  771. home_dirs = list(self.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory').order_by('-last_modified'))
  772. parent_home_dir = home_dirs.pop()
  773. for dir in home_dirs:
  774. dir.children.exclude(name='.Trash').update(parent_directory=parent_home_dir)
  775. dir.delete()
  776. return parent_home_dir
  777. def get_by_path(self, user, path):
  778. """
  779. This can be an expensive operation b/c we have to traverse the path tree, so if possible, request a document by UUID
  780. NOTE: get_by_path only works for the owner's documents since it is based off the user's home directory
  781. """
  782. cleaned_path = path.rstrip('/')
  783. doc = Document2.objects.get_home_directory(user)
  784. if cleaned_path:
  785. path_tokens = cleaned_path.split('/')[1:]
  786. for token in path_tokens:
  787. try:
  788. doc = doc.children.get(name=token)
  789. except Document2.DoesNotExist:
  790. raise FilesystemException(_('Requested invalid path for user %s: %s') % (user.username, path))
  791. except Document2.MultipleObjectsReturned:
  792. raise FilesystemException(_('Duplicate documents found for user %s at path: %s') % (user.username, path))
  793. doc.can_read_or_exception(user)
  794. return doc
  795. def create_user_directories(self, user):
  796. """
  797. Creates user home and Trash directories if they do not exist and move any orphan documents to home directory
  798. :param user: User object
  799. """
  800. # Edge-case if the user has a legacy home directory with path-name
  801. Directory.objects.filter(name='/', owner=user).update(name=Document2.HOME_DIR)
  802. # Get or create home and Trash directories for all users
  803. home_dir, created = Directory.objects.get_or_create(name=Document2.HOME_DIR, owner=user)
  804. if created:
  805. LOG.info('Successfully created home directory for user: %s' % user.username)
  806. trash_dir, created = Directory.objects.get_or_create(name=Document2.TRASH_DIR, owner=user, parent_directory=home_dir)
  807. if created:
  808. LOG.info('Successfully created trash directory for user: %s' % user.username)
  809. # For any directories or documents that do not have a parent directory, assign it to home directory
  810. count = 0
  811. for doc in Document2.objects.filter(owner=user).filter(parent_directory=None).exclude(id__in=[home_dir.id, trash_dir.id]):
  812. doc.parent_directory = home_dir
  813. doc.save()
  814. count += 1
  815. LOG.info("Moved %d documents to home directory for user: %s" % (count, user.username))
  816. return home_dir
  817. class Document2(models.Model):
  818. HOME_DIR = ''
  819. TRASH_DIR = '.Trash'
  820. EXAMPLES_DIR = 'examples'
  821. owner = models.ForeignKey(auth_models.User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Creator.'), related_name='doc2_owner')
  822. name = models.CharField(default='', max_length=255)
  823. description = models.TextField(default='')
  824. uuid = models.CharField(default=uuid_default, max_length=36, db_index=True)
  825. type = models.CharField(default='', max_length=32, db_index=True, help_text=_t('Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...'))
  826. data = models.TextField(default='{}')
  827. extra = models.TextField(default='')
  828. search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
  829. # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
  830. last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
  831. version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
  832. is_history = models.BooleanField(default=False, db_index=True)
  833. is_managed = models.BooleanField(default=False, db_index=True, verbose_name=_t('If managed under the cover by Hue and never by the user')) # Aka isTask
  834. is_trashed = models.NullBooleanField(default=False, db_index=True, verbose_name=_t('True if trashed'))
  835. dependencies = models.ManyToManyField('self', symmetrical=False, related_name='dependents', db_index=True)
  836. parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
  837. doc = GenericRelation(Document, related_query_name='doc_doc') # Compatibility with Hue 3
  838. objects = Document2Manager()
  839. class Meta:
  840. unique_together = ('uuid', 'version', 'is_history')
  841. ordering = ["-last_modified", "name"]
  842. def __unicode__(self):
  843. res = '%s - %s - %s - %s' % (force_unicode(self.name), self.owner, self.type, self.uuid)
  844. return force_unicode(res)
  845. @property
  846. def data_dict(self):
  847. if not self.data:
  848. self.data = json.dumps({})
  849. data_python = json.loads(self.data)
  850. return data_python
  851. @property
  852. def path(self):
  853. quoted_name = urllib.quote(self.name.encode('utf-8'))
  854. if self.parent_directory:
  855. return '%s/%s' % (self.parent_directory.path, quoted_name)
  856. else:
  857. return quoted_name
  858. @property
  859. def dirname(self):
  860. return os.path.dirname(self.path) or '/'
  861. @property
  862. def is_directory(self):
  863. return self.type == 'directory'
  864. @property
  865. def is_home_directory(self):
  866. return self.is_directory and self.parent_directory == None and self.name == self.HOME_DIR
  867. @property
  868. def is_trash_directory(self):
  869. return self.is_directory and self.name == self.TRASH_DIR
  870. def natural_key(self):
  871. return (self.uuid, self.version, self.is_history)
  872. def copy(self, name, owner, description=None):
  873. copy_doc = self
  874. copy_doc.pk = None
  875. copy_doc.id = None
  876. copy_doc.uuid = uuid_default()
  877. copy_doc.name = name
  878. copy_doc.owner = owner
  879. if description:
  880. copy_doc.description = description
  881. copy_doc.save()
  882. return copy_doc
  883. def update_data(self, post_data):
  884. data_dict = self.data_dict
  885. data_dict.update(post_data)
  886. self.data = json.dumps(data_dict)
  887. def get_absolute_url(self):
  888. url = None
  889. try:
  890. if self.type == 'oozie-coordinator2':
  891. url = reverse('oozie:edit_coordinator') + '?coordinator=' + str(self.id)
  892. elif self.type == 'oozie-bundle2':
  893. url = reverse('oozie:edit_bundle') + '?bundle=' + str(self.id)
  894. elif self.type.startswith('query'):
  895. url = '/editor' + '?editor=' + str(self.id)
  896. elif self.type == 'directory':
  897. url = '/home2' + '?uuid=' + self.uuid
  898. elif self.type == 'notebook':
  899. url = reverse('notebook:notebook') + '?notebook=' + str(self.id)
  900. elif self.type == 'search-dashboard':
  901. url = reverse('search:index') + '?collection=' + str(self.id)
  902. elif self.type == 'link-pigscript':
  903. url = reverse('pig:index') + '#edit/%s' % self.data_dict.get('object_id', '')
  904. elif self.type == 'link-workflow':
  905. url = '/jobsub/#edit-design/%s' % self.data_dict.get('object_id', '')
  906. else:
  907. url = reverse('oozie:edit_workflow') + '?workflow=' + str(self.id)
  908. except NoReverseMatch:
  909. LOG.warn('Could not perform reverse lookup for type %s, app may be blacklisted.' % self.type)
  910. return url
  911. def to_dict(self):
  912. return {
  913. 'owner': self.owner.username,
  914. 'name': self.name,
  915. 'path': self.path or '/',
  916. 'description': self.description,
  917. 'uuid': self.uuid,
  918. 'id': self.id,
  919. 'doc1_id': self.doc.get().id if self.doc.exists() else -1,
  920. 'parent_uuid': self.parent_directory.uuid if self.parent_directory else None,
  921. 'type': self.type,
  922. 'perms': self._massage_permissions(),
  923. 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT) if self.last_modified else None,
  924. 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()) if self.last_modified else None,
  925. 'is_managed': self.is_managed,
  926. 'isSelected': False,
  927. 'absoluteUrl': self.get_absolute_url(),
  928. }
  929. def get_history(self):
  930. return self.dependencies.filter(is_history=True).order_by('-last_modified')
  931. def add_to_history(self, user, data_dict):
  932. doc_id = self.id # Need to copy as the clone messes it
  933. history_doc = self.copy(name=self.name, owner=user)
  934. history_doc.update_data({'history': data_dict})
  935. history_doc.is_history = True
  936. history_doc.last_modified = None
  937. history_doc.save()
  938. Document2.objects.get(id=doc_id).dependencies.add(history_doc)
  939. return history_doc
  940. def save(self, *args, **kwargs):
  941. # Set document parent to home directory if parent directory isn't specified
  942. if not self.parent_directory and not self.is_home_directory and not self.is_trash_directory:
  943. home_dir = Document2.objects.get_home_directory(self.owner)
  944. self.parent_directory = home_dir
  945. # Run validations
  946. self.validate()
  947. # Redact query if needed
  948. self._redact_query()
  949. if self.search:
  950. self.search = self.search[:DOCUMENT2_SEARCH_MAX_LENGTH]
  951. super(Document2, self).save(*args, **kwargs)
  952. # Inherit shared permissions from parent directory, must be done after save b/c new doc needs ID
  953. self.inherit_permissions()
  954. def validate(self):
  955. # Validate home and Trash directories are only created once per user and cannot be created or modified after
  956. if self.name in [Document2.HOME_DIR, Document2.TRASH_DIR] and self.type == 'directory' and \
  957. Document2.objects.filter(name=self.name, owner=self.owner, type='directory').exists():
  958. raise FilesystemException(_('Cannot create or modify directory with name: %s') % self.name)
  959. # Validate that parent directory does not create cycle
  960. if self._contains_cycle():
  961. raise FilesystemException(_('Cannot save document %s under parent directory %s due to circular dependency') %
  962. (self.name, self.parent_directory.uuid))
  963. def inherit_permissions(self):
  964. if self.parent_directory is not None:
  965. parent_perms = Document2Permission.objects.filter(doc=self.parent_directory)
  966. for perm in parent_perms:
  967. self.share(self.owner, name=perm.perms, users=perm.users.all(), groups=perm.groups.all())
  968. def move(self, directory, user):
  969. if not directory.is_directory:
  970. raise FilesystemException(_('Target with UUID %s is not a directory') % directory.uuid)
  971. if directory.can_write_or_exception(user=user):
  972. # Restore last_modified date after save
  973. original_last_modified = self.last_modified
  974. self.parent_directory = directory
  975. self.save()
  976. # Use update instead of save() so that last_modified date is not modified automatically
  977. Document2.objects.filter(id=self.id).update(last_modified=original_last_modified)
  978. return self
  979. def trash(self):
  980. try:
  981. trash_dir = Directory.objects.get(name=self.TRASH_DIR, owner=self.owner)
  982. self.move(trash_dir, self.owner)
  983. self.is_trashed = True
  984. self.save()
  985. if self.is_directory:
  986. children_ids = self._get_child_ids_recursively(self.id)
  987. Document2.objects.filter(id__in=children_ids).update(is_trashed=True)
  988. except Document2.MultipleObjectsReturned:
  989. LOG.error('Multiple Trash directories detected. Merging all into one.')
  990. trash_dirs = list(Directory.objects.filter(name=self.TRASH_DIR, owner=self.owner).order_by('-last_modified'))
  991. parent_trash_dir = trash_dirs.pop()
  992. for dir in trash_dirs:
  993. dir.children.update(parent_directory=parent_trash_dir)
  994. dir.delete()
  995. self.move(parent_trash_dir, self.owner)
  996. def restore(self):
  997. # Currently restoring any doucment to /home
  998. home_dir = Document2.objects.get_home_directory(self.owner)
  999. self.move(home_dir, self.owner)
  1000. self.is_trashed = False
  1001. self.save()
  1002. if self.is_directory:
  1003. children_ids = self._get_child_ids_recursively(self.id)
  1004. Document2.objects.filter(id__in=children_ids).update(is_trashed=False)
  1005. def _get_child_ids_recursively(self, directory_id):
  1006. """
  1007. Returns the list of all children ids for a given directory id recursively, excluding history documents
  1008. """
  1009. directory = Directory.objects.get(id=directory_id)
  1010. children_ids = []
  1011. for child in directory.children.filter(is_history=False).filter(is_managed=False):
  1012. children_ids.append(child.id)
  1013. if child.is_directory:
  1014. children_ids.extend(self._get_child_ids_recursively(child.id))
  1015. return children_ids
  1016. def can_read(self, user):
  1017. perm = self.get_permission('read')
  1018. has_read_permissions = perm.user_has_access(user) if perm else False
  1019. return is_admin(user) or self.owner == user or self.can_write(user) or has_read_permissions
  1020. def can_read_or_exception(self, user):
  1021. if self.can_read(user):
  1022. return True
  1023. else:
  1024. raise PopupException(_("Document does not exist or you don't have the permission to access it."), error_code=401)
  1025. def can_write(self, user):
  1026. perm = self.get_permission('write')
  1027. has_write_permissions = perm.user_has_access(user) if perm else False
  1028. return is_admin(user) or self.owner == user or has_write_permissions or (self.parent_directory and self.parent_directory.can_write(user))
  1029. def can_write_or_exception(self, user):
  1030. if self.can_write(user):
  1031. return True
  1032. else:
  1033. raise PopupException(_("Document does not exist or you don't have the permission to access it."))
  1034. def get_permission(self, perm='read'):
  1035. try:
  1036. return Document2Permission.objects.get(doc=self, perms=perm)
  1037. except Document2Permission.DoesNotExist:
  1038. return None
  1039. def share(self, user, name='read', users=None, groups=None):
  1040. try:
  1041. with transaction.atomic():
  1042. self.update_permission(user, name, users, groups)
  1043. # For directories, update all children recursively with same permissions
  1044. for child in self.children.all():
  1045. child.share(user, name, users, groups)
  1046. except Exception, e:
  1047. raise PopupException(_("Failed to share document: %s") % e)
  1048. return self
  1049. def update_permission(self, user, name='read', users=None, groups=None):
  1050. # Check if user has access to grant permissions
  1051. if users or groups:
  1052. if name == 'read':
  1053. self.can_read_or_exception(user)
  1054. elif name == 'write':
  1055. self.can_write_or_exception(user)
  1056. else:
  1057. raise ValueError(_('Invalid permission type: %s') % name)
  1058. perm, created = Document2Permission.objects.get_or_create(doc=self, perms=name)
  1059. perm.users = []
  1060. if users is not None:
  1061. perm.users = users
  1062. perm.groups = []
  1063. if groups is not None:
  1064. perm.groups = groups
  1065. perm.save()
  1066. def _get_doc1(self, doc2_type=None):
  1067. if not doc2_type:
  1068. doc2_type = self.type
  1069. try:
  1070. doc = self.doc.get()
  1071. except Exception, e:
  1072. LOG.error('Exception when retrieving document object for saved query: %s' % e)
  1073. doc = Document.objects.link(
  1074. self,
  1075. owner=self.owner,
  1076. name=self.name,
  1077. description=self.description,
  1078. extra=doc2_type
  1079. )
  1080. return doc
  1081. def _massage_permissions(self):
  1082. """
  1083. Returns the permissions for a given document as a dictionary
  1084. """
  1085. permissions = {
  1086. 'read': {'users': [], 'groups': []},
  1087. 'write': {'users': [], 'groups': []}
  1088. }
  1089. read_perms = self.get_permission(perm='read')
  1090. write_perms = self.get_permission(perm='write')
  1091. if read_perms:
  1092. permissions.update(read_perms.to_dict())
  1093. if write_perms:
  1094. permissions.update(write_perms.to_dict())
  1095. return permissions
  1096. def _redact_query(self):
  1097. """
  1098. Optionally mask out the query from being saved to the database. This is because if the database contains sensitive
  1099. information like personally identifiable information, that information could be leaked into the Hue database and
  1100. logfiles.
  1101. """
  1102. if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
  1103. data_dict = self.data_dict
  1104. snippets = data_dict.get('snippets', [])
  1105. for snippet in snippets:
  1106. if snippet['type'] in ('hive', 'impala'): # TODO: Pull SQL types from canonical lookup
  1107. redacted_statement_raw = global_redaction_engine.redact(snippet['statement_raw'])
  1108. if snippet['statement_raw'] != redacted_statement_raw:
  1109. snippet['statement_raw'] = redacted_statement_raw
  1110. snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
  1111. snippet['is_redacted'] = True
  1112. self.data = json.dumps(data_dict)
  1113. self.search = global_redaction_engine.redact(self.search)
  1114. def _contains_cycle(self):
  1115. """
  1116. Uses Floyd's cycle-detection algorithm to detect a cycle (aka Tortoise and Hare)
  1117. https://en.wikipedia.org/wiki/Cycle_detection#Tortoise_and_hare
  1118. """
  1119. # Test base case where self.uuid == self.parent_directory.uuid first
  1120. if self.parent_directory is not None and self.parent_directory.uuid == self.uuid:
  1121. return True
  1122. slow = self
  1123. fast = self
  1124. while True:
  1125. slow = slow.parent_directory
  1126. if slow and slow.uuid == self.uuid:
  1127. slow = self
  1128. if fast.parent_directory is not None:
  1129. if fast.parent_directory.uuid == self.uuid:
  1130. fast = self.parent_directory.parent_directory
  1131. else:
  1132. fast = fast.parent_directory.parent_directory
  1133. else:
  1134. return False
  1135. if slow is None or fast is None:
  1136. return False
  1137. if slow == fast:
  1138. return True
  1139. class DirectoryManager(Document2Manager):
  1140. def get_queryset(self):
  1141. return super(DirectoryManager, self).get_queryset().filter(type='directory')
  1142. class Directory(Document2):
  1143. # e.g. name = '/' or '/dir1/dir2/f3'
  1144. objects = DirectoryManager()
  1145. class Meta:
  1146. proxy = True
  1147. def get_children_documents(self):
  1148. """
  1149. Returns the children documents for a given directory, excluding history documents
  1150. """
  1151. documents = self.children.filter(is_history=False).filter(is_managed=False) # TODO: perms
  1152. return documents
  1153. def get_children_and_shared_documents(self, user):
  1154. """
  1155. Returns the children and shared documents for a given directory, excluding history documents
  1156. """
  1157. # Get documents that are direct children, or shared with but not owned by the current user
  1158. documents = Document2.objects.filter(
  1159. Q(parent_directory=self) |
  1160. ( (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
  1161. ~Q(owner=user) )
  1162. )
  1163. documents = documents.exclude(is_history=True).exclude(is_managed=True)
  1164. # Excluding all trashed docs across users
  1165. documents = documents.exclude(is_trashed=True)
  1166. # Optimizing roll up for /home by checking only with directories instead of all documents
  1167. # For all other directories roll up is done in _filter_documents()
  1168. directories_all = Document2.objects.filter(type='directory').exclude(id=self.id)
  1169. directories_inside_home = directories_all.filter(
  1170. (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
  1171. ~Q(owner=user)
  1172. )
  1173. documents = documents.exclude(parent_directory__in=directories_inside_home)
  1174. return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
  1175. def save(self, *args, **kwargs):
  1176. self.type = 'directory'
  1177. super(Directory, self).save(*args, **kwargs)
  1178. class Document2Permission(models.Model):
  1179. """
  1180. Combine either:
  1181. - regular perms (listed)
  1182. - link
  1183. """
  1184. READ_PERM = 'read'
  1185. WRITE_PERM = 'write'
  1186. COMMENT_PERM = 'comment'
  1187. doc = models.ForeignKey(Document2)
  1188. users = models.ManyToManyField(auth_models.User, db_index=True, db_table='documentpermission2_users')
  1189. groups = models.ManyToManyField(auth_models.Group, db_index=True, db_table='documentpermission2_groups')
  1190. perms = models.CharField(default=READ_PERM, max_length=10, db_index=True, choices=( # one perm
  1191. (READ_PERM, 'read'),
  1192. (WRITE_PERM, 'write'),
  1193. (COMMENT_PERM, 'comment'), # PLAYER PERM?
  1194. ))
  1195. # link = models.CharField(default=uuid_default, max_length=255, unique=True) # Short link like dropbox
  1196. # embed
  1197. class Meta:
  1198. unique_together = ('doc', 'perms')
  1199. def to_dict(self):
  1200. return {
  1201. self.perms: {
  1202. 'users': [{'id': perm_user.id, 'username': perm_user.username} for perm_user in self.users.all()],
  1203. 'groups': [{'id': perm_group.id, 'name': perm_group.name} for perm_group in self.groups.all()]
  1204. }
  1205. }
  1206. def user_has_access(self, user):
  1207. """
  1208. Returns true if the given user has permissions based on users, groups, or all flag
  1209. """
  1210. return self.groups.filter(id__in=user.groups.all()).exists() or user in self.users.all()
  1211. def get_cluster_config(user):
  1212. return Cluster(user).get_app_config().get_config()
  1213. # Aka 'Atus'
  1214. ANALYTIC_DB = 'altus'
  1215. class ClusterConfig():
  1216. """
  1217. Configuration of the apps and engines that each individual user sees on the core Hue.
  1218. Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
  1219. TODO: rename to HueConfig
  1220. TODO: get list of contexts dynamically
  1221. """
  1222. def __init__(self, user, apps=None, cluster_type='ini'):
  1223. self.user = user
  1224. self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
  1225. self.cluster_type = cluster_type
  1226. def refreshConfig(self):
  1227. # TODO: reload "some ini sections"
  1228. pass
  1229. def get_config(self):
  1230. app_config = self.get_apps()
  1231. editors = app_config.get('editor')
  1232. main_button_action = self.get_main_quick_action(app_config)
  1233. if main_button_action.get('is_sql'):
  1234. default_sql_interpreter = main_button_action['type']
  1235. else:
  1236. default_sql_interpreter = editors and editors['default_sql_interpreter']
  1237. return {
  1238. 'app_config': app_config,
  1239. 'main_button_action': main_button_action,
  1240. 'button_actions': [
  1241. app for app in [
  1242. editors,
  1243. app_config.get('dashboard'),
  1244. app_config.get('scheduler')
  1245. ] if app is not None
  1246. ],
  1247. 'default_sql_interpreter': default_sql_interpreter,
  1248. 'cluster_type': self.cluster_type
  1249. }
  1250. def get_apps(self):
  1251. apps = OrderedDict([app for app in [
  1252. ('editor', self._get_editor()),
  1253. ('dashboard', self._get_dashboard()),
  1254. ('catalogs', self._get_catalogs()),
  1255. ('browser', self._get_browser()),
  1256. ('scheduler', self._get_scheduler()),
  1257. ('sdkapps', self._get_sdk_apps()),
  1258. ('home', self._get_home()),
  1259. ] if app[1]])
  1260. return apps
  1261. def get_main_quick_action(self, apps):
  1262. if not apps:
  1263. raise PopupException(_('No permission to any app.'))
  1264. default_app = apps.values()[0]
  1265. default_interpreter = default_app.get('interpreters')
  1266. try:
  1267. user_default_app = json.loads(UserPreferences.objects.get(user=self.user, key='default_app').value)
  1268. if apps.get(user_default_app['app']):
  1269. default_interpreter = []
  1270. default_app = apps[user_default_app['app']]
  1271. if default_app.get('interpreters'):
  1272. interpreters = [interpreter for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']]
  1273. if interpreters:
  1274. default_interpreter = interpreters
  1275. except UserPreferences.DoesNotExist:
  1276. pass
  1277. except Exception:
  1278. LOG.exception('Could not load back default app')
  1279. if default_interpreter:
  1280. return default_interpreter[0]
  1281. else:
  1282. return default_app
  1283. def _get_home(self):
  1284. return {
  1285. 'name': 'home',
  1286. 'displayName': _('Home'),
  1287. 'buttonName': _('Documents'),
  1288. 'interpreters': [],
  1289. 'page': '/home'
  1290. }
  1291. def _get_editor(self):
  1292. interpreters = []
  1293. _interpreters = get_ordered_interpreters(self.user)
  1294. if ANALYTIC_DB in self.cluster_type:
  1295. _interpreters = [interpreter for interpreter in _interpreters if interpreter['type'] in ('impala')] #, 'hive', 'spark2', 'pyspark', 'mapreduce')]
  1296. for interpreter in _interpreters:
  1297. if interpreter['interface'] != 'hms':
  1298. interpreters.append({
  1299. 'name': interpreter['name'],
  1300. 'type': interpreter['type'],
  1301. 'displayName': interpreter['name'],
  1302. 'buttonName': _('Query'),
  1303. 'tooltip': _('%s Query') % interpreter['type'].title(),
  1304. 'page': '/editor/?type=%(type)s' % interpreter,
  1305. 'is_sql': interpreter['is_sql'],
  1306. })
  1307. if SHOW_NOTEBOOKS.get() and ANALYTIC_DB not in self.cluster_type:
  1308. try:
  1309. first_non_sql_index = [interpreter['is_sql'] for interpreter in interpreters].index(False)
  1310. except ValueError:
  1311. first_non_sql_index = len(interpreters) if all([interpreter['is_sql'] for interpreter in interpreters]) else 0
  1312. interpreters.insert(first_non_sql_index, {
  1313. 'name': 'notebook',
  1314. 'type': 'notebook',
  1315. 'displayName': 'Notebook',
  1316. 'buttonName': _('Notebook'),
  1317. 'tooltip': _('Notebook'),
  1318. 'page': '/notebook',
  1319. 'is_sql': False,
  1320. })
  1321. if interpreters:
  1322. return {
  1323. 'name': 'editor',
  1324. 'displayName': _('Editor'),
  1325. 'buttonName': _('Query'),
  1326. 'interpreters': interpreters,
  1327. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1328. 'page': interpreters[0]['page'],
  1329. 'default_sql_interpreter': next((interpreter['type'] for interpreter in interpreters if interpreter.get('is_sql')), 'hive')
  1330. }
  1331. else:
  1332. return None
  1333. def _get_catalogs(self):
  1334. interpreters = []
  1335. _interpreters = get_ordered_interpreters(self.user)
  1336. for interpreter in _interpreters:
  1337. if interpreter['interface'] == 'hms':
  1338. interpreters.append({
  1339. 'name': interpreter['name'],
  1340. 'type': interpreter['type'],
  1341. 'displayName': interpreter['name'],
  1342. 'buttonName': _('Query'),
  1343. 'tooltip': _('%s Query') % interpreter['type'].title(),
  1344. 'page': '/editor/?type=%(type)s' % interpreter,
  1345. 'is_sql': interpreter['is_sql'],
  1346. 'is_catalog': interpreter['is_catalog'],
  1347. })
  1348. return interpreters if interpreters else None
  1349. def _get_dashboard(self):
  1350. interpreters = get_engines(self.user)
  1351. _interpreters = []
  1352. if interpreters and ANALYTIC_DB not in self.cluster_type:
  1353. if HAS_REPORT_ENABLED.get():
  1354. _interpreters.append({
  1355. 'type': 'report',
  1356. 'displayName': 'Report',
  1357. 'buttonName': 'Report',
  1358. 'page': '/dashboard/new_search?engine=report',
  1359. 'tooltip': _('Report'),
  1360. 'is_sql': False
  1361. })
  1362. _interpreters.extend([{
  1363. 'type': interpreter['type'],
  1364. 'displayName': interpreter['type'].title(),
  1365. 'buttonName': interpreter['type'].title(),
  1366. 'page': '/dashboard/new_search?engine=%(type)s' % interpreter,
  1367. 'tooltip': _('%s Dashboard') % interpreter['type'].title(),
  1368. 'is_sql': True
  1369. } for interpreter in interpreters
  1370. ])
  1371. return {
  1372. 'name': 'dashboard',
  1373. 'displayName': _('Dashboard'),
  1374. 'buttonName': _('Dashboard'),
  1375. 'interpreters': _interpreters,
  1376. 'page': '/dashboard/new_search?engine=%(type)s' % interpreters[0]
  1377. }
  1378. else:
  1379. return None
  1380. def _get_browser(self):
  1381. interpreters = []
  1382. if has_hdfs_enabled() and 'filebrowser' in self.apps and ANALYTIC_DB not in self.cluster_type:
  1383. interpreters.append({
  1384. 'type': 'hdfs',
  1385. 'displayName': _('Files'),
  1386. 'buttonName': _('Browse'),
  1387. 'tooltip': _('Files'),
  1388. 'page': '/filebrowser/' + (not self.user.is_anonymous() and 'view=' + urllib.quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or '')
  1389. })
  1390. if is_s3_enabled() and 'filebrowser' in self.apps and has_s3_access(self.user) and not IS_EMBEDDED.get():
  1391. interpreters.append({
  1392. 'type': 's3',
  1393. 'displayName': _('S3'),
  1394. 'buttonName': _('Browse'),
  1395. 'tooltip': _('S3'),
  1396. 'page': '/filebrowser/view=' + urllib.quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1397. })
  1398. if is_adls_enabled() and 'filebrowser' in self.apps and has_adls_access(self.user) and ANALYTIC_DB not in self.cluster_type:
  1399. interpreters.append({
  1400. 'type': 'adls',
  1401. 'displayName': _('ADLS'),
  1402. 'buttonName': _('Browse'),
  1403. 'tooltip': _('ADLS'),
  1404. 'page': '/filebrowser/view=' + urllib.quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
  1405. })
  1406. if 'metastore' in self.apps:
  1407. interpreters.append({
  1408. 'type': 'tables',
  1409. 'displayName': _('Tables'),
  1410. 'buttonName': _('Browse'),
  1411. 'tooltip': _('Tables'),
  1412. 'page': '/metastore/tables'
  1413. })
  1414. if 'search' in self.apps and ANALYTIC_DB not in self.cluster_type:
  1415. interpreters.append({
  1416. 'type': 'indexes',
  1417. 'displayName': _('Indexes'),
  1418. 'buttonName': _('Dashboard'),
  1419. 'tooltip': _('Indexes'),
  1420. 'page': '/indexer/'
  1421. })
  1422. if 'jobbrowser' in self.apps:
  1423. from hadoop.cluster import get_default_yarncluster # Circular loop
  1424. title = _('Jobs') if ANALYTIC_DB not in self.cluster_type else _('Queries')
  1425. if get_default_yarncluster():
  1426. interpreters.append({
  1427. 'type': 'yarn',
  1428. 'displayName': title,
  1429. 'buttonName': title,
  1430. 'tooltip': title,
  1431. 'page': '/jobbrowser/'
  1432. })
  1433. if has_kafka() and ANALYTIC_DB not in self.cluster_type:
  1434. interpreters.append({
  1435. 'type': 'kafka',
  1436. 'displayName': _('Streams'),
  1437. 'buttonName': _('Browse'),
  1438. 'tooltip': _('Kafka'),
  1439. 'page': '/kafka/'
  1440. })
  1441. if 'hbase' in self.apps and ANALYTIC_DB not in self.cluster_type:
  1442. interpreters.append({
  1443. 'type': 'hbase',
  1444. 'displayName': _('HBase'),
  1445. 'buttonName': _('Browse'),
  1446. 'tooltip': _('HBase'),
  1447. 'page': '/hbase/'
  1448. })
  1449. if 'security' in self.apps and not IS_EMBEDDED.get():
  1450. interpreters.append({
  1451. 'type': 'security',
  1452. 'displayName': _('Security'),
  1453. 'buttonName': _('Browse'),
  1454. 'tooltip': _('Security'),
  1455. 'page': '/security/hive'
  1456. })
  1457. if 'sqoop' in self.apps and ANALYTIC_DB not in self.cluster_type:
  1458. from sqoop.conf import IS_ENABLED
  1459. if IS_ENABLED.get():
  1460. interpreters.append({
  1461. 'type': 'sqoop',
  1462. 'displayName': _('Sqoop'),
  1463. 'buttonName': _('Browse'),
  1464. 'tooltip': _('Sqoop'),
  1465. 'page': '/sqoop'
  1466. })
  1467. if interpreters:
  1468. return {
  1469. 'name': 'browser',
  1470. 'displayName': _('Browsers'),
  1471. 'buttonName': _('Browse'),
  1472. 'interpreters': interpreters,
  1473. 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
  1474. }
  1475. else:
  1476. return None
  1477. def _get_scheduler(self):
  1478. interpreters = [{
  1479. 'type': 'oozie-workflow',
  1480. 'displayName': _('Workflow'),
  1481. 'buttonName': _('Schedule'),
  1482. 'tooltip': _('Workflow'),
  1483. 'page': '/oozie/editor/workflow/new/'
  1484. }, {
  1485. 'type': 'oozie-coordinator',
  1486. 'displayName': _('Schedule'),
  1487. 'buttonName': _('Schedule'),
  1488. 'tooltip': _('Schedule'),
  1489. 'page': '/oozie/editor/coordinator/new/'
  1490. }, {
  1491. 'type': 'oozie-bundle',
  1492. 'displayName': _('Bundle'),
  1493. 'buttonName': _('Schedule'),
  1494. 'tooltip': _('Bundle'),
  1495. 'page': '/oozie/editor/bundle/new/'
  1496. }
  1497. ]
  1498. if 'oozie' in self.apps and not (self.user.has_hue_permission(action="disable_editor_access", app="oozie") and not is_admin(self.user)) and ANALYTIC_DB not in self.cluster_type:
  1499. return {
  1500. 'name': 'oozie',
  1501. 'displayName': _('Scheduler'),
  1502. 'buttonName': _('Schedule'),
  1503. 'interpreters': interpreters,
  1504. 'page': interpreters[0]['page']
  1505. }
  1506. else:
  1507. return None
  1508. def _get_sdk_apps(self):
  1509. current_app, other_apps, apps_list = _get_apps(self.user)
  1510. interpreters = []
  1511. for other in other_apps:
  1512. interpreters.append({
  1513. 'type': other.display_name,
  1514. 'displayName': other.nice_name,
  1515. 'buttonName': other.nice_name,
  1516. 'tooltip': other.nice_name,
  1517. 'page': '/%s' % other.display_name
  1518. })
  1519. if interpreters:
  1520. return {
  1521. 'name': 'other',
  1522. 'displayName': _('Other Apps'),
  1523. 'buttonName': _('Other'),
  1524. 'interpreters': interpreters,
  1525. }
  1526. else:
  1527. return None
  1528. def get_hive_metastore_interpreters(self):
  1529. return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter == 'hive' or interpreter == 'hms']
  1530. class Cluster():
  1531. def __init__(self, user):
  1532. self.user = user
  1533. self.clusters = get_clusters(user)
  1534. if len(self.clusters) == 1:
  1535. self.data = self.clusters.values()[0]
  1536. elif IS_K8S_ONLY.get():
  1537. self.data = self.clusters['AltusV2']
  1538. self.data['type'] = 'altus' # To show simplified UI
  1539. elif IS_MULTICLUSTER_ONLY.get():
  1540. self.data = self.clusters['Altus']
  1541. else:
  1542. self.data = self.clusters[CLUSTER_ID.get()]
  1543. def get_type(self):
  1544. return self.data['type']
  1545. def get_config(self, name):
  1546. return self.clusters[name]
  1547. def get_app_config(self):
  1548. return ClusterConfig(self.user, cluster_type=self.get_type())
  1549. def _get_apps(user, section=None):
  1550. current_app = None
  1551. other_apps = []
  1552. if user.is_authenticated():
  1553. apps_list = appmanager.get_apps_dict(user)
  1554. apps = apps_list.values()
  1555. for app in apps:
  1556. if app.display_name not in [
  1557. 'beeswax', 'hive', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
  1558. 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1:
  1559. other_apps.append(app)
  1560. if section == app.display_name:
  1561. current_app = app
  1562. else:
  1563. apps_list = []
  1564. return current_app, other_apps, apps_list
  1565. def get_user_preferences(user, key=None):
  1566. if key is not None:
  1567. try:
  1568. x = UserPreferences.objects.get(user=user, key=key)
  1569. return {key: x.value}
  1570. except UserPreferences.DoesNotExist:
  1571. return None
  1572. else:
  1573. return dict((x.key, x.value) for x in UserPreferences.objects.filter(user=user))
  1574. def set_user_preferences(user, key, value):
  1575. try:
  1576. x = UserPreferences.objects.get(user=user, key=key)
  1577. except UserPreferences.DoesNotExist:
  1578. x = UserPreferences(user=user, key=key)
  1579. x.value = value
  1580. x.save()
  1581. return x
  1582. def get_data_link(meta):
  1583. link = None
  1584. if not meta.get('type'):
  1585. pass
  1586. elif meta['type'] == 'hbase':
  1587. link = '/hbase/#Cluster/%(table)s/query/%(row_key)s' % meta
  1588. if 'col' in meta:
  1589. link += '[%(fam)s:%(col)s]' % meta
  1590. elif 'fam' in meta:
  1591. link += '[%(fam)s]' % meta
  1592. elif meta['type'] == 'hdfs':
  1593. link = '/filebrowser/view=%(path)s' % meta # Could add a byte #
  1594. elif meta['type'] == 'link':
  1595. link = meta['link']
  1596. elif meta['type'] == 'hive':
  1597. link = '/metastore/table/%(database)s/%(table)s' % meta # Could also add col=val
  1598. return link