| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167 |
- #!/usr/bin/env python
- # Licensed to Cloudera, Inc. under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. Cloudera, Inc. licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- from __future__ import absolute_import
- from future import standard_library
- standard_library.install_aliases()
- from builtins import next
- from builtins import object
- import calendar
- import json
- import logging
- import os
- import sys
- import uuid
- from collections import OrderedDict
- from itertools import chain
- from django.db import connection, models, transaction
- from django.db.models import Q
- from django.db.models.query import QuerySet
- from django.contrib.auth.validators import UnicodeUsernameValidator
- from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey
- from django.contrib.contenttypes.models import ContentType
- from django.contrib.staticfiles.storage import staticfiles_storage
- from django.urls import reverse, NoReverseMatch
- from django.utils.translation import ugettext as _, ugettext_lazy as _t
- from dashboard.conf import get_engines, HAS_REPORT_ENABLED
- from kafka.conf import has_kafka
- from notebook.conf import DEFAULT_LIMIT, SHOW_NOTEBOOKS, get_ordered_interpreters
- from useradmin.models import User, Group, get_organization
- from useradmin.organization import _fitered_queryset
- from desktop import appmanager
- from desktop.auth.backend import is_admin
- from desktop.conf import get_clusters, CLUSTER_ID, IS_MULTICLUSTER_ONLY, IS_K8S_ONLY, ENABLE_ORGANIZATIONS, ENABLE_PROMETHEUS,\
- has_connectors, TASK_SERVER, ENABLE_GIST, APP_BLACKLIST
- from desktop.lib import fsmanager
- from desktop.lib.connectors.api import _get_installed_connectors
- from desktop.lib.i18n import force_unicode
- from desktop.lib.exceptions_renderable import PopupException
- from desktop.lib.paths import get_run_root, SAFE_CHARACTERS_URI_COMPONENTS
- from desktop.redaction import global_redaction_engine
- from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH, HUE_DESKTOP_VERSION
- if sys.version_info[0] > 2:
- from urllib.parse import quote as urllib_quote
- else:
- from urllib import quote as urllib_quote
- LOG = logging.getLogger(__name__)
- SAMPLE_USER_ID = 1100713
- SAMPLE_USER_INSTALL = 'hue'
- SAMPLE_USER_OWNERS = ['hue', 'sample']
- UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
- HUE_VERSION = None
- def uuid_default():
- return str(uuid.uuid4())
- def hue_version():
- global HUE_VERSION
- if HUE_VERSION is None:
- HUE_VERSION = HUE_DESKTOP_VERSION
- p = get_run_root('cloudera', 'cdh_version.properties')
- if os.path.exists(p):
- build_version = _version_from_properties(open(p))
- if build_version:
- HUE_VERSION = '%s - %s' % (HUE_VERSION, build_version)
- return HUE_VERSION
- def _version_from_properties(f):
- return dict(line.strip().split('=') for line in f.readlines() if len(line.strip().split('=')) == 2).get('cloudera.cdh.release')
- def get_sample_user_install(user):
- if ENABLE_ORGANIZATIONS.get():
- organization = get_organization(email=user.email)
- if organization.is_multi_user:
- return SAMPLE_USER_INSTALL + '@' + organization.domain
- else:
- return organization.domain
- else:
- return SAMPLE_USER_INSTALL
- ###################################################################################################
- # Custom Settings
- ###################################################################################################
- PREFERENCE_IS_WELCOME_TOUR_SEEN = 'is_welcome_tour_seen'
- if not ENABLE_ORGANIZATIONS.get():
- class HueUser(User):
- class Meta(object):
- proxy = True
- def __init__(self, *args, **kwargs):
- self._meta.get_field(
- 'username'
- ).validators[0] = UnicodeUsernameValidator()
- super(User, self).__init__(*args, **kwargs)
- class UserPreferences(models.Model):
- """Holds arbitrary key/value strings."""
- user = models.ForeignKey(User)
- key = models.CharField(max_length=20)
- value = models.TextField(max_length=4096)
- class Settings(models.Model):
- collect_usage = models.BooleanField(db_index=True, default=True)
- tours_and_tutorials = models.BooleanField(db_index=True, default=True)
- @classmethod
- def get_settings(cls):
- settings, created = Settings.objects.get_or_create(id=1)
- return settings
- class DefaultConfigurationManager(models.Manager):
- def get_configuration_for_user(self, app, user):
- """
- :param app: app name
- :param user: User object
- :return: DefaultConfiguration for user, or first group found, or default for app, or None
- """
- try:
- return super(DefaultConfigurationManager, self).get(app=app, user=user)
- except DefaultConfiguration.DoesNotExist:
- pass
- configs = super(DefaultConfigurationManager, self).get_queryset().filter(app=app, groups__in=user.groups.all())
- if configs.count() > 0:
- return configs[0]
- try:
- return super(DefaultConfigurationManager, self).get(app=app, is_default=True)
- except DefaultConfiguration.DoesNotExist:
- pass
- return None
- class DefaultConfiguration(models.Model):
- """
- Default values for configuration properties for a given app/editor
- Can be designated as default for all users by is_default flag, or for a specific group or user
- """
- app = models.CharField(max_length=32, null=False, db_index=True, help_text=_t('App that this configuration belongs to.'))
- properties = models.TextField(default='[]', help_text=_t('JSON-formatted default properties values.'))
- is_default = models.BooleanField(default=False, db_index=True)
- groups = models.ManyToManyField(Group, db_index=True, db_table='defaultconfiguration_groups')
- user = models.ForeignKey(User, blank=True, null=True, db_index=True)
- objects = DefaultConfigurationManager()
- class Meta(object):
- ordering = ["app", "-is_default", "user"]
- @property
- def properties_list(self):
- """
- :return: Deserialized properties as a list of property objects
- """
- if not self.properties:
- self.properties = json.dumps([])
- return json.loads(self.properties)
- @property
- def properties_dict(self):
- """
- :return: Deserialized properties as a dict of key: value pairs
- """
- if not self.properties:
- self.properties = json.dumps([])
- return dict((prop['key'], prop['value']) for prop in json.loads(self.properties))
- def to_dict(self):
- return {
- 'id': self.id,
- 'app': self.app,
- 'properties': self.properties_list,
- 'is_default': self.is_default,
- 'group_ids': [group.id for group in self.groups.all()],
- 'user': self.user.username if self.user else None
- }
- ###################################################################################################
- # Document1
- ###################################################################################################
- class DocumentTagManager(models.Manager):
- def get_tags(self, user):
- return self.filter(owner=user).distinct()
- def create_tag(self, owner, tag_name):
- if tag_name in DocumentTag.RESERVED:
- raise Exception(_("Can't add %s: it is a reserved tag.") % tag_name)
- else:
- tag, created = DocumentTag.objects.get_or_create(tag=tag_name, owner=owner)
- return tag
- def _get_tag(self, user, name):
- tag, created = DocumentTag.objects.get_or_create(owner=user, tag=name)
- return tag
- def get_default_tag(self, user):
- return self._get_tag(user, DocumentTag.DEFAULT)
- def get_trash_tag(self, user):
- return self._get_tag(user, DocumentTag.TRASH)
- def get_history_tag(self, user):
- return self._get_tag(user, DocumentTag.HISTORY)
- def get_example_tag(self, user):
- return self._get_tag(user, DocumentTag.EXAMPLE)
- def get_imported2_tag(self, user):
- return self._get_tag(user, DocumentTag.IMPORTED2)
- def tag(self, owner, doc_id, tag_name='', tag_id=None):
- try:
- tag = DocumentTag.objects.get(id=tag_id, owner=owner)
- if tag.tag in DocumentTag.RESERVED:
- raise Exception(_("Can't add %s: it is a reserved tag.") % tag)
- except DocumentTag.DoesNotExist:
- tag = self._get_tag(user=owner, name=tag_name)
- doc = Document.objects.get_doc_for_writing(doc_id, owner)
- doc.add_tag(tag)
- return tag
- def untag(self, tag_id, owner, doc_id):
- tag = DocumentTag.objects.get(id=tag_id, owner=owner)
- if tag.tag in DocumentTag.RESERVED:
- raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
- doc = Document.objects.get_doc_for_writing(doc_id, owner=owner)
- doc.remove_tag(tag)
- def delete_tag(self, tag_id, owner):
- tag = DocumentTag.objects.get(id=tag_id, owner=owner)
- default_tag = DocumentTag.objects.get_default_tag(owner)
- if tag.tag in DocumentTag.RESERVED:
- raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
- else:
- tag.delete()
- for doc in Document.objects.get_docs(owner).filter(tags=None):
- doc.add_tag(default_tag)
- def update_tags(self, owner, doc_id, tag_ids):
- doc = Document.objects.get_doc_for_writing(doc_id, owner)
- for tag in doc.tags.all():
- if tag.tag not in DocumentTag.RESERVED:
- doc.remove_tag(tag)
- for tag_id in tag_ids:
- tag = DocumentTag.objects.get(id=tag_id, owner=owner)
- if tag.tag not in DocumentTag.RESERVED:
- doc.add_tag(tag)
- return doc
- class DocumentTag(models.Model):
- """
- Reserved tags can't be manually removed by the user.
- """
- owner = models.ForeignKey(User, db_index=True)
- tag = models.SlugField()
- DEFAULT = 'default' # Always there
- TRASH = 'trash' # There when the document is trashed
- HISTORY = 'history' # There when the document is a submission history
- EXAMPLE = 'example' # Hue examples
- IMPORTED2 = 'imported2' # Was imported to document2
- RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE, IMPORTED2)
- objects = DocumentTagManager()
- class Meta(object):
- unique_together = ('owner', 'tag')
- def __unicode__(self):
- return force_unicode('%s') % (self.tag,)
- class DocumentManager(models.Manager):
- def documents(self, user):
- return Document.objects.filter(
- Q(owner=user) |
- Q(documentpermission__users=user) |
- Q(documentpermission__groups__in=user.groups.all())
- ).defer('description', 'extra').distinct()
- def get_docs(self, user, model_class=None, extra=None, qfilter=None):
- docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
- if model_class is not None:
- ct = ContentType.objects.get_for_model(model_class)
- docs = docs.filter(content_type=ct)
- if extra is not None:
- docs = docs.filter(extra=extra)
- if qfilter is not None:
- docs = docs.filter(qfilter)
- return docs
- def get_doc_for_writing(self, doc_id, user):
- """Fetch a document and confirm that this user can write to it."""
- doc = Document.objects.documents(user).get(id=doc_id)
- doc.can_write_or_exception(user)
- return doc
- def trashed_docs(self, model_class, user):
- tag = DocumentTag.objects.get_trash_tag(user=user)
- return Document.objects.get_docs(user, model_class).filter(tags__in=[tag]).order_by('-last_modified')
- def trashed(self, model_class, user):
- docs = self.trashed_docs(model_class, user)
- return [job.content_object for job in docs if job.content_object]
- def available_docs(self, model_class, user, with_history=False):
- exclude = [DocumentTag.objects.get_trash_tag(user=user)]
- if not with_history:
- exclude.append(DocumentTag.objects.get_history_tag(user=user))
- return Document.objects.get_docs(user, model_class).exclude(tags__in=exclude).order_by('-last_modified')
- def history_docs(self, model_class, user):
- include = [DocumentTag.objects.get_history_tag(user=user)]
- exclude = [DocumentTag.objects.get_trash_tag(user=user)]
- return Document.objects.get_docs(user, model_class).filter(tags__in=include).exclude(tags__in=exclude).order_by('-last_modified')
- def available(self, model_class, user, with_history=False):
- docs = self.available_docs(model_class, user, with_history)
- return [doc.content_object for doc in docs if doc.content_object]
- def can_read_or_exception(self, user, doc_class, doc_id, exception_class=PopupException):
- if doc_id is None:
- return
- try:
- ct = ContentType.objects.get_for_model(doc_class)
- doc = Document.objects.get(object_id=doc_id, content_type=ct)
- if doc.can_read(user):
- return doc
- else:
- message = _("Permission denied. %(username)s does not have the permissions required to access document %(id)s") % \
- {'username': user.username, 'id': doc.id}
- raise exception_class(message)
- except Document.DoesNotExist:
- raise exception_class(_('Document %(id)s does not exist') % {'id': doc_id})
- def can_read(self, user, doc_class, doc_id):
- ct = ContentType.objects.get_for_model(doc_class)
- doc = Document.objects.get(object_id=doc_id, content_type=ct)
- return doc.can_read(user)
- def link(self, content_object, owner, name='', description='', extra=''):
- if not content_object.doc.exists():
- doc = Document.objects.create(
- content_object=content_object,
- owner=owner,
- name=name,
- description=description,
- extra=extra
- )
- tag = DocumentTag.objects.get_default_tag(user=owner)
- doc.tags.add(tag)
- return doc
- else:
- LOG.warn('Object %s already has documents: %s' % (content_object, content_object.doc.all()))
- return content_object.doc.all()[0]
- def sync(self, doc2_only=True):
- def find_jobs_with_no_doc(model):
- jobs = model.objects.filter(doc__isnull=True)
- if model == Document2:
- jobs = jobs.exclude(is_history=True)
- return jobs.select_related('owner')
- def find_oozie_jobs_with_no_doc(model):
- return model.objects.filter(doc__isnull=True).exclude(name__exact='').select_related('owner')
- table_names = connection.introspection.table_names()
- try:
- from oozie.models import Workflow, Coordinator, Bundle
- if \
- Workflow._meta.db_table in table_names or \
- Coordinator._meta.db_table in table_names or \
- Bundle._meta.db_table in table_names:
- with transaction.atomic():
- for job in chain(
- find_oozie_jobs_with_no_doc(Workflow),
- find_oozie_jobs_with_no_doc(Coordinator),
- find_oozie_jobs_with_no_doc(Bundle)):
- doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description)
- if job.is_trashed:
- doc.send_to_trash()
- if job.is_shared:
- doc.share_to_default()
- if hasattr(job, 'managed'):
- if not job.managed:
- doc.extra = 'jobsub'
- doc.save()
- except Exception as e:
- LOG.exception('error syncing oozie')
- try:
- from beeswax.models import SavedQuery
- if SavedQuery._meta.db_table in table_names:
- with transaction.atomic():
- for job in find_jobs_with_no_doc(SavedQuery):
- doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
- if job.is_trashed:
- doc.send_to_trash()
- except Exception as e:
- LOG.exception('error syncing beeswax')
- try:
- from pig.models import PigScript
- if PigScript._meta.db_table in table_names:
- with transaction.atomic():
- for job in find_jobs_with_no_doc(PigScript):
- Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
- except Exception as e:
- LOG.exception('error syncing pig')
- try:
- from search.models import Collection
- if Collection._meta.db_table in table_names:
- with transaction.atomic():
- for dashboard in Collection.objects.all():
- if 'collection' in dashboard.properties_dict:
- col_dict = dashboard.properties_dict['collection']
- if not 'uuid' in col_dict:
- _uuid = str(uuid.uuid4())
- col_dict['uuid'] = _uuid
- dashboard.update_properties({'collection': col_dict})
- if dashboard.owner is None:
- from useradmin.models import install_sample_user
- owner = install_sample_user()
- else:
- owner = dashboard.owner
- dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
- Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
- dashboard.save()
- except Exception as e:
- LOG.exception('error syncing search')
- try:
- if Document2._meta.db_table in table_names:
- with transaction.atomic():
- for job in find_jobs_with_no_doc(Document2):
- if job.type == 'oozie-workflow2':
- extra = 'workflow2'
- elif job.type == 'oozie-coordinator2':
- extra = 'coordinator2'
- elif job.type == 'oozie-bundle2':
- extra = 'bundle2'
- elif job.type == 'notebook':
- extra = 'notebook'
- elif job.type == 'search-dashboard':
- extra = 'search-dashboard'
- else:
- extra = ''
- doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description, extra=extra)
- except Exception as e:
- LOG.exception('error syncing Document2')
- if not doc2_only and Document._meta.db_table in table_names:
- # Make sure doc have at least a tag
- try:
- for doc in Document.objects.filter(tags=None):
- default_tag = DocumentTag.objects.get_default_tag(doc.owner)
- doc.tags.add(default_tag)
- except Exception as e:
- LOG.exception('error adding at least one tag to docs')
- # Make sure all the sample user documents are shared.
- try:
- with transaction.atomic():
- for doc in Document.objects.filter(owner__username__in=SAMPLE_USER_OWNERS):
- doc.share_to_default()
- tag = DocumentTag.objects.get_example_tag(user=doc.owner)
- doc.tags.add(tag)
- doc_last_modified = doc.last_modified
- doc.save()
- Document.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
- except Exception as e:
- LOG.exception('error sharing sample user documents')
- # For now remove the default tag from the examples
- try:
- for doc in Document.objects.filter(tags__tag=DocumentTag.EXAMPLE):
- default_tag = DocumentTag.objects.get_default_tag(doc.owner)
- doc.tags.remove(default_tag)
- except Exception as e:
- LOG.exception('error removing default tags')
- # ------------------------------------------------------------------------
- LOG.info('Looking for documents that have no object')
- # Delete documents with no object.
- try:
- with transaction.atomic():
- # First, delete all the documents that don't have a content type
- docs = Document.objects.filter(content_type=None)
- if docs:
- LOG.info('Deleting %s doc(s) that do not have a content type' % docs.count())
- docs.delete()
- # Next, it's possible that there are documents pointing at a non-existing
- # content_type. We need to do a left join to find these records, but we
- # can't do this directly in django. To get around writing wrap sql (which
- # might not be portable), we'll use an aggregate to count up all the
- # associated content_types, and delete the documents that have a count of
- # zero.
- #
- # Note we're counting `content_type__name` to force the join.
- docs = Document.objects \
- .values('id') \
- .annotate(content_type_count=models.Count('content_type__name')) \
- .filter(content_type_count=0)
- if docs:
- LOG.info('Deleting %s doc(s) that have invalid content types' % docs.count())
- docs.delete()
- # Finally we need to delete documents with no associated content object.
- # This is tricky because of our use of generic foreign keys. So to do
- # this a bit more efficiently, we'll start with a query of all the
- # documents, then step through each content type and and filter out all
- # the documents it's referencing from our document query. Messy, but it
- # works.
- # TODO: This can be several 100k entries for large databases.
- # Need to figure out a better way to handle this scenario.
- docs = Document.objects.all()
- for content_type in ContentType.objects.all():
- model_class = content_type.model_class()
- # Ignore any types that don't have a model.
- if model_class is None:
- continue
- # Ignore types that don't have a table yet.
- if model_class._meta.db_table not in table_names:
- continue
- # Ignore classes that don't have a 'doc'.
- if not hasattr(model_class, 'doc'):
- continue
- # First create a query that grabs all the document ids for this type.
- docs_from_content = model_class.objects.values('doc__id')
- # Next, filter these from our document query.
- docs = docs.exclude(id__in=docs_from_content)
- if docs.exists():
- LOG.info('Deleting %s documents' % docs.count())
- docs.delete()
- except Exception as e:
- LOG.exception('Error in sync while attempting to delete documents with no object: %s' % e)
- class Document(models.Model):
- owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
- name = models.CharField(default='', max_length=255)
- description = models.TextField(default='')
- last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Last modified'))
- version = models.SmallIntegerField(default=1, verbose_name=_t('Schema version'))
- extra = models.TextField(default='')
- tags = models.ManyToManyField(DocumentTag, db_index=True)
- content_type = models.ForeignKey(ContentType)
- object_id = models.PositiveIntegerField()
- content_object = GenericForeignKey('content_type', 'object_id')
- objects = DocumentManager()
- class Meta(object):
- unique_together = ('content_type', 'object_id')
- def __unicode__(self):
- return force_unicode('%s %s %s') % (self.content_type, self.name, self.owner)
- def is_editable(self, user):
- """Deprecated by can_read"""
- return self.can_write(user)
- def can_edit_or_exception(self, user, exception_class=PopupException):
- """Deprecated by can_write_or_exception"""
- return self.can_write_or_exception(user, exception_class)
- def add_tag(self, tag):
- self.tags.add(tag)
- def remove_tag(self, tag):
- self.tags.remove(tag)
- def is_trashed(self):
- return DocumentTag.objects.get_trash_tag(user=self.owner) in self.tags.all()
- def is_historic(self):
- return DocumentTag.objects.get_history_tag(user=self.owner) in self.tags.all()
- def send_to_trash(self):
- tag = DocumentTag.objects.get_trash_tag(user=self.owner)
- self.tags.add(tag)
- def restore_from_trash(self):
- tag = DocumentTag.objects.get_trash_tag(user=self.owner)
- self.tags.remove(tag)
- def add_to_history(self):
- tag = DocumentTag.objects.get_history_tag(user=self.owner)
- self.tags.add(tag)
- def remove_from_history(self):
- tag = DocumentTag.objects.get_history_tag(user=self.owner)
- self.tags.remove(tag)
- def share_to_default(self, name='read'):
- DocumentPermission.objects.share_to_default(self, name=name)
- def can_read(self, user):
- return is_admin(user) or self.owner == user or Document.objects.get_docs(user).filter(id=self.id).exists()
- def can_write(self, user):
- perm = self.list_permissions('write')
- return is_admin(user) or self.owner == user or perm.groups.filter(id__in=user.groups.all()).exists() or user in perm.users.all()
- def can_read_or_exception(self, user, exception_class=PopupException):
- if self.can_read(user):
- return True
- else:
- raise exception_class(_("Document does not exist or you don't have the permission to access it."))
- def can_write_or_exception(self, user, exception_class=PopupException):
- if self.can_write(user):
- return True
- else:
- raise exception_class(_("Document does not exist or you don't have the permission to access it."))
- def copy(self, content_object, name, owner, description=None):
- if content_object:
- copy_doc = self
- copy_doc.pk = None
- copy_doc.id = None
- copy_doc.name = name
- copy_doc.owner = owner
- if description:
- copy_doc.description = description
- copy_doc = Document.objects.link(content_object,
- owner=copy_doc.owner,
- name=copy_doc.name,
- description=copy_doc.description,
- extra=copy_doc.extra)
- # Update reverse Document relation to new copy
- if content_object.doc.get():
- content_object.doc.get().delete()
- content_object.doc.add(copy_doc)
- copy_doc.save()
- return copy_doc
- else:
- raise PopupException(_("Document copy method requires a content_object argument."))
- @property
- def icon(self):
- apps = appmanager.get_apps_dict()
- try:
- if self.extra == 'workflow2':
- return staticfiles_storage.url('oozie/art/icon_oozie_workflow_48.png')
- elif self.extra == 'coordinator2':
- return staticfiles_storage.url('oozie/art/icon_oozie_coordinator_48.png')
- elif self.extra == 'bundle2':
- return staticfiles_storage.url('oozie/art/icon_oozie_bundle_48.png')
- elif self.extra == 'notebook':
- return staticfiles_storage.url('notebook/art/icon_notebook_48.png')
- elif self.extra.startswith('query'):
- if self.extra == 'query-impala':
- return staticfiles_storage.url(apps['impala'].icon_path)
- else:
- return staticfiles_storage.url(apps['beeswax'].icon_path)
- elif self.extra.startswith('search'):
- return staticfiles_storage.url('search/art/icon_search_48.png')
- elif self.content_type.app_label == 'beeswax':
- if self.extra == '0':
- return staticfiles_storage.url(apps['beeswax'].icon_path)
- elif self.extra == '3':
- return staticfiles_storage.url(apps['spark'].icon_path)
- else:
- return staticfiles_storage.url(apps['impala'].icon_path)
- elif self.content_type.app_label == 'oozie':
- if self.extra == 'jobsub':
- return staticfiles_storage.url(apps['jobsub'].icon_path)
- else:
- return staticfiles_storage.url(self.content_type.model_class().ICON)
- elif self.content_type.app_label in apps:
- return staticfiles_storage.url(apps[self.content_type.app_label].icon_path)
- else:
- return staticfiles_storage.url('desktop/art/icon_hue_48.png')
- except Exception as e:
- LOG.warn(force_unicode(e))
- return staticfiles_storage.url('desktop/art/icon_hue_48.png')
- def share(self, users, groups, name='read'):
- DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=True)
- def unshare(self, users, groups, name='read'):
- DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=False)
- def sync_permissions(self, perms_dict):
- """
- Set who else or which other group can interact with the document.
- Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
- """
- for name, perm in perms_dict.items():
- users = groups = None
- if perm.get('user_ids'):
- users = User.objects.in_bulk(perm.get('user_ids'))
- else:
- users = []
- if perm.get('group_ids'):
- groups = Group.objects.in_bulk(perm.get('group_ids'))
- else:
- groups = []
- DocumentPermission.objects.sync(document=self, name=name, users=users, groups=groups)
- def list_permissions(self, perm='read'):
- return DocumentPermission.objects.list(document=self, perm=perm)
- def to_dict(self):
- return {
- 'owner': self.owner.username,
- 'name': self.name,
- 'description': self.description,
- 'uuid': None, # no uuid == v1
- 'id': self.id,
- 'doc1_id': self.id,
- 'object_id': self.object_id,
- 'type': str(self.content_type),
- 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT),
- 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()),
- 'isSelected': False
- }
- class DocumentPermissionManager(models.Manager):
- def _check_perm(self, name):
- perms = (DocumentPermission.READ_PERM, DocumentPermission.WRITE_PERM)
- if name not in perms:
- perms_string = ' and '.join(', '.join(perms).rsplit(', ', 1))
- raise PopupException(_('Only %s permissions are supported, not %s.') % (perms_string, name))
- def share_to_default(self, document, name='read'):
- from useradmin.models import get_default_user_group # Remove build dependency
- self._check_perm(name)
- if name == DocumentPermission.WRITE_PERM:
- perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.WRITE_PERM)
- else:
- perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.READ_PERM)
- default_group = get_default_user_group()
- if default_group:
- perm.groups.add(default_group)
- def update(self, document, name='read', users=None, groups=None, add=True):
- self._check_perm(name)
- perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
- if users is not None:
- if add:
- perm.users.add(*users)
- else:
- perm.users.remove(*users)
- if groups is not None:
- if add:
- perm.groups.add(*groups)
- else:
- perm.groups.remove(*groups)
- if not perm.users and not perm.groups:
- perm.delete()
- def sync(self, document, name='read', users=None, groups=None):
- self._check_perm(name)
- perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)
- if users is not None:
- perm.users = []
- perm.users = users
- perm.save()
- if groups is not None:
- perm.groups = []
- perm.groups = groups
- perm.save()
- if not users and not groups:
- perm.delete()
- def list(self, document, perm='read'):
- perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=perm)
- return perm
- class DocumentPermission(models.Model):
- READ_PERM = 'read'
- WRITE_PERM = 'write'
- doc = models.ForeignKey(Document)
- users = models.ManyToManyField(User, db_index=True, db_table='documentpermission_users')
- groups = models.ManyToManyField(Group, db_index=True, db_table='documentpermission_groups')
- perms = models.CharField(default=READ_PERM, max_length=10, choices=( # one perm
- (READ_PERM, 'read'),
- (WRITE_PERM, 'write'),
- ))
- objects = DocumentPermissionManager()
- class Meta(object):
- unique_together = ('doc', 'perms')
- ###################################################################################################
- # Document2
- ###################################################################################################
- class FilesystemException(Exception):
- pass
- class Document2QueryMixin(object):
- def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False, include_shared_links=False):
- """
- Returns all documents that are owned or shared with the user.
- :param perms: both, shared, owned. Defaults to both.
- :param include_history: boolean flag to return history documents. Defaults to False.
- :param include_trashed: boolean flag to return trashed documents. Defaults to True.
- :param include_managed: boolean flag to return documents generated by Hue. Defaults to False.
- :param include_shared_links: boolean flag to include documents shared by links. Defaults to False.
- """
- if perms == 'both':
- docs = self.filter(
- Q(owner=user) |
- Q(document2permission__users=user) |
- Q(document2permission__groups__in=user.groups.all()) |
- (Q(document2permission__is_link_on=include_shared_links) & Q(document2permission__is_link_on=True))
- )
- elif perms == 'shared':
- docs = self.filter(
- Q(document2permission__users=user) |
- Q(document2permission__groups__in=user.groups.all()) |
- (Q(document2permission__is_link_on=include_shared_links) & Q(document2permission__is_link_on=True))
- )
- else: # Only return documents owned by the user
- docs = self.filter(owner=user)
- if not include_history:
- docs = docs.exclude(is_history=True)
- if not include_managed:
- docs = docs.exclude(is_managed=True)
- if not include_trashed:
- docs = docs.exclude(is_trashed=True)
- return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
- def search_documents(self, types=None, search_text=None, order_by=None):
- """
- Search for documents based on type filters, search_text or order_by and return a queryset of document objects
- :param types: list of Document2 types (e.g. - query-hive, directory, etc)
- :param search_text: text to search on in the name and description fields
- :param order_by: order by field (e.g. -last_modified, type)
- """
- documents = self
- if types and isinstance(types, list) and types[0] != 'all':
- documents = documents.filter(type__in=types)
- if search_text:
- documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) | Q(search__icontains=search_text))
- if order_by: # TODO: Validate that order_by is a valid sort parameter
- documents = documents.order_by(order_by)
- return documents
- class Document2QuerySet(QuerySet, Document2QueryMixin):
- pass
- class Document2Manager(models.Manager, Document2QueryMixin):
- def get_queryset(self):
- """Make sure to restrict to only organization's documents"""
- return _fitered_queryset(
- Document2QuerySet(self.model, using=self._db),
- by_owner=True
- )
- # TODO prevent get() in favor of this
- def document(self, user, doc_id):
- return self.documents(user, include_trashed=True, include_history=True, include_shared_links=True).get(id=doc_id)
- def get_by_natural_key(self, uuid, version, is_history):
- return self.get(uuid=uuid, version=version, is_history=is_history)
- def get_by_uuid(self, user, uuid, perm_type='read'):
- """
- Since UUID is not a unique field, but part of a composite unique key, this returns the latest version by UUID.
- This should always be used in place of Document2.objects.get(uuid=) when a single document is expected.
- :param user: User to check permissions against
- :param uuid
- :param perm_type: permission type to check against
- """
- docs = self.filter(uuid=uuid).order_by('-last_modified')
- if not docs.exists():
- raise FilesystemException(_('Document with UUID %s not found.') % uuid)
- latest_doc = docs[0]
- if perm_type == 'write':
- latest_doc.can_write_or_exception(user)
- else:
- latest_doc.can_read_or_exception(user)
- return latest_doc
- def get_history(self, user, doc_type, include_trashed=False):
- return self.documents(user, perms='owned', include_history=True, include_trashed=include_trashed).filter(type=doc_type, is_history=True)
- def get_tasks_history(self, user):
- return self.documents(user, perms='owned', include_history=True, include_trashed=False, include_managed=True).filter(is_history=True, is_managed=True).exclude(name='pig-app-hue-script').exclude(type='oozie-workflow2')
- def get_home_directory(self, user):
- try:
- return self.get(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory')
- except Document2.DoesNotExist:
- return self.create_user_directories(user)
- except Document2.MultipleObjectsReturned:
- LOG.error('Multiple Home directories detected. Merging all into one.')
- home_dirs = list(self.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory').order_by('-last_modified'))
- parent_home_dir = home_dirs.pop()
- for dir in home_dirs:
- dir.children.exclude(name='.Trash').update(parent_directory=parent_home_dir)
- dir.delete()
- return parent_home_dir
- def get_gist_directory(self, user):
- home_dir = self.get_home_directory(user)
- gist_dir, created = Directory.objects.get_or_create(name=Document2.GIST_DIR, owner=user, parent_directory=home_dir)
- if created:
- LOG.info('Successfully created gist directory for user: %s' % user.username)
- return gist_dir
- def get_by_path(self, user, path):
- """
- This can be an expensive operation b/c we have to traverse the path tree, so if possible, request a document by UUID
- NOTE: get_by_path only works for the owner's documents since it is based off the user's home directory
- """
- cleaned_path = path.rstrip('/')
- doc = Document2.objects.get_home_directory(user)
- if cleaned_path:
- path_tokens = cleaned_path.split('/')[1:]
- for token in path_tokens:
- try:
- doc = doc.children.get(name=token)
- except Document2.DoesNotExist:
- raise FilesystemException(_('Requested invalid path for user %s: %s') % (user.username, path))
- except Document2.MultipleObjectsReturned:
- raise FilesystemException(_('Duplicate documents found for user %s at path: %s') % (user.username, path))
- doc.can_read_or_exception(user)
- return doc
- def create_user_directories(self, user):
- """
- Creates user home and Trash directories if they do not exist and move any orphan documents to home directory
- :param user: User object
- """
- # Edge-case if the user has a legacy home directory with path-name
- Directory.objects.filter(name='/', owner=user).update(name=Document2.HOME_DIR)
- # Get or create home and Trash directories for all users
- home_dir, created = Directory.objects.get_or_create(name=Document2.HOME_DIR, owner=user)
- if created:
- LOG.info('Successfully created home directory for user: %s' % user.username)
- trash_dir, created = Directory.objects.get_or_create(name=Document2.TRASH_DIR, owner=user, parent_directory=home_dir)
- if created:
- LOG.info('Successfully created trash directory for user: %s' % user.username)
- # For any directories or documents that do not have a parent directory, assign it to home directory
- count = 0
- for doc in Document2.objects.filter(owner=user).filter(parent_directory=None).exclude(id__in=[home_dir.id, trash_dir.id]):
- doc.parent_directory = home_dir
- doc.save()
- count += 1
- LOG.info("Moved %d documents to home directory for user: %s" % (count, user.username))
- return home_dir
- class Document2(models.Model):
- HOME_DIR = ''
- GIST_DIR = 'Gist'
- TRASH_DIR = '.Trash'
- EXAMPLES_DIR = 'examples'
- owner = models.ForeignKey(User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Creator.'), related_name='doc2_owner')
- name = models.CharField(default='', max_length=255)
- description = models.TextField(default='')
- uuid = models.CharField(default=uuid_default, max_length=36, db_index=True)
- type = models.CharField(default='', max_length=32, db_index=True, help_text=_t('Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...'))
- data = models.TextField(default='{}')
- extra = models.TextField(default='')
- search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
- # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
- last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
- version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
- is_history = models.BooleanField(default=False, db_index=True)
- is_managed = models.BooleanField(default=False, db_index=True, verbose_name=_t('If managed under the cover by Hue and never by the user')) # Aka isTask
- is_trashed = models.NullBooleanField(default=False, db_index=True, verbose_name=_t('True if trashed'))
- dependencies = models.ManyToManyField('self', symmetrical=False, related_name='dependents', db_index=True)
- parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
- doc = GenericRelation(Document, related_query_name='doc_doc') # Compatibility with Hue 3
- objects = Document2Manager()
- class Meta(object):
- unique_together = ('uuid', 'version', 'is_history')
- ordering = ["-last_modified", "name"]
- def __unicode__(self):
- res = '%s - %s - %s - %s' % (force_unicode(self.name), self.owner, self.type, self.uuid)
- return force_unicode(res)
- @property
- def data_dict(self):
- if not self.data:
- self.data = json.dumps({})
- data_python = json.loads(self.data)
- return data_python
- @property
- def path(self):
- quoted_name = urllib_quote(self.name.encode('utf-8'))
- if self.parent_directory:
- return '%s/%s' % (self.parent_directory.path, quoted_name)
- else:
- return quoted_name
- @property
- def dirname(self):
- return os.path.dirname(self.path) or '/'
- @property
- def is_directory(self):
- return self.type == 'directory'
- @property
- def is_home_directory(self):
- return self.is_directory and self.parent_directory == None and self.name == self.HOME_DIR
- @property
- def is_trash_directory(self):
- return self.is_directory and self.name == self.TRASH_DIR
- def natural_key(self):
- return (self.uuid, self.version, self.is_history)
- def copy(self, name, owner, description=None):
- copy_doc = self
- copy_doc.pk = None
- copy_doc.id = None
- copy_doc.uuid = uuid_default()
- copy_doc.name = name
- copy_doc.owner = owner
- if description:
- copy_doc.description = description
- copy_doc.save()
- return copy_doc
- def update_data(self, post_data):
- data_dict = self.data_dict
- data_dict.update(post_data)
- self.data = json.dumps(data_dict)
- def get_absolute_url(self):
- url = None
- try:
- if self.type == 'oozie-coordinator2':
- url = reverse('oozie:edit_coordinator') + '?coordinator=' + str(self.id)
- elif self.type == 'oozie-bundle2':
- url = reverse('oozie:edit_bundle') + '?bundle=' + str(self.id)
- elif self.type.startswith('query'):
- url = '/editor' + '?editor=' + str(self.id)
- elif self.type == 'directory':
- url = '/home2' + '?uuid=' + self.uuid
- elif self.type == 'gist':
- url = '/hue/gist?uuid=' + str(self.uuid)
- elif self.type == 'notebook':
- url = reverse('notebook:notebook') + '?notebook=' + str(self.id)
- elif self.type == 'search-dashboard':
- url = reverse('search:index') + '?collection=' + str(self.id)
- elif self.type == 'link-pigscript':
- url = reverse('pig:index') + '#edit/%s' % self.data_dict.get('object_id', '')
- elif self.type == 'link-workflow':
- url = '/jobsub/#edit-design/%s' % self.data_dict.get('object_id', '')
- else:
- url = reverse('oozie:edit_workflow') + '?workflow=' + str(self.id)
- except NoReverseMatch:
- LOG.warn('Could not perform reverse lookup for type %s, app may be blacklisted.' % self.type)
- return url
- def to_dict(self):
- return {
- 'owner': self.owner.username,
- 'name': self.name,
- 'path': self.path or '/',
- 'description': self.description,
- 'uuid': self.uuid,
- 'id': self.id,
- 'doc1_id': self.doc.get().id if self.doc.exists() else -1,
- 'parent_uuid': self.parent_directory.uuid if self.parent_directory else None,
- 'type': self.type,
- 'perms': self._massage_permissions(),
- 'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT) if self.last_modified else None,
- 'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()) if self.last_modified else None,
- 'is_managed': self.is_managed,
- 'isSelected': False,
- 'absoluteUrl': self.get_absolute_url(),
- }
- def get_history(self):
- return self.dependencies.filter(is_history=True).order_by('-last_modified')
- def add_to_history(self, user, data_dict):
- doc_id = self.id # Need to copy as the clone messes it
- history_doc = self.copy(name=self.name, owner=user)
- history_doc.update_data({'history': data_dict})
- history_doc.is_history = True
- history_doc.last_modified = None
- history_doc.save()
- Document2.objects.get(id=doc_id).dependencies.add(history_doc)
- return history_doc
- def save(self, *args, **kwargs):
- # Set document parent to home directory if parent directory isn't specified
- if not self.parent_directory and not self.is_home_directory and not self.is_trash_directory:
- home_dir = Document2.objects.get_home_directory(self.owner)
- self.parent_directory = home_dir
- # Run validations
- self.validate()
- # Redact query if needed
- self._redact_query()
- if self.search:
- self.search = self.search[:DOCUMENT2_SEARCH_MAX_LENGTH]
- super(Document2, self).save(*args, **kwargs)
- # Inherit shared permissions from parent directory, must be done after save b/c new doc needs ID
- self.inherit_permissions()
- def validate(self):
- # Validate home and Trash directories are only created once per user and cannot be created or modified after
- if self.name in [Document2.HOME_DIR, Document2.TRASH_DIR] and self.type == 'directory' and \
- Document2.objects.filter(name=self.name, owner=self.owner, type='directory').exists():
- raise FilesystemException(_('Cannot create or modify directory with name: %s') % self.name)
- # Validate that parent directory does not create cycle
- if self._contains_cycle():
- raise FilesystemException(_('Cannot save document %s under parent directory %s due to circular dependency') %
- (self.name, self.parent_directory.uuid))
- def inherit_permissions(self):
- if self.parent_directory is not None:
- parent_perms = Document2Permission.objects.filter(doc=self.parent_directory)
- for perm in parent_perms:
- self.share(self.owner, name=perm.perms, users=perm.users.all(), groups=perm.groups.all())
- def move(self, directory, user):
- if not directory.is_directory:
- raise FilesystemException(_('Target with UUID %s is not a directory') % directory.uuid)
- if directory.can_write_or_exception(user=user):
- # Restore last_modified date after save
- original_last_modified = self.last_modified
- self.parent_directory = directory
- self.save()
- # Use update instead of save() so that last_modified date is not modified automatically
- Document2.objects.filter(id=self.id).update(last_modified=original_last_modified)
- return self
- def trash(self):
- try:
- trash_dir = Directory.objects.get(name=self.TRASH_DIR, owner=self.owner)
- self.move(trash_dir, self.owner)
- self.is_trashed = True
- self.save()
- if self.is_directory:
- children_ids = self._get_child_ids_recursively(self.id)
- Document2.objects.filter(id__in=children_ids).update(is_trashed=True)
- except Document2.MultipleObjectsReturned:
- LOG.error('Multiple Trash directories detected. Merging all into one.')
- trash_dirs = list(Directory.objects.filter(name=self.TRASH_DIR, owner=self.owner).order_by('-last_modified'))
- parent_trash_dir = trash_dirs.pop()
- for dir in trash_dirs:
- dir.children.update(parent_directory=parent_trash_dir)
- dir.delete()
- self.move(parent_trash_dir, self.owner)
- def restore(self):
- # Currently restoring any doucment to /home
- home_dir = Document2.objects.get_home_directory(self.owner)
- self.move(home_dir, self.owner)
- self.is_trashed = False
- self.save()
- if self.is_directory:
- children_ids = self._get_child_ids_recursively(self.id)
- Document2.objects.filter(id__in=children_ids).update(is_trashed=False)
- def _get_child_ids_recursively(self, directory_id):
- """
- Returns the list of all children ids for a given directory id recursively, excluding history documents
- """
- directory = Directory.objects.get(id=directory_id)
- children_ids = []
- for child in directory.children.filter(is_history=False).filter(is_managed=False):
- children_ids.append(child.id)
- if child.is_directory:
- children_ids.extend(self._get_child_ids_recursively(child.id))
- return children_ids
- def can_read(self, user):
- return \
- self.owner == user \
- or is_admin(user) \
- or any([perm.user_has_access(user) for perm in self.get_permissions('read')]) \
- or self.can_write(user)
- def can_read_or_exception(self, user):
- if self.can_read(user):
- return True
- else:
- raise PopupException(_("Document does not exist or you don't have the permission to access it."), error_code=401)
- def can_write(self, user):
- return \
- self.owner == user \
- or is_admin(user) \
- or any([perm.user_has_access(user) for perm in self.get_permissions('write')]) \
- or (self.parent_directory and self.parent_directory.can_write(user))
- def can_write_or_exception(self, user):
- if self.can_write(user):
- return True
- else:
- raise PopupException(_("Document does not exist or you don't have the permission to access it."))
- def get_permission(self, perm='read'):
- try:
- return Document2Permission.objects.get(doc=self, perms=perm)
- except Document2Permission.DoesNotExist:
- return None
- def get_permissions(self, perm='read'):
- '''
- Return the sub permissions that make one of the two top privileges.
- e.g. 'read' and 'link_read' perms give the global 'read' privilege.
- '''
- if perm == 'read':
- perms = Q(perms=Document2Permission.READ_PERM) | (Q(perms=Document2Permission.LINK_READ_PERM) & Q(is_link_on=True))
- elif perm == 'write':
- perms = Q(perms=Document2Permission.WRITE_PERM) | (Q(perms=Document2Permission.LINK_WRITE_PERM) & Q(is_link_on=True))
- else:
- raise PopupException(_("Permission name %s is invalid.") % perm)
- return Document2Permission.objects.filter(Q(doc=self) & perms)
- def share(self, user, name='read', users=None, groups=None, is_link_on=False):
- try:
- with transaction.atomic():
- self.update_permission(user, name, users, groups, is_link_on)
- # For directories, update all children recursively with same permissions
- for child in self.children.all():
- child.share(user, name, users, groups, is_link_on)
- except Exception as e:
- raise PopupException(_("Failed to share document: %s") % e)
- return self
- def share_link(self, user, perm='read'):
- if perm == 'read':
- self.share(user, name=Document2Permission.LINK_READ_PERM, is_link_on=True)
- self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=False)
- elif perm == 'write':
- self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=True)
- else:
- self.share(user, name=Document2Permission.LINK_READ_PERM, is_link_on=False)
- self.share(user, name=Document2Permission.LINK_WRITE_PERM, is_link_on=False)
- return self
- def update_permission(self, user, name='read', users=None, groups=None, is_link_on=False):
- # Check if user has access to grant permissions
- if users or groups:
- if name == 'read' or name == 'link_read':
- self.can_read_or_exception(user)
- elif name == 'write' or name == 'link_write':
- self.can_write_or_exception(user)
- else:
- raise ValueError(_('Invalid permission type: %s') % name)
- perm, created = Document2Permission.objects.get_or_create(doc=self, perms=name)
- perm.users = []
- if users is not None:
- perm.users = users
- perm.groups = []
- if groups is not None:
- perm.groups = groups
- perm.is_link_on = is_link_on
- perm.save()
- def _get_doc1(self, doc2_type=None):
- if not doc2_type:
- doc2_type = self.type
- try:
- doc = self.doc.get()
- except Exception as e:
- LOG.error('Exception when retrieving document object for saved query: %s' % e)
- doc = Document.objects.link(
- self,
- owner=self.owner,
- name=self.name,
- description=self.description,
- extra=doc2_type
- )
- return doc
- def _massage_permissions(self):
- """
- Returns the permissions for a given document as a dictionary
- """
- permissions = {
- 'read': {'users': [], 'groups': []},
- 'write': {'users': [], 'groups': []},
- 'link_sharing_on': False,
- 'link_read': False,
- 'link_write': False,
- }
- read_perm = self.get_permission(perm='read')
- write_perm = self.get_permission(perm='write')
- if read_perm:
- permissions.update(read_perm.to_dict())
- if write_perm:
- permissions.update(write_perm.to_dict())
- link_read_perm = self.get_permission(perm=Document2Permission.LINK_READ_PERM)
- link_write_perm = self.get_permission(perm=Document2Permission.LINK_WRITE_PERM)
- if link_read_perm:
- permissions['link_read'] = link_read_perm.is_link_on
- if link_write_perm:
- permissions['link_write'] = link_write_perm.is_link_on
- permissions['link_sharing_on'] = permissions['link_read'] or permissions['link_write']
- return permissions
- def _redact_query(self):
- """
- Optionally mask out the query from being saved to the database. This is because if the database contains sensitive
- information like personally identifiable information, that information could be leaked into the Hue database and
- logfiles.
- """
- if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
- data_dict = self.data_dict
- snippets = data_dict.get('snippets', [])
- for snippet in snippets:
- if snippet['type'] in ('hive', 'impala'): # TODO: Pull SQL types from canonical lookup
- redacted_statement_raw = global_redaction_engine.redact(snippet['statement_raw'])
- if snippet['statement_raw'] != redacted_statement_raw:
- snippet['statement_raw'] = redacted_statement_raw
- snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
- snippet['is_redacted'] = True
- self.data = json.dumps(data_dict)
- self.search = global_redaction_engine.redact(self.search)
- def _contains_cycle(self):
- """
- Uses Floyd's cycle-detection algorithm to detect a cycle (aka Tortoise and Hare)
- https://en.wikipedia.org/wiki/Cycle_detection#Tortoise_and_hare
- """
- # Test base case where self.uuid == self.parent_directory.uuid first
- if self.parent_directory is not None and self.parent_directory.uuid == self.uuid:
- return True
- slow = self
- fast = self
- while True:
- slow = slow.parent_directory
- if slow and slow.uuid == self.uuid:
- slow = self
- if fast.parent_directory is not None:
- if fast.parent_directory.uuid == self.uuid:
- fast = self.parent_directory.parent_directory
- else:
- fast = fast.parent_directory.parent_directory
- else:
- return False
- if slow is None or fast is None:
- return False
- if slow == fast:
- return True
- class DirectoryManager(Document2Manager):
- def get_queryset(self):
- return super(DirectoryManager, self).get_queryset().filter(type='directory')
- class Directory(Document2):
- # e.g. name = '/' or '/dir1/dir2/f3'
- objects = DirectoryManager()
- class Meta(object):
- proxy = True
- def get_children_documents(self):
- """
- Returns the children documents for a given directory, excluding history documents
- """
- documents = self.children.filter(is_history=False).filter(is_managed=False) # TODO: perms
- return documents
- def get_children_and_shared_documents(self, user):
- """
- Returns the children and shared documents for a given directory, excluding history documents
- """
- # Get documents that are direct children, or shared with but not owned by the current user
- documents = Document2.objects.filter(
- Q(parent_directory=self) |
- ( (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
- ~Q(owner=user) )
- )
- documents = documents.exclude(is_history=True).exclude(is_managed=True)
- # Excluding all trashed docs across users
- documents = documents.exclude(is_trashed=True)
- # Optimizing roll up for /home by checking only with directories instead of all documents
- # For all other directories roll up is done in _filter_documents()
- directories_all = Document2.objects.filter(type='directory').exclude(id=self.id)
- directories_inside_home = directories_all.filter(
- (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
- ~Q(owner=user)
- )
- documents = documents.exclude(parent_directory__in=directories_inside_home)
- return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
- def save(self, *args, **kwargs):
- self.type = 'directory'
- super(Directory, self).save(*args, **kwargs)
- class Document2Permission(models.Model):
- """
- Combine either:
- - regular perms (listed)
- - link
- """
- READ_PERM = 'read'
- WRITE_PERM = 'write'
- COMMENT_PERM = 'comment'
- LINK_READ_PERM = 'link_read'
- LINK_WRITE_PERM = 'link_write'
- doc = models.ForeignKey(Document2)
- users = models.ManyToManyField(User, db_index=True, db_table='documentpermission2_users')
- groups = models.ManyToManyField(Group, db_index=True, db_table='documentpermission2_groups')
- perms = models.CharField(default=READ_PERM, max_length=10, db_index=True, choices=( # One perm
- (READ_PERM, 'read'),
- (WRITE_PERM, 'write'),
- (COMMENT_PERM, 'comment'), # Unused
- (LINK_READ_PERM, 'link read'),
- (LINK_WRITE_PERM, 'link write'),
- ))
- is_link_on = models.BooleanField(default=False)
- class Meta(object):
- unique_together = ('doc', 'perms')
- def __str__(self):
- return force_unicode('Document: %s, Name: %s, Users: %s, Groups: %s, is_link_on: %s') % (
- self.doc, self.perms, self.users, self.groups, self.is_link_on
- )
- def to_dict(self):
- return {
- self.perms: {
- 'users': [{'id': perm_user.id, 'username': perm_user.username} for perm_user in self.users.all()],
- 'groups': [{'id': perm_group.id, 'name': perm_group.name} for perm_group in self.groups.all()]
- }
- }
- def user_has_access(self, user):
- """
- Returns true if the given user has permissions based on users, groups or all flag.
- """
- return self.is_link_on or user in self.users.all() or self.groups.filter(id__in=user.groups.all()).exists()
- def get_cluster_config(user):
- return Cluster(user).get_app_config().get_config()
- class ClusterConfig(object):
- """
- Configuration of the apps and engines that each individual user sees on the core Hue.
- Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
- TODO: rename to HueConfig
- TODO: get list of contexts dynamically
- """
- def __init__(self, user, apps=None, cluster_type='ini'):
- self.user = user
- self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
- self.cluster_type = cluster_type
- def get_config(self):
- app_config = self.get_apps()
- editors = app_config.get('editor')
- main_button_action = self.get_main_quick_action(app_config)
- if main_button_action.get('is_sql'):
- default_sql_interpreter = main_button_action['type']
- else:
- default_sql_interpreter = editors and editors['default_sql_interpreter']
- return {
- 'app_config': app_config,
- 'main_button_action': main_button_action,
- 'button_actions': [
- app for app in [
- editors,
- app_config.get('dashboard'),
- app_config.get('scheduler')
- ] if app is not None
- ],
- 'default_sql_interpreter': default_sql_interpreter,
- 'cluster_type': self.cluster_type,
- 'has_computes': self.cluster_type in ('altus', 'snowball') # or any grouped engine connectors
- }
- def get_apps(self):
- apps = OrderedDict([
- app for app in [
- ('editor', self._get_editor()),
- ('dashboard', self._get_dashboard()),
- ('catalogs', self._get_catalogs()),
- ('browser', self._get_browser()),
- ('scheduler', self._get_scheduler()),
- ('sdkapps', self._get_sdk_apps()),
- ('home', self._get_home()),
- ] if app[1]
- ])
- return apps
- def get_main_quick_action(self, apps):
- if not apps:
- raise PopupException(_('No permission to any app.'))
- default_app = list(apps.values())[0]
- default_interpreter = default_app.get('interpreters')
- try:
- user_default_app = json.loads(UserPreferences.objects.get(user=self.user, key='default_app').value)
- if apps.get(user_default_app['app']):
- default_interpreter = []
- default_app = apps[user_default_app['app']]
- if default_app.get('interpreters'):
- interpreters = [interpreter for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']]
- if interpreters:
- default_interpreter = interpreters
- except UserPreferences.DoesNotExist:
- pass
- except Exception:
- LOG.exception('Could not load back default app')
- if default_interpreter:
- return default_interpreter[0]
- else:
- return default_app
- def _get_home(self):
- return {
- 'name': 'home',
- 'displayName': _('Home'),
- 'buttonName': _('Saved Queries') if has_connectors() else _('Documents'),
- 'interpreters': [],
- 'page': '/home'
- }
- def _get_editor(self):
- interpreters = []
- _interpreters = get_ordered_interpreters(self.user)
- for interpreter in _interpreters:
- if interpreter['interface'] != 'hms':
- interpreters.append({
- 'name': interpreter['name'],
- 'type': interpreter['type'],
- 'displayName': interpreter['name'],
- 'buttonName': _('Query'),
- 'tooltip': _('%s Query') % interpreter['type'].title(),
- 'page': '/editor/?type=%(type)s' % interpreter,
- 'is_sql': interpreter['is_sql'],
- 'dialect': interpreter['dialect'],
- 'dialect_properties': interpreter.get('dialect_properties'),
- })
- if SHOW_NOTEBOOKS.get():
- try:
- first_non_sql_index = [interpreter['is_sql'] for interpreter in interpreters].index(False)
- except ValueError:
- first_non_sql_index = len(interpreters) if all([interpreter['is_sql'] for interpreter in interpreters]) else 0
- interpreters.insert(first_non_sql_index, {
- 'name': 'notebook',
- 'type': 'notebook',
- 'displayName': 'Notebook',
- 'buttonName': _('Notebook'),
- 'tooltip': _('Notebook'),
- 'page': '/notebook',
- 'is_sql': False,
- 'dialect': 'notebook'
- })
- if interpreters:
- return {
- 'name': 'editor',
- 'displayName': _('Editor'),
- 'buttonName': _('Query'),
- 'interpreters': interpreters,
- 'default_limit': None if DEFAULT_LIMIT.get() == 0 else DEFAULT_LIMIT.get(),
- 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
- 'page': interpreters[0]['page'],
- 'default_sql_interpreter': next((interpreter['type'] for interpreter in interpreters if interpreter.get('is_sql')), 'hive')
- }
- else:
- return None
- def _get_catalogs(self):
- '''
- Any connector implementing Notebook Assist API.
- Currently SQL connectors, Hive Metastore. Possibly Metadata Catalogs later.
- '''
- interpreters = []
- _interpreters = get_ordered_interpreters(self.user)
- for interpreter in _interpreters:
- if interpreter['is_sql'] or interpreter['interface'] == 'hms':
- interpreters.append({
- 'name': interpreter['name'],
- 'type': interpreter['type'],
- 'displayName': interpreter['name'],
- 'buttonName': _('Query'),
- 'tooltip': _('%s Query') % interpreter['type'].title(),
- 'page': '/editor/?type=%(type)s' % interpreter,
- 'is_sql': interpreter['is_sql'],
- 'is_catalog': True,
- })
- return interpreters if interpreters else None
- def _get_dashboard(self):
- interpreters = get_engines(self.user)
- _interpreters = []
- if interpreters:
- if HAS_REPORT_ENABLED.get():
- _interpreters.append({
- 'type': 'report',
- 'displayName': 'Report',
- 'buttonName': 'Report',
- 'page': '/dashboard/new_search?engine=report',
- 'tooltip': _('Report'),
- 'is_sql': False
- })
- _interpreters.extend([{
- 'type': interpreter['type'],
- 'displayName': interpreter['type'].title(),
- 'buttonName': interpreter['type'].title(),
- 'page': '/dashboard/new_search?engine=%(type)s' % interpreter,
- 'tooltip': _('%s Dashboard') % interpreter['type'].title(),
- 'is_sql': True
- } for interpreter in interpreters
- ])
- return {
- 'name': 'dashboard',
- 'displayName': _('Dashboard'),
- 'buttonName': _('Dashboard'),
- 'interpreters': _interpreters,
- 'page': '/dashboard/new_search?engine=%(type)s' % interpreters[0]
- }
- else:
- return None
- def _get_browser(self):
- interpreters = []
- hdfs_connectors = []
- if has_connectors():
- hdfs_connectors = [
- connector['nice_name'] for connector in _get_installed_connectors(category='browsers', dialect='hdfs', interface='rest')
- ]
- elif 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('hdfs', self.user):
- hdfs_connectors.append(_('Files'))
- for hdfs_connector in hdfs_connectors:
- interpreters.append({
- 'type': 'hdfs',
- 'displayName': hdfs_connector,
- 'buttonName': _('Browse'),
- 'tooltip': hdfs_connector,
- 'page': '/filebrowser/' + (
- not self.user.is_anonymous() and
- 'view=' + urllib_quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or ''
- )
- })
- if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('s3a', self.user):
- interpreters.append({
- 'type': 's3',
- 'displayName': _('S3'),
- 'buttonName': _('Browse'),
- 'tooltip': _('S3'),
- 'page': '/filebrowser/view=' + urllib_quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
- })
- if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('adl', self.user):
- interpreters.append({
- 'type': 'adls',
- 'displayName': _('ADLS'),
- 'buttonName': _('Browse'),
- 'tooltip': _('ADLS'),
- 'page': '/filebrowser/view=' + urllib_quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
- })
- if 'filebrowser' in self.apps and fsmanager.is_enabled_and_has_access('abfs', self.user):
- from azure.abfs.__init__ import get_home_dir_for_ABFS
- interpreters.append({
- 'type': 'abfs',
- 'displayName': _('ABFS'),
- 'buttonName': _('Browse'),
- 'tooltip': _('ABFS'),
- 'page': '/filebrowser/view=' + urllib_quote(get_home_dir_for_ABFS().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
- })
- if 'metastore' in self.apps:
- interpreters.append({
- 'type': 'tables',
- 'displayName': _('Tables'),
- 'buttonName': _('Browse'),
- 'tooltip': _('Tables'),
- 'page': '/metastore/tables'
- })
- if 'search' in self.apps:
- interpreters.append({
- 'type': 'indexes',
- 'displayName': _('Indexes'),
- 'buttonName': _('Dashboard'),
- 'tooltip': _('Indexes'),
- 'page': '/indexer/'
- })
- if 'jobbrowser' in self.apps:
- from hadoop.cluster import get_default_yarncluster # Circular loop
- if get_default_yarncluster():
- interpreters.append({
- 'type': 'yarn',
- 'displayName': _('Jobs'),
- 'buttonName': _('Jobs'),
- 'tooltip': _('Jobs'),
- 'page': '/jobbrowser/'
- })
- if has_kafka():
- interpreters.append({
- 'type': 'kafka',
- 'displayName': _('Streams'),
- 'buttonName': _('Browse'),
- 'tooltip': _('Kafka'),
- 'page': '/kafka/'
- })
- if 'hbase' in self.apps:
- interpreters.append({
- 'type': 'hbase',
- 'displayName': _('HBase'),
- 'buttonName': _('Browse'),
- 'tooltip': _('HBase'),
- 'page': '/hbase/'
- })
- if 'security' in self.apps:
- interpreters.append({
- 'type': 'security',
- 'displayName': _('Security'),
- 'buttonName': _('Browse'),
- 'tooltip': _('Security'),
- 'page': '/security/hive'
- })
- if 'indexer' in self.apps and 'filebrowser' in self.apps and self.user.has_hue_permission(action="access:importer", app="indexer") \
- and 'importer' not in APP_BLACKLIST.get():
- interpreters.append({
- 'type': 'importer',
- 'displayName': _('Importer'),
- 'buttonName': _('Import'),
- 'tooltip': _('Importer'),
- 'page': '/indexer/importer'
- })
- if 'sqoop' in self.apps:
- from sqoop.conf import IS_ENABLED
- if IS_ENABLED.get():
- interpreters.append({
- 'type': 'sqoop',
- 'displayName': _('Sqoop'),
- 'buttonName': _('Browse'),
- 'tooltip': _('Sqoop'),
- 'page': '/sqoop'
- })
- if interpreters:
- return {
- 'name': 'browser',
- 'displayName': _('Browsers'),
- 'buttonName': _('Browse'),
- 'interpreters': interpreters,
- 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
- }
- else:
- return None
- def _get_scheduler(self):
- interpreters = []
- if 'oozie' in self.apps and not (self.user.has_hue_permission(action="disable_editor_access", app="oozie") and not is_admin(self.user)):
- interpreters.extend([{
- 'type': 'oozie-workflow',
- 'displayName': _('Workflow'),
- 'buttonName': _('Schedule'),
- 'tooltip': _('Workflow'),
- 'page': '/oozie/editor/workflow/new/'
- }, {
- 'type': 'oozie-coordinator',
- 'displayName': _('Schedule'),
- 'buttonName': _('Schedule'),
- 'tooltip': _('Schedule'),
- 'page': '/oozie/editor/coordinator/new/'
- }, {
- 'type': 'oozie-bundle',
- 'displayName': _('Bundle'),
- 'buttonName': _('Schedule'),
- 'tooltip': _('Bundle'),
- 'page': '/oozie/editor/bundle/new/'
- }
- ])
- if TASK_SERVER.BEAT_ENABLED.get():
- interpreters.append({
- 'type': 'celery-beat',
- 'displayName': _('Scheduled Tasks'),
- 'buttonName': _('Scheduled Tasks'),
- 'tooltip': _('Scheduled Tasks'),
- 'page': '/jobbrowser/'
- }
- )
- if interpreters:
- return {
- 'name': 'oozie',
- 'displayName': _('Scheduler'),
- 'buttonName': _('Schedule'),
- 'interpreters': interpreters,
- 'interpreter_names': [interpreter['type'] for interpreter in interpreters],
- 'page': interpreters[0]['page']
- }
- else:
- return None
- def _get_sdk_apps(self):
- current_app, other_apps, apps_list = _get_apps(self.user)
- interpreters = []
- for other in other_apps:
- interpreters.append({
- 'type': other.display_name,
- 'displayName': other.nice_name,
- 'buttonName': other.nice_name,
- 'tooltip': other.nice_name,
- 'page': '/%s' % other.display_name
- })
- if interpreters:
- return {
- 'name': 'other',
- 'displayName': _('Other Apps'),
- 'buttonName': _('Other'),
- 'interpreters': interpreters,
- }
- else:
- return None
- def get_hive_metastore_interpreters(self):
- return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter['type'] == 'hive' or interpreter['type'] == 'hms']
- class Cluster(object):
- def __init__(self, user):
- self.user = user
- self.clusters = get_clusters(user)
- if IS_MULTICLUSTER_ONLY.get():
- self.data = self.clusters['Altus'] # Backward compatibility
- else:
- self.data = list(self.clusters.values())[0] # Next: CLUSTER_ID.get() or user persisted
- def get_type(self):
- return self.data['type']
- def get_config(self, name):
- return self.clusters[name]
- def get_app_config(self):
- return ClusterConfig(self.user, cluster_type=self.get_type())
- def _get_apps(user, section=None):
- current_app = None
- other_apps = []
- if user.is_authenticated():
- apps_list = appmanager.get_apps_dict(user)
- apps = list(apps_list.values())
- for app in apps:
- if app.display_name not in [
- 'beeswax', 'hive', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
- 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'
- ] and app.menu_index != -1:
- other_apps.append(app)
- if section == app.display_name:
- current_app = app
- else:
- apps_list = []
- return current_app, other_apps, apps_list
- def get_user_preferences(user, key=None):
- if key is not None:
- try:
- x = UserPreferences.objects.get(user=user, key=key)
- return {key: x.value}
- except UserPreferences.DoesNotExist:
- return None
- else:
- return dict((x.key, x.value) for x in UserPreferences.objects.filter(user=user))
- def set_user_preferences(user, key, value):
- try:
- x = UserPreferences.objects.get(user=user, key=key)
- except UserPreferences.DoesNotExist:
- x = UserPreferences(user=user, key=key)
- x.value = value
- x.save()
- return x
- def get_data_link(meta):
- link = None
- if not meta.get('type'):
- pass
- elif meta['type'] == 'hbase':
- link = '/hbase/#Cluster/%(table)s/query/%(row_key)s' % meta
- if 'col' in meta:
- link += '[%(fam)s:%(col)s]' % meta
- elif 'fam' in meta:
- link += '[%(fam)s]' % meta
- elif meta['type'] == 'hdfs':
- link = '/filebrowser/view=%(path)s' % meta # Could add a byte #
- elif meta['type'] == 'link':
- link = meta['link']
- elif meta['type'] == 'hive':
- link = '/metastore/table/%(database)s/%(table)s' % meta # Could also add col=val
- return link
- def _get_gist_document(uuid):
- return Document2.objects.get(uuid=uuid, type='gist') # Workaround until there is a share to all permission
- def __paginate(page, limit, queryset):
- if limit > 0:
- offset = (page - 1) * limit
- last = offset + limit
- queryset = queryset.all()[offset:last]
- return {
- 'documents': queryset,
- 'page': page,
- 'limit': limit
- }
|