Преглед на файлове

HUE-3507 [oozie] List workflows displays trashed workflows

Also refactors Document2Manager to use custom QuerySet so and mixin so manager methods can be chained
Jenny Kim преди 10 години
родител
ревизия
a56862d
променени са 5 файла, в които са добавени 105 реда и са изтрити 31 реда
  1. 22 0
      apps/oozie/src/oozie/tests2.py
  2. 42 13
      apps/oozie/src/oozie/views/editor2.py
  3. 8 3
      desktop/core/src/desktop/api2.py
  4. 32 14
      desktop/core/src/desktop/models.py
  5. 1 1
      desktop/core/src/desktop/urls.py

+ 22 - 0
apps/oozie/src/oozie/tests2.py

@@ -393,6 +393,28 @@ LIMIT $limit"""))
       remove_from_group("test", group)
 
 
+  def test_list_editor_workflows(self):
+    wf_doc = save_temp_workflow(MockOozieApi.JSON_WORKFLOW_LIST[5], self.user)
+    reset = ENABLE_V2.set_for_testing(True)
+    try:
+      response = self.c.get(reverse('oozie:list_editor_workflows'))
+      assert_equal(response.status_code, 200)
+      data = json.loads(response.context['workflows_json'])
+      uuids = [doc['uuid'] for doc in data]
+      assert_true(wf_doc.uuid in uuids, data)
+
+      # Trash workflow and verify it no longer appears in list
+      response = self.c.post('/desktop/api2/doc/delete', {'uuid': json.dumps(wf_doc.uuid)})
+      response = self.c.get(reverse('oozie:list_editor_workflows'))
+      assert_equal(response.status_code, 200)
+      data = json.loads(response.context['workflows_json'])
+      uuids = [doc['uuid'] for doc in data]
+      assert_false(wf_doc.uuid in uuids, data)
+    finally:
+      reset()
+      wf_doc.delete()
+
+
 class TestExternalWorkflowGraph():
 
   def setUp(self):

+ 42 - 13
apps/oozie/src/oozie/views/editor2.py

@@ -51,7 +51,11 @@ LOG = logging.getLogger(__name__)
 
 @check_editor_access_permission
 def list_editor_workflows(request):
-  workflows = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='workflow2')]
+  if USE_NEW_EDITOR.get():
+    docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-workflow2'])
+    workflows = [doc.to_dict() for doc in docs]
+  else:
+    workflows = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='workflow2')]
 
   workflows_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldWorklow, request.user) if job.managed]
   if workflows_v1:
@@ -263,14 +267,23 @@ def new_node(request):
 
 
 def _get_workflows(user):
-  return [{
-        'name': workflow.name,
-        'owner': workflow.owner.username,
-        'value': workflow.uuid,
-        'id': workflow.id
-      } for workflow in [d.content_object for d in Document.objects.get_docs(user, Document2, extra='workflow2').order_by('-id')]
+  if USE_NEW_EDITOR.get():
+    workflows = [{
+          'name': workflow.name,
+          'owner': workflow.owner.username,
+          'value': workflow.uuid,
+          'id': workflow.id
+        } for workflow in [doc for doc in Document2.objects.documents(user).search_documents(types=['oozie-workflow2']).order_by('-id')]
+      ]
+  else:
+    workflows = [{
+          'name': workflow.name,
+          'owner': workflow.owner.username,
+          'value': workflow.uuid,
+          'id': workflow.id
+        } for workflow in [d.content_object for d in Document.objects.get_docs(user, Document2, extra='workflow2').order_by('-id')]
     ]
-
+  return workflows
 
 @check_editor_access_permission
 def add_node(request):
@@ -435,7 +448,11 @@ def _submit_workflow(user, fs, jt, workflow, mapping):
 
 @check_editor_access_permission
 def list_editor_coordinators(request):
-  coordinators = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
+  if USE_NEW_EDITOR.get():
+    docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-coordinator2'])
+    coordinators = [doc.to_dict() for doc in docs]
+  else:
+    coordinators = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
 
   coordinators_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldCoordinator, request.user)]
   if coordinators_v1:
@@ -476,8 +493,12 @@ def edit_coordinator(request):
   except Exception, e:
     LOG.error(smart_str(e))
 
-  workflows = [dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)])
-                    for d in Document.objects.available_docs(Document2, request.user).filter(extra='workflow2')]
+  if USE_NEW_EDITOR.get():
+    workflows = [dict([('uuid', d.uuid), ('name', d.name)])
+                      for d in Document2.objects.documents(request.user).search_documents(types=['oozie-workflow2'])]
+  else:
+    workflows = [dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)])
+                      for d in Document.objects.available_docs(Document2, request.user).filter(extra='workflow2')]
 
   if coordinator_id and not filter(lambda a: a['uuid'] == coordinator.data['properties']['workflow'], workflows):
     raise PopupException(_('You don\'t have access to the workflow of this coordinator.'))
@@ -646,7 +667,11 @@ def _submit_coordinator(request, coordinator, mapping):
 
 @check_editor_access_permission
 def list_editor_bundles(request):
-  bundles = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='bundle2')]
+  if USE_NEW_EDITOR.get():
+    docs = Document2.objects.documents(request.user).search_documents(types=['oozie-bundle2'])
+    bundles = [doc.to_dict() for doc in docs]
+  else:
+    bundles = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='bundle2')]
 
   bundles_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldBundle, request.user)]
   if bundles_v1:
@@ -670,7 +695,11 @@ def edit_bundle(request):
     bundle = Bundle()
     bundle.set_workspace(request.user)
 
-  coordinators = [dict([('id', d.content_object.id), ('uuid', d.content_object.uuid), ('name', d.content_object.name)])
+  if USE_NEW_EDITOR.get():
+    coordinators = [dict([('id', d.id), ('uuid', d.uuid), ('name', d.name)])
+                      for d in Document2.objects.documents(request.user).search_documents(types=['oozie-coordinator2'])]
+  else:
+    coordinators = [dict([('id', d.content_object.id), ('uuid', d.content_object.uuid), ('name', d.content_object.name)])
                       for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
 
   return render('editor2/bundle_editor.mako', request, {

+ 8 - 3
desktop/core/src/desktop/api2.py

@@ -84,12 +84,18 @@ def search_documents(request):
 
   perms = request.GET.get('perms', 'both').lower()
   include_history = json.loads(request.GET.get('include_history', 'false'))
+  include_trashed = json.loads(request.GET.get('include_trashed', 'true'))
   flatten = json.loads(request.GET.get('flatten', 'true'))
 
   if perms not in ['owned', 'shared', 'both']:
     raise PopupException(_('Invalid value for perms, acceptable values are: owned, shared, both.'))
 
-  documents = Document2.objects.documents(user=request.user, perms=perms, include_history=include_history)
+  documents = Document2.objects.documents(
+    user=request.user,
+    perms=perms,
+    include_history=include_history,
+    include_trashed=include_trashed
+  )
 
   # Refine results
   response.update(_filter_documents(request, queryset=documents, flatten=flatten))
@@ -450,8 +456,7 @@ def _filter_documents(request, queryset, flatten=True):
   sort = request.GET.get('sort', '-last_modified')
   search_text = request.GET.get('text', None)
 
-  documents = Document2.objects.refine_documents(
-      documents=queryset,
+  documents = queryset.search_documents(
       types=type_filters,
       search_text=search_text,
       order_by=sort)

+ 32 - 14
desktop/core/src/desktop/models.py

@@ -16,13 +16,12 @@
 # limitations under the License.
 
 import calendar
-import logging
 import json
+import logging
 import os
 import re
 import uuid
 
-from datetime import datetime
 from itertools import chain
 
 from django.contrib.auth import models as auth_models
@@ -32,6 +31,7 @@ from django.contrib.staticfiles.storage import staticfiles_storage
 from django.core.urlresolvers import reverse, NoReverseMatch
 from django.db import connection, models, transaction
 from django.db.models import Q
+from django.db.models.query import QuerySet
 from django.template.defaultfilters import urlencode
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
@@ -756,45 +756,49 @@ class FilesystemException(Exception):
   pass
 
 
-class Document2Manager(models.Manager):
-
-  # TODO prevent get
-  def document(self, user, doc_id):
-    return self.documents(user).get(id=doc_id)
+class Document2QueryMixin(object):
 
-  def documents(self, user, perms='both', include_history=False):
+  def documents(self, user, perms='both', include_history=False, include_trashed=False):
     """
     Returns all documents that are owned or shared with the user.
     :param perms: both, shared, owned. Defaults to both.
     :param include_history: boolean flag to return history documents. Defaults to False.
+    :param include_trashed: boolean flag to return trashed documents. Defaults to True.
     """
     if perms == 'both':
-      docs = Document2.objects.filter(
+      docs = self.filter(
         Q(owner=user) |
         Q(document2permission__users=user) |
         Q(document2permission__groups__in=user.groups.all())
       )
     elif perms == 'shared':
-      docs = Document2.objects.filter(
+      docs = self.filter(
         Q(document2permission__users=user) |
         Q(document2permission__groups__in=user.groups.all())
       )
     else:  # only return documents owned by the user
-      docs = Document2.objects.filter(owner=user)
+      docs = self.filter(owner=user)
 
     if not include_history:
       docs = docs.exclude(is_history=True)
 
+    if not include_trashed:
+      # Since the Trash folder can have multiple directory levels, we need to check full path and exclude those IDs
+      trashed_ids = [doc.id for doc in docs if Document2.TRASH_DIR in doc.path]
+      docs = docs.exclude(id__in=trashed_ids)
+
     return docs.defer('description', 'data', 'extra').distinct().order_by('-last_modified')
 
-  def refine_documents(self, documents, types=None, search_text=None, order_by=None):
+
+  def search_documents(self, types=None, search_text=None, order_by=None):
     """
-    Refines a queryset of document objects by type filters, search_text or order_by
-    :param documents: queryset of Document2 objects
+    Search for documents based on type filters, search_text or order_by and return a queryset of document objects
     :param types: list of Document2 types (e.g. - query-hive, directory, etc)
     :param search_text: text to search on in the name and description fields
     :param order_by: order by field (e.g. -last_modified, type)
     """
+    documents = self
+
     if types and isinstance(types, list):
       documents = documents.filter(type__in=types)
 
@@ -806,6 +810,20 @@ class Document2Manager(models.Manager):
 
     return documents
 
+
+class Document2QuerySet(QuerySet, Document2QueryMixin):
+    pass
+
+
+class Document2Manager(models.Manager, Document2QueryMixin):
+
+  def get_query_set(self):
+    return Document2QuerySet(self.model, using=self._db)
+
+  # TODO prevent get
+  def document(self, user, doc_id):
+    return self.documents(user, include_trashed=True).get(id=doc_id)
+
   def get_by_natural_key(self, uuid, version, is_history):
     return self.get(uuid=uuid, version=version, is_history=is_history)
 

+ 1 - 1
desktop/core/src/desktop/urls.py

@@ -34,12 +34,12 @@ desktop.lib.metrics.file_reporter.start_file_reporter()
 
 from django.conf import settings
 from django.conf.urls import include, patterns
-from django.conf.urls.static import static
 from django.contrib import admin
 
 from desktop import appmanager
 from desktop.conf import METRICS, USE_NEW_EDITOR
 
+
 # Django expects handler404 and handler500 to be defined.
 # django.conf.urls provides them. But we want to override them.
 # Also see http://code.djangoproject.com/ticket/5350