HUE-3507 [oozie] List workflows displays trashed workflows

Also refactors Document2Manager to use custom QuerySet so and mixin so manager methods can be chained

apps/oozie/src/oozie/tests2.py

@@ -393,6 +393,28 @@ LIMIT $limit"""))
       remove_from_group("test", group)
 
 
+  def test_list_editor_workflows(self):
+    wf_doc = save_temp_workflow(MockOozieApi.JSON_WORKFLOW_LIST[5], self.user)
+    reset = ENABLE_V2.set_for_testing(True)
+    try:
+      response = self.c.get(reverse('oozie:list_editor_workflows'))
+      assert_equal(response.status_code, 200)
+      data = json.loads(response.context['workflows_json'])
+      uuids = [doc['uuid'] for doc in data]
+      assert_true(wf_doc.uuid in uuids, data)
+
+      # Trash workflow and verify it no longer appears in list
+      response = self.c.post('/desktop/api2/doc/delete', {'uuid': json.dumps(wf_doc.uuid)})
+      response = self.c.get(reverse('oozie:list_editor_workflows'))
+      assert_equal(response.status_code, 200)
+      data = json.loads(response.context['workflows_json'])
+      uuids = [doc['uuid'] for doc in data]
+      assert_false(wf_doc.uuid in uuids, data)
+    finally:
+      reset()
+      wf_doc.delete()
+
+
 class TestExternalWorkflowGraph():
 
   def setUp(self):

apps/oozie/src/oozie/views/editor2.py

@@ -51,7 +51,11 @@ LOG = logging.getLogger(__name__)
 
 @check_editor_access_permission
 def list_editor_workflows(request):
-  workflows = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='workflow2')]
+  if USE_NEW_EDITOR.get():
+    docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-workflow2'])
+    workflows = [doc.to_dict() for doc in docs]
+  else:
+    workflows = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='workflow2')]
 
   workflows_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldWorklow, request.user) if job.managed]
   if workflows_v1:
@@ -263,14 +267,23 @@ def new_node(request):
 
 
 def _get_workflows(user):
-  return [{
-        'name': workflow.name,
-        'owner': workflow.owner.username,
-        'value': workflow.uuid,
-        'id': workflow.id
-      } for workflow in [d.content_object for d in Document.objects.get_docs(user, Document2, extra='workflow2').order_by('-id')]
+  if USE_NEW_EDITOR.get():
+    workflows = [{
+          'name': workflow.name,
+          'owner': workflow.owner.username,
+          'value': workflow.uuid,
+          'id': workflow.id
+        } for workflow in [doc for doc in Document2.objects.documents(user).search_documents(types=['oozie-workflow2']).order_by('-id')]
+      ]
+  else:
+    workflows = [{
+          'name': workflow.name,
+          'owner': workflow.owner.username,
+          'value': workflow.uuid,
+          'id': workflow.id
+        } for workflow in [d.content_object for d in Document.objects.get_docs(user, Document2, extra='workflow2').order_by('-id')]
     ]
-
+  return workflows
 
 @check_editor_access_permission
 def add_node(request):
@@ -435,7 +448,11 @@ def _submit_workflow(user, fs, jt, workflow, mapping):
 
 @check_editor_access_permission
 def list_editor_coordinators(request):
-  coordinators = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
+  if USE_NEW_EDITOR.get():
+    docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-coordinator2'])
+    coordinators = [doc.to_dict() for doc in docs]
+  else:
+    coordinators = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
 
   coordinators_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldCoordinator, request.user)]
   if coordinators_v1:
@@ -476,8 +493,12 @@ def edit_coordinator(request):
   except Exception, e:
     LOG.error(smart_str(e))
 
-  workflows = [dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)])
-                    for d in Document.objects.available_docs(Document2, request.user).filter(extra='workflow2')]
+  if USE_NEW_EDITOR.get():
+    workflows = [dict([('uuid', d.uuid), ('name', d.name)])
+                      for d in Document2.objects.documents(request.user).search_documents(types=['oozie-workflow2'])]
+  else:
+    workflows = [dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)])
+                      for d in Document.objects.available_docs(Document2, request.user).filter(extra='workflow2')]
 
   if coordinator_id and not filter(lambda a: a['uuid'] == coordinator.data['properties']['workflow'], workflows):
     raise PopupException(_('You don\'t have access to the workflow of this coordinator.'))
@@ -646,7 +667,11 @@ def _submit_coordinator(request, coordinator, mapping):
 
 @check_editor_access_permission
 def list_editor_bundles(request):
-  bundles = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='bundle2')]
+  if USE_NEW_EDITOR.get():
+    docs = Document2.objects.documents(request.user).search_documents(types=['oozie-bundle2'])
+    bundles = [doc.to_dict() for doc in docs]
+  else:
+    bundles = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='bundle2')]
 
   bundles_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldBundle, request.user)]
   if bundles_v1:
@@ -670,7 +695,11 @@ def edit_bundle(request):
     bundle = Bundle()
     bundle.set_workspace(request.user)
 
-  coordinators = [dict([('id', d.content_object.id), ('uuid', d.content_object.uuid), ('name', d.content_object.name)])
+  if USE_NEW_EDITOR.get():
+    coordinators = [dict([('id', d.id), ('uuid', d.uuid), ('name', d.name)])
+                      for d in Document2.objects.documents(request.user).search_documents(types=['oozie-coordinator2'])]
+  else:
+    coordinators = [dict([('id', d.content_object.id), ('uuid', d.content_object.uuid), ('name', d.content_object.name)])
                       for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
 
   return render('editor2/bundle_editor.mako', request, {

desktop/core/src/desktop/api2.py

@@ -84,12 +84,18 @@ def search_documents(request):
 
   perms = request.GET.get('perms', 'both').lower()
   include_history = json.loads(request.GET.get('include_history', 'false'))
+  include_trashed = json.loads(request.GET.get('include_trashed', 'true'))
   flatten = json.loads(request.GET.get('flatten', 'true'))
 
   if perms not in ['owned', 'shared', 'both']:
     raise PopupException(_('Invalid value for perms, acceptable values are: owned, shared, both.'))
 
-  documents = Document2.objects.documents(user=request.user, perms=perms, include_history=include_history)
+  documents = Document2.objects.documents(
+    user=request.user,
+    perms=perms,
+    include_history=include_history,
+    include_trashed=include_trashed
+  )
 
   # Refine results
   response.update(_filter_documents(request, queryset=documents, flatten=flatten))
@@ -450,8 +456,7 @@ def _filter_documents(request, queryset, flatten=True):
   sort = request.GET.get('sort', '-last_modified')
   search_text = request.GET.get('text', None)
 
-  documents = Document2.objects.refine_documents(
-      documents=queryset,
+  documents = queryset.search_documents(
       types=type_filters,
       search_text=search_text,
       order_by=sort)

desktop/core/src/desktop/models.py

@@ -16,13 +16,12 @@
 # limitations under the License.
 
 import calendar
-import logging
 import json
+import logging
 import os
 import re
 import uuid
 
-from datetime import datetime
 from itertools import chain
 
 from django.contrib.auth import models as auth_models
@@ -32,6 +31,7 @@ from django.contrib.staticfiles.storage import staticfiles_storage
 from django.core.urlresolvers import reverse, NoReverseMatch
 from django.db import connection, models, transaction
 from django.db.models import Q
+from django.db.models.query import QuerySet
 from django.template.defaultfilters import urlencode
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
@@ -756,45 +756,49 @@ class FilesystemException(Exception):
   pass
 
 
-class Document2Manager(models.Manager):
-
-  # TODO prevent get
-  def document(self, user, doc_id):
-    return self.documents(user).get(id=doc_id)
+class Document2QueryMixin(object):
 
-  def documents(self, user, perms='both', include_history=False):
+  def documents(self, user, perms='both', include_history=False, include_trashed=False):
     """
     Returns all documents that are owned or shared with the user.
     :param perms: both, shared, owned. Defaults to both.
     :param include_history: boolean flag to return history documents. Defaults to False.
+    :param include_trashed: boolean flag to return trashed documents. Defaults to True.
     """
     if perms == 'both':
-      docs = Document2.objects.filter(
+      docs = self.filter(
         Q(owner=user) |
         Q(document2permission__users=user) |
         Q(document2permission__groups__in=user.groups.all())
       )
     elif perms == 'shared':
-      docs = Document2.objects.filter(
+      docs = self.filter(
         Q(document2permission__users=user) |
         Q(document2permission__groups__in=user.groups.all())
       )
     else:  # only return documents owned by the user
-      docs = Document2.objects.filter(owner=user)
+      docs = self.filter(owner=user)
 
     if not include_history:
       docs = docs.exclude(is_history=True)
 
+    if not include_trashed:
+      # Since the Trash folder can have multiple directory levels, we need to check full path and exclude those IDs
+      trashed_ids = [doc.id for doc in docs if Document2.TRASH_DIR in doc.path]
+      docs = docs.exclude(id__in=trashed_ids)
+
     return docs.defer('description', 'data', 'extra').distinct().order_by('-last_modified')
 
-  def refine_documents(self, documents, types=None, search_text=None, order_by=None):
+
+  def search_documents(self, types=None, search_text=None, order_by=None):
     """
-    Refines a queryset of document objects by type filters, search_text or order_by
-    :param documents: queryset of Document2 objects
+    Search for documents based on type filters, search_text or order_by and return a queryset of document objects
     :param types: list of Document2 types (e.g. - query-hive, directory, etc)
     :param search_text: text to search on in the name and description fields
     :param order_by: order by field (e.g. -last_modified, type)
     """
+    documents = self
+
     if types and isinstance(types, list):
       documents = documents.filter(type__in=types)
 
@@ -806,6 +810,20 @@ class Document2Manager(models.Manager):
 
     return documents
 
+
+class Document2QuerySet(QuerySet, Document2QueryMixin):
+    pass
+
+
+class Document2Manager(models.Manager, Document2QueryMixin):
+
+  def get_query_set(self):
+    return Document2QuerySet(self.model, using=self._db)
+
+  # TODO prevent get
+  def document(self, user, doc_id):
+    return self.documents(user, include_trashed=True).get(id=doc_id)
+
   def get_by_natural_key(self, uuid, version, is_history):
     return self.get(uuid=uuid, version=version, is_history=is_history)
 

desktop/core/src/desktop/urls.py

@@ -34,12 +34,12 @@ desktop.lib.metrics.file_reporter.start_file_reporter()
 
 from django.conf import settings
 from django.conf.urls import include, patterns
-from django.conf.urls.static import static
 from django.contrib import admin
 
 from desktop import appmanager
 from desktop.conf import METRICS, USE_NEW_EDITOR
 
+
 # Django expects handler404 and handler500 to be defined.
 # django.conf.urls provides them. But we want to override them.
 # Also see http://code.djangoproject.com/ticket/5350