Browse Source

HUe-950 [jobsub] Add Document support

Romain Rigaux 12 years ago
parent
commit
ddb1afb96e

+ 0 - 4
apps/jobsub/src/jobsub/models.py

@@ -16,10 +16,6 @@
 # limitations under the License.
 
 import logging
-try:
-  import json
-except ImportError:
-  import simplejson
 
 from django.db import models
 from django.core import urlresolvers

+ 7 - 7
apps/jobsub/src/jobsub/templates/designs.mako

@@ -60,7 +60,7 @@ ${ commonheader(None, "jobsub", user) | n,unicode }
 
 <div class="container-fluid">
   <div class="card card-small">
-  <h1 class="card-heading simple">${_('Job Designs')}</h1>
+  <h1 class="card-heading simple">${_('Designs')}</h1>
 
   <%actionbar:render>
     <%def name="search()">
@@ -84,22 +84,20 @@ ${ commonheader(None, "jobsub", user) | n,unicode }
         </div>
       <!-- /ko -->
       <!-- ko if: inTrash -->
-        <button id="restore-designs" class="btn" title="${_('Restore')}" data-bind="enable: selectedDesignObjects().length > 0"><i class="icon-cloud-upload"></i> ${_('Restore')}</button>
-        <button id="destroy-designs" class="btn" title="${_('Delete forever')}" data-bind="enable: selectedDesignObjects().length > 0"><i class="icon-bolt"></i> ${_('Delete forever')}</button>
+        <button id="restore-designs" disabled="disabled" class="btn" title="${_('Restore')}" data-bind="enable: selectedDesignObjects().length > 0"><i class="icon-cloud-upload"></i> ${_('Restore')}</button>
+        <button id="destroy-designs" disabled="disabled" class="btn" title="${_('Delete forever')}" data-bind="enable: selectedDesignObjects().length > 0"><i class="icon-bolt"></i> ${_('Delete forever')}</button>
       <!-- /ko -->
       </div>
     </%def>
 
     <%def name="creation()">
       <div class="btn-toolbar" style="display: inline; vertical-align: middle">
-        <button id="home" class="btn" title="${_('Home')}" data-bind="visible: isEditing"><i class="icon-home"></i> ${_('View designs')}</button>
       <!-- ko if: inTrash -->
+        <button disabled="disabled" type="button" id="purge-trashed-designs" class="btn" title="${ _('Delete all the designs') }"><i class="icon-fire"></i> ${ _('Empty trash') }</button>
         &nbsp;&nbsp;
-        <button type="button" id="purge-trashed-designs" class="btn" title="${ _('Delete all the designs') }"><i class="icon-fire"></i> ${ _('Empty trash') }</button>
       <!-- /ko -->
+      <button id="home" class="btn" title="${_('Home')}" data-bind="visible: isEditing"><i class="icon-home"></i> ${_('View designs')}</button>
       <!-- ko ifnot: inTrash -->
-        <a href="#trashed-designs" class="btn"><i class="icon-trash"></i> ${ _('View trash') }</a>
-        &nbsp;&nbsp;
         <div id="new-action-dropdown" class="btn-group" style="vertical-align: middle">
           <a href="#" class="btn new-action-link dropdown-toggle" title="${_('New action')}" data-toggle="dropdown">
             <i class="icon-plus-sign"></i> ${_('New action')}
@@ -141,6 +139,8 @@ ${ commonheader(None, "jobsub", user) | n,unicode }
             </li>
           </ul>
         </div>
+        &nbsp;&nbsp;
+        <a href="#trashed-designs" class="btn"><i class="icon-trash"></i> ${ _('View trash') }</a>
       <!-- /ko -->
 
       </div>

+ 25 - 12
apps/jobsub/src/jobsub/tests.py

@@ -28,6 +28,8 @@ from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 
 from desktop.lib.django_test_util import make_logged_in_client
+from desktop.models import Document
+
 from liboozie.oozie_api_test import OozieServerProvider
 from oozie.models import Workflow, Node, Action, Start, Kill, End, Link
 
@@ -41,6 +43,7 @@ class TestJobsubWithHadoop(OozieServerProvider):
     self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir, '/user/jobsub_test')
     self.cluster.fs.do_as_superuser(self.cluster.fs.chmod, '/user/jobsub_test', 0777, True)
     self.client = make_logged_in_client(username='jobsub_test')
+    self.user = User.objects.get(username='jobsub_test')
 
     # Ensure access to MR folder.
     # Need to chmod because jobs are submitted as a
@@ -112,47 +115,57 @@ class TestJobsubWithHadoop(OozieServerProvider):
 
   def test_delete_design(self):
     # Trash
-    assert_equal(1, Workflow.objects.available().count())
+    n_available = Document.objects.available_docs(Workflow, self.user).count()
+    n_trashed = Document.objects.trashed_docs(Workflow, self.user).count()
+
     response = self.client.post(reverse('jobsub.views.delete_design',
       kwargs={'design_id': self.design.id}),
       follow=True,
       HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+
     assert_equal(response.status_code, 200)
-    assert_equal(0, Workflow.objects.available().count())
-    assert_equal(1, Workflow.objects.trashed().count())
+    assert_equal(n_available - 1, Document.objects.available_docs(Workflow, self.user).count())
+    assert_equal(n_trashed + 1, Document.objects.trashed_docs(Workflow, self.user).count())
 
     # Destroy
     response = self.client.post(reverse('jobsub.views.delete_design',
       kwargs={'design_id': self.design.id}) + '?skip_trash',
       follow=True,
       HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+
     assert_equal(response.status_code, 200)
-    assert_equal(0, Workflow.objects.available().count())
-    assert_equal(0, Workflow.objects.trashed().count())
+    assert_equal(n_available - 1, Document.objects.available_docs(Workflow, self.user).count())
+    assert_equal(n_trashed, Document.objects.trashed_docs(Workflow, self.user).count())
 
   def test_clone_design(self):
-    assert_equal(1, Workflow.objects.available().count())
+    n_available = Document.objects.available_docs(Workflow, self.user).count()
+
     response = self.client.post(reverse('jobsub.views.clone_design',
       kwargs={'design_id': self.design.id}),
       follow=True,
       HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+
     assert_equal(response.status_code, 200)
-    assert_equal(2, Workflow.objects.available().count())
+    assert_equal(n_available + 1, Document.objects.available_docs(Workflow, self.user).count())
 
   def test_restore_design(self):
-    assert_equal(1, Workflow.objects.available().count())
+    n_available = Document.objects.available_docs(Workflow, self.user).count()
+    n_trashed = Document.objects.trashed_docs(Workflow, self.user).count()
+
     response = self.client.post(reverse('jobsub.views.delete_design',
       kwargs={'design_id': self.design.id}),
       follow=True,
       HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+
     assert_equal(response.status_code, 200)
-    assert_equal(0, Workflow.objects.available().count())
-    assert_equal(1, Workflow.objects.trashed().count())
+    assert_equal(n_available - 1, Document.objects.available_docs(Workflow, self.user).count())
+    assert_equal(n_trashed + 1, Document.objects.trashed_docs(Workflow, self.user).count())
 
     response = self.client.post(reverse('jobsub.views.restore_design',
       kwargs={'design_id': self.design.id}),
       follow=True,
       HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+
     assert_equal(response.status_code, 200)
-    assert_equal(1, Workflow.objects.available().count())
-    assert_equal(0, Workflow.objects.trashed().count())
+    assert_equal(n_available, Document.objects.available_docs(Workflow, self.user).count())
+    assert_equal(n_trashed, Document.objects.trashed_docs(Workflow, self.user).count())

+ 30 - 16
apps/jobsub/src/jobsub/views.py

@@ -14,6 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 """
 Views for JobSubmission.
 
@@ -35,6 +36,7 @@ from desktop.lib.django_util import render, render_json
 from desktop.lib.exceptions import StructuredException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.log.access import access_warn
+from desktop.models import Document
 
 from oozie.models import Workflow
 from oozie.forms import design_form_by_type
@@ -54,7 +56,8 @@ def _list_designs(request, owner, name, order_by='-last_modified'):
     order_by    - Order by string in django ORM format
     is_trashed  - Boolean filter for trash or available
   """
-  data = Workflow.objects.filter(managed=False)
+  data = Document.objects.get_docs(request.user, Workflow, extra='jobsub')
+
   if owner:
       data = data.filter(owner__username__icontains=owner)
   if name:
@@ -62,20 +65,22 @@ def _list_designs(request, owner, name, order_by='-last_modified'):
   data = data.order_by(order_by)
 
   designs = []
-  for design in data:
-      ko_design = {
-        'id': design.id,
-        'owner': design.owner.username,
-        # Design name is validated by workflow and node forms.
-        'name': design.name,
-        'description': design.description,
-        'node_type': design.start.get_child('to').node_type,
-        'last_modified': py_time.mktime(design.last_modified.timetuple()),
-        'editable': design.owner.id == request.user.id,
-        'is_shared': design.is_shared,
-        'is_trashed': design.is_trashed
-      }
-      designs.append(ko_design)
+  for doc in data:
+    design = doc.content_object
+
+    ko_design = {
+      'id': design.id,
+      'owner': design.owner.username,
+      # Design name is validated by workflow and node forms.
+      'name': design.name,
+      'description': design.description,
+      'node_type': design.start.get_child('to').node_type,
+      'last_modified': py_time.mktime(design.last_modified.timetuple()),
+      'editable': design.owner.id == request.user.id,
+      'is_shared': design.is_shared,
+      'is_trashed': doc.is_trashed()
+    }
+    designs.append(ko_design)
 
   return designs
 
@@ -87,7 +92,6 @@ def list_designs(request):
     owner       - Substring filter by owner field
     name        - Substring filter by design name field
   '''
-  data = Workflow.objects.filter(managed=False)
   owner = request.GET.get('owner', '')
   name = request.GET.get('name', '')
 
@@ -102,6 +106,7 @@ def list_designs(request):
       'name': name
     })
 
+
 def _get_design(design_id):
   """Raise PopupException if design doesn't exist"""
   try:
@@ -109,6 +114,7 @@ def _get_design(design_id):
   except Workflow.DoesNotExist:
     raise PopupException(_("Workflow not found"))
 
+
 def _check_permission(request, owner_name, error_msg, allow_root=False):
   """Raise PopupException if user doesn't have permission to modify the design"""
   if request.user.username != owner_name:
@@ -117,6 +123,7 @@ def _check_permission(request, owner_name, error_msg, allow_root=False):
     access_warn(request, error_msg)
     raise PopupException(_("Permission denied. You are not the owner."))
 
+
 def delete_design(request, design_id):
   if request.method != 'POST':
     raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)
@@ -211,6 +218,9 @@ def _save_design(design_id, data):
 
   workflow.save()
 
+  if workflow.doc.exists():
+    workflow.doc.update(name=workflow.name, description=workflow.description)
+
 
 def new_design(request, node_type):
   """
@@ -245,6 +255,10 @@ def new_design(request, node_type):
   workflow.description = request.POST.get('description')
   workflow.save()
 
+  doc = workflow.doc.get()
+  doc.extra='jobsub'
+  doc.save()
+
   # Save design again to update all fields.
   data = format_dict_field_values(request.POST.copy())
   _save_design(workflow.id, data)

+ 4 - 1
apps/oozie/src/oozie/models.py

@@ -417,7 +417,10 @@ class Workflow(Job):
     return 'workflow.xml'
 
   def get_absolute_url(self):
-    return reverse('oozie:edit_workflow', kwargs={'workflow': self.id}) + '#editWorkflow'
+    if self.doc.get().extra == 'jobsub':
+      return '/jobsub/#edit-design/%s' % self.id
+    else:
+      return reverse('oozie:edit_workflow', kwargs={'workflow': self.id}) + '#editWorkflow'
 
   def get_hierarchy(self):
     node = Start.objects.get(workflow=self) # Uncached version of start.

+ 2 - 1
apps/zookeeper/src/zookeeper/templates/clients.mako

@@ -23,9 +23,10 @@
 
 ${ commonheader(_('Clients'), app_name, user) | n,unicode }
 ${ shared.menubar() }
+
 <%
   _breadcrumbs = [
-    [_("Home"), url('zookeeper:index')],
+    [_("Clusters"), url('zookeeper:index')],
     [cluster['nice_name'].lower(), url('zookeeper:view', id=cluster['id'])],
     [host + ":" + port, url('zookeeper:clients', id=cluster['id'], host=host + ":" + port)]
   ]

+ 5 - 5
apps/zookeeper/src/zookeeper/templates/index.mako

@@ -26,11 +26,11 @@ ${ shared.menubar() }
 
 <%
   _breadcrumbs = [
-    ["Home", url('zookeeper:index')]
+    ["Clusters", url('zookeeper:index')]
   ]
 %>
 
-%if len(clusters)==0:
+% if not clusters:
   <div class="container-fluid">
     <div class="card">
       <h1 class="card-heading simple">${ _('There are currently no clusters to browse.') }</h1>
@@ -43,10 +43,10 @@ ${ shared.menubar() }
     </div>
     </div>
   </div>
-%else:
-${ shared.header(_breadcrumbs, clusters) }
+% else:
+  ${ shared.header(_breadcrumbs, clusters) }
 % for c in clusters:
-  <h3 class="card-heading simple simpler">${ _('Cluster') } ${ c }</h3>
+  <h3 class="card-heading simple simpler">${ c }</h3>
   <p>
     </br>
     &nbsp;&nbsp;<a href="${ url('zookeeper:view', id=c) }">${ _("Znode Hierarchy") }</a> 

+ 1 - 1
apps/zookeeper/src/zookeeper/templates/tree.mako

@@ -27,7 +27,7 @@ ${ shared.menubar() }
 <%
   _split = path.split("/");
   _breadcrumbs = [
-    [_("Home"), url('zookeeper:index')],
+    [_("Clusters"), url('zookeeper:index')],
     [cluster['nice_name'].lower(), url('zookeeper:view', id=cluster['id'])],
   ]
   for idx, p in enumerate(_split):

+ 2 - 1
apps/zookeeper/src/zookeeper/templates/view.mako

@@ -26,10 +26,11 @@ ${ shared.menubar() }
 
 <%
   _breadcrumbs = [
-    [_("Home"), url('zookeeper:index')],
+    [_("Clusters"), url('zookeeper:index')],
     [cluster['nice_name'].lower(), url('zookeeper:view', id=cluster['id'])]
   ]
 %>
+
 <%def name="show_stats(stats)">
   <thead>
   <tr>

+ 53 - 20
desktop/core/src/desktop/models.py

@@ -59,7 +59,7 @@ class DocumentTagManager(models.Manager):
     if tag_name in DocumentTag.RESERVED:
       raise Exception(_("Can't add %s: it is a reserved tag.") % tag_name)
     else:
-      tag, created = DocumentTag.objects.create(tag=tag_name, owner=owner)
+      tag, created = DocumentTag.objects.get_or_create(tag=tag_name, owner=owner)
       return tag
 
   def get_default_tag(self, user):
@@ -147,17 +147,25 @@ class DocumentManager(models.Manager):
   def documents(self, user):
     return Document.objects.filter(Q(owner=user) | Q(documentpermission__users=user) | Q(documentpermission__groups__in=user.groups.all()))
 
-  def get_docs(self, user):
-    return Document.objects.documents(user).exclude(name='pig-app-hue-script')
+  def get_docs(self, user, model_class=None, extra=None):
+    docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
+    
+    if model_class is not None:
+      ct = ContentType.objects.get_for_model(model_class)
+      docs = docs.filter(content_type=ct)
+
+    if extra is not None:
+      docs = docs.filter(extra=extra)
+
+    return docs
 
   def get_doc(self, doc_id, user):
     return Document.objects.documents(user).get(id=doc_id)
 
   def trashed_docs(self, model_class, user):
-    ct = ContentType.objects.get_for_model(model_class)
     tag = DocumentTag.objects.get_trash_tag(user=user)
 
-    return Document.objects.get_docs(user).filter(content_type=ct).filter(tags__in=[tag]).order_by('-last_modified')
+    return Document.objects.get_docs(user, model_class).filter(tags__in=[tag]).order_by('-last_modified')
 
   def trashed(self, model_class, user):
     docs = self.trashed_docs(model_class, user)
@@ -165,11 +173,10 @@ class DocumentManager(models.Manager):
     return [job.content_object for job in docs if job.content_object]
 
   def available_docs(self, model_class, user):
-    ct = ContentType.objects.get_for_model(model_class)
     trash = DocumentTag.objects.get_trash_tag(user=user)
     history = DocumentTag.objects.get_history_tag(user=user)
 
-    return Document.objects.get_docs(user).filter(content_type=ct).exclude(tags__in=[trash, history]).order_by('-last_modified')
+    return Document.objects.get_docs(user, model_class).exclude(tags__in=[trash, history]).order_by('-last_modified')  
 
   def available(self, model_class, user):
     docs = self.available_docs(model_class, user)
@@ -199,24 +206,32 @@ class DocumentManager(models.Manager):
     return doc.is_accessible(user)
 
   def link(self, content_object, owner, name='', description='', extra=''):
-    doc = Document.objects.create(
-              content_object=content_object,
-              owner=owner,
-              name=name,
-              description=description,
-              extra=extra
-          )
-
-    tag = DocumentTag.objects.get_default_tag(user=owner)
-    doc.tags.add(tag)
-
-    return doc
+    if not content_object.doc.exists():
+      doc = Document.objects.create(
+                content_object=content_object,
+                owner=owner,
+                name=name,
+                description=description,
+                extra=extra
+            )
+  
+      tag = DocumentTag.objects.get_default_tag(user=owner)
+      doc.tags.add(tag)
+      return doc
+    else:
+      LOG.warn('Already a document %s for %s' % (content_object.doc.all(), content_object))
+      return content_object.doc.all()[0]
 
   def sync(self):
+
     try:
       from oozie.models import Workflow, Coordinator, Bundle
 
       for job in list(chain(Workflow.objects.all(), Coordinator.objects.all(), Bundle.objects.all())):
+        if job.doc.count() > 1:
+          LOG.warn('Deleting duplicate document %s for %s' % (job.doc.all(), job))
+          job.doc.all().delete()
+
         if not job.doc.exists():
           doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description)
           tag = DocumentTag.objects.get_default_tag(owner=job.owner)
@@ -225,11 +240,19 @@ class DocumentManager(models.Manager):
             doc.send_to_trash()
           if job.is_shared:
             DocumentPermission.objects.share_to_default(doc)
+          if hasattr(job, 'managed'):
+            if not job.managed:
+              doc.extra = 'jobsub'
+              doc.save()              
     except Exception, e:
       LOG.warn(force_unicode(e))
 
     try:
       for job in SavedQuery.objects.all():
+        LOG.warn('Deleting duplicate document %s for %s' % (job.doc.all(), job))
+        if job.doc.count() > 1:
+          job.doc.all().delete()
+
         if not job.doc.exists():
           doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
           tag = DocumentTag.objects.get_default_tag(owner=job.owner)
@@ -243,6 +266,10 @@ class DocumentManager(models.Manager):
       from pig.models import PigScript
 
       for job in PigScript.objects.all():
+        LOG.warn('Deleting duplicate document %s for %s' % (job.doc.all(), job))
+        if job.doc.count() > 1:
+          job.doc.all().delete()
+
         if not job.doc.exists():
           doc = Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
           tag = DocumentTag.objects.get_default_tag(owner=job.owner)
@@ -286,6 +313,9 @@ class Document(models.Model):
   def remove_tag(self, tag):
     self.tags.remove(tag)
 
+  def is_trashed(self):
+    return DocumentTag.objects.get_trash_tag(user=self.owner) in self.tags.all()
+
   def send_to_trash(self):
     tag = DocumentTag.objects.get_trash_tag(user=self.owner)
     self.tags.add(tag)
@@ -343,7 +373,10 @@ class Document(models.Model):
         else:
           return apps['impala'].icon_path
       elif self.content_type.app_label == 'oozie':
-        return self.content_type.model_class().ICON
+        if self.extra == 'jobsub':
+          return apps['jobsub'].icon_path
+        else:
+          return self.content_type.model_class().ICON
       elif self.content_type.app_label in apps:
         return apps[self.content_type.app_label].icon_path
       else:

+ 7 - 3
desktop/core/src/desktop/templates/home.mako

@@ -118,9 +118,13 @@ ${ commonheader(_('Welcome Home'), "home", user) | n,unicode }
                 % endif
               </ul>
            </li>
-           <li class="viewTrash toggableSection"><a href="javascript:void(0)"><i class="icon-trash"></i> ${_('Trash')} <span id="trashCounter" class="badge pull-right">0</span></a></li>
-           <li class="viewHistory toggableSection"><a href="javascript:void(0)"><i class="icon-time"></i> ${_('History')} <span id="historyCounter" class="badge pull-right">0</span></a></li>
-          <li class="nav-header tag-header">${_('Tags')} <div id="editTags" style="display: inline;cursor: pointer;margin-left: 6px"><i class="icon-tags"></i></div> </li>
+           <li class="viewTrash toggableSection">
+             <a href="javascript:void(0)"><i class="icon-trash"></i> ${_('Trash')} <span id="trashCounter" class="badge pull-right">0</span></a>
+           </li>
+           <li class="viewHistory toggableSection">
+             <a href="javascript:void(0)"><i class="icon-time"></i> ${_('History')} <span id="historyCounter" class="badge pull-right">0</span></a>
+           </li>
+          <li class="nav-header tag-header">${_('Tags')} <div id="editTags" style="display: inline;cursor: pointer;margin-left: 6px" title="${ _('Edit tags') }"><i class="icon-tags"></i></div> </li>
           % for tag in tags:
             %if tag.tag != "trash" and tag.tag != "history":
             <li class="toggleTag white" data-tag="${ tag.tag }"><div class="hueCheckbox pull-left"></div>${ tag.tag }</li>