Эх сурвалжийг харах

[doc2] Doc2 API and model refactor with migration for new parent FK

Includes:
- changing Document2 name to simple name instead of full path
- parent FK to parent directories
- management command and sanity check to create user directories
- updating move and trash operations
- update APIs to generally use UUIDs
- allow get_documents to work for both directories and documents
Jenny Kim 9 жил өмнө
parent
commit
4188f2a649

+ 2 - 0
apps/oozie/src/oozie/tests.py

@@ -285,6 +285,7 @@ class OozieMockBase(object):
     grant_access("test", "test", "oozie")
     add_to_group("test")
     self.user = User.objects.get(username='test')
+
     self.wf = create_workflow(self.c, self.user)
     self.original_fs = originalCluster.FS_CACHE["default"]
     originalCluster.FS_CACHE["default"] = MockFs()
@@ -391,6 +392,7 @@ class OozieBase(OozieServerProvider):
     grant_access("test", "test", "oozie")
     add_to_group("test")
     self.cluster = OozieServerProvider.cluster
+
     self.install_examples()
 
   def install_examples(self):

+ 68 - 78
desktop/core/src/desktop/api2.py

@@ -18,7 +18,6 @@
 import logging
 import json
 import tempfile
-import time
 import StringIO
 import zipfile
 
@@ -27,17 +26,15 @@ from django.core import management
 from django.db import transaction
 from django.http import HttpResponse
 from django.shortcuts import redirect
-from django.utils import html
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_POST
 
 from beeswax.models import SavedQuery
 from desktop.lib.django_util import JsonResponse
+from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.export_csvxls import make_response
 from desktop.lib.i18n import smart_str, force_unicode
-from desktop.models import Document2, Document, Directory, DocumentTag, import_saved_beeswax_query
-from desktop.lib.exceptions_renderable import PopupException
-from hadoop.fs.hadoopfs import Hdfs
+from desktop.models import Document2, Document, Directory, DocumentTag, FilesystemException, import_saved_beeswax_query
 
 
 LOG = logging.getLogger(__name__)
@@ -63,7 +60,7 @@ def api_error_handler(func):
 @api_error_handler
 def get_documents(request):
   """
-  Returns all documents and directories found in the given path (required) and current user.
+  Returns all documents and directories found for the given uuid or path and current user.
   Optional params:
     page=<n>    - Controls pagination. Defaults to 1.
     limit=<n>   - Controls limit per page. Defaults to all.
@@ -74,19 +71,15 @@ def get_documents(request):
                   Default to "-last_modified".
     text=<frag> - Search for fragment "frag" in names and descriptions.
   """
-  path = request.GET.get('path', '/') # Expects path to be a Directory for now
+  path = request.GET.get('path', '/')
+  uuid = request.GET.get('uuid')
 
-  try:
-    directory = Directory.objects.get(owner=request.user, name=path) # TODO perms
-  except Directory.DoesNotExist, e:
-    if path == '/':
-      directory, created = Directory.objects.get_or_create(name='/', owner=request.user)
-      directory.dependencies.add(*Document2.objects.filter(owner=request.user).exclude(id=directory.id))
-    else:
-      raise e
+  if uuid:
+    document = Document2.objects.get(uuid=uuid)
+  else:  # Find by path
+    document = Document2.objects.get_by_path(user=request.user, path=path)
 
-  parent_path = path.rstrip('/').rsplit('/', 1)[0] or '/'
-  parent = directory.dependencies.get(name=parent_path) if path != '/' else None
+  # TODO perms
 
   # Get querystring filters if any
   page = int(request.GET.get('page', 1))
@@ -95,20 +88,24 @@ def get_documents(request):
   sort = request.GET.get('sort', '-last_modified')
   search_text = request.GET.get('text', None)
 
-  documents = directory.documents(types=type_filters, search_text=search_text, order_by=sort)
-  count = documents.count()
+  # Get children documents if this is a directory
+  children = None
+  count = 0
+  if document.is_directory:
+    directory = Directory.objects.get(id=document.id)
+    children = directory.documents(types=type_filters, search_text=search_text, order_by=sort)
+    count = children.count()
 
   # Paginate
-  if limit > 0:
+  if children and limit > 0:
     offset = (page - 1) * limit
     last = offset + limit
-    documents = documents.all()[offset:last]
+    children = children.all()[offset:last]
 
   return JsonResponse({
-      'path': path,
-      'directory': directory.to_dict(),
-      'parent': parent.to_dict() if parent else None,
-      'documents': [doc.to_dict() for doc in documents if doc != parent],
+      'document': document.to_dict(),
+      'parent': document.parent_directory.to_dict() if document.parent_directory else None,
+      'children': [doc.to_dict() for doc in children] if children else [],
       'page': page,
       'limit': limit,
       'count': count,
@@ -125,6 +122,9 @@ def _convert_documents(user):
   from beeswax.models import HQL, IMPALA, RDBMS
 
   with transaction.atomic():
+    # If user does not have a home directory, we need to create one and import any orphan documents to it
+    Document2.objects.create_user_directories(user)
+
     docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA, RDBMS])
 
     imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
@@ -136,7 +136,7 @@ def _convert_documents(user):
         imported_tag  # No already imported docs
     ])
 
-    root_doc, created = Directory.objects.get_or_create(name='/', owner=user)
+    root_doc, created = Directory.objects.get_or_create(name='', owner=user)
     imported_docs = []
 
     for doc in docs:
@@ -153,7 +153,7 @@ def _convert_documents(user):
           raise e
 
     if imported_docs:
-      root_doc.dependencies.add(*imported_docs)
+      root_doc.children.add(*imported_docs)
 
 
 @api_error_handler
@@ -170,18 +170,15 @@ def get_document(request):
 @api_error_handler
 @require_POST
 def move_document(request):
-  source_doc_id = json.loads(request.POST.get('source_doc_id'))
-  destination_doc_id = json.loads(request.POST.get('destination_doc_id'))
-
-  # destination exists + is dir?
-  source = Document2.objects.document(request.user, doc_id=source_doc_id)
-  destination = Document2.objects.document(request.user, doc_id=destination_doc_id)
+  source_doc_uuid = json.loads(request.POST.get('source_doc_uuid'))
+  destination_doc_uuid = json.loads(request.POST.get('destination_doc_uuid'))
 
-  if destination.type != 'directory':
-    raise PopupException(_('Destination is not a directory'))
+  if not source_doc_uuid or not destination_doc_uuid:
+    raise PopupException(_('move_document requires source_doc_uuid and destination_doc_uuid'))
 
+  source = Directory.objects.get(uuid=source_doc_uuid)
+  destination = Directory.objects.get(uuid=destination_doc_uuid)
   source.move(destination, request.user)
-  source.save()
 
   return JsonResponse({'status': 0})
 
@@ -189,32 +186,49 @@ def move_document(request):
 @api_error_handler
 @require_POST
 def create_directory(request):
-  parent_path = json.loads(request.POST.get('parent_path'))
+  parent_uuid = json.loads(request.POST.get('parent_uuid'))
   name = json.loads(request.POST.get('name'))
 
-  parent_dir = Directory.objects.get(owner=request.user, name=parent_path)
+  if not parent_uuid or not name:
+    raise PopupException(_('create_directory requires parent_uuid and name'))
 
-  path = Hdfs.normpath(parent_path + '/' + name)
-  file_doc = Directory.objects.create(name=path, owner=request.user)
-  parent_dir.dependencies.add(file_doc)
+  parent_dir = Directory.objects.get(uuid=parent_uuid)
+  # TODO: Check permissions and move to manager
+  directory = Directory.objects.create(name=name, owner=request.user, parent_directory=parent_dir)
 
   return JsonResponse({
       'status': 0,
-      'file': file_doc.to_dict()
+      'directory': directory.to_dict()
   })
 
 
 @api_error_handler
 @require_POST
 def delete_document(request):
-  document_id = json.loads(request.POST.get('doc_id'))
-  skip_trash = json.loads(request.POST.get('skip_trash', 'false')) # TODO always false currently
+  """
+  Accepts a uuid and optional skip_trash parameter
+
+  (Default) skip_trash=false, flags a document as trashed
+  skip_trash=true, deletes it permanently along with any history dependencies
 
-  document = Document2.objects.document(request.user, doc_id=document_id)
-  if document.type == 'directory' and document.dependencies.count() > 1:
-    raise PopupException(_('Directory is not empty'))
+  If directory and skip_trash=false, all dependencies will also be flagged as trash
+  If directory and skip_trash=true, directory must be empty (no dependencies)
+  """
+  uuid = json.loads(request.POST.get('uuid'))
+  skip_trash = json.loads(request.POST.get('skip_trash', 'false'))
 
-  document.delete()
+  if not uuid:
+    raise PopupException(_('delete_document requires uuid'))
+
+  document = Document2.objects.get(uuid=uuid)
+
+  if skip_trash:
+    # TODO: check if document is in the .Trash folder, if not raise exception
+    if document.is_directory and document.has_children:
+      raise PopupException(_('Directory is not empty'))
+    document.delete()
+  else:
+    document.trash()  # TODO: get number of docs trashed
 
   return JsonResponse({
       'status': 0,
@@ -229,10 +243,13 @@ def share_document(request):
 
   Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
   """
-  perms_dict = json.loads(request.POST['data'])
-  doc_id = json.loads(request.POST['doc_id'])
+  perms_dict = json.loads(request.POST.get('data'))
+  uuid = json.loads(request.POST.get('uuid'))
+
+  if not uuid:
+    raise PopupException(_('share_document requires uuid'))
 
-  doc = Document2.objects.document(request.user, doc_id)
+  doc = Document2.objects.get(uuid=uuid)
 
   for name, perm in perms_dict.iteritems():
     users = groups = None
@@ -253,33 +270,6 @@ def share_document(request):
   })
 
 
-def _massage_doc_for_json(document, user, with_data=False):
-
-  massaged_doc = {
-    'id': document.id,
-    'uuid': document.uuid,
-
-    'owner': document.owner.username,
-    'type': html.conditional_escape(document.type),
-    'name': html.conditional_escape(document.name),
-    'description': html.conditional_escape(document.description),
-
-    'isMine': document.owner == user,
-    'lastModified': document.last_modified.strftime("%x %X"),
-    'lastModifiedInMillis': time.mktime(document.last_modified.timetuple()),
-    'version': document.version,
-    'is_history': document.is_history,
-
-    # tags
-    # dependencies
-  }
-
-  if with_data:
-    massaged_doc['data'] = document.data_dict
-
-  return massaged_doc
-
-
 def export_documents(request):
   if request.GET.get('documents'):
     selection = json.loads(request.GET.get('documents'))

+ 70 - 0
desktop/core/src/desktop/management/commands/create_user_directories.py

@@ -0,0 +1,70 @@
+
+# adapted from django-extensions (http://code.google.com/p/django-command-extensions/)
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from optparse import make_option
+
+from django.contrib.auth.models import User
+from django.core.management.base import CommandError, BaseCommand
+from django.utils.translation import ugettext_lazy as _
+
+from desktop.models import Document2
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  """
+  Creates home and Trash directories for users as needed, or specific user if username is provided
+
+  If no arguments are provided, this command will loop through all users and create a home and Trash directory for the
+  user if they don't exist. It will then move any orphan documents to the home directory.
+
+  If --username is specified, it will only perform the operation for the specific user.
+  """
+
+  help = _("Creates home and Trash directories for users as needed, or specific user if username is provided.")
+
+  option_list = BaseCommand.option_list + (
+    make_option('--username', help=_("Username of user to create directories for."), action='store', default=None),
+  )
+
+  def handle(self, *args, **options):
+    users = User.objects.all()
+
+    if options['username']:
+      try:
+        user = User.objects.get(username=options['username'])
+        users = [user]
+      except Exception, e:
+        msg = 'Failed to get user with username %s: %s' % (options['username'], str(e))
+        self.stdout.write(msg)
+        LOG.exception(msg)
+
+    for user in users:
+      try:
+        msg = 'Attempting to create user directories for user: %s' % user.username
+        self.stdout.write(msg)
+        LOG.debug(msg)
+        Document2.objects.create_user_directories(user)
+      except Exception, e:
+        msg = 'Failed to create user directories for user %s: %s' % (user.username, str(e))
+        self.stdout.write(msg)
+        LOG.warn(msg)

+ 127 - 0
desktop/core/src/desktop/migrations/0018_auto__add_field_document2_parent_directory.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Document2.parent_directory'
+        db.add_column(u'desktop_document2', 'parent_directory',
+                      self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='children', null=True, to=orm['desktop.Document2']),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'Document2.parent_directory'
+        db.delete_column(u'desktop_document2', 'parent_directory_id')
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'desktop.document': {
+            'Meta': {'unique_together': "(('content_type', 'object_id'),)", 'object_name': 'Document'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc_owner'", 'to': u"orm['auth.User']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['desktop.DocumentTag']", 'db_index': 'True', 'symmetrical': 'False'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'})
+        },
+        u'desktop.document2': {
+            'Meta': {'ordering': "['-last_modified']", 'unique_together': "(('uuid', 'version', 'is_history'),)", 'object_name': 'Document2'},
+            'data': ('django.db.models.fields.TextField', [], {'default': "'{}'"}),
+            'dependencies': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'dependencies_rel_+'", 'db_index': 'True', 'to': u"orm['desktop.Document2']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_history': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc2_owner'", 'to': u"orm['auth.User']"}),
+            'parent_directory': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': u"orm['desktop.Document2']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'tags_rel_+'", 'db_index': 'True', 'to': u"orm['desktop.Document2']"}),
+            'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '32', 'db_index': 'True'}),
+            'uuid': ('django.db.models.fields.CharField', [], {'default': "'f1f73b8a-8114-4fee-bb9a-414fd2bd2320'", 'max_length': '36', 'db_index': 'True'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1', 'db_index': 'True'})
+        },
+        u'desktop.document2permission': {
+            'Meta': {'unique_together': "(('doc', 'perms'),)", 'object_name': 'Document2Permission'},
+            'all': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'doc': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['desktop.Document2']"}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'documentpermission2_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'perms': ('django.db.models.fields.CharField', [], {'default': "'read'", 'max_length': '10', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.User']", 'db_table': "'documentpermission2_users'", 'symmetrical': 'False'})
+        },
+        u'desktop.documentpermission': {
+            'Meta': {'unique_together': "(('doc', 'perms'),)", 'object_name': 'DocumentPermission'},
+            'doc': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['desktop.Document']"}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'documentpermission_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'perms': ('django.db.models.fields.CharField', [], {'default': "'read'", 'max_length': '10'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.User']", 'db_table': "'documentpermission_users'", 'symmetrical': 'False'})
+        },
+        u'desktop.documenttag': {
+            'Meta': {'unique_together': "(('owner', 'tag'),)", 'object_name': 'DocumentTag'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'tag': ('django.db.models.fields.SlugField', [], {'max_length': '50'})
+        },
+        u'desktop.settings': {
+            'Meta': {'object_name': 'Settings'},
+            'collect_usage': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'tours_and_tutorials': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'})
+        },
+        u'desktop.userpreferences': {
+            'Meta': {'object_name': 'UserPreferences'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'key': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'value': ('django.db.models.fields.TextField', [], {'max_length': '4096'})
+        }
+    }
+
+    complete_apps = ['desktop']

+ 137 - 40
desktop/core/src/desktop/models.py

@@ -18,6 +18,7 @@
 import calendar
 import logging
 import json
+import os
 import uuid
 
 from itertools import chain
@@ -29,7 +30,9 @@ from django.contrib.staticfiles.storage import staticfiles_storage
 from django.core.urlresolvers import reverse
 from django.db import connection, models, transaction
 from django.db.models import Q
-from django.forms import ValidationError
+from django.db.models.signals import pre_save, pre_delete, post_save
+from django.dispatch import receiver
+from django.template.defaultfilters import urlencode
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
 from desktop import appmanager
@@ -45,6 +48,12 @@ SAMPLE_USER_ID = 1100713
 SAMPLE_USER_INSTALL = 'hue'
 SAMPLE_USER_OWNERS = ['hue', 'sample']
 
+UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
+
+
+def uuid_default():
+  return str(uuid.uuid4())
+
 
 class UserPreferences(models.Model):
   """Holds arbitrary key/value strings."""
@@ -460,9 +469,6 @@ class DocumentManager(models.Manager):
           docs.delete()
 
 
-UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
-
-
 class Document(models.Model):
   owner = models.ForeignKey(auth_models.User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
   name = models.CharField(default='', max_length=255)
@@ -734,13 +740,16 @@ class DocumentPermission(models.Model):
     (WRITE_PERM, 'write'),
   ))
 
-
   objects = DocumentPermissionManager()
 
   class Meta:
     unique_together = ('doc', 'perms')
 
 
+class FilesystemException(Exception):
+  pass
+
+
 class Document2Manager(models.Manager):
 
   # TODO prevent get
@@ -748,6 +757,10 @@ class Document2Manager(models.Manager):
     return self.documents(user).get(id=doc_id)
 
   def documents(self, user):
+    """
+    Returns all documents that are either owned or shared with the user
+    WARNING: Do NOT use this if you ONLY need documents that are owned by the user!
+    """
     return Document2.objects.filter(
         Q(owner=user) |
         Q(document2permission__users=user) |
@@ -755,21 +768,68 @@ class Document2Manager(models.Manager):
         Q(document2permission__all=True)
     ).distinct().order_by('-last_modified')
 
-  def directory(self, user, path):
-    return self.documents(user).get(type='directory', name=path)
-
   def get_by_natural_key(self, uuid, version, is_history):
     return self.get(uuid=uuid, version=version, is_history=is_history)
 
   def get_history(self, user, doc_type):
     return self.documents(user).filter(type=doc_type, is_history=True)
 
+  def get_home_directory(self, user):
+    try:
+      return self.get(owner=user, parent_directory=None, name='', type='directory')
+    except Document2.DoesNotExist:
+      return self.create_user_directories(user)
 
-def uuid_default():
-  return str(uuid.uuid4())
+  def get_by_path(self, user, path):
+    """
+    This can be an expensive operation b/c we have to traverse the path tree, so if possible, request a document by UUID
+    """
+    cleaned_path = path.rstrip('/')
+    doc = Document2.objects.get_home_directory(user)
+    if cleaned_path:
+      path_tokens = cleaned_path.split('/')[1:]
+      for token in path_tokens:
+        try:
+          doc = doc.children.get(name=token)
+        except Document2.DoesNotExist:
+          raise FilesystemException(_('Requested invalid path for user %s: %s') % (user.username, path))
+
+    return doc
+
+  def create_user_directories(self, user):
+    """
+    Creates user home and Trash directories if they do not exist and move any orphan documents to home directory
+    :param user: User object
+    """
+    # Edge-case if the user has a legacy home directory with path-name
+    Directory.objects.filter(name='/', owner=user).update(name='')
+
+    # Get or create home and Trash directories for all users
+    home_dir, created = Directory.objects.get_or_create(name='', owner=user)
+
+    if created:
+      LOG.info('Successfully created home directory for user: %s' % user.username)
+
+    trash_dir, created = Directory.objects.get_or_create(name=Document2.TRASH_DIR, owner=user, parent_directory=home_dir)
+
+    if created:
+      LOG.info('Successfully created trash directory for user: %s' % user.username)
+
+    # For any directories or documents that do not have a parent directory, assign it to home directory
+    count = 0
+    for doc in Document2.objects.filter(owner=user).filter(parent_directory=None).exclude(id__in=[home_dir.id, trash_dir.id]):
+      doc.parent_directory = home_dir
+      doc.save()
+      count += 1
+
+    LOG.info("Moved %d documents to home directory for user: %s" % (count, user.username))
+    return home_dir
 
 
 class Document2(models.Model):
+
+  TRASH_DIR = '.Trash'
+
   owner = models.ForeignKey(auth_models.User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Creator.'), related_name='doc2_owner')
   name = models.CharField(default='', max_length=255)
   description = models.TextField(default='')
@@ -783,10 +843,12 @@ class Document2(models.Model):
   last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
   version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
   is_history = models.BooleanField(default=False, db_index=True)
-  # is_trashed
 
   tags = models.ManyToManyField('self', db_index=True)
   dependencies = models.ManyToManyField('self', db_index=True)
+
+  parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
+
   doc = generic.GenericRelation(Document, related_name='doc_doc') # Compatibility with Hue 3
 
   objects = Document2Manager()
@@ -795,17 +857,43 @@ class Document2(models.Model):
     unique_together = ('uuid', 'version', 'is_history')
     ordering = ["-last_modified"]
 
-  def natural_key(self):
-    return (self.uuid, self.version, self.is_history)
+  def __str__(self):
+    res = '%s - %s - %s' % (force_unicode(self.name), self.owner, self.uuid)
+    return force_unicode(res)
 
   @property
   def data_dict(self):
     if not self.data:
       self.data = json.dumps({})
     data_python = json.loads(self.data)
-
     return data_python
 
+  @property
+  def path(self):
+    if self.parent_directory:
+      return '%s/%s' % (self.parent_directory.path, self.name)
+    else:
+      return self.name
+
+  @property
+  def dirname(self):
+    return os.path.dirname(self.path) or '/'
+
+  @property
+  def is_directory(self):
+    return self.type == 'directory'
+
+  @property
+  def is_home_directory(self):
+    return self.is_directory and self.parent_directory == None and self.name == ''
+
+  @property
+  def is_trash_directory(self):
+    return self.is_directory and self.name == self.TRASH_DIR
+
+  def natural_key(self):
+    return (self.uuid, self.version, self.is_history)
+
   def copy(self, name, owner, description=None):
     copy_doc = self
 
@@ -817,20 +905,13 @@ class Document2(models.Model):
     if description:
       copy_doc.description = description
     copy_doc.save()
-
     return copy_doc
 
   def update_data(self, post_data):
     data_dict = self.data_dict
-
     data_dict.update(post_data)
-
     self.data = json.dumps(data_dict)
 
-  def __str__(self):
-    res = '%s - %s - %s' % (force_unicode(self.name), self.owner, self.uuid)
-    return force_unicode(res)
-
   def get_absolute_url(self):
     if self.type == 'oozie-coordinator2':
       return reverse('oozie:edit_coordinator') + '?coordinator=' + str(self.id)
@@ -839,7 +920,7 @@ class Document2(models.Model):
     elif self.type.startswith('query'):
       return reverse('notebook:editor') + '?editor=' + str(self.id)
     elif self.type == 'directory':
-      return '/home2' + '?path=' + self.name
+      return '/home2' + '?uuid=' + self.uuid
     elif self.type == 'notebook':
       return reverse('notebook:notebook') + '?notebook=' + str(self.id)
     elif self.type == 'search-dashboard':
@@ -851,6 +932,7 @@ class Document2(models.Model):
     doc_dict = {
       'owner': self.owner.username,
       'name': self.name,
+      'path': urlencode(self.path or '/'),
       'description': self.description,
       'uuid': self.uuid,
       'id': self.id,
@@ -910,19 +992,30 @@ class Document2(models.Model):
             snippet['is_redacted'] = True
       self.data = json.dumps(data_dict)
 
+    # TODO: Validate name, shouldn't contain slashes
+    # TODO: Prevent documents with same name and location from being saved
+    # TODO: Prevent Home and Trash directories from being deleted
+    # TODO: Prevent creating home or trash directories in any location
+
+    # Save document to home directory if parent directory isn't specified
+    if not self.parent_directory and not self.is_home_directory and not self.is_trash_directory:
+      home_dir = Document2.objects.get_home_directory(self.owner)
+      self.parent_directory = home_dir
+
     super(Document2, self).save(*args, **kwargs)
 
   def move(self, directory, user):
-    # get dir and remove
-    old_directory = Document2.objects.get(type='directory', dependencies=self.pk)
-    if old_directory.can_write_or_exception(user=user):
-      old_directory.dependencies.remove(self)
-
-    # add to new dir
     if directory.can_write_or_exception(user=user):
-      directory.dependencies.add(self)
+      self.parent_directory = directory
+      self.save()
+
+  def trash(self):
+    trash_dir = Directory.objects.get(name=self.TRASH_DIR, owner=self.owner)
+    self.move(trash_dir, self.owner)
+
+  # TODO: erase/purge
 
-    self.name = directory.name + '/' + self.name.split('/')[-1]
+  # TODO: restore
 
   def share(self, user, name='read', users=None, groups=None):
     # TODO check in settings if user can sync, re-share, which perms...
@@ -943,6 +1036,7 @@ class Document2(models.Model):
       perm.delete()
 
   def list_permissions(self, perm='read'):
+    # FIXME This is causing an integrity error b/c it is called by doc2 api potentially concurrently for new docs
     perm, created = Document2Permission.objects.get_or_create(doc=self, perms=perm)
     return perm
 
@@ -967,15 +1061,22 @@ class Document2(models.Model):
       }
     }
 
+class DirectoryManager(models.Manager):
+
+  def get_queryset(self):
+    return super(DirectoryManager, self).get_queryset().filter(type='directory')
+
+
 class Directory(Document2):
   # e.g. name = '/' or '/dir1/dir2/f3'
 
+  objects = DirectoryManager()
+
   class Meta:
     proxy = True
 
-
   def documents(self, types=None, search_text=None, order_by=None):
-    documents = self.dependencies.all()  # TODO: perms
+    documents = self.children.all()  # TODO: perms
 
     if types and isinstance(types, list):
       documents = documents.filter(type__in=types)
@@ -988,22 +1089,19 @@ class Directory(Document2):
 
     return documents
 
-
-  def parent(self):
-    return Document2.objects.get(type='directory', dependencies=[self.pk]) # or name__startswith=self.name
-
-
   def save(self, *args, **kwargs):
     self.type = 'directory'
 
     try:
       directory = Directory.objects.get(name=self.name, owner=self.owner, type='directory')
       if directory.pk != self.pk:
-        raise ValidationError('Directory for owner %s named %s already exists' % (self.name, self.owner))
+        raise FilesystemException(_('Directory for owner %s at path already exists') % (self.owner, self.path))
     except Directory.DoesNotExist:
       pass  # no conflicts
     except Directory.MultipleObjectsReturned:
-      raise Exception('Found multiple directories for owner %s named %s' % (self.owner, self.name))
+      directory_ids = [dir.id for dir in Directory.objects.filter(name=self.name, owner=self.owner, type='directory')]
+      raise FilesystemException(_('Found multiple directories for owner %s at path %s with IDs: [%s]') %
+                                (self.owner, self.path, ', '.join(directory_ids)))
 
     super(Directory, self).save(*args, **kwargs)
 
@@ -1086,4 +1184,3 @@ def _convert_type(btype, bdata):
     return 'spark'
   else:
     return 'hive'
-

+ 12 - 27
desktop/core/src/desktop/tests.py

@@ -53,7 +53,7 @@ from desktop.lib.conf import validate_path
 from desktop.lib.django_util import TruncatingModel
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.test_utils import grant_access
-from desktop.models import Document, Document2, get_data_link
+from desktop.models import Directory, Document, Document2, get_data_link
 from desktop.redaction import logfilter
 from desktop.redaction.engine import RedactionPolicy, RedactionRule
 from desktop.views import check_config, home
@@ -849,19 +849,17 @@ class TestDocument(object):
     make_logged_in_client(username="copy_owner", groupname="test_doc", recreate=True, is_superuser=False)
     self.copy_user = User.objects.get(username="copy_owner")
 
-    # Get count of existing Document objects
-    self.doc2_count = Document2.objects.count()
-    self.doc1_count = Document.objects.count()
-
     self.document2 = Document2.objects.create(name='Test Document2',
                                               type='search-dashboard',
                                               owner=self.user,
                                               description='Test Document2')
+
     self.document = Document.objects.link(content_object=self.document2,
                                           owner=self.user,
                                           name='Test Document',
                                           description='Test Document',
                                           extra='test')
+
     self.document.save()
     self.document2.doc.add(self.document)
 
@@ -871,17 +869,22 @@ class TestDocument(object):
     test_docs.delete()
 
   def test_document_create(self):
-    assert_equal(Document2.objects.count(), self.doc2_count + 1)
-    assert_equal(Document.objects.count(), self.doc1_count + 1)
+    assert_true(Document2.objects.filter(name='Test Document2').exists())
+    assert_true(Document.objects.filter(name='Test Document').exists())
     assert_equal(Document2.objects.get(name='Test Document2').id, self.document2.id)
     assert_equal(Document.objects.get(name='Test Document').id, self.document.id)
 
   def test_document_copy(self):
     name = 'Test Document2 Copy'
+
+    self.doc2_count = Document2.objects.count()
+    self.doc1_count = Document.objects.count()
+
     doc2 = self.document2.copy(name=name, owner=self.copy_user, description=self.document2.description)
+    doc = self.document.copy(doc2, name=name, owner=self.copy_user, description=self.document2.description)
 
-    # Test that copying a Document2 object creates another object
-    assert_equal(Document2.objects.count(), self.doc2_count + 2)
+    # Test that copying creates another object
+    assert_equal(Document2.objects.count(), self.doc2_count + 1)
     assert_equal(Document.objects.count(), self.doc1_count + 1)
 
     # Test that the content object is not pointing to the same object
@@ -894,12 +897,6 @@ class TestDocument(object):
     assert_equal(Document2.objects.filter(name=name).count(), 1)
     assert_equal(doc2.description, self.document2.description)
 
-    doc = self.document.copy(doc2, name=name, owner=self.copy_user, description=self.document2.description)
-
-    # Test that copying a Document object creates another Document2 and Document object
-    assert_equal(Document2.objects.count(), self.doc2_count + 2)
-    assert_equal(Document.objects.count(), self.doc1_count + 2)
-
     # Test that the content object is not pointing to the same object
     assert_not_equal(self.document.content_object, doc.content_object)
 
@@ -910,18 +907,6 @@ class TestDocument(object):
     assert_equal(Document.objects.filter(name=name).count(), 1)
     assert_equal(doc.description, self.document.description)
 
-  def test_add_to_history(self):
-    assert_equal(len(self.document2.get_history()), 0)
-
-    doc_id = self.document2.id
-    history_doc = self.document2.add_to_history(self.user, {'key1': 'val1'})
-
-    assert_equal(len(Document2.objects.get(id=doc_id).dependencies.all()), 1) # Need to get original document dynamically
-    assert_equal(len(Document2.objects.get(id=doc_id).get_history()), 1)
-
-    assert_equal(history_doc, Document2.objects.get(id=doc_id).get_history()[0])
-
-    assert_not_equal(doc_id, history_doc.id)
 
   def test_redact_statements(self):
     old_policies = redaction.global_redaction_engine.policies

+ 25 - 21
desktop/core/src/desktop/tests_doc2.py

@@ -41,7 +41,7 @@ class TestDocument2(object):
     response = self.client.get('/desktop/api2/docs/')
     data = json.loads(response.content)
 
-    assert_equal('/', data['path'], data)
+    assert_equal('/', data['document']['path'], data)
 
 
   def test_document_create(self):
@@ -80,65 +80,69 @@ class TestDocument2(object):
 
 
   def test_directory_create(self):
-    response = self.client.post('/desktop/api2/doc/mkdir', {'parent_path': json.dumps('/'), 'name': json.dumps('test_mkdir')})
+    home_dir = Document2.objects.get_home_directory(self.user)
+    response = self.client.post('/desktop/api2/doc/mkdir', {'parent_uuid': json.dumps(home_dir.uuid), 'name': json.dumps('test_mkdir')})
     data = json.loads(response.content)
 
     assert_equal(0, data['status'], data)
+    assert_true('directory' in data)
+    assert_equal(data['directory']['name'], 'test_mkdir', data)
+    assert_equal(data['directory']['type'], 'directory', data)
 
 
   def test_directory_move(self):
-    response = self.client.post('/desktop/api2/doc/mkdir', {'parent_path': json.dumps('/'), 'name': json.dumps('test_mv')})
+    home_dir = Document2.objects.get_home_directory(self.user)
+    response = self.client.post('/desktop/api2/doc/mkdir', {'parent_uuid': json.dumps(home_dir.uuid), 'name': json.dumps('test_mv')})
     data = json.loads(response.content)
     assert_equal(0, data['status'], data)
 
-    response = self.client.post('/desktop/api2/doc/mkdir', {'parent_path': json.dumps('/'), 'name': json.dumps('test_mv_dst')})
+    response = self.client.post('/desktop/api2/doc/mkdir', {'parent_uuid': json.dumps(home_dir.uuid), 'name': json.dumps('test_mv_dst')})
     data = json.loads(response.content)
     assert_equal(0, data['status'], data)
 
     response = self.client.post('/desktop/api2/doc/move', {
-        'source_doc_id': json.dumps(Directory.objects.get(owner=self.user, name='/test_mv').id),
-        'destination_doc_id': json.dumps(Directory.objects.get(owner=self.user, name='/test_mv_dst').id)
+        'source_doc_uuid': json.dumps(Directory.objects.get(owner=self.user, name='test_mv').uuid),
+        'destination_doc_uuid': json.dumps(Directory.objects.get(owner=self.user, name='test_mv_dst').uuid)
     })
     data = json.loads(response.content)
 
     assert_equal(0, data['status'], data)
-
-    assert_true(Directory.objects.filter(owner=self.user, name='/test_mv_dst/test_mv').exists())
+    assert_equal(Directory.objects.get(name='test_mv', owner=self.user).path, '/test_mv_dst/test_mv')
 
 
   def test_directory_documents(self):
-    home_dir = Directory.objects.get(owner=self.user, name='/')
+    home_dir = Directory.objects.get(owner=self.user, name='')
 
-    dir1 = Directory.objects.create(name='/test_dir1', owner=self.user)
-    dir2 = Directory.objects.create(name='/test_dir2', owner=self.user)
-    query1 = Document2.objects.create(name='query1', type='query-hive', owner=self.user, data={})
-    query2 = Document2.objects.create(name='query2', type='query-hive', owner=self.user, data={})
+    dir1 = Directory.objects.create(name='test_dir1', owner=self.user)
+    dir2 = Directory.objects.create(name='test_dir2', owner=self.user)
+    query1 = Document2.objects.create(name='query1.sql', type='query-hive', owner=self.user, data={})
+    query2 = Document2.objects.create(name='query2.sql', type='query-hive', owner=self.user, data={})
     children = [dir1, dir2, query1, query2]
 
-    home_dir.dependencies.add(*children)
+    home_dir.children.add(*children)
 
     # Test that all children directories and documents are returned
     response = self.client.get('/desktop/api2/docs', {'path': '/'})
     data = json.loads(response.content)
-    assert_true('documents' in data)
-    assert_equal(4, data['count'])
+    assert_true('children' in data)
+    assert_equal(5, data['count'])  # This includes the 4 docs and .Trash
 
     # Test filter type
     response = self.client.get('/desktop/api2/docs', {'path': '/', 'type': ['directory']})
     data = json.loads(response.content)
     assert_equal(['directory'], data['types'])
-    assert_equal(2, data['count'])
-    assert_true(all(doc['type'] == 'directory' for doc in data['documents']))
+    assert_equal(3, data['count'])
+    assert_true(all(doc['type'] == 'directory' for doc in data['children']))
 
     # Test search text
     response = self.client.get('/desktop/api2/docs', {'path': '/', 'text': 'query'})
     data = json.loads(response.content)
     assert_equal('query', data['text'])
     assert_equal(2, data['count'])
-    assert_true(all(doc['name'].startswith('query') for doc in data['documents']))
+    assert_true(all('query' in doc['name'] for doc in data['children']))
 
     # Test pagination with limit
     response = self.client.get('/desktop/api2/docs', {'path': '/', 'page': 2, 'limit': 2})
     data = json.loads(response.content)
-    assert_equal(4, data['count'])
-    assert_equal(2, len(data['documents']))
+    assert_equal(5, data['count'])
+    assert_equal(2, len(data['children']))

+ 1 - 1
desktop/libs/notebook/src/notebook/decorators.py

@@ -63,7 +63,7 @@ def check_document_modify_permission():
           doc2 = Document2.objects.get(id=notebook['id'])
           doc2.doc.get().can_write_or_exception(request.user)
       except Document.DoesNotExist:
-        raise PopupException(_('Job %(id)s does not exist') % {'id': notebook.get('id')})
+        raise PopupException(_('Document %(id)s does not exist') % {'id': notebook.get('id')})
 
       return view_func(request, *args, **kwargs)
     return wraps(view_func)(decorate)