Browse Source

HUE-3065 [doc2] Enables filtering, searching, offset and limit on documents

Jenny Kim 9 years ago
parent
commit
472d9a3

+ 39 - 9
desktop/core/src/desktop/api2.py

@@ -62,6 +62,18 @@ def api_error_handler(func):
 
 @api_error_handler
 def get_documents(request):
+  """
+  Returns all documents and directories found in the given path (required) and current user.
+  Optional params:
+    page=<n>    - Controls pagination. Defaults to 1.
+    limit=<n>   - Controls limit per page. Defaults to all.
+    type=<type> - Show documents of given type(s) (directory, query-hive, query-impala, query-mysql, etc). Default to all.
+    sort=<key>  - Sort by the attribute <key>, which is one of:
+                    "name", "type", "owner", "last_modified"
+                  Accepts the form "-last_modified", which sorts in descending order.
+                  Default to "-last_modified".
+    text=<frag> - Search for fragment "frag" in names and descriptions.
+  """
   path = request.GET.get('path', '/') # Expects path to be a Directory for now
 
   try:
@@ -73,18 +85,36 @@ def get_documents(request):
     else:
       raise e
 
-  parent_path = path.rsplit('/', 1)[0] or '/'
+  parent_path = path.rstrip('/').rsplit('/', 1)[0] or '/'
   parent = directory.dependencies.get(name=parent_path) if path != '/' else None
 
+  # Get querystring filters if any
+  page = int(request.GET.get('page', 1))
+  limit = int(request.GET.get('limit', 0))
+  type_filters = request.GET.getlist('type', None)
+  sort = request.GET.get('sort', '-last_modified')
+  search_text = request.GET.get('text', None)
+
+  documents, count = directory.documents(types=type_filters, search_text=search_text, page=page, limit=limit, order_by=sort)
+
   return JsonResponse({
+      'path': path,
       'directory': directory.to_dict(),
       'parent': parent.to_dict() if parent else None,
-      'documents': [doc.to_dict() for doc in directory.documents() if doc != parent],
-      'path': path
+      'documents': [doc.to_dict() for doc in documents if doc != parent],
+      'page': page,
+      'limit': limit,
+      'count': count,
+      'types': type_filters,
+      'sort': sort,
+      'text': search_text
   })
 
 
-def _import_documents1(user):
+def _convert_documents(user):
+  """
+  Given a user, converts any existing Document objects to Document2 objects
+  """
   from beeswax.models import HQL, IMPALA, RDBMS
 
   with transaction.atomic():
@@ -93,10 +123,10 @@ def _import_documents1(user):
     imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
 
     docs = docs.exclude(tags__in=[
-        DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
-        DocumentTag.objects.get_history_tag(user=user), # No history yet
-        DocumentTag.objects.get_example_tag(user=user), # No examples
-        imported_tag # No already imported docs
+        DocumentTag.objects.get_trash_tag(user=user),  # No trashed docs
+        DocumentTag.objects.get_history_tag(user=user),  # No history yet
+        DocumentTag.objects.get_example_tag(user=user),  # No examples
+        imported_tag  # No already imported docs
     ])
 
     root_doc, created = Directory.objects.get_or_create(name='/', owner=user)
@@ -157,6 +187,7 @@ def _massage_permissions(document):
       }
     }
 
+
 @api_error_handler
 @require_POST
 def move_document(request):
@@ -312,7 +343,6 @@ def export_documents(request):
     return make_response(f.getvalue(), 'json', 'hue-documents')
 
 
-
 def import_documents(request):
   if request.FILES.get('documents'):
     documents = request.FILES['documents'].read()

+ 24 - 2
desktop/core/src/desktop/models.py

@@ -951,6 +951,7 @@ class Directory(Document2):
   class Meta:
     proxy = True
 
+
   def save(self, *args, **kwargs):
     if self.type != 'directory':
       self.type = 'directory'
@@ -961,11 +962,32 @@ class Directory(Document2):
     if Document2.objects.filter(type='directory', owner=self.owner, name=self.name).count() > 1:
       raise ValidationError(_('Same directory %s for %s already exist') % (self.owner, self.name))
 
+
   def parent(self):
     return Document2.objects.get(type='directory', dependencies=[self.pk]) # or name__startswith=self.name
 
-  def documents(self):
-    return self.dependencies.all() # TODO perms
+
+  def documents(self, types=None, search_text=None, page=1, limit=0, order_by=None):
+    documents = self.dependencies.all()  # TODO: perms
+
+    if types and isinstance(types, list):
+      documents = documents.filter(type__in=types)
+
+    if search_text:
+      documents = documents.filter(Q(name__icontains=search_text) |
+                                   Q(description__icontains=search_text))
+
+    if order_by:  # TODO: Validate that order_by is a valid sort parameter
+      documents = documents.order_by(order_by)
+
+    count = documents.count()
+
+    if limit > 0:
+      offset = (page - 1) * limit
+      last = offset + limit
+      documents = documents.all()[offset:last]
+
+    return documents, count
 
 
 class Document2Permission(models.Model):

+ 33 - 1
desktop/core/src/desktop/tests_doc2.py

@@ -23,10 +23,11 @@ from django.contrib.auth.models import User
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
+from desktop.models import import_saved_beeswax_query, Directory, Document2
 
 from beeswax.models import SavedQuery
 from beeswax.design import hql_query
-from desktop.models import import_saved_beeswax_query, Directory
+
 
 
 class TestDocument2(object):
@@ -103,3 +104,34 @@ class TestDocument2(object):
     assert_equal(0, data['status'], data)
 
     assert_true(Directory.objects.filter(owner=self.user, name='/test_mv_dst/test_mv').exists())
+
+
+  def test_directory_documents(self):
+    home_dir = Directory.objects.get(owner=self.user, name='/')
+
+    dir1 = Directory.objects.create(name='/test_dir1', owner=self.user)
+    dir2 = Directory.objects.create(name='/test_dir2', owner=self.user)
+    query1 = Document2.objects.create(name='query1', type='query-hive', owner=self.user, data={})
+    query2 = Document2.objects.create(name='query2', type='query-hive', owner=self.user, data={})
+    children = [dir1, dir2, query1, query2]
+
+    home_dir.dependencies.add(*children)
+
+    # Test that all children directories and documents are returned
+    documents, count = home_dir.documents()
+    assert_equal(4, count, documents)
+
+    # Test filter type
+    documents, count = home_dir.documents(types=['directory'])
+    assert_equal(2, count, documents)
+    assert_true(all(doc.type == 'directory' for doc in documents))
+
+    # Test search text
+    documents, count = home_dir.documents(search_text='query')
+    assert_equal(2, count, documents)
+    assert_true(all(doc.name.startswith('query') for doc in documents))
+
+    # Test pagination with limit
+    documents, count = home_dir.documents(page=2, limit=2)
+    assert_equal(4, count, documents)
+    assert_equal(2, len(documents))

+ 2 - 2
desktop/core/src/desktop/views.py

@@ -40,7 +40,7 @@ import desktop.conf
 import desktop.log.log_buffer
 
 from desktop.api import massaged_tags_for_json, massaged_documents_for_json, _get_docs
-from desktop.api2 import _import_documents1
+from desktop.api2 import _convert_documents
 from desktop.lib import django_mako
 from desktop.lib.conf import GLOBAL_CONFIG, BoundConfig
 from desktop.lib.django_util import JsonResponse, login_notrequired, render_json, render
@@ -74,7 +74,7 @@ def home(request):
 
 
 def home2(request):
-  _import_documents1(request.user)
+  _convert_documents(request.user)
 
   apps = appmanager.get_apps_dict(request.user)