Jelajahi Sumber

HUE-3500 [doc2] Unify dependency relationships representation

Also allows dependencies to transparently be exported with exported documents
Jenny Kim 9 tahun lalu
induk
melakukan
69bcd6061d

+ 26 - 1
desktop/core/src/desktop/api2.py

@@ -317,7 +317,12 @@ def export_documents(request):
   if not request.user.is_superuser:
   if not request.user.is_superuser:
     docs = docs.filter(owner=request.user)
     docs = docs.filter(owner=request.user)
   docs = docs.filter(id__in=selection).order_by('-id')
   docs = docs.filter(id__in=selection).order_by('-id')
-  doc_ids = docs.values_list('id', flat=True)
+
+  # Add any dependencies to the set of exported documents
+  export_doc_set = _get_dependencies(docs)
+
+  # Get PKs of documents to export
+  doc_ids = [doc.pk for doc in export_doc_set]
 
 
   f = StringIO.StringIO()
   f = StringIO.StringIO()
 
 
@@ -413,6 +418,26 @@ def import_documents(request):
     return JsonResponse({'message': stdout.getvalue()})
     return JsonResponse({'message': stdout.getvalue()})
 
 
 
 
+def _get_dependencies(documents):
+  """
+  Given a list of Document2 objects, perform a depth-first search and return a set of documents with all
+   dependencies included
+  :param doc_set:
+  """
+  doc_set = set()
+
+  for doc in documents:
+    stack = [doc]
+    while stack:
+      curr_doc = stack.pop()
+      if curr_doc not in doc_set:
+        doc_set.add(curr_doc)
+        deps_set = set(curr_doc.dependencies.all())
+        stack.extend(deps_set - doc_set)
+
+  return doc_set
+
+
 def _filter_documents(request, queryset, flatten=True):
 def _filter_documents(request, queryset, flatten=True):
   """
   """
   Given optional querystring params extracted from the request, filter the given queryset of documents and return a
   Given optional querystring params extracted from the request, filter the given queryset of documents and return a

+ 1 - 1
desktop/core/src/desktop/models.py

@@ -908,7 +908,7 @@ class Document2(models.Model):
   version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
   version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
   is_history = models.BooleanField(default=False, db_index=True)
   is_history = models.BooleanField(default=False, db_index=True)
 
 
-  dependencies = models.ManyToManyField('self', db_index=True)
+  dependencies = models.ManyToManyField('self', symmetrical=False, related_name='dependents', db_index=True)
 
 
   parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
   parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
 
 

+ 32 - 0
desktop/core/src/desktop/tests_doc2.py

@@ -773,6 +773,38 @@ class TestDocument2ImportExport(object):
     self.home_dir = Document2.objects.get_home_directory(user=self.user)
     self.home_dir = Document2.objects.get_home_directory(user=self.user)
     self.not_me_home_dir = Document2.objects.get_home_directory(user=self.user_not_me)
     self.not_me_home_dir = Document2.objects.get_home_directory(user=self.user_not_me)
 
 
+  def test_export_documents(self):
+    query1 = Document2.objects.create(name='query1.sql', type='query-hive', owner=self.user, data={}, parent_directory=self.home_dir)
+    query2 = Document2.objects.create(name='query2.sql', type='query-hive', owner=self.user, data={}, parent_directory=self.home_dir)
+    workflow = Document2.objects.create(name='test.wf', type='oozie-workflow2', owner=self.user, data={}, parent_directory=self.home_dir)
+    workflow.dependencies.add(query1)
+    workflow.dependencies.add(query2)
+
+    # Test that exporting workflow should export all dependencies
+    response = self.client.get('/desktop/api2/doc/export/', {'documents': json.dumps([workflow.id]), 'format': 'json'})
+    documents = json.loads(response.content)
+    documents = json.loads(documents)
+
+    assert_equal(3, len(documents))
+    assert_true('test.wf' in [doc['fields']['name'] for doc in documents])
+    assert_true('query1.sql' in [doc['fields']['name'] for doc in documents])
+    assert_true('query2.sql' in [doc['fields']['name'] for doc in documents])
+
+    # Test that exporting multiple workflows with overlapping dependencies works
+    workflow2 = Document2.objects.create(name='test2.wf', type='oozie-workflow2', owner=self.user, data={}, parent_directory=self.home_dir)
+    workflow2.dependencies.add(query1)
+
+    response = self.client.get('/desktop/api2/doc/export/', {'documents': json.dumps([workflow.id, workflow2.id]), 'format': 'json'})
+    documents = json.loads(response.content)
+    documents = json.loads(documents)
+
+    assert_equal(4, len(documents))
+    assert_true('test.wf' in [doc['fields']['name'] for doc in documents])
+    assert_true('test2.wf' in [doc['fields']['name'] for doc in documents])
+    assert_true('query1.sql' in [doc['fields']['name'] for doc in documents])
+    assert_true('query2.sql' in [doc['fields']['name'] for doc in documents])
+
+
   def test_import_owned_document(self):
   def test_import_owned_document(self):
     owned_query = Document2.objects.create(
     owned_query = Document2.objects.create(
       name='query.sql',
       name='query.sql',