Преглед изворни кода

HUE-6295 [doc2] Avoid unrelated DB calls in sync_documents after import

1. Remove the default tag from examples
2. Delete documents with no object

are running on > 100k entries each in large databases, which is crashing
the import_documents functionality. This will prevent doing any calls
to the DB which are unrelated to the imported documents.
krish пре 8 година
родитељ
комит
59402b8

+ 1 - 1
desktop/core/src/desktop/management/commands/sync_documents.py

@@ -27,5 +27,5 @@ class Command(NoArgsCommand):
   def handle_noargs(self, **options):
     logging.info('Running syncing document command.')
     self.stdout.write('Syncing document objects...\n')
-    Document.objects.sync()
+    Document.objects.sync(doc2_only=False)
     self.stdout.write('Documents synced.\n')

+ 4 - 2
desktop/core/src/desktop/models.py

@@ -377,7 +377,7 @@ class DocumentManager(models.Manager):
       LOG.warn('Object %s already has documents: %s' % (content_object, content_object.doc.all()))
       return content_object.doc.all()[0]
 
-  def sync(self):
+  def sync(self, doc2_only=True):
 
     def find_jobs_with_no_doc(model):
       jobs = model.objects.filter(doc__isnull=True)
@@ -483,7 +483,7 @@ class DocumentManager(models.Manager):
       LOG.exception('error syncing Document2')
 
 
-    if Document._meta.db_table in table_names:
+    if not doc2_only and Document._meta.db_table in table_names:
       # Make sure doc have at least a tag
       try:
         for doc in Document.objects.filter(tags=None):
@@ -551,6 +551,8 @@ class DocumentManager(models.Manager):
           # the documents it's referencing from our document query. Messy, but it
           # works.
 
+          # TODO: This can be several 100k entries for large databases.
+          # Need to figure out a better way to handle this scenario.
           docs = Document.objects.all()
 
           for content_type in ContentType.objects.all():