Sfoglia il codice sorgente

HUE-7402 [core] Truncate SEARCH Column in Document2 for Oracle database

  Searching saved documents in Oracle returns following error:
    DatabaseError: ORA-06502: PL/SQL: numeric or value error: character string buffer too small
  This is caused by DBMS_LOB.SUBSTR(%s, 4000) in Django framework django/db/backends/oracle/base.py
  Django has a ticket for this issue but unfixed: https://code.djangoproject.com/ticket/11580.
  Buffer size 4000 limit the length of field equals or less than 2000 characters.
Ying Chen 8 anni fa
parent
commit
417a9cc

+ 135 - 0
desktop/core/src/desktop/migrations/0027_truncate_documents.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+from desktop.conf import DATABASE
+from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        if DATABASE.ENGINE.get().endswith('oracle'):
+          db.execute("UPDATE DESKTOP_DOCUMENT2 SET SEARCH = SUBSTR(SEARCH, 1, %s)", [DOCUMENT2_SEARCH_MAX_LENGTH])
+
+    def backwards(self, orm):
+        pass
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'desktop.defaultconfiguration': {
+            'Meta': {'ordering': "['app', '-is_default', 'user']", 'object_name': 'DefaultConfiguration'},
+            'app': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'defaultconfiguration_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_default': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        u'desktop.document': {
+            'Meta': {'unique_together': "(('content_type', 'object_id'),)", 'object_name': 'Document'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc_owner'", 'to': u"orm['auth.User']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['desktop.DocumentTag']", 'db_index': 'True', 'symmetrical': 'False'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'})
+        },
+        u'desktop.document2': {
+            'Meta': {'ordering': "['-last_modified', 'name']", 'unique_together': "(('uuid', 'version', 'is_history'),)", 'object_name': 'Document2'},
+            'data': ('django.db.models.fields.TextField', [], {'default': "'{}'"}),
+            'dependencies': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'related_name': "'dependents'", 'symmetrical': 'False', 'to': u"orm['desktop.Document2']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_history': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'is_managed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'is_trashed': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc2_owner'", 'to': u"orm['auth.User']"}),
+            'parent_directory': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': u"orm['desktop.Document2']"}),
+            'search': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '32', 'db_index': 'True'}),
+            'uuid': ('django.db.models.fields.CharField', [], {'default': "'19659599-4bc7-40a6-820b-869364381201'", 'max_length': '36', 'db_index': 'True'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1', 'db_index': 'True'})
+        },
+        u'desktop.document2permission': {
+            'Meta': {'unique_together': "(('doc', 'perms'),)", 'object_name': 'Document2Permission'},
+            'doc': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['desktop.Document2']"}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'documentpermission2_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'perms': ('django.db.models.fields.CharField', [], {'default': "'read'", 'max_length': '10', 'db_index': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.User']", 'db_table': "'documentpermission2_users'", 'symmetrical': 'False'})
+        },
+        u'desktop.documentpermission': {
+            'Meta': {'unique_together': "(('doc', 'perms'),)", 'object_name': 'DocumentPermission'},
+            'doc': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['desktop.Document']"}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.Group']", 'db_table': "'documentpermission_groups'", 'symmetrical': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'perms': ('django.db.models.fields.CharField', [], {'default': "'read'", 'max_length': '10'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'db_index': 'True', 'to': u"orm['auth.User']", 'db_table': "'documentpermission_users'", 'symmetrical': 'False'})
+        },
+        u'desktop.documenttag': {
+            'Meta': {'unique_together': "(('owner', 'tag'),)", 'object_name': 'DocumentTag'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'tag': ('django.db.models.fields.SlugField', [], {'max_length': '50'})
+        },
+        u'desktop.settings': {
+            'Meta': {'object_name': 'Settings'},
+            'collect_usage': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'tours_and_tutorials': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'})
+        },
+        u'desktop.userpreferences': {
+            'Meta': {'object_name': 'UserPreferences'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'key': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'value': ('django.db.models.fields.TextField', [], {'max_length': '4096'})
+        }
+    }
+
+    complete_apps = ['desktop']

+ 4 - 0
desktop/core/src/desktop/models.py

@@ -52,6 +52,7 @@ from desktop.lib.i18n import force_unicode
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.paths import get_run_root
 from desktop.redaction import global_redaction_engine
+from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH
 
 
 LOG = logging.getLogger(__name__)
@@ -1211,6 +1212,9 @@ class Document2(models.Model):
     # Redact query if needed
     self._redact_query()
 
+    if self.search:
+      self.search = self.search[:DOCUMENT2_SEARCH_MAX_LENGTH]
+
     super(Document2, self).save(*args, **kwargs)
 
     # Inherit shared permissions from parent directory, must be done after save b/c new doc needs ID

+ 10 - 0
desktop/core/src/desktop/settings.py

@@ -506,3 +506,13 @@ if not desktop.conf.DATABASE_LOGGING.get():
     BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(cursor, self)
 
   disable_database_logging()
+
+############################################################
+# Searching saved documents in Oracle returns following error:
+#   DatabaseError: ORA-06502: PL/SQL: numeric or value error: character string buffer too small
+# This is caused by DBMS_LOB.SUBSTR(%s, 4000) in Django framework django/db/backends/oracle/base.py
+# Django has a ticket for this issue but unfixed: https://code.djangoproject.com/ticket/11580.
+# Buffer size 4000 limit the length of field equals or less than 2000 characters.
+#
+# For performance reasons and to avoid searching in huge fields, we also truncate to a max length
+DOCUMENT2_SEARCH_MAX_LENGTH = 2000