Răsfoiți Sursa

HUE-2538 [beeswax] Redact the QueryHistory query if redaction is enabled

Erick Tryzelaar 11 ani în urmă
părinte
comite
f3c43fc

+ 140 - 0
apps/beeswax/src/beeswax/migrations/0009_auto__add_field_savedquery_is_redacted__add_field_queryhistory_is_reda.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'SavedQuery.is_redacted'
+        db.add_column('beeswax_savedquery', 'is_redacted',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+        # Adding field 'QueryHistory.is_redacted'
+        db.add_column('beeswax_queryhistory', 'is_redacted',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'SavedQuery.is_redacted'
+        db.delete_column('beeswax_savedquery', 'is_redacted')
+
+        # Deleting field 'QueryHistory.is_redacted'
+        db.delete_column('beeswax_queryhistory', 'is_redacted')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'beeswax.metainstall': {
+            'Meta': {'object_name': 'MetaInstall'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'installed_example': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'beeswax.queryhistory': {
+            'Meta': {'ordering': "['-submission_date']", 'object_name': 'QueryHistory'},
+            'design': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['beeswax.SavedQuery']", 'null': 'True'}),
+            'has_results': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_redacted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_state': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'log_context': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}),
+            'modified_row_count': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'operation_type': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'query': ('django.db.models.fields.TextField', [], {}),
+            'query_type': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'server_guid': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '1024', 'null': 'True'}),
+            'server_host': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}),
+            'server_id': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}),
+            'server_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}),
+            'server_port': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'server_type': ('django.db.models.fields.CharField', [], {'default': "'beeswax'", 'max_length': '128'}),
+            'statement_number': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'submission_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'})
+        },
+        'beeswax.savedquery': {
+            'Meta': {'ordering': "['-mtime']", 'object_name': 'SavedQuery'},
+            'data': ('django.db.models.fields.TextField', [], {'max_length': '65536'}),
+            'desc': ('django.db.models.fields.TextField', [], {'max_length': '1024'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_auto': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'is_redacted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_trashed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'mtime': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'type': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'beeswax.session': {
+            'Meta': {'object_name': 'Session'},
+            'application': ('django.db.models.fields.CharField', [], {'default': "'beeswax'", 'max_length': '128'}),
+            'guid': ('django.db.models.fields.TextField', [], {'max_length': "'100'"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_used': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'secret': ('django.db.models.fields.TextField', [], {'max_length': "'100'"}),
+            'server_protocol_version': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status_code': ('django.db.models.fields.PositiveSmallIntegerField', [], {})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'desktop.document': {
+            'Meta': {'object_name': 'Document'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'extra': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'doc_owner'", 'to': "orm['auth.User']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['desktop.DocumentTag']", 'db_index': 'True', 'symmetrical': 'False'}),
+            'version': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'})
+        },
+        'desktop.documenttag': {
+            'Meta': {'object_name': 'DocumentTag'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'tag': ('django.db.models.fields.SlugField', [], {'max_length': '50'})
+        }
+    }
+
+    complete_apps = ['beeswax']

+ 56 - 3
apps/beeswax/src/beeswax/models.py

@@ -18,6 +18,7 @@
 import base64
 import datetime
 import logging
+import json
 
 from django.db import models
 from django.contrib.auth.models import User
@@ -29,6 +30,7 @@ from enum import Enum
 
 from librdbms.server import dbms as librdbms_dbms
 
+from desktop.redaction import global_redaction_engine
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document
 
@@ -80,6 +82,8 @@ class QueryHistory(models.Model):
   design = models.ForeignKey('SavedQuery', to_field='id', null=True) # Some queries (like read/create table) don't have a design
   notify = models.BooleanField(default=False)                        # Notify on completion
 
+  is_redacted = models.BooleanField(default=False)
+
 
   class Meta:
     ordering = ['-submission_date']
@@ -167,6 +171,23 @@ class QueryHistory(models.Model):
   def set_to_expired(self):
     self.last_state = QueryHistory.STATE.expired.index
 
+  def save(self, *args, **kwargs):
+    """
+    Override `save` to optionally mask out the query from being saved to the
+    database. This is because if the beeswax database contains sensitive
+    information like personally identifiable information, that information
+    could be leaked into the Hue database and logfiles.
+    """
+
+    if global_redaction_engine.is_enabled():
+      redacted_query = global_redaction_engine.redact(self.query)
+
+      if self.query != redacted_query:
+        self.query = redacted_query
+        self.is_redacted = True
+
+    super(QueryHistory, self).save(*args, **kwargs)
+
 
 def make_query_context(type, info):
   """
@@ -243,6 +264,8 @@ class SavedQuery(models.Model):
   is_trashed = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is trashed'),
                                    help_text=_t('If this query is trashed.'))
 
+  is_redacted = models.BooleanField(default=False)
+
   doc = generic.GenericRelation(Document, related_name='hql_doc')
 
   class Meta:
@@ -271,13 +294,18 @@ class SavedQuery(models.Model):
     design = SavedQuery(owner=owner, type=query_type)
     design.name = SavedQuery.DEFAULT_NEW_DESIGN_NAME
     design.desc = ''
-    design.data = data
+
+    if global_redaction_engine.is_enabled():
+      design.data = global_redaction_engine.redact(data)
+    else:
+      design.data = data
+
     design.is_auto = True
     design.save()
 
     Document.objects.link(design, owner=design.owner, extra=design.type, name=design.name, description=design.desc)
-    design.doc.get().add_to_history()    
-    
+    design.doc.get().add_to_history()
+
     return design
 
   @staticmethod
@@ -320,6 +348,31 @@ class SavedQuery(models.Model):
   def get_absolute_url(self):
     return reverse(QueryHistory.get_type_name(self.type) + ':execute_design', kwargs={'design_id': self.id})
 
+  def save(self, *args, **kwargs):
+    """
+    Override `save` to optionally mask out the query from being saved to the
+    database. This is because if the beeswax database contains sensitive
+    information like personally identifiable information, that information
+    could be leaked into the Hue database and logfiles.
+    """
+
+    if global_redaction_engine.is_enabled():
+      data = json.loads(self.data)
+
+      try:
+        query = data['query']['query']
+      except KeyError:
+        pass
+      else:
+        redacted_query = global_redaction_engine.redact(query)
+
+        if query != redacted_query:
+          data['query']['query'] = redacted_query
+          self.is_redacted = True
+          self.data = json.dumps(data)
+
+    super(SavedQuery, self).save(*args, **kwargs)
+
 
 class SessionManager(models.Manager):
   def get_session(self, user, application='beeswax'):

+ 76 - 0
apps/beeswax/src/beeswax/tests.py

@@ -39,6 +39,8 @@ from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.db import transaction
 
+from desktop import redaction
+from desktop.redaction import logfilter
 from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
 from desktop.lib.test_utils import grant_access, add_to_group
 from desktop.lib.security_util import get_localhost_name
@@ -1494,6 +1496,41 @@ for x in sys.stdin:
     assert_true('<th>foo</th>' in resp.content, resp.content)
     assert_true([0, '0x0'] in resp.context['sample'], resp.context['sample'])
 
+  def test_redacting_queries(self):
+    c = make_logged_in_client()
+
+    rule = r'ssn=::ssn=\d{3}-\d{2}-\d{4}::ssn=XXX-XX-XXXX'
+    old_rules = redaction.global_redaction_engine.rules
+    redaction.global_redaction_engine.rules = []
+    redaction.global_redaction_engine.add_rules_from_string(rule)
+
+    logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
+
+    try:
+      # Make sure redacted queries are redacted.
+      query = 'SELECT "ssn=123-45-6789"'
+      expected_query = 'SELECT "ssn=XXX-XX-XXXX"'
+
+      resp = make_query(c, query)
+      content = json.loads(resp.content)
+      query_id = content['id']
+      history = beeswax.models.QueryHistory.objects.get(pk=query_id)
+      assert_equal(history.query, expected_query)
+      assert_true(history.is_redacted)
+
+      # Make sure unredacted queries are not redacted.
+      query = 'SELECT "hello"'
+      expected_query = 'SELECT "hello"'
+
+      resp = make_query(c, query)
+      content = json.loads(resp.content)
+      query_id = content['id']
+      history = beeswax.models.QueryHistory.objects.get(pk=query_id)
+      assert_equal(history.query, expected_query)
+      assert_false(history.is_redacted)
+    finally:
+      redaction.global_redaction_engine.rules = old_rules
+
 
 def test_import_gzip_reader():
   """Test the gzip reader in create table"""
@@ -2079,6 +2116,45 @@ class TestWithMockedServer(object):
     assert_true(sql_escaped in resp.content, resp.content)
     assert_false(sql in resp.content, resp.content)
 
+  def test_redact_saved_design(self):
+    rule = r'ssn=::ssn=\d{3}-\d{2}-\d{4}::ssn=XXX-XX-XXXX'
+    old_rules = redaction.global_redaction_engine.rules
+    redaction.global_redaction_engine.rules = []
+    redaction.global_redaction_engine.add_rules_from_string(rule)
+
+    logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
+
+    try:
+      # Make sure redacted queries are redacted.
+      query = 'SELECT "ssn=123-45-6789"'
+      expected_query = 'SELECT "ssn=XXX-XX-XXXX"'
+
+      response = _make_query(self.client, query, submission_type='Save', name='My Name 1', desc='My Description')
+      content = json.loads(response.content)
+      design_id = content['design_id']
+
+      design = SavedQuery.get(id=design_id)
+      data = json.loads(design.data)
+
+      assert_equal(data['query']['query'], expected_query)
+      assert_true(design.is_redacted)
+
+      # Make sure unredacted queries are not redacted.
+      query = 'SELECT "hello"'
+      expected_query = 'SELECT "hello"'
+
+      response = _make_query(self.client, query, submission_type='Save', name='My Name 2', desc='My Description')
+      content = json.loads(response.content)
+      design_id = content['design_id']
+
+      design = SavedQuery.get(id=design_id)
+      data = json.loads(design.data)
+
+      assert_equal(data['query']['query'], expected_query)
+      assert_false(design.is_redacted)
+    finally:
+      redaction.global_redaction_engine.rules = old_rules
+
 
 class TestDesign():