Răsfoiți Sursa

HUE-1167 [impala] Query server address depend on the type of the query

There is no perfect solution as some queries don't have a SavedQuery but we need
to store the type (Beeswax or Impala) of the executed query.
Romain Rigaux 12 ani în urmă
părinte
comite
bc9067a

+ 3 - 1
apps/beeswax/src/beeswax/design.py

@@ -66,7 +66,7 @@ class HQLdesign(object):
   _FILE_RES_ATTRS = [ 'type', 'path' ]
   _FUNCTIONS_ATTRS = [ 'name', 'class_name' ]
 
-  def __init__(self, form=None):
+  def __init__(self, form=None, query_type=None):
     """Initialize the design from a valid form data."""
     if form is not None:
       assert isinstance(form, MultiForm)
@@ -75,6 +75,8 @@ class HQLdesign(object):
           settings = normalize_formset_dict(form.settings, HQLdesign._SETTINGS_ATTRS),
           file_resources = normalize_formset_dict(form.file_resources, HQLdesign._FILE_RES_ATTRS),
           functions = normalize_formset_dict(form.functions, HQLdesign._FUNCTIONS_ATTRS))
+      if query_type is not None:
+        self._data_dict['query']['type'] = query_type
 
   def dumps(self):
     """Returns the serialized form of the design in a string"""

+ 110 - 0
apps/beeswax/src/beeswax/migrations/0008_auto__add_field_queryhistory_query_type.py

@@ -0,0 +1,110 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+
+        # Adding field 'QueryHistory.query_type'
+        db.add_column('beeswax_queryhistory', 'query_type', self.gf('django.db.models.fields.SmallIntegerField')(default=0), keep_default=False)
+
+
+    def backwards(self, orm):
+
+        # Deleting field 'QueryHistory.query_type'
+        db.delete_column('beeswax_queryhistory', 'query_type')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'beeswax.metainstall': {
+            'Meta': {'object_name': 'MetaInstall'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'installed_example': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'})
+        },
+        'beeswax.queryhistory': {
+            'Meta': {'object_name': 'QueryHistory'},
+            'design': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['beeswax.SavedQuery']", 'null': 'True'}),
+            'has_results': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_state': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'log_context': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}),
+            'modified_row_count': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'operation_type': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'query': ('django.db.models.fields.TextField', [], {}),
+            'query_type': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'server_guid': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '1024', 'null': 'True'}),
+            'server_host': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}),
+            'server_id': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}),
+            'server_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}),
+            'server_port': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'server_type': ('django.db.models.fields.CharField', [], {'default': "'beeswax'", 'max_length': '128'}),
+            'statement_number': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'submission_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'})
+        },
+        'beeswax.savedquery': {
+            'Meta': {'object_name': 'SavedQuery'},
+            'data': ('django.db.models.fields.TextField', [], {'max_length': '65536'}),
+            'desc': ('django.db.models.fields.TextField', [], {'max_length': '1024'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_auto': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True', 'blank': 'True'}),
+            'is_trashed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True', 'blank': 'True'}),
+            'mtime': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'type': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'beeswax.session': {
+            'Meta': {'object_name': 'Session'},
+            'application': ('django.db.models.fields.CharField', [], {'default': "'beeswax'", 'max_length': '128'}),
+            'guid': ('django.db.models.fields.TextField', [], {'max_length': "'100'"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_used': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'secret': ('django.db.models.fields.TextField', [], {'max_length': "'100'"}),
+            'server_protocol_version': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status_code': ('django.db.models.fields.PositiveSmallIntegerField', [], {})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        }
+    }
+
+    complete_apps = ['beeswax']

+ 10 - 5
apps/beeswax/src/beeswax/models.py

@@ -29,7 +29,6 @@ from enum import Enum
 
 from desktop.lib.exceptions_renderable import PopupException
 
-from beeswax.conf import SERVER_INTERFACE
 from beeswax.design import HQLdesign, hql_query
 from beeswaxd.ttypes import QueryHandle as BeeswaxdQueryHandle, QueryState
 from TCLIService.ttypes import TSessionHandle, THandleIdentifier,\
@@ -43,6 +42,7 @@ QUERY_SUBMISSION_TIMEOUT = datetime.timedelta(0, 60 * 60)               # 1 hour
 # Constants for DB fields, hue ini
 BEESWAX = 'beeswax'
 HIVE_SERVER2 = 'hiveserver2'
+QUERY_TYPES = (HQL, IMPALA) = range(2)
 
 
 class QueryHistory(models.Model):
@@ -70,6 +70,7 @@ class QueryHistory(models.Model):
   server_port = models.SmallIntegerField(help_text=_('Port of the query server.'), default=0)
   server_name = models.CharField(max_length=128, help_text=_('Name of the query server.'), default='')
   server_type = models.CharField(max_length=128, help_text=_('Type of the query server.'), default=BEESWAX, choices=SERVER_TYPE)
+  query_type = models.SmallIntegerField(help_text=_('Type of the query.'), default=HQL, choices=((HQL, 'HQL'), (IMPALA, 'IMPALA')))
 
   design = models.ForeignKey('SavedQuery', to_field='id', null=True) # Some queries (like read/create table) don't have a design
   notify = models.BooleanField(default=False)                        # Notify on completion
@@ -79,7 +80,7 @@ class QueryHistory(models.Model):
 
   @staticmethod
   def build(*args, **kwargs):
-    if SERVER_INTERFACE.get() == HIVE_SERVER2:
+    if kwargs['server_type'] == HIVE_SERVER2:
       return HiveServerQueryHistory(*args, **kwargs)
     else:
       return BeeswaxQueryHistory(*args, **kwargs)
@@ -87,7 +88,6 @@ class QueryHistory(models.Model):
   def get_full_object(self):
     if self.server_type == HiveServerQueryHistory.node_type:
       return HiveServerQueryHistory.objects.get(id=self.id)
-    # Default is Beeswax
     else:
       return BeeswaxQueryHistory.objects.get(id=self.id)
 
@@ -98,11 +98,16 @@ class QueryHistory(models.Model):
     else:
       return HiveServerQueryHistory.objects.get(id=id)
 
+  def get_type_name(self):
+    if self.query_type == 1:
+      return 'impala'
+    else:
+      return 'beeswax'
 
   def get_query_server_config(self):
     from beeswax.server.dbms import get_query_server_config
 
-    query_server = get_query_server_config(self.server_type)
+    query_server = get_query_server_config(self.get_type_name())
     query_server.update({
         'server_name': self.server_name,
         'server_host': self.server_host,
@@ -261,7 +266,7 @@ class SavedQuery(models.Model):
   """
   DEFAULT_NEW_DESIGN_NAME = _('My saved query')
   AUTO_DESIGN_SUFFIX = _(' (new)')
-  TYPES = (HQL, IMPALA) = range(2)
+  TYPES = QUERY_TYPES
   TYPES_MAPPING = {'beeswax': HQL, 'hql': HQL, 'impala': IMPALA}
 
   type = models.IntegerField(null=False)

+ 13 - 10
apps/beeswax/src/beeswax/server/dbms.py

@@ -256,16 +256,18 @@ class Dbms:
     hql_query = query.hql_query
     if query_history is None:
       query_history = QueryHistory.build(
-                                  owner=self.client.user,
-                                  query=hql_query,
-                                  server_host='%(server_host)s' % self.client.query_server,
-                                  server_port='%(server_port)d' % self.client.query_server,
-                                  server_name='%(server_name)s' % self.client.query_server,
-                                  server_type=self.server_type,
-                                  last_state=QueryHistory.STATE.submitted.index,
-                                  design=design,
-                                  notify=query.query.get('email_notify', False),
-                                  statement_number=0)
+          owner=self.client.user,
+          query=hql_query,
+          server_host='%(server_host)s' % self.client.query_server,
+          server_port='%(server_port)d' % self.client.query_server,
+          server_name='%(server_name)s' % self.client.query_server,
+          server_type=self.server_type,
+          last_state=QueryHistory.STATE.submitted.index,
+          design=design,
+          notify=query.query.get('email_notify', False),
+          query_type=query.query['type'],
+          statement_number=0
+      )
       query_history.save()
 
       LOG.debug("Made new QueryHistory id %s user %s query: %s..." % (query_history.id, self.client.user, query_history.query[:25]))
@@ -291,6 +293,7 @@ class Dbms:
     query_history.has_results = handle.has_result_set
     query_history.modified_row_count = handle.modified_row_count
     query_history.log_context = handle.log_context
+    query_history.query_type = query.query['type']
     query_history.set_to_running()
     query_history.save()
 

+ 0 - 1
apps/beeswax/src/beeswax/tests.py

@@ -155,7 +155,6 @@ class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
     assert_true("A base for other temporary directories" in response_verbose.content)
 
   def test_query_with_resource(self):
-    raise SkipTest
     script = self.cluster.fs.open("/square.py", "w")
     script.write(
       """#!/usr/bin/python

+ 3 - 3
apps/beeswax/src/beeswax/views.py

@@ -80,9 +80,9 @@ def save_design(request, form, type, design, explicit_save):
   """
   assert form.saveform.is_valid()
 
-  if type == models.SavedQuery.HQL:
+  if type == models.HQL:
     design_cls = beeswax.design.HQLdesign
-  elif type == models.SavedQuery.IMPALA:
+  elif type == models.IMPALA:
     design_cls = beeswax.design.HQLdesign
   else:
     raise ValueError(_('Invalid design type %(type)s') % {'type': type})
@@ -411,7 +411,7 @@ def execute_query(request, design_id=None):
           return parameterization
 
         try:
-          query = HQLdesign(form)
+          query = HQLdesign(form, query_type=query_type)
           if to_explain:
             return explain_directly(request, query, design, query_server)
           else: